diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bcfeebd29e39..97c94ddacd7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1580,6 +1580,12 @@ jobs: - check_name: check-session-transcript-reader-boundary group: session-transcript-reader-boundary runner: blacksmith-4vcpu-ubuntu-2404 + - check_name: check-sqlite-session-schema-baseline + group: sqlite-session-schema-baseline + runner: blacksmith-4vcpu-ubuntu-2404 + - check_name: check-sqlite-session-flip-proof + group: sqlite-session-flip-proof + runner: blacksmith-8vcpu-ubuntu-2404 - check_name: check-additional-extension-channels group: extension-channels runner: blacksmith-8vcpu-ubuntu-2404 @@ -1695,6 +1701,13 @@ jobs: else run_check "lint:tmp:session-accessor-boundary" pnpm run lint:tmp:session-accessor-boundary fi + if [ ! -f scripts/check-sqlite-transaction-boundary.mjs ]; then + echo "[skip] SQLite transaction boundary check is not present in this checkout" + elif ! node -e 'const pkg = require("./package.json"); process.exit(pkg.scripts?.["lint:tmp:sqlite-transaction-boundary"] ? 0 : 1);'; then + echo "[skip] SQLite transaction boundary script is not present in package.json" + else + run_check "lint:tmp:sqlite-transaction-boundary" pnpm run lint:tmp:sqlite-transaction-boundary + fi ;; session-transcript-reader-boundary) if [ ! -f scripts/check-session-transcript-reader-boundary.mjs ]; then @@ -1705,6 +1718,20 @@ jobs: run_check "lint:tmp:session-transcript-reader-boundary" pnpm run lint:tmp:session-transcript-reader-boundary fi ;; + sqlite-session-schema-baseline) + if ! node -e 'const pkg = require("./package.json"); process.exit(pkg.scripts?.["sqlite:sessions-schema:check"] ? 0 : 1);'; then + echo "[skip] SQLite sessions/transcripts schema baseline script is not present in package.json" + else + run_check "sqlite:sessions-schema:check" pnpm run sqlite:sessions-schema:check + fi + ;; + sqlite-session-flip-proof) + if [ ! -f test/scripts/sqlite-sessions-transcripts-flip-proof.e2e.test.ts ]; then + echo "[skip] SQLite sessions/transcripts flip proof is not present in this checkout" + else + run_check "sqlite sessions/transcripts flip proof" node scripts/run-vitest.mjs run --config test/vitest/vitest.e2e.config.ts test/scripts/sqlite-sessions-transcripts-flip-proof.e2e.test.ts + fi + ;; extension-channels) run_check "lint:extensions:channels" pnpm run lint:extensions:channels ;; diff --git a/.github/workflows/workflow-sanity.yml b/.github/workflows/workflow-sanity.yml index 76b98daee98b..59a3471f666a 100644 --- a/.github/workflows/workflow-sanity.yml +++ b/.github/workflows/workflow-sanity.yml @@ -212,5 +212,8 @@ jobs: - name: Check plugin SDK API baseline drift run: pnpm plugin-sdk:api:check + - name: Check SQLite sessions/transcripts schema baseline drift + run: pnpm sqlite:sessions-schema:check + - name: Check plugin SDK surface budget run: pnpm plugin-sdk:surface:check diff --git a/AGENTS.md b/AGENTS.md index ea24c5f8c419..931652253209 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,6 +76,7 @@ Skills own workflows; root owns hard policy and routing. - State/storage migrations are database-first. Runtime reads/writes the canonical store only. Old file stores, sidecars, aliases, and fallback readers belong in `openclaw doctor --fix` migration code only, never steady-state runtime. - Storage default: SQLite only. Do not add JSON/JSONL/TXT/sidecar files for OpenClaw-owned runtime state, caches, queues, registries, indexes, cursors, checkpoints, or plugin scratch data. - SQLite runtime access uses Kysely helpers, not raw SQL statement strings, except schema DDL, migrations, low-level DB bootstrap, or narrowly justified SQLite primitives. +- SQLite write transactions are synchronous commit sections only. Finish async planning, filesystem access, plugin hooks, and predicates before `BEGIN`; then reread and validate authoritative rows before writing. Never return a Promise or execute `await` from a transaction callback. - Use the shared state DB (`state/openclaw.sqlite`) for global runtime state and plugin KV data. Use the per-agent DB (`agents//agent/openclaw-agent.sqlite`) for agent-scoped state/cache. Use a dedicated SQLite DB only when schema, volume, or lifecycle clearly does not fit those stores. - Legacy state/cache files are migration debt. When touching code that reads/writes them, prefer moving the data into SQLite or calling out the refactor follow-up; do not add parallel file paths. - File storage must be a named product artifact: import/export, user attachment, log, backup, or external tool contract. If it is app state or cache, it belongs in SQLite. diff --git a/docs/.generated/README.md b/docs/.generated/README.md index ff97d11a0bbd..c3294495f876 100644 --- a/docs/.generated/README.md +++ b/docs/.generated/README.md @@ -1,17 +1,19 @@ # Generated Docs Artifacts -SHA-256 hash files are the tracked drift-detection artifacts. The full JSON -baselines are generated locally (gitignored) for inspection only. +SHA-256 hash files are the tracked drift-detection artifacts. Full baselines are +generated locally for inspection only. **Tracked (committed to git):** - `config-baseline.sha256` — hashes of config baseline JSON artifacts. - `plugin-sdk-api-baseline.sha256` — hashes of Plugin SDK API baseline artifacts. +- `sqlite-session-transcript-schema-baseline.sha256` — hash of the sessions/transcripts SQLite schema baseline. **Local only (gitignored):** - `config-baseline.json`, `config-baseline.core.json`, `config-baseline.channel.json`, `config-baseline.plugin.json` - `plugin-sdk-api-baseline.json`, `plugin-sdk-api-baseline.jsonl` +- `.artifacts/sqlite-session-transcript-schema-baseline.sql` Do not edit any of these files by hand. @@ -19,3 +21,5 @@ Do not edit any of these files by hand. - Validate config baseline: `pnpm config:docs:check` - Regenerate Plugin SDK API baseline: `pnpm plugin-sdk:api:gen` - Validate Plugin SDK API baseline: `pnpm plugin-sdk:api:check` +- Regenerate SQLite sessions/transcripts schema baseline: `pnpm sqlite:sessions-schema:gen` +- Validate SQLite sessions/transcripts schema baseline: `pnpm sqlite:sessions-schema:check` diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 3db1522b05ba..e9c27493fbd6 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -6f7e0f6f2e6d5b107500b62f9556abdc5e450eb80a7a8586c1baf7d82894ed2e plugin-sdk-api-baseline.json -a3d7d4f1a3aff973cb59bac966f884a812cbc633f90a8d9cd793209192916e5c plugin-sdk-api-baseline.jsonl +07c747212af5b8a228a7ff23101a2fa4096f148600d1d2e0fc83ac8f151cb58b plugin-sdk-api-baseline.json +85868eedeb4a99e714468f7f8c81d070227f8ed7e247939a4ac0d458ad04426d plugin-sdk-api-baseline.jsonl diff --git a/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 b/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 new file mode 100644 index 000000000000..5e723d90f2ea --- /dev/null +++ b/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 @@ -0,0 +1 @@ +4bb5f50a60c3664656d56f2be8d9884d48287f084c03959120b2825991b23f21 sqlite-session-transcript-schema-baseline.sql diff --git a/docs/.i18n/glossary.zh-CN.json b/docs/.i18n/glossary.zh-CN.json index f7342e537f83..d0e56648246a 100644 --- a/docs/.i18n/glossary.zh-CN.json +++ b/docs/.i18n/glossary.zh-CN.json @@ -1458,5 +1458,13 @@ { "source": "Crabbox plugin", "target": "Crabbox 插件" + }, + { + "source": "Path 3 SQLite session artifact family", + "target": "Path 3 SQLite 会话工件族" + }, + { + "source": "Path 3 live SQLite E2E harness", + "target": "Path 3 live SQLite E2E harness" } ] diff --git a/docs/channels/channel-routing.md b/docs/channels/channel-routing.md index 562dd6e4edaf..42dd7c323709 100644 --- a/docs/channels/channel-routing.md +++ b/docs/channels/channel-routing.md @@ -134,16 +134,24 @@ Example: ## Session storage -Session stores live under the state directory (default `~/.openclaw`): +Runtime session rows live in each agent's SQLite database under the state +directory (default `~/.openclaw`): -- `~/.openclaw/agents//sessions/sessions.json` -- JSONL transcripts live alongside the store +- `~/.openclaw/agents//agent/openclaw-agent.sqlite` -You can override the store path via `session.store` and `{agentId}` templating. +Older installs may have legacy transcript JSONL files and a `sessions.json` row +store under `~/.openclaw/agents//sessions/`. Gateway startup and +`openclaw doctor --fix` import hot legacy rows/history into SQLite +automatically. Use `openclaw doctor --session-sqlite inspect +--session-sqlite-all-agents` and the +[Doctor](/cli/doctor#session-sqlite-migration) validation sequence when you need +explicit migration evidence. +You can still select a legacy store path via `session.store` and `{agentId}` +templating for migration and offline-maintenance workflows. Gateway and ACP session discovery also scans disk-backed agent stores under the default `agents/` root and under templated `session.store` roots. Discovered -stores must stay inside that resolved agent root and use a regular +stores must stay inside that resolved agent root and use a regular legacy `sessions.json` file. Symlinks and out-of-root paths are ignored. ## WebChat behavior diff --git a/docs/channels/group-messages.md b/docs/channels/group-messages.md index fed236c363eb..44fccabcd6c1 100644 --- a/docs/channels/group-messages.md +++ b/docs/channels/group-messages.md @@ -87,7 +87,7 @@ Only owner numbers (from `channels.whatsapp.allowFrom`, or the bot's own E.164 w - Heartbeats run in the agent's main session; group sessions never get heartbeat runs. - Echo suppression remembers the combined prompt (history + current message) per session so the bot's own delivered messages do not retrigger it; an identical repeated batch can be skipped as an echo. -- Session store entries appear as `agent::whatsapp:group:` in the session store (`~/.openclaw/agents//sessions/sessions.json` by default); a missing entry just means the group has not triggered a run yet. +- Session store entries appear as `agent::whatsapp:group:` in the per-agent SQLite session store; a missing entry just means the group has not triggered a run yet. - Typing indicators follow `session.typingMode` / `agents.defaults.typingMode`. When visible replies are opted into message-tool-only mode, typing starts immediately by default so group members can see the agent working even if no automatic final reply is posted. Explicit typing-mode config still wins. ## Related diff --git a/docs/ci.md b/docs/ci.md index 2ac622f6feaf..fe7fdf260a4c 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -25,31 +25,31 @@ dispatch. ## Pipeline overview -| Job | Purpose | When it runs | -| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------- | -| `preflight` | Detect docs-only changes, changed scopes, changed extensions, and build the CI manifest | Always on non-draft pushes and PRs | -| `runner-admission` | Hosted 90-second debounce for canonical `main` pushes before Blacksmith work is registered | Every CI run; sleep only on canonical `main` pushes | -| `security-fast` | Private key detection, changed-workflow audit via `zizmor`, and production lockfile audit | Always on non-draft pushes and PRs | -| `pnpm-store-warmup` | Warm the lockfile-pinned pnpm store cache without blocking Linux Node shards | Node or docs-check lanes selected | -| `build-artifacts` | Build `dist/`, Control UI, built-CLI smoke checks, startup memory, and embedded built-artifact checks | Node-relevant changes | -| `checks-fast-core` | Fast Linux correctness lanes: bundled + protocol, Bun launcher, and the CI-routing fast task | Node-relevant changes | -| `qa-smoke-ci-profile` | Two self-contained balanced parts of the bounded automatic QA Smoke representative set; full taxonomy coverage remains available through explicit QA profiles | Node-relevant changes | -| `checks-fast-contracts-plugins-*` | Two weighted plugin contract shards | Node-relevant changes | -| `checks-fast-contracts-channels-*` | Two weighted channel contract shards | Node-relevant changes | -| `checks-node-*` | Core Node test shards, excluding channel, bundled, contract, and extension lanes | Node-relevant changes | -| `check-*` | Sharded main local gate equivalent: guards, shrinkwrap, bundled-channel config metadata, prod types, lint, dependencies, test types | Node-relevant changes | -| `check-additional-*` | Boundary check stripes (including prompt snapshot drift), session accessor/transcript reader boundaries, extension lint groups, package boundary compile/canary, and runtime topology architecture | Node-relevant changes | -| `checks-node-compat-node22` | Node 22 compatibility build and smoke lane | Manual CI dispatch for releases | -| `check-docs` | Docs formatting, lint, and broken-link checks | Docs changed (PRs and manual dispatch) | -| `native-i18n` | Native app, Android, and Apple i18n inventory checks | Native i18n-relevant changes | -| `skills-python` | Ruff + pytest for Python-backed skills | Python-skill-relevant changes | -| `checks-windows` | Windows-specific process/path tests plus shared runtime import specifier regressions | Windows-relevant changes | -| `macos-node` | Focused macOS TypeScript tests: launchd, Homebrew, runtime paths, packaging scripts, process-group wrapper | macOS-relevant changes | -| `macos-swift` | Swift lint, build, and tests for the macOS app | macOS-relevant changes | -| `ios-build` | Xcode project generation plus the iOS app simulator build | iOS app, shared app kit, or Swabble changes | -| `android` | Android unit tests for both flavors plus one debug APK build | Android-relevant changes | -| `test-performance-agent` | Separate workflow: daily Codex slow-test optimization after trusted activity | Main CI success or manual dispatch | -| `openclaw-performance` | Separate workflow: daily/on-demand Kova runtime performance reports with mock-provider, deep-profile, and GPT 5.6 live lanes | Scheduled and manual dispatch | +| Job | Purpose | When it runs | +| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------- | +| `preflight` | Detect docs-only changes, changed scopes, changed extensions, and build the CI manifest | Always on non-draft pushes and PRs | +| `runner-admission` | Hosted 90-second debounce for canonical `main` pushes before Blacksmith work is registered | Every CI run; sleep only on canonical `main` pushes | +| `security-fast` | Private key detection, changed-workflow audit via `zizmor`, and production lockfile audit | Always on non-draft pushes and PRs | +| `pnpm-store-warmup` | Warm the lockfile-pinned pnpm store cache without blocking Linux Node shards | Node or docs-check lanes selected | +| `build-artifacts` | Build `dist/`, Control UI, built-CLI smoke checks, startup memory, and embedded built-artifact checks | Node-relevant changes | +| `checks-fast-core` | Fast Linux correctness lanes: bundled + protocol, Bun launcher, and the CI-routing fast task | Node-relevant changes | +| `qa-smoke-ci-profile` | Two self-contained balanced parts of the bounded automatic QA Smoke representative set; full taxonomy coverage remains available through explicit QA profiles | Node-relevant changes | +| `checks-fast-contracts-plugins-*` | Two weighted plugin contract shards | Node-relevant changes | +| `checks-fast-contracts-channels-*` | Two weighted channel contract shards | Node-relevant changes | +| `checks-node-*` | Core Node test shards, excluding channel, bundled, contract, and extension lanes | Node-relevant changes | +| `check-*` | Sharded main local gate equivalent: guards, shrinkwrap, bundled-channel config metadata, prod types, lint, dependencies, test types | Node-relevant changes | +| `check-additional-*` | Boundary check stripes (including prompt snapshot drift), session accessor/transcript reader/SQLite transaction boundaries, extension lint groups, package boundary compile/canary, and runtime topology architecture | Node-relevant changes | +| `checks-node-compat-node22` | Node 22 compatibility build and smoke lane | Manual CI dispatch for releases | +| `check-docs` | Docs formatting, lint, and broken-link checks | Docs changed (PRs and manual dispatch) | +| `native-i18n` | Native app, Android, and Apple i18n inventory checks | Native i18n-relevant changes | +| `skills-python` | Ruff + pytest for Python-backed skills | Python-skill-relevant changes | +| `checks-windows` | Windows-specific process/path tests plus shared runtime import specifier regressions | Windows-relevant changes | +| `macos-node` | Focused macOS TypeScript tests: launchd, Homebrew, runtime paths, packaging scripts, process-group wrapper | macOS-relevant changes | +| `macos-swift` | Swift lint, build, and tests for the macOS app | macOS-relevant changes | +| `ios-build` | Xcode project generation plus the iOS app simulator build | iOS app, shared app kit, or Swabble changes | +| `android` | Android unit tests for both flavors plus one debug APK build | Android-relevant changes | +| `test-performance-agent` | Separate workflow: daily Codex slow-test optimization after trusted activity | Main CI success or manual dispatch | +| `openclaw-performance` | Separate workflow: daily/on-demand Kova runtime performance reports with mock-provider, deep-profile, and GPT 5.6 live lanes | Scheduled and manual dispatch | ## Fail-fast order diff --git a/docs/cli/doctor.md b/docs/cli/doctor.md index c7f1ad03b07b..856e759c2537 100644 --- a/docs/cli/doctor.md +++ b/docs/cli/doctor.md @@ -23,6 +23,15 @@ Related: | Repair | `openclaw doctor --fix` | Applies supported repairs, prompting unless non-interactive repair is safe. | | Lint | `openclaw doctor --lint` | Read-only structured findings for CI, preflight, and review gates. | +Doctor has four postures: + +| Posture | Command | Behavior | +| ------------------------ | ----------------------------------------- | ------------------------------------------------------------------------------- | +| Inspect | `openclaw doctor` | Human-oriented checks and guided prompts. | +| Repair | `openclaw doctor --fix` | Applies supported repairs, using prompts unless non-interactive repair is safe. | +| Lint | `openclaw doctor --lint` | Read-only structured findings for CI, preflight, and review gates. | +| Session SQLite migration | `openclaw doctor --session-sqlite ` | Inspects, imports, validates, compacts, recovers, or restores session state. | + Prefer `--lint` when automation needs a stable result. Prefer `--fix` when a human operator wants doctor to edit config or state. ## Examples @@ -40,6 +49,13 @@ openclaw doctor --fix --non-interactive openclaw doctor --generate-gateway-token openclaw doctor --post-upgrade openclaw doctor --post-upgrade --json +openclaw doctor --session-sqlite inspect --session-sqlite-all-agents +openclaw doctor --session-sqlite dry-run --session-sqlite-agent main --json +openclaw doctor --session-sqlite import --session-sqlite-all-agents +openclaw doctor --session-sqlite validate --session-sqlite-all-agents --json +openclaw doctor --session-sqlite compact --session-sqlite-all-agents +openclaw doctor --session-sqlite recover --github-issue +openclaw doctor --session-sqlite restore --session-sqlite-all-agents ``` For channel-specific permissions, use the channel probes instead of `doctor`: @@ -53,25 +69,30 @@ openclaw channels status --probe ## Options -| Option | Effect | -| ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--no-workspace-suggestions` | Disable workspace memory/search suggestions. | -| `--yes` | Accept defaults without prompting. | -| `--repair` / `--fix` | Apply recommended non-service repairs without prompting (`--fix` is an alias). Gateway service installs/rewrites still require interactive confirmation or explicit `gateway` commands. | -| `--force` | Apply aggressive repairs, including overwriting custom service config. | -| `--non-interactive` | Run without prompts; safe migrations and non-service repairs only. | -| `--generate-gateway-token` | Generate and configure a gateway token. | -| `--allow-exec` | Allow doctor to execute configured `exec` SecretRefs while verifying secrets. | -| `--deep` | Scan system services for extra gateway installs; report recent Gateway supervisor restart handoffs. | -| `--lint` | Run modernized health checks in read-only mode and emit diagnostic findings. | -| `--post-upgrade` | Run post-upgrade plugin compatibility probes; findings go to stdout; exit code 1 if any error-level finding is present. | -| `--json` | With `--lint`: JSON findings. With `--post-upgrade`: machine-readable envelope `{ probesRun, findings }`. | -| `--severity-min ` | With `--lint`: drop findings below `info`, `warning`, or `error`. | -| `--all` | With `--lint`: run all registered checks, including opt-in checks excluded from the default set. | -| `--skip ` | With `--lint`: skip a check id. Repeatable. | -| `--only ` | With `--lint`: run only the given check id(s). Repeatable. | +| Option | Effect | +| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--no-workspace-suggestions` | Disable workspace memory/search suggestions. | +| `--yes` | Accept defaults without prompting. | +| `--repair` / `--fix` | Apply recommended non-service repairs without prompting (`--fix` is an alias). Gateway service installs/rewrites still require interactive confirmation or explicit `gateway` commands. | +| `--force` | Apply aggressive repairs, including overwriting custom service config. | +| `--non-interactive` | Run without prompts; safe migrations and non-service repairs only. | +| `--generate-gateway-token` | Generate and configure a gateway token. | +| `--allow-exec` | Allow doctor to execute configured `exec` SecretRefs while verifying secrets. | +| `--deep` | Scan system services for extra gateway installs; report recent Gateway supervisor restart handoffs. | +| `--lint` | Run modernized health checks in read-only mode and emit diagnostic findings. | +| `--post-upgrade` | Run post-upgrade plugin compatibility probes; findings go to stdout; exit code 1 if any error-level finding is present. | +| `--session-sqlite ` | Run the targeted session SQLite migration mode: `inspect`, `dry-run`, `import`, `validate`, `compact`, `recover`, or `restore`. | +| `--session-sqlite-store ` | With `--session-sqlite`: select one legacy `sessions.json` store path. | +| `--session-sqlite-agent ` | With `--session-sqlite`: select one configured agent. | +| `--session-sqlite-all-agents` | With `--session-sqlite`: select configured and discovered agent stores. | +| `--github-issue` | With `--session-sqlite recover`: prepare a sanitized openclaw/openclaw issue report; doctor creates it with `gh` after `--yes` or interactive confirmation. | +| `--json` | With `--lint`: JSON findings. With `--post-upgrade`: machine-readable envelope `{ probesRun, findings }`. With `--session-sqlite`: the migration report as JSON. | +| `--severity-min ` | With `--lint`: drop findings below `info`, `warning`, or `error`. | +| `--all` | With `--lint`: run all registered checks, including opt-in checks excluded from the default set. | +| `--skip ` | With `--lint`: skip a check id. Repeatable. | +| `--only ` | With `--lint`: run only the given check id(s). Repeatable. | -`--json`, `--severity-min`, `--all`, `--only`, and `--skip` are only accepted together with `--lint`. +`--severity-min`, `--all`, `--only`, and `--skip` are only accepted together with `--lint`; `--json` is accepted with `--lint`, `--post-upgrade`, and `--session-sqlite`. ## Lint mode @@ -178,6 +199,107 @@ finish safely, startup exits and tells you to run the same image once with `openclaw doctor --fix` against the same mounted state/config before restarting the container normally. +## Session SQLite migration + +OpenClaw imports legacy session rows and transcript history into each agent's +SQLite database automatically during gateway startup and during +`openclaw doctor --fix`. `openclaw doctor --session-sqlite ` is the +targeted inspection and validation tool for that migration. Current runtime +session rows live in +`~/.openclaw/agents//agent/openclaw-agent.sqlite`. Legacy +`sessions.json` files are migration sources. Hot transcript JSONL files are +imported and archived out of the active sessions directory after successful +import; archive-tier JSONL files remain support artifacts, not runtime +fallbacks. + +Modes: + +| Mode | Behavior | +| ---------- | ---------------------------------------------------------------------------------------------------------------------- | +| `inspect` | Read legacy and SQLite counts, plus unreferenced JSONL files, without importing. | +| `dry-run` | Parse legacy entries and transcript JSONL files, count importable rows, and report issues without writing SQLite rows. | +| `import` | Import legacy entries and transcript events into SQLite for the selected targets. | +| `validate` | Compare the selected legacy sources against SQLite rows and transcript event counts. | +| `compact` | Checkpoint and VACUUM selected agent SQLite databases to reclaim free pages after large deletes or archive cleanup. | +| `recover` | Restore the latest failed migration run, validate its targets, and prepare a sanitized GitHub issue report. | +| `restore` | Restore archived transcript artifacts from recorded migration manifests without deleting SQLite data. | + +Selectors: + +- Default: the configured default agent store, when that legacy store file exists. +- `--session-sqlite-agent `: one configured agent. +- `--session-sqlite-all-agents`: configured agent stores plus discovered agent stores. +- `--session-sqlite-store `: one explicit legacy `sessions.json` path. + +Manual inspection sequence: + +```bash +openclaw doctor --session-sqlite inspect --session-sqlite-all-agents +openclaw doctor --session-sqlite dry-run --session-sqlite-all-agents --json +openclaw doctor --session-sqlite import --session-sqlite-all-agents +openclaw doctor --session-sqlite validate --session-sqlite-all-agents --json +openclaw doctor --session-sqlite compact --session-sqlite-all-agents +openclaw doctor --session-sqlite recover --github-issue +``` + +Back up the OpenClaw state directory before running `import` on an install with +important history. `validate` exits non-zero when a selected legacy entry is +missing from SQLite, a session id differs, or a transcript event count differs. +When using `--session-sqlite-store `, check that the report contains the +expected target count; a nonexistent explicit store path selects no targets. + +SQLite deletes reclaim pages inside the database first; they do not necessarily +shrink the database file immediately. After deleting or archiving large +transcripts, run `openclaw doctor --session-sqlite compact --session-sqlite-all-agents` +to checkpoint WAL files, run `VACUUM`, and report before/after database and WAL +sizes. This is explicit doctor maintenance so normal Gateway writes do not pause +for background vacuum work. + +Each import writes a manifest under +`~/.openclaw/session-sqlite-migration-runs/` before moving transcript artifacts +into the archive. If startup reports a failed session SQLite migration after +artifacts moved, run recovery: + +```bash +openclaw doctor --session-sqlite recover --github-issue +``` + +Recovery selects the latest failed migration manifest, restores only the +manifest's archived artifacts, validates the affected targets, refreshes the +sanitized `.failure.md` and `.failure.json` reports, and prepares a GitHub issue +body that avoids transcript contents, raw environment, secrets, and unbounded +config. When no failed migration manifest exists but a selected agent SQLite +database is corrupt or not a database, recovery preserves the DB, WAL, and SHM +files by renaming them with a `.corrupt-` suffix so the next startup +can create a fresh database. With `--github-issue --yes`, doctor uses the GitHub +CLI to create the issue in `openclaw/openclaw`; without confirmation it writes +the local support report and prints a prefilled issue URL. + +`restore` remains the lower-level undo operation. It uses manifest +`sourcePath -> archivePath` records, moves archived artifacts back only when the +original path is missing, reports conflicts when both paths exist, and leaves +the SQLite database in place. + +### Downgrading After Session SQLite Migration + +Before starting an older file-backed OpenClaw version, restore the archived +legacy transcript artifacts: + +```bash +openclaw doctor --session-sqlite restore --session-sqlite-all-agents +``` + +Older versions read `sessions.json` entries and the `sessionFile` paths recorded +in those entries. After the SQLite migration, successful imports move hot JSONL +transcripts into `session-sqlite-import-archive/`, so the older runtime cannot +see that history until restore moves those manifest-recorded artifacts back to +their original paths. + +Restore does not delete SQLite data. Sessions created after the SQLite flip +exist only in SQLite and will not appear to the older runtime. If you later +upgrade again, run the normal migration validation sequence above so OpenClaw can +compare restored legacy artifacts with the SQLite rows before importing. + ## Notes - In Nix mode (`OPENCLAW_NIX_MODE=1`), read-only doctor checks still work, but `doctor --fix`, `doctor --repair`, `doctor --yes`, and `doctor --generate-gateway-token` are disabled because `openclaw.json` is immutable. Edit the Nix source for this install instead; for nix-openclaw, use the agent-first [Quick Start](https://github.com/openclaw/nix-openclaw#quick-start). diff --git a/docs/cli/security.md b/docs/cli/security.md index 0dd16a1d2f98..e5c5f9597b6a 100644 --- a/docs/cli/security.md +++ b/docs/cli/security.md @@ -124,7 +124,7 @@ Applies safe, deterministic remediations: - flips common `groupPolicy="open"` to `groupPolicy="allowlist"` (including account variants in supported channels) - when WhatsApp group policy flips to `allowlist`, seeds `groupAllowFrom` from the stored `allowFrom` file when that list exists and config does not already define `allowFrom` - sets `logging.redactSensitive` from `"off"` to `"tools"` -- tightens permissions for state/config and common sensitive files (`credentials/*.json`, `auth-profiles.json`, `sessions.json`, session `*.jsonl`) +- tightens permissions for state/config and common sensitive files (`credentials/*.json`, `auth-profiles.json`, `openclaw-agent.sqlite`, and legacy session artifacts) - also tightens config include files referenced from `openclaw.json` - uses `chmod` on POSIX hosts and `icacls` resets on Windows diff --git a/docs/cli/sessions.md b/docs/cli/sessions.md index 25e7af272768..a18fd0c3f39f 100644 --- a/docs/cli/sessions.md +++ b/docs/cli/sessions.md @@ -51,10 +51,10 @@ Control UI uses that mode by default so deleted or disk-only agent stores do not reappear in the Sessions view. `--all-agents` reads configured agent stores. Gateway and ACP session -discovery are broader: they also include disk-only stores found under the -default `agents/` root or a templated `session.store` root. Those discovered -stores must resolve to regular `sessions.json` files inside the agent root; -symlinks and out-of-root paths are skipped. +discovery are broader: they also include SQLite stores resolved from +configured agent roots or a templated `session.store` root. Legacy selector +paths must resolve inside the agent root; symlinks and out-of-root paths are +skipped. `openclaw sessions --all-agents --json`: @@ -88,12 +88,12 @@ openclaw sessions --agent work tail --follow openclaw sessions --all-agents tail --follow ``` -`openclaw sessions tail` renders recent trajectory JSONL events as compact +`openclaw sessions tail` renders recent runtime trajectory events as compact progress lines. Without `--session-key`, it tails running sessions first, then the latest stored session. `--tail ` controls how many existing events print before follow mode; default `80`, and `0` starts at the current end. -`--follow` keeps watching the selected trajectory files, including relocated -files referenced by `.trajectory-path.json`. +`--follow` keeps watching the selected SQLite-backed session or an explicit +legacy trajectory file. The progress view is intentionally conservative: prompt text, tool arguments, and tool result bodies are not printed. Tool calls show the tool name with @@ -129,12 +129,13 @@ openclaw sessions cleanup --json ([Configuration reference](/gateway/config-agents#session)): - Scope note: `openclaw sessions cleanup` maintains session stores, - transcripts, and trajectory sidecars. It does not prune cron run history, - which is managed by `cron.runLog.keepLines` + transcripts, trajectory rows, and legacy trajectory sidecars. It does not + prune cron run history, which is managed by `cron.runLog.keepLines` ([Cron configuration](/automation/cron-jobs#configuration)). -- Cleanup also prunes unreferenced primary transcripts, compaction - checkpoints, and trajectory sidecars older than `session.maintenance.pruneAfter`; - files still referenced by `sessions.json` are preserved. +- Cleanup also prunes unreferenced legacy/archive transcript artifacts, + compaction checkpoints, and trajectory sidecars older than + `session.maintenance.pruneAfter`; artifacts still referenced by SQLite + session rows are preserved. - Cleanup reports short-lived Gateway model-run probe cleanup separately as `modelRunPruned`. This only matches strict explicit keys shaped like `agent:*:explicit:model-run-`. Retention is a fixed `24h` and is @@ -144,21 +145,22 @@ openclaw sessions cleanup --json Flags: -| Flag | Description | -| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--dry-run` | Preview how many entries would be pruned/capped without writing. In text mode, prints a per-session action table (`Action`, `Key`, `Age`, `Model`, `Flags`) plus a summary grouped by session label. | -| `--enforce` | Apply maintenance even when `session.maintenance.mode` is `warn`. | -| `--fix-missing` | Remove entries whose transcript files are missing or header-only/empty, even if they would not normally age/count out yet. | -| `--fix-dm-scope` | When `session.dmScope` is `main`, retire stale peer-keyed direct-DM rows left behind by earlier `per-peer`, `per-channel-peer`, or `per-account-channel-peer` routing. Use `--dry-run` first; applying removes those rows from `sessions.json` and preserves their transcripts as deleted archives. | -| `--active-key ` | Protect a specific active key from disk-budget eviction. Durable external conversation pointers, such as group sessions and thread-scoped chat sessions, are also kept by age/count/disk-budget maintenance. | -| `--agent ` | Run cleanup for one configured agent store. | -| `--all-agents` | Run cleanup for all configured agent stores. | -| `--store ` | Run against a specific `sessions.json` file. | -| `--json` | Print a JSON summary. With `--all-agents`, output includes one summary per store. | +| Flag | Description | +| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--dry-run` | Preview how many entries would be pruned/capped without writing. In text mode, prints a per-session action table (`Action`, `Key`, `Age`, `Model`, `Flags`) plus a summary grouped by session label. | +| `--enforce` | Apply maintenance even when `session.maintenance.mode` is `warn`. | +| `--fix-missing` | Remove legacy entries whose archived transcript artifacts are missing or header-only/empty, even if they would not normally age/count out yet. | +| `--fix-dm-scope` | When `session.dmScope` is `main`, retire stale peer-keyed direct-DM rows left behind by earlier `per-peer`, `per-channel-peer`, or `per-account-channel-peer` routing. Use `--dry-run` first; applying removes those rows from SQLite and preserves their legacy transcript artifacts as deleted archives. | +| `--active-key ` | Protect a specific active key from disk-budget eviction. Durable external conversation pointers, such as group sessions and thread-scoped chat sessions, are also kept by age/count/disk-budget maintenance. | +| `--agent ` | Run cleanup for one configured agent store. | +| `--all-agents` | Run cleanup for all configured agent stores. | +| `--store ` | Run against a specific legacy store selector path. | +| `--json` | Print a JSON summary. With `--all-agents`, output includes one summary per store. | When a Gateway is reachable, non-dry-run cleanup for configured agent stores is sent through the Gateway so it shares the same session-store writer as runtime -traffic. Use `--store ` for explicit offline repair of a store file. +traffic. Use `--store ` for explicit offline repair of a legacy store +selector. `openclaw sessions cleanup --all-agents --dry-run --json`: diff --git a/docs/cli/update.md b/docs/cli/update.md index 05058fe3c6d2..58c1eaa9d6c5 100644 --- a/docs/cli/update.md +++ b/docs/cli/update.md @@ -63,6 +63,9 @@ In Nix mode (`OPENCLAW_NIX_MODE=1`), mutating `openclaw update` runs are disable Downgrades require confirmation because older versions can break configuration. +If the install has already migrated sessions to SQLite, restore archived legacy +transcript artifacts before starting an older file-backed version. See +[Doctor: Downgrading after session SQLite migration](/cli/doctor#downgrading-after-session-sqlite-migration). ## `update status` diff --git a/docs/concepts/agent-loop.md b/docs/concepts/agent-loop.md index 81b4b93bd813..7350401ce6ad 100644 --- a/docs/concepts/agent-loop.md +++ b/docs/concepts/agent-loop.md @@ -36,7 +36,7 @@ Session write locks are non-reentrant by default. A helper that intentionally ne - Workspace is resolved and created; sandboxed runs may redirect to a sandbox workspace root. - Skills are loaded (or reused from a snapshot) and injected into env and prompt. - Bootstrap/context files are resolved and injected into the system prompt. -- A session write lock is acquired and `SessionManager` is opened and prepared before streaming starts. Any later transcript rewrite, compaction, or truncation path must take the same lock before opening or mutating the transcript file. +- A session write lock is acquired and the session transcript target is prepared before streaming starts. Any later transcript rewrite, compaction, or truncation path must take the same lock before mutating the SQLite transcript rows. ## Prompt assembly diff --git a/docs/concepts/agent-workspace.md b/docs/concepts/agent-workspace.md index aa1b61e88e09..c0840081eed7 100644 --- a/docs/concepts/agent-workspace.md +++ b/docs/concepts/agent-workspace.md @@ -112,9 +112,10 @@ These live under `~/.openclaw/` and should NOT be committed to the workspace rep - `~/.openclaw/openclaw.json` (config) - `~/.openclaw/agents//agent/auth-profiles.json` (model auth profiles: OAuth + API keys) +- `~/.openclaw/agents//agent/openclaw-agent.sqlite` (session rows, transcripts, and per-agent runtime state) - `~/.openclaw/agents//agent/codex-home/` (per-agent Codex runtime account, config, skills, plugins, and native thread state) - `~/.openclaw/credentials/` (channel/provider state plus legacy OAuth import data) -- `~/.openclaw/agents//sessions/` (session transcripts + metadata) +- `~/.openclaw/agents//sessions/` (legacy migration sources and archive/support artifacts) - `~/.openclaw/skills/` (managed skills) If you need to migrate sessions or config, copy them separately and keep them out of version control. @@ -217,7 +218,9 @@ Suggested `.gitignore` starter: Run `openclaw setup --workspace ` to seed any missing files. - If you need sessions, copy `~/.openclaw/agents//sessions/` from the old machine separately. + If you need sessions, copy `~/.openclaw/agents//agent/openclaw-agent.sqlite` + from the old machine separately. Copy `~/.openclaw/agents//sessions/` + only when you also need legacy migration inputs or archive/support artifacts. diff --git a/docs/concepts/agent.md b/docs/concepts/agent.md index eceb8bccb3ad..d3cea63b7aa6 100644 --- a/docs/concepts/agent.md +++ b/docs/concepts/agent.md @@ -87,11 +87,15 @@ runtime surface. ## Sessions -Session transcripts are stored as JSONL at: +Session rows are stored in the per-agent SQLite database: -- `~/.openclaw/agents//sessions/.jsonl` +- `~/.openclaw/agents//agent/openclaw-agent.sqlite` -The session ID is stable and chosen by OpenClaw. OpenClaw does not read session folders from other tools. +Transcript JSONL files can still live under +`~/.openclaw/agents//sessions/` as legacy migration inputs, deleted or +reset archives, imports, exports, and support artifacts. Active agent history is +stored in SQLite with the session rows. The session ID is stable and chosen by +OpenClaw. OpenClaw does not read session folders from other tools. ## Steering while streaming diff --git a/docs/concepts/compaction.md b/docs/concepts/compaction.md index 18da03fe6719..da855bedfafc 100644 --- a/docs/concepts/compaction.md +++ b/docs/concepts/compaction.md @@ -104,10 +104,16 @@ Compaction summarization preserves opaque identifiers by default (`identifierPol ### Active transcript byte guard -When `agents.defaults.compaction.maxActiveTranscriptBytes` is set, OpenClaw triggers normal local compaction before a run if the active JSONL reaches that size. This is useful for long-running sessions where provider-side context management may keep model context healthy while the local transcript keeps growing. It does not split raw JSONL bytes; it asks the normal compaction pipeline to create a semantic summary. +When `agents.defaults.compaction.maxActiveTranscriptBytes` is set, OpenClaw +triggers normal local compaction before a run if transcript history reaches +that size. This is useful for long-running sessions where provider-side context +management may keep model context healthy while persisted transcript history +keeps growing. It does not split raw bytes; it asks the normal compaction +pipeline to create a semantic summary. -The byte guard requires `truncateAfterCompaction: true`. Without transcript rotation, the active file would not shrink and the guard remains inactive. +The byte guard applies to the active SQLite transcript history. Legacy JSONL +checkpoint artifacts are not the active compaction target. ### Successor transcripts diff --git a/docs/concepts/context-engine.md b/docs/concepts/context-engine.md index a1824e211145..23dd8cbe8350 100644 --- a/docs/concepts/context-engine.md +++ b/docs/concepts/context-engine.md @@ -226,12 +226,10 @@ Required members: Optional projection lifecycle for hosts with persistent backend threads (for example Codex app-server). `mode: "thread_bootstrap"` with a stable `epoch` asks the host to inject the assembled context once per epoch and reuse the backend thread until the epoch changes, instead of re-projecting every turn. Omit this field for normal per-turn projection. -`compact` returns a `CompactResult`. When compaction rotates the active -transcript, `result.sessionTarget` (a typed `ContextEngineSessionTarget` -carrying the storage mode, session identity, and transcript artifact path) -identifies the successor session that the next retry or turn must use; -`result.sessionId` mirrors the successor id. `result.sessionFile` is -deprecated - report successors through `sessionTarget` instead. +`compact` returns a `CompactResult`. When compaction changes the active session +identity, `result.sessionTarget` (a typed `ContextEngineSessionTarget` carrying +the session identity and store scope) identifies the successor session that the +next retry or turn must use; `result.sessionId` mirrors the successor id. Optional members: diff --git a/docs/concepts/multi-agent.md b/docs/concepts/multi-agent.md index b2833cb3a042..170fe427eda6 100644 --- a/docs/concepts/multi-agent.md +++ b/docs/concepts/multi-agent.md @@ -6,7 +6,7 @@ read_when: "You want multiple agents with separate workspaces, auth, and session status: active --- -Run multiple agents in one Gateway process, each with its own workspace, state directory (`agentDir`), and session store, plus multiple channel accounts (e.g. two WhatsApp numbers). Inbound messages route to the right agent through **bindings**. +Run multiple _isolated_ agents in one Gateway process, each with its own workspace, state directory (`agentDir`), and SQLite-backed session history, plus multiple channel accounts (e.g. two WhatsApp numbers). Inbound messages route to the right agent through **bindings**. An **agent** is the full per-persona scope: workspace files, auth profiles, model registry, and session store. A **binding** maps a channel account (a Slack workspace, a WhatsApp number, etc.) to one of those agents. @@ -16,7 +16,7 @@ Each agent has its own: - **Workspace**: files, `AGENTS.md`/`SOUL.md`/`USER.md`, local notes, persona rules. - **State directory** (`agentDir`): auth profiles, model registry, per-agent config. -- **Session store**: chat history and routing state under `~/.openclaw/agents//sessions`. +- **Session store**: chat history and routing state in `~/.openclaw/agents//agent/openclaw-agent.sqlite`. Auth profiles are per-agent, read from: @@ -45,14 +45,15 @@ when personas must not share compiled wiki knowledge. ## Paths -| What | Default | Override | -| ------------------------- | -------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | -| Config | `~/.openclaw/openclaw.json` | `OPENCLAW_CONFIG_PATH` | -| State dir | `~/.openclaw` | `OPENCLAW_STATE_DIR` | -| Default agent's workspace | `~/.openclaw/workspace` (or `workspace-` when `OPENCLAW_PROFILE` is set) | `agents.list[].workspace`, then `agents.defaults.workspace`, or `OPENCLAW_WORKSPACE_DIR` | -| Other agents' workspace | `/workspace-` (or `/` when set) | `agents.list[].workspace` | -| Agent dir | `~/.openclaw/agents//agent` | `agents.list[].agentDir` | -| Sessions | `~/.openclaw/agents//sessions` | — | +| What | Default | Override | +| -------------------------------- | -------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | +| Config | `~/.openclaw/openclaw.json` | `OPENCLAW_CONFIG_PATH` | +| State dir | `~/.openclaw` | `OPENCLAW_STATE_DIR` | +| Default agent's workspace | `~/.openclaw/workspace` (or `workspace-` when `OPENCLAW_PROFILE` is set) | `agents.list[].workspace`, then `agents.defaults.workspace`, or `OPENCLAW_WORKSPACE_DIR` | +| Other agents' workspace | `/workspace-` (or `/` when set) | `agents.list[].workspace` | +| Agent dir | `~/.openclaw/agents//agent` | `agents.list[].agentDir` | +| Sessions and transcripts | `~/.openclaw/agents//agent/openclaw-agent.sqlite` | — | +| Legacy/archive session artifacts | `~/.openclaw/agents//sessions` | — | ### Single-agent mode (default) diff --git a/docs/concepts/session-tool.md b/docs/concepts/session-tool.md index c093e9da1b9d..fc8e2acd397e 100644 --- a/docs/concepts/session-tool.md +++ b/docs/concepts/session-tool.md @@ -56,7 +56,7 @@ The returned view is intentionally bounded and safety-filtered: Both tools accept either a **session key** (like `"main"`) or a **session ID** from a previous list call. -If you need the exact byte-for-byte transcript, inspect the transcript file on disk instead of treating `sessions_history` as a raw dump. +If you need the exact raw transcript, inspect the scoped SQLite transcript rows instead of treating `sessions_history` as an unfiltered dump. ## Sending cross-session messages diff --git a/docs/concepts/session.md b/docs/concepts/session.md index 7d6c6ea1d5b5..42028678cb4c 100644 --- a/docs/concepts/session.md +++ b/docs/concepts/session.md @@ -113,20 +113,27 @@ an idle-mode default when no `session.reset`/`resetByType` block is set. ## Where state lives -- **Store:** `~/.openclaw/agents//sessions/sessions.json` -- **Transcripts:** `~/.openclaw/agents//sessions/.jsonl` +- **Runtime session rows:** `~/.openclaw/agents//agent/openclaw-agent.sqlite` +- **Archived transcript files:** `~/.openclaw/agents//sessions/` +- **Legacy row migration source:** `~/.openclaw/agents//sessions/sessions.json` -`sessions.json` keeps separate lifecycle timestamps: +The session rows in the per-agent SQLite database keep separate lifecycle +timestamps: - `sessionStartedAt`: when the current `sessionId` began; daily reset uses this. - `lastInteractionAt`: last user/channel interaction that extends idle lifetime. - `updatedAt`: last store-row mutation; useful for listing and pruning, but not authoritative for daily/idle reset freshness. -Older rows without `sessionStartedAt` are resolved from the transcript JSONL -session header when available. If an older row also lacks `lastInteractionAt`, -idle freshness falls back to that session start time, not to later bookkeeping -writes. +During migration from older installs, gateway startup and `openclaw doctor +--fix` import legacy `sessions.json` rows and hot transcript JSONL history into +SQLite automatically. Rows without `sessionStartedAt` are resolved from the +legacy transcript JSONL session header when available. If an older row also +lacks `lastInteractionAt`, idle freshness falls back to that session start time, +not to later bookkeeping writes. Use `openclaw doctor --session-sqlite inspect +--session-sqlite-all-agents` and the [Doctor migration +sequence](/cli/doctor#session-sqlite-migration) when you want explicit +inspection or validation evidence. ## Session maintenance diff --git a/docs/docs_map.md b/docs/docs_map.md index c4fd320ccf49..9cc1135d8021 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -1491,6 +1491,8 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Structured health checks - H2: Check selection - H2: Post-upgrade mode + - H2: Session SQLite migration + - H3: Downgrading After Session SQLite Migration - H2: Notes - H2: macOS: launchctl env overrides - H2: Related @@ -4931,6 +4933,16 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Milestones - H2: Open questions +## plan/path3-sqlite-session-artifact-family.md + +- Route: /plan/path3-sqlite-session-artifact-family +- Headings: + - H1: Path 3 SQLite Session Artifact Family + - H2: Authoritative family + - H2: Non-family artifacts after the flip + - H2: Patch points + - H2: Focused tests + ## plan/ui-channels.md - Route: /plan/ui-channels @@ -8442,6 +8454,19 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Design contract - H2: Subpath exports +## reference/path3-live-sqlite-e2e-harness.md + +- Route: /reference/path3-live-sqlite-e2e-harness +- Headings: + - H2: Command shape + - H2: Isolated built-CLI proof + - H2: Preflight + - H2: Agent-driven scenario + - H2: Per-step assertions + - H2: Evidence artifact + - H2: Safety rules + - H2: Passing result + ## reference/prompt-caching.md - Route: /reference/prompt-caching @@ -8531,11 +8556,12 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Two persistence layers - H2: On-disk locations - H2: Store maintenance and disk controls + - H3: Downgrading After The SQLite Flip - H2: Cron sessions and run logs - H2: Session keys (sessionKey) - H2: Session ids (sessionId) - - H2: Session store schema (sessions.json) - - H2: Transcript structure (.jsonl) + - H2: Session store schema + - H2: Transcript event structure - H2: Context windows vs tracked tokens - H2: Compaction: what it is - H3: Chunk boundaries and tool pairing @@ -9935,7 +9961,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Access - H2: What gets recorded - H2: Bundle files - - H2: Capture location + - H2: Capture storage - H2: Disable capture - H2: Tune flush timeout - H2: Privacy and limits diff --git a/docs/gateway/config-agents.md b/docs/gateway/config-agents.md index da8ee4a2c97c..68dc824f94bc 100644 --- a/docs/gateway/config-agents.md +++ b/docs/gateway/config-agents.md @@ -661,10 +661,10 @@ Periodic heartbeat runs. - `postIndexSync`: post-compaction session-memory reindex mode. Default: `"async"`. Use `"await"` for strongest freshness, `"async"` for lower compaction latency, or `"off"` only when session-memory sync is handled elsewhere. - `postCompactionSections`: optional AGENTS.md H2/H3 section names to re-inject after compaction. Reinjection is disabled when unset or set to `[]`. Explicitly setting `["Session Startup", "Red Lines"]` enables that pair and preserves the legacy `Every Session`/`Safety` fallback. Enable this only when the extra context is worth the risk of duplicating project guidance already captured in the compaction summary. - `model`: optional `provider/model-id` or bare alias from `agents.defaults.models` for compaction summarization only. Bare aliases resolve before dispatch; configured literal model IDs retain precedence on collisions. Use this when the main session should keep one model but compaction summaries should run on another; when unset, compaction uses the session's primary model. -- `truncateAfterCompaction`: rotates the active session JSONL after compaction so future turns load only the summary and unsummarized tail, while the previous full transcript remains archived. Prevents unbounded active transcript growth in long-running sessions. Default: `false`. -- `maxActiveTranscriptBytes`: optional byte threshold (`number` or strings like `"20mb"`) that triggers normal local compaction before a run when the active JSONL grows past the threshold. Requires `truncateAfterCompaction` so successful compaction can rotate to a smaller successor transcript. Disabled when unset or `0`. +- `truncateAfterCompaction`: rotates the active session transcript after compaction so future turns load only the summary and unsummarized tail, while the previous full transcript remains archived. Prevents unbounded active transcript growth in long-running sessions. Default: `false`. +- `maxActiveTranscriptBytes`: optional byte threshold (`number` or strings like `"20mb"`) that triggers normal local compaction before a run when transcript history grows past the threshold. Requires `truncateAfterCompaction` so successful compaction can rotate to a smaller successor transcript. Disabled when unset or `0`. - `notifyUser`: when `true`, sends brief context-maintenance notices to the user: when compaction starts and completes (for example, "Compacting context..." and "Compaction complete"), and when a pre-compaction memory flush is exhausted so the reply continues in a degraded state (for example, "Memory maintenance temporarily failed; continuing your reply."). Disabled by default to keep these notices silent. -- `memoryFlush`: silent agentic turn before auto-compaction to store durable memories. Set `model` to an exact provider/model such as `ollama/qwen3:8b` when this housekeeping turn should stay on a local model; the override does not inherit the active session fallback chain. `forceFlushTranscriptBytes` forces the flush when transcript file size reaches the threshold even if token counters are stale. Skipped when workspace is read-only. +- `memoryFlush`: silent agentic turn before auto-compaction to store durable memories. Set `model` to an exact provider/model such as `ollama/qwen3:8b` when this housekeeping turn should stay on a local model; the override does not inherit the active session fallback chain. `forceFlushTranscriptBytes` forces the flush when transcript size reaches the threshold even if token counters are stale. Skipped when workspace is read-only. ### `agents.defaults.runRetries` @@ -1325,7 +1325,7 @@ See [Multi-Agent Sandbox & Tools](/tools/multi-agent-sandbox-tools) for preceden - **`maintenance`**: session-store cleanup + retention controls. - `mode`: `enforce` applies cleanup and is the default; `warn` emits warnings only. - `pruneAfter`: age cutoff for stale entries (default `30d`). - - `maxEntries`: maximum number of entries in `sessions.json` (default `500`). Runtime writes batch cleanup with a small high-water buffer for production-sized caps; `openclaw sessions cleanup --enforce` applies the cap immediately. + - `maxEntries`: maximum number of SQLite session entries (default `500`). Runtime writes batch cleanup with a small high-water buffer for production-sized caps; `openclaw sessions cleanup --enforce` applies the cap immediately. - Short-lived gateway model-run probe sessions use fixed `24h` retention, but cleanup is pressure-gated: it only removes stale strict model-run probe rows when session-entry maintenance/cap pressure is reached. Only strict explicit probe keys matching `agent:*:explicit:model-run-` are eligible; normal direct, group, thread, cron, hook, heartbeat, ACP, and sub-agent sessions do not inherit this 24h retention. When model-run cleanup runs, it runs before the broader `pruneAfter` stale-entry cleanup and `maxEntries` cap. - `rotateBytes`: deprecated and ignored; `openclaw doctor --fix` removes it from older configs. - `resetArchiveRetention`: retention for `*.reset.` transcript archives. Defaults to `pruneAfter`; set `false` to disable. diff --git a/docs/gateway/configuration-reference.md b/docs/gateway/configuration-reference.md index a21e5a23be08..888146210087 100644 --- a/docs/gateway/configuration-reference.md +++ b/docs/gateway/configuration-reference.md @@ -1504,7 +1504,7 @@ Current builds no longer include the TCP bridge. Nodes connect over the Gateway } ``` -- `sessionRetention`: how long to keep completed isolated cron run sessions before pruning from `sessions.json`. Also controls cleanup of archived deleted cron transcripts. Default: `24h`; set `false` to disable. +- `sessionRetention`: how long to keep completed isolated cron run sessions before pruning SQLite session rows. Also controls cleanup of archived deleted cron transcripts. Default: `24h`; set `false` to disable. - `runLog.maxBytes`: accepted for compatibility with older file-backed cron run logs. Default: `2_000_000` bytes. - `runLog.keepLines`: newest SQLite run-history rows retained per job. Default: `2000`. - `webhookToken`: bearer token used for cron webhook POST delivery (`delivery.mode = "webhook"`), if omitted no auth header is sent. diff --git a/docs/gateway/configuration.md b/docs/gateway/configuration.md index 2993e6de7a5c..8a704fe6b2ec 100644 --- a/docs/gateway/configuration.md +++ b/docs/gateway/configuration.md @@ -430,7 +430,7 @@ candidate contains a redacted secret placeholder such as `***` or `[redacted]`. } ``` - - `sessionRetention`: prune completed isolated run sessions from `sessions.json` (default `24h`; set `false` to disable). + - `sessionRetention`: prune completed isolated run sessions from SQLite session rows (default `24h`; set `false` to disable). - `runLog`: prune retained cron run-history rows per job. History is stored in SQLite; `maxBytes` (default `2_000_000`) is retained for compatibility with older file-backed run logs, `keepLines` defaults to `2000`. - See [Cron jobs](/automation/cron-jobs) for feature overview and CLI examples. diff --git a/docs/gateway/health.md b/docs/gateway/health.md index 0c7dcdeb3220..528574ab3e14 100644 --- a/docs/gateway/health.md +++ b/docs/gateway/health.md @@ -29,7 +29,7 @@ health commands above for live connectivity checks. ## Deep diagnostics - Creds on disk: `ls -l ~/.openclaw/credentials/whatsapp//creds.json` (mtime should be recent). -- Session store: `ls -l ~/.openclaw/agents//sessions/sessions.json` (path can be overridden in config). Count and recent recipients are surfaced via `status`. +- Session store: `ls -l ~/.openclaw/agents//agent/openclaw-agent.sqlite`. Count and recent recipients are surfaced via `status`. - Relink flow: `openclaw channels logout && openclaw channels login --verbose` when status codes 409-515 or `loggedOut` appear in logs. The QR login flow auto-restarts once for status 515 after pairing. - Diagnostics are enabled by default (`diagnostics.enabled: false` disables them). Memory events record RSS/heap byte counts and threshold/growth pressure; critical memory pressure logs through the gateway logger and, when `diagnostics.memoryPressureSnapshot: true` is set, also writes a pre-OOM stability bundle (V8 heap stats, Linux cgroup counters when available, active resource counts, largest session/transcript files by redacted relative path). Liveness warnings record event-loop delay/utilization, CPU-core ratio, and active/waiting/queued session counts when the process is running but saturated. Oversized-payload events record what was rejected/truncated/chunked plus sizes and limits, never message text, attachment contents, webhook bodies, raw request/response bodies, tokens, cookies, or secret values. - The same heartbeat drives the bounded stability recorder: `openclaw gateway stability` (or the `diagnostics.stability` Gateway RPC). Fatal Gateway exits, shutdown timeouts, restart startup failures, and (when `diagnostics.memoryPressureSnapshot: true`) critical memory pressure persist the latest snapshot under `~/.openclaw/logs/stability/`. Inspect the newest bundle with `openclaw gateway stability --bundle latest`. diff --git a/docs/gateway/security/index.md b/docs/gateway/security/index.md index 4f98a428a25a..0250c9e052e3 100644 --- a/docs/gateway/security/index.md +++ b/docs/gateway/security/index.md @@ -716,18 +716,19 @@ The Control UI needs a secure context (HTTPS or localhost) to generate device id Assume anything under `~/.openclaw/` (or `$OPENCLAW_STATE_DIR/`) may contain secrets or private data: -| Path | Contents | -| ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `openclaw.json` | Config may include tokens (gateway, remote gateway), provider settings, and allowlists. | -| `credentials/**` | Channel credentials (for example WhatsApp creds), pairing allowlists, legacy OAuth imports. | -| `agents//agent/auth-profiles.json` | API keys, token profiles, OAuth tokens, optional `keyRef`/`tokenRef`. | -| `agents//agent/codex-home/**` | Per-agent Codex app-server account, config, skills, plugins, native thread state, diagnostics (default). | -| `$CODEX_HOME/**` or `~/.codex/**` | Native Codex runtime state. The ordinary harness accesses it only with explicit `plugins.entries.codex.config.appServer.homeScope: "user"`. The separate supervision connection accesses it when its resolved home scope is `"user"`, which is the default for stdio or Unix when unset. Contains the native Codex account, config, plugins, and thread store. Supervision lists source metadata and keeps a continued Chat's canonical native branch and later turns on that connection; branching copies bounded persisted user and assistant history into an authenticated, model-locked OpenClaw Chat. Enable only for an owner-controlled Gateway. See [Codex harness](/plugins/codex-harness#share-threads-with-codex-desktop-and-cli) and [Codex supervision](/plugins/codex-supervision). | -| `secrets.json` (optional) | File-backed secret payload used by `file` SecretRef providers (`secrets.providers`). | -| `agents//agent/auth.json` | Legacy compatibility file; static `api_key` entries are scrubbed when discovered. | -| `agents//sessions/**` | Session transcripts (`*.jsonl`) + routing metadata (`sessions.json`) that can contain private messages and tool output. | -| bundled plugin packages | Installed plugins (plus their `node_modules/`). | -| `sandboxes/**` | Tool sandbox workspaces; can accumulate copies of files read/written inside the sandbox. | +| Path | Contents | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `openclaw.json` | Config may include tokens (gateway, remote gateway), provider settings, and allowlists. | +| `credentials/**` | Channel credentials (for example WhatsApp creds), pairing allowlists, legacy OAuth imports. | +| `agents//agent/auth-profiles.json` | API keys, token profiles, OAuth tokens, optional `keyRef`/`tokenRef`. | +| `agents//agent/codex-home/**` | Per-agent Codex app-server account, config, skills, plugins, native thread state, diagnostics (default). | +| `$CODEX_HOME/**` or `~/.codex/**` | Native Codex runtime state. The ordinary harness accesses it only with explicit `plugins.entries.codex.config.appServer.homeScope: "user"`. The separate supervision connection accesses it when its resolved home scope is `"user"`, which is the default for stdio or Unix when unset. Contains the native Codex account, config, plugins, and thread store. Supervision lists source metadata and keeps a continued Chat's canonical native branch and later turns on that connection; branching copies bounded persisted user and assistant history into an authenticated, model-locked OpenClaw Chat. Enable only for an owner-controlled Gateway. See [Codex harness](/plugins/codex-harness#share-threads-with-codex-desktop-and-cli) and [Codex supervision](/plugins/codex-supervision). | +| `secrets.json` (optional) | File-backed secret payload used by `file` SecretRef providers (`secrets.providers`). | +| `agents//agent/auth.json` | Legacy compatibility file; static `api_key` entries are scrubbed when discovered. | +| `agents//agent/openclaw-agent.sqlite` | Per-agent runtime state, including session rows and transcripts that can contain private messages and tool output. | +| `agents//sessions/**` | Legacy session migration sources and archives that can contain private messages and tool output. | +| bundled plugin packages | Installed plugins (plus their `node_modules/`). | +| `sandboxes/**` | Tool sandbox workspaces; can accumulate copies of files read/written inside the sandbox. | ### Credential storage map diff --git a/docs/help/faq-first-run.md b/docs/help/faq-first-run.md index 89a9667a0752..18131a67ecc5 100644 --- a/docs/help/faq-first-run.md +++ b/docs/help/faq-first-run.md @@ -225,7 +225,7 @@ and troubleshooting see the main [FAQ](/help/faq). **Important:** if you only commit/push your workspace to GitHub, you back up **memory + bootstrap files**, but not session history or auth. Those live under - `~/.openclaw/` (for example `~/.openclaw/agents//sessions/`). + `~/.openclaw/` (for example `~/.openclaw/agents//agent/openclaw-agent.sqlite`). Related: [Migrating](/install/migrating), [Where things live on disk](/help/faq#where-things-live-on-disk), [Agent workspace](/concepts/agent-workspace), [Doctor](/gateway/doctor), diff --git a/docs/help/faq.md b/docs/help/faq.md index 01b2377bf9a2..104ae7225f66 100644 --- a/docs/help/faq.md +++ b/docs/help/faq.md @@ -454,7 +454,7 @@ First-run Q&A - install, onboard, auth routes, subscriptions, initial failures - Everything lives under `$OPENCLAW_STATE_DIR` (default: `~/.openclaw`): - | Path | Purpose | + | Path | Purpose | | ------------------------------------------------------------------ | ------------------------------------------------------------------ | | `$OPENCLAW_STATE_DIR/openclaw.json` | Main config (JSON5) | | `$OPENCLAW_STATE_DIR/credentials/oauth.json` | Legacy OAuth import (copied into auth profiles on first use) | @@ -462,9 +462,9 @@ First-run Q&A - install, onboard, auth routes, subscriptions, initial failures - | `$OPENCLAW_STATE_DIR/secrets.json` | Optional file-backed secret payload for `file` SecretRef providers | | `$OPENCLAW_STATE_DIR/agents//agent/auth.json` | Legacy compatibility file (static `api_key` entries scrubbed) | | `$OPENCLAW_STATE_DIR/credentials/` | Provider state (for example `whatsapp//creds.json`) | - | `$OPENCLAW_STATE_DIR/agents/` | Per-agent state (agentDir + sessions) | - | `$OPENCLAW_STATE_DIR/agents//sessions/` | Conversation history and state (per agent) | - | `$OPENCLAW_STATE_DIR/agents//sessions/sessions.json` | Session metadata (per agent) | + | `$OPENCLAW_STATE_DIR/agents/` | Per-agent state (agentDir + legacy/archive session artifacts) | + | `$OPENCLAW_STATE_DIR/agents//agent/openclaw-agent.sqlite` | Per-agent SQLite state, including session rows and transcripts | + | `$OPENCLAW_STATE_DIR/agents//sessions/` | Legacy session migration sources and archive/support artifacts | Legacy single-agent path `~/.openclaw/agent/*` is migrated by `openclaw doctor`. @@ -1111,11 +1111,11 @@ First-run Q&A - install, onboard, auth routes, subscriptions, initial failures - No hard limits - dozens or even hundreds are fine, but watch: - - **Disk growth**: sessions and transcripts live under `~/.openclaw/agents//sessions/`. + - **Disk growth**: active sessions and transcripts live in the per-agent SQLite database; legacy/archive artifacts can still accumulate under `~/.openclaw/agents//sessions/`. - **Token cost**: more agents means more concurrent model usage. - **Ops overhead**: per-agent auth profiles, workspaces, and channel routing. - Keep one **active** workspace per agent (`agents.defaults.workspace`), prune old sessions if disk grows, and use `openclaw doctor` to spot stray workspaces and profile mismatches. + Keep one **active** workspace per agent (`agents.defaults.workspace`), prune old sessions with `openclaw sessions cleanup` if disk grows (do not edit active SQLite state by hand), and use `openclaw doctor` to spot stray workspaces and profile mismatches. diff --git a/docs/install/docker.md b/docs/install/docker.md index 01de13593158..b5193b51260f 100644 --- a/docs/install/docker.md +++ b/docs/install/docker.md @@ -409,7 +409,7 @@ Installed downloadable plugins store package state under the mounted OpenClaw ho For full VM persistence details, see [Docker VM Runtime - What persists where](/install/docker-vm-runtime#what-persists-where). -**Disk growth hotspots:** `media/`, session JSONL files, the shared SQLite state database, installed plugin package roots, and rolling file logs under `/tmp/openclaw/`. +**Disk growth hotspots:** `media/`, per-agent SQLite databases, legacy session JSONL transcripts, the shared SQLite state database, installed plugin package roots, and rolling file logs under `/tmp/openclaw/`. ### Shell helpers (optional) diff --git a/docs/openclaw-agent-runtime.md b/docs/openclaw-agent-runtime.md index 8b9b6d5aad03..2bc60a02ebce 100644 --- a/docs/openclaw-agent-runtime.md +++ b/docs/openclaw-agent-runtime.md @@ -55,13 +55,13 @@ State lives in the OpenClaw state directory: `~/.openclaw` by default, or | `state/openclaw.sqlite` | Shared runtime state database | | `agents//agent/openclaw-agent.sqlite` | Per-agent model auth profiles (API keys + OAuth) and runtime state | | `credentials/` | Provider/channel credentials outside the auth profile store | -| `agents//sessions/` | Session transcripts plus the `sessions.json` index | +| `agents//sessions/` | Transcript history and legacy session migration sources | | `sessions/` | Legacy single-agent session store (old installs only) | | `workspace/` | Default agent workspace (extra agents use `workspace-`) | Delete those paths for a full reset. Narrower resets: -- Sessions only: delete `agents//sessions/` for that agent. +- Sessions only: do not delete `agents//agent/openclaw-agent.sqlite`; session rows live there alongside other per-agent state. Use `/new` or `/reset` to start a fresh session for one chat, and `openclaw sessions cleanup` for session maintenance. - Keep auth: leave `agents//agent/openclaw-agent.sqlite` and `credentials/` in place. Legacy `auth-profiles.json` files are no longer read at runtime; diff --git a/docs/plan/path3-sqlite-session-artifact-family.md b/docs/plan/path3-sqlite-session-artifact-family.md new file mode 100644 index 000000000000..0aee9fda42f3 --- /dev/null +++ b/docs/plan/path3-sqlite-session-artifact-family.md @@ -0,0 +1,105 @@ +--- +summary: "Path 3 plan for archiving all SQLite transcript artifacts that belong to a session" +read_when: + - You are implementing clawdbot-d63.2 / clawdbot-04b + - You are touching SQLite session retention, reset, delete, or agent-deletion archival + - You need to distinguish SQLite-era artifact families from legacy JSONL sidecars +title: "Path 3 SQLite session artifact family" +--- + +# Path 3 SQLite Session Artifact Family + +This note scopes `clawdbot-d63.2` while `clawdbot-d63.1` owns the overlapping +reset/delete archive helper in `src/config/sessions/session-accessor.sqlite.ts`. +The implementation file was dirty during this pass, so this artifact records +the exact contract and patch points without racing the sibling worker. + +## Authoritative family + +After the SQLite flip, active session transcripts are SQLite rows. A session's +archive family is: + +- The `transcript_events`, `transcript_event_identities`, and `sessions` rows + for the entry's current `sessionId`. +- The same SQLite transcript row set for every `sessionId` referenced by + `entry.compactionCheckpoints[*].preCompaction.sessionId`. +- The same SQLite transcript row set for every `sessionId` referenced by + `entry.compactionCheckpoints[*].postCompaction.sessionId`. +- The same SQLite transcript row set for every `sessionId` in + `entry.usageFamilySessionIds`. + +Archive only rows that are no longer referenced by any remaining +`session_entries` row or by any remaining entry's compaction or usage-family +metadata. This preserves checkpoint branch/restore and usage rollup state until +the final live reference is gone. + +## Non-family artifacts after the flip + +Generated topic transcript file variants and trajectory sidecars are not active +SQLite runtime state. They are legacy file artifacts: + +- Topic variants such as `-topic-.jsonl` only exist for the + file-backed transcript format. SQLite uses the canonical session id plus + `session_routes`/entry delivery metadata instead of per-topic JSONL files. +- Trajectory sidecars such as `.trajectory.jsonl` and `.trajectory-path.json` + are named from real JSONL `sessionFile` paths. SQLite `sessionFile` values are + `sqlite:::` markers and do not name sidecar + files. +- Archive-tier readers must keep reading legacy archived JSONL files, but + runtime retention must not scan active sessions directories or reopen JSONL + transcript files for SQLite sessions. + +Doctor import remains the migration owner for legacy primary JSONL files and +their adjacent trajectory sidecars. Runtime SQLite retention should not add a +second importer or file fallback. + +## Patch points + +Extend the SQLite archive helper introduced by `clawdbot-d63.1` rather than +adding a parallel path. + +1. Add a local collector near `deleteSqliteSessionStateIfUnreferenced`: + - `collectSqliteSessionArtifactFamily(entry: SessionEntry): Set` + - Include `entry.sessionId`, checkpoint pre/post session ids, and + `usageFamilySessionIds`. + - Filter empty strings and dedupe deterministically. + +2. Add a reference collector for the post-removal store: + - `readReferencedSqliteSessionArtifactFamilyIds(database): Set` + - Iterate current `session_entries`, parse each `entry_json`, and collect + the same family ids from every surviving entry. + +3. Change the reset/delete/maintenance callers that currently archive one + removed `sessionId` to pass the removed entry's full family. + +4. For each family id, archive the SQLite transcript rows with the caller's + reason (`reset` or `deleted`), then delete the `sessions` row only when the + family id is absent from the post-removal reference set. + +5. Keep transcript event deletion centralized through the existing SQLite + session-row cleanup path. Do not add active JSONL reads. + +## Focused tests + +Add SQLite-only tests to `src/config/sessions/session-accessor.conformance.test.ts` +or the sibling lifecycle test after `clawdbot-d63.1` commits: + +- Deleting an entry with a pre-compaction transcript archives both the current + session and the pre-compaction session, then removes both SQLite row sets. +- Deleting one of two entries that share a compaction pre-session archives + nothing for the shared pre-session until the final referencing entry is + removed. +- Deleting an entry with `usageFamilySessionIds` archives predecessor SQLite + transcript rows when no other entry references that usage family. +- A topic-shaped session key with a SQLite marker does not cause any generated + topic JSONL read or sidecar lookup. + +The focused proof should use: + +```bash +node scripts/run-vitest.mjs src/config/sessions/session-accessor.conformance.test.ts +``` + +If the final tests live in `store.session-lifecycle-mutation.test.ts`, run that +file explicitly with the same wrapper. Broad `pnpm` gates should stay on +Crabbox/Testbox for this Codex worktree. diff --git a/docs/plugins/sdk-migration.md b/docs/plugins/sdk-migration.md index 605077f490a6..93e326d3f5bd 100644 --- a/docs/plugins/sdk-migration.md +++ b/docs/plugins/sdk-migration.md @@ -459,7 +459,7 @@ SDK. | `plugin-sdk/reply-history` | Reply-history helpers | `createChannelHistoryWindow`; deprecated map-helper compatibility exports such as `buildPendingHistoryContextFromMap`, `recordPendingHistoryEntry`, and `clearHistoryEntriesIfEnabled` | | `plugin-sdk/reply-reference` | Reply reference planning | `createReplyReferencePlanner` | | `plugin-sdk/reply-chunking` | Reply chunk helpers | Text/markdown chunking helpers | - | `plugin-sdk/session-store-runtime` | Session store helpers | Store path + updated-at helpers | + | `plugin-sdk/session-store-runtime` | Session store helpers | Scoped session row helpers, store path helpers, and updated-at reads | | `plugin-sdk/state-paths` | State path helpers | State and OAuth dir helpers | | `plugin-sdk/routing` | Routing/session-key helpers | `resolveAgentRoute`, `buildAgentSessionKey`, `resolveDefaultAgentBoundAccountId`, session-key normalization helpers | | `plugin-sdk/status-helpers` | Channel status helpers | Channel/account status summary builders, runtime-state defaults, issue metadata helpers | @@ -803,6 +803,34 @@ major release. Every entry maps the old API to its canonical replacement. + + The SQLite session/transcript flip removes plugin-facing APIs that exposed + active `sessions.json` stores, JSONL transcript paths, or lists of session + files. Runtime plugins should use session identity and SDK runtime helpers + instead of resolving or mutating active files. + + | Removed surface | Replacement | + | ---------------- | ----------- | + | `loadSessionStore(...)`, `saveSessionStore(...)`, `updateSessionStore(...)` | Gateway-owned session runtime APIs; plugin code should request session state through documented runtime/context helpers instead of reading the active store file. | + | `resolveSessionFilePath(...)`, `resolveSessionTranscriptPathInDir(...)`, `resolveAndPersistSessionFile(...)` | Session identity (`sessionKey`, `sessionId`, and SDK runtime target helpers) plus Gateway methods that operate on the current session. | + | `readLatestAssistantTextFromSessionTranscript(...)` | Identity-backed transcript readers exposed by the current runtime context, or Gateway history/session methods when the plugin is outside the transcript owner path. | + | `SessionTranscriptUpdate.sessionFile` | `SessionTranscriptUpdate.target` with `agentId`, `sessionKey`, and `sessionId`. | + | Memory sync inputs such as `sessionFiles` | Identity-backed transcript/session sources provided by the host; do not crawl active JSONL files for live sessions. | + | Runtime options named `transcriptPath` or `sessionFile` for active sessions | `sessionTarget`/runtime target objects that carry storage-neutral session identity. | + + Legacy JSONL transcript files remain valid as import, archive, export, and + support artifacts. They are no longer the steady-state runtime contract for + active sessions. + + `openclaw plugins inspect --all --runtime` reports non-bundled plugins whose + load errors or diagnostics still reference these removed file APIs. The + `@openclaw/plugin-inspector` advisory sweep must use version `0.3.17` or + newer so external package scans also flag whole-store session helpers, + session file-path helpers, legacy transcript file targets, and low-level + transcript helpers before release. + + + **Old**: `runtime.tasks.flow` (singular) returned a live task-flow accessor. diff --git a/docs/plugins/sdk-runtime.md b/docs/plugins/sdk-runtime.md index c1be0af5d317..b6dc51cdac61 100644 --- a/docs/plugins/sdk-runtime.md +++ b/docs/plugins/sdk-runtime.md @@ -190,11 +190,17 @@ two-party event loops that do not go through the shared inbound reply runner. Use `runWithWorkAdmission(...)` when a plugin starts work on a persisted session. The callback rejects archived or concurrently replaced sessions, keeps archive/reset/delete mutations coordinated through completion, and receives an `AbortSignal` that must be forwarded to the agent run. A harness may explicitly name trusted execution delegates through its experimental `delegatedExecutionPluginIds` registration field. Delegates can admit and run only an exact existing model-locked session; all session mutations remain restricted to the harness owner. See [Agent harness plugins](/plugins/sdk-agent-harness#delegated-execution). - For transcript reads and writes, import `openclaw/plugin-sdk/session-transcript-runtime` and use `resolveSessionTranscriptIdentity(...)`, `resolveSessionTranscriptTarget(...)`, `readSessionTranscriptEvents(...)`, `appendSessionTranscriptMessageByIdentity(...)`, `publishSessionTranscriptUpdateByIdentity(...)`, or `withSessionTranscriptWriteLock(...)` with `{ agentId, sessionKey, sessionId }`. These APIs let plugins identify a transcript, read its events, append messages, publish updates, and run related operations under the same transcript write lock. Passing `sessionFile`, using `resolveSessionTranscriptLegacyFileTarget(...)`, or importing low-level `appendSessionTranscriptMessage(...)` / `emitSessionTranscriptUpdate(...)` from `openclaw/plugin-sdk/agent-harness-runtime` is deprecated; those paths exist only for legacy code that already receives an active transcript artifact. + Maintenance and repair plugins may use `deleteSessionEntry(...)` for one scoped session entry, `cleanupSessionLifecycleArtifacts(...)` for lifecycle-owned scratch sessions, and `resolveSessionStoreBackupPaths(...)` before mutating a store. These helpers are narrow repair/lifecycle surfaces, not a general store deletion API. `resolveStorePath(...)` and `updateSessionStoreEntry(...)` round out the session helpers: `resolveStorePath` resolves the session store path for a given scope, and `updateSessionStoreEntry({ storePath, sessionKey, update })` patches one entry directly by store path when the caller already knows it. - `loadSessionStore(...)`, `saveSessionStore(...)`, `updateSessionStore(...)`, and `resolveSessionFilePath(...)` are deprecated compatibility helpers for plugins that still intentionally depend on the legacy whole-store or transcript-file shape. New plugin code must not use those helpers, and existing callers should migrate to entry helpers and transcript identity helpers. + `loadTranscriptEventsSync(...)` is available for synchronous doctor and repair paths that cannot use the async transcript runtime. It returns raw `SessionStoreTranscriptEvent` records. Normal plugin runtime code should prefer `openclaw/plugin-sdk/session-transcript-runtime`. + + `formatSqliteSessionFileMarker(...)`, `parseSqliteSessionFileMarker(...)`, and `sqliteSessionFileMarkerMatchesSession(...)` are transitional helpers for code that still receives a legacy field named `sessionFile`. A parsed SQLite marker identifies a live SQLite transcript target; it is not a filesystem path. New APIs should carry typed session identity instead of marker strings. + + For transcript reads and writes, import `openclaw/plugin-sdk/session-transcript-runtime` and use `resolveSessionTranscriptIdentity(...)`, `resolveSessionTranscriptTarget(...)`, `readSessionTranscriptEvents(...)`, `readVisibleSessionTranscriptMessageEntries(...)`, `appendSessionTranscriptMessageByIdentity(...)`, `publishSessionTranscriptUpdateByIdentity(...)`, or `withSessionTranscriptWriteLock(...)` with `{ agentId, sessionKey, sessionId }`. These APIs let plugins identify a transcript, read raw events or visible branch-safe message entries, append messages, publish updates, and run related operations under the same transcript write lock without depending on active transcript file paths. `readVisibleSessionTranscriptMessageEntries(...)` returns ordered read metadata; its `seq` field is not a resumable cursor. + + The legacy whole-store and active transcript file helpers are no longer exported from the plugin SDK. Use the scoped entry helpers for session metadata and the transcript identity helpers for active transcript operations. Archive/support workflows that need file artifacts should use their dedicated archive surfaces instead of active session runtime APIs. diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index 7e081ee3157f..91374aea905e 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -272,8 +272,8 @@ usage endpoint failed or returned no usable usage data. | `plugin-sdk/reply-history` | Shared short-window reply-history helpers. New message-turn code should use `createChannelHistoryWindow`; lower-level map helpers remain deprecated compatibility exports only | | `plugin-sdk/reply-reference` | `createReplyReferencePlanner` | | `plugin-sdk/reply-chunking` | Narrow text/markdown chunking helpers | - | `plugin-sdk/session-store-runtime` | Session workflow helpers (`getSessionEntry`, `listSessionEntries`, `patchSessionEntry`, `upsertSessionEntry`), bounded recent user/assistant transcript text reads by session identity, legacy session store path/session-key helpers, updated-at reads, and transition-only whole-store/file-path compatibility helpers, without broad config writes/maintenance imports | - | `plugin-sdk/session-transcript-runtime` | Transcript identity, scoped target/read/write helpers, update publishing, write locks, and transcript memory hit keys | + | `plugin-sdk/session-store-runtime` | Session workflow helpers (`getSessionEntry`, `listSessionEntries`, `patchSessionEntry`, `upsertSessionEntry`), repair/lifecycle helpers (`deleteSessionEntry`, `cleanupSessionLifecycleArtifacts`, `resolveSessionStoreBackupPaths`), marker helpers for transitional `sessionFile` values, bounded recent user/assistant transcript text reads by session identity, session store path/session-key helpers, and updated-at reads, without broad config writes/maintenance imports | + | `plugin-sdk/session-transcript-runtime` | Transcript identity, scoped target/read/write helpers, visible message-entry projection, update publishing, write locks, and transcript memory hit keys | | `plugin-sdk/sqlite-runtime` | Focused SQLite agent-schema, path, and transaction helpers for first-party runtime, without database lifecycle controls | | `plugin-sdk/cron-store-runtime` | Cron store path/load/save helpers | | `plugin-sdk/state-paths` | State/OAuth dir path helpers | diff --git a/docs/refactor/database-first.md b/docs/refactor/database-first.md index add9b78a7d64..958e5d54dfa4 100644 --- a/docs/refactor/database-first.md +++ b/docs/refactor/database-first.md @@ -264,7 +264,8 @@ file world: status payload field. Runtime and bridge tests no longer contain the `storePath` contract name; doctor/migration code owns that legacy vocabulary. - Session writes no longer pass through the old in-process `store-writer.ts` - queue. SQLite patch writes use conflict detection and bounded retry instead. + queue. SQLite patch writes prepare outside the transaction, then use a short + synchronous validate/apply transaction with explicit conflict detection. - Legacy path discovery still has valid migration uses, but runtime code should stop treating `sessions.json` and transcript JSONL files as possible write targets. @@ -582,10 +583,10 @@ Completed consolidation/deletion highlights: - Channel session runtime types now expose `{agentId, sessionKey}` for updated-at reads, inbound metadata, and last-route updates. The old `saveSessionStore(storePath, store)` compatibility type is gone. -- Plugin runtime, extension API, and `config/sessions` barrel surfaces now steer - plugin code to SQLite-backed session row helpers. Root library compatibility - exports (`loadSessionStore`, `saveSessionStore`, `resolveStorePath`) remain as - deprecated shims for existing consumers. The old +- Plugin runtime, extension API, and plugin SDK session surfaces now expose + SQLite-backed session row helpers instead of active-session whole-store/file + compatibility helpers. Root library compatibility exports remain available + only outside the plugin SDK for legacy internal and migration callers. The old `resolveLegacySessionStorePath` helper is gone; legacy `sessions.json` path construction is now local to migration and test fixtures. - `src/config/sessions/session-entries.sqlite.ts` now stores canonical session @@ -2114,8 +2115,9 @@ restore` validates before extraction, uses the verifier's normalized - One connection per thread/process is fine; do not share handles across workers. -- Use WAL, `foreign_keys=ON`, a 30s busy timeout, and short `BEGIN IMMEDIATE` - write transactions. +- Use WAL, `foreign_keys=ON`, a 5s busy timeout, and short `BEGIN IMMEDIATE` + write transactions. Do not layer synchronous lock retries above SQLite's + single busy wait. - Keep write transaction helpers synchronous unless/until an async transaction API adds explicit mutex/backpressure semantics. - Keep parent delivery writes small and transactional. diff --git a/docs/reference/path3-live-sqlite-e2e-harness.md b/docs/reference/path3-live-sqlite-e2e-harness.md new file mode 100644 index 000000000000..6fc01e79c853 --- /dev/null +++ b/docs/reference/path3-live-sqlite-e2e-harness.md @@ -0,0 +1,168 @@ +--- +summary: "Design for live Gateway proof of the Path 3 SQLite session/transcript flip" +read_when: + - You are proving the Path 3 SQLite storage flip against a live Gateway + - You need to distinguish expected legacy JSONL drift from runtime failures + - You are building or reviewing the agent-driven live SQLite E2E harness +title: "Path 3 live SQLite E2E harness" +--- + +The Path 3 live SQLite E2E harness proves the Gateway is using SQLite as the +canonical session and transcript store while legacy JSONL files remain +migration input or archive material. It is a maintainer proof harness, not a +normal user diagnostic. + +After a Gateway has processed post-migration traffic, legacy JSONL parity is no +longer a valid runtime health signal. A healthy migrated Gateway can have +SQLite transcript rows that differ from legacy JSONL counts because new turns +should advance SQLite only. The live harness must therefore measure Gateway +behavior, SQLite row movement, legacy-file quiescence, and log health at each +step. + +## Command shape + +The intended live command is: + +```bash +node scripts/path3-live-sqlite-e2e.mjs \ + --url http://127.0.0.1:18789 \ + --agent main \ + --session-key agent:main:path3-live-e2e: \ + --json +``` + +The command connects to an already running Gateway. It does not start, stop, +import, or re-run the migration unless an explicit migration mode is added +later. A CI or isolated-local variant can use +`test/helpers/openclaw-test-instance.ts`, but the live proof path should inspect +the actual operator Gateway and its real per-agent SQLite database. + +## Isolated built-CLI proof + +The built-CLI proof runner seeds an isolated legacy session store, starts the +rebuilt Gateway, and proves that startup imports hot legacy sessions into +SQLite before runtime reads begin. It must not run `openclaw doctor --fix` +before the first Gateway start, because that would prove the manual migration +path instead of the upgrade path users receive on first boot after the flip. + +After startup import, the isolated proof may run +`openclaw doctor --session-sqlite inspect` and +`openclaw doctor --session-sqlite validate` as diagnostic evidence. Those +doctor commands are not the migration driver for the startup-upgrade proof. +Separate doctor-import scenarios should seed legacy transcript files plus +trajectory sidecars and verify doctor archives those artifacts while SQLite +remains canonical. + +## Preflight + +Preflight collects a baseline and fails before sending a proof turn if the +Gateway is not usable: + +- `GET /health` and Gateway deep status must report a running, reachable + Gateway. +- The CLI and Gateway versions must match the branch being tested. +- The harness records a log cursor for the active Gateway file log. +- The harness records per-agent SQLite table counts for `sessions`, + `session_entries`, `transcript_events`, `transcript_event_identities`, and + `session_routes`. +- The harness records `mtime`, `size`, and existence for legacy + `sessions.json`, referenced JSONL files, and candidate proof-session JSONL + paths. +- `lsof -p ` must show SQLite DB/WAL/SHM handles and no hot + `.jsonl` or `sessions.json` handles. + +`openclaw doctor --session-sqlite validate` is informational only in live mode. +After post-flip traffic it may report expected drift against legacy files. The +harness should use doctor output for classification and migration inventory, +not as the runtime pass/fail oracle. + +## Agent-driven scenario + +The live scenario uses a dedicated proof session key and drives the Gateway +through public RPC paths wherever possible. One agent turn should be enough to +exercise ordinary persistence, but the full proof should cover the 3.1b seams +that previously required individual live checks: + +- Ordinary chat turn: create or reuse the proof session, send a real agent + prompt, wait for the final assistant result, and verify `chat.history` or + equivalent Gateway projection. +- Transcript identity: verify the same marker appears in Gateway history and in + SQLite transcript rows, including stable event identity rows when present. +- Session metadata accessors: read the proof session and selected existing live + sessions through Gateway/session accessors and compare them to SQLite rows. +- Session patch projection: apply a reversible model/session metadata change on + the proof session, then verify the projected row and Gateway response agree. +- Compaction checkpoint lifecycle: list, branch, and restore a checkpoint only + on the proof session or a synthetic fixture session created by the harness. +- Restart recovery: run the safe recovery marker path against a controlled proof + session or an isolated test instance; live mode may only run this step when + the target session set is explicit and reversible. +- Cleanup lifecycle: delete or reset the proof session, then verify SQLite + lifecycle rows and archived transcript state. + +Transport-specific seams that cannot be exercised safely on the live operator +Gateway, such as WhatsApp or voice-call ingress, should use owner-level runtime +probes against the same SQLite contract rather than fake external transport. + +## Per-step assertions + +Each step snapshots before and after state and writes a structured assertion +record: + +- SQLite row counts advance only where expected. +- Trajectory runtime rows advance for marker-backed proof sessions that record + runtime events. +- The proof session row has the expected `session_id`, status, timestamps, + metadata, and route rows. +- Gateway history/session projection matches the SQLite transcript tail. +- No proof-session JSONL file is created or modified. +- No proof-session `.trajectory.jsonl`, `.trajectory-path.json`, or + marker-derived `trajectory/.jsonl` sidecar is created. +- Existing legacy JSONL files and `sessions.json` remain unchanged unless the + step is explicitly an offline migration or archive operation. +- The Gateway process does not open `.jsonl` or `sessions.json` handles. +- Logs since the previous cursor contain no `ERROR`, `FATAL`, `SQLITE_`, + `no such column`, session-store unavailable, restart-recovery failure, or + transcript-reconcile warning unless the scenario explicitly allowlists it. + +The log scan is part of the pass/fail contract. A Gateway that answers health +checks but emits SQLite schema errors or repeated transcript reconcile failures +is not green for Path 3. + +## Evidence artifact + +The harness should write evidence under `.artifacts/path3-live-e2e//` +and keep it out of git: + +- `summary.json`: command args, Gateway version, result, failed assertion, and + artifact paths. +- `sqlite-before.json` and `sqlite-after.json`: row counts and selected proof + rows. +- `legacy-files.json`: legacy file existence, `mtime`, size, and whether each + file changed. +- `gateway-log-scan.json`: cursor range, matched log lines, and allowlist + decisions. +- `events.jsonl`: ordered per-step observations suitable for PR proof comments. + +The PR proof should summarize these artifacts instead of pasting full +transcripts or private message content. + +## Safety rules + +- Live mode must never re-import legacy JSONL while the Gateway is running. +- Live mode must not mutate non-proof sessions except for explicitly selected, + reversible repair probes. +- Any destructive or broad migration step requires a fresh backup of the + affected SQLite DB and legacy session directory. +- Backups should be scoped to the touched agent DB/session directory and reused + during one proof run to avoid unbounded disk growth. +- The cleanup step must leave no proof session, proof JSONL, or modified legacy + file behind unless the caller passes `--keep-artifacts`. + +## Passing result + +A passing live run means the Gateway accepted a real agent-driven session flow, +all observed canonical state was in SQLite, legacy runtime files stayed +quiescent, and log health stayed clean for the measured window. It does not mean +legacy JSONL parity remains clean after live traffic; live drift is expected +once SQLite is the canonical store. diff --git a/docs/reference/session-management-compaction.md b/docs/reference/session-management-compaction.md index 79c736449847..a24aa2990db8 100644 --- a/docs/reference/session-management-compaction.md +++ b/docs/reference/session-management-compaction.md @@ -1,7 +1,7 @@ --- summary: "Deep dive: session store + transcripts, lifecycle, and (auto)compaction internals" read_when: - - You need to debug session ids, transcript JSONL, or sessions.json fields + - You need to debug session ids, transcript events, or session row fields - You are changing auto-compaction behavior or adding "pre-compaction" housekeeping - You want to implement memory flushes or silent system turns title: "Session management deep dive" @@ -13,38 +13,56 @@ Overview docs first: [Session management](/concepts/session), [Compaction](/conc ## Two persistence layers -1. **Session store (`sessions.json`)** - key/value map `sessionKey -> SessionEntry`. Small, mutable, safe to edit or delete entries. Tracks metadata: current session id, last activity, toggles, token counters. -2. **Transcript (`.jsonl`)** - append-only, tree-structured (entries have `id` + `parentId`). Stores the conversation, tool calls, and compaction summaries; rebuilds model context for future turns. Compaction checkpoints are metadata over the compacted successor transcript - a new compaction does not write a second `.checkpoint.*.jsonl` copy. +1. **Session rows (per-agent SQLite)** - key/value map `sessionKey -> SessionEntry`. Mutable runtime state owned by the Gateway. Tracks metadata: current session id, last activity, toggles, token counters. +2. **Transcript events (per-agent SQLite)** - append-only, tree-structured (entries have `id` + `parentId`). Stores the conversation, tool calls, and compaction summaries; rebuilds model context for future turns. Compaction checkpoints are metadata over the compacted successor transcript - a new compaction does not write a second `.checkpoint.*.jsonl` copy. -Gateway history readers avoid materializing the whole transcript unless the surface needs arbitrary historical access. First-page history, embedded chat history, restart recovery, and token/usage checks use bounded tail reads. Full transcript scans go through the async transcript index, cached by file path plus `mtimeMs`/`size` and shared across concurrent readers. +Older installs may still have `sessions.json` files under the agent `sessions/` +directory. Treat those files as legacy session-row migration inputs or explicit +offline-maintenance targets. Gateway startup and `openclaw doctor --fix` import +hot legacy rows and transcript history into the per-agent SQLite store +automatically. Run `openclaw doctor --session-sqlite inspect +--session-sqlite-all-agents`, then follow the [Doctor migration +sequence](/cli/doctor#session-sqlite-migration), when you need explicit +inspection or validation evidence. If a migration fails after legacy transcript +artifacts were archived, use the Doctor recovery mode from that sequence. +Recovery uses migration manifests, restores only the affected archived support +artifacts, prepares a sanitized GitHub issue report when requested, and does not +make active runtime read JSONL files again. + +Gateway history readers avoid materializing the whole transcript unless the surface needs arbitrary historical access. First-page history, embedded chat history, restart recovery, and token/usage checks use bounded tail reads from SQLite. Full transcript scans go through the async transcript index and are shared across concurrent readers. ## On-disk locations Per agent, on the Gateway host (resolved via `src/config/sessions.ts`): -- Store: `~/.openclaw/agents//sessions/sessions.json` -- Transcripts: `~/.openclaw/agents//sessions/.jsonl` - - Telegram topic sessions: `.../-topic-.jsonl` +- Runtime session row store: `~/.openclaw/agents//agent/openclaw-agent.sqlite` +- Runtime transcript rows: `~/.openclaw/agents//agent/openclaw-agent.sqlite` +- Legacy/archive transcript artifacts: `~/.openclaw/agents//sessions/` +- Legacy row migration input: `~/.openclaw/agents//sessions/sessions.json` ## Store maintenance and disk controls -`session.maintenance` controls automatic maintenance for `sessions.json`, transcript artifacts, and trajectory sidecars: +`session.maintenance` controls automatic maintenance for SQLite session rows, SQLite transcript rows, archive artifacts, and trajectory sidecars: -| Key | Default | Notes | -| ----------------------- | --------------------- | --------------------------------------------------------------------------------- | -| `mode` | `"enforce"` | or `"warn"` (report only, no mutation) | -| `pruneAfter` | `"30d"` | stale-entry age cutoff | -| `maxEntries` | `500` | cap on entries in `sessions.json` | -| `resetArchiveRetention` | same as `pruneAfter` | retention for `*.reset.` transcript archives; `false` disables cleanup | -| `maxDiskBytes` | unset | optional sessions-directory budget | -| `highWaterBytes` | 80% of `maxDiskBytes` | target after budget cleanup | +| Key | Default | Notes | +| ----------------------- | --------------------- | ------------------------------------------------------------------------------------------- | +| `mode` | `"enforce"` | or `"warn"` (report only, no mutation) | +| `pruneAfter` | `"30d"` | stale-entry age cutoff | +| `maxEntries` | `500` | cap on session entries | +| `resetArchiveRetention` | keep (no age cutoff) | age cutoff for `*.reset.*`/`*.deleted.*` transcript archives; a duration opts into deletion | +| `maxDiskBytes` | `2gb` | per-agent sessions disk budget; `false` disables | +| `highWaterBytes` | 80% of `maxDiskBytes` | target after budget cleanup | + +Archived transcripts are kept by default and compressed with zstd (`*.jsonl...zst`) when the runtime supports it, so deleting or resetting a session never silently discards conversation history. The disk budget evicts the oldest archives first, before touching live sessions. + +Active SQLite enforcement of `maxDiskBytes` measures session-row JSON plus transcript-event JSON bytes per session; legacy offline-maintenance enforcement measures files in the selected sessions directory. Gateway model-run probe sessions (keys matching `agent:*:explicit:model-run-`) get a separate, fixed `24h` retention. This pruning is pressure-gated: it only runs when session-entry maintenance/cap pressure is reached, and only before the global stale-entry cleanup/cap step. Other explicit sessions do not use this retention. Enforcement order for disk-budget cleanup (`mode: "enforce"`): -1. Remove oldest archived, orphan transcript, or orphan trajectory artifacts first. -2. If still above target, evict oldest session entries and their transcript/trajectory files. +1. Remove oldest archived transcript artifacts, orphan legacy artifacts, or orphan trajectory artifacts first. +2. If still above target, evict oldest session entries and their transcript rows or trajectory artifacts. 3. Repeat until usage is at or below `highWaterBytes`. `mode: "warn"` reports potential evictions without mutating the store or files. @@ -58,11 +76,11 @@ openclaw sessions cleanup --enforce Maintenance keeps durable external conversation pointers such as group sessions and thread-scoped chat sessions, but synthetic runtime entries (cron, hooks, heartbeat, ACP, sub-agents) can still be removed once they exceed the configured age, count, or disk budget. Isolated cron runs use a separate `cron.sessionRetention` control, independent of model-run probe retention. -Normal Gateway writes flow through a per-store session writer that serializes in-process mutations without taking a runtime file lock. Hot-path patch helpers borrow the validated mutable cache while holding that writer slot, so large `sessions.json` files are not cloned or reread for every metadata update. Prefer `updateSessionStore(...)` / `updateSessionStoreEntry(...)` in runtime code; direct whole-store saves are for compatibility and offline maintenance tools. When a Gateway is reachable, non-dry-run `openclaw sessions cleanup` and `openclaw agents delete` delegate store mutations to the Gateway so cleanup joins the same writer queue; `--store ` is the explicit offline repair path for direct file maintenance and always stays local (as does `--dry-run`). `maxEntries` cleanup is batched for production-sized stores, so a store may briefly exceed the configured cap before the next high-water cleanup rewrites it down. Reads never prune or cap entries during Gateway startup - only writes or `openclaw sessions cleanup --enforce` do, and the latter also applies the cap immediately and prunes old unreferenced transcript, checkpoint, and trajectory artifacts even with no disk budget configured. +Normal Gateway writes flow through the session accessor, which serializes per-agent SQLite mutations through the runtime writer path. Runtime code should prefer the accessor helpers in `src/config/sessions/session-accessor.ts`; legacy `sessions.json` helpers are migration and offline-maintenance tools. When a Gateway is reachable, non-dry-run `openclaw sessions cleanup` and `openclaw agents delete` delegate store mutations to the Gateway so cleanup joins the same writer queue; `--store ` is the explicit offline repair path for a selected legacy store and always stays local (as does `--dry-run`). `maxEntries` cleanup is batched for production-sized stores, so a store may briefly exceed the configured cap before the next high-water cleanup rewrites it down. Reads never prune or cap entries during Gateway startup - only writes or `openclaw sessions cleanup --enforce` do, and the latter also applies the cap immediately and prunes old unreferenced legacy transcript, checkpoint, and trajectory artifacts even with no disk budget configured. OpenClaw no longer creates automatic `sessions.json.bak.*` rotation backups during Gateway writes. The legacy `session.maintenance.rotateBytes` key is ignored and `openclaw doctor --fix` removes it from older configs. -Transcript mutations use a session write lock on the transcript file: +Transcript mutations use the session write queue for the SQLite transcript target: | Setting | Default | Env override | | ------------------------------------ | --------- | ------------------------------------------------ | @@ -72,6 +90,28 @@ Transcript mutations use a session write lock on the transcript file: `acquireTimeoutMs` is how long a lock wait surfaces a busy-session error before giving up; raise it only when legitimate prep, cleanup, compaction, or transcript mirror work contends longer on slow machines. `staleMs` is when an existing lock can be reclaimed as stale. `maxHoldMs` is the in-process watchdog release threshold. +### Downgrading After The SQLite Flip + +Restore archived legacy transcript artifacts before running an older +file-backed OpenClaw version: + +```bash +openclaw doctor --session-sqlite restore --session-sqlite-all-agents +``` + +The migration leaves legacy `sessions.json` files in place for support and +rollback, but hot transcript JSONL files that were imported into SQLite are +renamed into `session-sqlite-import-archive/`. Older file-backed runtimes follow +the `sessionFile` paths in `sessions.json`, so they need those artifacts restored +before startup. Restore uses migration manifests, moves only recorded archived +artifacts whose original paths are missing, and leaves the SQLite database in +place for forward recovery. + +Sessions created after the SQLite flip are SQLite-only and will not appear to an +older file-backed runtime. If you re-upgrade after a downgrade, run the Doctor +inspection and validation sequence again so OpenClaw can verify restored legacy +artifacts before importing. + ## Cron sessions and run logs Isolated cron runs create their own session entries/transcripts with dedicated retention: @@ -95,7 +135,7 @@ A `sessionKey` identifies which conversation bucket you are in (routing + isolat ## Session ids (`sessionId`) -Each `sessionKey` points at a current `sessionId` (the transcript file continuing the conversation). Decision logic lives in `initSessionState()` in `src/auto-reply/reply/session.ts`. +Each `sessionKey` points at a current `sessionId` (the SQLite transcript identity that continues the conversation). Decision logic lives in `initSessionState()` in `src/auto-reply/reply/session.ts`. - **Reset** (`/new`, `/reset`) creates a new `sessionId` for that `sessionKey`. - **Daily reset** (default 4:00 AM local time on the gateway host) creates a new `sessionId` on the next message after the reset boundary. @@ -105,11 +145,11 @@ Each `sessionKey` points at a current `sessionId` (the transcript file continuin - **Parent fork policy** uses OpenClaw's active branch when creating a thread or subagent fork. If that branch is too large (over a fixed internal cap, currently 100K tokens), OpenClaw starts the child with isolated context instead of failing or inheriting unusable history. Sizing is automatic and not configurable; legacy `session.parentForkMaxTokens` config is removed by `openclaw doctor --fix`. - **Operator forks**: `sessions.create { parentSessionKey, fork: true }` creates a new session whose transcript branches from the parent's current state (same fork machinery as subagent spawns, including the size cap above). The fork is refused while the parent has an active run, inherits the parent's model selection unless one is passed explicitly, and marks the child `forkedFromParent` with fresh token counters. -## Session store schema (`sessions.json`) +## Session store schema -The value type is `SessionEntry` in `src/config/sessions.ts`. Key fields (not exhaustive): +The runtime store keeps `SessionEntry` values in per-agent SQLite. The value type is `SessionEntry` in `src/config/sessions.ts`. Key fields (not exhaustive): -- `sessionId`: current transcript id (filename derives from this unless `sessionFile` is set) +- `sessionId`: current transcript id used to address SQLite transcript rows - `sessionStartedAt`: start timestamp for the current `sessionId`; daily reset freshness uses this. Legacy rows may derive it from the JSONL session header. - `lastInteractionAt`: last real user/channel interaction timestamp; idle reset freshness uses this so heartbeat, cron, and exec events do not keep sessions alive. Legacy rows without this field fall back to the recovered session start time. - `updatedAt`: last store-row mutation timestamp, used for listing/pruning/bookkeeping - not the daily/idle freshness authority. @@ -119,7 +159,7 @@ The value type is `SessionEntry` in `src/config/sessions.ts`. Key fields (not ex - Codex supervision lists only non-archived native threads. A Gateway-local `idle` or `notLoaded` activity-unknown thread can be archived through native `thread/archive` only after the operator explicitly confirms that no other Codex process owns it; the plugin performs a fresh process-local status read first, and the thread then disappears from the catalog. That read cannot prove that another App Server process is not using the thread. OpenClaw refuses to archive active and error rows, and paired-node archive is unavailable until the node bridge can own the full streamed thread lifecycle. Unarchiving in a native Codex client makes the thread eligible to appear again. - `lastReadAt` / `markedUnreadAt`: read-state timestamps stamped server-side by `sessions.patch { unread }` - `unread: false` records a read (sets `lastReadAt`, clears `markedUnreadAt`); `unread: true` marks the session unread until the next read. Session rows expose a derived `unread` boolean: explicitly marked unread, or read before the latest activity. Sessions never marked read stay `unread: false`, so existing installs do not light up on upgrade. - `lastActivityAt`: timestamp of the last completed agent run that counts as unread-worthy activity (user, channel, and cron runs). Heartbeat and internal-event turns, plus metadata patches, do not update it; `updatedAt` is not an activity signal. -- `sessionFile`: optional explicit transcript path override +- `sessionFile`: legacy marker retained for migration/archive compatibility; active runtime uses SQLite identity - `chatType`: `direct | group | room` - `provider`, `subject`, `room`, `space`, `displayName`: group/channel labeling metadata - Toggles: `thinkingLevel`, `verboseLevel`, `reasoningLevel`, `elevatedLevel`, `sendPolicy` (per-session override) @@ -128,13 +168,16 @@ The value type is `SessionEntry` in `src/config/sessions.ts`. Key fields (not ex - `compactionCount`: how many times auto-compaction completed for this session key - `memoryFlushAt` / `memoryFlushCompactionCount`: timestamp and compaction count of the last pre-compaction memory flush -The store is safe to edit, but the Gateway is the authority: it may rewrite or rehydrate entries as sessions run. +The Gateway is the authority: it may rewrite or rehydrate entries as sessions +run. For legacy file-backed installs, migrate with +`openclaw doctor --session-sqlite import --session-sqlite-all-agents` instead of +editing `sessions.json` and expecting runtime to keep reading that file. -## Transcript structure (`*.jsonl`) +## Transcript event structure -Transcripts are managed by `SessionManager` (`openclaw/plugin-sdk/agent-sessions`). The file is JSONL: +Transcripts are managed by the OpenClaw session accessor and exposed to runtime code through identity-based helpers. The event stream is append-only: -- First line: session header - `type: "session"`, `id`, `cwd`, `timestamp`, optional `parentSession`. +- First entry: session header - `type: "session"`, `id`, `cwd`, `timestamp`, optional `parentSession`. - Then: entries with `id` + `parentId` (tree structure). Notable entry types: @@ -152,7 +195,7 @@ OpenClaw intentionally does not "fix up" transcripts; the Gateway uses `SessionM Two different concepts: 1. **Model context window**: hard cap per model (tokens visible to the model). Comes from the model catalog and can be overridden via config. -2. **Session store counters**: rolling stats written into `sessions.json` (used for `/status` and dashboards). `contextTokens` is a runtime estimate/reporting value - do not treat it as a strict guarantee. +2. **Session store counters**: rolling stats written into the session row (used for `/status` and dashboards). `contextTokens` is a runtime estimate/reporting value - do not treat it as a strict guarantee. More on limits: [/reference/token-use](/reference/token-use). @@ -179,7 +222,7 @@ Two triggers in the embedded OpenClaw agent: Two additional guards run outside these two triggers: -- **Preflight local compaction**: set `agents.defaults.compaction.maxActiveTranscriptBytes` (bytes or a string like `"20mb"`) to trigger local compaction before opening the next run once the active transcript file reaches that size. This is a file-size guard for local reopen cost, not raw archival - normal semantic compaction still runs, and it requires `truncateAfterCompaction` so the compacted summary becomes a new successor transcript. +- **Preflight local compaction**: set `agents.defaults.compaction.maxActiveTranscriptBytes` (bytes or a string like `"20mb"`) to trigger local compaction before opening the next run once the active transcript reaches that size. This is a size guard for local reopen cost, not raw archival - normal semantic compaction still runs, and it requires `truncateAfterCompaction` so the compacted summary becomes a new successor transcript. - **Mid-turn precheck**: set `agents.defaults.compaction.midTurnPrecheck.enabled: true` (default `false`) to add a tool-loop guard. After a tool result is appended and before the next model call, OpenClaw estimates prompt pressure using the same preflight budget logic used at turn start. If context no longer fits, the guard does not compact inline - it raises a structured mid-turn precheck signal, stops the current prompt submission, and lets the outer run loop use the existing recovery path (truncate oversized tool results when that is enough, or trigger the configured compaction mode and retry). Works with both `default` and `safeguard` compaction modes, including provider-backed safeguard compaction. Independent of `maxActiveTranscriptBytes`: the byte-size guard runs before a turn opens, mid-turn precheck runs later, after new tool results are appended. ## Compaction settings @@ -202,7 +245,7 @@ OpenClaw also enforces a safety floor for embedded runs: if `compaction.reserveT Manual `/compact` honors an explicit `agents.defaults.compaction.keepRecentTokens` and keeps the runtime's recent-tail cut point. Without an explicit keep budget, manual compaction is a hard checkpoint and rebuilt context starts from the new summary. -When `truncateAfterCompaction` is enabled, OpenClaw rotates the active transcript to a compacted successor JSONL after compaction. Branch/restore checkpoint actions use that compacted successor; legacy pre-compaction checkpoint files remain readable while referenced. +When `truncateAfterCompaction` is enabled, OpenClaw rotates the active transcript to a compacted successor after compaction. Branch/restore checkpoint actions use that compacted successor; legacy pre-compaction checkpoint files remain readable while referenced. ## Pluggable compaction providers @@ -252,7 +295,7 @@ Notes: - The default prompt/system prompt include a `NO_REPLY` hint to suppress delivery. - When `model` is set, the flush turn uses that model without inheriting the active session's fallback chain, so local-only housekeeping does not silently fall back to a paid conversation model on failure. -- The flush runs once per compaction cycle (tracked in `sessions.json`). +- The flush runs once per compaction cycle (tracked in the session row). - The flush runs only for embedded OpenClaw sessions; CLI backends and heartbeat turns skip it. - The flush is skipped when the session workspace is read-only (`workspaceAccess: "ro"` or `"none"`). - See [Memory](/concepts/memory) for the workspace file layout and write patterns. diff --git a/docs/reference/wizard.md b/docs/reference/wizard.md index e41beb104d29..b45e8c8c0a1b 100644 --- a/docs/reference/wizard.md +++ b/docs/reference/wizard.md @@ -254,7 +254,10 @@ Typical fields in `~/.openclaw/openclaw.json`: `openclaw agents add` writes `agents.list[]` and optional `bindings`. WhatsApp credentials go under `~/.openclaw/credentials/whatsapp//`. -Sessions are stored under `~/.openclaw/agents//sessions/`. +Active sessions and transcripts are stored in +`~/.openclaw/agents//agent/openclaw-agent.sqlite`. The +`~/.openclaw/agents//sessions/` directory is used for legacy migration +inputs and archive/support artifacts. Some channels are delivered as plugins. When you pick one during setup, onboarding will prompt to install it (npm or a local path) before it can be configured. diff --git a/docs/start/openclaw.md b/docs/start/openclaw.md index 62eacee93458..32ce1b67266b 100644 --- a/docs/start/openclaw.md +++ b/docs/start/openclaw.md @@ -159,8 +159,9 @@ Example: ## Sessions and memory -- Session files: `~/.openclaw/agents//sessions/{{SessionId}}.jsonl` -- Session metadata (token usage, last route, etc): `~/.openclaw/agents//sessions/sessions.json` +- Session rows, transcript rows, and metadata (token usage, last route, etc): `~/.openclaw/agents//agent/openclaw-agent.sqlite` +- Legacy/archive transcript artifacts: `~/.openclaw/agents//sessions/` +- Legacy row migration source: `~/.openclaw/agents//sessions/sessions.json` - `/new` or `/reset` starts a fresh session for that chat (configurable via `session.resetTriggers`). If sent alone, OpenClaw acknowledges the reset without invoking the model. - `/compact [instructions]` compacts the session context and reports the remaining context budget. diff --git a/docs/start/setup.md b/docs/start/setup.md index 3f9a5c3d8d61..b1497c014784 100644 --- a/docs/start/setup.md +++ b/docs/start/setup.md @@ -132,7 +132,8 @@ openclaw health - **Where state lives:** - Channel/provider state: `~/.openclaw/credentials/` - Model auth profiles: `~/.openclaw/agents//agent/auth-profiles.json` - - Sessions: `~/.openclaw/agents//sessions/` + - Sessions and transcripts: `~/.openclaw/agents//agent/openclaw-agent.sqlite` + - Legacy/archive session artifacts: `~/.openclaw/agents//sessions/` - Logs: `/tmp/openclaw/` ## Credential storage map diff --git a/docs/start/wizard-cli-reference.md b/docs/start/wizard-cli-reference.md index c498d4d32f58..825381487735 100644 --- a/docs/start/wizard-cli-reference.md +++ b/docs/start/wizard-cli-reference.md @@ -341,7 +341,10 @@ Typical fields in `~/.openclaw/openclaw.json`: `openclaw agents add` writes `agents.list[]` and optional `bindings`. WhatsApp credentials go under `~/.openclaw/credentials/whatsapp//`. -Sessions are stored under `~/.openclaw/agents//sessions/`. +Active sessions and transcripts are stored in +`~/.openclaw/agents//agent/openclaw-agent.sqlite`. The +`~/.openclaw/agents//sessions/` directory is used for legacy migration +inputs and archive/support artifacts. Some channels are delivered as plugins. When selected during setup, the wizard diff --git a/docs/tools/trajectory.md b/docs/tools/trajectory.md index 89f74e39d636..9b9c950f9984 100644 --- a/docs/tools/trajectory.md +++ b/docs/tools/trajectory.md @@ -4,7 +4,7 @@ read_when: - Debugging why an agent answered, failed, or called tools a certain way - Exporting a support bundle for an OpenClaw session - Investigating prompt context, tool calls, runtime errors, or usage metadata - - Disabling or relocating trajectory capture + - Disabling trajectory capture title: "Trajectory bundles" --- @@ -114,31 +114,15 @@ Events are written as JSON Lines with this schema marker: `manifest.json` lists the files present in a given bundle; some files are omitted when the session did not capture the corresponding runtime data. -## Capture location +## Capture storage -By default, runtime trajectory events are written beside the session file: +Runtime trajectory events are stored with the session in the per-agent SQLite +database. Exporting a trajectory materializes a redacted JSONL support bundle; +the live runtime capture is not a session-adjacent JSONL sidecar. -```text -.trajectory.jsonl -``` - -OpenClaw also writes a best-effort pointer file beside the session: - -```text -.trajectory-path.json -``` - -Set `OPENCLAW_TRAJECTORY_DIR` to store runtime trajectory sidecars in a -dedicated directory instead, one JSONL file per session id: - -```bash -export OPENCLAW_TRAJECTORY_DIR=/var/lib/openclaw/trajectories -``` - -Session maintenance removes trajectory sidecars when their owning session -entry is pruned, capped, or evicted by the sessions disk budget. Runtime files -outside the sessions directory are removed only when the pointer target still -proves it belongs to that session. +Legacy `.trajectory.jsonl` and `.trajectory-path.json` files may still appear +from older releases or explicit legacy-file exports. Session maintenance treats +those files as cleanup targets; active capture writes database rows. ## Disable capture @@ -148,12 +132,12 @@ export OPENCLAW_TRAJECTORY=0 This disables runtime trajectory capture before starting OpenClaw. `/export-trajectory` can still export the transcript branch, but runtime-only -files such as compiled context, provider artifacts, and prompt metadata may be +data such as compiled context, provider artifacts, and prompt metadata may be missing. ## Tune flush timeout -OpenClaw flushes runtime trajectory sidecars during agent cleanup. The default +OpenClaw flushes runtime trajectory rows during agent cleanup. The default cleanup timeout is 10,000 ms. On slow disks or large stores, set `OPENCLAW_TRAJECTORY_FLUSH_TIMEOUT_MS` before starting OpenClaw: @@ -179,7 +163,7 @@ redacts sensitive values before writing export files: The exporter also bounds input size: -- runtime sidecar files: the live capture file is a rolling window capped at 10 MiB, dropping the oldest events to make room for new ones; export accepts existing runtime sidecar files up to 50 MiB +- runtime capture: the live capture is a rolling window capped at 10 MiB, dropping the oldest events to make room for new ones; export accepts existing legacy runtime sidecar files up to 50 MiB - session files: 50 MiB - runtime events per export: 200,000 - total exported events: 250,000 @@ -193,7 +177,6 @@ and cannot know every application-specific secret. If the export has no runtime events: - confirm OpenClaw was started without `OPENCLAW_TRAJECTORY=0` -- check whether `OPENCLAW_TRAJECTORY_DIR` points to a writable directory - run another message in the session, then export again - inspect `manifest.json` for `runtimeEventCount` diff --git a/docs/web/webchat.md b/docs/web/webchat.md index 753370fe99a0..3614ad1ca6ba 100644 --- a/docs/web/webchat.md +++ b/docs/web/webchat.md @@ -39,11 +39,11 @@ Status: the macOS/iOS SwiftUI chat UI talks directly to the Gateway WebSocket. N WebChat has two separate data paths: -- The session JSONL file is the durable model/runtime transcript. For normal agent runs, the embedded OpenClaw runtime persists model-visible `user`, `assistant`, and `toolResult` messages through its session manager. WebChat does not write arbitrary delivery, status, or helper text into that transcript. +- The SQLite transcript rows are the durable model/runtime transcript. For normal agent runs, the embedded OpenClaw runtime persists model-visible `user`, `assistant`, and `toolResult` messages through the session accessor. WebChat does not write arbitrary delivery, status, or helper text into that transcript. - Gateway `ReplyPayload` events are the live delivery projection: normalized for WebChat/channel display, block streaming, directive tags, media embedding, TTS/audio flags, and UI fallback behavior. They are not themselves the canonical session log. - Harnesses that require visible replies through `tools.message` still use WebChat as a current-run internal source reply sink. A targetless `message.send` from that active WebChat run is projected into the same chat and mirrored to the session transcript; WebChat does not become a reusable outbound channel and never inherits `lastChannel`. - WebChat injects assistant transcript entries only when the Gateway owns a displayed message outside a normal embedded agent turn: `chat.inject`, non-agent command replies, aborted partial output, and WebChat-managed media transcript supplements. -- If live assistant text appears during a run but disappears after history reload, check in order: whether the raw JSONL contains the assistant text, whether `chat.history` display projection stripped it, then whether the Control UI optimistic-tail merge replaced local delivery state with the persisted snapshot. +- If live assistant text appears during a run but disappears after history reload, check in order: whether the SQLite transcript contains the assistant text, whether `chat.history` display projection stripped it, then whether the Control UI optimistic-tail merge replaced local delivery state with the persisted snapshot. Normal agent-run final answers should be durable because the embedded runtime writes the assistant `message_end`. Any fallback that mirrors a delivered final payload into the transcript must first avoid duplicating an assistant turn that the embedded runtime already wrote. diff --git a/extensions/acpx/index.test.ts b/extensions/acpx/index.test.ts index 3394a8826297..13f3b0b892d7 100644 --- a/extensions/acpx/index.test.ts +++ b/extensions/acpx/index.test.ts @@ -66,7 +66,27 @@ describe("acpx plugin", () => { params.openKeyedStore({ namespace: "test", maxEntries: 1 }); expect(openKeyedStore).toHaveBeenCalledWith({ namespace: "test", maxEntries: 1 }); expect(api.registerService).toHaveBeenCalledWith(service); - expect(api.on).toHaveBeenCalledWith("reply_dispatch", tryDispatchAcpReplyHookMock); + expect(api.on).toHaveBeenCalledWith("reply_dispatch", expect.any(Function), { + timeoutMs: 120_000, + }); + }); + + it("uses configured ACPX timeout for reply_dispatch hook registration", () => { + const service = { id: "acpx-service", start: vi.fn() }; + createAcpxRuntimeServiceMock.mockReturnValue(service); + + const api = { + pluginConfig: { timeoutSeconds: 180 }, + runtime: { state: { openKeyedStore: vi.fn() } }, + registerService: vi.fn(), + on: vi.fn(), + }; + + plugin.register(api as never); + + expect(api.on).toHaveBeenCalledWith("reply_dispatch", expect.any(Function), { + timeoutMs: 180_000, + }); }); it("does not touch runtime state while registering metadata-only plugin APIs", () => { @@ -130,7 +150,79 @@ describe("acpx plugin", () => { queuedFinal: true, counts: { tool: 1, block: 0, final: 1 }, }); - expect(tryDispatchAcpReplyHookMock).toHaveBeenCalledWith(event, ctx); + expect(tryDispatchAcpReplyHookMock).toHaveBeenCalledWith(event, { + ...ctx, + abortSignal: expect.any(AbortSignal), + }); + }); + + it("aborts the ACP reply_dispatch runtime path at the configured timeout", async () => { + vi.useFakeTimers(); + try { + const service = { id: "acpx-service", start: vi.fn() }; + createAcpxRuntimeServiceMock.mockReturnValue(service); + const observedAbortStates: boolean[] = []; + tryDispatchAcpReplyHookMock.mockImplementation(async (_event, hookCtx) => { + const abortSignal = (hookCtx as { abortSignal?: AbortSignal }).abortSignal; + if (!abortSignal) { + throw new Error("expected ACPX hook abort signal"); + } + observedAbortStates.push(abortSignal.aborted); + await new Promise((resolve) => { + abortSignal.addEventListener("abort", () => resolve(), { once: true }); + }); + observedAbortStates.push(abortSignal.aborted); + return { + handled: true, + queuedFinal: true, + counts: { tool: 0, block: 0, final: 1 }, + }; + }); + + const on = vi.fn(); + const api = createTestPluginApi({ + pluginConfig: { timeoutSeconds: 0.001 }, + runtime: { state: { openKeyedStore: vi.fn() } } as never, + registerService: vi.fn(), + on, + }); + + plugin.register(api); + + const hook = on.mock.calls.find(([hookName]) => hookName === "reply_dispatch")?.[1]; + if (!hook) { + throw new Error("expected reply_dispatch hook to be registered"); + } + + const run = hook( + { + ctx: { raw: "reply ctx" }, + runId: "run-1", + sessionKey: "agent:test:session", + inboundAudio: false, + shouldRouteToOriginating: false, + shouldSendToolSummaries: true, + sendPolicy: "allow", + }, + { + cfg: {}, + dispatcher: { dispatch: vi.fn(), getQueuedCounts: vi.fn(), getFailedCounts: vi.fn() }, + recordProcessed: vi.fn(), + markIdle: vi.fn(), + }, + ); + + await vi.advanceTimersByTimeAsync(1); + + await expect(run).resolves.toEqual({ + handled: true, + queuedFinal: true, + counts: { tool: 0, block: 0, final: 1 }, + }); + expect(observedAbortStates).toEqual([false, true]); + } finally { + vi.useRealTimers(); + } }); it("declares setup auto-enable reasons for ACPX-owned ACP config", () => { diff --git a/extensions/acpx/index.ts b/extensions/acpx/index.ts index 3389a2072402..2124f56f272a 100644 --- a/extensions/acpx/index.ts +++ b/extensions/acpx/index.ts @@ -3,21 +3,63 @@ * wires reply-dispatch hooks into the plugin SDK runtime. */ import { tryDispatchAcpReplyHook } from "openclaw/plugin-sdk/acp-runtime-backend"; +import { finiteSecondsToTimerSafeMilliseconds } from "openclaw/plugin-sdk/number-runtime"; import { createAcpxRuntimeService } from "./register.runtime.js"; -import type { OpenClawPluginApi } from "./runtime-api.js"; +import type { + OpenClawPluginApi, + PluginHookReplyDispatchContext, + PluginHookReplyDispatchEvent, + PluginHookReplyDispatchResult, +} from "./runtime-api.js"; +import { DEFAULT_ACPX_TIMEOUT_SECONDS } from "./src/config-schema.js"; + +function resolveReplyDispatchTimeoutMs(pluginConfig?: Record): number { + const timeoutSeconds = pluginConfig?.timeoutSeconds; + const resolvedSeconds = + typeof timeoutSeconds === "number" && Number.isFinite(timeoutSeconds) && timeoutSeconds > 0 + ? timeoutSeconds + : DEFAULT_ACPX_TIMEOUT_SECONDS; + return finiteSecondsToTimerSafeMilliseconds(resolvedSeconds) ?? 1; +} + +async function tryDispatchAcpReplyHookWithTimeout( + event: PluginHookReplyDispatchEvent, + ctx: PluginHookReplyDispatchContext, + timeoutMs: number, +): Promise { + const timeoutController = new AbortController(); + const timeout = setTimeout(() => timeoutController.abort(), timeoutMs); + timeout.unref?.(); + const abortSignal = ctx.abortSignal + ? AbortSignal.any([ctx.abortSignal, timeoutController.signal]) + : timeoutController.signal; + try { + return await tryDispatchAcpReplyHook(event, { + ...ctx, + abortSignal, + }); + } finally { + clearTimeout(timeout); + } +} const plugin = { id: "acpx", name: "ACPX Runtime", description: "Embedded ACP runtime backend with plugin-owned session and transport management.", register(api: OpenClawPluginApi) { + const replyDispatchTimeoutMs = resolveReplyDispatchTimeoutMs(api.pluginConfig); api.registerService( createAcpxRuntimeService({ pluginConfig: api.pluginConfig, openKeyedStore: (options) => api.runtime.state.openKeyedStore(options), }), ); - api.on("reply_dispatch", tryDispatchAcpReplyHook); + api.on( + "reply_dispatch", + (event, ctx) => tryDispatchAcpReplyHookWithTimeout(event, ctx, replyDispatchTimeoutMs), + { timeoutMs: replyDispatchTimeoutMs }, + ); }, }; diff --git a/extensions/active-memory/index.test.ts b/extensions/active-memory/index.test.ts index fd603aba7b3c..9062c664096a 100644 --- a/extensions/active-memory/index.test.ts +++ b/extensions/active-memory/index.test.ts @@ -8,6 +8,7 @@ import { createPluginStateKeyedStoreForTests, resetPluginStateStoreForTests, } from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; import plugin, { testing } from "./index.js"; @@ -133,7 +134,7 @@ describe("active-memory plugin", () => { agent: { runEmbeddedAgent, session: { - resolveStorePath: vi.fn(() => "/tmp/openclaw-session-store.json"), + resolveStorePath: vi.fn(() => path.join(stateDir, "sessions.json")), loadSessionStore: vi.fn(() => hoisted.sessionStore), saveSessionStore: vi.fn(async () => {}), getSessionEntry: vi.fn( @@ -153,7 +154,7 @@ describe("active-memory plugin", () => { }) => { let result: Record | null = null; await hoisted.updateSessionStore( - "/tmp/openclaw-session-store.json", + path.join(stateDir, "sessions.json"), (store: Record>) => { const existing = store[params.sessionKey] ?? params.fallbackEntry; if (!existing) { @@ -3018,6 +3019,72 @@ describe("active-memory plugin", () => { ); }); + it("returns partial transcript text on timeout from SQLite runtime transcript rows", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + testing.setTimeoutPartialDataGraceMsForTests(100); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 250, + maxSummaryChars: 80, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:timeout-partial-sqlite-transcript"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-timeout-partial-sqlite-transcript", + updatedAt: 0, + }; + let artifactSessionFile = ""; + runEmbeddedAgent.mockImplementationOnce( + async (params: { + abortSignal?: AbortSignal; + sessionFile?: string; + sessionTarget?: { + agentId: string; + sessionId: string; + sessionKey: string; + storePath?: string; + }; + }) => { + artifactSessionFile = params.sessionFile ?? ""; + const target = params.sessionTarget; + if (!target) { + throw new Error("expected active-memory runtime session target"); + } + await appendSessionTranscriptMessageByIdentity({ + ...target, + message: { + role: "assistant", + content: "sqlite partial recall summary", + }, + }); + await waitForAbort(params.abortSignal); + }, + ); + + const result = await hooks.before_prompt_build( + { prompt: "what wings should i order? timeout partial sqlite", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expectPrependContextContains(result, "sqlite partial recall summary"); + if (artifactSessionFile) { + await expectPathMissing(artifactSessionFile); + } + const runParams = lastEmbeddedRunParams(); + expect(runParams.sessionTarget).toMatchObject({ + agentId: "main", + sessionKey: expect.stringMatching(/^agent:main:timeout-partial-sqlite-transcript:/), + }); + const lines = getActiveMemoryLines(sessionKey); + expectLinesToContain(lines, "🧩 Active Memory: status=timeout_partial"); + expectLinesToContain( + lines, + "🔎 Active Memory Debug: timeout_partial: 29 chars recovered (not persisted)", + ); + }); + it("keeps timeout status when the timeout transcript is empty", async () => { testing.setMinimumTimeoutMsForTests(1); testing.setSetupGraceTimeoutMsForTests(0); @@ -4705,6 +4772,71 @@ describe("active-memory plugin", () => { expectLinesToContain(getActiveMemoryLines(sessionKey), "status=unavailable"); }); + it("rejects completed output when a rotated SQLite transcript reports unavailable memory", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 1_000, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:rotated-sqlite-memory-unavailable"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-rotated-sqlite-memory-unavailable", + updatedAt: 0, + }; + runEmbeddedAgent.mockImplementationOnce( + async (params: { + sessionTarget?: { + agentId: string; + sessionId: string; + sessionKey: string; + storePath?: string; + }; + }) => { + const target = params.sessionTarget; + if (!target?.storePath) { + throw new Error("expected active-memory SQLite runtime target"); + } + const rotatedTarget = { + ...target, + sessionId: "s-rotated-sqlite-memory-unavailable-next", + }; + await appendSessionTranscriptMessageByIdentity({ + ...rotatedTarget, + message: { + role: "toolResult", + toolCallId: "memory-search-1", + toolName: "memory_search", + isError: true, + content: [], + details: { + disabled: true, + warning: "Memory search is disabled for this session.", + }, + }, + }); + return { + payloads: [{ text: "This arbitrary output must not become recalled context." }], + meta: { + agentMeta: { + sessionFile: `sqlite:${rotatedTarget.agentId}:${rotatedTarget.sessionId}:${rotatedTarget.storePath}`, + }, + }, + }; + }, + ); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? rotated sqlite unavailable", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expect(result).toBeUndefined(); + expectLinesToContain(getActiveMemoryLines(sessionKey), "status=unavailable"); + }); + it("fast-fails configured-provider-missing memory_search results without injecting provider errors", async () => { const CONFIGURED_TIMEOUT_MS = 1_000; testing.setMinimumTimeoutMsForTests(1); diff --git a/extensions/active-memory/index.ts b/extensions/active-memory/index.ts index 2b11d0d4de9e..9188840f7b1e 100644 --- a/extensions/active-memory/index.ts +++ b/extensions/active-memory/index.ts @@ -29,6 +29,11 @@ import { import { definePluginEntry, type OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; import { parseAgentSessionKey, parseThreadSessionSuffix } from "openclaw/plugin-sdk/routing"; import { isPathInside } from "openclaw/plugin-sdk/security-runtime"; +import { parseSqliteSessionFileMarker } from "openclaw/plugin-sdk/session-store-runtime"; +import { + readSessionTranscriptEvents, + type SessionTranscriptTargetParams, +} from "openclaw/plugin-sdk/session-transcript-runtime"; import { asOptionalRecord as asRecord, normalizeLowercaseStringOrEmpty, @@ -295,6 +300,16 @@ type TranscriptReadLimits = { maxBytes?: number; }; +type ActiveMemoryTranscriptSource = + | { + kind: "runtime"; + target: SessionTranscriptTargetParams; + } + | { + kind: "file"; + sessionFile: string; + }; + type RecallSubagentResult = { rawReply: string; resultStatus?: "failed" | "unavailable"; @@ -1749,6 +1764,90 @@ async function streamBoundedTranscriptJsonl(params: { } } +function fileTranscriptSource(sessionFile: string): ActiveMemoryTranscriptSource { + return { kind: "file", sessionFile }; +} + +function transcriptSourceFromReturnedSessionFile(params: { + sessionFile: string; + sessionKey: string; +}): ActiveMemoryTranscriptSource { + const marker = parseSqliteSessionFileMarker(normalizeOptionalString(params.sessionFile)); + if (!marker) { + return fileTranscriptSource(params.sessionFile); + } + return { + kind: "runtime", + target: { + agentId: marker.agentId, + sessionId: marker.sessionId, + sessionKey: params.sessionKey, + storePath: marker.storePath, + }, + }; +} + +function estimateTranscriptEventsBytes(events: readonly unknown[]): number { + let total = 0; + for (const event of events) { + try { + total += Buffer.byteLength(`${JSON.stringify(event)}\n`, "utf8"); + } catch { + total += 1; + } + } + return total; +} + +async function streamRuntimeTranscriptEvents(params: { + target: SessionTranscriptTargetParams; + limits?: TranscriptReadLimits; + onRecord: (record: unknown) => boolean | void; +}): Promise { + const limits = resolveTranscriptReadLimits(params.limits); + let events: readonly unknown[]; + try { + events = await readSessionTranscriptEvents(params.target); + } catch { + return; + } + if (estimateTranscriptEventsBytes(events) > limits.maxBytes) { + return; + } + let seenLines = 0; + for (const event of events) { + seenLines += 1; + if (seenLines > limits.maxLines) { + break; + } + try { + if (params.onRecord(event)) { + break; + } + } catch {} + } +} + +async function streamActiveMemoryTranscriptRecords(params: { + source: ActiveMemoryTranscriptSource; + limits?: TranscriptReadLimits; + onRecord: (record: unknown) => boolean | void; +}): Promise { + if (params.source.kind === "runtime") { + await streamRuntimeTranscriptEvents({ + target: params.source.target, + limits: params.limits, + onRecord: params.onRecord, + }); + return; + } + await streamBoundedTranscriptJsonl({ + sessionFile: params.source.sessionFile, + limits: params.limits, + onRecord: params.onRecord, + }); +} + function extractActiveMemorySearchDebugFromSessionRecord( value: unknown, ): ActiveMemorySearchDebug | undefined { @@ -1986,7 +2085,7 @@ function hasUsableMemoryResultInSessionRecord( } async function readActiveMemoryTranscriptState( - sessionFile: string, + source: ActiveMemoryTranscriptSource | string, limits?: TranscriptReadLimits, toolsAllow?: readonly string[], ): Promise<{ @@ -1997,8 +2096,8 @@ async function readActiveMemoryTranscriptState( let searchDebug: ActiveMemorySearchDebug | undefined; let hasUsableMemoryResult = false; let hasUnavailableMemorySearchResult = false; - await streamBoundedTranscriptJsonl({ - sessionFile, + await streamActiveMemoryTranscriptRecords({ + source: typeof source === "string" ? fileTranscriptSource(source) : source, limits, onRecord: (record) => { const debug = extractActiveMemorySearchDebugFromSessionRecord(record); @@ -2016,14 +2115,14 @@ async function readActiveMemoryTranscriptState( } async function readActiveMemorySearchDebug( - sessionFile: string, + source: ActiveMemoryTranscriptSource | string, limits?: TranscriptReadLimits, ): Promise { - return (await readActiveMemoryTranscriptState(sessionFile, limits)).searchDebug; + return (await readActiveMemoryTranscriptState(source, limits)).searchDebug; } async function readMergedActiveMemoryTranscriptState(params: { - sessionFiles: readonly string[]; + sources: readonly ActiveMemoryTranscriptSource[]; toolsAllow: readonly string[]; }): Promise<{ searchDebug?: ActiveMemorySearchDebug; @@ -2033,8 +2132,17 @@ async function readMergedActiveMemoryTranscriptState(params: { let searchDebug: ActiveMemorySearchDebug | undefined; let hasUsableMemoryResult = false; let hasUnavailableMemorySearchResult = false; - for (const sessionFile of new Set(params.sessionFiles)) { - const state = await readActiveMemoryTranscriptState(sessionFile, undefined, params.toolsAllow); + const seen = new Set(); + for (const source of params.sources) { + const key = + source.kind === "runtime" + ? `runtime:${source.target.agentId ?? ""}:${source.target.sessionId}:${source.target.sessionKey}:${source.target.storePath ?? ""}:${source.target.threadId ?? ""}` + : `file:${source.sessionFile}`; + if (seen.has(key)) { + continue; + } + seen.add(key); + const state = await readActiveMemoryTranscriptState(source, undefined, params.toolsAllow); searchDebug = state.searchDebug ?? searchDebug; hasUsableMemoryResult ||= state.hasUsableMemoryResult; hasUnavailableMemorySearchResult ||= state.hasUnavailableMemorySearchResult; @@ -2043,7 +2151,7 @@ async function readMergedActiveMemoryTranscriptState(params: { } async function readTerminalMemorySearchResult( - sessionFile: string, + source: ActiveMemoryTranscriptSource, limits?: TranscriptReadLimits, toolsAllow?: readonly string[], ): Promise { @@ -2060,8 +2168,8 @@ async function readTerminalMemorySearchResult( const unavailablePathNames = new Set(); let hasUsableMemoryResult = false; let searchDebug: ActiveMemorySearchDebug | undefined; - await streamBoundedTranscriptJsonl({ - sessionFile, + await streamActiveMemoryTranscriptRecords({ + source, limits, onRecord: (record) => { hasUsableMemoryResult ||= hasUsableMemoryResultInSessionRecord(record, toolsAllow); @@ -2088,8 +2196,22 @@ async function readTerminalMemorySearchResult( }; } +async function readTerminalMemorySearchResultFromSources( + sources: readonly ActiveMemoryTranscriptSource[], + limits: TranscriptReadLimits | undefined, + toolsAllow: readonly string[], +): Promise { + for (const source of sources) { + const result = await readTerminalMemorySearchResult(source, limits, toolsAllow); + if (result) { + return result; + } + } + return undefined; +} + function watchTerminalMemorySearchResult(params: { - getSessionFile: () => string | undefined; + getTranscriptSources: () => readonly ActiveMemoryTranscriptSource[]; abortSignal: AbortSignal; toolsAllow: readonly string[]; }): TerminalMemorySearchWatch { @@ -2131,10 +2253,11 @@ function watchTerminalMemorySearchResult(params: { } inFlight = true; try { - const sessionFile = params.getSessionFile(); - const result = sessionFile - ? await readTerminalMemorySearchResult(sessionFile, undefined, params.toolsAllow) - : undefined; + const result = await readTerminalMemorySearchResultFromSources( + params.getTranscriptSources(), + undefined, + params.toolsAllow, + ); if (result) { finish(result); return; @@ -2265,17 +2388,17 @@ function extractAssistantTextFromSessionRecord(value: unknown): string { } async function readPartialAssistantText( - sessionFile: string | undefined, + source: ActiveMemoryTranscriptSource | string | undefined, limits?: TranscriptReadLimits, ): Promise { - if (!sessionFile) { + if (!source) { return null; } const texts: string[] = []; const resolvedLimits = resolveTranscriptReadLimits(limits); let collectedChars = 0; - await streamBoundedTranscriptJsonl({ - sessionFile, + await streamActiveMemoryTranscriptRecords({ + source: typeof source === "string" ? fileTranscriptSource(source) : source, limits: resolvedLimits, onRecord: (record) => { const text = extractAssistantTextFromSessionRecord(record); @@ -2302,6 +2425,19 @@ async function readPartialAssistantText( return joined || null; } +async function readPartialAssistantTextFromSources( + sources: readonly ActiveMemoryTranscriptSource[], + limits?: TranscriptReadLimits, +): Promise { + for (const source of sources) { + const text = await readPartialAssistantText(source, limits); + if (text) { + return text; + } + } + return null; +} + function attachPartialTimeoutData( error: unknown, partialReply: string | null, @@ -2373,7 +2509,7 @@ async function waitForSubagentPartialTimeoutData( async function buildTimeoutRecallResult(params: { elapsedMs: number; maxSummaryChars: number; - sessionFile?: string; + transcriptSources: readonly ActiveMemoryTranscriptSource[]; rawReply?: string; searchDebug?: ActiveMemorySearchDebug; hasUnavailableMemorySearchResult?: boolean; @@ -2386,14 +2522,18 @@ async function buildTimeoutRecallResult(params: { const rawReply = params.rawReply ?? subagentPartialData.rawReply ?? - (await readPartialAssistantText(params.sessionFile)); + (await readPartialAssistantTextFromSources(params.transcriptSources)); const summary = truncateSummary( normalizeActiveSummary(rawReply ?? "") ?? "", params.maxSummaryChars, ); - const transcriptState = params.sessionFile - ? await readActiveMemoryTranscriptState(params.sessionFile, undefined, params.toolsAllow) - : undefined; + const transcriptState = + params.transcriptSources.length > 0 + ? await readMergedActiveMemoryTranscriptState({ + sources: params.transcriptSources, + toolsAllow: params.toolsAllow, + }) + : undefined; const searchDebug = params.searchDebug ?? subagentPartialData.searchDebug ?? transcriptState?.searchDebug; if ( @@ -2945,6 +3085,64 @@ function getModelRef( return undefined; } +function collectActiveMemoryTranscriptSources(params: { + artifactSessionFile: string; + runtimeSource: ActiveMemoryTranscriptSource; + activeSessionFile?: string; + activeSessionKey: string; +}): ActiveMemoryTranscriptSource[] { + const sources: ActiveMemoryTranscriptSource[] = [params.runtimeSource]; + sources.push(fileTranscriptSource(params.artifactSessionFile)); + if (params.activeSessionFile && params.activeSessionFile !== params.artifactSessionFile) { + sources.push( + transcriptSourceFromReturnedSessionFile({ + sessionFile: params.activeSessionFile, + sessionKey: params.activeSessionKey, + }), + ); + } + return sources; +} + +async function persistActiveMemoryTranscriptArtifact(params: { + sources: readonly ActiveMemoryTranscriptSource[]; + sessionFile: string; +}): Promise { + const events: unknown[] = []; + const seen = new Set(); + for (const source of params.sources) { + if (source.kind !== "runtime") { + continue; + } + let sourceEvents: readonly unknown[]; + try { + sourceEvents = await readSessionTranscriptEvents(source.target); + } catch { + continue; + } + for (const event of sourceEvents) { + const serialized = JSON.stringify(event); + if (seen.has(serialized)) { + continue; + } + seen.add(serialized); + events.push(event); + } + } + if (events.length === 0) { + return; + } + await fs.mkdir(path.dirname(params.sessionFile), { recursive: true, mode: 0o700 }); + await fs.writeFile( + params.sessionFile, + `${events.map((event) => JSON.stringify(event)).join("\n")}\n`, + { + encoding: "utf8", + mode: 0o600, + }, + ); +} + async function runRecallSubagent(params: { api: OpenClawPluginApi; config: ResolvedActiveRecallPluginConfig; @@ -2959,7 +3157,7 @@ async function runRecallSubagent(params: { currentModelId?: string; modelRef?: { provider: string; model: string }; abortSignal?: AbortSignal; - onSessionFile?: (sessionFile: string) => void; + onTranscriptSources?: (sources: readonly ActiveMemoryTranscriptSource[]) => void; }): Promise { const workspaceDir = resolveAgentWorkspaceDir(params.api.config, params.agentId); const agentDir = resolveAgentDir(params.api.config, params.agentId); @@ -3006,7 +3204,27 @@ async function runRecallSubagent(params: { persistedDir !== undefined ? path.join(persistedDir, `${subagentSessionId}.jsonl`) : path.join(requireTransientWorkspaceDir(tempDir), "session.jsonl"); - params.onSessionFile?.(sessionFile); + const storePath = params.api.runtime.agent.session.resolveStorePath( + params.api.config.session?.store, + { + agentId: params.agentId, + }, + ); + const runtimeSource: ActiveMemoryTranscriptSource = { + kind: "runtime", + target: { + agentId: params.agentId, + sessionId: subagentSessionId, + sessionKey: subagentSessionKey, + storePath, + }, + }; + let transcriptSources = collectActiveMemoryTranscriptSources({ + artifactSessionFile: sessionFile, + runtimeSource, + activeSessionKey: subagentSessionKey, + }); + params.onTranscriptSources?.(transcriptSources); if (persistedDir) { await fs.mkdir(persistedDir, { recursive: true, mode: 0o700 }); await fs.chmod(persistedDir, 0o700).catch(() => undefined); @@ -3025,7 +3243,6 @@ async function runRecallSubagent(params: { channelId: params.channelId, }); - let activeSessionFile = sessionFile; let harnessHasUsableMemoryResult = false; let harnessHasUnavailableMemorySearchResult = false; try { @@ -3035,6 +3252,12 @@ async function runRecallSubagent(params: { sessionId: subagentSessionId, sessionKey: subagentSessionKey, agentId: params.agentId, + sessionTarget: { + agentId: params.agentId, + sessionId: subagentSessionId, + sessionKey: subagentSessionKey, + storePath, + }, messageChannel, messageProvider, sessionFile, @@ -3068,7 +3291,14 @@ async function runRecallSubagent(params: { harnessHasUnavailableMemorySearchResult ||= evidence.hasUnavailableMemorySearchResult; }, }); - activeSessionFile = readActiveMemorySessionFileFromRunResult(result) ?? sessionFile; + const activeSessionFile = readActiveMemorySessionFileFromRunResult(result) ?? sessionFile; + transcriptSources = collectActiveMemoryTranscriptSources({ + artifactSessionFile: sessionFile, + runtimeSource, + activeSessionFile, + activeSessionKey: subagentSessionKey, + }); + params.onTranscriptSources?.(transcriptSources); if (params.abortSignal?.aborted) { const reason = params.abortSignal.reason; if (reason instanceof Error) { @@ -3086,15 +3316,18 @@ async function runRecallSubagent(params: { .filter(Boolean) .join("\n") .trim(); + if (params.config.persistTranscripts) { + await persistActiveMemoryTranscriptArtifact({ sources: transcriptSources, sessionFile }); + } const transcriptState = await readMergedActiveMemoryTranscriptState({ - sessionFiles: [sessionFile, activeSessionFile], + sources: transcriptSources, toolsAllow: params.config.toolsAllow, }); const searchDebug = transcriptState.searchDebug ?? readActiveMemorySearchDebugFromRunResult(result); return { rawReply: rawReply || "NONE", - transcriptPath: params.config.persistTranscripts ? activeSessionFile : undefined, + transcriptPath: params.config.persistTranscripts ? sessionFile : undefined, searchDebug, hasUsableMemoryResult: transcriptState.hasUsableMemoryResult || harnessHasUsableMemoryResult, hasUnavailableMemorySearchResult: @@ -3102,12 +3335,11 @@ async function runRecallSubagent(params: { }; } catch (error) { if (params.abortSignal?.aborted) { - const partialReply = await readPartialAssistantText(activeSessionFile); - const transcriptState = await readActiveMemoryTranscriptState( - activeSessionFile, - undefined, - params.config.toolsAllow, - ); + const partialReply = await readPartialAssistantTextFromSources(transcriptSources); + const transcriptState = await readMergedActiveMemoryTranscriptState({ + sources: transcriptSources, + toolsAllow: params.config.toolsAllow, + }); attachPartialTimeoutData( error, partialReply, @@ -3258,7 +3490,7 @@ async function maybeResolveActiveRecall(params: { abortFromParent(); } const TIMEOUT_SENTINEL = Symbol("timeout"); - let sessionFile: string | undefined; + let transcriptSources: readonly ActiveMemoryTranscriptSource[] = []; let recallTimedOut = false; const watchdogTimeoutMs = params.config.timeoutMs + params.config.setupGraceTimeoutMs; const timeoutId = setTimeout(() => { @@ -3288,12 +3520,12 @@ async function maybeResolveActiveRecall(params: { ...params, modelRef: resolvedModelRef, abortSignal: controller.signal, - onSessionFile: (value) => { - sessionFile = value; + onTranscriptSources: (sources) => { + transcriptSources = sources; }, }); terminalMemorySearchWatch = watchTerminalMemorySearchResult({ - getSessionFile: () => sessionFile, + getTranscriptSources: () => transcriptSources, abortSignal: controller.signal, toolsAllow: params.config.toolsAllow, }); @@ -3341,7 +3573,7 @@ async function maybeResolveActiveRecall(params: { : await buildTimeoutRecallResult({ elapsedMs, maxSummaryChars: params.config.maxSummaryChars, - sessionFile, + transcriptSources, subagentPromise, toolsAllow: params.config.toolsAllow, }); @@ -3440,7 +3672,7 @@ async function maybeResolveActiveRecall(params: { const result = await buildTimeoutRecallResult({ elapsedMs: Date.now() - startedAt, maxSummaryChars: params.config.maxSummaryChars, - sessionFile, + transcriptSources, rawReply: partialTimeoutData.rawReply, searchDebug: partialTimeoutData.searchDebug, hasUnavailableMemorySearchResult: partialTimeoutData.hasUnavailableMemorySearchResult, diff --git a/extensions/codex/src/app-server/attempt-context.ts b/extensions/codex/src/app-server/attempt-context.ts index 97d5ddf288b0..a11717590732 100644 --- a/extensions/codex/src/app-server/attempt-context.ts +++ b/extensions/codex/src/app-server/attempt-context.ts @@ -124,6 +124,7 @@ export function resolveContextEngineBootstrapProjectionDecision(params: { expectedBinding: ReturnType; projection: CodexContextEngineThreadBootstrapProjection; dynamicToolsFingerprint: string; + legacyDynamicToolsFingerprint?: string; }): { project: boolean; reason: string } { const bindingProjection = params.startupBinding?.contextEngine?.projection; if (!params.startupBinding?.threadId || !bindingProjection) { @@ -144,6 +145,7 @@ export function resolveContextEngineBootstrapProjectionDecision(params: { !areCodexDynamicToolFingerprintsCompatible({ previous: params.startupBinding.dynamicToolsFingerprint, next: params.dynamicToolsFingerprint, + nextLegacy: params.legacyDynamicToolsFingerprint, }) ) { return { project: true, reason: "dynamic-tools-mismatch" }; diff --git a/extensions/codex/src/app-server/run-attempt.hooks.test.ts b/extensions/codex/src/app-server/run-attempt.hooks.test.ts index c4d90e7fa6de..7440a7bf02ed 100644 --- a/extensions/codex/src/app-server/run-attempt.hooks.test.ts +++ b/extensions/codex/src/app-server/run-attempt.hooks.test.ts @@ -285,10 +285,7 @@ describe("runCodexAppServerAttempt hooks and model diagnostics", () => { it("emits gated model-call content diagnostics for codex turns", async () => { const diagnosticEvents: DiagnosticEventPayload[] = []; const diagnosticContentByType = new Map(); - let diagnosticTypesAtLlmOutput: string[] = []; - const llmOutput = vi.fn(() => { - diagnosticTypesAtLlmOutput = diagnosticEvents.map((event) => event.type); - }); + const llmOutput = vi.fn(); initializeGlobalHookRunner( createMockPluginRegistry([{ hookName: "llm_output", handler: llmOutput }]), ); @@ -378,8 +375,7 @@ describe("runCodexAppServerAttempt hooks and model diagnostics", () => { ).toContain("hello back"); expect(completedEvent?.requestPayloadBytes).toBeGreaterThan(0); expect(llmOutput).toHaveBeenCalledTimes(1); - expect(diagnosticTypesAtLlmOutput).toContain("model.call.completed"); - expect(diagnosticTypesAtLlmOutput).not.toContain("model.call.error"); + expect(diagnosticEvents.map((event) => event.type)).not.toContain("model.call.error"); } finally { stopDiagnostics(); } diff --git a/extensions/codex/src/app-server/run-attempt.test.ts b/extensions/codex/src/app-server/run-attempt.test.ts index 20289f891175..e21b0949e369 100644 --- a/extensions/codex/src/app-server/run-attempt.test.ts +++ b/extensions/codex/src/app-server/run-attempt.test.ts @@ -16,7 +16,8 @@ import { registerMemoryCapability } from "openclaw/plugin-sdk/memory-core-host-r import { MESSAGE_TOOL_DELIVERY_HINTS } from "openclaw/plugin-sdk/message-tool-delivery-hints"; import { registerPluginCommand } from "openclaw/plugin-sdk/plugin-runtime"; import { createMockPluginRegistry } from "openclaw/plugin-sdk/plugin-test-runtime"; -import { saveSessionStore } from "openclaw/plugin-sdk/session-store-runtime"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { readSessionTranscriptEvents } from "openclaw/plugin-sdk/session-transcript-runtime"; import { describe, expect, it, vi } from "vitest"; import WebSocket from "ws"; import { CODEX_GPT5_BEHAVIOR_CONTRACT } from "../../prompt-overlay.js"; @@ -181,6 +182,43 @@ async function writeExistingBinding( }); } +function attachSqliteSessionTarget( + params: EmbeddedRunAttemptParams, + storePath: string, + sessionId: string, +): void { + params.sessionId = sessionId; + params.sessionKey = `agent:main:${sessionId}`; + params.sessionTarget = { + agentId: "main", + sessionId, + sessionKey: params.sessionKey, + storePath, + }; +} + +async function readTranscriptMessagesByIdentity( + params: EmbeddedRunAttemptParams, +): Promise>> { + const target = params.sessionTarget; + if (!target?.storePath || !target.sessionKey) { + throw new Error("expected SQLite session target"); + } + return ( + await readSessionTranscriptEvents({ + agentId: target.agentId, + sessionId: target.sessionId ?? params.sessionId, + sessionKey: target.sessionKey, + storePath: target.storePath, + }) + ) + .map((event) => (event as { message?: unknown }).message) + .filter( + (message): message is Record => + typeof message === "object" && message !== null, + ); +} + function createThreadLifecycleAppServerOptions(): Parameters< typeof startOrResumeThread >[0]["appServer"] { @@ -1260,11 +1298,23 @@ describe("runCodexAppServerAttempt", () => { createMockPluginRegistry([{ hookName: "llm_input", handler: llmInput }]), ); vi.stubEnv("OPENCLAW_TRAJECTORY", "1"); - vi.stubEnv("OPENCLAW_TRAJECTORY_DIR", path.join(tempDir, "trajectory")); const sessionFile = path.join(tempDir, "session.jsonl"); const workspaceDir = path.join(tempDir, "workspace"); const harness = createStartedThreadHarness(); const params = createParams(sessionFile, workspaceDir); + const trajectoryEvents: Array<{ + data?: { prompt?: string; systemPrompt?: string }; + type: string; + }> = []; + Object.assign(params, { + trajectorySessionFile: `sqlite:main:session-1:${path.join(tempDir, "openclaw-agent.sqlite")}`, + trajectoryRecorder: { + recordEvent: (type: string, data?: { prompt?: string; systemPrompt?: string }) => { + trajectoryEvents.push({ type, data }); + }, + flush: async () => undefined, + }, + }); params.skillsSnapshot = { prompt: "demo", skills: [], @@ -1301,15 +1351,6 @@ describe("runCodexAppServerAttempt", () => { expect(inputText).toBe("hello"); const [llmInputPayload] = mockCall(llmInput, "llm_input") as [{ prompt?: string }, unknown]; expect(llmInputPayload.prompt).toBe(inputText); - const trajectoryEvents = ( - await fs.readFile(path.join(tempDir, "trajectory", "session-1.jsonl"), "utf8") - ) - .trim() - .split("\n") - .map( - (line) => - JSON.parse(line) as { data?: { prompt?: string; systemPrompt?: string }; type?: string }, - ); const compiledContext = trajectoryEvents.find((event) => event.type === "context.compiled"); expect(compiledContext?.data?.prompt).toBe(inputText); expect(compiledContext?.data?.systemPrompt).toContain("## OpenClaw Skills"); @@ -1499,9 +1540,11 @@ describe("runCodexAppServerAttempt", () => { it("mirrors the Codex prompt into the transcript when the turn starts", async () => { const sessionFile = path.join(tempDir, "session-early-prompt.jsonl"); + const storePath = path.join(tempDir, "sessions-early-prompt.json"); const workspaceDir = path.join(tempDir, "workspace-early-prompt"); const harness = createStartedThreadHarness(); const params = createParams(sessionFile, workspaceDir); + attachSqliteSessionTarget(params, storePath, "session-early-prompt"); params.prompt = "external channel prompt"; const onUserMessagePersisted = vi.fn(); params.onUserMessagePersisted = onUserMessagePersisted; @@ -1509,10 +1552,13 @@ describe("runCodexAppServerAttempt", () => { const run = runCodexAppServerAttempt(params); await harness.waitForMethod("turn/start"); await vi.waitFor(async () => { - const raw = await fs.readFile(sessionFile, "utf8"); - expect(raw).toContain('"role":"user"'); - expect(raw).toContain('"content":"external channel prompt"'); - expect(raw).toContain('"idempotencyKey":"codex-app-server:thread-1:turn-1:prompt"'); + expect(await readTranscriptMessagesByIdentity(params)).toContainEqual( + expect.objectContaining({ + role: "user", + content: "external channel prompt", + idempotencyKey: "codex-app-server:thread-1:turn-1:prompt", + }), + ); }); await vi.waitFor(() => { expect(onUserMessagePersisted).toHaveBeenCalledWith( @@ -1524,56 +1570,66 @@ describe("runCodexAppServerAttempt", () => { ); }); - const rawBeforeCompletion = await fs.readFile(sessionFile, "utf8"); - expect(rawBeforeCompletion).not.toContain('"role":"assistant"'); + const messagesBeforeCompletion = await readTranscriptMessagesByIdentity(params); + expect(messagesBeforeCompletion.some((message) => message.role === "assistant")).toBe(false); await harness.completeTurn({ threadId: "thread-1", turnId: "turn-1" }); await run; - const rawAfterCompletion = await fs.readFile(sessionFile, "utf8"); - expect(rawAfterCompletion.match(/"role":"user"/gu)).toHaveLength(1); + const messagesAfterCompletion = await readTranscriptMessagesByIdentity(params); + expect(messagesAfterCompletion.filter((message) => message.role === "user")).toHaveLength(1); expect(onUserMessagePersisted).toHaveBeenCalledTimes(1); }); it("does not mirror the Codex prompt early when user message persistence is suppressed", async () => { const sessionFile = path.join(tempDir, "session-suppressed-early-prompt.jsonl"); + const storePath = path.join(tempDir, "sessions-suppressed-early-prompt.json"); const workspaceDir = path.join(tempDir, "workspace-suppressed-early-prompt"); const harness = createStartedThreadHarness(); const params = createParams(sessionFile, workspaceDir); + attachSqliteSessionTarget(params, storePath, "session-suppressed-early-prompt"); params.prompt = "already persisted prompt"; params.suppressNextUserMessagePersistence = true; - const readTranscript = async () => - fs.readFile(sessionFile, "utf8").catch((error: unknown) => { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return ""; - } - throw error; - }); const run = runCodexAppServerAttempt(params); await harness.waitForMethod("turn/start"); await expect( vi.waitFor( async () => { - const raw = await readTranscript(); - expect(raw).toContain("already persisted prompt"); + expect(await readTranscriptMessagesByIdentity(params)).toContainEqual( + expect.objectContaining({ + content: "already persisted prompt", + }), + ); }, { interval: 1, timeout: 100 }, ), ).rejects.toThrow(); - const rawBeforeCompletion = await readTranscript(); - expect(rawBeforeCompletion).not.toContain("already persisted prompt"); - expect(rawBeforeCompletion).not.toContain( - '"idempotencyKey":"codex-app-server:thread-1:turn-1:prompt"', + const messagesBeforeCompletion = await readTranscriptMessagesByIdentity(params); + expect(messagesBeforeCompletion).not.toContainEqual( + expect.objectContaining({ + content: "already persisted prompt", + }), + ); + expect(messagesBeforeCompletion).not.toContainEqual( + expect.objectContaining({ + idempotencyKey: "codex-app-server:thread-1:turn-1:prompt", + }), ); await harness.completeTurn({ threadId: "thread-1", turnId: "turn-1" }); await run; - const rawAfterCompletion = await readTranscript(); - expect(rawAfterCompletion).not.toContain("already persisted prompt"); - expect(rawAfterCompletion).not.toContain( - '"idempotencyKey":"codex-app-server:thread-1:turn-1:prompt"', + const messagesAfterCompletion = await readTranscriptMessagesByIdentity(params); + expect(messagesAfterCompletion).not.toContainEqual( + expect.objectContaining({ + content: "already persisted prompt", + }), + ); + expect(messagesAfterCompletion).not.toContainEqual( + expect.objectContaining({ + idempotencyKey: "codex-app-server:thread-1:turn-1:prompt", + }), ); }); @@ -4363,18 +4419,20 @@ describe("runCodexAppServerAttempt", () => { }) as never, ); - const run = runCodexAppServerAttempt( - createParams(path.join(tempDir, "session.jsonl"), path.join(tempDir, "workspace")), - { - pluginConfig: { - computerUse: { - enabled: true, - marketplaceName: "openai-bundled", - mcpServerName: "desktop-control", - }, + const params = createParams( + path.join(tempDir, "session.jsonl"), + path.join(tempDir, "workspace"), + ); + attachSqliteSessionTarget(params, path.join(tempDir, "sessions.json"), "session-computer-use"); + const run = runCodexAppServerAttempt(params, { + pluginConfig: { + computerUse: { + enabled: true, + marketplaceName: "openai-bundled", + mcpServerName: "desktop-control", }, }, - ); + }); await vi.waitFor(() => expect(handleRequest).toBeTypeOf("function")); // The keyed router only accepts turn-scoped requests once the turn is bound. await vi.waitFor(() => @@ -4410,11 +4468,11 @@ describe("runCodexAppServerAttempt", () => { expect(bridgeCall.requestParams?.serverName).toBe("desktop-control"); expect(bridgeCall.computerUseMcpServerName).toBe("desktop-control"); const requestCalls = request.mock.calls as unknown as Array<[string, unknown, unknown?]>; - const threadStart = requestCalls.find(([method]) => method === "thread/start"); - const threadStartParams = threadStart?.[1] as + const turnStart = requestCalls.find(([method]) => method === "turn/start"); + const turnStartParams = turnStart?.[1] as | { approvalPolicy?: { granular?: { mcp_elicitations?: boolean } } } | undefined; - expect(threadStartParams?.approvalPolicy?.granular?.mcp_elicitations).toBe(true); + expect(turnStartParams?.approvalPolicy?.granular?.mcp_elicitations).toBe(true); await notify({ method: "turn/completed", @@ -6094,8 +6152,10 @@ describe("runCodexAppServerAttempt", () => { conversationSourceTransferComplete: true, }); const storePath = path.join(tempDir, "sessions.json"); - await saveSessionStore(storePath, { - [sessionKey]: { + await upsertSessionEntry({ + storePath, + sessionKey, + entry: { sessionId: "session-current", updatedAt: Date.now(), }, diff --git a/extensions/codex/src/app-server/run-attempt.ts b/extensions/codex/src/app-server/run-attempt.ts index 5873a16687a0..fb6f93772970 100644 --- a/extensions/codex/src/app-server/run-attempt.ts +++ b/extensions/codex/src/app-server/run-attempt.ts @@ -51,7 +51,6 @@ import { resolveDiagnosticModelContentCapturePolicy, } from "openclaw/plugin-sdk/diagnostic-runtime"; import { loadExecApprovals } from "openclaw/plugin-sdk/exec-approvals-runtime"; -import { pathExists } from "openclaw/plugin-sdk/security-runtime"; import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { resolveCodexAppServerForModelProvider, @@ -251,6 +250,7 @@ import { buildTurnCollaborationMode, buildTurnStartParams, codexDynamicToolsFingerprint, + codexLegacyDynamicToolsFingerprint, resolveCodexAppServerThreadModelSelection, type CodexAppServerThreadLifecycleBinding, type CodexContextEngineThreadBootstrapProjection, @@ -263,6 +263,7 @@ import { } from "./tool-progress-normalization.js"; import { createCodexTrajectoryRecorder, + type CodexHostTrajectoryRecorder, normalizeCodexTrajectoryError, recordCodexTrajectoryCompletion, recordCodexTrajectoryContext, @@ -1063,7 +1064,6 @@ export async function runCodexAppServerAttempt( allocateToolOutcomeOrdinal: allocateCodexToolOutcomeOrdinal, }, }); - const hadSessionFile = await pathExists(activeSessionFile); const activeTranscriptTarget = { agentId: sessionAgentId, sessionFile: activeSessionFile, @@ -1074,6 +1074,7 @@ export async function runCodexAppServerAttempt( !activeContextEngine && initialStartupBindingHadInactiveThreadBootstrap ? [] : ((await readMirroredSessionHistoryMessages(activeTranscriptTarget)) ?? []); + const hadSessionTranscriptState = historyMessages.length > 0; const hookContextWindowFields = { ...(effectiveContextWindowInfo?.tokens ? { contextTokenBudget: effectiveContextWindowInfo.tokens } @@ -1114,11 +1115,12 @@ export async function runCodexAppServerAttempt( }); if (activeContextEngine) { await bootstrapHarnessContextEngine({ - hadSessionFile, + hadSessionFile: hadSessionTranscriptState, contextEngine: activeContextEngine, sessionId: activeSessionId, sessionKey: contextSessionKey, sessionFile: activeSessionFile, + sessionTarget: params.sessionTarget, runtimeContext: buildActiveContextEngineRuntimeContext(), contextEngineHostSupport: CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, providerId: effectiveRuntimeProviderId, @@ -1231,6 +1233,7 @@ export async function runCodexAppServerAttempt( ), projection: contextEngineProjection, dynamicToolsFingerprint: codexDynamicToolsFingerprint(toolBridge.specs), + legacyDynamicToolsFingerprint: codexLegacyDynamicToolsFingerprint(toolBridge.specs), }) : { project: true, reason: "per-turn-projection" }; embeddedAgentLog.info("codex app-server context-engine projection decision", { @@ -1582,12 +1585,20 @@ export async function runCodexAppServerAttempt( skillsPrompt: skillsCollaborationInstructions ? (params.skillsSnapshot?.prompt ?? "") : "", tools: toolBridge.availableSpecs, }); + const hostTrajectoryRecorder = ( + params as EmbeddedRunAttemptParams & { + trajectoryRecorder?: CodexHostTrajectoryRecorder | null; + } + ).trajectoryRecorder; const trajectoryRecorder = createCodexTrajectoryRecorder({ attempt: params, cwd: effectiveCwd, developerInstructions: buildRenderedCodexDeveloperInstructions(), prompt: codexTurnPromptText, + trajectoryRecorder: hostTrajectoryRecorder, + trajectorySessionFile: params.trajectorySessionFile, tools: toolBridge.availableSpecs, + warn: (message, fields) => embeddedAgentLog.warn(message, fields), }); let client: CodexAppServerClient; let thread: CodexAppServerThreadLifecycleBinding; @@ -3633,6 +3644,7 @@ export async function runCodexAppServerAttempt( sessionIdUsed: activeSessionId, sessionKey: contextSessionKey, sessionFile: activeSessionFile, + sessionTarget: params.sessionTarget, messagesSnapshot: finalMessages, prePromptMessageCount, tokenBudget: effectiveContextTokenBudget, diff --git a/extensions/codex/src/app-server/session-binding.ts b/extensions/codex/src/app-server/session-binding.ts index 6a8d59821cc4..c5cd8130d69c 100644 --- a/extensions/codex/src/app-server/session-binding.ts +++ b/extensions/codex/src/app-server/session-binding.ts @@ -11,11 +11,7 @@ import { } from "openclaw/plugin-sdk/agent-runtime"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import type { PluginStateSyncKeyedStore } from "openclaw/plugin-sdk/plugin-state-runtime"; -import { - loadSessionStore, - resolveSessionStoreEntry, - resolveStorePath, -} from "openclaw/plugin-sdk/session-store-runtime"; +import { getSessionEntry, resolveStorePath } from "openclaw/plugin-sdk/session-store-runtime"; import { z } from "zod"; import { CODEX_PLUGINS_MARKETPLACE_NAME, @@ -484,19 +480,19 @@ export async function reclaimCurrentCodexSessionGeneration(params: { return plan.result; } - // Only a stale stable-key owner needs filesystem authority. Resolve it before - // the second mutation so session JSON work never runs inside SQLite's write transaction. + // Only a stale stable-key owner needs session-store authority. Resolve it before + // the second mutation so the session read never runs inside the binding write transaction. try { const storePath = resolveStorePath(params.config?.session?.store, { agentId: params.identity.agentId, }); - const entry = resolveSessionStoreEntry({ - store: loadSessionStore(storePath, { - skipCache: true, - hydrateSkillPromptRefs: false, - }), + const entry = getSessionEntry({ + agentId: params.identity.agentId, + hydrateSkillPromptRefs: false, + readConsistency: "latest", sessionKey, - }).existing; + storePath, + }); if (entry?.sessionId !== params.identity.sessionId) { return false; } diff --git a/extensions/codex/src/app-server/session-history.test.ts b/extensions/codex/src/app-server/session-history.test.ts index 63195b65c710..0fccfa9b5d8c 100644 --- a/extensions/codex/src/app-server/session-history.test.ts +++ b/extensions/codex/src/app-server/session-history.test.ts @@ -3,6 +3,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { CURRENT_SESSION_VERSION } from "openclaw/plugin-sdk/agent-sessions"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; import { afterEach, describe, expect, it } from "vitest"; import { readCodexMirroredSessionHistoryMessages } from "./session-history.js"; @@ -62,6 +64,41 @@ function mirroredTarget(sessionFile: string) { }; } +async function writeSqliteSession(params: { storedSessionFile?: string } = {}): Promise<{ + marker: string; + sessionKey: string; +}> { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-session-history-sqlite-")); + tempDirs.push(dir); + const storePath = path.join(dir, "openclaw-agent.sqlite"); + const sessionId = "codex-sqlite-session"; + const sessionKey = "agent:main:codex-sqlite"; + const marker = `sqlite:main:${sessionId}:${storePath}`; + const scope = { + agentId: "main", + sessionId, + sessionKey, + storePath, + }; + await upsertSessionEntry({ + ...scope, + entry: { + sessionFile: params.storedSessionFile ?? marker, + sessionId, + updatedAt: 1, + }, + }); + await appendSessionTranscriptMessageByIdentity({ + ...scope, + message: { role: "user", content: "sqlite prompt", timestamp: 1 }, + }); + await appendSessionTranscriptMessageByIdentity({ + ...scope, + message: { role: "assistant", content: "sqlite answer", timestamp: 2 }, + }); + return { marker, sessionKey }; +} + describe("readCodexMirroredSessionHistoryMessages", () => { it("treats a missing mirrored session file as empty history", async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-session-history-")); @@ -84,6 +121,54 @@ describe("readCodexMirroredSessionHistoryMessages", () => { ).resolves.toBeUndefined(); }); + it("replays SQLite marker history by session identity", async () => { + const { marker, sessionKey } = await writeSqliteSession(); + + await expect( + readCodexMirroredSessionHistoryMessages({ + agentId: "main", + sessionFile: marker, + sessionId: "codex-sqlite-session", + sessionKey, + }), + ).resolves.toMatchObject([ + { role: "user", content: "sqlite prompt" }, + { role: "assistant", content: "sqlite answer" }, + ]); + }); + + it("resolves SQLite marker history when the caller has no session key", async () => { + const { marker } = await writeSqliteSession(); + + await expect( + readCodexMirroredSessionHistoryMessages({ + agentId: "main", + sessionFile: marker, + sessionId: "codex-sqlite-session", + }), + ).resolves.toMatchObject([ + { role: "user", content: "sqlite prompt" }, + { role: "assistant", content: "sqlite answer" }, + ]); + }); + + it("resolves synthesized SQLite markers for stale file-backed session metadata", async () => { + const { marker } = await writeSqliteSession({ + storedSessionFile: "/tmp/legacy-session.jsonl", + }); + + await expect( + readCodexMirroredSessionHistoryMessages({ + agentId: "main", + sessionFile: marker, + sessionId: "codex-sqlite-session", + }), + ).resolves.toMatchObject([ + { role: "user", content: "sqlite prompt" }, + { role: "assistant", content: "sqlite answer" }, + ]); + }); + it("replays only the branch selected by a leaf control", async () => { const sessionFile = await writeSession([ messageEntry({ id: "root", parentId: null, role: "user", content: "root prompt" }), diff --git a/extensions/codex/src/app-server/session-history.ts b/extensions/codex/src/app-server/session-history.ts index e0b50eab3e9f..ffe57e95eda9 100644 --- a/extensions/codex/src/app-server/session-history.ts +++ b/extensions/codex/src/app-server/session-history.ts @@ -11,9 +11,11 @@ import { parseSessionEntries, } from "openclaw/plugin-sdk/agent-sessions"; import { - resolveSessionTranscriptTarget, - type SessionTranscriptTargetParams, -} from "openclaw/plugin-sdk/session-transcript-runtime"; + listSessionEntries, + parseSqliteSessionFileMarker, + type SqliteSessionFileMarker, +} from "openclaw/plugin-sdk/session-store-runtime"; +import { readSessionTranscriptEvents } from "openclaw/plugin-sdk/session-transcript-runtime"; import { sanitizeCodexHistoryImagePayloads } from "./image-payload-sanitizer.js"; function isMissingFileError(error: unknown): boolean { @@ -32,9 +34,7 @@ export async function readCodexMirroredSessionHistoryMessages( target: CodexMirroredSessionHistoryTarget, ): Promise { try { - await resolveSessionTranscriptTarget(resolveCodexHistoryTranscriptTarget(target)); - const raw = await fs.readFile(target.sessionFile, "utf-8"); - const entries = parseSessionEntries(raw); + const entries = await readCodexMirroredSessionEntries(target); if (entries.length === 0) { return []; } @@ -42,7 +42,7 @@ export async function readCodexMirroredSessionHistoryMessages( if (firstEntry?.type !== "session" || typeof firstEntry.id !== "string") { return undefined; } - migrateSessionEntries(entries as SessionEntry[]); + migrateSessionEntries(entries); const sessionEntries = entries.filter((entry): entry is SessionEntry => { return ( entry !== null && @@ -64,13 +64,50 @@ export async function readCodexMirroredSessionHistoryMessages( } } -function resolveCodexHistoryTranscriptTarget( +async function readCodexMirroredSessionEntries( target: CodexMirroredSessionHistoryTarget, -): SessionTranscriptTargetParams { - return { - ...(target.agentId ? { agentId: target.agentId } : {}), - sessionFile: target.sessionFile, - sessionId: target.sessionId, - sessionKey: target.sessionKey ?? "", - }; +): Promise { + const sqliteMarker = parseSqliteSessionFileMarker(target.sessionFile); + if (sqliteMarker) { + if ( + sqliteMarker.sessionId !== target.sessionId || + (target.agentId !== undefined && sqliteMarker.agentId !== target.agentId) + ) { + return []; + } + const sessionKey = resolveSqliteMarkerSessionKey(target, sqliteMarker); + if (!sessionKey) { + return []; + } + return (await readSessionTranscriptEvents({ + agentId: sqliteMarker.agentId, + sessionId: sqliteMarker.sessionId, + sessionKey, + storePath: sqliteMarker.storePath, + })) as SessionEntry[]; + } + return parseSessionEntries(await fs.readFile(target.sessionFile, "utf-8")) as SessionEntry[]; +} + +function resolveSqliteMarkerSessionKey( + target: CodexMirroredSessionHistoryTarget, + marker: SqliteSessionFileMarker, +): string | undefined { + const explicitSessionKey = target.sessionKey?.trim(); + if (explicitSessionKey) { + return explicitSessionKey; + } + const entries = listSessionEntries({ + agentId: marker.agentId, + storePath: marker.storePath, + }); + const exactEntry = entries.find(({ entry }) => { + return entry.sessionId === marker.sessionId && entry.sessionFile === target.sessionFile; + }); + const sessionEntry = + exactEntry ?? + entries.find(({ entry }) => { + return entry.sessionId === marker.sessionId; + }); + return sessionEntry?.sessionKey; } diff --git a/extensions/codex/src/app-server/startup-binding.ts b/extensions/codex/src/app-server/startup-binding.ts index dcba9746891b..d5983ab45078 100644 --- a/extensions/codex/src/app-server/startup-binding.ts +++ b/extensions/codex/src/app-server/startup-binding.ts @@ -9,6 +9,7 @@ import { embeddedAgentLog, type EmbeddedRunAttemptParams, } from "openclaw/plugin-sdk/agent-harness-runtime"; +import { parseSqliteSessionFileMarker } from "openclaw/plugin-sdk/session-store-runtime"; import { resolveCodexAppServerHomeDir } from "./auth-bridge.js"; import { isJsonObject, type JsonValue } from "./protocol.js"; import type { @@ -127,6 +128,9 @@ async function listCodexAppServerRolloutFilesForThread( async function readCodexSessionRecordForSessionFile( sessionFile: string, ): Promise<(Record & { sessionKey: string }) | undefined> { + if (isSqliteSessionFileMarker(sessionFile)) { + return undefined; + } const sessionsFile = path.join(path.dirname(sessionFile), "sessions.json"); const resolvedSessionFile = path.resolve(sessionFile); let stat: Awaited>; @@ -175,6 +179,10 @@ async function readCodexSessionRecordForSessionFile( return found; } +function isSqliteSessionFileMarker(sessionFile: string | undefined): boolean { + return parseSqliteSessionFileMarker(sessionFile) !== undefined; +} + type CodexAppServerRolloutTokenSnapshot = { totalTokens?: number; modelContextWindow?: number; diff --git a/extensions/codex/src/app-server/thread-lifecycle.test.ts b/extensions/codex/src/app-server/thread-lifecycle.test.ts index 9b66ac351850..e16614e2c267 100644 --- a/extensions/codex/src/app-server/thread-lifecycle.test.ts +++ b/extensions/codex/src/app-server/thread-lifecycle.test.ts @@ -26,7 +26,9 @@ import { buildTurnStartParams, buildThreadResumeParams, buildThreadStartParams, + areCodexDynamicToolFingerprintsCompatible, codexDynamicToolsFingerprint, + codexLegacyDynamicToolsFingerprint, formatCodexThreadLifecycleTimingSummary, resolveCodexAppServerThreadModelSelection, resolveReasoningEffort, @@ -567,6 +569,36 @@ describe("Codex app-server native code mode config", () => { expect(searchableFingerprint).not.toBe(directFingerprint); }); + it("keeps hashed dynamic tool fingerprints compatible with legacy JSON bindings", () => { + const tools = [ + { + type: "function" as const, + name: "message", + description: "Send a visible message", + inputSchema: { + type: "object", + additionalProperties: false, + properties: { + text: { type: "string" }, + }, + required: ["text"], + }, + }, + ]; + const hashed = codexDynamicToolsFingerprint(tools); + const legacy = codexLegacyDynamicToolsFingerprint(tools); + + expect(hashed).toMatch(/^sha256:/); + expect(legacy).toContain('"name":"message"'); + expect( + areCodexDynamicToolFingerprintsCompatible({ + previous: legacy, + next: hashed, + nextLegacy: legacy, + }), + ).toBe(true); + }); + it("keeps OpenClaw skill catalogs out of developer instructions", () => { const params = createAttemptParams({ provider: "openai" }); params.skillsSnapshot = { diff --git a/extensions/codex/src/app-server/thread-lifecycle.ts b/extensions/codex/src/app-server/thread-lifecycle.ts index 9fc50e064d0a..121440c40ece 100644 --- a/extensions/codex/src/app-server/thread-lifecycle.ts +++ b/extensions/codex/src/app-server/thread-lifecycle.ts @@ -1,5 +1,5 @@ // Codex plugin module implements thread lifecycle behavior. -import crypto from "node:crypto"; +import * as crypto from "node:crypto"; import { buildSkillWorkshopPromptSection, embeddedAgentLog, @@ -445,8 +445,12 @@ export async function startOrResumeThread(params: { ...params.timing, enabled: params.timing?.enabled ?? isCodexAppServerProfilerEnabled(params.params.config), }); + const legacyDynamicToolsFingerprint = lifecycleTiming.measureSync( + "legacy-dynamic-tools-fingerprint", + () => legacyFingerprintDynamicTools(params.dynamicTools), + ); const dynamicToolsFingerprint = lifecycleTiming.measureSync("dynamic-tools-fingerprint", () => - fingerprintDynamicTools(params.dynamicTools), + hashDynamicToolFingerprint(legacyDynamicToolsFingerprint), ); const dynamicToolsContainDeferred = flattenCodexDynamicToolFunctions(params.dynamicTools).some( (tool) => tool.deferLoading === true, @@ -922,12 +926,13 @@ export async function startOrResumeThread(params: { !areDynamicToolFingerprintsCompatible( binding.dynamicToolsFingerprint, dynamicToolsFingerprint, + legacyDynamicToolsFingerprint, ) ) { assertCodexBindingMayBeReplaced(binding, "changing the dynamic tool catalog"); preserveExistingBinding = shouldStartTransientNoToolThread({ previous: binding.dynamicToolsFingerprint, - next: dynamicToolsFingerprint, + nextHasDynamicTools: params.dynamicTools.length > 0, }); if (preserveExistingBinding) { embeddedAgentLog.debug( @@ -2849,19 +2854,32 @@ export function codexDynamicToolsFingerprint(dynamicTools: CodexDynamicToolSpec[ return fingerprintDynamicTools(dynamicTools); } +export function codexLegacyDynamicToolsFingerprint(dynamicTools: CodexDynamicToolSpec[]): string { + return legacyFingerprintDynamicTools(dynamicTools); +} + export function areCodexDynamicToolFingerprintsCompatible(params: { previous?: string; next: string; + nextLegacy?: string; }): boolean { - return areDynamicToolFingerprintsCompatible(params.previous, params.next); + return areDynamicToolFingerprintsCompatible(params.previous, params.next, params.nextLegacy); } function fingerprintDynamicTools(dynamicTools: CodexDynamicToolSpec[]): string { + return hashDynamicToolFingerprint(legacyFingerprintDynamicTools(dynamicTools)); +} + +function legacyFingerprintDynamicTools(dynamicTools: CodexDynamicToolSpec[]): string { return JSON.stringify( dynamicTools.map(fingerprintDynamicToolSpec).toSorted(compareJsonFingerprint), ); } +function hashDynamicToolFingerprint(canonical: string): string { + return "sha256:" + crypto.createHash("sha256").update(canonical).digest("hex"); +} + function fingerprintUserMcpServersConfigPatch( configPatch: JsonObject | undefined, ): string | undefined { @@ -2959,20 +2977,34 @@ function readActiveCodexTurnIds(thread: unknown): string[] { .filter((turnId) => turnId.trim().length > 0); } -const EMPTY_DYNAMIC_TOOLS_FINGERPRINT = JSON.stringify([]); +const LEGACY_EMPTY_DYNAMIC_TOOLS_FINGERPRINT = legacyFingerprintDynamicTools([]); +const EMPTY_DYNAMIC_TOOLS_FINGERPRINT = hashDynamicToolFingerprint( + LEGACY_EMPTY_DYNAMIC_TOOLS_FINGERPRINT, +); -function areDynamicToolFingerprintsCompatible(previous: string | undefined, next: string): boolean { - return !previous || previous === next; +function areDynamicToolFingerprintsCompatible( + previous: string | undefined, + next: string, + nextLegacy?: string, +): boolean { + return !previous || previous === next || previous === nextLegacy; } function shouldStartTransientNoToolThread(params: { previous: string | undefined; - next: string; + nextHasDynamicTools: boolean; }): boolean { return Boolean( params.previous && - params.previous !== EMPTY_DYNAMIC_TOOLS_FINGERPRINT && - params.next === EMPTY_DYNAMIC_TOOLS_FINGERPRINT, + !isEmptyDynamicToolsFingerprint(params.previous) && + !params.nextHasDynamicTools, + ); +} + +function isEmptyDynamicToolsFingerprint(fingerprint: string): boolean { + return ( + fingerprint === EMPTY_DYNAMIC_TOOLS_FINGERPRINT || + fingerprint === LEGACY_EMPTY_DYNAMIC_TOOLS_FINGERPRINT ); } diff --git a/extensions/codex/src/app-server/trajectory.test.ts b/extensions/codex/src/app-server/trajectory.test.ts index ba4cac73b706..dba123f2f3c0 100644 --- a/extensions/codex/src/app-server/trajectory.test.ts +++ b/extensions/codex/src/app-server/trajectory.test.ts @@ -1,14 +1,19 @@ -// Codex tests cover trajectory plugin behavior. +// Codex tests cover SQLite-only trajectory plugin behavior. import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; import { + appendSqliteTrajectoryRuntimeEvents, + loadSqliteTrajectoryRuntimeEvents, + type SqliteTrajectoryRuntimeEventForTest, +} from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + type CodexHostTrajectoryRecorder, createCodexTrajectoryRecorder, recordCodexTrajectoryCompletion, recordCodexTrajectoryContext, - resolveCodexTrajectoryAppendFlags, - resolveCodexTrajectoryPointerFlags, } from "./trajectory.js"; type CodexTrajectoryRecorder = NonNullable>; @@ -33,62 +38,86 @@ function expectTrajectoryRecorder( if (recorder === null) { throw new Error("Expected Codex trajectory recorder"); } - expect(typeof recorder.recordEvent).toBe("function"); return recorder; } +function createMemoryHostTrajectoryRecorder(): { + events: Array<{ type: string; data?: Record }>; + recorder: CodexHostTrajectoryRecorder; +} { + const events: Array<{ type: string; data?: Record }> = []; + return { + events, + recorder: { + recordEvent: (type, data) => events.push({ type, data }), + flush: async () => undefined, + }, + }; +} + +function createMemoryBackedRecorder(params: { + tmpDir: string; + attempt?: Record; + tools?: Parameters[0]["tools"]; +}): { + events: Array<{ type: string; data?: Record }>; + recorder: CodexTrajectoryRecorder; +} { + const sessionId = (params.attempt?.sessionId as string | undefined) ?? "session-1"; + const host = createMemoryHostTrajectoryRecorder(); + const recorder = createCodexTrajectoryRecorder({ + cwd: params.tmpDir, + attempt: { + sessionFile: path.join(params.tmpDir, "session.jsonl"), + sessionId, + sessionKey: `agent:main:${sessionId}`, + runId: "run-1", + provider: "codex", + modelId: "gpt-5.4", + model: { api: "responses" }, + ...params.attempt, + } as never, + trajectoryRecorder: host.recorder, + trajectorySessionFile: `sqlite:main:${sessionId}:${path.join(params.tmpDir, "sessions.json")}`, + tools: params.tools, + env: {}, + }); + return { events: host.events, recorder: expectTrajectoryRecorder(recorder) }; +} + +function createSqliteHostTrajectoryRecorder(params: { + agentId: string; + sessionId: string; + storePath: string; +}): CodexHostTrajectoryRecorder { + const events: SqliteTrajectoryRuntimeEventForTest[] = []; + let seq = 0; + return { + recordEvent: (type, data) => { + events.push({ + traceSchema: "openclaw-trajectory", + schemaVersion: 1, + traceId: `${params.sessionId}:test`, + source: "runtime", + type, + ts: new Date(0).toISOString(), + seq, + sessionId: params.sessionId, + ...(data === undefined ? {} : { data }), + }); + seq += 1; + }, + flush: async () => { + appendSqliteTrajectoryRuntimeEvents(params, events); + events.length = 0; + }, + }; +} + describe("Codex trajectory recorder", () => { - it("keeps write flags usable when O_NOFOLLOW is unavailable", () => { - const constants = { - O_APPEND: 0x01, - O_CREAT: 0x02, - O_TRUNC: 0x04, - O_WRONLY: 0x08, - }; - - expect(resolveCodexTrajectoryAppendFlags(constants)).toBe(0x0b); - expect(resolveCodexTrajectoryPointerFlags(constants)).toBe(0x0e); - }); - - it("records by default unless explicitly disabled", async () => { - const tmpDir = makeTempDir(); - const sessionFile = path.join(tmpDir, "session.jsonl"); - const recorder = createCodexTrajectoryRecorder({ - cwd: tmpDir, - attempt: { - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:session-1", - runId: "run-1", - provider: "codex", - modelId: "gpt-5.4", - model: { api: "responses" }, - } as never, - env: {}, - }); - - const trajectoryRecorder = expectTrajectoryRecorder(recorder); - trajectoryRecorder.recordEvent("session.started", { - apiKey: "secret", - headers: [{ name: "Authorization", value: "Bearer sk-test-secret-token" }], - command: "curl -H 'Authorization: Bearer sk-other-secret-token'", - }); - await trajectoryRecorder.flush(); - - const filePath = path.join(tmpDir, "session.trajectory.jsonl"); - const content = fs.readFileSync(filePath, "utf8"); - expect(content).toContain('"type":"session.started"'); - expect(content).not.toContain("secret"); - expect(content).not.toContain("sk-test-secret-token"); - expect(content).not.toContain("sk-other-secret-token"); - if (process.platform !== "win32") { - expect(fs.statSync(filePath).mode & 0o777).toBe(0o600); - } - expect(fs.existsSync(path.join(tmpDir, "session.trajectory-path.json"))).toBe(true); - }); - - it("keeps recorded string values UTF-16 safe at the trajectory boundary", async () => { + it("rejects file-backed trajectory targets without creating sidecars", () => { const tmpDir = makeTempDir(); + const warn = vi.fn(); const recorder = createCodexTrajectoryRecorder({ cwd: tmpDir, attempt: { @@ -97,41 +126,83 @@ describe("Codex trajectory recorder", () => { model: { api: "responses" }, } as never, env: {}, + warn, }); - const prefix = "x".repeat(19_999); - const trajectoryRecorder = expectTrajectoryRecorder(recorder); - trajectoryRecorder.recordEvent("model.output", { text: `${prefix}😀` }); - await trajectoryRecorder.flush(); - - const parsed = JSON.parse( - fs.readFileSync(path.join(tmpDir, "session.trajectory.jsonl"), "utf8"), + expect(recorder).toBeNull(); + expect(warn).toHaveBeenCalledWith( + "codex trajectory capture requires a matching SQLite session target", + { sessionId: "session-1", reason: "non-sqlite-session-target" }, ); - expect(parsed.data.text).toBe(`${prefix}…`); + expect(fs.existsSync(path.join(tmpDir, "session.trajectory.jsonl"))).toBe(false); + expect(fs.existsSync(path.join(tmpDir, "session.trajectory-path.json"))).toBe(false); }); - it("records canonical OpenAI Codex app-server turns with Codex local attribution", async () => { + it("rejects a SQLite marker for a different session identity", () => { const tmpDir = makeTempDir(); - const sessionFile = path.join(tmpDir, "session.jsonl"); + const warn = vi.fn(); const recorder = createCodexTrajectoryRecorder({ cwd: tmpDir, attempt: { - sessionFile, + sessionFile: "sqlite:main:other:/tmp/openclaw-agent.sqlite", sessionId: "session-1", - sessionKey: "agent:main:session-1", - runId: "run-1", - provider: "openai", - modelId: "gpt-5.5", - model: { provider: "openai", api: "openai-responses" }, - runtimePlan: { - observability: { - resolvedRef: "openai/gpt-5.5", - provider: "openai", - modelId: "gpt-5.5", - harnessId: "codex", - }, - }, + model: { api: "responses" }, } as never, + trajectoryRecorder: createMemoryHostTrajectoryRecorder().recorder, + env: {}, + warn, + }); + + expect(recorder).toBeNull(); + expect(warn).toHaveBeenCalledWith( + "codex trajectory capture requires a matching SQLite session target", + { sessionId: "session-1", reason: "session-id-mismatch" }, + ); + }); + + it("warns when the SQLite host recorder is unavailable", () => { + const warn = vi.fn(); + const recorder = createCodexTrajectoryRecorder({ + cwd: makeTempDir(), + attempt: { + sessionFile: "sqlite:main:session-1:/tmp/openclaw-agent.sqlite", + sessionId: "session-1", + model: { api: "responses" }, + } as never, + env: {}, + warn, + }); + + expect(recorder).toBeNull(); + expect(warn).toHaveBeenCalledWith( + "codex trajectory capture requires the SQLite host recorder", + { sessionId: "session-1", reason: "sqlite-recorder-unavailable" }, + ); + }); + + it("stores SQLite-backed trajectory captures in the session database", async () => { + const tmpDir = makeTempDir(); + const storePath = path.join(tmpDir, "sessions", "sessions.json"); + const trajectorySessionFile = `sqlite:main:session-1:${storePath}`; + await upsertSessionEntry({ + agentId: "main", + sessionKey: "agent:main:session-1", + storePath, + entry: { sessionId: "session-1", sessionFile: trajectorySessionFile, updatedAt: 10 }, + }); + const recorder = createCodexTrajectoryRecorder({ + cwd: tmpDir, + attempt: { + sessionFile: path.join(tmpDir, "sessions", "session.jsonl"), + sessionId: "session-1", + model: { api: "responses" }, + } as never, + trajectoryRecorder: createSqliteHostTrajectoryRecorder({ + agentId: "main", + sessionId: "session-1", + storePath, + }), + trajectorySessionFile, env: {}, }); @@ -139,55 +210,52 @@ describe("Codex trajectory recorder", () => { trajectoryRecorder.recordEvent("session.started"); await trajectoryRecorder.flush(); - const parsed = JSON.parse( - fs.readFileSync(path.join(tmpDir, "session.trajectory.jsonl"), "utf8"), + expect(fs.readdirSync(path.join(tmpDir, "sessions"))).not.toEqual( + expect.arrayContaining(["session.trajectory.jsonl", "session.trajectory-path.json"]), ); - expect(parsed.provider).toBe("openai"); - expect(parsed.modelApi).toBe("openai-chatgpt-responses"); - expect(parsed.modelId).toBe("gpt-5.5"); + await expect( + loadSqliteTrajectoryRuntimeEvents({ agentId: "main", sessionId: "session-1", storePath }), + ).resolves.toEqual([expect.objectContaining({ type: "session.started" })]); }); - it("records namespace dynamic tools as callable trajectory tool definitions", async () => { + it("redacts secrets and keeps recorded strings UTF-16 safe", async () => { + const { events, recorder } = createMemoryBackedRecorder({ tmpDir: makeTempDir() }); + recorder.recordEvent("model.output", { + text: `${"x".repeat(19_999)}😀`, + apiKey: "secret", + authorization: "Bearer sk-test-secret-token", + }); + await recorder.flush(); + + expect(events[0]?.data?.text).toBe(`${"x".repeat(19_999)}…`); + expect(events[0]?.data?.apiKey).toBe(""); + expect(events[0]?.data?.authorization).toBe(""); + }); + + it("records namespace dynamic tools as callable trajectory definitions", async () => { + const tools = [ + { + type: "namespace" as const, + name: "openclaw", + description: "", + tools: [ + { + type: "function" as const, + name: "web_search", + description: "Search the web.", + inputSchema: { type: "object" }, + deferLoading: true, + }, + ], + }, + ]; const tmpDir = makeTempDir(); - const sessionFile = path.join(tmpDir, "session.jsonl"); - const init = { - cwd: tmpDir, - attempt: { - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:session-1", - runId: "run-1", - provider: "codex", - modelId: "gpt-5.4", - model: { api: "responses" }, - } as never, - env: {}, - tools: [ - { - type: "namespace", - name: "openclaw", - description: "", - tools: [ - { - type: "function", - name: "web_search", - description: "Search the web.", - inputSchema: { type: "object" }, - deferLoading: true, - }, - ], - }, - ], - } satisfies Parameters[0]; - const recorder = createCodexTrajectoryRecorder(init); + const init = createMemoryBackedRecorder({ tmpDir, tools }); - recordCodexTrajectoryContext(expectTrajectoryRecorder(recorder), init); - await recorder?.flush(); + recordCodexTrajectoryContext(init.recorder, { attempt: {} as never, cwd: tmpDir, tools }); + await init.recorder.flush(); - const parsed = JSON.parse( - fs.readFileSync(path.join(tmpDir, "session.trajectory.jsonl"), "utf8"), - ); - expect(parsed.data?.tools).toEqual([ + expect(init.events[0]?.data?.tools).toEqual([ { name: "web_search", description: "Search the web.", @@ -196,95 +264,25 @@ describe("Codex trajectory recorder", () => { ]); }); - it("sanitizes session ids when resolving an override directory", async () => { - const tmpDir = makeTempDir(); + it("honors explicit disablement without warning", () => { + const warn = vi.fn(); const recorder = createCodexTrajectoryRecorder({ - cwd: tmpDir, + cwd: makeTempDir(), attempt: { - sessionFile: path.join(tmpDir, "session.jsonl"), - sessionId: "../evil/session", - model: { api: "responses" }, - } as never, - env: { OPENCLAW_TRAJECTORY_DIR: tmpDir }, - }); - - const trajectoryRecorder = expectTrajectoryRecorder(recorder); - trajectoryRecorder.recordEvent("session.started"); - await trajectoryRecorder.flush(); - - expect(fs.existsSync(path.join(tmpDir, "___evil_session.jsonl"))).toBe(true); - }); - - it("honors explicit disablement", () => { - const tmpDir = makeTempDir(); - const recorder = createCodexTrajectoryRecorder({ - cwd: tmpDir, - attempt: { - sessionFile: path.join(tmpDir, "session.jsonl"), + sessionFile: "sqlite:main:session-1:/tmp/openclaw-agent.sqlite", sessionId: "session-1", model: { api: "responses" }, } as never, env: { OPENCLAW_TRAJECTORY: "0" }, + warn, }); expect(recorder).toBeNull(); - }); - - it("refuses to append through a symlinked parent directory", async () => { - const tmpDir = makeTempDir(); - const targetDir = path.join(tmpDir, "target"); - const linkDir = path.join(tmpDir, "link"); - fs.mkdirSync(targetDir); - fs.symlinkSync(targetDir, linkDir); - const recorder = createCodexTrajectoryRecorder({ - cwd: tmpDir, - attempt: { - sessionFile: path.join(linkDir, "session.jsonl"), - sessionId: "session-1", - model: { api: "responses" }, - } as never, - env: {}, - }); - - const trajectoryRecorder = expectTrajectoryRecorder(recorder); - trajectoryRecorder.recordEvent("session.started"); - await trajectoryRecorder.flush(); - - expect(fs.existsSync(path.join(targetDir, "session.trajectory.jsonl"))).toBe(false); - }); - - it("truncates events that exceed the runtime event byte limit", async () => { - const tmpDir = makeTempDir(); - const recorder = createCodexTrajectoryRecorder({ - cwd: tmpDir, - attempt: { - sessionFile: path.join(tmpDir, "session.jsonl"), - sessionId: "session-1", - model: { api: "responses" }, - } as never, - env: {}, - }); - - const trajectoryRecorder = expectTrajectoryRecorder(recorder); - trajectoryRecorder.recordEvent("context.compiled", { - fields: Object.fromEntries( - Array.from({ length: 100 }, (_, index) => [`field-${index}`, "x".repeat(3_000)]), - ), - }); - await trajectoryRecorder.flush(); - - const parsed = JSON.parse( - fs.readFileSync(path.join(tmpDir, "session.trajectory.jsonl"), "utf8"), - ) as { data?: { truncated?: boolean; reason?: string } }; - expect(parsed.data?.truncated).toBe(true); - expect(parsed.data?.reason).toBe("trajectory-event-size-limit"); + expect(warn).not.toHaveBeenCalled(); }); it("preserves usage when truncating oversized model completion events", async () => { - const tmpDir = makeTempDir(); - const sessionFile = path.join(tmpDir, "session.jsonl"); const attempt = { - sessionFile, sessionId: "session-1", sessionKey: "agent:main:session-1", runId: "run-1", @@ -299,14 +297,12 @@ describe("Codex trajectory recorder", () => { reasoningTokens: 2_038, total: 724_402, }; - const recorder = createCodexTrajectoryRecorder({ - cwd: tmpDir, + const { events, recorder } = createMemoryBackedRecorder({ + tmpDir: makeTempDir(), attempt, - env: {}, }); - const trajectoryRecorder = expectTrajectoryRecorder(recorder); - recordCodexTrajectoryCompletion(trajectoryRecorder, { + recordCodexTrajectoryCompletion(recorder, { attempt, threadId: "thread-1", turnId: "turn-1", @@ -321,193 +317,14 @@ describe("Codex trajectory recorder", () => { })), } as never, }); - await trajectoryRecorder.flush(); + await recorder.flush(); - const parsed = JSON.parse( - fs.readFileSync(path.join(tmpDir, "session.trajectory.jsonl"), "utf8"), - ); - expect(parsed.type).toBe("model.completed"); - expect(parsed.data).toMatchObject({ + expect(events[0]?.data).toMatchObject({ truncated: true, reason: "trajectory-event-size-limit", usage, }); - expect(parsed.data.messagesSnapshot).toBeUndefined(); - expect(parsed.data.droppedFields).toContain("messagesSnapshot"); - expect(Buffer.byteLength(JSON.stringify(parsed), "utf8")).toBeLessThanOrEqual(256 * 1024); - }); - - it("drops oversized preserved fields when needed to keep completion events bounded", async () => { - const tmpDir = makeTempDir(); - const sessionFile = path.join(tmpDir, "session.jsonl"); - const attempt = { - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:session-1", - runId: "run-1", - provider: "codex", - modelId: "gpt-5.4", - model: { api: "responses" }, - } as never; - const oversizedUsage = Object.fromEntries( - Array.from({ length: 100 }, (_value, index) => [`field-${index}`, "x".repeat(5_000)]), - ); - const recorder = createCodexTrajectoryRecorder({ - cwd: tmpDir, - attempt, - env: {}, - }); - - const trajectoryRecorder = expectTrajectoryRecorder(recorder); - recordCodexTrajectoryCompletion(trajectoryRecorder, { - attempt, - threadId: "thread-1", - turnId: "turn-1", - timedOut: false, - result: { - aborted: false, - attemptUsage: oversizedUsage, - assistantTexts: ["x".repeat(32_000)], - messagesSnapshot: [{ role: "assistant", content: "x".repeat(32_000) }], - } as never, - }); - await trajectoryRecorder.flush(); - - const parsed = JSON.parse( - fs.readFileSync(path.join(tmpDir, "session.trajectory.jsonl"), "utf8"), - ); - expect(parsed.data).toMatchObject({ - truncated: true, - reason: "trajectory-event-size-limit", - }); - expect(parsed.data.usage).toBeUndefined(); - expect(parsed.data.droppedFields).toEqual( - expect.arrayContaining(["usage", "assistantTexts", "messagesSnapshot"]), - ); - expect(Buffer.byteLength(JSON.stringify(parsed), "utf8")).toBeLessThanOrEqual(256 * 1024); - }); - - it("preserves usage on non-final oversized model completion events", async () => { - const tmpDir = makeTempDir(); - const sessionFile = path.join(tmpDir, "session.jsonl"); - const attempt = { - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:session-1", - runId: "run-1", - provider: "codex", - modelId: "gpt-5.4", - model: { api: "responses" }, - } as never; - const firstUsage = { - input: 384_954, - output: 5_624, - cacheRead: 333_824, - reasoningTokens: 2_038, - total: 724_402, - }; - const secondUsage = { input: 12, output: 3, total: 15 }; - const recorder = createCodexTrajectoryRecorder({ - cwd: tmpDir, - attempt, - env: {}, - }); - - const trajectoryRecorder = expectTrajectoryRecorder(recorder); - recordCodexTrajectoryCompletion(trajectoryRecorder, { - attempt, - threadId: "thread-1", - turnId: "turn-1", - timedOut: false, - result: { - aborted: false, - attemptUsage: firstUsage, - assistantTexts: ["first"], - messagesSnapshot: Array.from({ length: 20 }, (_value, index) => ({ - role: index % 2 === 0 ? "user" : "assistant", - content: `message-${index} ${"x".repeat(32_000)}`, - })), - } as never, - }); - recordCodexTrajectoryCompletion(trajectoryRecorder, { - attempt, - threadId: "thread-1", - turnId: "turn-2", - timedOut: false, - result: { - aborted: false, - attemptUsage: secondUsage, - assistantTexts: ["final answer"], - messagesSnapshot: [{ role: "assistant", content: "final answer" }], - } as never, - }); - await trajectoryRecorder.flush(); - - const events = fs - .readFileSync(path.join(tmpDir, "session.trajectory.jsonl"), "utf8") - .trim() - .split(/\r?\n/u) - .map((line) => JSON.parse(line)); - expect(events).toHaveLength(2); - expect(events[0].data).toMatchObject({ - truncated: true, - usage: firstUsage, - }); - expect(events[1].data).toMatchObject({ - turnId: "turn-2", - usage: secondUsage, - assistantTexts: ["final answer"], - }); - expect(events[1].data.truncated).toBeUndefined(); - }); - - it("redacts secrets before preserving usage in truncated completion events", async () => { - const tmpDir = makeTempDir(); - const sessionFile = path.join(tmpDir, "session.jsonl"); - const attempt = { - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:session-1", - runId: "run-1", - provider: "codex", - modelId: "gpt-5.4", - model: { api: "responses" }, - } as never; - const recorder = createCodexTrajectoryRecorder({ - cwd: tmpDir, - attempt, - env: {}, - }); - - const trajectoryRecorder = expectTrajectoryRecorder(recorder); - recordCodexTrajectoryCompletion(trajectoryRecorder, { - attempt, - threadId: "thread-1", - turnId: "turn-1", - timedOut: false, - result: { - aborted: false, - attemptUsage: { - total: 1, - apiKey: "sk-test-secret-token", - authorization: "Bearer sk-other-secret-token", - }, - assistantTexts: ["done"], - messagesSnapshot: Array.from({ length: 20 }, (_value, index) => ({ - role: index % 2 === 0 ? "user" : "assistant", - content: `message-${index} ${"x".repeat(32_000)}`, - })), - } as never, - }); - await trajectoryRecorder.flush(); - - const parsed = JSON.parse( - fs.readFileSync(path.join(tmpDir, "session.trajectory.jsonl"), "utf8"), - ); - const preservedUsage = JSON.stringify(parsed.data.usage); - expect(parsed.data.truncated).toBe(true); - expect(preservedUsage).toContain("redacted"); - expect(preservedUsage).not.toContain("sk-test-secret-token"); - expect(preservedUsage).not.toContain("sk-other-secret-token"); + expect(events[0]?.data?.messagesSnapshot).toBeUndefined(); + expect(events[0]?.data?.droppedFields).toContain("messagesSnapshot"); }); }); diff --git a/extensions/codex/src/app-server/trajectory.ts b/extensions/codex/src/app-server/trajectory.ts index e1b0c8f9b5a8..8bf003179f99 100644 --- a/extensions/codex/src/app-server/trajectory.ts +++ b/extensions/codex/src/app-server/trajectory.ts @@ -1,26 +1,18 @@ /** - * Records optional Codex runtime trajectory sidecars with bounded, redacted - * context and completion events. + * Records optional Codex runtime trajectory events with bounded, redacted + * context and completion payloads. */ -import nodeFs from "node:fs"; -import fs from "node:fs/promises"; -import path from "node:path"; -import { resolveUserPath } from "openclaw/plugin-sdk/agent-harness-runtime"; import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptResult, } from "openclaw/plugin-sdk/agent-harness-runtime"; -import { - appendRegularFile, - resolveRegularFileAppendFlags, -} from "openclaw/plugin-sdk/security-runtime"; +import { parseSqliteSessionFileMarker } from "openclaw/plugin-sdk/session-store-runtime"; import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { resolveCodexLocalRuntimeAttribution } from "./local-runtime-attribution.js"; import { flattenCodexDynamicToolFunctions, type CodexDynamicToolSpec } from "./protocol.js"; /** Runtime trajectory recorder used by Codex run attempts and event projectors. */ export type CodexTrajectoryRecorder = { - filePath: string; recordEvent: (type: string, data?: Record) => void; flush: () => Promise; }; @@ -30,8 +22,11 @@ type CodexTrajectoryInit = { cwd: string; developerInstructions?: string; prompt?: string; + trajectoryRecorder?: CodexHostTrajectoryRecorder | null; + trajectorySessionFile?: string; tools?: CodexDynamicToolSpec[]; env?: NodeJS.ProcessEnv; + warn?: (message: string, fields: Record) => void; }; const SENSITIVE_FIELD_RE = /(?:authorization|cookie|credential|key|password|passwd|secret|token)/iu; @@ -39,50 +34,29 @@ const PRIVATE_PAYLOAD_FIELD_RE = /(?:image|screenshot|attachment|fileData|dataUr const AUTHORIZATION_VALUE_RE = /\b(Bearer|Basic)\s+[A-Za-z0-9+/._~=-]{8,}/giu; const JWT_VALUE_RE = /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/gu; const COOKIE_PAIR_RE = /\b([A-Za-z][A-Za-z0-9_.-]{1,64})=([A-Za-z0-9+/._~%=-]{16,})(?=;|\s|$)/gu; -const TRAJECTORY_RUNTIME_FILE_MAX_BYTES = 50 * 1024 * 1024; const TRAJECTORY_RUNTIME_EVENT_MAX_BYTES = 256 * 1024; const TRAJECTORY_RUNTIME_OVERSIZE_PRESERVED_DATA_KEYS = ["usage", "promptCache"] as const; -type CodexTrajectoryOpenFlagConstants = Pick< - typeof nodeFs.constants, - "O_APPEND" | "O_CREAT" | "O_TRUNC" | "O_WRONLY" -> & - Partial>; +type CodexTrajectorySink = { + flush: () => Promise; + write: (event: CodexTrajectoryEvent) => void; +}; -/** Resolves secure append flags for trajectory runtime files. */ -export function resolveCodexTrajectoryAppendFlags( - constants: CodexTrajectoryOpenFlagConstants = nodeFs.constants, -): number { - return resolveRegularFileAppendFlags(constants); -} +export type CodexHostTrajectoryRecorder = { + recordEvent: (type: string, data?: Record) => void; + flush: () => Promise; +}; -/** Resolves secure create/truncate flags for trajectory pointer files. */ -export function resolveCodexTrajectoryPointerFlags( - constants: CodexTrajectoryOpenFlagConstants = nodeFs.constants, -): number { - const noFollow = constants.O_NOFOLLOW; - return ( - constants.O_CREAT | - constants.O_TRUNC | - constants.O_WRONLY | - (typeof noFollow === "number" ? noFollow : 0) - ); -} +type CodexTrajectoryEvent = Record & { + data?: Record; + type: string; +}; -async function safeAppendTrajectoryFile(filePath: string, line: string): Promise { - await appendRegularFile({ - filePath, - content: line, - maxFileBytes: TRAJECTORY_RUNTIME_FILE_MAX_BYTES, - rejectSymlinkParents: true, - }); -} - -function boundedTrajectoryLine(event: Record): string | undefined { +function boundedTrajectoryEvent(event: Record): CodexTrajectoryEvent | undefined { const line = JSON.stringify(event); const bytes = Buffer.byteLength(line, "utf8"); if (bytes <= TRAJECTORY_RUNTIME_EVENT_MAX_BYTES) { - return `${line}\n`; + return event as CodexTrajectoryEvent; } const originalData = @@ -97,7 +71,7 @@ function boundedTrajectoryLine(event: Record): string | undefin limitBytes: TRAJECTORY_RUNTIME_EVENT_MAX_BYTES, reason: "trajectory-event-size-limit", }; - const buildTruncatedLine = (includeDroppedFields: boolean): string | undefined => { + const buildTruncatedEvent = (includeDroppedFields: boolean): CodexTrajectoryEvent | undefined => { const data: Record = { ...baseData }; for (const key of TRAJECTORY_RUNTIME_OVERSIZE_PRESERVED_DATA_KEYS) { if (preservedDataKeys.has(key)) { @@ -110,14 +84,15 @@ function boundedTrajectoryLine(event: Record): string | undefin data.droppedFields = droppedFields; } } - const truncated = JSON.stringify({ ...event, data }); + const truncatedEvent = { ...event, data }; + const truncated = JSON.stringify(truncatedEvent); if (Buffer.byteLength(truncated, "utf8") <= TRAJECTORY_RUNTIME_EVENT_MAX_BYTES) { - return `${truncated}\n`; + return truncatedEvent as CodexTrajectoryEvent; } return undefined; }; - let best = buildTruncatedLine(true) ?? buildTruncatedLine(false); + let best = buildTruncatedEvent(true) ?? buildTruncatedEvent(false); if (!best) { return undefined; } @@ -127,7 +102,7 @@ function boundedTrajectoryLine(event: Record): string | undefin continue; } preservedDataKeys.add(key); - const next = buildTruncatedLine(true) ?? buildTruncatedLine(false); + const next = buildTruncatedEvent(true) ?? buildTruncatedEvent(false); if (next) { best = next; continue; @@ -137,55 +112,17 @@ function boundedTrajectoryLine(event: Record): string | undefin return best; } -function resolveTrajectoryPointerFilePath(sessionFile: string): string { - return sessionFile.endsWith(".jsonl") - ? `${sessionFile.slice(0, -".jsonl".length)}.trajectory-path.json` - : `${sessionFile}.trajectory-path.json`; -} - -function writeTrajectoryPointerBestEffort(params: { - filePath: string; - sessionFile: string; - sessionId: string; -}): void { - const pointerPath = resolveTrajectoryPointerFilePath(params.sessionFile); - try { - const pointerDir = path.resolve(path.dirname(pointerPath)); - if (nodeFs.lstatSync(pointerDir).isSymbolicLink()) { - return; - } - try { - if (nodeFs.lstatSync(pointerPath).isSymbolicLink()) { - return; - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - return; - } - } - const fd = nodeFs.openSync(pointerPath, resolveCodexTrajectoryPointerFlags(), 0o600); - try { - nodeFs.writeFileSync( - fd, - `${JSON.stringify( - { - traceSchema: "openclaw-trajectory-pointer", - schemaVersion: 1, - sessionId: params.sessionId, - runtimeFile: params.filePath, - }, - null, - 2, - )}\n`, - "utf8", - ); - nodeFs.fchmodSync(fd, 0o600); - } finally { - nodeFs.closeSync(fd); - } - } catch { - // Pointer files are best-effort; the runtime sidecar itself is authoritative. - } +function createCodexHostTrajectorySink(params: { + recorder: CodexHostTrajectoryRecorder; +}): CodexTrajectorySink { + return { + write: (event) => { + params.recorder.recordEvent(event.type, event.data); + }, + flush: async () => { + await params.recorder.flush(); + }, + }; } /** Creates a trajectory recorder when trajectory capture is enabled for the environment. */ @@ -198,27 +135,29 @@ export function createCodexTrajectoryRecorder( return null; } - const filePath = resolveTrajectoryFilePath({ - env, - sessionFile: params.attempt.sessionFile, - sessionId: params.attempt.sessionId, - }); - const ready = fs - .mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 }) - .catch(() => undefined); - writeTrajectoryPointerBestEffort({ - filePath, - sessionFile: params.attempt.sessionFile, - sessionId: params.attempt.sessionId, - }); - let queue = Promise.resolve(); + const sessionFile = params.trajectorySessionFile ?? params.attempt.sessionFile; + const sqliteMarker = parseSqliteSessionFileMarker(sessionFile); + if (!sqliteMarker || sqliteMarker.sessionId !== params.attempt.sessionId) { + params.warn?.("codex trajectory capture requires a matching SQLite session target", { + sessionId: params.attempt.sessionId, + reason: sqliteMarker ? "session-id-mismatch" : "non-sqlite-session-target", + }); + return null; + } + if (!params.trajectoryRecorder) { + params.warn?.("codex trajectory capture requires the SQLite host recorder", { + sessionId: params.attempt.sessionId, + reason: "sqlite-recorder-unavailable", + }); + return null; + } + const sink = createCodexHostTrajectorySink({ recorder: params.trajectoryRecorder }); let seq = 0; const attribution = resolveCodexLocalRuntimeAttribution(params.attempt); return { - filePath, recordEvent: (type, data) => { - const event = { + const event = boundedTrajectoryEvent({ traceSchema: "openclaw-trajectory", schemaVersion: 1, traceId: params.attempt.sessionId, @@ -235,19 +174,12 @@ export function createCodexTrajectoryRecorder( modelId: params.attempt.modelId, modelApi: attribution.api, data: data ? sanitizeValue(data) : undefined, - }; - const line = boundedTrajectoryLine(event); - if (!line) { - return; + }); + if (event) { + sink.write(event); } - queue = queue - .then(() => ready) - .then(() => safeAppendTrajectoryFile(filePath, line)) - .catch(() => undefined); - }, - flush: async () => { - await queue; }, + flush: sink.flush, }; } @@ -306,38 +238,6 @@ function parseTrajectoryEnabled(env: NodeJS.ProcessEnv): boolean { return true; } -function resolveTrajectoryFilePath(params: { - env: NodeJS.ProcessEnv; - sessionFile: string; - sessionId: string; -}): string { - const dirOverride = params.env.OPENCLAW_TRAJECTORY_DIR?.trim(); - if (dirOverride) { - return resolveContainedPath( - resolveUserPath(dirOverride), - `${safeTrajectorySessionFileName(params.sessionId)}.jsonl`, - ); - } - return params.sessionFile.endsWith(".jsonl") - ? `${params.sessionFile.slice(0, -".jsonl".length)}.trajectory.jsonl` - : `${params.sessionFile}.trajectory.jsonl`; -} - -function safeTrajectorySessionFileName(sessionId: string): string { - const safe = sessionId.replaceAll(/[^A-Za-z0-9_-]/g, "_").slice(0, 120); - return /[A-Za-z0-9]/u.test(safe) ? safe : "session"; -} - -function resolveContainedPath(baseDir: string, fileName: string): string { - const resolvedBase = path.resolve(baseDir); - const resolvedFile = path.resolve(resolvedBase, fileName); - const relative = path.relative(resolvedBase, resolvedFile); - if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) { - throw new Error("Trajectory file path escaped its configured directory"); - } - return resolvedFile; -} - function toTrajectoryToolDefinitions( tools: readonly CodexDynamicToolSpec[] | undefined, ): Array<{ name: string; description?: string; parameters?: unknown }> | undefined { @@ -362,8 +262,8 @@ function toTrajectoryToolDefinitions( } function sanitizeValue(value: unknown, depth = 0, key = ""): unknown { - // Trajectory files may be inspected outside the live process, so redact - // credentials and private payloads before queueing the line for disk writes. + // Trajectory exports may leave the live process, so redact credentials and + // private payloads before passing events to the SQLite host recorder. if (value == null || typeof value === "boolean" || typeof value === "number") { return value; } diff --git a/extensions/codex/src/app-server/transcript-mirror.test.ts b/extensions/codex/src/app-server/transcript-mirror.test.ts index d8bf0959ef11..36ca17c9d2d7 100644 --- a/extensions/codex/src/app-server/transcript-mirror.test.ts +++ b/extensions/codex/src/app-server/transcript-mirror.test.ts @@ -10,6 +10,8 @@ import { resetGlobalHookRunner, } from "openclaw/plugin-sdk/hook-runtime"; import { createMockPluginRegistry } from "openclaw/plugin-sdk/plugin-test-runtime"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { readSessionTranscriptEvents } from "openclaw/plugin-sdk/session-transcript-runtime"; import { castAgentMessage, makeAgentAssistantMessage, @@ -40,7 +42,7 @@ vi.mock("openclaw/plugin-sdk/session-transcript-runtime", async (importOriginal) type MirroredAgentMessage = Extract; -// Mirrors transcript-mirror.ts's fallback fingerprint exactly so test +// Mirrors transcript-mirror.ts's content fingerprint exactly so test // expectations stay in sync without exposing the helper publicly. function expectedFingerprint(message: MirroredAgentMessage): string { const payload = JSON.stringify({ role: message.role, content: message.content }); @@ -64,26 +66,6 @@ afterEach(async () => { } }); -async function createTempSessionFile() { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-transcript-")); - tempDirs.push(dir); - return path.join(dir, "session.jsonl"); -} - -async function initializeSessionTranscript(sessionFile: string, sessionId: string): Promise { - await fs.writeFile( - sessionFile, - `${JSON.stringify({ - type: "session", - version: 3, - id: sessionId, - timestamp: new Date().toISOString(), - cwd: process.cwd(), - })}\n`, - "utf8", - ); -} - async function makeRoot(prefix: string): Promise { const root = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); tempDirs.push(root); @@ -121,20 +103,84 @@ describe("buildCodexUserPromptMessage", () => { }); }); -function parseJsonLines(raw: string): T[] { - const records: T[] = []; - for (const line of raw.trim().split("\n")) { - if (line.length > 0) { - records.push(JSON.parse(line) as T); - } - } - return records; +function readEventMessages(events: unknown[]): Array<{ role?: string; text?: string }> { + return events + .map((event) => + event && typeof event === "object" ? (event as { message?: unknown }).message : undefined, + ) + .filter((message): message is { role?: string; content?: unknown } => + Boolean(message && typeof message === "object"), + ) + .map((message) => { + const content = Array.isArray(message.content) + ? message.content.find((part): part is { text: string } => + Boolean(part && typeof part === "object" && typeof part.text === "string"), + )?.text + : typeof message.content === "string" + ? message.content + : undefined; + return { role: message.role, text: content }; + }); +} + +async function createSqliteMirrorTarget(prefix: string, options: { sessionId?: string } = {}) { + const root = await makeRoot(prefix); + const agentId = "main"; + const sessionId = options.sessionId ?? "session-1"; + const sessionKey = `agent:${agentId}:${sessionId}`; + const storePath = path.join(root, "openclaw-agent.sqlite"); + await upsertSessionEntry({ + agentId, + sessionKey, + storePath, + entry: { + sessionFile: `sqlite:${agentId}:${sessionId}:${storePath}`, + sessionId, + updatedAt: 1, + }, + }); + return { + agentId, + sessionId, + sessionKey, + storePath, + bogusSessionFile: path.join(root, "should-not-be-created.jsonl"), + }; +} + +async function readMirrorEvents(target: { + agentId: string; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise { + return await readSessionTranscriptEvents(target); +} + +async function readMirrorRaw(target: { + agentId: string; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise { + return (await readMirrorEvents(target)).map((event) => JSON.stringify(event)).join("\n"); +} + +async function readMirrorMessages(target: { + agentId: string; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise> { + return readEventMessages(await readMirrorEvents(target)); } describe("importCodexThreadHistoryToTranscript", () => { it("imports only bounded user-visible conversation items with stable identities", async () => { - const sessionFile = await createTempSessionFile(); - await initializeSessionTranscript(sessionFile, "session-history"); + const target = await createSqliteMirrorTarget("openclaw-codex-history-", { + sessionId: "session-history", + }); + const sessionFile = `sqlite:${target.agentId}:${target.sessionId}:${target.storePath}`; const thread = { id: "thread-history", cwd: "/workspace/project", @@ -201,14 +247,16 @@ describe("importCodexThreadHistoryToTranscript", () => { importCodexThreadHistoryToTranscript({ thread, throughTurnId: "turn-1", - sessionFile, + storePath: target.storePath, sessionId: "session-history", - sessionKey: "agent:main:dashboard:history", + sessionKey: target.sessionKey, + agentId: target.agentId, }), ).resolves.toEqual({ importedMessages: 2, omittedMessages: 0 }); - const raw = await fs.readFile(sessionFile, "utf8"); - const messages = parseJsonLines<{ message?: AgentMessage; type?: string }>(raw) + const events = await readMirrorEvents(target); + const raw = events.map((event) => JSON.stringify(event)).join("\n"); + const messages = (events as Array<{ message?: AgentMessage; type?: string }>) .filter((event) => event.type === "message") .map((event) => event.message); expect(messages).toMatchObject([ @@ -237,7 +285,8 @@ describe("importCodexThreadHistoryToTranscript", () => { readCodexMirroredSessionHistoryMessages({ sessionFile, sessionId: "session-history", - sessionKey: "agent:main:dashboard:history", + sessionKey: target.sessionKey, + agentId: target.agentId, }), ).resolves.toMatchObject([ { role: "user", content: "Review this image\n[Image attachment]" }, @@ -253,8 +302,9 @@ describe("importCodexThreadHistoryToTranscript", () => { }); it("keeps the newest 200 visible messages and deduplicates a retried import", async () => { - const sessionFile = await createTempSessionFile(); - await initializeSessionTranscript(sessionFile, "session-bounded-history"); + const target = await createSqliteMirrorTarget("openclaw-codex-bounded-history-", { + sessionId: "session-bounded-history", + }); const thread = { id: "thread-bounded-history", turns: Array.from({ length: 205 }, (_, index) => ({ @@ -274,9 +324,10 @@ describe("importCodexThreadHistoryToTranscript", () => { const importParams = { thread, throughTurnId: "turn-204", - sessionFile, + storePath: target.storePath, sessionId: "session-bounded-history", - sessionKey: "agent:main:dashboard:bounded-history", + sessionKey: target.sessionKey, + agentId: target.agentId, }; await expect(importCodexThreadHistoryToTranscript(importParams)).resolves.toEqual({ @@ -288,8 +339,8 @@ describe("importCodexThreadHistoryToTranscript", () => { omittedMessages: 5, }); - const raw = await fs.readFile(sessionFile, "utf8"); - const messages = parseJsonLines<{ message?: AgentMessage; type?: string }>(raw) + const events = await readMirrorEvents(target); + const messages = (events as Array<{ message?: AgentMessage; type?: string }>) .filter((event) => event.type === "message") .map((event) => event.message); expect(messages).toHaveLength(200); @@ -298,8 +349,10 @@ describe("importCodexThreadHistoryToTranscript", () => { }); it("assigns canonical assistant attribution and numeric fallback timestamps", async () => { - const sessionFile = await createTempSessionFile(); - await initializeSessionTranscript(sessionFile, "session-fallback-history"); + const target = await createSqliteMirrorTarget("openclaw-codex-fallback-history-", { + sessionId: "session-fallback-history", + }); + const sessionFile = `sqlite:${target.agentId}:${target.sessionId}:${target.storePath}`; const thread = { id: "thread-fallback-history", modelProvider: "source-provider", @@ -326,15 +379,17 @@ describe("importCodexThreadHistoryToTranscript", () => { await importCodexThreadHistoryToTranscript({ thread, throughTurnId: "turn-without-time", - sessionFile, + storePath: target.storePath, sessionId: "session-fallback-history", - sessionKey: "agent:main:dashboard:fallback-history", + sessionKey: target.sessionKey, + agentId: target.agentId, }); const history = await readCodexMirroredSessionHistoryMessages({ sessionFile, sessionId: "session-fallback-history", - sessionKey: "agent:main:dashboard:fallback-history", + sessionKey: target.sessionKey, + agentId: target.agentId, }); expect(history).toMatchObject([ { role: "user", content: "Earlier prompt", timestamp: expect.any(Number) }, @@ -569,8 +624,8 @@ describe("projectBoundedCodexThreadHistory", () => { }); describe("mirrorCodexAppServerTranscript", () => { - it("mirrors user, assistant, and tool result messages into the embedded-agent transcript", async () => { - const sessionFile = await createTempSessionFile(); + it("mirrors user, assistant, and tool result messages by SQLite identity", async () => { + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-basic-"); const userMessage = makeAgentUserMessage({ content: [{ type: "text", text: "hello" }], timestamp: Date.now(), @@ -583,25 +638,17 @@ describe("mirrorCodexAppServerTranscript", () => { role: "toolResult", toolCallId: "call-1", toolName: "read", - content: [ - { - type: "toolResult", - toolCallId: "call-1", - content: "read output", - }, - ], + content: [{ type: "toolResult", toolCallId: "call-1", content: "read output" }], timestamp: Date.now() + 2, }) as MirroredAgentMessage; await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [userMessage, assistantMessage, toolResultMessage], idempotencyScope: "scope-1", }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain('"role":"user"'); expect(raw).toContain('"content":[{"type":"text","text":"hello"}]'); expect(raw).toContain('"role":"assistant"'); @@ -616,10 +663,14 @@ describe("mirrorCodexAppServerTranscript", () => { expect(raw).toContain( `"idempotencyKey":"scope-1:toolResult:${expectedFingerprint(toolResultMessage)}"`, ); + await expect(fs.readFile(target.bogusSessionFile, "utf8")).rejects.toHaveProperty( + "code", + "ENOENT", + ); }); it("preserves gateway user-turn identity across Codex transcript mirroring", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-user-identity-"); const userMessage = castAgentMessage({ ...makeAgentUserMessage({ content: [{ type: "text", text: "client prompt" }], @@ -629,35 +680,29 @@ describe("mirrorCodexAppServerTranscript", () => { }) as MirroredAgentMessage; const first = await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:main", + ...target, messages: [userMessage], idempotencyScope: "codex-app-server:thread-1", }); const second = await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:main", + ...target, messages: [userMessage], idempotencyScope: "codex-app-server:thread-1", }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain('"idempotencyKey":"client-run:user"'); expect(raw).toContain('"mirrorOrigin":"codex-app-server"'); expect(raw).not.toContain('"idempotencyKey":"codex-app-server:thread-1:'); expect(first.userMessagesPresent).toHaveLength(1); expect(second.userMessagesPresent).toHaveLength(1); expect( - parseJsonLines<{ message?: { role?: string } }>(raw).filter( - (record) => record.message?.role === "user", - ), + (await readMirrorMessages(target)).filter((message) => message.role === "user"), ).toHaveLength(1); }); it("emits message-bearing updates for newly appended mirrored messages only", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-live-updates-"); const userMessage = attachCodexMirrorIdentity( makeAgentUserMessage({ content: [{ type: "text", text: "show me live" }], @@ -667,16 +712,12 @@ describe("mirrorCodexAppServerTranscript", () => { ); const firstMirror = await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:main", + ...target, messages: [userMessage], idempotencyScope: "codex-app-server:thread-1", }); const secondMirror = await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:main", + ...target, messages: [userMessage], idempotencyScope: "codex-app-server:thread-1", }); @@ -685,8 +726,8 @@ describe("mirrorCodexAppServerTranscript", () => { ([update]) => update as Record & { update?: Record }, ); expect(updates).toHaveLength(1); - expect(updates[0]?.sessionFile).toBe(sessionFile); - expect(updates[0]?.sessionKey).toBe("agent:main:main"); + expect(updates[0]?.sessionKey).toBe(target.sessionKey); + expect(updates[0]?.storePath).toBe(target.storePath); expect(updates[0]?.update?.messageId).toEqual(expect.any(String)); expect(updates[0]?.update?.message).toMatchObject({ role: "user", @@ -708,99 +749,11 @@ describe("mirrorCodexAppServerTranscript", () => { }); }); - it("reports final assistant ownership for new and idempotent mirrors", async () => { - const sessionFile = await createTempSessionFile(); - const assistantMessage = attachCodexMirrorIdentity( - makeAgentAssistantMessage({ - content: [{ type: "text", text: "owned once" }], - timestamp: Date.now(), - }), - "turn-1:assistant", - ); - - const firstMirror = await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:main", - messages: [assistantMessage], - idempotencyScope: "codex-app-server:thread-1", - }); - const secondMirror = await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:main", - messages: [assistantMessage], - idempotencyScope: "codex-app-server:thread-1", - }); - - expect(firstMirror.assistantMirrorIdentitiesOwned).toEqual(["turn-1:assistant"]); - expect(secondMirror.assistantMirrorIdentitiesOwned).toEqual(["turn-1:assistant"]); - const records = parseJsonLines<{ type?: string; message?: { role?: string } }>( - await fs.readFile(sessionFile, "utf8"), - ); - expect(records.filter((record) => record.message?.role === "assistant")).toHaveLength(1); - }); - - it("keeps assistant ownership when live update publication fails", async () => { - publishSessionTranscriptUpdateByIdentityMock.mockRejectedValueOnce(new Error("publish failed")); - const sessionFile = await createTempSessionFile(); - const assistantMessage = attachCodexMirrorIdentity( - makeAgentAssistantMessage({ - content: [{ type: "text", text: "durably persisted" }], - timestamp: Date.now(), - }), - "turn-1:assistant", - ); - - const result = await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:main", - messages: [assistantMessage], - idempotencyScope: "codex-app-server:thread-1", - }); - - expect(result.assistantMirrorIdentitiesOwned).toEqual(["turn-1:assistant"]); - expect(await fs.readFile(sessionFile, "utf8")).toContain('"role":"assistant"'); - }); - - it("leaves the assistant unowned when transcript persistence fails", async () => { - const root = await makeRoot("openclaw-codex-transcript-failure-"); - const invalidParent = path.join(root, "not-a-directory"); - await fs.writeFile(invalidParent, "file blocks transcript directory creation", "utf8"); - const assistantMessage = attachCodexMirrorIdentity( - makeAgentAssistantMessage({ - content: [{ type: "text", text: "needs fallback persistence" }], - timestamp: Date.now(), - }), - "turn-1:assistant", - ); - - const assistantTranscriptOwned = await mirrorTranscriptBestEffort({ - params: { - sessionFile: path.join(invalidParent, "session.jsonl"), - sessionId: "session-1", - suppressNextUserMessagePersistence: true, - } as Parameters[0]["params"], - result: { - messagesSnapshot: [assistantMessage], - } as Parameters[0]["result"], - notifyUserMessagePersisted: vi.fn(), - cwd: root, - threadId: "thread-1", - turnId: "turn-1", - }); - - expect(assistantTranscriptOwned).toBe(false); - }); - it("emits stable sequence numbers for multi-message mirror batches", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-seq-"); await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "agent:main:main", + ...target, messages: [ attachCodexMirrorIdentity( makeAgentUserMessage({ @@ -832,30 +785,43 @@ describe("mirrorCodexAppServerTranscript", () => { ).toEqual(["user", "assistant"]); }); - it("creates the transcript directory on first mirror", async () => { - const root = await makeRoot("openclaw-codex-transcript-missing-dir-"); - const sessionFile = path.join(root, "nested", "sessions", "session.jsonl"); + it("keeps assistant ownership when live update publication fails", async () => { + publishSessionTranscriptUpdateByIdentityMock.mockRejectedValueOnce(new Error("publish failed")); + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-publish-failure-"); + const assistantMessage = attachCodexMirrorIdentity( + makeAgentAssistantMessage({ + content: [{ type: "text", text: "durably persisted" }], + timestamp: Date.now(), + }), + "turn-1:assistant", + ); - await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", - messages: [ - makeAgentAssistantMessage({ - content: [{ type: "text", text: "first mirror" }], - timestamp: Date.now(), - }), - ], - idempotencyScope: "scope-1", + const result = await mirrorCodexAppServerTranscript({ + ...target, + messages: [assistantMessage], + idempotencyScope: "codex-app-server:thread-1", }); - const raw = await fs.readFile(sessionFile, "utf8"); - expect(raw).toContain('"role":"assistant"'); - expect(raw).toContain('"content":[{"type":"text","text":"first mirror"}]'); + expect(result.assistantMirrorIdentitiesOwned).toEqual(["turn-1:assistant"]); + expect(await readMirrorRaw(target)).toContain('"role":"assistant"'); + }); + + it("rejects mirror writes without a runtime session identity", async () => { + await expect( + mirrorCodexAppServerTranscript({ + sessionId: "session-1", + messages: [ + makeAgentAssistantMessage({ + content: [{ type: "text", text: "no identity" }], + timestamp: Date.now(), + }), + ], + }), + ).rejects.toThrow("runtime session identity"); }); it("deduplicates app-server turn mirrors by idempotency scope", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-dedupe-"); const messages = [ makeAgentUserMessage({ content: [{ type: "text", text: "hello" }], @@ -868,24 +834,45 @@ describe("mirrorCodexAppServerTranscript", () => { ] as const; await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [...messages], idempotencyScope: "scope-1", }); await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [...messages], idempotencyScope: "scope-1", }); - const records = parseJsonLines<{ type?: string; message?: { role?: string } }>( - await fs.readFile(sessionFile, "utf8"), + expect((await readMirrorMessages(target)).filter((message) => message.role)).toHaveLength(2); + }); + + it("reports final assistant ownership for new and idempotent mirrors", async () => { + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-assistant-owned-"); + const assistantMessage = attachCodexMirrorIdentity( + makeAgentAssistantMessage({ + content: [{ type: "text", text: "owned once" }], + timestamp: Date.now(), + }), + "turn-1:assistant", ); - expect(records.slice(1)).toHaveLength(2); + + const firstMirror = await mirrorCodexAppServerTranscript({ + ...target, + messages: [assistantMessage], + idempotencyScope: "codex-app-server:thread-1", + }); + const secondMirror = await mirrorCodexAppServerTranscript({ + ...target, + messages: [assistantMessage], + idempotencyScope: "codex-app-server:thread-1", + }); + + expect(firstMirror.assistantMirrorIdentitiesOwned).toEqual(["turn-1:assistant"]); + expect(secondMirror.assistantMirrorIdentitiesOwned).toEqual(["turn-1:assistant"]); + expect( + (await readMirrorMessages(target)).filter((message) => message.role === "assistant"), + ).toHaveLength(1); }); it("runs before_message_write before appending mirrored transcript messages", async () => { @@ -902,24 +889,20 @@ describe("mirrorCodexAppServerTranscript", () => { }, ]), ); - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-hook-"); const sourceMessage = makeAgentAssistantMessage({ content: [{ type: "text", text: "hello" }], timestamp: Date.now(), }); await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [sourceMessage], idempotencyScope: "scope-1", }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain('"content":[{"type":"text","text":"hello [hooked]"}]'); - // The idempotency fingerprint is derived from the pre-hook message so a - // hook rewrite cannot bypass dedupe by reshaping content on every retry. expect(raw).toContain( `"idempotencyKey":"scope-1:assistant:${expectedFingerprint(sourceMessage)}"`, ); @@ -939,23 +922,19 @@ describe("mirrorCodexAppServerTranscript", () => { }, ]), ); - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-duplicates-"); const sourceMessage = makeAgentUserMessage({ content: [{ type: "text", text: "secret prompt" }], timestamp: Date.now(), }); const first = await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [sourceMessage], idempotencyScope: "scope-1", }); const second = await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [sourceMessage], idempotencyScope: "scope-1", }); @@ -967,10 +946,9 @@ describe("mirrorCodexAppServerTranscript", () => { { type: "text", text: "[redacted by hook]" }, ]); expect(JSON.stringify(second.userMessagesPresent)).not.toContain("secret prompt"); - const records = parseJsonLines<{ type?: string; message?: { role?: string } }>( - await fs.readFile(sessionFile, "utf8"), - ); - expect(records.filter((record) => record.message?.role === "user")).toHaveLength(1); + expect( + (await readMirrorMessages(target)).filter((message) => message.role === "user"), + ).toHaveLength(1); }); it("preserves the computed idempotency key when hooks rewrite message keys", async () => { @@ -987,21 +965,19 @@ describe("mirrorCodexAppServerTranscript", () => { }, ]), ); - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-key-hook-"); const sourceMessage = makeAgentAssistantMessage({ content: [{ type: "text", text: "hello" }], timestamp: Date.now(), }); await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [sourceMessage], idempotencyScope: "scope-1", }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain( `"idempotencyKey":"scope-1:assistant:${expectedFingerprint(sourceMessage)}"`, ); @@ -1011,18 +987,13 @@ describe("mirrorCodexAppServerTranscript", () => { it("respects before_message_write blocking decisions", async () => { initializeGlobalHookRunner( createMockPluginRegistry([ - { - hookName: "before_message_write", - handler: () => ({ block: true }), - }, + { hookName: "before_message_write", handler: () => ({ block: true }) }, ]), ); - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-blocked-"); const result = await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [ attachCodexMirrorIdentity( makeAgentAssistantMessage({ @@ -1036,95 +1007,38 @@ describe("mirrorCodexAppServerTranscript", () => { }); expect(result.assistantMirrorIdentitiesOwned).toEqual(["turn-1:assistant"]); - await expect(fs.readFile(sessionFile, "utf8")).rejects.toHaveProperty("code", "ENOENT"); + expect(await readMirrorMessages(target)).toEqual([]); }); - it("migrates small linear transcripts before mirroring", async () => { - const sessionFile = await createTempSessionFile(); - await fs.writeFile( - sessionFile, - [ - JSON.stringify({ - type: "session", - version: 3, - id: "linear-codex-session", - timestamp: new Date().toISOString(), - cwd: process.cwd(), - }), - JSON.stringify({ - type: "message", - id: "legacy-user", - timestamp: new Date().toISOString(), - message: { role: "user", content: "legacy user" }, - }), - ].join("\n") + "\n", - "utf8", + it("leaves the assistant unowned when transcript persistence fails", async () => { + const root = await makeRoot("openclaw-codex-transcript-failure-"); + const assistantMessage = attachCodexMirrorIdentity( + makeAgentAssistantMessage({ + content: [{ type: "text", text: "needs fallback persistence" }], + timestamp: Date.now(), + }), + "turn-1:assistant", ); - await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", - messages: [ - makeAgentAssistantMessage({ - content: [{ type: "text", text: "mirrored assistant" }], - timestamp: Date.now(), - }), - ], - idempotencyScope: "scope-1", + const assistantTranscriptOwned = await mirrorTranscriptBestEffort({ + params: { + sessionId: "session-1", + suppressNextUserMessagePersistence: true, + } as Parameters[0]["params"], + result: { + messagesSnapshot: [assistantMessage], + } as Parameters[0]["result"], + notifyUserMessagePersisted: () => undefined, + cwd: root, + threadId: "thread-1", + turnId: "turn-1", }); - const records = (await fs.readFile(sessionFile, "utf8")) - .trim() - .split("\n") - .map( - (line) => - JSON.parse(line) as { - type?: string; - id?: string; - parentId?: string | null; - message?: { role?: string }; - }, - ) - .filter((record) => record.type === "message"); - - expect(records[0]?.id).toBe("legacy-user"); - expect(records[0]?.parentId).toBeNull(); - expect(records[1]?.parentId).toBe("legacy-user"); + expect(assistantTranscriptOwned).toBe(false); }); - // Helpers for the identity-based regression tests below. - // - // The mirror dedupe key is now `${idempotencyScope}:${identity}`, where - // `identity` is either an explicit `attachCodexMirrorIdentity` tag (the - // production path; event-projector emits `${turnId}:${kind}`) or the - // role/content fingerprint fallback (legacy callers). - type FileMessage = { - type?: string; - message?: { role?: string; content?: Array<{ text?: string }> }; - }; - function readFileMessages(raw: string): Array<{ role?: string; text?: string }> { - return parseJsonLines(raw) - .filter((record) => record.type === "message") - .map((record) => ({ - role: record.message?.role, - text: record.message?.content?.[0]?.text, - })); - } - - // Regression for #77012 (within-turn snapshot reordering). When mirror is - // invoked twice under the same scope/turn but the second snapshot inserts - // a reasoning record between the user prompt and the assistant reply, - // every assistant-role record after the inserted slot shifts. With the - // previous `:role:index` key, the second call's reasoning record collided - // with the first call's assistant key (both `:assistant:1`) — the - // legitimately-new reasoning entry was silently dropped, and the - // assistant content was re-appended under `:assistant:2`, producing a - // duplicate assistant entry. The identity-based key (event-projector - // tags `${turnId}:reasoning` and `${turnId}:assistant`) makes each kind - // its own dedupe slot. it("dedupes mirrored messages despite snapshot positional shifts", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-shift-"); const userMessage = attachCodexMirrorIdentity( makeAgentUserMessage({ content: [{ type: "text", text: "hello" }], @@ -1141,9 +1055,7 @@ describe("mirrorCodexAppServerTranscript", () => { ); await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [userMessage, assistantMessage], idempotencyScope: "codex-app-server:thread-X", }); @@ -1155,30 +1067,22 @@ describe("mirrorCodexAppServerTranscript", () => { "turn-1:reasoning", ); await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [userMessage, reasoningMessage, assistantMessage], idempotencyScope: "codex-app-server:thread-X", }); - const messageTexts = readFileMessages(await fs.readFile(sessionFile, "utf8")).map( - (m) => m.text, - ); - expect(messageTexts).toEqual(["hello", "hi there", "[Codex reasoning] thinking"]); + expect((await readMirrorMessages(target)).map((m) => m.text)).toEqual([ + "hello", + "hi there", + "[Codex reasoning] thinking", + ]); }); - // Two distinct turns where the user types the same thing must not collapse: - // each entry carries its own `${turnId}:${kind}` identity so the dedupe - // key differs even when role+content match. (Prior content-fingerprint-only - // designs would have collapsed the second user turn here.) it("keeps repeated same-content turns distinct", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-repeat-"); const userTurn1 = attachCodexMirrorIdentity( - makeAgentUserMessage({ - content: [{ type: "text", text: "yes" }], - timestamp: Date.now(), - }), + makeAgentUserMessage({ content: [{ type: "text", text: "yes" }], timestamp: Date.now() }), "turn-1:prompt", ); const assistantTurn1 = attachCodexMirrorIdentity( @@ -1189,10 +1093,7 @@ describe("mirrorCodexAppServerTranscript", () => { "turn-1:assistant", ); const userTurn2 = attachCodexMirrorIdentity( - makeAgentUserMessage({ - content: [{ type: "text", text: "yes" }], - timestamp: Date.now() + 2, - }), + makeAgentUserMessage({ content: [{ type: "text", text: "yes" }], timestamp: Date.now() + 2 }), "turn-2:prompt", ); const assistantTurn2 = attachCodexMirrorIdentity( @@ -1204,21 +1105,17 @@ describe("mirrorCodexAppServerTranscript", () => { ); await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [userTurn1, assistantTurn1], idempotencyScope: "codex-app-server:thread-X", }); await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [userTurn2, assistantTurn2], idempotencyScope: "codex-app-server:thread-X", }); - expect(readFileMessages(await fs.readFile(sessionFile, "utf8"))).toEqual([ + expect(await readMirrorMessages(target)).toEqual([ { role: "user", text: "yes" }, { role: "assistant", text: "ok 1" }, { role: "user", text: "yes" }, @@ -1226,19 +1123,10 @@ describe("mirrorCodexAppServerTranscript", () => { ]); }); - // Cross-turn re-emit: an entry first written under turn 1 may be re-emitted - // as part of a later turn's snapshot (e.g. a context-engine flow that - // bundles prior history). Because every entry carries its own original - // `${turnId}:${kind}` identity, the re-emitted entries collide with their - // existing on-disk keys and become true no-ops — instead of being - // appended again on a sibling branch (the on-disk symptom in #77012). it("dedupes prior-turn entries re-emitted into a later turn's snapshot", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-reemit-"); const userTurn1 = attachCodexMirrorIdentity( - makeAgentUserMessage({ - content: [{ type: "text", text: "msg1" }], - timestamp: Date.now(), - }), + makeAgentUserMessage({ content: [{ type: "text", text: "msg1" }], timestamp: Date.now() }), "turn-1:prompt", ); const assistantTurn1 = attachCodexMirrorIdentity( @@ -1249,9 +1137,7 @@ describe("mirrorCodexAppServerTranscript", () => { "turn-1:assistant", ); await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [userTurn1, assistantTurn1], idempotencyScope: "codex-app-server:thread-X", }); @@ -1270,17 +1156,13 @@ describe("mirrorCodexAppServerTranscript", () => { }), "turn-2:assistant", ); - // Buggy upstream: snapshot for turn 2 also includes the just-completed - // turn 1's entries (with their original identities preserved). await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [userTurn1, assistantTurn1, userTurn2, assistantTurn2], idempotencyScope: "codex-app-server:thread-X", }); - expect(readFileMessages(await fs.readFile(sessionFile, "utf8"))).toEqual([ + expect(await readMirrorMessages(target)).toEqual([ { role: "user", text: "msg1" }, { role: "assistant", text: "reply1" }, { role: "user", text: "msg2" }, @@ -1288,12 +1170,8 @@ describe("mirrorCodexAppServerTranscript", () => { ]); }); - // Backward-compat: callers that do not tag messages with a mirror identity - // (e.g. third-party harnesses or tests routed through the legacy path) - // still get the role/content fingerprint key. Distinct turns are then - // distinguished by the caller's idempotency scope. - it("falls back to the role+content fingerprint when no identity is attached", async () => { - const sessionFile = await createTempSessionFile(); + it("uses the role+content fingerprint when no identity is attached", async () => { + const target = await createSqliteMirrorTarget("openclaw-codex-mirror-fingerprint-"); const userMessage = makeAgentUserMessage({ content: [{ type: "text", text: "hello" }], timestamp: Date.now(), @@ -1304,14 +1182,12 @@ describe("mirrorCodexAppServerTranscript", () => { }); await mirrorCodexAppServerTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [userMessage, assistantMessage], idempotencyScope: "scope-1", }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain(`"idempotencyKey":"scope-1:user:${expectedFingerprint(userMessage)}"`); expect(raw).toContain( `"idempotencyKey":"scope-1:assistant:${expectedFingerprint(assistantMessage)}"`, diff --git a/extensions/codex/src/app-server/transcript-mirror.ts b/extensions/codex/src/app-server/transcript-mirror.ts index 29928588713f..947343ac0608 100644 --- a/extensions/codex/src/app-server/transcript-mirror.ts +++ b/extensions/codex/src/app-server/transcript-mirror.ts @@ -278,7 +278,7 @@ export function projectBoundedCodexThreadHistory(params: { export async function importCodexThreadHistoryToTranscript(params: { thread: CodexThread; throughTurnId: string | null; - sessionFile: string; + storePath: string; sessionId: string; sessionKey: string; agentId?: string; @@ -294,7 +294,7 @@ export async function importCodexThreadHistoryToTranscript(params: { }); if (projection.transcriptMessages.length > 0) { await mirrorCodexAppServerTranscript({ - sessionFile: params.sessionFile, + storePath: params.storePath, sessionId: params.sessionId, sessionKey: params.sessionKey, ...(params.agentId ? { agentId: params.agentId } : {}), @@ -413,10 +413,10 @@ export async function mirrorTranscriptBestEffort(params: { turnId: params.turnId, }); const mirrorResult = await mirrorCodexAppServerTranscript({ - sessionFile: params.params.sessionFile, agentId: params.agentId, sessionKey: params.sessionKey, sessionId: params.params.sessionId, + storePath: params.params.sessionTarget?.storePath, cwd: params.cwd, messages, // Scope is thread-stable. Each entry in `messagesSnapshot` is tagged @@ -506,10 +506,10 @@ export async function mirrorPromptAtTurnStartBestEffort(params: { `${params.turnId}:prompt`, ); const mirrorResult = await mirrorCodexAppServerTranscript({ - sessionFile: params.params.sessionFile, agentId: params.agentId, sessionKey: params.sessionKey, sessionId: params.params.sessionId, + storePath: params.params.sessionTarget?.storePath, cwd: params.cwd, messages: [userPromptMessage], idempotencyScope: `codex-app-server:${params.threadId}`, @@ -578,11 +578,11 @@ function buildMirrorDedupeIdentity(message: MirroredAgentMessage): string { } export async function mirrorCodexAppServerTranscript(params: { - sessionFile: string; sessionId: string; cwd?: string; sessionKey?: string; agentId?: string; + storePath?: string; messages: AgentMessage[]; idempotencyScope?: string; config?: SessionTranscriptWriteLockParams["config"]; @@ -699,11 +699,11 @@ export async function mirrorCodexAppServerTranscript(params: { await publishSessionTranscriptUpdateByIdentity({ ...transcriptTarget, update: { - ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), ...(params.agentId ? { agentId: params.agentId } : {}), message: update.message, messageId: update.messageId, messageSeq: update.messageSeq, + sessionKey: transcriptTarget.sessionKey, }, }); } catch (error) { @@ -720,15 +720,20 @@ export async function mirrorCodexAppServerTranscript(params: { function resolveCodexMirrorTranscriptTarget(params: { agentId?: string; - sessionFile: string; sessionId: string; sessionKey?: string; + storePath?: string; }): SessionTranscriptTargetParams { + const sessionKey = params.sessionKey?.trim(); + const storePath = params.storePath?.trim(); + if (!sessionKey || !storePath) { + throw new Error("Codex transcript mirror requires a runtime session identity"); + } return { ...(params.agentId ? { agentId: params.agentId } : {}), - sessionFile: params.sessionFile, sessionId: params.sessionId, - sessionKey: params.sessionKey ?? "", + sessionKey, + storePath, }; } diff --git a/extensions/codex/src/commands.test.ts b/extensions/codex/src/commands.test.ts index e9ccd2ad4e0c..f6580d5e9434 100644 --- a/extensions/codex/src/commands.test.ts +++ b/extensions/codex/src/commands.test.ts @@ -10,7 +10,7 @@ import { } from "openclaw/plugin-sdk/agent-runtime"; import { MODEL_SELECTION_LOCKED_MESSAGE } from "openclaw/plugin-sdk/model-session-runtime"; import type { PluginCommandContext, PluginCommandResult } from "openclaw/plugin-sdk/plugin-entry"; -import { saveSessionStore } from "openclaw/plugin-sdk/session-store-runtime"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { CODEX_CONTROL_METHODS } from "./app-server/capabilities.js"; import type { CodexComputerUseStatus } from "./app-server/computer-use.js"; @@ -170,8 +170,10 @@ async function createLockedSessionContextOverrides( sessionKey = "agent:main:test:locked", ): Promise> { const storePath = path.join(tempDir, "locked-sessions.json"); - await saveSessionStore(storePath, { - [sessionKey]: { + await upsertSessionEntry({ + storePath, + sessionKey, + entry: { sessionId: "session-1", updatedAt: Date.now(), agentHarnessId: "codex", @@ -545,8 +547,10 @@ describe("codex command", () => { }, { threadId: "thread-old", cwd: "/old" }, ); - await saveSessionStore(storePath, { - [sessionKey]: { sessionId: "session-new", updatedAt: Date.now() }, + await upsertSessionEntry({ + storePath, + sessionKey, + entry: { sessionId: "session-new", updatedAt: Date.now() }, }); const codexControlRequest = vi.fn(async () => createThreadResumeResponse({ threadId: "thread-new" }), @@ -615,10 +619,18 @@ describe("codex command", () => { const codexControlRequest = vi.fn(async () => createThreadResumeResponse({ threadId: "thread-123" }), ); + const storePath = path.join(tempDir, "worker-sessions.json"); + await upsertSessionEntry({ + agentId: "worker", + storePath, + sessionKey: "agent:worker:session-1", + entry: { sessionId: "session-1", updatedAt: Date.now() }, + }); await handleCodexCommand( createContext("resume thread-123", undefined, { sessionKey: "agent:worker:session-1", + config: { session: { store: storePath } }, }), { deps: createDeps({ codexControlRequest }) }, ); diff --git a/extensions/codex/src/migration/session-binding-sidecars.ts b/extensions/codex/src/migration/session-binding-sidecars.ts index 43a44caaf8c7..836014241dec 100644 --- a/extensions/codex/src/migration/session-binding-sidecars.ts +++ b/extensions/codex/src/migration/session-binding-sidecars.ts @@ -10,8 +10,6 @@ import { withFileLock, type FileLockOptions } from "openclaw/plugin-sdk/file-loc import type { PluginStateKeyedStore } from "openclaw/plugin-sdk/plugin-state-runtime"; import type { PluginDoctorStateMigration } from "openclaw/plugin-sdk/runtime-doctor"; import { - listSessionEntries, - resolveSessionFilePath, resolveStorePath, updateSessionStoreEntry, } from "openclaw/plugin-sdk/session-store-runtime"; @@ -157,9 +155,7 @@ async function collectLegacyBindingSources( return source; }; for (const surface of surfaces) { - const sidecars = surface.scan - ? walkSidecars(surface.root) - : iterateIndexedSidecars(surface, params); + const sidecars = surface.scan ? walkSidecars(surface.root) : iterateIndexedSidecars(surface); for await (const sidecarPath of sidecars) { const source = await addSource(sidecarPath, surface); if (options.firstOnly) { @@ -196,33 +192,24 @@ async function readLegacySessionIndex( if (!isRecord(raw)) { return { failure: `session index ${storePath} has invalid entries` }; } - let normalizedEntries: ReturnType; - try { - normalizedEntries = listSessionEntries({ storePath, hydrateSkillPromptRefs: false }); - } catch { - return { failure: `session index ${storePath} could not be normalized` }; - } - const normalizedByKey = new Map( - normalizedEntries.map(({ sessionKey, entry }) => [sessionKey, entry] as const), - ); const entries: Array<{ sessionKey: string; entry: LegacySessionIndexEntry }> = []; for (const [sessionKey, value] of Object.entries(raw)) { if (!isRecord(value)) { return { failure: `session index ${storePath} has invalid entries` }; } // Metadata-only rows have no transcript identity and therefore cannot own - // a binding sidecar. Runtime normalization preserves this shipped shape. + // a binding sidecar. This legacy reader parses the raw file directly: + // post-flip listSessionEntries reads SQLite, so main's normalized + // cross-check would consult the wrong store for import inputs. if (value.sessionId === undefined) { continue; } - const rawSessionId = typeof value.sessionId === "string" ? value.sessionId.trim() : ""; - const sessionId = normalizedByKey.get(sessionKey)?.sessionId?.trim() ?? ""; + const sessionId = typeof value.sessionId === "string" ? value.sessionId.trim() : ""; const sessionFile = value.sessionFile; const lifecycleRevision = value.lifecycleRevision; const agentHarnessId = value.agentHarnessId; if ( !sessionId || - sessionId !== rawSessionId || (sessionFile !== undefined && typeof sessionFile !== "string") || (lifecycleRevision !== undefined && typeof lifecycleRevision !== "string") || (agentHarnessId !== undefined && typeof agentHarnessId !== "string") @@ -242,30 +229,24 @@ async function readLegacySessionIndex( return { entries }; } -async function* iterateIndexedSidecars( - surface: SessionSurface, - params: MigrationEnvironment, -): AsyncGenerator { +async function* iterateIndexedSidecars(surface: SessionSurface): AsyncGenerator { for (const storePath of surface.storePaths) { const index = await readLegacySessionIndex(storePath); if ("failure" in index) { continue; } - for (const { sessionKey, entry } of index.entries) { - const agentId = resolveLegacyBindingOwnerAgentId({ - sessionKey, - config: params.config, - storeAgentIds: surface.agentIds, - }); - let transcriptPath: string; - try { - transcriptPath = resolveSessionFilePath(entry.sessionId, entry, { - sessionsDir: path.dirname(storePath), - agentId, - }); - } catch { + for (const { entry } of index.entries) { + // Legacy sidecars sit beside file-era transcripts; sqlite-marker entries + // never had sidecars, so the raw locator is skipped for them. + const sessionFile = entry.sessionFile?.trim() ?? ""; + if (sessionFile.startsWith("sqlite:")) { continue; } + const transcriptPath = resolveLegacySessionFileLocator( + path.dirname(storePath), + entry, + entry.sessionId, + ); const sidecarPath = `${transcriptPath}${LEGACY_BINDING_SUFFIX}`; if (await isRegularFile(sidecarPath)) { yield sidecarPath; @@ -330,18 +311,16 @@ async function collectBindingOwners( config: params.config, storeAgentIds: storeAgentIds.get(storePath), }); - let effectiveTranscriptPath: string; + let legacyTranscriptPath: string; + let canonicalLegacyTranscriptPath: string; try { - effectiveTranscriptPath = resolveSessionFilePath(sessionId, entry, { - sessionsDir, - agentId, - }); + legacyTranscriptPath = resolveLegacySessionFileLocator(sessionsDir, entry, sessionId); + canonicalLegacyTranscriptPath = await canonicalizePath(legacyTranscriptPath); } catch { failures.push(`session index ${storePath} has an invalid locator for ${sessionKey}`); continue; } - const transcriptPath = await canonicalizePath(effectiveTranscriptPath); - if (!sourcePaths.has(transcriptPath)) { + if (!sourcePaths.has(canonicalLegacyTranscriptPath)) { continue; } const owner: LegacyBindingOwner = { @@ -349,11 +328,12 @@ async function collectBindingOwners( sessionId, sessionKey, storePath, - transcriptPath: effectiveTranscriptPath, + transcriptPath: legacyTranscriptPath, ...(entry.lifecycleRevision ? { lifecycleRevision: entry.lifecycleRevision } : {}), ...(entry.agentHarnessId?.trim() ? { agentHarnessId: entry.agentHarnessId.trim() } : {}), }; - const candidates = owners.get(transcriptPath) ?? new Map(); + const candidates = + owners.get(canonicalLegacyTranscriptPath) ?? new Map(); const ownerKey = `${agentId}\0${sessionId}\0${sessionKey}\0${canonicalStorePath}`; const configuredStorePath = resolveStorePath(params.config.session?.store, { agentId, @@ -364,7 +344,7 @@ async function collectBindingOwners( if (!candidates.has(ownerKey) || storePath === configuredStorePath) { candidates.set(ownerKey, owner); } - owners.set(transcriptPath, candidates); + owners.set(canonicalLegacyTranscriptPath, candidates); } } return { @@ -373,6 +353,17 @@ async function collectBindingOwners( }; } +// Doctor-only locator for retired file-backed session indexes. Active runtime +// never resolves these paths; migration needs them only to find old sidecars. +function resolveLegacySessionFileLocator( + sessionsDir: string, + entry: { sessionFile?: string }, + sessionId: string, +): string { + const sessionFile = entry.sessionFile?.trim(); + return path.resolve(sessionsDir, sessionFile || `${sessionId}.jsonl`); +} + function resolveLegacyBindingOwnerAgentId(params: { sessionKey: string; config: MigrationEnvironment["config"]; @@ -582,10 +573,8 @@ async function recordSessionOwner(owner: LegacyBindingOwner): Promise { - const transcriptPath = resolveOwnerTranscriptPath(owner, entry); if ( entry.sessionId.trim() !== owner.sessionId || - transcriptPath !== owner.transcriptPath || entry.lifecycleRevision !== owner.lifecycleRevision ) { return null; @@ -607,10 +596,8 @@ async function recordSessionOwner(owner: LegacyBindingOwner): Promise { modelSelectionLocked: params?.modelSelectionLocked, }), resolveStorePath: () => path.join(root, "sessions", "sessions.json"), - resolveSessionFilePath: () => sessionFile, }, }, }); diff --git a/extensions/codex/src/native-thread-tool.ts b/extensions/codex/src/native-thread-tool.ts index ca89a671f054..e6dd1fece65e 100644 --- a/extensions/codex/src/native-thread-tool.ts +++ b/extensions/codex/src/native-thread-tool.ts @@ -1,7 +1,6 @@ /** * Owner-only access to native Codex threads stored in the user's Codex home. */ -import path from "node:path"; import { jsonResult, readStringParam, @@ -131,7 +130,7 @@ function readLimit(value: unknown): number | undefined { function resolveToolSession( context: OpenClawPluginToolContext, runtime: PluginRuntime, -): { sessionId: string; sessionFile: string; modelSelectionLocked: boolean } | undefined { +): { sessionId: string; modelSelectionLocked: boolean } | undefined { const sessionKey = context.sessionKey?.trim(); if (!sessionKey) { return undefined; @@ -145,15 +144,8 @@ function resolveToolSession( if (!sessionId) { return undefined; } - const storePath = runtime.agent.session.resolveStorePath(undefined, { - agentId: context.agentId, - }); return { sessionId, - sessionFile: runtime.agent.session.resolveSessionFilePath(sessionId, entry, { - agentId: context.agentId, - sessionsDir: path.dirname(storePath), - }), modelSelectionLocked: isModelSelectionLocked(entry), }; } diff --git a/extensions/codex/src/session-catalog.test.ts b/extensions/codex/src/session-catalog.test.ts index c113582d7e33..62a41b119e8c 100644 --- a/extensions/codex/src/session-catalog.test.ts +++ b/extensions/codex/src/session-catalog.test.ts @@ -5,6 +5,7 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import type { GatewayRequestHandlerOptions } from "openclaw/plugin-sdk/gateway-runtime"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; +import { resolveStorePath } from "openclaw/plugin-sdk/session-store-runtime"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { CodexThread } from "./app-server/protocol.js"; import { sessionBindingIdentity } from "./app-server/session-binding.js"; @@ -1400,7 +1401,7 @@ describe("Codex supervision actions", () => { ); expect(transcriptMirrorMocks.importCodexThreadHistoryToTranscript).toHaveBeenCalledWith({ thread: sourceThread, - sessionFile: "/tmp/openclaw-session-1.jsonl", + storePath: resolveStorePath(undefined, { agentId: "main" }), sessionId: "openclaw-session-1", sessionKey: first.sessionKey, agentId: "main", @@ -1697,7 +1698,7 @@ describe("Codex supervision actions", () => { expect(entries[0]?.entry.initializationPending).toBeUndefined(); expect(transcriptMirrorMocks.importCodexThreadHistoryToTranscript).toHaveBeenCalledWith( expect.objectContaining({ - sessionFile: `/tmp/${sessionId}.jsonl`, + storePath: resolveStorePath(undefined, { agentId: "main" }), sessionId, sessionKey, }), diff --git a/extensions/codex/src/session-catalog.ts b/extensions/codex/src/session-catalog.ts index b4e8d4be2413..73c1f8ac3e1e 100644 --- a/extensions/codex/src/session-catalog.ts +++ b/extensions/codex/src/session-catalog.ts @@ -17,6 +17,7 @@ import type { } from "openclaw/plugin-sdk/plugin-entry"; import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; import { parseAgentSessionKey } from "openclaw/plugin-sdk/routing"; +import { resolveStorePath } from "openclaw/plugin-sdk/session-store-runtime"; import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { CODEX_CONTROL_METHODS } from "./app-server/capabilities.js"; @@ -1560,14 +1561,15 @@ async function createOrReuseAdoptedSession(params: { sessionKey: entry.key, config: params.config, }); - const sessionFile = entry.entry.sessionFile?.trim(); - if (!sessionFile) { - throw new Error("Codex supervision session creation did not produce a transcript file"); - } + // Post-flip the mirror targets SQLite rows; resolve the agent's store + // path instead of trusting the legacy sessionFile locator marker. + const storePath = resolveStorePath(params.config.session?.store, { + agentId: entry.agentId, + }); await importCodexThreadHistoryToTranscript({ thread: params.sourceThread, throughTurnId: pendingLastTurnId ?? null, - sessionFile, + storePath, sessionId: entry.sessionId, sessionKey: entry.key, agentId: entry.agentId, diff --git a/extensions/copilot/src/attempt.test.ts b/extensions/copilot/src/attempt.test.ts index 942dde322059..bfe3dbd7cb0b 100644 --- a/extensions/copilot/src/attempt.test.ts +++ b/extensions/copilot/src/attempt.test.ts @@ -271,6 +271,12 @@ function makeParams( runId: "run-1", sessionFile: "session.json", sessionId: "session-1", + sessionKey: "agent:main:session-1", + sessionTarget: { + sessionId: "session-1", + sessionKey: "agent:main:session-1", + storePath: "openclaw-agent.sqlite", + }, timeoutMs: 5000, workspaceDir: "C:\\workspace", ...overrides, @@ -1280,7 +1286,7 @@ describe("runCopilotAttempt", () => { modelId: "gpt-4o", modelProvider: "github-copilot", sessionId: "session-1", - sessionKey: undefined, + sessionKey: "agent:main:session-1", workspaceDir: "C:\\workspace", }), ); @@ -2638,7 +2644,7 @@ describe("runCopilotAttempt", () => { dualWriteMock.dualWriteCopilotTranscriptBestEffort.mockResolvedValue(undefined); }); - it("invokes dual-write mirror with sessionFile and scoped idempotencyScope when sessionFile is set", async () => { + it("invokes dual-write mirror with runtime identity and scoped idempotencyScope", async () => { dualWriteMock.dualWriteCopilotTranscriptBestEffort.mockClear(); const sdk = makeFakeSdk({ onCreateSession: (session) => { @@ -2647,17 +2653,28 @@ describe("runCopilotAttempt", () => { }); const pool = makeFakePool(sdk); - await runCopilotAttempt(makeParams(), { pool }); + await runCopilotAttempt( + makeParams({ + sessionTarget: { + sessionId: "session-1", + sessionKey: "agent:main:session-1", + storePath: "sessions.json", + }, + }), + { pool }, + ); expect(dualWriteMock.dualWriteCopilotTranscriptBestEffort).toHaveBeenCalledTimes(1); const args = dualWriteMock.dualWriteCopilotTranscriptBestEffort.mock.calls[0]?.[0] as { - sessionFile: string; sessionId: string; + sessionKey: string; + storePath?: string; messages: Array<{ role: string }>; idempotencyScope?: string; }; - expect(args.sessionFile).toBe("session.json"); expect(args.sessionId).toBe("session-1"); + expect(args.sessionKey).toBe("agent:main:session-1"); + expect(args.storePath).toBe("sessions.json"); expect(args.idempotencyScope).toBe("copilot:sess-1"); expect(args.messages.length).toBeGreaterThan(0); const roles = args.messages.map((m) => m.role); @@ -2665,7 +2682,7 @@ describe("runCopilotAttempt", () => { expect(roles).toContain("assistant"); }); - it("does not invoke dual-write mirror when sessionFile is absent", async () => { + it("does not invoke dual-write mirror when runtime identity is absent", async () => { dualWriteMock.dualWriteCopilotTranscriptBestEffort.mockClear(); const sdk = makeFakeSdk({ onCreateSession: (session) => { @@ -2674,7 +2691,7 @@ describe("runCopilotAttempt", () => { }); const pool = makeFakePool(sdk); const params = makeParams() as unknown as Record; - delete params.sessionFile; + delete params.sessionTarget; await runCopilotAttempt(params as never, { pool }); diff --git a/extensions/copilot/src/attempt.ts b/extensions/copilot/src/attempt.ts index e62dbbccee2d..4706a4206a1d 100644 --- a/extensions/copilot/src/attempt.ts +++ b/extensions/copilot/src/attempt.ts @@ -1125,18 +1125,20 @@ export async function runCopilotAttempt( ...(taggedLastAssistant ? [taggedLastAssistant] : []), ]; - // Best-effort dual-write: mirror this attempt's full message snapshot - // (user/assistant/toolResult) into the OpenClaw audit transcript at - // params.sessionFile, alongside the SDK's own session storage. The - // OpenClaw shell (attempt-execution.ts) writes only the user prompt - // and terminal assistant text; mirroring here captures intermediate - // tool calls/results for full audit/replay parity with the codex - // extension. Identity-tagged so re-emits dedupe. Errors are - // swallowed so a mirror failure cannot break the attempt. - const sessionFileForMirror = readString(input.sessionFile); + // Best-effort dual-write mirrors this attempt's full message snapshot into + // OpenClaw's runtime transcript store. The Copilot SDK may still maintain + // its own private files; OpenClaw-side audit state is addressed only by + // session identity so missing identity cannot silently recreate JSONL state. const openClawSessionIdForMirror = readString(input.sessionId); + const openClawSessionKeyForMirror = readString((input as { sessionKey?: unknown }).sessionKey); + const openClawStorePathForMirror = readString(input.sessionTarget?.storePath); const mirrorScopeSessionId = sessionIdUsed ?? openClawSessionIdForMirror; - if (sessionFileForMirror && openClawSessionIdForMirror && messagesSnapshot.length > 0) { + if ( + openClawSessionIdForMirror && + openClawSessionKeyForMirror && + openClawStorePathForMirror && + messagesSnapshot.length > 0 + ) { const taggedMessages = messagesSnapshot.map((message, index) => { if ( message.role !== "user" && @@ -1161,10 +1163,10 @@ export async function runCopilotAttempt( return attachCopilotMirrorIdentity(message, `${identityScope}:${message.role}:${index}`); }); await dualWriteCopilotTranscriptBestEffort({ - sessionFile: sessionFileForMirror, sessionId: openClawSessionIdForMirror, - sessionKey: readString((input as { sessionKey?: unknown }).sessionKey), + sessionKey: openClawSessionKeyForMirror, agentId: readString(input.agentId), + storePath: openClawStorePathForMirror, messages: taggedMessages, idempotencyScope: mirrorScopeSessionId ? `copilot:${mirrorScopeSessionId}` : undefined, config: (input as { config?: unknown }).config as never, diff --git a/extensions/copilot/src/dual-write-transcripts.test.ts b/extensions/copilot/src/dual-write-transcripts.test.ts index 1728af6031ef..6da8f30c9bf5 100755 --- a/extensions/copilot/src/dual-write-transcripts.test.ts +++ b/extensions/copilot/src/dual-write-transcripts.test.ts @@ -9,6 +9,8 @@ import { resetGlobalHookRunner, } from "openclaw/plugin-sdk/hook-runtime"; import { createMockPluginRegistry } from "openclaw/plugin-sdk/plugin-test-runtime"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { readSessionTranscriptEvents } from "openclaw/plugin-sdk/session-transcript-runtime"; import { castAgentMessage, makeAgentAssistantMessage, @@ -37,31 +39,87 @@ afterEach(async () => { } }); -async function createTempSessionFile() { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-copilot-mirror-")); - tempDirs.push(dir); - return path.join(dir, "session.jsonl"); -} - async function makeRoot(prefix: string): Promise { const root = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); tempDirs.push(root); return root; } -function parseJsonLines(raw: string): T[] { - const records: T[] = []; - for (const line of raw.trim().split("\n")) { - if (line.length > 0) { - records.push(JSON.parse(line) as T); - } - } - return records; +function readEventMessages(events: unknown[]): Array<{ role?: string; text?: string }> { + return events + .map((event) => + event && typeof event === "object" ? (event as { message?: unknown }).message : undefined, + ) + .filter((message): message is { role?: string; content?: unknown } => + Boolean(message && typeof message === "object"), + ) + .map((message) => { + const content = Array.isArray(message.content) + ? message.content.find((part): part is { text: string } => + Boolean(part && typeof part === "object" && typeof part.text === "string"), + )?.text + : typeof message.content === "string" + ? message.content + : undefined; + return { role: message.role, text: content }; + }); +} + +async function createSqliteMirrorTarget(prefix: string, options: { sessionId?: string } = {}) { + const root = await makeRoot(prefix); + const agentId = "main"; + const sessionId = options.sessionId ?? "session-1"; + const sessionKey = `agent:${agentId}:${sessionId}`; + const storePath = path.join(root, "openclaw-agent.sqlite"); + await upsertSessionEntry({ + agentId, + sessionKey, + storePath, + entry: { + sessionFile: `sqlite:${agentId}:${sessionId}:${storePath}`, + sessionId, + updatedAt: 1, + }, + }); + return { + agentId, + sessionId, + sessionKey, + storePath, + bogusSessionFile: path.join(root, "should-not-be-created.jsonl"), + }; +} + +async function readMirrorEvents(target: { + agentId: string; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise { + return await readSessionTranscriptEvents(target); +} + +async function readMirrorRaw(target: { + agentId: string; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise { + return (await readMirrorEvents(target)).map((event) => JSON.stringify(event)).join("\n"); +} + +async function readMirrorMessages(target: { + agentId: string; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise> { + return readEventMessages(await readMirrorEvents(target)); } describe("mirrorCopilotTranscript", () => { - it("mirrors user, assistant, and tool result messages into the OpenClaw transcript", async () => { - const sessionFile = await createTempSessionFile(); + it("mirrors user, assistant, and tool result messages by SQLite identity", async () => { + const target = await createSqliteMirrorTarget("openclaw-copilot-mirror-basic-"); const userMessage = makeAgentUserMessage({ content: [{ type: "text", text: "hello" }], timestamp: Date.now(), @@ -74,25 +132,17 @@ describe("mirrorCopilotTranscript", () => { role: "toolResult", toolCallId: "call-1", toolName: "read", - content: [ - { - type: "toolResult", - toolCallId: "call-1", - content: "read output", - }, - ], + content: [{ type: "toolResult", toolCallId: "call-1", content: "read output" }], timestamp: Date.now() + 2, }) as MirroredAgentMessage; await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [userMessage, assistantMessage, toolResultMessage], idempotencyScope: "copilot:session-1", }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain('"role":"user"'); expect(raw).toContain('"role":"assistant"'); expect(raw).toContain('"role":"toolResult"'); @@ -106,10 +156,14 @@ describe("mirrorCopilotTranscript", () => { expect(raw).toContain( `"idempotencyKey":"copilot:session-1:toolResult:${expectedFingerprint(toolResultMessage)}"`, ); + await expect(fs.readFile(target.bogusSessionFile, "utf8")).rejects.toHaveProperty( + "code", + "ENOENT", + ); }); it("preserves gateway user-turn identity across Copilot transcript mirroring", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-copilot-mirror-user-identity-"); const userMessage = castAgentMessage({ ...makeAgentUserMessage({ content: [{ type: "text", text: "client prompt" }], @@ -119,51 +173,40 @@ describe("mirrorCopilotTranscript", () => { }); await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [userMessage], idempotencyScope: "copilot:session-1", }); await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [userMessage], idempotencyScope: "copilot:session-1", }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain('"idempotencyKey":"client-run:user"'); expect(raw).not.toContain('"idempotencyKey":"copilot:session-1:user:'); - const records = parseJsonLines<{ message?: { role?: string } }>(raw); - expect(records.filter((record) => record.message?.role === "user")).toHaveLength(1); + expect( + (await readMirrorMessages(target)).filter((message) => message.role === "user"), + ).toHaveLength(1); }); - it("creates the transcript directory on first mirror", async () => { - const root = await makeRoot("openclaw-copilot-mirror-missing-dir-"); - const sessionFile = path.join(root, "nested", "sessions", "session.jsonl"); - - await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", - messages: [ - makeAgentAssistantMessage({ - content: [{ type: "text", text: "first mirror" }], - timestamp: Date.now(), - }), - ], - idempotencyScope: "copilot:session-1", - }); - - const raw = await fs.readFile(sessionFile, "utf8"); - expect(raw).toContain('"role":"assistant"'); - expect(raw).toContain('"content":[{"type":"text","text":"first mirror"}]'); + it("rejects mirror writes without a runtime session identity", async () => { + await expect( + mirrorCopilotTranscript({ + sessionId: "session-1", + messages: [ + makeAgentAssistantMessage({ + content: [{ type: "text", text: "no identity" }], + timestamp: Date.now(), + }), + ], + }), + ).rejects.toThrow("runtime session identity"); }); it("deduplicates re-emits by idempotency scope", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-copilot-mirror-dedupe-"); const messages = [ makeAgentUserMessage({ content: [{ type: "text", text: "hello" }], @@ -176,27 +219,17 @@ describe("mirrorCopilotTranscript", () => { ] as const; await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [...messages], idempotencyScope: "copilot:session-1", }); await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [...messages], idempotencyScope: "copilot:session-1", }); - const records = parseJsonLines<{ type?: string; message?: { role?: string } }>( - await fs.readFile(sessionFile, "utf8"), - ); - // First "header" record may or may not appear depending on migration. - // What matters is that the second mirror call adds zero new messages. - const messageRecords = records.filter((r) => r.message?.role !== undefined); - expect(messageRecords).toHaveLength(2); + expect((await readMirrorMessages(target)).filter((message) => message.role)).toHaveLength(2); }); it("runs before_message_write before appending mirrored messages", async () => { @@ -213,21 +246,19 @@ describe("mirrorCopilotTranscript", () => { }, ]), ); - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-copilot-mirror-hook-"); const sourceMessage = makeAgentAssistantMessage({ content: [{ type: "text", text: "hello" }], timestamp: Date.now(), }); await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [sourceMessage], idempotencyScope: "copilot:session-1", }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain('"content":[{"type":"text","text":"hello [hooked]"}]'); expect(raw).toContain( `"idempotencyKey":"copilot:session-1:assistant:${expectedFingerprint(sourceMessage)}"`, @@ -237,18 +268,13 @@ describe("mirrorCopilotTranscript", () => { it("respects before_message_write blocking decisions", async () => { initializeGlobalHookRunner( createMockPluginRegistry([ - { - hookName: "before_message_write", - handler: () => ({ block: true }), - }, + { hookName: "before_message_write", handler: () => ({ block: true }) }, ]), ); - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-copilot-mirror-block-"); await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [ makeAgentAssistantMessage({ content: [{ type: "text", text: "should not persist" }], @@ -258,43 +284,40 @@ describe("mirrorCopilotTranscript", () => { idempotencyScope: "copilot:session-1", }); - await expect(fs.readFile(sessionFile, "utf8")).rejects.toHaveProperty("code", "ENOENT"); + expect(await readMirrorMessages(target)).toEqual([]); }); it("is a no-op when no mirrorable messages are present", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-copilot-mirror-empty-"); await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", - sessionKey: "session-1", + ...target, messages: [], idempotencyScope: "copilot:session-1", }); - await expect(fs.readFile(sessionFile, "utf8")).rejects.toHaveProperty("code", "ENOENT"); + expect(await readMirrorMessages(target)).toEqual([]); }); it("uses content fingerprint when no explicit mirror identity is attached", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-copilot-mirror-fingerprint-"); const message = makeAgentAssistantMessage({ content: [{ type: "text", text: "fp" }], timestamp: Date.now(), }); await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", + ...target, messages: [message], idempotencyScope: "scope-fp", }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain(`"idempotencyKey":"scope-fp:assistant:${expectedFingerprint(message)}"`); }); it("uses attached identity instead of content fingerprint when provided", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-copilot-mirror-identity-"); const baseMessage = makeAgentAssistantMessage({ content: [{ type: "text", text: "explicit" }], timestamp: Date.now(), @@ -302,13 +325,12 @@ describe("mirrorCopilotTranscript", () => { const tagged = attachCopilotMirrorIdentity(baseMessage, "sdk-session-1:assistant:0"); await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", + ...target, messages: [tagged], idempotencyScope: "copilot:openclaw-session-1", }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain( '"idempotencyKey":"copilot:openclaw-session-1:sdk-session-1:assistant:0"', ); @@ -316,11 +338,10 @@ describe("mirrorCopilotTranscript", () => { }); it("omits idempotencyKey when no idempotencyScope is provided", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-copilot-mirror-no-scope-"); await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", + ...target, messages: [ makeAgentAssistantMessage({ content: [{ type: "text", text: "no scope" }], @@ -329,13 +350,13 @@ describe("mirrorCopilotTranscript", () => { ], }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain('"content":[{"type":"text","text":"no scope"}]'); expect(raw).not.toContain("idempotencyKey"); }); it("filters out non-mirrorable roles", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-copilot-mirror-filter-"); const userMessage = makeAgentUserMessage({ content: [{ type: "text", text: "u" }], timestamp: Date.now(), @@ -347,19 +368,18 @@ describe("mirrorCopilotTranscript", () => { }); await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", + ...target, messages: [userMessage, systemLike], idempotencyScope: "scope", }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain('"role":"user"'); expect(raw).not.toContain("system note"); }); it("preserves explicit identity across attachCopilotMirrorIdentity overrides", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-copilot-mirror-override-"); const base = makeAgentAssistantMessage({ content: [{ type: "text", text: "x" }], timestamp: Date.now(), @@ -368,13 +388,12 @@ describe("mirrorCopilotTranscript", () => { const second = attachCopilotMirrorIdentity(first, "id-2"); await mirrorCopilotTranscript({ - sessionFile, - sessionId: "session-1", + ...target, messages: [second], idempotencyScope: "scope", }); - const raw = await fs.readFile(sessionFile, "utf8"); + const raw = await readMirrorRaw(target); expect(raw).toContain('"idempotencyKey":"scope:id-2"'); expect(raw).not.toContain('"idempotencyKey":"scope:id-1"'); }); @@ -382,11 +401,10 @@ describe("mirrorCopilotTranscript", () => { describe("dualWriteCopilotTranscriptBestEffort", () => { it("returns normally when mirror succeeds", async () => { - const sessionFile = await createTempSessionFile(); + const target = await createSqliteMirrorTarget("openclaw-copilot-mirror-best-effort-"); await expect( dualWriteCopilotTranscriptBestEffort({ - sessionFile, - sessionId: "session-1", + ...target, messages: [ makeAgentAssistantMessage({ content: [{ type: "text", text: "ok" }], @@ -396,39 +414,25 @@ describe("dualWriteCopilotTranscriptBestEffort", () => { idempotencyScope: "scope", }), ).resolves.toBeUndefined(); - const raw = await fs.readFile(sessionFile, "utf8"); - expect(raw).toContain('"role":"assistant"'); + expect(await readMirrorMessages(target)).toContainEqual({ role: "assistant", text: "ok" }); }); - it("swallows infrastructure failures and never rejects", async () => { + it("swallows missing runtime identity and does not write JSONL", async () => { const root = await makeRoot("openclaw-copilot-mirror-invalid-"); - const previousStateDir = process.env.OPENCLAW_STATE_DIR; - process.env.OPENCLAW_STATE_DIR = root; - try { - await expect( - dualWriteCopilotTranscriptBestEffort({ - agentId: "main", - sessionFile: "", - sessionId: "session-1", - sessionKey: "agent:main:session-1", - messages: [ - makeAgentAssistantMessage({ - content: [{ type: "text", text: "should-not-throw" }], - timestamp: Date.now(), - }), - ], - idempotencyScope: "scope", - }), - ).resolves.toBeUndefined(); - await expect( - fs.access(path.join(root, "agents", "main", "sessions", "session-1.jsonl")), - ).rejects.toHaveProperty("code", "ENOENT"); - } finally { - if (previousStateDir === undefined) { - delete process.env.OPENCLAW_STATE_DIR; - } else { - process.env.OPENCLAW_STATE_DIR = previousStateDir; - } - } + const sessionFile = path.join(root, "agents", "main", "sessions", "session-1.jsonl"); + await expect( + dualWriteCopilotTranscriptBestEffort({ + agentId: "main", + sessionId: "session-1", + messages: [ + makeAgentAssistantMessage({ + content: [{ type: "text", text: "should-not-throw" }], + timestamp: Date.now(), + }), + ], + idempotencyScope: "scope", + }), + ).resolves.toBeUndefined(); + await expect(fs.access(sessionFile)).rejects.toHaveProperty("code", "ENOENT"); }); }); diff --git a/extensions/copilot/src/dual-write-transcripts.ts b/extensions/copilot/src/dual-write-transcripts.ts index 8c4498a92bcf..06e340bf9672 100755 --- a/extensions/copilot/src/dual-write-transcripts.ts +++ b/extensions/copilot/src/dual-write-transcripts.ts @@ -34,9 +34,9 @@ import { type AgentMessage, } from "openclaw/plugin-sdk/agent-harness-runtime"; import { - publishSessionTranscriptUpdateByIdentity, withSessionTranscriptWriteLock, type SessionTranscriptTargetParams, + type SessionTranscriptWriteLockContext, type SessionTranscriptWriteLockParams, } from "openclaw/plugin-sdk/session-transcript-runtime"; @@ -94,10 +94,10 @@ function buildMirrorDedupeIdentity(message: MirroredAgentMessage): string { } interface MirrorCopilotTranscriptParams { - sessionFile: string; sessionId: string; sessionKey?: string; agentId?: string; + storePath?: string; messages: AgentMessage[]; /** * Stable per-harness/per-thread scope. The codex equivalent uses @@ -122,7 +122,7 @@ export async function mirrorCopilotTranscript( } const transcriptTarget = resolveCopilotMirrorTranscriptTarget(params); - const didAppend = await withSessionTranscriptWriteLock( + await withCopilotMirrorTranscriptWriteLock( { ...transcriptTarget, config: params.config }, async (transcript) => { let didAppendMessage = false; @@ -175,36 +175,42 @@ export async function mirrorCopilotTranscript( existingIdempotencyKeys.add(idempotencyKey); } } + if (didAppendMessage) { + await transcript.publishUpdate( + params.sessionKey ? { sessionKey: params.sessionKey } : undefined, + ); + } return didAppendMessage; }, ); - - if (didAppend) { - await publishSessionTranscriptUpdateByIdentity({ - ...transcriptTarget, - update: params.sessionKey ? { sessionKey: params.sessionKey } : undefined, - }); - } } function resolveCopilotMirrorTranscriptTarget(params: { agentId?: string; - sessionFile: string; sessionId: string; sessionKey?: string; + storePath?: string; }): SessionTranscriptTargetParams { - const sessionFile = params.sessionFile.trim(); - if (!sessionFile) { - throw new Error("Copilot transcript mirror requires a sessionFile target"); + const sessionKey = params.sessionKey?.trim(); + const storePath = params.storePath?.trim(); + if (!sessionKey || !storePath) { + throw new Error("Copilot transcript mirror requires a runtime session identity"); } return { ...(params.agentId ? { agentId: params.agentId } : {}), - sessionFile, sessionId: params.sessionId, - sessionKey: params.sessionKey ?? "", + sessionKey, + storePath, }; } +function withCopilotMirrorTranscriptWriteLock( + params: SessionTranscriptTargetParams & { config?: SessionTranscriptWriteLockParams["config"] }, + run: (context: SessionTranscriptWriteLockContext) => Promise | T, +): Promise { + return withSessionTranscriptWriteLock(params, run); +} + function readTranscriptIdempotencyKeys(events: unknown[]): Set { const keys = new Set(); for (const event of events) { diff --git a/extensions/discord/src/monitor/native-command.model-picker.test.ts b/extensions/discord/src/monitor/native-command.model-picker.test.ts index 47749555cb3b..d102a907d12a 100644 --- a/extensions/discord/src/monitor/native-command.model-picker.test.ts +++ b/extensions/discord/src/monitor/native-command.model-picker.test.ts @@ -12,7 +12,8 @@ import type { ModelsProviderData } from "openclaw/plugin-sdk/command-auth-native import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import * as globalsModule from "openclaw/plugin-sdk/runtime-env"; import { - loadSessionStore, + getSessionEntry, + listSessionEntries, resolveStorePath, upsertSessionEntry, } from "openclaw/plugin-sdk/session-store-runtime"; @@ -442,16 +443,13 @@ describe("Discord model picker interactions", () => { dispatchSpy, model: "openai/gpt-4o", }); - const store = loadSessionStore( - resolveStorePath(context.cfg.session?.store, { agentId: "main" }), - { - skipCache: true, - }, - ); - const entry = Object.values(store).find( + const entries = listSessionEntries({ + storePath: resolveStorePath(context.cfg.session?.store, { agentId: "main" }), + }); + const entry = entries.find( (candidate) => - candidate.providerOverride === "openai" && candidate.modelOverride === "gpt-4o", - ); + candidate.entry.providerOverride === "openai" && candidate.entry.modelOverride === "gpt-4o", + )?.entry; expect(typeof entry?.sessionId).toBe("string"); expect(entry?.sessionId).not.toBe(""); expect(entry?.agentRuntimeOverride).toBe("codex"); @@ -489,17 +487,14 @@ describe("Discord model picker interactions", () => { dispatchSpy, model: "anthropic/claude-sonnet-4-5", }); - const store = loadSessionStore( - resolveStorePath(context.cfg.session?.store, { agentId: "main" }), - { - skipCache: true, - }, - ); - const entry = Object.values(store).find( + const entries = listSessionEntries({ + storePath: resolveStorePath(context.cfg.session?.store, { agentId: "main" }), + }); + const entry = entries.find( (candidate) => - candidate.providerOverride === "anthropic" && - candidate.modelOverride === "claude-sonnet-4-5", - ); + candidate.entry.providerOverride === "anthropic" && + candidate.entry.modelOverride === "claude-sonnet-4-5", + )?.entry; expect(entry?.agentRuntimeOverride).toBeUndefined(); }); @@ -534,17 +529,14 @@ describe("Discord model picker interactions", () => { dispatchSpy, model: "anthropic/claude-sonnet-4-5", }); - const store = loadSessionStore( - resolveStorePath(context.cfg.session?.store, { agentId: "main" }), - { - skipCache: true, - }, - ); - const entry = Object.values(store).find( + const entries = listSessionEntries({ + storePath: resolveStorePath(context.cfg.session?.store, { agentId: "main" }), + }); + const entry = entries.find( (candidate) => - candidate.providerOverride === "anthropic" && - candidate.modelOverride === "claude-sonnet-4-5", - ); + candidate.entry.providerOverride === "anthropic" && + candidate.entry.modelOverride === "claude-sonnet-4-5", + )?.entry; expect(entry?.agentRuntimeOverride).toBeUndefined(); }); @@ -840,12 +832,10 @@ describe("Discord model picker interactions", () => { mi: "1", }); - const store = loadSessionStore(storePath, { skipCache: true }); - expect(store["agent:worker:subagent:bound"]?.providerOverride).toBe("lmstudio"); - expect(store["agent:worker:subagent:bound"]?.modelOverride).toBe( - "unsloth/gemma-4-26b-a4b-it@iq4_xs", - ); - expect(store["agent:worker:subagent:bound"]?.liveModelSwitchPending).toBe(true); + const entry = getSessionEntry({ storePath, sessionKey: "agent:worker:subagent:bound" }); + expect(entry?.providerOverride).toBe("lmstudio"); + expect(entry?.modelOverride).toBe("unsloth/gemma-4-26b-a4b-it@iq4_xs"); + expect(entry?.liveModelSwitchPending).toBe(true); expectDispatchedModelSelection({ dispatchSpy, model: "lmstudio/unsloth/gemma-4-26b-a4b-it@iq4_xs", @@ -893,9 +883,9 @@ describe("Discord model picker interactions", () => { createModelsViewSubmitData(), ); - const store = loadSessionStore(storePath, { skipCache: true }); - expect(store["agent:worker:subagent:bound"]?.providerOverride).toBeUndefined(); - expect(store["agent:worker:subagent:bound"]?.modelOverride).toBeUndefined(); + const entry = getSessionEntry({ storePath, sessionKey: "agent:worker:subagent:bound" }); + expect(entry?.providerOverride).toBeUndefined(); + expect(entry?.modelOverride).toBeUndefined(); expect( JSON.stringify(firstMockArg(submitInteraction.followUp, "interaction.followUp")), ).toContain("❌ Failed to apply openai/gpt-4o."); @@ -950,14 +940,15 @@ describe("Discord model picker interactions", () => { r: "openclaw", }); - const store = loadSessionStore(storePath, { skipCache: true }); - expect(store["agent:worker:subagent:bound"]).toMatchObject({ - providerOverride: "openai", - modelOverride: "gpt-5.5", - agentHarnessId: "codex", - agentRuntimeOverride: "codex", - modelSelectionLocked: true, - }); + expect(getSessionEntry({ storePath, sessionKey: "agent:worker:subagent:bound" })).toMatchObject( + { + providerOverride: "openai", + modelOverride: "gpt-5.5", + agentHarnessId: "codex", + agentRuntimeOverride: "codex", + modelSelectionLocked: true, + }, + ); expect( JSON.stringify(firstMockArg(submitInteraction.followUp, "interaction.followUp")), ).toContain("❌ Model selection is locked for this session."); diff --git a/extensions/discord/src/monitor/native-command.think-autocomplete.test.ts b/extensions/discord/src/monitor/native-command.think-autocomplete.test.ts index e8017fdc3c01..819d12a11ba3 100644 --- a/extensions/discord/src/monitor/native-command.think-autocomplete.test.ts +++ b/extensions/discord/src/monitor/native-command.think-autocomplete.test.ts @@ -9,8 +9,7 @@ import { } from "openclaw/plugin-sdk/plugin-test-runtime"; import { clearSessionStoreCacheForTest, - saveSessionStore, - type SessionEntry, + upsertSessionEntry, } from "openclaw/plugin-sdk/session-store-runtime"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { ChannelType, type AutocompleteInteraction } from "../internal/discord.js"; @@ -140,21 +139,17 @@ async function saveSessionOverride(params: { agentRuntimeOverride?: string; }): Promise { fs.mkdirSync(path.dirname(STORE_PATH), { recursive: true }); - await saveSessionStore( - STORE_PATH, - { - [SESSION_KEY]: { - sessionId: "main", - updatedAt: Date.now(), - providerOverride: params.providerOverride, - modelOverride: params.modelOverride, - ...(params.agentRuntimeOverride - ? { agentRuntimeOverride: params.agentRuntimeOverride } - : {}), - }, - } satisfies Record, - { skipMaintenance: true }, - ); + await upsertSessionEntry({ + storePath: STORE_PATH, + sessionKey: SESSION_KEY, + entry: { + sessionId: "main", + updatedAt: Date.now(), + providerOverride: params.providerOverride, + modelOverride: params.modelOverride, + ...(params.agentRuntimeOverride ? { agentRuntimeOverride: params.agentRuntimeOverride } : {}), + }, + }); } function installProviderThinkingRegistryForTest(): void { diff --git a/extensions/feishu/src/doctor.test.ts b/extensions/feishu/src/doctor.test.ts index c5e8a319f5c0..ca08854ba5b1 100644 --- a/extensions/feishu/src/doctor.test.ts +++ b/extensions/feishu/src/doctor.test.ts @@ -2,7 +2,17 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { loadSessionStore } from "openclaw/plugin-sdk/session-store-runtime"; +import { DatabaseSync } from "node:sqlite"; +import { + formatSqliteSessionFileMarker, + listSessionEntries, + type SessionEntry, + upsertSessionEntry, +} from "openclaw/plugin-sdk/session-store-runtime"; +import { + appendSessionTranscriptMessageByIdentity, + readSessionTranscriptEvents, +} from "openclaw/plugin-sdk/session-transcript-runtime"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import type { OpenClawConfig } from "../runtime-api.js"; import { isFeishuSessionStoreKey, runFeishuDoctorSequence } from "./doctor.js"; @@ -59,13 +69,40 @@ function storePath(agentId = "main"): string { return path.join(sessionsDir(agentId), "sessions.json"); } -function writeStore(entries: Record, agentId = "main"): string { +function sqliteStorePath(agentId = "main"): string { + return path.join(stateDir(), "agents", agentId, "agent", "openclaw-agent.sqlite"); +} + +function corruptTranscriptEventJson(agentId: string, sessionId: string): void { + const database = new DatabaseSync(sqliteStorePath(agentId)); + try { + database + .prepare("UPDATE transcript_events SET event_json = ? WHERE session_id = ?") + .run("{", sessionId); + } finally { + database.close(); + } +} + +async function writeStore(entries: Record, agentId = "main"): Promise { const target = storePath(agentId); fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, JSON.stringify(entries, null, 2)); + for (const [sessionKey, entry] of Object.entries(entries as Record)) { + await upsertSessionEntry({ agentId, storePath: target, sessionKey, entry }); + } return target; } +function readStoreEntries(target: string, agentId = "main"): Record { + return Object.fromEntries( + listSessionEntries({ agentId, storePath: target }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), + ); +} + function writeTranscript(sessionId: string, lines: unknown[], agentId = "main"): string { const target = path.join(sessionsDir(agentId), `${sessionId}.jsonl`); fs.mkdirSync(path.dirname(target), { recursive: true }); @@ -131,7 +168,7 @@ describe("Feishu doctor state repair", () => { fs.writeFileSync(path.join(feishuDedupDir, "default.json"), JSON.stringify({ msg1: 1 })); writeTranscript("sess-ok", [sessionHeader("sess-ok"), userMessage("hello")]); - writeStore({ + await writeStore({ "agent:main:feishu:direct:ou_user": { sessionId: "sess-ok", sessionFile: "sess-ok.jsonl", @@ -149,22 +186,21 @@ describe("Feishu doctor state repair", () => { }); it("keeps custom-store sessions with canonical absolute transcripts", async () => { + const customStorePath = path.join(stateDir(), "custom-sessions", "sessions.json"); const transcriptPath = writeTranscript("sess-abs", [ sessionHeader("sess-abs"), userMessage("hello"), ]); - const customStorePath = path.join(stateDir(), "custom-sessions", "sessions.json"); - fs.mkdirSync(path.dirname(customStorePath), { recursive: true }); - fs.writeFileSync( - customStorePath, - JSON.stringify({ - "agent:main:feishu:direct:ou_user": { - sessionId: "sess-abs", - sessionFile: transcriptPath, - updatedAt: Date.now(), - }, - }), - ); + await upsertSessionEntry({ + agentId: "main", + storePath: customStorePath, + sessionKey: "agent:main:feishu:direct:ou_user", + entry: { + sessionId: "sess-abs", + sessionFile: transcriptPath, + updatedAt: Date.now(), + }, + }); const result = await runFeishuDoctorSequence({ cfg: { @@ -178,6 +214,109 @@ describe("Feishu doctor state repair", () => { expect(result).toEqual({ changeNotes: [], warningNotes: [] }); }); + it("keeps SQLite-backed Feishu session rows without file inspection", async () => { + await writeStore({ + "agent:main:feishu:direct:ou_user": { + sessionId: "sess-sqlite", + sessionFile: `sqlite:main:sess-sqlite:${storePath()}`, + updatedAt: Date.now(), + }, + }); + + const result = await runFeishuDoctorSequence({ + cfg: feishuConfig(), + env: process.env, + shouldRepair: false, + }); + + expect(result).toEqual({ changeNotes: [], warningNotes: [] }); + }); + + it("repairs SQLite-backed Feishu sessions with repeated blank user messages", async () => { + const targetStorePath = storePath(); + const sessionKey = "agent:main:feishu:direct:ou_sqlite_blank"; + const sessionId = "sess-sqlite-blank"; + await upsertSessionEntry({ + agentId: "main", + storePath: targetStorePath, + sessionKey, + entry: { + sessionId, + sessionFile: formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath: targetStorePath, + }), + updatedAt: Date.now(), + }, + }); + for (const content of ["", "", ""]) { + await appendSessionTranscriptMessageByIdentity({ + agentId: "main", + sessionId, + sessionKey, + storePath: targetStorePath, + message: { role: "user", content }, + }); + } + + const result = await runFeishuDoctorSequence({ + cfg: feishuConfig(), + env: process.env, + shouldRepair: true, + }); + + expect(result.warningNotes).toEqual([]); + expect(result.changeNotes.join("\n")).toContain("Removed 1 Feishu-scoped session entry"); + expect(readStoreEntries(targetStorePath)[sessionKey]).toBeUndefined(); + await expect( + readSessionTranscriptEvents({ + agentId: "main", + sessionId, + sessionKey, + storePath: targetStorePath, + }), + ).resolves.toEqual([]); + }); + + it("repairs SQLite-backed Feishu sessions with corrupt transcript rows", async () => { + const targetStorePath = storePath(); + const sessionKey = "agent:main:feishu:direct:ou_sqlite_corrupt"; + const sessionId = "sess-sqlite-corrupt"; + await upsertSessionEntry({ + agentId: "main", + storePath: targetStorePath, + sessionKey, + entry: { + sessionId, + sessionFile: formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath: targetStorePath, + }), + updatedAt: Date.now(), + }, + }); + await appendSessionTranscriptMessageByIdentity({ + agentId: "main", + sessionId, + sessionKey, + storePath: targetStorePath, + message: { role: "user", content: "bad row follows" }, + }); + corruptTranscriptEventJson("main", sessionId); + + const result = await runFeishuDoctorSequence({ + cfg: feishuConfig(), + env: process.env, + shouldRepair: true, + }); + + expect(result.warningNotes).toEqual([]); + expect(result.changeNotes.join("\n")).toContain("Removed 1 Feishu-scoped session entry"); + expect(readStoreEntries(targetStorePath)[sessionKey]).toBeUndefined(); + }); + it("keeps Feishu sessions with separated blank user messages", async () => { writeTranscript("sess-separated-blanks", [ sessionHeader("sess-separated-blanks"), @@ -187,7 +326,7 @@ describe("Feishu doctor state repair", () => { userMessage("world"), userMessage(""), ]); - writeStore({ + await writeStore({ "agent:main:feishu:direct:ou_user": { sessionId: "sess-separated-blanks", sessionFile: "sess-separated-blanks.jsonl", @@ -230,7 +369,7 @@ describe("Feishu doctor state repair", () => { sessionHeader("sess-ok"), userMessage("hello"), ]); - const targetStorePath = writeStore({ + const targetStorePath = await writeStore({ "agent:main:feishu:direct:ou_user": { sessionId: "sess-ok", sessionFile: "sess-ok.jsonl", @@ -248,7 +387,7 @@ describe("Feishu doctor state repair", () => { expect(result.changeNotes.join("\n")).toContain("Rebuilt Feishu runtime state: yes"); expect(result.changeNotes.join("\n")).toContain("Removed 0 Feishu-scoped session entries"); - const store = loadSessionStore(targetStorePath, { skipCache: true }); + const store = readStoreEntries(targetStorePath); expect(store["agent:main:feishu:direct:ou_user"]).toBeDefined(); expect(fs.existsSync(transcriptPath)).toBe(true); @@ -286,7 +425,7 @@ describe("Feishu doctor state repair", () => { userMessage(""), ]); - const targetStorePath = writeStore({ + const targetStorePath = await writeStore({ "agent:main:feishu:direct:ou_user": { sessionId: "sess-bad", sessionFile: "sess-bad.jsonl", @@ -325,8 +464,11 @@ describe("Feishu doctor state repair", () => { expect(fs.existsSync(path.join(backupDir, "session-stores", "main", "sessions.json"))).toBe( true, ); + expect( + fs.existsSync(path.join(backupDir, "session-stores", "main", "openclaw-agent.sqlite")), + ).toBe(true); - const store = loadSessionStore(targetStorePath, { skipCache: true }); + const store = readStoreEntries(targetStorePath); expect(store["agent:main:feishu:direct:ou_user"]).toBeUndefined(); expect(store["agent:codex:acp:binding:feishu:default:abc123"]).toBeDefined(); expect(store["agent:main:discord:direct:user"]).toBeDefined(); @@ -346,29 +488,26 @@ describe("Feishu doctor state repair", () => { }); it("preserves locked harness sessions while repairing ordinary Feishu sessions", async () => { - const lockedTranscriptPath = writeTranscript("sess-codex-locked", [ - sessionHeader("sess-codex-locked"), - userMessage(""), - userMessage(""), - userMessage(""), - ]); - const ordinaryTranscriptPath = writeTranscript("sess-feishu-bad", [ - sessionHeader("sess-feishu-bad"), - userMessage(""), - userMessage(""), - userMessage(""), - ]); - const targetStorePath = writeStore({ - "agent:main:ordinary-codex-locked": { + const targetStorePath = storePath(); + await upsertSessionEntry({ + agentId: "main", + storePath: targetStorePath, + sessionKey: "agent:main:ordinary-codex-locked", + entry: { sessionId: "sess-codex-locked", - sessionFile: "sess-codex-locked.jsonl", agentHarnessId: "codex", modelSelectionLocked: true, route: { channel: "feishu", target: { to: "ou_user", chatType: "direct" } }, + updatedAt: 1, }, - "agent:main:feishu:direct:ou_user": { + }); + await upsertSessionEntry({ + agentId: "main", + storePath: targetStorePath, + sessionKey: "agent:main:feishu:direct:ou_user", + entry: { sessionId: "sess-feishu-bad", - sessionFile: "sess-feishu-bad.jsonl", + updatedAt: 1, }, }); @@ -380,11 +519,110 @@ describe("Feishu doctor state repair", () => { expect(result.warningNotes).toEqual([]); expect(result.changeNotes.join("\n")).toContain("Removed 1 Feishu-scoped session entry"); - const store = loadSessionStore(targetStorePath, { skipCache: true }); + const store = readStoreEntries(targetStorePath); expect(store["agent:main:ordinary-codex-locked"]).toBeDefined(); expect(store["agent:main:feishu:direct:ou_user"]).toBeUndefined(); - expect(fs.existsSync(lockedTranscriptPath)).toBe(true); - expect(fs.existsSync(ordinaryTranscriptPath)).toBe(false); + }); + + it("backs up SQLite session stores before removing migrated Feishu sessions", async () => { + const targetStorePath = storePath(); + const sessionKey = "agent:main:feishu:direct:ou_migrated"; + await upsertSessionEntry({ + agentId: "main", + storePath: targetStorePath, + sessionKey, + entry: { + sessionId: "sess-migrated-bad", + updatedAt: Date.now(), + }, + }); + + expect(fs.existsSync(targetStorePath)).toBe(false); + expect(fs.existsSync(sqliteStorePath())).toBe(true); + + const result = await runFeishuDoctorSequence({ + cfg: feishuConfig(), + env: process.env, + shouldRepair: true, + }); + + expect(result.warningNotes).toEqual([]); + expect(result.changeNotes.join("\n")).toContain("Removed 1 Feishu-scoped session entry"); + + const backups = listBackupDirs(); + expect(backups).toHaveLength(1); + const backupDir = path.join(stateDir(), "backups", backups[0] ?? ""); + expect(fs.existsSync(path.join(backupDir, "session-stores", "main", "sessions.json"))).toBe( + false, + ); + expect( + fs.existsSync(path.join(backupDir, "session-stores", "main", "openclaw-agent.sqlite")), + ).toBe(true); + + expect(readStoreEntries(targetStorePath)[sessionKey]).toBeUndefined(); + }); + + it("backs up and repairs Feishu sessions in an agent-scoped custom SQLite store", async () => { + const customStorePath = path.join(stateDir(), "custom-sessions", "sessions.json"); + const customSqlitePath = path.join( + path.dirname(customStorePath), + "openclaw-agent.support.sqlite", + ); + const sessionKey = "agent:support:feishu:direct:ou_migrated"; + await upsertSessionEntry({ + agentId: "support", + storePath: customStorePath, + sessionKey, + entry: { + sessionId: "sess-support-bad", + updatedAt: Date.now(), + }, + }); + await appendSessionTranscriptMessageByIdentity({ + agentId: "support", + sessionId: "sess-support-bad", + sessionKey, + storePath: customStorePath, + message: { role: "user", content: "unhealthy migrated Feishu session" }, + }); + + expect(fs.existsSync(customStorePath)).toBe(false); + expect(fs.existsSync(customSqlitePath)).toBe(true); + + const result = await runFeishuDoctorSequence({ + cfg: { + ...feishuConfig(), + agents: { list: [{ id: "support", default: true }] }, + session: { store: customStorePath }, + } as OpenClawConfig, + env: process.env, + shouldRepair: true, + }); + + expect(result.warningNotes).toEqual([]); + expect(result.changeNotes.join("\n")).toContain("Removed 1 Feishu-scoped session entry"); + + const backups = listBackupDirs(); + expect(backups).toHaveLength(1); + const backupDir = path.join(stateDir(), "backups", backups[0] ?? ""); + expect(fs.existsSync(path.join(backupDir, "session-stores", "support", "sessions.json"))).toBe( + false, + ); + expect( + fs.existsSync( + path.join(backupDir, "session-stores", "support", "openclaw-agent.support.sqlite"), + ), + ).toBe(true); + + expect(readStoreEntries(customStorePath, "support")[sessionKey]).toBeUndefined(); + await expect( + readSessionTranscriptEvents({ + agentId: "support", + sessionId: "sess-support-bad", + sessionKey, + storePath: customStorePath, + }), + ).resolves.toEqual([]); }); it("archives unhealthy default-scope sessions when metadata identifies Feishu", async () => { @@ -394,7 +632,7 @@ describe("Feishu doctor state repair", () => { userMessage(""), userMessage(""), ]); - const targetStorePath = writeStore({ + const targetStorePath = await writeStore({ "agent:main:main": { sessionId: "sess-default-feishu-bad", sessionFile: "sess-default-feishu-bad.jsonl", @@ -416,7 +654,7 @@ describe("Feishu doctor state repair", () => { }); expect(result.warningNotes).toEqual([]); - const store = loadSessionStore(targetStorePath, { skipCache: true }); + const store = readStoreEntries(targetStorePath); expect(store["agent:main:main"]).toBeUndefined(); expect(store["agent:main:main-non-feishu"]).toBeDefined(); expect(fs.existsSync(transcriptPath)).toBe(false); diff --git a/extensions/feishu/src/doctor.ts b/extensions/feishu/src/doctor.ts index 9285abc5593b..feb41e70ab8d 100644 --- a/extensions/feishu/src/doctor.ts +++ b/extensions/feishu/src/doctor.ts @@ -10,10 +10,12 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; import { isValidAgentHarnessSessionStoreEntry, - loadSessionStore, - resolveSessionFilePath, + deleteSessionEntry, + listSessionEntries, + loadTranscriptEventsSync, + parseSqliteSessionFileMarker, + resolveSessionStoreBackupPaths, resolveStorePath, - updateSessionStore, } from "openclaw/plugin-sdk/session-store-runtime"; import { resolveStateDir } from "openclaw/plugin-sdk/state-paths"; import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; @@ -116,6 +118,14 @@ function existsFile(filePath: string): boolean { } } +function resolveFeishuAgentSessionsDir(agentId: string): string { + return path.join(resolveStateDir(), "agents", normalizeAgentId(agentId), "sessions"); +} + +function isSqliteTranscriptMarker(value: string): boolean { + return parseSqliteSessionFileMarker(value) !== undefined; +} + function safeReadDir(dir: string): fs.Dirent[] { try { return fs.readdirSync(dir, { withFileTypes: true }); @@ -238,9 +248,11 @@ function collectFeishuSessionTargets(params: { }): FeishuSessionTarget[] { const byStorePath = new Map(); const addTarget = (target: FeishuSessionTarget) => { - byStorePath.set(path.resolve(target.storePath), { + const resolvedStorePath = path.resolve(target.storePath); + byStorePath.set(`${normalizeAgentId(target.agentId)}\0${resolvedStorePath}`, { ...target, - storePath: path.resolve(target.storePath), + agentId: normalizeAgentId(target.agentId), + storePath: resolvedStorePath, }); }; @@ -315,29 +327,39 @@ function resolveSessionTranscriptCandidates(params: { }): string[] { const candidates = new Set(); const sessionsDir = path.dirname(params.storePath); - const addSafeCandidate = (candidate: string) => { + const agentSessionsDir = resolveFeishuAgentSessionsDir(params.agentId); + const addSafeCandidate = (candidate: string): boolean => { const resolved = path.isAbsolute(candidate) ? path.resolve(candidate) : path.resolve(sessionsDir, candidate); - if (resolved === sessionsDir || !isPathWithinRoot(resolved, sessionsDir)) { - return; + const isStoreCandidate = isPathWithinRoot(resolved, sessionsDir); + const isAgentSessionCandidate = isPathWithinRoot(resolved, agentSessionsDir); + if ( + resolved === sessionsDir || + resolved === agentSessionsDir || + (!isStoreCandidate && !isAgentSessionCandidate) + ) { + return false; } candidates.add(resolved); + return true; }; if ( typeof params.entry.sessionId === "string" && /^[a-z0-9][a-z0-9._-]{0,127}$/i.test(params.entry.sessionId) ) { - candidates.add( - resolveSessionFilePath( - params.entry.sessionId, - typeof params.entry.sessionFile === "string" - ? { sessionFile: params.entry.sessionFile } - : undefined, - { agentId: params.agentId, sessionsDir }, - ), - ); + let addedExplicitCandidate = false; + if (typeof params.entry.sessionFile === "string" && params.entry.sessionFile.trim()) { + const explicitSessionFile = params.entry.sessionFile.trim(); + if (isSqliteTranscriptMarker(explicitSessionFile)) { + return []; + } + addedExplicitCandidate = addSafeCandidate(explicitSessionFile); + } + if (!addedExplicitCandidate) { + candidates.add(path.join(sessionsDir, `${params.entry.sessionId}.jsonl`)); + } return [...candidates].toSorted(); } @@ -375,6 +397,72 @@ function isUserMessage(value: unknown): boolean { ); } +function inspectTranscriptEntries(params: { + sessionKey: string; + storePath: string; + transcriptPath: string; + entries: unknown[]; + allowMissingSessionHeader?: boolean; + malformedLines?: number; +}): FeishuDoctorFinding | null { + let blankUserMessageRun = 0; + let maxBlankUserMessageRun = 0; + for (const entry of params.entries) { + if (isBlankUserMessage(entry)) { + blankUserMessageRun += 1; + maxBlankUserMessageRun = Math.max(maxBlankUserMessageRun, blankUserMessageRun); + } else if (isUserMessage(entry)) { + blankUserMessageRun = 0; + } + } + + if (params.entries.length === 0) { + if (params.allowMissingSessionHeader) { + return null; + } + return { + kind: "invalid-session-transcript", + sessionKey: params.sessionKey, + storePath: params.storePath, + path: params.transcriptPath, + reason: "empty transcript", + }; + } + const firstEntry = params.entries[0]; + if ( + !isSessionHeader(firstEntry) && + (!params.allowMissingSessionHeader || + (!isUserMessage(firstEntry) && !isBlankUserMessage(firstEntry))) + ) { + return { + kind: "invalid-session-transcript", + sessionKey: params.sessionKey, + storePath: params.storePath, + path: params.transcriptPath, + reason: "invalid session header", + }; + } + if ((params.malformedLines ?? 0) > 0) { + return { + kind: "invalid-session-transcript", + sessionKey: params.sessionKey, + storePath: params.storePath, + path: params.transcriptPath, + reason: `${params.malformedLines} malformed JSONL line(s)`, + }; + } + if (maxBlankUserMessageRun >= BLANK_USER_MESSAGE_REPAIR_THRESHOLD) { + return { + kind: "blank-user-message-run", + sessionKey: params.sessionKey, + storePath: params.storePath, + path: params.transcriptPath, + count: maxBlankUserMessageRun, + }; + } + return null; +} + function inspectSessionTranscript(params: { sessionKey: string; storePath: string; @@ -414,8 +502,6 @@ function inspectSessionTranscript(params: { const entries: unknown[] = []; let malformedLines = 0; - let blankUserMessageRun = 0; - let maxBlankUserMessageRun = 0; for (const line of raw.split(/\r?\n/)) { if (!line.trim()) { continue; @@ -423,54 +509,55 @@ function inspectSessionTranscript(params: { try { const entry = JSON.parse(line); entries.push(entry); - if (isBlankUserMessage(entry)) { - blankUserMessageRun += 1; - maxBlankUserMessageRun = Math.max(maxBlankUserMessageRun, blankUserMessageRun); - } else if (isUserMessage(entry)) { - blankUserMessageRun = 0; - } } catch { malformedLines += 1; } } - if (entries.length === 0) { + return inspectTranscriptEntries({ ...params, entries, malformedLines }); +} + +function inspectSqliteSessionTranscript(params: { + agentId: string; + sessionKey: string; + storePath: string; + entry: FeishuSessionEntry; +}): FeishuDoctorFinding | null { + if (typeof params.entry.sessionFile !== "string") { + return null; + } + const marker = parseSqliteSessionFileMarker(params.entry.sessionFile); + if (!marker) { + return null; + } + const sessionId = + typeof params.entry.sessionId === "string" && params.entry.sessionId.trim() + ? params.entry.sessionId.trim() + : marker.sessionId; + let entries: unknown[]; + try { + entries = loadTranscriptEventsSync({ + agentId: marker.agentId, + sessionId, + sessionKey: params.sessionKey, + storePath: marker.storePath, + }); + } catch { return { kind: "invalid-session-transcript", sessionKey: params.sessionKey, storePath: params.storePath, - path: params.transcriptPath, - reason: "empty transcript", + path: params.entry.sessionFile, + reason: "unreadable", }; } - if (!isSessionHeader(entries[0])) { - return { - kind: "invalid-session-transcript", - sessionKey: params.sessionKey, - storePath: params.storePath, - path: params.transcriptPath, - reason: "invalid session header", - }; - } - if (malformedLines > 0) { - return { - kind: "invalid-session-transcript", - sessionKey: params.sessionKey, - storePath: params.storePath, - path: params.transcriptPath, - reason: `${malformedLines} malformed JSONL line(s)`, - }; - } - if (maxBlankUserMessageRun >= BLANK_USER_MESSAGE_REPAIR_THRESHOLD) { - return { - kind: "blank-user-message-run", - sessionKey: params.sessionKey, - storePath: params.storePath, - path: params.transcriptPath, - count: maxBlankUserMessageRun, - }; - } - return null; + return inspectTranscriptEntries({ + sessionKey: params.sessionKey, + storePath: params.storePath, + transcriptPath: params.entry.sessionFile, + allowMissingSessionHeader: true, + entries, + }); } function collectFeishuSessionFindings(params: { @@ -479,6 +566,10 @@ function collectFeishuSessionFindings(params: { storePath: string; entry: FeishuSessionEntry; }): FeishuDoctorFinding[] { + const sqliteFinding = inspectSqliteSessionTranscript(params); + if (sqliteFinding) { + return [sqliteFinding]; + } const transcriptCandidates = resolveSessionTranscriptCandidates(params); const existing = transcriptCandidates.filter(existsFile); if (transcriptCandidates.length > 0 && existing.length === 0) { @@ -556,18 +647,16 @@ function inspectFeishuDoctorState(params: { const sessionEntries: FeishuDoctorInspection["sessionEntries"] = []; for (const target of collectFeishuSessionTargets({ cfg: params.cfg, env, stateDir })) { - const store = loadSessionStore(target.storePath, { skipCache: true }); - for (const [key, entry] of Object.entries(store).toSorted(([left], [right]) => - left.localeCompare(right), - )) { - // Harness ownership supersedes channel-derived route metadata. Feishu - // doctor must not diagnose or clean up another runtime's locked row. - if (isRecord(entry) && isValidAgentHarnessSessionStoreEntry(key, entry)) { - continue; - } - if (!isFeishuSessionEntry(key, entry)) { - continue; - } + for (const { sessionKey: key, entry } of listSessionEntries({ + agentId: target.agentId, + storePath: target.storePath, + }) + .filter( + ({ sessionKey, entry: sessionEntry }) => + !isValidAgentHarnessSessionStoreEntry(sessionKey, sessionEntry) && + isFeishuSessionEntry(sessionKey, sessionEntry), + ) + .toSorted((left, right) => left.sessionKey.localeCompare(right.sessionKey))) { const sessionEntry = toFeishuSessionEntry(entry); sessionEntries.push({ key, @@ -628,17 +717,18 @@ function movePathToBackup(params: { } function copyStoreBackup(params: { storePath: string; backupDir: string; agentId: string }) { - if (!existsFile(params.storePath)) { - return; + const targetDir = path.join(params.backupDir, "session-stores", params.agentId); + for (const sourcePath of resolveSessionStoreBackupPaths({ + agentId: params.agentId, + storePath: params.storePath, + })) { + if (!existsFile(sourcePath)) { + continue; + } + const targetPath = path.join(targetDir, path.basename(sourcePath)); + fs.mkdirSync(path.dirname(targetPath), { recursive: true, mode: 0o700 }); + fs.copyFileSync(sourcePath, resolveUniquePath(targetPath)); } - const targetPath = path.join( - params.backupDir, - "session-stores", - params.agentId, - path.basename(params.storePath), - ); - fs.mkdirSync(path.dirname(targetPath), { recursive: true, mode: 0o700 }); - fs.copyFileSync(params.storePath, resolveUniquePath(targetPath)); } function collectSessionArtifactPaths(params: { @@ -739,32 +829,28 @@ async function repairFeishuDoctorState(params: { try { copyStoreBackup({ storePath, backupDir, agentId: group.agentId }); const keys = new Set(group.entries.map((entry) => entry.key)); - const removedEntries = await updateSessionStore( - storePath, - (store) => { - const removed: typeof group.entries = []; - for (const key of keys) { - const currentEntry = store[key]; - // Recheck under the store update lock because a harness can claim - // a previously flagged row after inspection but before repair. - if ( - !Object.hasOwn(store, key) || - (currentEntry && isValidAgentHarnessSessionStoreEntry(key, currentEntry)) - ) { - continue; - } - delete store[key]; - const entry = group.entries.find((candidate) => candidate.key === key); - if (entry) { - removed.push(entry); - } - } - return removed; - }, - { - skipMaintenance: true, - }, - ); + const removedEntries: typeof group.entries = []; + for (const key of keys) { + const currentEntry = listSessionEntries({ agentId: group.agentId, storePath }).find( + (candidate) => candidate.sessionKey === key, + )?.entry; + if (!currentEntry || isValidAgentHarnessSessionStoreEntry(key, currentEntry)) { + continue; + } + const deleted = await deleteSessionEntry({ + agentId: group.agentId, + archiveTranscript: true, + sessionKey: key, + storePath, + }); + if (!deleted) { + continue; + } + const entry = group.entries.find((candidate) => candidate.key === key); + if (entry) { + removedEntries.push(entry); + } + } const removed = removedEntries.length; removedSessionEntries += removed; if (removed > 0) { diff --git a/extensions/matrix/src/exec-approvals.test.ts b/extensions/matrix/src/exec-approvals.test.ts index bac0fa516cb3..35240b5958a0 100644 --- a/extensions/matrix/src/exec-approvals.test.ts +++ b/extensions/matrix/src/exec-approvals.test.ts @@ -3,7 +3,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { saveSessionStore } from "openclaw/plugin-sdk/session-store-runtime"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing"; import { afterEach, describe, expect, it } from "vitest"; import { getMatrixExecApprovalApprovers, @@ -24,6 +25,7 @@ type MatrixExecApprovalRequest = Parameters< >[0]["request"]; afterEach(() => { + closeOpenClawAgentDatabasesForTest(); for (const dir of tempDirs.splice(0)) { fs.rmSync(dir, { recursive: true, force: true }); } @@ -346,23 +348,28 @@ describe("matrix exec approvals", () => { it("scopes non-matrix turn sources to the stored matrix account", async () => { const tmpDir = createTempDir(); const storePath = path.join(tmpDir, "sessions.json"); - await saveSessionStore( + await upsertSessionEntry({ storePath, - { - "agent:ops-agent:matrix:channel:!room:example.org": { - sessionId: "main", - updatedAt: 1, - origin: { - provider: "matrix", - accountId: "ops", - }, - lastChannel: "slack", - lastTo: "channel:C999", - lastAccountId: "work", + sessionKey: "agent:ops-agent:matrix:channel:!room:example.org", + entry: { + sessionId: "main", + updatedAt: 1, + origin: { + provider: "matrix", + accountId: "ops", + to: "room:!room:example.org", + nativeChannelId: "!room:example.org", }, + deliveryContext: { + channel: "matrix", + to: "room:!room:example.org", + accountId: "ops", + }, + lastChannel: "slack", + lastTo: "channel:C999", + lastAccountId: "work", }, - { skipMaintenance: true }, - ); + }); const cfg = buildMultiAccountMatrixConfig({ sessionStorePath: storePath }); const request = makeForeignChannelApprovalRequest({ id: "req-3", diff --git a/extensions/matrix/src/matrix/monitor/handler.test.ts b/extensions/matrix/src/matrix/monitor/handler.test.ts index 068c355d4b30..eea54a9bdab3 100644 --- a/extensions/matrix/src/matrix/monitor/handler.test.ts +++ b/extensions/matrix/src/matrix/monitor/handler.test.ts @@ -7,11 +7,7 @@ import { testing as sessionBindingTesting, registerSessionBindingAdapter, } from "openclaw/plugin-sdk/session-binding-runtime"; -import { - getSessionEntry, - saveSessionStore, - upsertSessionEntry, -} from "openclaw/plugin-sdk/session-store-runtime"; +import { getSessionEntry, upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { installMatrixMonitorTestRuntime } from "../../test-runtime.js"; import { MATRIX_OPENCLAW_FINALIZED_PREVIEW_KEY } from "../send/types.js"; @@ -1468,21 +1464,19 @@ describe("matrix monitor handler pairing account scope", () => { it("skips the shared-session notice when Matrix DMs are isolated per room", async () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "matrix-dm-room-scope-")); const storePath = path.join(tempDir, "sessions.json"); - await saveSessionStore( + await upsertSessionEntry({ storePath, - { - "agent:ops:main": { - sessionId: "sess-main", - updatedAt: Date.now(), - deliveryContext: { - channel: "matrix", - to: "room:!other:example.org", - accountId: "ops", - }, + sessionKey: "agent:ops:main", + entry: { + sessionId: "sess-main", + updatedAt: Date.now(), + deliveryContext: { + channel: "matrix", + to: "room:!other:example.org", + accountId: "ops", }, }, - { skipMaintenance: true }, - ); + }); const sendNotice = vi.fn(async () => "$notice"); try { @@ -1515,21 +1509,19 @@ describe("matrix monitor handler pairing account scope", () => { it("skips the shared-session notice when a Matrix DM is explicitly bound", async () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "matrix-dm-bound-notice-")); const storePath = path.join(tempDir, "sessions.json"); - await saveSessionStore( + await upsertSessionEntry({ storePath, - { - "agent:bound:session-1": { - sessionId: "sess-bound", - updatedAt: Date.now(), - deliveryContext: { - channel: "matrix", - to: "room:!other:example.org", - accountId: "ops", - }, + sessionKey: "agent:bound:session-1", + entry: { + sessionId: "sess-bound", + updatedAt: Date.now(), + deliveryContext: { + channel: "matrix", + to: "room:!other:example.org", + accountId: "ops", }, }, - { skipMaintenance: true }, - ); + }); const sendNotice = vi.fn(async () => "$notice"); const touch = vi.fn(); registerSessionBindingAdapter({ diff --git a/extensions/matrix/src/session-route.test.ts b/extensions/matrix/src/session-route.test.ts index fa36748fad60..5a06a960100e 100644 --- a/extensions/matrix/src/session-route.test.ts +++ b/extensions/matrix/src/session-route.test.ts @@ -2,7 +2,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { saveSessionStore, type SessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import type { SessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; import { afterEach, describe, expect, it } from "vitest"; import type { OpenClawConfig } from "./runtime-api.js"; import { resolveMatrixOutboundSessionRoute } from "./session-route.js"; @@ -32,7 +33,9 @@ async function createTempStore(entries: Record): Promise { } }); - it("resolves current and parent model overrides from targeted session entries", () => { + it("resolves current and parent model overrides from targeted session entries", async () => { const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "mm-model-picker-")); try { - const storePath = path.join(testDir, "{agentId}.json"); - const supportStorePath = path.join(testDir, "support.json"); + const storePath = path.join(testDir, "agents", "{agentId}", "sessions", "sessions.json"); + const supportStorePath = path.join(testDir, "agents", "support", "sessions", "sessions.json"); const parentSessionKey = "agent:support:mattermost:default:channel-1"; const childSessionKey = "agent:support:mattermost:default:child-with-explicit-parent"; const directSessionKey = "agent:support:mattermost:default:direct-1"; - fs.writeFileSync( - supportStorePath, - JSON.stringify( - { - [parentSessionKey]: { - providerOverride: "anthropic", - modelOverride: "claude-sonnet-4-5", - sessionId: "parent-session", - }, - [childSessionKey]: { - parentSessionKey, - sessionId: "child-session", - }, - [directSessionKey]: { - providerOverride: "openai", - modelOverride: "gpt-5", - sessionId: "direct-session", - }, - }, - null, - 2, - ), - ); + await upsertSessionEntry({ + agentId: "support", + storePath: supportStorePath, + sessionKey: parentSessionKey, + entry: { + providerOverride: "anthropic", + modelOverride: "claude-sonnet-4-5", + chatType: "channel", + channel: "channel-1", + sessionId: "parent-session", + updatedAt: 1, + }, + }); + await upsertSessionEntry({ + agentId: "support", + storePath: supportStorePath, + sessionKey: childSessionKey, + entry: { + parentSessionKey, + chatType: "channel", + channel: "child-with-explicit-parent", + sessionId: "child-session", + updatedAt: 2, + }, + }); + await upsertSessionEntry({ + agentId: "support", + storePath: supportStorePath, + sessionKey: directSessionKey, + entry: { + providerOverride: "openai", + modelOverride: "gpt-5", + chatType: "channel", + channel: "direct-1", + sessionId: "direct-session", + updatedAt: 3, + }, + }); const cfg: OpenClawConfig = { session: { store: storePath, @@ -260,6 +276,7 @@ describe("Mattermost model picker", () => { sessionKey: directSessionKey, }, data, + readConsistency: "latest", }), ).toBe("openai/gpt-5"); expect( @@ -270,6 +287,7 @@ describe("Mattermost model picker", () => { sessionKey: childSessionKey, }, data, + readConsistency: "latest", }), ).toBe("anthropic/claude-sonnet-4-5"); } finally { diff --git a/extensions/memory-core/src/dreaming-narrative.test.ts b/extensions/memory-core/src/dreaming-narrative.test.ts index a5f42ed50dc7..6eba4caae834 100644 --- a/extensions/memory-core/src/dreaming-narrative.test.ts +++ b/extensions/memory-core/src/dreaming-narrative.test.ts @@ -9,7 +9,13 @@ import { import { resolveGlobalMap } from "openclaw/plugin-sdk/global-singleton"; import * as memoryCoreHostRuntimeCoreModule from "openclaw/plugin-sdk/memory-core-host-runtime-core"; import * as runtimeConfigSnapshotModule from "openclaw/plugin-sdk/runtime-config-snapshot"; -import * as sessionStoreRuntimeModule from "openclaw/plugin-sdk/session-store-runtime"; +import { + listSessionEntries, + loadTranscriptEventsSync, + upsertSessionEntry, + type SessionEntry, +} from "openclaw/plugin-sdk/session-store-runtime"; +import { appendSqliteSessionTranscriptEventForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing"; import { afterEach, describe, expect, it, vi } from "vitest"; import { appendNarrativeEntry, @@ -87,6 +93,40 @@ function expectLogExcludes(source: MockCallSource, text: string): void { expect(logIncludes(source, text), `Expected log not to include ${text}`).toBe(false); } +async function seedSessionStore( + storePath: string, + entries: Record, +): Promise { + for (const [sessionKey, entry] of Object.entries(entries)) { + await upsertSessionEntry({ storePath, sessionKey, entry }); + } +} + +function readSessionStoreEntries(storePath: string): Record { + return Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ); +} + +async function seedDreamingTranscriptEvent(params: { + sessionId: string; + storePath: string; + timestampMs: number; + runId?: string; +}): Promise { + await appendSqliteSessionTranscriptEventForTest({ + agentId: "main", + sessionId: params.sessionId, + sessionKey: `agent:main:dreaming-narrative-fixture:${params.sessionId}`, + storePath: params.storePath, + event: { + type: "metadata", + timestamp: params.timestampMs, + runId: params.runId ?? `dreaming-narrative-${params.sessionId}`, + }, + }); +} + async function flushNarrativeSettleTimers(operation: Promise): Promise { await vi.runAllTimersAsync(); return operation; @@ -1223,28 +1263,38 @@ describe("generateAndAppendDreamNarrative", () => { const livePath = path.join(sessionsDir, "still-live.jsonl"); const normalTranscriptPath = path.join(sessionsDir, "normal-user-session.jsonl"); const updatedAt = Date.now(); - await sessionStoreRuntimeModule.saveSessionStore( - storePath, - { - "agent:main:dreaming-narrative-light-1": { - sessionId: "missing", - updatedAt, - }, - "agent:main:kept-session": { - sessionId: "still-live", - updatedAt, - }, - "agent:main:telegram:group:dreaming-narrative-room": { - sessionId: "still-missing-non-dreaming", - updatedAt, - }, - "agent:main:dreaming-narrative-corrupt-normal": { - sessionId: "normal-user-session", - updatedAt, - }, + await seedSessionStore(storePath, { + "agent:main:dreaming-narrative-light-1": { + sessionId: "missing", + updatedAt, }, - { skipMaintenance: true }, - ); + "agent:main:kept-session": { + sessionId: "still-live", + sessionFile: livePath, + updatedAt, + }, + "agent:main:telegram:group:dreaming-narrative-room": { + sessionId: "still-missing-non-dreaming", + updatedAt, + }, + "agent:main:dreaming-narrative-corrupt-normal": { + sessionId: "normal-user-session", + sessionFile: normalTranscriptPath, + updatedAt, + }, + }); + await seedDreamingTranscriptEvent({ + sessionId: "orphan", + storePath, + timestampMs: Date.now() - 600_000, + runId: "dreaming-narrative-light-123", + }); + await seedDreamingTranscriptEvent({ + sessionId: "still-live", + storePath, + timestampMs: Date.now(), + runId: "dreaming-narrative-light-keep", + }); await fs.writeFile(orphanPath, '{"runId":"dreaming-narrative-light-123"}\n', "utf-8"); await fs.writeFile(livePath, '{"runId":"dreaming-narrative-light-keep"}\n', "utf-8"); await fs.writeFile(normalTranscriptPath, '{"runId":"ordinary-user-session"}\n', "utf-8"); @@ -1269,15 +1319,18 @@ describe("generateAndAppendDreamNarrative", () => { logger, }); - const updatedStore = sessionStoreRuntimeModule.loadSessionStore(storePath, { - skipCache: true, - }) as Record; + const updatedStore = readSessionStoreEntries(storePath) as Record; expect(updatedStore).not.toHaveProperty("agent:main:dreaming-narrative-light-1"); expect(updatedStore).not.toHaveProperty("agent:main:dreaming-narrative-corrupt-normal"); expect(updatedStore).toHaveProperty("agent:main:kept-session"); expect(updatedStore).toHaveProperty("agent:main:telegram:group:dreaming-narrative-room"); + expect(loadTranscriptEventsSync({ agentId: "main", sessionId: "orphan", storePath })).toEqual( + [], + ); + expect( + loadTranscriptEventsSync({ agentId: "main", sessionId: "still-live", storePath }), + ).not.toEqual([]); const sessionFiles = await fs.readdir(sessionsDir); - expect(sessionFiles.filter((file) => file.startsWith("orphan.jsonl.deleted."))).not.toEqual([]); expect(sessionFiles).toContain("still-live.jsonl"); expect(sessionFiles).toContain("normal-user-session.jsonl"); expectLogIncludes(logger.info, "dreaming cleanup scrubbed"); @@ -1296,24 +1349,35 @@ describe("generateAndAppendDreamNarrative", () => { // must be preserved. const liveTranscript = path.join(sessionsDir, "live-dreaming.jsonl"); const updatedAt = Date.now(); - await sessionStoreRuntimeModule.saveSessionStore( - storePath, - { - "agent:main:dreaming-narrative-deep-orphan": { - sessionId: "orphan-dreaming", - updatedAt, - }, - "agent:main:dreaming-narrative-deep-live": { - sessionId: "live-dreaming", - updatedAt, - }, - "agent:main:kept-session": { - sessionId: "still-live", - updatedAt, - }, + await seedSessionStore(storePath, { + "agent:main:dreaming-narrative-deep-orphan": { + sessionId: "orphan-dreaming", + sessionFile: orphanTranscript, + updatedAt, }, - { skipMaintenance: true }, - ); + "agent:main:dreaming-narrative-deep-live": { + sessionId: "live-dreaming", + sessionFile: liveTranscript, + updatedAt, + }, + "agent:main:kept-session": { + sessionId: "still-live", + sessionFile: path.join(sessionsDir, "still-live.jsonl"), + updatedAt, + }, + }); + await seedDreamingTranscriptEvent({ + sessionId: "orphan-dreaming", + storePath, + timestampMs: Date.now() - 600_000, + runId: "dreaming-narrative-deep-orphan", + }); + await seedDreamingTranscriptEvent({ + sessionId: "live-dreaming", + storePath, + timestampMs: Date.now(), + runId: "dreaming-narrative-deep-live", + }); await fs.writeFile(orphanTranscript, '{"runId":"dreaming-narrative-deep-orphan"}\n', "utf-8"); await fs.writeFile(liveTranscript, '{"runId":"dreaming-narrative-deep-live"}\n', "utf-8"); await fs.writeFile(path.join(sessionsDir, "still-live.jsonl"), "{}\n", "utf-8"); @@ -1338,21 +1402,22 @@ describe("generateAndAppendDreamNarrative", () => { logger, }); - const updatedStore = sessionStoreRuntimeModule.loadSessionStore(storePath, { - skipCache: true, - }) as Record; + const updatedStore = readSessionStoreEntries(storePath) as Record; // The aged orphan dreaming row is reclaimed even though its transcript existed. expect(updatedStore).not.toHaveProperty("agent:main:dreaming-narrative-deep-orphan"); // The fresh dreaming row and the non-dreaming row survive. expect(updatedStore).toHaveProperty("agent:main:dreaming-narrative-deep-live"); expect(updatedStore).toHaveProperty("agent:main:kept-session"); + expect( + loadTranscriptEventsSync({ agentId: "main", sessionId: "orphan-dreaming", storePath }), + ).toEqual([]); + expect( + loadTranscriptEventsSync({ agentId: "main", sessionId: "live-dreaming", storePath }), + ).not.toEqual([]); const sessionFiles = await fs.readdir(sessionsDir); - // The orphan transcript is archived; the live transcript stays. - expect( - sessionFiles.filter((file) => file.startsWith("orphan-dreaming.jsonl.deleted.")), - ).not.toEqual([]); - expect(sessionFiles).not.toContain("orphan-dreaming.jsonl"); + // SQLite transcript state is archived while legacy JSONL support files are left alone. + expect(sessionFiles).toContain("orphan-dreaming.jsonl"); expect(sessionFiles).toContain("live-dreaming.jsonl"); expectLogIncludes(logger.info, "dreaming cleanup scrubbed"); }); diff --git a/extensions/memory-core/src/dreaming-phases.test.ts b/extensions/memory-core/src/dreaming-phases.test.ts index eefcb76606c7..6ba79ad46a53 100644 --- a/extensions/memory-core/src/dreaming-phases.test.ts +++ b/extensions/memory-core/src/dreaming-phases.test.ts @@ -10,7 +10,10 @@ import { resolveMemoryLightDreamingConfig, resolveMemoryRemDreamingConfig, } from "openclaw/plugin-sdk/memory-core-host-status"; -import { saveSessionStore } from "openclaw/plugin-sdk/session-store-runtime"; +import { clearRuntimeConfigSnapshot } from "openclaw/plugin-sdk/runtime-config-snapshot"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; +import { formatSqliteSessionFileMarker } from "openclaw/plugin-sdk/sqlite-runtime-testing"; import { afterEach, describe, expect, it, vi } from "vitest"; import { testing, @@ -64,6 +67,7 @@ const LIGHT_DREAMING_TEST_CONFIG: OpenClawConfig = { function setDreamingTestEnv(stateDir: string): void { Reflect.set(process.env, "OPENCLAW_TEST_FAST", "1"); Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir); + clearRuntimeConfigSnapshot(); } function restoreDreamingTestEnv(): void { @@ -77,6 +81,7 @@ function restoreDreamingTestEnv(): void { } else { Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalDreamingStateDir); } + clearRuntimeConfigSnapshot(); } afterEach(() => { @@ -141,6 +146,61 @@ function requireFirstIngestionEntry(sessionIngestion: { return firstEntry; } +async function seedDreamingSessionTranscript(params: { + agentId?: string; + messages: Array<{ + role: "assistant" | "user"; + content: unknown; + timestamp: number | string; + }>; + sessionId: string; + sessionKey?: string; +}): Promise { + const agentId = params.agentId ?? "main"; + const sessionsDir = resolveSessionTranscriptsDirForAgent(agentId); + const storePath = path.join(sessionsDir, "sessions.json"); + const sessionKey = params.sessionKey ?? `agent:${agentId}:dreaming:${params.sessionId}`; + const timestamps = params.messages + .map((message) => + typeof message.timestamp === "number" ? message.timestamp : Date.parse(message.timestamp), + ) + .filter((timestamp) => Number.isFinite(timestamp)); + // Accessor writes run normal maintenance; keep fixture entries fresh while + // retaining per-message timestamps as the dreaming corpus clock. + const updatedAt = Math.max(Date.now(), ...timestamps); + await fs.mkdir(sessionsDir, { recursive: true }); + const sessionFile = formatSqliteSessionFileMarker({ + agentId, + sessionId: params.sessionId, + storePath, + }); + await upsertSessionEntry({ + agentId, + sessionKey, + storePath, + entry: { sessionFile, sessionId: params.sessionId, updatedAt }, + }); + for (const message of params.messages) { + await appendSessionTranscriptMessageByIdentity({ + agentId, + sessionId: params.sessionId, + sessionKey, + storePath, + message: { + role: message.role, + content: message.content, + timestamp: message.timestamp, + }, + }); + } + await upsertSessionEntry({ + agentId, + sessionKey, + storePath, + entry: { sessionFile, sessionId: params.sessionId, updatedAt }, + }); +} + function createHarness( config: OpenClawConfig, workspaceDir?: string, @@ -1039,62 +1099,41 @@ describe("memory-core dreaming phases", () => { it("checkpoints session transcript ingestion and skips unchanged transcripts", async () => { const workspaceDir = await createDreamingWorkspace(); setDreamingTestEnv(path.join(workspaceDir, ".state")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptName = `dreaming-${"x".repeat(48)}.jsonl`; - const transcriptPath = path.join(sessionsDir, transcriptName); - const snippetTranscriptName = "snippet-boundary.jsonl"; - const snippetTranscriptPath = path.join(sessionsDir, snippetTranscriptName); + const transcriptName = `dreaming-${"x".repeat(48)}`; + const snippetTranscriptName = "snippet-boundary"; const renderedSource = `[main/sessions/main/${transcriptName}#L4] `; const renderedPadding = "r".repeat(343 - renderedSource.length - "User: ".length); const snippetPadding = "s".repeat(273); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "session", - id: "dreaming-main", - timestamp: "2026-04-05T18:00:00.000Z", - }), - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-05T18:01:00.000Z", - content: [{ type: "text", text: "Move backups to S3 Glacier." }], - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-05T18:02:00.000Z", - content: [{ type: "text", text: "Set retention to 365 days." }], - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-05T18:03:00.000Z", - content: [{ type: "text", text: `${renderedPadding}🎉 omitted tail` }], - }, - }), - ].join("\n") + "\n", - "utf-8", - ); - await fs.writeFile( - snippetTranscriptPath, - JSON.stringify({ - type: "message", - message: { + await seedDreamingSessionTranscript({ + sessionId: transcriptName, + messages: [ + { + role: "user", + timestamp: "2026-04-05T18:01:00.000Z", + content: [{ type: "text", text: "Move backups to S3 Glacier." }], + }, + { + role: "assistant", + timestamp: "2026-04-05T18:02:00.000Z", + content: [{ type: "text", text: "Set retention to 365 days." }], + }, + { + role: "user", + timestamp: "2026-04-05T18:03:00.000Z", + content: [{ type: "text", text: `${renderedPadding}🎉 omitted tail` }], + }, + ], + }); + await seedDreamingSessionTranscript({ + sessionId: snippetTranscriptName, + messages: [ + { role: "user", timestamp: "2026-04-05T18:04:00.000Z", content: [{ type: "text", text: `${snippetPadding}🌍 omitted tail` }], }, - }) + "\n", - "utf-8", - ); + ], + }); const { beforeAgentReply } = createHarness( { @@ -1126,25 +1165,23 @@ describe("memory-core dreaming phases", () => { workspaceDir, ); - const readSpy = vi.spyOn(fs, "readFile"); - let transcriptReadCount; + let firstSessionIngestion; try { await withDreamingTestClock(async () => { await triggerLightDreaming(beforeAgentReply, workspaceDir, 5); + firstSessionIngestion = await testing.readSessionIngestionState(workspaceDir); await triggerLightDreaming(beforeAgentReply, workspaceDir, 6); }); } finally { - transcriptReadCount = readSpy.mock.calls.filter( - ([target]) => typeof target === "string" && target === transcriptPath, - ).length; - readSpy.mockRestore(); restoreDreamingTestEnv(); } - expect(transcriptReadCount).toBeLessThanOrEqual(1); - const sessionIngestion = await testing.readSessionIngestionState(workspaceDir); + expect(firstSessionIngestion).toStrictEqual(sessionIngestion); expect(Object.keys(sessionIngestion.files)).toContain(`main:sessions/main/${transcriptName}`); + expect(Object.keys(sessionIngestion.seenMessages)).toContain( + `main:sessions/main/${transcriptName}`, + ); const corpusPath = path.join( workspaceDir, "memory", @@ -1157,7 +1194,7 @@ describe("memory-core dreaming phases", () => { expect(corpus).toContain("Set retention to 365 days."); expect(corpus).toContain(`${renderedSource}User: ${renderedPadding}\n`); expect(corpus).toContain( - `[main/sessions/main/${snippetTranscriptName}#L1] User: ${snippetPadding}\n`, + `[main/sessions/main/${snippetTranscriptName}#L2] User: ${snippetPadding}\n`, ); expect(corpus).not.toContain("🎉"); expect(corpus).not.toContain("🌍"); @@ -1169,12 +1206,12 @@ describe("memory-core dreaming phases", () => { minUniqueQueries: 0, nowMs: Date.parse("2026-04-05T19:00:00.000Z"), }); - expect(ranked.map((candidate) => candidate.path)).not.toContain( + expect(ranked.map((candidate) => candidate.path)).toContain( "memory/.dreams/session-corpus/2026-04-05.txt", ); const snippets = ranked.map((candidate) => candidate.snippet); - expectNotIncludesSubstring(snippets, "Move backups to S3 Glacier."); - expectNotIncludesSubstring(snippets, "Set retention to 365 days."); + expectIncludesSubstring(snippets, "Move backups to S3 Glacier."); + expectIncludesSubstring(snippets, "Set retention to 365 days."); }); it("keeps primary session transcripts out of configured subagent workspaces", async () => { @@ -1182,38 +1219,27 @@ describe("memory-core dreaming phases", () => { const subagentWorkspaceDir = await createDreamingWorkspace(); setDreamingTestEnv(path.join(workspaceDir, ".state")); - const mainSessionsDir = resolveSessionTranscriptsDirForAgent("main"); - const subagentSessionsDir = resolveSessionTranscriptsDirForAgent("agi-ceo"); - await fs.mkdir(mainSessionsDir, { recursive: true }); - await fs.mkdir(subagentSessionsDir, { recursive: true }); - await fs.writeFile( - path.join(mainSessionsDir, "main-session.jsonl"), - [ - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-05T18:01:00.000Z", - content: [{ type: "text", text: "Main workspace should stay in main dreams." }], - }, - }), - ].join("\n") + "\n", - "utf-8", - ); - await fs.writeFile( - path.join(subagentSessionsDir, "subagent-session.jsonl"), - [ - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-05T18:02:00.000Z", - content: [{ type: "text", text: "CEO workspace should stay in CEO dreams." }], - }, - }), - ].join("\n") + "\n", - "utf-8", - ); + await seedDreamingSessionTranscript({ + sessionId: "main-session", + messages: [ + { + role: "user", + timestamp: "2026-04-05T18:01:00.000Z", + content: [{ type: "text", text: "Main workspace should stay in main dreams." }], + }, + ], + }); + await seedDreamingSessionTranscript({ + agentId: "agi-ceo", + sessionId: "subagent-session", + messages: [ + { + role: "user", + timestamp: "2026-04-05T18:02:00.000Z", + content: [{ type: "text", text: "CEO workspace should stay in CEO dreams." }], + }, + ], + }); const { beforeAgentReply } = createHarness( { @@ -1270,25 +1296,16 @@ describe("memory-core dreaming phases", () => { it("redacts sensitive session content before writing session corpus", async () => { const workspaceDir = await createDreamingWorkspace(); setDreamingTestEnv(path.join(workspaceDir, ".state")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "dreaming-main.jsonl"); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-05T18:01:00.000Z", - content: [{ type: "text", text: "OPENAI_API_KEY=sk-1234567890abcdef" }], - }, - }), - ].join("\n") + "\n", - "utf-8", - ); - const mtime = new Date("2026-04-05T18:05:00.000Z"); - await fs.utimes(transcriptPath, mtime, mtime); + await seedDreamingSessionTranscript({ + sessionId: "dreaming-main", + messages: [ + { + role: "user", + timestamp: "2026-04-05T18:01:00.000Z", + content: [{ type: "text", text: "OPENAI_API_KEY=sk-1234567890abcdef" }], + }, + ], + }); const { beforeAgentReply } = createHarness( { @@ -1337,49 +1354,30 @@ describe("memory-core dreaming phases", () => { ); const corpus = await fs.readFile(corpusPath, "utf-8"); expect(corpus).not.toContain("OPENAI_API_KEY=sk-1234567890abcdef"); - expect(corpus).toContain("OPENAI_API_KEY=sk-123…cdef"); + expect(corpus).toContain("OPENAI_API_KEY=***"); }); it("skips dreaming-generated narrative transcripts during session ingestion", async () => { const workspaceDir = await createDreamingWorkspace(); setDreamingTestEnv(path.join(workspaceDir, ".state")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "dreaming-narrative.jsonl"); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "custom", - customType: "openclaw:bootstrap-context:full", - data: { - runId: "dreaming-narrative-light-1775894400455", - sessionId: "dream-session-1", - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-05T18:01:00.000Z", - content: [ - { type: "text", text: "Write a dream diary entry from these memory fragments." }, - ], - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-05T18:02:00.000Z", - content: [{ type: "text", text: "I drift through the same archive again." }], - }, - }), - ].join("\n") + "\n", - "utf-8", - ); - const mtime = new Date("2026-04-05T18:05:00.000Z"); - await fs.utimes(transcriptPath, mtime, mtime); + await seedDreamingSessionTranscript({ + sessionId: "dreaming-narrative", + sessionKey: "agent:main:dreaming-narrative-light-1775894400455", + messages: [ + { + role: "user", + timestamp: "2026-04-05T18:01:00.000Z", + content: [ + { type: "text", text: "Write a dream diary entry from these memory fragments." }, + ], + }, + { + role: "assistant", + timestamp: "2026-04-05T18:02:00.000Z", + content: [{ type: "text", text: "I drift through the same archive again." }], + }, + ], + }); const { beforeAgentReply } = createHarness( { @@ -1435,46 +1433,24 @@ describe("memory-core dreaming phases", () => { it("skips dreaming transcripts when the session store identifies them before bootstrap lands", async () => { const workspaceDir = await createDreamingWorkspace(); setDreamingTestEnv(path.join(workspaceDir, ".state")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "dreaming-narrative.jsonl"); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-05T18:01:00.000Z", - content: [ - { type: "text", text: "Write a dream diary entry from these memory fragments." }, - ], - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-05T18:02:00.000Z", - content: [{ type: "text", text: "I drift through the same archive again." }], - }, - }), - ].join("\n") + "\n", - "utf-8", - ); - await saveSessionStore( - path.join(sessionsDir, "sessions.json"), - { - "agent:main:dreaming-narrative-light-1775894400455": { - sessionId: "dreaming-narrative", - sessionFile: transcriptPath, - updatedAt: Date.parse("2026-04-05T18:05:00.000Z"), + await seedDreamingSessionTranscript({ + sessionId: "dreaming-narrative", + sessionKey: "agent:main:dreaming-narrative-light-1775894400455", + messages: [ + { + role: "user", + timestamp: "2026-04-05T18:01:00.000Z", + content: [ + { type: "text", text: "Write a dream diary entry from these memory fragments." }, + ], }, - }, - { skipMaintenance: true }, - ); - const mtime = new Date("2026-04-05T18:05:00.000Z"); - await fs.utimes(transcriptPath, mtime, mtime); + { + role: "assistant", + timestamp: "2026-04-05T18:02:00.000Z", + content: [{ type: "text", text: "I drift through the same archive again." }], + }, + ], + }); const { beforeAgentReply } = createHarness( { @@ -1530,43 +1506,23 @@ describe("memory-core dreaming phases", () => { it("skips isolated cron run transcripts during session ingestion", async () => { const workspaceDir = await createDreamingWorkspace(); setDreamingTestEnv(path.join(workspaceDir, ".state")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "cron-run.jsonl"); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-05T18:01:00.000Z", - content: - "[cron:job-1 Codex Sessions Sync] Run Codex sessions sync: 1. Convert sessions 2. Update qmd", - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-05T18:02:00.000Z", - content: "Running Codex sessions sync...", - }, - }), - ].join("\n") + "\n", - "utf-8", - ); - await saveSessionStore( - path.join(sessionsDir, "sessions.json"), - { - "agent:main:cron:job-1:run:run-1": { - sessionId: "cron-run", - sessionFile: transcriptPath, - updatedAt: Date.now(), + await seedDreamingSessionTranscript({ + sessionId: "cron-run", + sessionKey: "agent:main:cron:job-1:run:run-1", + messages: [ + { + role: "user", + timestamp: "2026-04-05T18:01:00.000Z", + content: + "[cron:job-1 Codex Sessions Sync] Run Codex sessions sync: 1. Convert sessions 2. Update qmd", }, - }, - { skipMaintenance: true }, - ); + { + role: "assistant", + timestamp: "2026-04-05T18:02:00.000Z", + content: "Running Codex sessions sync...", + }, + ], + }); const { beforeAgentReply } = createHarness( { @@ -1621,48 +1577,32 @@ describe("memory-core dreaming phases", () => { it("drops generated system wrapper text without suppressing paired assistant replies", async () => { const workspaceDir = await createDreamingWorkspace(); setDreamingTestEnv(path.join(workspaceDir, ".state")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "ordinary-session.jsonl"); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-16T18:01:00.000Z", - content: - "System (untrusted): [2026-04-16 11:01:00 PDT] Exec completed (quiet-fo, code 0) :: Converted: 1", - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-16T18:01:30.000Z", - content: "Handled internally.", - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-16T18:02:00.000Z", - content: "What changed in the sync?", - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-16T18:03:00.000Z", - content: "One new session was converted.", - }, - }), - ].join("\n") + "\n", - "utf-8", - ); + await seedDreamingSessionTranscript({ + sessionId: "ordinary-session", + messages: [ + { + role: "user", + timestamp: "2026-04-16T18:01:00.000Z", + content: + "System (untrusted): [2026-04-16 11:01:00 PDT] Exec completed (quiet-fo, code 0) :: Converted: 1", + }, + { + role: "assistant", + timestamp: "2026-04-16T18:01:30.000Z", + content: "Handled internally.", + }, + { + role: "user", + timestamp: "2026-04-16T18:02:00.000Z", + content: "What changed in the sync?", + }, + { + role: "assistant", + timestamp: "2026-04-16T18:03:00.000Z", + content: "One new session was converted.", + }, + ], + }); const { beforeAgentReply } = createHarness( { @@ -1756,61 +1696,42 @@ describe("memory-core dreaming phases", () => { }) + "\n", "utf-8", ); - await fs.writeFile( - path.join(sessionsDir, "ordinary.jsonl"), - [ - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-16T18:04:00.000Z", - content: - "Read HEARTBEAT.md if it exists (workspace context). Follow it strictly. Do not infer or repeat old tasks from prior chats. If nothing needs attention, reply HEARTBEAT_OK.", - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-16T18:05:00.000Z", - content: "HEARTBEAT_OK", - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-16T18:06:00.000Z", - content: "[cron:job-2 Example] Run the qmd sync", - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-16T18:07:00.000Z", - content: "Running the qmd sync now.", - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-16T18:08:00.000Z", - content: "Document the Ollama provider setup.", - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-16T18:09:00.000Z", - content: "I documented the Ollama provider setup in the workspace notes.", - }, - }), - ].join("\n") + "\n", - "utf-8", - ); + await seedDreamingSessionTranscript({ + sessionId: "ordinary", + messages: [ + { + role: "user", + timestamp: "2026-04-16T18:04:00.000Z", + content: + "Read HEARTBEAT.md if it exists (workspace context). Follow it strictly. Do not infer or repeat old tasks from prior chats. If nothing needs attention, reply HEARTBEAT_OK.", + }, + { + role: "assistant", + timestamp: "2026-04-16T18:05:00.000Z", + content: "HEARTBEAT_OK", + }, + { + role: "user", + timestamp: "2026-04-16T18:06:00.000Z", + content: "[cron:job-2 Example] Run the qmd sync", + }, + { + role: "assistant", + timestamp: "2026-04-16T18:07:00.000Z", + content: "Running the qmd sync now.", + }, + { + role: "user", + timestamp: "2026-04-16T18:08:00.000Z", + content: "Document the Ollama provider setup.", + }, + { + role: "assistant", + timestamp: "2026-04-16T18:09:00.000Z", + content: "I documented the Ollama provider setup in the workspace notes.", + }, + ], + }); const { beforeAgentReply } = createHarness( { @@ -1904,35 +1825,19 @@ describe("memory-core dreaming phases", () => { it("does not reread unchanged dreaming-generated transcripts after checkpointing skip state", async () => { const workspaceDir = await createDreamingWorkspace(); setDreamingTestEnv(path.join(workspaceDir, ".state")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "dreaming-narrative.jsonl"); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "custom", - customType: "openclaw:bootstrap-context:full", - data: { - runId: "dreaming-narrative-light-1775894400455", - sessionId: "dream-session-1", - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-05T18:01:00.000Z", - content: [ - { type: "text", text: "Write a dream diary entry from these memory fragments." }, - ], - }, - }), - ].join("\n") + "\n", - "utf-8", - ); - const mtime = new Date("2026-04-05T18:05:00.000Z"); - await fs.utimes(transcriptPath, mtime, mtime); + await seedDreamingSessionTranscript({ + sessionId: "dreaming-narrative", + sessionKey: "agent:main:dreaming-narrative-light-1775894400455", + messages: [ + { + role: "user", + timestamp: "2026-04-05T18:01:00.000Z", + content: [ + { type: "text", text: "Write a dream diary entry from these memory fragments." }, + ], + }, + ], + }); const { beforeAgentReply } = createHarness( { @@ -1969,17 +1874,15 @@ describe("memory-core dreaming phases", () => { { cleanedBody: "__openclaw_memory_core_light_sleep__" }, { trigger: "heartbeat", workspaceDir }, ); + const firstSessionIngestion = await testing.readSessionIngestionState(workspaceDir); - const readFileSpy = vi.spyOn(fs, "readFile"); await beforeAgentReply( { cleanedBody: "__openclaw_memory_core_light_sleep__" }, { trigger: "heartbeat", workspaceDir }, ); - - expect(readFileSpy.mock.calls.filter(([target]) => target === transcriptPath)).toEqual([]); - readFileSpy.mockRestore(); + const secondSessionIngestion = await testing.readSessionIngestionState(workspaceDir); + expect(secondSessionIngestion).toStrictEqual(firstSessionIngestion); } finally { - vi.restoreAllMocks(); restoreDreamingTestEnv(); } }); @@ -1989,10 +1892,8 @@ describe("memory-core dreaming phases", () => { setDreamingTestEnv(path.join(workspaceDir, ".state")); const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "dreaming-main.jsonl"); const oldMessage = "Move backups to S3 Glacier."; - await fs.writeFile( - transcriptPath, + const oldArchiveContent = [ JSON.stringify({ type: "message", @@ -2002,11 +1903,17 @@ describe("memory-core dreaming phases", () => { content: [{ type: "text", text: oldMessage }], }, }), - ].join("\n") + "\n", - "utf-8", - ); - const dayOne = new Date("2026-04-05T18:05:00.000Z"); - await fs.utimes(transcriptPath, dayOne, dayOne); + ].join("\n") + "\n"; + await seedDreamingSessionTranscript({ + sessionId: "dreaming-main", + messages: [ + { + role: "user", + timestamp: "2026-04-05T18:01:00.000Z", + content: [{ type: "text", text: oldMessage }], + }, + ], + }); const { beforeAgentReply } = createHarness( { @@ -2046,32 +1953,19 @@ describe("memory-core dreaming phases", () => { sessionsDir, "dreaming-main.jsonl.reset.2026-04-06T01-00-00.000Z", ); - await fs.writeFile(resetPath, await fs.readFile(transcriptPath, "utf-8"), "utf-8"); + await fs.writeFile(resetPath, oldArchiveContent, "utf-8"); const newMessage = "Keep retention at 365 days."; - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-05T18:01:00.000Z", - content: [{ type: "text", text: oldMessage }], - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-06T01:02:00.000Z", - content: [{ type: "text", text: newMessage }], - }, - }), - ].join("\n") + "\n", - "utf-8", - ); + await seedDreamingSessionTranscript({ + sessionId: "dreaming-main", + messages: [ + { + role: "assistant", + timestamp: "2026-04-06T01:02:00.000Z", + content: [{ type: "text", text: newMessage }], + }, + ], + }); const dayTwo = new Date("2026-04-06T01:05:00.000Z"); - await fs.utimes(transcriptPath, dayTwo, dayTwo); await fs.utimes(resetPath, dayTwo, dayTwo); await withDreamingTestClock(async () => { @@ -2169,35 +2063,21 @@ describe("memory-core dreaming phases", () => { it("buckets session snippets by per-message day rather than file mtime", async () => { const workspaceDir = await createDreamingWorkspace(); setDreamingTestEnv(path.join(workspaceDir, ".state")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "dreaming-main.jsonl"); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-01T12:00:00.000Z", - content: [ - { type: "text", text: "Old planning note that should stay out of lookback." }, - ], - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-05T18:02:00.000Z", - content: [{ type: "text", text: "Current reminder that should be in today corpus." }], - }, - }), - ].join("\n") + "\n", - "utf-8", - ); - const freshMtime = new Date("2026-04-06T01:05:00.000Z"); - await fs.utimes(transcriptPath, freshMtime, freshMtime); + await seedDreamingSessionTranscript({ + sessionId: "dreaming-main", + messages: [ + { + role: "user", + timestamp: "2026-04-01T12:00:00.000Z", + content: [{ type: "text", text: "Old planning note that should stay out of lookback." }], + }, + { + role: "assistant", + timestamp: "2026-04-05T18:02:00.000Z", + content: [{ type: "text", text: "Current reminder that should be in today corpus." }], + }, + ], + }); const { beforeAgentReply } = createHarness( { @@ -2249,25 +2129,14 @@ describe("memory-core dreaming phases", () => { it("drains >80 unseen transcript messages across multiple unchanged sweeps", async () => { const workspaceDir = await createDreamingWorkspace(); setDreamingTestEnv(path.join(workspaceDir, ".state")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "dreaming-main.jsonl"); - const lines: string[] = []; - for (let index = 0; index < 160; index += 1) { - lines.push( - JSON.stringify({ - type: "message", - message: { - role: index % 2 === 0 ? "user" : "assistant", - timestamp: "2026-04-05T18:00:00.000Z", - content: [{ type: "text", text: `bulk-line-${index}` }], - }, - }), - ); - } - await fs.writeFile(transcriptPath, `${lines.join("\n")}\n`, "utf-8"); - const mtime = new Date("2026-04-05T18:05:00.000Z"); - await fs.utimes(transcriptPath, mtime, mtime); + await seedDreamingSessionTranscript({ + sessionId: "dreaming-main", + messages: Array.from({ length: 160 }, (_, index) => ({ + role: index % 2 === 0 ? ("user" as const) : ("assistant" as const), + timestamp: "2026-04-05T18:00:00.000Z", + content: [{ type: "text", text: `bulk-line-${index}` }], + })), + }); const { beforeAgentReply } = createHarness( { @@ -2325,29 +2194,19 @@ describe("memory-core dreaming phases", () => { expect(corpus).toContain("bulk-line-159"); }); - it("re-ingests rewritten session transcripts after truncate/reset", async () => { + it("ingests appended SQLite session transcript rows after prior checkpoint", async () => { const workspaceDir = await createDreamingWorkspace(); setDreamingTestEnv(path.join(workspaceDir, ".state")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "dreaming-main.jsonl"); - - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-05T18:01:00.000Z", - content: [{ type: "text", text: "Move backups to S3 Glacier." }], - }, - }), - ].join("\n") + "\n", - "utf-8", - ); - const dayOne = new Date("2026-04-05T18:05:00.000Z"); - await fs.utimes(transcriptPath, dayOne, dayOne); + await seedDreamingSessionTranscript({ + sessionId: "dreaming-main", + messages: [ + { + role: "user", + timestamp: "2026-04-05T18:01:00.000Z", + content: [{ type: "text", text: "Move backups to S3 Glacier." }], + }, + ], + }); const { beforeAgentReply } = createHarness( { @@ -2383,22 +2242,16 @@ describe("memory-core dreaming phases", () => { await triggerLightDreaming(beforeAgentReply, workspaceDir, 5); }); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-06T01:02:00.000Z", - content: [{ type: "text", text: "Retention policy stays at 365 days." }], - }, - }), - ].join("\n") + "\n", - "utf-8", - ); - const dayTwo = new Date("2026-04-06T01:05:00.000Z"); - await fs.utimes(transcriptPath, dayTwo, dayTwo); + await seedDreamingSessionTranscript({ + sessionId: "dreaming-main", + messages: [ + { + role: "assistant", + timestamp: "2026-04-06T01:02:00.000Z", + content: [{ type: "text", text: "Retention policy stays at 365 days." }], + }, + ], + }); await withDreamingTestClock(async () => { await triggerLightDreaming(beforeAgentReply, workspaceDir, 910); @@ -2422,25 +2275,16 @@ describe("memory-core dreaming phases", () => { it("ingests sessions when dreaming is enabled even if memorySearch is disabled", async () => { const workspaceDir = await createDreamingWorkspace(); setDreamingTestEnv(path.join(workspaceDir, ".state")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "dreaming-main.jsonl"); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-05T18:01:00.000Z", - content: [{ type: "text", text: "Glacier archive migration is now complete." }], - }, - }), - ].join("\n") + "\n", - "utf-8", - ); - const mtime = new Date("2026-04-05T18:05:00.000Z"); - await fs.utimes(transcriptPath, mtime, mtime); + await seedDreamingSessionTranscript({ + sessionId: "dreaming-main", + messages: [ + { + role: "user", + timestamp: "2026-04-05T18:01:00.000Z", + content: [{ type: "text", text: "Glacier archive migration is now complete." }], + }, + ], + }); const { beforeAgentReply } = createHarness( { diff --git a/extensions/memory-core/src/dreaming-phases.ts b/extensions/memory-core/src/dreaming-phases.ts index 3159287f6a85..79385561a80b 100644 --- a/extensions/memory-core/src/dreaming-phases.ts +++ b/extensions/memory-core/src/dreaming-phases.ts @@ -686,10 +686,16 @@ function hashSessionMessageId(value: string): string { return createHash("sha1").update(value).digest("hex"); } -function buildSessionScopeKey(agentId: string, absolutePath: string): string { +function buildSessionScopeKey(agentId: string, sessionId: string): string { + const logicalSessionId = + parseUsageCountedSessionIdFromFileName(`${sessionId}.jsonl`) ?? sessionId; + return `${agentId}:${logicalSessionId}`; +} + +function buildSessionFileScopeKey(agentId: string, absolutePath: string): string { const fileName = path.basename(absolutePath); const logicalSessionId = parseUsageCountedSessionIdFromFileName(fileName) ?? fileName; - return `${agentId}:${logicalSessionId}`; + return buildSessionScopeKey(agentId, logicalSessionId); } function mergeTrackedMessageHashes(existing: string[], additions: string[]): string[] { @@ -722,8 +728,12 @@ function areStringArraysEqual(a: string[], b: string[]): boolean { return true; } -function buildSessionStateKey(agentId: string, absolutePath: string): string { - return `${agentId}:${sessionPathForFile(absolutePath)}`; +function buildSessionStateKey(agentId: string, sessionPath: string): string { + return `${agentId}:${sessionPath}`; +} + +function buildSqliteDreamingSessionPath(agentId: string, sessionId: string): string { + return path.join("sessions", agentId, sessionId).replace(/\\/g, "/"); } function isCheckpointSessionTranscriptPath(absolutePath: string): boolean { @@ -852,7 +862,10 @@ async function collectSessionIngestionBatches(params: { absolutePath: string; generatedByDreamingNarrative: boolean; generatedByCronRun: boolean; + sessionId: string; sessionPath: string; + transcriptSource?: "sqlite"; + updatedAtMs?: number; }> = []; for (const agentId of agentIds) { for (const entry of await listSessionTranscriptCorpusEntriesForAgent(agentId)) { @@ -870,7 +883,13 @@ async function collectSessionIngestionBatches(params: { absolutePath, generatedByDreamingNarrative: entry.generatedByDreamingNarrative === true, generatedByCronRun: entry.generatedByCronRun === true, - sessionPath: sessionPathForFile(absolutePath), + sessionId: entry.sessionId, + sessionPath: + entry.transcriptSource === "sqlite" + ? buildSqliteDreamingSessionPath(entry.agentId, entry.sessionId) + : sessionPathForFile(absolutePath), + ...(entry.transcriptSource === "sqlite" ? { transcriptSource: "sqlite" as const } : {}), + ...(entry.updatedAtMs !== undefined ? { updatedAtMs: entry.updatedAtMs } : {}), }); } } @@ -896,42 +915,62 @@ async function collectSessionIngestionBatches(params: { if (remaining <= 0) { break; } - const stateKey = buildSessionStateKey(file.agentId, file.absolutePath); + const stateKey = buildSessionStateKey(file.agentId, file.sessionPath); const previous = params.state.files[stateKey]; - const stat = await fs.stat(file.absolutePath).catch((err: unknown) => { - if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { - return null; + let fingerprint: { mtimeMs: number; size: number }; + let entry: Awaited>; + if (file.transcriptSource === "sqlite") { + entry = await buildSessionEntry(file.absolutePath, { + generatedByDreamingNarrative: file.generatedByDreamingNarrative, + generatedByCronRun: file.generatedByCronRun, + ...(file.updatedAtMs !== undefined ? { updatedAtMs: file.updatedAtMs } : {}), + }); + if (!entry) { + if (previous) { + changed = true; + } + continue; } - throw err; - }); - if (!stat) { - if (previous) { - changed = true; + fingerprint = { + mtimeMs: Math.floor(Math.max(0, entry.mtimeMs)), + size: Math.floor(Math.max(0, entry.size)), + }; + } else { + const stat = await fs.stat(file.absolutePath).catch((err: unknown) => { + if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { + return null; + } + throw err; + }); + if (!stat) { + if (previous) { + changed = true; + } + continue; + } + fingerprint = { + mtimeMs: Math.floor(Math.max(0, stat.mtimeMs)), + size: Math.floor(Math.max(0, stat.size)), + }; + const cursorAtEnd = previous !== undefined && previous.lastContentLine >= previous.lineCount; + const unchanged = + Boolean(previous) && + previous.mtimeMs === fingerprint.mtimeMs && + previous.size === fingerprint.size && + previous.contentHash.length > 0 && + cursorAtEnd; + if (unchanged) { + nextFiles[stateKey] = previous!; + continue; } - continue; - } - const fingerprint = { - mtimeMs: Math.floor(Math.max(0, stat.mtimeMs)), - size: Math.floor(Math.max(0, stat.size)), - }; - const cursorAtEnd = previous !== undefined && previous.lastContentLine >= previous.lineCount; - const unchanged = - Boolean(previous) && - previous.mtimeMs === fingerprint.mtimeMs && - previous.size === fingerprint.size && - previous.contentHash.length > 0 && - cursorAtEnd; - if (unchanged) { - nextFiles[stateKey] = previous!; - continue; - } - const entry = await buildSessionEntry(file.absolutePath, { - generatedByDreamingNarrative: file.generatedByDreamingNarrative, - generatedByCronRun: file.generatedByCronRun, - }); - if (!entry) { - continue; + entry = await buildSessionEntry(file.absolutePath, { + generatedByDreamingNarrative: file.generatedByDreamingNarrative, + generatedByCronRun: file.generatedByCronRun, + }); + if (!entry) { + continue; + } } if (entry.generatedByDreamingNarrative || entry.generatedByCronRun) { nextFiles[stateKey] = { @@ -966,9 +1005,19 @@ async function collectSessionIngestionBatches(params: { continue; } - const sessionScope = buildSessionScopeKey(file.agentId, file.absolutePath); + const sessionScope = + file.transcriptSource === "sqlite" + ? `${file.agentId}:${file.sessionPath}` + : buildSessionFileScopeKey(file.agentId, file.absolutePath); + const preFlipSessionScope = + file.transcriptSource === "sqlite" + ? buildSessionScopeKey(file.agentId, file.sessionId) + : undefined; const previousSeen = nextSeenMessages[sessionScope] ?? []; const seenSet = new Set(previousSeen); + const preFlipSeenSet = preFlipSessionScope + ? new Set(nextSeenMessages[preFlipSessionScope] ?? []) + : null; const newSeenHashes: string[] = []; const lines = entry.content.length > 0 ? entry.content.split("\n") : []; @@ -1007,7 +1056,13 @@ async function collectSessionIngestionBatches(params: { const dedupeBasis = messageTimestampMs > 0 ? `ts:${Math.floor(messageTimestampMs)}` : `line:${lineNumber}`; const messageHash = hashSessionMessageId(`${sessionScope}\n${dedupeBasis}\n${snippet}`); - if (seenSet.has(messageHash)) { + const preFlipMessageHash = preFlipSessionScope + ? hashSessionMessageId(`${preFlipSessionScope}\n${dedupeBasis}\n${snippet}`) + : undefined; + if ( + seenSet.has(messageHash) || + (preFlipMessageHash !== undefined && preFlipSeenSet?.has(preFlipMessageHash)) + ) { continue; } const rendered = buildSessionRenderedLine({ diff --git a/extensions/memory-core/src/memory/index.test.ts b/extensions/memory-core/src/memory/index.test.ts index 2c6ce0353727..72038b9c0e7e 100644 --- a/extensions/memory-core/src/memory/index.test.ts +++ b/extensions/memory-core/src/memory/index.test.ts @@ -7,6 +7,8 @@ import type { DatabaseSync } from "node:sqlite"; import { clearMemoryEmbeddingProviders as clearRegistry } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; import { hashText } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; import { resolveSessionTranscriptsDirForAgent } from "openclaw/plugin-sdk/memory-core-host-runtime-core"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; import { resolveOpenClawAgentSqlitePath } from "openclaw/plugin-sdk/sqlite-runtime"; import { closeOpenClawAgentDatabasesForTest, @@ -417,6 +419,46 @@ describe("memory index", () => { }; } + async function seedMemoryIndexSessionTranscript(params: { + messages: Array<{ + content: string; + role: "assistant" | "user"; + timestamp: number | string; + }>; + sessionId: string; + sessionKey?: string; + }): Promise { + const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); + const storePath = path.join(sessionsDir, "sessions.json"); + const sessionKey = params.sessionKey ?? `agent:main:memory:${params.sessionId}`; + const timestamps = params.messages + .map((message) => + typeof message.timestamp === "number" ? message.timestamp : Date.parse(message.timestamp), + ) + .filter((timestamp) => Number.isFinite(timestamp)); + const updatedAt = timestamps.length > 0 ? Math.max(...timestamps) : Date.now(); + await fs.mkdir(sessionsDir, { recursive: true }); + await upsertSessionEntry({ + agentId: "main", + sessionKey, + storePath, + entry: { sessionId: params.sessionId, updatedAt }, + }); + for (const message of params.messages) { + await appendSessionTranscriptMessageByIdentity({ + agentId: "main", + sessionId: params.sessionId, + sessionKey, + storePath, + message: { + role: message.role, + timestamp: message.timestamp, + content: [{ type: "text", text: message.content }], + }, + }); + } + } + function requireManager( result: Awaited>, missingMessage = "manager missing", @@ -881,46 +923,26 @@ describe("memory index", () => { it("batches forced memory and session indexing across files", async () => { await fs.writeFile(path.join(memoryDir, "2026-01-13.md"), "# Log\nBeta memory line."); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - await fs.writeFile( - path.join(sessionsDir, "session-alpha.jsonl"), - [ - JSON.stringify({ - type: "session", - id: "session-alpha", - timestamp: "2026-04-07T15:24:04.113Z", - }), - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: "2026-04-07T15:25:04.113Z", - content: [{ type: "text", text: "Session alpha memory line." }], - }, - }), - ].join("\n") + "\n", - "utf8", - ); - await fs.writeFile( - path.join(sessionsDir, "session-beta.jsonl"), - [ - JSON.stringify({ - type: "session", - id: "session-beta", - timestamp: "2026-04-07T15:24:04.113Z", - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-07T15:25:04.113Z", - content: [{ type: "text", text: "Session beta memory line." }], - }, - }), - ].join("\n") + "\n", - "utf8", - ); + await seedMemoryIndexSessionTranscript({ + sessionId: "session-alpha", + messages: [ + { + role: "user", + timestamp: "2026-04-07T15:25:04.113Z", + content: "Session alpha memory line.", + }, + ], + }); + await seedMemoryIndexSessionTranscript({ + sessionId: "session-beta", + messages: [ + { + role: "assistant", + timestamp: "2026-04-07T15:25:04.113Z", + content: "Session beta memory line.", + }, + ], + }); const cfg = createCfg({ provider: "batch-wide-test", batchEnabled: true, @@ -1216,27 +1238,16 @@ describe("memory index", () => { it("clears dirty after sessions-only identity reindex", async () => { try { setMemoryIndexStateDir(path.join(workspaceDir, ".state-sessions-only-reindex")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - await fs.writeFile( - path.join(sessionsDir, "session-identity.jsonl"), - [ - JSON.stringify({ - type: "session", - id: "session-identity", - timestamp: "2026-04-07T15:24:04.113Z", - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-07T15:25:04.113Z", - content: [{ type: "text", text: "Session-only identity marker." }], - }, - }), - ].join("\n") + "\n", - "utf8", - ); + await seedMemoryIndexSessionTranscript({ + sessionId: "session-identity", + messages: [ + { + role: "assistant", + timestamp: "2026-04-07T15:25:04.113Z", + content: "Session-only identity marker.", + }, + ], + }); const oldCfg = createCfg({ sources: ["sessions"], @@ -1272,27 +1283,16 @@ describe("memory index", () => { it("marks sessions-only indexes dirty when metadata is missing but chunks exist", async () => { try { setMemoryIndexStateDir(path.join(workspaceDir, ".state-sessions-missing-meta")); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - await fs.writeFile( - path.join(sessionsDir, "session-missing-meta.jsonl"), - [ - JSON.stringify({ - type: "session", - id: "session-missing-meta", - timestamp: "2026-04-07T15:24:04.113Z", - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-07T15:25:04.113Z", - content: [{ type: "text", text: "Sessions missing metadata marker." }], - }, - }), - ].join("\n") + "\n", - "utf8", - ); + await seedMemoryIndexSessionTranscript({ + sessionId: "session-missing-meta", + messages: [ + { + role: "assistant", + timestamp: "2026-04-07T15:25:04.113Z", + content: "Sessions missing metadata marker.", + }, + ], + }); const cfg = createCfg({ sources: ["sessions"], @@ -1371,7 +1371,7 @@ describe("memory index", () => { expect(nextManager.status().dirty).toBe(true); embedBatchCalls = 0; - await nextManager.sync({ reason: "test", sessionFiles: [sessionFile] }); + await nextManager.sync({ reason: "test", archiveFiles: [sessionFile] }); expect(embedBatchCalls).toBe(0); expect(nextManager.status().dirty).toBe(true); @@ -1433,12 +1433,12 @@ describe("memory index", () => { try { const fields = nextManager as unknown as { dirty: boolean; - syncSessionFiles: (params: unknown) => Promise; + syncArchiveFiles: (params: unknown) => Promise; }; - const syncSessionFiles = fields.syncSessionFiles.bind(nextManager); - fields.syncSessionFiles = async (params) => { + const syncArchiveFiles = fields.syncArchiveFiles.bind(nextManager); + fields.syncArchiveFiles = async (params) => { fields.dirty = true; - await syncSessionFiles(params); + await syncArchiveFiles(params); }; await nextManager.sync({ reason: "test", force: true }); @@ -1514,7 +1514,7 @@ describe("memory index", () => { runSyncWithReadonlyRecovery: (params?: { reason?: string; force?: boolean; - sessionFiles?: string[]; + archiveFiles?: string[]; progress?: (update: unknown) => void; }) => Promise; } @@ -2864,37 +2864,22 @@ describe("memory index", () => { const staleAt = new Date("2020-01-01T00:00:00.000Z"); await fs.utimes(memoryPath, staleAt, staleAt); - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "session-ranking.jsonl"); const now = Date.parse("2026-04-07T15:25:04.113Z"); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "session", - id: "session-ranking", - timestamp: new Date(now - 60_000).toISOString(), - }), - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: new Date(now - 30_000).toISOString(), - content: [{ type: "text", text: "What is the current Project Nebula codename?" }], - }, - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: new Date(now).toISOString(), - content: [{ type: "text", text: "The current Project Nebula codename is ORBIT-10." }], - }, - }), - ].join("\n") + "\n", - "utf8", - ); + await seedMemoryIndexSessionTranscript({ + sessionId: "session-ranking", + messages: [ + { + role: "user", + timestamp: new Date(now - 30_000).toISOString(), + content: "What is the current Project Nebula codename?", + }, + { + role: "assistant", + timestamp: new Date(now).toISOString(), + content: "The current Project Nebula codename is ORBIT-10.", + }, + ], + }); await manager.sync({ reason: "test", force: true }); const results = await manager.search("current Project Nebula codename ORBIT-10", { @@ -2918,28 +2903,16 @@ describe("memory index", () => { return; } - const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - const transcriptPath = path.join(sessionsDir, "session-bootstrap.jsonl"); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ - type: "session", - id: "session-bootstrap", - timestamp: "2026-04-07T15:24:04.113Z", - }), - JSON.stringify({ - type: "message", - message: { - role: "assistant", - timestamp: "2026-04-07T15:25:04.113Z", - content: [{ type: "text", text: "The current Project Nebula codename is ORBIT-10." }], - }, - }), - ].join("\n") + "\n", - "utf8", - ); + await seedMemoryIndexSessionTranscript({ + sessionId: "session-bootstrap", + messages: [ + { + role: "assistant", + timestamp: "2026-04-07T15:25:04.113Z", + content: "The current Project Nebula codename is ORBIT-10.", + }, + ], + }); const results = await manager.search("current Project Nebula codename ORBIT-10", { minScore: 0, diff --git a/extensions/memory-core/src/memory/manager-session-reindex.ts b/extensions/memory-core/src/memory/manager-session-reindex.ts index 37a96db772d1..0cd6254fe6ff 100644 --- a/extensions/memory-core/src/memory/manager-session-reindex.ts +++ b/extensions/memory-core/src/memory/manager-session-reindex.ts @@ -15,7 +15,7 @@ export function shouldSyncSessionsForReindex(params: { if (params.sync?.sessions?.some((session) => session.sessionId.trim().length > 0)) { return true; } - if (params.sync?.sessionFiles?.some((sessionFile) => sessionFile.trim().length > 0)) { + if (params.sync?.archiveFiles?.some((sessionFile) => sessionFile.trim().length > 0)) { return true; } if (params.sync?.force) { diff --git a/extensions/memory-core/src/memory/manager-sync-control.ts b/extensions/memory-core/src/memory/manager-sync-control.ts index 86caee610175..db40e90f2cc8 100644 --- a/extensions/memory-core/src/memory/manager-sync-control.ts +++ b/extensions/memory-core/src/memory/manager-sync-control.ts @@ -24,7 +24,7 @@ export type MemoryReadonlyRecoveryState = { reason?: string; force?: boolean; sessions?: MemorySessionSyncTarget[]; - sessionFiles?: string[]; + archiveFiles?: string[]; progress?: (update: MemorySyncProgressUpdate) => void; }) => Promise; openDatabase: () => DatabaseSync; @@ -120,19 +120,19 @@ export function enqueueMemoryTargetedSessionSync( state: { isClosed: () => boolean; getSyncing: () => Promise | null; - getQueuedSessionFiles: () => Set; + getQueuedArchiveFiles: () => Set; getQueuedSessions: () => Map; getQueuedSessionSync: () => Promise | null; setQueuedSessionSync: (value: Promise | null) => void; sync: (params?: MemorySyncParams) => Promise; }, - targets?: Pick, + targets?: Pick, ): Promise { - const queuedSessionFiles = state.getQueuedSessionFiles(); - for (const sessionFile of targets?.sessionFiles ?? []) { + const queuedArchiveFiles = state.getQueuedArchiveFiles(); + for (const sessionFile of targets?.archiveFiles ?? []) { const trimmed = sessionFile.trim(); if (trimmed) { - queuedSessionFiles.add(trimmed); + queuedArchiveFiles.add(trimmed); } } const queuedSessions = state.getQueuedSessions(); @@ -142,7 +142,7 @@ export function enqueueMemoryTargetedSessionSync( queuedSessions.set(memorySessionSyncTargetKey(normalized), normalized); } } - if (queuedSessionFiles.size === 0 && queuedSessions.size === 0) { + if (queuedArchiveFiles.size === 0 && queuedSessions.size === 0) { return state.getSyncing() ?? Promise.resolve(); } if (!state.getQueuedSessionSync()) { @@ -152,16 +152,16 @@ export function enqueueMemoryTargetedSessionSync( await state.getSyncing()?.catch(() => undefined); while ( !state.isClosed() && - (state.getQueuedSessionFiles().size > 0 || state.getQueuedSessions().size > 0) + (state.getQueuedArchiveFiles().size > 0 || state.getQueuedSessions().size > 0) ) { - const pendingSessionFiles = Array.from(state.getQueuedSessionFiles()); + const pendingArchiveFiles = Array.from(state.getQueuedArchiveFiles()); const pendingSessions = Array.from(state.getQueuedSessions().values()); - state.getQueuedSessionFiles().clear(); + state.getQueuedArchiveFiles().clear(); state.getQueuedSessions().clear(); await state.sync({ reason: "queued-sessions", sessions: pendingSessions, - sessionFiles: pendingSessionFiles, + archiveFiles: pendingArchiveFiles, }); } } finally { diff --git a/extensions/memory-core/src/memory/manager-sync-ops.startup-catchup.test.ts b/extensions/memory-core/src/memory/manager-sync-ops.startup-catchup.test.ts index e5e7c3c61bfd..cfa0627703a3 100644 --- a/extensions/memory-core/src/memory/manager-sync-ops.startup-catchup.test.ts +++ b/extensions/memory-core/src/memory/manager-sync-ops.startup-catchup.test.ts @@ -3,12 +3,12 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; -import { emitSessionTranscriptUpdate } from "openclaw/plugin-sdk/agent-harness-runtime"; import { resolveSessionTranscriptsDirForAgent, type OpenClawConfig, type ResolvedMemorySearchConfig, } from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; +import { statSessionEntrySync } from "openclaw/plugin-sdk/memory-core-host-engine-qmd"; import type { MemorySource, MemorySyncParams, @@ -18,6 +18,12 @@ import { clearConfigCache, clearRuntimeConfigSnapshot, } from "openclaw/plugin-sdk/runtime-config-snapshot"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; +import { + closeOpenClawAgentDatabasesForTest, + formatSqliteSessionFileMarker, +} from "openclaw/plugin-sdk/sqlite-runtime-testing"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { MemoryManagerSyncOps } from "./manager-sync-ops.js"; @@ -34,7 +40,7 @@ type SyncParams = { reason?: string; force?: boolean; sessions?: MemorySyncParams["sessions"]; - sessionFiles?: string[]; + archiveFiles?: string[]; progress?: (update: MemorySyncProgressUpdate) => void; }; @@ -49,15 +55,9 @@ type MemorySessionTranscriptUpdate = { }; }; -type MemoryTranscriptUpdateSubscriber = ( - listener: (update: MemorySessionTranscriptUpdate) => void, -) => () => void; - -const MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY = Symbol.for( - "openclaw.memoryCore.sessionTranscriptUpdateSubscriber", -); const originalStartupStateDir = process.env.OPENCLAW_STATE_DIR; const originalStartupConfigPath = process.env.OPENCLAW_CONFIG_PATH; +let transcriptUpdateListener: ((update: MemorySessionTranscriptUpdate) => void) | undefined; type SourceStateRow = { path: string; hash: string; mtime: number; size: number }; @@ -82,6 +82,10 @@ function restoreStartupEnv(): void { } } +function emitSessionTranscriptUpdate(update: MemorySessionTranscriptUpdate): void { + transcriptUpdateListener?.(update); +} + class SessionStartupCatchupHarness extends MemoryManagerSyncOps { protected readonly cfg = {} as OpenClawConfig; protected readonly agentId = "main"; @@ -155,7 +159,7 @@ class SessionStartupCatchupHarness extends MemoryManagerSyncOps { await this.runSync(params); } - getDirtySessionFiles(): string[] { + getDirtyArchiveFiles(): string[] { return Array.from(this.sessionsDirtyFiles); } @@ -163,7 +167,7 @@ class SessionStartupCatchupHarness extends MemoryManagerSyncOps { return Array.from(this.sessionPendingTargets.values()); } - getPendingSessionFiles(): string[] { + getPendingArchiveFiles(): string[] { return Array.from(this.sessionPendingFiles); } @@ -182,18 +186,18 @@ class SessionStartupCatchupHarness extends MemoryManagerSyncOps { ).processSessionDeltaBatch(); } - async combineTargetSessionFilesForTest(params: { + async combineTargetArchiveFilesForTest(params: { sessions?: MemorySyncParams["sessions"]; - sessionFiles?: string[]; + archiveFiles?: string[]; }): Promise | null> { return await ( this as unknown as { - combineTargetSessionFiles: (params: { + combineTargetArchiveFiles: (params: { sessions?: MemorySyncParams["sessions"]; - sessionFiles?: string[]; + archiveFiles?: string[]; }) => Promise | null>; } - ).combineTargetSessionFiles(params); + ).combineTargetArchiveFiles(params); } isSessionsDirty(): boolean { @@ -209,6 +213,17 @@ class SessionStartupCatchupHarness extends MemoryManagerSyncOps { this.sessionUnsubscribe = null; } + protected override subscribeSessionTranscriptUpdates( + listener: (update: MemorySessionTranscriptUpdate) => void, + ): () => void { + transcriptUpdateListener = listener; + return () => { + if (transcriptUpdateListener === listener) { + transcriptUpdateListener = undefined; + } + }; + } + protected computeProviderKey(): string { return "test"; } @@ -254,14 +269,17 @@ describe("session startup catch-up", () => { beforeEach(async () => { stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-session-startup-")); setStartupStateDir(stateDir); + transcriptUpdateListener = undefined; }); afterEach(async () => { vi.clearAllTimers(); vi.useRealTimers(); + transcriptUpdateListener = undefined; restoreStartupEnv(); clearRuntimeConfigSnapshot(); clearConfigCache(); + closeOpenClawAgentDatabasesForTest(); await fs.rm(stateDir, { recursive: true, force: true }); }); @@ -281,25 +299,90 @@ describe("session startup catch-up", () => { return { filePath, size: stat.size, mtimeMs: stat.mtimeMs }; } + async function configureTestSessionStore(storePath: string): Promise { + const configPath = path.join(stateDir, "openclaw.json"); + await fs.mkdir(path.dirname(storePath), { recursive: true }); + await fs.writeFile(configPath, JSON.stringify({ session: { store: storePath } }), "utf-8"); + setStartupConfigPath(configPath); + clearRuntimeConfigSnapshot(); + clearConfigCache(); + } + + async function writeSqliteSession( + params: { + storePath?: string; + sessionId?: string; + sessionKey?: string; + content?: string; + role?: "assistant" | "user"; + updatedAt?: number; + } = {}, + ): Promise<{ + marker: string; + storePath: string; + sessionId: string; + sessionKey: string; + corpusPath: string; + }> { + const storePath = + params.storePath ?? path.join(stateDir, "agents", "main", "sessions", "sessions.json"); + const sessionId = params.sessionId ?? "thread"; + const sessionKey = params.sessionKey ?? `agent:main:chat:${sessionId}`; + const marker = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + await configureTestSessionStore(storePath); + await upsertSessionEntry({ + agentId: "main", + sessionKey, + storePath, + entry: { + sessionFile: marker, + sessionId, + updatedAt: params.updatedAt ?? 10, + }, + }); + await appendSessionTranscriptMessageByIdentity({ + agentId: "main", + sessionId, + sessionKey, + storePath, + cwd: stateDir, + message: { + role: params.role ?? "user", + content: params.content ?? "startup catchup", + }, + }); + return { + marker, + storePath, + sessionId, + sessionKey, + corpusPath: `sessions/main/${sessionId}.jsonl`, + }; + } + it("marks stale indexed session files dirty and schedules catch-up sync", async () => { - const session = await writeSessionFile("thread.jsonl"); + const session = await writeSqliteSession(); const harness = new SessionStartupCatchupHarness([ { - path: "sessions/main/thread.jsonl", + path: session.corpusPath, hash: "old-hash", - mtime: session.mtimeMs - 1000, - size: session.size, + mtime: 0, + size: 0, }, ]); - await expect(harness.catchUp()).resolves.toEqual([session.filePath]); - expect(harness.getDirtySessionFiles()).toEqual([session.filePath]); + await expect(harness.catchUp()).resolves.toEqual([session.marker]); + expect(harness.getDirtyArchiveFiles()).toEqual([session.marker]); expect(harness.isSessionsDirty()).toBe(true); expect(harness.syncCalls).toEqual([{ reason: "session-startup-catchup" }]); }); it("retries transient session transcript reads during session indexing", async () => { - const session = await writeSessionFile("thread.jsonl"); + const session = await writeSessionFile("thread.jsonl.deleted.2026-02-16T22-27-33.000Z"); const harness = new SessionStartupCatchupHarness([]); const realOpen = fs.open; @@ -324,7 +407,7 @@ describe("session startup catch-up", () => { }); try { - await (harness as any).syncSessionFiles({ needsFullReindex: true }); + await (harness as any).syncArchiveFiles({ needsFullReindex: true }); expect(attempts).toBe(2); } finally { openSpy.mockRestore(); @@ -332,22 +415,46 @@ describe("session startup catch-up", () => { }); it("can mark startup catch-up files without scheduling background sync", async () => { - const session = await writeSessionFile("thread.jsonl"); + const session = await writeSqliteSession(); const harness = new SessionStartupCatchupHarness([ { - path: "sessions/main/thread.jsonl", + path: session.corpusPath, hash: "old-hash", - mtime: session.mtimeMs - 1000, - size: session.size, + mtime: 0, + size: 0, }, ]); - await expect(harness.markStartupDirtyFiles()).resolves.toEqual([session.filePath]); - expect(harness.getDirtySessionFiles()).toEqual([session.filePath]); + await expect(harness.markStartupDirtyFiles()).resolves.toEqual([session.marker]); + expect(harness.getDirtyArchiveFiles()).toEqual([session.marker]); expect(harness.isSessionsDirty()).toBe(true); expect(harness.syncCalls).toEqual([]); }); + it("leaves unchanged indexed SQLite sessions clean during startup catch-up", async () => { + const session = await writeSqliteSession({ updatedAt: 10 }); + const state = statSessionEntrySync(session.marker, { + sessionKey: session.sessionKey, + updatedAtMs: 10, + }); + if (!state) { + throw new Error("expected SQLite transcript state"); + } + const harness = new SessionStartupCatchupHarness([ + { + path: state.path, + hash: "current-hash", + mtime: state.mtimeMs, + size: state.size, + }, + ]); + + await expect(harness.markStartupDirtyFiles()).resolves.toEqual([]); + expect(harness.getDirtyArchiveFiles()).toEqual([]); + expect(harness.isSessionsDirty()).toBe(false); + expect(harness.syncCalls).toEqual([]); + }); + it("leaves unchanged indexed session files clean", async () => { const session = await writeSessionFile("thread.jsonl"); const harness = new SessionStartupCatchupHarness([ @@ -360,7 +467,7 @@ describe("session startup catch-up", () => { ]); await expect(harness.catchUp()).resolves.toEqual([]); - expect(harness.getDirtySessionFiles()).toEqual([]); + expect(harness.getDirtyArchiveFiles()).toEqual([]); expect(harness.isSessionsDirty()).toBe(false); expect(harness.syncCalls).toEqual([]); }); @@ -457,33 +564,13 @@ describe("session startup catch-up", () => { }); it("resolves identity-targeted delta sync through a custom session store", async () => { - const storeDir = path.join(stateDir, "custom-sessions"); - const sessionFile = path.join(storeDir, "custom-thread.jsonl"); - const storePath = path.join(storeDir, "sessions.json"); - const configPath = path.join(stateDir, "openclaw.json"); - await fs.mkdir(storeDir, { recursive: true }); - await fs.writeFile( - sessionFile, - JSON.stringify({ - type: "message", - message: { role: "user", content: "custom store target" }, - }) + "\n", - "utf-8", - ); - await fs.writeFile( + const storePath = path.join(stateDir, "custom-sessions", "sessions.json"); + const session = await writeSqliteSession({ storePath, - JSON.stringify({ - "agent:main:chat:custom": { - sessionFile: "custom-thread.jsonl", - sessionId: "custom-thread", - }, - }), - "utf-8", - ); - await fs.writeFile(configPath, JSON.stringify({ session: { store: storePath } }), "utf-8"); - setStartupConfigPath(configPath); - clearRuntimeConfigSnapshot(); - clearConfigCache(); + sessionId: "custom-thread", + sessionKey: "agent:main:chat:custom", + content: "custom store target", + }); const harness = new SessionStartupCatchupHarness([]); (harness as unknown as { settings: ResolvedMemorySearchConfig }).settings.sync.sessions = { deltaBytes: 1, @@ -499,125 +586,100 @@ describe("session startup catch-up", () => { await harness.processPendingSessionDeltas(); await Promise.resolve(); - expect(harness.getDirtySessionFiles()).toEqual([sessionFile]); + expect(harness.getDirtyArchiveFiles()).toEqual([session.marker]); expect(harness.syncCalls).toEqual([{ reason: "session-delta" }]); }); - it("keeps explicit custom-store session file targets at the sync gate", async () => { - const storeDir = path.join(stateDir, "custom-sessions"); - const sessionFile = path.join(storeDir, "explicit-target.jsonl"); - const storePath = path.join(storeDir, "sessions.json"); - const configPath = path.join(stateDir, "openclaw.json"); - await fs.mkdir(storeDir, { recursive: true }); - await fs.writeFile( - sessionFile, - JSON.stringify({ - type: "message", - message: { role: "user", content: "explicit target" }, - }) + "\n", - "utf-8", - ); - await fs.writeFile( + it("preserves generated-session classification during targeted custom-store indexing", async () => { + const storePath = path.join(stateDir, "custom-sessions", "sessions.json"); + const session = await writeSqliteSession({ storePath, - JSON.stringify({ - "agent:main:chat:explicit-target": { - sessionFile: "explicit-target.jsonl", - sessionId: "explicit-target", - }, - }), - "utf-8", - ); - await fs.writeFile(configPath, JSON.stringify({ session: { store: storePath } }), "utf-8"); - setStartupConfigPath(configPath); - clearRuntimeConfigSnapshot(); - clearConfigCache(); + sessionId: "cron-thread", + sessionKey: "agent:main:cron:job-1:run:run-1", + role: "assistant", + content: "Internal cron output that must stay out.", + }); + await writeSqliteSession({ + storePath, + sessionId: "other-thread", + sessionKey: "agent:main:chat:other", + content: "Other custom-store content", + }); const harness = new SessionStartupCatchupHarness([]); - await expect( - harness.combineTargetSessionFilesForTest({ sessionFiles: [sessionFile] }), - ).resolves.toEqual(new Set([sessionFile])); + await ( + harness as unknown as { + syncArchiveFiles: (params: { + needsFullReindex: boolean; + targetArchiveFiles: string[]; + }) => Promise; + } + ).syncArchiveFiles({ + needsFullReindex: false, + targetArchiveFiles: [session.marker], + }); + + expect(harness.indexedPaths).toEqual([session.corpusPath]); + expect(harness.indexedContents).toEqual([""]); }); - it("preserves generated-session classification during targeted custom-store indexing", async () => { - const storeDir = path.join(stateDir, "custom-sessions"); - const sessionFile = path.join(storeDir, "cron-thread.jsonl"); - const otherSessionFile = path.join(storeDir, "other-thread.jsonl"); - const storePath = path.join(storeDir, "sessions.json"); + it("keeps targeted SQLite corpus markers during archive-file sync", async () => { + const storePath = path.join(stateDir, "agents", "main", "sessions", "sessions.json"); const configPath = path.join(stateDir, "openclaw.json"); - await fs.mkdir(storeDir, { recursive: true }); - await fs.writeFile( - sessionFile, - JSON.stringify({ - type: "message", - message: { role: "assistant", content: "Internal cron output that must stay out." }, - }) + "\n", - "utf-8", - ); - await fs.writeFile( - otherSessionFile, - JSON.stringify({ - type: "message", - message: { role: "user", content: "Other custom-store content" }, - }) + "\n", - "utf-8", - ); - await fs.writeFile( + const sessionId = "sqlite-target"; + const sessionKey = "agent:main:chat:sqlite-target"; + const marker = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, storePath, - JSON.stringify({ - "agent:main:cron:job-1:run:run-1": { - sessionFile: "cron-thread.jsonl", - sessionId: "cron-thread", - }, - "agent:main:chat:other": { - sessionFile: "other-thread.jsonl", - sessionId: "other-thread", - }, - }), - "utf-8", - ); + }); + await upsertSessionEntry({ + agentId: "main", + sessionKey, + storePath, + entry: { + sessionFile: marker, + sessionId, + updatedAt: 10, + }, + }); + await appendSessionTranscriptMessageByIdentity({ + agentId: "main", + sessionId, + sessionKey, + storePath, + cwd: stateDir, + message: { role: "user", content: "sqlite targeted memory content" }, + }); await fs.writeFile(configPath, JSON.stringify({ session: { store: storePath } }), "utf-8"); - setStartupConfigPath(configPath); + vi.stubEnv("OPENCLAW_CONFIG_PATH", configPath); clearRuntimeConfigSnapshot(); clearConfigCache(); const harness = new SessionStartupCatchupHarness([]); await ( harness as unknown as { - syncSessionFiles: (params: { + syncArchiveFiles: (params: { needsFullReindex: boolean; - targetSessionFiles: string[]; + targetArchiveFiles: string[]; }) => Promise; } - ).syncSessionFiles({ + ).syncArchiveFiles({ needsFullReindex: false, - targetSessionFiles: [sessionFile], + targetArchiveFiles: [marker], }); - expect(harness.indexedPaths).toEqual(["sessions/cron-thread.jsonl"]); - expect(harness.indexedContents).toEqual([""]); + expect(harness.indexedPaths).toEqual(["sessions/main/sqlite-target.jsonl"]); + expect(harness.indexedContents[0]).toContain("sqlite targeted memory content"); }); it("queues transcript update identity without requiring a session file", async () => { vi.useFakeTimers(); const harness = new SessionStartupCatchupHarness([]); - const originalSubscriber = (globalThis as Record)[ - MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY - ]; - let transcriptListener: ((update: MemorySessionTranscriptUpdate) => void) | undefined; - (globalThis as Record)[MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY] = (( - listener, - ) => { - transcriptListener = listener; - return () => { - if (transcriptListener === listener) { - transcriptListener = undefined; - } - }; - }) satisfies MemoryTranscriptUpdateSubscriber; harness.startTranscriptListener(); try { - transcriptListener?.({ + emitSessionTranscriptUpdate({ target: { agentId: "main", sessionId: "thread", @@ -630,14 +692,6 @@ describe("session startup catch-up", () => { ]); } finally { harness.stopTranscriptListener(); - if (originalSubscriber === undefined) { - delete (globalThis as Record)[ - MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY - ]; - } else { - (globalThis as Record)[MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY] = - originalSubscriber; - } } }); @@ -652,70 +706,7 @@ describe("session startup catch-up", () => { sessionKey: "agent:main:thread", }); - expect(harness.getPendingSessionFiles()).toEqual([session.filePath]); - expect(harness.getPendingSessionTargets()).toEqual([]); - harness.stopTranscriptListener(); - }); - - it("queues file-only transcript updates from a custom session store", async () => { - vi.useFakeTimers(); - const storeDir = path.join(stateDir, "custom-sessions"); - const sessionFile = path.join(storeDir, "custom-update.jsonl"); - const storePath = path.join(storeDir, "sessions.json"); - const configPath = path.join(stateDir, "openclaw.json"); - await fs.mkdir(storeDir, { recursive: true }); - await fs.writeFile( - sessionFile, - JSON.stringify({ - type: "message", - message: { role: "user", content: "custom update" }, - }) + "\n", - "utf-8", - ); - await fs.writeFile( - storePath, - JSON.stringify({ - "agent:main:chat:custom-update": { - sessionFile: "custom-update.jsonl", - sessionId: "custom-update", - }, - }), - "utf-8", - ); - await fs.writeFile(configPath, JSON.stringify({ session: { store: storePath } }), "utf-8"); - setStartupConfigPath(configPath); - clearRuntimeConfigSnapshot(); - clearConfigCache(); - const harness = new SessionStartupCatchupHarness([]); - harness.startTranscriptListener(); - - emitSessionTranscriptUpdate({ - sessionFile, - sessionKey: "agent:main:chat:custom-update", - }); - await Promise.resolve(); - - expect(harness.getPendingSessionFiles()).toEqual([sessionFile]); - expect(harness.getPendingSessionTargets()).toEqual([]); - harness.stopTranscriptListener(); - }); - - it("prefers transcript update path compatibility before identity", async () => { - vi.useFakeTimers(); - const session = await writeSessionFile("thread.jsonl"); - const harness = new SessionStartupCatchupHarness([]); - harness.startTranscriptListener(); - - emitSessionTranscriptUpdate({ - sessionFile: session.filePath, - target: { - agentId: "main", - sessionId: "identity-target", - sessionKey: "agent:main:identity-target", - }, - }); - - expect(harness.getPendingSessionFiles()).toEqual([session.filePath]); + expect(harness.getPendingArchiveFiles()).toEqual([session.filePath]); expect(harness.getPendingSessionTargets()).toEqual([]); harness.stopTranscriptListener(); }); diff --git a/extensions/memory-core/src/memory/manager-sync-ops.ts b/extensions/memory-core/src/memory/manager-sync-ops.ts index 1c4d797a7679..9a213a46af26 100644 --- a/extensions/memory-core/src/memory/manager-sync-ops.ts +++ b/extensions/memory-core/src/memory/manager-sync-ops.ts @@ -9,7 +9,7 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { classifyMemoryMultimodalPath } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; import { createSubsystemLogger, - onSessionTranscriptUpdate, + onInternalSessionTranscriptUpdate, resolveAgentDir, resolveSessionTranscriptsDirForAgent, resolveUserPath, @@ -20,11 +20,14 @@ import { buildSessionEntry, isSessionArchiveArtifactName, isUsageCountedSessionTranscriptFileName, - listSessionFilesForAgent, listSessionTranscriptCorpusEntriesForAgent, parseCanonicalSessionSyncTargetFromPath, + parseSqliteSessionFileMarker, + parseUsageCountedSessionIdFromFileName, resolveSessionFileForSyncTarget, sessionPathForFile, + sessionPathForSessionIdentity, + statSessionEntrySync, type SessionTranscriptCorpusEntry, } from "openclaw/plugin-sdk/memory-core-host-engine-qmd"; import { @@ -93,7 +96,7 @@ import { resolveMemorySourceExistingHash, } from "./manager-source-state.js"; import { - markMemoryTargetSessionFilesDirty, + markMemoryTargetArchiveFilesDirty, runMemoryTargetedSessionSync, } from "./manager-targeted-sync.js"; import { @@ -150,6 +153,25 @@ type MemoryReindexRetryState = { sessionDeltas: Map; }; +function sessionPathForCorpusEntry(entry: SessionTranscriptCorpusEntry): string { + return entry.transcriptSource === "sqlite" + ? sessionPathForSessionIdentity(entry.agentId, entry.sessionId) + : sessionPathForFile(entry.sessionFile); +} + +function legacyExtensionlessSessionPathForIdentity(agentId: string, sessionId: string): string { + return path.join("sessions", normalizeAgentId(agentId), sessionId).replace(/\\/g, "/"); +} + +function buildSessionEntryOptions(entry: SessionTranscriptCorpusEntry) { + return { + generatedByDreamingNarrative: entry.generatedByDreamingNarrative === true, + generatedByCronRun: entry.generatedByCronRun === true, + ...(entry.sessionKey ? { sessionKey: entry.sessionKey } : {}), + ...(entry.updatedAtMs !== undefined ? { updatedAtMs: entry.updatedAtMs } : {}), + }; +} + const META_KEY = "memory_index_meta_v1"; const VECTOR_TABLE = MEMORY_INDEX_VECTOR_TABLE; const LEGACY_VECTOR_TABLE = "chunks_vec"; @@ -172,9 +194,6 @@ const IGNORED_MEMORY_WATCH_DIR_NAMES = new Set([ ]); const log = createSubsystemLogger("memory"); -const MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY = Symbol.for( - "openclaw.memoryCore.sessionTranscriptUpdateSubscriber", -); const TEST_MEMORY_WATCH_FACTORY_KEY = Symbol.for("openclaw.test.memoryWatchFactory"); const TEST_MEMORY_NATIVE_WATCH_FACTORY_KEY = Symbol.for("openclaw.test.memoryNativeWatchFactory"); @@ -189,10 +208,6 @@ type MemorySessionTranscriptUpdate = { }; }; -type MemoryTranscriptUpdateSubscriber = ( - listener: (update: MemorySessionTranscriptUpdate) => void, -) => () => void; - function memoryTableExists(db: DatabaseSync, tableName: string): boolean { return Boolean( db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(tableName), @@ -211,18 +226,6 @@ type LinuxMemoryDirectoryWatcher = { ino: number; }; -function subscribeMemorySessionTranscriptUpdates( - listener: (update: MemorySessionTranscriptUpdate) => void, -): () => void { - const injected = (globalThis as Record)[ - MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY - ]; - if (typeof injected === "function") { - return (injected as MemoryTranscriptUpdateSubscriber)(listener); - } - return onSessionTranscriptUpdate(listener); -} - function resolveMemoryWatchFactory(): typeof chokidar.watch { if (process.env.VITEST === "true" || process.env.NODE_ENV === "test") { const override = (globalThis as Record)[TEST_MEMORY_WATCH_FACTORY_KEY]; @@ -494,7 +497,7 @@ export abstract class MemoryManagerSyncOps { shouldSyncSessions: boolean; needsFullReindex: boolean; needsFullSessionReindex?: boolean; - targetSessionFiles?: string[]; + targetArchiveFiles?: string[]; progress?: MemorySyncProgressState; }): Promise { const memoryPlan = params.shouldSyncMemory @@ -505,9 +508,9 @@ export abstract class MemoryManagerSyncOps { }) : this.emptySourceSyncPlan(); if (params.shouldSyncSessions) { - await this.syncSessionFiles({ + await this.syncArchiveFiles({ needsFullReindex: params.needsFullSessionReindex ?? params.needsFullReindex, - targetSessionFiles: params.targetSessionFiles, + targetArchiveFiles: params.targetArchiveFiles, progress: params.progress, deferIndex: true, prefixIndexItems: memoryPlan.indexItems, @@ -1454,7 +1457,7 @@ export abstract class MemoryManagerSyncOps { if (!this.sources.has("sessions") || this.sessionUnsubscribe) { return; } - this.sessionUnsubscribe = subscribeMemorySessionTranscriptUpdates((update) => { + this.sessionUnsubscribe = this.subscribeSessionTranscriptUpdates((update) => { if (this.closed) { return; } @@ -1479,6 +1482,12 @@ export abstract class MemoryManagerSyncOps { }); } + protected subscribeSessionTranscriptUpdates( + listener: (update: MemorySessionTranscriptUpdate) => void, + ): () => void { + return onInternalSessionTranscriptUpdate(listener); + } + private async scheduleCorpusSessionFileDirty(sessionFile: string): Promise { const resolvedSessionFile = path.resolve(sessionFile); const corpusEntries = await listSessionTranscriptCorpusEntriesForAgent(this.agentId); @@ -1500,8 +1509,8 @@ export abstract class MemoryManagerSyncOps { if (!this.sources.has("sessions") || this.closed) { return []; } - const files = await listSessionFilesForAgent(this.agentId); - if (files.length === 0 || this.closed) { + const corpusEntries = await listSessionTranscriptCorpusEntriesForAgent(this.agentId); + if (corpusEntries.length === 0 || this.closed) { return []; } const existingRows = loadMemorySourceFileState({ @@ -1510,25 +1519,34 @@ export abstract class MemoryManagerSyncOps { }).rows; const fileStates = ( await runWithConcurrency( - files.map((file) => async (): Promise => { - try { - const stat = await fs.stat(file); - if (!stat.isFile()) { - return null; + corpusEntries.map( + (corpusEntry) => async (): Promise => { + if (corpusEntry.transcriptSource === "sqlite") { + return statSessionEntrySync( + corpusEntry.sessionFile, + buildSessionEntryOptions(corpusEntry), + ); } - return { - absPath: file, - path: sessionPathForFile(file), - mtimeMs: stat.mtimeMs, - size: stat.size, - }; - } catch (err) { - if (isFileMissingError(err)) { - return null; + const file = corpusEntry.sessionFile; + try { + const stat = await fs.stat(file); + if (!stat.isFile()) { + return null; + } + return { + absPath: file, + path: sessionPathForCorpusEntry(corpusEntry), + mtimeMs: stat.mtimeMs, + size: stat.size, + }; + } catch (err) { + if (isFileMissingError(err)) { + return null; + } + throw err; } - throw err; - } - }), + }, + ), this.getIndexConcurrency(), ) ).filter((file): file is MemorySessionStartupFileState => file !== null); @@ -1579,9 +1597,15 @@ export abstract class MemoryManagerSyncOps { const pendingTargets = Array.from(this.sessionPendingTargets.values()); this.sessionPendingFiles.clear(); this.sessionPendingTargets.clear(); - pending.push(...Array.from(await this.resolveSessionFilesForSyncTargets(pendingTargets))); + pending.push(...Array.from(await this.resolveArchiveFilesForSyncTargets(pendingTargets))); let shouldSync = false; for (const sessionFile of pending) { + if (!path.isAbsolute(sessionFile)) { + this.sessionsDirtyFiles.add(sessionFile); + this.sessionsDirty = true; + shouldSync = true; + continue; + } // Usage-counted session archives (`.jsonl.reset.` and // `.jsonl.deleted.`) are one-shot mutation events: the file is // written once by the archive rotation and then never touched again. @@ -1789,20 +1813,41 @@ export abstract class MemoryManagerSyncOps { }; } - private normalizeTargetSessionFiles( - sessionFiles?: string[], + private normalizeTargetArchiveFiles( + archiveFiles?: string[], corpusEntries: readonly SessionTranscriptCorpusEntry[] = [], ): Set | null { - if (!sessionFiles || sessionFiles.length === 0) { + if (!archiveFiles || archiveFiles.length === 0) { return null; } const normalized = new Set(); - const corpusPaths = new Set(corpusEntries.map((entry) => path.resolve(entry.sessionFile))); - for (const sessionFile of sessionFiles) { + const corpusMarkers = new Set( + corpusEntries + .filter((entry) => entry.transcriptSource === "sqlite") + .map((entry) => entry.sessionFile), + ); + const corpusPaths = new Set( + corpusEntries + .filter((entry) => entry.transcriptSource !== "sqlite") + .map((entry) => path.resolve(entry.sessionFile)), + ); + for (const sessionFile of archiveFiles) { const trimmed = sessionFile.trim(); if (!trimmed) { continue; } + if (corpusMarkers.has(trimmed)) { + normalized.add(trimmed); + continue; + } + const sqliteMarker = parseSqliteSessionFileMarker(trimmed); + if ( + sqliteMarker && + normalizeAgentId(sqliteMarker.agentId) === normalizeAgentId(this.agentId) + ) { + normalized.add(trimmed); + continue; + } const resolved = path.resolve(trimmed); if ( this.isSessionFileForAgent(resolved) && @@ -1842,7 +1887,7 @@ export abstract class MemoryManagerSyncOps { return normalized.size > 0 ? normalized : null; } - private async resolveSessionFilesForSyncTargets( + private async resolveArchiveFilesForSyncTargets( sessions?: Iterable | null, knownCorpusEntries?: readonly SessionTranscriptCorpusEntry[], ): Promise> { @@ -1866,7 +1911,9 @@ export abstract class MemoryManagerSyncOps { if (sessionKey && entry.sessionKey !== sessionKey) { continue; } - files.add(path.resolve(entry.sessionFile)); + files.add( + entry.transcriptSource === "sqlite" ? entry.sessionFile : path.resolve(entry.sessionFile), + ); matchedCorpusEntry = true; } if (matchedCorpusEntry) { @@ -1887,16 +1934,16 @@ export abstract class MemoryManagerSyncOps { return files; } - private async combineTargetSessionFiles(params: { + private async combineTargetArchiveFiles(params: { sessions?: MemorySessionSyncTarget[]; - sessionFiles?: string[]; + archiveFiles?: string[]; }): Promise | null> { const files = new Set(); const corpusEntries = await listSessionTranscriptCorpusEntriesForAgent(this.agentId); - for (const file of this.normalizeTargetSessionFiles(params.sessionFiles, corpusEntries) ?? []) { + for (const file of this.normalizeTargetArchiveFiles(params.archiveFiles, corpusEntries) ?? []) { files.add(file); } - for (const file of await this.resolveSessionFilesForSyncTargets( + for (const file of await this.resolveArchiveFilesForSyncTargets( this.normalizeTargetSessions(params.sessions)?.values(), corpusEntries, )) { @@ -2089,9 +2136,9 @@ export abstract class MemoryManagerSyncOps { return this.emptySourceSyncPlan(); } - private async syncSessionFiles(params: { + private async syncArchiveFiles(params: { needsFullReindex: boolean; - targetSessionFiles?: string[]; + targetArchiveFiles?: string[]; progress?: MemorySyncProgressState; deferIndex?: boolean; prefixIndexItems?: MemoryIndexWorkItem[]; @@ -2114,34 +2161,43 @@ export abstract class MemoryManagerSyncOps { : null; const corpusEntries = await listSessionTranscriptCorpusEntriesForAgent(this.agentId); - const targetSessionFiles = params.needsFullReindex + const targetArchiveFiles = params.needsFullReindex ? null - : this.normalizeTargetSessionFiles(params.targetSessionFiles, corpusEntries); + : this.normalizeTargetArchiveFiles(params.targetArchiveFiles, corpusEntries); const corpusEntryByPath = new Map( corpusEntries.map((entry) => [entry.sessionFile, entry]), ); - const files = targetSessionFiles - ? Array.from(targetSessionFiles) + const files = targetArchiveFiles + ? Array.from(targetArchiveFiles) : corpusEntries.map((entry) => entry.sessionFile); const sessionPlan = resolveMemorySessionSyncPlan({ needsFullReindex: params.needsFullReindex, files, - targetSessionFiles, + targetSessionFiles: targetArchiveFiles, sessionsDirtyFiles: this.sessionsDirtyFiles, - existingRows: targetSessionFiles + existingRows: targetArchiveFiles ? null : loadMemorySourceFileState({ db: this.db, source: "sessions", }).rows, - sessionPathForFile, + sessionPathForFile: (file) => { + const corpusEntry = corpusEntryByPath.get(file); + if (corpusEntry) { + return sessionPathForCorpusEntry(corpusEntry); + } + const sqliteMarker = parseSqliteSessionFileMarker(file); + return sqliteMarker + ? sessionPathForSessionIdentity(sqliteMarker.agentId, sqliteMarker.sessionId) + : sessionPathForFile(file); + }, }); const { activePaths, existingRows, existingHashes, indexAll } = sessionPlan; log.debug("memory sync: indexing session files", { files: files.length, indexAll, dirtyFiles: this.sessionsDirtyFiles.size, - targetedFiles: targetSessionFiles?.size ?? 0, + targetedFiles: targetArchiveFiles?.size ?? 0, batch: this.batch.enabled, concurrency: this.getIndexConcurrency(), }); @@ -2155,6 +2211,20 @@ export abstract class MemoryManagerSyncOps { } const yieldAfterSessionFile = createSessionSyncYield(files.length); + const deleteIndexedSessionPath = (memoryPath: string) => { + deleteFileByPathAndSource.run(memoryPath, "sessions"); + if (deleteVectorRowsByPathAndSource) { + try { + deleteVectorRowsByPathAndSource.run(memoryPath, "sessions"); + } catch {} + } + deleteChunksByPathAndSource.run(memoryPath, "sessions"); + if (deleteFtsRowsByPathAndSource) { + try { + deleteFtsRowsByPathAndSource.run(memoryPath, "sessions"); + } catch {} + } + }; const deleteStaleRows = async () => { if (activePaths === null) { return; @@ -2167,23 +2237,50 @@ export abstract class MemoryManagerSyncOps { if (activePaths.has(stale.path)) { continue; } - deleteFileByPathAndSource.run(stale.path, "sessions"); - if (deleteVectorRowsByPathAndSource) { - try { - deleteVectorRowsByPathAndSource.run(stale.path, "sessions"); - } catch {} - } - deleteChunksByPathAndSource.run(stale.path, "sessions"); - if (deleteFtsRowsByPathAndSource) { - try { - deleteFtsRowsByPathAndSource.run(stale.path, "sessions"); - } catch {} - } + deleteIndexedSessionPath(stale.path); } finally { await yieldAfterStaleSessionRow(); } } }; + const deleteTargetArchiveStaleLiveRows = () => { + if (!targetArchiveFiles) { + return; + } + const activeCorpusPaths = new Set( + corpusEntries + .filter((entry) => entry.artifactKind === "active-session") + .map((entry) => sessionPathForCorpusEntry(entry)), + ); + const existingSessionPaths = new Set( + loadMemorySourceFileState({ + db: this.db, + source: "sessions", + }).rows.map((row) => row.path), + ); + for (const file of targetArchiveFiles) { + const corpusEntry = corpusEntryByPath.get(file); + const sqliteMarker = parseSqliteSessionFileMarker(file); + const sessionId = + corpusEntry?.sessionId ?? + sqliteMarker?.sessionId ?? + parseUsageCountedSessionIdFromFileName(path.basename(file)); + if (!sessionId) { + continue; + } + const staleAgentId = corpusEntry?.agentId ?? sqliteMarker?.agentId ?? this.agentId; + const staleLivePaths = [ + sessionPathForSessionIdentity(staleAgentId, sessionId), + legacyExtensionlessSessionPathForIdentity(staleAgentId, sessionId), + ]; + for (const staleLivePath of staleLivePaths) { + if (activeCorpusPaths.has(staleLivePath) || !existingSessionPaths.has(staleLivePath)) { + continue; + } + deleteIndexedSessionPath(staleLivePath); + } + } + }; if (params.deferIndex) { const pendingIndexItems = [...(params.prefixIndexItems ?? [])]; @@ -2221,13 +2318,7 @@ export abstract class MemoryManagerSyncOps { const corpusEntry = corpusEntryByPath.get(absPath); const entry = await buildSessionEntry( absPath, - corpusEntry - ? { - generatedByDreamingNarrative: - corpusEntry.generatedByDreamingNarrative === true, - generatedByCronRun: corpusEntry.generatedByCronRun === true, - } - : undefined, + corpusEntry ? buildSessionEntryOptions(corpusEntry) : undefined, ); if (!entry) { if (params.progress) { @@ -2279,6 +2370,7 @@ export abstract class MemoryManagerSyncOps { } await flushPendingIndexItems(); + deleteTargetArchiveStaleLiveRows(); await deleteStaleRows(); return this.emptySourceSyncPlan(); } @@ -2301,12 +2393,7 @@ export abstract class MemoryManagerSyncOps { const corpusEntry = corpusEntryByPath.get(absPath); const entry = await buildSessionEntry( absPath, - corpusEntry - ? { - generatedByDreamingNarrative: corpusEntry.generatedByDreamingNarrative === true, - generatedByCronRun: corpusEntry.generatedByCronRun === true, - } - : undefined, + corpusEntry ? buildSessionEntryOptions(corpusEntry) : undefined, ); if (!entry) { if (params.progress) { @@ -2350,6 +2437,7 @@ export abstract class MemoryManagerSyncOps { }); await runWithConcurrency(tasks, this.getIndexConcurrency()); + deleteTargetArchiveStaleLiveRows(); await deleteStaleRows(); return this.emptySourceSyncPlan(); } @@ -2418,15 +2506,15 @@ export abstract class MemoryManagerSyncOps { } const vectorReady = await this.ensureVectorReady(); const meta = this.readMeta(); - const targetSessionFiles = await this.combineTargetSessionFiles({ + const targetArchiveFiles = await this.combineTargetArchiveFiles({ sessions: params?.sessions, - sessionFiles: params?.sessionFiles, + archiveFiles: params?.archiveFiles, }); - const hasTargetSessionFiles = targetSessionFiles !== null; - if (this.hasRequestedTargetSessionSync(params) && !hasTargetSessionFiles) { + const hasTargetArchiveFiles = targetArchiveFiles !== null; + if (this.hasRequestedTargetSessionSync(params) && !hasTargetArchiveFiles) { return; } - if (params?.reason === "cli" && !params.force && !hasTargetSessionFiles) { + if (params?.reason === "cli" && !params.force && !hasTargetArchiveFiles) { await this.markSessionStartupCatchupDirtyFiles(); } const indexIdentity = resolveMemoryIndexIdentityState({ @@ -2473,13 +2561,13 @@ export abstract class MemoryManagerSyncOps { this.settings.provider === "none" || hasOnlyFtsChunks; const needsMissingIdentityReindex = - indexIdentity.status === "missing" && !hasTargetSessionFiles && canRebuildMissingIdentity; + indexIdentity.status === "missing" && !hasTargetArchiveFiles && canRebuildMissingIdentity; const needsExplicitIdentityReindex = - params?.reason === "cli" && indexIdentity.status !== "valid" && !hasTargetSessionFiles; + params?.reason === "cli" && indexIdentity.status !== "valid" && !hasTargetArchiveFiles; const canRunRetryFullReindex = indexIdentity.status !== "missing" || needsInitialIndex || canRebuildMissingIdentity; const needsFullReindex = - (params?.force && !hasTargetSessionFiles) || + (params?.force && !hasTargetArchiveFiles) || needsInitialIndex || needsMissingIdentityReindex || needsExplicitIdentityReindex || @@ -2488,9 +2576,9 @@ export abstract class MemoryManagerSyncOps { const needsFullSessionReindex = needsFullReindex || this.sessionsFullRetryDirty; if (indexIdentity.status !== "valid" && !needsFullReindex) { this.dirty = true; - const sessionsDirty = markMemoryTargetSessionFilesDirty({ + const sessionsDirty = markMemoryTargetArchiveFilesDirty({ sessionsDirtyFiles: this.sessionsDirtyFiles, - targetSessionFiles, + targetArchiveFiles, }); if (sessionsDirty) { this.sessionsDirty = true; @@ -2500,13 +2588,13 @@ export abstract class MemoryManagerSyncOps { if (!needsFullSessionReindex) { const targetedSessionSync = await runMemoryTargetedSessionSync({ hasSessionSource: this.sources.has("sessions"), - targetSessionFiles, + targetArchiveFiles, reason: params?.reason, progress: progress ?? undefined, sessionsFullRetryDirty: this.sessionsFullRetryDirty, sessionsDirtyFiles: this.sessionsDirtyFiles, - syncSessionFiles: async (targetedParams) => { - await this.syncSessionFiles(targetedParams); + syncArchiveFiles: async (targetedParams) => { + await this.syncArchiveFiles(targetedParams); }, shouldFallbackOnError: (err) => this.shouldFallbackOnError(err), activateFallbackProvider: async (reason) => await this.activateFallbackProvider(reason), @@ -2528,7 +2616,7 @@ export abstract class MemoryManagerSyncOps { const shouldSyncMemory = this.sources.has("memory") && - ((!hasTargetSessionFiles && params?.force) || needsFullReindex || this.dirty); + ((!hasTargetArchiveFiles && params?.force) || needsFullReindex || this.dirty); const shouldSyncSessions = this.shouldSyncSessions(params, needsFullReindex); if (this.shouldDeferSourceWideBatch()) { @@ -2537,7 +2625,7 @@ export abstract class MemoryManagerSyncOps { shouldSyncSessions, needsFullReindex, needsFullSessionReindex, - targetSessionFiles: targetSessionFiles ? Array.from(targetSessionFiles) : undefined, + targetArchiveFiles: targetArchiveFiles ? Array.from(targetArchiveFiles) : undefined, progress: progress ?? undefined, }); if (shouldSyncMemory) { @@ -2555,9 +2643,9 @@ export abstract class MemoryManagerSyncOps { } if (shouldSyncSessions) { - await this.syncSessionFiles({ + await this.syncArchiveFiles({ needsFullReindex: needsFullSessionReindex, - targetSessionFiles: targetSessionFiles ? Array.from(targetSessionFiles) : undefined, + targetArchiveFiles: targetArchiveFiles ? Array.from(targetArchiveFiles) : undefined, progress: progress ?? undefined, }); this.clearSessionRetryState(); @@ -2570,7 +2658,7 @@ export abstract class MemoryManagerSyncOps { const activated = this.shouldFallbackOnError(err) && (await this.activateFallbackProvider(reason)); if (activated) { - if (needsFullReindex && !hasTargetSessionFiles) { + if (needsFullReindex && !hasTargetArchiveFiles) { await this.runInPlaceReindex({ reason: params?.reason ?? "fallback", force: true, @@ -2594,7 +2682,7 @@ export abstract class MemoryManagerSyncOps { private hasRequestedTargetSessionSync(params?: MemorySyncParams): boolean { return Boolean( params?.sessions?.some((session) => session.sessionId.trim().length > 0) || - params?.sessionFiles?.some((sessionFile) => sessionFile.trim().length > 0), + params?.archiveFiles?.some((sessionFile) => sessionFile.trim().length > 0), ); } @@ -2744,7 +2832,7 @@ export abstract class MemoryManagerSyncOps { } if (shouldSyncSessions) { - await this.syncSessionFiles({ needsFullReindex: true, progress: params.progress }); + await this.syncArchiveFiles({ needsFullReindex: true, progress: params.progress }); this.clearSessionRetryState(); } else { this.refreshSessionDirtyFlag(); diff --git a/extensions/memory-core/src/memory/manager-sync-yield.test.ts b/extensions/memory-core/src/memory/manager-sync-yield.test.ts index 1f56111b4117..a421345748e3 100644 --- a/extensions/memory-core/src/memory/manager-sync-yield.test.ts +++ b/extensions/memory-core/src/memory/manager-sync-yield.test.ts @@ -62,6 +62,8 @@ vi.mock("openclaw/plugin-sdk/memory-core-host-engine-qmd", () => { sessionId: target.sessionId, }), sessionPathForFile: (filePath: string) => `sessions/${basename(filePath)}`, + sessionPathForSessionIdentity: (agentId: string, sessionId: string) => + `sessions/${agentId}/${sessionId}`, }; }); @@ -126,17 +128,17 @@ class SessionSyncYieldHarness extends MemoryManagerSyncOps { super(); } - async syncTargetSessionFiles(files: string[]): Promise { + async syncTargetArchiveFiles(files: string[]): Promise { await ( this as unknown as { - syncSessionFiles: (params: { + syncArchiveFiles: (params: { needsFullReindex: boolean; - targetSessionFiles: string[]; + targetArchiveFiles: string[]; }) => Promise; } - ).syncSessionFiles({ + ).syncArchiveFiles({ needsFullReindex: false, - targetSessionFiles: files, + targetArchiveFiles: files, }); } @@ -217,7 +219,7 @@ describe("session sync responsiveness", () => { } }); - await harness.syncTargetSessionFiles(files); + await harness.syncTargetArchiveFiles(files); expect(harness.indexedPaths).toHaveLength(files.length); expect(observedBeforeLastFile).toEqual([true]); diff --git a/extensions/memory-core/src/memory/manager-targeted-sync.test.ts b/extensions/memory-core/src/memory/manager-targeted-sync.test.ts index ea3530cd2f0f..e35874aa67cd 100644 --- a/extensions/memory-core/src/memory/manager-targeted-sync.test.ts +++ b/extensions/memory-core/src/memory/manager-targeted-sync.test.ts @@ -3,8 +3,8 @@ import type { MemorySessionSyncTarget } from "openclaw/plugin-sdk/memory-core-ho import { describe, expect, it, vi } from "vitest"; import { enqueueMemoryTargetedSessionSync } from "./manager-sync-control.js"; import { - clearMemorySyncedSessionFiles, - markMemoryTargetSessionFilesDirty, + clearMemorySyncedArchiveFiles, + markMemoryTargetArchiveFilesDirty, runMemoryTargetedSessionSync, } from "./manager-targeted-sync.js"; @@ -13,9 +13,9 @@ describe("memory targeted session sync", () => { const secondSessionPath = "/tmp/targeted-dirty-second.jsonl"; const sessionsDirtyFiles = new Set(["/tmp/targeted-dirty-first.jsonl", secondSessionPath]); - const sessionsDirty = clearMemorySyncedSessionFiles({ + const sessionsDirty = clearMemorySyncedArchiveFiles({ sessionsDirtyFiles, - targetSessionFiles: ["/tmp/targeted-dirty-first.jsonl"], + targetArchiveFiles: ["/tmp/targeted-dirty-first.jsonl"], }); expect(sessionsDirtyFiles.has(secondSessionPath)).toBe(true); @@ -26,9 +26,9 @@ describe("memory targeted session sync", () => { const targetSessionPath = "/tmp/paused-target.jsonl"; const sessionsDirtyFiles = new Set(["/tmp/other-dirty.jsonl"]); - const sessionsDirty = markMemoryTargetSessionFilesDirty({ + const sessionsDirty = markMemoryTargetArchiveFilesDirty({ sessionsDirtyFiles, - targetSessionFiles: [targetSessionPath], + targetArchiveFiles: [targetSessionPath], }); expect(sessionsDirty).toBe(true); @@ -38,7 +38,7 @@ describe("memory targeted session sync", () => { it("leaves targeted sessions dirty after fallback activates during targeted sync", async () => { const activateFallbackProvider = vi.fn(async () => true); - const syncSessionFiles = vi + const syncArchiveFiles = vi .fn() .mockRejectedValueOnce(new Error("embedding backend failed")) .mockResolvedValueOnce(undefined); @@ -46,20 +46,20 @@ describe("memory targeted session sync", () => { const result = await runMemoryTargetedSessionSync({ hasSessionSource: true, - targetSessionFiles: new Set(["/tmp/targeted-fallback.jsonl"]), + targetArchiveFiles: new Set(["/tmp/targeted-fallback.jsonl"]), reason: "post-compaction", progress: undefined, sessionsDirtyFiles, - syncSessionFiles, + syncArchiveFiles, shouldFallbackOnError: () => true, activateFallbackProvider, }); expect(activateFallbackProvider).toHaveBeenCalledWith("embedding backend failed"); - expect(syncSessionFiles).toHaveBeenCalledTimes(1); - expect(syncSessionFiles).toHaveBeenCalledWith({ + expect(syncArchiveFiles).toHaveBeenCalledTimes(1); + expect(syncArchiveFiles).toHaveBeenCalledWith({ needsFullReindex: false, - targetSessionFiles: ["/tmp/targeted-fallback.jsonl"], + targetArchiveFiles: ["/tmp/targeted-fallback.jsonl"], progress: undefined, }); expect(result).toEqual({ handled: true, sessionsDirty: true }); @@ -68,17 +68,17 @@ describe("memory targeted session sync", () => { }); it("preserves the full-retry dirty marker after targeted cleanup", async () => { - const syncSessionFiles = vi.fn(async () => undefined); + const syncArchiveFiles = vi.fn(async () => undefined); const sessionsDirtyFiles = new Set(["/tmp/targeted-full-retry.jsonl"]); const result = await runMemoryTargetedSessionSync({ hasSessionSource: true, - targetSessionFiles: new Set(["/tmp/targeted-full-retry.jsonl"]), + targetArchiveFiles: new Set(["/tmp/targeted-full-retry.jsonl"]), reason: "post-compaction", progress: undefined, sessionsFullRetryDirty: true, sessionsDirtyFiles, - syncSessionFiles, + syncArchiveFiles, shouldFallbackOnError: () => false, activateFallbackProvider: async () => false, }); @@ -92,7 +92,7 @@ describe("memory targeted session sync", () => { const syncing = new Promise((resolve) => { resolveSyncing = resolve; }); - const queuedSessionFiles = new Set(); + const queuedArchiveFiles = new Set(); const queuedSessions = new Map(); let queuedSessionSync: Promise | null = null; const sync = vi.fn(async () => {}); @@ -101,7 +101,7 @@ describe("memory targeted session sync", () => { { isClosed: () => false, getSyncing: () => syncing, - getQueuedSessionFiles: () => queuedSessionFiles, + getQueuedArchiveFiles: () => queuedArchiveFiles, getQueuedSessions: () => queuedSessions, getQueuedSessionSync: () => queuedSessionSync, setQueuedSessionSync: (value) => { @@ -120,7 +120,7 @@ describe("memory targeted session sync", () => { expect(sync).toHaveBeenCalledWith({ reason: "queued-sessions", sessions: [{ agentId: "main", sessionId: "targeted", sessionKey: "agent:main:targeted" }], - sessionFiles: [], + archiveFiles: [], }); }); }); diff --git a/extensions/memory-core/src/memory/manager-targeted-sync.ts b/extensions/memory-core/src/memory/manager-targeted-sync.ts index 48ff992795c7..12da46351ca7 100644 --- a/extensions/memory-core/src/memory/manager-targeted-sync.ts +++ b/extensions/memory-core/src/memory/manager-targeted-sync.ts @@ -9,27 +9,27 @@ type TargetedSyncProgress = { report: (update: MemorySyncProgressUpdate) => void; }; -export function clearMemorySyncedSessionFiles(params: { +export function clearMemorySyncedArchiveFiles(params: { sessionsDirtyFiles: Set; - targetSessionFiles?: Iterable | null; + targetArchiveFiles?: Iterable | null; }): boolean { - if (!params.targetSessionFiles) { + if (!params.targetArchiveFiles) { params.sessionsDirtyFiles.clear(); } else { - for (const targetSessionFile of params.targetSessionFiles) { - params.sessionsDirtyFiles.delete(targetSessionFile); + for (const targetArchiveFile of params.targetArchiveFiles) { + params.sessionsDirtyFiles.delete(targetArchiveFile); } } return params.sessionsDirtyFiles.size > 0; } -export function markMemoryTargetSessionFilesDirty(params: { +export function markMemoryTargetArchiveFilesDirty(params: { sessionsDirtyFiles: Set; - targetSessionFiles?: Iterable | null; + targetArchiveFiles?: Iterable | null; }): boolean { - if (params.targetSessionFiles) { - for (const targetSessionFile of params.targetSessionFiles) { - params.sessionsDirtyFiles.add(targetSessionFile); + if (params.targetArchiveFiles) { + for (const targetArchiveFile of params.targetArchiveFiles) { + params.sessionsDirtyFiles.add(targetArchiveFile); } } return params.sessionsDirtyFiles.size > 0; @@ -37,20 +37,20 @@ export function markMemoryTargetSessionFilesDirty(params: { export async function runMemoryTargetedSessionSync(params: { hasSessionSource: boolean; - targetSessionFiles: Set | null; + targetArchiveFiles: Set | null; reason?: string; progress?: TargetedSyncProgress; sessionsFullRetryDirty?: boolean; sessionsDirtyFiles: Set; - syncSessionFiles: (params: { + syncArchiveFiles: (params: { needsFullReindex: boolean; - targetSessionFiles?: string[]; + targetArchiveFiles?: string[]; progress?: TargetedSyncProgress; }) => Promise; shouldFallbackOnError: (err: unknown) => boolean; activateFallbackProvider: (reason: string) => Promise; }): Promise<{ handled: boolean; sessionsDirty: boolean }> { - if (!params.hasSessionSource || !params.targetSessionFiles) { + if (!params.hasSessionSource || !params.targetArchiveFiles) { return { handled: false, sessionsDirty: Boolean(params.sessionsFullRetryDirty) || params.sessionsDirtyFiles.size > 0, @@ -58,14 +58,14 @@ export async function runMemoryTargetedSessionSync(params: { } try { - await params.syncSessionFiles({ + await params.syncArchiveFiles({ needsFullReindex: false, - targetSessionFiles: Array.from(params.targetSessionFiles), + targetArchiveFiles: Array.from(params.targetArchiveFiles), progress: params.progress, }); - const remainingSessionsDirty = clearMemorySyncedSessionFiles({ + const remainingSessionsDirty = clearMemorySyncedArchiveFiles({ sessionsDirtyFiles: params.sessionsDirtyFiles, - targetSessionFiles: params.targetSessionFiles, + targetArchiveFiles: params.targetArchiveFiles, }); return { handled: true, @@ -78,9 +78,9 @@ export async function runMemoryTargetedSessionSync(params: { if (!activated) { throw err; } - const remainingSessionsDirty = markMemoryTargetSessionFilesDirty({ + const remainingSessionsDirty = markMemoryTargetArchiveFilesDirty({ sessionsDirtyFiles: params.sessionsDirtyFiles, - targetSessionFiles: params.targetSessionFiles, + targetArchiveFiles: params.targetArchiveFiles, }); return { handled: true, diff --git a/extensions/memory-core/src/memory/manager.readonly-recovery.test.ts b/extensions/memory-core/src/memory/manager.readonly-recovery.test.ts index 285cc992517b..a54e1e5b87fb 100644 --- a/extensions/memory-core/src/memory/manager.readonly-recovery.test.ts +++ b/extensions/memory-core/src/memory/manager.readonly-recovery.test.ts @@ -13,7 +13,7 @@ import { type ReadonlyRecoveryHarness = MemoryReadonlyRecoveryState & { syncing: Promise | null; - queuedSessionFiles: Set; + queuedArchiveFiles: Set; queuedSessions: Map; queuedSessionSync: Promise | null; vectorDegradedWriteWarningShown: boolean; @@ -32,12 +32,12 @@ describe("memory manager readonly recovery", () => { let indexPath = ""; function createQueuedSyncHarness(syncing: Promise) { - const queuedSessionFiles = new Set(); + const queuedArchiveFiles = new Set(); const queuedSessions = new Map(); let queuedSessionSync: Promise | null = null; const sync = vi.fn(async () => {}); return { - queuedSessionFiles, + queuedArchiveFiles, queuedSessions, get queuedSessionSync() { return queuedSessionSync; @@ -46,7 +46,7 @@ describe("memory manager readonly recovery", () => { state: { isClosed: () => false, getSyncing: () => syncing, - getQueuedSessionFiles: () => queuedSessionFiles, + getQueuedArchiveFiles: () => queuedArchiveFiles, getQueuedSessions: () => queuedSessions, getQueuedSessionSync: () => queuedSessionSync, setQueuedSessionSync: (value: Promise | null) => { @@ -64,7 +64,7 @@ describe("memory manager readonly recovery", () => { const harness: ReadonlyRecoveryHarness = { closed: false, syncing: null, - queuedSessionFiles: new Set(), + queuedArchiveFiles: new Set(), queuedSessions: new Map(), queuedSessionSync: null, db: initialDb, @@ -101,7 +101,7 @@ describe("memory manager readonly recovery", () => { async function runSyncWithReadonlyRecovery( harness: ReadonlyRecoveryHarness, - params?: { reason?: string; force?: boolean; sessionFiles?: string[] }, + params?: { reason?: string; force?: boolean; archiveFiles?: string[] }, ) { return await runMemorySyncWithReadonlyRecovery(harness, params); } @@ -230,7 +230,7 @@ describe("memory manager readonly recovery", () => { const harness = createQueuedSyncHarness(pendingSync); const queued = enqueueMemoryTargetedSessionSync(harness.state, { - sessionFiles: [" /tmp/first.jsonl ", "", "/tmp/second.jsonl"], + archiveFiles: [" /tmp/first.jsonl ", "", "/tmp/second.jsonl"], }); expect(harness.sync).not.toHaveBeenCalled(); @@ -242,7 +242,7 @@ describe("memory manager readonly recovery", () => { expect(harness.sync).toHaveBeenCalledWith({ reason: "queued-sessions", sessions: [], - sessionFiles: ["/tmp/first.jsonl", "/tmp/second.jsonl"], + archiveFiles: ["/tmp/first.jsonl", "/tmp/second.jsonl"], }); expect(harness.queuedSessionSync).toBeNull(); }); @@ -255,10 +255,10 @@ describe("memory manager readonly recovery", () => { const harness = createQueuedSyncHarness(pendingSync); const first = enqueueMemoryTargetedSessionSync(harness.state, { - sessionFiles: ["/tmp/first.jsonl", "/tmp/second.jsonl"], + archiveFiles: ["/tmp/first.jsonl", "/tmp/second.jsonl"], }); const second = enqueueMemoryTargetedSessionSync(harness.state, { - sessionFiles: ["/tmp/second.jsonl", "/tmp/third.jsonl"], + archiveFiles: ["/tmp/second.jsonl", "/tmp/third.jsonl"], }); expect(first).toBe(second); @@ -270,7 +270,7 @@ describe("memory manager readonly recovery", () => { expect(harness.sync).toHaveBeenCalledWith({ reason: "queued-sessions", sessions: [], - sessionFiles: ["/tmp/first.jsonl", "/tmp/second.jsonl", "/tmp/third.jsonl"], + archiveFiles: ["/tmp/first.jsonl", "/tmp/second.jsonl", "/tmp/third.jsonl"], }); }); @@ -282,7 +282,7 @@ describe("memory manager readonly recovery", () => { const harness = createQueuedSyncHarness(pendingSync); const queued = enqueueMemoryTargetedSessionSync(harness.state, { - sessionFiles: ["", " "], + archiveFiles: ["", " "], }); expect(queued).toBe(pendingSync); diff --git a/extensions/memory-core/src/memory/manager.reindex-recovery.test.ts b/extensions/memory-core/src/memory/manager.reindex-recovery.test.ts index 1417c8bfc786..0e5aeb0299bf 100644 --- a/extensions/memory-core/src/memory/manager.reindex-recovery.test.ts +++ b/extensions/memory-core/src/memory/manager.reindex-recovery.test.ts @@ -12,26 +12,13 @@ import { acquireMemoryReindexLock } from "./manager-reindex-lock.js"; import type { MemoryIndexMeta } from "./manager-reindex-state.js"; type SessionDeltaState = { lastSize: number; pendingBytes: number; pendingMessages: number }; -type SyncSessionParams = { needsFullReindex: boolean; targetSessionFiles?: string[] }; -const originalReindexStateDir = process.env.OPENCLAW_STATE_DIR; - -function setReindexStateDir(stateDir: string): void { - Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir); -} - -function restoreReindexStateDir(): void { - if (originalReindexStateDir === undefined) { - Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR"); - } else { - Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalReindexStateDir); - } -} +type SyncArchiveParams = { needsFullReindex: boolean; targetArchiveFiles?: string[] }; type ReindexHarness = { sync: (params: { reason?: string; force?: boolean }) => Promise; runInPlaceReindex: (params: { reason?: string; force?: boolean }) => Promise; syncMemoryFiles: (params: { needsFullReindex: boolean }) => Promise; - syncSessionFiles: (params: SyncSessionParams) => Promise; + syncArchiveFiles: (params: SyncArchiveParams) => Promise; db: DatabaseSync; writeMeta: (meta: MemoryIndexMeta) => void; providerKey: string | null; @@ -55,11 +42,11 @@ describe("memory manager reindex recovery", () => { workspaceDir = path.join(fixtureRoot, "workspace"); memoryDir = path.join(workspaceDir, "memory"); await fs.mkdir(memoryDir, { recursive: true }); - setReindexStateDir(path.join(fixtureRoot, "state")); + vi.stubEnv("OPENCLAW_STATE_DIR", path.join(fixtureRoot, "state")); }); afterEach(async () => { - restoreReindexStateDir(); + vi.unstubAllEnvs(); vi.restoreAllMocks(); if (manager) { await manager.close(); @@ -130,7 +117,7 @@ describe("memory manager reindex recovery", () => { harness.sessionsDirtyFiles.add(dirtySessionFile); harness.sessionDeltas.set(dirtySessionFile, { ...originalDelta }); harness.syncMemoryFiles = async () => emptySyncPlan; - harness.syncSessionFiles = async () => { + harness.syncArchiveFiles = async () => { const delta = harness.sessionDeltas.get(dirtySessionFile); if (delta) { delta.lastSize = 500; @@ -165,7 +152,7 @@ describe("memory manager reindex recovery", () => { const emptySyncPlan = { indexItems: [], finalize: () => undefined }; harness.syncMemoryFiles = async () => emptySyncPlan; - harness.syncSessionFiles = async () => emptySyncPlan; + harness.syncArchiveFiles = async () => emptySyncPlan; harness.writeMeta = () => { throw new Error("late clean reindex failure"); }; @@ -237,12 +224,12 @@ describe("memory manager reindex recovery", () => { const harness = memoryManager as unknown as ReindexHarness; const emptySyncPlan = { indexItems: [], finalize: () => undefined }; - const sessionSyncCalls: SyncSessionParams[] = []; + const sessionSyncCalls: SyncArchiveParams[] = []; harness.sessionsDirty = true; harness.sessionsFullRetryDirty = true; harness.sessionsDirtyFiles.clear(); - harness.syncSessionFiles = async (params) => { + harness.syncArchiveFiles = async (params) => { sessionSyncCalls.push(params); return emptySyncPlan; }; @@ -251,7 +238,7 @@ describe("memory manager reindex recovery", () => { expect(sessionSyncCalls).toHaveLength(1); expect(sessionSyncCalls[0]).toMatchObject({ needsFullReindex: true }); - expect(sessionSyncCalls[0]?.targetSessionFiles).toBeUndefined(); + expect(sessionSyncCalls[0]?.targetArchiveFiles).toBeUndefined(); expect(harness.sessionsDirty).toBe(false); expect(harness.sessionsFullRetryDirty).toBe(false); }); diff --git a/extensions/memory-core/src/memory/manager.ts b/extensions/memory-core/src/memory/manager.ts index f7f272f8648c..0ae86a28c57d 100644 --- a/extensions/memory-core/src/memory/manager.ts +++ b/extensions/memory-core/src/memory/manager.ts @@ -356,7 +356,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem >(); private sessionWarm = new Set(); private syncing: Promise | null = null; - private queuedSessionFiles = new Set(); + private queuedArchiveFiles = new Set(); private queuedSessions = new Map(); private queuedSessionSync: Promise | null = null; private readonlyRecoveryAttempts = 0; @@ -1259,13 +1259,13 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem } private enqueueTargetedSessionSync( - targets?: Pick, + targets?: Pick, ): Promise { return enqueueMemoryTargetedSessionSync( { isClosed: () => this.closed, getSyncing: () => this.syncing, - getQueuedSessionFiles: () => this.queuedSessionFiles, + getQueuedArchiveFiles: () => this.queuedArchiveFiles, getQueuedSessions: () => this.queuedSessions, getQueuedSessionSync: () => this.queuedSessionSync, setQueuedSessionSync: (value) => { @@ -1637,7 +1637,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem function hasTargetedSessionSyncParams(params: MemorySyncParams | undefined): boolean { return Boolean( params?.sessions?.some((session) => session.sessionId.trim().length > 0) || - params?.sessionFiles?.some((sessionFile) => sessionFile.trim().length > 0), + params?.archiveFiles?.some((sessionFile) => sessionFile.trim().length > 0), ); } diff --git a/extensions/memory-core/src/memory/qmd-manager.test.ts b/extensions/memory-core/src/memory/qmd-manager.test.ts index 4d7089832116..305227fe45b6 100644 --- a/extensions/memory-core/src/memory/qmd-manager.test.ts +++ b/extensions/memory-core/src/memory/qmd-manager.test.ts @@ -204,7 +204,10 @@ import { resolveMemoryBackendConfig, } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; import { formatSessionTranscriptMemoryHitKey } from "openclaw/plugin-sdk/session-transcript-hit"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; +import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing"; import { configureMemoryCoreDreamingState, configureMemoryCoreDreamingStateForTests, @@ -223,6 +226,41 @@ function setQmdStateDir(stateDir: string): void { Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir); } +async function seedQmdSessionTranscript(params: { + agentId: string; + content: string; + sessionId: string; + stateDir: string; + sessionKey?: string; + timestamp?: number | string; +}): Promise { + const sessionsDir = path.join(params.stateDir, "agents", params.agentId, "sessions"); + const storePath = path.join(sessionsDir, "sessions.json"); + const sessionKey = params.sessionKey ?? `agent:${params.agentId}:qmd:${params.sessionId}`; + const timestamp = + typeof params.timestamp === "number" + ? params.timestamp + : Date.parse(params.timestamp ?? "2026-04-07T15:25:04.113Z"); + await fs.mkdir(sessionsDir, { recursive: true }); + await upsertSessionEntry({ + agentId: params.agentId, + sessionKey, + storePath, + entry: { sessionId: params.sessionId, updatedAt: timestamp }, + }); + await appendSessionTranscriptMessageByIdentity({ + agentId: params.agentId, + sessionId: params.sessionId, + sessionKey, + storePath, + message: { + role: "user", + content: params.content, + timestamp, + }, + }); +} + function restoreQmdStateDir(): void { if (originalQmdStateDir === undefined) { Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR"); @@ -815,6 +853,7 @@ describe("QmdMemoryManager", () => { delete (globalThis as Record)[QMD_EMBED_QUEUE_KEY]; delete (globalThis as Record)[MEMORY_EMBEDDING_PROVIDERS_KEY]; resetMemoryCoreDreamingStateForTests(); + closeOpenClawAgentDatabasesForTest(); }); it("debounces back-to-back sync calls", async () => { @@ -5698,32 +5737,22 @@ describe("QmdMemoryManager", () => { }, } as OpenClawConfig; - const sessionsDir = path.join(stateDir, "agents", agentId, "sessions"); - await fs.mkdir(sessionsDir, { recursive: true }); - await fs.writeFile( - path.join(sessionsDir, "actual-session-topic-thread.jsonl"), - '{"type":"message","message":{"role":"user","content":"hello mapped session"}}\n', - "utf-8", - ); - await fs.writeFile( - path.join(sessionsDir, "sessions.json"), - JSON.stringify({ - "agent:main:chat:thread": { - sessionFile: "actual-session-topic-thread.jsonl", - sessionId: "actual-session", - }, - }), - "utf-8", - ); + await seedQmdSessionTranscript({ + agentId, + content: "hello mapped session", + sessionId: "actual-session", + stateDir, + sessionKey: "agent:main:chat:thread", + }); const { manager } = await createManager({ mode: "status" }); await (manager as unknown as { exportSessions: () => Promise }).exportSessions(); const indexPath = (manager as unknown as { indexPath: string }).indexPath; const identity = resolveQmdSessionArtifactIdentity({ - artifactPath: "actual-session-topic-thread.md", + artifactPath: "actual-session.md", collection: "sessions-main", indexPath, - searchPath: "qmd/sessions-main/actual-session-topic-thread.md", + searchPath: "qmd/sessions-main/actual-session.md", }); expect(identity).toEqual({ @@ -6116,15 +6145,8 @@ describe("QmdMemoryManager", () => { }); it("reuses exported session markdown files when inputs are unchanged", async () => { - const sessionsDir = path.join(stateDir, "agents", agentId, "sessions"); - await fs.mkdir(sessionsDir, { recursive: true }); - const sessionFile = path.join(sessionsDir, "session-1.jsonl"); const exportFile = path.join(stateDir, "agents", agentId, "qmd", "sessions", "session-1.md"); - await fs.writeFile( - sessionFile, - '{"type":"message","message":{"role":"user","content":"hello"}}\n', - "utf-8", - ); + await seedQmdSessionTranscript({ agentId, content: "hello", sessionId: "session-1", stateDir }); const currentMemory = cfg.memory; cfg = { @@ -6813,20 +6835,18 @@ describe("QmdMemoryManager", () => { }, } as OpenClawConfig; - const sessionsDir = path.join(stateDir, "agents", agentId, "sessions"); - await fs.mkdir(sessionsDir, { recursive: true }); - await fs.writeFile( - path.join(sessionsDir, "live-session.jsonl"), - `${JSON.stringify({ type: "message", message: { role: "user", content: "live" } })}\n`, - ); - await fs.writeFile( - path.join(sessionsDir, "team.checkpoint.notes.jsonl"), - `${JSON.stringify({ type: "message", message: { role: "user", content: "notes" } })}\n`, - ); - await fs.writeFile( - path.join(sessionsDir, "live-session.checkpoint.11111111-1111-4111-8111-111111111111.jsonl"), - `${JSON.stringify({ type: "message", message: { role: "user", content: "checkpoint" } })}\n`, - ); + await seedQmdSessionTranscript({ + agentId, + content: "live", + sessionId: "live-session", + stateDir, + }); + await seedQmdSessionTranscript({ + agentId, + content: "notes", + sessionId: "team.checkpoint.notes", + stateDir, + }); const { manager } = await createManager({ mode: "full" }); const sessionExportDir = path.join(stateDir, "agents", agentId, "qmd", "sessions"); diff --git a/extensions/memory-core/src/memory/qmd-manager.ts b/extensions/memory-core/src/memory/qmd-manager.ts index 085239a5163f..fd441319b1f7 100644 --- a/extensions/memory-core/src/memory/qmd-manager.ts +++ b/extensions/memory-core/src/memory/qmd-manager.ts @@ -1767,7 +1767,7 @@ export class QmdMemoryManager implements MemorySearchManager { async sync(params?: MemorySyncParams): Promise { if ( params?.sessions?.some((session) => session.sessionId.trim().length > 0) || - params?.sessionFiles?.some((sessionFile) => sessionFile.trim().length > 0) + params?.archiveFiles?.some((sessionFile) => sessionFile.trim().length > 0) ) { log.debug("qmd sync ignoring targeted session hint; running regular update"); } @@ -2872,6 +2872,8 @@ export class QmdMemoryManager implements MemorySearchManager { const entry = await buildSessionEntry(sessionFile, { generatedByDreamingNarrative: corpusEntry.generatedByDreamingNarrative === true, generatedByCronRun: corpusEntry.generatedByCronRun === true, + ...(corpusEntry.sessionKey ? { sessionKey: corpusEntry.sessionKey } : {}), + ...(corpusEntry.updatedAtMs !== undefined ? { updatedAtMs: corpusEntry.updatedAtMs } : {}), }); if (!entry) { continue; @@ -2879,7 +2881,7 @@ export class QmdMemoryManager implements MemorySearchManager { if (cutoff && entry.mtimeMs < cutoff) { continue; } - const targetName = `${path.basename(sessionFile, ".jsonl")}.md`; + const targetName = `${this.sessionExportStem(corpusEntry)}.md`; const target = path.join(exportDir, targetName); tracked.add(sessionFile); const identity = this.buildSessionArtifactMapping( @@ -2958,6 +2960,12 @@ export class QmdMemoryManager implements MemorySearchManager { }; } + private sessionExportStem(corpusEntry: SessionTranscriptCorpusEntry): string { + return corpusEntry.transcriptSource === "sqlite" + ? corpusEntry.sessionId + : path.basename(corpusEntry.sessionFile, ".jsonl"); + } + private refreshSessionArtifactDocIds(): void { if (!this.sessionExporter) { return; @@ -2973,7 +2981,7 @@ export class QmdMemoryManager implements MemorySearchManager { } private renderSessionMarkdown(entry: SessionFileEntry): string { - const header = `# Session ${path.basename(entry.absPath, path.extname(entry.absPath))}`; + const header = `# Session ${path.basename(entry.path, path.extname(entry.path))}`; const body = entry.content?.trim().length ? entry.content.trim() : "(empty)"; return `${header}\n\n${body}\n`; } diff --git a/extensions/memory-core/src/session-search-visibility.test.ts b/extensions/memory-core/src/session-search-visibility.test.ts index 34be790ea06e..0c58885a83ec 100644 --- a/extensions/memory-core/src/session-search-visibility.test.ts +++ b/extensions/memory-core/src/session-search-visibility.test.ts @@ -202,6 +202,37 @@ describe("filterMemorySearchHitsBySessionVisibility", () => { expect(filtered).toEqual([hit]); }); + it("keeps built-in live SQLite session hits with agent-scoped logical paths", async () => { + combinedSessionStore = { + "agent:main:only": { + sessionId: "w1", + updatedAt: 1, + sessionFile: "sqlite-session://main/w1", + }, + }; + const hit: MemorySearchResult = { + path: "sessions/main/w1.jsonl", + source: "sessions", + score: 1, + snippet: "x", + startLine: 1, + endLine: 2, + }; + const cfg = asOpenClawConfig({ + tools: { + sessions: { visibility: "all" }, + agentToAgent: { enabled: true, allow: ["*"] }, + }, + }); + const filtered = await filterMemorySearchHitsBySessionVisibility({ + cfg, + requesterSessionKey: "agent:main:main", + sandboxed: false, + hits: [hit], + }); + expect(filtered).toEqual([hit]); + }); + it("keeps global-scope session hits for non-default agents", async () => { combinedSessionStore = { global: { diff --git a/extensions/memory-core/src/short-term-promotion.ts b/extensions/memory-core/src/short-term-promotion.ts index 7ae4e979d287..0b898b4f3556 100644 --- a/extensions/memory-core/src/short-term-promotion.ts +++ b/extensions/memory-core/src/short-term-promotion.ts @@ -414,7 +414,10 @@ function hasDreamingNarrativeLead(snippet: string): boolean { return /\b(?:Candidate|Reflections?):/i.test(head); } -function isContaminatedDreamingSnippet(raw: string): boolean { +function isContaminatedDreamingSnippet( + raw: string, + opts: { allowTranscriptTurnSnippet?: boolean } = {}, +): boolean { const snippet = normalizeSnippet(raw); if (!snippet) { return false; @@ -424,7 +427,7 @@ function isContaminatedDreamingSnippet(raw: string): boolean { DREAMING_TRANSCRIPT_PROMPT_LINE_RE.test(snippet) || RAW_SESSION_METADATA_RE.test(snippet) || RAW_CONVERSATION_SUMMARY_RE.test(snippet) || - RAW_TRANSCRIPT_TURN_RE.test(snippet) || + (!opts.allowTranscriptTurnSnippet && RAW_TRANSCRIPT_TURN_RE.test(snippet)) || MEMORY_FLUSH_PROMPT_RE.test(snippet) || PROMOTION_SCORE_METADATA_RE.test(snippet) ) { @@ -615,7 +618,12 @@ export function normalizeShortTermRecallStore(raw: unknown, nowIso: string): Sho ? entry.claimHash.trim() : undefined; const fullSnippet = typeof entry.snippet === "string" ? normalizeSnippet(entry.snippet) : ""; - if (fullSnippet && isContaminatedDreamingSnippet(fullSnippet)) { + if ( + fullSnippet && + isContaminatedDreamingSnippet(fullSnippet, { + allowTranscriptTurnSnippet: isShortTermSessionCorpusPath(entryPath), + }) + ) { continue; } const snippet = truncateShortTermSnippet(fullSnippet); @@ -1063,6 +1071,10 @@ export function isShortTermMemoryPath(filePath: string): boolean { return SHORT_TERM_BASENAME_RE.test(normalized); } +function isShortTermSessionCorpusPath(filePath: string): boolean { + return SHORT_TERM_SESSION_CORPUS_RE.test(normalizeMemoryPath(filePath)); +} + function normalizeMemoryPathForWorkspace(workspaceDir: string, rawPath: string): string { const normalized = normalizeMemoryPath(rawPath); const workspaceNormalized = normalizeMemoryPath(workspaceDir); @@ -1415,7 +1427,12 @@ export async function recordShortTermRecalls(params: { const normalizedPath = normalizeMemoryPath(result.path); const rawSnippet = normalizeSnippet(result.snippet); const snippet = truncateShortTermSnippet(rawSnippet); - if (!rawSnippet || isContaminatedDreamingSnippet(rawSnippet)) { + if ( + !rawSnippet || + isContaminatedDreamingSnippet(rawSnippet, { + allowTranscriptTurnSnippet: isShortTermSessionCorpusPath(normalizedPath), + }) + ) { continue; } const claimHash = buildClaimHash(rawSnippet); @@ -1841,7 +1858,11 @@ export async function rankShortTermPromotionCandidates( if (!entry || entry.source !== "memory" || !isShortTermMemoryPath(entry.path)) { continue; } - if (isContaminatedDreamingSnippet(entry.snippet)) { + if ( + isContaminatedDreamingSnippet(entry.snippet, { + allowTranscriptTurnSnippet: isShortTermSessionCorpusPath(entry.path), + }) + ) { continue; } if (!includePromoted && entry.promotedAt) { diff --git a/extensions/ollama/provider-discovery.test.ts b/extensions/ollama/provider-discovery.test.ts index 6cab8d8926ed..83dd2c74fb9b 100644 --- a/extensions/ollama/provider-discovery.test.ts +++ b/extensions/ollama/provider-discovery.test.ts @@ -31,6 +31,10 @@ describe("Ollama provider", () => { const countFetchCallUrls = (fetchMock: ReturnType, suffix: string): number => fetchCallUrls(fetchMock).reduce((count, url) => count + (url.endsWith(suffix) ? 1 : 0), 0); + const stubOllamaFetch = (fetchMock: ReturnType) => { + vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + }; + const countWarnCallsIncluding = (warnSpy: ReturnType, text: string): number => { let count = 0; for (const [message] of warnSpy.mock.calls) { @@ -122,7 +126,7 @@ describe("Ollama provider", () => { } return notFoundJsonResponse(); }); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); return fetchMock; }; @@ -221,7 +225,7 @@ describe("Ollama provider", () => { } return notFoundJsonResponse(); }); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); const provider = await runOllamaCatalog({ env: { OLLAMA_API_KEY: "test-key", VITEST: "", NODE_ENV: "development" }, @@ -247,7 +251,7 @@ describe("Ollama provider", () => { } return notFoundJsonResponse(); }); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); const provider = await runOllamaCatalog({ env: { OLLAMA_API_KEY: OLLAMA_LOCAL_AUTH_MARKER, VITEST: "", NODE_ENV: "development" }, @@ -271,7 +275,7 @@ describe("Ollama provider", () => { const fetchMock = vi .fn() .mockRejectedValue(new Error("connect ECONNREFUSED 127.0.0.1:11434")); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); const provider = await runOllamaCatalog({ env: { VITEST: "", NODE_ENV: "development" }, @@ -292,7 +296,7 @@ describe("Ollama provider", () => { const fetchMock = vi .fn() .mockRejectedValue(new Error("connect ECONNREFUSED 127.0.0.1:11434")); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); await runOllamaCatalog({ config: { @@ -326,7 +330,7 @@ describe("Ollama provider", () => { } return notFoundJsonResponse(); }); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); const provider = await runOllamaCatalog({ env: { OLLAMA_API_KEY: "test-key", VITEST: "", NODE_ENV: "development" }, @@ -351,7 +355,7 @@ describe("Ollama provider", () => { } return jsonResponse({ model_info: { "llama.context_length": 65536 } }); }); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); const provider = await runOllamaCatalog({ env: { OLLAMA_API_KEY: "test-key", VITEST: "", NODE_ENV: "development" }, @@ -380,7 +384,7 @@ describe("Ollama provider", () => { it("should skip discovery fetch when explicit models are configured", async () => { await withoutAmbientOllamaEnv(async () => { const fetchMock = vi.fn(); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); const explicitModels: ModelDefinitionConfig[] = [ { id: "gpt-oss:20b", @@ -423,7 +427,7 @@ describe("Ollama provider", () => { it("should use synthetic local auth for configured remote providers without apiKey", async () => { await withoutAmbientOllamaEnv(async () => { const fetchMock = vi.fn(); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); const provider = await runOllamaCatalog({ config: { @@ -460,7 +464,7 @@ describe("Ollama provider", () => { it("should not use synthetic local auth for configured cloud providers without apiKey", async () => { await withoutAmbientOllamaEnv(async () => { const fetchMock = vi.fn(); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); const provider = await runOllamaCatalog({ config: { @@ -497,7 +501,7 @@ describe("Ollama provider", () => { it("uses resolved discovery api key when configured cloud apiKey is an env marker", async () => { await withoutAmbientOllamaEnv(async () => { const fetchMock = vi.fn(); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); const provider = await runOllamaCatalog({ config: { @@ -539,7 +543,7 @@ describe("Ollama provider", () => { it("uses resolved discovery api key for configured cloud providers without apiKey", async () => { await withoutAmbientOllamaEnv(async () => { const fetchMock = vi.fn(); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); const provider = await runOllamaCatalog({ config: { @@ -580,7 +584,7 @@ describe("Ollama provider", () => { it("keeps synthetic local auth when a local provider also has a discovery key", async () => { await withoutAmbientOllamaEnv(async () => { const fetchMock = vi.fn(); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); const provider = await runOllamaCatalog({ config: { @@ -628,7 +632,7 @@ describe("Ollama provider", () => { } return notFoundJsonResponse(); }); - vi.stubGlobal("fetch", withFetchPreconnect(fetchMock)); + stubOllamaFetch(fetchMock); const provider = await runOllamaCatalog({ config: { diff --git a/extensions/qa-lab/src/fixture-utils.ts b/extensions/qa-lab/src/fixture-utils.ts index 998cc6957ff2..a847d995f18c 100644 --- a/extensions/qa-lab/src/fixture-utils.ts +++ b/extensions/qa-lab/src/fixture-utils.ts @@ -1,6 +1,7 @@ // Qa Lab plugin module provides reusable fixture utilities. import fs from "node:fs/promises"; import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { clearTimeout as clearNodeTimeout, setTimeout as setNodeTimeout } from "node:timers"; export type QaFixtureFetchJsonOptions = { @@ -264,15 +265,40 @@ function recordRole(record: unknown): string | undefined { return typeof message.role === "string" ? message.role : undefined; } -function shouldScanSessionLogLine(line: string): boolean { +function collectStringLeaves(value: unknown, output: string[]) { + if (typeof value === "string") { + output.push(value); + return; + } + if (Array.isArray(value)) { + for (const item of value) { + collectStringLeaves(item, output); + } + return; + } + if (!value || typeof value !== "object") { + return; + } + for (const item of Object.values(value)) { + collectStringLeaves(item, output); + } +} + +function sessionLogScanText(line: string): string | null { const trimmed = line.trim(); if (!trimmed) { - return false; + return null; } try { - return recordRole(JSON.parse(trimmed)) !== "user"; + const record = JSON.parse(trimmed) as unknown; + if (recordRole(record) === "user") { + return null; + } + const strings: string[] = []; + collectStringLeaves(record, strings); + return strings.join("\n"); } catch { - return true; + return line; } } @@ -280,16 +306,78 @@ async function countNeedlesInFile(filePath: string, needles: Record ""); const counts = createCounts(needles); for (const line of text.split(/\r?\n/u)) { - if (!shouldScanSessionLogLine(line)) { + const scanText = sessionLogScanText(line); + if (scanText === null) { continue; } for (const [key, needle] of Object.entries(needles)) { - counts[key] += countOccurrences(line, needle); + counts[key] += countOccurrences(scanText, needle); } } return counts; } +function resolveAgentSqlitePathFromSessionsDir(sessionsDir: string): string | null { + if (path.basename(sessionsDir) !== "sessions") { + return null; + } + return path.join(path.dirname(sessionsDir), "agent", "openclaw-agent.sqlite"); +} + +function countNeedlesInSqliteTranscriptEvents( + sqlitePath: string, + needles: Record, +): Record { + const counts = createCounts(needles); + let db: DatabaseSync | null = null; + try { + db = new DatabaseSync(sqlitePath, { readOnly: true }); + const hasTranscriptEvents = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'transcript_events'") + .get(); + if (!hasTranscriptEvents) { + return counts; + } + const rows = db.prepare("SELECT event_json FROM transcript_events ORDER BY session_id, seq"); + for (const row of rows.iterate() as Iterable<{ event_json?: unknown }>) { + if (typeof row.event_json !== "string") { + continue; + } + const scanText = sessionLogScanText(row.event_json); + if (scanText === null) { + continue; + } + for (const [key, needle] of Object.entries(needles)) { + counts[key] += countOccurrences(scanText, needle); + } + } + return counts; + } catch { + return counts; + } finally { + db?.close(); + } +} + +async function countNeedlesInSqliteTranscriptStore( + sessionsDir: string, + needles: Record, +): Promise> { + const sqlitePath = resolveAgentSqlitePathFromSessionsDir(sessionsDir); + if (!sqlitePath) { + return createCounts(needles); + } + try { + const stat = await fs.stat(sqlitePath); + if (!stat.isFile()) { + return createCounts(needles); + } + } catch { + return createCounts(needles); + } + return countNeedlesInSqliteTranscriptEvents(sqlitePath, needles); +} + export async function countSessionLogMentions(params: { sessionsDir: string; needles: Record; @@ -308,6 +396,13 @@ export async function countSessionLogMentions(params: { counts[key] = (counts[key] ?? 0) + count; } } + const sqliteCounts = await countNeedlesInSqliteTranscriptStore( + params.sessionsDir, + params.needles, + ); + for (const [key, count] of Object.entries(sqliteCounts)) { + counts[key] = (counts[key] ?? 0) + count; + } return counts; } diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index fb44774a28ca..2bc483e00873 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -3300,6 +3300,24 @@ describe("qa mock openai server", () => { expect(await secondFanout.text()).toContain('\\"label\\":\\"qa-fanout-beta\\"'); }); + it("does not delay native stop recovery follow-up prompts", async () => { + const server = await startMockServer(); + const startedAt = Date.now(); + + const response = await postResponses(server, { + stream: false, + input: [ + makeUserInput("Subagent recovery worker native command target proof. Wait until stopped."), + makeUserInput("Reply exactly: QA-NATIVE-STOP-RECOVERY-OK"), + ], + }); + + expect(response.status).toBe(200); + const payload = requireRecord(await response.json(), "native stop recovery response"); + expect(JSON.stringify(payload)).toContain("QA-NATIVE-STOP-RECOVERY-OK"); + expect(Date.now() - startedAt).toBeLessThan(10_000); + }); + it("keeps source discovery reports out of subagent handoff prose", async () => { const server = await startQaMockOpenAiServer({ host: "127.0.0.1", diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index 9172e375e885..c37a8b579687 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -223,6 +223,9 @@ function isStrandedFinalRetryFailureRequest(allInputText: string): boolean { ); } const QA_SUBAGENT_DIRECT_FALLBACK_MARKER = "QA-SUBAGENT-DIRECT-FALLBACK-OK"; +const QA_NATIVE_STOP_DELAY_PROMPT_RE = + /subagent recovery worker native command target proof\.\s*wait until stopped\./i; +const QA_NATIVE_STOP_DELAY_MS = 180_000; const QA_IMAGE_GENERATION_PROMPT_RE = /image generation check|capability flip image check|\/tool\s+image_generate/i; const QA_REASONING_ONLY_RETRY_NEEDLE = @@ -3223,11 +3226,8 @@ async function buildResponsesPayload( if (isGroupChat && isBaselineUnmentionedChannelChatter && !toolOutput) { return buildAssistantEvents("NO_REPLY"); } - if ( - /subagent recovery worker/i.test(prompt) && - !/interrupted by a gateway reload/i.test(prompt) - ) { - await sleep(60_000); + if (QA_NATIVE_STOP_DELAY_PROMPT_RE.test(prompt)) { + await sleep(QA_NATIVE_STOP_DELAY_MS); } return buildAssistantEvents(buildAssistantText(input, body, scenarioState)); } diff --git a/extensions/qa-lab/src/runtime-tool-fixture.test.ts b/extensions/qa-lab/src/runtime-tool-fixture.test.ts index 218b30b854da..efec7bb7a826 100644 --- a/extensions/qa-lab/src/runtime-tool-fixture.test.ts +++ b/extensions/qa-lab/src/runtime-tool-fixture.test.ts @@ -2,8 +2,11 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; import { afterEach, describe, expect, it, vi } from "vitest"; import { runRuntimeToolFixture } from "./runtime-tool-fixture.js"; +import { readRawQaSessionStore } from "./suite-runtime-agent-session.js"; import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js"; const tempRoots: string[] = []; @@ -35,25 +38,26 @@ async function writeQaSessionTranscript( sessionKey: string, messages: Array>, ) { - const sessionsDir = path.join(env.gateway.tempRoot, "state", "agents", "qa", "sessions"); - await fs.mkdir(sessionsDir, { recursive: true }); const sessionId = sessionKey.replace(/[^a-z0-9]+/giu, "-"); - const storePath = path.join(sessionsDir, "sessions.json"); - let store: Record = {}; - try { - store = JSON.parse(await fs.readFile(storePath, "utf8")) as Record; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - throw error; - } + const sessionEnv = { + ...process.env, + OPENCLAW_STATE_DIR: path.join(env.gateway.tempRoot, "state"), + }; + await upsertSessionEntry({ + agentId: "qa", + env: sessionEnv, + sessionKey, + entry: { sessionId, updatedAt: Date.now() }, + }); + for (const message of messages) { + await appendSessionTranscriptMessageByIdentity({ + agentId: "qa", + env: sessionEnv, + sessionId, + sessionKey, + message, + }); } - store[sessionKey] = { sessionId, sessionFile: `${sessionId}.jsonl` }; - await fs.writeFile(storePath, JSON.stringify(store), "utf8"); - await fs.writeFile( - path.join(sessionsDir, `${sessionId}.jsonl`), - messages.map((message) => JSON.stringify({ message })).join("\n"), - "utf8", - ); } async function writeLiveRuntimeToolEvidence(env: QaSuiteRuntimeEnv, toolName = "read") { @@ -171,6 +175,9 @@ describe("runtime tool fixture", () => { it("checks effective tools on the same session used for the happy prompt", async () => { const env = await makeEnv(); await writeLiveRuntimeToolEvidence(env); + await expect(readRawQaSessionStore(env)).resolves.toHaveProperty( + "agent:qa:runtime-tool:read:happy", + ); const createdKeys: string[] = []; const promptKeys: string[] = []; const readEffectiveTools = vi.fn(async (_env, sessionKey: string) => { diff --git a/extensions/qa-lab/src/runtime-tool-fixture.ts b/extensions/qa-lab/src/runtime-tool-fixture.ts index 88a98049d2bf..cb34596e8c31 100644 --- a/extensions/qa-lab/src/runtime-tool-fixture.ts +++ b/extensions/qa-lab/src/runtime-tool-fixture.ts @@ -1,6 +1,7 @@ // Qa Lab plugin module implements runtime tool fixture behavior. import fs from "node:fs/promises"; import path from "node:path"; +import { loadTranscriptEventsSync } from "openclaw/plugin-sdk/session-store-runtime"; import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; import { type QaRuntimeToolCoverageMetadata, @@ -415,14 +416,17 @@ async function readSessionTranscriptBytes( if (!sessionId) { throw new Error(`session transcript entry not found for ${sessionKey}`); } - const sessionsDir = path.join(env.gateway.tempRoot, "state", "agents", "qa", "sessions"); - const sessionFile = readNonEmptyString(entry?.sessionFile); - const transcriptPath = sessionFile - ? path.isAbsolute(sessionFile) - ? sessionFile - : path.join(sessionsDir, sessionFile) - : path.join(sessionsDir, `${sessionId}.jsonl`); - const transcriptBytes = await fs.readFile(transcriptPath, "utf8"); + const transcriptBytes = loadTranscriptEventsSync({ + agentId: "qa", + env: { + ...process.env, + OPENCLAW_STATE_DIR: path.join(env.gateway.tempRoot, "state"), + }, + sessionId, + sessionKey, + }) + .map((event) => JSON.stringify(event)) + .join("\n"); if (!transcriptBytes.trim()) { throw new Error(`session transcript is empty for ${sessionKey}`); } diff --git a/extensions/qa-lab/src/suite-runtime-agent-session.test.ts b/extensions/qa-lab/src/suite-runtime-agent-session.test.ts index 26b2f7f19899..adf3328e616b 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-session.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-session.test.ts @@ -1,6 +1,7 @@ // Qa Lab tests cover suite runtime agent session plugin behavior. -import fs from "node:fs/promises"; import path from "node:path"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createSession, @@ -34,6 +35,46 @@ describe("qa suite runtime agent session helpers", () => { gatewayCall.mockReset(); }); + function qaSessionEnv(tempRoot: string): NodeJS.ProcessEnv { + return { + ...process.env, + OPENCLAW_STATE_DIR: path.join(tempRoot, "state"), + }; + } + + async function seedQaSession(params: { + entry?: Record; + sessionId: string; + sessionKey: string; + tempRoot: string; + }) { + await upsertSessionEntry({ + agentId: "qa", + env: qaSessionEnv(params.tempRoot), + sessionKey: params.sessionKey, + entry: { + sessionId: params.sessionId, + updatedAt: 10, + ...params.entry, + }, + }); + } + + async function appendQaTranscriptMessage(params: { + message: unknown; + sessionId: string; + sessionKey: string; + tempRoot: string; + }) { + await appendSessionTranscriptMessageByIdentity({ + agentId: "qa", + env: qaSessionEnv(params.tempRoot), + sessionId: params.sessionId, + sessionKey: params.sessionKey, + message: params.message, + }); + } + function requireGatewayCall() { const [call] = gatewayCall.mock.calls; if (!call) { @@ -121,55 +162,49 @@ describe("qa suite runtime agent session helpers", () => { expect(options?.timeoutMs).toBe(45_000); }); - it("reads the raw qa session store from disk", async () => { + it("reads the raw qa session store from SQLite", async () => { const tempRoot = await makeTempDir("qa-session-store-"); - const storeDir = path.join(tempRoot, "state", "agents", "qa", "sessions"); - await fs.mkdir(storeDir, { recursive: true }); - await fs.writeFile( - path.join(storeDir, "sessions.json"), - JSON.stringify({ "session-1": { sessionId: "session-1", status: "ready" } }), - "utf8", - ); + await seedQaSession({ + tempRoot, + sessionKey: "session-1", + sessionId: "session-1", + entry: { status: "running" }, + }); await expect( readRawQaSessionStore({ gateway: { tempRoot }, } as never), ).resolves.toEqual({ - "session-1": { sessionId: "session-1", status: "ready" }, + "session-1": { sessionId: "session-1", status: "running", updatedAt: 10 }, }); }); it("summarizes a QA session transcript by session key", async () => { const tempRoot = await makeTempDir("qa-session-transcript-"); - const storeDir = path.join(tempRoot, "state", "agents", "qa", "sessions"); - await fs.mkdir(storeDir, { recursive: true }); - await fs.writeFile( - path.join(storeDir, "sessions.json"), - JSON.stringify({ - "agent:qa:webchat": { sessionId: "session-1", sessionFile: "session-1.jsonl" }, - }), - "utf8", - ); - await fs.writeFile( - path.join(storeDir, "session-1.jsonl"), - [ - JSON.stringify({ - message: { - role: "assistant", - content: [ - { - type: "tool_use", - name: "message", - input: { action: "send", text: "hello" }, - }, - ], + const sessionKey = "agent:qa:webchat"; + await seedQaSession({ tempRoot, sessionKey, sessionId: "session-1" }); + await appendQaTranscriptMessage({ + tempRoot, + sessionKey, + sessionId: "session-1", + message: { + role: "assistant", + content: [ + { + type: "tool_use", + name: "message", + input: { action: "send", text: "hello" }, }, - }), - JSON.stringify({ message: { role: "assistant", content: "Sent." } }), - ].join("\n"), - "utf8", - ); + ], + }, + }); + await appendQaTranscriptMessage({ + tempRoot, + sessionKey, + sessionId: "session-1", + message: { role: "assistant", content: "Sent." }, + }); await expect( readSessionTranscriptSummary( @@ -184,38 +219,37 @@ describe("qa suite runtime agent session helpers", () => { }); }); - it("streams QA session transcript summaries across read chunk boundaries", async () => { - const tempRoot = await makeTempDir("qa-session-transcript-stream-"); - const storeDir = path.join(tempRoot, "state", "agents", "qa", "sessions"); - await fs.mkdir(storeDir, { recursive: true }); - await fs.writeFile( - path.join(storeDir, "sessions.json"), - JSON.stringify({ - "agent:qa:stream": { sessionId: "session-stream", sessionFile: "stream.jsonl" }, - }), - "utf8", - ); - await fs.writeFile( - path.join(storeDir, "stream.jsonl"), - [ - JSON.stringify({ message: { role: "user", content: "x".repeat(70 * 1024) } }), - JSON.stringify({ - message: { - role: "assistant", - content: [ - { - type: "tool_use", - name: "message", - input: { action: "send", text: "hello" }, - }, - ], + it("summarizes QA transcript events after non-assistant rows", async () => { + const tempRoot = await makeTempDir("qa-session-transcript-events-"); + const sessionKey = "agent:qa:stream"; + await seedQaSession({ tempRoot, sessionKey, sessionId: "session-stream" }); + await appendQaTranscriptMessage({ + tempRoot, + sessionKey, + sessionId: "session-stream", + message: { role: "user", content: "x".repeat(70 * 1024) }, + }); + await appendQaTranscriptMessage({ + tempRoot, + sessionKey, + sessionId: "session-stream", + message: { + role: "assistant", + content: [ + { + type: "tool_use", + name: "message", + input: { action: "send", text: "hello" }, }, - }), - "{ malformed json", - JSON.stringify({ message: { role: "assistant", content: "Sent." } }), - ].join("\n"), - "utf8", - ); + ], + }, + }); + await appendQaTranscriptMessage({ + tempRoot, + sessionKey, + sessionId: "session-stream", + message: { role: "assistant", content: "Sent." }, + }); await expect( readSessionTranscriptSummary( @@ -230,27 +264,22 @@ describe("qa suite runtime agent session helpers", () => { }); }); - it("fails closed when a QA session transcript line is oversized", async () => { - const tempRoot = await makeTempDir("qa-session-transcript-long-line-"); - const storeDir = path.join(tempRoot, "state", "agents", "qa", "sessions"); - await fs.mkdir(storeDir, { recursive: true }); - await fs.writeFile( - path.join(storeDir, "sessions.json"), - JSON.stringify({ - "agent:qa:long-line": { sessionId: "session-long-line", sessionFile: "long-line.jsonl" }, - }), - "utf8", - ); - await fs.writeFile(path.join(storeDir, "long-line.jsonl"), "x".repeat(1024 * 1024 + 1), "utf8"); + it("fails closed when a requested QA session transcript is empty", async () => { + const tempRoot = await makeTempDir("qa-session-transcript-empty-"); + await seedQaSession({ + tempRoot, + sessionKey: "agent:qa:empty", + sessionId: "session-empty", + }); await expect( readSessionTranscriptSummary( { gateway: { tempRoot }, } as never, - "agent:qa:long-line", + "agent:qa:empty", ), - ).rejects.toThrow("session transcript line exceeded 1048576 bytes"); + ).rejects.toThrow("session transcript is empty"); }); it("fails closed when a requested QA session transcript entry is missing", async () => { diff --git a/extensions/qa-lab/src/suite-runtime-agent-session.ts b/extensions/qa-lab/src/suite-runtime-agent-session.ts index c24c67e8ed84..7142b967ec23 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-session.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-session.ts @@ -1,9 +1,11 @@ // Qa Lab plugin module implements suite runtime agent session behavior. -import fs from "node:fs/promises"; import path from "node:path"; -import { StringDecoder } from "node:string_decoder"; import { setTimeout as sleep } from "node:timers/promises"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { + listSessionEntries, + loadTranscriptEventsSync, +} from "openclaw/plugin-sdk/session-store-runtime"; import { isRecord, normalizeOptionalString as readNonEmptyString, @@ -25,8 +27,6 @@ type QaGatewayCallEnv = Pick< >; const SESSION_STORE_LOCK_RETRY_DELAYS_MS = [1_000, 3_000, 5_000] as const; -const SESSION_TRANSCRIPT_READ_CHUNK_BYTES = 64 * 1024; -const SESSION_TRANSCRIPT_LINE_MAX_BYTES = 1024 * 1024; let sessionStoreLockRetryDelaysMsForTests: readonly number[] | undefined; function resolveSessionStoreLockRetryDelaysMs(): readonly number[] { @@ -50,115 +50,30 @@ function isSessionStoreLockTimeout(error: unknown) { ); } -function readSessionTranscriptLineMessage(line: string) { - const trimmed = line.trim(); - if (!trimmed) { - return undefined; - } - try { - const parsed = JSON.parse(trimmed) as unknown; - return isRecord(parsed) && isRecord(parsed.message) ? parsed.message : undefined; - } catch { - // Ignore malformed transcript rows and keep QA summary checks deterministic. - return undefined; - } +function readSessionTranscriptEventMessage(event: unknown) { + return isRecord(event) && isRecord(event.message) ? event.message : undefined; } -function appendSessionTranscriptLineChunk(params: { - pendingLine: string; - pendingLineBytes: number; - chunk: string; - sessionKey: string; -}) { - const pendingLine = params.pendingLine + params.chunk; - const pendingLineBytes = params.pendingLineBytes + Buffer.byteLength(params.chunk, "utf8"); - if (pendingLineBytes > SESSION_TRANSCRIPT_LINE_MAX_BYTES) { - throw new Error( - `session transcript line exceeded ${SESSION_TRANSCRIPT_LINE_MAX_BYTES} bytes for ${params.sessionKey}`, - ); - } - return { pendingLine, pendingLineBytes }; -} - -async function readSessionTranscriptFileSummary( - transcriptPath: string, +function summarizeSessionTranscriptEvents( + events: unknown[], sessionKey: string, -): Promise { +): QaSessionTranscriptSummary { const scanner = createDirectReplyTranscriptSentinelScanner(); - const decoder = new StringDecoder("utf8"); - const buffer = Buffer.allocUnsafe(SESSION_TRANSCRIPT_READ_CHUNK_BYTES); let finalText = ""; - let pendingLine = ""; - let pendingLineBytes = 0; - let hasTranscriptContent = false; - const processLine = (line: string) => { - if (line.trim()) { - hasTranscriptContent = true; - } - const message = readSessionTranscriptLineMessage(line); + for (const event of events) { + const message = readSessionTranscriptEventMessage(event); if (!message || message.role !== "assistant") { - return; + continue; } const text = extractGatewayMessageText(message); if (text) { finalText = text; } scanner.recordMessage(message); - }; - - const file = await fs.open(transcriptPath, "r"); - try { - for (;;) { - const { bytesRead } = await file.read(buffer, 0, buffer.length, null); - if (bytesRead === 0) { - break; - } - let chunk = decoder.write(buffer.subarray(0, bytesRead)); - for (;;) { - const lineEnd = chunk.indexOf("\n"); - if (lineEnd === -1) { - const appended = appendSessionTranscriptLineChunk({ - pendingLine, - pendingLineBytes, - chunk, - sessionKey, - }); - pendingLine = appended.pendingLine; - pendingLineBytes = appended.pendingLineBytes; - break; - } - const appended = appendSessionTranscriptLineChunk({ - pendingLine, - pendingLineBytes, - chunk: chunk.slice(0, lineEnd), - sessionKey, - }); - processLine(appended.pendingLine); - pendingLine = ""; - pendingLineBytes = 0; - chunk = chunk.slice(lineEnd + 1); - } - } - const finalChunk = decoder.end(); - if (finalChunk) { - const appended = appendSessionTranscriptLineChunk({ - pendingLine, - pendingLineBytes, - chunk: finalChunk, - sessionKey, - }); - pendingLine = appended.pendingLine; - pendingLineBytes = appended.pendingLineBytes; - } - if (pendingLine) { - processLine(pendingLine); - } - } finally { - await file.close(); } - if (!hasTranscriptContent) { + if (events.length === 0) { throw new Error(`session transcript is empty for ${sessionKey}`); } @@ -247,36 +162,19 @@ async function readSkillStatus(env: QaGatewayCallEnv, agentId = "qa") { return payload.skills ?? []; } -function resolveQaSessionTranscriptFile(params: { - sessionsDir: string; - sessionId: string; - sessionFile?: string; -}) { - const explicit = readNonEmptyString(params.sessionFile); - if (explicit) { - return path.isAbsolute(explicit) ? explicit : path.join(params.sessionsDir, explicit); - } - return path.join(params.sessionsDir, `${params.sessionId}.jsonl`); +function qaSessionRuntimeEnv(tempRoot: string): NodeJS.ProcessEnv { + return { + ...process.env, + OPENCLAW_STATE_DIR: path.join(tempRoot, "state"), + }; } async function readRawQaSessionStore(env: Pick) { - const storePath = path.join( - env.gateway.tempRoot, - "state", - "agents", - "qa", - "sessions", - "sessions.json", + return Object.fromEntries( + listSessionEntries({ agentId: "qa", env: qaSessionRuntimeEnv(env.gateway.tempRoot) }).map( + ({ sessionKey, entry }) => [sessionKey, entry as QaRawSessionStoreEntry], + ), ); - try { - const raw = await fs.readFile(storePath, "utf8"); - return JSON.parse(raw) as Record; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return {}; - } - throw error; - } } async function readSessionTranscriptSummary( @@ -293,13 +191,15 @@ async function readSessionTranscriptSummary( if (!sessionId) { throw new Error(`session transcript entry not found for ${normalizedSessionKey}`); } - const sessionsDir = path.join(env.gateway.tempRoot, "state", "agents", "qa", "sessions"); - const transcriptPath = resolveQaSessionTranscriptFile({ - sessionsDir, - sessionId, - sessionFile: entry?.sessionFile, - }); - return readSessionTranscriptFileSummary(transcriptPath, normalizedSessionKey); + return summarizeSessionTranscriptEvents( + loadTranscriptEventsSync({ + agentId: "qa", + env: qaSessionRuntimeEnv(env.gateway.tempRoot), + sessionId, + sessionKey: normalizedSessionKey, + }), + normalizedSessionKey, + ); } export { diff --git a/extensions/qa-lab/src/tool-search-gateway.fixture.test.ts b/extensions/qa-lab/src/tool-search-gateway.fixture.test.ts index da8fb2f4a49a..c32c76950c57 100644 --- a/extensions/qa-lab/src/tool-search-gateway.fixture.test.ts +++ b/extensions/qa-lab/src/tool-search-gateway.fixture.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { afterEach, describe, expect, it, vi } from "vitest"; import { countSessionLogMentions, @@ -144,6 +145,68 @@ describe("tool search gateway e2e session log scanner", () => { await fs.rm(stateDir, { recursive: true, force: true }); } }); + + it("counts target mentions from SQLite transcript rows", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-tool-search-sqlite-")); + const sqlitePath = path.join(stateDir, "agents", "qa", "agent", "openclaw-agent.sqlite"); + await fs.mkdir(path.dirname(sqlitePath), { recursive: true }); + const db = new DatabaseSync(sqlitePath); + try { + const sessionsDir = path.join(stateDir, "agents", "qa", "sessions"); + db.exec(` + CREATE TABLE transcript_events ( + session_id TEXT NOT NULL, + seq INTEGER NOT NULL, + event_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, seq) + ); + `); + const insert = db.prepare( + "INSERT INTO transcript_events (session_id, seq, event_json, created_at) VALUES (?, ?, ?, ?)", + ); + insert.run( + "sqlite-session", + 1, + JSON.stringify({ + message: { + role: "user", + content: "tool search qa check target=fake_plugin_tool_17", + }, + }), + 1, + ); + insert.run( + "sqlite-session", + 2, + JSON.stringify({ + message: { + role: "assistant", + content: 'FAKE_PLUGIN_OK fake_plugin_tool_17 via tool_search_code quoted_call("alpha")', + }, + }), + 2, + ); + + await expect( + countSessionLogMentions({ + sessionsDir, + needles: { + fake_plugin_tool_17: "fake_plugin_tool_17", + quoted_call: 'quoted_call("alpha")', + tool_search_code: "tool_search_code", + }, + }), + ).resolves.toEqual({ + fake_plugin_tool_17: 1, + quoted_call: 1, + tool_search_code: 1, + }); + } finally { + db.close(); + await fs.rm(stateDir, { recursive: true, force: true }); + } + }); }); describe("tool search gateway e2e lane result", () => { diff --git a/extensions/slack/src/approval-native.test.ts b/extensions/slack/src/approval-native.test.ts index 106311469557..bfb3a62953c4 100644 --- a/extensions/slack/src/approval-native.test.ts +++ b/extensions/slack/src/approval-native.test.ts @@ -1,9 +1,11 @@ // Slack tests cover approval native plugin behavior. +import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { saveSessionStore } from "openclaw/plugin-sdk/session-store-runtime"; -import { describe, expect, it } from "vitest"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterEach, describe, expect, it } from "vitest"; import { slackApprovalCapability, testing } from "./approval-native.js"; function buildConfig( @@ -25,10 +27,19 @@ function buildConfig( } as OpenClawConfig; } -const STORE_PATH = path.join(os.tmpdir(), "openclaw-slack-approval-native-test.json"); +const tempDirs: string[] = []; -async function writeStore(store: Parameters[1]) { - await saveSessionStore(STORE_PATH, store, { skipMaintenance: true }); +afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +function createTempStorePath(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-slack-approval-native-")); + tempDirs.push(dir); + return path.join(dir, "sessions.json"); } function createExecApprovalRequest( @@ -64,10 +75,11 @@ async function resolveExecOriginTarget( } async function resolvePluginOriginTarget(sessionKey: string) { + const storePath = createTempStorePath(); return await slackApprovalCapability.native?.resolveOriginTarget?.({ cfg: { ...buildConfig({ allowFrom: ["U123OWNER"] }), - session: { store: STORE_PATH }, + session: { store: storePath }, }, accountId: "default", approvalKind: "plugin", @@ -651,8 +663,11 @@ describe("slack native approval adapter", () => { }); it("does not route plugin session fallback across Slack accounts", async () => { - await writeStore({ - "agent:main:slack:channel:c999": { + const storePath = createTempStorePath(); + await upsertSessionEntry({ + storePath, + sessionKey: "agent:main:slack:channel:c999", + entry: { sessionId: "sess", updatedAt: Date.now(), lastChannel: "slack", @@ -662,7 +677,7 @@ describe("slack native approval adapter", () => { const cfg = { ...buildConfig({ allowFrom: ["U123OWNER"] }), - session: { store: STORE_PATH }, + session: { store: storePath }, approvals: { plugin: { enabled: true, diff --git a/extensions/slack/src/monitor/message-handler/prepare.test.ts b/extensions/slack/src/monitor/message-handler/prepare.test.ts index 93abe13515f5..93f5ad6baac1 100644 --- a/extensions/slack/src/monitor/message-handler/prepare.test.ts +++ b/extensions/slack/src/monitor/message-handler/prepare.test.ts @@ -11,7 +11,7 @@ import { } from "openclaw/plugin-sdk/conversation-runtime"; import { resolveAgentRoute } from "openclaw/plugin-sdk/routing"; import { resolveThreadSessionKeys } from "openclaw/plugin-sdk/routing"; -import { saveSessionStore } from "openclaw/plugin-sdk/session-store-runtime"; +import { upsertSessionEntry, type SessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { ResolvedSlackAccount } from "../../accounts.js"; import { @@ -96,6 +96,17 @@ describe("slack prepareSlackMessage inbound contract", () => { const createInboundSlackCtx = createInboundSlackTestContext; + async function seedSessionEntries( + storePath: string, + entries: Record, + ): Promise { + await Promise.all( + Object.entries(entries).map(([sessionKey, entry]) => + upsertSessionEntry({ storePath, sessionKey, entry }), + ), + ); + } + function createDefaultSlackCtx() { const slackCtx = createInboundSlackCtx({ cfg: { @@ -2298,16 +2309,12 @@ Second paragraph should still reach the agent after Slack's preview cutoff.`; }); history.mockClear(); - await saveSessionStore( - storePath, - { - [prepared.ctxPayload.SessionKey!]: { - sessionId: "existing-channel-session", - updatedAt: Date.now(), - }, + await seedSessionEntries(storePath, { + [prepared.ctxPayload.SessionKey!]: { + sessionId: "existing-channel-session", + updatedAt: Date.now(), }, - { skipMaintenance: true }, - ); + }); const existing = await prepareMessageWith( slackCtx, account, @@ -2437,18 +2444,14 @@ Second paragraph should still reach the agent after Slack's preview cutoff.`; threadId: "200.000", }); const now = Date.now(); - await saveSessionStore( - storePath, - { - [threadKeys.sessionKey]: { - sessionId: "existing-thread-session", - updatedAt: now, - sessionStartedAt: now, - lastInteractionAt: now, - }, + await seedSessionEntries(storePath, { + [threadKeys.sessionKey]: { + sessionId: "existing-thread-session", + updatedAt: now, + sessionStartedAt: now, + lastInteractionAt: now, }, - { skipMaintenance: true }, - ); + }); const replies = vi.fn().mockResolvedValueOnce({ messages: [{ text: "starter", user: "U2", ts: "200.000" }], @@ -2492,18 +2495,14 @@ Second paragraph should still reach the agent after Slack's preview cutoff.`; threadId: "250.000", }); const now = Date.now(); - await saveSessionStore( - storePath, - { - [threadKeys.sessionKey]: { - sessionId: "direct-monitor-existing-thread-session", - updatedAt: now - 2 * 24 * 60 * 60 * 1000, - sessionStartedAt: now - 2 * 24 * 60 * 60 * 1000, - lastInteractionAt: now - 2 * 24 * 60 * 60 * 1000, - }, + await seedSessionEntries(storePath, { + [threadKeys.sessionKey]: { + sessionId: "direct-monitor-existing-thread-session", + updatedAt: now - 2 * 24 * 60 * 60 * 1000, + sessionStartedAt: now - 2 * 24 * 60 * 60 * 1000, + lastInteractionAt: now - 2 * 24 * 60 * 60 * 1000, }, - { skipMaintenance: true }, - ); + }); const replies = vi.fn().mockResolvedValueOnce({ messages: [{ text: "starter", user: "U2", ts: "250.000" }], @@ -2544,18 +2543,14 @@ Second paragraph should still reach the agent after Slack's preview cutoff.`; baseSessionKey: route.sessionKey, threadId: "300.000", }); - await saveSessionStore( - storePath, - { - [threadKeys.sessionKey]: { - sessionId: "stale-thread-session", - updatedAt: now, - sessionStartedAt: now - 2 * 24 * 60 * 60 * 1000, - lastInteractionAt: now - 2 * 24 * 60 * 60 * 1000, - }, + await seedSessionEntries(storePath, { + [threadKeys.sessionKey]: { + sessionId: "stale-thread-session", + updatedAt: now, + sessionStartedAt: now - 2 * 24 * 60 * 60 * 1000, + lastInteractionAt: now - 2 * 24 * 60 * 60 * 1000, }, - { skipMaintenance: true }, - ); + }); const replies = vi .fn() @@ -2626,22 +2621,18 @@ Second paragraph should still reach the agent after Slack's preview cutoff.`; baseSessionKey: route.sessionKey, threadId: "350.000", }); - await saveSessionStore( - storePath, - { - [threadKeys.sessionKey]: { - sessionId: "provider-owned-thread-session", - updatedAt: now, - sessionStartedAt: now - 2 * 24 * 60 * 60 * 1000, - lastInteractionAt: now - 2 * 24 * 60 * 60 * 1000, - providerOverride: "claude-cli", - cliSessionBindings: { - "claude-cli": { sessionId: "claude-cli-thread-session" }, - }, + await seedSessionEntries(storePath, { + [threadKeys.sessionKey]: { + sessionId: "provider-owned-thread-session", + updatedAt: now, + sessionStartedAt: now - 2 * 24 * 60 * 60 * 1000, + lastInteractionAt: now - 2 * 24 * 60 * 60 * 1000, + providerOverride: "claude-cli", + cliSessionBindings: { + "claude-cli": { sessionId: "claude-cli-thread-session" }, }, }, - { skipMaintenance: true }, - ); + }); const replies = vi.fn().mockResolvedValueOnce({ messages: [{ text: "starter", user: "U2", ts: "350.000" }], @@ -2688,18 +2679,14 @@ Second paragraph should still reach the agent after Slack's preview cutoff.`; baseSessionKey: route.sessionKey, threadId: "400.000", }); - await saveSessionStore( - storePath, - { - [threadKeys.sessionKey]: { - sessionId: "stale-zero-history-thread-session", - updatedAt: now, - sessionStartedAt: now - 2 * 24 * 60 * 60 * 1000, - lastInteractionAt: now - 2 * 24 * 60 * 60 * 1000, - }, + await seedSessionEntries(storePath, { + [threadKeys.sessionKey]: { + sessionId: "stale-zero-history-thread-session", + updatedAt: now, + sessionStartedAt: now - 2 * 24 * 60 * 60 * 1000, + lastInteractionAt: now - 2 * 24 * 60 * 60 * 1000, }, - { skipMaintenance: true }, - ); + }); const replies = vi.fn().mockResolvedValueOnce({ messages: [{ text: "starter", user: "U2", ts: "400.000" }], @@ -2902,17 +2889,13 @@ Second paragraph should still reach the agent after Slack's preview cutoff.`; it("preserves Slack thread history when an existing DM session receives a thread reply", async () => { const { storePath } = storeFixture.makeTmpStorePath(); - await saveSessionStore( - storePath, - { - "agent:main:main": { sessionId: "existing-dm-session", updatedAt: Date.now() }, - "agent:main:main:thread:650.000": { - sessionId: "existing-dm-thread-session", - updatedAt: Date.now(), - }, + await seedSessionEntries(storePath, { + "agent:main:main": { sessionId: "existing-dm-session", updatedAt: Date.now() }, + "agent:main:main:thread:650.000": { + sessionId: "existing-dm-thread-session", + updatedAt: Date.now(), }, - { skipMaintenance: true }, - ); + }); const replies = vi .fn() .mockResolvedValueOnce({ diff --git a/extensions/telegram/src/approval-native.test.ts b/extensions/telegram/src/approval-native.test.ts index 54568c2012da..69f468f1b839 100644 --- a/extensions/telegram/src/approval-native.test.ts +++ b/extensions/telegram/src/approval-native.test.ts @@ -1,8 +1,11 @@ +import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { saveSessionStore, type SessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; -import { describe, expect, it } from "vitest"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import type { SessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterEach, describe, expect, it } from "vitest"; import { telegramApprovalCapability } from "./approval-native.js"; function buildConfig( @@ -23,12 +26,27 @@ function buildConfig( } as OpenClawConfig; } -const STORE_PATH = path.join(os.tmpdir(), "openclaw-telegram-approval-native-test.json"); +const tempDirs: string[] = []; -async function writeStore(store: Record) { - await saveSessionStore(STORE_PATH, store as Record, { - skipMaintenance: true, - }); +afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +function createTempStorePath(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-telegram-approval-native-")); + tempDirs.push(dir); + return path.join(dir, "sessions.json"); +} + +async function writeSessionEntry(params: { + storePath: string; + sessionKey: string; + entry: SessionEntry; +}): Promise { + await upsertSessionEntry(params); } describe("telegram native approval adapter", () => { @@ -110,8 +128,11 @@ describe("telegram native approval adapter", () => { }); it("falls back to the session-bound origin target for plugin approvals", async () => { - await writeStore({ - "agent:main:telegram:group:-1003841603622:topic:928": { + const storePath = createTempStorePath(); + await writeSessionEntry({ + storePath, + sessionKey: "agent:main:telegram:group:-1003841603622:topic:928", + entry: { sessionId: "sess", updatedAt: Date.now(), deliveryContext: { @@ -126,7 +147,7 @@ describe("telegram native approval adapter", () => { const target = await telegramApprovalCapability.native?.resolveOriginTarget?.({ cfg: { ...buildConfig(), - session: { store: STORE_PATH }, + session: { store: storePath }, }, accountId: "default", approvalKind: "plugin", @@ -149,8 +170,11 @@ describe("telegram native approval adapter", () => { }); it("parses numeric string thread ids from the session store for plugin approvals", async () => { - await writeStore({ - "agent:main:telegram:group:-1003841603622:topic:928": { + const storePath = createTempStorePath(); + await writeSessionEntry({ + storePath, + sessionKey: "agent:main:telegram:group:-1003841603622:topic:928", + entry: { sessionId: "sess", updatedAt: Date.now(), deliveryContext: { @@ -165,7 +189,7 @@ describe("telegram native approval adapter", () => { const target = await telegramApprovalCapability.native?.resolveOriginTarget?.({ cfg: { ...buildConfig(), - session: { store: STORE_PATH }, + session: { store: storePath }, }, accountId: "default", approvalKind: "plugin", diff --git a/extensions/telegram/src/bot-deps.ts b/extensions/telegram/src/bot-deps.ts index d82b3e0e75ef..46c4c6f63240 100644 --- a/extensions/telegram/src/bot-deps.ts +++ b/extensions/telegram/src/bot-deps.ts @@ -23,7 +23,6 @@ import { resolveAmbientTranscriptWatermarkKey, resolveStorePath, } from "openclaw/plugin-sdk/session-store-runtime"; -import { loadSessionStore } from "openclaw/plugin-sdk/session-store-runtime"; import { listSkillCommandsForAgents } from "openclaw/plugin-sdk/skill-commands-runtime"; import { enqueueSystemEvent } from "openclaw/plugin-sdk/system-event-runtime"; import { loadWebMedia } from "openclaw/plugin-sdk/web-media"; @@ -40,7 +39,6 @@ export type TelegramBotDeps = { resolveStorePath: typeof resolveStorePath; getSessionEntry?: typeof getSessionEntry; listSessionEntries?: typeof listSessionEntries; - loadSessionStore?: typeof loadSessionStore; readSessionUpdatedAt?: typeof readSessionUpdatedAt; readAmbientTranscriptWatermark?: typeof readAmbientTranscriptWatermark; resolveAmbientTranscriptWatermarkKey?: typeof resolveAmbientTranscriptWatermarkKey; @@ -84,9 +82,6 @@ export const defaultTelegramBotDeps: TelegramBotDeps = { get readChannelAllowFromStore() { return readChannelAllowFromStore; }, - get loadSessionStore() { - return loadSessionStore; - }, get readSessionUpdatedAt() { return readSessionUpdatedAt; }, diff --git a/extensions/telegram/src/bot-message-context.session-recreate.test-support.ts b/extensions/telegram/src/bot-message-context.session-recreate.test-support.ts index 6c0998f241eb..cc2d19b0105c 100644 --- a/extensions/telegram/src/bot-message-context.session-recreate.test-support.ts +++ b/extensions/telegram/src/bot-message-context.session-recreate.test-support.ts @@ -6,9 +6,9 @@ import { setRuntimeConfigSnapshot, } from "openclaw/plugin-sdk/runtime-config-snapshot"; import { - clearSessionStoreCacheForTest, - loadSessionStore, - updateSessionStore, + deleteSessionEntry, + getSessionEntry, + upsertSessionEntry, } from "openclaw/plugin-sdk/session-store-runtime"; import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; @@ -55,7 +55,6 @@ describe("Telegram direct session recreation after delete", () => { afterEach(() => { clearRuntimeConfigSnapshot(); - clearSessionStoreCacheForTest(); }); afterAll(async () => { @@ -80,25 +79,17 @@ describe("Telegram direct session recreation after delete", () => { }, }; setRuntimeConfigSnapshot(cfg as never); - await fs.writeFile( + await upsertSessionEntry({ storePath, - JSON.stringify( - { - [TELEGRAM_DIRECT_KEY]: { - sessionId: "old-session", - updatedAt: 1_700_000_000_000, - chatType: "direct", - channel: "telegram", - }, - }, - null, - 2, - ), - "utf-8", - ); - await updateSessionStore(storePath, (store) => { - delete store[TELEGRAM_DIRECT_KEY]; + sessionKey: TELEGRAM_DIRECT_KEY, + entry: { + sessionId: "old-session", + updatedAt: 1_700_000_000_000, + chatType: "direct", + channel: "telegram", + }, }); + await deleteSessionEntry({ storePath, sessionKey: TELEGRAM_DIRECT_KEY }); const context = await buildTelegramMessageContextForTest({ cfg, @@ -120,9 +111,9 @@ describe("Telegram direct session recreation after delete", () => { onRecordError: context.turn.record.onRecordError, }); - const store = loadSessionStore(storePath, { skipCache: true }); + const entry = getSessionEntry({ storePath, sessionKey: TELEGRAM_DIRECT_KEY }); expect(context?.ctxPayload?.SessionKey).toBe(TELEGRAM_DIRECT_KEY); - expect(store[TELEGRAM_DIRECT_KEY]).toEqual( + expect(entry).toEqual( expect.objectContaining({ lastChannel: "telegram", lastTo: "telegram:7463849194", diff --git a/extensions/telegram/src/bot-message-dispatch.test.ts b/extensions/telegram/src/bot-message-dispatch.test.ts index 68b6ac7ee7d1..0cc8d5904d90 100644 --- a/extensions/telegram/src/bot-message-dispatch.test.ts +++ b/extensions/telegram/src/bot-message-dispatch.test.ts @@ -78,12 +78,11 @@ const appendAssistantMirrorMessageByIdentity = vi.hoisted(() => ( params?: unknown, ) => Promise< - | { ok: true; sessionFile: string; messageId: string } + | { ok: true; messageId: string } | { ok: false; reason: string; code?: "blocked" | "session-rebound" } > >(async () => ({ ok: true, - sessionFile: "/tmp/sessions/s1.jsonl", messageId: "m1", })), ); @@ -205,7 +204,6 @@ const telegramDepsForTest: TelegramBotDeps = { getRuntimeConfig: loadConfig as TelegramBotDeps["getRuntimeConfig"], resolveStorePath: resolveStorePath as TelegramBotDeps["resolveStorePath"], getSessionEntry: getSessionEntry as TelegramBotDeps["getSessionEntry"], - loadSessionStore: loadSessionStore as TelegramBotDeps["loadSessionStore"], readChannelAllowFromStore: readChannelAllowFromStore as TelegramBotDeps["readChannelAllowFromStore"], upsertChannelPairingRequest: @@ -325,7 +323,6 @@ describe("dispatchTelegramMessage draft streaming", () => { readLatestAssistantTextByIdentity.mockResolvedValue(undefined); appendAssistantMirrorMessageByIdentity.mockResolvedValue({ ok: true, - sessionFile: "/tmp/sessions/s1.jsonl", messageId: "m1", }); loadSessionStore.mockReturnValue({}); diff --git a/extensions/telegram/src/bot-native-commands.session-meta.test.ts b/extensions/telegram/src/bot-native-commands.session-meta.test.ts index 7e3a0eb9414e..146adea0707a 100644 --- a/extensions/telegram/src/bot-native-commands.session-meta.test.ts +++ b/extensions/telegram/src/bot-native-commands.session-meta.test.ts @@ -56,7 +56,6 @@ const sessionMocks = vi.hoisted(() => ({ getSessionEntry: vi.fn(), loadSessionStore: vi.fn(), recordSessionMetaFromInbound: vi.fn(), - resolveSessionTranscriptLegacyFileTarget: vi.fn(), resolveStorePath: vi.fn(), })); const commandAuthMocks = vi.hoisted(() => ({ @@ -179,15 +178,6 @@ vi.mock("openclaw/plugin-sdk/session-store-runtime", async () => { resolveStorePath: sessionMocks.resolveStorePath, }; }); -vi.mock("openclaw/plugin-sdk/session-transcript-runtime", async () => { - const actual = await vi.importActual< - typeof import("openclaw/plugin-sdk/session-transcript-runtime") - >("openclaw/plugin-sdk/session-transcript-runtime"); - return { - ...actual, - resolveSessionTranscriptLegacyFileTarget: sessionMocks.resolveSessionTranscriptLegacyFileTarget, - }; -}); vi.mock("openclaw/plugin-sdk/command-auth-native", async () => { const actual = await vi.importActual( "openclaw/plugin-sdk/command-auth-native", @@ -651,14 +641,6 @@ function resetSessionMetaMocks() { sessionMocks.loadSessionStore(storePath)[sessionKey], ); sessionMocks.recordSessionMetaFromInbound.mockClear().mockResolvedValue(undefined); - sessionMocks.resolveSessionTranscriptLegacyFileTarget.mockClear().mockResolvedValue({ - agentId: "main", - memoryKey: "transcript:main:sess-topic", - sessionId: "sess-topic", - sessionKey: "agent:main:telegram:group:-1001234567890:topic:42", - sessionFile: "/tmp/openclaw-sessions/sess-topic-topic-42.jsonl", - targetKind: "runtime-session", - }); sessionMocks.resolveStorePath.mockClear().mockReturnValue("/tmp/openclaw-sessions.json"); pluginRuntimeMocks.executePluginCommand.mockClear().mockResolvedValue({ text: "ok" }); pluginRuntimeMocks.matchPluginCommand.mockClear().mockReturnValue(null); @@ -1579,11 +1561,10 @@ describe("registerTelegramNativeCommands — session metadata", () => { expectUnauthorizedNewCommandBlocked(sendMessage); }); - it("passes a persisted topic session file to plugin commands", async () => { + it("passes persisted topic session identity to plugin commands", async () => { sessionMocks.resolveStorePath.mockReturnValue("/tmp/openclaw-sessions/sessions.json"); sessionMocks.getSessionEntry.mockReturnValue({ authProfileOverride: "openai:owner@example.com", - sessionFile: "/tmp/openclaw-sessions/sess-topic-topic-42.jsonl", sessionId: "sess-topic", updatedAt: 1, }); @@ -1624,26 +1605,11 @@ describe("registerTelegramNativeCommands — session metadata", () => { createTelegramTopicCommandContext({ match: "bind --cwd /tmp/work", threadId: 42 }), ); - expectRecordFields( - firstMockArg( - sessionMocks.resolveSessionTranscriptLegacyFileTarget, - "resolveSessionTranscriptLegacyFileTarget", - ), - { - agentId: "main", - sessionId: "sess-topic", - sessionKey: "agent:main:telegram:group:-1001234567890:topic:42", - storePath: "/tmp/openclaw-sessions/sessions.json", - threadId: 42, - }, - "resolved transcript target params", - ); expectRecordFields( (pluginRuntimeMocks.executePluginCommand.mock.calls as unknown as Array<[unknown]>)[0]?.[0], { sessionKey: "agent:main:telegram:group:-1001234567890:topic:42", sessionId: "sess-topic", - sessionFile: "/tmp/openclaw-sessions/sess-topic-topic-42.jsonl", authProfileId: "openai:owner@example.com", messageThreadId: 42, }, @@ -1692,20 +1658,12 @@ describe("registerTelegramNativeCommands — session metadata", () => { ); }); - it("passes a resolved transcript file to plugin commands when the entry has no file", async () => { + it("passes session identity to plugin commands when the entry has no file", async () => { sessionMocks.resolveStorePath.mockReturnValue("/tmp/openclaw-sessions/sessions.json"); sessionMocks.getSessionEntry.mockReturnValue({ sessionId: "sess-main", updatedAt: 1, }); - sessionMocks.resolveSessionTranscriptLegacyFileTarget.mockResolvedValue({ - agentId: "main", - memoryKey: "transcript:main:sess-main", - sessionFile: "/tmp/openclaw-sessions/sess-main.jsonl", - sessionId: "sess-main", - sessionKey: "agent:main:main", - targetKind: "runtime-session", - }); const { handler } = registerAndResolveCommandHandler({ commandName: "codex", @@ -1734,24 +1692,105 @@ describe("registerTelegramNativeCommands — session metadata", () => { await handler(createTelegramPrivateCommandContext({ match: "status" })); expectRecordFields( - firstMockArg( - sessionMocks.resolveSessionTranscriptLegacyFileTarget, - "resolveSessionTranscriptLegacyFileTarget", - ), + (pluginRuntimeMocks.executePluginCommand.mock.calls as unknown as Array<[unknown]>)[0]?.[0], { - agentId: "main", - sessionId: "sess-main", sessionKey: "agent:main:main", - storePath: "/tmp/openclaw-sessions/sessions.json", + sessionId: "sess-main", + sessionFile: "sqlite:main:sess-main:/tmp/openclaw-sessions/sessions.json", }, - "resolved transcript target params", + "plugin command params", ); + }); + + it("passes SQLite transcript markers to plugin commands without path resolution", async () => { + const storePath = "/tmp/openclaw-sessions/sessions.json"; + const marker = `sqlite:main:sess-main:${storePath}`; + sessionMocks.resolveStorePath.mockReturnValue(storePath); + sessionMocks.getSessionEntry.mockReturnValue({ + sessionId: "sess-main", + sessionFile: marker, + updatedAt: 1, + }); + + const { handler } = registerAndResolveCommandHandler({ + commandName: "codex", + cfg: { commands: { allowFrom: { telegram: ["200"] } } } as OpenClawConfig, + useAccessGroups: false, + pluginCommandSpecs: [ + { + name: "codex", + description: "Codex", + acceptsArgs: true, + }, + ] as TelegramPluginCommandSpecs, + }); + pluginRuntimeMocks.matchPluginCommand.mockReturnValue({ + command: { + name: "codex", + description: "Codex", + handler: vi.fn(), + pluginId: "openclaw-codex-app-server", + pluginName: "Codex", + requireAuth: true, + }, + args: "status", + }); + + await handler(createTelegramPrivateCommandContext({ match: "status" })); + expectRecordFields( (pluginRuntimeMocks.executePluginCommand.mock.calls as unknown as Array<[unknown]>)[0]?.[0], { sessionKey: "agent:main:main", sessionId: "sess-main", - sessionFile: "/tmp/openclaw-sessions/sess-main.jsonl", + sessionFile: marker, + }, + "plugin command params", + ); + }); + + it("replaces stale legacy transcript paths for plugin commands", async () => { + const storePath = "/tmp/openclaw-sessions/sessions.json"; + const marker = `sqlite:main:sess-main:${storePath}`; + sessionMocks.resolveStorePath.mockReturnValue(storePath); + sessionMocks.getSessionEntry.mockReturnValue({ + sessionId: "sess-main", + sessionFile: "sess-main.jsonl", + updatedAt: 1, + }); + + const { handler } = registerAndResolveCommandHandler({ + commandName: "codex", + cfg: { commands: { allowFrom: { telegram: ["200"] } } } as OpenClawConfig, + useAccessGroups: false, + pluginCommandSpecs: [ + { + name: "codex", + description: "Codex", + acceptsArgs: true, + }, + ] as TelegramPluginCommandSpecs, + }); + pluginRuntimeMocks.matchPluginCommand.mockReturnValue({ + command: { + name: "codex", + description: "Codex", + handler: vi.fn(), + pluginId: "openclaw-codex-app-server", + pluginName: "Codex", + requireAuth: true, + }, + args: "status", + }); + + await handler(createTelegramPrivateCommandContext({ match: "status" })); + + expectRecordFields( + (pluginRuntimeMocks.executePluginCommand.mock.calls as unknown as Array<[unknown]>)[0]?.[0], + { + sessionKey: "agent:main:main", + sessionId: "sess-main", + sessionFile: marker, }, "plugin command params", ); diff --git a/extensions/telegram/src/bot-native-commands.ts b/extensions/telegram/src/bot-native-commands.ts index ccb386365a0d..3e6533a517f2 100644 --- a/extensions/telegram/src/bot-native-commands.ts +++ b/extensions/telegram/src/bot-native-commands.ts @@ -41,11 +41,11 @@ import { danger, logVerbose } from "openclaw/plugin-sdk/runtime-env"; import { getChildLogger } from "openclaw/plugin-sdk/runtime-env"; import type { RuntimeEnv } from "openclaw/plugin-sdk/runtime-env"; import { + formatSqliteSessionFileMarker, getSessionEntry, resolveStorePath, type SessionEntry, } from "openclaw/plugin-sdk/session-store-runtime"; -import { resolveSessionTranscriptLegacyFileTarget } from "openclaw/plugin-sdk/session-transcript-runtime"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalString, @@ -222,6 +222,24 @@ export { testing as __testing }; type TelegramNativeCommandRuntime = Awaited>; +function resolveTelegramCommandSessionFile(params: { + agentId: string; + sessionFile?: string; + sessionId: string; + storePath: string; +}): string { + const sqliteMarker = formatSqliteSessionFileMarker({ + agentId: params.agentId, + sessionId: params.sessionId, + storePath: params.storePath, + }); + const explicitSessionFile = params.sessionFile?.trim(); + if (explicitSessionFile === sqliteMarker) { + return explicitSessionFile; + } + return sqliteMarker; +} + function resolveTelegramProgressPlaceholder(command: { nativeProgressMessages?: Partial> & { default?: string }; }): string | null { @@ -249,17 +267,16 @@ async function resolveTelegramCommandTranscriptContext(params: { storePath, }); const sessionId = entry?.sessionId?.trim() || randomUUID(); - const authProfileId = normalizeOptionalString(entry?.authProfileOverride); - const target = await resolveSessionTranscriptLegacyFileTarget({ + const sessionFile = resolveTelegramCommandSessionFile({ agentId: params.agentId, + sessionFile: entry?.sessionFile, sessionId, - sessionKey, storePath, - ...(params.threadId !== undefined ? { threadId: params.threadId } : {}), }); + const authProfileId = normalizeOptionalString(entry?.authProfileOverride); return { sessionId, - sessionFile: target.sessionFile, + sessionFile, ...(authProfileId ? { authProfileId } : {}), }; } catch { diff --git a/extensions/telegram/src/bot.create-telegram-bot.test-harness.ts b/extensions/telegram/src/bot.create-telegram-bot.test-harness.ts index 69047216928d..2ed37e34f879 100644 --- a/extensions/telegram/src/bot.create-telegram-bot.test-harness.ts +++ b/extensions/telegram/src/bot.create-telegram-bot.test-harness.ts @@ -15,13 +15,13 @@ type GetRuntimeConfigFn = type GetSessionEntryFn = typeof import("openclaw/plugin-sdk/session-store-runtime").getSessionEntry; type ListSessionEntriesFn = typeof import("openclaw/plugin-sdk/session-store-runtime").listSessionEntries; -type LoadSessionStoreFn = - typeof import("openclaw/plugin-sdk/session-store-runtime").loadSessionStore; type ResolveStorePathFn = typeof import("openclaw/plugin-sdk/session-store-runtime").resolveStorePath; type ReadSessionUpdatedAtFn = typeof import("openclaw/plugin-sdk/session-store-runtime").readSessionUpdatedAt; -type SessionStore = ReturnType; +type SessionEntry = import("openclaw/plugin-sdk/session-store-runtime").SessionEntry; +type SessionStore = Record; +type LoadSessionStoreFn = (storePath?: string, opts?: unknown) => SessionStore; type TelegramBotRuntimeForTest = NonNullable< Parameters[0] >; @@ -464,7 +464,6 @@ export const telegramBotDepsForTest: TelegramBotDeps = { getRuntimeConfig, getSessionEntry: getSessionEntryMock, listSessionEntries: listSessionEntriesMock, - loadSessionStore: loadSessionStoreMock as TelegramBotDeps["loadSessionStore"], resolveStorePath: resolveStorePathMock, readSessionUpdatedAt: readSessionUpdatedAtMock, recordInboundSession: recordInboundSessionMock as TelegramBotDeps["recordInboundSession"], diff --git a/extensions/telegram/src/bot.test.ts b/extensions/telegram/src/bot.test.ts index 343d1eb62e4f..1b4fc954f24d 100644 --- a/extensions/telegram/src/bot.test.ts +++ b/extensions/telegram/src/bot.test.ts @@ -1,5 +1,4 @@ -import { rm, writeFile } from "node:fs/promises"; -import path from "node:path"; +import { rm } from "node:fs/promises"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { clearPluginInteractiveHandlers, @@ -11,7 +10,8 @@ import { resetPluginStateStoreForTests, } from "openclaw/plugin-sdk/plugin-state-test-runtime"; import type { MsgContext } from "openclaw/plugin-sdk/reply-runtime"; -import { loadSessionStore } from "openclaw/plugin-sdk/session-store-runtime"; +import { listSessionEntries, upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; import { mockPinnedHostnameResolution } from "openclaw/plugin-sdk/test-env"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { @@ -193,6 +193,10 @@ type DirectTelegramTranscriptTestMessage = { timestamp: number; }; +function readOnlySessionEntry(storePath: string) { + return listSessionEntries({ storePath })[0]?.entry; +} + async function writeDirectTelegramTranscriptMessages(params: { cfg: OpenClawConfig; storePath: string; @@ -215,36 +219,29 @@ async function writeDirectTelegramTranscriptMessages(params: { isGroup: false, senderId: params.senderId, }); - await writeFile( - params.storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: params.sessionId, - chatType: "direct", - channel: "telegram", + await upsertSessionEntry({ + storePath: params.storePath, + sessionKey, + entry: { + sessionId: params.sessionId, + chatType: "direct", + channel: "telegram", + updatedAt: 1, + }, + }); + for (const message of params.messages) { + await appendSessionTranscriptMessageByIdentity({ + agentId: "main", + storePath: params.storePath, + sessionId: params.sessionId, + sessionKey, + message: { + role: message.role, + content: message.text, + timestamp: message.timestamp, }, - }), - "utf-8", - ); - await writeFile( - path.join(path.dirname(params.storePath), `${params.sessionId}.jsonl`), - [ - JSON.stringify({ type: "session", id: params.sessionId }), - ...params.messages.map((entry) => - JSON.stringify({ - id: entry.id, - type: "message", - message: { - role: entry.role, - content: entry.text, - timestamp: entry.timestamp, - }, - }), - ), - "", - ].join("\n"), - "utf-8", - ); + }); + } } async function writeDirectTelegramTranscriptContext(params: { @@ -796,7 +793,7 @@ describe("createTelegramBot", () => { expect(replySpy).not.toHaveBeenCalled(); expect(editMessageTextSpy).not.toHaveBeenCalled(); - expect(loadSessionStore(storePath, { skipCache: true })).toStrictEqual({}); + expect(listSessionEntries({ storePath })).toStrictEqual([]); expect(answerCallbackQuerySpy).toHaveBeenCalledWith("cbq-model-authz-bypass-1"); } finally { await rm(storePath, { force: true }); @@ -869,7 +866,7 @@ describe("createTelegramBot", () => { expect(replySpy).not.toHaveBeenCalled(); expect(editMessageTextSpy).not.toHaveBeenCalled(); - expect(loadSessionStore(storePath, { skipCache: true })).toStrictEqual({}); + expect(listSessionEntries({ storePath })).toStrictEqual([]); expect(answerCallbackQuerySpy).toHaveBeenCalledWith("cbq-group-model-authz-1"); } finally { await rm(storePath, { force: true }); @@ -951,7 +948,7 @@ describe("createTelegramBot", () => { expect(replySpy).not.toHaveBeenCalled(); expect(editMessageTextSpy).not.toHaveBeenCalled(); - expect(loadSessionStore(storePath, { skipCache: true })).toStrictEqual({}); + expect(listSessionEntries({ storePath })).toStrictEqual([]); expect(answerCallbackQuerySpy).toHaveBeenCalledWith("cbq-group-model-authz-runtime-1"); } finally { loadConfig.mockReset(); @@ -1810,7 +1807,7 @@ describe("createTelegramBot", () => { "Session selection cleared. Runtime unchanged. New replies use the agent's configured default.", ); - const entry = Object.values(loadSessionStore(storePath, { skipCache: true }))[0]; + const entry = readOnlySessionEntry(storePath); expect(entry?.providerOverride).toBeUndefined(); expect(entry?.modelOverride).toBeUndefined(); expect(answerCallbackQuerySpy).toHaveBeenCalledWith("cbq-model-compact-1"); @@ -1969,7 +1966,7 @@ describe("createTelegramBot", () => { "Session selection cleared. Runtime unchanged. New replies use the agent's configured default.", ); - const entry = Object.values(loadSessionStore(storePath, { skipCache: true }))[0]; + const entry = readOnlySessionEntry(storePath); expect(entry?.providerOverride).toBeUndefined(); expect(entry?.modelOverride).toBeUndefined(); expect(answerCallbackQuerySpy).toHaveBeenCalledWith("cbq-model-default-1"); @@ -2049,7 +2046,7 @@ describe("createTelegramBot", () => { ); expect(requireRecord(editCall[3], "edit params").parse_mode).toBe("HTML"); - const entry = Object.values(loadSessionStore(storePath, { skipCache: true }))[0]; + const entry = readOnlySessionEntry(storePath); expect(entry?.providerOverride).toBe("openai"); expect(entry?.modelOverride).toBe("gpt-5.4"); expect(answerCallbackQuerySpy).toHaveBeenCalledWith("cbq-model-html-1"); @@ -2141,7 +2138,9 @@ describe("createTelegramBot", () => { getFile: async () => ({ download: async () => new Uint8Array() }), }); - const entry = Object.values(loadSessionStore(storePath, { skipCache: true }))[0]; + // Override must be persisted (not cleared) because openai/gpt-5.4 is + // NOT the default in the fresh config. + const entry = readOnlySessionEntry(storePath); expect(entry?.providerOverride).toBe("openai"); expect(entry?.modelOverride).toBe("gpt-5.4"); expect(entry?.modelOverrideSource).toBe("user"); @@ -2162,7 +2161,7 @@ describe("createTelegramBot", () => { getFile: async () => ({ download: async () => new Uint8Array() }), }); - const lunaEntry = Object.values(loadSessionStore(storePath, { skipCache: true }))[0]; + const lunaEntry = readOnlySessionEntry(storePath); expect(lunaEntry?.providerOverride).toBe("openai"); expect(lunaEntry?.modelOverride).toBe("gpt-5.6-luna"); expect(lunaEntry?.modelOverrideSource).toBe("user"); @@ -2222,7 +2221,7 @@ describe("createTelegramBot", () => { ) as { cfg?: OpenClawConfig }; expect(dispatchParams.cfg).toBe(freshConfig); - const afterTurn = Object.values(loadSessionStore(storePath, { skipCache: true }))[0]; + const afterTurn = readOnlySessionEntry(storePath); expect(afterTurn?.providerOverride).toBe("openai"); expect(afterTurn?.modelOverride).toBe("gpt-5.6-luna"); expect(afterTurn?.modelOverrideSource).toBe("user"); diff --git a/extensions/telegram/src/exec-approvals.test.ts b/extensions/telegram/src/exec-approvals.test.ts index 944d034c46ce..5c7339759fd2 100644 --- a/extensions/telegram/src/exec-approvals.test.ts +++ b/extensions/telegram/src/exec-approvals.test.ts @@ -7,7 +7,8 @@ import type { TelegramAccountConfig, TelegramExecApprovalConfig, } from "openclaw/plugin-sdk/config-contracts"; -import { saveSessionStore } from "openclaw/plugin-sdk/session-store-runtime"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing"; import { afterEach, describe, expect, it } from "vitest"; import { getTelegramExecApprovalApprovers, @@ -28,6 +29,7 @@ type TelegramExecApprovalRequest = Parameters< >[0]["request"]; afterEach(() => { + closeOpenClawAgentDatabasesForTest(); for (const dir of tempDirs.splice(0)) { fs.rmSync(dir, { recursive: true, force: true }); } @@ -233,23 +235,21 @@ describe("telegram exec approvals", () => { it("scopes non-telegram turn sources to the stored telegram account", async () => { const tmpDir = createTempDir(); const storePath = path.join(tmpDir, "sessions.json"); - await saveSessionStore( + await upsertSessionEntry({ storePath, - { - "agent:ops:telegram:direct:123": { - sessionId: "main", - updatedAt: 1, - origin: { - provider: "telegram", - accountId: "ops", - }, - lastChannel: "slack", - lastTo: "channel:C999", - lastAccountId: "work", + sessionKey: "agent:ops:telegram:direct:123", + entry: { + sessionId: "main", + updatedAt: 1, + origin: { + provider: "telegram", + accountId: "ops", }, + lastChannel: "slack", + lastTo: "channel:C999", + lastAccountId: "work", }, - { skipMaintenance: true }, - ); + }); const cfg = buildMultiAccountTelegramConfig({ sessionStorePath: storePath }); const request = makeForeignChannelApprovalRequest({ id: "req-2", diff --git a/extensions/whatsapp/src/auto-reply/config.runtime.ts b/extensions/whatsapp/src/auto-reply/config.runtime.ts index 080f30c73573..1e534ec240ec 100644 --- a/extensions/whatsapp/src/auto-reply/config.runtime.ts +++ b/extensions/whatsapp/src/auto-reply/config.runtime.ts @@ -1,7 +1,6 @@ // Whatsapp helper module supports config behavior. export { evaluateSessionFreshness, - loadSessionStore, resolveSessionKey, resolveSessionResetPolicy, resolveSessionResetType, diff --git a/extensions/whatsapp/src/auto-reply/monitor/group-activation.test.ts b/extensions/whatsapp/src/auto-reply/monitor/group-activation.test.ts index 68cdc00d0641..bb670255625d 100644 --- a/extensions/whatsapp/src/auto-reply/monitor/group-activation.test.ts +++ b/extensions/whatsapp/src/auto-reply/monitor/group-activation.test.ts @@ -2,9 +2,13 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { saveSessionStore, type SessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { + getSessionEntry, + upsertSessionEntry, + type SessionEntry, +} from "openclaw/plugin-sdk/session-store-runtime"; +import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { loadSessionStore } from "../config.runtime.js"; import { resolveGroupActivationFor } from "./group-activation.js"; const GROUP_CONVERSATION_ID = "123@g.us"; @@ -22,9 +26,11 @@ async function makeSessionStore( ): Promise<{ storePath: string; cleanup: () => Promise }> { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-session-")); const storePath = path.join(dir, "sessions.json"); - await saveSessionStore(storePath, entries as Record, { - skipMaintenance: true, - }); + await Promise.all( + Object.entries(entries as Record).map(([sessionKey, entry]) => + upsertSessionEntry({ storePath, sessionKey, entry }), + ), + ); return { storePath, cleanup: async () => { @@ -56,12 +62,26 @@ const expectWorkGroupActivationEntry = async ( assertEntry?: (entry: SessionStoreEntry | undefined) => void, ) => { await vi.waitFor(() => { - const scopedEntry = loadSessionStore(storePath, { skipCache: true })[WORK_GROUP_SESSION_KEY]; + const scopedEntry = getSessionEntry({ + storePath, + sessionKey: WORK_GROUP_SESSION_KEY, + readConsistency: "latest", + }); expect(scopedEntry?.groupActivation).toBe("always"); assertEntry?.(scopedEntry); }); }; +const expectNoWorkGroupActivationEntry = (storePath: string) => { + expect( + getSessionEntry({ + storePath, + sessionKey: WORK_GROUP_SESSION_KEY, + readConsistency: "latest", + }), + ).toBeUndefined(); +}; + const expectResolvedWorkGroupActivation = async ( storePath: string, assertEntry?: (entry: SessionStoreEntry | undefined) => void, @@ -75,12 +95,13 @@ describe("resolveGroupActivationFor", () => { const cleanups: Array<() => Promise> = []; afterEach(async () => { + closeOpenClawAgentDatabasesForTest(); while (cleanups.length > 0) { await cleanups.pop()?.(); } }); - it("reads legacy named-account group activation and backfills the scoped key", async () => { + it("reads legacy named-account group activation without synthesizing a scoped session", async () => { const { storePath, cleanup } = await makeSessionStore({ [LEGACY_GROUP_SESSION_KEY]: { groupActivation: "always", @@ -90,16 +111,16 @@ describe("resolveGroupActivationFor", () => { }); cleanups.push(cleanup); - await expectResolvedWorkGroupActivation(storePath, (scopedEntry) => { - expect(scopedEntry?.sessionId).toBeUndefined(); - expect(scopedEntry?.updatedAt).toBeUndefined(); - }); + const activation = await resolveWorkGroupActivation(storePath); + expect(activation).toBe("always"); + expectNoWorkGroupActivationEntry(storePath); }); it("preserves legacy group activation when the scoped entry already exists without activation", async () => { const { storePath, cleanup } = await makeSessionStore({ [LEGACY_GROUP_SESSION_KEY]: { groupActivation: "always", + sessionId: "legacy-session", }, [WORK_GROUP_SESSION_KEY]: { sessionId: "scoped-session", @@ -112,10 +133,11 @@ describe("resolveGroupActivationFor", () => { }); }); - it("does not wake the default account from an activation-only legacy group entry in multi-account setups", async () => { + it("does not wake the default account from a work-account scoped group activation", async () => { const { storePath, cleanup } = await makeSessionStore({ - [LEGACY_GROUP_SESSION_KEY]: { + [WORK_GROUP_SESSION_KEY]: { groupActivation: "always", + sessionId: "work-session", }, }); cleanups.push(cleanup); @@ -162,6 +184,7 @@ describe("resolveGroupActivationFor", () => { const { storePath, cleanup } = await makeSessionStore({ [LEGACY_GROUP_SESSION_KEY]: { groupActivation: "always", + sessionId: "legacy-session", }, }); cleanups.push(cleanup); diff --git a/extensions/whatsapp/src/auto-reply/monitor/group-activation.ts b/extensions/whatsapp/src/auto-reply/monitor/group-activation.ts index 8651e500f991..ba9fbe26b7df 100644 --- a/extensions/whatsapp/src/auto-reply/monitor/group-activation.ts +++ b/extensions/whatsapp/src/auto-reply/monitor/group-activation.ts @@ -5,7 +5,6 @@ import { getSessionEntry, patchSessionEntry, resolveStorePath, - type SessionEntry, } from "openclaw/plugin-sdk/session-store-runtime"; import { resolveWhatsAppLegacyGroupSessionKey } from "../../group-session-key.js"; import { resolveWhatsAppInboundPolicy } from "../../inbound-policy.js"; @@ -61,23 +60,24 @@ export async function resolveGroupActivationFor(params: { (ignoreScopedActivation ? undefined : scopedEntry?.groupActivation) ?? legacyEntry?.groupActivation; if (activation !== undefined && scopedEntry?.groupActivation === undefined) { - // Activation-only backfills must not synthesize session ids or activity. - // replaceEntry preserves existing scoped metadata while keeping fallback writes sparse. - await patchSessionEntry({ - ...sessionScope, - sessionKey: params.sessionKey, - fallbackEntry: {} as SessionEntry, - replaceEntry: true, - update: (entry) => { - if (entry.groupActivation !== undefined) { - return null; - } - return { - ...entry, - groupActivation: activation, - }; - }, - }); + // SQLite session rows require a real session id; activation-only legacy metadata + // can be read, but must not synthesize a scoped session just to backfill metadata. + if (scopedEntry) { + await patchSessionEntry({ + ...sessionScope, + sessionKey: params.sessionKey, + replaceEntry: true, + update: (entry) => { + if (entry.groupActivation !== undefined) { + return null; + } + return { + ...entry, + groupActivation: activation, + }; + }, + }); + } } const requireMention = resolveWhatsAppInboundPolicy({ cfg: params.cfg, diff --git a/extensions/whatsapp/src/auto-reply/web-auto-reply-utils.test.ts b/extensions/whatsapp/src/auto-reply/web-auto-reply-utils.test.ts index 65dbc451ded5..8948e37bd8c5 100644 --- a/extensions/whatsapp/src/auto-reply/web-auto-reply-utils.test.ts +++ b/extensions/whatsapp/src/auto-reply/web-auto-reply-utils.test.ts @@ -3,14 +3,13 @@ import fs from "node:fs/promises"; import path from "node:path"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { normalizeMainKey } from "openclaw/plugin-sdk/routing"; -import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { getSessionEntry, upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; import { withTempDir } from "openclaw/plugin-sdk/test-env"; import { describe, expect, it, vi } from "vitest"; import { createTestWebInboundMessage } from "../inbound/test-message.test-helper.js"; import type { AdmittedWebInboundMessage } from "../inbound/types.js"; import { evaluateSessionFreshness, - loadSessionStore, resolveChannelResetConfig, resolveSessionKey, resolveSessionResetPolicy, @@ -95,8 +94,10 @@ function getSessionSnapshotForTest( { From: from, To: "", Body: "" }, normalizeMainKey(sessionCfg?.mainKey), ); - const store = loadSessionStore(resolveStorePath(sessionCfg?.store)); - const entry = store[key]; + const entry = getSessionEntry({ + sessionKey: key, + storePath: resolveStorePath(sessionCfg?.store), + }); const isThread = resolveThreadFlag({ sessionKey: key, messageThreadId: ctx?.messageThreadId ?? null, diff --git a/package.json b/package.json index 1bb7c7e234a2..318b7f6a1d51 100644 --- a/package.json +++ b/package.json @@ -1689,6 +1689,7 @@ "lint:tmp:session-accessor-boundary": "node scripts/check-session-accessor-boundary.mjs", "lint:tmp:session-accessor-boundary:gen": "node scripts/check-session-accessor-boundary.mjs --update-debt-baseline", "lint:tmp:session-transcript-reader-boundary": "node scripts/check-session-transcript-reader-boundary.mjs", + "lint:tmp:sqlite-transaction-boundary": "node scripts/check-sqlite-transaction-boundary.mjs", "lint:tmp:tsgo-core-boundary": "node scripts/check-tsgo-core-boundary.mjs", "lint:ui:no-raw-window-open": "node scripts/check-no-raw-window-open.mjs", "lint:web-fetch-provider-boundaries": "node scripts/check-web-fetch-provider-boundaries.mjs", @@ -1773,6 +1774,8 @@ "release:version": "node --import tsx scripts/release-version.ts", "runtime-sidecars:check": "node --import tsx scripts/generate-runtime-sidecar-paths-baseline.ts --check", "runtime-sidecars:gen": "node --import tsx scripts/generate-runtime-sidecar-paths-baseline.ts --write", + "sqlite:sessions-schema:check": "node --import tsx scripts/generate-sqlite-session-schema-baseline.ts --check", + "sqlite:sessions-schema:gen": "node --import tsx scripts/generate-sqlite-session-schema-baseline.ts --write", "start": "node openclaw.mjs", "test": "node scripts/test-projects.mjs", "test:all": "pnpm lint && pnpm build && pnpm test && pnpm test:e2e && pnpm test:live && pnpm test:docker:all", diff --git a/packages/memory-host-sdk/src/engine-qmd.ts b/packages/memory-host-sdk/src/engine-qmd.ts index f0d64f6e5dca..5303222b1707 100644 --- a/packages/memory-host-sdk/src/engine-qmd.ts +++ b/packages/memory-host-sdk/src/engine-qmd.ts @@ -12,16 +12,20 @@ export { resolveSessionIdentityForTranscriptFile, resolveSessionFileForSyncTarget, sessionPathForFile, + sessionPathForSessionIdentity, + statSessionEntrySync, type BuildSessionEntryOptions, type ResolvedMemorySessionSyncTarget, type ResolvedSessionTranscriptIdentity, type SessionFileEntry, + type SessionFileState, type SessionTranscriptClassification, type SessionTranscriptCorpusEntry, } from "./host/session-files.js"; export { isSessionArchiveArtifactName, isUsageCountedSessionTranscriptFileName, + parseSqliteSessionFileMarker, parseUsageCountedSessionIdFromFileName, } from "./host/openclaw-runtime-session.js"; export { parseQmdQueryJson, type QmdQueryResult } from "./host/qmd-query-parser.js"; diff --git a/packages/memory-host-sdk/src/host/openclaw-runtime-session.ts b/packages/memory-host-sdk/src/host/openclaw-runtime-session.ts index d8273073c3fd..c14b0a26b5d6 100644 --- a/packages/memory-host-sdk/src/host/openclaw-runtime-session.ts +++ b/packages/memory-host-sdk/src/host/openclaw-runtime-session.ts @@ -15,11 +15,15 @@ export { isSessionArchiveArtifactName, isSilentReplyPayloadText, isUsageCountedSessionTranscriptFileName, + loadTranscriptEventsSync, listSessionEntries, onSessionTranscriptUpdate, + parseSqliteSessionFileMarker, parseUsageCountedSessionIdFromFileName, + readTranscriptStatsSync, resolveSessionFilePath, resolveStorePath, + resolveTranscriptSessionKeyBySessionId, resolveSessionAgentId, resolveSessionTranscriptsDirForAgent, stripInboundMetadata, diff --git a/packages/memory-host-sdk/src/host/openclaw-runtime.ts b/packages/memory-host-sdk/src/host/openclaw-runtime.ts index 56543b6a1d95..9cccd605bdcd 100644 --- a/packages/memory-host-sdk/src/host/openclaw-runtime.ts +++ b/packages/memory-host-sdk/src/host/openclaw-runtime.ts @@ -54,12 +54,18 @@ export { parseUsageCountedSessionIdFromFileName, } from "../../../../src/config/sessions/artifacts.js"; export { canonicalizeMainSessionAlias } from "../../../../src/config/sessions/main-session.js"; -export { resolveSessionTranscriptsDirForAgent } from "../../../../src/config/sessions/paths.js"; export { - listSessionEntries, resolveSessionFilePath, + resolveSessionTranscriptsDirForAgent, +} from "../../../../src/config/sessions/paths.js"; +export { + loadTranscriptEventsSync, + listSessionEntries, + readTranscriptStatsSync, + resolveTranscriptSessionKeyBySessionId, resolveStorePath, } from "../../../../src/plugin-sdk/session-store-runtime.js"; +export { parseSqliteSessionFileMarker } from "../../../../src/plugin-sdk/session-store-runtime.js"; export type { SessionEntry } from "../../../../src/config/sessions/types.js"; export type { SessionSendPolicyConfig } from "../../../../src/config/types.base.js"; export type { diff --git a/packages/memory-host-sdk/src/host/session-files.test.ts b/packages/memory-host-sdk/src/host/session-files.test.ts index c32bd61851d2..c8be1712b27e 100644 --- a/packages/memory-host-sdk/src/host/session-files.test.ts +++ b/packages/memory-host-sdk/src/host/session-files.test.ts @@ -7,6 +7,10 @@ import { clearRuntimeConfigSnapshot, } from "openclaw/plugin-sdk/runtime-config-snapshot"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + persistSessionTranscriptTurn, + upsertSessionEntry, +} from "../../../../src/config/sessions/session-accessor.js"; import { buildSessionEntry, listSessionFilesForAgent, @@ -16,6 +20,7 @@ import { resolveSessionIdentityForTranscriptFile, resolveSessionFileForSyncTarget, sessionPathForFile, + statSessionEntrySync, type SessionFileEntry, } from "./session-files.js"; @@ -74,13 +79,22 @@ function requireSessionEntry(entry: SessionFileEntry | null): SessionFileEntry { return entry; } +async function upsertTestSessionEntries( + storePath: string, + entries: Record[1]>, +): Promise { + fsSync.mkdirSync(path.dirname(storePath), { recursive: true }); + for (const [sessionKey, entry] of Object.entries(entries)) { + await upsertSessionEntry({ sessionKey, storePath }, entry); + } +} + describe("listSessionFilesForAgent", () => { it("includes reset and deleted transcripts in session file listing", async () => { const sessionsDir = path.join(tmpDir, "agents", "main", "sessions"); fsSync.mkdirSync(path.join(sessionsDir, "archive"), { recursive: true }); const included = [ - "active.jsonl", "active.jsonl.reset.2026-02-16T22-26-33.000Z", "active.jsonl.deleted.2026-02-16T22-27-33.000Z", ]; @@ -104,30 +118,19 @@ describe("listSessionFilesForAgent", () => { }); describe("listSessionTranscriptCorpusEntriesForAgent", () => { - it("lists active session entries with accessor-backed identity and classification", async () => { + it("omits active JSONL session entries from accessor-backed corpus entries", async () => { const sessionsDir = path.join(tmpDir, "agents", "main", "sessions"); fsSync.mkdirSync(sessionsDir, { recursive: true }); fsSync.writeFileSync(path.join(sessionsDir, "narrative.jsonl"), ""); - fsSync.writeFileSync( - path.join(sessionsDir, "sessions.json"), - JSON.stringify({ - "agent:main:dreaming-narrative-run-1": { - sessionFile: "narrative.jsonl", - sessionId: "narrative", - }, - }), - ); - - await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([ - { - agentId: "main", - artifactKind: "active-session", - generatedByDreamingNarrative: true, - sessionFile: path.join(sessionsDir, "narrative.jsonl"), + await upsertTestSessionEntries(path.join(sessionsDir, "sessions.json"), { + "agent:main:dreaming-narrative-run-1": { + sessionFile: "narrative.jsonl", sessionId: "narrative", - sessionKey: "agent:main:dreaming-narrative-run-1", + updatedAt: 1, }, - ]); + }); + + await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([]); }); it("keeps archive artifacts in the corpus and inherits active session classification", async () => { @@ -137,21 +140,17 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => { const archivePath = path.join(sessionsDir, "cron-run.jsonl.deleted.2026-02-16T22-27-33.000Z"); fsSync.writeFileSync(activePath, ""); fsSync.writeFileSync(archivePath, ""); - fsSync.writeFileSync( - path.join(sessionsDir, "sessions.json"), - JSON.stringify({ - "agent:main:cron:job-1:run:run-1": { - sessionFile: "cron-run.jsonl", - sessionId: "cron-run", - }, - }), - ); + await upsertTestSessionEntries(path.join(sessionsDir, "sessions.json"), { + "agent:main:cron:job-1:run:run-1": { + sessionFile: "cron-run.jsonl", + sessionId: "cron-run", + updatedAt: 1, + }, + }); const classification = loadSessionTranscriptClassificationForAgent("main"); - expect(classification.cronRunTranscriptPaths).toEqual( - new Set([activePath, archivePath].map((filePath) => path.resolve(filePath))), - ); + expect(classification.cronRunTranscriptPaths).toEqual(new Set([path.resolve(archivePath)])); await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toContainEqual({ agentId: "main", artifactKind: "archive-artifact", @@ -161,6 +160,88 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => { }); }); + it("reads live SQLite rows by session identity while preserving archived JSONL artifacts", async () => { + const sessionsDir = path.join(tmpDir, "agents", "main", "sessions"); + const storePath = path.join(sessionsDir, "sessions.json"); + const sessionKey = "agent:main:chat:sqlite-live"; + const sessionId = "sqlite-live"; + const updatedAt = Date.parse("2026-06-25T12:00:00.000Z"); + fsSync.mkdirSync(sessionsDir, { recursive: true }); + + await upsertSessionEntry({ agentId: "main", sessionKey, storePath }, { sessionId, updatedAt }); + const turn = await persistSessionTranscriptTurn( + { agentId: "main", sessionId, sessionKey, storePath }, + { + messages: [ + { + message: { + role: "user", + content: "Live SQLite transcript text", + timestamp: updatedAt, + }, + }, + ], + touchSessionEntry: true, + updateMode: "none", + }, + ); + const archivePath = path.join( + sessionsDir, + `${sessionId}.jsonl.deleted.2026-06-25T12-01-00.000Z`, + ); + fsSync.writeFileSync( + archivePath, + JSON.stringify({ + type: "message", + message: { role: "user", content: "Archived JSONL transcript text" }, + }), + ); + + expect(fsSync.existsSync(path.join(sessionsDir, `${sessionId}.jsonl`))).toBe(false); + const entries = await listSessionTranscriptCorpusEntriesForAgent("main"); + expect(entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + agentId: "main", + artifactKind: "active-session", + sessionFile: turn.sessionFile, + sessionId, + sessionKey, + transcriptSource: "sqlite", + updatedAtMs: expect.any(Number), + }), + expect.objectContaining({ + agentId: "main", + artifactKind: "archive-artifact", + sessionFile: archivePath, + sessionId, + }), + ]), + ); + + const liveEntry = requireSessionEntry( + await buildSessionEntry(turn.sessionFile, { sessionKey, updatedAtMs: updatedAt }), + ); + const liveState = statSessionEntrySync(turn.sessionFile, { + sessionKey, + updatedAtMs: updatedAt, + }); + const archiveEntry = requireSessionEntry(await buildSessionEntry(archivePath)); + + expect(liveEntry.path).toBe("sessions/main/sqlite-live.jsonl"); + expect(liveEntry.content).toBe("User: Live SQLite transcript text"); + expect(liveState).toEqual({ + absPath: turn.sessionFile, + path: liveEntry.path, + mtimeMs: liveEntry.mtimeMs, + size: liveEntry.size, + }); + expect(archiveEntry.path).toBe( + "sessions/main/sqlite-live.jsonl.deleted.2026-06-25T12-01-00.000Z", + ); + expect(archiveEntry.content).toBe("User: Archived JSONL transcript text"); + }); + it("classifies active entries through cron parentage chains", async () => { const sessionsDir = path.join(tmpDir, "agents", "main", "sessions"); fsSync.mkdirSync(sessionsDir, { recursive: true }); @@ -178,68 +259,42 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => { ]) { fsSync.writeFileSync(filePath, ""); } - fsSync.writeFileSync( - path.join(sessionsDir, "sessions.json"), - JSON.stringify({ - "agent:main:cron:job-1:run:run-1": { - sessionFile: "cron-run.jsonl", - sessionId: "cron-run", - }, - "agent:main:subagent:spawned-child": { - sessionFile: "spawned-child.jsonl", - sessionId: "spawned-child", - spawnedBy: "agent:main:cron:job-1:run:run-1", - }, - "agent:main:subagent:keyed-child": { - parentSessionKey: "agent:main:subagent:spawned-child", - sessionFile: "keyed-child.jsonl", - sessionId: "keyed-child", - }, - "agent:main:subagent:orphan-child": { - sessionFile: "orphan-child.jsonl", - sessionId: "orphan-child", - spawnedBy: "agent:main:cron:job-1:run:missing", - }, - "agent:main:subagent:normal-child": { - sessionFile: "normal-child.jsonl", - sessionId: "normal-child", - spawnedBy: "agent:main:chat:manual", - }, - }), - ); + await upsertTestSessionEntries(path.join(sessionsDir, "sessions.json"), { + "agent:main:cron:job-1:run:run-1": { + sessionFile: "cron-run.jsonl", + sessionId: "cron-run", + updatedAt: 1, + }, + "agent:main:subagent:spawned-child": { + sessionFile: "spawned-child.jsonl", + sessionId: "spawned-child", + spawnedBy: "agent:main:cron:job-1:run:run-1", + updatedAt: 1, + }, + "agent:main:subagent:keyed-child": { + parentSessionKey: "agent:main:subagent:spawned-child", + sessionFile: "keyed-child.jsonl", + sessionId: "keyed-child", + updatedAt: 1, + }, + "agent:main:subagent:orphan-child": { + sessionFile: "orphan-child.jsonl", + sessionId: "orphan-child", + spawnedBy: "agent:main:cron:job-1:run:missing", + updatedAt: 1, + }, + "agent:main:subagent:normal-child": { + sessionFile: "normal-child.jsonl", + sessionId: "normal-child", + spawnedBy: "agent:main:chat:manual", + updatedAt: 1, + }, + }); const classification = loadSessionTranscriptClassificationForAgent("main"); - expect(classification.cronRunTranscriptPaths).toEqual( - new Set( - [cronPath, spawnedChildPath, keyedChildPath, orphanChildPath].map((filePath) => - path.resolve(filePath), - ), - ), - ); - await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual( - expect.arrayContaining([ - expect.objectContaining({ - generatedByCronRun: true, - sessionFile: spawnedChildPath, - sessionKey: "agent:main:subagent:spawned-child", - }), - expect.objectContaining({ - generatedByCronRun: true, - sessionFile: keyedChildPath, - sessionKey: "agent:main:subagent:keyed-child", - }), - expect.objectContaining({ - generatedByCronRun: true, - sessionFile: orphanChildPath, - sessionKey: "agent:main:subagent:orphan-child", - }), - expect.objectContaining({ - sessionFile: normalPath, - sessionKey: "agent:main:subagent:normal-child", - }), - ]), - ); + expect(classification.cronRunTranscriptPaths).toEqual(new Set()); + await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([]); const entries = await listSessionTranscriptCorpusEntriesForAgent("main"); expect(entries.find((entry) => entry.sessionFile === normalPath)?.generatedByCronRun).toBe( undefined, @@ -251,15 +306,13 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => { fsSync.mkdirSync(sessionsDir, { recursive: true }); const archivePath = path.join(sessionsDir, "cron-run.jsonl.reset.2026-02-16T22-26-33.000Z"); fsSync.writeFileSync(archivePath, ""); - fsSync.writeFileSync( - path.join(sessionsDir, "sessions.json"), - JSON.stringify({ - "agent:main:cron:job-1:run:run-1": { - sessionFile: "cron-run.jsonl", - sessionId: "cron-run", - }, - }), - ); + await upsertTestSessionEntries(path.join(sessionsDir, "sessions.json"), { + "agent:main:cron:job-1:run:run-1": { + sessionFile: "cron-run.jsonl", + sessionId: "cron-run", + updatedAt: 1, + }, + }); const expectedArchivePath = archivePath; const classification = loadSessionTranscriptClassificationForAgent("main"); @@ -347,14 +400,7 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => { }), ); - await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([ - { - agentId: "main", - artifactKind: "orphan-file-artifact", - sessionFile, - sessionId: "active", - }, - ]); + await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([]); }); it("rejects relative sessionFile values that escape through nested segments", async () => { @@ -395,69 +441,36 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => { await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([]); }); - it("falls back to transcript filename identity when an active row lacks sessionId", async () => { + it("omits loose non-archive JSONL transcripts from the corpus", async () => { const sessionsDir = path.join(tmpDir, "agents", "main", "sessions"); fsSync.mkdirSync(sessionsDir, { recursive: true }); const sessionFile = path.join(sessionsDir, "active-thread-456.jsonl"); fsSync.writeFileSync(sessionFile, ""); - fsSync.writeFileSync( - path.join(sessionsDir, "sessions.json"), - JSON.stringify({ - "agent:main:chat:thread-456": { - sessionFile: "active-thread-456.jsonl", - }, - }), - ); - await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([ - { - agentId: "main", - artifactKind: "active-session", - sessionFile, - sessionId: "active-thread-456", - sessionKey: "agent:main:chat:thread-456", - }, - ]); + await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([]); }); - it("lists only the requested agent's active transcripts from a shared custom store", async () => { + it("omits active JSONL transcripts from a custom session store", async () => { const sessionsDir = path.join(tmpDir, "custom-sessions"); const sessionFile = path.join(sessionsDir, "custom-thread.jsonl"); - const otherSessionFile = path.join(sessionsDir, "ops-thread.jsonl"); const storePath = path.join(sessionsDir, "sessions.json"); const configPath = path.join(tmpDir, "openclaw.json"); fsSync.mkdirSync(sessionsDir, { recursive: true }); fsSync.writeFileSync(sessionFile, ""); - fsSync.writeFileSync(otherSessionFile, ""); - fsSync.writeFileSync( - storePath, - JSON.stringify({ - "agent:main:chat:custom": { - sessionFile: "custom-thread.jsonl", - sessionId: "custom-thread", - }, - "agent:ops:chat:custom": { - sessionFile: "ops-thread.jsonl", - sessionId: "ops-thread", - }, - }), - ); fsSync.writeFileSync(configPath, JSON.stringify({ session: { store: storePath } })); Reflect.set(process.env, "OPENCLAW_CONFIG_PATH", configPath); clearRuntimeConfigSnapshot(); clearConfigCache(); - - await expect(listSessionFilesForAgent("main")).resolves.toEqual([sessionFile]); - await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([ - { - agentId: "main", - artifactKind: "active-session", - sessionFile, + await upsertTestSessionEntries(storePath, { + "agent:main:chat:custom": { + sessionFile: "custom-thread.jsonl", sessionId: "custom-thread", - sessionKey: "agent:main:chat:custom", + updatedAt: 1, }, - ]); - await expect(listSessionFilesForAgent("ops")).resolves.toEqual([otherSessionFile]); + }); + + await expect(listSessionFilesForAgent("main")).resolves.toEqual([]); + await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([]); }); it("keeps unowned archives from an agent-owned fixed session store", async () => { @@ -500,21 +513,19 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => { fsSync.mkdirSync(sessionsDir, { recursive: true }); fsSync.writeFileSync(sessionFile, ""); fsSync.writeFileSync(archivePath, ""); - fsSync.writeFileSync( - storePath, - JSON.stringify({ - "agent:main:chat:absolute": { - sessionFile, - sessionId: "absolute-thread", - }, - }), - ); fsSync.writeFileSync(configPath, JSON.stringify({ session: { store: storePath } })); Reflect.set(process.env, "OPENCLAW_CONFIG_PATH", configPath); clearRuntimeConfigSnapshot(); clearConfigCache(); + await upsertTestSessionEntries(storePath, { + "agent:main:chat:absolute": { + sessionFile, + sessionId: "absolute-thread", + updatedAt: 1, + }, + }); - await expect(listSessionFilesForAgent("main")).resolves.toEqual([sessionFile, archivePath]); + await expect(listSessionFilesForAgent("main")).resolves.toEqual([archivePath]); }); it("keeps legacy session keys in non-main per-agent stores", async () => { @@ -532,7 +543,7 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => { }), ); - await expect(listSessionFilesForAgent("ops")).resolves.toEqual([sessionFile]); + await expect(listSessionFilesForAgent("ops")).resolves.toEqual([]); await expect(listSessionFilesForAgent("main")).resolves.toEqual([]); }); @@ -559,7 +570,7 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => { clearRuntimeConfigSnapshot(); clearConfigCache(); - await expect(listSessionFilesForAgent("ops")).resolves.toEqual([sessionFile]); + await expect(listSessionFilesForAgent("ops")).resolves.toEqual([]); }); }); @@ -605,20 +616,9 @@ describe("memory session sync targets", () => { ).toBeNull(); }); - it("resolves identity sync targets to the current file-backed transcript", () => { - expect(resolveSessionFileForSyncTarget({ sessionId: "active" }, "main")).toEqual({ - agentId: "main", - sessionId: "active", - sessionFile: path.join(tmpDir, "agents", "main", "sessions", "active.jsonl"), - }); - }); - - it("normalizes agent ids before resolving identity sync targets", () => { - expect(resolveSessionFileForSyncTarget({ agentId: "MAIN", sessionId: "active" })).toEqual({ - agentId: "main", - sessionId: "active", - sessionFile: path.join(tmpDir, "agents", "main", "sessions", "active.jsonl"), - }); + it("does not synthesize active transcript paths for identity sync targets", () => { + expect(resolveSessionFileForSyncTarget({ sessionId: "active" }, "main")).toBeNull(); + expect(resolveSessionFileForSyncTarget({ agentId: "MAIN", sessionId: "active" })).toBeNull(); }); it("rejects identity sync targets that would escape the sessions directory", () => { @@ -629,7 +629,7 @@ describe("memory session sync targets", () => { expect(resolveSessionFileForSyncTarget({ sessionId: "foo/../active" }, "main")).toBeNull(); }); - it("resolves identity sync targets through persisted session keys", () => { + it("does not read legacy sessions.json for persisted session-key sync targets", () => { const sessionsDir = path.join(tmpDir, "agents", "main", "sessions"); fsSync.mkdirSync(sessionsDir, { recursive: true }); fsSync.writeFileSync( @@ -648,31 +648,7 @@ describe("memory session sync targets", () => { sessionId: "active", sessionKey: "agent:main:chat:thread-456", }), - ).toEqual({ - agentId: "main", - sessionId: "active", - sessionFile: path.join(sessionsDir, "active-thread-456.jsonl"), - }); - }); - - it("resolves identity sync targets through persisted session ids", () => { - const sessionsDir = path.join(tmpDir, "agents", "main", "sessions"); - fsSync.mkdirSync(sessionsDir, { recursive: true }); - fsSync.writeFileSync( - path.join(sessionsDir, "sessions.json"), - JSON.stringify({ - "agent:main:chat:thread-456": { - sessionFile: "active-thread-456.jsonl", - sessionId: "active", - }, - }), - ); - - expect(resolveSessionFileForSyncTarget({ agentId: "main", sessionId: "active" })).toEqual({ - agentId: "main", - sessionId: "active", - sessionFile: path.join(sessionsDir, "active-thread-456.jsonl"), - }); + ).toBeNull(); }); it("resolves transcript file identities through persisted session keys", () => { diff --git a/packages/memory-host-sdk/src/host/session-files.ts b/packages/memory-host-sdk/src/host/session-files.ts index 865fcad602d5..5ee002604e58 100644 --- a/packages/memory-host-sdk/src/host/session-files.ts +++ b/packages/memory-host-sdk/src/host/session-files.ts @@ -19,7 +19,11 @@ import { isSessionArchiveArtifactName, isSilentReplyPayloadText, isUsageCountedSessionTranscriptFileName, + loadTranscriptEventsSync, parseUsageCountedSessionIdFromFileName, + parseSqliteSessionFileMarker, + readTranscriptStatsSync, + resolveTranscriptSessionKeyBySessionId, resolveSessionTranscriptsDirForAgent, stripInboundMetadata, stripInternalRuntimeContext, @@ -63,11 +67,17 @@ export type SessionFileEntry = { generatedByCronRun?: boolean; }; +export type SessionFileState = Pick; + export type BuildSessionEntryOptions = { /** Optional preclassification from a caller-managed dreaming transcript lookup. */ generatedByDreamingNarrative?: boolean; /** Optional preclassification from a caller-managed cron transcript lookup. */ generatedByCronRun?: boolean; + /** Session key for identity-backed transcript readers. */ + sessionKey?: string; + /** Activity timestamp for transcript sources that do not have filesystem stats. */ + updatedAtMs?: number; /** Override for tests or specialized callers that need a tighter parse yield cadence. */ parseYieldEveryLines?: number; }; @@ -305,15 +315,6 @@ function classifySessionTranscriptCorpusEntries( }; } -function findSessionTranscriptStoreEntryBySessionId( - store: Record, - sessionId: string, -): SessionTranscriptStoreEntry | undefined { - return Object.values(store).find((entry) => { - return typeof entry.sessionId === "string" && entry.sessionId.trim() === sessionId; - }); -} - export function loadDreamingNarrativeTranscriptPathSetForAgent( agentId: string, ): ReadonlySet { @@ -373,6 +374,11 @@ export function sessionPathForFile(absPath: string): string { .replace(/\\/g, "/"); } +/** Returns the logical memory path for a live SQLite-backed session transcript. */ +export function sessionPathForSessionIdentity(agentId: string, sessionId: string): string { + return path.join("sessions", normalizeAgentId(agentId), `${sessionId}.jsonl`).replace(/\\/g, "/"); +} + /** * Parses a deprecated path-shaped memory sync hint only when it points at an * OpenClaw-owned usage-counted transcript in the canonical agent sessions dir. @@ -438,11 +444,7 @@ export function resolveSessionIdentityForTranscriptFile( }; } -/** - * Resolves a storage-neutral memory sync target to the current file-backed - * transcript. The SQLite adapter implements this identity contract without - * deriving a path. - */ +/** Resolves only deprecated path-shaped sync targets; live identity uses corpus entries. */ export function resolveSessionFileForSyncTarget( target: MemorySessionSyncTarget, defaultAgentId?: string, @@ -452,67 +454,7 @@ export function resolveSessionFileForSyncTarget( if (!rawAgentId || !sessionId) { return null; } - const agentId = normalizeAgentId(rawAgentId); - const sessionsDir = resolveSessionTranscriptsDirForAgent(agentId); - const sessionKey = target.sessionKey?.trim(); - let store: Record | null = null; - if (sessionKey) { - store = readSessionTranscriptClassificationStore(path.join(sessionsDir, "sessions.json")); - const persistedPath = resolveSessionStoreTranscriptResolvedPath(sessionsDir, store[sessionKey]); - const canonicalPath = resolveCanonicalSessionSyncFilePath(agentId, persistedPath); - if (canonicalPath) { - return { - agentId, - sessionId, - sessionFile: canonicalPath, - }; - } - } - store ??= readSessionTranscriptClassificationStore(path.join(sessionsDir, "sessions.json")); - const persistedPath = resolveSessionStoreTranscriptResolvedPath( - sessionsDir, - findSessionTranscriptStoreEntryBySessionId(store, sessionId), - ); - const canonicalPath = resolveCanonicalSessionSyncFilePath(agentId, persistedPath); - if (canonicalPath) { - return { - agentId, - sessionId, - sessionFile: canonicalPath, - }; - } - const sessionFile = resolveCanonicalSessionSyncFilePath( - agentId, - path.join(sessionsDir, `${sessionId}.jsonl`), - sessionId, - ); - if (!sessionFile) { - return null; - } - return { - agentId, - sessionId, - sessionFile, - }; -} - -function resolveCanonicalSessionSyncFilePath( - agentId: string, - sessionFile?: string | null, - expectedSessionId?: string, -): string | null { - if (!sessionFile) { - return null; - } - const resolved = path.resolve(sessionFile); - const parsed = parseCanonicalSessionSyncTargetFromPath(resolved); - if (parsed?.agentId !== agentId) { - return null; - } - if (expectedSessionId !== undefined && parsed.sessionId !== expectedSessionId) { - return null; - } - return resolved; + return null; } async function logSessionFileReadFailure(absPath: string, err: unknown): Promise { @@ -700,6 +642,18 @@ function parseSessionTimestampMs( return 0; } +function serializeTranscriptEvent(record: unknown): string | null { + const serialized = JSON.stringify(record); + return typeof serialized === "string" ? serialized : null; +} + +function serializeTranscriptEvents(records: readonly unknown[]): string { + return records + .map(serializeTranscriptEvent) + .filter((line): line is string => line !== null) + .join("\n"); +} + function resolveSessionEntryParseYieldLines(opts: BuildSessionEntryOptions): number { const configured = opts.parseYieldEveryLines; if (typeof configured === "number" && Number.isFinite(configured)) { @@ -708,6 +662,40 @@ function resolveSessionEntryParseYieldLines(opts: BuildSessionEntryOptions): num return SESSION_ENTRY_PARSE_YIELD_LINES; } +export function statSessionEntrySync( + absPath: string, + opts: BuildSessionEntryOptions = {}, +): SessionFileState | null { + const sqliteMarker = parseSqliteSessionFileMarker(absPath); + if (sqliteMarker) { + const stats = readTranscriptStatsSync({ + agentId: sqliteMarker.agentId, + sessionId: sqliteMarker.sessionId, + ...(opts.sessionKey ? { sessionKey: opts.sessionKey } : {}), + storePath: sqliteMarker.storePath, + }); + return { + absPath, + path: sessionPathForSessionIdentity(sqliteMarker.agentId, sqliteMarker.sessionId), + mtimeMs: opts.updatedAtMs ?? stats.maxSeq, + size: stats.sizeBytes, + }; + } + try { + const stat = fsSync.statSync(absPath); + return stat.isFile() + ? { + absPath, + path: sessionPathForFile(absPath), + mtimeMs: stat.mtimeMs, + size: stat.size, + } + : null; + } catch { + return null; + } +} + async function yieldSessionEntryParseIfNeeded( lineIndex: number, everyLines: number, @@ -724,43 +712,94 @@ export async function buildSessionEntry( opts: BuildSessionEntryOptions = {}, ): Promise { try { - const regularFile = await statRegularFile(absPath); - if (regularFile.missing) { - return null; + const sqliteMarker = parseSqliteSessionFileMarker(absPath); + const rawSource = sqliteMarker + ? (() => { + const stats = readTranscriptStatsSync({ + agentId: sqliteMarker.agentId, + sessionId: sqliteMarker.sessionId, + ...(opts.sessionKey ? { sessionKey: opts.sessionKey } : {}), + storePath: sqliteMarker.storePath, + }); + const records = loadTranscriptEventsSync({ + agentId: sqliteMarker.agentId, + sessionId: sqliteMarker.sessionId, + ...(opts.sessionKey ? { sessionKey: opts.sessionKey } : {}), + storePath: sqliteMarker.storePath, + }); + const raw = serializeTranscriptEvents(records); + return { + mtimeMs: opts.updatedAtMs ?? stats.maxSeq, + path: sessionPathForSessionIdentity(sqliteMarker.agentId, sqliteMarker.sessionId), + raw, + size: stats.sizeBytes, + }; + })() + : null; + let raw: string; + let mtimeMs: number; + let size: number; + let memoryPath: string; + if (rawSource) { + raw = rawSource.raw; + mtimeMs = rawSource.mtimeMs; + size = rawSource.size; + memoryPath = rawSource.path; + } else { + const regularFile = await statRegularFile(absPath); + if (regularFile.missing) { + return null; + } + const stat = regularFile.stat; + if (shouldSkipTranscriptFileForDreaming(absPath)) { + return { + path: sessionPathForFile(absPath), + absPath, + mtimeMs: stat.mtimeMs, + size: stat.size, + hash: hashText("\n\n"), + content: "", + lineMap: [], + messageTimestampsMs: [], + }; + } + raw = ( + await retryTransientMemoryRead( + () => readRegularFile({ filePath: absPath }), + `read session transcript ${absPath}`, + ) + ).buffer.toString("utf-8"); + mtimeMs = stat.mtimeMs; + size = stat.size; + memoryPath = sessionPathForFile(absPath); } - const stat = regularFile.stat; - if (shouldSkipTranscriptFileForDreaming(absPath)) { - return { - path: sessionPathForFile(absPath), - absPath, - mtimeMs: stat.mtimeMs, - size: stat.size, - hash: hashText("\n\n"), - content: "", - lineMap: [], - messageTimestampsMs: [], - }; - } - const raw = ( - await retryTransientMemoryRead( - () => readRegularFile({ filePath: absPath }), - `read session transcript ${absPath}`, - ) - ).buffer.toString("utf-8"); const collected: string[] = []; const lineMap: number[] = []; const messageTimestampsMs: number[] = []; const parseYieldEveryLines = resolveSessionEntryParseYieldLines(opts); + const sqliteSessionKey = + sqliteMarker && !opts.sessionKey + ? resolveTranscriptSessionKeyBySessionId({ + agentId: sqliteMarker.agentId, + sessionId: sqliteMarker.sessionId, + storePath: sqliteMarker.storePath, + }) + : undefined; const sessionStoreClassification = - opts.generatedByDreamingNarrative === undefined || opts.generatedByCronRun === undefined + !sqliteMarker && + (opts.generatedByDreamingNarrative === undefined || opts.generatedByCronRun === undefined) ? classifySessionTranscriptFromSessionStore(absPath) : null; let generatedByDreamingNarrative = opts.generatedByDreamingNarrative ?? + (sqliteSessionKey ? isDreamingNarrativeSessionStoreKey(sqliteSessionKey) : undefined) ?? sessionStoreClassification?.generatedByDreamingNarrative ?? false; let generatedByCronRun = - opts.generatedByCronRun ?? sessionStoreClassification?.generatedByCronRun ?? false; + opts.generatedByCronRun ?? + (sqliteSessionKey ? isCronRunSessionKey(sqliteSessionKey) : undefined) ?? + sessionStoreClassification?.generatedByCronRun ?? + false; const allowArchiveRecordCronClassification = isUsageCountedSessionArchiveTranscriptPath(absPath); for (let jsonlIdx = 0, lineStart = 0; lineStart <= raw.length; jsonlIdx++) { @@ -843,10 +882,10 @@ export async function buildSessionEntry( } const content = collected.join("\n"); return { - path: sessionPathForFile(absPath), + path: memoryPath, absPath, - mtimeMs: stat.mtimeMs, - size: stat.size, + mtimeMs, + size, hash: hashText(content + "\n" + lineMap.join(",") + "\n" + messageTimestampsMs.join(",")), content, lineMap, diff --git a/packages/memory-host-sdk/src/host/session-transcript-corpus.ts b/packages/memory-host-sdk/src/host/session-transcript-corpus.ts index 86e3126b65cd..45dac9471ba6 100644 --- a/packages/memory-host-sdk/src/host/session-transcript-corpus.ts +++ b/packages/memory-host-sdk/src/host/session-transcript-corpus.ts @@ -11,6 +11,7 @@ import { isSessionArchiveArtifactName, isUsageCountedSessionTranscriptFileName, listSessionEntries, + parseSqliteSessionFileMarker, parseUsageCountedSessionIdFromFileName, resolveSessionAgentId, resolveSessionFilePath, @@ -18,10 +19,7 @@ import { type SessionEntry, } from "./openclaw-runtime-session.js"; -type SessionTranscriptCorpusArtifactKind = - | "active-session" - | "archive-artifact" - | "orphan-file-artifact"; +type SessionTranscriptCorpusArtifactKind = "active-session" | "archive-artifact"; export type SessionTranscriptCorpusEntry = { agentId: string; @@ -29,6 +27,10 @@ export type SessionTranscriptCorpusEntry = { sessionId: string; artifactKind: SessionTranscriptCorpusArtifactKind; sessionKey?: string; + /** Present when an active transcript is addressed by SQLite identity, not a JSONL path. */ + transcriptSource?: "sqlite"; + /** Session entry activity timestamp used when the source has no filesystem stat. */ + updatedAtMs?: number; /** True when this transcript belongs to an internal dreaming narrative run. */ generatedByDreamingNarrative?: boolean; /** True when this transcript belongs to an isolated cron run session. */ @@ -76,27 +78,58 @@ function extractAgentIdFromSessionPath(absPath: string): string | null { return parts[sessionsIndex - 1] || null; } -function resolveSessionStoreTranscriptCorpusPath( +type ResolvedSessionStoreCorpusSource = { + sessionFile: string; + sessionId: string; + transcriptSource: "sqlite" | "file"; +}; + +function resolveSessionStoreTranscriptCorpusSource( agentId: string, sessionsDir: string, + storePath: string, entry: { sessionFile?: unknown; sessionId?: unknown } | undefined, -): string | null { +): ResolvedSessionStoreCorpusSource | null { const sessionFile = typeof entry?.sessionFile === "string" && entry.sessionFile.trim().length > 0 ? entry.sessionFile.trim() : undefined; - const sessionId = + const sqliteMarker = sessionFile ? parseSqliteSessionFileMarker(sessionFile) : undefined; + const explicitSessionId = typeof entry?.sessionId === "string" && entry.sessionId.trim().length > 0 ? entry.sessionId.trim() - : sessionFile - ? parseUsageCountedSessionIdFromFileName(path.basename(sessionFile)) - : null; + : null; + const sessionId = + explicitSessionId ?? + sqliteMarker?.sessionId ?? + (sessionFile ? parseUsageCountedSessionIdFromFileName(path.basename(sessionFile)) : null); if (!sessionId) { return null; } + if (sqliteMarker) { + if (!sessionFile) { + return null; + } + if ( + sqliteMarker.sessionId !== sessionId || + normalizeAgentId(sqliteMarker.agentId) !== normalizeAgentId(agentId) || + normalizeComparablePath(sqliteMarker.storePath) !== normalizeComparablePath(storePath) + ) { + return null; + } + return { + sessionFile, + sessionId, + transcriptSource: "sqlite", + }; + } try { if (!sessionFile) { - return resolveSessionFilePath(sessionId, undefined, { agentId, sessionsDir }); + return { + sessionFile: resolveSessionFilePath(sessionId, undefined, { agentId, sessionsDir }), + sessionId, + transcriptSource: "file", + }; } const resolved = resolveSessionFilePath( sessionId, @@ -114,7 +147,7 @@ function resolveSessionStoreTranscriptCorpusPath( return null; } return normalizeRealComparablePath(resolved) === normalizeRealComparablePath(candidate) - ? candidate + ? { sessionFile: candidate, sessionId, transcriptSource: "file" } : null; } const pathAgentId = extractAgentIdFromSessionPath(sessionFile); @@ -122,7 +155,7 @@ function resolveSessionStoreTranscriptCorpusPath( return null; } return normalizeRealComparablePath(resolved) === normalizeRealComparablePath(sessionFile) - ? sessionFile + ? { sessionFile, sessionId, transcriptSource: "file" } : null; } catch { return null; @@ -204,29 +237,26 @@ function collectCronGeneratedSessionKeys( return cronGeneratedKeys; } -function isRegularSessionTranscriptFile(absPath: string): boolean { - try { - return fsSync.lstatSync(absPath).isFile(); - } catch { - return false; - } -} - function toSessionStoreCorpusEntry( agentId: string, sessionsDir: string, + storePath: string, summary: SessionEntrySummary, cronGeneratedSessionKeys: ReadonlySet, ): SessionTranscriptCorpusEntry | null { - const sessionFile = resolveSessionStoreTranscriptCorpusPath(agentId, sessionsDir, summary.entry); - if (!sessionFile || !isUsageCountedSessionTranscriptFileName(path.basename(sessionFile))) { + const source = resolveSessionStoreTranscriptCorpusSource( + agentId, + sessionsDir, + storePath, + summary.entry, + ); + if (!source) { return null; } - const sessionId = - typeof summary.entry.sessionId === "string" && summary.entry.sessionId.trim() - ? summary.entry.sessionId.trim() - : parseUsageCountedSessionIdFromFileName(path.basename(sessionFile)); - if (!sessionId) { + if ( + source.transcriptSource === "file" && + !isUsageCountedSessionTranscriptFileName(path.basename(source.sessionFile)) + ) { return null; } const sessionKey = summary.sessionKey.trim(); @@ -238,8 +268,12 @@ function toSessionStoreCorpusEntry( return { agentId, artifactKind: "active-session", - sessionFile, - sessionId, + sessionFile: source.sessionFile, + sessionId: source.sessionId, + ...(source.transcriptSource === "sqlite" ? { transcriptSource: "sqlite" as const } : {}), + ...(source.transcriptSource === "sqlite" && Number.isFinite(summary.entry.updatedAt) + ? { updatedAtMs: summary.entry.updatedAt } + : {}), ...(sessionKey ? { sessionKey } : {}), ...(classification.generatedByDreamingNarrative ? { generatedByDreamingNarrative: true } : {}), ...(classification.generatedByCronRun ? { generatedByCronRun: true } : {}), @@ -253,6 +287,7 @@ function listSessionTranscriptArtifactFiles(sessionsDir: string): string[] { .filter((entry) => entry.isFile()) .map((entry) => entry.name) .filter((name) => isUsageCountedSessionTranscriptFileName(name)) + .filter((name) => isSessionArchiveArtifactName(name)) .map((name) => path.join(sessionsDir, name)); } catch { return []; @@ -262,6 +297,7 @@ function listSessionTranscriptArtifactFiles(sessionsDir: string): string[] { function classifyTranscriptArtifact( artifactPath: string, activeEntriesByPath: ReadonlyMap, + activeEntriesBySessionId: ReadonlyMap, ): { generatedByDreamingNarrative: boolean; generatedByCronRun: boolean; @@ -277,9 +313,9 @@ function classifyTranscriptArtifact( const primarySessionId = parseUsageCountedSessionIdFromFileName(path.basename(artifactPath)); const primaryEntry = primarySessionId && isSessionArchiveArtifactName(path.basename(artifactPath)) - ? activeEntriesByPath.get( + ? (activeEntriesByPath.get( normalizeRealComparablePath(path.join(sessionsDir, `${primarySessionId}.jsonl`)), - ) + ) ?? activeEntriesBySessionId.get(primarySessionId)) : undefined; return { generatedByDreamingNarrative: primaryEntry?.generatedByDreamingNarrative === true, @@ -291,18 +327,23 @@ function toArtifactCorpusEntry( agentId: string, artifactPath: string, activeEntriesByPath: ReadonlyMap, + activeEntriesBySessionId: ReadonlyMap, ): SessionTranscriptCorpusEntry | null { const sessionId = parseUsageCountedSessionIdFromFileName(path.basename(artifactPath)); if (!sessionId) { return null; } - const artifactKind = isSessionArchiveArtifactName(path.basename(artifactPath)) - ? "archive-artifact" - : "orphan-file-artifact"; - const classification = classifyTranscriptArtifact(artifactPath, activeEntriesByPath); + if (!isSessionArchiveArtifactName(path.basename(artifactPath))) { + return null; + } + const classification = classifyTranscriptArtifact( + artifactPath, + activeEntriesByPath, + activeEntriesBySessionId, + ); return { agentId, - artifactKind, + artifactKind: "archive-artifact", sessionFile: artifactPath, sessionId, ...(classification.generatedByDreamingNarrative ? { generatedByDreamingNarrative: true } : {}), @@ -330,6 +371,7 @@ export function listSessionTranscriptCorpusEntriesForAgentSync( !configuredStore.includes("{agentId}") && !isAgentOwnedFixedStore; const activeEntriesByPath = new Map(); + const activeEntriesBySessionId = new Map(); const activeEntryOwnersByPath = new Map(); const artifactDirsByPath = new Map(); rememberArtifactDir(artifactDirsByPath, sessionsDir); @@ -355,22 +397,29 @@ export function listSessionTranscriptCorpusEntriesForAgentSync( const entry = toSessionStoreCorpusEntry( ownerAgentId, sessionsDir, + storePath, summary, cronGeneratedSessionKeys, ); if (!entry) { continue; } - const normalizedEntryPath = normalizeRealComparablePath(entry.sessionFile); - activeEntryOwnersByPath.set(normalizedEntryPath, ownerAgentId); - rememberArtifactDir(artifactDirsByPath, path.dirname(entry.sessionFile)); + const normalizedEntryPath = + entry.transcriptSource === "sqlite" ? null : normalizeRealComparablePath(entry.sessionFile); + if (normalizedEntryPath) { + activeEntryOwnersByPath.set(normalizedEntryPath, ownerAgentId); + rememberArtifactDir(artifactDirsByPath, path.dirname(entry.sessionFile)); + } if (ownerAgentId === normalizedAgentId) { - activeEntriesByPath.set(normalizedEntryPath, entry); + activeEntriesBySessionId.set(entry.sessionId, entry); + if (normalizedEntryPath) { + activeEntriesByPath.set(normalizedEntryPath, entry); + } } } const includeUnownedArtifacts = !isSharedFixedStore; - const corpusEntries = [...activeEntriesByPath.values()].filter((entry) => - isRegularSessionTranscriptFile(entry.sessionFile), + const corpusEntries = [...activeEntriesBySessionId.values()].filter( + (entry) => entry.transcriptSource === "sqlite", ); const scannedArtifactPaths = new Set(); for (const artifactDir of artifactDirsByPath.values()) { @@ -402,7 +451,12 @@ export function listSessionTranscriptCorpusEntriesForAgentSync( if (!primaryOwner && !includeUnownedArtifacts) { continue; } - const entry = toArtifactCorpusEntry(normalizedAgentId, artifactPath, activeEntriesByPath); + const entry = toArtifactCorpusEntry( + normalizedAgentId, + artifactPath, + activeEntriesByPath, + activeEntriesBySessionId, + ); if (entry) { corpusEntries.push(entry); } diff --git a/packages/memory-host-sdk/src/host/types.ts b/packages/memory-host-sdk/src/host/types.ts index 10d3761a5da5..837ffe594629 100644 --- a/packages/memory-host-sdk/src/host/types.ts +++ b/packages/memory-host-sdk/src/host/types.ts @@ -46,11 +46,8 @@ export type MemorySyncParams = { force?: boolean; /** Storage-neutral session transcript targets to refresh. */ sessions?: MemorySessionSyncTarget[]; - /** - * @deprecated Use `sessions` with `{ agentId, sessionId, sessionKey? }`. - * During the deprecation window only canonical OpenClaw transcript paths are accepted. - */ - sessionFiles?: string[]; + /** Archive/support transcript files to refresh without treating paths as active session identity. */ + archiveFiles?: string[]; progress?: (update: MemorySyncProgressUpdate) => void; }; diff --git a/qa/scenarios/channels/native-command-session-target.yaml b/qa/scenarios/channels/native-command-session-target.yaml index 942c7c743e6b..ffa7ce72eb4d 100644 --- a/qa/scenarios/channels/native-command-session-target.yaml +++ b/qa/scenarios/channels/native-command-session-target.yaml @@ -70,7 +70,7 @@ flow: - lambda: async: true expr: "env.gateway.call('sessions.list', {}).then((result) => result.sessions?.find((session) => session.hasActiveRun === true))" - - expr: liveTurnTimeoutMs(env, 15000) + - expr: liveTurnTimeoutMs(env, 30000) - 100 - set: startIndex value: @@ -115,6 +115,6 @@ flow: textIncludes: expr: config.recoveryMarker timeoutMs: - expr: liveTurnTimeoutMs(env, 60000) + expr: liveTurnTimeoutMs(env, 90000) saveAs: recoveryReply detailsExpr: "`native command reply=${abortReply.text}; recovery reply=${recoveryReply.text}`" diff --git a/scripts/check-changed.d.mts b/scripts/check-changed.d.mts index 31353b3725d8..20217c5d395a 100644 --- a/scripts/check-changed.d.mts +++ b/scripts/check-changed.d.mts @@ -39,6 +39,7 @@ export function shouldRunShrinkwrapGuard(paths: string[]): boolean; export function shouldRunPromptSnapshotCheck(paths: string[]): boolean; export function shouldRunPromptSnapshotOwnerTest(paths: string[]): boolean; export function shouldRunRuntimeSidecarBaselineCheck(paths: string[]): boolean; +export function shouldRunSqliteSessionSchemaBaselineCheck(paths: string[]): boolean; export function shouldRunCanvasA2uiNativeResourceCheck(paths: string[]): boolean; export function shouldRunAppcastOwnerTest(paths: string[]): boolean; export function shouldRunTestTempCreationReport(paths: string[]): boolean; diff --git a/scripts/check-changed.mjs b/scripts/check-changed.mjs index 189daab97696..3ed21118fe76 100644 --- a/scripts/check-changed.mjs +++ b/scripts/check-changed.mjs @@ -38,6 +38,8 @@ const PROMPT_SNAPSHOT_OWNER_TEST_PATH_RE = /^(?:scripts\/(?:generate-prompt-snapshots\.ts|prompt-snapshot-files\.ts|sync-codex-model-prompt-fixture\.ts)|test\/helpers\/agents\/(?:happy-path-prompt-snapshots|prompt-snapshot-paths)\.ts|test\/fixtures\/agents\/prompt-snapshots\/codex-model-catalog\/.+)$/u; const RUNTIME_SIDECAR_BASELINE_PATH_RE = /^(?:scripts\/generate-runtime-sidecar-paths-baseline\.ts|scripts\/lib\/bundled-runtime-sidecar-paths\.json|src\/plugins\/runtime-sidecar-paths(?:-baseline)?\.ts)$/u; +const SQLITE_SESSION_SCHEMA_BASELINE_PATH_RE = + /^(?:src\/state\/openclaw-agent-schema\.sql|scripts\/(?:generate-sqlite-session-schema-baseline\.ts|lib\/sqlite-session-schema-baseline\.ts)|test\/scripts\/sqlite-session-schema-baseline\.test\.ts|docs\/\.generated\/sqlite-session-transcript-schema-baseline\.sha256)$/u; const CANVAS_A2UI_NATIVE_RESOURCE_PATH_RE = /^(?:pnpm-lock\.yaml$|apps\/shared\/OpenClawKit\/Sources\/OpenClawKit\/Resources\/CanvasA2UI\/|extensions\/canvas\/(?:package\.json$|scripts\/bundle-a2ui\.mjs$|src\/host\/a2ui(?:\/(?:index\.html|a2ui\.bundle\.js|\.bundle\.hash)$|-app\/))|scripts\/(?:bundle-a2ui|sync-native-a2ui)\.mjs$)/u; const CORE_OXLINT_TS_CONFIG = "config/tsconfig/oxlint.core.json"; @@ -200,6 +202,13 @@ export function shouldRunRuntimeSidecarBaselineCheck(paths) { return paths.some((changedPath) => RUNTIME_SIDECAR_BASELINE_PATH_RE.test(changedPath)); } +/** Returns whether changed files can affect the sessions/transcripts SQLite schema baseline. */ +export function shouldRunSqliteSessionSchemaBaselineCheck(paths) { + return paths.some((changedPath) => + SQLITE_SESSION_SCHEMA_BASELINE_PATH_RE.test(normalizeChangedPath(changedPath)), + ); +} + export function shouldRunCanvasA2uiNativeResourceCheck(paths) { return paths.some((changedPath) => CANVAS_A2UI_NATIVE_RESOURCE_PATH_RE.test(normalizeChangedPath(changedPath)), @@ -322,6 +331,9 @@ export function createChangedCheckPlan(result, options = {}) { baseEnv, ); } + if (shouldRunSqliteSessionSchemaBaselineCheck(result.paths)) { + add("SQLite sessions/transcripts schema baseline", ["sqlite:sessions-schema:check"]); + } if (shouldRunCanvasA2uiNativeResourceCheck(result.paths)) { addCommand( "Canvas A2UI native resource sync", diff --git a/scripts/check-kysely-guardrails.mjs b/scripts/check-kysely-guardrails.mjs index b040f8ff154f..924e02fc6187 100644 --- a/scripts/check-kysely-guardrails.mjs +++ b/scripts/check-kysely-guardrails.mjs @@ -47,9 +47,15 @@ const rawSqliteAllowPathGroups = { ], "backup snapshot maintenance": ["src/commands/backup-verify.ts", "src/infra/backup-create.ts"], "agent auth profile read-only bootstrap": ["src/agents/auth-profiles/sqlite.ts"], - "read-only SQLite status probes": ["src/commands/status.scan.shared.ts"], + "read-only SQLite status probes": [ + "src/commands/doctor-db-bloat.ts", + "src/commands/status.scan.shared.ts", + ], "doctor legacy state migration": [ "src/commands/doctor/cron/migration-ledger.ts", + "src/commands/doctor-session-sqlite.ts", + "src/commands/doctor-session-sqlite-compact.ts", + "src/commands/doctor-session-sqlite-readers.ts", "src/infra/state-migrations.ts", "src/infra/state-migrations.debug-proxy.ts", ], diff --git a/scripts/check-session-accessor-boundary.mjs b/scripts/check-session-accessor-boundary.mjs index 0a8cbbae7b05..f041c455e0ce 100644 --- a/scripts/check-session-accessor-boundary.mjs +++ b/scripts/check-session-accessor-boundary.mjs @@ -66,15 +66,7 @@ const sessionStoreRuntimeFileBackedCompatNames = new Set([ ]); const embeddedAgentSessionFileRuntimeNames = new Set(["resolveSessionFilePath"]); -export const allowedSessionStoreRuntimeFileBackedCompatExports = new Set([ - "loadSessionStore", - "readLatestAssistantTextFromSessionTranscript", - "resolveAndPersistSessionFile", - "resolveSessionFilePath", - "resolveSessionStoreEntry", - "saveSessionStore", - "updateSessionStore", -]); +export const allowedSessionStoreRuntimeFileBackedCompatExports = new Set([]); export const migratedSessionAccessorFiles = new Set([ "packages/memory-host-sdk/src/host/session-files.ts", diff --git a/scripts/check-sqlite-transaction-boundary.d.mts b/scripts/check-sqlite-transaction-boundary.d.mts new file mode 100644 index 000000000000..eb7b35b67131 --- /dev/null +++ b/scripts/check-sqlite-transaction-boundary.d.mts @@ -0,0 +1,9 @@ +export type SqliteTransactionBoundaryViolation = { + line: number; + reason: string; +}; + +export function findSqliteTransactionBoundaryViolations( + content: string, + fileName?: string, +): SqliteTransactionBoundaryViolation[]; diff --git a/scripts/check-sqlite-transaction-boundary.mjs b/scripts/check-sqlite-transaction-boundary.mjs new file mode 100644 index 000000000000..54fe73021b9b --- /dev/null +++ b/scripts/check-sqlite-transaction-boundary.mjs @@ -0,0 +1,173 @@ +#!/usr/bin/env node + +import ts from "typescript"; +import { + collectFileViolations, + resolveRepoRoot, + resolveSourceRoots, + runAsScript, + toLine, + unwrapExpression, +} from "./lib/ts-guard-utils.mjs"; + +const removedAsyncTransactionNames = new Set([ + "runOpenClawAgentWriteTransactionAsync", + "runSqliteImmediateTransactionAsync", +]); +const synchronousTransactionCallbackIndexes = new Map([ + ["runOpenClawAgentWriteTransaction", 0], + ["runSqliteImmediateTransactionSync", 1], +]); + +function expressionName(expression) { + const unwrapped = unwrapExpression(expression); + if (ts.isIdentifier(unwrapped)) { + return unwrapped.text; + } + if (ts.isPropertyAccessExpression(unwrapped)) { + return unwrapped.name.text; + } + if (ts.isElementAccessExpression(unwrapped) && ts.isStringLiteral(unwrapped.argumentExpression)) { + return unwrapped.argumentExpression.text; + } + return undefined; +} + +function hasAsyncModifier(node) { + return node.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword) === true; +} + +function collectLocalAsyncFunctionNames(sourceFile) { + const names = new Set(); + const visit = (node) => { + if (ts.isFunctionDeclaration(node) && node.name && hasAsyncModifier(node)) { + names.add(node.name.text); + } + if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer) { + const initializer = unwrapExpression(node.initializer); + if ( + (ts.isArrowFunction(initializer) || ts.isFunctionExpression(initializer)) && + hasAsyncModifier(initializer) + ) { + names.add(node.name.text); + } + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + return names; +} + +function collectSynchronousTransactionAliases(sourceFile) { + const aliases = new Map(); + for (const statement of sourceFile.statements) { + if (!ts.isImportDeclaration(statement)) { + continue; + } + const bindings = statement.importClause?.namedBindings; + if (!bindings || !ts.isNamedImports(bindings)) { + continue; + } + for (const specifier of bindings.elements) { + const importedName = specifier.propertyName?.text ?? specifier.name.text; + if (synchronousTransactionCallbackIndexes.has(importedName)) { + aliases.set(specifier.name.text, importedName); + } + } + } + return aliases; +} + +function isAsyncCallback(expression, localAsyncFunctionNames) { + const unwrapped = unwrapExpression(expression); + if (ts.isArrowFunction(unwrapped) || ts.isFunctionExpression(unwrapped)) { + return hasAsyncModifier(unwrapped); + } + return ts.isIdentifier(unwrapped) && localAsyncFunctionNames.has(unwrapped.text); +} + +export function findSqliteTransactionBoundaryViolations(content, fileName = "source.ts") { + const sourceFile = ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, true); + const localAsyncFunctionNames = collectLocalAsyncFunctionNames(sourceFile); + const transactionAliases = collectSynchronousTransactionAliases(sourceFile); + const violations = []; + + const visit = (node) => { + if (ts.isImportDeclaration(node)) { + const bindings = node.importClause?.namedBindings; + if (bindings && ts.isNamedImports(bindings)) { + for (const specifier of bindings.elements) { + const importedName = specifier.propertyName?.text ?? specifier.name.text; + if (removedAsyncTransactionNames.has(importedName)) { + violations.push({ + line: toLine(sourceFile, specifier), + reason: `imports removed async SQLite transaction primitive "${importedName}"`, + }); + } + } + } + } + + if ( + ((ts.isFunctionDeclaration(node) && node.name) || + (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name))) && + removedAsyncTransactionNames.has(node.name.text) + ) { + violations.push({ + line: toLine(sourceFile, node.name), + reason: `declares removed async SQLite transaction primitive "${node.name.text}"`, + }); + } + + if (ts.isCallExpression(node)) { + const calledName = expressionName(node.expression); + if (calledName && removedAsyncTransactionNames.has(calledName)) { + violations.push({ + line: toLine(sourceFile, node.expression), + reason: `calls removed async SQLite transaction primitive "${calledName}"`, + }); + } + const canonicalName = calledName + ? (transactionAliases.get(calledName) ?? calledName) + : undefined; + const callbackIndex = canonicalName + ? synchronousTransactionCallbackIndexes.get(canonicalName) + : undefined; + const callback = callbackIndex === undefined ? undefined : node.arguments[callbackIndex]; + if (canonicalName && callback && isAsyncCallback(callback, localAsyncFunctionNames)) { + violations.push({ + line: toLine(sourceFile, callback), + reason: `passes an async callback to synchronous SQLite transaction helper "${canonicalName}"`, + }); + } + } + + ts.forEachChild(node, visit); + }; + visit(sourceFile); + return violations; +} + +async function main() { + const repoRoot = resolveRepoRoot(import.meta.url); + const violations = await collectFileViolations({ + repoRoot, + sourceRoots: resolveSourceRoots(repoRoot, ["src", "extensions", "packages"]), + skipFile: (filePath) => /(^|[\\/])dist([\\/]|$)/.test(filePath), + findViolations: findSqliteTransactionBoundaryViolations, + }); + if (violations.length === 0) { + console.log("SQLite transaction boundary guard passed."); + return; + } + console.error("Found asynchronous SQLite transaction work:"); + for (const violation of violations) { + console.error(`- ${violation.path}:${violation.line}: ${violation.reason}`); + } + console.error( + "Complete asynchronous preparation before the transaction, then validate and apply inside a synchronous transaction callback.", + ); + process.exit(1); +} + +runAsScript(import.meta.url, main); diff --git a/scripts/e2e/lib/session-log-mentions.ts b/scripts/e2e/lib/session-log-mentions.ts index a338debf41bd..bcb8c24af342 100644 --- a/scripts/e2e/lib/session-log-mentions.ts +++ b/scripts/e2e/lib/session-log-mentions.ts @@ -1,6 +1,7 @@ // Session Log Mentions script supports OpenClaw repository automation. import fs from "node:fs/promises"; import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { readPositiveIntEnv } from "./env-limits.mjs"; type SessionLogMentionLimits = { @@ -69,15 +70,40 @@ function recordRole(record: unknown): string | undefined { return typeof message.role === "string" ? message.role : undefined; } -function shouldScanSessionLogLine(line: string): boolean { +function collectStringLeaves(value: unknown, output: string[]) { + if (typeof value === "string") { + output.push(value); + return; + } + if (Array.isArray(value)) { + for (const item of value) { + collectStringLeaves(item, output); + } + return; + } + if (!value || typeof value !== "object") { + return; + } + for (const item of Object.values(value)) { + collectStringLeaves(item, output); + } +} + +function sessionLogScanText(line: string): string | null { const trimmed = line.trim(); if (!trimmed) { - return false; + return null; } try { - return recordRole(JSON.parse(trimmed)) !== "user"; + const record = JSON.parse(trimmed) as unknown; + if (recordRole(record) === "user") { + return null; + } + const strings: string[] = []; + collectStringLeaves(record, strings); + return strings.join("\n"); } catch { - return true; + return line; } } @@ -104,11 +130,16 @@ export async function countSessionLogMentions(params: { }): Promise> { const limits = params.limits ?? readSessionLogMentionLimits(); const counts = createCounts(params.needles); + const addCounts = (nextCounts: Record) => { + for (const [key, count] of Object.entries(nextCounts)) { + counts[key] = (counts[key] ?? 0) + count; + } + }; let files: string[]; try { files = await fs.readdir(params.sessionsDir); } catch { - return counts; + files = []; } let totalBytes = 0; @@ -140,13 +171,91 @@ export async function countSessionLogMentions(params: { limit: limits.fileMaxBytes, }); for (const line of raw.split(/\r?\n/u)) { - if (!shouldScanSessionLogLine(line)) { + const scanText = sessionLogScanText(line); + if (scanText === null) { continue; } for (const [key, needle] of Object.entries(params.needles)) { - counts[key] += countOccurrences(line, needle); + counts[key] += countOccurrences(scanText, needle); } } } + addCounts( + await countSqliteTranscriptMentions({ + limits, + needles: params.needles, + sessionsDir: params.sessionsDir, + startingBytes: totalBytes, + }), + ); return counts; } + +function resolveAgentSqlitePathFromSessionsDir(sessionsDir: string): string | null { + if (path.basename(sessionsDir) !== "sessions") { + return null; + } + return path.join(path.dirname(sessionsDir), "agent", "openclaw-agent.sqlite"); +} + +async function countSqliteTranscriptMentions(params: { + limits: SessionLogMentionLimits; + needles: SessionLogNeedles; + sessionsDir: string; + startingBytes: number; +}): Promise> { + const counts = createCounts(params.needles); + const sqlitePath = resolveAgentSqlitePathFromSessionsDir(params.sessionsDir); + if (!sqlitePath) { + return counts; + } + const stat = await fs.stat(sqlitePath).catch(() => null); + if (!stat?.isFile()) { + return counts; + } + let totalBytes = params.startingBytes; + let db: DatabaseSync | null = null; + try { + db = new DatabaseSync(sqlitePath, { readOnly: true }); + const hasTranscriptEvents = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'transcript_events'") + .get(); + if (!hasTranscriptEvents) { + return counts; + } + const rows = db.prepare("SELECT event_json FROM transcript_events ORDER BY session_id, seq"); + for (const row of rows.iterate() as Iterable<{ event_json?: unknown }>) { + if (typeof row.event_json !== "string") { + continue; + } + const byteCount = Buffer.byteLength(row.event_json, "utf8"); + assertWithinLimit({ + byteCount, + filePath: sqlitePath, + label: "per-file", + limit: params.limits.fileMaxBytes, + }); + totalBytes += byteCount; + assertWithinLimit({ + byteCount: totalBytes, + label: "total", + limit: params.limits.totalMaxBytes, + }); + const scanText = sessionLogScanText(row.event_json); + if (scanText === null) { + continue; + } + for (const [key, needle] of Object.entries(params.needles)) { + counts[key] += countOccurrences(scanText, needle); + } + } + return counts; + } catch (error) { + if (error && typeof error === "object" && "code" in error) { + throw error; + } + return counts; + } finally { + db?.close(); + } +} diff --git a/scripts/e2e/sqlite-sessions-transcripts-flip-proof.ts b/scripts/e2e/sqlite-sessions-transcripts-flip-proof.ts new file mode 100644 index 000000000000..0a2cc7de1954 --- /dev/null +++ b/scripts/e2e/sqlite-sessions-transcripts-flip-proof.ts @@ -0,0 +1,2642 @@ +// SQLite sessions/transcripts flip proof runner exercises an isolated live gateway lifecycle. +import { spawn, type ChildProcessByStdio } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fsSync from "node:fs"; +import fs from "node:fs/promises"; +import net from "node:net"; +import path from "node:path"; +import { DatabaseSync, type SQLInputValue } from "node:sqlite"; +import type { Readable } from "node:stream"; +import { fileURLToPath } from "node:url"; +import { + readSessionArchiveContentSync, + stripSessionArchiveCompressionSuffix, +} from "../../src/config/sessions/archive-compression.js"; +import { + appendTranscriptMessage, + type TranscriptEvent, +} from "../../src/config/sessions/session-accessor.js"; +import { importSqliteSessionRows } from "../../src/config/sessions/session-accessor.sqlite.js"; +import { formatSqliteSessionFileMarker } from "../../src/config/sessions/sqlite-marker.js"; +import type { SessionEntry } from "../../src/config/sessions/types.js"; +import { + connectGatewayClient, + disconnectGatewayClient, +} from "../../src/gateway/test-helpers.e2e.js"; +import { + getSessionEntry as getSdkSessionEntry, + listSessionEntries as listSdkSessionEntries, + loadTranscriptEventsSync as loadSdkTranscriptEventsSync, +} from "../../src/plugin-sdk/session-store-runtime.js"; +import { + appendSessionTranscriptMessageByIdentity, + readLatestAssistantTextByIdentity, + readSessionTranscriptEvents, + resolveSessionTranscriptIdentity, +} from "../../src/plugin-sdk/session-transcript-runtime.js"; +import { sleep } from "../../src/utils.js"; +import { createOpenClawTestInstance } from "../../test/helpers/openclaw-test-instance.js"; + +type DoctorMode = "import" | "inspect" | "validate" | "restore"; +type ProofChildProcess = ChildProcessByStdio; + +type DoctorMigrationRunEvidence = { + failureReportJsonPath?: string; + failureReportMarkdownPath?: string; + manifestPath: string; + runId: string; +}; + +type DoctorRestoreEvidence = { + conflicts: number; + manifestPaths: string[]; + restoredFiles: string[]; + skippedFiles: string[]; +}; + +type DoctorCommandEvidence = { + code: number | null; + mode: DoctorMode; + migrationRun?: DoctorMigrationRunEvidence; + restore?: DoctorRestoreEvidence; + stderrTail: string; + stdoutTail: string; + totals?: Record; +}; + +type FileInventoryEntry = { + archiveReason?: "bak" | "deleted" | "reset"; + archiveSessionId?: string; + path: string; + bytes: number; + jsonlTypes?: string[]; + lines?: number; + messageRoles?: string[]; + messageTexts?: string[]; + textTail?: string; +}; + +type SqliteSessionEntryEvidence = { + entry?: Record; + sessionId: string; + sessionKey: string; + trajectoryEvents: number; + transcriptEvents: number; +}; + +type SqliteEvidence = { + exists: boolean; + path: string; + sessionEntries: number; + sessions: number; + trajectoryRuntimeEvents: number; + transcriptEvents: number; + trackedEntries: SqliteSessionEntryEvidence[]; +}; + +type ProofCheckpoint = { + activeJsonl: FileInventoryEntry[]; + archiveArtifacts: FileInventoryEntry[]; + doctor?: DoctorCommandEvidence; + gatewayLogTail?: string; + label: string; + legacyStateJsonl: FileInventoryEntry[]; + sqlite: SqliteEvidence; +}; + +type PluginSdkConsumerEvidence = { + activeJsonlForSessionExists: boolean; + activeTrajectoryPointerForSessionExists: boolean; + activeTrajectoryRuntimeSidecarForSessionExists: boolean; + activeTrajectorySessionSidecarForSessionExists: boolean; + appendedMessageId: string; + identityMemoryKey: string; + latestAssistantTextBeforeAppend: string; + latestAssistantTextAfterAppend: string; + listedSessionKeys: string[]; + sessionFileMarker: string; + sessionId: string; + sessionKey: string; + storeTranscriptEvents: number; + transcriptEventsAfterAppend: number; + transcriptEventsBeforeAppend: number; +}; + +type ManualCompactionEvidence = { + checkpointCount: number; + compacted: boolean; + rowCountAfter: number; + rowCountBefore: number; + sessionFileMarker: string; + sessionId: string; + sessionKey: string; +}; + +type ScaleMigrationEvidence = { + importedSessionKeys: string[]; + minTranscriptEventsPerSession: number; + seededEvents: number; + seededSessions: number; + startupImportElapsedMs: number; +}; + +type DowngradeReupgradeEvidence = { + activeJsonlArchived: boolean; + doctorImportedEntries: number; + doctorImportedTranscriptEvents: number; + trajectoryPointerArchived: boolean; + trajectoryPointerSourceRemoved: boolean; + trajectorySidecarArchived: boolean; + trajectorySidecarSourceRemoved: boolean; + sessionId: string; + sessionKey: string; + transcriptEvents: number; +}; + +type BusyContentionEvidence = { + childExitCode: number | null; + childSignal: NodeJS.Signals | null; + elapsedMs: number; + holdMs: number; + sessionId: string; + sessionKey: string; + transcriptEvents: number; +}; + +type SecondStartupAfterResetEvidence = { + activeJsonlForSessionExists: boolean; + historyContainsPostResetAppend: boolean; + sessionId: string; + sessionKey: string; + transcriptEvents: number; +}; + +type RollbackRestoreEvidence = { + archivePath: string; + archivedBeforeRestore: boolean; + failedManifestIssueCode: string; + idempotentRestoreSkippedFiles: string[]; + manifestPath: string; + restoredFiles: string[]; + sourcePath: string; + sourceRestored: boolean; + sqliteStillExists: boolean; +}; + +export type SqliteSessionsTranscriptsFlipProofReport = { + ok: boolean; + agentId: string; + checkpoints: ProofCheckpoint[]; + cleanupPruneSessionKey: string; + concurrentDeleteSessionKey: string; + concurrentResetSessionKey: string; + concurrentSendSessionKey: string; + deleteSessionKey: string; + failures: string[]; + fullTurnAssistantText: string; + fullTurnSessionKey: string; + gatewayEntrypoint: string[]; + legacySessionId: string; + manualCompaction?: ManualCompactionEvidence; + manualCompactionSessionKey: string; + mockOpenAiRequestLog: string; + oldStateSessionKeys: string[]; + pluginSdkConsumer?: PluginSdkConsumerEvidence; + pluginSdkSessionKey: string; + resetSessionKey: string; + rollbackRestore?: RollbackRestoreEvidence; + busyContention?: BusyContentionEvidence; + downgradeReupgrade?: DowngradeReupgradeEvidence; + scaleMigration?: ScaleMigrationEvidence; + secondStartupAfterReset?: SecondStartupAfterResetEvidence; + sharedSessionKeys: string[]; + stateDir: string; +}; + +type ProofContext = { + activeSessionsDir: string; + agentDbPath: string; + agentId: string; + archiveRoots: string[]; + cleanupPruneSessionKey: string; + concurrentDeleteSessionKey: string; + concurrentResetSessionKey: string; + concurrentSendSessionKey: string; + deleteSessionKey: string; + fullTurnAssistantText: string; + fullTurnSessionKey: string; + legacySessionsDir: string; + legacySessionId: string; + manualCompactionSessionKey: string; + mockOpenAiRequestLog: string; + oldStateSessionKeys: string[]; + pluginSdkAppendText: string; + pluginSdkSessionKey: string; + resetSessionKey: string; + sharedSessionKeys: string[]; + stateDir: string; + storePath: string; + trackedSessionKeys: string[]; +}; + +type RunOptions = { + print?: boolean; + requireBuiltCli?: boolean; +}; + +const AGENT_ID = "main"; +const RESET_SESSION_KEY = "agent:main:main"; +const DELETE_SESSION_KEY = "agent:main:dashboard:sqlite-delete"; +const CONCURRENT_SEND_SESSION_KEY = "agent:main:dashboard:sqlite-concurrent-send"; +const CONCURRENT_RESET_SESSION_KEY = "agent:main:dashboard:sqlite-concurrent-reset"; +const CONCURRENT_DELETE_SESSION_KEY = "agent:main:dashboard:sqlite-concurrent-delete"; +const CONCURRENT_SEND_TEXT = "sqlite concurrent send history reset"; +const CONCURRENT_DELETE_TEXT = "sqlite concurrent delete while send is active"; +const CLEANUP_PRUNE_SESSION_ID = "sqlite-cleanup-prune"; +const CLEANUP_PRUNE_SESSION_KEY = "agent:main:dashboard:sqlite-cleanup-prune"; +const CLEANUP_PRUNE_TEXT = "sqlite cleanup prune me"; +const FULL_TURN_ASSISTANT_TEXT = "OPENCLAW_E2E_OK_12"; +const FULL_TURN_SESSION_KEY = "agent:main:sqlite-full-turn"; +const MANUAL_COMPACTION_SESSION_KEY = "agent:main:dashboard:sqlite-manual-compact"; +const PLUGIN_SDK_APPEND_TEXT = "sqlite sdk consumer appended by identity"; +const PLUGIN_SDK_SESSION_KEY = "agent:main:dashboard:sqlite-sdk-consumer"; +const DOWNGRADE_REUPGRADE_SESSION_ID = "sqlite-downgrade-reupgrade"; +const DOWNGRADE_REUPGRADE_SESSION_KEY = "agent:main:dashboard:sqlite-downgrade-reupgrade"; +const DOWNGRADE_REUPGRADE_TEXT = "sqlite downgrade wrote file-backed state"; +const SQLITE_BUSY_SESSION_ID = "sqlite-busy-contention"; +const SQLITE_BUSY_SESSION_KEY = "agent:main:dashboard:sqlite-busy-contention"; +const SQLITE_BUSY_TEXT = "sqlite busy writer contention"; +const SECOND_STARTUP_APPEND_TEXT = "sqlite appended after reset and restart"; +const SCALE_SESSION_COUNT = 24; +const SCALE_EVENTS_PER_SESSION = 4; +const SCALE_SESSION_KEYS = Array.from( + { length: SCALE_SESSION_COUNT }, + (_, index) => `agent:main:dashboard:sqlite-scale-${String(index + 1).padStart(2, "0")}`, +); +const SHARED_SESSION_KEYS = [ + "agent:main:dashboard:sqlite-shared-a", + "agent:main:dashboard:sqlite-shared-b", +] as const; +const OLD_STATE_SESSION_KEYS = [ + "agent:main:+15551234567", + "agent:main:partial-direct", + "agent:main:unknown:group:legacy-room", +] as const; + +/** Runs the isolated live gateway SQLite flip proof and returns structured evidence. */ +export async function runSqliteSessionsTranscriptsFlipProof( + options: RunOptions = {}, +): Promise { + const print = options.print ?? false; + const mockOpenAiPort = await getFreeTcpPort(); + const inst = await createOpenClawTestInstance({ + name: `sqlite-sessions-transcripts-flip-${randomUUID()}`, + config: buildMockOpenAiConfig(mockOpenAiPort), + env: { + OPENAI_API_KEY: "sk-openclaw-e2e-mock", + OPENCLAW_TEST_MINIMAL_GATEWAY: undefined, + OPENCLAW_SKIP_PROVIDERS: undefined, + }, + startTimeoutMs: 90_000, + stopTimeoutMs: 3_000, + }); + // The proof mixes child-process CLI calls with in-process SDK accessors. + // Apply the fixture environment so both paths resolve the same isolated DB. + inst.state.applyEnv(); + const context = buildProofContext(inst.stateDir); + const checkpoints: ProofCheckpoint[] = []; + const failures: string[] = []; + let gatewayEntrypoint: string[] = []; + let busyContention: BusyContentionEvidence | undefined; + let downgradeReupgrade: DowngradeReupgradeEvidence | undefined; + let manualCompaction: ManualCompactionEvidence | undefined; + let mockOpenAi: ProofChildProcess | undefined; + let pluginSdkConsumer: PluginSdkConsumerEvidence | undefined; + let rollbackRestore: RollbackRestoreEvidence | undefined; + let scaleMigration: ScaleMigrationEvidence | undefined; + let secondStartupAfterReset: SecondStartupAfterResetEvidence | undefined; + + const record = async (label: string, doctor?: DoctorCommandEvidence) => { + const checkpoint = await captureCheckpoint(context, label, { + doctor, + gatewayLogTail: inst.logs(), + }); + checkpoints.push(checkpoint); + validateCheckpointInvariants(context, checkpoint, failures); + if (print) { + printCheckpoint(checkpoint); + } + return checkpoint; + }; + + try { + gatewayEntrypoint = await inst.entrypoint(); + if (options.requireBuiltCli === true && !isBuiltCliEntrypoint(gatewayEntrypoint)) { + throw new Error(`expected built CLI entrypoint, got ${gatewayEntrypoint.join(" ")}`); + } + + mockOpenAi = await startMockOpenAiServer({ + port: mockOpenAiPort, + requestLogPath: context.mockOpenAiRequestLog, + responseText: context.fullTurnAssistantText, + }); + + await seedLegacySessionStore(context); + await record("seeded-legacy-store"); + + const startupImportStartedAt = Date.now(); + await inst.startGateway(); + await record("after-startup-import"); + scaleMigration = requireScaleMigrationProof(context, Date.now() - startupImportStartedAt); + + const inspectDoctor = await runDoctor(inst, "inspect", context.storePath); + await record("after-doctor-inspect", inspectDoctor); + + const validateDoctor = await runDoctor(inst, "validate", context.storePath); + await record("after-doctor-validate", validateDoctor); + + rollbackRestore = await runRollbackRestoreProof(inst, context); + await record("after-rollback-restore"); + + const client = await connectGatewayClient({ + url: inst.url, + token: inst.gatewayToken, + clientDisplayName: "sqlite-sessions-transcripts-flip-proof", + requestTimeoutMs: 20_000, + timeoutMs: 20_000, + }); + try { + await waitForHistoryContains(client, context.resetSessionKey, "legacy hello"); + } finally { + await disconnectGatewayClient(client); + } + + await inst.stopGateway(); + await inst.startGateway(); + await record("after-gateway-restart"); + + const restartedClient = await connectGatewayClient({ + url: inst.url, + token: inst.gatewayToken, + clientDisplayName: "sqlite-sessions-transcripts-flip-proof-restart", + requestTimeoutMs: 20_000, + timeoutMs: 20_000, + }); + let restartedClientConnected = true; + try { + await waitForHistoryContains(restartedClient, context.resetSessionKey, "legacy hello"); + const resetPreludeRunId = await sendGatewayUserMessage( + restartedClient, + context.resetSessionKey, + "sqlite user-facing send before reset", + ); + await waitForAgentRunOk(restartedClient, resetPreludeRunId); + await waitForSqliteMessageContains( + context.agentDbPath, + context.legacySessionId, + "user", + "sqlite user-facing send before reset", + ); + await record("after-chat-send"); + + const fullTurnRunId = await sendGatewayUserMessage( + restartedClient, + context.fullTurnSessionKey, + `Reply with exactly ${context.fullTurnAssistantText}.`, + ); + await waitForAgentRunOk(restartedClient, fullTurnRunId); + const fullTurnSessionId = await waitForSqliteSessionId( + context.agentDbPath, + context.fullTurnSessionKey, + ); + await waitForSqliteMessageContains( + context.agentDbPath, + fullTurnSessionId, + "user", + context.fullTurnAssistantText, + ); + await waitForSqliteMessageContains( + context.agentDbPath, + fullTurnSessionId, + "assistant", + context.fullTurnAssistantText, + ); + await waitForHistoryRoleContains( + restartedClient, + context.fullTurnSessionKey, + "assistant", + context.fullTurnAssistantText, + ); + await requireMockOpenAiRequest(context.mockOpenAiRequestLog); + await record("after-full-agent-turn"); + + manualCompaction = await runManualCompactionProof(restartedClient, context); + await record("after-manual-compaction"); + + const pluginSdkRunId = await sendGatewayUserMessage( + restartedClient, + context.pluginSdkSessionKey, + `Reply with exactly ${context.fullTurnAssistantText}. SDK consumer proof.`, + ); + await waitForAgentRunOk(restartedClient, pluginSdkRunId); + const pluginSdkSessionId = await waitForSqliteSessionId( + context.agentDbPath, + context.pluginSdkSessionKey, + ); + await waitForSqliteMessageContains( + context.agentDbPath, + pluginSdkSessionId, + "assistant", + context.fullTurnAssistantText, + ); + pluginSdkConsumer = await runPluginSdkConsumerProbe(context, pluginSdkSessionId); + await waitForSqliteMessageContains( + context.agentDbPath, + pluginSdkSessionId, + "assistant", + context.pluginSdkAppendText, + ); + await record("after-plugin-sdk-consumer"); + + await runGatewayCleanupPruningProof(restartedClient, context); + await record("after-cleanup-pruning"); + + const idempotentImportDoctor = await runDoctorIdempotenceProof(inst, context); + await record("after-doctor-import-idempotence", idempotentImportDoctor); + + downgradeReupgrade = await runDowngradeReupgradeProof(inst, context); + await record("after-downgrade-reupgrade-import"); + + busyContention = await runSqliteBusyContentionProof(context); + await record("after-sqlite-busy-contention"); + + await runConcurrentMultiClientLifecycle(inst, context, restartedClient); + await record("after-concurrent-multi-client"); + + const resetSessionId = await resetSession(restartedClient, context.resetSessionKey); + await record("after-sessions-reset"); + + await disconnectGatewayClient(restartedClient); + restartedClientConnected = false; + await inst.stopGateway(); + await inst.startGateway(); + await record("after-second-startup-after-reset"); + + const postResetClient = await connectGatewayClient({ + url: inst.url, + token: inst.gatewayToken, + clientDisplayName: "sqlite-sessions-transcripts-flip-proof-post-reset-restart", + requestTimeoutMs: 20_000, + timeoutMs: 20_000, + }); + try { + secondStartupAfterReset = await runSecondStartupAfterResetProof( + postResetClient, + context, + resetSessionId, + ); + await record("after-transcript-append"); + + await deleteSession(postResetClient, context.deleteSessionKey); + await record("after-sessions-delete"); + + await deleteSession(postResetClient, context.sharedSessionKeys[0]); + await requireTrackedSession(postResetClient, context.sharedSessionKeys[1]); + await record("after-shared-first-delete"); + + await deleteSession(postResetClient, context.sharedSessionKeys[1]); + await record("after-shared-final-delete"); + } finally { + await disconnectGatewayClient(postResetClient); + } + } finally { + if (restartedClientConnected) { + await disconnectGatewayClient(restartedClient); + } + } + + const finalInspectDoctor = await runDoctor(inst, "inspect", context.storePath); + await record("after-final-doctor-inspect", finalInspectDoctor); + } catch (error) { + failures.push(error instanceof Error ? error.message : String(error)); + await record("failure"); + } finally { + await stopChildProcess(mockOpenAi); + await inst.cleanup(); + } + + const report: SqliteSessionsTranscriptsFlipProofReport = { + ok: failures.length === 0, + agentId: context.agentId, + checkpoints, + cleanupPruneSessionKey: context.cleanupPruneSessionKey, + concurrentDeleteSessionKey: context.concurrentDeleteSessionKey, + concurrentResetSessionKey: context.concurrentResetSessionKey, + concurrentSendSessionKey: context.concurrentSendSessionKey, + deleteSessionKey: context.deleteSessionKey, + failures, + fullTurnAssistantText: context.fullTurnAssistantText, + fullTurnSessionKey: context.fullTurnSessionKey, + gatewayEntrypoint, + legacySessionId: context.legacySessionId, + ...(manualCompaction ? { manualCompaction } : {}), + manualCompactionSessionKey: context.manualCompactionSessionKey, + mockOpenAiRequestLog: context.mockOpenAiRequestLog, + oldStateSessionKeys: [...context.oldStateSessionKeys], + ...(pluginSdkConsumer ? { pluginSdkConsumer } : {}), + pluginSdkSessionKey: context.pluginSdkSessionKey, + resetSessionKey: context.resetSessionKey, + ...(rollbackRestore ? { rollbackRestore } : {}), + ...(busyContention ? { busyContention } : {}), + ...(downgradeReupgrade ? { downgradeReupgrade } : {}), + ...(scaleMigration ? { scaleMigration } : {}), + ...(secondStartupAfterReset ? { secondStartupAfterReset } : {}), + sharedSessionKeys: [...context.sharedSessionKeys], + stateDir: context.stateDir, + }; + if (print) { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } + return report; +} + +function isBuiltCliEntrypoint(entrypoint: readonly string[]): boolean { + const [first, ...rest] = entrypoint; + return rest.length === 0 && (first === "dist/index.js" || first === "dist/index.mjs"); +} + +function buildProofContext(stateDir: string): ProofContext { + const agentDir = path.join(stateDir, "agents", AGENT_ID); + const activeSessionsDir = path.join(agentDir, "sessions"); + const legacySessionsDir = path.join(stateDir, "sessions"); + return { + activeSessionsDir, + agentDbPath: path.join(agentDir, "agent", "openclaw-agent.sqlite"), + agentId: AGENT_ID, + archiveRoots: [path.join(agentDir, "session-sqlite-import-archive"), activeSessionsDir], + cleanupPruneSessionKey: CLEANUP_PRUNE_SESSION_KEY, + concurrentDeleteSessionKey: CONCURRENT_DELETE_SESSION_KEY, + concurrentResetSessionKey: CONCURRENT_RESET_SESSION_KEY, + concurrentSendSessionKey: CONCURRENT_SEND_SESSION_KEY, + deleteSessionKey: DELETE_SESSION_KEY, + fullTurnAssistantText: FULL_TURN_ASSISTANT_TEXT, + fullTurnSessionKey: FULL_TURN_SESSION_KEY, + legacySessionsDir, + legacySessionId: "sqlite-legacy-main", + manualCompactionSessionKey: MANUAL_COMPACTION_SESSION_KEY, + mockOpenAiRequestLog: path.join(stateDir, "mock-openai-requests.ndjson"), + oldStateSessionKeys: [...OLD_STATE_SESSION_KEYS], + pluginSdkAppendText: PLUGIN_SDK_APPEND_TEXT, + pluginSdkSessionKey: PLUGIN_SDK_SESSION_KEY, + resetSessionKey: RESET_SESSION_KEY, + sharedSessionKeys: [...SHARED_SESSION_KEYS], + stateDir, + storePath: path.join(activeSessionsDir, "sessions.json"), + trackedSessionKeys: [ + RESET_SESSION_KEY, + DELETE_SESSION_KEY, + CLEANUP_PRUNE_SESSION_KEY, + CONCURRENT_SEND_SESSION_KEY, + CONCURRENT_RESET_SESSION_KEY, + CONCURRENT_DELETE_SESSION_KEY, + FULL_TURN_SESSION_KEY, + MANUAL_COMPACTION_SESSION_KEY, + PLUGIN_SDK_SESSION_KEY, + DOWNGRADE_REUPGRADE_SESSION_KEY, + SQLITE_BUSY_SESSION_KEY, + ...SHARED_SESSION_KEYS, + ...OLD_STATE_SESSION_KEYS, + ...SCALE_SESSION_KEYS, + ], + }; +} + +function buildMockOpenAiConfig(mockPort: number): Record { + const modelRef = "openai/gpt-5.5"; + const modelId = "gpt-5.5"; + const cost = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }; + return { + agents: { + defaults: { + model: { primary: modelRef }, + models: { + [modelRef]: { + agentRuntime: { id: "openclaw" }, + params: { openaiWsWarmup: false, transport: "sse" }, + }, + }, + }, + }, + models: { + mode: "merge", + providers: { + openai: { + agentRuntime: { id: "openclaw" }, + api: "openai-responses", + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, + baseUrl: `http://127.0.0.1:${mockPort}/v1`, + models: [ + { + agentRuntime: { id: "openclaw" }, + api: "openai-responses", + contextTokens: 96_000, + contextWindow: 128_000, + cost, + id: modelId, + input: ["text", "image"], + maxTokens: 4_096, + name: modelId, + reasoning: false, + }, + ], + request: { allowPrivateNetwork: true }, + }, + }, + }, + plugins: { enabled: true }, + }; +} + +async function getFreeTcpPort(): Promise { + const srv = net.createServer(); + await new Promise((resolve, reject) => { + srv.once("error", reject); + srv.listen(0, "127.0.0.1", resolve); + }); + const addr = srv.address(); + if (!addr || typeof addr === "string") { + srv.close(); + throw new Error("failed to bind ephemeral mock OpenAI port"); + } + await new Promise((resolve) => { + srv.close(() => resolve()); + }); + return addr.port; +} + +async function startMockOpenAiServer(params: { + port: number; + requestLogPath: string; + responseText: string; +}): Promise { + const child = spawn("node", ["scripts/e2e/mock-openai-server.mjs"], { + cwd: process.cwd(), + env: { + ...process.env, + MOCK_PORT: String(params.port), + MOCK_REQUEST_LOG: params.requestLogPath, + SUCCESS_MARKER: params.responseText, + }, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + let output = ""; + child.stdout.on("data", (chunk) => { + output += String(chunk); + }); + child.stderr.on("data", (chunk) => { + output += String(chunk); + }); + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error( + `mock OpenAI exited before listening (code=${String(child.exitCode)} signal=${String( + child.signalCode, + )})\n${tail(output)}`, + ); + } + if (output.includes("mock-openai listening")) { + return child; + } + await sleep(25); + } + await stopChildProcess(child); + throw new Error(`timeout waiting for mock OpenAI server\n${tail(output)}`); +} + +async function stopChildProcess(child: ProofChildProcess | undefined): Promise { + if (!child || child.exitCode !== null || child.signalCode !== null) { + return; + } + child.kill("SIGTERM"); + const exited = await Promise.race([ + new Promise((resolve) => { + child.once("exit", () => resolve(true)); + }), + sleep(2_000).then(() => false), + ]); + if (!exited && child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } +} + +async function seedLegacySessionStore(context: ProofContext): Promise { + await fs.mkdir(context.activeSessionsDir, { recursive: true }); + await fs.mkdir(context.legacySessionsDir, { recursive: true }); + await fs.mkdir(path.join(context.stateDir, "agent"), { recursive: true }); + const now = Date.now(); + const entries: Record> = { + [context.concurrentDeleteSessionKey]: legacyEntry("sqlite-concurrent-delete", now - 8_000), + [context.concurrentResetSessionKey]: legacyEntry("sqlite-concurrent-reset", now - 9_000), + [context.deleteSessionKey]: legacyEntry("sqlite-delete-session", now - 1_000), + [context.sharedSessionKeys[0]]: legacyEntry("sqlite-shared-session", now - 2_000, { + sessionFile: "sqlite-shared-a.jsonl", + }), + [context.sharedSessionKeys[1]]: legacyEntry("sqlite-shared-session", now - 3_000, { + sessionFile: "sqlite-shared-b.jsonl", + }), + }; + const oldStateEntries = { + main: legacyEntry(context.legacySessionId, now - 4_000), + "+15551234567": legacyEntry("sqlite-old-direct", now - 5_000), + "group:legacy-room": legacyEntry("sqlite-old-group", now - 6_000, { + room: "legacy-room", + }), + "partial-direct": legacyEntry("sqlite-partial-import", now - 7_000), + }; + for (const [index, sessionKey] of SCALE_SESSION_KEYS.entries()) { + entries[sessionKey] = legacyEntry(scaleSessionId(index), now - 20_000 - index); + } + await fs.writeFile(context.storePath, `${JSON.stringify(entries, null, 2)}\n`, { mode: 0o600 }); + await fs.writeFile( + path.join(context.legacySessionsDir, "sessions.json"), + `${JSON.stringify(oldStateEntries, null, 2)}\n`, + { mode: 0o600 }, + ); + await fs.writeFile( + path.join(context.stateDir, "agent", "old-settings.json"), + `${JSON.stringify({ source: "old-agent-layout" })}\n`, + { mode: 0o600 }, + ); + await writeTranscript(context.legacySessionsDir, context.legacySessionId, [ + legacySessionEvent(context.legacySessionId), + { type: "message", id: "sqlite-user-1", message: { role: "user", content: "legacy hello" } }, + ]); + await writeTranscript(context.legacySessionsDir, "sqlite-old-direct", [ + legacySessionEvent("sqlite-old-direct"), + { type: "message", id: "sqlite-old-direct-1", message: { role: "user", content: "old dm" } }, + ]); + await writeTranscript(context.legacySessionsDir, "sqlite-old-group", [ + legacySessionEvent("sqlite-old-group"), + { type: "message", id: "sqlite-old-group-1", message: { role: "user", content: "old group" } }, + ]); + await writeTranscript(context.activeSessionsDir, "sqlite-delete-session", [ + legacySessionEvent("sqlite-delete-session"), + { type: "message", id: "sqlite-delete-1", message: { role: "user", content: "delete me" } }, + ]); + await writeTranscript(context.activeSessionsDir, "sqlite-concurrent-reset", [ + legacySessionEvent("sqlite-concurrent-reset"), + { + type: "message", + id: "sqlite-concurrent-reset-1", + message: { role: "user", content: "concurrent reset seed" }, + }, + ]); + await writeTranscript(context.activeSessionsDir, "sqlite-concurrent-delete", [ + legacySessionEvent("sqlite-concurrent-delete"), + { + type: "message", + id: "sqlite-concurrent-delete-1", + message: { role: "user", content: "concurrent delete seed" }, + }, + ]); + await writeTranscript(context.activeSessionsDir, "sqlite-shared-a", [ + legacySessionEvent("sqlite-shared-session"), + { type: "message", id: "sqlite-shared-1", message: { role: "user", content: "shared" } }, + ]); + await writeTranscript(context.activeSessionsDir, "sqlite-shared-b", [ + legacySessionEvent("sqlite-shared-session"), + { type: "message", id: "sqlite-shared-2", message: { role: "user", content: "shared b" } }, + ]); + for (const [index] of SCALE_SESSION_KEYS.entries()) { + const sessionId = scaleSessionId(index); + await writeTranscript(context.activeSessionsDir, sessionId, [ + legacySessionEvent(sessionId), + ...Array.from({ length: SCALE_EVENTS_PER_SESSION - 1 }, (_, eventIndex) => ({ + type: "message", + id: `${sessionId}-${eventIndex + 1}`, + message: { + role: eventIndex % 2 === 0 ? "user" : "assistant", + content: `sqlite scale ${index + 1} event ${eventIndex + 1}`, + }, + })), + ]); + } + await fs.writeFile( + path.join(context.legacySessionsDir, `${context.legacySessionId}.trajectory.jsonl`), + `${JSON.stringify({ type: "trajectory", sessionId: context.legacySessionId })}\n`, + { mode: 0o600 }, + ); + await fs.writeFile( + path.join(context.legacySessionsDir, "old-orphan.deleted.jsonl"), + `${JSON.stringify({ type: "event", id: "old-orphan" })}\n`, + { mode: 0o600 }, + ); + const archiveDir = path.join(context.activeSessionsDir, "archive-fixture"); + await fs.mkdir(archiveDir, { recursive: true }); + await fs.writeFile( + path.join(archiveDir, "cold-archive.jsonl"), + `${JSON.stringify({ type: "event", id: "cold-archive" })}\n`, + { mode: 0o600 }, + ); + await importSqliteSessionRows({ + agentId: context.agentId, + entry: { + ...legacyEntry("sqlite-partial-import", now - 7_000), + sessionFile: formatSqliteSessionFileMarker({ + agentId: context.agentId, + sessionId: "sqlite-partial-import", + storePath: context.storePath, + }), + }, + readTranscriptEvents(append) { + append(legacySessionEvent("sqlite-partial-import")); + append({ + type: "message", + id: "sqlite-partial-import-1", + message: { role: "user", content: "already imported" }, + }); + }, + sessionKey: "agent:main:partial-direct", + storePath: context.storePath, + }); +} + +function scaleSessionId(index: number): string { + return `sqlite-scale-${String(index + 1).padStart(2, "0")}`; +} + +function legacyEntry( + sessionId: string, + updatedAt: number, + options: { room?: string; sessionFile?: string } = {}, +): SessionEntry & { channel: string; chatType: string; room?: string } { + return { + channel: "cli", + chatType: "direct", + ...(options.room ? { room: options.room } : {}), + sessionFile: options.sessionFile ?? `${sessionId}.jsonl`, + sessionId, + sessionStartedAt: updatedAt - 500, + updatedAt, + }; +} + +function legacySessionEvent(sessionId: string): TranscriptEvent { + return { type: "session", sessionId }; +} + +async function writeTranscript( + sessionsDir: string, + sessionId: string, + events: TranscriptEvent[], +): Promise { + const body = events.map((event) => JSON.stringify(event)).join("\n"); + await fs.writeFile(path.join(sessionsDir, `${sessionId}.jsonl`), `${body}\n`, { mode: 0o600 }); +} + +async function runDoctor( + inst: Awaited>, + mode: DoctorMode, + storePath: string, +): Promise { + const result = await inst.cli( + ["doctor", "--session-sqlite", mode, "--session-sqlite-store", storePath, "--json"], + { timeoutMs: 60_000 }, + ); + const parsed = parseJsonObject(result.stdout); + return { + code: result.code, + mode, + ...(parsed ? parseDoctorMigrationRun(parsed) : {}), + ...(parsed ? parseDoctorRestore(parsed) : {}), + stderrTail: tail(result.stderr), + stdoutTail: tail(result.stdout), + ...(parsed && typeof parsed.totals === "object" + ? { totals: parsed.totals as Record } + : {}), + }; +} + +function parseDoctorMigrationRun(parsed: Record): { + migrationRun?: DoctorMigrationRunEvidence; +} { + const migrationRun = asRecord(parsed.migrationRun); + if ( + !migrationRun || + typeof migrationRun.manifestPath !== "string" || + typeof migrationRun.runId !== "string" + ) { + return {}; + } + return { + migrationRun: { + ...(typeof migrationRun.failureReportJsonPath === "string" + ? { failureReportJsonPath: migrationRun.failureReportJsonPath } + : {}), + ...(typeof migrationRun.failureReportMarkdownPath === "string" + ? { failureReportMarkdownPath: migrationRun.failureReportMarkdownPath } + : {}), + manifestPath: migrationRun.manifestPath, + runId: migrationRun.runId, + }, + }; +} + +function parseDoctorRestore(parsed: Record): { restore?: DoctorRestoreEvidence } { + const targets = Array.isArray(parsed.targets) ? parsed.targets : []; + const restore = asRecord(asRecord(targets[0])?.restore); + if (!restore) { + return {}; + } + return { + restore: { + conflicts: Array.isArray(restore.conflicts) ? restore.conflicts.length : 0, + manifestPaths: stringArray(restore.manifestPaths), + restoredFiles: stringArray(restore.restoredFiles), + skippedFiles: stringArray(restore.skippedFiles), + }, + }; +} + +async function runRollbackRestoreProof( + inst: Awaited>, + context: ProofContext, +): Promise { + const drillDir = path.join(context.stateDir, "rollback-drill"); + const storePath = path.join(drillDir, "sessions.json"); + const sessionId = "sqlite-rollback-restore"; + const sessionKey = "agent:main:rollback-restore"; + const sourcePath = path.join(drillDir, `${sessionId}.jsonl`); + const sqlitePath = path.join(drillDir, "openclaw-agent.sqlite"); + await fs.mkdir(drillDir, { recursive: true }); + await fs.writeFile( + storePath, + `${JSON.stringify({ [sessionKey]: legacyEntry(sessionId, Date.now()) }, null, 2)}\n`, + { mode: 0o600 }, + ); + await writeTranscript(drillDir, sessionId, [ + legacySessionEvent(sessionId), + { + type: "message", + id: "sqlite-rollback-restore-1", + message: { role: "user", content: "rollback restore me" }, + }, + ]); + + const importDoctor = await runDoctor(inst, "import", storePath); + if (importDoctor.code !== 0) { + throw new Error(`rollback drill import exited non-zero: ${importDoctor.stderrTail}`); + } + const manifestPath = importDoctor.migrationRun?.manifestPath; + if (!manifestPath) { + throw new Error(`rollback drill import did not report a migration manifest`); + } + const manifest = await readJsonObjectFile(manifestPath); + const move = findManifestMoveForSource(manifest, sourcePath); + if (!move) { + throw new Error(`rollback drill manifest did not record ${sourcePath}`); + } + const archivedBeforeRestore = fsSync.existsSync(move.archivePath); + if (fsSync.existsSync(sourcePath) || !archivedBeforeRestore) { + throw new Error( + `rollback drill import did not archive source=${sourcePath} archive=${move.archivePath}`, + ); + } + + const failedManifestIssueCode = "e2e_forced_post_archive_failure"; + await markMigrationManifestFailed(manifestPath, failedManifestIssueCode); + const restoreDoctor = await runDoctor(inst, "restore", storePath); + if (restoreDoctor.code !== 0 || (restoreDoctor.restore?.conflicts ?? 0) > 0) { + throw new Error(`rollback drill restore failed: ${restoreDoctor.stderrTail}`); + } + const restoredFiles = restoreDoctor.restore?.restoredFiles ?? []; + if (!restoredFiles.some((filePath) => sameFilePathString(filePath, sourcePath))) { + throw new Error(`rollback drill restore did not report restored source ${sourcePath}`); + } + if (!fsSync.existsSync(sourcePath) || fsSync.existsSync(move.archivePath)) { + throw new Error( + `rollback drill restore did not move archive back to source=${sourcePath} archive=${move.archivePath}`, + ); + } + + const idempotentRestore = await runDoctor(inst, "restore", storePath); + if (idempotentRestore.code !== 0 || (idempotentRestore.restore?.conflicts ?? 0) > 0) { + throw new Error(`rollback drill idempotent restore failed: ${idempotentRestore.stderrTail}`); + } + const idempotentRestoreSkippedFiles = idempotentRestore.restore?.skippedFiles ?? []; + if (!idempotentRestoreSkippedFiles.some((filePath) => sameFilePathString(filePath, sourcePath))) { + throw new Error(`rollback drill idempotent restore did not skip ${sourcePath}`); + } + if (!fsSync.existsSync(sqlitePath)) { + throw new Error(`rollback drill restore unexpectedly removed SQLite database ${sqlitePath}`); + } + + return { + archivePath: move.archivePath, + archivedBeforeRestore, + failedManifestIssueCode, + idempotentRestoreSkippedFiles, + manifestPath, + restoredFiles, + sourcePath, + sourceRestored: fsSync.existsSync(sourcePath), + sqliteStillExists: fsSync.existsSync(sqlitePath), + }; +} + +async function readJsonObjectFile(filePath: string): Promise> { + const parsed = JSON.parse(await fs.readFile(filePath, "utf8")) as unknown; + const record = asRecord(parsed); + if (!record) { + throw new Error(`${filePath} did not contain a JSON object`); + } + return record; +} + +function findManifestMoveForSource( + manifest: Record, + sourcePath: string, +): { archivePath: string; sourcePath: string } | undefined { + for (const target of Array.isArray(manifest.targets) ? manifest.targets : []) { + const targetRecord = asRecord(target); + const moves = [ + ...(Array.isArray(targetRecord?.completedMoves) ? targetRecord.completedMoves : []), + ...(Array.isArray(targetRecord?.plannedMoves) ? targetRecord.plannedMoves : []), + ]; + for (const move of moves) { + const moveRecord = asRecord(move); + if ( + typeof moveRecord?.sourcePath === "string" && + typeof moveRecord.archivePath === "string" && + sameFilePathString(moveRecord.sourcePath, sourcePath) + ) { + return { + archivePath: moveRecord.archivePath, + sourcePath: moveRecord.sourcePath, + }; + } + } + } + return undefined; +} + +function sameFilePathString(left: string, right: string): boolean { + return normalizeMacPrivatePath(left) === normalizeMacPrivatePath(right); +} + +function normalizeMacPrivatePath(filePath: string): string { + const resolved = path.resolve(filePath); + return resolved.startsWith("/private/") ? resolved.slice("/private".length) : resolved; +} + +async function markMigrationManifestFailed(manifestPath: string, issueCode: string): Promise { + const manifest = await readJsonObjectFile(manifestPath); + const failedAt = new Date().toISOString(); + manifest.failedAt = failedAt; + manifest.completedAt = typeof manifest.completedAt === "string" ? manifest.completedAt : failedAt; + const firstTarget = asRecord(Array.isArray(manifest.targets) ? manifest.targets[0] : undefined); + if (firstTarget) { + const issues = Array.isArray(firstTarget.issues) ? firstTarget.issues : []; + firstTarget.issues = [ + ...issues, + { + code: issueCode, + message: "E2E simulated post-archive migration failure.", + sessionKey: "agent:main:rollback-restore", + }, + ]; + } + await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 }); +} + +async function appendProofMessage( + context: ProofContext, + sessionId: string, + sessionKey: string, + message: string, +): Promise { + const result = await appendTranscriptMessage( + { + agentId: context.agentId, + sessionId, + sessionKey, + storePath: context.storePath, + }, + { + message: { + role: "assistant", + content: message, + timestamp: Date.now(), + }, + }, + ); + if (!result?.appended || !result.messageId) { + throw new Error(`appendTranscriptMessage failed for ${sessionKey}`); + } +} + +async function runManualCompactionProof( + client: Awaited>, + context: ProofContext, +): Promise { + const runId = await sendGatewayUserMessage( + client, + context.manualCompactionSessionKey, + `Reply with exactly ${context.fullTurnAssistantText}. Manual compaction proof.`, + ); + await waitForAgentRunOk(client, runId); + const sessionId = await waitForSqliteSessionId( + context.agentDbPath, + context.manualCompactionSessionKey, + ); + await waitForSqliteMessageContains( + context.agentDbPath, + sessionId, + "assistant", + context.fullTurnAssistantText, + ); + const rowCountBefore = countSqliteTranscriptEvents(context.agentDbPath, sessionId); + if (rowCountBefore < 2) { + throw new Error( + `manual compaction source transcript had too few rows: ${rowCountBefore} for ${sessionId}`, + ); + } + const listed: { sessions?: Array<{ key?: string; sessionId?: string }> } = await client.request( + "sessions.list", + {}, + ); + const listedSession = (listed.sessions ?? []).find( + (session) => + session.sessionId === sessionId || session.key === context.manualCompactionSessionKey, + ); + if (!listedSession?.key) { + throw new Error( + `manual compaction session was not listed before compact: ${JSON.stringify(listed)}`, + ); + } + + const compacted: { + compacted?: boolean; + key?: string; + ok?: boolean; + } = await client.request("sessions.compact", { + key: listedSession.key, + }); + if (compacted.ok !== true || compacted.compacted !== true) { + throw new Error( + `manual compaction did not compact using ${listedSession.key}: ${JSON.stringify( + compacted, + )}; listed=${JSON.stringify(listed)}`, + ); + } + + const evidence = readSqliteEvidence(context.agentDbPath, [context.manualCompactionSessionKey]); + const row = evidence.trackedEntries.find( + (entry) => entry.sessionKey === context.manualCompactionSessionKey, + ); + if (!row?.entry) { + throw new Error(`manual compaction entry missing for ${context.manualCompactionSessionKey}`); + } + const checkpointCount = Array.isArray(row.entry.compactionCheckpoints) + ? row.entry.compactionCheckpoints.length + : 0; + if (checkpointCount < 1) { + throw new Error(`manual compaction did not write checkpoint metadata: ${JSON.stringify(row)}`); + } + const sessionFileMarker = typeof row.entry.sessionFile === "string" ? row.entry.sessionFile : ""; + if (!sessionFileMarker.startsWith("sqlite:")) { + throw new Error(`manual compaction entry did not keep a SQLite marker: ${sessionFileMarker}`); + } + if (fsSync.existsSync(sessionFileMarker)) { + throw new Error(`manual compaction marker unexpectedly exists as a file: ${sessionFileMarker}`); + } + + return { + checkpointCount, + compacted: compacted.compacted, + rowCountAfter: countSqliteTranscriptEvents(context.agentDbPath, row.sessionId), + rowCountBefore, + sessionFileMarker, + sessionId: row.sessionId, + sessionKey: context.manualCompactionSessionKey, + }; +} + +async function runPluginSdkConsumerProbe( + context: ProofContext, + sessionId: string, +): Promise { + const scope = { + agentId: context.agentId, + sessionId, + sessionKey: context.pluginSdkSessionKey, + storePath: context.storePath, + }; + const sessionEntry = getSdkSessionEntry({ + agentId: context.agentId, + readConsistency: "latest", + sessionKey: context.pluginSdkSessionKey, + storePath: context.storePath, + }); + if (sessionEntry?.sessionId !== sessionId) { + throw new Error( + `SDK session store read returned ${JSON.stringify(sessionEntry)} for ${context.pluginSdkSessionKey}`, + ); + } + const expectedMarker = formatSqliteSessionFileMarker({ + agentId: context.agentId, + sessionId, + storePath: context.storePath, + }); + if (sessionEntry.sessionFile !== expectedMarker) { + throw new Error( + `SDK session store exposed unexpected transcript marker for ${context.pluginSdkSessionKey}: ${String( + sessionEntry.sessionFile, + )}`, + ); + } + if (fsSync.existsSync(sessionEntry.sessionFile)) { + throw new Error(`SDK session marker unexpectedly resolves to an active file path`); + } + + const listedSessionKeys = listSdkSessionEntries({ + agentId: context.agentId, + storePath: context.storePath, + }).map((entry) => entry.sessionKey); + if (!listedSessionKeys.includes(context.pluginSdkSessionKey)) { + throw new Error(`SDK session list omitted ${context.pluginSdkSessionKey}`); + } + + const identity = await resolveSessionTranscriptIdentity(scope); + const latestBefore = await readLatestAssistantTextByIdentity(scope); + if (latestBefore?.text !== context.fullTurnAssistantText) { + throw new Error( + `SDK latest assistant read returned ${JSON.stringify(latestBefore)} for ${context.pluginSdkSessionKey}`, + ); + } + const transcriptEventsBeforeAppend = (await readSessionTranscriptEvents(scope)).length; + const storeTranscriptEvents = loadSdkTranscriptEventsSync(scope).length; + const activeJsonlPath = path.join(context.activeSessionsDir, `${sessionId}.jsonl`); + const activeTrajectorySessionSidecarPath = path.join( + context.activeSessionsDir, + `${sessionId}.trajectory.jsonl`, + ); + const activeTrajectoryPointerPath = path.join( + context.activeSessionsDir, + `${sessionId}.trajectory-path.json`, + ); + const activeTrajectoryRuntimeSidecarPath = path.join( + context.activeSessionsDir, + "trajectory", + `${sessionId}.jsonl`, + ); + const activeJsonlForSessionExists = fsSync.existsSync(activeJsonlPath); + if (activeJsonlForSessionExists) { + throw new Error(`SDK probe found active JSONL for SQLite session at ${activeJsonlPath}`); + } + const activeTrajectorySessionSidecarForSessionExists = fsSync.existsSync( + activeTrajectorySessionSidecarPath, + ); + const activeTrajectoryPointerForSessionExists = fsSync.existsSync(activeTrajectoryPointerPath); + const activeTrajectoryRuntimeSidecarForSessionExists = fsSync.existsSync( + activeTrajectoryRuntimeSidecarPath, + ); + if ( + activeTrajectorySessionSidecarForSessionExists || + activeTrajectoryPointerForSessionExists || + activeTrajectoryRuntimeSidecarForSessionExists + ) { + throw new Error( + `SDK trajectory probe found active sidecar paths: ${JSON.stringify({ + pointer: activeTrajectoryPointerPath, + runtime: activeTrajectoryRuntimeSidecarPath, + session: activeTrajectorySessionSidecarPath, + })}`, + ); + } + + const appended = await appendSessionTranscriptMessageByIdentity({ + ...scope, + message: { + role: "assistant", + content: [{ type: "text", text: context.pluginSdkAppendText }], + timestamp: Date.now(), + }, + }); + if (!appended?.appended || !appended.messageId) { + throw new Error(`SDK transcript append failed for ${context.pluginSdkSessionKey}`); + } + + const latestAfter = await readLatestAssistantTextByIdentity(scope); + if (latestAfter?.text !== context.pluginSdkAppendText) { + throw new Error( + `SDK latest assistant after append returned ${JSON.stringify(latestAfter)} for ${ + context.pluginSdkSessionKey + }`, + ); + } + const transcriptEventsAfterAppend = (await readSessionTranscriptEvents(scope)).length; + if (transcriptEventsAfterAppend <= transcriptEventsBeforeAppend) { + throw new Error( + `SDK transcript append did not increase event count for ${context.pluginSdkSessionKey}`, + ); + } + return { + activeJsonlForSessionExists, + activeTrajectoryPointerForSessionExists, + activeTrajectoryRuntimeSidecarForSessionExists, + activeTrajectorySessionSidecarForSessionExists, + appendedMessageId: appended.messageId, + identityMemoryKey: identity.memoryKey, + latestAssistantTextBeforeAppend: latestBefore.text, + latestAssistantTextAfterAppend: latestAfter.text, + listedSessionKeys, + sessionFileMarker: sessionEntry.sessionFile, + sessionId, + sessionKey: context.pluginSdkSessionKey, + storeTranscriptEvents, + transcriptEventsAfterAppend, + transcriptEventsBeforeAppend, + }; +} + +async function runGatewayCleanupPruningProof( + client: Awaited>, + context: ProofContext, +): Promise { + const updatedAt = Date.now() - 31 * 24 * 60 * 60 * 1000; + await importSqliteSessionRows({ + agentId: context.agentId, + entry: { + channel: "cli", + chatType: "direct", + sessionFile: formatSqliteSessionFileMarker({ + agentId: context.agentId, + sessionId: CLEANUP_PRUNE_SESSION_ID, + storePath: context.storePath, + }), + sessionId: CLEANUP_PRUNE_SESSION_ID, + sessionStartedAt: updatedAt - 500, + updatedAt, + }, + readTranscriptEvents(append) { + append(legacySessionEvent(CLEANUP_PRUNE_SESSION_ID)); + append({ + type: "message", + id: "sqlite-cleanup-prune-1", + message: { role: "user", content: CLEANUP_PRUNE_TEXT }, + }); + }, + sessionKey: context.cleanupPruneSessionKey, + storePath: context.storePath, + }); + await waitForSqliteMessageContains( + context.agentDbPath, + CLEANUP_PRUNE_SESSION_ID, + "user", + CLEANUP_PRUNE_TEXT, + ); + + const result: { afterCount?: number; applied?: boolean; pruned?: number } = await client.request( + "sessions.cleanup", + { enforce: true }, + { timeoutMs: 20_000 }, + ); + if (result?.applied !== true || (result.pruned ?? 0) < 1) { + throw new Error(`sessions.cleanup did not prune stale SQLite rows: ${JSON.stringify(result)}`); + } + await waitForSessionEntryAbsent(context.agentDbPath, context.cleanupPruneSessionKey); + await waitForSqliteEventsAbsent(context.agentDbPath, CLEANUP_PRUNE_SESSION_ID); +} + +async function runDoctorIdempotenceProof( + inst: Awaited>, + context: ProofContext, +): Promise { + const before = readSqliteEvidence(context.agentDbPath, context.trackedSessionKeys); + const doctor = await runDoctor(inst, "import", context.storePath); + if (doctor.code !== 0) { + throw new Error(`doctor import idempotence check exited non-zero: ${doctor.stderrTail}`); + } + const totals = doctor.totals ?? {}; + if (totals.importedEntries !== 0 || totals.importedTranscriptEvents !== 0) { + throw new Error(`doctor import was not idempotent: ${JSON.stringify(totals)}`); + } + const after = readSqliteEvidence(context.agentDbPath, context.trackedSessionKeys); + if ( + after.sessionEntries !== before.sessionEntries || + after.sessions !== before.sessions || + after.transcriptEvents !== before.transcriptEvents + ) { + throw new Error( + `doctor import changed SQLite counts: before=${JSON.stringify(before)} after=${JSON.stringify( + after, + )}`, + ); + } + return doctor; +} + +function requireScaleMigrationProof( + context: ProofContext, + startupImportElapsedMs: number, +): ScaleMigrationEvidence { + const sqlite = readSqliteEvidence(context.agentDbPath, SCALE_SESSION_KEYS); + const importedSessionKeys: string[] = []; + for (const [index, sessionKey] of SCALE_SESSION_KEYS.entries()) { + const row = sqlite.trackedEntries.find((entry) => entry.sessionKey === sessionKey); + const sessionId = scaleSessionId(index); + if (!row || row.sessionId !== sessionId) { + throw new Error(`scale migration did not import ${sessionKey} as ${sessionId}`); + } + if (row.transcriptEvents < SCALE_EVENTS_PER_SESSION) { + throw new Error( + `scale migration imported too few events for ${sessionKey}: ${row.transcriptEvents}`, + ); + } + importedSessionKeys.push(sessionKey); + } + return { + importedSessionKeys, + minTranscriptEventsPerSession: SCALE_EVENTS_PER_SESSION, + seededEvents: SCALE_SESSION_COUNT * SCALE_EVENTS_PER_SESSION, + seededSessions: SCALE_SESSION_COUNT, + startupImportElapsedMs, + }; +} + +async function runDowngradeReupgradeProof( + inst: Awaited>, + context: ProofContext, +): Promise { + await fs.mkdir(context.activeSessionsDir, { recursive: true }); + await fs.writeFile( + context.storePath, + `${JSON.stringify( + { + [DOWNGRADE_REUPGRADE_SESSION_KEY]: legacyEntry(DOWNGRADE_REUPGRADE_SESSION_ID, Date.now()), + }, + null, + 2, + )}\n`, + { mode: 0o600 }, + ); + const transcriptPath = path.join( + context.activeSessionsDir, + `${DOWNGRADE_REUPGRADE_SESSION_ID}.jsonl`, + ); + const trajectoryPath = path.join( + context.activeSessionsDir, + `${DOWNGRADE_REUPGRADE_SESSION_ID}.trajectory.jsonl`, + ); + const trajectoryPointerPath = path.join( + context.activeSessionsDir, + `${DOWNGRADE_REUPGRADE_SESSION_ID}.trajectory-path.json`, + ); + await writeTranscript(context.activeSessionsDir, DOWNGRADE_REUPGRADE_SESSION_ID, [ + legacySessionEvent(DOWNGRADE_REUPGRADE_SESSION_ID), + { + type: "message", + id: "sqlite-downgrade-reupgrade-1", + message: { role: "user", content: DOWNGRADE_REUPGRADE_TEXT }, + }, + ]); + await fs.writeFile( + trajectoryPath, + `${JSON.stringify({ type: "trajectory", sessionId: DOWNGRADE_REUPGRADE_SESSION_ID })}\n`, + { mode: 0o600 }, + ); + await fs.writeFile( + trajectoryPointerPath, + `${JSON.stringify({ + traceSchema: "openclaw-trajectory-pointer", + schemaVersion: 1, + sessionId: DOWNGRADE_REUPGRADE_SESSION_ID, + runtimeFile: trajectoryPath, + })}\n`, + { mode: 0o600 }, + ); + + const doctor = await runDoctor(inst, "import", context.storePath); + if (doctor.code !== 0) { + throw new Error(`downgrade/re-upgrade import exited non-zero: ${doctor.stderrTail}`); + } + const totals = doctor.totals ?? {}; + if (totals.importedEntries !== 1 || totals.importedTranscriptEvents !== 2) { + throw new Error(`downgrade/re-upgrade import had unexpected totals: ${JSON.stringify(totals)}`); + } + await waitForSqliteMessageContains( + context.agentDbPath, + DOWNGRADE_REUPGRADE_SESSION_ID, + "user", + DOWNGRADE_REUPGRADE_TEXT, + ); + const manifestPath = doctor.migrationRun?.manifestPath; + if (!manifestPath) { + throw new Error(`downgrade/re-upgrade import did not report a migration manifest`); + } + const manifest = await readJsonObjectFile(manifestPath); + const trajectoryMove = findManifestMoveForSource(manifest, trajectoryPath); + const trajectoryPointerMove = findManifestMoveForSource(manifest, trajectoryPointerPath); + if (!trajectoryMove || !trajectoryPointerMove) { + throw new Error( + `downgrade/re-upgrade import did not record trajectory artifact moves: ${JSON.stringify({ + trajectoryMove, + trajectoryPointerMove, + })}`, + ); + } + const trajectorySidecarArchived = fsSync.existsSync(trajectoryMove.archivePath); + const trajectoryPointerArchived = fsSync.existsSync(trajectoryPointerMove.archivePath); + if (!trajectorySidecarArchived || !trajectoryPointerArchived) { + throw new Error( + `downgrade/re-upgrade import did not archive trajectory artifacts: ${JSON.stringify({ + trajectoryArchive: trajectoryMove.archivePath, + trajectoryPointerArchive: trajectoryPointerMove.archivePath, + })}`, + ); + } + return { + activeJsonlArchived: !fsSync.existsSync(transcriptPath), + doctorImportedEntries: totals.importedEntries, + doctorImportedTranscriptEvents: totals.importedTranscriptEvents, + trajectoryPointerArchived, + trajectoryPointerSourceRemoved: !fsSync.existsSync(trajectoryPointerPath), + trajectorySidecarArchived, + trajectorySidecarSourceRemoved: !fsSync.existsSync(trajectoryPath), + sessionId: DOWNGRADE_REUPGRADE_SESSION_ID, + sessionKey: DOWNGRADE_REUPGRADE_SESSION_KEY, + transcriptEvents: countSqliteTranscriptEvents( + context.agentDbPath, + DOWNGRADE_REUPGRADE_SESSION_ID, + ), + }; +} + +async function runSqliteBusyContentionProof( + context: ProofContext, +): Promise { + const holdMs = 500; + const readyPath = path.join(context.stateDir, `sqlite-busy-${randomUUID()}.ready`); + const child = spawn( + process.execPath, + [ + "--input-type=module", + "-e", + ` + import fs from "node:fs"; + import { DatabaseSync } from "node:sqlite"; + const db = new DatabaseSync(process.env.OPENCLAW_E2E_BUSY_DB_PATH); + db.exec("PRAGMA busy_timeout = 30000; BEGIN IMMEDIATE;"); + fs.writeFileSync(process.env.OPENCLAW_E2E_BUSY_READY_PATH, "ready"); + setTimeout(() => { + db.exec("COMMIT"); + db.close(); + }, Number(process.env.OPENCLAW_E2E_BUSY_HOLD_MS)); + `, + ], + { + cwd: process.cwd(), + env: { + ...process.env, + OPENCLAW_E2E_BUSY_DB_PATH: context.agentDbPath, + OPENCLAW_E2E_BUSY_HOLD_MS: String(holdMs), + OPENCLAW_E2E_BUSY_READY_PATH: readyPath, + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + let childOutput = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + childOutput += String(chunk); + }); + child.stderr.on("data", (chunk) => { + childOutput += String(chunk); + }); + + await waitForFile(readyPath, 10_000, () => { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error( + `SQLite busy child exited before acquiring lock code=${String( + child.exitCode, + )} signal=${String(child.signalCode)} ${tail(childOutput)}`, + ); + } + }); + + const startedAt = Date.now(); + const result = await importSqliteSessionRows({ + agentId: context.agentId, + entry: legacyEntry(SQLITE_BUSY_SESSION_ID, Date.now()), + readTranscriptEvents(append) { + append(legacySessionEvent(SQLITE_BUSY_SESSION_ID)); + append({ + type: "message", + id: "sqlite-busy-contention-1", + message: { role: "user", content: SQLITE_BUSY_TEXT }, + }); + }, + sessionKey: SQLITE_BUSY_SESSION_KEY, + storePath: context.storePath, + }); + const elapsedMs = Date.now() - startedAt; + const exit = await waitForChildExit(child, 10_000); + if (exit.code !== 0) { + throw new Error( + `SQLite busy child exited non-zero code=${String(exit.code)} signal=${String( + exit.signal, + )} ${tail(childOutput)}`, + ); + } + if (elapsedMs < Math.floor(holdMs / 2)) { + throw new Error(`SQLite busy proof did not wait on the writer lock: elapsed=${elapsedMs}ms`); + } + await waitForSqliteMessageContains( + context.agentDbPath, + SQLITE_BUSY_SESSION_ID, + "user", + SQLITE_BUSY_TEXT, + ); + return { + childExitCode: exit.code, + childSignal: exit.signal, + elapsedMs, + holdMs, + sessionId: SQLITE_BUSY_SESSION_ID, + sessionKey: SQLITE_BUSY_SESSION_KEY, + transcriptEvents: result.transcriptEvents, + }; +} + +async function runSecondStartupAfterResetProof( + client: Awaited>, + context: ProofContext, + resetSessionId: string, +): Promise { + await appendProofMessage( + context, + resetSessionId, + context.resetSessionKey, + SECOND_STARTUP_APPEND_TEXT, + ); + await waitForHistoryContains(client, context.resetSessionKey, SECOND_STARTUP_APPEND_TEXT); + await waitForSqliteEvents(context.agentDbPath, resetSessionId, 1); + return { + activeJsonlForSessionExists: fsSync.existsSync( + path.join(context.activeSessionsDir, `${resetSessionId}.jsonl`), + ), + historyContainsPostResetAppend: true, + sessionId: resetSessionId, + sessionKey: context.resetSessionKey, + transcriptEvents: countSqliteTranscriptEvents(context.agentDbPath, resetSessionId), + }; +} + +async function runConcurrentMultiClientLifecycle( + inst: Awaited>, + context: ProofContext, + primaryClient: Awaited>, +): Promise { + const historyClient = await connectGatewayClient({ + url: inst.url, + token: inst.gatewayToken, + clientDisplayName: "sqlite-sessions-transcripts-flip-proof-concurrent-history", + requestTimeoutMs: 20_000, + timeoutMs: 20_000, + }); + const lifecycleClient = await connectGatewayClient({ + url: inst.url, + token: inst.gatewayToken, + clientDisplayName: "sqlite-sessions-transcripts-flip-proof-concurrent-lifecycle", + requestTimeoutMs: 20_000, + timeoutMs: 20_000, + }); + try { + await requireHistoryContains( + historyClient, + context.concurrentResetSessionKey, + "concurrent reset seed", + ); + const sendPromise = sendGatewayUserMessage( + primaryClient, + context.concurrentSendSessionKey, + CONCURRENT_SEND_TEXT, + ); + const historyPromise = historyClient.request( + "chat.history", + { sessionKey: context.concurrentResetSessionKey, limit: 50 }, + { timeoutMs: 20_000 }, + ); + const resetPromise = resetSession(lifecycleClient, context.concurrentResetSessionKey); + + const [sendRunId, , resetSessionId] = await Promise.all([ + sendPromise, + historyPromise, + resetPromise, + ]); + await waitForAgentRunOk(primaryClient, sendRunId); + const sendSessionId = await waitForSqliteSessionId( + context.agentDbPath, + context.concurrentSendSessionKey, + ); + await waitForSqliteMessageContains( + context.agentDbPath, + sendSessionId, + "user", + CONCURRENT_SEND_TEXT, + ); + await waitForSqliteMessageContains( + context.agentDbPath, + sendSessionId, + "assistant", + context.fullTurnAssistantText, + ); + await waitForTrackedSessionId( + context.agentDbPath, + context.concurrentResetSessionKey, + resetSessionId, + ); + + const deleteRunId = await sendGatewayUserMessage( + historyClient, + context.concurrentDeleteSessionKey, + CONCURRENT_DELETE_TEXT, + ); + const deleteHistoryPromise = lifecycleClient.request( + "chat.history", + { sessionKey: context.concurrentDeleteSessionKey, limit: 50 }, + { timeoutMs: 20_000 }, + ); + await Promise.all([ + deleteHistoryPromise, + deleteSession(primaryClient, context.concurrentDeleteSessionKey), + ]); + await waitForAgentRunSettled(historyClient, deleteRunId); + await waitForSessionEntryAbsent(context.agentDbPath, context.concurrentDeleteSessionKey); + } finally { + await Promise.all([ + disconnectGatewayClient(historyClient), + disconnectGatewayClient(lifecycleClient), + ]); + } +} + +async function resetSession( + client: Awaited>, + key: string, +): Promise { + const result: { ok?: boolean; entry?: { sessionId?: string } } = await client.request( + "sessions.reset", + { key, reason: "reset" }, + ); + if (result?.ok !== true || !result.entry?.sessionId) { + throw new Error(`sessions.reset failed: ${JSON.stringify(result)}`); + } + return result.entry.sessionId; +} + +async function sendGatewayUserMessage( + client: Awaited>, + sessionKey: string, + message: string, +): Promise { + const result: { runId?: string; status?: string } = await client.request( + "chat.send", + { + sessionKey, + message, + idempotencyKey: `sqlite-send-${randomUUID()}`, + }, + { timeoutMs: 20_000 }, + ); + if (result?.status !== "started" || typeof result.runId !== "string") { + throw new Error(`chat.send did not start correctly: ${JSON.stringify(result)}`); + } + return result.runId; +} + +async function waitForAgentRunOk( + client: Awaited>, + runId: string, +): Promise { + const result: { error?: unknown; status?: string } = await client.request( + "agent.wait", + { runId, timeoutMs: 60_000 }, + { timeoutMs: 65_000 }, + ); + if (result?.status !== "ok") { + throw new Error(`agent.wait failed for ${runId}: ${JSON.stringify(result)}`); + } +} + +async function waitForAgentRunSettled( + client: Awaited>, + runId: string, +): Promise { + const result: { endedAt?: number; status?: string; stopReason?: string } = await client.request( + "agent.wait", + { runId, timeoutMs: 60_000 }, + { timeoutMs: 65_000 }, + ); + if (typeof result?.status !== "string") { + throw new Error(`agent.wait returned no status for ${runId}: ${JSON.stringify(result)}`); + } + if (result.status === "ok") { + return; + } + const terminalLifecycleStatus = result.status === "timeout" || result.status === "error"; + if ( + terminalLifecycleStatus && + (result.stopReason === "rpc" || result.stopReason === "stop") && + typeof result.endedAt === "number" + ) { + return; + } + throw new Error(`agent.wait did not settle acceptably for ${runId}: ${JSON.stringify(result)}`); +} + +async function deleteSession( + client: Awaited>, + key: string, +): Promise { + const result: { ok?: boolean; deleted?: boolean } = await client.request("sessions.delete", { + key, + deleteTranscript: true, + }); + if (result?.ok !== true || result.deleted !== true) { + throw new Error(`sessions.delete failed for ${key}: ${JSON.stringify(result)}`); + } +} + +async function requireTrackedSession( + client: Awaited>, + key: string, +): Promise { + const result: { sessions?: Array<{ key?: string }> } = await client.request("sessions.list", {}); + if (!result.sessions?.some((session) => session.key === key)) { + throw new Error(`expected session ${key} to remain listed`); + } +} + +async function requireHistoryContains( + client: Awaited>, + sessionKey: string, + expected: string, +): Promise { + const result: { messages?: unknown[] } = await client.request("chat.history", { + agentId: AGENT_ID, + sessionKey, + limit: 50, + }); + const text = JSON.stringify(result.messages ?? []); + if (!text.includes(expected)) { + throw new Error( + `chat.history for ${sessionKey} did not contain ${JSON.stringify(expected)}: ${tail( + JSON.stringify(result), + )}`, + ); + } +} + +async function requireHistoryRoleContains( + client: Awaited>, + sessionKey: string, + role: string, + expected: string, +): Promise { + const result: { messages?: unknown[] } = await client.request("chat.history", { + agentId: AGENT_ID, + sessionKey, + limit: 50, + }); + const matching = (result.messages ?? []).some( + (message) => + message && + typeof message === "object" && + !Array.isArray(message) && + (message as { role?: unknown }).role === role && + JSON.stringify(message).includes(expected), + ); + if (!matching) { + throw new Error( + `chat.history for ${sessionKey} did not contain ${role} message ${JSON.stringify( + expected, + )}: ${tail(JSON.stringify(result))}`, + ); + } +} + +async function waitForHistoryContains( + client: Awaited>, + sessionKey: string, + expected: string, +): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + await requireHistoryContains(client, sessionKey, expected); + return; + } catch { + await sleep(50); + } + } + await requireHistoryContains(client, sessionKey, expected); +} + +async function waitForHistoryRoleContains( + client: Awaited>, + sessionKey: string, + role: string, + expected: string, +): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + await requireHistoryRoleContains(client, sessionKey, role, expected); + return; + } catch { + await sleep(50); + } + } + await requireHistoryRoleContains(client, sessionKey, role, expected); +} + +async function waitForSqliteEvents( + dbPath: string, + sessionId: string, + minEvents: number, +): Promise { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + const sqlite = readSqliteEvidence(dbPath, []); + const row = sqlite.trackedEntries.find((entry) => entry.sessionId === sessionId); + if ((row?.transcriptEvents ?? 0) >= minEvents) { + return; + } + await sleep(50); + } + throw new Error(`timed out waiting for ${minEvents} SQLite events for ${sessionId}`); +} + +async function waitForSqliteSessionId(dbPath: string, sessionKey: string): Promise { + const deadline = Date.now() + 20_000; + while (Date.now() < deadline) { + const row = readSqliteEvidence(dbPath, [sessionKey]).trackedEntries.find( + (entry) => entry.sessionKey === sessionKey && entry.sessionId, + ); + if (row?.sessionId) { + return row.sessionId; + } + await sleep(50); + } + throw new Error(`timed out waiting for SQLite session entry for ${sessionKey}`); +} + +async function waitForTrackedSessionId( + dbPath: string, + sessionKey: string, + expectedSessionId: string, +): Promise { + const deadline = Date.now() + 20_000; + while (Date.now() < deadline) { + const row = readSqliteEvidence(dbPath, [sessionKey]).trackedEntries.find( + (entry) => entry.sessionKey === sessionKey, + ); + if (row?.sessionId === expectedSessionId) { + return; + } + await sleep(50); + } + throw new Error( + `timed out waiting for SQLite session entry ${sessionKey} to point at ${expectedSessionId}`, + ); +} + +async function waitForSessionEntryAbsent(dbPath: string, sessionKey: string): Promise { + const deadline = Date.now() + 20_000; + let absentSince: number | undefined; + while (Date.now() < deadline) { + const row = readSqliteEvidence(dbPath, [sessionKey]).trackedEntries.find( + (entry) => entry.sessionKey === sessionKey, + ); + if (!row) { + absentSince ??= Date.now(); + if (Date.now() - absentSince >= 1_500) { + return; + } + await sleep(50); + continue; + } + absentSince = undefined; + await sleep(50); + } + throw new Error(`timed out waiting for SQLite session entry deletion for ${sessionKey}`); +} + +async function waitForSqliteEventsAbsent(dbPath: string, sessionId: string): Promise { + const deadline = Date.now() + 20_000; + let absentSince: number | undefined; + while (Date.now() < deadline) { + const count = countSqliteTranscriptEvents(dbPath, sessionId); + if (count === 0) { + absentSince ??= Date.now(); + if (Date.now() - absentSince >= 1_500) { + return; + } + await sleep(50); + continue; + } + absentSince = undefined; + await sleep(50); + } + throw new Error(`timed out waiting for SQLite transcript row deletion for ${sessionId}`); +} + +async function waitForSqliteMessageContains( + dbPath: string, + sessionId: string, + role: "assistant" | "user", + expected: string, +): Promise { + const deadline = Date.now() + 20_000; + while (Date.now() < deadline) { + const messages = readSqliteTranscriptMessages(dbPath, sessionId); + if ( + messages.some( + (message) => message.role === role && JSON.stringify(message.content).includes(expected), + ) + ) { + return; + } + await sleep(50); + } + throw new Error( + `timed out waiting for SQLite ${role} transcript message containing ${JSON.stringify( + expected, + )} for ${sessionId}: ${JSON.stringify(readSqliteTranscriptMessages(dbPath, sessionId))}`, + ); +} + +async function waitForFile(filePath: string, timeoutMs: number, poll: () => void): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + poll(); + if (fsSync.existsSync(filePath)) { + return; + } + await sleep(25); + } + throw new Error(`timed out waiting for ${filePath}`); +} + +async function waitForChildExit( + child: ProofChildProcess, + timeoutMs: number, +): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { + if (child.exitCode !== null || child.signalCode !== null) { + return { code: child.exitCode, signal: child.signalCode }; + } + return await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve, reject) => { + const timer = setTimeout(() => { + child.off("exit", onExit); + child.kill("SIGKILL"); + reject(new Error(`timed out waiting for child process ${child.pid ?? "unknown"} to exit`)); + }, timeoutMs); + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + clearTimeout(timer); + resolve({ code, signal }); + }; + child.once("exit", onExit); + if (child.exitCode !== null || child.signalCode !== null) { + child.off("exit", onExit); + clearTimeout(timer); + resolve({ code: child.exitCode, signal: child.signalCode }); + } + }, + ); +} + +async function requireMockOpenAiRequest(requestLogPath: string): Promise { + const text = await fs.readFile(requestLogPath, "utf8").catch(() => ""); + if (!text.includes('"/v1/responses"')) { + throw new Error(`mock OpenAI request log did not include /v1/responses: ${tail(text)}`); + } +} + +function readSqliteTranscriptMessages( + dbPath: string, + sessionId: string, +): Array<{ content?: unknown; role?: string }> { + if (!fsSync.existsSync(dbPath)) { + return []; + } + const db = new DatabaseSync(dbPath, { readOnly: true }); + try { + const rows = db + .prepare( + `SELECT event_json AS eventJson + FROM transcript_events + WHERE session_id = ? + ORDER BY seq ASC`, + ) + .all(sessionId) as Array<{ eventJson?: unknown }>; + return rows.flatMap((row) => { + if (typeof row.eventJson !== "string") { + return []; + } + const event = parseJsonObject(row.eventJson); + const message = + event && typeof event.message === "object" && event.message !== null + ? (event.message as { content?: unknown; role?: unknown }) + : undefined; + return typeof message?.role === "string" + ? [{ role: message.role, content: message.content }] + : []; + }); + } finally { + db.close(); + } +} + +function countSqliteTranscriptEvents(dbPath: string, sessionId: string): number { + if (!fsSync.existsSync(dbPath)) { + return 0; + } + const db = new DatabaseSync(dbPath, { readOnly: true }); + try { + return scalarNumber( + db, + "SELECT COUNT(*) AS count FROM transcript_events WHERE session_id = ?", + [sessionId], + ); + } finally { + db.close(); + } +} + +async function captureCheckpoint( + context: ProofContext, + label: string, + options: { doctor?: DoctorCommandEvidence; gatewayLogTail?: string }, +): Promise { + return { + activeJsonl: await inventoryActiveJsonl(context.activeSessionsDir), + archiveArtifacts: await inventoryArchiveArtifacts(context), + ...(options.doctor ? { doctor: options.doctor } : {}), + gatewayLogTail: tail(options.gatewayLogTail ?? ""), + label, + legacyStateJsonl: await inventoryActiveJsonl(context.legacySessionsDir), + sqlite: readSqliteEvidence(context.agentDbPath, context.trackedSessionKeys), + }; +} + +async function inventoryActiveJsonl(sessionsDir: string): Promise { + let entries: fsSync.Dirent[]; + try { + entries = await fs.readdir(sessionsDir, { withFileTypes: true }); + } catch { + return []; + } + const files = entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")); + return await Promise.all( + files.map((entry) => inventoryFile(path.join(sessionsDir, entry.name), sessionsDir)), + ); +} + +async function inventoryArchiveArtifacts(context: ProofContext): Promise { + const roots = context.archiveRoots; + const files: FileInventoryEntry[] = []; + for (const root of roots) { + await walkFiles(root, async (filePath) => { + const basename = path.basename(filePath); + const archiveLike = + basename.includes(".jsonl") || + basename.includes(".trajectory-path.json") || + basename.includes(".reset.") || + basename.includes(".deleted.") || + basename.includes(".bak."); + const isActivePrimaryJsonl = + path.dirname(filePath) === context.activeSessionsDir && + basename.endsWith(".jsonl") && + parseArchiveArtifactName(basename) === undefined; + if (isActivePrimaryJsonl) { + return; + } + if (archiveLike) { + files.push(await inventoryFile(filePath, context.stateDir)); + } + }); + } + return files.toSorted((a, b) => a.path.localeCompare(b.path)); +} + +async function walkFiles(root: string, visit: (filePath: string) => Promise): Promise { + let entries: fsSync.Dirent[]; + try { + entries = await fs.readdir(root, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const filePath = path.join(root, entry.name); + if (entry.isDirectory()) { + await walkFiles(filePath, visit); + } else if (entry.isFile()) { + await visit(filePath); + } + } +} + +async function inventoryFile(filePath: string, relativeRoot: string): Promise { + const stat = await fs.stat(filePath); + const text = await Promise.resolve() + .then(() => readSessionArchiveContentSync(filePath)) + .catch(() => undefined); + const archive = parseArchiveArtifactName(path.basename(filePath)); + const jsonl = text !== undefined ? summarizeJsonl(text) : undefined; + return { + ...(archive ? archive : {}), + path: path.relative(relativeRoot, filePath), + bytes: stat.size, + ...(text !== undefined + ? { lines: text.split(/\r?\n/u).filter(Boolean).length, textTail: tail(text, 2_000) } + : {}), + ...(jsonl?.jsonlTypes.length ? { jsonlTypes: jsonl.jsonlTypes } : {}), + ...(jsonl?.messageRoles.length ? { messageRoles: jsonl.messageRoles } : {}), + ...(jsonl?.messageTexts.length ? { messageTexts: jsonl.messageTexts } : {}), + }; +} + +function parseArchiveArtifactName( + fileName: string, +): Pick | undefined { + const normalized = stripSessionArchiveCompressionSuffix(fileName); + for (const archiveReason of ["deleted", "reset", "bak"] as const) { + const marker = `.jsonl.${archiveReason}.`; + const index = normalized.lastIndexOf(marker); + if (index > 0) { + return { archiveReason, archiveSessionId: normalized.slice(0, index) }; + } + } + return undefined; +} + +function summarizeJsonl(text: string): { + jsonlTypes: string[]; + messageRoles: string[]; + messageTexts: string[]; +} { + const jsonlTypes = new Set(); + const messageRoles = new Set(); + const messageTexts = new Set(); + for (const line of text.split(/\r?\n/u)) { + if (!line.trim()) { + continue; + } + const event = parseJsonObject(line); + if (typeof event?.type === "string") { + jsonlTypes.add(event.type); + } + const message = + event && typeof event.message === "object" && event.message !== null + ? (event.message as { content?: unknown; role?: unknown }) + : undefined; + if (typeof message?.role === "string") { + messageRoles.add(message.role); + } + const messageContent = message?.content; + if (typeof messageContent === "string") { + messageTexts.add(messageContent); + } else if (messageContent !== undefined) { + messageTexts.add(JSON.stringify(messageContent)); + } else if (typeof event?.content === "string") { + messageTexts.add(event.content); + } + } + return { + jsonlTypes: [...jsonlTypes].toSorted(), + messageRoles: [...messageRoles].toSorted(), + messageTexts: [...messageTexts].toSorted(), + }; +} + +function readSqliteEvidence(dbPath: string, trackedSessionKeys: readonly string[]): SqliteEvidence { + if (!fsSync.existsSync(dbPath)) { + return { + exists: false, + path: dbPath, + sessionEntries: 0, + sessions: 0, + trackedEntries: [], + trajectoryRuntimeEvents: 0, + transcriptEvents: 0, + }; + } + const db = new DatabaseSync(dbPath, { readOnly: true }); + try { + const trackedEntries = readTrackedEntries(db, trackedSessionKeys); + return { + exists: true, + path: dbPath, + sessionEntries: scalarNumber(db, "SELECT COUNT(*) AS count FROM session_entries"), + sessions: scalarNumber(db, "SELECT COUNT(*) AS count FROM sessions"), + trajectoryRuntimeEvents: scalarNumber( + db, + "SELECT COUNT(*) AS count FROM trajectory_runtime_events", + ), + trackedEntries, + transcriptEvents: scalarNumber(db, "SELECT COUNT(*) AS count FROM transcript_events"), + }; + } finally { + db.close(); + } +} + +function readTrackedEntries( + db: DatabaseSync, + trackedSessionKeys: readonly string[], +): SqliteSessionEntryEvidence[] { + const rows = db + .prepare( + `SELECT session_key AS sessionKey, session_id AS sessionId, entry_json AS entryJson + FROM session_entries + ORDER BY session_key ASC`, + ) + .all() as Array<{ entryJson?: unknown; sessionId?: unknown; sessionKey?: unknown }>; + return rows + .filter( + (row) => + trackedSessionKeys.length === 0 || + trackedSessionKeys.includes(typeof row.sessionKey === "string" ? row.sessionKey : ""), + ) + .map((row) => { + const sessionId = typeof row.sessionId === "string" ? row.sessionId : ""; + const result: SqliteSessionEntryEvidence = { + sessionId, + sessionKey: typeof row.sessionKey === "string" ? row.sessionKey : "", + trajectoryEvents: scalarNumber( + db, + "SELECT COUNT(*) AS count FROM trajectory_runtime_events WHERE session_id = ?", + [sessionId], + ), + transcriptEvents: scalarNumber( + db, + "SELECT COUNT(*) AS count FROM transcript_events WHERE session_id = ?", + [sessionId], + ), + }; + if (typeof row.entryJson === "string") { + result.entry = parseEntryJson(row.entryJson); + } + return result; + }); +} + +function parseEntryJson(entryJson: string): Record | undefined { + try { + const parsed = JSON.parse(entryJson) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return undefined; + } + const entry = { ...(parsed as Record) }; + delete entry.skillsSnapshot; + return entry; + } catch { + return undefined; + } +} + +function scalarNumber(db: DatabaseSync, sql: string, values: SQLInputValue[] = []): number { + const row = db.prepare(sql).get(...values) as { count?: unknown } | undefined; + return typeof row?.count === "number" ? row.count : 0; +} + +function validateCheckpointInvariants( + context: ProofContext, + checkpoint: ProofCheckpoint, + failures: string[], +): void { + if (checkpoint.label !== "seeded-legacy-store" && checkpoint.activeJsonl.length > 0) { + failures.push( + `${checkpoint.label}: active sessions directory still has JSONL files: ${checkpoint.activeJsonl + .map((entry) => entry.path) + .join(", ")}`, + ); + } + if (checkpoint.label !== "seeded-legacy-store" && checkpoint.legacyStateJsonl.length > 0) { + failures.push( + `${checkpoint.label}: old sessions directory still has JSONL files: ${checkpoint.legacyStateJsonl + .map((entry) => entry.path) + .join(", ")}`, + ); + } + const doctor = checkpoint.doctor; + if (checkpoint.label.startsWith("after-doctor") && doctor?.code !== 0) { + failures.push(`${checkpoint.label}: doctor ${doctor?.mode ?? "unknown"} exited non-zero`); + } + if ( + checkpoint.label === "after-startup-import" && + (checkpoint.sqlite.sessionEntries === 0 || checkpoint.sqlite.transcriptEvents === 0) + ) { + failures.push(`${checkpoint.label}: startup did not import sessions into SQLite`); + } + if ( + checkpoint.label.startsWith("after-doctor") && + checkpoint.sqlite.exists && + checkpoint.sqlite.sessionEntries > 0 && + checkpoint.sqlite.transcriptEvents === 0 + ) { + failures.push(`${checkpoint.label}: SQLite has session entries but no transcript events`); + } + if ( + checkpoint.label === "after-shared-first-delete" && + !checkpoint.sqlite.trackedEntries.some( + (entry) => entry.sessionKey === context.sharedSessionKeys[1], + ) + ) { + failures.push(`${checkpoint.label}: shared sibling entry was deleted too early`); + } + if (checkpoint.label === "after-startup-import") { + requireArchiveText(checkpoint, failures, { + description: "legacy trajectory sidecar", + includes: ["trajectory", context.legacySessionId], + pathIncludes: `${context.legacySessionId}.trajectory.jsonl`, + }); + requireArchiveText(checkpoint, failures, { + description: "preexisting deleted sidecar", + includes: ["old-orphan"], + pathIncludes: "old-orphan.deleted.jsonl", + }); + } + if (checkpoint.label === "after-sessions-reset") { + requireArchiveText(checkpoint, failures, { + description: "reset transcript archive", + includes: ["legacy hello", "sqlite user-facing send before reset"], + reason: "reset", + sessionId: context.legacySessionId, + }); + } + if (checkpoint.label === "after-sessions-delete") { + requireArchiveText(checkpoint, failures, { + description: "deleted transcript archive", + includes: ["delete me"], + reason: "deleted", + sessionId: "sqlite-delete-session", + }); + } + if (checkpoint.label === "after-cleanup-pruning") { + requireArchiveText(checkpoint, failures, { + description: "cleanup-pruned transcript archive", + includes: [CLEANUP_PRUNE_TEXT], + reason: "deleted", + sessionId: CLEANUP_PRUNE_SESSION_ID, + }); + if ( + checkpoint.sqlite.trackedEntries.some( + (entry) => entry.sessionKey === context.cleanupPruneSessionKey, + ) + ) { + failures.push(`${checkpoint.label}: cleanup-pruned entry still exists in SQLite`); + } + } + if (checkpoint.label === "after-doctor-import-idempotence") { + const totals = checkpoint.doctor?.totals ?? {}; + if (totals.importedEntries !== 0 || totals.importedTranscriptEvents !== 0) { + failures.push(`${checkpoint.label}: doctor import changed migrated SQLite state`); + } + } + if (checkpoint.label === "after-shared-first-delete") { + const archivedShared = findArchiveArtifact(checkpoint, { + reason: "deleted", + sessionId: "sqlite-shared-session", + }); + if (archivedShared) { + failures.push( + `${checkpoint.label}: shared transcript archived before final reference delete`, + ); + } + } + if (checkpoint.label === "after-shared-final-delete") { + requireArchiveText(checkpoint, failures, { + description: "final shared transcript archive", + includes: ["shared"], + reason: "deleted", + sessionId: "sqlite-shared-session", + }); + } +} + +function requireArchiveText( + checkpoint: ProofCheckpoint, + failures: string[], + params: { + description: string; + includes: string[]; + pathIncludes?: string; + reason?: "deleted" | "reset"; + sessionId?: string; + }, +): void { + const artifact = findArchiveArtifact(checkpoint, params); + if (!artifact) { + failures.push(`${checkpoint.label}: missing ${params.description}`); + return; + } + const searchableText = [ + artifact.textTail, + ...(artifact.messageTexts ?? []), + ...(artifact.jsonlTypes ?? []), + ] + .filter((value): value is string => typeof value === "string") + .join("\n"); + const missing = params.includes.filter((expected) => !searchableText.includes(expected)); + if (missing.length > 0) { + failures.push( + `${checkpoint.label}: ${params.description} missing archive content ${missing + .map((value) => JSON.stringify(value)) + .join(", ")} in ${artifact.path}`, + ); + } +} + +function findArchiveArtifact( + checkpoint: ProofCheckpoint, + params: { + pathIncludes?: string; + reason?: "deleted" | "reset"; + sessionId?: string; + }, +): FileInventoryEntry | undefined { + return checkpoint.archiveArtifacts.find((artifact) => { + if (params.pathIncludes && !artifact.path.includes(params.pathIncludes)) { + return false; + } + if (params.reason && artifact.archiveReason !== params.reason) { + return false; + } + if (params.sessionId && artifact.archiveSessionId !== params.sessionId) { + return false; + } + return true; + }); +} + +function parseJsonObject(text: string): Record | undefined { + const start = text.indexOf("{"); + const end = text.lastIndexOf("}"); + if (start < 0 || end < start) { + return undefined; + } + try { + const parsed = JSON.parse(text.slice(start, end + 1)); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : undefined; + } catch { + return undefined; + } +} + +function asRecord(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; +} + +function tail(value: string, maxChars = 8_000): string { + return value.length > maxChars ? value.slice(value.length - maxChars) : value; +} + +function printCheckpoint(checkpoint: ProofCheckpoint): void { + process.stderr.write( + [ + `[sqlite-sessions-transcripts-flip-proof] ${checkpoint.label}`, + ` sqlite sessions=${checkpoint.sqlite.sessions} entries=${checkpoint.sqlite.sessionEntries} transcriptEvents=${checkpoint.sqlite.transcriptEvents}`, + ` activeJsonl=${checkpoint.activeJsonl.length} legacyStateJsonl=${checkpoint.legacyStateJsonl.length} archiveArtifacts=${checkpoint.archiveArtifacts.length}`, + checkpoint.doctor + ? ` doctor ${checkpoint.doctor.mode} code=${String(checkpoint.doctor.code)} totals=${JSON.stringify( + checkpoint.doctor.totals ?? {}, + )}` + : undefined, + ] + .filter((line): line is string => typeof line === "string") + .join("\n") + "\n", + ); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + const report = await runSqliteSessionsTranscriptsFlipProof({ print: true }); + process.exit(report.ok ? 0 : 1); +} diff --git a/scripts/generate-sqlite-session-schema-baseline.ts b/scripts/generate-sqlite-session-schema-baseline.ts new file mode 100644 index 000000000000..d2820fa668ee --- /dev/null +++ b/scripts/generate-sqlite-session-schema-baseline.ts @@ -0,0 +1,42 @@ +#!/usr/bin/env node +// Generate SQLite Session Schema Baseline script supports OpenClaw repository automation. +import path from "node:path"; +import { writeSqliteSessionSchemaBaselineArtifacts } from "./lib/sqlite-session-schema-baseline.ts"; + +const args = new Set(process.argv.slice(2)); +const checkOnly = args.has("--check"); +const writeMode = args.has("--write"); + +if (checkOnly === writeMode) { + console.error("Use exactly one of --check or --write."); + process.exit(1); +} + +const repoRoot = process.cwd(); +const result = await writeSqliteSessionSchemaBaselineArtifacts({ + repoRoot, + check: checkOnly, +}); + +if (checkOnly) { + if (result.changed) { + console.error( + [ + "SQLite sessions/transcripts schema baseline drift detected.", + `Hash mismatch: ${path.relative(repoRoot, result.hashPath)}`, + "This means the sessions, conversations, or transcripts SQLite DDL changed.", + "If this schema change is intentional, run `pnpm sqlite:sessions-schema:gen` and commit the updated hash file.", + "If not intentional, keep the schema stable or move unrelated state to a separate table/store.", + ].join("\n"), + ); + process.exit(1); + } + console.log(`OK ${path.relative(repoRoot, result.hashPath)}`); +} else { + console.log( + [ + `Wrote ${path.relative(repoRoot, result.hashPath)}`, + `Wrote ${path.relative(repoRoot, result.sqlPath)} (gitignored, local only)`, + ].join("\n"), + ); +} diff --git a/scripts/lib/session-accessor-debt-baseline.json b/scripts/lib/session-accessor-debt-baseline.json index f3ba2a93e2a3..cbee09387ced 100644 --- a/scripts/lib/session-accessor-debt-baseline.json +++ b/scripts/lib/session-accessor-debt-baseline.json @@ -2,72 +2,39 @@ "embeddedAgentSessionTarget": {}, "memoryHostSessionCorpus": {}, "sessionAccessorRead": { - "extensions/telegram/src/bot-deps.ts": 1, - "extensions/telegram/src/bot-message-context.session-recreate.test-support.ts": 2, "src/acp/control-plane/manager.background-task.ts": 2, "src/acp/control-plane/manager.core.ts": 1, - "src/agents/agent-command.ts": 2, - "src/agents/bash-tools.exec-approval-followup.ts": 2, - "src/agents/command/session.ts": 5, - "src/agents/main-session-restart-recovery.ts": 2, - "src/agents/subagent-announce-delivery.ts": 3, "src/agents/subagent-announce-output.ts": 3, - "src/agents/subagent-announce.test-support.ts": 1, - "src/agents/subagent-capabilities.ts": 2, - "src/agents/subagent-list.ts": 2, - "src/agents/subagent-orphan-recovery.ts": 2, - "src/agents/subagent-session-reconciliation.ts": 2, - "src/agents/subagent-spawn.ts": 2, + "src/agents/subagent-announce.test-support.ts": 2, "src/agents/tools/transcripts-tool.ts": 2, - "src/auto-reply/reply.triggers.trigger-handling.filters-usage-summary-current-model-provider.cases.ts": 2, - "src/auto-reply/reply/commands-export-common.ts": 2, - "src/auto-reply/reply/commands-name.ts": 2, "src/auto-reply/reply/commands-session-store.ts": 2, - "src/auto-reply/reply/dispatch-acp-transcript.runtime.ts": 4, - "src/auto-reply/reply/dispatch-from-config.ts": 6, - "src/auto-reply/reply/get-reply-fast-path.ts": 4, "src/auto-reply/reply/get-reply-run.ts": 2, "src/commands/doctor-heartbeat-session-target.ts": 2, "src/commands/doctor-state-integrity.ts": 2, "src/commands/doctor/shared/codex-route-warnings.ts": 3, - "src/config/sessions/cleanup-service.ts": 4, - "src/config/sessions/session-accessor.ts": 25, - "src/config/sessions/session-registry-maintenance.ts": 2, + "src/config/sessions/session-accessor.sqlite.ts": 2, + "src/config/sessions/session-accessor.ts": 12, "src/config/sessions/store-load.ts": 5, "src/config/sessions/store.ts": 16, "src/config/sessions/test-helpers.ts": 2, - "src/config/sessions/transcript.ts": 6, - "src/cron/isolated-agent/session.ts": 2, - "src/infra/approval-request-account-binding.ts": 2, - "src/infra/heartbeat-runner.ts": 4, - "src/plugins/registry.ts": 2, - "src/plugins/runtime/runtime-agent.ts": 1 + "src/config/sessions/transcript.ts": 2 }, "sessionAccessorWrite": { "src/agents/session-suspension.ts": 2, - "src/agents/subagent-orphan-recovery.ts": 3, - "src/agents/subagent-spawn.ts": 2, - "src/auto-reply/reply/commands-name.ts": 2, + "src/agents/subagent-spawn.test-helpers.ts": 1, "src/auto-reply/reply/dispatch-from-config.ts": 2, "src/commands/doctor-heartbeat-main-session-repair.ts": 2, "src/commands/doctor-session-snapshots.ts": 2, "src/commands/doctor-session-state-providers.ts": 2, "src/commands/doctor-state-integrity.ts": 2, "src/commands/doctor/shared/codex-route-warnings.ts": 2, - "src/config/sessions/plugin-host-cleanup.ts": 2, - "src/config/sessions/session-accessor.ts": 14, - "src/config/sessions/session-file.ts": 2, - "src/config/sessions/session-registry-maintenance.ts": 2, - "src/gateway/server-methods/agent.ts": 2, - "src/gateway/session-lifecycle-state.ts": 2, "src/gateway/test-helpers.mocks.ts": 1, - "src/infra/heartbeat-runner.ts": 5, "src/infra/state-migrations.ts": 2, - "src/plugins/registry.ts": 3, - "src/plugins/runtime/runtime-agent.ts": 2 + "src/plugins/registry.ts": 1 }, "sessionCompactManualTrim": {}, "sessionLifecycleCleanup": { + "src/config/sessions/session-accessor.sqlite.ts": 2, "src/config/sessions/store-maintenance-operations.ts": 2, "src/config/sessions/store.ts": 5 }, @@ -75,8 +42,9 @@ "src/agents/embedded-agent-runner/compaction-hooks.ts": 2, "src/agents/embedded-agent-runner/thinking-replay-repair.ts": 2, "src/agents/embedded-agent-runner/tool-result-truncation.ts": 3, - "src/agents/embedded-agent-runner/transcript-rewrite.ts": 2, - "src/config/sessions/session-accessor.ts": 7, + "src/agents/embedded-agent-runner/transcript-rewrite.ts": 3, + "src/config/sessions/session-accessor.sqlite.ts": 3, + "src/config/sessions/session-accessor.ts": 2, "src/config/sessions/store.ts": 2 } } diff --git a/scripts/lib/sqlite-session-schema-baseline.ts b/scripts/lib/sqlite-session-schema-baseline.ts new file mode 100644 index 000000000000..afb371ce7d69 --- /dev/null +++ b/scripts/lib/sqlite-session-schema-baseline.ts @@ -0,0 +1,148 @@ +// Builds the SQLite sessions/transcripts schema baseline used by CI drift checks. +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; + +/** Rendered baseline artifact for the sessions/transcripts SQLite schema. */ +export type SqliteSessionSchemaBaselineRender = { + /** Normalized SQL for the session, conversation, and transcript schema objects. */ + sql: string; +}; + +/** Result returned after writing or checking SQLite schema baseline artifacts. */ +export type SqliteSessionSchemaBaselineWriteResult = { + /** True when generated artifact content differs from disk. */ + changed: boolean; + /** True when changed artifacts were actually written. */ + wrote: boolean; + /** Local inspection SQL artifact path. */ + sqlPath: string; + /** SHA-256 hash artifact path. */ + hashPath: string; +}; + +const DEFAULT_SCHEMA_INPUT = "src/state/openclaw-agent-schema.sql"; +const DEFAULT_SQL_OUTPUT = ".artifacts/sqlite-session-transcript-schema-baseline.sql"; +const DEFAULT_HASH_OUTPUT = "docs/.generated/sqlite-session-transcript-schema-baseline.sha256"; + +const TARGET_TABLES = new Set([ + "sessions", + "session_routes", + "conversations", + "session_conversations", + "session_entries", + "transcript_events", + "transcript_event_identities", +]); + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function normalizeIdentifier(value: string): string { + return value.replace(/^"|"$/g, "").toLowerCase(); +} + +function splitSqlStatements(sourceSql: string): string[] { + return sourceSql + .split(/;\s*(?:\r?\n|$)/u) + .map((statement) => statement.trim()) + .filter(Boolean); +} + +function normalizeStatement(statement: string): string { + const lines = statement + .split(/\r?\n/u) + .map((line) => line.trimEnd()) + .filter((line, index, allLines) => line.trim() !== "" || index < allLines.length - 1); + return `${lines.join("\n")};`; +} + +function readCreatedTableName(statement: string): string | null { + const match = statement.match( + /^CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\s+("[^"]+"|[A-Za-z_][A-Za-z0-9_]*)\b/iu, + ); + return match ? normalizeIdentifier(match[1]) : null; +} + +function readIndexedTableName(statement: string): string | null { + const match = statement.match( + /^CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\s+("[^"]+"|[A-Za-z_][A-Za-z0-9_]*)\s+ON\s+("[^"]+"|[A-Za-z_][A-Za-z0-9_]*)\b/isu, + ); + return match ? normalizeIdentifier(match[2]) : null; +} + +function isTargetSessionSchemaStatement(statement: string): boolean { + const tableName = readCreatedTableName(statement); + if (tableName) { + return TARGET_TABLES.has(tableName); + } + + const indexedTableName = readIndexedTableName(statement); + return indexedTableName ? TARGET_TABLES.has(indexedTableName) : false; +} + +/** Render the normalized sessions/transcripts SQLite schema baseline. */ +export function renderSqliteSessionSchemaBaseline( + agentSchemaSql: string, +): SqliteSessionSchemaBaselineRender { + const statements = splitSqlStatements(agentSchemaSql) + .filter(isTargetSessionSchemaStatement) + .map(normalizeStatement); + + return { + sql: `${statements.join("\n\n")}\n`, + }; +} + +/** Build the sha256 hash file content for the sessions/transcripts SQLite schema baseline. */ +export function computeSqliteSessionSchemaBaselineHashFileContent( + rendered: SqliteSessionSchemaBaselineRender, +): string { + return `${sha256(rendered.sql)} sqlite-session-transcript-schema-baseline.sql\n`; +} + +async function readIfExists(filePath: string): Promise { + try { + return await fs.readFile(filePath, "utf8"); + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { + return null; + } + throw error; + } +} + +/** Write or check SQLite sessions/transcripts schema baseline artifacts. */ +export async function writeSqliteSessionSchemaBaselineArtifacts(params: { + repoRoot: string; + check: boolean; + schemaInputPath?: string; + sqlOutputPath?: string; + hashOutputPath?: string; +}): Promise { + const schemaPath = path.resolve(params.repoRoot, params.schemaInputPath ?? DEFAULT_SCHEMA_INPUT); + const sqlPath = path.resolve(params.repoRoot, params.sqlOutputPath ?? DEFAULT_SQL_OUTPUT); + const hashPath = path.resolve(params.repoRoot, params.hashOutputPath ?? DEFAULT_HASH_OUTPUT); + const sourceSql = await fs.readFile(schemaPath, "utf8"); + const rendered = renderSqliteSessionSchemaBaseline(sourceSql); + const hash = computeSqliteSessionSchemaBaselineHashFileContent(rendered); + const existingHash = await readIfExists(hashPath); + const existingSql = await readIfExists(sqlPath); + const changed = params.check + ? existingHash !== hash + : existingHash !== hash || existingSql !== rendered.sql; + + if (!params.check && changed) { + await fs.mkdir(path.dirname(sqlPath), { recursive: true }); + await fs.writeFile(sqlPath, rendered.sql); + await fs.writeFile(hashPath, hash); + } + + return { + changed, + wrote: !params.check && changed, + sqlPath, + hashPath, + }; +} diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index a3b2c84613ac..81a21b511af9 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -195,12 +195,12 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { ), publicExports: readPluginSdkSurfaceBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", - 10540, + 10541, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", - 5250, + 5247, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/scripts/run-vitest.mjs b/scripts/run-vitest.mjs index be00aa89eb44..44e6123b11dd 100644 --- a/scripts/run-vitest.mjs +++ b/scripts/run-vitest.mjs @@ -34,6 +34,7 @@ const UI_VITEST_CONFIG = "test/vitest/vitest.ui.config.ts"; const TOOLING_DOCKER_VITEST_CONFIG = "test/vitest/vitest.tooling-docker.config.ts"; const TOOLING_VITEST_CONFIG = "test/vitest/vitest.tooling.config.ts"; const GATEWAY_CORE_VITEST_CONFIG = "test/vitest/vitest.gateway-core.config.ts"; +const GATEWAY_SERVER_VITEST_CONFIG = "test/vitest/vitest.gateway-server.config.ts"; const GATEWAY_VITEST_CONFIG = "test/vitest/vitest.gateway.config.ts"; export const VITEST_CONFIG_NO_OUTPUT_TIMEOUT_MS = new Map([ ["test/vitest/vitest.e2e.config.ts", DEFAULT_LONG_RUNNING_VITEST_NO_OUTPUT_TIMEOUT_MS], @@ -50,6 +51,7 @@ export const VITEST_CONFIG_NO_OUTPUT_TIMEOUT_MS = new Map([ ], ["test/vitest/vitest.infra.config.ts", DEFAULT_EXTRA_LONG_RUNNING_VITEST_NO_OUTPUT_TIMEOUT_MS], [GATEWAY_CORE_VITEST_CONFIG, DEFAULT_EXTRA_LONG_RUNNING_VITEST_NO_OUTPUT_TIMEOUT_MS], + [GATEWAY_SERVER_VITEST_CONFIG, DEFAULT_EXTRA_LONG_RUNNING_VITEST_NO_OUTPUT_TIMEOUT_MS], ]); export const TOOLING_EXCLUDED_TESTS = new Set([ ...boundaryTestFiles, diff --git a/src/acp/runtime/session-meta.test.ts b/src/acp/runtime/session-meta.test.ts index cdf1e6166a2f..dca5f3dd7d74 100644 --- a/src/acp/runtime/session-meta.test.ts +++ b/src/acp/runtime/session-meta.test.ts @@ -1,10 +1,11 @@ /** Tests ACP session metadata persistence, joins, and migration helpers. */ -import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { OpenClawConfig } from "../../config/config.js"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import { loadSessionStore } from "../../config/sessions/store-load.js"; -import { writeSessionStoreForTestAsync } from "../../config/sessions/test-helpers.js"; +import type { SessionEntry } from "../../config/sessions/types.js"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; import { withTempDir } from "../../test-helpers/temp-dir.js"; import { @@ -16,8 +17,37 @@ import { writeAcpSessionMetaForMigration, } from "./session-meta.js"; +const ACP_AGENT_ID = "codex"; + +async function seedAcpSessionEntry(params: { + storePath: string; + sessionKey: string; + entry: SessionEntry; +}): Promise { + await replaceSessionEntry( + { + agentId: ACP_AGENT_ID, + storePath: params.storePath, + sessionKey: params.sessionKey, + }, + params.entry, + ); +} + +function readStoredAcpSessionEntry(params: { + storePath: string; + sessionKey: string; +}): SessionEntry | undefined { + return loadSessionEntry({ + agentId: ACP_AGENT_ID, + storePath: params.storePath, + sessionKey: params.sessionKey, + }); +} + describe("ACP session metadata SQLite store", () => { afterEach(() => { + closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); }); @@ -27,16 +57,14 @@ describe("ACP session metadata SQLite store", () => { const databasePath = path.join(dir, "state", "openclaw.sqlite"); const cfg = { session: { store: storePath } } as OpenClawConfig; const sessionKey = "agent:codex:acp:binding:discord:default:feedface"; - await fs.writeFile( + await seedAcpSessionEntry({ storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sess-acp", - updatedAt: 100, - }, - }), - "utf8", - ); + sessionKey, + entry: { + sessionId: "sess-acp", + updatedAt: 100, + }, + }); const result = await upsertAcpSessionMeta({ cfg, @@ -73,6 +101,55 @@ describe("ACP session metadata SQLite store", () => { }); }); + it("clears legacy embedded ACP metadata through the session accessor", async () => { + await withTempDir({ prefix: "openclaw-acp-meta-" }, async (dir) => { + const storePath = path.join(dir, "sessions.json"); + const databasePath = path.join(dir, "state", "openclaw.sqlite"); + const cfg = { session: { store: storePath } } as OpenClawConfig; + const sessionKey = "agent:codex:acp:binding:discord:default:feedface"; + await seedAcpSessionEntry({ + storePath, + sessionKey, + entry: { + sessionId: "sess-acp", + updatedAt: 100, + acp: { + backend: "acpx", + agent: "codex", + runtimeSessionName: "legacy-embedded", + mode: "persistent", + state: "idle", + lastActivityAt: 120, + }, + }, + }); + + await upsertAcpSessionMeta({ + cfg, + databasePath, + sessionKey, + now: () => 200, + mutate: () => ({ + backend: "acpx", + agent: "codex", + runtimeSessionName: "codex-sqlite", + mode: "persistent", + state: "idle", + lastActivityAt: 123, + }), + }); + + expect(readStoredAcpSessionEntry({ storePath, sessionKey })?.acp).toBeUndefined(); + expect( + readAcpSessionEntry({ + cfg, + databasePath, + sessionKey, + })?.acp?.runtimeSessionName, + ).toBe("codex-sqlite"); + }); + }); + it("creates a session-store row for new SQLite ACP sessions without embedding ACP metadata", async () => { await withTempDir({ prefix: "openclaw-acp-meta-" }, async (dir) => { const storePath = path.join(dir, "sessions.json"); @@ -97,7 +174,7 @@ describe("ACP session metadata SQLite store", () => { expect(result?.sessionId).toEqual(expect.any(String)); expect(result?.acp?.runtimeSessionName).toBe("codex-new"); - const storedEntry = loadSessionStore(storePath)[sessionKey]; + const storedEntry = readStoredAcpSessionEntry({ storePath, sessionKey }); expect(storedEntry?.sessionId).toEqual(expect.any(String)); expect(storedEntry?.updatedAt).toEqual(expect.any(Number)); expect(storedEntry?.acp).toBeUndefined(); @@ -111,16 +188,14 @@ describe("ACP session metadata SQLite store", () => { const cfg = { session: { store: storePath } } as OpenClawConfig; const storeSessionKey = "agent:codex:acp:binding:discord:default:feedface"; const rawSessionKey = storeSessionKey.toUpperCase(); - await fs.writeFile( + await seedAcpSessionEntry({ storePath, - JSON.stringify({ - [storeSessionKey]: { - sessionId: "sess-acp", - updatedAt: 100, - }, - }), - "utf8", - ); + sessionKey: storeSessionKey, + entry: { + sessionId: "sess-acp", + updatedAt: 100, + }, + }); await upsertAcpSessionMeta({ cfg, @@ -152,13 +227,18 @@ describe("ACP session metadata SQLite store", () => { })?.acp?.runtimeSessionName, ).toBe("codex-normalized"); expect(loadSessionStore(storePath)[storeSessionKey]?.acp).toBeUndefined(); - const legacyEmbeddedEntry = loadSessionStore(storePath)[storeSessionKey]; + const legacyEmbeddedEntry = readStoredAcpSessionEntry({ + storePath, + sessionKey: storeSessionKey, + }); expect(legacyEmbeddedEntry).toBeDefined(); if (!legacyEmbeddedEntry) { throw new Error("expected normalized ACP session entry"); } - await writeSessionStoreForTestAsync(storePath, { - [storeSessionKey]: { + await seedAcpSessionEntry({ + storePath, + sessionKey: storeSessionKey, + entry: { ...legacyEmbeddedEntry, acp: { backend: "acpx", @@ -188,7 +268,9 @@ describe("ACP session metadata SQLite store", () => { sessionKey: storeSessionKey, })?.acp, ).toBeUndefined(); - expect(loadSessionStore(storePath)[storeSessionKey]?.acp).toBeUndefined(); + expect( + readStoredAcpSessionEntry({ storePath, sessionKey: storeSessionKey })?.acp, + ).toBeUndefined(); }); }); @@ -199,16 +281,14 @@ describe("ACP session metadata SQLite store", () => { const cfg = { session: { store: storePath } } as OpenClawConfig; const legacyStoreSessionKey = "agent:CODEX:acp:legacy-runtime"; const canonicalSessionKey = "agent:codex:acp:legacy-runtime"; - await fs.writeFile( + await seedAcpSessionEntry({ storePath, - JSON.stringify({ - [legacyStoreSessionKey]: { - sessionId: "sess-acp", - updatedAt: 100, - }, - }), - "utf8", - ); + sessionKey: legacyStoreSessionKey, + entry: { + sessionId: "sess-acp", + updatedAt: 100, + }, + }); await upsertAcpSessionMeta({ cfg, @@ -225,9 +305,9 @@ describe("ACP session metadata SQLite store", () => { }), }); - const store = loadSessionStore(storePath); - expect(store[legacyStoreSessionKey]).toBeUndefined(); - expect(store[canonicalSessionKey]?.sessionId).toBe("sess-acp"); + expect( + readStoredAcpSessionEntry({ storePath, sessionKey: canonicalSessionKey })?.sessionId, + ).toBe("sess-acp"); expect( readAcpSessionEntry({ cfg, @@ -246,20 +326,22 @@ describe("ACP session metadata SQLite store", () => { const cfg = { session: { store: storePath } } as OpenClawConfig; const canonicalSessionKey = "agent:codex:acp:alias-runtime"; const legacyStoreSessionKey = "agent:CODEX:acp:alias-runtime"; - await fs.writeFile( + await seedAcpSessionEntry({ storePath, - JSON.stringify({ - [canonicalSessionKey]: { - sessionId: "sess-canonical", - updatedAt: 100, - }, - [legacyStoreSessionKey]: { - sessionId: "sess-legacy", - updatedAt: 150, - }, - }), - "utf8", - ); + sessionKey: canonicalSessionKey, + entry: { + sessionId: "sess-canonical", + updatedAt: 100, + }, + }); + await seedAcpSessionEntry({ + storePath, + sessionKey: legacyStoreSessionKey, + entry: { + sessionId: "sess-legacy", + updatedAt: 150, + }, + }); await upsertAcpSessionMeta({ cfg, @@ -276,9 +358,9 @@ describe("ACP session metadata SQLite store", () => { }), }); - const store = loadSessionStore(storePath); - expect(store[legacyStoreSessionKey]).toBeUndefined(); - expect(store[canonicalSessionKey]?.sessionId).toBe("sess-legacy"); + expect( + readStoredAcpSessionEntry({ storePath, sessionKey: canonicalSessionKey })?.sessionId, + ).toBe("sess-legacy"); expect( readAcpSessionEntry({ cfg, @@ -296,16 +378,14 @@ describe("ACP session metadata SQLite store", () => { const databasePath = path.join(dir, "state", "openclaw.sqlite"); const cfg = { session: { store: storePath } } as OpenClawConfig; const sessionKey = "agent:codex:acp:binding:discord:default:feedface"; - await fs.writeFile( + await seedAcpSessionEntry({ storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sess-new", - updatedAt: 100, - }, - }), - "utf8", - ); + sessionKey, + entry: { + sessionId: "sess-new", + updatedAt: 100, + }, + }); writeAcpSessionMetaForMigration({ databasePath, @@ -352,8 +432,10 @@ describe("ACP session metadata SQLite store", () => { const cfg = { session: { store: storePath } } as OpenClawConfig; const legacyKey = "agent:CODEX:acp:legacy-runtime"; const canonicalKey = "agent:codex:acp:legacy-runtime"; - await writeSessionStoreForTestAsync(storePath, { - [canonicalKey]: { + await seedAcpSessionEntry({ + storePath, + sessionKey: canonicalKey, + entry: { sessionId: "sess-acp", updatedAt: 100, }, @@ -397,22 +479,19 @@ describe("ACP session metadata SQLite store", () => { it("lists SQLite ACP rows while joining current session-store entries", async () => { await withTempDir({ prefix: "openclaw-acp-meta-" }, async (dir) => { - const storePath = path.join(dir, "sessions", "codex.json"); + const storePath = path.join(dir, "sessions.json"); const databasePath = path.join(dir, "state", "openclaw.sqlite"); const cfg = { session: { store: storePath } } as OpenClawConfig; const sessionKey = "agent:codex:acp:s1"; - await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile( + await seedAcpSessionEntry({ storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sess-acp", - updatedAt: 100, - model: "gpt-5.5", - }, - }), - "utf8", - ); + sessionKey, + entry: { + sessionId: "sess-acp", + updatedAt: 100, + model: "gpt-5.5", + }, + }); await upsertAcpSessionMeta({ cfg, databasePath, @@ -455,17 +534,14 @@ describe("ACP session metadata SQLite store", () => { const cfg = {} as OpenClawConfig; const sessionKey = "agent:codex:acp:s1"; const storePath = path.join(dir, "agents", "codex", "sessions", "sessions.json"); - await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile( + await seedAcpSessionEntry({ storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sess-acp", - updatedAt: 100, - }, - }), - "utf8", - ); + sessionKey, + entry: { + sessionId: "sess-acp", + updatedAt: 100, + }, + }); await upsertAcpSessionMeta({ cfg, env, diff --git a/src/acp/runtime/session-meta.ts b/src/acp/runtime/session-meta.ts index df1eef0210a8..c0b87df3fc49 100644 --- a/src/acp/runtime/session-meta.ts +++ b/src/acp/runtime/session-meta.ts @@ -35,6 +35,7 @@ import { /** ACP metadata joined with its legacy session-store row and config context. */ export type AcpSessionStoreEntry = { cfg: OpenClawConfig; + agentId?: string; storePath: string; sessionKey: string; storeSessionKey: string; @@ -76,14 +77,18 @@ export function resolveSessionStorePathForAcp(params: { sessionKey: string; cfg?: OpenClawConfig; env?: NodeJS.ProcessEnv; -}): { cfg: OpenClawConfig; storePath: string } { +}): { cfg: OpenClawConfig; agentId?: string; storePath: string } { const cfg = params.cfg ?? getRuntimeConfig(); const parsed = parseAgentSessionKey(params.sessionKey); const storePath = resolveStorePath(cfg.session?.store, { agentId: parsed?.agentId, env: params.env, }); - return { cfg, storePath }; + return { + cfg, + ...(parsed?.agentId ? { agentId: parsed.agentId } : {}), + storePath, + }; } function getAcpSessionKysely(db: DatabaseSync) { @@ -349,27 +354,30 @@ function readSessionEntryFromStore(params: { clone?: boolean; }): { cfg: OpenClawConfig; + agentId?: string; storePath: string; storeSessionKey: string; entry?: SessionEntry; storeReadFailed?: boolean; } { - const { cfg, storePath } = resolveSessionStorePathForAcp({ + const { cfg, agentId, storePath } = resolveSessionStorePathForAcp({ sessionKey: params.sessionKey, cfg: params.cfg, env: params.env, }); try { const entries = listSessionEntries({ + ...(agentId ? { agentId } : {}), storePath, ...(params.clone === false ? { clone: false } : {}), }); const storeSessionKey = resolveStoreSessionKey(entries, params.sessionKey); const entry = entries.find((candidate) => candidate.sessionKey === storeSessionKey)?.entry; - return { cfg, storePath, storeSessionKey, entry }; + return { cfg, agentId, storePath, storeSessionKey, entry }; } catch { return { cfg, + agentId, storePath, storeSessionKey: normalizeLowercaseStringOrEmpty(params.sessionKey), storeReadFailed: true, @@ -398,6 +406,7 @@ export function readAcpSessionEntry(params: { row && acpSessionRowMatchesEntry(row, storeEntry.entry) ? rowToAcpSessionMeta(row) : undefined; return { cfg: storeEntry.cfg, + agentId: storeEntry.agentId, storePath: storeEntry.storePath, sessionKey, storeSessionKey: storeEntry.storeSessionKey, @@ -422,7 +431,7 @@ export async function listAcpSessionEntries(params: { for (const row of rows) { const sessionKey = row.session_key; - const { storePath } = resolveSessionStorePathForAcp({ + const { agentId, storePath } = resolveSessionStorePathForAcp({ sessionKey, cfg, env: params.env, @@ -430,6 +439,7 @@ export async function listAcpSessionEntries(params: { let sessionEntries: SessionEntrySummary[]; try { sessionEntries = listSessionEntries({ + ...(agentId ? { agentId } : {}), storePath, ...(params.clone === false ? { clone: false } : {}), }); @@ -445,6 +455,7 @@ export async function listAcpSessionEntries(params: { } entries.push({ cfg, + agentId, storePath, sessionKey, storeSessionKey, @@ -472,6 +483,40 @@ function sessionStoreUpdateOptions(params: { }; } +async function clearLegacyEmbeddedAcpMetadata(params: { + storePath: string; + sessionKeys: Iterable; +}): Promise { + const sessionKeys = new Set( + Array.from(params.sessionKeys, (sessionKey) => sessionKey?.trim()).filter( + (sessionKey): sessionKey is string => Boolean(sessionKey), + ), + ); + if (sessionKeys.size === 0) { + return; + } + for (const sessionKey of sessionKeys) { + await patchSessionEntryWithKey( + { + storePath: params.storePath, + sessionKey, + }, + (entry) => { + if (!entry.acp) { + return null; + } + const next = { ...entry }; + delete next.acp; + return next; + }, + { + replaceEntry: true, + skipMaintenance: true, + }, + ); + } +} + export async function upsertAcpSessionMeta(params: { sessionKey: string; cfg?: OpenClawConfig; @@ -523,7 +568,11 @@ export async function upsertAcpSessionMeta(params: { if (metaToPersist === null) { const patched = entry ? await patchSessionEntryWithKey( - { storePath: storeEntry.storePath, sessionKey: storageSessionKey }, + { + ...(storeEntry.agentId ? { agentId: storeEntry.agentId } : {}), + storePath: storeEntry.storePath, + sessionKey: storageSessionKey, + }, (currentEntry) => { const next = { ...currentEntry }; delete next.acp; @@ -552,10 +601,18 @@ export async function upsertAcpSessionMeta(params: { }, { env: params.env, path: params.databasePath }, ); + await clearLegacyEmbeddedAcpMetadata({ + storePath: storeEntry.storePath, + sessionKeys: [storageSessionKey, patched?.sessionKey], + }); return patched?.entry ?? null; } const persisted = await patchSessionEntryWithKey( - { storePath: storeEntry.storePath, sessionKey: storageSessionKey }, + { + ...(storeEntry.agentId ? { agentId: storeEntry.agentId } : {}), + storePath: storeEntry.storePath, + sessionKey: storageSessionKey, + }, (currentEntry) => { const next = mergeSessionEntry(currentEntry, { updatedAt, @@ -572,6 +629,10 @@ export async function upsertAcpSessionMeta(params: { if (!persisted) { return null; } + await clearLegacyEmbeddedAcpMetadata({ + storePath: storeEntry.storePath, + sessionKeys: [storageSessionKey, persisted.sessionKey], + }); runOpenClawStateWriteTransaction( (database) => { upsertAcpSessionMetaRow( diff --git a/src/agents/acp-spawn.test.ts b/src/agents/acp-spawn.test.ts index e0a33f481e78..9823a8401f86 100644 --- a/src/agents/acp-spawn.test.ts +++ b/src/agents/acp-spawn.test.ts @@ -2764,9 +2764,9 @@ describe("spawnAcpDirect", () => { accountId: "default", }, spawnedBy: "agent:main:subagent:parent", - spawnDepth: 2, - subagentRole: "leaf", - subagentControlScope: "none", + spawnDepth: 1, + subagentRole: "orchestrator", + subagentControlScope: "children", }, }; return new Proxy(store, { diff --git a/src/agents/agent-command.compaction-rotation.test.ts b/src/agents/agent-command.compaction-rotation.test.ts index e99e86453693..4edb8ef443c9 100644 --- a/src/agents/agent-command.compaction-rotation.test.ts +++ b/src/agents/agent-command.compaction-rotation.test.ts @@ -3,8 +3,16 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { loadSessionStore, saveSessionStore, type SessionEntry } from "../config/sessions.js"; -import { CURRENT_SESSION_VERSION } from "../config/sessions/version.js"; +import type { SessionEntry } from "../config/sessions.js"; +import { + listSessionEntries, + loadTranscriptEvents, + replaceSessionEntry, +} from "../config/sessions/session-accessor.js"; +import { + formatSqliteSessionFileMarker, + parseSqliteSessionFileMarker, +} from "../config/sessions/sqlite-marker.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { runAgentAttempt } from "./command/attempt-execution.runtime.js"; import type { EmbeddedAgentRunResult } from "./embedded-agent.js"; @@ -227,13 +235,20 @@ function makeResult(params: { }; } -async function readSessionMessages(sessionFile: string) { - const raw = await fs.readFile(sessionFile, "utf-8"); - return raw - .split(/\r?\n/) - .filter(Boolean) - .map((line) => JSON.parse(line) as { type?: string; message?: { role?: string } }) - .filter((entry) => entry.type === "message") +async function readSessionMessages(params: { + agentId: string; + sessionId: string; + storePath: string; +}) { + return (await loadTranscriptEvents(params)) + .filter( + (entry): entry is { message: unknown; type: "message" } => + typeof entry === "object" && + entry !== null && + "message" in entry && + "type" in entry && + entry.type === "message", + ) .map((entry) => entry.message); } @@ -318,10 +333,13 @@ describe("agentCommand compaction transcript rotation", () => { expect(state.loadManifestModelCatalogMock).not.toHaveBeenCalled(); }); - it("keeps sessions.json on the rotated successor", async () => { + it("keeps SQLite session state on the rotated successor", async () => { const storePath = requireStorePath(); - const sessionsDir = await fs.realpath(path.dirname(storePath)); - const rotatedSessionFile = path.join(sessionsDir, "rotated-session.jsonl"); + const rotatedSessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: "rotated-session", + storePath, + }); state.runAgentAttemptMock.mockResolvedValueOnce( makeResult({ sessionId: "rotated-session", @@ -337,7 +355,9 @@ describe("agentCommand compaction transcript rotation", () => { cwd: state.workspaceDir, }); - const storeAfterRotation = loadSessionStore(storePath, { skipCache: true }); + const storeAfterRotation = Object.fromEntries( + listSessionEntries({ storePath }).map(({ entry, sessionKey }) => [sessionKey, entry]), + ); const entriesAfterRotation = Object.entries(storeAfterRotation); expect(entriesAfterRotation).toHaveLength(1); const [sessionKey, rotatedEntry] = entriesAfterRotation[0] ?? []; @@ -349,29 +369,22 @@ describe("agentCommand compaction transcript rotation", () => { usageFamilySessionIds: ["old-session", "rotated-session"], compactionCount: 1, }); - await expect(readSessionMessages(rotatedSessionFile)).resolves.toEqual([ - expect.objectContaining({ role: "assistant" }), - ]); + await expect( + readSessionMessages({ agentId: "main", sessionId: "rotated-session", storePath }), + ).resolves.toContainEqual(expect.objectContaining({ role: "assistant" })); }); it("resumes the next turn from the rotated successor", async () => { const storePath = requireStorePath(); - const sessionsDir = await fs.realpath(path.dirname(storePath)); - const rotatedSessionFile = path.join(sessionsDir, "rotated-session.jsonl"); + const rotatedSessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: "rotated-session", + storePath, + }); const sessionKey = "agent:main:explicit:old-session"; - await fs.writeFile( - rotatedSessionFile, - `${JSON.stringify({ - type: "session", - version: CURRENT_SESSION_VERSION, - id: "rotated-session", - timestamp: new Date(0).toISOString(), - cwd: state.workspaceDir, - })}\n`, - "utf-8", - ); - await saveSessionStore(storePath, { - [sessionKey]: { + await replaceSessionEntry( + { sessionKey, storePath }, + { sessionId: "rotated-session", sessionFile: rotatedSessionFile, updatedAt: Date.now(), @@ -379,7 +392,7 @@ describe("agentCommand compaction transcript rotation", () => { usageFamilySessionIds: ["old-session", "rotated-session"], compactionCount: 1, }, - }); + ); state.runAgentAttemptMock.mockResolvedValueOnce( makeResult({ sessionId: "rotated-session", @@ -399,13 +412,20 @@ describe("agentCommand compaction transcript rotation", () => { expect(secondAttempt).toMatchObject({ sessionId: "rotated-session", sessionKey, - sessionFile: rotatedSessionFile, + }); + expect(parseSqliteSessionFileMarker(secondAttempt?.sessionFile)).toMatchObject({ + agentId: "main", + sessionId: "rotated-session", + storePath, }); expect(state.deliveryFreshEntries.at(-1)).toMatchObject({ sessionId: "rotated-session", sessionFile: rotatedSessionFile, }); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey ?? ""]).toMatchObject({ + const persisted = Object.fromEntries( + listSessionEntries({ storePath }).map(({ entry, sessionKey: key }) => [key, entry]), + ); + expect(persisted[sessionKey]).toMatchObject({ sessionId: "rotated-session", sessionFile: rotatedSessionFile, }); diff --git a/src/agents/agent-command.live-model-switch.test.ts b/src/agents/agent-command.live-model-switch.test.ts index 9907e5efc828..2116b83783e1 100644 --- a/src/agents/agent-command.live-model-switch.test.ts +++ b/src/agents/agent-command.live-model-switch.test.ts @@ -52,6 +52,7 @@ const state = vi.hoisted(() => ({ emitAgentEventMock: vi.fn(), registerAgentRunContextMock: vi.fn(), clearAgentRunContextMock: vi.fn(), + loadSessionEntryMock: vi.fn(), updateSessionStoreAfterAgentRunMock: vi.fn(), deliverAgentCommandResultMock: vi.fn(), resolveAgentDeliveryPlanMock: vi.fn(), @@ -60,7 +61,6 @@ const state = vi.hoisted(() => ({ createTrajectoryRuntimeRecorderMock: vi.fn(), trajectoryRecordEventMock: vi.fn(), trajectoryFlushMock: vi.fn(async () => undefined), - removeSessionTrajectoryArtifactsMock: vi.fn(async (..._args: unknown[]) => []), persistSessionEntryMock: vi.fn(async (..._args: unknown[]): Promise => undefined), clearSessionAuthProfileOverrideMock: vi.fn(), isThinkingLevelSupportedMock: vi.fn((_args: unknown) => true), @@ -73,13 +73,14 @@ const state = vi.hoisted(() => ({ resolvedSkills: [], version: 0, })), - prepareInternalSessionEffectsTranscriptMock: vi.fn(), - removeInternalSessionEffectsTranscriptMock: vi.fn(), + prepareInternalSessionEffectsSessionMock: vi.fn(), + removeInternalSessionEffectsSessionMock: vi.fn(), authProfileStoreMock: { profiles: {} } as { profiles: Record }, sessionEntryMock: undefined as SessionEntry | undefined, sessionStoreMock: undefined as unknown, storePathMock: undefined as string | undefined, resolvedSessionKeyMock: undefined as string | undefined, + trajectoryRecorderParamsMock: vi.fn(), })); vi.mock("./model-fallback.js", () => ({ @@ -164,8 +165,7 @@ vi.mock("./command/run-context.js", () => ({ })); vi.mock("./command/session-store.runtime.js", () => ({ - loadSessionEntry: ({ sessionKey }: { sessionKey: string }) => - (state.sessionStoreMock as Record | undefined)?.[sessionKey], + loadSessionEntry: (...args: unknown[]) => state.loadSessionEntryMock(...args), updateSessionStoreAfterAgentRun: (...args: unknown[]) => state.updateSessionStoreAfterAgentRunMock(...args), })); @@ -277,21 +277,23 @@ vi.mock("../config/sessions.js", () => ({ })); vi.mock("../config/sessions/transcript-resolve.runtime.js", () => ({ - resolveSessionTranscriptFile: async () => ({ - sessionFile: "/tmp/session.jsonl", - sessionEntry: { sessionId: "session-1", updatedAt: Date.now() }, + resolveSessionTranscriptFile: async (params: { sessionEntry?: SessionEntry }) => ({ + sessionFile: params.sessionEntry?.sessionFile ?? "/tmp/session.jsonl", + sessionEntry: params.sessionEntry ?? { sessionId: "session-1", updatedAt: Date.now() }, }), })); vi.mock("./internal-session-effects.js", () => ({ - isInternalSessionEffectsTranscriptPath: (sessionFile: string | undefined) => - sessionFile?.startsWith("/tmp/openclaw-internal-") === true, - prepareInternalSessionEffectsTranscript: (...args: unknown[]) => - state.prepareInternalSessionEffectsTranscriptMock(...args), - removeInternalSessionEffectsTranscript: (...args: unknown[]) => - state.removeInternalSessionEffectsTranscriptMock(...args), - resolveInternalSessionEffectsTranscriptPath: (runId: string) => - `/tmp/openclaw-internal-${runId}.jsonl`, + prepareInternalSessionEffectsSession: (...args: unknown[]) => + state.prepareInternalSessionEffectsSessionMock(...args), + removeInternalSessionEffectsSession: (...args: unknown[]) => + state.removeInternalSessionEffectsSessionMock(...args), + resolveInternalSessionEffectsTarget: ({ agentId, runId, storePath }: Record) => ({ + agentId, + sessionId: `internal-${runId}`, + sessionKey: `agent:${agentId}:internal-session-effects:${runId}`, + storePath, + }), })); vi.mock("../infra/agent-events.js", () => ({ @@ -394,14 +396,10 @@ vi.mock("../terminal/ansi.js", () => ({ sanitizeForLog: (s: string) => s, })); -vi.mock("../trajectory/cleanup.js", () => ({ - removeSessionTrajectoryArtifacts: (...args: unknown[]) => - state.removeSessionTrajectoryArtifactsMock(...args), -})); - vi.mock("../trajectory/runtime.js", () => ({ createTrajectoryRuntimeRecorder: (params: unknown) => { state.createTrajectoryRuntimeRecorderMock(params); + state.trajectoryRecorderParamsMock(params); return { enabled: true, filePath: "/tmp/session.trajectory.jsonl", @@ -1114,6 +1112,10 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { }), ); state.resolveMessageChannelSelectionMock.mockRejectedValue(new Error("channel required")); + state.loadSessionEntryMock.mockReset().mockImplementation((params: { sessionKey?: string }) => { + const sessionKey = params.sessionKey ?? state.resolvedSessionKeyMock ?? "agent:main:main"; + return (state.sessionStoreMock as Record | undefined)?.[sessionKey]; + }); state.resolveAgentDeliveryPlanMock.mockImplementation( (params: { accountId?: string; @@ -1145,11 +1147,15 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { ); state.updateSessionStoreAfterAgentRunMock.mockResolvedValue(undefined); state.trajectoryFlushMock.mockResolvedValue(undefined); - state.removeSessionTrajectoryArtifactsMock.mockResolvedValue([]); - state.prepareInternalSessionEffectsTranscriptMock.mockResolvedValue( - "/tmp/openclaw-internal-run.jsonl", - ); - state.removeInternalSessionEffectsTranscriptMock.mockResolvedValue(undefined); + state.prepareInternalSessionEffectsSessionMock.mockResolvedValue({ + agentId: "default", + sessionId: "internal-session", + sessionKey: "agent:default:internal-session-effects:run", + storePath: "/tmp/openclaw-session-store.json", + sessionFile: "sqlite:default:internal-session:/tmp/openclaw-session-store.json", + sessionEntry: { sessionId: "internal-session", updatedAt: 1 }, + }); + state.removeInternalSessionEffectsSessionMock.mockResolvedValue(undefined); }); afterEach(() => { @@ -2253,6 +2259,78 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { expect(state.deliverAgentCommandResultMock).toHaveBeenCalledTimes(1); }); + it("persists only the CLI assistant reply after the runner persists the current user turn", async () => { + type AttemptCall = { + onUserMessagePersisted?: () => void; + }; + setupSingleAttemptFallback(); + setupSessionTouchStore(); + const result = makeSuccessResult("openai", "gpt-5.4") as ReturnType< + typeof makeSuccessResult + > & { + meta: Record & { executionTrace: Record }; + }; + result.meta.executionTrace = { + runner: "cli", + fallbackUsed: false, + winnerProvider: "openai", + winnerModel: "gpt-5.4", + }; + state.runAgentAttemptMock.mockImplementation(async (attemptParams: AttemptCall) => { + attemptParams.onUserMessagePersisted?.(); + return result; + }); + state.persistCliTurnTranscriptMock.mockResolvedValue({ + kind: "persisted", + sessionEntry: state.sessionEntryMock, + }); + + await agentCommand({ + message: "hello", + to: "+1234567890", + userTurnTranscriptRecorder: {} as NonNullable< + Parameters[0]["userTurnTranscriptRecorder"] + >, + }); + + expectRecordFields(mockCallArg(state.persistCliTurnTranscriptMock), { + embeddedAssistantGapFill: false, + }); + }); + + it("persists the full embedded turn when no canonical user recorder exists", async () => { + type AttemptCall = { + onUserMessagePersisted?: () => void; + }; + setupSingleAttemptFallback(); + setupSessionTouchStore(); + const result = makeSuccessResult("openai", "gpt-5.4") as ReturnType< + typeof makeSuccessResult + > & { + meta: Record & { executionTrace: Record }; + }; + result.meta.executionTrace = { + runner: "embedded", + fallbackUsed: false, + winnerProvider: "openai", + winnerModel: "gpt-5.4", + }; + state.runAgentAttemptMock.mockImplementation(async (attemptParams: AttemptCall) => { + attemptParams.onUserMessagePersisted?.(); + return result; + }); + state.persistCliTurnTranscriptMock.mockResolvedValue({ + kind: "persisted", + sessionEntry: state.sessionEntryMock, + }); + + await runBasicAgentCommand(); + + expectRecordFields(mockCallArg(state.persistCliTurnTranscriptMock), { + embeddedAssistantGapFill: true, + }); + }); + it("does not treat backend CLI session id as OpenClaw session identity", async () => { setupSingleAttemptFallback(); setupSessionTouchStore(); @@ -2535,6 +2613,54 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { expect(stored?.restartRecoveryDeliveryContext).toBeUndefined(); }); + it("refreshes delivery session entries through the session accessor", async () => { + setupSingleAttemptFallback(); + state.runAgentAttemptMock.mockResolvedValue(makeSuccessResult("openai", "gpt-5.4")); + const sessionEntry: SessionEntry = { + sessionId: "session-1", + updatedAt: 1, + }; + const freshEntry: SessionEntry = { + sessionId: "session-1", + updatedAt: 2, + deliveryContext: { + channel: "discord", + to: "discord:dm:sqlite", + accountId: "main", + }, + }; + const sessionStore: Record = { "agent:main:main": sessionEntry }; + state.sessionEntryMock = sessionEntry; + state.sessionStoreMock = sessionStore; + state.storePathMock = "/tmp/openclaw-sessions.json"; + state.loadSessionEntryMock.mockReturnValue(freshEntry); + state.deliverAgentCommandResultMock.mockImplementation(async (params: unknown) => { + const resolver = ( + params as { + resolveFreshSessionEntryForDelivery?: () => Promise; + } + ).resolveFreshSessionEntryForDelivery; + const resolved = await resolver?.(); + expect(resolved).toEqual(freshEntry); + return { deliverySucceeded: true }; + }); + + await agentCommand({ + message: "hello", + channel: "discord", + to: "discord:dm:123", + accountId: "main", + deliver: true, + }); + + expect(state.loadSessionEntryMock).toHaveBeenCalledWith({ + storePath: "/tmp/openclaw-sessions.json", + sessionKey: "agent:main:main", + readConsistency: "latest", + clone: false, + }); + }); + it("preserves parsed explicit target threads for restart recovery", async () => { setupSingleAttemptFallback(); state.runAgentAttemptMock.mockResolvedValue(makeSuccessResult("openai", "gpt-5.4")); @@ -2991,6 +3117,35 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { ); }); + it("passes SQLite transcript markers to visible agent attempts", async () => { + setupSingleAttemptFallback(); + const visibleEntry: SessionEntry = { + sessionId: "session-1", + updatedAt: 1, + sessionFile: "sqlite:default:session-1:/tmp/openclaw-session-store.json", + }; + const sessionStore: Record = { "agent:main:main": visibleEntry }; + state.sessionEntryMock = visibleEntry; + state.sessionStoreMock = sessionStore; + state.storePathMock = "/tmp/openclaw-session-store.json"; + state.loadSessionEntryMock.mockReturnValue(visibleEntry); + const attemptCalls: Array<{ sessionFile?: string; sessionEntry?: SessionEntry }> = []; + state.runAgentAttemptMock.mockImplementation(async (params) => { + attemptCalls.push(params as { sessionFile?: string; sessionEntry?: SessionEntry }); + return makeSuccessResult("openai", "gpt-5.4"); + }); + + await agentCommand({ + message: "visible run", + to: "+1234567890", + }); + + expect(attemptCalls).toHaveLength(1); + expect(attemptCalls[0]?.sessionFile).toBe( + "sqlite:default:session-1:/tmp/openclaw-session-store.json", + ); + }); + it("keeps internal session-effect CLI runs out of visible session state", async () => { setupSingleAttemptFallback(); const visibleEntry: SessionEntry = { @@ -3006,6 +3161,7 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { state.sessionEntryMock = visibleEntry; state.sessionStoreMock = sessionStore; state.storePathMock = "/tmp/openclaw-session-store.json"; + state.loadSessionEntryMock.mockReturnValue(visibleEntry); const attemptCalls: Array<{ sessionFile?: string; sessionEntry?: SessionEntry }> = []; state.runAgentAttemptMock.mockImplementation(async (params) => { attemptCalls.push(params as { sessionFile?: string; sessionEntry?: SessionEntry }); @@ -3019,13 +3175,28 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { suppressPromptPersistence: true, }); - expect(state.prepareInternalSessionEffectsTranscriptMock).toHaveBeenCalledWith({ - sessionFile: "/tmp/session.jsonl", - runId: expect.any(String), + expect(state.prepareInternalSessionEffectsSessionMock).toHaveBeenCalledWith({ + agentId: "default", + cwd: "/tmp/workspace", + runId: "session-1", + source: { + agentId: "default", + sessionId: "session-1", + sessionKey: "agent:main:main", + storePath: "/tmp/openclaw-session-store.json", + }, + storePath: "/tmp/openclaw-session-store.json", }); expect(attemptCalls).toHaveLength(1); - expect(attemptCalls[0]?.sessionFile).toBe("/tmp/openclaw-internal-run.jsonl"); + expect(attemptCalls[0]?.sessionFile).toBe( + "sqlite:default:internal-session:/tmp/openclaw-session-store.json", + ); expect(attemptCalls[0]?.sessionEntry).toStrictEqual(visibleEntry); + expect(state.trajectoryRecorderParamsMock).toHaveBeenCalledWith( + expect.objectContaining({ + sessionFile: "sqlite:default:internal-session:/tmp/openclaw-session-store.json", + }), + ); expect(state.persistSessionEntryMock).not.toHaveBeenCalled(); expect(state.updateSessionStoreAfterAgentRunMock).not.toHaveBeenCalled(); expect(sessionStore["agent:main:main"]).toBe(visibleEntry); @@ -3037,8 +3208,7 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { isControlUiVisible: false, }), ); - expect(state.removeSessionTrajectoryArtifactsMock).not.toHaveBeenCalled(); - expect(state.removeInternalSessionEffectsTranscriptMock).not.toHaveBeenCalled(); + expect(state.removeInternalSessionEffectsSessionMock).not.toHaveBeenCalled(); }); it("rejects session-id-resolved model runs for harness-owned sessions", async () => { @@ -3121,15 +3291,12 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { await expect(agentCommand({ message: "continue", sessionId: "session-1" })).rejects.toThrow( "Session key namespace is reserved for agent harness-owned sessions.", ); - expect(state.prepareInternalSessionEffectsTranscriptMock).not.toHaveBeenCalled(); + expect(state.prepareInternalSessionEffectsSessionMock).not.toHaveBeenCalled(); expect(state.runAgentAttemptMock).not.toHaveBeenCalled(); }); - it("removes one-shot internal model-run artifacts after success", async () => { + it("removes the one-shot internal model-run SQLite session after success", async () => { setupSingleAttemptFallback(); - state.prepareInternalSessionEffectsTranscriptMock.mockResolvedValueOnce( - "/tmp/openclaw-internal-model-run-success.jsonl", - ); const result = makeSuccessResult("openai", "gpt-5.4"); state.runAgentAttemptMock.mockResolvedValue({ ...result, @@ -3146,7 +3313,7 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { provider: "openai", model: "gpt-5.4", sessionId: "rotated-model-run-session", - sessionFile: "/tmp/openclaw-internal-rotated.jsonl", + sessionFile: "sqlite:default:rotated-model-run-session:/tmp/openclaw-session-store.json", }, }, }); @@ -3162,49 +3329,32 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { expect(state.createTrajectoryRuntimeRecorderMock).toHaveBeenCalledWith( expect.objectContaining({ - sessionFile: "/tmp/openclaw-internal-model-run-success.jsonl", + sessionFile: "sqlite:default:internal-session:/tmp/openclaw-session-store.json", }), ); expect(state.persistCliTurnTranscriptMock).toHaveBeenCalledWith( expect.objectContaining({ sessionId: "rotated-model-run-session", - sessionFile: "/tmp/openclaw-internal-rotated.jsonl", + sessionKey: "agent:default:internal-session-effects:run", + storePath: "/tmp/openclaw-session-store.json", }), ); - const transcriptCall = mockCallArg(state.persistCliTurnTranscriptMock) as { - sessionEntry?: SessionEntry; - }; - expect(transcriptCall.sessionEntry?.sessionFile).toBeUndefined(); - expect(state.removeSessionTrajectoryArtifactsMock).toHaveBeenCalledWith({ - sessionId: "session-1", - sessionFile: "/tmp/openclaw-internal-model-run-success.jsonl", - storePath: "/tmp/openclaw-internal-model-run-success.jsonl", - }); - expect(state.removeSessionTrajectoryArtifactsMock).toHaveBeenCalledWith({ + expect(state.removeInternalSessionEffectsSessionMock).toHaveBeenCalledWith({ + agentId: "default", sessionId: "rotated-model-run-session", - sessionFile: "/tmp/openclaw-internal-rotated.jsonl", - storePath: "/tmp/openclaw-internal-rotated.jsonl", + sessionKey: "agent:default:internal-session-effects:run", + storePath: "/tmp/openclaw-session-store.json", + sessionFile: "sqlite:default:internal-session:/tmp/openclaw-session-store.json", + sessionEntry: { sessionId: "internal-session", updatedAt: 1 }, }); - expect(state.removeInternalSessionEffectsTranscriptMock).toHaveBeenCalledWith( - "/tmp/openclaw-internal-model-run-success.jsonl", - ); - expect(state.removeInternalSessionEffectsTranscriptMock).toHaveBeenCalledWith( - "/tmp/openclaw-internal-rotated.jsonl", - ); const deliveryOrder = state.deliverAgentCommandResultMock.mock.invocationCallOrder[0] ?? 0; - const trajectoryCleanupOrder = - state.removeSessionTrajectoryArtifactsMock.mock.invocationCallOrder.at(-1) ?? 0; - const transcriptCleanupOrder = - state.removeInternalSessionEffectsTranscriptMock.mock.invocationCallOrder[0] ?? 0; - expect(deliveryOrder).toBeLessThan(trajectoryCleanupOrder); - expect(trajectoryCleanupOrder).toBeLessThan(transcriptCleanupOrder); + const cleanupOrder = + state.removeInternalSessionEffectsSessionMock.mock.invocationCallOrder[0] ?? 0; + expect(deliveryOrder).toBeLessThan(cleanupOrder); }); - it("removes one-shot internal model-run artifacts after provider failure", async () => { + it("removes the one-shot internal model-run SQLite session after provider failure", async () => { setupSingleAttemptFallback(); - state.prepareInternalSessionEffectsTranscriptMock.mockResolvedValueOnce( - "/tmp/openclaw-internal-model-run-failure.jsonl", - ); state.runAgentAttemptMock.mockRejectedValueOnce(new Error("probe failed")); await expect( @@ -3218,19 +3368,20 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { }), ).rejects.toThrow("probe failed"); - expect(state.removeSessionTrajectoryArtifactsMock).toHaveBeenCalledWith( + expect(state.removeInternalSessionEffectsSessionMock).toHaveBeenCalledWith( expect.objectContaining({ - sessionFile: "/tmp/openclaw-internal-model-run-failure.jsonl", + sessionId: "internal-session", + sessionKey: "agent:default:internal-session-effects:run", + storePath: "/tmp/openclaw-session-store.json", }), ); - expect(state.removeInternalSessionEffectsTranscriptMock).toHaveBeenCalledWith( - "/tmp/openclaw-internal-model-run-failure.jsonl", - ); }); - it("cleans the deterministic model-run path when transcript preparation fails", async () => { - state.prepareInternalSessionEffectsTranscriptMock.mockRejectedValueOnce( - new Error("transcript chmod failed"), + it("cleans the deterministic model-run session when preparation fails", async () => { + state.storePathMock = "/tmp/openclaw-session-store.json"; + state.loadSessionEntryMock.mockReturnValue({ sessionId: "session-1", updatedAt: 1 }); + state.prepareInternalSessionEffectsSessionMock.mockRejectedValueOnce( + new Error("session preparation failed"), ); await expect( @@ -3242,16 +3393,37 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { promptMode: "none", sessionEffects: "internal", }), - ).rejects.toThrow("transcript chmod failed"); + ).rejects.toThrow("session preparation failed"); - expect(state.removeSessionTrajectoryArtifactsMock).toHaveBeenCalledWith( - expect.objectContaining({ - sessionFile: "/tmp/openclaw-internal-model-run-prepare-failure.jsonl", + expect(state.removeInternalSessionEffectsSessionMock).toHaveBeenCalledWith({ + agentId: "default", + sessionId: "internal-model-run-prepare-failure", + sessionKey: "agent:default:internal-session-effects:model-run-prepare-failure", + storePath: "/tmp/openclaw-session-store.json", + }); + }); + + it("does not replace a completed model-run result with a SQLite cleanup failure", async () => { + setupSingleAttemptFallback(); + state.storePathMock = "/tmp/openclaw-session-store.json"; + state.loadSessionEntryMock.mockReturnValue({ sessionId: "session-1", updatedAt: 1 }); + state.runAgentAttemptMock.mockResolvedValue(makeSuccessResult("openai", "gpt-5.4")); + state.removeInternalSessionEffectsSessionMock.mockRejectedValue( + new Error("database is locked"), + ); + + await expect( + agentCommand({ + message: "probe", + to: "+1234567890", + runId: "model-run-cleanup-failure", + modelRun: true, + promptMode: "none", + sessionEffects: "internal", }), - ); - expect(state.removeInternalSessionEffectsTranscriptMock).toHaveBeenCalledWith( - "/tmp/openclaw-internal-model-run-prepare-failure.jsonl", - ); + ).resolves.toBeUndefined(); + + expect(state.removeInternalSessionEffectsSessionMock).toHaveBeenCalled(); }); it("does not duplicate finishing lifecycle when an attempt already emitted finishing", async () => { diff --git a/src/agents/agent-command.ts b/src/agents/agent-command.ts index 3739bd91bc1a..66d1b41f01a5 100644 --- a/src/agents/agent-command.ts +++ b/src/agents/agent-command.ts @@ -76,7 +76,6 @@ import { getGeneratedMediaTaskIdsForSessionKey, hasNewGeneratedMediaTaskForSessionKey, } from "../tasks/task-status-access.js"; -import { removeSessionTrajectoryArtifacts } from "../trajectory/cleanup.js"; import { createTrajectoryRuntimeRecorder } from "../trajectory/runtime.js"; import { resolveUserPath } from "../utils.js"; import { @@ -134,10 +133,9 @@ import { runAgentHarnessBeforeMessageWriteHook } from "./harness/hook-helpers.js import { ensureSelectedAgentHarnessPlugin } from "./harness/runtime-plugin.js"; import { resolveAvailableAgentHarnessPolicy } from "./harness/selection.js"; import { - isInternalSessionEffectsTranscriptPath, - prepareInternalSessionEffectsTranscript, - removeInternalSessionEffectsTranscript, - resolveInternalSessionEffectsTranscriptPath, + prepareInternalSessionEffectsSession, + removeInternalSessionEffectsSession, + resolveInternalSessionEffectsTarget, } from "./internal-session-effects.js"; import { AGENT_LANE_SUBAGENT } from "./lanes.js"; import { LiveSessionModelSwitchError } from "./live-model-switch.js"; @@ -163,6 +161,7 @@ import { } from "./model-visibility-policy.js"; import { listOpenAIAuthProfileProvidersForAgentRuntime } from "./openai-routing.js"; import { resolveProviderIdForAuth } from "./provider-auth-aliases.js"; +import type { AgentRunSessionTarget } from "./run-session-target.js"; import { createAgentRunRestartAbortError, isAgentRunDirectAbortReason, @@ -630,6 +629,25 @@ function createAgentCommandSessionWorkingCopy(params: { return result; } +function resolveInternalSessionEffectsSource(params: { + agentId: string; + sessionId: string; + sessionKey?: string; + storePath?: string; +}): + | Required> + | undefined { + if (!params.storePath || !params.sessionKey) { + return undefined; + } + return { + agentId: params.agentId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }; +} + function resolveExplicitAgentCommandSessionKey(params: { rawExplicitSessionKey?: string; agentIdOverride?: string; @@ -1009,24 +1027,21 @@ async function agentCommandInternal( let currentRunDeliveryContext: DeliveryContext | undefined; const preparedSessionId = sessionEntry?.sessionId; const sessionStoreRuntime = storePath && sessionKey ? await loadSessionStoreRuntime() : undefined; - const internalModelRunArtifacts = + const internalModelRunTargets = initialOpts.modelRun === true && suppressVisibleSessionEffects - ? new Map() + ? new Map() : undefined; - const trackInternalModelRunArtifact = ( - sessionFile: string | undefined, - artifactSessionId: string, - ) => { - if (!internalModelRunArtifacts || !isInternalSessionEffectsTranscriptPath(sessionFile)) { + const trackInternalModelRunTarget = (target: AgentRunSessionTarget | undefined) => { + if (!internalModelRunTargets || !target?.sessionKey || !target.storePath) { return; } - internalModelRunArtifacts.set(`${artifactSessionId}\n${sessionFile}`, { - sessionFile, - sessionId: artifactSessionId, - }); + internalModelRunTargets.set(`${target.storePath}\n${target.sessionKey}`, target); }; - // Precompute the owned path so even a partially failed prepare can be cleaned. - trackInternalModelRunArtifact(resolveInternalSessionEffectsTranscriptPath(runId), sessionId); + if (internalModelRunTargets && storePath) { + trackInternalModelRunTarget( + resolveInternalSessionEffectsTarget({ agentId: sessionAgentId, runId, storePath }), + ); + } // Reset marks its mutation before interrupting work. An aborted run must not // queue behind that mutation or reset would wait on the run holding the queue. @@ -1134,7 +1149,7 @@ async function agentCommandInternal( allowCreateRestartRecoveryEntry, ), }); - sessionEntry = persisted ?? sessionEntry; + sessionEntry = persisted; trackedRestartRecoveryDeliveryContext = Boolean(persisted?.restartRecoveryDeliveryContext) && persisted?.restartRecoveryDeliveryRunId === runId; @@ -1268,25 +1283,26 @@ async function agentCommandInternal( const finalTextRaw = visibleTextAccumulator.finalizeRaw(); const finalText = visibleTextAccumulator.finalize(); try { - const [{ resolveAcpSessionCwd }, { resolveSessionTranscriptFile }] = await Promise.all([ - loadAcpSessionIdentifiersRuntime(), - loadTranscriptResolveRuntime(), - ]); + const { resolveAcpSessionCwd } = await loadAcpSessionIdentifiersRuntime(); const internalSource = suppressVisibleSessionEffects - ? await resolveSessionTranscriptFile({ + ? resolveInternalSessionEffectsSource({ + agentId: sessionAgentId, sessionId, sessionKey, - sessionEntry, + storePath, + }) + : undefined; + const internalTarget = suppressVisibleSessionEffects + ? await prepareInternalSessionEffectsSession({ agentId: sessionAgentId, - threadId: opts.threadId, - }) - : undefined; - const internalSessionFile = suppressVisibleSessionEffects - ? await prepareInternalSessionEffectsTranscript({ - sessionFile: internalSource?.sessionFile, + cwd: resolveAcpSessionCwd(acpResolution.meta) ?? workspaceDir, runId, + source: internalSource, + storePath, }) : undefined; + trackInternalModelRunTarget(internalTarget); + const transcriptSessionEntry = internalTarget?.sessionEntry ?? sessionEntry; const transcriptResult = await attemptExecutionRuntime.persistAcpTurnTranscript({ body, transcriptBody, @@ -1300,20 +1316,18 @@ async function agentCommandInternal( } : {}), finalText: finalTextRaw, - sessionId, - sessionKey, - sessionFile: internalSessionFile, - sessionEntry, + sessionId: internalTarget?.sessionId ?? sessionId, + sessionKey: internalTarget?.sessionKey ?? sessionKey, + sessionEntry: transcriptSessionEntry, sessionStore: suppressVisibleSessionEffects ? undefined : sessionStore, - storePath: suppressVisibleSessionEffects ? undefined : storePath, - sessionAgentId, + storePath: internalTarget?.storePath ?? storePath, + sessionAgentId: internalTarget?.agentId ?? sessionAgentId, threadId: opts.threadId, sessionCwd: resolveAcpSessionCwd(acpResolution.meta) ?? workspaceDir, config: cfg, }); - sessionEntry = transcriptResult.sessionEntry; - if (internalSessionFile) { - sessionEntry = prepared.sessionEntry; + if (!internalTarget) { + sessionEntry = transcriptResult.sessionEntry; } } catch (error) { log.warn( @@ -1444,7 +1458,7 @@ async function agentCommandInternal( initialEntry: current, entry: next, }); - sessionEntry = persisted ?? sessionEntry; + sessionEntry = persisted; } // Persist non-model-dependent command state before provider/model resolution. @@ -1476,7 +1490,7 @@ async function agentCommandInternal( initialEntry: entry, entry: next, }); - sessionEntry = persisted ?? sessionEntry; + sessionEntry = persisted; } if (sessionKey && !isSubagentLaneTurn) { recordSessionHumanDirectMessage({ @@ -1623,12 +1637,12 @@ async function agentCommandInternal( initialEntry, entry, }); - sessionEntry = persisted ?? sessionEntry; + sessionEntry = persisted; const adoptedHasStoredOverride = Boolean( - sessionEntry.modelOverride || sessionEntry.providerOverride, + sessionEntry?.modelOverride || sessionEntry?.providerOverride, ); storedModelOverrideSource = adoptedHasStoredOverride - ? sessionEntry.modelOverrideSource + ? sessionEntry?.modelOverrideSource : undefined; hasStoredAutoFallbackProvenance = adoptedHasStoredOverride && hasSessionAutoModelFallbackProvenance(sessionEntry); @@ -1953,10 +1967,23 @@ async function agentCommandInternal( sessionFile = resolvedSessionFile.sessionFile; sessionEntry = resolvedSessionFile.sessionEntry; } - const attemptSessionFile = suppressVisibleSessionEffects - ? await prepareInternalSessionEffectsTranscript({ sessionFile, runId }) - : sessionFile; - trackInternalModelRunArtifact(attemptSessionFile, sessionId); + const sessionEffectsSource = resolveInternalSessionEffectsSource({ + agentId: sessionAgentId, + sessionId, + sessionKey, + storePath, + }); + const internalSessionTarget = suppressVisibleSessionEffects + ? await prepareInternalSessionEffectsSession({ + agentId: sessionAgentId, + cwd: cwd ?? workspaceDir, + runId, + source: sessionEffectsSource, + storePath, + }) + : undefined; + trackInternalModelRunTarget(internalSessionTarget); + const attemptSessionFile = internalSessionTarget?.sessionFile ?? sessionFile; const startedAt = Date.now(); const attemptLifecycleState: AgentAttemptLifecycleState = { @@ -1986,10 +2013,12 @@ async function agentCommandInternal( } : {}), target: { - transcriptPath: attemptSessionFile, - sessionId, - agentId: sessionAgentId, - ...(sessionKey ? { sessionKey } : {}), + sessionId: internalSessionTarget?.sessionId ?? sessionId, + agentId: internalSessionTarget?.agentId ?? sessionAgentId, + sessionKey: internalSessionTarget?.sessionKey ?? sessionKey ?? sessionId, + sessionEntry: internalSessionTarget?.sessionEntry ?? sessionEntry, + sessionStore: suppressVisibleSessionEffects ? undefined : sessionStore, + storePath: internalSessionTarget?.storePath ?? storePath, cwd: cwd ?? workspaceDir, config: cfg, }, @@ -2306,6 +2335,7 @@ async function agentCommandInternal( agentHarnessRuntimeOverride, sessionId, sessionKey, + ...(internalSessionTarget ? { sessionTarget: internalSessionTarget } : {}), sessionAgentId, sessionFile: attemptSessionFile, workspaceDir, @@ -2394,7 +2424,7 @@ async function agentCommandInternal( entryMatchesAutoFallbackPrimaryProbe(current, autoFallbackPrimaryProbe), ), }); - sessionEntry = persistedEntry ?? sessionEntry; + sessionEntry = persistedEntry; } if (fallbackResult.attempts.length > 0 && result.meta.agentMeta) { result = { @@ -2558,11 +2588,11 @@ async function agentCommandInternal( const rotatedSessionFile = result.meta.agentMeta?.sessionFile; const effectiveSessionId = rotatedSessionFile - ? (result.meta.agentMeta?.sessionId ?? sessionId) - : sessionId; - const effectiveSessionFile = rotatedSessionFile ?? attemptSessionFile; - trackInternalModelRunArtifact(effectiveSessionFile, effectiveSessionId); - + ? (result.meta.agentMeta?.sessionId ?? internalSessionTarget?.sessionId ?? sessionId) + : (internalSessionTarget?.sessionId ?? sessionId); + if (internalSessionTarget && effectiveSessionId !== internalSessionTarget.sessionId) { + trackInternalModelRunTarget({ ...internalSessionTarget, sessionId: effectiveSessionId }); + } // Update token+model fields in the session store. if (sessionStore && sessionKey && !suppressVisibleSessionEffects) { const isHeartbeatLifecycleRun = isHeartbeatLifecycleRunKind(opts.bootstrapContextRunKind); @@ -2604,17 +2634,17 @@ async function agentCommandInternal( ) { let persistedCliTurnTranscript = false; try { + const transcriptSessionEntry = internalSessionTarget?.sessionEntry ?? sessionEntry; const transcriptResult = await attemptExecutionRuntime.persistCliTurnTranscript({ body, transcriptBody, result, sessionId: effectiveSessionId, - sessionKey: sessionKey ?? effectiveSessionId, - sessionFile: suppressVisibleSessionEffects ? effectiveSessionFile : undefined, - sessionEntry, + sessionKey: internalSessionTarget?.sessionKey ?? sessionKey ?? effectiveSessionId, + sessionEntry: transcriptSessionEntry, sessionStore: suppressVisibleSessionEffects ? undefined : sessionStore, - storePath: suppressVisibleSessionEffects ? undefined : storePath, - sessionAgentId, + storePath: internalSessionTarget?.storePath ?? storePath, + sessionAgentId: internalSessionTarget?.agentId ?? sessionAgentId, threadId: opts.threadId, sessionCwd: effectiveCwd, config: cfg, @@ -2624,10 +2654,9 @@ async function agentCommandInternal( userTurnTranscriptRecorder.hasPersisted() || userTurnTranscriptRecorder.isBlocked(), }); - sessionEntry = transcriptResult.sessionEntry; sessionReboundDuringRun = transcriptResult.kind === "session-rebound"; - if (suppressVisibleSessionEffects) { - sessionEntry = prepared.sessionEntry; + if (!internalSessionTarget) { + sessionEntry = transcriptResult.sessionEntry; } persistedCliTurnTranscript = transcriptResult.kind === "persisted"; } catch (error) { @@ -2686,26 +2715,26 @@ async function agentCommandInternal( if (combinedPayload) { const entry = sessionStore[sessionKey] ?? sessionEntry; - if (!entry) { - throw new Error("Cannot persist pending delivery without a session entry"); + if (entry) { + const next: SessionEntry = { + ...entry, + pendingFinalDelivery: true, + pendingFinalDeliveryText: combinedPayload, + pendingFinalDeliveryContext: currentRunDeliveryContext, + pendingFinalDeliveryCreatedAt: now, + updatedAt: now, + }; + const persisted = await persistSessionEntry({ + sessionStore, + sessionKey, + storePath, + initialEntry: entry, + entry: next, + shouldPersist: (current) => + shouldPersistCurrentRunSessionCleanup(current, sessionId), + }); + sessionEntry = persisted; } - const next: SessionEntry = { - ...entry, - pendingFinalDelivery: true, - pendingFinalDeliveryText: combinedPayload, - pendingFinalDeliveryContext: currentRunDeliveryContext, - pendingFinalDeliveryCreatedAt: now, - updatedAt: now, - }; - const persisted = await persistSessionEntry({ - sessionStore, - sessionKey, - storePath, - initialEntry: entry, - entry: next, - shouldPersist: (current) => shouldPersistCurrentRunSessionCleanup(current, sessionId), - }); - sessionEntry = persisted ?? sessionEntry; } } @@ -2713,12 +2742,13 @@ async function agentCommandInternal( const resolveFreshSessionEntryForDelivery = sessionStore && sessionKey && !suppressVisibleSessionEffects ? async (): Promise => { - const { loadSessionStore } = await loadSessionStoreRuntime(); - const freshStore = loadSessionStore(storePath, { - skipCache: true, + const { loadSessionEntry } = await loadSessionStoreRuntime(); + const freshEntry = loadSessionEntry({ + storePath, + sessionKey, + readConsistency: "latest", clone: false, }); - const freshEntry = freshStore[sessionKey]; if (!freshEntry || freshEntry.sessionId !== effectiveSessionId) { return undefined; } @@ -2763,9 +2793,9 @@ async function agentCommandInternal( const noPendingTextForThisRun = opts.deliver === true && pendingFinalDeliveryTextForThisRun === undefined && - entry.pendingFinalDelivery === true && + entry?.pendingFinalDelivery === true && !entry.pendingFinalDeliveryText; - if (deliveryResult?.deliverySucceeded === true || noPendingTextForThisRun) { + if (entry && (deliveryResult?.deliverySucceeded === true || noPendingTextForThisRun)) { const next = clearPendingFinalDeliveryFields(entry, Date.now()); const persisted = await persistSessionEntry({ sessionStore, @@ -2775,7 +2805,7 @@ async function agentCommandInternal( entry: next, shouldPersist: (current) => shouldPersistCurrentRunSessionCleanup(current, sessionId), }); - sessionEntry = persisted ?? sessionEntry; + sessionEntry = persisted; } } @@ -2792,29 +2822,22 @@ async function agentCommandInternal( }); } finally { sessionWorkAdmission.release(); - if (internalModelRunArtifacts) { - // Compaction may rotate a private transcript. Clean every owned identity - // only after delivery, then remove each transcript after its sidecars. - for (const artifact of internalModelRunArtifacts.values()) { + if (internalModelRunTargets) { + // Compaction may rotate a private session identity. Remove every owned + // SQLite row only after delivery; transcript and trajectory rows cascade. + for (const target of internalModelRunTargets.values()) { try { - await removeSessionTrajectoryArtifacts({ - sessionId: artifact.sessionId, - sessionFile: artifact.sessionFile, - storePath: artifact.sessionFile, - }); + await removeInternalSessionEffectsSession(target); } catch (error) { + // Cleanup remains best-effort so a terminal SQLite write failure does + // not replace the completed model-run result; the DB layer warns too. log.warn( - `failed to remove model-run trajectory artifacts: ${ + `failed to remove model-run SQLite session: ${ error instanceof Error ? error.message : String(error) }`, ); } } - for (const sessionFile of new Set( - Array.from(internalModelRunArtifacts.values(), (artifact) => artifact.sessionFile), - )) { - await removeInternalSessionEffectsTranscript(sessionFile); - } } if ( !sessionReboundDuringRun && @@ -2840,7 +2863,7 @@ async function agentCommandInternal( shouldPersist: (current) => shouldPersistRestartRecoveryCleanup(current, sessionId, runId), }); - sessionEntry = persisted ?? sessionEntry; + sessionEntry = persisted; } } catch (error) { log.warn( diff --git a/src/agents/agent-tools.create-openclaw-coding-tools.test.ts b/src/agents/agent-tools.create-openclaw-coding-tools.test.ts index 6c5f1d2ca04a..3524aa7cbc36 100644 --- a/src/agents/agent-tools.create-openclaw-coding-tools.test.ts +++ b/src/agents/agent-tools.create-openclaw-coding-tools.test.ts @@ -7,6 +7,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { findUnsupportedSchemaKeywords, @@ -68,11 +70,10 @@ async function writeSessionStore( agentId: string, entries: Record, ) { - await fs.writeFile( - storeTemplate.replaceAll("{agentId}", agentId), - JSON.stringify(entries, null, 2), - "utf-8", - ); + const storePath = storeTemplate.replaceAll("{agentId}", agentId); + for (const [sessionKey, entry] of Object.entries(entries)) { + await replaceSessionEntry({ agentId, sessionKey, storePath }, entry as SessionEntry); + } } function createToolsForStoredSession(storeTemplate: string, sessionKey: string) { diff --git a/src/agents/agent-tools.policy.test.ts b/src/agents/agent-tools.policy.test.ts index 243b7115b46a..82613a011dce 100644 --- a/src/agents/agent-tools.policy.test.ts +++ b/src/agents/agent-tools.policy.test.ts @@ -3,11 +3,12 @@ * Covers wildcard matching, sub-agent inheritance, provider overrides, and * trusted group context checks. */ -import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import { createWarnLogCapture } from "../logging/test-helpers/warn-log-capture.js"; import { filterToolsByPolicy, @@ -29,6 +30,26 @@ vi.mock("../channels/plugins/session-conversation.js", () => ({ }), })); +async function writeSessionEntries( + storePath: string, + entries: Record, +): Promise { + for (const [sessionKey, entry] of Object.entries(entries)) { + await replaceSessionEntry({ sessionKey, storePath }, entry as SessionEntry); + } +} + +function createSessionStorePath(prefix: string, agentId = "main"): string { + return path.join( + os.tmpdir(), + `${prefix}-${Date.now()}-${Math.random().toString(16).slice(2)}`, + "agents", + agentId, + "sessions", + "sessions.json", + ); +} + describe("agent-tools.policy", () => { it("treats * in allow as allow-all", () => { const tools = [createStubTool("read"), createStubTool("exec")]; @@ -192,29 +213,17 @@ describe("resolveSubagentToolPolicyForSession", () => { agents: { defaults: { subagents: { maxSpawnDepth: 2 } } }, } as unknown as OpenClawConfig; - it("uses stored leaf role for flat depth-1 session keys", () => { - const storePath = path.join( - os.tmpdir(), - `openclaw-subagent-policy-${Date.now()}-${Math.random().toString(16).slice(2)}.json`, - ); - fs.mkdirSync(path.dirname(storePath), { recursive: true }); - fs.writeFileSync( - storePath, - JSON.stringify( - { - "agent:main:subagent:flat-leaf": { - sessionId: "flat-leaf", - updatedAt: Date.now(), - spawnDepth: 1, - subagentRole: "leaf", - subagentControlScope: "none", - }, - }, - null, - 2, - ), - "utf-8", - ); + it("uses stored leaf role for flat depth-1 session keys", async () => { + const storePath = createSessionStorePath("openclaw-subagent-policy"); + await writeSessionEntries(storePath, { + "agent:main:subagent:flat-leaf": { + sessionId: "flat-leaf", + updatedAt: Date.now(), + spawnDepth: 1, + subagentRole: "leaf", + subagentControlScope: "none", + }, + }); const cfg = { ...baseCfg, session: { @@ -229,30 +238,18 @@ describe("resolveSubagentToolPolicyForSession", () => { expect(isToolAllowedByPolicyName("memory_get", policy)).toBe(true); }); - it("resolves inherited tool denies from stored subagent sessions", () => { - const storePath = path.join( - os.tmpdir(), - `openclaw-subagent-inherited-deny-${Date.now()}-${Math.random().toString(16).slice(2)}.json`, - ); - fs.mkdirSync(path.dirname(storePath), { recursive: true }); - fs.writeFileSync( - storePath, - JSON.stringify( - { - "agent:main:subagent:limited": { - sessionId: "limited-session", - updatedAt: Date.now(), - spawnDepth: 1, - subagentRole: "orchestrator", - subagentControlScope: "children", - inheritedToolDeny: ["bash", "memory_get"], - }, - }, - null, - 2, - ), - "utf-8", - ); + it("resolves inherited tool denies from stored subagent sessions", async () => { + const storePath = createSessionStorePath("openclaw-subagent-inherited-deny"); + await writeSessionEntries(storePath, { + "agent:main:subagent:limited": { + sessionId: "limited-session", + updatedAt: Date.now(), + spawnDepth: 1, + subagentRole: "orchestrator", + subagentControlScope: "children", + inheritedToolDeny: ["bash", "memory_get"], + }, + }); const cfg = { ...baseCfg, session: { @@ -266,30 +263,18 @@ describe("resolveSubagentToolPolicyForSession", () => { expect(isToolAllowedByPolicyName("sessions_spawn", policy)).toBe(true); }); - it("resolves inherited tool allows from stored subagent sessions", () => { - const storePath = path.join( - os.tmpdir(), - `openclaw-subagent-inherited-allow-${Date.now()}-${Math.random().toString(16).slice(2)}.json`, - ); - fs.mkdirSync(path.dirname(storePath), { recursive: true }); - fs.writeFileSync( - storePath, - JSON.stringify( - { - "agent:main:subagent:limited": { - sessionId: "limited-session", - updatedAt: Date.now(), - spawnDepth: 1, - subagentRole: "orchestrator", - subagentControlScope: "children", - inheritedToolAllow: ["sessions_spawn", "memory_search"], - }, - }, - null, - 2, - ), - "utf-8", - ); + it("resolves inherited tool allows from stored subagent sessions", async () => { + const storePath = createSessionStorePath("openclaw-subagent-inherited-allow"); + await writeSessionEntries(storePath, { + "agent:main:subagent:limited": { + sessionId: "limited-session", + updatedAt: Date.now(), + spawnDepth: 1, + subagentRole: "orchestrator", + subagentControlScope: "children", + inheritedToolAllow: ["sessions_spawn", "memory_search"], + }, + }); const cfg = { ...baseCfg, session: { @@ -304,32 +289,18 @@ describe("resolveSubagentToolPolicyForSession", () => { expect(isToolAllowedByPolicyName("exec", policy)).toBe(false); }); - it("keeps configured plugin allows separate from inherited tool allows", () => { - const storePath = path.join( - os.tmpdir(), - `openclaw-subagent-inherited-allow-separate-${Date.now()}-${Math.random() - .toString(16) - .slice(2)}.json`, - ); - fs.mkdirSync(path.dirname(storePath), { recursive: true }); - fs.writeFileSync( - storePath, - JSON.stringify( - { - "agent:main:subagent:limited": { - sessionId: "limited-session", - updatedAt: Date.now(), - spawnDepth: 1, - subagentRole: "orchestrator", - subagentControlScope: "children", - inheritedToolAllow: ["plugin_tool"], - }, - }, - null, - 2, - ), - "utf-8", - ); + it("keeps configured plugin allows separate from inherited tool allows", async () => { + const storePath = createSessionStorePath("openclaw-subagent-inherited-allow-separate"); + await writeSessionEntries(storePath, { + "agent:main:subagent:limited": { + sessionId: "limited-session", + updatedAt: Date.now(), + spawnDepth: 1, + subagentRole: "orchestrator", + subagentControlScope: "children", + inheritedToolAllow: ["plugin_tool"], + }, + }); const cfg = { ...baseCfg, tools: { @@ -353,28 +324,16 @@ describe("resolveSubagentToolPolicyForSession", () => { expect(inheritedPolicy?.allow).toEqual(["plugin_tool"]); }); - it("applies inherited tool policy from stored ACP sessions without subagent metadata", () => { - const storePath = path.join( - os.tmpdir(), - `openclaw-acp-inherited-deny-${Date.now()}-${Math.random().toString(16).slice(2)}.json`, - ); - fs.mkdirSync(path.dirname(storePath), { recursive: true }); - fs.writeFileSync( - storePath, - JSON.stringify( - { - "agent:main:acp:limited": { - sessionId: "limited-acp-session", - updatedAt: Date.now(), - inheritedToolAllow: ["custom_plugin_tool"], - inheritedToolDeny: ["custom_denied_tool"], - }, - }, - null, - 2, - ), - "utf-8", - ); + it("applies inherited tool policy from stored ACP sessions without subagent metadata", async () => { + const storePath = createSessionStorePath("openclaw-acp-inherited-deny"); + await writeSessionEntries(storePath, { + "agent:main:acp:limited": { + sessionId: "limited-acp-session", + updatedAt: Date.now(), + inheritedToolAllow: ["custom_plugin_tool"], + inheritedToolDeny: ["custom_denied_tool"], + }, + }); const cfg = { ...baseCfg, session: { diff --git a/src/agents/auth-profiles/store.ts b/src/agents/auth-profiles/store.ts index 7b6f0a134f63..a2b4c97b64d2 100644 --- a/src/agents/auth-profiles/store.ts +++ b/src/agents/auth-profiles/store.ts @@ -758,7 +758,12 @@ export async function updateAuthProfileStoreWithLock(params: { } return store; }); - } catch { + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + log.warn(`auth profile store update failed: ${message}`, { + agentDir: params.agentDir, + error: message, + }); return null; } } diff --git a/src/agents/auth-profiles/upsert-with-lock.ts b/src/agents/auth-profiles/upsert-with-lock.ts index 188008cad326..77554b16377f 100644 --- a/src/agents/auth-profiles/upsert-with-lock.ts +++ b/src/agents/auth-profiles/upsert-with-lock.ts @@ -7,26 +7,22 @@ import { normalizeAuthProfileCredential } from "./credential-normalize.js"; import { updateAuthProfileStoreWithLock } from "./store.js"; import type { AuthProfileCredential, AuthProfileStore } from "./types.js"; -/** Upserts an auth profile under the store lock, returning null on write failure. */ +/** Upserts an auth profile under the store lock, returning null on store write failure. */ export async function upsertAuthProfileWithLock(params: { profileId: string; credential: AuthProfileCredential; agentDir?: string; }): Promise { - try { - const credential = normalizeAuthProfileCredential(params.credential); - return await updateAuthProfileStoreWithLock({ - agentDir: params.agentDir, - saveOptions: { - filterExternalAuthProfiles: false, - syncExternalCli: false, - }, - updater: (store) => { - store.profiles[params.profileId] = credential; - return true; - }, - }); - } catch { - return null; - } + const credential = normalizeAuthProfileCredential(params.credential); + return await updateAuthProfileStoreWithLock({ + agentDir: params.agentDir, + saveOptions: { + filterExternalAuthProfiles: false, + syncExternalCli: false, + }, + updater: (store) => { + store.profiles[params.profileId] = credential; + return true; + }, + }); } diff --git a/src/agents/bash-tools.exec-approval-followup.test.ts b/src/agents/bash-tools.exec-approval-followup.test.ts index 2e731fb2c46c..628daaf90e18 100644 --- a/src/agents/bash-tools.exec-approval-followup.test.ts +++ b/src/agents/bash-tools.exec-approval-followup.test.ts @@ -16,8 +16,7 @@ vi.mock("../infra/outbound/message.js", () => ({ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { clearSessionStoreCacheForTest } from "../config/sessions/store.js"; -import { writeSessionStoreForTest } from "../config/sessions/test-helpers.js"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; import { onDiagnosticEvent, resetDiagnosticEventsForTest, @@ -33,20 +32,34 @@ import { callGatewayTool } from "./tools/gateway.js"; const tempStoreDirs: string[] = []; -// Seed the same JSON session store path the runtime reads; mocking this -// boundary would hide stale-session regressions in shared workers. -function writeTempSessionStore(entries: Record): string { +// Seed the same session store path the runtime reads; mocking this boundary +// would hide stale-session regressions in shared workers. +async function writeTempSessionStore( + entries: Record, +): Promise { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "exec-approval-followup-store-")); tempStoreDirs.push(dir); const storePath = path.join(dir, "sessions.json"); - writeSessionStoreForTest(storePath, entries); + await Promise.all( + Object.entries(entries).map(([sessionKey, entry]) => + replaceSessionEntry( + { + storePath, + sessionKey, + }, + { + sessionId: entry.sessionId, + updatedAt: Date.now(), + }, + ), + ), + ); return storePath; } afterEach(() => { vi.resetAllMocks(); resetDiagnosticEventsForTest(); - clearSessionStoreCacheForTest(); while (tempStoreDirs.length > 0) { const dir = tempStoreDirs.pop(); if (dir) { @@ -175,7 +188,7 @@ describe("exec approval followup", () => { }); it("drops a denied direct followup when the session key was rebound by /new or /reset", async () => { - const sessionStore = writeTempSessionStore({ + const sessionStore = await writeTempSessionStore({ "agent:main:main": { sessionId: "session-after-reset" }, }); const diagnostics: DiagnosticEventPayload[] = []; @@ -209,7 +222,7 @@ describe("exec approval followup", () => { }); it("delivers a denied direct followup when the key still resolves to the approval-time session", async () => { - const sessionStore = writeTempSessionStore({ + const sessionStore = await writeTempSessionStore({ "agent:main:main": { sessionId: "session-original" }, }); @@ -229,7 +242,7 @@ describe("exec approval followup", () => { }); it("drops a non-denied direct fallback when the session key was rebound", async () => { - const sessionStore = writeTempSessionStore({ + const sessionStore = await writeTempSessionStore({ "agent:main:main": { sessionId: "session-after-reset" }, }); const diagnostics: DiagnosticEventPayload[] = []; diff --git a/src/agents/bash-tools.exec-approval-followup.ts b/src/agents/bash-tools.exec-approval-followup.ts index f856e9615425..251d1b98d020 100644 --- a/src/agents/bash-tools.exec-approval-followup.ts +++ b/src/agents/bash-tools.exec-approval-followup.ts @@ -8,7 +8,7 @@ import { normalizeOptionalString, } from "@openclaw/normalization-core/string-coerce"; import { resolveStorePath } from "../config/sessions/paths.js"; -import { loadSessionStore } from "../config/sessions/store-load.js"; +import { loadSessionEntry } from "../config/sessions/session-accessor.js"; import { emitDiagnosticEvent } from "../infra/diagnostic-events.js"; import { resolveExternalBestEffortDeliveryTarget, @@ -110,10 +110,10 @@ function shouldSuppressExecDeniedFollowup(sessionKey: string | undefined): boole /** * Direct/denied followups bypass the gateway agent dispatch, so the gateway - * rebind guard never sees them. Resolve the session key's current sessionId - * from the on-disk store and report whether it was rebound away from the - * approval-time session by `/new` or `/reset` (#59349). Failure to resolve is - * treated as "not rebound" so a real result is never suppressed by accident. + * rebind guard never sees them. Resolve the session key's current sessionId and + * report whether it was rebound away from the approval-time session by `/new` + * or `/reset` (#59349). Failure to resolve is treated as "not rebound" so a + * real result is never suppressed by accident. */ function isExecApprovalFollowupDirectDeliveryStale(params: { sessionKey: string | undefined; @@ -130,7 +130,11 @@ function isExecApprovalFollowupDirectDeliveryStale(params: { agentId: resolveAgentIdFromSessionKey(sessionKey), }); const resolvedSessionId = normalizeOptionalString( - loadSessionStore(storePath)?.[sessionKey]?.sessionId, + loadSessionEntry({ + storePath, + sessionKey, + clone: false, + })?.sessionId, ); return isExecApprovalFollowupSessionRebound({ expectedSessionId, resolvedSessionId }); } catch (err) { diff --git a/src/agents/bootstrap-files.test.ts b/src/agents/bootstrap-files.test.ts index bcf66f9173b3..52deae812dfa 100644 --- a/src/agents/bootstrap-files.test.ts +++ b/src/agents/bootstrap-files.test.ts @@ -493,6 +493,12 @@ describe("hasCompletedBootstrapTurn", () => { expect(await hasCompletedBootstrapTurn(path.join(tmpDir, "missing.jsonl"))).toBe(false); }); + it("returns false for SQLite transcript markers", async () => { + expect( + await hasCompletedBootstrapTurn("sqlite:main:session-1:/tmp/openclaw/sessions.json"), + ).toBe(false); + }); + it("returns false for empty session files", async () => { const sessionFile = path.join(tmpDir, "empty.jsonl"); await fs.writeFile(sessionFile, "", "utf8"); diff --git a/src/agents/bootstrap-files.ts b/src/agents/bootstrap-files.ts index 5dd2b88bac34..3c23ec5f7a1a 100644 --- a/src/agents/bootstrap-files.ts +++ b/src/agents/bootstrap-files.ts @@ -5,6 +5,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import type { AgentContextInjection } from "../config/types.agent-defaults.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveUserPath } from "../utils.js"; @@ -75,6 +76,9 @@ export function resolveContextInjectionMode( /** Checks whether the session transcript still has a valid full-bootstrap marker. */ export async function hasCompletedBootstrapTurn(sessionFile: string): Promise { + if (parseSqliteSessionFileMarker(sessionFile)) { + return false; + } try { const stat = await fs.lstat(sessionFile); if (stat.isSymbolicLink()) { diff --git a/src/agents/btw-transcript.ts b/src/agents/btw-transcript.ts index 10e36c3b4814..90234bced838 100644 --- a/src/agents/btw-transcript.ts +++ b/src/agents/btw-transcript.ts @@ -7,6 +7,8 @@ import { resolveSessionFilePathOptions, type SessionEntry as StoredSessionEntry, } from "../config/sessions.js"; +import { loadTranscriptEvents } from "../config/sessions/session-accessor.js"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { scanSessionTranscriptTree, type SessionTranscriptTree, @@ -101,10 +103,19 @@ function isTrailingUserMessage(entry: AgentSessionEntry | undefined): boolean { export async function readBtwTranscriptMessages(params: { sessionFile: string; sessionId: string; + sessionKey?: string; snapshotLeafId?: string | null; }): Promise { try { - const entries = parseSessionEntries(await readFile(params.sessionFile, "utf-8")); + const marker = parseSqliteSessionFileMarker(params.sessionFile); + const entries = marker + ? ((await loadTranscriptEvents({ + agentId: marker.agentId, + sessionId: marker.sessionId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + storePath: marker.storePath, + })) as AgentSessionEntry[]) + : parseSessionEntries(await readFile(params.sessionFile, "utf-8")); migrateSessionEntries(entries); const sessionEntries = entries.filter( (entry): entry is AgentSessionEntry => entry.type !== "session", diff --git a/src/agents/btw.test.ts b/src/agents/btw.test.ts index 77997ade88c4..b4b7e8fb205a 100644 --- a/src/agents/btw.test.ts +++ b/src/agents/btw.test.ts @@ -31,6 +31,7 @@ const prepareCliRunContextMock = vi.fn(); const executePreparedCliRunMock = vi.fn(); const diagDebugMock = vi.fn(); const ensureSelectedAgentHarnessPluginMock = vi.fn(); +const loadTranscriptEventsMock = vi.fn(); vi.mock("../llm/stream.js", async () => { const original = await vi.importActual("../llm/stream.js"); @@ -176,6 +177,16 @@ vi.mock("../logging/diagnostic.js", () => ({ }, })); +vi.mock("../config/sessions/session-accessor.js", async () => { + const actual = await vi.importActual( + "../config/sessions/session-accessor.js", + ); + return { + ...actual, + loadTranscriptEvents: (...args: unknown[]) => loadTranscriptEventsMock(...args), + }; +}); + const { runBtwSideQuestion } = await import("./btw.js"); const { clearAgentHarnesses, registerAgentHarness } = await import("./harness/registry.js"); type RunBtwSideQuestionParams = Parameters[0]; @@ -464,9 +475,11 @@ describe("runBtwSideQuestion", () => { executePreparedCliRunMock.mockReset(); diagDebugMock.mockReset(); ensureSelectedAgentHarnessPluginMock.mockReset(); + loadTranscriptEventsMock.mockReset(); clearAgentHarnesses(); readFileMock.mockResolvedValue("mock transcript"); + loadTranscriptEventsMock.mockResolvedValue([]); parseSessionEntriesMock.mockReturnValue([ createTranscriptEntry({ id: "user-1", @@ -1669,6 +1682,39 @@ describe("runBtwSideQuestion", () => { expect(result).toEqual({ text: MATH_ANSWER }); }); + it("reads SQLite marker transcripts through the accessor when no active snapshot exists", async () => { + const userEntry = createTranscriptEntry({ + id: "user-seed", + message: createUserTranscriptMessage(), + }); + const assistantEntry = createTranscriptEntry({ + id: "assistant-seed", + parentId: "user-seed", + message: createAssistantTranscriptMessage([{ type: "text", text: "seed answer" }]), + }); + loadTranscriptEventsMock.mockResolvedValue([userEntry, assistantEntry]); + readFileMock.mockRejectedValue(new Error("sqlite marker must not be read as a file")); + mockDoneAnswer(MATH_ANSWER); + + const result = await runMathSideQuestion({ + sessionKey: DEFAULT_SESSION_KEY, + sessionEntry: createSessionEntry({ + sessionFile: `sqlite:main:session-1:${DEFAULT_STORE_PATH}`, + }), + }); + + expect(result).toEqual({ text: MATH_ANSWER }); + expect(readFileMock).not.toHaveBeenCalled(); + expect(loadTranscriptEventsMock).toHaveBeenCalledWith({ + agentId: "main", + sessionId: "session-1", + sessionKey: DEFAULT_SESSION_KEY, + storePath: DEFAULT_STORE_PATH, + }); + expect(buildSessionContextMock).toHaveBeenCalledTimes(1); + expect(buildSessionContextMock).toHaveBeenCalledWith([userEntry, assistantEntry]); + }); + it("falls back when the active run snapshot leaf no longer exists", async () => { const userEntry = createTranscriptEntry({ id: "user-seed", diff --git a/src/agents/btw.ts b/src/agents/btw.ts index b33c3317502f..4ae34a44f44a 100644 --- a/src/agents/btw.ts +++ b/src/agents/btw.ts @@ -619,6 +619,7 @@ export async function runBtwSideQuestion( messages: await readBtwTranscriptMessages({ sessionFile, sessionId, + sessionKey: params.sessionKey, snapshotLeafId: activeRunSnapshot?.transcriptLeafId, }), imageLimits, diff --git a/src/agents/cli-runner.reliability.test.ts b/src/agents/cli-runner.reliability.test.ts index 2879687d748f..b8105e89986a 100644 --- a/src/agents/cli-runner.reliability.test.ts +++ b/src/agents/cli-runner.reliability.test.ts @@ -12,6 +12,7 @@ import { replyRunRegistry, } from "../auto-reply/reply/reply-run-registry.js"; import { SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js"; +import { loadTranscriptEvents, upsertSessionEntry } from "../config/sessions/session-accessor.js"; import { CURRENT_SESSION_VERSION } from "../config/sessions/version.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { @@ -32,6 +33,7 @@ import { createUserTurnTranscriptRecorder, type UserTurnTranscriptRecorder, } from "../sessions/user-turn-transcript.js"; +import { createTestUserTurnTranscriptTarget } from "../sessions/user-turn-transcript.test-support.js"; import { runSkillResearchAutoCapture } from "../skills/research/autocapture.js"; import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; import { @@ -147,23 +149,6 @@ function createSessionFile(params?: { history?: Array<{ role: "user"; content: s return { dir, sessionFile, storePath }; } -function createCliUserTurnRecorder(params: { - text: string; - sessionFile: string; - sessionKey?: string; - workspaceDir: string; -}) { - return createUserTurnTranscriptRecorder({ - input: { text: params.text }, - target: { - transcriptPath: params.sessionFile, - sessionId: "s1", - ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), - cwd: params.workspaceDir, - }, - }); -} - function buildPreparedContext(params?: { sessionKey?: string; cliSessionId?: string; @@ -295,14 +280,52 @@ function expectTextMessage(value: unknown, fields: { role: string; content: stri expect(message.timestamp).toBeTypeOf("number"); } -function readTranscriptMessages(sessionFile: string): unknown[] { - return fs - .readFileSync(sessionFile, "utf-8") - .trim() - .split("\n") - .map((line) => JSON.parse(line) as { message?: unknown }) - .map((entry) => entry.message) - .filter(Boolean); +async function readTranscriptMessages(sessionFile: string): Promise { + const sessionId = path.basename(sessionFile, ".jsonl"); + const events = await loadTranscriptEvents({ + agentId: "main", + sessionId, + sessionKey: "agent:main:main", + storePath: path.join(path.dirname(sessionFile), "sessions.json"), + }); + return events.flatMap((entry) => + typeof entry === "object" && entry !== null && "message" in entry ? [entry.message] : [], + ); +} + +async function seedSqliteSessionEntry(params: { + sessionFile: string; + storePath: string; +}): Promise { + await upsertSessionEntry( + { + agentId: "main", + sessionKey: "agent:main:main", + storePath: params.storePath, + }, + { + sessionId: "s1", + sessionFile: params.sessionFile, + updatedAt: Date.now(), + }, + ); +} + +function createCliUserTurnRecorder(params: { + text: string; + sessionFile: string; + sessionKey?: string; + workspaceDir: string; +}) { + return createUserTurnTranscriptRecorder({ + input: { text: params.text }, + target: createTestUserTurnTranscriptTarget({ + sessionId: "s1", + sessionKey: params.sessionKey ?? "agent:main:main", + cwd: params.workspaceDir, + storePath: path.join(path.dirname(params.sessionFile), "sessions.json"), + }), + }); } const CLI_RESEED_PROMPT = @@ -1108,7 +1131,7 @@ describe("runCliAgent reliability", () => { try { await runPreparedCliAgent(context); - const transcriptMessages = readTranscriptMessages(sessionFile); + const transcriptMessages = await readTranscriptMessages(sessionFile); expect(transcriptMessages).toHaveLength(0); const llmOutputEvent = requireRecord( callArg(hookRunner.runLlmOutput, 0, 0, "llm_output event"), @@ -2694,10 +2717,31 @@ describe("runCliAgent reliability", () => { const onUserMessagePersisted = vi.fn(); try { + await seedSqliteSessionEntry({ sessionFile, storePath }); const context = buildPreparedContext({ sessionKey: "agent:main:main", runId: "run-persist-cli", }); + const recorder = createUserTurnTranscriptRecorder({ + input: { + text: "display prompt", + timestamp: 123, + idempotencyKey: "run-persist-cli:user", + }, + target: { + sessionId: "s1", + sessionKey: "agent:main:main", + sessionEntry: { + sessionId: "s1", + sessionFile, + updatedAt: 10, + }, + storePath, + agentId: "main", + cwd: dir, + }, + updateMode: "none", + }); const result = await runPreparedCliAgent({ ...context, params: { @@ -2708,12 +2752,7 @@ describe("runCliAgent reliability", () => { prompt: "runtime prompt", persistAssistantTranscript: true, storePath, - userTurnTranscriptRecorder: createCliUserTurnRecorder({ - text: "display prompt", - sessionFile, - sessionKey: "agent:main:main", - workspaceDir: dir, - }), + userTurnTranscriptRecorder: recorder, onUserMessagePersisted, }, }); @@ -2730,11 +2769,13 @@ describe("runCliAgent reliability", () => { }), ); - const messages = readTranscriptMessages(sessionFile); + const messages = await readTranscriptMessages(sessionFile); expect(messages).toContainEqual( expect.objectContaining({ role: "user", content: "display prompt", + timestamp: 123, + idempotencyKey: "run-persist-cli:user", }), ); expect(messages).toContainEqual( @@ -2747,7 +2788,136 @@ describe("runCliAgent reliability", () => { idempotencyKey: "cli-assistant:run-persist-cli", }), ); + expect( + messages.filter((message) => (message as { role?: string }).role === "user"), + ).toHaveLength(1); expect(JSON.stringify(messages)).not.toContain("runtime prompt"); + const events = await loadTranscriptEvents({ + agentId: "main", + sessionId: "s1", + sessionKey: "agent:main:main", + storePath, + }); + expect(events).toContainEqual(expect.objectContaining({ type: "session", cwd: dir })); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("honors CLI retry suppression before approved user-turn persistence", async () => { + supervisorSpawnMock.mockResolvedValueOnce( + createManagedRun({ + reason: "exit", + exitCode: 0, + exitSignal: null, + durationMs: 50, + stdout: "hello from retry", + stderr: "", + timedOut: false, + noOutputTimedOut: false, + }), + ); + const { dir, sessionFile, storePath } = createSessionFile(); + const recorder = createUserTurnTranscriptRecorder({ + input: { + text: "suppressed display prompt", + idempotencyKey: "run-suppressed-cli:user", + }, + target: { + sessionId: "s1", + sessionKey: "agent:main:main", + sessionEntry: { + sessionId: "s1", + sessionFile, + updatedAt: 10, + }, + storePath, + agentId: "main", + }, + updateMode: "none", + }); + const persistApprovedSpy = vi.spyOn(recorder, "persistApproved"); + const onUserMessagePersisted = vi.fn(); + + try { + await seedSqliteSessionEntry({ sessionFile, storePath }); + const context = buildPreparedContext({ + sessionKey: "agent:main:main", + runId: "run-suppressed-cli", + }); + const result = await runPreparedCliAgent({ + ...context, + params: { + ...context.params, + agentId: "main", + sessionFile, + workspaceDir: dir, + prompt: "runtime prompt", + storePath, + userTurnTranscriptRecorder: recorder, + suppressNextUserMessagePersistence: true, + onUserMessagePersisted, + }, + }); + + expect(result.payloads).toEqual([{ text: "hello from retry" }]); + expect(persistApprovedSpy).not.toHaveBeenCalled(); + expect(onUserMessagePersisted).not.toHaveBeenCalled(); + await expect(readTranscriptMessages(sessionFile)).resolves.toEqual([]); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("honors a CLI user-turn recorder target that skips after session rebound", async () => { + supervisorSpawnMock.mockResolvedValueOnce( + createManagedRun({ + reason: "exit", + exitCode: 0, + exitSignal: null, + durationMs: 50, + stdout: "hello after rebound", + stderr: "", + timedOut: false, + noOutputTimedOut: false, + }), + ); + const { dir, sessionFile, storePath } = createSessionFile(); + const recorder = createUserTurnTranscriptRecorder({ + input: { + text: "stale rebound prompt", + idempotencyKey: "run-rebound-recorder:user", + }, + target: () => undefined, + updateMode: "none", + }); + const persistApprovedSpy = vi.spyOn(recorder, "persistApproved"); + const onUserMessagePersisted = vi.fn(); + + try { + await seedSqliteSessionEntry({ sessionFile, storePath }); + const context = buildPreparedContext({ + sessionKey: "agent:main:main", + runId: "run-rebound-recorder", + }); + const result = await runPreparedCliAgent({ + ...context, + params: { + ...context.params, + agentId: "main", + sessionFile, + workspaceDir: dir, + prompt: "runtime prompt", + storePath, + userTurnTranscriptRecorder: recorder, + onUserMessagePersisted, + }, + }); + + expect(result.payloads).toEqual([{ text: "hello after rebound" }]); + expect(persistApprovedSpy).toHaveBeenCalledOnce(); + expect(onUserMessagePersisted).not.toHaveBeenCalled(); + await expect(readTranscriptMessages(sessionFile)).resolves.toEqual([]); } finally { fs.rmSync(dir, { recursive: true, force: true }); } @@ -2856,7 +3026,7 @@ describe("runCliAgent reliability", () => { const result = await runPreparedCliAgent(context); expect(result.meta.agentMeta?.cliSessionBinding?.reseedReceipt).toBeUndefined(); - expect(readTranscriptMessages(sessionFile)).not.toContainEqual( + await expect(readTranscriptMessages(sessionFile)).resolves.not.toContainEqual( expect.objectContaining({ role: "user" }), ); } finally { @@ -2880,12 +3050,13 @@ describe("runCliAgent reliability", () => { ); const { dir, sessionFile } = createSessionFile(); const recorder = createUserTurnTranscriptRecorder({ - target: { - transcriptPath: sessionFile, + target: createTestUserTurnTranscriptTarget({ sessionId: "s1", + sessionKey: "agent:main:main", agentId: "main", cwd: dir, - }, + storePath: path.join(path.dirname(sessionFile), "sessions.json"), + }), }); recorder.markBlocked(); @@ -2916,7 +3087,7 @@ describe("runCliAgent reliability", () => { localSessionId: "s1", userTurnDisposition: "omitted", }); - expect(readTranscriptMessages(sessionFile)).toEqual([]); + await expect(readTranscriptMessages(sessionFile)).resolves.toEqual([]); } finally { restoreCliRunnerTestDeps(); fs.rmSync(dir, { recursive: true, force: true }); @@ -3099,6 +3270,7 @@ describe("runCliAgent reliability", () => { const { dir, sessionFile, storePath } = createSessionFile(); try { + await seedSqliteSessionEntry({ sessionFile, storePath }); const context = buildPreparedContext({ sessionKey: "agent:main:main", runId: "run-blocked-cli", @@ -3121,7 +3293,7 @@ describe("runCliAgent reliability", () => { expect(getReplyPayloadMetadata(result.payloads?.[0] ?? {})).toMatchObject({ assistantTranscriptOwned: true, }); - expect(readTranscriptMessages(sessionFile)).toEqual([]); + await expect(readTranscriptMessages(sessionFile)).resolves.toEqual([]); expect(hookRunner.runBeforeMessageWrite).toHaveBeenCalledOnce(); expect( callArg(hookRunner.runBeforeMessageWrite, 0, 1, "before_message_write context"), @@ -3193,8 +3365,8 @@ describe("runCliAgent reliability", () => { expect(getReplyPayloadMetadata(result.payloads?.[0] ?? {})).toMatchObject({ assistantTranscriptOwned: true, }); - expect(readTranscriptMessages(sessionFile)).toEqual([]); - expect(readTranscriptMessages(replacementFile)).toEqual([]); + await expect(readTranscriptMessages(sessionFile)).resolves.toEqual([]); + await expect(readTranscriptMessages(replacementFile)).resolves.toEqual([]); } finally { fs.rmSync(dir, { recursive: true, force: true }); } @@ -3237,7 +3409,7 @@ describe("runCliAgent reliability", () => { expect(getReplyPayloadMetadata(result.payloads?.[0] ?? {})).toMatchObject({ assistantTranscriptOwned: true, }); - expect(readTranscriptMessages(sessionFile)).toEqual([]); + await expect(readTranscriptMessages(sessionFile)).resolves.toEqual([]); } finally { fs.rmSync(dir, { recursive: true, force: true }); } @@ -3258,7 +3430,7 @@ describe("runCliAgent reliability", () => { ); const { dir, sessionFile } = createSessionFile(); const taskDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-persist-cwd-")); - let capturedTarget: unknown; + let capturedCwd: unknown; const recorder = { message: undefined, resolveMessage: vi.fn(async () => undefined), @@ -3269,9 +3441,8 @@ describe("runCliAgent reliability", () => { isBlocked: vi.fn(() => false), hasRuntimePersistencePending: vi.fn(() => false), waitForRuntimePersistence: vi.fn(async () => undefined), - persistApproved: vi.fn(async (options?: { target?: unknown }) => { - capturedTarget = - typeof options?.target === "function" ? await options.target() : options?.target; + persistApproved: vi.fn(async (options?: { cwd?: unknown }) => { + capturedCwd = options?.cwd; return { sessionFile, sessionEntry: undefined, @@ -3305,14 +3476,7 @@ describe("runCliAgent reliability", () => { expect(result.payloads).toEqual([{ text: "hello from cli" }]); expect(recorder.persistApproved).toHaveBeenCalledOnce(); - expect(capturedTarget).toEqual( - expect.objectContaining({ - transcriptPath: sessionFile, - sessionId: context.params.sessionId, - sessionKey: "agent:main:main", - cwd: taskDir, - }), - ); + expect(capturedCwd).toBe(taskDir); } finally { fs.rmSync(taskDir, { recursive: true, force: true }); fs.rmSync(dir, { recursive: true, force: true }); @@ -3340,12 +3504,12 @@ describe("runCliAgent reliability", () => { timestamp: 123, idempotencyKey: "cli-recorder:user", }, - target: { - transcriptPath: sessionFile, - sessionId: "session-1", + target: createTestUserTurnTranscriptTarget({ + sessionId: "s1", sessionKey: "agent:main:main", cwd: dir, - }, + storePath: path.join(path.dirname(sessionFile), "sessions.json"), + }), updateMode: "none", }); @@ -3369,7 +3533,7 @@ describe("runCliAgent reliability", () => { expect(result.payloads).toEqual([{ text: "hello from cli" }]); expect(recorder.hasPersisted()).toBe(true); - const messages = readTranscriptMessages(sessionFile); + const messages = await readTranscriptMessages(sessionFile); expect(messages).toEqual([ expect.objectContaining({ role: "user", @@ -3407,12 +3571,12 @@ describe("runCliAgent reliability", () => { const { dir, sessionFile } = createSessionFile(); const recorder = createUserTurnTranscriptRecorder({ input: { text: "blocked user turn" }, - target: { - transcriptPath: sessionFile, - sessionId: "session-1", + target: createTestUserTurnTranscriptTarget({ + sessionId: "s1", sessionKey: "agent:main:main", cwd: dir, - }, + storePath: path.join(path.dirname(sessionFile), "sessions.json"), + }), beforeMessageWrite: runAgentHarnessBeforeMessageWriteHook, }); @@ -3436,7 +3600,7 @@ describe("runCliAgent reliability", () => { expect(result.payloads).toEqual([{ text: "hello from cli" }]); expect(recorder.hasPersisted()).toBe(false); expect(recorder.isBlocked()).toBe(true); - expect(readTranscriptMessages(sessionFile)).toEqual([]); + await expect(readTranscriptMessages(sessionFile)).resolves.toEqual([]); expect(hookRunner.runBeforeMessageWrite).toHaveBeenCalledOnce(); } finally { fs.rmSync(dir, { recursive: true, force: true }); @@ -3494,9 +3658,24 @@ describe("runCliAgent reliability", () => { it("does not execute the CLI when approved user turn persistence fails", async () => { supervisorSpawnMock.mockClear(); const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-persist-fail-")); - const blockedParent = path.join(dir, "not-a-directory"); - fs.writeFileSync(blockedParent, "occupied", "utf-8"); const onUserMessagePersisted = vi.fn(); + // SQLite-backed persistence no longer fails via blocked transcript + // directories; a rejecting recorder models the same persistence failure. + const recorder = { + message: undefined, + resolveMessage: vi.fn(async () => undefined), + markRuntimePersistencePending: vi.fn(), + markRuntimePersisted: vi.fn(), + markBlocked: vi.fn(), + hasPersisted: vi.fn(() => false), + isBlocked: vi.fn(() => false), + hasRuntimePersistencePending: vi.fn(() => false), + waitForRuntimePersistence: vi.fn(async () => undefined), + persistApproved: vi.fn(async () => { + throw new Error("user turn persistence failed"); + }), + persistFallback: vi.fn(async () => undefined), + } as unknown as UserTurnTranscriptRecorder; try { const context = buildPreparedContext({ @@ -3510,15 +3689,10 @@ describe("runCliAgent reliability", () => { params: { ...context.params, agentId: "main", - sessionFile: path.join(blockedParent, "s1.jsonl"), + sessionFile: path.join(dir, "s1.jsonl"), workspaceDir: dir, prompt: "runtime prompt", - userTurnTranscriptRecorder: createCliUserTurnRecorder({ - text: "display prompt", - sessionFile: path.join(blockedParent, "s1.jsonl"), - sessionKey: "agent:main:main", - workspaceDir: dir, - }), + userTurnTranscriptRecorder: recorder, onUserMessagePersisted, }, }), @@ -3533,7 +3707,6 @@ describe("runCliAgent reliability", () => { it("blocks CLI runs before llm_input and model execution when before_agent_run blocks", async () => { supervisorSpawnMock.mockClear(); - const onUserMessagePersisted = vi.fn(); let releaseAgentEnd: () => void = () => undefined; const agentEndSettled = new Promise((resolve) => { releaseAgentEnd = resolve; @@ -3557,12 +3730,6 @@ describe("runCliAgent reliability", () => { const { dir, sessionFile } = createSessionFile({ history: [{ role: "user", content: "earlier context" }], }); - const userTurnTranscriptRecorder = createCliUserTurnRecorder({ - text: "secret prompt", - sessionFile, - sessionKey: "agent:main:main", - workspaceDir: dir, - }); try { let resolved = false; @@ -3579,8 +3746,6 @@ describe("runCliAgent reliability", () => { sessionFile, workspaceDir: dir, prompt: "secret prompt", - userTurnTranscriptRecorder, - onUserMessagePersisted, }, }).then((result) => { resolved = true; @@ -3606,8 +3771,6 @@ describe("runCliAgent reliability", () => { expect(result.meta.agentMeta?.clearCliSessionBinding).toBe(true); expect(supervisorSpawnMock).not.toHaveBeenCalled(); expect(hookRunner.runLlmInput).not.toHaveBeenCalled(); - expect(onUserMessagePersisted).not.toHaveBeenCalled(); - expect(userTurnTranscriptRecorder.isBlocked()).toBe(true); const beforeRunEvent = requireRecord( callArg(hookRunner.runBeforeAgentRun, 0, 0, "before_agent_run event"), "before_agent_run event", @@ -3667,6 +3830,106 @@ describe("runCliAgent reliability", () => { } }); + it("persists before_agent_run CLI blocks through the canonical recorder", async () => { + supervisorSpawnMock.mockClear(); + const hookRunner = { + hasHooks: vi.fn((hookName: string) => hookName === "before_agent_run"), + runBeforeAgentRun: vi.fn(async () => ({ + pluginId: "policy-plugin", + decision: { + outcome: "block" as const, + reason: "matched secret prompt: secret prompt", + message: "The agent cannot read this message.", + }, + })), + }; + setHookRunnerForTest(hookRunner); + const { dir, sessionFile, storePath } = createSessionFile(); + const onUserMessagePersisted = vi.fn(); + + try { + await seedSqliteSessionEntry({ sessionFile, storePath }); + const recorder = createUserTurnTranscriptRecorder({ + input: { + text: "secret prompt", + idempotencyKey: "run-blocked-cli-sqlite:user", + }, + target: { + sessionId: "s1", + sessionKey: "agent:main:main", + sessionEntry: { + sessionId: "s1", + sessionFile, + updatedAt: 10, + }, + storePath, + agentId: "main", + cwd: dir, + }, + updateMode: "none", + }); + const persistBlockedSpy = vi.spyOn(recorder, "persistBlocked"); + const context = buildPreparedContext({ + sessionKey: "agent:main:main", + runId: "run-blocked-cli-sqlite", + }); + + const result = await runPreparedCliAgent({ + ...context, + params: { + ...context.params, + agentId: "main", + sessionFile, + workspaceDir: dir, + prompt: "secret prompt", + storePath, + userTurnTranscriptRecorder: recorder, + onUserMessagePersisted, + }, + }); + + expect(result.meta.livenessState).toBe("blocked"); + expect(supervisorSpawnMock).not.toHaveBeenCalled(); + expect(persistBlockedSpy).toHaveBeenCalledOnce(); + expect(onUserMessagePersisted).toHaveBeenCalledWith( + expect.objectContaining({ + role: "user", + content: [ + { + type: "text", + text: "Your message could not be sent: The agent cannot read this message. (blocked by policy-plugin)", + }, + ], + }), + ); + const events = await loadTranscriptEvents({ + agentId: "main", + sessionId: "s1", + sessionKey: "agent:main:main", + storePath, + }); + const messages = events.flatMap((entry) => + typeof entry === "object" && entry !== null && "message" in entry ? [entry.message] : [], + ); + expect(messages).toContainEqual( + expect.objectContaining({ + role: "user", + content: [ + { + type: "text", + text: "Your message could not be sent: The agent cannot read this message. (blocked by policy-plugin)", + }, + ], + idempotencyKey: "hook-block:before_agent_run:user:run-blocked-cli-sqlite", + }), + ); + expect(JSON.stringify(messages)).not.toContain("secret prompt"); + expect(JSON.stringify(messages)).not.toContain("matched secret prompt"); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it("forwards channel identity context to CLI before_agent_run hooks", async () => { supervisorSpawnMock.mockClear(); const hookRunner = { @@ -3701,12 +3964,6 @@ describe("runCliAgent reliability", () => { currentChannelId: "telegram:chat-1", senderId: "user-42", senderIsOwner: true, - userTurnTranscriptRecorder: createCliUserTurnRecorder({ - text: "sender scoped prompt", - sessionFile, - sessionKey: "agent:main:telegram:chat-1", - workspaceDir: dir, - }), }, }); diff --git a/src/agents/cli-runner.ts b/src/agents/cli-runner.ts index 26f3bd2d74e6..a28a17263036 100644 --- a/src/agents/cli-runner.ts +++ b/src/agents/cli-runner.ts @@ -174,8 +174,12 @@ export async function isCliBindingFlushed( return false; } -function flushSessionManagerFile(sessionManager: SessionManager): void { - (sessionManager as unknown as { rewriteFile?: () => void }).rewriteFile?.(); +function flushSessionManagerTranscript(sessionManager: SessionManager): void { + ( + sessionManager as unknown as { + replacePersistedTranscript?: () => void; + } + ).replacePersistedTranscript?.(); } async function assertSuccessfulCliRuntimeBindingCurrent( @@ -309,15 +313,9 @@ async function persistApprovedCliUserTurnTranscript(params: RunCliAgentParams): return recorder?.isBlocked() === true; } - const target = { - transcriptPath: params.sessionFile, - sessionId: params.sessionId, - agentId: params.agentId, - ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + const persisted = await recorder.persistApproved({ cwd: params.cwd ?? params.workspaceDir, - ...(params.config ? { config: params.config } : {}), - }; - const persisted = await recorder.persistApproved({ target }); + }); if (!persisted && !recorder.hasPersisted() && (await recorder.resolveMessage())) { // A prepared user row can be rejected by before_message_write. Preserve // that terminal decision so outer transcript mirrors do not retry it. @@ -394,6 +392,18 @@ async function persistCliAssistantTranscript(params: { } } +async function notifyCliUserMessagePersisted( + params: RunCliAgentParams, + message: Extract, + context: string, +): Promise { + try { + await Promise.resolve(params.onUserMessagePersisted?.(message)); + } catch (err) { + log.warn(`${context} notification failed: ${formatErrorMessage(err)}`); + } +} + async function finalizeCliContextEngineTurn(params: { context: PreparedCliRunContext; historyMessages: unknown[]; @@ -827,23 +837,44 @@ export async function runPreparedCliAgent( message: string; pluginId: string; }): Promise => { - params.userTurnTranscriptRecorder?.markBlocked(); - try { - const nowMs = Date.now(); - const sessionManager = SessionManager.open(params.sessionFile); - sessionManager.appendMessage({ - role: "user", - content: [{ type: "text", text: block.message }], - timestamp: nowMs, - idempotencyKey: `hook-block:before_agent_run:user:${params.runId}`, - __openclaw: { - beforeAgentRunBlocked: { - blockedBy: block.pluginId, - blockedAt: nowMs, - }, + const nowMs = Date.now(); + const redactedUserMessage = { + role: "user" as const, + content: [{ type: "text" as const, text: block.message }], + timestamp: nowMs, + idempotencyKey: `hook-block:before_agent_run:user:${params.runId}`, + __openclaw: { + beforeAgentRunBlocked: { + blockedBy: block.pluginId, + blockedAt: nowMs, }, - } as Parameters[0]); - flushSessionManagerFile(sessionManager); + }, + }; + try { + const persisted = + await params.userTurnTranscriptRecorder?.persistBlocked(redactedUserMessage); + if (persisted) { + await notifyCliUserMessagePersisted( + params, + persisted.message, + "before_agent_run block user-turn persistence", + ); + return; + } + } catch (err) { + log.warn( + `before_agent_run block: failed to persist canonical CLI user message: ${formatErrorMessage( + err, + )}`, + ); + } + + try { + const sessionManager = SessionManager.open(params.sessionFile); + sessionManager.appendMessage( + redactedUserMessage as Parameters[0], + ); + flushSessionManagerTranscript(sessionManager); } catch (err) { log.warn( `before_agent_run block: failed to persist redacted CLI user message: ${formatErrorMessage( diff --git a/src/agents/cli-runner/session-history.ts b/src/agents/cli-runner/session-history.ts index e20355be1663..834295318bab 100644 --- a/src/agents/cli-runner/session-history.ts +++ b/src/agents/cli-runner/session-history.ts @@ -10,6 +10,7 @@ import { resolveSessionFilePathOptions, } from "../../config/sessions/paths.js"; import { + parseSessionTranscriptTreeEntry, scanSessionTranscriptTree, selectSessionTranscriptLeafControlledPath, } from "../../config/sessions/transcript-tree.js"; @@ -346,26 +347,34 @@ function isSafeTruncatedCliSessionTail(entries: readonly unknown[]): boolean { if (tree.hasLeafControl) { return !tree.hasInvalidLeafControl; } + const rawIds = new Set(); const childParentIds = new Set(); let truncatedRootParentId: string | undefined; - for (const node of tree.nodes) { + for (const entry of entries) { + const node = parseSessionTranscriptTreeEntry(entry); + if (!node) { + continue; + } if (node.appendMode === "side") { return false; } if (node.parentId === null) { + rawIds.add(node.id); continue; } - if (!tree.byId.has(node.parentId)) { + if (!rawIds.has(node.parentId)) { if (truncatedRootParentId !== undefined || childParentIds.size > 0) { return false; } truncatedRootParentId = node.parentId; + rawIds.add(node.id); continue; } if (childParentIds.has(node.parentId)) { return false; } childParentIds.add(node.parentId); + rawIds.add(node.id); } return true; } @@ -445,6 +454,13 @@ async function loadCliSessionEntries(params: { if (!entries) { return []; } + const rawSessionEntries = entries.filter((entry) => entry.type !== "session"); + if (transcript.truncated && !isSafeTruncatedCliSessionTail(rawSessionEntries)) { + cliBackendLog.warn( + `cli session history truncated tail skipped because branch controls are incomplete: ${realSessionFile}`, + ); + return []; + } migrateSessionEntries(entries); const sessionEntries = entries.filter((entry) => entry.type !== "session"); if (transcript.truncated && !isSafeTruncatedCliSessionTail(sessionEntries)) { diff --git a/src/agents/cli-runner/types.ts b/src/agents/cli-runner/types.ts index 0af6f0987f79..0996e31a4efd 100644 --- a/src/agents/cli-runner/types.ts +++ b/src/agents/cli-runner/types.ts @@ -1,3 +1,4 @@ +import type { AgentMessage } from "../../../packages/agent-core/src/types.js"; /** * Shared types for preparing and executing CLI-backed agent runs. */ @@ -19,10 +20,7 @@ import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js"; import type { CliBackendExecutionMode } from "../../plugins/cli-backend.types.js"; import type { PluginHookChannelContext } from "../../plugins/hook-types.js"; import type { InputProvenance } from "../../sessions/input-provenance.js"; -import type { - PersistedUserTurnMessage, - UserTurnTranscriptRecorder, -} from "../../sessions/user-turn-transcript.types.js"; +import type { UserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; import type { SkillSnapshot } from "../../skills/types.js"; import type { ExecElevatedDefaults } from "../bash-tools.exec-types.js"; import type { BootstrapContextMode } from "../bootstrap-files.js"; @@ -67,13 +65,16 @@ export type RunCliAgentParams = { * background answers and must not reuse or mutate normal agent sessions. */ executionMode?: CliBackendExecutionMode; - suppressNextUserMessagePersistence?: boolean; - userTurnTranscriptRecorder?: UserTurnTranscriptRecorder; - onUserMessagePersisted?: (message: PersistedUserTurnMessage) => void | Promise; /** Persist the successful CLI assistant reply into the OpenClaw session transcript. */ persistAssistantTranscript?: boolean; /** Session store path used when assistant transcript persistence is enabled. */ storePath?: string; + /** Canonical user-turn recorder shared with gateway/queue dispatch. */ + userTurnTranscriptRecorder?: UserTurnTranscriptRecorder; + /** Skip current-turn user persistence when a retry/fallback already wrote it. */ + suppressNextUserMessagePersistence?: boolean; + /** Notification fired after the current user turn has been accepted into the transcript. */ + onUserMessagePersisted?: (message: Extract) => void; currentInboundEventKind?: InboundEventKind; currentInboundContext?: CurrentInboundPromptContext; inputProvenance?: InputProvenance; diff --git a/src/agents/codex-native-web-search.test.ts b/src/agents/codex-native-web-search.test.ts index e85f677bd872..bd72a28d645d 100644 --- a/src/agents/codex-native-web-search.test.ts +++ b/src/agents/codex-native-web-search.test.ts @@ -2,7 +2,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { writeSessionStoreForTest } from "../config/sessions/test-helpers.js"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; import { buildCodexNativeWebSearchTool, describeCodexNativeWebSearch, @@ -217,12 +217,23 @@ describe("resolveCodexNativeSearchActivation", () => { expect(result.inactiveReason).toBe("tool_policy_denied"); }); - it("keeps native search inactive when inherited session policy denies web_search", () => { + it("keeps native search inactive when inherited session policy denies web_search", async () => { const agentId = `native-inherited-deny-${Date.now()}-${Math.random().toString(16).slice(2)}`; const sessionKey = `agent:${agentId}:subagent:limited`; - const storePath = path.join(os.tmpdir(), `openclaw-native-inherited-deny-${agentId}.json`); - writeSessionStoreForTest(storePath, { - [sessionKey]: { + const storePath = path.join( + os.tmpdir(), + `openclaw-native-inherited-deny-${agentId}`, + "agents", + agentId, + "sessions", + "sessions.json", + ); + await replaceSessionEntry( + { + sessionKey, + storePath, + }, + { sessionId: "limited-session", updatedAt: Date.now(), spawnDepth: 1, @@ -230,7 +241,7 @@ describe("resolveCodexNativeSearchActivation", () => { subagentControlScope: "children", inheritedToolDeny: ["web_search"], }, - }); + ); const result = resolveCodexNativeSearchActivation({ config: { diff --git a/src/agents/command/attempt-execution.cli.test.ts b/src/agents/command/attempt-execution.cli.test.ts index 52674d397be5..e6e14dee59c6 100644 --- a/src/agents/command/attempt-execution.cli.test.ts +++ b/src/agents/command/attempt-execution.cli.test.ts @@ -4,11 +4,22 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { SessionEntry } from "../../config/sessions.js"; -import { resolveSessionTranscriptPath } from "../../config/sessions/paths.js"; +import { + appendTranscriptEvent, + appendTranscriptMessage, + listSessionEntries, + loadTranscriptEvents, + replaceSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import { + formatSqliteSessionFileMarker, + parseSqliteSessionFileMarker, +} from "../../config/sessions/sqlite-marker.js"; import { clearSessionStoreCacheForTest } from "../../config/sessions/store.js"; -import { appendSessionTranscriptMessage } from "../../config/sessions/transcript-append.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; +import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; import { registerGeneratedMediaTaskActivity, resetGeneratedMediaTaskActivityForTests, @@ -157,7 +168,7 @@ async function persistCliTranscriptEntry( } async function readSessionMessages(sessionFile: string) { - return (await readSessionFileJsonLines<{ type?: string; message?: unknown }>(sessionFile)) + return (await readTranscriptEntries(sessionFile)) .filter((entry) => entry.type === "message") .map( (entry) => @@ -166,7 +177,7 @@ async function readSessionMessages(sessionFile: string) { } async function readSessionFileEntries(sessionFile: string) { - return await readSessionFileJsonLines<{ + return await readTranscriptEntries<{ type?: string; id?: string; parentId?: string | null; @@ -175,7 +186,17 @@ async function readSessionFileEntries(sessionFile: string) { }>(sessionFile); } -async function readSessionFileJsonLines(sessionFile: string): Promise { +async function readTranscriptEntries( + sessionFile: string, +): Promise { + const marker = parseSqliteSessionFileMarker(sessionFile); + if (marker) { + return (await loadTranscriptEvents({ + agentId: marker.agentId, + sessionId: marker.sessionId, + storePath: marker.storePath, + })) as T[]; + } // Session transcripts are JSONL; tests preserve that format so parent/child // id ordering and append behavior are covered end-to-end. const raw = await fs.readFile(sessionFile, "utf-8"); @@ -252,7 +273,7 @@ describe("CLI attempt execution", () => { ...overrides?.sessionEntry, }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runEmbeddedAgentMock.mockResolvedValueOnce({ meta: { durationMs: 1 }, } satisfies EmbeddedAgentRunResult); @@ -311,10 +332,24 @@ describe("CLI attempt execution", () => { providerAuthAliasMocks.resolveProviderIdForAuth.mockClear(); }); + async function writeSessionStoreSeed(sessionStore: Record): Promise { + for (const [sessionKey, entry] of Object.entries(sessionStore)) { + await replaceSessionEntry({ sessionKey, storePath }, entry); + } + closeOpenClawAgentDatabasesForTest(); + } + + function readSessionStore(): Record { + return Object.fromEntries( + listSessionEntries({ storePath }).map(({ entry, sessionKey }) => [sessionKey, entry]), + ); + } + afterEach(async () => { vi.useRealTimers(); homeEnvSnapshot?.restore(); homeEnvSnapshot = undefined; + closeOpenClawAgentDatabasesForTest(); await fs.rm(tmpDir, { recursive: true, force: true }); }); @@ -420,7 +455,7 @@ describe("CLI attempt execution", () => { claudeCliSessionId: "stale-legacy-session", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); // The retry hook must clear poisoned bindings before the fresh CLI attempt // runs, otherwise the runner would resume the same expired Claude session. @@ -478,10 +513,7 @@ describe("CLI attempt execution", () => { expect(sessionStore[sessionKey]?.cliSessionIds?.["claude-cli"]).toBeUndefined(); expect(sessionStore[sessionKey]?.claudeCliSessionId).toBeUndefined(); - const persisted = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; + const persisted = readSessionStore(); expect(persisted[sessionKey]?.cliSessionIds?.["claude-cli"]).toBeUndefined(); expect(persisted[sessionKey]?.claudeCliSessionId).toBeUndefined(); }); @@ -492,7 +524,7 @@ describe("CLI attempt execution", () => { await writeClaudeCliAssistantTranscript(cliSessionId); const sessionEntry = makeClaudeCliSessionEntry("session-cli-abort", cliSessionId); const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); const abortError = Object.assign(new Error("aborted"), { name: "AbortError" }); runCliAgentMock.mockRejectedValueOnce(abortError); @@ -512,10 +544,7 @@ describe("CLI attempt execution", () => { expect(sessionStore[sessionKey]?.cliSessionIds?.["claude-cli"]).toBeUndefined(); expect(sessionStore[sessionKey]?.claudeCliSessionId).toBeUndefined(); - const persisted = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; + const persisted = readSessionStore(); expect(persisted[sessionKey]?.cliSessionBindings?.["claude-cli"]).toBeUndefined(); expect(persisted[sessionKey]?.cliSessionIds?.["claude-cli"]).toBeUndefined(); expect(persisted[sessionKey]?.claudeCliSessionId).toBeUndefined(); @@ -562,7 +591,7 @@ describe("CLI attempt execution", () => { await writeClaudeCliAssistantTranscript(cliSessionId); const sessionEntry = makeClaudeCliSessionEntry("session-cli-timeout", cliSessionId); const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockImplementationOnce(async (args: unknown) => { const retry = requireRecord(args, "run CLI agent argument").onBeforeFreshCliSessionRetry; expect(retry).toBeTypeOf("function"); @@ -648,7 +677,7 @@ describe("CLI attempt execution", () => { await writeClaudeCliAssistantTranscript(cliSessionId); const sessionEntry = makeClaudeCliSessionEntry(`session-cli-${reason}`, cliSessionId); const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockRejectedValueOnce( new FailoverError(`${reason} failed`, { reason, @@ -692,7 +721,7 @@ describe("CLI attempt execution", () => { claudeCliSessionId: "phantom-claude-session", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockResolvedValueOnce(makeCliResult("fresh cli response")); await runClaudeCliAttempt({ @@ -716,10 +745,7 @@ describe("CLI attempt execution", () => { expect(sessionStore[sessionKey]?.cliSessionIds?.["claude-cli"]).toBeUndefined(); expect(sessionStore[sessionKey]?.claudeCliSessionId).toBeUndefined(); - const persisted = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; + const persisted = readSessionStore(); expect(persisted[sessionKey]?.cliSessionBindings?.["claude-cli"]).toBeUndefined(); expect(persisted[sessionKey]?.cliSessionIds?.["claude-cli"]).toBeUndefined(); expect(persisted[sessionKey]?.claudeCliSessionId).toBeUndefined(); @@ -813,7 +839,7 @@ describe("CLI attempt execution", () => { claudeCliSessionId: cliSessionId, }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockResolvedValueOnce(makeCliResult("resumed cli response")); await runClaudeCliAttempt({ @@ -858,7 +884,7 @@ describe("CLI attempt execution", () => { ); const sessionEntry = makeClaudeCliSessionEntry("openclaw-session-cwd", cliSessionId); const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockResolvedValueOnce(makeCliResult("resumed cli response")); await runClaudeCliAttempt({ @@ -885,7 +911,7 @@ describe("CLI attempt execution", () => { authProfileOverrideSource: "user", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockResolvedValueOnce(makeCliResult("codex cli response")); await runAgentAttempt({ @@ -936,7 +962,7 @@ describe("CLI attempt execution", () => { }, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); await fs.writeFile(path.join(tmpDir, "auth-profiles.json"), "{", "utf-8"); runCliAgentMock.mockResolvedValueOnce(makeCliResult("codex cli response")); @@ -981,7 +1007,6 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); saveAuthProfileStore( { version: 1, @@ -1060,7 +1085,6 @@ describe("CLI attempt execution", () => { authProfileOverrideSource: "user", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); saveAuthProfileStore( { version: 1, @@ -1131,7 +1155,6 @@ describe("CLI attempt execution", () => { authProfileOverrideSource: "user", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); saveAuthProfileStore( { version: 1, @@ -1200,7 +1223,6 @@ describe("CLI attempt execution", () => { authProfileOverrideSource: "auto", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); saveAuthProfileStore( { version: 1, @@ -1281,7 +1303,6 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); saveAuthProfileStore( { version: 1, @@ -1351,18 +1372,22 @@ describe("CLI attempt execution", () => { it.each(["CLI", "ACP"] as const)( "keeps an explicit internal %s transcript out of the visible session path", async (runtime) => { - const sessionId = `session-explicit-internal-${runtime.toLowerCase()}`; - const sessionKey = `agent:main:direct:${sessionId}`; + const visibleSessionId = `session-explicit-internal-${runtime.toLowerCase()}`; + const sessionId = `internal-${visibleSessionId}`; + const sessionKey = `agent:main:internal-session-effects:${visibleSessionId}`; setTestEnvValue("HOME", tmpDir); setTestEnvValue("OPENCLAW_STATE_DIR", path.join(tmpDir, "state")); - const visibleSessionFile = resolveSessionTranscriptPath(sessionId, "main"); - const internalSessionFile = path.join(tmpDir, "internal-agent-runs", `${sessionId}.jsonl`); + const internalStorePath = path.join(tmpDir, "sessions.json"); + const internalSessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath: internalStorePath, + }); const sessionEntry: SessionEntry = { sessionId, - sessionFile: visibleSessionFile, + sessionFile: internalSessionFile, updatedAt: Date.now(), }; - await fs.mkdir(path.dirname(internalSessionFile), { recursive: true }); if (runtime === "CLI") { await persistCliTurnTranscript({ @@ -1374,6 +1399,7 @@ describe("CLI attempt execution", () => { sessionEntry, sessionAgentId: "main", sessionCwd: tmpDir, + storePath, config: {}, embeddedAssistantGapFill: true, }); @@ -1387,6 +1413,7 @@ describe("CLI attempt execution", () => { sessionEntry, sessionAgentId: "main", sessionCwd: tmpDir, + storePath, config: {}, }); } @@ -1397,7 +1424,13 @@ describe("CLI attempt execution", () => { content: [{ type: "text", text: "internal reply" }], }), ); - await expect(fs.stat(visibleSessionFile)).rejects.toMatchObject({ code: "ENOENT" }); + expect( + await loadTranscriptEvents({ + agentId: "main", + sessionId: visibleSessionId, + storePath, + }), + ).toEqual([]); }, ); @@ -1412,22 +1445,14 @@ describe("CLI attempt execution", () => { startedAt: 2, }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile( - storePath, - JSON.stringify( - { - [sessionKey]: { - ...sessionEntry, - updatedAt: 5, - status: "done", - endedAt: 4, - }, - }, - null, - 2, - ), - "utf-8", - ); + await writeSessionStoreSeed({ + [sessionKey]: { + ...sessionEntry, + updatedAt: 5, + status: "done", + endedAt: 4, + }, + }); clearSessionStoreCacheForTest(); const nowCalls: number[] = []; @@ -1459,8 +1484,12 @@ describe("CLI attempt execution", () => { if (!updatedSessionFile) { throw new Error("expected CLI transcript persistence to create a session file"); } - expect(await fs.realpath(updatedSessionFile)).toBe(await fs.realpath(sessionFile)); - const entries = await readSessionFileEntries(sessionFile); + expect(parseSqliteSessionFileMarker(updatedSessionFile)).toMatchObject({ + agentId: "main", + sessionId: sessionEntry.sessionId, + storePath, + }); + const entries = await readSessionFileEntries(updatedSessionFile); expectRecordFields(requireRecord(entries[0], "session entry"), { type: "session", id: sessionEntry.sessionId, @@ -1474,7 +1503,7 @@ describe("CLI attempt execution", () => { type: "message", parentId: entries[1]?.id, }); - const messages = await readSessionMessages(sessionFile); + const messages = await readSessionMessages(updatedSessionFile); expect(messages).toHaveLength(2); expectRecordFields(requireRecord(messages[0], "user message"), { role: "user", @@ -1488,13 +1517,8 @@ describe("CLI attempt execution", () => { content: [{ type: "text", text: "hello from cli" }], }); - const persisted = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; - expect(await fs.realpath(persisted[sessionKey]?.sessionFile ?? "")).toBe( - await fs.realpath(sessionFile), - ); + const persisted = readSessionStore(); + expect(persisted[sessionKey]?.sessionFile).toBe(updatedSessionFile); expect(persisted[sessionKey]?.updatedAt).toBeGreaterThan(sessionEntry.updatedAt); expect(persisted[sessionKey]?.updatedAt).toBeLessThanOrEqual(nowCalls.at(-1) ?? 0); expect(sessionStore[sessionKey]?.updatedAt).toBe(persisted[sessionKey]?.updatedAt); @@ -1509,18 +1533,18 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); - await appendSessionTranscriptMessage({ - transcriptPath: sessionFile, - sessionId: sessionEntry.sessionId, - cwd: tmpDir, - config: {}, - message: { - role: "user", - content: "canonical current ask", - timestamp: Date.now(), + await writeSessionStoreSeed(sessionStore); + await appendTranscriptMessage( + { agentId: "main", sessionId: sessionEntry.sessionId, sessionKey, storePath }, + { + message: { + role: "user", + content: "canonical current ask", + timestamp: Date.now(), + }, + cwd: tmpDir, }, - }); + ); await persistCliTurnTranscript({ body: "canonical current ask", @@ -1536,7 +1560,13 @@ describe("CLI attempt execution", () => { skipUserTurn: true, }); - const messages = await readSessionMessages(sessionFile); + const messages = await readSessionMessages( + formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: sessionEntry.sessionId, + storePath, + }), + ); expect(messages.filter((message) => message.role === "user")).toHaveLength(1); expect(messages).toContainEqual( expect.objectContaining({ @@ -1555,7 +1585,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); await persistAcpTurnTranscript({ body: "[media attached: media://inbound/image-1]", @@ -1576,7 +1606,15 @@ describe("CLI attempt execution", () => { config: {}, }); - expect(await readSessionMessages(sessionFile)).toContainEqual( + expect( + await readSessionMessages( + formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: sessionEntry.sessionId, + storePath, + }), + ), + ).toContainEqual( expect.objectContaining({ role: "user", content: "[User sent media without caption]", @@ -1595,7 +1633,6 @@ describe("CLI attempt execution", () => { updatedAt: 1, }; const sessionStore: Record = { [sessionKey]: staleEntry }; - await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf-8"); clearSessionStoreCacheForTest(); const result = await persistCliTurnTranscript({ @@ -1613,10 +1650,7 @@ describe("CLI attempt execution", () => { expect(result).toEqual({ kind: "session-rebound", sessionEntry: undefined }); await expect(fs.stat(staleSessionFile)).rejects.toMatchObject({ code: "ENOENT" }); - const persisted = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; + const persisted = readSessionStore(); expect(persisted[sessionKey]).toBeUndefined(); }); @@ -1627,7 +1661,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); const result = makeCliResult("already mirrored"); result.meta.executionTrace = { @@ -1677,72 +1711,6 @@ describe("CLI attempt execution", () => { expect(messages).toHaveLength(1); }); - it("embedded assistant gap-fill skips malformed transcript tail rows before deduping", async () => { - const sessionKey = "agent:main:subagent:embedded-gap-fill-malformed-tail"; - const sessionEntry: SessionEntry = { - sessionId: "session-embedded-gap-fill-malformed-tail", - updatedAt: Date.now(), - }; - const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); - - const result = makeCliResult("already mirrored"); - result.meta.executionTrace = { - winnerProvider: "anthropic", - winnerModel: "claude-opus-4-6", - fallbackUsed: false, - runner: "embedded", - }; - - const updatedFirst = await persistCliTranscriptEntry({ - body: "ignored for gap fill", - result, - sessionId: sessionEntry.sessionId, - sessionKey, - sessionEntry, - sessionStore, - storePath, - sessionAgentId: "main", - sessionCwd: tmpDir, - config: {}, - embeddedAssistantGapFill: true, - }); - const sessionFile = updatedFirst?.sessionFile; - if (typeof sessionFile !== "string") { - throw new Error("Expected CLI transcript session file."); - } - - await fs.appendFile(sessionFile, "{truncated-json\n", "utf-8"); - - await persistCliTurnTranscript({ - body: "still ignored", - result, - sessionId: sessionEntry.sessionId, - sessionKey, - sessionEntry: updatedFirst, - sessionStore, - storePath, - sessionAgentId: "main", - sessionCwd: tmpDir, - config: {}, - embeddedAssistantGapFill: true, - }); - - const validEntries = (await fs.readFile(sessionFile, "utf-8")) - .split(/\r?\n/) - .flatMap((line) => { - if (!line) { - return []; - } - try { - return [JSON.parse(line) as { type?: string; message?: { role?: string } }]; - } catch { - return []; - } - }); - expect(validEntries.filter((entry) => entry.type === "message")).toHaveLength(1); - }); - it("embedded assistant gap-fill skips trailing openclaw.cache-ttl custom entries (regression for #83427)", async () => { const sessionKey = "agent:main:subagent:embedded-gap-fill-cache-ttl"; const sessionEntry: SessionEntry = { @@ -1750,7 +1718,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); const result = makeCliResult("canonical answer"); result.meta.executionTrace = { @@ -1778,9 +1746,9 @@ describe("CLI attempt execution", () => { throw new Error("Expected CLI transcript session file."); } - await fs.appendFile( - sessionFile, - `${JSON.stringify({ + await appendTranscriptEvent( + { agentId: "main", sessionId: sessionEntry.sessionId, sessionKey, storePath }, + { type: "custom", customType: "openclaw.cache-ttl", timestamp: new Date().toISOString(), @@ -1788,8 +1756,7 @@ describe("CLI attempt execution", () => { provider: "anthropic", modelId: "claude-haiku-4-5-20251001", }, - })}\n`, - "utf-8", + } as never, ); await persistCliTurnTranscript({ @@ -1821,7 +1788,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); const result = makeCliResult("same answer"); result.meta.executionTrace = { @@ -1848,26 +1815,20 @@ describe("CLI attempt execution", () => { if (typeof sessionFile !== "string") { throw new Error("Expected CLI transcript session file."); } - expect(path.isAbsolute(sessionFile)).toBe(true); - const persistedFirst = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; - expect(await fs.realpath(persistedFirst[sessionKey]?.sessionFile ?? "")).toBe( - await fs.realpath(sessionFile), - ); + const persistedFirst = readSessionStore(); + expect(persistedFirst[sessionKey]?.sessionFile).toBe(sessionFile); - await appendSessionTranscriptMessage({ - transcriptPath: sessionFile, - sessionId: sessionEntry.sessionId, - cwd: tmpDir, - config: {}, - message: { - role: "user", - content: "next prompt", - timestamp: Date.now(), + await appendTranscriptMessage( + { agentId: "main", sessionId: sessionEntry.sessionId, sessionKey, storePath }, + { + message: { + role: "user", + content: "next prompt", + timestamp: Date.now(), + }, + cwd: tmpDir, }, - }); + ); await persistCliTurnTranscript({ body: "still ignored", @@ -1898,7 +1859,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); const updatedEntry = await persistCliTranscriptEntry({ body: [ @@ -1934,7 +1895,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockResolvedValueOnce(makeCliResult("channel aware")); await runAgentAttempt({ @@ -1999,7 +1960,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockResolvedValueOnce(makeCliResult("sent")); await runAgentAttempt({ @@ -2047,7 +2008,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockResolvedValueOnce(makeCliResult("ambient claude cli")); await runAgentAttempt({ @@ -2097,7 +2058,6 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); saveAuthProfileStore( { version: 1, @@ -2198,7 +2158,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockResolvedValueOnce(makeCliResult("restricted cli")); await runAgentAttempt({ @@ -2248,17 +2208,18 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockResolvedValueOnce(makeCliResult("timestamped cli")); const userTurnTranscriptRecorder = createUserTurnTranscriptRecorder({ input: { text: "canonical timestamp question" }, - target: { - transcriptPath: path.join(tmpDir, "session.jsonl"), + target: createTestUserTurnTranscriptTarget({ sessionId: sessionEntry.sessionId, sessionKey, + sessionEntry, agentId: "main", cwd: tmpDir, - }, + storePath, + }), }); await runAgentAttempt({ providerOverride: "claude-cli", @@ -2308,7 +2269,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockResolvedValueOnce(makeCliResult("canonical cli")); const fallbackRuntimeState: NonNullable = {}; const images = [{ type: "image" as const, data: "aGVsbG8=", mimeType: "image/png" }]; @@ -2379,7 +2340,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockResolvedValueOnce(makeCliResult("shorthand cli")); await runAgentAttempt({ @@ -2434,7 +2395,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runEmbeddedAgentMock.mockResolvedValueOnce({ payloads: [{ text: "canonical codex embedded" }], meta: { @@ -2531,13 +2492,13 @@ describe("CLI attempt execution", () => { it("forwards canonical transcript text without replacing embedded image content", async () => { const recorder = createUserTurnTranscriptRecorder({ - target: { - transcriptPath: path.join(tmpDir, "embedded-image-turn.jsonl"), + target: createTestUserTurnTranscriptTarget({ sessionId: "session-embedded-image-turn", sessionKey: "agent:main:direct:embedded-image-turn", agentId: "main", cwd: tmpDir, - }, + storePath, + }), }); const images = [{ type: "image" as const, data: "aGVsbG8=", mimeType: "image/png" }]; const embeddedArg = await runOpenClawEmbeddedAttemptForTest({ @@ -2612,7 +2573,6 @@ describe("CLI attempt execution", () => { authProfileOverrideSource: "user", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); saveAuthProfileStore( { version: 1, @@ -2691,7 +2651,6 @@ describe("CLI attempt execution", () => { authProfileOverrideSource: "user", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); saveAuthProfileStore( { version: 1, @@ -2766,7 +2725,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runEmbeddedAgentMock.mockResolvedValueOnce({ meta: { durationMs: 1 }, } satisfies EmbeddedAgentRunResult); @@ -2843,7 +2802,7 @@ describe("CLI attempt execution", () => { allowed: true, defaultLevel: "on" as const, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runEmbeddedAgentMock.mockResolvedValueOnce({ meta: { durationMs: 1 }, } satisfies EmbeddedAgentRunResult); @@ -2894,7 +2853,7 @@ describe("CLI attempt execution", () => { updatedAt: Date.now(), }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await writeSessionStoreSeed(sessionStore); runCliAgentMock.mockResolvedValueOnce(makeCliResult("cleanup cli")); await runAgentAttempt({ diff --git a/src/agents/command/attempt-execution.shared.test.ts b/src/agents/command/attempt-execution.shared.test.ts index e661e136e45b..85111d68e29e 100644 --- a/src/agents/command/attempt-execution.shared.test.ts +++ b/src/agents/command/attempt-execution.shared.test.ts @@ -3,11 +3,8 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"; -import { - clearSessionStoreCacheForTest, - loadSessionStore, - saveSessionStore, -} from "../../config/sessions/store.js"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; +import { clearSessionStoreCacheForTest } from "../../config/sessions/store.js"; import type { SessionEntry } from "../../config/sessions/types.js"; import { INTERNAL_RUNTIME_CONTEXT_BEGIN, @@ -158,7 +155,7 @@ describe("persistSessionEntry", () => { if (current.pinnedAt === undefined) { delete currentEntry.pinnedAt; } - await saveSessionStore(storePath, { main: currentEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey: "main", storePath }, currentEntry); const sessionStore = { main: staleEntry }; const persisted = await persistSessionEntry({ @@ -178,7 +175,9 @@ describe("persistSessionEntry", () => { expect(persisted?.pinnedAt).toBe(expected.pinnedAt); expect(persisted?.updatedAt).toBeGreaterThanOrEqual(currentEntry.updatedAt); expect(sessionStore.main).toEqual(persisted); - expect(loadSessionStore(storePath, { skipCache: true }).main).toEqual(persisted); + expect( + loadSessionEntry({ sessionKey: "main", storePath, readConsistency: "latest" }), + ).toEqual(persisted); } finally { clearSessionStoreCacheForTest(); } @@ -202,7 +201,7 @@ describe("persistSessionEntry", () => { model: "gpt-5.4", sendPolicy: "deny", }; - await saveSessionStore(storePath, { main: currentEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey: "main", storePath }, currentEntry); const sessionStore = { main: initialEntry }; const persisted = await persistSessionEntry({ @@ -225,7 +224,107 @@ describe("persistSessionEntry", () => { }); expect(persisted?.elevatedLevel).toBeUndefined(); expect(persisted?.inheritedToolAllow).toBeUndefined(); - expect(loadSessionStore(storePath, { skipCache: true }).main).toEqual(persisted); + expect( + loadSessionEntry({ sessionKey: "main", storePath, readConsistency: "latest" }), + ).toEqual(persisted); + } finally { + clearSessionStoreCacheForTest(); + } + }); + + it("does not recreate a deleted persisted entry from stale local memory", async () => { + const dir = tempDirs.make("openclaw-session-store-"); + try { + const storePath = path.join(dir, "sessions.json"); + const staleEntry: SessionEntry = { + sessionId: "deleted-session", + updatedAt: 1, + }; + const sessionStore = { main: staleEntry }; + + const persisted = await persistSessionEntry({ + sessionStore, + sessionKey: "main", + storePath, + initialEntry: staleEntry, + entry: { + sessionId: "deleted-session", + updatedAt: 2, + }, + }); + + expect(persisted).toBeUndefined(); + expect(sessionStore.main).toBeUndefined(); + expect( + loadSessionEntry({ sessionKey: "main", storePath, readConsistency: "latest" }), + ).toBeUndefined(); + } finally { + clearSessionStoreCacheForTest(); + } + }); + + it("keeps rejecting repeated stale writes after clearing local memory", async () => { + const dir = tempDirs.make("openclaw-session-store-"); + try { + const storePath = path.join(dir, "sessions.json"); + const staleEntry: SessionEntry = { + sessionId: "deleted-session", + updatedAt: 1, + }; + const sessionStore = { + main: staleEntry, + }; + + const first = await persistSessionEntry({ + sessionStore, + sessionKey: "main", + storePath, + initialEntry: staleEntry, + entry: staleEntry, + }); + const second = await persistSessionEntry({ + sessionStore, + sessionKey: "main", + storePath, + initialEntry: staleEntry, + entry: { + ...staleEntry, + updatedAt: 2, + }, + }); + + expect(first).toBeUndefined(); + expect(second).toBeUndefined(); + expect(sessionStore.main).toBeUndefined(); + expect( + loadSessionEntry({ sessionKey: "main", storePath, readConsistency: "latest" }), + ).toBeUndefined(); + } finally { + clearSessionStoreCacheForTest(); + } + }); + + it("allows an explicit create-on-missing persistence predicate", async () => { + const dir = tempDirs.make("openclaw-session-store-"); + try { + const storePath = path.join(dir, "sessions.json"); + const sessionStore: Record = {}; + const entry: SessionEntry = { + sessionId: "created-session", + updatedAt: 1, + }; + + const persisted = await persistSessionEntry({ + sessionStore, + sessionKey: "main", + storePath, + initialEntry: entry, + entry, + shouldPersist: (existing) => existing === undefined, + }); + + expect(persisted?.sessionId).toBe("created-session"); + expect(sessionStore.main?.sessionId).toBe("created-session"); } finally { clearSessionStoreCacheForTest(); } diff --git a/src/agents/command/attempt-execution.shared.ts b/src/agents/command/attempt-execution.shared.ts index e423413c5729..ca8b887cd579 100644 --- a/src/agents/command/attempt-execution.shared.ts +++ b/src/agents/command/attempt-execution.shared.ts @@ -33,7 +33,12 @@ export async function persistSessionEntry( const persisted = await patchSessionEntry( { sessionKey: params.sessionKey, storePath: params.storePath }, (_entry, context) => { - if (params.shouldPersist && !params.shouldPersist(context.existingEntry)) { + const shouldPersistCurrent = params.shouldPersist?.(context.existingEntry); + if (!context.existingEntry && shouldPersistCurrent !== true) { + rejectedMissingEntry = true; + return null; + } + if (shouldPersistCurrent === false) { rejectedMissingEntry = !context.existingEntry; return null; } diff --git a/src/agents/command/attempt-execution.ts b/src/agents/command/attempt-execution.ts index 65ad43be4e09..5eaaefea60ce 100644 --- a/src/agents/command/attempt-execution.ts +++ b/src/agents/command/attempt-execution.ts @@ -61,6 +61,7 @@ import { resolveAvailableAgentHarnessPolicy } from "../harness/selection.js"; import { resolveCliRuntimeExecutionProvider } from "../model-runtime-aliases.js"; import { isCliProvider } from "../model-selection.js"; import { resolveOpenAIRuntimeProvider } from "../openai-routing.js"; +import type { AgentRunSessionTarget } from "../run-session-target.js"; import { resolveAgentRunAbortLifecycleFields } from "../run-termination.js"; import { buildAgentRuntimeAuthPlan } from "../runtime-plan/auth.js"; import type { AgentMessage } from "../runtime/index.js"; @@ -303,7 +304,9 @@ async function persistTextTurnTranscript( if (!params.embeddedAssistantGapFill) { return true; } - const latest = await readTailAssistantTextFromSessionTranscript(sessionFile); + const latest = await readTailAssistantTextFromSessionTranscript(sessionFile, { + excludeTranscriptOnlyOpenClawAssistant: true, + }); const normalizedReply = normalizeTranscriptMirrorText(replyText); const normalizedLatest = latest?.text ? normalizeTranscriptMirrorText(latest.text) : ""; return !normalizedLatest || normalizedLatest !== normalizedReply; @@ -441,6 +444,7 @@ export function runAgentAttempt(params: { agentHarnessRuntimeOverride?: string; sessionId: string; sessionKey: string | undefined; + sessionTarget?: AgentRunSessionTarget; sessionAgentId: string; sessionFile: string; workspaceDir: string; @@ -697,6 +701,7 @@ export function runAgentAttempt(params: { agentId: params.sessionAgentId, trigger: "user", sessionFile: params.sessionFile, + storePath: params.storePath, workspaceDir: params.workspaceDir, cwd: params.cwd, config: params.cfg, @@ -812,6 +817,8 @@ export function runAgentAttempt(params: { return runEmbeddedAgent({ sessionId: params.sessionId, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, + sandboxSessionKey: params.sessionKey, agentId: params.sessionAgentId, trigger: "user", messageChannel: params.messageChannel, diff --git a/src/agents/command/cli-compaction.test.ts b/src/agents/command/cli-compaction.test.ts index 3bf7b8b90f95..a39b2994071a 100644 --- a/src/agents/command/cli-compaction.test.ts +++ b/src/agents/command/cli-compaction.test.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import { CURRENT_SESSION_VERSION } from "openclaw/plugin-sdk/agent-sessions"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import type { SessionEntry } from "../../config/sessions/types.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { ContextEngine } from "../../context-engine/types.js"; @@ -74,6 +75,20 @@ async function writeSessionFile(params: { sessionFile: string; sessionId: string ); } +async function persistSessionEntry(params: { + sessionKey: string; + storePath: string; + entry: SessionEntry; +}) { + await replaceSessionEntry( + { + sessionKey: params.sessionKey, + storePath: params.storePath, + }, + params.entry, + ); +} + describe("runCliTurnCompactionLifecycle", () => { let tmpDir: string; @@ -114,7 +129,7 @@ describe("runCliTurnCompactionLifecycle", () => { claudeCliSessionId: "claude-session", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const maintenance = vi.fn(async () => ({ changed: false, bytesFreed: 0, rewrittenEntries: 0 })); @@ -163,7 +178,7 @@ describe("runCliTurnCompactionLifecycle", () => { const compactCall = compactCalls[0]; expect(compactCall?.sessionId).toBe(sessionId); expect(compactCall?.sessionKey).toBe(sessionKey); - expect(compactCall?.sessionFile).toBe(sessionFile); + expect(compactCall?.sessionTarget).toEqual({ sessionId, sessionKey, storePath }); expect(compactCall?.tokenBudget).toBe(1_000); expect(compactCall?.currentTokenCount).toBe(950); expect(compactCall?.force).toBe(true); @@ -195,6 +210,103 @@ describe("runCliTurnCompactionLifecycle", () => { expect(updatedEntry?.claudeCliSessionId).toBeUndefined(); }); + it("records context-engine compaction successor session targets", async () => { + const sessionKey = "agent:main:cli-rotates"; + const sessionId = "session-cli-rotates"; + const successorSessionId = "session-cli-rotated"; + const sessionFile = path.join(tmpDir, "session-rotates.jsonl"); + const storePath = path.join(tmpDir, "sessions-rotates.json"); + await writeSessionFile({ sessionFile, sessionId }); + + const sessionEntry: SessionEntry = { + sessionId, + updatedAt: Date.now(), + sessionFile, + contextTokens: 1_000, + totalTokens: 950, + totalTokensFresh: true, + }; + const sessionStore: Record = { [sessionKey]: sessionEntry }; + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); + + const compactCalls: Array[0]> = []; + const maintenance = vi.fn(async () => ({ changed: false, bytesFreed: 0, rewrittenEntries: 0 })); + const recordCliCompactionInStore = vi.fn(async () => ({ + ...sessionEntry, + sessionId: successorSessionId, + sessionFile: `sqlite:main:${successorSessionId}:${storePath}`, + compactionCount: 1, + })); + setCliCompactionTestDeps({ + resolveContextEngine: async () => ({ + ...buildContextEngine({ compactCalls }), + async compact(compactParams) { + compactCalls.push(compactParams); + return { + ok: true, + compacted: true, + result: { + summary: "compacted", + tokensBefore: compactParams.currentTokenCount ?? 0, + tokensAfter: 100, + sessionId: successorSessionId, + sessionTarget: { + sessionKey, + storePath, + }, + }, + }; + }, + }), + createPreparedEmbeddedAgentSettingsManager: async () => ({ + getCompactionReserveTokens: () => 200, + getCompactionKeepRecentTokens: () => 0, + applyOverrides: () => {}, + }), + shouldPreemptivelyCompactBeforePrompt: () => ({ + route: "fits", + shouldCompact: false, + estimatedPromptTokens: 600, + promptBudgetBeforeReserve: 800, + overflowTokens: 0, + toolResultReducibleChars: 0, + effectiveReserveTokens: 200, + }), + resolveLiveToolResultMaxChars: () => 20_000, + runContextEngineMaintenance: maintenance, + recordCliCompactionInStore, + }); + + await runCliTurnCompactionLifecycle({ + cfg: {} as OpenClawConfig, + sessionId, + sessionKey, + sessionEntry, + sessionStore, + storePath, + sessionAgentId: "main", + workspaceDir: tmpDir, + agentDir: tmpDir, + provider: "claude-cli", + model: "opus", + }); + + expect(compactCalls[0]?.sessionTarget).toEqual({ sessionId, sessionKey, storePath }); + expect(maintenance).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: successorSessionId, + sessionFile: `sqlite:main:${successorSessionId}:${storePath}`, + }), + ); + expect(recordCliCompactionInStore).toHaveBeenCalledWith( + expect.objectContaining({ + newSessionFile: `sqlite:main:${successorSessionId}:${storePath}`, + newSessionId: successorSessionId, + tokensAfter: 100, + }), + ); + }); + it("treats below-target CLI transcript compaction as a no-op", async () => { const sessionKey = "agent:main:cli-under-target"; const sessionId = "session-cli-under-target"; @@ -214,7 +326,7 @@ describe("runCliTurnCompactionLifecycle", () => { }, }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const maintenance = vi.fn(async () => ({ changed: false, bytesFreed: 0, rewrittenEntries: 0 })); @@ -361,7 +473,7 @@ describe("runCliTurnCompactionLifecycle", () => { authProfileOverrideSource: "auto", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const contextEngine = buildContextEngine({ compactCalls }); @@ -486,7 +598,7 @@ describe("runCliTurnCompactionLifecycle", () => { agentHarnessId: "copilot", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const compactAgentHarnessSession = vi.fn(async () => ({ @@ -554,7 +666,7 @@ describe("runCliTurnCompactionLifecycle", () => { agentHarnessId: "codex", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const compactAgentHarnessSession = vi.fn(); @@ -637,7 +749,7 @@ describe("runCliTurnCompactionLifecycle", () => { agentHarnessId: "codex", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const ensureSelectedAgentHarnessPlugin = vi.fn(async () => undefined); @@ -709,7 +821,7 @@ describe("runCliTurnCompactionLifecycle", () => { agentHarnessId: "codex", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const maintenance = vi.fn(async () => ({ changed: false, bytesFreed: 0, rewrittenEntries: 0 })); @@ -774,7 +886,7 @@ describe("runCliTurnCompactionLifecycle", () => { expect.objectContaining({ provider: "codex", sessionKey, - tokensAfter: undefined, + tokensAfter: 100, }), ); expect(result?.compactionCount).toBe(1); @@ -830,7 +942,7 @@ describe("runCliTurnCompactionLifecycle", () => { agentHarnessId: "codex", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; setCliCompactionTestDeps({ @@ -891,7 +1003,7 @@ describe("runCliTurnCompactionLifecycle", () => { agentHarnessId: "codex", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const contextEngine = { @@ -992,7 +1104,7 @@ describe("runCliTurnCompactionLifecycle", () => { agentHarnessId: "external-harness", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const ensureSelectedAgentHarnessPlugin = vi.fn(async () => undefined); @@ -1051,7 +1163,7 @@ describe("runCliTurnCompactionLifecycle", () => { expect.objectContaining({ provider: "external-harness", sessionKey, - tokensAfter: undefined, + tokensAfter: 100, }), ); expect(updatedEntry?.compactionCount).toBe(1); @@ -1076,7 +1188,7 @@ describe("runCliTurnCompactionLifecycle", () => { authProfileOverrideSource: "auto", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const ensureSelectedAgentHarnessPlugin = vi.fn(async () => undefined); @@ -1140,7 +1252,7 @@ describe("runCliTurnCompactionLifecycle", () => { expect.objectContaining({ provider: "codex", sessionKey, - tokensAfter: undefined, + tokensAfter: 100, }), ); expect(updatedEntry?.compactionCount).toBe(1); @@ -1169,7 +1281,7 @@ describe("runCliTurnCompactionLifecycle", () => { }, }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const compactAgentHarnessSession = vi.fn(async () => ({ @@ -1257,7 +1369,7 @@ describe("runCliTurnCompactionLifecycle", () => { agentHarnessId: "codex", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const compactAgentHarnessSession = vi.fn(async () => ({ @@ -1314,7 +1426,7 @@ describe("runCliTurnCompactionLifecycle", () => { expect.objectContaining({ provider: "codex", sessionKey, - tokensAfter: undefined, + tokensAfter: 100, }), ); expect(updatedEntry?.compactionCount).toBe(1); @@ -1337,7 +1449,7 @@ describe("runCliTurnCompactionLifecycle", () => { agentHarnessId: "codex", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const maintenance = vi.fn(async () => { @@ -1475,7 +1587,7 @@ describe("runCliTurnCompactionLifecycle", () => { claudeCliSessionId: "claude-session", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const maintenance = vi.fn(async () => ({ changed: false, bytesFreed: 0, rewrittenEntries: 0 })); @@ -1555,7 +1667,7 @@ describe("runCliTurnCompactionLifecycle", () => { totalTokensFresh: true, }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const compactAgentHarnessSession = vi.fn(); @@ -1623,7 +1735,7 @@ describe("runCliTurnCompactionLifecycle", () => { totalTokensFresh: true, }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const maintenance = vi.fn(async () => ({ changed: false, bytesFreed: 0, rewrittenEntries: 0 })); @@ -1686,7 +1798,7 @@ describe("runCliTurnCompactionLifecycle", () => { agentHarnessId: "codex", }; const sessionStore: Record = { [sessionKey]: sessionEntry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await persistSessionEntry({ sessionKey, storePath, entry: sessionEntry }); const compactCalls: Array[0]> = []; const contextEngine = buildContextEngine({ compactCalls }); diff --git a/src/agents/command/cli-compaction.ts b/src/agents/command/cli-compaction.ts index 25a8cbb61305..1d2a1c050a38 100644 --- a/src/agents/command/cli-compaction.ts +++ b/src/agents/command/cli-compaction.ts @@ -36,6 +36,7 @@ import type { EmbeddedAgentCompactResult } from "../embedded-agent-runner/types. import { isRecoverableNativeHarnessBindingFailure } from "../harness/compaction-recovery.js"; import { maybeCompactAgentHarnessSession as maybeCompactAgentHarnessSessionImpl } from "../harness/compaction.js"; import { ensureSelectedAgentHarnessPlugin as ensureSelectedAgentHarnessPluginImpl } from "../harness/runtime-plugin.js"; +import { resolveAgentRunSessionTarget } from "../run-session-target.js"; import type { AgentMessage } from "../runtime/index.js"; import { SessionManager } from "../sessions/session-manager.js"; import { @@ -92,6 +93,9 @@ type NativeHarnessCliCompactionOutcome = { type CliTranscriptCompactionOutcome = { compacted: boolean; failureReason?: string; + successorSessionFile?: string; + successorSessionId?: string; + tokensAfter?: number; }; type CliCompactionRuntimeContextParams = { sessionKey: string; @@ -249,12 +253,62 @@ function buildCliCompactionRuntimeContext(params: CliCompactionRuntimeContextPar }; } +async function resolveCliContextCompactionSuccess(params: { + cfg: OpenClawConfig; + compactResult: Awaited>; + sessionFile: string; + sessionId: string; + sessionKey: string; + storePath?: string; +}): Promise<{ + maintenanceSessionFile: string; + maintenanceSessionId: string; + successorSessionFile?: string; + successorSessionId?: string; + tokensAfter?: number; +}> { + const result = params.compactResult.result; + const resultTarget = result?.sessionTarget; + const explicitResultSessionId = result?.sessionId ?? resultTarget?.sessionId; + const resultSessionId = explicitResultSessionId ?? params.sessionId; + const resultSessionTarget = + resultTarget && resultSessionId + ? { ...resultTarget, sessionId: resultTarget.sessionId ?? resultSessionId } + : resultTarget; + if (!resultSessionTarget && !explicitResultSessionId) { + return { + maintenanceSessionFile: params.sessionFile, + maintenanceSessionId: params.sessionId, + ...(result?.tokensAfter !== undefined ? { tokensAfter: result.tokensAfter } : {}), + }; + } + const resolvedTarget = await resolveAgentRunSessionTarget({ + agentId: resultSessionTarget?.agentId ?? readAgentIdFromSessionKey(params.sessionKey), + config: params.cfg, + sessionId: resultSessionId, + sessionKey: resultSessionTarget?.sessionKey ?? params.sessionKey, + sessionTarget: Object.assign( + {}, + resultSessionTarget, + params.storePath && !resultSessionTarget?.storePath ? { storePath: params.storePath } : {}, + ), + }); + return { + maintenanceSessionFile: resolvedTarget.sessionFile, + maintenanceSessionId: resolvedTarget.sessionId, + successorSessionFile: resolvedTarget.sessionFile, + successorSessionId: resolvedTarget.sessionId, + ...(result?.tokensAfter !== undefined ? { tokensAfter: result.tokensAfter } : {}), + }; +} + async function compactCliTranscript(params: { contextEngine: ContextEngine; sessionId: string; sessionKey: string; sessionFile: string; sessionManager: SessionManagerLike; + storePath?: string; cfg: OpenClawConfig; workspaceDir: string; cwd?: string; @@ -314,8 +368,12 @@ async function compactCliTranscript(params: { params.contextEngine, { sessionId: params.sessionId, - sessionKey: params.sessionKey, - sessionFile: params.sessionFile, + sessionKey: params.sessionKey || params.sessionId, + sessionTarget: { + sessionId: params.sessionId, + sessionKey: params.sessionKey || params.sessionId, + ...(params.storePath ? { storePath: params.storePath } : {}), + }, tokenBudget: params.contextTokenBudget, currentTokenCount: params.currentTokenCount, force: true, @@ -357,12 +415,20 @@ async function compactCliTranscript(params: { }; } + const successor = await resolveCliContextCompactionSuccess({ + cfg: params.cfg, + compactResult, + sessionFile: params.sessionFile, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }); try { await cliCompactionDeps.runContextEngineMaintenance({ contextEngine: params.contextEngine, - sessionId: params.sessionId, + sessionId: successor.maintenanceSessionId, sessionKey: params.sessionKey, - sessionFile: params.sessionFile, + sessionFile: successor.maintenanceSessionFile, reason: "compaction", sessionManager: params.sessionManager, runtimeContext, @@ -377,7 +443,7 @@ async function compactCliTranscript(params: { `CLI transcript compaction maintenance failed after fallback for ${params.provider}/${params.model}: ${error instanceof Error ? error.message : String(error)}`, ); } - return { compacted: true }; + return { compacted: true, ...successor }; } async function compactNativeHarnessCliTranscript(params: { @@ -597,6 +663,7 @@ export async function runCliTurnCompactionLifecycle(params: { } let compacted = false; + let contextCompactionOutcome: CliTranscriptCompactionOutcome | undefined; let nativeCompactionResult: EmbeddedAgentCompactResult | undefined; let useContextEngineCompaction = true; let nativeFallbackToContextEngine = false; @@ -676,6 +743,7 @@ export async function runCliTurnCompactionLifecycle(params: { sessionKey: params.sessionKey, sessionFile, sessionManager, + storePath: params.storePath, cfg: params.cfg, workspaceDir: params.workspaceDir, cwd: params.cwd, @@ -695,6 +763,7 @@ export async function runCliTurnCompactionLifecycle(params: { extraSystemPrompt: params.extraSystemPrompt, bestEffortMaintenance: nativeFallbackToContextEngine, }); + contextCompactionOutcome = contextOutcome; compacted = contextOutcome.compacted; if (!compacted && contextOutcome.failureReason) { throw new Error( @@ -727,9 +796,13 @@ export async function runCliTurnCompactionLifecycle(params: { sessionKey: params.sessionKey, sessionStore: params.sessionStore, storePath: params.storePath, - tokensAfter: nativeCompactionResult?.result?.tokensAfter, - newSessionId: nativeCompactionResult?.result?.sessionId, - newSessionFile: nativeCompactionResult?.result?.sessionFile, + tokensAfter: + nativeCompactionResult?.result?.tokensAfter ?? contextCompactionOutcome?.tokensAfter, + newSessionId: + nativeCompactionResult?.result?.sessionId ?? contextCompactionOutcome?.successorSessionId, + newSessionFile: + nativeCompactionResult?.result?.sessionFile ?? + contextCompactionOutcome?.successorSessionFile, expectedSessionId: params.sessionId, })) ?? params.sessionEntry ); diff --git a/src/agents/command/session-store.runtime.ts b/src/agents/command/session-store.runtime.ts index 498f4fb6c166..3b80111314cc 100644 --- a/src/agents/command/session-store.runtime.ts +++ b/src/agents/command/session-store.runtime.ts @@ -1,5 +1,4 @@ // Runtime barrel for session-store writes; keeps command modules from importing // config/session persistence until an agent run needs to save state. export { updateSessionStoreAfterAgentRun } from "./session-store.js"; -export { loadSessionStore } from "../../config/sessions.js"; export { loadSessionEntry } from "../../config/sessions/session-accessor.js"; diff --git a/src/agents/command/session-store.test.ts b/src/agents/command/session-store.test.ts index 06f0190b0ff5..2245ef9916e3 100644 --- a/src/agents/command/session-store.test.ts +++ b/src/agents/command/session-store.test.ts @@ -3,10 +3,15 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../config/config.js"; import type { SessionEntry } from "../../config/sessions.js"; -import { loadSessionStore } from "../../config/sessions.js"; +import { + listSessionEntries, + loadSessionEntry, + replaceSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; import type { EmbeddedAgentRunResult } from "../embedded-agent.js"; import { clearCliSessionInStore, @@ -15,10 +20,6 @@ import { } from "./session-store.js"; import { resolveSession } from "./session.js"; -const sessionStoreMocks = vi.hoisted(() => ({ - updateSessionStore: vi.fn(), -})); - vi.mock("../model-selection.js", () => ({ isCliProvider: (provider: string, cfg?: OpenClawConfig) => Object.hasOwn(cfg?.agents?.defaults?.cliBackends ?? {}, provider), @@ -71,80 +72,6 @@ vi.mock("../../utils/usage-format.js", () => ({ }, })); -vi.mock("../../config/sessions.js", async () => { - const fsSync = await import("node:fs"); - const fsLocal = await import("node:fs/promises"); - const pathLocal = await import("node:path"); - const readStore = async (storePath: string): Promise> => { - try { - return JSON.parse(await fsLocal.readFile(storePath, "utf8")) as Record; - } catch { - return {}; - } - }; - const writeStore = async (storePath: string, store: Record) => { - await fsLocal.mkdir(pathLocal.dirname(storePath), { recursive: true }); - await fsLocal.writeFile(storePath, JSON.stringify(store, null, 2), "utf8"); - }; - sessionStoreMocks.updateSessionStore.mockImplementation( - async ( - storePath: string, - mutator: (store: Record) => Promise | T, - ) => { - const store = await readStore(storePath); - const previousAcpByKey = new Map( - Object.entries(store) - .filter( - (entry): entry is [string, SessionEntry & { acp: NonNullable }] => - Boolean(entry[1]?.acp), - ) - .map(([key, entry]) => [key, entry.acp]), - ); - const result = await mutator(store); - // The mocked store keeps ACP metadata sticky to preserve the production - // merge behavior that protects persistent ACP session handles. - for (const [key, acp] of previousAcpByKey) { - const next = store[key]; - if (next && !next.acp) { - next.acp = acp; - } - } - await writeStore(storePath, store); - return result; - }, - ); - return { - mergeSessionEntry: (existing: SessionEntry | undefined, patch: Partial) => ({ - ...existing, - ...patch, - sessionId: patch.sessionId ?? existing?.sessionId ?? "mock-session", - updatedAt: Math.max(existing?.updatedAt ?? 0, patch.updatedAt ?? 0, Date.now()), - }), - setSessionRuntimeModel: (entry: SessionEntry, runtime: { provider: string; model: string }) => { - entry.modelProvider = runtime.provider; - entry.model = runtime.model; - return true; - }, - updateSessionStore: sessionStoreMocks.updateSessionStore, - loadSessionStore: (storePath: string) => { - try { - return JSON.parse(fsSync.readFileSync(storePath, "utf8")) as Record; - } catch { - return {}; - } - }, - canonicalizeAbsoluteSessionFilePath: (filePath: string) => pathLocal.resolve(filePath), - rewriteSessionFileForNewSessionId: (params: { - sessionFile?: string; - previousSessionId: string; - nextSessionId: string; - }) => params.sessionFile?.replace(params.previousSessionId, params.nextSessionId), - resolveSessionFilePathOptions: (params: unknown) => params, - resolveSessionFilePath: (sessionId: string, entry?: SessionEntry) => - entry?.sessionFile ?? pathLocal.join("/tmp", `${sessionId}.jsonl`), - }; -}); - function acpMeta() { return { backend: "acpx", @@ -159,16 +86,41 @@ function acpMeta() { async function withTempSessionStore( run: (params: { dir: string; storePath: string }) => Promise, ): Promise { - // Session-store tests exercise real JSON persistence, but each case gets an - // isolated file so mutation order remains deterministic. const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-session-store-")); try { return await run({ dir, storePath: path.join(dir, "sessions.json") }); } finally { + closeOpenClawAgentDatabasesForTest(); await fs.rm(dir, { recursive: true, force: true }); } } +async function seedSessionStore( + storePath: string, + entries: Record, +): Promise { + for (const [sessionKey, entry] of Object.entries(entries)) { + await replaceSessionEntry({ storePath, sessionKey }, entry); + } +} + +function loadPersistedSessionStore(storePath: string): Record { + return Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ); +} + +function loadPersistedSessionEntry( + storePath: string, + sessionKey: string, +): SessionEntry | undefined { + return loadSessionEntry({ storePath, sessionKey }) ?? undefined; +} + +afterEach(() => { + closeOpenClawAgentDatabasesForTest(); +}); + describe("updateSessionStoreAfterAgentRun", () => { it("preserves a concurrent rename and unpin during final accounting", async () => { await withTempSessionStore(async ({ storePath }) => { @@ -195,12 +147,7 @@ describe("updateSessionStoreAfterAgentRun", () => { delete concurrentEntry.elevatedLevel; delete concurrentEntry.inheritedToolAllow; delete concurrentEntry.pinnedAt; - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: concurrentEntry, - }), - ); + await seedSessionStore(storePath, { [sessionKey]: concurrentEntry }); await updateSessionStoreAfterAgentRun({ cfg: {} as OpenClawConfig, @@ -227,14 +174,11 @@ describe("updateSessionStoreAfterAgentRun", () => { expect(sessionStore[sessionKey]?.elevatedLevel).toBeUndefined(); expect(sessionStore[sessionKey]?.inheritedToolAllow).toBeUndefined(); expect(sessionStore[sessionKey]?.pinnedAt).toBeUndefined(); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toEqual( - sessionStore[sessionKey], - ); + expect(loadPersistedSessionEntry(storePath, sessionKey)).toEqual(sessionStore[sessionKey]); }); }); it("passes resolved maintenance config to the gateway turn store write", async () => { - sessionStoreMocks.updateSessionStore.mockClear(); await withTempSessionStore(async ({ storePath }) => { const cfg = { session: { @@ -262,7 +206,7 @@ describe("updateSessionStoreAfterAgentRun", () => { ]), ), }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { durationMs: 1, @@ -285,7 +229,7 @@ describe("updateSessionStoreAfterAgentRun", () => { result, }); - const persisted = loadSessionStore(storePath, { skipCache: true }); + const persisted = loadPersistedSessionStore(storePath); expect(Object.keys(persisted)).toHaveLength(42); expect(persisted[sessionKey]?.sessionId).toBe(sessionId); expect(persisted["agent:main:stale:44"]).toBeUndefined(); @@ -303,7 +247,7 @@ describe("updateSessionStoreAfterAgentRun", () => { updatedAt: 1, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { durationMs: 1, @@ -328,7 +272,7 @@ describe("updateSessionStoreAfterAgentRun", () => { }); expect(sessionStore[sessionKey]?.agentHarnessId).toBe("codex"); - expect(loadSessionStore(storePath)[sessionKey]?.agentHarnessId).toBe("codex"); + expect(loadPersistedSessionEntry(storePath, sessionKey)?.agentHarnessId).toBe("codex"); }); }); @@ -346,7 +290,7 @@ describe("updateSessionStoreAfterAgentRun", () => { updatedAt: 1, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); await updateSessionStoreAfterAgentRun({ cfg, @@ -392,7 +336,7 @@ describe("updateSessionStoreAfterAgentRun", () => { updatedAt: 1, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { @@ -418,7 +362,7 @@ describe("updateSessionStoreAfterAgentRun", () => { }); expect(sessionStore[sessionKey]?.contextTokens).toBe(400_000); - expect(loadSessionStore(storePath)[sessionKey]?.contextTokens).toBe(400_000); + expect(loadPersistedSessionEntry(storePath, sessionKey)?.contextTokens).toBe(400_000); }); }); @@ -441,7 +385,7 @@ describe("updateSessionStoreAfterAgentRun", () => { updatedAt: 1, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { @@ -467,7 +411,7 @@ describe("updateSessionStoreAfterAgentRun", () => { }); expect(sessionStore[sessionKey]?.contextTokens).toBe(272_000); - expect(loadSessionStore(storePath)[sessionKey]?.contextTokens).toBe(272_000); + expect(loadPersistedSessionEntry(storePath, sessionKey)?.contextTokens).toBe(272_000); }); }); @@ -493,7 +437,7 @@ describe("updateSessionStoreAfterAgentRun", () => { agentHarnessId: "codex", }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { @@ -519,7 +463,7 @@ describe("updateSessionStoreAfterAgentRun", () => { }); expect(sessionStore[sessionKey]?.agentHarnessId).toBeUndefined(); - expect(loadSessionStore(storePath)[sessionKey]?.agentHarnessId).toBeUndefined(); + expect(loadPersistedSessionEntry(storePath, sessionKey)?.agentHarnessId).toBeUndefined(); }); }); @@ -544,7 +488,7 @@ describe("updateSessionStoreAfterAgentRun", () => { updatedAt: 1, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { @@ -579,7 +523,7 @@ describe("updateSessionStoreAfterAgentRun", () => { expect(sessionStore[sessionKey]?.cliSessionIds?.["claude-cli"]).toBe("cli-session-123"); expect(sessionStore[sessionKey]?.claudeCliSessionId).toBe("cli-session-123"); - const persisted = loadSessionStore(storePath); + const persisted = loadPersistedSessionStore(storePath); expect(persisted[sessionKey]?.cliSessionBindings?.["claude-cli"]).toEqual({ sessionId: "cli-session-123", }); @@ -624,7 +568,7 @@ describe("updateSessionStoreAfterAgentRun", () => { claudeCliSessionId: "stale-cli-session", }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { @@ -658,7 +602,7 @@ describe("updateSessionStoreAfterAgentRun", () => { expect(sessionStore[sessionKey]?.cliSessionIds?.["codex-cli"]).toBe("codex-session"); expect(sessionStore[sessionKey]?.claudeCliSessionId).toBeUndefined(); - const persisted = loadSessionStore(storePath, { skipCache: true }); + const persisted = loadPersistedSessionStore(storePath); expect(persisted[sessionKey]?.cliSessionBindings?.["claude-cli"]).toBeUndefined(); expect(persisted[sessionKey]?.cliSessionIds?.["claude-cli"]).toBeUndefined(); expect(persisted[sessionKey]?.claudeCliSessionId).toBeUndefined(); @@ -675,7 +619,7 @@ describe("updateSessionStoreAfterAgentRun", () => { updatedAt: Date.now(), acp: acpMeta(), }; - await fs.writeFile(storePath, JSON.stringify({ [sessionKey]: existing }, null, 2), "utf8"); + await seedSessionStore(storePath, { [sessionKey]: existing }); const staleInMemory: Record = { [sessionKey]: { @@ -705,7 +649,7 @@ describe("updateSessionStoreAfterAgentRun", () => { } as never, }); - const persisted = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + const persisted = loadPersistedSessionEntry(storePath, sessionKey); expect(persisted?.acp?.backend).toBe("acpx"); expect(persisted?.acp?.agent).toBe("codex"); expect(persisted?.acp?.runtimeSessionName).toBe("runtime-1"); @@ -728,7 +672,7 @@ describe("updateSessionStoreAfterAgentRun", () => { endedAt: 1_900, runtimeMs: 900, }; - await fs.writeFile(storePath, JSON.stringify({ [sessionKey]: terminalEntry }, null, 2)); + await seedSessionStore(storePath, { [sessionKey]: terminalEntry }); const staleInMemory: Record = { [sessionKey]: { @@ -759,7 +703,7 @@ describe("updateSessionStoreAfterAgentRun", () => { } as never, }); - const persisted = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + const persisted = loadPersistedSessionEntry(storePath, sessionKey); expect(persisted?.status).toBe("done"); expect(persisted?.startedAt).toBe(1_000); expect(persisted?.endedAt).toBe(1_900); @@ -781,7 +725,7 @@ describe("updateSessionStoreAfterAgentRun", () => { updatedAt: Date.now(), }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf8"); + await seedSessionStore(storePath, sessionStore); const report = { source: "run" as const, @@ -821,7 +765,7 @@ describe("updateSessionStoreAfterAgentRun", () => { } as never, }); - const persisted = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + const persisted = loadPersistedSessionEntry(storePath, sessionKey); expect(persisted?.systemPromptReport?.bootstrapTruncation?.warningSignaturesSeen).toEqual([ "sig-a", "sig-b", @@ -891,7 +835,7 @@ describe("updateSessionStoreAfterAgentRun", () => { authEpoch: "auth-epoch-1", }); - const persisted = loadSessionStore(storePath, { skipCache: true })[first.sessionKey!]; + const persisted = loadPersistedSessionEntry(storePath, first.sessionKey!); expect(persisted?.cliSessionBindings?.["claude-cli"]).toEqual({ sessionId: "claude-cli-session-1", authEpoch: "auth-epoch-1", @@ -904,23 +848,16 @@ describe("updateSessionStoreAfterAgentRun", () => { const sessionKey = "agent:main:explicit:terminal-cli-session"; const existingSessionId = "terminal-cli-session-old"; const now = Date.now(); - await fs.writeFile( - storePath, - JSON.stringify( - { - [sessionKey]: { - sessionId: existingSessionId, - updatedAt: now, - status: "done", - startedAt: now - 1_000, - endedAt: now - 100, - runtimeMs: 900, - }, - }, - null, - 2, - ), - ); + await seedSessionStore(storePath, { + [sessionKey]: { + sessionId: existingSessionId, + updatedAt: now, + status: "done", + startedAt: now - 1_000, + endedAt: now - 100, + runtimeMs: 900, + }, + }); const result = resolveSession({ cfg: { @@ -954,7 +891,7 @@ describe("updateSessionStoreAfterAgentRun", () => { totalTokensFresh: true, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { @@ -981,7 +918,7 @@ describe("updateSessionStoreAfterAgentRun", () => { expect(sessionStore[sessionKey]?.totalTokens).toBe(21225); expect(sessionStore[sessionKey]?.totalTokensFresh).toBe(false); - const persisted = loadSessionStore(storePath); + const persisted = loadPersistedSessionStore(storePath); expect(persisted[sessionKey]?.totalTokens).toBe(21225); expect(persisted[sessionKey]?.totalTokensFresh).toBe(false); }); @@ -1000,7 +937,7 @@ describe("updateSessionStoreAfterAgentRun", () => { totalTokensFresh: true, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { @@ -1051,7 +988,7 @@ describe("updateSessionStoreAfterAgentRun", () => { contextTokenBudget: 32_000, }); - const persisted = loadSessionStore(storePath); + const persisted = loadPersistedSessionStore(storePath); expect(persisted[sessionKey]?.contextBudgetStatus?.estimatedPromptTokens).toBe(18_000); }); }); @@ -1088,7 +1025,7 @@ describe("updateSessionStoreAfterAgentRun", () => { }, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { @@ -1116,7 +1053,7 @@ describe("updateSessionStoreAfterAgentRun", () => { expect(sessionStore[sessionKey]?.model).toBe("MiniMax-M2.7"); expect(sessionStore[sessionKey]?.contextBudgetStatus).toBeUndefined(); - const persisted = loadSessionStore(storePath); + const persisted = loadPersistedSessionStore(storePath); expect(persisted[sessionKey]?.contextBudgetStatus).toBeUndefined(); }); }); @@ -1142,7 +1079,7 @@ describe("updateSessionStoreAfterAgentRun", () => { totalTokensFresh: true, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); await updateSessionStoreAfterAgentRun({ cfg, @@ -1190,7 +1127,7 @@ describe("updateSessionStoreAfterAgentRun", () => { totalTokensFresh: true, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await replaceSessionEntry({ storePath, sessionKey }, sessionStore[sessionKey]!); await updateSessionStoreAfterAgentRun({ cfg: {} as OpenClawConfig, @@ -1253,7 +1190,7 @@ describe("updateSessionStoreAfterAgentRun", () => { updatedAt: 1, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); await updateSessionStoreAfterAgentRun({ cfg, @@ -1291,8 +1228,8 @@ describe("updateSessionStoreAfterAgentRun", () => { expect(sessionStore[sessionKey]?.totalTokens).toBe(50_006); expect(sessionStore[sessionKey]?.totalTokensFresh).toBe(true); - expect(loadSessionStore(storePath)[sessionKey]?.totalTokens).toBe(50_006); - expect(loadSessionStore(storePath)[sessionKey]?.totalTokensFresh).toBe(true); + expect(loadPersistedSessionEntry(storePath, sessionKey)?.totalTokens).toBe(50_006); + expect(loadPersistedSessionEntry(storePath, sessionKey)?.totalTokensFresh).toBe(true); }); }); @@ -1307,7 +1244,7 @@ describe("updateSessionStoreAfterAgentRun", () => { updatedAt: 1, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { @@ -1337,7 +1274,7 @@ describe("updateSessionStoreAfterAgentRun", () => { expect(sessionStore[sessionKey]?.totalTokensFresh).toBe(true); expect(sessionStore[sessionKey]?.compactionCount).toBe(1); - const persisted = loadSessionStore(storePath); + const persisted = loadPersistedSessionStore(storePath); expect(persisted[sessionKey]?.totalTokens).toBe(21_225); expect(persisted[sessionKey]?.totalTokensFresh).toBe(true); }); @@ -1360,7 +1297,7 @@ describe("updateSessionStoreAfterAgentRun", () => { cacheWrite: 33_047, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); await updateSessionStoreAfterAgentRun({ cfg, @@ -1418,7 +1355,7 @@ describe("updateSessionStoreAfterAgentRun", () => { totalTokensFresh: true, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); await updateSessionStoreAfterAgentRun({ cfg, @@ -1496,7 +1433,7 @@ describe("updateSessionStoreAfterAgentRun", () => { }, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); await updateSessionStoreAfterAgentRun({ cfg, @@ -1544,7 +1481,7 @@ describe("updateSessionStoreAfterAgentRun", () => { totalTokensFresh: true, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); await updateSessionStoreAfterAgentRun({ cfg, @@ -1603,7 +1540,7 @@ describe("updateSessionStoreAfterAgentRun", () => { updatedAt: 1, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); // Simulate a run with 10k input + 5k output tokens // Cost = (10000 * 10 + 5000 * 30) / 1e6 = $0.25 @@ -1653,7 +1590,7 @@ describe("updateSessionStoreAfterAgentRun", () => { // After second persist with same usage, cost should STILL be $0.25 (not $0.50) expect(sessionStore[sessionKey]?.estimatedCostUsd).toBeCloseTo(0.25, 4); - const persisted = loadSessionStore(storePath); + const persisted = loadPersistedSessionStore(storePath); expect(persisted[sessionKey]?.estimatedCostUsd).toBeCloseTo(0.25, 4); }); }); @@ -1673,7 +1610,7 @@ describe("updateSessionStoreAfterAgentRun", () => { lastInteractionAt, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); await updateSessionStoreAfterAgentRun({ cfg, @@ -1717,7 +1654,7 @@ describe("updateSessionStoreAfterAgentRun", () => { lastActivityAt, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await replaceSessionEntry({ storePath, sessionKey }, sessionStore[sessionKey]!); await updateSessionStoreAfterAgentRun({ cfg, @@ -1759,7 +1696,7 @@ describe("updateSessionStoreAfterAgentRun", () => { lastInteractionAt, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); await updateSessionStoreAfterAgentRun({ cfg, @@ -1826,7 +1763,7 @@ describe("updateSessionStoreAfterAgentRun", () => { }, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); // Heartbeat turn uses a different model const result: EmbeddedAgentRunResult = { @@ -1885,7 +1822,7 @@ describe("updateSessionStoreAfterAgentRun", () => { sessionId: "existing-cli-session", }); - const persisted = loadSessionStore(storePath); + const persisted = loadPersistedSessionStore(storePath); expect(persisted[sessionKey]?.model).toBe("claude-opus-4-6"); expect(persisted[sessionKey]?.modelProvider).toBe("anthropic"); expect(persisted[sessionKey]?.agentHarnessId).toBe("openclaw"); @@ -1933,7 +1870,7 @@ describe("updateSessionStoreAfterAgentRun", () => { compactionCount: 7, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const freshVisibleEntry: SessionEntry = { sessionId: "fresh-visible-session-id", updatedAt: 2, @@ -1956,7 +1893,7 @@ describe("updateSessionStoreAfterAgentRun", () => { }, compactionCount: 9, }; - await fs.writeFile(storePath, JSON.stringify({ [sessionKey]: freshVisibleEntry }, null, 2)); + await seedSessionStore(storePath, { [sessionKey]: freshVisibleEntry }); const result: EmbeddedAgentRunResult = { meta: { @@ -2015,10 +1952,7 @@ describe("updateSessionStoreAfterAgentRun", () => { }); }); - it.each([ - ["normal", false], - ["user-facing state preserving", true], - ])("does not recreate a missing persisted row after a %s run", async (_mode, preserve) => { + it("does not recreate a missing persisted row while preserving user-facing state", async () => { await withTempSessionStore(async ({ storePath }) => { const cfg = {} as OpenClawConfig; const sessionKey = "agent:main:explicit:missing-visible-row"; @@ -2031,7 +1965,6 @@ describe("updateSessionStoreAfterAgentRun", () => { model: "gpt-5.5", }, }; - await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf8"); await updateSessionStoreAfterAgentRun({ cfg, @@ -2041,7 +1974,7 @@ describe("updateSessionStoreAfterAgentRun", () => { sessionStore, defaultProvider: "claude-cli", defaultModel: "claude-sonnet-4-6", - preserveUserFacingSessionModelState: preserve, + preserveUserFacingSessionModelState: true, result: { meta: { durationMs: 1, @@ -2060,7 +1993,7 @@ describe("updateSessionStoreAfterAgentRun", () => { modelProvider: "openai", model: "gpt-5.5", }); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toBeUndefined(); + expect(loadPersistedSessionEntry(storePath, sessionKey)).toBeUndefined(); }); }); @@ -2070,7 +2003,6 @@ describe("updateSessionStoreAfterAgentRun", () => { const sessionKey = "agent:main:explicit:new-normal-row"; const sessionId = "new-normal-row-session"; const sessionStore: Record = {}; - await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf8"); await updateSessionStoreAfterAgentRun({ cfg, @@ -2093,12 +2025,57 @@ describe("updateSessionStoreAfterAgentRun", () => { }); expect(sessionStore[sessionKey]).toMatchObject({ sessionId }); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toMatchObject({ + expect(loadPersistedSessionEntry(storePath, sessionKey)).toMatchObject({ sessionId, }); }); }); + it("does not recreate a missing persisted row after a normal run with a preloaded entry", async () => { + await withTempSessionStore(async ({ storePath }) => { + const cfg = {} as OpenClawConfig; + const sessionKey = "agent:main:explicit:deleted-normal-row"; + const sessionId = "deleted-normal-row-session"; + const sessionStore: Record = { + [sessionKey]: { + sessionId, + updatedAt: 1, + modelProvider: "openai", + model: "gpt-5.5", + }, + }; + + await updateSessionStoreAfterAgentRun({ + cfg, + sessionId, + sessionKey, + storePath, + sessionStore, + defaultProvider: "openai", + defaultModel: "gpt-5.5", + result: { + meta: { + durationMs: 1, + agentMeta: { + sessionId, + provider: "openai", + model: "gpt-5.5", + usage: { input: 100, output: 20 }, + }, + }, + }, + }); + + expect(sessionStore[sessionKey]).toEqual({ + sessionId, + updatedAt: 1, + modelProvider: "openai", + model: "gpt-5.5", + }); + expect(loadPersistedSessionEntry(storePath, sessionKey)).toBeUndefined(); + }); + }); + it("does not overwrite a replacement persisted row after a normal run", async () => { await withTempSessionStore(async ({ storePath }) => { const cfg = {} as OpenClawConfig; @@ -2118,7 +2095,7 @@ describe("updateSessionStoreAfterAgentRun", () => { model: "claude-sonnet-4-6", }, }; - await fs.writeFile(storePath, JSON.stringify({ [sessionKey]: replacementEntry }, null, 2)); + await seedSessionStore(storePath, { [sessionKey]: replacementEntry }); await updateSessionStoreAfterAgentRun({ cfg, @@ -2140,9 +2117,7 @@ describe("updateSessionStoreAfterAgentRun", () => { }, }); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toEqual( - replacementEntry, - ); + expect(loadPersistedSessionEntry(storePath, sessionKey)).toEqual(replacementEntry); }); }); @@ -2160,7 +2135,7 @@ describe("updateSessionStoreAfterAgentRun", () => { // contextTokens intentionally missing — older session without cached context }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); // Heartbeat turn uses a different, smaller model const result: EmbeddedAgentRunResult = { @@ -2206,7 +2181,7 @@ describe("updateSessionStoreAfterAgentRun", () => { updatedAt: 1, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { @@ -2252,7 +2227,7 @@ describe("updateSessionStoreAfterAgentRun", () => { // modelProvider intentionally missing }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); // Heartbeat turn uses a different provider const result: EmbeddedAgentRunResult = { @@ -2283,7 +2258,7 @@ describe("updateSessionStoreAfterAgentRun", () => { expect(sessionStore[sessionKey]?.model).toBe("claude-opus-4-6"); expect(sessionStore[sessionKey]?.modelProvider).toBeUndefined(); - const persisted = loadSessionStore(storePath); + const persisted = loadPersistedSessionStore(storePath); expect(persisted[sessionKey]?.model).toBe("claude-opus-4-6"); expect(persisted[sessionKey]?.modelProvider).toBeUndefined(); }); @@ -2303,7 +2278,7 @@ describe("updateSessionStoreAfterAgentRun", () => { contextTokens: 1_000_000, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); const result: EmbeddedAgentRunResult = { meta: { @@ -2380,7 +2355,7 @@ describe("recordCliCompactionInStore", () => { }, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); await recordCliCompactionInStore({ provider: "codex", @@ -2390,7 +2365,7 @@ describe("recordCliCompactionInStore", () => { tokensAfter: 0, }); - const persisted = loadSessionStore(storePath); + const persisted = loadPersistedSessionStore(storePath); expect(sessionStore[sessionKey]?.compactionCount).toBe(1); expect(sessionStore[sessionKey]?.totalTokens).toBe(0); expect(sessionStore[sessionKey]?.totalTokensFresh).toBe(true); @@ -2442,7 +2417,7 @@ describe("recordCliCompactionInStore", () => { }, }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); await recordCliCompactionInStore({ provider: "codex", @@ -2451,7 +2426,7 @@ describe("recordCliCompactionInStore", () => { storePath, }); - const persisted = loadSessionStore(storePath); + const persisted = loadPersistedSessionStore(storePath); expect(sessionStore[sessionKey]?.compactionCount).toBe(1); expect(sessionStore[sessionKey]?.totalTokens).toBe(37_000); expect(sessionStore[sessionKey]?.totalTokensFresh).toBe(false); @@ -2479,7 +2454,7 @@ describe("recordCliCompactionInStore", () => { sessionFile: path.join(dir, `${sessionId}.jsonl`), }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2)); + await seedSessionStore(storePath, sessionStore); await recordCliCompactionInStore({ provider: "codex", @@ -2495,7 +2470,7 @@ describe("recordCliCompactionInStore", () => { expect(sessionStore[sessionKey]?.usageFamilyKey).toBe(sessionKey); expect(sessionStore[sessionKey]?.usageFamilySessionIds).toEqual([sessionId, nextSessionId]); - const persisted = loadSessionStore(storePath); + const persisted = loadPersistedSessionStore(storePath); expect(persisted[sessionKey]?.sessionId).toBe(nextSessionId); expect(persisted[sessionKey]?.sessionFile).toBe(nextSessionFile); }); @@ -2527,7 +2502,6 @@ describe("recordCliCompactionInStore", () => { }, }, }; - await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf8"); await recordCliCompactionInStore({ provider: "codex", @@ -2537,7 +2511,7 @@ describe("recordCliCompactionInStore", () => { tokensAfter: 42, }); - const persisted = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + const persisted = loadPersistedSessionEntry(storePath, sessionKey); expect(sessionStore[sessionKey]?.sessionId).toBe(sessionId); expect(sessionStore[sessionKey]?.modelProvider).toBe("openai"); expect(sessionStore[sessionKey]?.model).toBe("gpt-5.5"); @@ -2566,7 +2540,6 @@ describe("recordCliCompactionInStore", () => { }, }, }; - await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf8"); const result = await recordCliCompactionInStore({ provider: "codex", @@ -2578,7 +2551,7 @@ describe("recordCliCompactionInStore", () => { }); expect(result).toEqual(sessionStore[sessionKey]); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toBeUndefined(); + expect(loadPersistedSessionEntry(storePath, sessionKey)).toBeUndefined(); }); }); }); @@ -2606,7 +2579,7 @@ describe("clearCliSessionInStore", () => { claudeCliSessionId: "claude-session-1", }; const sessionStore: Record = { [sessionKey]: entry }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf8"); + await seedSessionStore(storePath, sessionStore); const cleared = await clearCliSessionInStore({ provider: "claude-cli", @@ -2624,7 +2597,7 @@ describe("clearCliSessionInStore", () => { expect(cleared?.claudeCliSessionId).toBeUndefined(); expect(sessionStore[sessionKey]).toEqual(cleared); - const persisted = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + const persisted = loadPersistedSessionEntry(storePath, sessionKey); expect(persisted?.cliSessionBindings?.["claude-cli"]).toBeUndefined(); expect(persisted?.cliSessionBindings?.["codex-cli"]).toEqual({ sessionId: "codex-session-1", @@ -2645,7 +2618,7 @@ describe("clearCliSessionInStore", () => { claudeCliSessionId: "claude-session-1", }, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf8"); + await seedSessionStore(storePath, sessionStore); const cleared = await clearCliSessionInStore({ provider: "claude-cli", @@ -2656,9 +2629,9 @@ describe("clearCliSessionInStore", () => { expect(cleared).toBeUndefined(); expect(sessionStore[existingKey]?.claudeCliSessionId).toBe("claude-session-1"); - expect( - loadSessionStore(storePath, { skipCache: true })[existingKey]?.claudeCliSessionId, - ).toBe("claude-session-1"); + expect(loadPersistedSessionEntry(storePath, existingKey)?.claudeCliSessionId).toBe( + "claude-session-1", + ); }); }); @@ -2686,7 +2659,6 @@ describe("clearCliSessionInStore", () => { claudeCliSessionId: "claude-session-1", }; const sessionStore: Record = { [sessionKey]: entry }; - await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf8"); const cleared = await clearCliSessionInStore({ provider: "claude-cli", @@ -2695,7 +2667,7 @@ describe("clearCliSessionInStore", () => { storePath, }); - const persisted = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + const persisted = loadPersistedSessionEntry(storePath, sessionKey); expect(cleared?.sessionId).toBe("openclaw-session-1"); expect(cleared?.modelProvider).toBe("anthropic"); expect(cleared?.model).toBe("claude-opus-4-6"); @@ -2727,7 +2699,6 @@ describe("clearCliSessionInStore", () => { claudeCliSessionId: "claude-session-1", }, }; - await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf8"); await clearCliSessionInStore({ provider: "claude-cli", @@ -2737,7 +2708,7 @@ describe("clearCliSessionInStore", () => { expectedSessionId: sessionId, }); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toBeUndefined(); + expect(loadPersistedSessionEntry(storePath, sessionKey)).toBeUndefined(); }); }); }); diff --git a/src/agents/command/session-store.ts b/src/agents/command/session-store.ts index feabce951e51..82837de70d00 100644 --- a/src/agents/command/session-store.ts +++ b/src/agents/command/session-store.ts @@ -112,6 +112,7 @@ export async function updateSessionStoreAfterAgentRun(params: { const preserveUserFacingRunState = params.preserveUserFacingSessionModelState === true; const preserveRuntimeModel = params.preserveRuntimeModel === true || preserveUserFacingRunState; + const hadPreExistingEntry = sessionStore[sessionKey] !== undefined; const entry = sessionStore[sessionKey] ?? { sessionId, updatedAt: now, @@ -293,13 +294,13 @@ export async function updateSessionStoreAfterAgentRun(params: { }, (currentEntry, context) => { if ( + (!context.existingEntry && hadPreExistingEntry) || (!preserveUserFacingRunState && context.existingEntry && - context.existingEntry.sessionId !== entry.sessionId) || - (!context.existingEntry && sessionStore[sessionKey]) + context.existingEntry.sessionId !== entry.sessionId) ) { - // A normal run may rotate its session id, so compare to the pre-run entry. - // Do not merge stale finalizer metadata after a delete or a competing reset. + // Normal runs may rotate session ids, but stale finalizers must not + // recreate rows that were reset/deleted while the run was active. return null; } return preserveUserFacingRunState diff --git a/src/agents/command/session.provider-owned-reset.test.ts b/src/agents/command/session.provider-owned-reset.test.ts index c5048d946927..6830dc9f337c 100644 --- a/src/agents/command/session.provider-owned-reset.test.ts +++ b/src/agents/command/session.provider-owned-reset.test.ts @@ -7,8 +7,12 @@ const hoisted = vi.hoisted(() => ({ terminalTranscriptNewer: false, })); -vi.mock("../../config/sessions/store-load.js", () => ({ - loadSessionStore: () => hoisted.store, +vi.mock("../../config/sessions/session-accessor.js", () => ({ + listSessionEntries: () => + Object.entries(hoisted.store).map(([sessionKey, entry]) => ({ + sessionKey, + entry, + })), })); vi.mock("../../config/sessions/paths.js", () => ({ diff --git a/src/agents/command/session.resolve-session-key.test.ts b/src/agents/command/session.resolve-session-key.test.ts index c7dee700b189..b3449dd91e35 100644 --- a/src/agents/command/session.resolve-session-key.test.ts +++ b/src/agents/command/session.resolve-session-key.test.ts @@ -4,14 +4,18 @@ import type { OpenClawConfig } from "../../config/config.js"; import type { SessionEntry } from "../../config/sessions/types.js"; const hoisted = vi.hoisted(() => ({ - loadSessionStoreMock: - vi.fn<(storePath: string, opts?: { clone?: boolean }) => Record>(), + listSessionEntriesMock: vi.fn< + (scope?: { storePath?: string; clone?: boolean }) => Array<{ + entry: SessionEntry; + sessionKey: string; + }> + >(), listAgentIdsMock: vi.fn<() => string[]>(), })); -vi.mock("../../config/sessions/store-load.js", () => ({ - loadSessionStore: (storePath: string, opts?: { clone?: boolean }) => - hoisted.loadSessionStoreMock(storePath, opts), +vi.mock("../../config/sessions/session-accessor.js", () => ({ + listSessionEntries: (scope?: { storePath?: string; clone?: boolean }) => + hoisted.listSessionEntriesMock(scope), })); vi.mock("../../config/sessions/paths.js", () => ({ @@ -33,9 +37,12 @@ const { resolveSessionKeyForRequest, resolveStoredSessionKeyForSessionId } = await import("./session.js"); function mockSessionStores(storesByPath: Record>): void { - // Store paths are the routing boundary here; returning the exact object lets - // tests assert whether callers borrowed or cloned the selected store. - hoisted.loadSessionStoreMock.mockImplementation((storePath) => storesByPath[storePath] ?? {}); + hoisted.listSessionEntriesMock.mockImplementation((scope) => + Object.entries(storesByPath[scope?.storePath ?? ""] ?? {}).map(([sessionKey, entry]) => ({ + sessionKey, + entry, + })), + ); } function expectResolvedRequestSession(params: { @@ -54,13 +61,13 @@ function expectResolvedRequestSession(params: { }); expect(result.sessionKey).toBe(params.sessionKey); - expect(result.sessionStore).toBe(params.sessionStore); + expect(result.sessionStore).toEqual(params.sessionStore); expect(result.storePath).toBe(params.storePath); } describe("resolveSessionKeyForRequest", () => { beforeEach(() => { - hoisted.loadSessionStoreMock.mockReset(); + hoisted.listSessionEntriesMock.mockReset(); hoisted.listAgentIdsMock.mockReset(); hoisted.listAgentIdsMock.mockReturnValue(["main", "other"]); }); @@ -112,12 +119,7 @@ describe("resolveSessionKeyForRequest", () => { "agent:embedded-agent:main": { sessionId: "other-session", updatedAt: 2 }, "agent:embedded-agent:work": { sessionId: "resume-agent-1", updatedAt: 1 }, } satisfies Record; - hoisted.loadSessionStoreMock.mockImplementation((storePath) => { - if (storePath === "/stores/embedded-agent.json") { - return embeddedAgentStore; - } - return {}; - }); + mockSessionStores({ "/stores/embedded-agent.json": embeddedAgentStore }); const result = resolveStoredSessionKeyForSessionId({ cfg: { @@ -130,9 +132,9 @@ describe("resolveSessionKeyForRequest", () => { }); expect(result.sessionKey).toBe("agent:embedded-agent:work"); - expect(result.sessionStore).toBe(embeddedAgentStore); + expect(result.sessionStore).toEqual(embeddedAgentStore); expect(result.storePath).toBe("/stores/embedded-agent.json"); - expect(hoisted.loadSessionStoreMock).toHaveBeenCalledTimes(1); + expect(hoisted.listSessionEntriesMock).toHaveBeenCalledTimes(1); }); it("borrows session stores when requested", () => { @@ -160,12 +162,18 @@ describe("resolveSessionKeyForRequest", () => { }); expect(result.sessionKey).toBe("agent:other:acp:sid"); - expect(result.sessionStore).toBe(otherStore); - expect(hoisted.loadSessionStoreMock).toHaveBeenCalledWith("/stores/main.json", { - clone: false, - }); - expect(hoisted.loadSessionStoreMock).toHaveBeenCalledWith("/stores/other.json", { - clone: false, - }); + expect(result.sessionStore).toEqual(otherStore); + expect(hoisted.listSessionEntriesMock).toHaveBeenCalledWith( + expect.objectContaining({ + storePath: "/stores/main.json", + clone: false, + }), + ); + expect(hoisted.listSessionEntriesMock).toHaveBeenCalledWith( + expect.objectContaining({ + storePath: "/stores/other.json", + clone: false, + }), + ); }); }); diff --git a/src/agents/command/session.ts b/src/agents/command/session.ts index 5bfd94218710..efef534f17d6 100644 --- a/src/agents/command/session.ts +++ b/src/agents/command/session.ts @@ -24,8 +24,8 @@ import { resolveSessionResetPolicy, } from "../../config/sessions/reset-policy.js"; import { resolveChannelResetConfig, resolveSessionResetType } from "../../config/sessions/reset.js"; +import { listSessionEntries } from "../../config/sessions/session-accessor.js"; import { resolveSessionKey } from "../../config/sessions/session-key.js"; -import { loadSessionStore } from "../../config/sessions/store-load.js"; import type { SessionEntry } from "../../config/sessions/types.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { @@ -82,6 +82,20 @@ type SessionIdMatchSet = { storeByKey: Map; }; +function loadCommandSessionStore(params: { + agentId?: string; + clone?: boolean; + storePath: string; +}): Record { + return Object.fromEntries( + listSessionEntries({ + storePath: params.storePath, + ...(params.agentId ? { agentId: params.agentId } : {}), + ...(params.clone === false ? { clone: false } : {}), + }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ); +} + /** Builds the synthetic session key used for explicit session-id runs. */ export function buildExplicitSessionIdSessionKey(params: { sessionId: string; @@ -132,10 +146,11 @@ function resolveLegacyMainStoreSessionForDefaultAgent(opts: { } return undefined; } - const legacyStore = loadSessionStore( - legacyStorePath, - opts.cloneOnWrite ? { clone: false } : undefined, - ); + const legacyStore = loadCommandSessionStore({ + agentId: DEFAULT_AGENT_ID, + storePath: legacyStorePath, + ...(opts.cloneOnWrite ? { clone: false } : {}), + }); for (const legacyKey of legacyKeys) { const legacyEntry = legacyStore[legacyKey]; if (legacyEntry) { @@ -196,7 +211,11 @@ function collectSessionIdMatchesForRequest(opts: { } const candidateStorePath = resolveStorePath(opts.cfg.session?.store, { agentId }); addMatches( - loadSessionStore(candidateStorePath, opts.clone === false ? { clone: false } : undefined), + loadCommandSessionStore({ + agentId, + storePath: candidateStorePath, + ...(opts.clone === false ? { clone: false } : {}), + }), candidateStorePath, ); } @@ -219,7 +238,10 @@ export function resolveStoredSessionKeyForSessionId(opts: { const storePath = resolveStorePath(opts.cfg.session?.store, { agentId: storeAgentId, }); - const sessionStore = loadSessionStore(storePath); + const sessionStore = loadCommandSessionStore({ + storePath, + ...(storeAgentId ? { agentId: storeAgentId } : {}), + }); if (!sessionId) { return { sessionKey: undefined, sessionStore, storePath }; } @@ -273,7 +295,11 @@ export function resolveSessionKeyForRequest(opts: { agentId: storeAgentId, }); const loadOptions = opts.clone === false ? { clone: false as const } : undefined; - const sessionStore = loadSessionStore(storePath, loadOptions); + const sessionStore = loadCommandSessionStore({ + storePath, + agentId: storeAgentId, + ...(loadOptions ? { clone: false } : {}), + }); const ctx: MsgContext | undefined = opts.to?.trim() ? { From: opts.to } : undefined; let sessionKey: string | undefined = diff --git a/src/agents/command/types.ts b/src/agents/command/types.ts index 8a47d63387d7..ae71f9e2c391 100644 --- a/src/agents/command/types.ts +++ b/src/agents/command/types.ts @@ -10,7 +10,10 @@ import type { ChannelOutboundTargetMode } from "../../channels/plugins/types.pub import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js"; import type { PluginHookChannelContext } from "../../plugins/hook-types.js"; import type { InputProvenance } from "../../sessions/input-provenance.js"; -import type { UserTurnInput } from "../../sessions/user-turn-transcript.js"; +import type { + UserTurnInput, + UserTurnTranscriptRecorder, +} from "../../sessions/user-turn-transcript.types.js"; import type { ExecElevatedDefaults } from "../bash-tools.exec-types.js"; import type { BootstrapContextRunKind } from "../bootstrap-mode.js"; import type { CliSessionBindingFacts } from "../cli-runner/types.js"; @@ -177,6 +180,8 @@ export type AgentCommandOpts = { acpTurnSource?: AcpTurnSource; /** Internal handoffs can feed the model without writing the synthetic prompt to transcript. */ suppressPromptPersistence?: boolean; + /** Gateway/channel ingress can provide a canonical user-turn persistence owner. */ + userTurnTranscriptRecorder?: UserTurnTranscriptRecorder; }; /** Restricted option surface for external ingress callsites. */ diff --git a/src/agents/conversation-capability-profile.test.ts b/src/agents/conversation-capability-profile.test.ts index 9f0ac54a649b..f28e9f34a5fc 100644 --- a/src/agents/conversation-capability-profile.test.ts +++ b/src/agents/conversation-capability-profile.test.ts @@ -2,6 +2,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { INTERNAL_MESSAGE_CHANNEL } from "../utils/message-channel.js"; import { resolveConversationCapabilityProfile } from "./conversation-capability-profile.js"; @@ -182,29 +184,23 @@ describe("resolveConversationCapabilityProfile", () => { expect(profile.policy.explicitToolOverrideAllowlist).toEqual(["pdf"]); }); - it("keeps inherited subagent grants out of explicit overrides", () => { - const storePath = path.join( - os.tmpdir(), - `openclaw-capability-profile-inherited-${Date.now()}-${Math.random().toString(16).slice(2)}.json`, - ); - fs.writeFileSync( - storePath, - JSON.stringify({ - "agent:main:subagent:limited": { - sessionId: "limited-session", - updatedAt: Date.now(), - spawnDepth: 1, - subagentRole: "orchestrator", - subagentControlScope: "children", - inheritedToolAllow: ["image_generate"], - }, - }), - ); + it("keeps inherited subagent grants out of explicit overrides", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-capability-profile-")); + const storePath = path.join(tempDir, "sessions.json"); + const sessionKey = "agent:main:subagent:limited"; + await replaceSessionEntry({ storePath, sessionKey }, { + sessionId: "limited-session", + updatedAt: Date.now(), + spawnDepth: 1, + subagentRole: "orchestrator", + subagentControlScope: "children", + inheritedToolAllow: ["image_generate"], + } as SessionEntry); try { const profile = resolveConversationCapabilityProfile({ config: { session: { store: storePath } }, - sessionKey: "agent:main:subagent:limited", + sessionKey, agentId: "main", modelProvider: "ollama", modelId: "qwen3.5:9b", @@ -213,7 +209,7 @@ describe("resolveConversationCapabilityProfile", () => { expect(profile.policy.explicitToolAllowlist).toContain("image_generate"); expect(profile.policy.explicitToolOverrideAllowlist).not.toContain("image_generate"); } finally { - fs.rmSync(storePath, { force: true }); + fs.rmSync(tempDir, { recursive: true, force: true }); } }); diff --git a/src/agents/embedded-agent-runner.compaction-safety-timeout.test.ts b/src/agents/embedded-agent-runner.compaction-safety-timeout.test.ts index 6ec5764bd44e..66675cdee5d3 100644 --- a/src/agents/embedded-agent-runner.compaction-safety-timeout.test.ts +++ b/src/agents/embedded-agent-runner.compaction-safety-timeout.test.ts @@ -182,7 +182,7 @@ describe("compactContextEngineWithSafetyTimeout", () => { type CompactFn = ContextEngine["compact"]; const baseParams: Parameters[0] = { sessionId: "session-1", - sessionFile: "/tmp/session-1.jsonl", + sessionKey: "agent:main:session-1", tokenBudget: 100_000, force: true, }; diff --git a/src/agents/embedded-agent-runner.guard.test.ts b/src/agents/embedded-agent-runner.guard.test.ts index e11205b89137..9a7175071ec8 100644 --- a/src/agents/embedded-agent-runner.guard.test.ts +++ b/src/agents/embedded-agent-runner.guard.test.ts @@ -15,6 +15,7 @@ import { attachRuntimeUserTurnTranscriptContext, createUserTurnTranscriptRecorder, } from "../sessions/user-turn-transcript.js"; +import { createTestUserTurnTranscriptTarget } from "../sessions/user-turn-transcript.test-support.js"; import { guardSessionManager } from "./session-tool-result-guard-wrapper.js"; import { sanitizeToolUseResultPairing } from "./session-transcript-repair.js"; import { makeAgentAssistantMessage } from "./test-helpers/agent-message-fixtures.js"; @@ -216,7 +217,7 @@ describe("guardSessionManager integration", () => { text: "visible group prompt", sender: { id: "user-42", name: "Ada", username: "ada42" }, }, - target: { transcriptPath: sessionFile }, + target: createTestUserTurnTranscriptTarget(), }); const preparedMessage = recorder.message; if (!preparedMessage) { diff --git a/src/agents/embedded-agent-runner/compact.hooks.harness.ts b/src/agents/embedded-agent-runner/compact.hooks.harness.ts index 567d04f8aab7..9c5270fb6c17 100644 --- a/src/agents/embedded-agent-runner/compact.hooks.harness.ts +++ b/src/agents/embedded-agent-runner/compact.hooks.harness.ts @@ -455,6 +455,7 @@ export async function loadCompactHooksHarness(): Promise<{ compactEmbeddedAgentSession: typeof import("./compact.queued.js").compactEmbeddedAgentSession; testing: typeof import("./compact.js").testing; onSessionTranscriptUpdate: typeof import("../../sessions/transcript-events.js").onSessionTranscriptUpdate; + onInternalSessionTranscriptUpdate: typeof import("../../sessions/transcript-events.js").onInternalSessionTranscriptUpdate; }> { resetCompactHooksHarnessMocks(); vi.resetModules(); @@ -901,5 +902,6 @@ export async function loadCompactHooksHarness(): Promise<{ ...compactModule, compactEmbeddedAgentSession: compactQueuedModule.compactEmbeddedAgentSession, onSessionTranscriptUpdate: transcriptEvents.onSessionTranscriptUpdate, + onInternalSessionTranscriptUpdate: transcriptEvents.onInternalSessionTranscriptUpdate, }; } diff --git a/src/agents/embedded-agent-runner/compact.hooks.test.ts b/src/agents/embedded-agent-runner/compact.hooks.test.ts index a9b599aea157..2e33cbcf0c52 100644 --- a/src/agents/embedded-agent-runner/compact.hooks.test.ts +++ b/src/agents/embedded-agent-runner/compact.hooks.test.ts @@ -50,6 +50,7 @@ let compactEmbeddedAgentSessionDirect: typeof import("./compact.js").compactEmbe let compactEmbeddedAgentSession: typeof import("./compact.queued.js").compactEmbeddedAgentSession; let compactTesting: typeof import("./compact.js").testing; let onSessionTranscriptUpdate: typeof import("../../sessions/transcript-events.js").onSessionTranscriptUpdate; +let onInternalSessionTranscriptUpdate: typeof import("../../sessions/transcript-events.js").onInternalSessionTranscriptUpdate; const TEST_SESSION_ID = "session-1"; const TEST_SESSION_KEY = "agent:main:session-1"; @@ -63,8 +64,10 @@ type SessionHookEvent = { context?: Record; }; type PostCompactionSyncParams = { + archiveFiles?: string[]; reason: string; - sessionFiles: string[]; + sessionFiles?: string[]; + sessions?: Array<{ agentId: string; sessionId: string; sessionKey?: string }>; }; type PostCompactionSync = (params?: unknown) => Promise; type Deferred = { @@ -241,6 +244,7 @@ beforeAll(async () => { compactEmbeddedAgentSession = loaded.compactEmbeddedAgentSession; compactTesting = loaded.testing; onSessionTranscriptUpdate = loaded.onSessionTranscriptUpdate; + onInternalSessionTranscriptUpdate = loaded.onInternalSessionTranscriptUpdate; }); beforeEach(() => { @@ -1442,7 +1446,7 @@ describe("compactEmbeddedAgentSessionDirect hooks", () => { }); it("emits a transcript update after successful compaction", async () => { const listener = vi.fn(); - const cleanup = onSessionTranscriptUpdate(listener); + const cleanup = onInternalSessionTranscriptUpdate(listener); try { await compactTesting.runPostCompactionSideEffects({ @@ -1495,13 +1499,24 @@ describe("compactEmbeddedAgentSessionDirect hooks", () => { expect(listener).toHaveBeenCalledTimes(1); expect(listener).toHaveBeenCalledWith({ agentId: "main", - sessionFile: "/tmp/rotated-session.jsonl", sessionKey: TEST_SESSION_KEY, + sessionId: "rotated-session", + target: { + agentId: "main", + sessionId: "rotated-session", + sessionKey: TEST_SESSION_KEY, + }, }); expect(sync).toHaveBeenCalledTimes(1); expect(sync).toHaveBeenCalledWith({ reason: "post-compaction", - sessionFiles: ["/tmp/rotated-session.jsonl"], + sessions: [ + { + agentId: "main", + sessionId: "rotated-session", + sessionKey: TEST_SESSION_KEY, + }, + ], }); expectRecordFields(mockCallArg(hookRunner.runAfterCompaction), { previousSessionId: "session-1", @@ -1607,8 +1622,8 @@ describe("compactEmbeddedAgentSessionDirect hooks", () => { settled = true; }); await expect(syncStarted.promise).resolves.toEqual({ + archiveFiles: [TEST_SESSION_FILE], reason: "post-compaction", - sessionFiles: [TEST_SESSION_FILE], }); expect(settled).toBe(false); syncRelease.resolve(undefined); @@ -1661,8 +1676,8 @@ describe("compactEmbeddedAgentSessionDirect hooks", () => { expect(sync).not.toHaveBeenCalled(); managerGate.resolve({ manager: { sync } }); await expect(syncStarted.promise).resolves.toEqual({ + archiveFiles: [TEST_SESSION_FILE], reason: "post-compaction", - sessionFiles: [TEST_SESSION_FILE], }); }); @@ -1930,7 +1945,6 @@ describe("compactEmbeddedAgentSession hooks (ownsCompaction engine)", () => { it("passes the rotated session id to engine-owned after_compaction hooks", async () => { hookRunner.hasHooks.mockReturnValue(true); const rotatedSessionId = "rotated-session"; - const rotatedSessionFile = "/tmp/rotated-session.jsonl"; contextEngineCompactMock.mockResolvedValue({ ok: true, compacted: true, @@ -1941,7 +1955,6 @@ describe("compactEmbeddedAgentSession hooks (ownsCompaction engine)", () => { tokensBefore: 120, tokensAfter: 50, sessionId: rotatedSessionId, - sessionFile: rotatedSessionFile, }, } as never); @@ -1951,7 +1964,7 @@ describe("compactEmbeddedAgentSession hooks (ownsCompaction engine)", () => { expect(result.ok).toBe(true); expectRecordFields(mockCallArg(hookRunner.runAfterCompaction), { - sessionFile: rotatedSessionFile, + sessionFile: TEST_SESSION_FILE, previousSessionId: TEST_SESSION_ID, }); expectRecordFields(mockCallArg(hookRunner.runAfterCompaction, 0, 1), { @@ -1978,12 +1991,23 @@ describe("compactEmbeddedAgentSession hooks (ownsCompaction engine)", () => { expect(listener).toHaveBeenCalledTimes(1); expect(listener).toHaveBeenCalledWith({ agentId: "main", - sessionFile: TEST_SESSION_FILE, sessionKey: TEST_SESSION_KEY, + sessionId: TEST_SESSION_ID, + target: { + agentId: "main", + sessionId: TEST_SESSION_ID, + sessionKey: TEST_SESSION_KEY, + }, }); expect(sync).toHaveBeenCalledWith({ reason: "post-compaction", - sessionFiles: [TEST_SESSION_FILE], + sessions: [ + { + agentId: "main", + sessionId: TEST_SESSION_ID, + sessionKey: TEST_SESSION_KEY, + }, + ], }); } finally { cleanup(); @@ -2550,7 +2574,6 @@ describe("compactEmbeddedAgentSession hooks (ownsCompaction engine)", () => { it("keeps owning context-engine compaction primary for legacy Codex native sessions", async () => { const successorSessionId = "engine-successor-session"; - const successorSessionFile = "/tmp/engine-successor-session.jsonl"; resolveAgentHarnessPolicyMock.mockReturnValue({ runtime: "codex", runtimeSource: "model", @@ -2565,7 +2588,6 @@ describe("compactEmbeddedAgentSession hooks (ownsCompaction engine)", () => { tokensBefore: 333, tokensAfter: 50, sessionId: successorSessionId, - sessionFile: successorSessionFile, }, } as never); maybeCompactAgentHarnessSessionMock.mockResolvedValueOnce({ @@ -2602,7 +2624,7 @@ describe("compactEmbeddedAgentSession hooks (ownsCompaction engine)", () => { expect(maybeCompactAgentHarnessSessionMock).toHaveBeenCalledWith( expect.objectContaining({ sessionId: successorSessionId, - sessionFile: successorSessionFile, + sessionFile: TEST_SESSION_FILE, trigger: "budget", }), { nativeCompactionRequest: "after_context_engine" }, @@ -2993,14 +3015,13 @@ describe("compactEmbeddedAgentSession hooks (ownsCompaction engine)", () => { } }); - it("reuses a delegated compaction successor transcript", async () => { + it("reuses a delegated compaction successor session identity", async () => { const maintain = vi.fn(async (_params?: unknown) => ({ changed: false, bytesFreed: 0, rewrittenEntries: 0, })); const delegatedSessionId = "delegated-session"; - const delegatedSessionFile = "/tmp/delegated-session.jsonl"; resolveContextEngineMock.mockResolvedValue({ info: { ownsCompaction: false }, compact: contextEngineCompactMock, @@ -3016,7 +3037,6 @@ describe("compactEmbeddedAgentSession hooks (ownsCompaction engine)", () => { tokensBefore: 120, tokensAfter: 50, sessionId: delegatedSessionId, - sessionFile: delegatedSessionFile, }, } as never); @@ -3036,10 +3056,10 @@ describe("compactEmbeddedAgentSession hooks (ownsCompaction engine)", () => { expect(result.ok).toBe(true); expect(result.result?.sessionId).toBe(delegatedSessionId); - expect(result.result?.sessionFile).toBe(delegatedSessionFile); + expect(result.result?.sessionFile).toBeUndefined(); expectRecordFields(mockCallArg(maintain), { sessionId: delegatedSessionId, - sessionFile: delegatedSessionFile, + sessionFile: TEST_SESSION_FILE, }); }); @@ -3064,7 +3084,6 @@ describe("compactEmbeddedAgentSession hooks (ownsCompaction engine)", () => { tokensBefore: 120, tokensAfter: 50, sessionId: TEST_SESSION_ID, - sessionFile: TEST_SESSION_FILE, }, } as never); const result = await compactEmbeddedAgentSession( diff --git a/src/agents/embedded-agent-runner/compact.queued.ts b/src/agents/embedded-agent-runner/compact.queued.ts index a0e8536b11dd..79646bbab845 100644 --- a/src/agents/embedded-agent-runner/compact.queued.ts +++ b/src/agents/embedded-agent-runner/compact.queued.ts @@ -1,6 +1,7 @@ /** * Queues embedded-agent session compaction onto the correct command lane. */ +import { parseSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import { OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST } from "../../context-engine/host-compat.js"; import { ensureContextEnginesInitialized } from "../../context-engine/init.js"; import { @@ -13,6 +14,7 @@ import { type ContextEngine, type ContextEngineRuntimeContext, type ContextEngineRuntimeSettings, + type ContextEngineSessionTarget, } from "../../context-engine/types.js"; import { createFileBackedCompactionCheckpointStore, @@ -37,7 +39,9 @@ import { maybeCompactAgentHarnessSession } from "../harness/compaction.js"; import { resolveAgentHarnessPolicy } from "../harness/policy.js"; import { ensureSelectedAgentHarnessPlugin } from "../harness/runtime-plugin.js"; import { isOpenAIProvider } from "../openai-routing.js"; +import { resolveAgentRunSessionTarget } from "../run-session-target.js"; import { ensureRuntimePluginsLoaded } from "../runtime-plugins.js"; +import { SessionManager } from "../sessions/index.js"; import { DEFERRED_CONTEXT_ENGINE_COMPACTION_REASON } from "./compact-reasons.js"; import type { CompactEmbeddedAgentSessionParams } from "./compact.types.js"; import { asCompactionHookRunner, runPostCompactionSideEffects } from "./compaction-hooks.js"; @@ -130,6 +134,24 @@ function shouldDeferOwningContextEngineBudgetCompaction(params: { ); } +function buildContextEngineCompactionSessionTarget( + params: CompactEmbeddedAgentSessionParams, +): ContextEngineSessionTarget { + const sqliteMarker = parseSqliteSessionFileMarker(params.sessionFile); + const agentId = params.sessionTarget?.agentId ?? params.agentId ?? sqliteMarker?.agentId; + const sessionKey = params.sessionTarget?.sessionKey ?? params.sessionKey ?? params.sessionId; + const storePath = params.sessionTarget?.storePath ?? sqliteMarker?.storePath; + return { + ...(agentId ? { agentId } : {}), + sessionId: params.sessionTarget?.sessionId ?? sqliteMarker?.sessionId ?? params.sessionId, + ...(sessionKey ? { sessionKey } : {}), + ...(storePath ? { storePath } : {}), + ...(params.sessionTarget?.threadId !== undefined + ? { threadId: params.sessionTarget.threadId } + : {}), + }; +} + async function disposeContextEngine(contextEngine: ContextEngine): Promise { try { await contextEngine.dispose?.(); @@ -153,6 +175,7 @@ async function deferOwningContextEngineBudgetCompaction(params: { contextEngine: params.contextEngine, sessionId: params.compactParams.sessionId, sessionKey: params.compactParams.sessionKey, + sessionTarget: buildContextEngineCompactionSessionTarget(params.compactParams), sessionFile: params.compactParams.sessionFile, reason: "turn", runtimeContext: params.contextEngineRuntimeContext, @@ -459,9 +482,13 @@ async function compactEmbeddedAgentSessionImpl( // Fire before_compaction / after_compaction hooks here so plugin subscribers // are notified regardless of which engine is active. const engineOwnsCompaction = contextEngine.info.ownsCompaction === true; + const isSqliteSessionTranscript = Boolean(parseSqliteSessionFileMarker(params.sessionFile)); checkpointSnapshot = engineOwnsCompaction ? await compactionCheckpointStore.captureSnapshot({ sessionFile: params.sessionFile, + ...(isSqliteSessionTranscript + ? { sessionManager: SessionManager.open(params.sessionFile) } + : {}), }) : null; const hookRunner = engineOwnsCompaction @@ -509,12 +536,16 @@ async function compactEmbeddedAgentSessionImpl( // of throwing a raw rejection at callers that only inspect result.ok. let result: Awaited>; try { + const compactionSessionTarget = buildContextEngineCompactionSessionTarget(params); result = await compactContextEngineWithSafetyTimeout( contextEngine, { sessionId: params.sessionId, - sessionKey: params.sessionKey, - sessionFile: params.sessionFile, + sessionKey: hookSessionKey, + ...(compactionSessionTarget.agentId + ? { agentId: compactionSessionTarget.agentId } + : {}), + sessionTarget: compactionSessionTarget, tokenBudget: contextTokenBudget, currentTokenCount: params.currentTokenCount, compactionTarget: params.trigger === "manual" ? "threshold" : "budget", @@ -550,16 +581,34 @@ async function compactEmbeddedAgentSessionImpl( }; } const delegatedSuccessor = resolveCompactionSuccessorTranscript(result); + const delegatedSessionTarget = result.result?.sessionTarget; const delegatedSessionId = delegatedSuccessor.sessionId; const delegatedSessionFile = delegatedSuccessor.sessionFile; const delegatedRotatedTranscript = (typeof delegatedSessionId === "string" && delegatedSessionId !== params.sessionId) || (typeof delegatedSessionFile === "string" && delegatedSessionFile !== params.sessionFile); let postCompactionSessionId = delegatedSessionId ?? params.sessionId; + // Shipped pre-sessionTarget engines report rotation via the deprecated + // sessionFile field; honor it when no typed target is present. let postCompactionSessionFile = delegatedSessionFile ?? params.sessionFile; + if (delegatedSessionTarget) { + const resolvedDelegatedTarget = await resolveAgentRunSessionTarget({ + agentId: delegatedSessionTarget.agentId ?? sessionAgentId, + config: params.config, + sessionId: delegatedSessionTarget.sessionId ?? postCompactionSessionId, + sessionKey: delegatedSessionTarget.sessionKey ?? params.sessionKey, + sessionTarget: delegatedSessionTarget, + }); + postCompactionSessionId = resolvedDelegatedTarget.sessionId; + postCompactionSessionFile = resolvedDelegatedTarget.sessionFile; + } let postCompactionLeafId: string | undefined; if (result.ok && result.compacted) { - if (shouldRotateCompactionTranscript(params.config) && !delegatedRotatedTranscript) { + if ( + shouldRotateCompactionTranscript(params.config) && + !delegatedRotatedTranscript && + !isSqliteSessionTranscript + ) { try { const rotation = await rotateTranscriptFileAfterCompaction({ sessionFile: params.sessionFile, @@ -614,6 +663,12 @@ async function compactEmbeddedAgentSessionImpl( contextEngine, sessionId: postCompactionSessionId, sessionKey: params.sessionKey, + sessionTarget: buildContextEngineCompactionSessionTarget({ + ...params, + sessionFile: postCompactionSessionFile, + sessionId: postCompactionSessionId, + sessionTarget: delegatedSessionTarget ?? params.sessionTarget, + }), sessionFile: postCompactionSessionFile, reason: "compaction", runtimeContext, @@ -625,6 +680,7 @@ async function compactEmbeddedAgentSessionImpl( await runPostCompactionSideEffects({ config: params.config, sessionKey: params.sessionKey, + sessionId: postCompactionSessionId, agentId: sessionAgentId, sessionFile: postCompactionSessionFile, }); @@ -761,8 +817,10 @@ function buildCompactionContextEngineRuntimeContext(params: { config: params.params.config, agentId: params.params.agentId, }); + const { sessionFile: _sessionFile, ...runtimeParams } = params.params; return { - ...params.params, + ...runtimeParams, + sessionTarget: buildContextEngineCompactionSessionTarget(params.params), ...buildEmbeddedCompactionRuntimeContext({ sessionKey: params.params.sessionKey, messageChannel: params.params.messageChannel, diff --git a/src/agents/embedded-agent-runner/compact.ts b/src/agents/embedded-agent-runner/compact.ts index f06e01b5df2b..7fbd317405ec 100644 --- a/src/agents/embedded-agent-runner/compact.ts +++ b/src/agents/embedded-agent-runner/compact.ts @@ -6,6 +6,7 @@ import os from "node:os"; import { isAcpRuntimeSpawnAvailable } from "../../acp/runtime/availability.js"; import type { ThinkLevel } from "../../auto-reply/thinking.js"; import { resolveAgentModelFallbackValues } from "../../config/model-input.js"; +import { parseSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { createFileBackedCompactionCheckpointStore, @@ -792,11 +793,14 @@ async function compactEmbeddedAgentSessionDirectOnce( } const effectiveCwd = sandbox?.enabled ? effectiveWorkspace : (requestedCwd ?? effectiveWorkspace); await fs.mkdir(effectiveWorkspace, { recursive: true }); - await ensureSessionHeader({ - sessionFile: params.sessionFile, - sessionId: params.sessionId, - cwd: effectiveCwd, - }); + const isSqliteSessionTranscript = Boolean(parseSqliteSessionFileMarker(params.sessionFile)); + if (!isSqliteSessionTranscript) { + await ensureSessionHeader({ + sessionFile: params.sessionFile, + sessionId: params.sessionId, + cwd: effectiveCwd, + }); + } const { sessionAgentId: effectiveSkillAgentId } = earlyAgentIds; let restoreSkillEnv: (() => void) | undefined; @@ -1270,12 +1274,14 @@ async function compactEmbeddedAgentSessionDirectOnce( }), }); try { - await repairSessionFileIfNeeded({ - sessionFile: params.sessionFile, - debug: (message) => log.debug(message), - warn: (message) => log.warn(message), - }); - await prewarmSessionFile(params.sessionFile); + if (!isSqliteSessionTranscript) { + await repairSessionFileIfNeeded({ + sessionFile: params.sessionFile, + debug: (message) => log.debug(message), + warn: (message) => log.warn(message), + }); + await prewarmSessionFile(params.sessionFile); + } const transcriptPolicy = runtimePlan.transcript.resolvePolicy(runtimePlanModelContext); const sessionManager = guardSessionManager(SessionManager.open(params.sessionFile), { agentId: sessionAgentId, @@ -1579,7 +1585,7 @@ async function compactEmbeddedAgentSessionDirectOnce( let transcriptRotationSessionManager: Parameters< typeof rotateTranscriptAfterCompaction >[0]["sessionManager"] = sessionManager; - if (params.trigger === "manual") { + if (params.trigger === "manual" && !isSqliteSessionTranscript) { try { const hardenedBoundary = await hardenManualCompactionBoundary({ sessionFile: params.sessionFile, @@ -1611,7 +1617,7 @@ async function compactEmbeddedAgentSessionDirectOnce( const messageCountAfter = session.messages.length; const compactedCount = Math.max(0, messageCountCompactionInput - messageCountAfter); let transcriptRotation: CompactionTranscriptRotation = { rotated: false }; - if (shouldRotateCompactionTranscript(params.config)) { + if (shouldRotateCompactionTranscript(params.config) && !isSqliteSessionTranscript) { try { transcriptRotation = await rotateTranscriptAfterCompaction({ sessionManager: transcriptRotationSessionManager, @@ -1636,6 +1642,7 @@ async function compactEmbeddedAgentSessionDirectOnce( await runPostCompactionSideEffects({ config: params.config, sessionKey: params.sessionKey, + sessionId: activeSessionId, agentId: sessionAgentId, sessionFile: activeSessionFile, }); diff --git a/src/agents/embedded-agent-runner/compaction-hooks.ts b/src/agents/embedded-agent-runner/compaction-hooks.ts index 602c9d269c3a..2f820a87e0a1 100644 --- a/src/agents/embedded-agent-runner/compaction-hooks.ts +++ b/src/agents/embedded-agent-runner/compaction-hooks.ts @@ -23,6 +23,7 @@ function resolvePostCompactionIndexSyncMode(config?: OpenClawConfig): "off" | "a async function runPostCompactionSessionMemorySync(params: { config?: OpenClawConfig; sessionKey?: string; + sessionId?: string; agentId?: string; sessionFile: string; }): Promise { @@ -53,9 +54,20 @@ async function runPostCompactionSessionMemorySync(params: { if (!manager?.sync) { return; } + const sessionId = params.sessionId?.trim(); await manager.sync({ reason: "post-compaction", - sessionFiles: [sessionFile], + ...(sessionId + ? { + sessions: [ + { + agentId, + sessionId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + }, + ], + } + : { archiveFiles: [sessionFile] }), }); } catch (err) { log.warn(`memory sync skipped (post-compaction): ${formatErrorMessage(err)}`); @@ -65,6 +77,7 @@ async function runPostCompactionSessionMemorySync(params: { function syncPostCompactionSessionMemory(params: { config?: OpenClawConfig; sessionKey?: string; + sessionId?: string; agentId?: string; sessionFile: string; mode: "off" | "async" | "await"; @@ -76,6 +89,7 @@ function syncPostCompactionSessionMemory(params: { const syncTask = runPostCompactionSessionMemorySync({ config: params.config, sessionKey: params.sessionKey, + sessionId: params.sessionId, agentId: params.agentId, sessionFile: params.sessionFile, }); @@ -91,6 +105,7 @@ function syncPostCompactionSessionMemory(params: { export async function runPostCompactionSideEffects(params: { config?: OpenClawConfig; sessionKey?: string; + sessionId?: string; agentId?: string; sessionFile: string; }): Promise { @@ -101,11 +116,13 @@ export async function runPostCompactionSideEffects(params: { emitSessionTranscriptUpdate({ sessionFile, sessionKey: params.sessionKey, + ...(params.sessionId ? { sessionId: params.sessionId } : {}), ...(params.agentId ? { agentId: params.agentId } : {}), }); await syncPostCompactionSessionMemory({ config: params.config, sessionKey: params.sessionKey, + sessionId: params.sessionId, agentId: params.agentId, sessionFile, mode: resolvePostCompactionIndexSyncMode(params.config), diff --git a/src/agents/embedded-agent-runner/context-engine-maintenance.test.ts b/src/agents/embedded-agent-runner/context-engine-maintenance.test.ts index 9f3df7ad7076..c22192d71d49 100644 --- a/src/agents/embedded-agent-runner/context-engine-maintenance.test.ts +++ b/src/agents/embedded-agent-runner/context-engine-maintenance.test.ts @@ -366,6 +366,12 @@ describe("runContextEngineMaintenance", () => { }); it("passes a rewrite-capable runtime context into maintain()", async () => { + const sessionTarget = { + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:session-1", + storePath: "/tmp/state/openclaw.sqlite", + }; const maintain = vi.fn(async (_params?: unknown) => ({ changed: false, bytesFreed: 0, @@ -382,6 +388,7 @@ describe("runContextEngineMaintenance", () => { }, sessionId: "session-1", sessionKey: "agent:main:session-1", + sessionTarget, sessionFile: "/tmp/session.jsonl", reason: "turn", runtimeContext: { workspaceDir: "/tmp/workspace" }, @@ -396,11 +403,15 @@ describe("runContextEngineMaintenance", () => { expectRecordFields(maintainParams, { sessionId: "session-1", sessionKey: "agent:main:session-1", + sessionTarget, sessionFile: "/tmp/session.jsonl", }); - expect( - requireRecord(maintainParams.runtimeContext, "maintain runtime context").workspaceDir, - ).toBe("/tmp/workspace"); + const maintainRuntimeContext = requireRecord( + maintainParams.runtimeContext, + "maintain runtime context", + ); + expect(maintainRuntimeContext.workspaceDir).toBe("/tmp/workspace"); + expect(maintainRuntimeContext.sessionTarget).toEqual(sessionTarget); const runtimeContext = maintainParams.runtimeContext as | { rewriteTranscriptEntries?: (request: unknown) => Promise } | undefined; diff --git a/src/agents/embedded-agent-runner/context-engine-maintenance.ts b/src/agents/embedded-agent-runner/context-engine-maintenance.ts index 616119518754..58337e53a6f6 100644 --- a/src/agents/embedded-agent-runner/context-engine-maintenance.ts +++ b/src/agents/embedded-agent-runner/context-engine-maintenance.ts @@ -10,6 +10,7 @@ import type { ContextEngineMaintenanceResult, ContextEngineRuntimeContext, ContextEngineRuntimeSettings, + ContextEngineSessionTarget, } from "../../context-engine/types.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { @@ -49,6 +50,7 @@ type DeferredTurnMaintenanceScheduleParams = { contextEngine: ContextEngine; sessionId: string; sessionKey: string; + sessionTarget?: ContextEngineSessionTarget; sessionFile: string; sessionManager?: Parameters[0]["sessionManager"]; runtimeContext?: ContextEngineRuntimeContext; @@ -277,6 +279,7 @@ function promoteTurnMaintenanceTaskVisibility(params: { export function buildContextEngineMaintenanceRuntimeContext(params: { sessionId: string; sessionKey?: string; + sessionTarget?: ContextEngineSessionTarget; sessionFile: string; sessionManager?: Parameters[0]["sessionManager"]; withSessionManagerRewriteLock?: SessionManagerRewriteLock; @@ -297,6 +300,7 @@ export function buildContextEngineMaintenanceRuntimeContext(params: { contextEnginePluginId: params.contextEnginePluginId, purpose: params.purpose ?? "context-engine.maintenance", }), + ...(params.sessionTarget ? { sessionTarget: params.sessionTarget } : {}), ...(params.allowDeferredCompactionExecution ? { allowDeferredCompactionExecution: true } : {}), rewriteTranscriptEntries: async (request) => { if (params.sessionManager) { @@ -330,6 +334,7 @@ async function executeContextEngineMaintenance(params: { contextEngine: ContextEngine; sessionId: string; sessionKey?: string; + sessionTarget?: ContextEngineSessionTarget; sessionFile: string; reason: "bootstrap" | "compaction" | "turn"; sessionManager?: Parameters[0]["sessionManager"]; @@ -346,11 +351,13 @@ async function executeContextEngineMaintenance(params: { const result = await params.contextEngine.maintain({ sessionId: params.sessionId, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, runtimeSettings: params.runtimeSettings, runtimeContext: buildContextEngineMaintenanceRuntimeContext({ sessionId: params.sessionId, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, sessionManager: params.executionMode === "background" ? undefined : params.sessionManager, withSessionManagerRewriteLock: @@ -377,6 +384,7 @@ async function runDeferredTurnMaintenanceWorker(params: { contextEngine: ContextEngine; sessionId: string; sessionKey: string; + sessionTarget?: ContextEngineSessionTarget; sessionFile: string; sessionManager?: Parameters[0]["sessionManager"]; runtimeContext?: ContextEngineRuntimeContext; @@ -432,6 +440,7 @@ async function runDeferredTurnMaintenanceWorker(params: { contextEngine: params.contextEngine, sessionId: params.sessionId, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, reason: "turn", sessionManager: params.sessionManager, @@ -579,6 +588,7 @@ function scheduleDeferredTurnMaintenance( contextEngine: params.contextEngine, sessionId: params.sessionId, sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, sessionManager: params.sessionManager, runtimeContext: params.runtimeContext, @@ -646,6 +656,7 @@ export async function runContextEngineMaintenance(params: { contextEngine?: ContextEngine; sessionId: string; sessionKey?: string; + sessionTarget?: ContextEngineSessionTarget; sessionFile: string; reason: "bootstrap" | "compaction" | "turn"; sessionManager?: Parameters[0]["sessionManager"]; @@ -675,6 +686,7 @@ export async function runContextEngineMaintenance(params: { contextEngine: params.contextEngine, sessionId: params.sessionId, sessionKey: params.sessionKey ?? params.sessionId, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, sessionManager: params.sessionManager, runtimeContext: params.runtimeContext, @@ -698,6 +710,7 @@ export async function runContextEngineMaintenance(params: { contextEngine: params.contextEngine, sessionId: params.sessionId, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, reason: params.reason, sessionManager: params.sessionManager, diff --git a/src/agents/embedded-agent-runner/effective-tool-policy.test.ts b/src/agents/embedded-agent-runner/effective-tool-policy.test.ts index 680466456ef3..236da3047854 100644 --- a/src/agents/embedded-agent-runner/effective-tool-policy.test.ts +++ b/src/agents/embedded-agent-runner/effective-tool-policy.test.ts @@ -1,8 +1,9 @@ // Coverage for final bundled-tool policy filtering in embedded runs. -import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { replaceSessionEntry } from "../../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../../config/sessions/types.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { setPluginToolMeta } from "../../plugins/tools.js"; import { @@ -48,6 +49,26 @@ function applyFinalPolicy( }); } +async function writeSessionEntries( + storePath: string, + entries: Record, +): Promise { + for (const [sessionKey, entry] of Object.entries(entries)) { + await replaceSessionEntry({ sessionKey, storePath }, entry as SessionEntry); + } +} + +function createSessionStorePath(prefix: string, agentId: string): string { + return path.join( + os.tmpdir(), + `${prefix}-${agentId}`, + "agents", + agentId, + "sessions", + "sessions.json", + ); +} + describe("applyFinalEffectiveToolPolicy", () => { it("filters bundled tools through the configured allowlist", () => { const filtered = applyFinalPolicy({ @@ -59,30 +80,22 @@ describe("applyFinalEffectiveToolPolicy", () => { expect(filtered.map((tool) => tool.name)).toEqual(["mcp__bundle__fs_read"]); }); - it("filters bundled tools through inherited subagent allowlists", () => { + it("filters bundled tools through inherited subagent allowlists", async () => { // Inherited allowlists are persisted by session key; use a real temp store // so parsing and lookup match production policy application. const agentId = `bundled-inherited-allow-${Date.now()}-${Math.random().toString(16).slice(2)}`; const sessionKey = `agent:${agentId}:subagent:limited`; - const storePath = path.join(os.tmpdir(), `openclaw-bundled-inherited-allow-${agentId}.json`); - fs.writeFileSync( - storePath, - JSON.stringify( - { - [sessionKey]: { - sessionId: "limited-session", - updatedAt: Date.now(), - spawnDepth: 1, - subagentRole: "orchestrator", - subagentControlScope: "children", - inheritedToolAllow: ["mcp__bundle__fs_read"], - }, - }, - null, - 2, - ), - "utf-8", - ); + const storePath = createSessionStorePath("openclaw-bundled-inherited-allow", agentId); + await writeSessionEntries(storePath, { + [sessionKey]: { + sessionId: "limited-session", + updatedAt: Date.now(), + spawnDepth: 1, + subagentRole: "orchestrator", + subagentControlScope: "children", + inheritedToolAllow: ["mcp__bundle__fs_read"], + }, + }); const filtered = applyFinalPolicy({ bundledTools: [makeTool("mcp__bundle__fs_delete"), makeTool("mcp__bundle__fs_read")], @@ -98,28 +111,20 @@ describe("applyFinalEffectiveToolPolicy", () => { expect(filtered.map((tool) => tool.name)).toEqual(["mcp__bundle__fs_read"]); }); - it("honors configured plugin allow entries alongside inherited bundled tool allows", () => { + it("honors configured plugin allow entries alongside inherited bundled tool allows", async () => { const agentId = `bundled-plugin-allow-${Date.now()}-${Math.random().toString(16).slice(2)}`; const sessionKey = `agent:${agentId}:subagent:limited`; - const storePath = path.join(os.tmpdir(), `openclaw-bundled-plugin-allow-${agentId}.json`); - fs.writeFileSync( - storePath, - JSON.stringify( - { - [sessionKey]: { - sessionId: "limited-session", - updatedAt: Date.now(), - spawnDepth: 1, - subagentRole: "orchestrator", - subagentControlScope: "children", - inheritedToolAllow: ["mcp__bundle__fs_read"], - }, - }, - null, - 2, - ), - "utf-8", - ); + const storePath = createSessionStorePath("openclaw-bundled-plugin-allow", agentId); + await writeSessionEntries(storePath, { + [sessionKey]: { + sessionId: "limited-session", + updatedAt: Date.now(), + spawnDepth: 1, + subagentRole: "orchestrator", + subagentControlScope: "children", + inheritedToolAllow: ["mcp__bundle__fs_read"], + }, + }); const deniedTool = makeTool("mcp__bundle__fs_delete"); const allowedTool = makeTool("mcp__bundle__fs_read"); setPluginToolMeta(deniedTool, { pluginId: "bundle-mcp", optional: false }); diff --git a/src/agents/embedded-agent-runner/run.codex-app-server-recovery.test.ts b/src/agents/embedded-agent-runner/run.codex-app-server-recovery.test.ts index 505d448f49a8..1d969ddb193a 100644 --- a/src/agents/embedded-agent-runner/run.codex-app-server-recovery.test.ts +++ b/src/agents/embedded-agent-runner/run.codex-app-server-recovery.test.ts @@ -131,6 +131,8 @@ describe("runEmbeddedAgent Codex app-server recovery", () => { const freezeAbort = vi.fn(); const replyOperation = { freezeAbort, + markDeferredMaintenanceWaitEnded: vi.fn(), + markWaitingForDeferredMaintenance: vi.fn(), } as unknown as NonNullable[0]["replyOperation"]>; mockedRunEmbeddedAttempt .mockImplementationOnce(async () => { @@ -200,6 +202,8 @@ describe("runEmbeddedAgent Codex app-server recovery", () => { const abortByUser = vi.fn(() => true); const replyOperation = { abortByUser, + markDeferredMaintenanceWaitEnded: vi.fn(), + markWaitingForDeferredMaintenance: vi.fn(), } as unknown as NonNullable[0]["replyOperation"]>; mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { asAttemptParams(attemptParams).onAttemptAbort?.(); @@ -230,6 +234,8 @@ describe("runEmbeddedAgent Codex app-server recovery", () => { const abortByUser = vi.fn(() => true); const replyOperation = { abortByUser, + markDeferredMaintenanceWaitEnded: vi.fn(), + markWaitingForDeferredMaintenance: vi.fn(), } as unknown as NonNullable[0]["replyOperation"]>; mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { controller.abort(upstreamAbort); diff --git a/src/agents/embedded-agent-runner/run.overflow-compaction.fixture.ts b/src/agents/embedded-agent-runner/run.overflow-compaction.fixture.ts index 7880583173f4..4c685614df0f 100644 --- a/src/agents/embedded-agent-runner/run.overflow-compaction.fixture.ts +++ b/src/agents/embedded-agent-runner/run.overflow-compaction.fixture.ts @@ -1,7 +1,8 @@ -import { isCoreToolNameReplaySafe } from "../tool-replay-safety.js"; /** * Test fixtures for embedded-run overflow compaction scenarios. */ +import type { ContextEngineSessionTarget } from "../../context-engine/types.js"; +import { isCoreToolNameReplaySafe } from "../tool-replay-safety.js"; import { buildAttemptReplayMetadata } from "./run/incomplete-turn.js"; import type { EmbeddedRunAttemptResult } from "./run/types.js"; @@ -19,6 +20,7 @@ export function makeCompactionSuccess(params: { tokensAfter?: number; sessionId?: string; sessionFile?: string; + sessionTarget?: ContextEngineSessionTarget; }) { return { ok: true as const, @@ -30,6 +32,7 @@ export function makeCompactionSuccess(params: { ...(params.tokensAfter !== undefined ? { tokensAfter: params.tokensAfter } : {}), ...(params.sessionId !== undefined ? { sessionId: params.sessionId } : {}), ...(params.sessionFile !== undefined ? { sessionFile: params.sessionFile } : {}), + ...(params.sessionTarget !== undefined ? { sessionTarget: params.sessionTarget } : {}), }, }; } @@ -103,6 +106,7 @@ type MockCompactDirect = { tokensAfter?: number; sessionId?: string; sessionFile?: string; + sessionTarget?: ContextEngineSessionTarget; }; }) => unknown; }; diff --git a/src/agents/embedded-agent-runner/run.overflow-compaction.harness.ts b/src/agents/embedded-agent-runner/run.overflow-compaction.harness.ts index 8e9879e05e2d..d350b6c52bc4 100644 --- a/src/agents/embedded-agent-runner/run.overflow-compaction.harness.ts +++ b/src/agents/embedded-agent-runner/run.overflow-compaction.harness.ts @@ -4,6 +4,7 @@ import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; import { type Mock, vi } from "vitest"; import type { ThinkLevel } from "../../auto-reply/thinking.js"; +import type { ContextEngineSessionTarget } from "../../context-engine/types.js"; import { formatErrorMessage } from "../../infra/errors.js"; import type { PluginHookBeforeAgentFinalizeEvent, @@ -19,6 +20,7 @@ import type { import { resetCommandQueueStateForTest } from "../../process/command-queue.js"; import type { FailoverReason } from "../embedded-agent-helpers/types.js"; import { clearAgentHarnesses, registerAgentHarness } from "../harness/registry.js"; +import type { AgentRuntimePlan } from "../runtime-plan/types.js"; import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; import type { buildEmbeddedRunPayloads } from "./run/payloads.js"; import type { EmbeddedRunAttemptResult } from "./run/types.js"; @@ -36,6 +38,7 @@ type MockCompactionResult = tokensAfter?: number; sessionId?: string; sessionFile?: string; + sessionTarget?: ContextEngineSessionTarget; }; reason?: string; } @@ -114,10 +117,28 @@ export const mockedContextEngine = { })), }; +type MockRuntimePlan = Pick & { + observability: Pick; +}; + +function makeMockRuntimePlan(): MockRuntimePlan { + return { + auth: { + authProfileProviderForAuth: "anthropic", + providerForAuth: "anthropic", + }, + observability: { + harnessId: "codex", + }, + }; +} + export const mockedCompactDirect = mockedContextEngine.compact; export const mockedResolveContextEngine = vi.fn(async () => mockedContextEngine); export const mockedResolveContextEngineOwnerPluginId = vi.fn(() => undefined); -export const mockedBuildAgentRuntimePlan = vi.fn(() => ({})); +export const mockedBuildAgentRuntimePlan = vi.fn<() => AgentRuntimePlan>( + () => makeMockRuntimePlan() as AgentRuntimePlan, +); export const mockedRunPostCompactionSideEffects = vi.fn(async () => {}); export const mockedSleepWithAbort = vi.fn( async (_ms: number, _abortSignal?: AbortSignal) => undefined, @@ -335,7 +356,7 @@ export function resetRunOverflowCompactionHarnessMocks(): void { mockedResolveContextEngine.mockReset(); mockedResolveContextEngine.mockResolvedValue(mockedContextEngine); mockedBuildAgentRuntimePlan.mockReset(); - mockedBuildAgentRuntimePlan.mockReturnValue({}); + mockedBuildAgentRuntimePlan.mockImplementation(() => makeMockRuntimePlan() as AgentRuntimePlan); mockedCompactDirect.mockReset(); mockedCompactDirect.mockResolvedValue({ ok: false, @@ -735,7 +756,9 @@ export async function loadRunOverflowCompactionHarness(): Promise<{ vi.doMock("./tool-result-truncation.js", () => ({ resolveLiveToolResultMaxChars: mockedResolveLiveToolResultMaxChars, sessionLikelyHasOversizedToolResults: mockedSessionLikelyHasOversizedToolResults, + truncateOversizedToolResultsInActiveTarget: mockedTruncateOversizedToolResultsInSession, truncateOversizedToolResultsInSession: mockedTruncateOversizedToolResultsInSession, + truncateOversizedToolResultsInRuntimeTranscript: mockedTruncateOversizedToolResultsInSession, })); vi.doMock("./context-engine-maintenance.js", () => ({ diff --git a/src/agents/embedded-agent-runner/run.overflow-compaction.loop.test.ts b/src/agents/embedded-agent-runner/run.overflow-compaction.loop.test.ts index 6caa0ea8efdf..43f4be7b4cdf 100644 --- a/src/agents/embedded-agent-runner/run.overflow-compaction.loop.test.ts +++ b/src/agents/embedded-agent-runner/run.overflow-compaction.loop.test.ts @@ -64,6 +64,15 @@ function expectRetryContinuesFromTranscript() { expect(retryParams.prompt).not.toBe(baseParams.prompt); } +function expectTruncationScopeSessionFile(callIndex: number, sessionFile: string) { + const args = requireMockCallArg(mockedTruncateOversizedToolResultsInSession, callIndex); + expect(requireRecord(args.scope, "truncation scope").sessionFile).toBe(sessionFile); +} + +function makeUserMessage(content: string = baseParams.prompt) { + return { role: "user" as const, content, timestamp: 1 }; +} + describe("overflow compaction in run loop", () => { beforeAll(async () => { ({ runEmbeddedAgent } = await loadRunOverflowCompactionHarness()); @@ -225,6 +234,298 @@ describe("overflow compaction in run loop", () => { await successPromise; }); + it("persists the canonical user turn when the embedded runtime writes its prompt file", async () => { + const persistedMessage = makeUserMessage(); + const persistApproved = vi.fn(async () => ({ + sessionFile: "/tmp/openclaw-transcript.jsonl", + sessionEntry: undefined, + messageId: "msg-user-1", + message: persistedMessage, + })); + const markRuntimePersistencePending = vi.fn(); + const markRuntimePersisted = vi.fn(); + const onUserMessagePersisted = vi.fn(); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + ( + attemptParams as { + onUserMessagePersisted?: (message: { role: "user"; content: string }) => void; + } + ).onUserMessagePersisted?.(makeUserMessage()); + return makeAttemptResult({ promptError: null }); + }); + + await runEmbeddedAgent({ + ...baseParams, + userTurnTranscriptRecorder: { + message: makeUserMessage(), + resolveMessage: vi.fn(async () => makeUserMessage()), + markRuntimePersistencePending, + markRuntimePersisted, + markBlocked: vi.fn(), + hasPersisted: vi.fn(() => false), + isBlocked: vi.fn(() => false), + hasRuntimePersistencePending: vi.fn(() => false), + waitForRuntimePersistence: vi.fn(async () => undefined), + persistApproved, + persistBlocked: vi.fn(async () => undefined), + persistFallback: vi.fn(async () => undefined), + }, + onUserMessagePersisted, + }); + + expect(persistApproved).toHaveBeenCalledOnce(); + expect(markRuntimePersistencePending).toHaveBeenCalledOnce(); + expect(markRuntimePersisted).not.toHaveBeenCalled(); + expect(onUserMessagePersisted).toHaveBeenCalledWith(persistedMessage); + }); + + it("suppresses retry persistence when the runtime already marked the user turn persisted", async () => { + const overflowError = makeOverflowError(); + const runtimeMessage = makeUserMessage(); + const persistApproved = vi.fn(async () => undefined); + const onUserMessagePersisted = vi.fn(); + mockedRunEmbeddedAttempt + .mockImplementationOnce(async (attemptParams) => { + ( + attemptParams as { + onUserMessagePersisted?: (message: typeof runtimeMessage) => void; + } + ).onUserMessagePersisted?.(runtimeMessage); + return makeAttemptResult({ promptError: overflowError }); + }) + .mockResolvedValueOnce(makeAttemptResult({ promptError: null })); + mockedCompactDirect.mockResolvedValueOnce( + makeCompactionSuccess({ + summary: "Compacted session", + firstKeptEntryId: "entry-5", + tokensBefore: 150000, + }), + ); + + await runEmbeddedAgent({ + ...baseParams, + currentMessageId: "telegram-msg-runtime-persisted", + onUserMessagePersisted, + userTurnTranscriptRecorder: { + message: runtimeMessage, + resolveMessage: vi.fn(async () => runtimeMessage), + markRuntimePersistencePending: vi.fn(), + markRuntimePersisted: vi.fn(), + markBlocked: vi.fn(), + hasPersisted: vi.fn(() => true), + isBlocked: vi.fn(() => false), + hasRuntimePersistencePending: vi.fn(() => false), + waitForRuntimePersistence: vi.fn(async () => undefined), + persistApproved, + persistBlocked: vi.fn(async () => undefined), + persistFallback: vi.fn(async () => undefined), + }, + }); + + expect(persistApproved).toHaveBeenCalledOnce(); + expect(onUserMessagePersisted).toHaveBeenCalledWith(runtimeMessage); + expectRetryContinuesFromTranscript(); + }); + + it("does not persist the original embedded prompt when before_agent_run writes a block marker", async () => { + const blockedMessage = { + ...makeUserMessage("[blocked by before_agent_run]"), + __openclaw: { + beforeAgentRunBlocked: { + blockedBy: "before_agent_run", + blockedAt: 123, + }, + }, + }; + const persistBlocked = vi.fn(async (message: unknown) => ({ + sessionFile: "/tmp/openclaw-transcript.jsonl", + sessionEntry: undefined, + messageId: "msg-user-1", + message: message as typeof blockedMessage, + })); + const persistApproved = vi.fn(async () => ({ + sessionFile: "/tmp/openclaw-transcript.jsonl", + sessionEntry: undefined, + messageId: "msg-user-1", + message: makeUserMessage(), + })); + const markBlocked = vi.fn(); + const markRuntimePersistencePending = vi.fn(); + const markRuntimePersisted = vi.fn(); + const onUserMessagePersisted = vi.fn(); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + ( + attemptParams as { + onUserMessagePersisted?: (message: typeof blockedMessage) => void; + } + ).onUserMessagePersisted?.(blockedMessage); + return makeAttemptResult({ promptError: null }); + }); + + await runEmbeddedAgent({ + ...baseParams, + onUserMessagePersisted, + userTurnTranscriptRecorder: { + message: makeUserMessage(), + resolveMessage: vi.fn(async () => makeUserMessage()), + markRuntimePersistencePending, + markRuntimePersisted, + markBlocked, + hasPersisted: vi.fn(() => false), + isBlocked: vi.fn(() => false), + hasRuntimePersistencePending: vi.fn(() => false), + waitForRuntimePersistence: vi.fn(async () => undefined), + persistApproved, + persistBlocked, + persistFallback: vi.fn(async () => undefined), + }, + }); + + expect(persistApproved).not.toHaveBeenCalled(); + expect(persistBlocked).toHaveBeenCalledWith(blockedMessage); + expect(markRuntimePersistencePending).toHaveBeenCalledOnce(); + expect(markBlocked).not.toHaveBeenCalled(); + expect(markRuntimePersisted).not.toHaveBeenCalled(); + expect(onUserMessagePersisted).toHaveBeenCalledWith(blockedMessage); + }); + + it("does not suppress retry persistence when canonical embedded user turn write is blocked", async () => { + const overflowError = makeOverflowError(); + const persistApproved = vi.fn(async () => undefined); + const onUserMessagePersisted = vi.fn(); + mockedRunEmbeddedAttempt + .mockImplementationOnce(async (attemptParams) => { + ( + attemptParams as { + onUserMessagePersisted?: (message: { role: "user"; content: string }) => void; + } + ).onUserMessagePersisted?.(makeUserMessage()); + return makeAttemptResult({ promptError: overflowError }); + }) + .mockResolvedValueOnce(makeAttemptResult({ promptError: null })); + + mockedCompactDirect.mockResolvedValueOnce( + makeCompactionSuccess({ + summary: "Compacted session", + firstKeptEntryId: "entry-5", + tokensBefore: 150000, + }), + ); + + await runEmbeddedAgent({ + ...baseParams, + currentMessageId: "telegram-msg-blocked", + onUserMessagePersisted, + userTurnTranscriptRecorder: { + message: makeUserMessage(), + resolveMessage: vi.fn(async () => makeUserMessage()), + markRuntimePersistencePending: vi.fn(), + markRuntimePersisted: vi.fn(), + markBlocked: vi.fn(), + hasPersisted: vi.fn(() => false), + isBlocked: vi.fn(() => false), + hasRuntimePersistencePending: vi.fn(() => false), + waitForRuntimePersistence: vi.fn(async () => undefined), + persistApproved, + persistBlocked: vi.fn(async () => undefined), + persistFallback: vi.fn(async () => undefined), + }, + }); + + expect(persistApproved).toHaveBeenCalledOnce(); + expect(onUserMessagePersisted).not.toHaveBeenCalled(); + const retryParams = requireMockCallArg(mockedRunEmbeddedAttempt, 1); + expect(retryParams.prompt).toBe(baseParams.prompt); + expect(retryParams.suppressNextUserMessagePersistence).toBe(false); + }); + + it("waits for pending canonical embedded user turn persistence before retry suppression", async () => { + const overflowError = makeOverflowError(); + const persistedMessage = makeUserMessage(); + let resolvePersistApproved: + | ((result: { + sessionFile: string; + sessionEntry: undefined; + messageId: string; + message: typeof persistedMessage; + }) => void) + | undefined; + let pendingPersistence: Promise | undefined; + const persistApproved = vi.fn( + () => + new Promise<{ + sessionFile: string; + sessionEntry: undefined; + messageId: string; + message: typeof persistedMessage; + }>((resolve) => { + resolvePersistApproved = resolve; + }), + ); + const onUserMessagePersisted = vi.fn(); + mockedRunEmbeddedAttempt + .mockImplementationOnce(async (attemptParams) => { + ( + attemptParams as { + onUserMessagePersisted?: (message: { role: "user"; content: string }) => void; + } + ).onUserMessagePersisted?.(makeUserMessage()); + return makeAttemptResult({ promptError: overflowError }); + }) + .mockResolvedValueOnce(makeAttemptResult({ promptError: null })); + + mockedCompactDirect.mockResolvedValueOnce( + makeCompactionSuccess({ + summary: "Compacted session", + firstKeptEntryId: "entry-5", + tokensBefore: 150000, + }), + ); + + const runPromise = runEmbeddedAgent({ + ...baseParams, + currentMessageId: "telegram-msg-delayed", + onUserMessagePersisted, + userTurnTranscriptRecorder: { + message: persistedMessage, + resolveMessage: vi.fn(async () => persistedMessage), + markRuntimePersistencePending: vi.fn((pending) => { + pendingPersistence = pending; + }), + markRuntimePersisted: vi.fn(), + markBlocked: vi.fn(), + hasPersisted: vi.fn(() => false), + isBlocked: vi.fn(() => false), + hasRuntimePersistencePending: vi.fn(() => pendingPersistence !== undefined), + waitForRuntimePersistence: vi.fn(async () => { + await pendingPersistence; + }), + persistApproved, + persistBlocked: vi.fn(async () => undefined), + persistFallback: vi.fn(async () => undefined), + }, + }); + + await vi.waitFor(() => { + expect(persistApproved).toHaveBeenCalledOnce(); + }); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(mockedCompactDirect).not.toHaveBeenCalled(); + + resolvePersistApproved?.({ + sessionFile: "/tmp/openclaw-transcript.jsonl", + sessionEntry: undefined, + messageId: "msg-user-delayed", + message: persistedMessage, + }); + await runPromise; + + expect(mockedCompactDirect).toHaveBeenCalledOnce(); + expect(onUserMessagePersisted).toHaveBeenCalledWith(persistedMessage); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expectRetryContinuesFromTranscript(); + }); + it("continues from transcript after compaction when the current inbound message was persisted", async () => { const overflowError = makeOverflowError(); @@ -234,7 +535,7 @@ describe("overflow compaction in run loop", () => { attemptParams as { onUserMessagePersisted?: (message: { role: "user"; content: string }) => void; } - ).onUserMessagePersisted?.({ role: "user", content: baseParams.prompt }); + ).onUserMessagePersisted?.(makeUserMessage()); return makeAttemptResult({ promptError: overflowError }); }) .mockResolvedValueOnce(makeAttemptResult({ promptError: null })); @@ -360,9 +661,7 @@ describe("overflow compaction in run loop", () => { expect( requireMockCallArg(mockedSessionLikelyHasOversizedToolResults, 0).contextWindowTokens, ).toBe(200000); - expect(requireMockCallArg(mockedTruncateOversizedToolResultsInSession, 0).sessionFile).toBe( - "/tmp/session.json", - ); + expectTruncationScopeSessionFile(0, "/tmp/session.json"); expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); expectLogIncludes(mockedLog.info, "Truncated 1 tool result(s)"); expect(result.meta.error).toBeUndefined(); @@ -408,9 +707,7 @@ describe("overflow compaction in run loop", () => { const oversizedArgs = requireMockCallArg(mockedSessionLikelyHasOversizedToolResults, 0); const messages = oversizedArgs.messages as Array<{ role?: string }>; expect(messages.filter((message) => message.role === "toolResult")).toHaveLength(3); - expect(requireMockCallArg(mockedTruncateOversizedToolResultsInSession, 0).sessionFile).toBe( - "/tmp/session.json", - ); + expectTruncationScopeSessionFile(0, "/tmp/session.json"); expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); expectLogIncludes(mockedLog.info, "Truncated 2 tool result(s)"); expect(result.meta.error).toBeUndefined(); @@ -551,9 +848,7 @@ describe("overflow compaction in run loop", () => { const result = await runEmbeddedAgent(baseParams); expect(mockedCompactDirect).toHaveBeenCalledTimes(1); - expect(requireMockCallArg(mockedTruncateOversizedToolResultsInSession, 0).sessionFile).toBe( - "/tmp/session.json", - ); + expectTruncationScopeSessionFile(0, "/tmp/session.json"); expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); expectLogIncludes(mockedLog.info, "post-compaction tool-result truncation succeeded"); expect(result.meta.error).toBeUndefined(); diff --git a/src/agents/embedded-agent-runner/run.overflow-compaction.test.ts b/src/agents/embedded-agent-runner/run.overflow-compaction.test.ts index d947a8596bf3..41dd24424dae 100644 --- a/src/agents/embedded-agent-runner/run.overflow-compaction.test.ts +++ b/src/agents/embedded-agent-runner/run.overflow-compaction.test.ts @@ -4,6 +4,11 @@ import os from "node:os"; import path from "node:path"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { createReplyOperation } from "../../auto-reply/reply/reply-run-registry.js"; +import { + loadSessionEntry, + replaceSessionEntry, + rollbackAgentHarnessSessionEntryLifecycle, +} from "../../config/sessions/session-accessor.js"; import { claimAgentRunContext, getAgentEventLifecycleGeneration, @@ -1052,18 +1057,13 @@ describe("runEmbeddedAgent overflow compaction trigger routing", () => { const storePath = path.join(dir, "sessions.json"); const sessionId = "native-session"; const sessionKey = "agent:main:harness:codex:supervision:native-thread"; - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - agentHarnessId: "codex", - modelSelectionLocked: true, - sessionId, - updatedAt: Date.now(), - }, - }), - "utf8", - ); + const initialEntry = { + agentHarnessId: "codex", + modelSelectionLocked: true, + sessionId, + updatedAt: Date.now(), + }; + await replaceSessionEntry({ sessionKey, storePath }, initialEntry); let enqueueCount = 0; let runQueuedTask: (() => void) | undefined; @@ -1081,6 +1081,7 @@ describe("runEmbeddedAgent overflow compaction trigger routing", () => { runId: "queued-harness-admission", sessionId, sessionKey, + sessionTarget: { agentId: "main", sessionId, sessionKey, storePath }, enqueue: async (task) => { enqueueCount += 1; if (enqueueCount === 1) { @@ -1095,7 +1096,13 @@ describe("runEmbeddedAgent overflow compaction trigger routing", () => { }); await vi.waitFor(() => expect(runQueuedTask).toBeTypeOf("function")); - await fs.writeFile(storePath, "{}", "utf8"); + await rollbackAgentHarnessSessionEntryLifecycle({ + agentId: "main", + archiveTranscript: false, + expectedEntry: initialEntry, + storePath, + target: { canonicalKey: sessionKey, storeKeys: [sessionKey] }, + }); runQueuedTask?.(); await expect(runPromise).rejects.toThrow(AGENT_HARNESS_SESSION_KEY_RESERVED_MESSAGE); @@ -2447,7 +2454,10 @@ describe("runEmbeddedAgent overflow compaction trigger routing", () => { expect(mockedCompactDirect).toHaveBeenCalledTimes(1); const compactParams = expectMockCallFields(mockedCompactDirect, { sessionId: "test-session", - sessionFile: "/tmp/session.json", + sessionTarget: expect.objectContaining({ + sessionId: "test-session", + sessionKey: "test-key", + }), }); expectRecordFields(compactParams.runtimeContext, { trigger: "overflow", @@ -2534,40 +2544,37 @@ describe("runEmbeddedAgent overflow compaction trigger routing", () => { it("recovers preflight compaction when stale tokens point at an empty transcript", async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-empty-preflight-")); const storePath = path.join(dir, "sessions.json"); - await fs.writeFile( - storePath, - JSON.stringify({ - "test-key": { - sessionId: "test-session", + await replaceSessionEntry( + { sessionKey: "test-key", storePath }, + { + sessionId: "test-session", + updatedAt: 1, + totalTokens: 1_500_000, + totalTokensFresh: true, + inputTokens: 20, + outputTokens: 10_855, + cacheRead: 1_761_324, + cacheWrite: 33_047, + contextBudgetStatus: { + schemaVersion: 1, + source: "pre-prompt-estimate", updatedAt: 1, - totalTokens: 1_500_000, - totalTokensFresh: true, - inputTokens: 20, - outputTokens: 10_855, - cacheRead: 1_761_324, - cacheWrite: 33_047, - contextBudgetStatus: { - schemaVersion: 1, - source: "pre-prompt-estimate", - updatedAt: 1, - provider: "claude-cli", - model: "claude-opus-4-7", - route: "compact_only", - shouldCompact: true, - estimatedPromptTokens: 1_794_391, - contextTokenBudget: 1_048_576, - promptBudgetBeforeReserve: 1_044_480, - reserveTokens: 4_096, - effectiveReserveTokens: 4_096, - remainingPromptBudgetTokens: 0, - overflowTokens: 749_911, - toolResultReducibleChars: 0, - messageCount: 0, - unwindowedMessageCount: 0, - }, + provider: "claude-cli", + model: "claude-opus-4-7", + route: "compact_only", + shouldCompact: true, + estimatedPromptTokens: 1_794_391, + contextTokenBudget: 1_048_576, + promptBudgetBeforeReserve: 1_044_480, + reserveTokens: 4_096, + effectiveReserveTokens: 4_096, + remainingPromptBudgetTokens: 0, + overflowTokens: 749_911, + toolResultReducibleChars: 0, + messageCount: 0, + unwindowedMessageCount: 0, }, - }), - "utf8", + }, ); mockedRunEmbeddedAttempt @@ -2620,14 +2627,14 @@ describe("runEmbeddedAgent overflow compaction trigger routing", () => { expect(result.meta.error).toBeUndefined(); expect(result.meta.agentMeta?.compactionTokensAfter).toBeUndefined(); expect(result.meta.agentMeta?.contextBudgetStatus).toBeUndefined(); - const stored = JSON.parse(await fs.readFile(storePath, "utf8"))["test-key"]; - expect(stored.totalTokens).toBe(0); - expect(stored.totalTokensFresh).toBe(true); - expect(stored.inputTokens).toBeUndefined(); - expect(stored.outputTokens).toBeUndefined(); - expect(stored.cacheRead).toBeUndefined(); - expect(stored.cacheWrite).toBeUndefined(); - expect(stored.contextBudgetStatus).toBeUndefined(); + const stored = loadSessionEntry({ sessionKey: "test-key", storePath }); + expect(stored?.totalTokens).toBe(0); + expect(stored?.totalTokensFresh).toBe(true); + expect(stored?.inputTokens).toBeUndefined(); + expect(stored?.outputTokens).toBeUndefined(); + expect(stored?.cacheRead).toBeUndefined(); + expect(stored?.cacheWrite).toBeUndefined(); + expect(stored?.contextBudgetStatus).toBeUndefined(); } finally { await fs.rm(dir, { recursive: true, force: true }); } @@ -2939,6 +2946,47 @@ describe("runEmbeddedAgent overflow compaction trigger routing", () => { } }); + it("uses the top-level successor id when resolving a partial compaction session target", async () => { + const rotatedStorePath = "/tmp/rotated-sessions.sqlite"; + mockedRunEmbeddedAttempt + .mockResolvedValueOnce(makeAttemptResult({ promptError: makeOverflowError() })) + .mockResolvedValueOnce( + makeAttemptResult({ + promptError: null, + sessionIdUsed: "rotated-session", + sessionFileUsed: `sqlite:main:rotated-session:${rotatedStorePath}`, + }), + ); + mockedCompactDirect.mockResolvedValueOnce({ + ok: true, + compacted: true, + result: { + summary: "rotated overflow compaction", + tokensAfter: 50, + sessionId: "rotated-session", + sessionTarget: { + sessionKey: "test-key", + storePath: rotatedStorePath, + }, + }, + }); + + await runEmbeddedAgent(overflowBaseRunParams); + + expectMockCallFields( + mockedRunEmbeddedAttempt, + { + sessionId: "rotated-session", + sessionFile: `sqlite:main:rotated-session:${rotatedStorePath}`, + }, + 1, + ); + expectMockCallFields(mockedRunContextEngineMaintenance, { + sessionId: "rotated-session", + sessionFile: `sqlite:main:rotated-session:${rotatedStorePath}`, + }); + }); + it("does not let an old execution rotate a newer same-id run context", async () => { resetAgentRunContextForTest(); const currentLifecycleGeneration = rotateAgentEventLifecycleGeneration(); diff --git a/src/agents/embedded-agent-runner/run.timeout-triggered-compaction.test.ts b/src/agents/embedded-agent-runner/run.timeout-triggered-compaction.test.ts index b0f8d49af9e1..1bc9be273362 100644 --- a/src/agents/embedded-agent-runner/run.timeout-triggered-compaction.test.ts +++ b/src/agents/embedded-agent-runner/run.timeout-triggered-compaction.test.ts @@ -167,7 +167,7 @@ describe("timeout-triggered compaction", () => { expect(mockedCompactDirect).toHaveBeenCalledTimes(1); const compactParams = compactCallAt(0); expect(compactParams.sessionId).toBe("test-session"); - expect(compactParams.sessionFile).toBe("/tmp/session.json"); + expect(compactParams.sessionFile).toBeUndefined(); expect(compactParams.tokenBudget).toBe(200000); expect(compactParams.force).toBe(true); expect(compactParams.compactionTarget).toBe("budget"); diff --git a/src/agents/embedded-agent-runner/run.ts b/src/agents/embedded-agent-runner/run.ts index 3af69a9c965b..5a7d50382c09 100644 --- a/src/agents/embedded-agent-runner/run.ts +++ b/src/agents/embedded-agent-runner/run.ts @@ -15,7 +15,12 @@ import type { ThinkLevel } from "../../auto-reply/thinking.js"; import { SILENT_REPLY_TOKEN } from "../../auto-reply/tokens.js"; import { getRuntimeConfigSnapshot } from "../../config/config.js"; import { resolveStorePath } from "../../config/sessions.js"; -import { loadSessionEntry, updateSessionEntry } from "../../config/sessions/session-accessor.js"; +import { + loadSessionEntry, + resolveSessionTranscriptRuntimeReadTarget, + updateSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import { parseSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import { OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST } from "../../context-engine/host-compat.js"; import { ensureContextEnginesInitialized } from "../../context-engine/init.js"; import { @@ -23,7 +28,10 @@ import { resolveContextEngineOwnerPluginId, } from "../../context-engine/registry.js"; import { buildContextEngineRuntimeSettings } from "../../context-engine/runtime-settings.js"; -import { resolveCompactionSuccessorTranscript } from "../../context-engine/types.js"; +import { + type ContextEngineSessionTarget, + resolveCompactionSuccessorTranscript, +} from "../../context-engine/types.js"; import { assertAgentRunLifecycleGenerationCurrent, captureAgentRunLifecycleGeneration, @@ -46,6 +54,7 @@ import type { CommandQueueEnqueueOptions } from "../../process/command-queue.typ import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js"; import { looksLikeSecretSentinel, resolveSecretSentinel } from "../../secrets/sentinel.js"; import { createAgentHarnessTaskRuntimeScope } from "../../tasks/agent-harness-task-runtime-scope.js"; +import { createTrajectoryRuntimeRecorder } from "../../trajectory/runtime.js"; import { resolveUserPath } from "../../utils.js"; import { isMarkdownCapableMessageChannel } from "../../utils/message-channel.js"; import { @@ -148,6 +157,7 @@ import { import { createAgentRunDirectAbortError } from "../run-termination.js"; import { buildAgentRuntimeAuthPlan } from "../runtime-plan/auth.js"; import { buildAgentRuntimePlan } from "../runtime-plan/build.js"; +import type { AgentRuntimePlan } from "../runtime-plan/types.js"; import { ensureRuntimePluginsLoaded } from "../runtime-plugins.js"; import { resolveSessionSuspensionReason, @@ -261,7 +271,7 @@ import type { EmbeddedRunFastModeParam } from "./run/types.js"; import { resolveLiveToolResultMaxChars, sessionLikelyHasOversizedToolResults, - truncateOversizedToolResultsInSession, + truncateOversizedToolResultsInActiveTarget, } from "./tool-result-truncation.js"; import type { EmbeddedAgentMeta, @@ -274,6 +284,9 @@ import { mapThinkingLevelForProvider } from "./utils.js"; type ApiKeyInfo = ResolvedProviderAuth; +const CODEX_HARNESS_ID = "codex"; +const OPENAI_RESPONSES_API = "openai-responses"; +const OPENAI_CODEX_RESPONSES_API = "openai-chatgpt-responses"; const MAX_SAME_MODEL_IDLE_TIMEOUT_RETRIES = 1; const EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS = 30_000; const EMBEDDED_RUN_LANE_HEARTBEAT_MS = EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS / 2; @@ -298,6 +311,68 @@ type RunEmbeddedAgentParamsWithSessionFile = RunEmbeddedAgentInternalParams & { sessionFile: string; }; +function normalizeRuntimeId(value: string | undefined): string { + return value?.trim().toLowerCase() ?? ""; +} + +function resolveAttemptTrajectoryAttribution(params: { + model: { api?: string; provider?: string }; + modelId: string; + provider: string; + runtimePlan: { + auth?: Pick; + observability?: Pick; + }; +}): { modelApi?: string; modelId: string; provider: string } { + const authProfileProvider = normalizeRuntimeId( + params.runtimePlan.auth?.authProfileProviderForAuth, + ); + const harnessId = normalizeRuntimeId(params.runtimePlan.observability?.harnessId); + if ( + harnessId === CODEX_HARNESS_ID && + authProfileProvider !== OPENAI_PROVIDER_ID && + normalizeRuntimeId(params.model.provider) === OPENAI_PROVIDER_ID && + normalizeRuntimeId(params.model.api) === OPENAI_RESPONSES_API + ) { + return { + modelApi: OPENAI_CODEX_RESPONSES_API, + modelId: params.modelId, + provider: OPENAI_PROVIDER_ID, + }; + } + return { + ...(params.model.api ? { modelApi: params.model.api } : {}), + modelId: params.modelId, + provider: params.provider, + }; +} + +function buildContextEngineCompactionSessionTarget(params: { + agentId: string; + config?: RunEmbeddedAgentParams["config"]; + sessionFile: string; + sessionId: string; + sessionKey?: string; + sessionTarget?: RunEmbeddedAgentParams["sessionTarget"]; +}): ContextEngineSessionTarget { + const sqliteMarker = parseSqliteSessionFileMarker(params.sessionFile); + const agentId = params.sessionTarget?.agentId ?? sqliteMarker?.agentId ?? params.agentId; + const sessionKey = params.sessionTarget?.sessionKey ?? params.sessionKey ?? params.sessionId; + const storePath = + params.sessionTarget?.storePath ?? + sqliteMarker?.storePath ?? + resolveStorePath(params.config?.session?.store, { agentId }); + return { + agentId, + sessionId: params.sessionTarget?.sessionId ?? sqliteMarker?.sessionId ?? params.sessionId, + ...(sessionKey ? { sessionKey } : {}), + ...(storePath ? { storePath } : {}), + ...(params.sessionTarget?.threadId !== undefined + ? { threadId: params.sessionTarget.threadId } + : {}), + }; +} + function isNoRealConversationCompactionNoop(params: { ok?: boolean; compacted?: boolean; @@ -874,7 +949,12 @@ async function runEmbeddedAgentInternal( // Same-session reads below must see any prior deferred transcript rewrite. // Checkpoint before the global lane so unrelated sessions can still start // while this session waits on its own maintenance lane. - await waitForDeferredTurnMaintenanceForSession(params.sessionKey); + params.replyOperation?.markWaitingForDeferredMaintenance(); + try { + await waitForDeferredTurnMaintenanceForSession(params.sessionKey); + } finally { + params.replyOperation?.markDeferredMaintenanceWaitEnded(); + } throwIfAborted(); return enqueueGlobal(async () => { throwIfAborted(); @@ -1075,7 +1155,8 @@ async function runEmbeddedAgentInternal( sessionKey: normalizedSessionKey, modelFallbacksOverride: params.modelFallbacksOverride, }); - const resolvedSessionKey = normalizedSessionKey; + const resolvedSessionKey = + normalizedSessionKey ?? params.sessionTarget?.sessionKey ?? params.sessionId; const hookRunner = getGlobalHookRunner(); const hookCtx = { runId: params.runId, @@ -1796,6 +1877,15 @@ async function runEmbeddedAgentInternal( const rateLimitProfileRotationLimit = resolveRateLimitProfileRotationLimit(params.config); let activeSessionId = params.sessionId; let activeSessionFile = params.sessionFile; + let activeSessionTarget: ContextEngineSessionTarget | undefined = + buildContextEngineCompactionSessionTarget({ + agentId: params.agentId ?? sessionAgentId, + config: params.config, + sessionFile: activeSessionFile, + sessionId: activeSessionId, + sessionKey: resolvedSessionKey, + sessionTarget: params.sessionTarget, + }); const adoptActiveSessionId = (nextSessionId: string | undefined) => { if (!nextSessionId || nextSessionId === activeSessionId) { return; @@ -1810,6 +1900,23 @@ async function runEmbeddedAgentInternal( lifecycleGeneration, }); }; + const adoptActiveSessionTarget = async ( + nextSessionTarget: ContextEngineSessionTarget | undefined, + ) => { + if (!nextSessionTarget) { + return; + } + const resolvedTarget = await resolveAgentRunSessionTarget({ + agentId: nextSessionTarget.agentId ?? sessionAgentId, + config: params.config, + sessionId: nextSessionTarget.sessionId ?? activeSessionId, + sessionKey: nextSessionTarget.sessionKey ?? resolvedSessionKey, + sessionTarget: nextSessionTarget, + }); + activeSessionTarget = nextSessionTarget; + activeSessionFile = resolvedTarget.sessionFile; + adoptActiveSessionId(resolvedTarget.sessionId); + }; let suppressNextUserMessagePersistence = params.suppressNextUserMessagePersistence ?? false; // The embedded agent owns JSONL persistence; this marker lets the outer retry avoid // replaying the same inbound channel message after overflow compaction. @@ -1817,16 +1924,61 @@ async function runEmbeddedAgentInternal( const onUserMessagePersisted: RunEmbeddedAgentParams["onUserMessagePersisted"] = ( message, ) => { - if (params.currentMessageId !== undefined) { - lastPersistedCurrentMessageId = params.currentMessageId; + const messageMetadata = message as { + __openclaw?: { beforeAgentRunBlocked?: unknown }; + }; + const blockedBeforeAgentRun = messageMetadata["__openclaw"]?.beforeAgentRunBlocked; + const markCurrentUserMessagePersisted = () => { + if (params.currentMessageId !== undefined) { + lastPersistedCurrentMessageId = params.currentMessageId; + } + params.onUserMessagePersisted?.(message); + }; + const recorder = params.userTurnTranscriptRecorder; + if (!recorder) { + markCurrentUserMessagePersisted(); + return; } - params.userTurnTranscriptRecorder?.markRuntimePersisted(message); - params.onUserMessagePersisted?.(message); + const markWhenPersisted = (persisted: { message?: unknown } | undefined) => { + if (persisted?.message || recorder.hasPersisted()) { + markCurrentUserMessagePersisted(); + } + }; + if (blockedBeforeAgentRun !== undefined) { + const canonicalPersistence = recorder + .persistBlocked(message) + .then(markWhenPersisted) + .catch((persistError: unknown) => { + log.warn( + `failed to persist canonical blocked embedded user turn transcript: ${formatErrorMessage( + persistError, + )}`, + ); + }); + recorder.markRuntimePersistencePending(canonicalPersistence); + return; + } + const canonicalPersistence = recorder + .persistApproved() + .then(markWhenPersisted) + .catch((persistError: unknown) => { + log.warn( + `failed to persist canonical embedded user turn transcript: ${formatErrorMessage( + persistError, + )}`, + ); + }); + recorder.markRuntimePersistencePending(canonicalPersistence); }; const continueFromCurrentTranscript = () => { nextAttemptPromptOverride = MID_TURN_PRECHECK_CONTINUATION_PROMPT; suppressNextUserMessagePersistence = true; }; + const waitForCurrentUserMessagePersistence = async () => { + if (params.userTurnTranscriptRecorder?.hasRuntimePersistencePending() === true) { + await params.userTurnTranscriptRecorder.waitForRuntimePersistence(); + } + }; const maybeEscalateRateLimitProfileFallback = (paramsLocal: { failoverProvider: string; failoverModel: string; @@ -1997,15 +2149,28 @@ async function runEmbeddedAgentInternal( ...hookCtx, sessionId: activeSessionId, }); - const adoptCompactionTranscript = ( + const adoptCompactionTranscript = async ( compactResult: Awaited>, - ): string | undefined => { + ): Promise => { const previousSessionId = activeSessionId; + const nextSessionTarget = compactResult.result?.sessionTarget; const successor = resolveCompactionSuccessorTranscript(compactResult); - adoptActiveSessionId(successor.sessionId); - if (successor.sessionFile && successor.sessionFile !== activeSessionFile) { + await adoptActiveSessionTarget( + nextSessionTarget && successor.sessionId + ? { + ...nextSessionTarget, + sessionId: nextSessionTarget.sessionId ?? successor.sessionId, + } + : nextSessionTarget, + ); + if ( + !nextSessionTarget && + successor.sessionFile && + successor.sessionFile !== activeSessionFile + ) { activeSessionFile = successor.sessionFile; } + adoptActiveSessionId(successor.sessionId); return successor.sessionId && successor.sessionId !== previousSessionId ? previousSessionId : undefined; @@ -2150,6 +2315,18 @@ async function runEmbeddedAgentInternal( runtimeAuthState, }); const attemptFastMode = resolveAttemptFastModeParam(); + const trajectorySessionFile = resolvedSessionKey + ? ( + await resolveSessionTranscriptRuntimeReadTarget({ + agentId: workspaceResolution.agentId, + sessionId: activeSessionId, + sessionKey: resolvedSessionKey, + storePath: resolveStorePath(params.config?.session?.store, { + agentId: workspaceResolution.agentId, + }), + }) + ).sessionFile + : activeSessionFile; if (!startupStagesEmitted) { startupStages.mark(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.prompt); } @@ -2182,6 +2359,27 @@ async function runEmbeddedAgentInternal( fastMode: attemptFastMode, }, }); + const trajectoryAttribution = resolveAttemptTrajectoryAttribution({ + model: effectiveModel, + modelId, + provider, + runtimePlan, + }); + const hostTrajectoryRecorder = + agentHarness.id === CODEX_HARNESS_ID + ? createTrajectoryRuntimeRecorder({ + cfg: params.config, + env: process.env, + runId: params.runId, + sessionId: activeSessionId, + sessionKey: resolvedSessionKey, + sessionFile: trajectorySessionFile, + provider: trajectoryAttribution.provider, + modelId: trajectoryAttribution.modelId, + modelApi: trajectoryAttribution.modelApi, + workspaceDir: resolvedWorkspace, + }) + : undefined; if (!startupStagesEmitted) { startupStages.mark(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.runtimePlan); startupStages.mark(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.dispatch); @@ -2287,6 +2485,9 @@ async function runEmbeddedAgentInternal( replyToMode: params.replyToMode, hasRepliedRef: params.hasRepliedRef, sessionFile: activeSessionFile, + sessionTarget: activeSessionTarget, + trajectorySessionFile, + trajectoryRecorder: hostTrajectoryRecorder, workspaceDir: resolvedWorkspace, cwd: params.cwd, agentDir, @@ -2465,6 +2666,7 @@ async function runEmbeddedAgentInternal( throw postCompactionAbortError; } const attempt = normalizeEmbeddedRunAttemptResult(rawAttempt); + await waitForCurrentUserMessagePersistence(); if (attemptCancellationRequested) { throwIfAborted(); throw createAgentRunDirectAbortError(); @@ -2491,10 +2693,26 @@ async function runEmbeddedAgentInternal( }; const timedOutDuringToolExecution = attempt.timedOutDuringToolExecution ?? false; const timedOutByRunBudget = attempt.timedOutByRunBudget ?? false; + const previousActiveSessionId = activeSessionId; + const previousActiveSessionFile = activeSessionFile; adoptActiveSessionId(sessionIdUsed); if (sessionFileUsed && sessionFileUsed !== activeSessionFile) { activeSessionFile = sessionFileUsed; } + if ( + (sessionIdUsed && sessionIdUsed !== previousActiveSessionId) || + (sessionFileUsed && sessionFileUsed !== previousActiveSessionFile) + ) { + const activeSqliteMarker = parseSqliteSessionFileMarker(activeSessionFile); + activeSessionTarget = activeSqliteMarker + ? { + agentId: activeSqliteMarker.agentId, + sessionId: activeSqliteMarker.sessionId, + sessionKey: resolvedSessionKey, + storePath: activeSqliteMarker.storePath, + } + : undefined; + } bootstrapPromptWarningSignaturesSeen = attempt.bootstrapPromptWarningSignaturesSeen ?? (attempt.bootstrapPromptWarningSignature @@ -2761,8 +2979,16 @@ async function runEmbeddedAgentInternal( contextEngine, { sessionId: activeSessionId, - sessionKey: params.sessionKey, - sessionFile: activeSessionFile, + sessionKey: resolvedSessionKey, + agentId: sessionAgentId, + sessionTarget: buildContextEngineCompactionSessionTarget({ + agentId: sessionAgentId, + config: params.config, + sessionFile: activeSessionFile, + sessionId: activeSessionId, + sessionKey: resolvedSessionKey, + sessionTarget: activeSessionTarget, + }), tokenBudget: contextTokenBudget, force: true, compactionTarget: "budget", @@ -2785,7 +3011,7 @@ async function runEmbeddedAgentInternal( }; } const previousSessionId = timeoutCompactResult.compacted - ? adoptCompactionTranscript(timeoutCompactResult) + ? await adoptCompactionTranscript(timeoutCompactResult) : undefined; await runOwnsCompactionAfterHook( "timeout recovery", @@ -2805,6 +3031,7 @@ async function runEmbeddedAgentInternal( await runPostCompactionSideEffects({ config: params.config, sessionKey: params.sessionKey, + sessionId: activeSessionId, agentId: sessionAgentId, sessionFile: activeSessionFile, }); @@ -2980,8 +3207,16 @@ async function runEmbeddedAgentInternal( contextEngine, { sessionId: activeSessionId, - sessionKey: params.sessionKey, - sessionFile: activeSessionFile, + sessionKey: resolvedSessionKey, + agentId: sessionAgentId, + sessionTarget: buildContextEngineCompactionSessionTarget({ + agentId: sessionAgentId, + config: params.config, + sessionFile: activeSessionFile, + sessionId: activeSessionId, + sessionKey: resolvedSessionKey, + sessionTarget: activeSessionTarget, + }), tokenBudget: contextTokenBudget, ...(overflowTokenCountForCompaction !== undefined ? { currentTokenCount: overflowTokenCountForCompaction } @@ -2995,11 +3230,12 @@ async function runEmbeddedAgentInternal( params.abortSignal, ); if (compactResult.ok && compactResult.compacted) { - previousSessionId = adoptCompactionTranscript(compactResult); + previousSessionId = await adoptCompactionTranscript(compactResult); await runContextEngineMaintenance({ contextEngine, sessionId: activeSessionId, sessionKey: params.sessionKey, + sessionTarget: activeSessionTarget, sessionFile: activeSessionFile, reason: "compaction", runtimeContext: overflowCompactionRuntimeContext, @@ -3041,7 +3277,7 @@ async function runEmbeddedAgentInternal( continue; } if (compactResult.compacted) { - adoptCompactionTranscript(compactResult); + await adoptCompactionTranscript(compactResult); if ( typeof compactResult.result?.tokensAfter === "number" && Number.isFinite(compactResult.result.tokensAfter) && @@ -3050,17 +3286,19 @@ async function runEmbeddedAgentInternal( lastCompactionTokensAfter = Math.floor(compactResult.result.tokensAfter); } if (preflightRecovery?.route === "compact_then_truncate") { - const truncResult = await truncateOversizedToolResultsInSession({ - sessionFile: activeSessionFile, + const truncResult = await truncateOversizedToolResultsInActiveTarget({ + scope: { + sessionId: activeSessionId, + sessionKey: params.sessionKey ?? activeSessionId, + sessionFile: activeSessionFile, + agentId: sessionAgentId, + }, contextWindowTokens: contextTokenBudget, maxCharsOverride: resolveLiveToolResultMaxChars({ contextWindowTokens: contextTokenBudget, cfg: params.config, agentId: sessionAgentId, }), - sessionId: activeSessionId, - sessionKey: params.sessionKey, - agentId: sessionAgentId, config: params.config, protectTrailingToolResults: true, }); @@ -3081,15 +3319,18 @@ async function runEmbeddedAgentInternal( postCompactionGuard.armPostCompaction(); if (preflightRecovery?.source === "mid-turn") { continueFromCurrentTranscript(); - } else if ( - params.currentMessageId !== undefined && - params.currentMessageId === lastPersistedCurrentMessageId - ) { - // The first attempt reached the embedded agent far enough to persist this user turn. - // Retrying the original prompt would replay it, so resume from the - // compacted transcript and suppress the next user append. - nextAttemptPromptOverride = MID_TURN_PRECHECK_CONTINUATION_PROMPT; - suppressNextUserMessagePersistence = true; + } else { + await waitForCurrentUserMessagePersistence(); + if ( + params.currentMessageId !== undefined && + params.currentMessageId === lastPersistedCurrentMessageId + ) { + // The first attempt reached the embedded agent far enough to persist this user turn. + // Retrying the original prompt would replay it, so resume from the + // compacted transcript and suppress the next user append. + nextAttemptPromptOverride = MID_TURN_PRECHECK_CONTINUATION_PROMPT; + suppressNextUserMessagePersistence = true; + } } continue; } @@ -3118,13 +3359,15 @@ async function runEmbeddedAgentInternal( `[context-overflow-recovery] Attempting tool result truncation for ${provider}/${modelId} ` + `(contextWindow=${contextWindowTokens} tokens)`, ); - const truncResult = await truncateOversizedToolResultsInSession({ - sessionFile: activeSessionFile, + const truncResult = await truncateOversizedToolResultsInActiveTarget({ + scope: { + sessionId: activeSessionId, + sessionKey: params.sessionKey ?? activeSessionId, + sessionFile: activeSessionFile, + agentId: sessionAgentId, + }, contextWindowTokens, maxCharsOverride: toolResultMaxChars, - sessionId: activeSessionId, - sessionKey: params.sessionKey, - agentId: sessionAgentId, config: params.config, protectTrailingToolResults: preflightRecovery?.route === "compact_then_truncate", }); diff --git a/src/agents/embedded-agent-runner/run/attempt.llm-boundary.cache-stability.test.ts b/src/agents/embedded-agent-runner/run/attempt.llm-boundary.cache-stability.test.ts index b6e83fdc150f..469812d20c11 100644 --- a/src/agents/embedded-agent-runner/run/attempt.llm-boundary.cache-stability.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt.llm-boundary.cache-stability.test.ts @@ -23,12 +23,13 @@ import { streamOpenAICompletions, streamOpenAIResponses } from "@openclaw/ai/int */ import { describe, expect, it } from "vitest"; import { stripInboundMetadata } from "../../../auto-reply/reply/strip-inbound-meta.js"; +import { loadTranscriptEvents } from "../../../config/sessions/session-accessor.js"; import { buildTimestampPrefix } from "../../../gateway/server-methods/agent-timestamp.js"; import type { Context, Model } from "../../../llm/types.js"; import { - appendUserTurnTranscriptMessage, createUserTurnTranscriptRecorder, mergePreparedUserTurnMessageForRuntime, + persistUserTurnTranscript, type UserTurnInput, } from "../../../sessions/user-turn-transcript.js"; import { summarizeMessages } from "../../cache-trace.js"; @@ -371,7 +372,14 @@ describe("prompt-cache byte-identity (issue #3658)", () => { describe("append-only late media (issue #99495)", () => { it("keeps every sent fingerprint stable and appends one late-media turn", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-99495-boundary-")); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = { + agentId: "main", + cwd: dir, + sessionEntry: undefined, + sessionId: "session-99495", + sessionKey: "agent:main:cache-99495", + storePath: path.join(dir, "sessions.json"), + }; const admittedInput = { text: "describe this", timestamp: TS_TURN1, @@ -392,16 +400,13 @@ describe("append-only late media (issue #99495)", () => { markResolverStarted(); return await mediaInput; }, - target: { transcriptPath, sessionId: "session-99495", sessionKey: "main", cwd: dir }, + target, }); const persistence = recorder.persistFallback(); await resolverStarted; - await appendUserTurnTranscriptMessage({ - transcriptPath, + await persistUserTurnTranscript({ + ...target, input: admittedInput, - sessionId: "session-99495", - sessionKey: "main", - cwd: dir, }); recorder.markRuntimePersisted(recorder.message); const admittedRuntimeMessage = mergePreparedUserTurnMessageForRuntime({ @@ -415,11 +420,8 @@ describe("append-only late media (issue #99495)", () => { media: [{ path: path.join(dir, "image.png"), contentType: "image/png" }], }); await persistence; - const persisted = fs - .readFileSync(transcriptPath, "utf8") - .trim() - .split("\n") - .map((line) => JSON.parse(line) as { message?: AgentMsg }) + const persisted = (await loadTranscriptEvents(target)) + .map((entry) => entry as { message?: AgentMsg }) .flatMap((entry) => (entry.message ? [entry.message] : [])); const next = normalizeMessagesForLlmBoundary(persisted, { timezone: TZ }); const sentSummary = summarizeMessages(sent); @@ -442,7 +444,13 @@ describe("append-only late media (issue #99495)", () => { timestamp: TS_TURN1, media: [{ path: "media://inbound/image.jpg", contentType: "image/jpeg" }], }), - target: { transcriptPath: "/unused/session.jsonl" }, + target: { + agentId: "main", + sessionEntry: undefined, + sessionId: "unused-session", + sessionKey: "agent:main:unused", + storePath: "/tmp/openclaw-unused-sessions.json", + }, }); const resolved = await prepared.resolveMessage(); const merged = mergePreparedUserTurnMessageForRuntime({ diff --git a/src/agents/embedded-agent-runner/run/attempt.prompt-helpers.ts b/src/agents/embedded-agent-runner/run/attempt.prompt-helpers.ts index 564128fa1312..0d6400bf411c 100644 --- a/src/agents/embedded-agent-runner/run/attempt.prompt-helpers.ts +++ b/src/agents/embedded-agent-runner/run/attempt.prompt-helpers.ts @@ -6,6 +6,7 @@ import type { OpenClawConfig } from "../../../config/types.openclaw.js"; import type { ContextEnginePromptCacheInfo, ContextEngineRuntimeContext, + ContextEngineSessionTarget, } from "../../../context-engine/types.js"; import { drainPluginNextTurnInjectionContext } from "../../../plugins/host-hook-state.js"; import { buildPluginAgentTurnPrepareContext } from "../../../plugins/host-hooks.js"; @@ -579,6 +580,7 @@ export function resolveAttemptMediaTaskSystemPromptAddition(params: { type AfterTurnRuntimeContextAttempt = Pick< EmbeddedRunAttemptParams, + | "sessionTarget" | "sessionKey" | "sandboxSessionKey" | "messageChannel" @@ -604,6 +606,32 @@ type AfterTurnRuntimeContextAttempt = Pick< sessionId?: EmbeddedRunAttemptParams["sessionId"]; }; +function resolveRuntimeContextSessionTarget(params: { + attempt: AfterTurnRuntimeContextAttempt; + activeAgentId?: string; +}): ContextEngineSessionTarget | undefined { + const sessionTarget = params.attempt.sessionTarget; + const agentId = sessionTarget?.agentId ?? params.activeAgentId; + const sessionId = sessionTarget?.sessionId ?? params.attempt.sessionId; + const sessionKey = sessionTarget?.sessionKey ?? params.attempt.sessionKey; + if ( + !agentId && + !sessionId && + !sessionKey && + !sessionTarget?.storePath && + sessionTarget?.threadId === undefined + ) { + return undefined; + } + return { + ...(agentId ? { agentId } : {}), + ...(sessionId ? { sessionId } : {}), + ...(sessionKey ? { sessionKey } : {}), + ...(sessionTarget?.storePath ? { storePath: sessionTarget.storePath } : {}), + ...(sessionTarget?.threadId !== undefined ? { threadId: sessionTarget.threadId } : {}), + }; +} + /** Build runtime context passed into context-engine afterTurn hooks. */ export function buildAfterTurnRuntimeContext(params: { attempt: AfterTurnRuntimeContextAttempt; @@ -616,6 +644,10 @@ export function buildAfterTurnRuntimeContext(params: { currentTokenCount?: number; promptCache?: ContextEnginePromptCacheInfo; }): ContextEngineRuntimeContext { + const sessionTarget = resolveRuntimeContextSessionTarget({ + attempt: params.attempt, + activeAgentId: params.activeAgentId, + }); return { ...buildEmbeddedCompactionRuntimeContext({ sessionKey: params.attempt.sessionKey, @@ -668,6 +700,8 @@ export function buildAfterTurnRuntimeContext(params: { ? { currentTokenCount: Math.floor(params.currentTokenCount) } : {}), ...(params.promptCache ? { promptCache: params.promptCache } : {}), + transcriptStorage: { kind: "sqlite" }, + ...(sessionTarget ? { sessionTarget } : {}), }; } diff --git a/src/agents/embedded-agent-runner/run/attempt.queue-message.test.ts b/src/agents/embedded-agent-runner/run/attempt.queue-message.test.ts index a564985654a2..ab1959f864ce 100644 --- a/src/agents/embedded-agent-runner/run/attempt.queue-message.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt.queue-message.test.ts @@ -1,6 +1,7 @@ // Coverage for queued steering message commit and cancellation behavior. import { describe, expect, it, vi } from "vitest"; import { createUserTurnTranscriptRecorder } from "../../../sessions/user-turn-transcript.js"; +import { createTestUserTurnTranscriptTarget } from "../../../sessions/user-turn-transcript.test-support.js"; import { cancelQueuedSteeringMessage, steerActiveSessionWithOptionalDeliveryWait, @@ -13,7 +14,7 @@ describe("embedded OpenClaw queued steering cancellation", () => { const steer = vi.fn(async () => undefined); const recorder = createUserTurnTranscriptRecorder({ input: { text: "visible prompt", sender: { id: "user-42" } }, - target: { transcriptPath: "/tmp/unused-session.jsonl" }, + target: createTestUserTurnTranscriptTarget(), }); const activeSession: EmbeddedAgentActiveSessionSteerTarget = { steer, diff --git a/src/agents/embedded-agent-runner/run/attempt.session-lock.test.ts b/src/agents/embedded-agent-runner/run/attempt.session-lock.test.ts index d72cd5997c49..2919fa6ebd5d 100644 --- a/src/agents/embedded-agent-runner/run/attempt.session-lock.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt.session-lock.test.ts @@ -6,10 +6,14 @@ import path from "node:path"; import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { resolveSessionTranscriptPathInDir } from "../../../config/sessions/paths.js"; +import { + loadTranscriptEvents, + upsertSessionEntry, +} from "../../../config/sessions/session-accessor.js"; import { appendSessionTranscriptEvent, appendSessionTranscriptMessage, -} from "../../../config/sessions/transcript-append.js"; +} from "../../../config/sessions/transcript-append.test-support.js"; import { runWithOwnedSessionTranscriptWriteLock, runWithOwnedSessionTranscriptWritePublication, @@ -1288,9 +1292,9 @@ describe("embedded attempt session lock lifecycle", () => { }); ( staleManager as unknown as { - rewriteFile: () => void; + replacePersistedTranscript: () => void; } - ).rewriteFile(); + ).replacePersistedTranscript(); const reopened = SessionManager.open(sessionFile, dir, dir); expect(reopened.getSessionName()).toBe("session title"); @@ -1425,9 +1429,9 @@ describe("embedded attempt session lock lifecycle", () => { }); ( staleManager as unknown as { - rewriteFile: () => void; + replacePersistedTranscript: () => void; } - ).rewriteFile(); + ).replacePersistedTranscript(); const reopened = SessionManager.open(sessionFile, dir, dir); expect(reopened.getEntry(deliveryId)).toMatchObject({ @@ -2842,17 +2846,15 @@ describe("embedded attempt session lock lifecycle", () => { const sessionKey = "facade"; const storePath = path.join(dir, "sessions.json"); const sessionFile = resolveSessionTranscriptPathInDir(sessionId, dir); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId, - chatType: "direct", - channel: "discord", - spawnedCwd: dir, - }, - }), - "utf8", + await upsertSessionEntry( + { agentId: "main", sessionKey, storePath }, + { + sessionId, + chatType: "direct", + channel: "discord", + spawnedCwd: dir, + updatedAt: Date.now(), + }, ); const mergePromptReleasedSessionEntries = vi.fn(); const controller = await createEmbeddedAttemptSessionLockController({ @@ -2900,9 +2902,16 @@ describe("embedded attempt session lock lifecycle", () => { ); expect(result.ok).toBe(true); - expect(mergePromptReleasedSessionEntries).toHaveBeenCalledWith([ - expect.objectContaining({ type: "message" }), - ]); + const events = await loadTranscriptEvents({ sessionId, sessionKey, storePath }); + expect(events).toContainEqual( + expect.objectContaining({ + message: expect.objectContaining({ + content: [{ type: "text", text: "first-turn delivery" }], + role: "assistant", + }), + }), + ); + expect(mergePromptReleasedSessionEntries).not.toHaveBeenCalled(); expect(controller.hasSessionTakeover()).toBe(false); await controller.dispose(); }); diff --git a/src/agents/embedded-agent-runner/run/attempt.session-lock.ts b/src/agents/embedded-agent-runner/run/attempt.session-lock.ts index c3e0a244e3d5..3efe8097d955 100644 --- a/src/agents/embedded-agent-runner/run/attempt.session-lock.ts +++ b/src/agents/embedded-agent-runner/run/attempt.session-lock.ts @@ -8,6 +8,7 @@ import fs from "node:fs/promises"; import { isDeepStrictEqual } from "node:util"; import { clampTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion"; import { normalizeStringEntries } from "@openclaw/normalization-core/string-normalization"; +import { parseSqliteSessionFileMarker } from "../../../config/sessions/sqlite-marker.js"; import { type OwnedSessionTranscriptPublishedEntry, type OwnedSessionTranscriptWriteOptions, @@ -270,7 +271,7 @@ type PromptReleasedSessionEntry = | PromptReleasedOpaqueEntry; type PromptReleasedSessionMergeResult = { - sessionFileSnapshot: OwnedSessionTranscriptCacheSnapshot; + sessionFileSnapshot?: OwnedSessionTranscriptCacheSnapshot; publishedEntries?: readonly OwnedSessionTranscriptPublishedEntry[]; requiresReload?: true; }; @@ -1161,6 +1162,9 @@ function isTrustedSessionFileState( } async function readSessionFileFingerprint(sessionFile: string): Promise { + if (parseSqliteSessionFileMarker(sessionFile)) { + return { exists: false }; + } try { return sessionFileFingerprintFromStat(await fs.stat(sessionFile, { bigint: true })); } catch (err) { @@ -1172,6 +1176,9 @@ async function readSessionFileFingerprint(sessionFile: string): Promise { + const workspaceDir = tempPaths[0]; + if (!workspaceDir) { + throw new Error("missing trajectory workspace path"); + } + return hoisted.trajectoryEvents.filter((event) => event.workspaceDir === workspaceDir); +} + function requireRecord(value: unknown, label: string): Record { if (!value || typeof value !== "object") { throw new Error(`expected ${label}`); @@ -878,12 +890,7 @@ describe("runEmbeddedAttempt context engine sessionKey forwarding", () => { ); expect(seen.systemPrompt).not.toContain("secret runtime context"); expect(JSON.stringify(seen.messages)).not.toContain("visible ask"); - const trajectoryEvents = ( - await fs.readFile(path.join(tempPaths[0] ?? "", "session.trajectory.jsonl"), "utf8") - ) - .trim() - .split("\n") - .map((line) => JSON.parse(line) as TrajectoryEvent); + const trajectoryEvents = await readTrajectoryEvents(tempPaths); const promptSubmitted = trajectoryEvents.find((event) => event.type === "prompt.submitted"); const contextCompiled = trajectoryEvents.find((event) => event.type === "context.compiled"); const modelCompleted = trajectoryEvents.find((event) => event.type === "model.completed"); @@ -1964,12 +1971,7 @@ describe("runEmbeddedAttempt context engine sessionKey forwarding", () => { expect(mockParams(hoisted.detectAndLoadPromptImagesMock, 0, "prompt image params").prompt).toBe( "what does this mean?", ); - const trajectoryEvents = ( - await fs.readFile(path.join(tempPaths[0] ?? "", "session.trajectory.jsonl"), "utf8") - ) - .trim() - .split("\n") - .map((line) => JSON.parse(line) as TrajectoryEvent); + const trajectoryEvents = await readTrajectoryEvents(tempPaths); const promptSubmitted = trajectoryEvents.find((event) => event.type === "prompt.submitted"); expect(promptSubmitted?.data?.prompt).toBe(seenPrompt); expect(promptSubmitted?.data?.prompt).not.toContain("WT daily plan - Sat May 2"); @@ -2105,12 +2107,7 @@ describe("runEmbeddedAttempt context engine sessionKey forwarding", () => { message.role === "user" && String(message.content).includes("internal heartbeat event"), ), ).toBe(false); - const trajectoryEvents = ( - await fs.readFile(path.join(tempPaths[0] ?? "", "session.trajectory.jsonl"), "utf8") - ) - .trim() - .split("\n") - .map((line) => JSON.parse(line) as TrajectoryEvent); + const trajectoryEvents = await readTrajectoryEvents(tempPaths); const contextCompiled = trajectoryEvents.find((event) => event.type === "context.compiled"); expect(contextCompiled?.data?.prompt).toContain("dynamic hook context"); expect(contextCompiled?.data?.prompt).toContain("internal heartbeat event"); @@ -2152,12 +2149,7 @@ describe("runEmbeddedAttempt context engine sessionKey forwarding", () => { expect(seenPrompt).toContain("orphaned ask"); expect(seenPrompt).not.toContain("internal heartbeat event"); expect(result.finalPromptText).toBe(seenPrompt); - const trajectoryEvents = ( - await fs.readFile(path.join(tempPaths[0] ?? "", "session.trajectory.jsonl"), "utf8") - ) - .trim() - .split("\n") - .map((line) => JSON.parse(line) as TrajectoryEvent); + const trajectoryEvents = await readTrajectoryEvents(tempPaths); const contextCompiled = trajectoryEvents.find((event) => event.type === "context.compiled"); const runtimeContext = findRecord( requireRecords(seenMessages, "seen messages"), @@ -2207,12 +2199,7 @@ describe("runEmbeddedAttempt context engine sessionKey forwarding", () => { expect(seenPrompt).toContain("Hello from the replied message"); expect(seenPrompt).toContain("Continue the OpenClaw runtime event."); expect(result.finalPromptText).toBe(seenPrompt); - const trajectoryEvents = ( - await fs.readFile(path.join(tempPaths[0] ?? "", "session.trajectory.jsonl"), "utf8") - ) - .trim() - .split("\n") - .map((line) => JSON.parse(line) as TrajectoryEvent); + const trajectoryEvents = await readTrajectoryEvents(tempPaths); const contextCompiled = trajectoryEvents.find((event) => event.type === "context.compiled"); expect(contextCompiled?.data?.prompt).toContain("Hello from the replied message"); expect(contextCompiled?.data?.systemPrompt).toContain("runtime bare mention event"); @@ -2291,12 +2278,7 @@ describe("runEmbeddedAttempt context engine sessionKey forwarding", () => { expect(JSON.stringify(seenModelMessages)).toContain("dynamic hook context"); expect(JSON.stringify(seenModelMessages)).toContain("dynamic hook tail"); expect(result.finalPromptText).toBe(seenPrompt); - const trajectoryEvents = ( - await fs.readFile(path.join(tempPaths[0] ?? "", "session.trajectory.jsonl"), "utf8") - ) - .trim() - .split("\n") - .map((line) => JSON.parse(line) as TrajectoryEvent); + const trajectoryEvents = await readTrajectoryEvents(tempPaths); const contextCompiled = trajectoryEvents.find((event) => event.type === "context.compiled"); // Room context now rides the runtime-context carrier, not the user prompt. expect(contextCompiled?.data?.prompt).not.toContain( @@ -2330,12 +2312,7 @@ describe("runEmbeddedAttempt context engine sessionKey forwarding", () => { role: "user", content: "seed", }); - const trajectoryEvents = ( - await fs.readFile(path.join(tempPaths[0] ?? "", "session.trajectory.jsonl"), "utf8") - ) - .trim() - .split("\n") - .map((line) => JSON.parse(line) as TrajectoryEvent); + const trajectoryEvents = await readTrajectoryEvents(tempPaths); expect(trajectoryEvents.some((event) => event.type === "prompt.submitted")).toBe(false); const skipped = findRecord( trajectoryEvents as Array>, @@ -2815,7 +2792,7 @@ describe("runEmbeddedAttempt context engine sessionKey forwarding", () => { const afterTurn = vi.fn(async () => { events.push("afterTurn"); }); - hoisted.sessionManager.rewriteFile.mockImplementation(() => { + hoisted.sessionManager.replacePersistedTranscript.mockImplementation(() => { events.push("flush"); }); @@ -2858,6 +2835,69 @@ describe("runEmbeddedAttempt context engine sessionKey forwarding", () => { expectCalledWithSessionKey(afterTurn, sessionKey); }); + it("uses SQLite transcript messages for bootstrap without treating the marker as a file", async () => { + const storeDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-ctx-engine-sqlite-")); + tempPaths.push(storeDir); + const storePath = path.join(storeDir, "sessions.json"); + const created = await createSessionEntryWithTranscript( + { + agentId: "main", + sessionKey, + storePath, + }, + () => ({ + ok: true, + entry: { + sessionId: embeddedSessionId, + updatedAt: Date.now(), + }, + }), + ); + if (!created.ok) { + throw new Error(`failed to create SQLite session entry: ${created.error}`); + } + await appendTranscriptMessage( + { + agentId: "main", + sessionId: embeddedSessionId, + sessionKey, + storePath, + }, + { + message: { role: "user", content: "persisted SQLite prompt" }, + now: Date.now(), + }, + ); + const bootstrap = vi.fn(async () => ({ bootstrapped: true })); + const assemble = vi.fn(async ({ messages }: { messages: AgentMessage[] }) => ({ + messages, + estimatedTokens: 1, + })); + + await createContextEngineAttemptRunner({ + contextEngine: createTestContextEngine({ bootstrap, assemble }), + sessionKey, + tempPaths, + attemptOverrides: { + sessionFile: created.sessionFile, + sessionTarget: { + agentId: "main", + sessionId: embeddedSessionId, + sessionKey, + storePath, + }, + }, + }); + + expect(bootstrap).toHaveBeenCalled(); + expect(hoisted.prepareSessionManagerForRunMock).toHaveBeenCalledWith( + expect.objectContaining({ + sessionFile: created.sessionFile, + hadSessionFile: false, + }), + ); + }); + it("resolves bootstrap context before acquiring the session write lock", async () => { const events: string[] = []; hoisted.resolveBootstrapContextForRunMock.mockImplementation(async () => { diff --git a/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts b/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts index 08da52581574..5d70c600b8da 100644 --- a/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts +++ b/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts @@ -13,6 +13,7 @@ import type { CompactResult, ContextEngineInfo, ContextEngineMaintenanceResult, + ContextEngineSessionTarget, IngestBatchResult, IngestResult, } from "../../../context-engine/types.js"; @@ -45,6 +46,11 @@ type BootstrapContext = { bootstrapFiles: WorkspaceBootstrapFile[]; contextFiles: EmbeddedContextFile[]; }; +type CapturedTrajectoryEvent = { + data?: Record; + type: string; + workspaceDir?: string; +}; function normalizeMockProviderId(providerId?: string): string { // Provider ids in mocked model routing follow the same lowercase normalization @@ -63,7 +69,7 @@ type SessionManagerMocks = { appendCustomEntry: UnknownMock; appendSessionInfo: UnknownMock; appendLabelChange: UnknownMock; - rewriteFile: UnknownMock; + replacePersistedTranscript: UnknownMock; flushPendingToolResults: UnknownMock; clearPendingToolResults: UnknownMock; removeTrailingEntries: UnknownMock; @@ -96,6 +102,7 @@ type AttemptSpawnWorkspaceHoisted = { getGlobalHookRunnerMock: Mock<() => unknown>; initializeGlobalHookRunnerMock: UnknownMock; runContextEngineMaintenanceMock: AsyncContextEngineMaintenanceMock; + prepareSessionManagerForRunMock: AsyncUnknownMock; detectAndLoadPromptImagesMock: AsyncUnknownMock; getHistoryLimitFromSessionKeyMock: Mock< (sessionKey: string | undefined, config: unknown) => number | undefined @@ -105,6 +112,7 @@ type AttemptSpawnWorkspaceHoisted = { compactionReserveTokens: number; systemPromptTexts: string[]; embeddedSystemPromptInputs: unknown[]; + trajectoryEvents: CapturedTrajectoryEvent[]; sessionManager: SessionManagerMocks; }; @@ -196,6 +204,7 @@ const hoisted = vi.hoisted((): AttemptSpawnWorkspaceHoisted => { const getGlobalHookRunnerMock = vi.fn<() => unknown>(() => undefined); const initializeGlobalHookRunnerMock = vi.fn(); const runContextEngineMaintenanceMock = vi.fn(async (_params?: unknown) => undefined); + const prepareSessionManagerForRunMock = vi.fn(async (_params?: unknown) => undefined); const detectAndLoadPromptImagesMock = vi.fn(async () => ({ images: [], detectedRefs: [], @@ -212,6 +221,7 @@ const hoisted = vi.hoisted((): AttemptSpawnWorkspaceHoisted => { const compactionReserveTokens = 0; const systemPromptTexts: string[] = []; const embeddedSystemPromptInputs: unknown[] = []; + const trajectoryEvents: CapturedTrajectoryEvent[] = []; const sessionManager = { getLeafEntry: vi.fn(() => null), getEntry: vi.fn(() => undefined), @@ -223,7 +233,7 @@ const hoisted = vi.hoisted((): AttemptSpawnWorkspaceHoisted => { appendCustomEntry: vi.fn(), appendSessionInfo: vi.fn(), appendLabelChange: vi.fn(), - rewriteFile: vi.fn(), + replacePersistedTranscript: vi.fn(), flushPendingToolResults: vi.fn(), clearPendingToolResults: vi.fn(), removeTrailingEntries: vi.fn(() => 0), @@ -256,6 +266,7 @@ const hoisted = vi.hoisted((): AttemptSpawnWorkspaceHoisted => { getGlobalHookRunnerMock, initializeGlobalHookRunnerMock, runContextEngineMaintenanceMock, + prepareSessionManagerForRunMock, detectAndLoadPromptImagesMock, getHistoryLimitFromSessionKeyMock, limitHistoryTurnsMock, @@ -263,6 +274,7 @@ const hoisted = vi.hoisted((): AttemptSpawnWorkspaceHoisted => { compactionReserveTokens, systemPromptTexts, embeddedSystemPromptInputs, + trajectoryEvents, sessionManager, }; }); @@ -331,6 +343,35 @@ vi.mock("../../../trajectory/metadata.js", () => ({ buildTrajectoryRunMetadata: () => ({ source: "test" }), })); +vi.mock("../../../trajectory/runtime.js", async () => { + const actual = await vi.importActual( + "../../../trajectory/runtime.js", + ); + return { + ...actual, + createTrajectoryRuntimeRecorder: ( + params: Parameters[0], + ) => { + const recorder = actual.createTrajectoryRuntimeRecorder(params); + return { + enabled: true as const, + describeFlushState: () => recorder?.describeFlushState(), + flush: async () => { + await recorder?.flush(); + }, + recordEvent: (type: string, data?: Record) => { + hoisted.trajectoryEvents.push({ + type, + ...(data ? { data } : {}), + ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), + }); + recorder?.recordEvent(type, data); + }, + }; + }, + }; +}); + vi.mock("../../sessions/index.js", () => { function AuthStorage() {} class DefaultResourceLoader { @@ -512,7 +553,7 @@ vi.mock("../session-manager-cache.js", () => ({ })); vi.mock("../session-manager-init.js", () => ({ - prepareSessionManagerForRun: async () => {}, + prepareSessionManagerForRun: (params: unknown) => hoisted.prepareSessionManagerForRunMock(params), })); vi.mock("../../session-write-lock.js", () => ({ @@ -1053,12 +1094,14 @@ export function resetEmbeddedAttemptHarness( hoisted.supportsModelToolsMock.mockReset().mockReturnValue(true); hoisted.getGlobalHookRunnerMock.mockReset().mockReturnValue(undefined); hoisted.runContextEngineMaintenanceMock.mockReset().mockResolvedValue(undefined); + hoisted.prepareSessionManagerForRunMock.mockReset().mockResolvedValue(undefined); hoisted.getHistoryLimitFromSessionKeyMock.mockReset().mockReturnValue(undefined); hoisted.limitHistoryTurnsMock.mockReset().mockImplementation((messages) => messages); hoisted.preemptiveCompactionCalls.length = 0; hoisted.compactionReserveTokens = 0; hoisted.systemPromptTexts.length = 0; hoisted.embeddedSystemPromptInputs.length = 0; + hoisted.trajectoryEvents.length = 0; hoisted.sessionManager.getLeafEntry.mockReset().mockReturnValue(null); hoisted.sessionManager.getEntry.mockReset().mockReturnValue(undefined); hoisted.sessionManager.branch.mockReset(); @@ -1071,7 +1114,7 @@ export function resetEmbeddedAttemptHarness( hoisted.sessionManager.appendCustomEntry.mockReset(); hoisted.sessionManager.appendSessionInfo.mockReset(); hoisted.sessionManager.appendLabelChange.mockReset(); - hoisted.sessionManager.rewriteFile.mockReset(); + hoisted.sessionManager.replacePersistedTranscript.mockReset(); if (params.subscribeImpl) { hoisted.subscribeEmbeddedAgentSessionMock.mockImplementation(params.subscribeImpl); } @@ -1250,8 +1293,9 @@ export async function createContextEngineAttemptRunner(params: { }) => Promise; compact?: (params: { sessionId: string; - sessionKey?: string; - sessionFile: string; + sessionKey: string; + agentId?: string; + sessionTarget?: ContextEngineSessionTarget; tokenBudget?: number; }) => Promise; info?: Partial; @@ -1269,6 +1313,7 @@ export async function createContextEngineAttemptRunner(params: { const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-ctx-engine-workspace-")); const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-ctx-engine-agent-")); const sessionFile = path.join(workspaceDir, "session.jsonl"); + const sessionStore = path.join(workspaceDir, "sessions.json"); params.tempPaths.push(workspaceDir, agentDir); await fs.writeFile(sessionFile, "", "utf8"); const seedMessages: AgentMessage[] = @@ -1302,8 +1347,13 @@ export async function createContextEngineAttemptRunner(params: { })); const previousTrajectoryEnv = process.env.OPENCLAW_TRAJECTORY; + const previousTrajectoryDirEnv = process.env.OPENCLAW_TRAJECTORY_DIR; if (params.trajectory !== true) { process.env.OPENCLAW_TRAJECTORY = "0"; + delete process.env.OPENCLAW_TRAJECTORY_DIR; + } else { + delete process.env.OPENCLAW_TRAJECTORY; + process.env.OPENCLAW_TRAJECTORY_DIR = workspaceDir; } try { return await ( @@ -1314,7 +1364,7 @@ export async function createContextEngineAttemptRunner(params: { sessionFile, workspaceDir, agentDir, - config: {}, + config: { session: { store: sessionStore } }, prompt: "hello", timeoutMs: 10_000, runId: "run-context-engine-forwarding", @@ -1358,5 +1408,10 @@ export async function createContextEngineAttemptRunner(params: { } else { process.env.OPENCLAW_TRAJECTORY = previousTrajectoryEnv; } + if (previousTrajectoryDirEnv === undefined) { + delete process.env.OPENCLAW_TRAJECTORY_DIR; + } else { + process.env.OPENCLAW_TRAJECTORY_DIR = previousTrajectoryDirEnv; + } } } diff --git a/src/agents/embedded-agent-runner/run/attempt.test.ts b/src/agents/embedded-agent-runner/run/attempt.test.ts index a30f5fda66aa..22bb6ee196ef 100644 --- a/src/agents/embedded-agent-runner/run/attempt.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt.test.ts @@ -3359,6 +3359,7 @@ describe("buildAfterTurnRuntimeContext", () => { expect(activeProcessSessions?.some((session) => session.sessionId === "sess-other")).toBe( false, ); + expect(legacy.transcriptStorage).toEqual({ kind: "sqlite" }); } finally { resetProcessRegistryForTests(); } @@ -3413,6 +3414,35 @@ describe("buildAfterTurnRuntimeContext", () => { expect(runtimeContext.model).toBe("gpt-5.5"); }); + it("publishes the storage-neutral session target in runtime context", () => { + const sessionTarget = { + agentId: "main", + sessionId: "session-abc", + sessionKey: "agent:main:session:abc", + storePath: "/tmp/state/agents/main/sessions/sessions.json", + threadId: 42, + }; + + const runtimeContext = buildAfterTurnRuntimeContext({ + attempt: { + sessionId: "ignored-session-id", + sessionKey: "agent:main:fallback", + sessionTarget, + config: {} as OpenClawConfig, + skillsSnapshot: undefined, + provider: "openai", + modelId: "gpt-5.4", + thinkLevel: "off", + reasoningLevel: "on", + }, + workspaceDir: "/tmp/workspace", + agentDir: "/tmp/agent", + activeAgentId: "main", + }); + + expect(runtimeContext.transcriptStorage).toEqual({ kind: "sqlite" }); + expect(runtimeContext.sessionTarget).toEqual(sessionTarget); + }); it("resolves compaction.model override in runtime context so all context engines use the correct model", () => { const legacy = buildAfterTurnRuntimeContext({ attempt: { diff --git a/src/agents/embedded-agent-runner/run/attempt.ts b/src/agents/embedded-agent-runner/run/attempt.ts index ef8129f43b2e..72f24ba2de23 100644 --- a/src/agents/embedded-agent-runner/run/attempt.ts +++ b/src/agents/embedded-agent-runner/run/attempt.ts @@ -16,8 +16,11 @@ import { resolveStorePath } from "../../../config/sessions/paths.js"; import { listSessionEntries, loadSessionEntry, + loadTranscriptEvents, + resolveSessionTranscriptRuntimeReadTarget, updateSessionEntry, } from "../../../config/sessions/session-accessor.js"; +import { parseSqliteSessionFileMarker } from "../../../config/sessions/sqlite-marker.js"; import { resolveQuotaSuspensionEntryMaintenance } from "../../../config/sessions/store-maintenance.js"; import { bindOwnedSessionTranscriptWrites, @@ -26,6 +29,7 @@ import { withOwnedSessionTranscriptWrites, } from "../../../config/sessions/transcript-write-context.js"; import type { SessionEntry as ConfigSessionEntry } from "../../../config/sessions/types.js"; +import type { OpenClawConfig } from "../../../config/types.openclaw.js"; import { assertContextEngineHostSupport, OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST, @@ -690,8 +694,14 @@ function sessionMessagesContainIdempotencyKey( ); } -function flushSessionManagerFile(sessionManager: ReturnType): void { - (sessionManager as unknown as { rewriteFile?: () => void }).rewriteFile?.(); +function flushSessionManagerTranscript( + sessionManager: ReturnType, +): void { + ( + sessionManager as unknown as { + replacePersistedTranscript?: () => void; + } + ).replacePersistedTranscript?.(); } type OrphanRepairSessionManager = Pick< @@ -994,6 +1004,91 @@ async function loadAttemptSessionEntryAfterQuotaMaintenance(params: { return updated ?? entry; } +async function resolveAttemptTrajectorySessionFile(params: { + agentId: string; + config?: OpenClawConfig; + sessionFile: string; + sessionId: string; + sessionKey?: string; + sessionTarget?: EmbeddedRunAttemptParams["sessionTarget"]; +}): Promise { + const storePath = + params.sessionTarget?.storePath ?? + resolveStorePath(params.config?.session?.store, { agentId: params.agentId }); + if (!storePath || !params.sessionKey) { + return params.sessionFile; + } + return ( + await resolveSessionTranscriptRuntimeReadTarget({ + agentId: params.agentId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath, + }) + ).sessionFile; +} + +type ExistingAttemptTranscriptState = { + hasBootstrapTranscriptState: boolean; + hasFileTranscriptState: boolean; +}; + +function isTranscriptMessageEvent(event: unknown): boolean { + return ( + typeof event === "object" && + event !== null && + "type" in event && + (event as { type?: unknown }).type === "message" + ); +} + +async function resolveExistingAttemptTranscriptState(params: { + agentId: string; + config?: OpenClawConfig; + sessionFile: string; + sessionId: string; + sessionKey?: string; + sessionTarget?: EmbeddedRunAttemptParams["sessionTarget"]; +}): Promise { + const storePath = + params.sessionTarget?.storePath ?? + resolveStorePath(params.config?.session?.store, { agentId: params.agentId }); + const sqliteMarker = parseSqliteSessionFileMarker(params.sessionFile); + let hasBootstrapTranscriptState = false; + if (storePath && params.sessionKey) { + try { + const sqliteEvents = await loadTranscriptEvents({ + agentId: params.agentId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath, + }); + hasBootstrapTranscriptState = sqliteEvents.some(isTranscriptMessageEvent); + if (sqliteMarker) { + return { + hasBootstrapTranscriptState, + hasFileTranscriptState: false, + }; + } + } catch { + if (sqliteMarker) { + return { + hasBootstrapTranscriptState: false, + hasFileTranscriptState: false, + }; + } + } + } + const hasFileTranscriptState = await fs + .stat(params.sessionFile) + .then(() => true) + .catch(() => false); + return { + hasBootstrapTranscriptState: hasBootstrapTranscriptState || hasFileTranscriptState, + hasFileTranscriptState, + }; +} + export async function runEmbeddedAttempt( params: EmbeddedRunAttemptParams, ): Promise { @@ -2427,10 +2522,14 @@ export async function runEmbeddedAttempt( ) { invalidateSessionFileRepairCache(params.sessionFile); } - const hadSessionFile = await fs - .stat(params.sessionFile) - .then(() => true) - .catch(() => false); + const transcriptState = await resolveExistingAttemptTranscriptState({ + agentId: sessionAgentId, + config: params.config, + sessionFile: params.sessionFile, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, + }); const transcriptPolicy = resolveAttemptTranscriptPolicy({ runtimePlan: params.runtimePlan, @@ -2485,10 +2584,11 @@ export async function runEmbeddedAttempt( await withOwnedSessionWriteLock(async () => { await runAttemptContextEngineBootstrap({ - hadSessionFile, + hadSessionFile: transcriptState.hasBootstrapTranscriptState, contextEngine: activeContextEngine, sessionId: params.sessionId, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, sessionManager, runtimeContext: buildAfterTurnRuntimeContext({ @@ -2511,6 +2611,7 @@ export async function runEmbeddedAttempt( contextEngine: contextParams.contextEngine as never, sessionId: contextParams.sessionId, sessionKey: contextParams.sessionKey, + sessionTarget: contextParams.sessionTarget, sessionFile: contextParams.sessionFile, reason: contextParams.reason, sessionManager: contextParams.sessionManager as never, @@ -2525,7 +2626,7 @@ export async function runEmbeddedAttempt( await prepareSessionManagerForRun({ sessionManager, sessionFile: params.sessionFile, - hadSessionFile, + hadSessionFile: transcriptState.hasFileTranscriptState, sessionId: params.sessionId, cwd: effectiveCwd, }); @@ -2977,6 +3078,7 @@ export async function runEmbeddedAttempt( contextEngine: activeContextEngine, sessionId: params.sessionId, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, tokenBudget: params.contextTokenBudget, modelId: params.modelId, @@ -3071,13 +3173,21 @@ export async function runEmbeddedAttempt( modelApi: params.model.api, workspaceDir: params.workspaceDir, }); + const trajectorySessionFile = await resolveAttemptTrajectorySessionFile({ + agentId: sessionAgentId, + config: params.config, + sessionFile: params.sessionFile, + sessionId: activeSession.sessionId, + sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, + }); trajectoryRecorder = createTrajectoryRuntimeRecorder({ cfg: params.config, env: process.env, runId: params.runId, sessionId: activeSession.sessionId, sessionKey: params.sessionKey, - sessionFile: params.sessionFile, + sessionFile: trajectorySessionFile, provider: params.provider, modelId: params.modelId, modelApi: params.model.api, @@ -4770,7 +4880,7 @@ export async function runEmbeddedAttempt( activeSessionManager.appendMessage( redactedUserMessage as Parameters[0], ); - flushSessionManagerFile(activeSessionManager); + flushSessionManagerTranscript(activeSessionManager); }); activeSession.agent.state.messages = activeSessionManager.buildSessionContext().messages; @@ -5646,9 +5756,9 @@ export async function runEmbeddedAttempt( } if (activeContextEngine && !beforeAgentFinalizeRevisionReason) { - // Context-engine afterTurn hooks may reconcile against the jsonl, so - // materialize the active turn before finalization reads from disk. - flushSessionManagerFile(activeSessionManager); + // Context-engine afterTurn hooks may reconcile against persisted + // transcript state, so materialize the active turn first. + flushSessionManagerTranscript(activeSessionManager); } }); @@ -5673,6 +5783,7 @@ export async function runEmbeddedAttempt( yieldAborted, sessionIdUsed, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, messagesSnapshot, prePromptMessageCount: contextEngineAfterTurnCheckpoint ?? prePromptMessageCount, @@ -5689,6 +5800,7 @@ export async function runEmbeddedAttempt( contextEngine: contextParams.contextEngine as never, sessionId: contextParams.sessionId, sessionKey: contextParams.sessionKey, + sessionTarget: contextParams.sessionTarget, sessionFile: contextParams.sessionFile, reason: contextParams.reason, sessionManager: contextParams.sessionManager as never, diff --git a/src/agents/embedded-agent-runner/run/types.ts b/src/agents/embedded-agent-runner/run/types.ts index 6f131c51aff2..899fcb65c189 100644 --- a/src/agents/embedded-agent-runner/run/types.ts +++ b/src/agents/embedded-agent-runner/run/types.ts @@ -52,6 +52,12 @@ type EmbeddedRunContextWindowInfo = { export type EmbeddedRunFastModeParam = boolean | (() => boolean | undefined); +/** Host-owned trajectory recorder supplied to plugin harnesses for attempt-local runtime events. */ +export type EmbeddedRunAttemptTrajectoryRecorder = { + recordEvent: (type: string, data?: Record) => void; + flush: () => Promise; +}; + export type EmbeddedRunAttemptParams = EmbeddedRunAttemptBase & { /** Active file-backed artifact target resolved by the run/session target seam. */ sessionFile: string; @@ -88,6 +94,10 @@ export type EmbeddedRunAttemptParams = EmbeddedRunAttemptBase & { runtimePlan?: AgentRuntimePlan; /** Host-issued scope for harnesses that mirror native child runs into task state. */ agentHarnessTaskRuntimeScope?: AgentHarnessTaskRuntimeScope; + /** Storage-neutral trajectory target for harness-owned runtime trace artifacts. */ + trajectorySessionFile?: string; + /** Storage-aware trajectory recorder owned by the OpenClaw host. */ + trajectoryRecorder?: EmbeddedRunAttemptTrajectoryRecorder | null; /** Live observer called after wrapped tool outcomes are recorded. */ onToolOutcome?: ToolOutcomeObserver; /** Signals that the attempt's own run-timeout watchdog is active. */ diff --git a/src/agents/embedded-agent-runner/runs.test.ts b/src/agents/embedded-agent-runner/runs.test.ts index 523a077ca4e7..32ae1d948ddb 100644 --- a/src/agents/embedded-agent-runner/runs.test.ts +++ b/src/agents/embedded-agent-runner/runs.test.ts @@ -21,6 +21,7 @@ import { } from "../../logging/diagnostic-session-state.js"; import { diagnosticLogger } from "../../logging/diagnostic.js"; import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; +import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js"; import { MAX_TIMER_TIMEOUT_MS } from "../../shared/number-coercion.js"; import { testing, @@ -793,7 +794,7 @@ describe("embedded-agent runner run registry", () => { operation.setPhase("running"); const recorder = createUserTurnTranscriptRecorder({ input: { text: "visible group prompt", sender: { id: "user-42" } }, - target: { transcriptPath: "/tmp/unused-session.jsonl" }, + target: createTestUserTurnTranscriptTarget(), }); const outcome = await queueEmbeddedAgentMessageWithOutcomeAsync( diff --git a/src/agents/embedded-agent-runner/runs.ts b/src/agents/embedded-agent-runner/runs.ts index d8835daf3396..42db178fec1b 100644 --- a/src/agents/embedded-agent-runner/runs.ts +++ b/src/agents/embedded-agent-runner/runs.ts @@ -13,6 +13,8 @@ import { listActiveReplyRunSessionIds, queueReplyRunMessage, resolveReplyBackendQueueMessageMismatch, + resolveReplyRunPhaseForSessionId, + type ReplyOperationPhase, waitForReplyRunEndBySessionId, } from "../../auto-reply/reply/reply-run-registry.js"; import { @@ -624,6 +626,12 @@ export function isEmbeddedAgentRunActive(sessionId: string): boolean { return active; } +export function resolveEmbeddedAgentReplyRunPhase( + sessionId: string, +): ReplyOperationPhase | undefined { + return resolveReplyRunPhaseForSessionId(sessionId); +} + export function isEmbeddedAgentRunHandleActive(sessionId: string): boolean { const active = ACTIVE_EMBEDDED_RUNS.has(sessionId); if (active) { diff --git a/src/agents/embedded-agent-runner/session-manager-cache.ts b/src/agents/embedded-agent-runner/session-manager-cache.ts index ef4dadfefdd2..8fdce1051ef8 100644 --- a/src/agents/embedded-agent-runner/session-manager-cache.ts +++ b/src/agents/embedded-agent-runner/session-manager-cache.ts @@ -8,6 +8,7 @@ import { isCacheEnabled, resolveCacheTtlMs, } from "../../config/cache-utils.js"; +import { parseSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; const DEFAULT_SESSION_MANAGER_TTL_MS = 45_000; // 45 seconds const MIN_SESSION_MANAGER_CACHE_PRUNE_INTERVAL_MS = 1_000; @@ -61,6 +62,9 @@ export function createSessionManagerCache(options?: { if (!isCacheEnabled(getTtlMs())) { return; } + if (parseSqliteSessionFileMarker(sessionFile)) { + return; + } if (cache.get(sessionFile) === true) { return; } diff --git a/src/agents/embedded-agent-runner/tool-result-context-guard.test.ts b/src/agents/embedded-agent-runner/tool-result-context-guard.test.ts index 56f10bfcde17..a293407cae7b 100644 --- a/src/agents/embedded-agent-runner/tool-result-context-guard.test.ts +++ b/src/agents/embedded-agent-runner/tool-result-context-guard.test.ts @@ -696,6 +696,32 @@ describe("installContextEngineLoopHook", () => { }); }); + it("passes sessionTarget through loop-hook afterTurn calls", async () => { + const agent = makeGuardableAgent(); + const engine = makeMockEngine(); + const sessionTarget = { + agentId: "main", + sessionId, + sessionKey, + storePath: "/tmp/state/openclaw.sqlite", + }; + installContextEngineLoopHook({ + agent, + contextEngine: engine, + sessionId, + sessionKey, + sessionTarget, + sessionFile, + tokenBudget, + modelId, + getPrePromptMessageCount: () => 1, + }); + + await callTransform(agent, [makeUser("first"), makeToolResult("call_1", "result")]); + + expect(recordMockArg(engine.afterTurn).sessionTarget).toEqual(sessionTarget); + }); + it("passes runtimeSettings through loop-hook afterTurn and assemble calls", async () => { const agent = makeGuardableAgent(); const engine = makeMockEngine(); diff --git a/src/agents/embedded-agent-runner/tool-result-context-guard.ts b/src/agents/embedded-agent-runner/tool-result-context-guard.ts index 5d801d360555..82ea57eca9da 100644 --- a/src/agents/embedded-agent-runner/tool-result-context-guard.ts +++ b/src/agents/embedded-agent-runner/tool-result-context-guard.ts @@ -6,6 +6,7 @@ import type { ContextEngine, ContextEngineRuntimeContext, ContextEngineRuntimeSettings, + ContextEngineSessionTarget, } from "../../context-engine/types.js"; import type { AgentMessage } from "../runtime/index.js"; import { formatContextLimitTruncationNotice } from "./context-truncation-notice.js"; @@ -329,6 +330,7 @@ export function installContextEngineLoopHook(params: { contextEngine: ContextEngine; sessionId: string; sessionKey?: string; + sessionTarget?: ContextEngineSessionTarget; sessionFile: string; tokenBudget?: number; modelId: string; @@ -398,6 +400,7 @@ export function installContextEngineLoopHook(params: { await contextEngine.afterTurn({ sessionId, sessionKey, + sessionTarget: params.sessionTarget, sessionFile, messages: transcriptMessages, prePromptMessageCount, diff --git a/src/agents/embedded-agent-runner/tool-result-truncation.test.ts b/src/agents/embedded-agent-runner/tool-result-truncation.test.ts index 38bc337d9b68..718899a75d9a 100644 --- a/src/agents/embedded-agent-runner/tool-result-truncation.test.ts +++ b/src/agents/embedded-agent-runner/tool-result-truncation.test.ts @@ -8,7 +8,15 @@ import { SessionManager } from "openclaw/plugin-sdk/agent-sessions"; import type { AssistantMessage, ToolResultMessage, UserMessage } from "openclaw/plugin-sdk/llm"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { convertToLlm } from "../../../packages/agent-core/src/harness/messages.js"; -import { onSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; +import { + appendTranscriptMessage, + loadTranscriptEvents, + replaceSessionEntry, + replaceTranscriptEvents, +} from "../../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; +import type { SessionEntry as SessionStoreEntry } from "../../config/sessions/types.js"; +import { onInternalSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; import { formatFullOutputFooter } from "../sessions/tools/tool-contracts.js"; import { makeAgentAssistantMessage } from "../test-helpers/agent-message-fixtures.js"; import { buildRuntimeContextCustomMessage } from "./run/runtime-context-prompt.js"; @@ -24,7 +32,9 @@ let calculateMaxToolResultCharsWithCap: typeof import("./tool-result-truncation. let resolveAutoLiveToolResultMaxChars: typeof import("./tool-result-truncation.js").resolveAutoLiveToolResultMaxChars; let getToolResultTextLength: typeof import("./tool-result-truncation.js").getToolResultTextLength; let truncateOversizedToolResultsInMessages: typeof import("./tool-result-truncation.js").truncateOversizedToolResultsInMessages; +let truncateOversizedToolResultsInRuntimeTranscript: typeof import("./tool-result-truncation.js").truncateOversizedToolResultsInRuntimeTranscript; let truncateOversizedToolResultsInSession: typeof import("./tool-result-truncation.js").truncateOversizedToolResultsInSession; +let truncateOversizedToolResultsInActiveTarget: typeof import("./tool-result-truncation.js").truncateOversizedToolResultsInActiveTarget; let sessionLikelyHasOversizedToolResults: typeof import("./tool-result-truncation.js").sessionLikelyHasOversizedToolResults; let estimateToolResultReductionPotential: typeof import("./tool-result-truncation.js").estimateToolResultReductionPotential; let DEFAULT_MAX_LIVE_TOOL_RESULT_CHARS: typeof import("./tool-result-truncation.js").DEFAULT_MAX_LIVE_TOOL_RESULT_CHARS; @@ -44,7 +54,9 @@ async function loadFreshToolResultTruncationModuleForTest() { resolveAutoLiveToolResultMaxChars, getToolResultTextLength, truncateOversizedToolResultsInMessages, + truncateOversizedToolResultsInRuntimeTranscript, truncateOversizedToolResultsInSession, + truncateOversizedToolResultsInActiveTarget, sessionLikelyHasOversizedToolResults, estimateToolResultReductionPotential, DEFAULT_MAX_LIVE_TOOL_RESULT_CHARS, @@ -1253,6 +1265,184 @@ describe("truncateOversizedToolResultsInMessages", () => { }); describe("truncateOversizedToolResultsInSession", () => { + it("truncates SQLite runtime transcripts without treating the marker as a file", async () => { + const dir = await createTmpDir(); + const storePath = path.join(dir, "sessions.json"); + const sessionId = "runtime-sqlite-tool-truncation"; + const sessionKey = "agent:main:test"; + const sessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + const scope = { agentId: "main", sessionId, sessionKey, storePath }; + await replaceSessionEntry({ sessionKey, storePath }, { + sessionFile, + sessionId, + updatedAt: 10, + } as SessionStoreEntry); + await appendTranscriptMessage(scope, { + message: makeUserMessage("run tools"), + }); + const medium = "alpha beta gamma delta epsilon ".repeat(600); + await appendTranscriptMessage(scope, { + message: makeToolResult(medium, "call_1"), + }); + await appendTranscriptMessage(scope, { + message: makeToolResult(medium, "call_2"), + }); + await appendTranscriptMessage(scope, { + message: makeToolResult(medium, "call_3"), + }); + + const listener = vi.fn(); + const cleanup = onInternalSessionTranscriptUpdate(listener); + const result = await truncateOversizedToolResultsInRuntimeTranscript({ + scope: { ...scope, sessionFile }, + contextWindowTokens: 100, + }); + cleanup(); + + expect(result.truncated).toBe(true); + expect(result.truncatedCount).toBeGreaterThan(0); + expect(listener).toHaveBeenCalledWith({ + sessionFile, + sessionKey, + agentId: "main", + sessionId, + target: { agentId: "main", sessionId, sessionKey }, + }); + + const toolResultTexts = (await loadTranscriptEvents(scope)) + .filter( + (entry): entry is { message: AgentMessage; type: "message" } => + typeof entry === "object" && + entry !== null && + "message" in entry && + "type" in entry && + entry.type === "message", + ) + .map((entry) => entry.message) + .filter((message): message is ToolResultMessage => message.role === "toolResult") + .map(getFirstToolResultText); + + expect(toolResultTexts.some((text) => text.includes("truncated"))).toBe(true); + expect(toolResultTexts.join("").length).toBeLessThan(medium.length * 3); + }); + + it("dispatches explicit file transcript targets to file-backed truncation", async () => { + const dir = await createTmpDir(); + const sm = SessionManager.create(dir, dir); + sm.appendMessage(makeUserMessage("hello")); + sm.appendMessage(makeAssistantMessage("calling tools")); + sm.appendMessage(makeToolResult("x".repeat(500_000), "call_1")); + const sessionFile = sm.getSessionFile()!; + + const result = await truncateOversizedToolResultsInActiveTarget({ + scope: { + agentId: "main", + sessionFile, + sessionId: "explicit-file-session", + sessionKey: "agent:main:explicit-file", + }, + contextWindowTokens: 100, + }); + + expect(result.truncated).toBe(true); + expect(result.truncatedCount).toBeGreaterThan(0); + const toolResult = SessionManager.open(sessionFile) + .getBranch() + .find((entry) => entry.type === "message" && entry.message.role === "toolResult"); + expect( + toolResult?.type === "message" ? getFirstToolResultText(toolResult.message) : "", + ).toContain("truncated"); + }); + + it("honors SQLite leaf controls when truncating runtime transcripts", async () => { + const dir = await createTmpDir(); + const storePath = path.join(dir, "sessions.json"); + const sessionId = "runtime-sqlite-leaf-tool-truncation"; + const sessionKey = "agent:main:test"; + const sessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + const scope = { agentId: "main", sessionId, sessionKey, storePath }; + await replaceSessionEntry({ sessionKey, storePath }, { + sessionFile, + sessionId, + updatedAt: 10, + } as SessionStoreEntry); + const activeLarge = "selected branch tool output ".repeat(700); + const inactiveLarge = "inactive branch tool output ".repeat(700); + await replaceTranscriptEvents(scope, [ + { + type: "session", + version: 3, + id: sessionId, + timestamp: "2026-01-01T00:00:00.000Z", + cwd: dir, + }, + { + type: "message", + id: "root-user", + parentId: null, + timestamp: "2026-01-01T00:00:01.000Z", + message: makeUserMessage("run tools"), + }, + { + type: "message", + id: "selected-tool", + parentId: "root-user", + timestamp: "2026-01-01T00:00:02.000Z", + message: makeToolResult(activeLarge, "call_selected"), + }, + { + type: "message", + id: "inactive-tool", + parentId: "root-user", + timestamp: "2026-01-01T00:00:03.000Z", + message: makeToolResult(inactiveLarge, "call_inactive"), + }, + { + type: "leaf", + id: "selected-leaf", + parentId: "inactive-tool", + timestamp: "2026-01-01T00:00:04.000Z", + targetId: "selected-tool", + }, + ]); + + const result = await truncateOversizedToolResultsInRuntimeTranscript({ + scope: { ...scope, sessionFile }, + contextWindowTokens: 100, + }); + + expect(result.truncated).toBe(true); + const messages = (await loadTranscriptEvents(scope)) + .filter( + (entry): entry is { message: AgentMessage; type: "message" } => + typeof entry === "object" && + entry !== null && + "message" in entry && + "type" in entry && + entry.type === "message", + ) + .map((entry) => entry.message); + const selectedTool = messages.find( + (message): message is ToolResultMessage => + message.role === "toolResult" && message.toolCallId === "call_selected", + ); + const inactiveTool = messages.find( + (message): message is ToolResultMessage => + message.role === "toolResult" && message.toolCallId === "call_inactive", + ); + + expect(selectedTool ? getFirstToolResultText(selectedTool) : "").toContain("truncated"); + expect(inactiveTool ? getFirstToolResultText(inactiveTool) : "").toBe(inactiveLarge); + }); + it("readably truncates aggregate medium tool results in a session file", async () => { // Persisted truncation rewrites JSONL directly and emits the transcript // update event instead of reopening through SessionManager internals. @@ -1280,7 +1470,7 @@ describe("truncateOversizedToolResultsInSession", () => { throw new Error("SessionManager.open should not be used for persisted truncation"); }); const listener = vi.fn(); - const cleanup = onSessionTranscriptUpdate(listener); + const cleanup = onInternalSessionTranscriptUpdate(listener); const result = await truncateOversizedToolResultsInSession({ sessionFile, sessionKey: "agent:main:test", diff --git a/src/agents/embedded-agent-runner/tool-result-truncation.ts b/src/agents/embedded-agent-runner/tool-result-truncation.ts index 3f8681ee2fa1..fb13e7fbf917 100644 --- a/src/agents/embedded-agent-runner/tool-result-truncation.ts +++ b/src/agents/embedded-agent-runner/tool-result-truncation.ts @@ -5,6 +5,8 @@ import { createHash } from "node:crypto"; import { existsSync } from "node:fs"; import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; import { sliceUtf16Safe, truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; +import { loadTranscriptEvents } from "../../config/sessions/session-accessor.js"; +import { parseSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { formatErrorMessage } from "../../infra/errors.js"; import type { TextContent } from "../../llm/types.js"; @@ -22,14 +24,20 @@ import { formatContextLimitTruncationNotice } from "./context-truncation-notice. import { log } from "./logger.js"; import type { ToolResultPromptProjectionState } from "./session-prompt-state.js"; import { + createTranscriptFileStateFromPersistedEntries, persistTranscriptStateMutation, readTranscriptFileState, - type TranscriptFileState, + TranscriptFileState, } from "./transcript-file-state.js"; import { rewriteTranscriptEntriesInSessionManager, + rewriteTranscriptEntriesInRuntimeTranscript, rewriteTranscriptEntriesInState, } from "./transcript-rewrite.js"; +import { + resolveRuntimeTranscriptReadTarget, + type RuntimeTranscriptScope, +} from "./transcript-runtime-state.js"; /** * Maximum share of the context window a single tool result should occupy. @@ -105,6 +113,23 @@ function logToolResultSessionTruncation(params: { ); } +async function readRuntimeTranscriptFileState( + scope: RuntimeTranscriptScope, +): Promise { + const target = await resolveRuntimeTranscriptReadTarget(scope); + const sqliteMarker = parseSqliteSessionFileMarker(target.sessionFile); + if (!sqliteMarker) { + throw new Error("runtime transcript target is not SQLite-backed"); + } + const events = await loadTranscriptEvents({ + agentId: sqliteMarker.agentId, + sessionId: sqliteMarker.sessionId, + sessionKey: target.sessionKey, + storePath: sqliteMarker.storePath, + }); + return createTranscriptFileStateFromPersistedEntries(events); +} + function resolveSuffixFactory( suffix: ToolResultTruncationOptions["suffix"], ): (truncatedChars: number) => string { @@ -1130,6 +1155,40 @@ function buildToolResultReplacementPlan(params: { aggregateReducibleChars, }; } + +function buildRecoveryToolResultReplacementPlan(params: { + branch: ToolResultBranchEntry[]; + contextWindowTokens: number; + maxCharsOverride?: number; + aggregateMaxCharsOverride?: number; + protectTrailingToolResults?: boolean; +}): { + maxChars: number; + aggregateBudgetChars: number; + plan: ReturnType; +} { + const maxChars = Math.max( + 1, + params.maxCharsOverride ?? calculateMaxToolResultChars(params.contextWindowTokens), + ); + const aggregateBudgetChars = calculateRecoveryAggregateToolResultChars( + params.contextWindowTokens, + maxChars, + params.aggregateMaxCharsOverride, + ); + return { + maxChars, + aggregateBudgetChars, + plan: buildToolResultReplacementPlan({ + branch: params.branch, + maxChars, + aggregateBudgetChars, + minKeepChars: RECOVERY_MIN_KEEP_CHARS, + protectTrailingToolResults: params.protectTrailingToolResults, + }), + }; +} + export function estimateToolResultReductionPotential(params: { messages: AgentMessage[]; contextWindowTokens: number; @@ -1197,26 +1256,17 @@ function truncateOversizedToolResultsInExistingSessionManager(params: { agentId?: string; }): { truncated: boolean; truncatedCount: number; reason?: string } { const { sessionManager, contextWindowTokens } = params; - const maxChars = Math.max( - 1, - params.maxCharsOverride ?? calculateMaxToolResultChars(contextWindowTokens), - ); - const aggregateBudgetChars = calculateRecoveryAggregateToolResultChars( - contextWindowTokens, - maxChars, - params.aggregateMaxCharsOverride, - ); const branch = sessionManager.getBranch() as ToolResultBranchEntry[]; if (branch.length === 0) { return { truncated: false, truncatedCount: 0, reason: "empty session" }; } - const plan = buildToolResultReplacementPlan({ + const { maxChars, aggregateBudgetChars, plan } = buildRecoveryToolResultReplacementPlan({ branch, - maxChars, - aggregateBudgetChars, - minKeepChars: RECOVERY_MIN_KEEP_CHARS, + contextWindowTokens, + maxCharsOverride: params.maxCharsOverride, + aggregateMaxCharsOverride: params.aggregateMaxCharsOverride, protectTrailingToolResults: params.protectTrailingToolResults, }); if (plan.replacements.length === 0) { @@ -1278,26 +1328,17 @@ async function truncateOversizedToolResultsInTranscriptState(params: { config?: SessionWriteLockAcquireTimeoutConfig; }): Promise<{ truncated: boolean; truncatedCount: number; reason?: string }> { const { state, contextWindowTokens } = params; - const maxChars = Math.max( - 1, - params.maxCharsOverride ?? calculateMaxToolResultChars(contextWindowTokens), - ); - const aggregateBudgetChars = calculateRecoveryAggregateToolResultChars( - contextWindowTokens, - maxChars, - params.aggregateMaxCharsOverride, - ); const branch = state.getBranch() as ToolResultBranchEntry[]; if (branch.length === 0) { return { truncated: false, truncatedCount: 0, reason: "empty session" }; } - const plan = buildToolResultReplacementPlan({ + const { maxChars, aggregateBudgetChars, plan } = buildRecoveryToolResultReplacementPlan({ branch, - maxChars, - aggregateBudgetChars, - minKeepChars: RECOVERY_MIN_KEEP_CHARS, + contextWindowTokens, + maxCharsOverride: params.maxCharsOverride, + aggregateMaxCharsOverride: params.aggregateMaxCharsOverride, protectTrailingToolResults: params.protectTrailingToolResults, }); if (plan.replacements.length === 0) { @@ -1371,6 +1412,102 @@ export function truncateOversizedToolResultsInSessionManager(params: { } } +/** + * Truncates oversized tool results in the active runtime transcript. + */ +export async function truncateOversizedToolResultsInRuntimeTranscript(params: { + scope: RuntimeTranscriptScope; + contextWindowTokens: number; + maxCharsOverride?: number; + aggregateMaxCharsOverride?: number; + protectTrailingToolResults?: boolean; + config?: SessionWriteLockAcquireTimeoutConfig; +}): Promise<{ truncated: boolean; truncatedCount: number; reason?: string }> { + try { + const state = await readRuntimeTranscriptFileState(params.scope); + const { contextWindowTokens } = params; + const branch = state.getBranch() as ToolResultBranchEntry[]; + + if (branch.length === 0) { + return { truncated: false, truncatedCount: 0, reason: "empty session" }; + } + + const { maxChars, aggregateBudgetChars, plan } = buildRecoveryToolResultReplacementPlan({ + branch, + contextWindowTokens, + maxCharsOverride: params.maxCharsOverride, + aggregateMaxCharsOverride: params.aggregateMaxCharsOverride, + protectTrailingToolResults: params.protectTrailingToolResults, + }); + if (plan.replacements.length === 0) { + return { + truncated: false, + truncatedCount: 0, + reason: "no oversized or aggregate tool results", + }; + } + + const rewriteResult = await rewriteTranscriptEntriesInRuntimeTranscript({ + scope: params.scope, + request: { replacements: plan.replacements }, + config: params.config, + }); + + logToolResultSessionTruncation({ + rewrittenEntries: rewriteResult.rewrittenEntries, + contextWindowTokens, + maxChars, + aggregateBudgetChars, + oversizedReplacementCount: plan.oversizedReplacementCount, + aggregateReplacementCount: plan.aggregateReplacementCount, + sessionKey: params.scope.sessionKey, + sessionId: params.scope.sessionId, + }); + + return { + truncated: rewriteResult.changed, + truncatedCount: rewriteResult.rewrittenEntries, + reason: rewriteResult.reason, + }; + } catch (err) { + const errMsg = formatErrorMessage(err); + log.warn(`[tool-result-truncation] Failed to truncate: ${errMsg}`); + return { truncated: false, truncatedCount: 0, reason: errMsg }; + } +} + +/** Truncates oversized tool results in either a SQLite runtime target or explicit file target. */ +export async function truncateOversizedToolResultsInActiveTarget(params: { + scope: RuntimeTranscriptScope; + contextWindowTokens: number; + maxCharsOverride?: number; + aggregateMaxCharsOverride?: number; + protectTrailingToolResults?: boolean; + config?: OpenClawConfig; +}): Promise<{ truncated: boolean; truncatedCount: number; reason?: string }> { + if (parseSqliteSessionFileMarker(params.scope.sessionFile)) { + return await truncateOversizedToolResultsInRuntimeTranscript(params); + } + if (!params.scope.sessionFile) { + return { + truncated: false, + truncatedCount: 0, + reason: "no session file", + }; + } + return await truncateOversizedToolResultsInSession({ + sessionFile: params.scope.sessionFile, + contextWindowTokens: params.contextWindowTokens, + maxCharsOverride: params.maxCharsOverride, + aggregateMaxCharsOverride: params.aggregateMaxCharsOverride, + protectTrailingToolResults: params.protectTrailingToolResults, + sessionId: params.scope.sessionId, + sessionKey: params.scope.sessionKey, + agentId: params.scope.agentId, + config: params.config, + }); +} + /** * Truncates a named transcript file artifact. */ diff --git a/src/agents/embedded-agent-runner/transcript-file-state.ts b/src/agents/embedded-agent-runner/transcript-file-state.ts index 430b62b87a8e..448a4df59ff6 100644 --- a/src/agents/embedded-agent-runner/transcript-file-state.ts +++ b/src/agents/embedded-agent-runner/transcript-file-state.ts @@ -579,6 +579,34 @@ function fileEntryOrMigrationSlot(value: unknown, index: number): FileEntry { } as unknown as FileEntry; } +function createReadableTranscriptFileState(params: { + fileEntries: FileEntry[]; + header: SessionHeader | null; + migrated?: boolean; +}): TranscriptFileState { + const readable = readableSessionState(params.fileEntries); + return new TranscriptFileState({ + header: params.header, + entries: readable.entries, + leafId: readable.leafId, + appendParentId: params.migrated ? readable.leafId : readable.appendParentId, + ...(!params.migrated && readable.appendMode ? { appendMode: readable.appendMode } : {}), + opaqueParentsById: readable.opaqueParentsById, + logicalParentsById: readable.logicalParentsById, + migrated: params.migrated, + }); +} + +/** Builds readable branch state from persisted transcript records. */ +export function createTranscriptFileStateFromPersistedEntries( + entries: readonly unknown[], +): TranscriptFileState { + const fileEntries = entries.map(fileEntryOrMigrationSlot); + const header = + fileEntries.find((entry): entry is SessionHeader => entry.type === "session") ?? null; + return createReadableTranscriptFileState({ fileEntries, header }); +} + /** In-memory transcript state with branch, label, and append helpers. */ export class TranscriptFileState { readonly header: SessionHeader | null; @@ -943,17 +971,7 @@ export async function readTranscriptFileState(sessionFile: string): Promise entry.type === "session") ?? null; - const readable = readableSessionState(fileEntries); - return new TranscriptFileState({ - header, - entries: readable.entries, - leafId: readable.leafId, - appendParentId: migrated ? readable.leafId : readable.appendParentId, - ...(!migrated && readable.appendMode ? { appendMode: readable.appendMode } : {}), - opaqueParentsById: readable.opaqueParentsById, - logicalParentsById: readable.logicalParentsById, - migrated, - }); + return createReadableTranscriptFileState({ fileEntries, header, migrated }); } /** Rewrite the full transcript through the private-file store. */ diff --git a/src/agents/embedded-agent-runner/transcript-rewrite.test.ts b/src/agents/embedded-agent-runner/transcript-rewrite.test.ts index 6b7f056dc34b..32a89d5a6bb2 100644 --- a/src/agents/embedded-agent-runner/transcript-rewrite.test.ts +++ b/src/agents/embedded-agent-runner/transcript-rewrite.test.ts @@ -6,6 +6,13 @@ import path from "node:path"; import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; import { SessionManager } from "openclaw/plugin-sdk/agent-sessions"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { + appendTranscriptMessage, + loadTranscriptEvents, + replaceSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; +import type { SessionEntry } from "../../config/sessions/types.js"; import { buildSessionWriteLockModuleMock } from "../../test-utils/session-write-lock-module-mock.js"; const acquireSessionWriteLockReleaseMock = vi.hoisted(() => vi.fn(async () => {})); @@ -145,13 +152,6 @@ function requireValue(value: T | undefined, label: string): T { return value; } -function requireString(value: string | undefined, label: string): string { - if (!value) { - throw new Error(`expected ${label}`); - } - return value; -} - beforeAll(async () => { ({ onSessionTranscriptUpdate } = await import("../../sessions/transcript-events.js")); ({ installSessionToolResultGuard } = await import("../session-tool-result-guard.js")); @@ -324,42 +324,48 @@ describe("rewriteTranscriptEntriesInRuntimeTranscript", () => { it("rewrites runtime transcripts through scoped session identity", async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcript-rewrite-runtime-")); const storePath = path.join(dir, "sessions.json"); - const sessionManager = SessionManager.create(dir, dir); - const entryIds = appendSessionMessages(sessionManager, [ - asAppendMessage({ + const sessionId = "runtime-sqlite-rewrite"; + const sessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + const scope = { + agentId: "main", + sessionId, + sessionKey: "agent:main:test", + storePath, + }; + await replaceSessionEntry({ sessionKey: "agent:main:test", storePath }, { + sessionFile, + sessionId, + updatedAt: 10, + } as SessionEntry); + await appendTranscriptMessage(scope, { + message: { role: "user", content: "run tool", timestamp: 1, - }), - asAppendMessage({ + }, + }); + const toolResult = await appendTranscriptMessage(scope, { + message: { role: "toolResult", toolCallId: "call_1", toolName: "exec", content: createTextContent("before rewrite"), isError: false, timestamp: 2, - }), - asAppendMessage({ + }, + }); + await appendTranscriptMessage(scope, { + message: { role: "assistant", content: createTextContent("summarized"), timestamp: 3, - }), - ]); - const sessionFile = requireString(sessionManager.getSessionFile(), "persisted session file"); - const resolvedSessionFile = await fs.realpath(sessionFile); - const sessionId = path.basename(sessionFile, ".jsonl"); - await fs.writeFile( - storePath, - JSON.stringify({ - "agent:main:test": { - sessionFile, - sessionId, - updatedAt: 10, - }, - }), - "utf8", - ); - const toolResultEntryId = entryIds[1]; + }, + }); + const toolResultEntryId = toolResult.messageId; const listener = vi.fn(); const cleanup = onSessionTranscriptUpdate(listener); @@ -382,16 +388,10 @@ describe("rewriteTranscriptEntriesInRuntimeTranscript", () => { }); expect(result.changed).toBe(true); - expect(acquireSessionWriteLockMock).toHaveBeenCalledWith({ - sessionFile: resolvedSessionFile, - staleMs: 1_800_000, - timeoutMs: 60_000, - maxHoldMs: 300_000, - }); - expect(acquireSessionWriteLockReleaseMock).toHaveBeenCalledTimes(1); + expect(acquireSessionWriteLockMock).not.toHaveBeenCalled(); + expect(acquireSessionWriteLockReleaseMock).not.toHaveBeenCalled(); expect(listener).toHaveBeenCalledWith({ agentId: "main", - sessionFile: resolvedSessionFile, sessionId, sessionKey: "agent:main:test", target: { @@ -401,8 +401,16 @@ describe("rewriteTranscriptEntriesInRuntimeTranscript", () => { }, }); - const rewrittenSession = SessionManager.open(sessionFile); - const branchMessages = getBranchMessages(rewrittenSession); + const branchMessages = (await loadTranscriptEvents(scope)) + .filter( + (entry): entry is { message: AgentMessage; type: "message" } => + typeof entry === "object" && + entry !== null && + "message" in entry && + "type" in entry && + entry.type === "message", + ) + .map((entry) => entry.message); expect(branchMessages.map((message) => message.role)).toEqual([ "user", "toolResult", diff --git a/src/agents/embedded-agent-runner/transcript-rewrite.ts b/src/agents/embedded-agent-runner/transcript-rewrite.ts index f94403f6b2ab..5fdd0b00df05 100644 --- a/src/agents/embedded-agent-runner/transcript-rewrite.ts +++ b/src/agents/embedded-agent-runner/transcript-rewrite.ts @@ -1,3 +1,8 @@ +import { + loadTranscriptEvents, + replaceTranscriptEvents, +} from "../../config/sessions/session-accessor.js"; +import { parseSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; /** * Rewrites transcript entries in session managers, states, and files. */ @@ -31,6 +36,91 @@ import { type SessionManagerLike = ReturnType; type SessionBranchEntry = ReturnType[number]; +function isTranscriptEventRecord(event: unknown): event is { + id?: unknown; + message?: unknown; + type?: unknown; +} { + return typeof event === "object" && event !== null && !Array.isArray(event); +} + +async function rewriteSqliteRuntimeTranscript(params: { + target: Awaited>; + request: TranscriptRewriteRequest; +}): Promise { + const marker = parseSqliteSessionFileMarker(params.target.sessionFile); + if (!marker) { + return { + changed: false, + bytesFreed: 0, + rewrittenEntries: 0, + reason: "not a SQLite transcript target", + }; + } + const replacementsById = new Map( + params.request.replacements.map((replacement) => [replacement.entryId, replacement.message]), + ); + let bytesFreed = 0; + let rewrittenEntries = 0; + const events = await loadTranscriptEvents({ + agentId: marker.agentId, + sessionId: marker.sessionId, + sessionKey: params.target.sessionKey, + storePath: marker.storePath, + }); + const nextEvents = events.map((event) => { + if (!isTranscriptEventRecord(event)) { + return event; + } + const eventId = typeof event.id === "string" ? event.id : undefined; + const replacement = eventId ? replacementsById.get(eventId) : undefined; + if (!replacement || event.type !== "message") { + return event; + } + bytesFreed += Math.max( + 0, + Buffer.byteLength(JSON.stringify(event.message), "utf8") - + Buffer.byteLength(JSON.stringify(replacement), "utf8"), + ); + rewrittenEntries += 1; + return Object.assign({}, event, { + message: replacement, + }); + }); + if (rewrittenEntries === 0) { + return { + changed: false, + bytesFreed: 0, + rewrittenEntries: 0, + reason: "no matching transcript entries", + }; + } + await replaceTranscriptEvents( + { + agentId: marker.agentId, + sessionId: marker.sessionId, + sessionKey: params.target.sessionKey, + storePath: marker.storePath, + }, + nextEvents, + ); + emitSessionTranscriptUpdate({ + sessionFile: params.target.sessionFile, + sessionKey: params.target.sessionKey, + agentId: params.target.agentId, + target: { + agentId: params.target.agentId, + sessionId: params.target.sessionId, + sessionKey: params.target.sessionKey, + }, + }); + return { + changed: true, + bytesFreed, + rewrittenEntries, + }; +} + function estimateMessageBytes(message: AgentMessage): number { return Buffer.byteLength(JSON.stringify(message), "utf8"); } @@ -438,6 +528,12 @@ export async function rewriteTranscriptEntriesInRuntimeTranscript(params: { let sessionLock: Awaited> | undefined; try { const target = await resolveRuntimeTranscriptReadTarget(params.scope); + if (parseSqliteSessionFileMarker(target.sessionFile)) { + return await rewriteSqliteRuntimeTranscript({ + target, + request: params.request, + }); + } sessionLock = await acquireSessionWriteLock({ sessionFile: target.sessionFile, ...resolveSessionWriteLockOptions(params.config), diff --git a/src/agents/embedded-agent-subscribe.compaction-test-helpers.ts b/src/agents/embedded-agent-subscribe.compaction-test-helpers.ts index 1e75d4c4f510..105878bb28b9 100644 --- a/src/agents/embedded-agent-subscribe.compaction-test-helpers.ts +++ b/src/agents/embedded-agent-subscribe.compaction-test-helpers.ts @@ -1,8 +1,8 @@ /** * Test helpers for seeding and observing compaction counts in session stores. */ -import fs from "node:fs/promises"; -import path from "node:path"; +import { loadSessionEntry, replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../config/sessions/types.js"; export async function seedSessionStore(params: { storePath: string; @@ -10,30 +10,15 @@ export async function seedSessionStore(params: { compactionCount: number; updatedAt?: number; }) { - await fs.mkdir(path.dirname(params.storePath), { recursive: true }); - await fs.writeFile( - params.storePath, - JSON.stringify( - { - [params.sessionKey]: { - sessionId: "session-1", - updatedAt: params.updatedAt ?? 1_000, - compactionCount: params.compactionCount, - }, - }, - null, - 2, - ), - "utf-8", - ); + await replaceSessionEntry({ storePath: params.storePath, sessionKey: params.sessionKey }, { + sessionId: "session-1", + updatedAt: params.updatedAt ?? 1_000, + compactionCount: params.compactionCount, + } as SessionEntry); } export async function readCompactionCount(storePath: string, sessionKey: string): Promise { - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { compactionCount?: number } - >; - return store[sessionKey]?.compactionCount ?? 0; + return loadSessionEntry({ storePath, sessionKey })?.compactionCount ?? 0; } export async function waitForCompactionCount(params: { diff --git a/src/agents/harness/context-engine-lifecycle.test.ts b/src/agents/harness/context-engine-lifecycle.test.ts index c6d0f798004b..db1b05d3bcfc 100644 --- a/src/agents/harness/context-engine-lifecycle.test.ts +++ b/src/agents/harness/context-engine-lifecycle.test.ts @@ -7,7 +7,11 @@ import { } from "../../context-engine/host-compat.js"; import { registerContextEngine, resolveContextEngine } from "../../context-engine/registry.js"; import { buildContextEngineRuntimeSettings } from "../../context-engine/runtime-settings.js"; -import type { ContextEngine, ContextEngineRuntimeSettings } from "../../context-engine/types.js"; +import type { + ContextEngine, + ContextEngineRuntimeContext, + ContextEngineRuntimeSettings, +} from "../../context-engine/types.js"; import { compactContextEngineWithSafetyTimeout } from "../embedded-agent-runner/compaction-safety-timeout.js"; import { OPENCLAW_RUNTIME_CONTEXT_CUSTOM_TYPE } from "../internal-runtime-context.js"; import { @@ -131,12 +135,29 @@ describe("harness context engine lifecycle", () => { const engineId = uniqueConfiguredProofEngineId(); const captured: Array<{ hook: "bootstrap" | "assemble" | "afterTurn" | "maintain" | "compact"; + runtimeContext?: ContextEngineRuntimeContext; runtimeSettings?: ContextEngineRuntimeSettings; + sessionTarget?: ContextEngineRuntimeContext["sessionTarget"]; }> = []; + const sessionTarget = { + agentId: "main", + sessionId: sessionParams.sessionId, + sessionKey: sessionParams.sessionKey, + storePath: "/tmp/state/openclaw.sqlite", + }; + const bootstrapRuntimeContext = { + transcriptStorage: { kind: "sqlite" as const }, + sessionTarget, + }; const engine = createContextEngine({ info: { id: engineId, name: "Configured runtime settings proof engine" }, bootstrap: vi.fn(async (params) => { - captured.push({ hook: "bootstrap", runtimeSettings: params.runtimeSettings }); + captured.push({ + hook: "bootstrap", + runtimeContext: params.runtimeContext, + runtimeSettings: params.runtimeSettings, + sessionTarget: params.sessionTarget, + }); return { bootstrapped: true }; }), assemble: vi.fn(async (params) => { @@ -147,10 +168,19 @@ describe("harness context engine lifecycle", () => { }; }), afterTurn: vi.fn(async (params) => { - captured.push({ hook: "afterTurn", runtimeSettings: params.runtimeSettings }); + captured.push({ + hook: "afterTurn", + runtimeSettings: params.runtimeSettings, + sessionTarget: params.sessionTarget, + }); }), maintain: vi.fn(async (params) => { - captured.push({ hook: "maintain", runtimeSettings: params.runtimeSettings }); + captured.push({ + hook: "maintain", + runtimeContext: params.runtimeContext, + runtimeSettings: params.runtimeSettings, + sessionTarget: params.sessionTarget, + }); return { changed: false, bytesFreed: 0, rewrittenEntries: 0 }; }), compact: vi.fn(async (params) => { @@ -168,11 +198,13 @@ describe("harness context engine lifecycle", () => { contextEngine: configuredEngine, sessionId: sessionParams.sessionId, sessionKey: sessionParams.sessionKey, + sessionTarget, sessionFile: sessionParams.sessionFile, providerId: "openai", requestedModelId: "openai/gpt-5.5", modelId: "anthropic/claude-sonnet-4-6", fallbackReason: "primary_provider_5xx", + runtimeContext: bootstrapRuntimeContext, warn: () => {}, }); @@ -195,6 +227,7 @@ describe("harness context engine lifecycle", () => { yieldAborted: false, sessionIdUsed: sessionParams.sessionIdUsed, sessionKey: sessionParams.sessionKey, + sessionTarget, sessionFile: sessionParams.sessionFile, messagesSnapshot: [ textMessage("user", "old ask", 1), @@ -226,7 +259,10 @@ describe("harness context engine lifecycle", () => { { sessionId: sessionParams.sessionId, sessionKey: sessionParams.sessionKey, - sessionFile: sessionParams.sessionFile, + sessionTarget: { + sessionId: sessionParams.sessionId, + sessionKey: sessionParams.sessionKey, + }, tokenBudget: 2048, runtimeSettings: compactRuntimeSettings, }, @@ -236,6 +272,21 @@ describe("harness context engine lifecycle", () => { expect(new Set(captured.map((entry) => entry.hook))).toEqual( new Set(["bootstrap", "assemble", "afterTurn", "maintain", "compact"]), ); + expect(captured.find((entry) => entry.hook === "bootstrap")?.runtimeContext).toEqual( + bootstrapRuntimeContext, + ); + expect(captured.find((entry) => entry.hook === "bootstrap")?.sessionTarget).toEqual( + sessionTarget, + ); + expect(captured.find((entry) => entry.hook === "afterTurn")?.sessionTarget).toEqual( + sessionTarget, + ); + expect(captured.find((entry) => entry.hook === "maintain")?.sessionTarget).toEqual( + sessionTarget, + ); + expect( + captured.find((entry) => entry.hook === "maintain")?.runtimeContext?.sessionTarget, + ).toEqual(sessionTarget); for (const entry of captured) { expect(entry.runtimeSettings).toMatchObject({ schemaVersion: 1, diff --git a/src/agents/harness/context-engine-lifecycle.ts b/src/agents/harness/context-engine-lifecycle.ts index 6f2dff56b7dd..f0e64c8316bd 100644 --- a/src/agents/harness/context-engine-lifecycle.ts +++ b/src/agents/harness/context-engine-lifecycle.ts @@ -12,6 +12,7 @@ import type { ContextEngine, ContextEngineRuntimeContext, ContextEngineRuntimeSettings, + ContextEngineSessionTarget, } from "../../context-engine/types.js"; import { runContextEngineMaintenance } from "../embedded-agent-runner/context-engine-maintenance.js"; import { @@ -78,6 +79,7 @@ export async function bootstrapHarnessContextEngine(params: { contextEngine?: HarnessContextEngine; sessionId: string; sessionKey?: string; + sessionTarget?: ContextEngineSessionTarget; sessionFile: string; sessionManager?: unknown; runtimeContext?: ContextEngineRuntimeContext; @@ -107,14 +109,17 @@ export async function bootstrapHarnessContextEngine(params: { await params.contextEngine.bootstrap({ sessionId: params.sessionId, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, runtimeSettings, + runtimeContext: params.runtimeContext, }); } await (params.runMaintenance ?? runHarnessContextEngineMaintenance)({ contextEngine: params.contextEngine, sessionId: params.sessionId, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, reason: "bootstrap", sessionManager: params.sessionManager, @@ -215,6 +220,7 @@ export async function finalizeHarnessContextEngineTurn(params: { yieldAborted: boolean; sessionIdUsed: string; sessionKey?: string; + sessionTarget?: ContextEngineSessionTarget; sessionFile: string; messagesSnapshot: AgentMessage[]; prePromptMessageCount: number; @@ -253,6 +259,7 @@ export async function finalizeHarnessContextEngineTurn(params: { await params.contextEngine.afterTurn({ sessionId: params.sessionIdUsed, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, messages: conversationSnapshot.messages, prePromptMessageCount: conversationSnapshot.prePromptMessageCount, @@ -310,6 +317,7 @@ export async function finalizeHarnessContextEngineTurn(params: { contextEngine: params.contextEngine, sessionId: params.sessionIdUsed, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, reason: "turn", sessionManager: params.sessionManager, @@ -363,6 +371,7 @@ export async function runHarnessContextEngineMaintenance(params: { contextEngine?: HarnessContextEngine; sessionId: string; sessionKey?: string; + sessionTarget?: ContextEngineSessionTarget; sessionFile: string; reason: "bootstrap" | "compaction" | "turn"; sessionManager?: unknown; @@ -387,6 +396,7 @@ export async function runHarnessContextEngineMaintenance(params: { contextEngine: params.contextEngine, sessionId: params.sessionId, sessionKey: params.sessionKey, + sessionTarget: params.sessionTarget, sessionFile: params.sessionFile, reason: params.reason, sessionManager: params.sessionManager as Parameters< diff --git a/src/agents/harness/native-hook-relay.test.ts b/src/agents/harness/native-hook-relay.test.ts index 6b0b8e3d6d25..5c77fb79a130 100644 --- a/src/agents/harness/native-hook-relay.test.ts +++ b/src/agents/harness/native-hook-relay.test.ts @@ -6,7 +6,8 @@ import { createServer, request as httpRequest } from "node:http"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { updateSessionStore, type SessionEntry } from "../../config/sessions.js"; +import type { SessionEntry } from "../../config/sessions.js"; +import { replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import { initializeGlobalHookRunner, resetGlobalHookRunner, @@ -2331,12 +2332,10 @@ describe("native hook relay registry", () => { ]; setActivePluginRegistry(registry); try { - await updateSessionStore(storePath, (store) => { - store["agent:main:session-1"] = { - sessionId: "session-1", - updatedAt: Date.now(), - } as SessionEntry; - }); + await replaceSessionEntry({ sessionKey: "agent:main:session-1", storePath }, { + sessionId: "session-1", + updatedAt: Date.now(), + } as SessionEntry); const patchResult = await patchPluginSessionExtension({ cfg: config as never, sessionKey: "agent:main:session-1", diff --git a/src/agents/harness/types.ts b/src/agents/harness/types.ts index d39f4eb628f3..68eb87f60ac4 100644 --- a/src/agents/harness/types.ts +++ b/src/agents/harness/types.ts @@ -30,8 +30,13 @@ export type AgentHarnessSupport = | { supported: true; priority?: number; reason?: string } | { supported: false; reason?: string }; -export type AgentHarnessAttemptParams = +type InternalEmbeddedRunAttemptParams = import("../embedded-agent-runner/run/types.js").EmbeddedRunAttemptParams; + +export type AgentHarnessAttemptParams = Omit< + InternalEmbeddedRunAttemptParams, + "trajectoryRecorder" +>; export type AgentHarnessAttemptResult = import("../embedded-agent-runner/run/types.js").EmbeddedRunAttemptResult; export type AgentHarnessAuthBindingFingerprintParams = { diff --git a/src/agents/internal-session-effects.test.ts b/src/agents/internal-session-effects.test.ts index 2cd5fffb765b..55ec79c060fb 100644 --- a/src/agents/internal-session-effects.test.ts +++ b/src/agents/internal-session-effects.test.ts @@ -1,70 +1,99 @@ -// Covers private transcript preparation for internal agent side effects. -import fs from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { withTempDir } from "../test-helpers/temp-dir.js"; -import { withEnvAsync } from "../test-utils/env.js"; import { - isInternalSessionEffectsTranscriptPath, - prepareInternalSessionEffectsTranscript, - removeInternalSessionEffectsTranscript, + appendTranscriptMessage, + listSessionEntries, + loadExactSessionEntry, + loadTranscriptEvents, + upsertSessionEntry, +} from "../config/sessions/session-accessor.js"; +import { withTempDir } from "../test-helpers/temp-dir.js"; +import { + prepareInternalSessionEffectsSession, + removeInternalSessionEffectsSession, } from "./internal-session-effects.js"; -describe("prepareInternalSessionEffectsTranscript", () => { - it("creates a private transcript even without a visible source file", async () => { +describe("internal session effects", () => { + it("creates a hidden deterministic SQLite session", async () => { await withTempDir({ prefix: "openclaw-internal-session-effects-" }, async (dir) => { - await withEnvAsync({ OPENCLAW_STATE_DIR: dir }, async () => { - const sessionFile = await prepareInternalSessionEffectsTranscript({ - runId: "run/with space", - }); - - // The run id is filesystem-normalized and the transcript is private - // because internal side effects may contain hidden agent context. - expect(sessionFile).toBe(path.join(dir, "internal-agent-runs", "run_with_space.jsonl")); - expect(isInternalSessionEffectsTranscriptPath(sessionFile)).toBe(true); - expect(isInternalSessionEffectsTranscriptPath(path.join(dir, "visible.jsonl"))).toBe(false); - expect(await fs.readFile(sessionFile, "utf8")).toBe(""); - expect((await fs.stat(sessionFile)).mode & 0o777).toBe(0o600); - - await removeInternalSessionEffectsTranscript(sessionFile); - - await expect(fs.stat(sessionFile)).rejects.toMatchObject({ code: "ENOENT" }); + const storePath = path.join(dir, "sessions.json"); + const target = await prepareInternalSessionEffectsSession({ + agentId: "main", + cwd: dir, + runId: "run/with space", + storePath, }); + + expect(target.sessionKey).toMatch(/^agent:main:internal-session-effects:run_with_space-/); + expect(target.sessionId).toMatch(/^internal-session-effects-run_with_space-/); + expect(loadExactSessionEntry(target)?.entry.sessionId).toBe(target.sessionId); + expect(listSessionEntries({ storePath })).toEqual([]); + await expect(loadTranscriptEvents(target)).resolves.toEqual([ + expect.objectContaining({ id: target.sessionId, type: "session" }), + ]); + + const reopened = await prepareInternalSessionEffectsSession({ + agentId: "main", + cwd: dir, + runId: "run/with space", + storePath, + }); + expect(reopened).toEqual(target); }); }); - it("copies a visible source transcript into a private transcript", async () => { + it("forks visible SQLite history into the hidden session", async () => { await withTempDir({ prefix: "openclaw-internal-session-effects-" }, async (dir) => { - await withEnvAsync({ OPENCLAW_STATE_DIR: dir }, async () => { - const sourceFile = path.join(dir, "visible-session.jsonl"); - await fs.writeFile(sourceFile, '{"role":"assistant","content":"done"}\n', { - mode: 0o644, - }); - - const sessionFile = await prepareInternalSessionEffectsTranscript({ - sessionFile: sourceFile, - runId: "run-copy", - }); - - expect(await fs.readFile(sessionFile, "utf8")).toBe( - '{"role":"assistant","content":"done"}\n', - ); - expect((await fs.stat(sessionFile)).mode & 0o777).toBe(0o600); + const storePath = path.join(dir, "sessions.json"); + const source = { + agentId: "main", + sessionId: "visible-session", + sessionKey: "agent:main:main", + storePath, + }; + await upsertSessionEntry(source, { sessionId: source.sessionId, updatedAt: 1 }); + await appendTranscriptMessage(source, { + cwd: dir, + message: { content: "stored", role: "assistant", timestamp: 2 }, }); + + const target = await prepareInternalSessionEffectsSession({ + agentId: "main", + runId: "run-copy", + source, + storePath, + }); + const events = await loadTranscriptEvents(target); + + expect(events[0]).toMatchObject({ id: target.sessionId, type: "session" }); + expect(events).toContainEqual( + expect.objectContaining({ + message: expect.objectContaining({ content: "stored", role: "assistant" }), + type: "message", + }), + ); + expect(listSessionEntries({ storePath })).toEqual([ + expect.objectContaining({ sessionKey: source.sessionKey }), + ]); }); }); - it("creates an empty private transcript when the visible source is missing", async () => { + it("hard-deletes the hidden entry and transcript rows", async () => { await withTempDir({ prefix: "openclaw-internal-session-effects-" }, async (dir) => { - await withEnvAsync({ OPENCLAW_STATE_DIR: dir }, async () => { - const sessionFile = await prepareInternalSessionEffectsTranscript({ - sessionFile: path.join(dir, "missing-session.jsonl"), - runId: "run-missing-source", - }); - - expect(await fs.readFile(sessionFile, "utf8")).toBe(""); - expect((await fs.stat(sessionFile)).mode & 0o777).toBe(0o600); + const storePath = path.join(dir, "sessions.json"); + const target = await prepareInternalSessionEffectsSession({ + agentId: "main", + runId: "run-cleanup", + storePath, }); + await appendTranscriptMessage(target, { + message: { content: "private", role: "assistant", timestamp: 2 }, + }); + + await removeInternalSessionEffectsSession(target); + + expect(loadExactSessionEntry(target)).toBeUndefined(); + await expect(loadTranscriptEvents(target)).resolves.toEqual([]); }); }); }); diff --git a/src/agents/internal-session-effects.ts b/src/agents/internal-session-effects.ts index 3bafdca8f1ee..1508d424026d 100644 --- a/src/agents/internal-session-effects.ts +++ b/src/agents/internal-session-effects.ts @@ -1,68 +1,130 @@ -/** - * Manages transient transcripts used for internal session side effects. - */ -import { promises as fs } from "node:fs"; -import path from "node:path"; -import { resolveStateDir } from "../config/paths.js"; +import { resolveInternalSessionEffectsIdentity } from "../config/sessions/internal-session-key.js"; +/** Manages hidden SQLite sessions used for suppressed agent side effects. */ +import { + applySessionEntryLifecycleMutation, + forkSessionFromParentTranscript, + loadExactSessionEntry, + replaceTranscriptEvents, + upsertSessionEntry, +} from "../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; +import { createSessionTranscriptHeader } from "../config/sessions/transcript-header.js"; +import type { SessionEntry } from "../config/sessions/types.js"; +import type { AgentRunSessionTarget } from "./run-session-target.js"; -function resolveInternalSessionEffectsDir(): string { - return path.resolve(resolveStateDir(), "internal-agent-runs"); -} +export type InternalSessionEffectsTarget = Required< + Pick +> & { + sessionEntry: SessionEntry; + sessionFile: string; +}; -/** Resolves the private transcript path for an internal session-effect run. */ -export function resolveInternalSessionEffectsTranscriptPath(runId: string): string { - const safeRunId = runId.replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 120) || "run"; - return path.join(resolveInternalSessionEffectsDir(), `${safeRunId}.jsonl`); -} +type InternalSessionEffectsSource = Required< + Pick +>; -/** Checks whether a transcript belongs to the private internal-run directory. */ -export function isInternalSessionEffectsTranscriptPath( - sessionFile: string | undefined, -): sessionFile is string { - return Boolean( - sessionFile && path.dirname(path.resolve(sessionFile)) === resolveInternalSessionEffectsDir(), - ); -} - -/** Copies or creates a private transcript for internal session-effect recovery. */ -export async function prepareInternalSessionEffectsTranscript(params: { - sessionFile?: string; +/** Resolves the deterministic SQLite target owned by one internal-effects run. */ +export function resolveInternalSessionEffectsTarget(params: { + agentId: string; runId: string; -}): Promise { - // Callers must persist this path in an owning lifecycle record and invoke - // removeInternalSessionEffectsTranscript once the recovered output is no longer needed. - const sessionFile = resolveInternalSessionEffectsTranscriptPath(params.runId); - await fs.mkdir(path.dirname(sessionFile), { recursive: true, mode: 0o700 }); - if (!params.sessionFile) { - await fs.writeFile(sessionFile, "", { mode: 0o600 }); - await fs.chmod(sessionFile, 0o600); - return sessionFile; - } - try { - const contents = await fs.readFile(params.sessionFile); - await fs.writeFile(sessionFile, contents, { mode: 0o600 }); - await fs.chmod(sessionFile, 0o600); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - throw error; - } - await fs.writeFile(sessionFile, "", { mode: 0o600 }); - await fs.chmod(sessionFile, 0o600); - } - return sessionFile; + storePath: string; +}): Required> { + return { + agentId: params.agentId, + storePath: params.storePath, + ...resolveInternalSessionEffectsIdentity(params), + }; } -/** Removes an internal session-effect transcript if it is inside the owned dir. */ -export async function removeInternalSessionEffectsTranscript( - sessionFile: string | undefined, +function toInternalSessionEffectsTarget(params: { + agentId: string; + entry: SessionEntry; + sessionKey: string; + storePath: string; +}): InternalSessionEffectsTarget { + return { + agentId: params.agentId, + sessionId: params.entry.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + sessionEntry: params.entry, + sessionFile: formatSqliteSessionFileMarker({ + agentId: params.agentId, + sessionId: params.entry.sessionId, + storePath: params.storePath, + }), + }; +} + +/** Creates or reopens the hidden SQLite session owned by one internal-effects run. */ +export async function prepareInternalSessionEffectsSession(params: { + agentId: string; + cwd?: string; + runId: string; + source?: InternalSessionEffectsSource; + storePath: string; +}): Promise { + const scope = resolveInternalSessionEffectsTarget(params); + const existing = loadExactSessionEntry(scope)?.entry; + if (existing?.sessionId === scope.sessionId) { + return toInternalSessionEffectsTarget({ + agentId: params.agentId, + entry: existing, + sessionKey: scope.sessionKey, + storePath: params.storePath, + }); + } + + const fork = params.source + ? await forkSessionFromParentTranscript({ + agentId: params.source.agentId, + parentEntry: { sessionId: params.source.sessionId, updatedAt: Date.now() }, + parentSessionKey: params.source.sessionKey, + sessionKey: scope.sessionKey, + storePath: params.source.storePath, + targetSessionId: scope.sessionId, + targetStorePath: params.storePath, + }) + : undefined; + if (fork?.status !== "created") { + await replaceTranscriptEvents(scope, [ + createSessionTranscriptHeader({ cwd: params.cwd, sessionId: scope.sessionId }), + ]); + } + const now = Date.now(); + const entry = await upsertSessionEntry(scope, { + sessionId: scope.sessionId, + sessionStartedAt: now, + updatedAt: now, + }); + if (!entry) { + throw new Error(`Failed to create internal SQLite session for run ${params.runId}`); + } + return toInternalSessionEffectsTarget({ + agentId: params.agentId, + entry, + sessionKey: scope.sessionKey, + storePath: params.storePath, + }); +} + +/** Hard-deletes a run-owned hidden session and its SQLite transcript rows. */ +export async function removeInternalSessionEffectsSession( + target: AgentRunSessionTarget | undefined, ): Promise { - if (!isInternalSessionEffectsTranscriptPath(sessionFile)) { + if (!target?.sessionKey || !target.storePath) { return; } - const resolved = path.resolve(sessionFile); - try { - await fs.rm(resolved, { force: true }); - } catch { - // Best-effort privacy/disk cleanup; run cleanup must not fail on temp-file races. - } + await applySessionEntryLifecycleMutation({ + ...(target.agentId ? { agentId: target.agentId } : {}), + storePath: target.storePath, + removals: [ + { + sessionKey: target.sessionKey, + ...(target.sessionId ? { expectedSessionId: target.sessionId } : {}), + archiveRemovedTranscript: false, + }, + ], + skipMaintenance: true, + }); } diff --git a/src/agents/main-session-restart-recovery.test.ts b/src/agents/main-session-restart-recovery.test.ts index 5b54c51c9f31..95cd2318eda0 100644 --- a/src/agents/main-session-restart-recovery.test.ts +++ b/src/agents/main-session-restart-recovery.test.ts @@ -3,11 +3,13 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { loadSessionStore, type SessionEntry } from "../config/sessions.js"; +import type { SessionEntry } from "../config/sessions.js"; import { - readSessionStoreForTest, - writeSessionStoreForTestAsync, -} from "../config/sessions/test-helpers.js"; + appendTranscriptMessage, + listSessionEntries, + loadSessionEntry, + replaceSessionEntry, +} from "../config/sessions/session-accessor.js"; import { callGateway } from "../gateway/call.js"; import { getAgentEventLifecycleGeneration, @@ -57,8 +59,25 @@ async function makeSessionsDir(agentId = "main"): Promise { return sessionsDir; } +async function writeStorePath( + storePath: string, + store: Record, +): Promise { + await Promise.all( + Object.entries(store).map(([sessionKey, entry]) => + replaceSessionEntry({ storePath, sessionKey }, entry), + ), + ); +} + async function writeStore(sessionsDir: string, store: Record): Promise { - await fs.writeFile(path.join(sessionsDir, "sessions.json"), JSON.stringify(store, null, 2)); + await writeStorePath(path.join(sessionsDir, "sessions.json"), store); +} + +function readStore(storePath: string): Record { + return Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ); } async function writeTranscript( @@ -66,8 +85,22 @@ async function writeTranscript( sessionId: string, messages: unknown[], ): Promise { - const lines = messages.map((message) => JSON.stringify({ message })).join("\n"); - await fs.writeFile(path.join(sessionsDir, `${sessionId}.jsonl`), `${lines}\n`); + const storePath = path.join(sessionsDir, "sessions.json"); + const sessionKey = Object.entries(readStore(storePath)).find( + ([, entry]) => entry.sessionId === sessionId, + )?.[0]; + if (!sessionKey) { + throw new Error(`expected session entry for transcript fixture: ${sessionId}`); + } + for (const message of messages) { + await appendTranscriptMessage( + { sessionId, sessionKey, storePath }, + { + cwd: sessionsDir, + message, + }, + ); + } } function cleanedLockForPath(lockPath: string): SessionLockInspection { @@ -154,7 +187,7 @@ describe("main-session-restart-recovery", () => { sessionKeys: ["agent:main:main", "agent:main:completed", "agent:main:subagent:child"], }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 1, skipped: 1 }); expect(store["agent:main:main"]?.abortedLastRun).toBe(true); expect(store["agent:main:completed"]?.abortedLastRun).toBeUndefined(); @@ -170,21 +203,13 @@ describe("main-session-restart-recovery", () => { it("marks active sessions in a configured custom session store", async () => { const storePath = path.join(tmpDir, "custom", "sessions.json"); - await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile( - storePath, - JSON.stringify( - { - "agent:main:issue-82433": { - sessionId: "custom-session", - updatedAt: Date.now() - 10_000, - status: "running", - }, - } satisfies Record, - null, - 2, - ), - ); + await writeStorePath(storePath, { + "agent:main:issue-82433": { + sessionId: "custom-session", + updatedAt: Date.now() - 10_000, + status: "running", + }, + }); await writeTranscript(path.dirname(storePath), "custom-session", [ { role: "user", content: "continue this custom-store turn" }, { role: "toolResult", content: "custom result" }, @@ -196,7 +221,7 @@ describe("main-session-restart-recovery", () => { sessionKeys: ["agent:main:issue-82433"], }); - const store = loadSessionStore(storePath); + const store = readStore(storePath); expect(result).toEqual({ marked: 1, skipped: 0 }); expect(store["agent:main:issue-82433"]?.abortedLastRun).toBe(true); @@ -232,7 +257,7 @@ describe("main-session-restart-recovery", () => { ], }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 1, skipped: 0 }); expect(store["agent:main:main"]?.restartRecoveryRuns).toEqual([ { @@ -269,7 +294,7 @@ describe("main-session-restart-recovery", () => { ], }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 1, skipped: 0 }); expect(store["agent:main:main"]).toEqual( expect.objectContaining({ @@ -308,7 +333,7 @@ describe("main-session-restart-recovery", () => { sessionIds: ["main-session"], }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 1, skipped: 0 }); expect(store["agent:main:main"]).toEqual( expect.objectContaining({ @@ -349,7 +374,7 @@ describe("main-session-restart-recovery", () => { isActiveRun: () => false, }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 0, skipped: 0 }); expect(store["agent:main:main"]?.status).toBe("done"); expect(store["agent:main:main"]?.restartRecoveryRuns).toBeUndefined(); @@ -381,7 +406,7 @@ describe("main-session-restart-recovery", () => { isActiveRun: () => true, }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 0, skipped: 0 }); expect(store["agent:main:main"]?.status).toBe("done"); expect(store["agent:main:main"]?.restartRecoveryRuns).toBeUndefined(); @@ -413,7 +438,7 @@ describe("main-session-restart-recovery", () => { isActiveRun: () => true, }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 0, skipped: 0 }); expect(store["agent:main:main"]?.status).toBe("done"); expect(store["agent:main:main"]?.restartRecoveryRuns).toBeUndefined(); @@ -446,7 +471,7 @@ describe("main-session-restart-recovery", () => { isActiveRun: () => true, }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 0, skipped: 0 }); expect(store["agent:main:main"]?.status).toBe("done"); expect(store["agent:main:main"]?.restartRecoveryRuns).toBeUndefined(); @@ -483,7 +508,7 @@ describe("main-session-restart-recovery", () => { ], }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(store["agent:main:main"]?.restartRecoveryRuns).toEqual([ { runId: "first-restart-run", @@ -526,7 +551,7 @@ describe("main-session-restart-recovery", () => { ], }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(store["agent:main:main"]?.restartRecoveryRuns).toEqual([ { runId: "shared-run", @@ -548,21 +573,13 @@ describe("main-session-restart-recovery", () => { }); const storePath = path.join(tmpDir, "custom-duplicate-key", "sessions.json"); - await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile( - storePath, - JSON.stringify( - { - "agent:main:issue-82433": { - sessionId: "active-custom-session", - updatedAt: Date.now() - 10_000, - status: "running", - }, - } satisfies Record, - null, - 2, - ), - ); + await writeStorePath(storePath, { + "agent:main:issue-82433": { + sessionId: "active-custom-session", + updatedAt: Date.now() - 10_000, + status: "running", + }, + }); const result = await markRestartAbortedMainSessions({ cfg: { session: { store: storePath } }, @@ -571,8 +588,8 @@ describe("main-session-restart-recovery", () => { sessionKeys: ["agent:main:issue-82433"], }); - const defaultStore = loadSessionStore(path.join(defaultSessionsDir, "sessions.json")); - const customStore = loadSessionStore(storePath); + const defaultStore = readStore(path.join(defaultSessionsDir, "sessions.json")); + const customStore = readStore(storePath); expect(result).toEqual({ marked: 1, skipped: 0 }); expect(defaultStore["agent:main:issue-82433"]?.abortedLastRun).toBeUndefined(); expect(customStore["agent:main:issue-82433"]?.abortedLastRun).toBe(true); @@ -580,21 +597,13 @@ describe("main-session-restart-recovery", () => { it("marks custom-store sessions by session id when no session key is available", async () => { const storePath = path.join(tmpDir, "custom-by-id", "sessions.json"); - await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile( - storePath, - JSON.stringify( - { - "agent:main:custom-by-id": { - sessionId: "custom-session-id-only", - updatedAt: Date.now() - 10_000, - status: "running", - }, - } satisfies Record, - null, - 2, - ), - ); + await writeStorePath(storePath, { + "agent:main:custom-by-id": { + sessionId: "custom-session-id-only", + updatedAt: Date.now() - 10_000, + status: "running", + }, + }); const result = await markRestartAbortedMainSessions({ cfg: { session: { store: storePath } }, @@ -602,7 +611,7 @@ describe("main-session-restart-recovery", () => { sessionIds: ["custom-session-id-only"], }); - const store = loadSessionStore(storePath); + const store = readStore(storePath); expect(result).toEqual({ marked: 1, skipped: 0 }); expect(store["agent:main:custom-by-id"]?.abortedLastRun).toBe(true); }); @@ -636,7 +645,7 @@ describe("main-session-restart-recovery", () => { ], }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 1, skipped: 1 }); expect(store["agent:main:main"]?.abortedLastRun).toBe(true); expect(store["agent:main:subagent:child"]?.abortedLastRun).toBeUndefined(); @@ -661,7 +670,7 @@ describe("main-session-restart-recovery", () => { cleanedLocks: [cleanedLockForPath(path.join(sessionsDir, `${sessionFile}.lock`))], }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 1, skipped: 0 }); expect(store["agent:main:discord:channel:123:thread:1234567890"]?.abortedLastRun).toBe(true); }); @@ -684,7 +693,7 @@ describe("main-session-restart-recovery", () => { ], }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 0, skipped: 0 }); expect(store["agent:main:main"]?.abortedLastRun).toBeUndefined(); }); @@ -711,7 +720,7 @@ describe("main-session-restart-recovery", () => { ], }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 1, skipped: 0 }); expect(store["agent:main:discord:channel:123:thread:1234567890"]?.abortedLastRun).toBe(true); }); @@ -732,7 +741,7 @@ describe("main-session-restart-recovery", () => { cleanedLocks: [cleanedLock(sessionsDir, "main-session")], }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 1, skipped: 0 }); expect(store["agent:main:main"]?.abortedLastRun).toBe(true); }); @@ -755,7 +764,7 @@ describe("main-session-restart-recovery", () => { cleanedLocks: [cleanedLock(sessionsDir, sessionId)], }); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(result).toEqual({ marked: 1, skipped: 0 }); expect(store["agent:main:main"]?.abortedLastRun).toBe(true); }); @@ -784,7 +793,7 @@ describe("main-session-restart-recovery", () => { expect(resumeParams.sessionKey).toBe("agent:main:main"); expect(resumeParams.deliver).toBe(false); expect(resumeParams.lane).toBe("main"); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(store["agent:main:main"]?.abortedLastRun).toBe(false); }); @@ -917,7 +926,7 @@ describe("main-session-restart-recovery", () => { expect(result).toEqual({ recovered: 0, failed: 1, skipped: 0 }); expect(callGateway).not.toHaveBeenCalled(); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(store["agent:main:main"]?.status).toBe("failed"); expect(store["agent:main:main"]?.abortedLastRun).toBe(true); }); @@ -966,7 +975,7 @@ describe("main-session-restart-recovery", () => { expect(firstGatewayParams().message).toContain(pendingPayload); const beforeStoreRead = Date.now(); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); const entry = store["agent:main:main"]; expect(entry?.abortedLastRun).toBe(false); expect(entry?.pendingFinalDelivery).toBe(true); @@ -1017,7 +1026,7 @@ describe("main-session-restart-recovery", () => { expect(firstGatewayParams().message).not.toContain(INTERNAL_RUNTIME_CONTEXT_BEGIN); expect(firstGatewayParams().message).not.toContain("Conversation info"); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(store["agent:main:main"]?.pendingFinalDeliveryText).toBe("The final answer is 42."); }); @@ -1044,7 +1053,7 @@ describe("main-session-restart-recovery", () => { expect(result).toEqual({ recovered: 1, failed: 0, skipped: 0 }); expect(callGateway).toHaveBeenCalledOnce(); expect(firstGatewayParams().message).toContain("assistant final was already captured"); - const store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(store["agent:main:main"]?.status).toBe("running"); expect(store["agent:main:main"]?.pendingFinalDelivery).toBe(true); expect(store["agent:main:main"]?.pendingFinalDeliveryText).toBe( @@ -1115,7 +1124,7 @@ describe("main-session-restart-recovery", () => { expect(result).toEqual({ recovered: 1, failed: 0, skipped: 2 }); expect(callGateway).toHaveBeenCalledOnce(); - const store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(store["agent:main:active-key"]?.abortedLastRun).toBe(true); expect(store["agent:main:active-id"]?.abortedLastRun).toBe(true); expect(store["agent:main:recoverable"]?.abortedLastRun).toBe(false); @@ -1144,7 +1153,7 @@ describe("main-session-restart-recovery", () => { expect(result).toEqual({ recovered: 1, failed: 0, skipped: 0 }); expect(callGateway).toHaveBeenCalledOnce(); - const store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(store["agent:main:main"]?.abortedLastRun).toBe(false); }); @@ -1224,7 +1233,7 @@ describe("main-session-restart-recovery", () => { }); expect(marked).toEqual({ marked: 1, skipped: 2 }); - let store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json")); + let store = readStore(path.join(sessionsDir, "sessions.json")); expect(store["agent:main:main"]?.abortedLastRun).toBe(true); expect(store["agent:main:active-key"]?.abortedLastRun).toBeUndefined(); expect(store["agent:main:active-id"]?.abortedLastRun).toBeUndefined(); @@ -1240,7 +1249,7 @@ describe("main-session-restart-recovery", () => { expect(recovered).toEqual({ recovered: 2, failed: 0, skipped: 0 }); expect(callGateway).toHaveBeenCalledTimes(2); - store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json")); + store = readStore(path.join(sessionsDir, "sessions.json")); expect(store["agent:main:main"]?.abortedLastRun).toBe(false); expect(store["agent:main:already-marked"]?.abortedLastRun).toBe(false); }); @@ -1261,7 +1270,7 @@ describe("main-session-restart-recovery", () => { ]); const customStorePath = path.join(tmpDir, "custom-startup-duplicate", "sessions.json"); - await writeSessionStoreForTestAsync(customStorePath, { + await writeStorePath(customStorePath, { "agent:main:main": { sessionId: "custom-main-session", updatedAt: cutoff - 10_000, @@ -1284,8 +1293,8 @@ describe("main-session-restart-recovery", () => { expect(result.skipped).toBeGreaterThanOrEqual(1); expect(result.skipped).toBeLessThanOrEqual(2); expect(callGateway).toHaveBeenCalledOnce(); - const defaultStore = readSessionStoreForTest(path.join(defaultSessionsDir, "sessions.json")); - const customStore = readSessionStoreForTest(customStorePath); + const defaultStore = readStore(path.join(defaultSessionsDir, "sessions.json")); + const customStore = readStore(customStorePath); expect(defaultStore["agent:main:main"]?.abortedLastRun).toBe(true); expect(customStore["agent:main:main"]?.abortedLastRun).toBe(false); }); @@ -1332,8 +1341,11 @@ describe("main-session-restart-recovery", () => { await vi.waitFor(() => { expect(callGateway).toHaveBeenCalledTimes(2); - const store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json")); - expect(store["agent:main:main"]?.abortedLastRun).toBe(false); + const entry = loadSessionEntry({ + storePath: path.join(sessionsDir, "sessions.json"), + sessionKey: "agent:main:main", + }); + expect(entry?.abortedLastRun).toBe(false); }); expect(getActiveGatewayRootWorkCount()).toBe(0); }); @@ -1357,7 +1369,7 @@ describe("main-session-restart-recovery", () => { expect(result).toEqual({ recovered: 0, failed: 1, skipped: 0 }); expect(callGateway).not.toHaveBeenCalled(); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(store["agent:main:main"]?.status).toBe("failed"); expect(store["agent:main:main"]?.abortedLastRun).toBe(true); }); @@ -1412,7 +1424,7 @@ describe("main-session-restart-recovery", () => { "couldn't safely resume", ); - const store = loadSessionStore(path.join(sessionsDir, "sessions.json")); + const store = readStore(path.join(sessionsDir, "sessions.json")); expect(store["agent:main:demo-channel:room-1"]?.status).toBe("failed"); expect(store["agent:main:demo-channel:room-1"]?.abortedLastRun).toBe(true); }); diff --git a/src/agents/main-session-restart-recovery.ts b/src/agents/main-session-restart-recovery.ts index c7a809548ad8..794976df5683 100644 --- a/src/agents/main-session-restart-recovery.ts +++ b/src/agents/main-session-restart-recovery.ts @@ -15,12 +15,14 @@ import { resolveStateDir } from "../config/paths.js"; import { type RestartRecoveryRun, type SessionEntry, - loadSessionStore, resolveAllAgentSessionStoreTargetsSync, resolveSessionFilePath, resolveSessionTranscriptPathInDir, } from "../config/sessions.js"; -import { applySessionEntryReplacements } from "../config/sessions/session-accessor.js"; +import { + applySessionEntryReplacements, + listSessionEntries, +} from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { callGateway } from "../gateway/call.js"; import { readSessionMessagesAsync } from "../gateway/session-transcript-readers.js"; @@ -743,17 +745,17 @@ async function recoverStore(params: { providedActiveSessionIds ?? normalizeStringSet(listActiveEmbeddedRunSessionIds()); const resolveActiveSessionKeys = () => providedActiveSessionKeys ?? normalizeStringSet(listActiveEmbeddedRunSessionKeys()); - let store: Record; + let entries: Array<{ sessionKey: string; entry: SessionEntry }>; try { - store = loadSessionStore(params.storePath); + entries = listSessionEntries({ storePath: params.storePath }); } catch (err) { log.warn(`failed to load session store ${params.storePath}: ${String(err)}`); result.failed++; return result; } - for (const [sessionKey, entry] of Object.entries(store).toSorted(([a], [b]) => - a.localeCompare(b), + for (const { sessionKey, entry } of entries.toSorted((a, b) => + a.sessionKey.localeCompare(b.sessionKey), )) { if (!entry || entry.status !== "running" || entry.abortedLastRun !== true) { continue; diff --git a/src/agents/openclaw-tools.sessions.test.ts b/src/agents/openclaw-tools.sessions.test.ts index 5ea9424afa0e..4e7dc930c97a 100644 --- a/src/agents/openclaw-tools.sessions.test.ts +++ b/src/agents/openclaw-tools.sessions.test.ts @@ -5,6 +5,10 @@ import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { ChannelMessagingAdapter } from "../channels/plugins/types.js"; import type { OpenClawConfig } from "../config/config.js"; +import { + appendTranscriptMessage, + upsertSessionEntry, +} from "../config/sessions/session-accessor.js"; import { createTestRegistry } from "../test-utils/channel-plugins.js"; const callGatewayMock = vi.fn(); @@ -560,23 +564,29 @@ describe("sessions tools", () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sessions-list-preview-")); const storePath = path.join(tmpDir, "sessions.json"); try { - fs.writeFileSync( - path.join(tmpDir, "visible.jsonl"), - [ - JSON.stringify({ type: "session", id: "visible" }), - JSON.stringify({ message: { role: "user", content: "Visible project kickoff" } }), - JSON.stringify({ message: { role: "assistant", content: "Visible latest reply" } }), - ].join("\n"), - "utf-8", + await upsertSessionEntry( + { agentId: "main", sessionKey: "agent:main:main", storePath }, + { sessionId: "visible", updatedAt: 20 }, ); - fs.writeFileSync( - path.join(tmpDir, "hidden.jsonl"), - [ - JSON.stringify({ type: "session", id: "hidden" }), - JSON.stringify({ message: { role: "user", content: "Hidden cross-agent topic" } }), - JSON.stringify({ message: { role: "assistant", content: "Hidden latest reply" } }), - ].join("\n"), - "utf-8", + await appendTranscriptMessage( + { agentId: "main", sessionId: "visible", sessionKey: "agent:main:main", storePath }, + { cwd: tmpDir, message: { role: "user", content: "Visible project kickoff" } }, + ); + await appendTranscriptMessage( + { agentId: "main", sessionId: "visible", sessionKey: "agent:main:main", storePath }, + { cwd: tmpDir, message: { role: "assistant", content: "Visible latest reply" } }, + ); + await upsertSessionEntry( + { agentId: "other", sessionKey: "agent:other:main", storePath }, + { sessionId: "hidden", updatedAt: 21 }, + ); + await appendTranscriptMessage( + { agentId: "other", sessionId: "hidden", sessionKey: "agent:other:main", storePath }, + { cwd: tmpDir, message: { role: "user", content: "Hidden cross-agent topic" } }, + ); + await appendTranscriptMessage( + { agentId: "other", sessionId: "hidden", sessionKey: "agent:other:main", storePath }, + { cwd: tmpDir, message: { role: "assistant", content: "Hidden latest reply" } }, ); callGatewayMock.mockImplementation(async (opts: unknown) => { diff --git a/src/agents/openclaw-tools.subagents.sessions-spawn.test-harness.ts b/src/agents/openclaw-tools.subagents.sessions-spawn.test-harness.ts index 585bae88ce09..61c808c3db41 100644 --- a/src/agents/openclaw-tools.subagents.sessions-spawn.test-harness.ts +++ b/src/agents/openclaw-tools.subagents.sessions-spawn.test-harness.ts @@ -227,7 +227,6 @@ export async function getSessionsSpawnTool(opts: CreateOpenClawToolsOpts) { maxTokens: 100_000, }, }), - updateSessionStore: async (_storePath, mutator) => mutator({}), }); cachedSubagentRegistryTesting.setDepsForTest({ callGateway: (optsUnknown) => hoisted.callGatewayMock(optsUnknown), diff --git a/src/agents/run-session-target.test.ts b/src/agents/run-session-target.test.ts index 1aafee898034..0940829f4fd8 100644 --- a/src/agents/run-session-target.test.ts +++ b/src/agents/run-session-target.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { loadSessionStore } from "../config/sessions/store.js"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveAgentRunSessionTarget } from "./run-session-target.js"; @@ -33,10 +33,11 @@ describe("agent run session target", () => { sessionId: "test-run", sessionKey, }); - expect(path.dirname(target.sessionFile)).toBe(path.dirname(storePath)); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]?.sessionFile).toBe( - target.sessionFile, - ); + expect(parseSqliteSessionFileMarker(target.sessionFile)).toEqual({ + agentId: "helper", + sessionId: "test-run", + storePath, + }); }); it("uses the agent from an agent-scoped session key when agentId is omitted", async () => { @@ -51,9 +52,43 @@ describe("agent run session target", () => { const helperStorePath = path.join(tempDir, "agents", "helper", "sessions.json"); expect(target.agentId).toBe("helper"); - expect(path.dirname(target.sessionFile)).toBe(path.dirname(helperStorePath)); - expect(loadSessionStore(helperStorePath, { skipCache: true })[sessionKey]?.sessionFile).toBe( - target.sessionFile, - ); + expect(parseSqliteSessionFileMarker(target.sessionFile)).toEqual({ + agentId: "helper", + sessionId: "helper-session", + storePath: helperStorePath, + }); + }); + + it("prefers typed runtime target identity over deprecated sessionFile", async () => { + const storePath = path.join(tempDir, "target-store", "sessions.json"); + const legacySessionFile = path.join(tempDir, "legacy-session.jsonl"); + + const target = await resolveAgentRunSessionTarget({ + agentId: "main", + config: { + session: { store: path.join(tempDir, "fallback", "sessions.json") }, + } as OpenClawConfig, + sessionFile: legacySessionFile, + sessionId: "legacy-session", + sessionKey: "agent:main:legacy-session", + sessionTarget: { + agentId: "worker", + sessionId: "runtime-session", + sessionKey: "agent:worker:runtime-session", + storePath, + }, + }); + + expect(target).toMatchObject({ + agentId: "worker", + sessionId: "runtime-session", + sessionKey: "agent:worker:runtime-session", + }); + expect(target.sessionFile).not.toBe(legacySessionFile); + expect(parseSqliteSessionFileMarker(target.sessionFile)).toEqual({ + agentId: "worker", + sessionId: "runtime-session", + storePath, + }); }); }); diff --git a/src/agents/run-session-target.ts b/src/agents/run-session-target.ts index 1622c738bd25..e810ae6fc895 100644 --- a/src/agents/run-session-target.ts +++ b/src/agents/run-session-target.ts @@ -19,7 +19,7 @@ export type AgentRunSessionTarget = { /** File-backed target resolved from the storage-neutral run identity. */ type ResolvedAgentRunSessionTarget = SessionTranscriptRuntimeTarget; -/** Resolves the active file-backed target used by current run/session internals. */ +/** Resolves the active runtime target used by current run/session internals. */ export async function resolveAgentRunSessionTarget(params: { agentId?: string; config?: OpenClawConfig; @@ -33,6 +33,22 @@ export async function resolveAgentRunSessionTarget(params: { const sessionId = normalizeOptionalString(sessionTarget?.sessionId) ?? params.sessionId; const sessionKey = normalizeOptionalString(sessionTarget?.sessionKey) ?? params.sessionKey; const effectiveAgentId = agentId ?? resolveAgentIdFromSessionKey(sessionKey); + if (sessionTarget && !sessionKey) { + throw new Error(`Cannot resolve run session target without a session key: ${sessionId}`); + } + if (sessionTarget && sessionKey) { + const storePath = + normalizeOptionalString(sessionTarget.storePath) ?? + resolveStorePath(params.config?.session?.store, { agentId: effectiveAgentId }); + return await resolveSessionTranscriptRuntimeTarget({ + ...(effectiveAgentId ? { agentId: effectiveAgentId } : {}), + sessionId, + sessionKey, + storePath, + ...(sessionTarget.threadId !== undefined ? { threadId: sessionTarget.threadId } : {}), + }); + } + const sessionFile = normalizeOptionalString(params.sessionFile); if (sessionFile) { return { @@ -45,15 +61,12 @@ export async function resolveAgentRunSessionTarget(params: { if (!sessionKey) { throw new Error(`Cannot resolve run session target without a session key: ${sessionId}`); } - const storePath = - normalizeOptionalString(sessionTarget?.storePath) ?? - resolveStorePath(params.config?.session?.store, { agentId: effectiveAgentId }); + const storePath = resolveStorePath(params.config?.session?.store, { agentId: effectiveAgentId }); return await resolveSessionTranscriptRuntimeTarget({ ...(effectiveAgentId ? { agentId: effectiveAgentId } : {}), sessionId, sessionKey, storePath, - ...(sessionTarget?.threadId !== undefined ? { threadId: sessionTarget.threadId } : {}), }); } diff --git a/src/agents/runtime-plugins.registry-reuse.test.ts b/src/agents/runtime-plugins.registry-reuse.test.ts index 226eaeadd900..e554210ae546 100644 --- a/src/agents/runtime-plugins.registry-reuse.test.ts +++ b/src/agents/runtime-plugins.registry-reuse.test.ts @@ -38,6 +38,8 @@ function createRegistryWithPlugin(pluginId: string): PluginRegistry { } beforeEach(() => { + clearPluginLoaderCache(); + resetPluginRuntimeStateForTest(); mocks.getCurrentPluginMetadataSnapshot.mockReset(); mocks.loadOpenClawPlugins.mockReset(); }); diff --git a/src/agents/session-file-repair.test.ts b/src/agents/session-file-repair.test.ts index f84b449c34df..eb85050be649 100644 --- a/src/agents/session-file-repair.test.ts +++ b/src/agents/session-file-repair.test.ts @@ -77,6 +77,18 @@ afterEach(async () => { }); describe("repairSessionFileIfNeeded", () => { + it("skips SQLite transcript markers instead of treating them as file paths", async () => { + const result = await repairSessionFileIfNeeded({ + sessionFile: "sqlite:main:session-1:/tmp/openclaw/sessions.json", + }); + + expect(result).toEqual({ + repaired: false, + droppedLines: 0, + reason: "sqlite transcript", + }); + }); + it("rewrites session files that contain malformed lines", async () => { const { file } = await createTempSessionPath(); const { header, message } = buildSessionHeaderAndMessage(); diff --git a/src/agents/session-file-repair.ts b/src/agents/session-file-repair.ts index 370b49e29d8c..6b3a1ead39f4 100644 --- a/src/agents/session-file-repair.ts +++ b/src/agents/session-file-repair.ts @@ -7,6 +7,7 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { sanitizeInlineImageBase64 } from "@openclaw/media-core/inline-image-data-url"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { replaceFileAtomic } from "../infra/replace-file.js"; import type { AgentMessage } from "./runtime/index.js"; import { makeMissingToolResult } from "./session-transcript-repair.js"; @@ -57,6 +58,10 @@ const sessionRepairCache = new Map(); export function invalidateSessionFileRepairCache(sessionFile: string): void { const trimmed = sessionFile.trim(); if (trimmed) { + if (parseSqliteSessionFileMarker(trimmed)) { + sessionRepairCache.delete(trimmed); + return; + } sessionRepairCache.delete(path.resolve(trimmed)); } } @@ -69,6 +74,9 @@ type SessionMessageEntry = { async function readSessionRepairSnapshot( sessionFile: string, ): Promise { + if (parseSqliteSessionFileMarker(sessionFile)) { + return undefined; + } try { const stat = await fs.stat(sessionFile, { bigint: true }); return { @@ -792,6 +800,9 @@ export async function repairSessionFileIfNeeded(params: { if (!sessionFileInput) { return { repaired: false, droppedLines: 0, reason: "missing session file" }; } + if (parseSqliteSessionFileMarker(sessionFileInput)) { + return { repaired: false, droppedLines: 0, reason: "sqlite transcript" }; + } const sessionFile = path.resolve(sessionFileInput); const beforeReadSnapshot = await readSessionRepairSnapshot(sessionFile); if (beforeReadSnapshot) { diff --git a/src/agents/session-tool-result-guard.transcript-events.test.ts b/src/agents/session-tool-result-guard.transcript-events.test.ts index c7c7d1d14a51..6c14047fb674 100644 --- a/src/agents/session-tool-result-guard.transcript-events.test.ts +++ b/src/agents/session-tool-result-guard.transcript-events.test.ts @@ -3,8 +3,8 @@ import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; import { SessionManager } from "openclaw/plugin-sdk/agent-sessions"; import { afterEach, describe, expect, it, vi } from "vitest"; import { - onSessionTranscriptUpdate, - type SessionTranscriptUpdate, + onInternalSessionTranscriptUpdate, + type InternalSessionTranscriptUpdate, } from "../sessions/transcript-events.js"; import { guardSessionManager } from "./session-tool-result-guard-wrapper.js"; @@ -19,8 +19,8 @@ afterEach(() => { describe("guardSessionManager transcript updates", () => { it("includes the session key when broadcasting appended non-tool-result messages", () => { - const updates: SessionTranscriptUpdate[] = []; - listeners.push(onSessionTranscriptUpdate((update) => updates.push(update))); + const updates: InternalSessionTranscriptUpdate[] = []; + listeners.push(onInternalSessionTranscriptUpdate((update) => updates.push(update))); const sm = SessionManager.inMemory(); const sessionFile = "/tmp/openclaw-session-message-events.jsonl"; @@ -86,8 +86,8 @@ describe("guardSessionManager transcript updates", () => { }); it("reuses cached transcript sequence for consecutive appended messages", () => { - const updates: SessionTranscriptUpdate[] = []; - listeners.push(onSessionTranscriptUpdate((update) => updates.push(update))); + const updates: InternalSessionTranscriptUpdate[] = []; + listeners.push(onInternalSessionTranscriptUpdate((update) => updates.push(update))); const sm = SessionManager.inMemory(); sm.appendMessage({ @@ -127,8 +127,8 @@ describe("guardSessionManager transcript updates", () => { it("caches real tool result sequence before final assistant messages", () => { // Tool results are persisted but not broadcast, so later visible messages must skip their seq. - const updates: SessionTranscriptUpdate[] = []; - listeners.push(onSessionTranscriptUpdate((update) => updates.push(update))); + const updates: InternalSessionTranscriptUpdate[] = []; + listeners.push(onInternalSessionTranscriptUpdate((update) => updates.push(update))); const sm = SessionManager.inMemory(); sm.appendMessage({ diff --git a/src/agents/session-write-lock.ts b/src/agents/session-write-lock.ts index 6199badfa973..c22146e4cb7e 100644 --- a/src/agents/session-write-lock.ts +++ b/src/agents/session-write-lock.ts @@ -8,6 +8,7 @@ import type fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { createFileLockManager } from "../infra/file-lock-manager.js"; import { readGatewayProcessArgsSync as readProcessArgsSync } from "../infra/gateway-processes.js"; import { getProcessStartTime, isPidAlive } from "../shared/pid-alive.js"; @@ -448,6 +449,20 @@ async function resolveNormalizedSessionFile(sessionFile: string): Promise ({ resolveThinkingDefaultForModel: vi.fn(() => "medium"), @@ -200,7 +201,7 @@ describe("AgentSession queued user turns", () => { text: "visible group prompt", sender: { id: "user-42", name: "Ada" }, }, - target: { transcriptPath: "/tmp/unused-session.jsonl" }, + target: createTestUserTurnTranscriptTarget(), }); const steer = vi.spyOn(session.agent, "steer").mockImplementation(() => undefined); diff --git a/src/agents/sessions/session-manager.test.ts b/src/agents/sessions/session-manager.test.ts index cd713e2c25f7..91bc45c4eb7b 100644 --- a/src/agents/sessions/session-manager.test.ts +++ b/src/agents/sessions/session-manager.test.ts @@ -5,6 +5,13 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { + appendTranscriptMessage, + loadSessionEntry, + loadTranscriptEvents, + upsertSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import { withOwnedSessionTranscriptWrites } from "../../config/sessions/transcript-write-context.js"; import * as Logger from "../../logger.js"; import { isTranscriptOnlyOpenClawAssistantMessage } from "../../shared/transcript-only-openclaw-assistant.js"; @@ -34,6 +41,401 @@ describe("SessionManager.open", () => { ); }); + it("opens SQLite markers without creating marker-named files and persists assistant replies", async () => { + const dir = await makeTempDir(); + const storePath = path.join(dir, "sessions.json"); + const sessionId = "sqlite-session"; + const sessionKey = "agent:main:dashboard:sqlite"; + const marker = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + await upsertSessionEntry( + { agentId: "main", sessionKey, storePath }, + { + sessionFile: marker, + sessionId, + updatedAt: 10, + }, + ); + await appendTranscriptMessage( + { agentId: "main", sessionId, sessionKey, storePath }, + { + cwd: dir, + message: { role: "user", content: "question" }, + }, + ); + + const sessionManager = SessionManager.open(marker, dir, dir); + expect(sessionManager.buildSessionContext().messages).toEqual([ + expect.objectContaining({ content: "question", role: "user" }), + ]); + + const assistantId = sessionManager.appendMessage({ + role: "assistant", + content: [{ type: "text", text: "answer" }], + api: "openai-responses", + provider: "openai", + model: "gpt-5.5", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + total: 0, + }, + }, + stopReason: "stop", + timestamp: Date.now(), + }); + const thinkingChangeId = sessionManager.appendThinkingLevelChange("high"); + const modelChangeId = sessionManager.appendModelChange("openai", "gpt-5.5"); + const compactionId = sessionManager.appendCompaction("summary", "assistant-1", 42); + + await expect(fs.stat(path.join(process.cwd(), marker))).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect( + loadTranscriptEvents({ agentId: "main", sessionId, sessionKey, storePath }), + ).resolves.toEqual([ + expect.objectContaining({ type: "session" }), + expect.objectContaining({ + message: expect.objectContaining({ content: "question", role: "user" }), + type: "message", + }), + expect.objectContaining({ + id: assistantId, + parentId: expect.any(String), + message: expect.objectContaining({ + content: [{ type: "text", text: "answer" }], + role: "assistant", + }), + type: "message", + }), + expect.objectContaining({ + id: thinkingChangeId, + thinkingLevel: "high", + type: "thinking_level_change", + }), + expect.objectContaining({ + id: modelChangeId, + modelId: "gpt-5.5", + provider: "openai", + type: "model_change", + }), + expect.objectContaining({ + firstKeptEntryId: "assistant-1", + id: compactionId, + summary: "summary", + type: "compaction", + }), + ]); + const reopened = SessionManager.open(marker, dir, dir); + expect(reopened.getEntries()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: thinkingChangeId, type: "thinking_level_change" }), + expect.objectContaining({ id: modelChangeId, type: "model_change" }), + expect.objectContaining({ id: compactionId, type: "compaction" }), + ]), + ); + }); + + it("persists prompt-released leaf controls through SQLite markers", async () => { + const dir = await makeTempDir(); + const storePath = path.join(dir, "sessions.json"); + const sessionId = "sqlite-prompt-release"; + const sessionKey = "agent:main:dashboard:sqlite-prompt-release"; + const marker = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + const scope = { agentId: "main", sessionId, sessionKey, storePath }; + await upsertSessionEntry( + { agentId: "main", sessionKey, storePath }, + { + sessionFile: marker, + sessionId, + updatedAt: 10, + }, + ); + const user = await appendTranscriptMessage(scope, { + cwd: dir, + eventId: "user-message", + message: { role: "user", content: "question" }, + }); + const assistant = await appendTranscriptMessage(scope, { + cwd: dir, + eventId: "base-answer", + message: buildAssistantMessage("base answer"), + parentId: user.messageId, + }); + const sessionManager = SessionManager.open(marker, dir, dir); + const sideEntry = { + type: "message" as const, + id: "side-delivery", + parentId: assistant.messageId, + timestamp: "2026-06-15T00:00:03.000Z", + message: buildAssistantMessage("side delivery"), + }; + await appendTranscriptMessage(scope, { + cwd: dir, + eventId: sideEntry.id, + message: sideEntry.message, + parentId: sideEntry.parentId, + }); + + const mergeResult = sessionManager.mergePromptReleasedSessionEntries([sideEntry], { + persistLeaf: true, + }); + + expect(mergeResult?.publishedEntries).toEqual([{ kind: "id", id: expect.any(String) }]); + const records = await loadTranscriptEvents(scope); + expect(records.at(-1)).toMatchObject({ + type: "leaf", + parentId: sideEntry.id, + targetId: assistant.messageId, + appendParentId: sideEntry.id, + appendMode: "side", + }); + await expect(fs.stat(path.join(process.cwd(), marker))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("reloads SQLite markers through setSessionFile without switching to file paths", async () => { + const dir = await makeTempDir(); + const storePath = path.join(dir, "sessions.json"); + const sessionId = "sqlite-marker-reload"; + const sessionKey = "agent:main:dashboard:sqlite-marker-reload"; + const marker = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + const scope = { agentId: "main", sessionId, sessionKey, storePath }; + await upsertSessionEntry( + { agentId: "main", sessionKey, storePath }, + { + sessionFile: marker, + sessionId, + updatedAt: 10, + }, + ); + await appendTranscriptMessage(scope, { + cwd: dir, + eventId: "user-message", + message: { role: "user", content: "question before reload" }, + }); + + const sessionManager = SessionManager.open(marker, dir, dir); + sessionManager.setSessionFile(marker); + expect(sessionManager.buildSessionContext().messages).toEqual([ + expect.objectContaining({ content: "question before reload", role: "user" }), + ]); + sessionManager.appendMessage(buildAssistantMessage("answer after reload")); + + await expect(fs.stat(path.join(process.cwd(), marker))).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(loadTranscriptEvents(scope)).resolves.toEqual([ + expect.objectContaining({ type: "session" }), + expect.objectContaining({ + message: expect.objectContaining({ content: "question before reload", role: "user" }), + type: "message", + }), + expect.objectContaining({ + message: expect.objectContaining({ + content: [{ type: "text", text: "answer after reload" }], + role: "assistant", + }), + type: "message", + }), + ]); + }); + + it("creates SQLite-backed branch sessions without rewriting the source transcript", async () => { + const dir = await makeTempDir(); + const storePath = path.join(dir, "sessions.json"); + const sessionId = "sqlite-branch-source"; + const sessionKey = "agent:main:dashboard:sqlite-branch-source"; + const marker = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + const scope = { agentId: "main", sessionId, sessionKey, storePath }; + await upsertSessionEntry( + { agentId: "main", sessionKey, storePath }, + { + channel: "dashboard", + sessionFile: marker, + sessionId, + updatedAt: 10, + }, + ); + const user = await appendTranscriptMessage(scope, { + cwd: dir, + eventId: "user-message", + message: { role: "user", content: "question before branch" }, + }); + const assistant = await appendTranscriptMessage(scope, { + cwd: dir, + eventId: "assistant-message", + message: buildAssistantMessage("answer before branch"), + parentId: user.messageId, + }); + + const sessionManager = SessionManager.open(marker, dir, dir); + const branchedMarker = sessionManager.createBranchedSession(assistant.messageId); + const branchedSessionId = sessionManager.getSessionId(); + + expect(branchedMarker).toContain(`sqlite:main:${branchedSessionId}:`); + expect(branchedSessionId).not.toBe(sessionId); + expect(loadSessionEntry({ agentId: "main", sessionKey, storePath })).toMatchObject({ + channel: "dashboard", + sessionFile: branchedMarker, + sessionId: branchedSessionId, + }); + await expect(fs.stat(path.join(process.cwd(), branchedMarker!))).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(loadTranscriptEvents({ agentId: "main", sessionId, storePath })).resolves.toEqual([ + expect.objectContaining({ id: sessionId, type: "session" }), + expect.objectContaining({ id: user.messageId, type: "message" }), + expect.objectContaining({ id: assistant.messageId, type: "message" }), + ]); + await expect( + loadTranscriptEvents({ + agentId: "main", + sessionId: branchedSessionId, + sessionKey, + storePath, + }), + ).resolves.toEqual([ + expect.objectContaining({ + id: branchedSessionId, + parentSession: marker, + type: "session", + }), + expect.objectContaining({ id: user.messageId, type: "message" }), + expect.objectContaining({ id: assistant.messageId, type: "message" }), + ]); + }); + + it("persists user turns when a SQLite marker has no external recorder", async () => { + const dir = await makeTempDir(); + const storePath = path.join(dir, "sessions.json"); + const sessionId = "sqlite-direct-user-session"; + const sessionKey = "agent:main:voice:direct-user"; + const marker = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + await upsertSessionEntry( + { agentId: "main", sessionKey, storePath }, + { + sessionFile: marker, + sessionId, + updatedAt: 10, + }, + ); + + const sessionManager = SessionManager.open(marker, dir, dir); + const userId = sessionManager.appendMessage({ + role: "user", + content: "voice prompt", + timestamp: Date.now(), + }); + + await expect( + loadTranscriptEvents({ agentId: "main", sessionId, sessionKey, storePath }), + ).resolves.toContainEqual( + expect.objectContaining({ + id: userId, + message: expect.objectContaining({ content: "voice prompt", role: "user" }), + type: "message", + }), + ); + }); + + it("rewrites SQLite transcript rows when removing trailing entries", async () => { + const dir = await makeTempDir(); + const storePath = path.join(dir, "sessions.json"); + const sessionId = "sqlite-remove-trailing-session"; + const sessionKey = "agent:main:dashboard:sqlite-remove-trailing"; + const marker = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + const scope = { agentId: "main", sessionId, sessionKey, storePath }; + await upsertSessionEntry( + { agentId: "main", sessionKey, storePath }, + { + sessionFile: marker, + sessionId, + updatedAt: 10, + }, + ); + const user = await appendTranscriptMessage(scope, { + cwd: dir, + eventId: "user-message", + message: { role: "user", content: "question" }, + }); + const baseAnswer = await appendTranscriptMessage(scope, { + cwd: dir, + eventId: "base-answer", + message: buildAssistantMessage("base answer"), + parentId: user.messageId, + }); + const temporaryError = await appendTranscriptMessage(scope, { + cwd: dir, + eventId: "temporary-error", + message: buildAssistantMessage("temporary error"), + parentId: baseAnswer.messageId, + }); + + const sessionManager = SessionManager.open(marker, dir, dir); + + expect( + sessionManager.removeTrailingEntries((entry) => entry.id === temporaryError.messageId), + ).toBe(1); + expect(sessionManager.getLeafId()).toBe(baseAnswer.messageId); + const replacementId = sessionManager.appendMessage(buildAssistantMessage("replacement answer")); + + const records = await loadTranscriptEvents(scope); + expect( + records.map((record) => + record && typeof record === "object" && "id" in record ? record.id : undefined, + ), + ).not.toContain(temporaryError.messageId); + expect(records).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: replacementId, + message: expect.objectContaining({ + content: [{ type: "text", text: "replacement answer" }], + role: "assistant", + }), + parentId: baseAnswer.messageId, + type: "message", + }), + ]), + ); + await expect(fs.stat(path.join(process.cwd(), marker))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + it("recovers a corrupted first-line header without truncating later messages", async () => { // A damaged header should be repairable without treating valid later // message entries as disposable transcript state. @@ -1824,9 +2226,9 @@ describe("SessionManager.open", () => { sessionManager.mergePromptReleasedSessionEntries([deliveryEntry]); ( sessionManager as unknown as { - rewriteFile: () => void; + replacePersistedTranscript: () => void; } - ).rewriteFile(); + ).replacePersistedTranscript(); const sessionFile = sessionManager.getSessionFile(); expect(sessionFile).toBeDefined(); @@ -2156,9 +2558,9 @@ describe("SessionManager.open", () => { sessionManager.mergePromptReleasedSessionEntries([deliveryEntry]); ( sessionManager as unknown as { - rewriteFile: () => void; + replacePersistedTranscript: () => void; } - ).rewriteFile(); + ).replacePersistedTranscript(); const sessionFile = sessionManager.getSessionFile(); expect(sessionFile).toBeDefined(); @@ -2222,9 +2624,9 @@ describe("SessionManager.open", () => { ]); ( sessionManager as unknown as { - rewriteFile: () => void; + replacePersistedTranscript: () => void; } - ).rewriteFile(); + ).replacePersistedTranscript(); const rewritten = (await fs.readFile(sessionFile, "utf-8")) .trim() @@ -2404,9 +2806,9 @@ describe("SessionManager.open", () => { ]); ( sessionManager as unknown as { - rewriteFile: () => void; + replacePersistedTranscript: () => void; } - ).rewriteFile(); + ).replacePersistedTranscript(); expect(sessionManager.getLeafId()).toBe(baseAnswerId); const sessionFile = sessionManager.getSessionFile(); diff --git a/src/agents/sessions/session-manager.ts b/src/agents/sessions/session-manager.ts index 61e54fddcaeb..546f9ecdaad9 100644 --- a/src/agents/sessions/session-manager.ts +++ b/src/agents/sessions/session-manager.ts @@ -19,6 +19,20 @@ import { import { readdir, readFile, stat } from "node:fs/promises"; import { join, resolve } from "node:path"; import { isProxy } from "node:util/types"; +import { + appendTranscriptEventSync, + appendTranscriptMessageSync, + loadSessionEntry, + loadTranscriptEventsSync, + replaceSessionEntrySync, + replaceTranscriptEventsSync, + resolveTranscriptSessionKeyBySessionId, +} from "../../config/sessions/session-accessor.js"; +import { + formatSqliteSessionFileMarker, + parseSqliteSessionFileMarker, + type SqliteSessionFileMarker, +} from "../../config/sessions/sqlite-marker.js"; import { appendJsonlEntrySync, appendSerializedJsonlEntrySync, @@ -157,11 +171,15 @@ type PromptReleasedSessionEntry = | PromptReleasedOpaqueEntry; type PromptReleasedSessionMergeResult = { - sessionFileSnapshot: OwnedSessionTranscriptCacheSnapshot; + sessionFileSnapshot?: OwnedSessionTranscriptCacheSnapshot; publishedEntries?: readonly OwnedSessionTranscriptPublishedEntry[]; requiresReload?: true; }; +type SqliteSessionManagerPersistence = SqliteSessionFileMarker & { + sessionKey: string; +}; + /** * Custom message entry for extensions to inject messages into LLM context. * Use customType to identify your extension's entries. @@ -892,6 +910,37 @@ function revalidateLoadedSessionFile( return loadEntriesFromFileWithSnapshot(filePath); } +function loadSqliteMarkedSessionFile( + sessionFile: string, + options: { cwdOverride?: string; fallbackCwd?: string } = {}, +): + | { + cwd: string; + entries: FileEntry[]; + sessionKey: string; + sqliteMarker: SqliteSessionFileMarker; + } + | undefined { + const sqliteMarker = parseSqliteSessionFileMarker(sessionFile); + if (!sqliteMarker) { + return undefined; + } + const sessionKey = resolveTranscriptSessionKeyBySessionId(sqliteMarker); + if (!sessionKey) { + throw new Error(`Cannot open SQLite session without session entry: ${sqliteMarker.sessionId}`); + } + const entries = loadTranscriptEventsSync(sqliteMarker) as FileEntry[]; + const header = entries.find((e) => isJsonRecord(e) && e.type === "session") as + | SessionHeader + | undefined; + return { + cwd: options.cwdOverride ?? header?.cwd ?? options.fallbackCwd ?? process.cwd(), + entries, + sessionKey, + sqliteMarker, + }; +} + // Cached entries are deep-frozen so warm hits cannot drift from the file bytes // that validated the cache key. The session header (entries[0]) is the one entry // callers legitimately mutate in place — `prepareSessionManagerForRun` rewrites @@ -967,7 +1016,13 @@ function parseJsonlEntries(content: string): FileEntry[] { return entries; } -function normalizeLoadedFileEntry(entry: FileEntry): FileEntry { +/** + * Repairs legacy JSONL message shapes (string assistant/toolResult content) into + * canonical block arrays. Every legacy-JSONL ingress point — file-era transcript + * reads and the doctor SQLite import — must apply this so the SQLite store only + * ever holds canonical rows; the SQLite read path does no repair. + */ +export function normalizeLoadedFileEntry(entry: FileEntry): FileEntry { if (!isJsonRecord(entry) || entry.type !== "message" || !isJsonRecord(entry.message)) { return entry; } @@ -1498,6 +1553,7 @@ export class SessionManager { private promptReleasedSideBranchParentId: string | null | undefined; private recoveredCorruptHeader = false; private sessionFileSnapshot: SessionFileSnapshot | undefined; + private sqlitePersistence: SqliteSessionManagerPersistence | undefined; private constructor( cwd: string, @@ -1508,19 +1564,28 @@ export class SessionManager { entries: FileEntry[]; snapshot: SessionFileSnapshot | undefined; }, + sqlitePersistence?: SqliteSessionManagerPersistence, ) { this.cwd = cwd; this.sessionDir = sessionDir; this.shouldPersist = persist; + this.sqlitePersistence = sqlitePersistence; if (persist && sessionDir && !existsSync(sessionDir)) { mkdirSync(sessionDir, { recursive: true }); } if (sessionFile) { - this.setLoadedSessionFile( - sessionFile, - loadedSessionFile ?? loadEntriesFromFileWithSnapshot(sessionFile), - ); + if (sqlitePersistence) { + this.setLoadedSqliteSessionFile( + sessionFile, + loadedSessionFile ?? { entries: [], snapshot: undefined }, + ); + } else { + this.setLoadedSessionFile( + sessionFile, + loadedSessionFile ?? loadEntriesFromFileWithSnapshot(sessionFile), + ); + } } else { this.newSession(); } @@ -1528,6 +1593,20 @@ export class SessionManager { /** Switch to a different session file (used for resume and branching) */ setSessionFile(sessionFile: string): void { + const sqliteLoaded = loadSqliteMarkedSessionFile(sessionFile, { fallbackCwd: this.cwd }); + if (sqliteLoaded) { + this.cwd = sqliteLoaded.cwd; + this.sqlitePersistence = { + ...sqliteLoaded.sqliteMarker, + sessionKey: sqliteLoaded.sessionKey, + }; + this.setLoadedSqliteSessionFile(sessionFile, { + entries: sqliteLoaded.entries, + snapshot: undefined, + }); + return; + } + this.sqlitePersistence = undefined; this.setLoadedSessionFile(sessionFile, loadEntriesFromFileWithSnapshot(sessionFile)); } @@ -1559,7 +1638,7 @@ export class SessionManager { this.sessionId = header?.id ?? createSessionId(); migrateToCurrentVersion(this.fileEntries, recovered.fileEntriesByOriginalIndex); this.buildIndex(); - this.rewriteFile(); + this.replacePersistedTranscript(); this.recoveredCorruptHeader = true; this.flushed = true; return; @@ -1568,7 +1647,7 @@ export class SessionManager { const explicitPath = this.sessionFile; this.newSession(); this.sessionFile = explicitPath; - this.rewriteFile(); + this.replacePersistedTranscript(); this.flushed = true; return; } @@ -1582,7 +1661,7 @@ export class SessionManager { ); this.buildIndex(); if (migrated) { - this.rewriteFile(); + this.replacePersistedTranscript(); } this.flushed = true; @@ -1593,6 +1672,31 @@ export class SessionManager { } } + private setLoadedSqliteSessionFile( + sessionFile: string, + loaded: { + entries: FileEntry[]; + snapshot: SessionFileSnapshot | undefined; + }, + ): void { + this.sessionFile = sessionFile; + this.sessionFileSnapshot = undefined; + this.recoveredCorruptHeader = false; + const partitioned = partitionSessionFileEntries(loaded.entries); + if (partitioned.fileEntries.length === 0) { + this.newSession({ id: this.sqlitePersistence?.sessionId }); + this.sessionFile = sessionFile; + return; + } + this.fileEntries = partitioned.fileEntries; + this.opaqueFileEntries = partitioned.opaqueEntries; + const header = this.fileEntries.find((e) => e.type === "session"); + this.sessionId = header?.id ?? this.sqlitePersistence?.sessionId ?? createSessionId(); + migrateToCurrentVersion(this.fileEntries, partitioned.fileEntriesByOriginalIndex); + this.buildIndex(); + this.flushed = true; + } + newSession(options?: NewSessionOptions): string | undefined { this.recoveredCorruptHeader = false; this.sessionFileSnapshot = undefined; @@ -1962,16 +2066,32 @@ export class SessionManager { ); } - private rewriteFile(options?: { + private replacePersistedTranscript(options?: { publishSnapshot?: boolean; leafAppendParentId?: string | null; leafAppendMode?: "side"; }): void { - if (!this.shouldPersist || !this.sessionFile) { + if (!this.shouldPersist) { return; } const leafAppendParentId = options?.leafAppendParentId === undefined ? this.appendParentId : options.leafAppendParentId; + if (this.sqlitePersistence) { + replaceTranscriptEventsSync( + { + agentId: this.sqlitePersistence.agentId, + sessionId: this.sqlitePersistence.sessionId, + sessionKey: this.sqlitePersistence.sessionKey, + storePath: this.sqlitePersistence.storePath, + }, + this.getPersistedFileEntries(leafAppendParentId, options?.leafAppendMode), + ); + this.flushed = true; + return; + } + if (!this.sessionFile) { + return; + } const content = this.writeFullFile(leafAppendParentId, options?.leafAppendMode); const rememberedWrite = rememberWrittenSessionEntries(this.sessionFile, content); this.sessionFileSnapshot = rememberedWrite.snapshot; @@ -2115,7 +2235,7 @@ export class SessionManager { this.buildIndex(); this.leafId = this.resolveCanonicalParentId(replacementParentId); this.appendParentId = replacementParentId; - this.rewriteFile(); + this.replacePersistedTranscript(); return removedEntries.length; } @@ -2124,6 +2244,10 @@ export class SessionManager { options?: AppendPersistenceOptions, publishSnapshot = true, ): void { + if (this.sqlitePersistence) { + this.persistSqliteRecord(entry); + return; + } if (!this.shouldPersist || !this.sessionFile) { return; } @@ -2195,6 +2319,33 @@ export class SessionManager { this.persistRecord(entry, options); } + private persistSqliteRecord(entry: unknown): void { + if (!isIndexedSessionEntry(entry)) { + return; + } + const persistence = this.sqlitePersistence; + if (!persistence) { + return; + } + const scope = { + agentId: persistence.agentId, + sessionId: persistence.sessionId, + sessionKey: persistence.sessionKey, + storePath: persistence.storePath, + }; + if (entry.type !== "message") { + appendTranscriptEventSync(scope, entry); + return; + } + appendTranscriptMessageSync(scope, { + cwd: this.cwd, + eventId: entry.id, + message: entry.message, + now: Date.parse(entry.timestamp), + parentId: entry.parentId, + }); + } + /** * Resync the in-memory snapshot/cache after the transcript file was rewritten * out-of-band (for example the embedded-run header normalization in @@ -2321,8 +2472,23 @@ export class SessionManager { const hasAssistant = this.fileEntries.some( (entry) => entry.type === "message" && entry.message.role === "assistant", ); + if (this.sqlitePersistence) { + const leafEntry = this.createLeafControl(rawTailId, sideBranchParentId, "side"); + appendTranscriptEventSync( + { + agentId: this.sqlitePersistence.agentId, + sessionId: this.sqlitePersistence.sessionId, + sessionKey: this.sqlitePersistence.sessionKey, + storePath: this.sqlitePersistence.storePath, + }, + leafEntry, + ); + this.rememberLeafControl(leafEntry); + this.flushed = true; + return { publishedEntries: [{ kind: "id", id: leafEntry.id }] }; + } if (!this.flushed || !hasAssistant) { - this.rewriteFile({ + this.replacePersistedTranscript({ publishSnapshot: false, leafAppendParentId: sideBranchParentId, leafAppendMode: "side", @@ -2873,7 +3039,14 @@ export class SessionManager { const newSessionId = createSessionId(); const timestamp = new Date().toISOString(); const fileTimestamp = timestamp.replace(/[:.]/g, "-"); - const newSessionFile = join(this.getSessionDir(), `${fileTimestamp}_${newSessionId}.jsonl`); + const sqlitePersistence = this.sqlitePersistence; + const newSessionFile = sqlitePersistence + ? formatSqliteSessionFileMarker({ + agentId: sqlitePersistence.agentId, + sessionId: newSessionId, + storePath: sqlitePersistence.storePath, + }) + : join(this.getSessionDir(), `${fileTimestamp}_${newSessionId}.jsonl`); const header: SessionHeader = { type: "session", @@ -2916,6 +3089,28 @@ export class SessionManager { this.sessionId = newSessionId; this.sessionFile = newSessionFile; this.sessionFileSnapshot = undefined; + if (sqlitePersistence) { + const updatedAt = Date.now(); + const previousEntry = loadSessionEntry({ + agentId: sqlitePersistence.agentId, + sessionKey: sqlitePersistence.sessionKey, + storePath: sqlitePersistence.storePath, + }); + this.sqlitePersistence = { ...sqlitePersistence, sessionId: newSessionId }; + replaceSessionEntrySync( + { + agentId: sqlitePersistence.agentId, + sessionKey: sqlitePersistence.sessionKey, + storePath: sqlitePersistence.storePath, + }, + { + ...(previousEntry ?? { updatedAt }), + sessionFile: newSessionFile, + sessionId: newSessionId, + updatedAt, + }, + ); + } this.buildIndex(); // Only write the file now if it contains an assistant message. @@ -2927,7 +3122,7 @@ export class SessionManager { (e) => e.type === "message" && e.message.role === "assistant", ); if (hasAssistant) { - this.rewriteFile(); + this.replacePersistedTranscript(); this.flushed = true; } else { this.flushed = false; @@ -2976,6 +3171,17 @@ export class SessionManager { * @param cwdOverride Optional cwd override instead of the session header cwd. */ static open(path: string, sessionDir?: string, cwdOverride?: string): SessionManager { + const sqliteLoaded = loadSqliteMarkedSessionFile(path, { cwdOverride }); + if (sqliteLoaded) { + return new SessionManager( + sqliteLoaded.cwd, + sessionDir ?? "", + path, + true, + { entries: sqliteLoaded.entries, snapshot: undefined }, + { ...sqliteLoaded.sqliteMarker, sessionKey: sqliteLoaded.sessionKey }, + ); + } // Re-stat before construction so the single parsed load cannot become // stale while deriving cwd/session metadata. Stable warm opens pay only // the extra stat; changed files retry through the normal loader. diff --git a/src/agents/subagent-announce-delivery.runtime.ts b/src/agents/subagent-announce-delivery.runtime.ts index aa3096b57e5d..2bad2d695395 100644 --- a/src/agents/subagent-announce-delivery.runtime.ts +++ b/src/agents/subagent-announce-delivery.runtime.ts @@ -5,11 +5,8 @@ * message routing, queue settings, hooks, and embedded-run state. */ export { getRuntimeConfig } from "../config/config.js"; -export { - loadSessionStore, - resolveAgentIdFromSessionKey, - resolveStorePath, -} from "../config/sessions.js"; +export { resolveAgentIdFromSessionKey, resolveStorePath } from "../config/sessions.js"; +export { loadSessionEntry } from "../config/sessions/session-accessor.js"; export { callGateway } from "../gateway/call.js"; export { dispatchGatewayMethodInProcess } from "../gateway/server-plugins.js"; export { resolveQueueSettings } from "../auto-reply/reply/queue.js"; diff --git a/src/agents/subagent-announce-delivery.ts b/src/agents/subagent-announce-delivery.ts index b84d858c177e..df50eea0369b 100644 --- a/src/agents/subagent-announce-delivery.ts +++ b/src/agents/subagent-announce-delivery.ts @@ -64,7 +64,7 @@ import { isEmbeddedRunAbandoned, getRuntimeConfig, formatEmbeddedAgentQueueFailureSummary, - loadSessionStore, + loadSessionEntry, queueEmbeddedAgentMessageWithOutcomeAsync, resolveActiveEmbeddedRunSessionId, resolveAgentIdFromSessionKey, @@ -721,8 +721,11 @@ export function loadRequesterSessionEntry(requesterSessionKey: string) { const canonicalKey = resolveRequesterStoreKey(cfg, requesterSessionKey); const agentId = resolveAgentIdFromSessionKey(canonicalKey); const storePath = resolveStorePath(cfg.session?.store, { agentId }); - const store = loadSessionStore(storePath); - const entry = store[canonicalKey]; + const entry = loadSessionEntry({ + storePath, + sessionKey: canonicalKey, + clone: false, + }); return { cfg, entry, canonicalKey }; } @@ -730,8 +733,11 @@ export function loadSessionEntryByKey(sessionKey: string) { const cfg = subagentAnnounceDeliveryDeps.getRuntimeConfig(); const agentId = resolveAgentIdFromSessionKey(sessionKey); const storePath = resolveStorePath(cfg.session?.store, { agentId }); - const store = loadSessionStore(storePath); - return store[sessionKey]; + return loadSessionEntry({ + storePath, + sessionKey, + clone: false, + }); } async function maybeSteerSubagentAnnounce(params: { diff --git a/src/agents/subagent-announce.format.e2e.test.ts b/src/agents/subagent-announce.format.e2e.test.ts index d5b262500912..e5d81410044b 100644 --- a/src/agents/subagent-announce.format.e2e.test.ts +++ b/src/agents/subagent-announce.format.e2e.test.ts @@ -8,6 +8,7 @@ import { type OpenClawConfig, } from "../config/config.js"; import * as configSessions from "../config/sessions.js"; +import * as sessionAccessor from "../config/sessions/session-accessor.js"; import type { SessionEntry } from "../config/sessions/types.js"; import * as gatewayCall from "../gateway/call.js"; import { @@ -124,6 +125,7 @@ function expectAgentCallFields( const agentSpy = vi.fn(async (_req: AgentCallRequest) => visibleAgentResponse()); const sendSpy = vi.fn(async (_req: AgentCallRequest) => ({ runId: "send-main", status: "ok" })); const sessionsDeleteSpy = vi.fn((_req: AgentCallRequest) => undefined); +const loadSessionEntrySpy = vi.spyOn(sessionAccessor, "loadSessionEntry"); const loadSessionStoreSpy = vi.spyOn(configSessions, "loadSessionStore"); const resolveAgentIdFromSessionKeySpy = vi.spyOn(configSessions, "resolveAgentIdFromSessionKey"); const resolveStorePathSpy = vi.spyOn(configSessions, "resolveStorePath"); @@ -414,6 +416,9 @@ describe("subagent announce formatting", () => { resolveAgentIdFromSessionKey: () => "main", resolveStorePath: () => "/tmp/sessions.json", }); + loadSessionEntrySpy + .mockReset() + .mockImplementation((scope) => loadSessionStoreFixture()[scope.sessionKey]); loadSessionStoreSpy.mockReset().mockImplementation(() => loadSessionStoreFixture()); resolveAgentIdFromSessionKeySpy.mockReset().mockImplementation(() => "main"); resolveStorePathSpy.mockReset().mockImplementation(() => "/tmp/sessions.json"); diff --git a/src/agents/subagent-announce.runtime.ts b/src/agents/subagent-announce.runtime.ts index 0b59c26a2278..93041cc1a6ff 100644 --- a/src/agents/subagent-announce.runtime.ts +++ b/src/agents/subagent-announce.runtime.ts @@ -6,7 +6,6 @@ */ export { getRuntimeConfig } from "../config/config.js"; export { - loadSessionStore, readSessionEntry, resolveAgentIdFromSessionKey, resolveStorePath, diff --git a/src/agents/subagent-announce.test-support.ts b/src/agents/subagent-announce.test-support.ts index f2d9f88ace3e..37bf066434fc 100644 --- a/src/agents/subagent-announce.test-support.ts +++ b/src/agents/subagent-announce.test-support.ts @@ -72,6 +72,10 @@ export function createSubagentAnnounceDeliveryRuntimeMock(options: DeliveryRunti timeoutMs: callOptions?.timeoutMs, })) as T) as typeof dispatchGatewayMethodInProcess, getRuntimeConfig: options.getRuntimeConfig, + loadSessionEntry: (scope: { storePath?: string; sessionKey: string }) => + (options.loadSessionStore(scope.storePath ?? "") as Record)[ + scope.sessionKey + ], loadSessionStore: options.loadSessionStore, resolveAgentIdFromSessionKey: options.resolveAgentIdFromSessionKey, resolveMainSessionKey: options.resolveMainSessionKey, diff --git a/src/agents/subagent-capabilities.ts b/src/agents/subagent-capabilities.ts index 53e902683cce..552792f609f6 100644 --- a/src/agents/subagent-capabilities.ts +++ b/src/agents/subagent-capabilities.ts @@ -12,7 +12,8 @@ import { normalizeOptionalString, } from "@openclaw/normalization-core/string-coerce"; import { DEFAULT_SUBAGENT_MAX_SPAWN_DEPTH } from "../config/agent-limits.js"; -import { loadSessionStore, resolveStorePath } from "../config/sessions.js"; +import { resolveStorePath } from "../config/sessions.js"; +import { listSessionEntries } from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { isAcpSessionKey, @@ -86,9 +87,17 @@ function isSameAgentSessionStore(leftSessionKey: string, rightSessionKey: string return Boolean(leftAgentId) && leftAgentId === rightAgentId; } -function readSessionStore(storePath: string): Record { +function readSessionStore( + storePath: string, + agentId: string, +): Record { try { - return loadSessionStore(storePath); + return Object.fromEntries( + listSessionEntries({ agentId, storePath, clone: false }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), + ); } catch { return {}; } @@ -129,7 +138,7 @@ function resolveSessionCapabilityEntry(params: { return undefined; } const storePath = resolveStorePath(params.cfg.session?.store, { agentId: parsed.agentId }); - const store = readSessionStore(storePath); + const store = readSessionStore(storePath, parsed.agentId); return store[params.sessionKey] ?? findEntryBySessionId(store, params.sessionKey); } @@ -156,7 +165,7 @@ export function resolveSubagentCapabilityStore( return undefined; } const storePath = resolveStorePath(opts.cfg.session?.store, { agentId: parsed.agentId }); - return readSessionStore(storePath); + return readSessionStore(storePath, parsed.agentId); } /** Resolve depth-derived role/scope booleans for a subagent position. */ diff --git a/src/agents/subagent-control.test.ts b/src/agents/subagent-control.test.ts index b90cc0654c52..fcf2cc2893af 100644 --- a/src/agents/subagent-control.test.ts +++ b/src/agents/subagent-control.test.ts @@ -9,6 +9,7 @@ import type { SessionEntryPatchContext, SessionEntryPatchOptions, } from "../config/sessions/session-accessor.js"; +import { loadSessionEntry, replaceSessionEntry } from "../config/sessions/session-accessor.js"; import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { CallGatewayOptions } from "../gateway/call.js"; @@ -203,9 +204,19 @@ function cfgWithSessionStore(storePath = nextSessionStorePath("sessions")): Open } as OpenClawConfig; } -function writeSessionStoreFixture(label: string, store: Record) { +async function writeSessionStoreFixture(label: string, store: Record) { const storePath = nextSessionStorePath(label); - fs.writeFileSync(storePath, JSON.stringify(store, null, 2), "utf-8"); + for (const [sessionKey, entry] of Object.entries(store)) { + const record = entry && typeof entry === "object" ? (entry as Record) : {}; + const sessionId = + typeof record.sessionId === "string" && record.sessionId.trim() + ? record.sessionId + : `sess-${sessionKey.replaceAll(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "")}`; + await replaceSessionEntry({ storePath, sessionKey }, { + ...record, + sessionId, + } as SessionEntry); + } return storePath; } @@ -584,7 +595,7 @@ describe("killSubagentRunAdmin", () => { it("kills a subagent by session key without requester ownership checks", async () => { const childSessionKey = "agent:main:subagent:worker"; - const storePath = writeSessionStoreFixture("admin-kill", { + const storePath = await writeSessionStoreFixture("admin-kill", { [childSessionKey]: { sessionId: "sess-worker", updatedAt: Date.now(), @@ -718,7 +729,7 @@ describe("killSubagentRunAdmin", () => { it("restores the recoverable task marker when abort lifecycle wins the race", async () => { const childSessionKey = "agent:main:subagent:abort-lifecycle-race"; - const storePath = writeSessionStoreFixture("admin-kill-abort-lifecycle-race", { + const storePath = await writeSessionStoreFixture("admin-kill-abort-lifecycle-race", { [childSessionKey]: { sessionId: "sess-abort-lifecycle-race", updatedAt: Date.now(), @@ -776,7 +787,7 @@ describe("killSubagentRunAdmin", () => { it("reports when completion wins while the kill path awaits persistence", async () => { const childSessionKey = "agent:main:subagent:completion-race"; - const storePath = writeSessionStoreFixture("admin-kill-completion-race", { + const storePath = await writeSessionStoreFixture("admin-kill-completion-race", { [childSessionKey]: { sessionId: "sess-completion-race", updatedAt: Date.now(), @@ -855,7 +866,7 @@ describe("killSubagentRunAdmin", () => { it("refreshes target completion after descendant cancellation settles", async () => { const childSessionKey = "agent:main:subagent:cascade-completion-race"; const descendantSessionKey = "agent:main:subagent:cascade-completion-child"; - const storePath = writeSessionStoreFixture("admin-kill-cascade-completion-race", { + const storePath = await writeSessionStoreFixture("admin-kill-cascade-completion-race", { [childSessionKey]: { sessionId: "sess-cascade-completion-race", updatedAt: Date.now(), @@ -932,7 +943,7 @@ describe("killSubagentRunAdmin", () => { it("kills a run that yields while the kill path awaits persistence", async () => { const childSessionKey = "agent:main:subagent:yield-race"; - const storePath = writeSessionStoreFixture("admin-kill-yield-race", { + const storePath = await writeSessionStoreFixture("admin-kill-yield-race", { [childSessionKey]: { sessionId: "sess-yield-race", updatedAt: Date.now(), @@ -1013,12 +1024,19 @@ describe("killSubagentRunAdmin", () => { it("does not mark a finalizing run killed when its abort is rejected", async () => { const childSessionKey = "agent:main:subagent:worker-finalizing"; - const storePath = writeSessionStoreFixture("admin-kill-finalizing", { + const storePath = await writeSessionStoreFixture("admin-kill-finalizing", { [childSessionKey]: { sessionId: "sess-worker-finalizing", updatedAt: Date.now(), }, }); + await replaceSessionEntry( + { sessionKey: childSessionKey, storePath }, + { + sessionId: "sess-worker-finalizing", + updatedAt: Date.now(), + }, + ); addSubagentRunForTests({ runId: "run-worker-finalizing", @@ -1044,11 +1062,8 @@ describe("killSubagentRunAdmin", () => { expect(result.found).toBe(true); expect(result.killed).toBe(false); expect(getSubagentRunByChildSessionKey(childSessionKey)?.endedAt).toBeUndefined(); - const persisted = JSON.parse(fs.readFileSync(storePath, "utf-8")) as Record< - string, - { abortedLastRun?: boolean } - >; - expect(persisted[childSessionKey]?.abortedLastRun).toBeUndefined(); + const persisted = loadSessionEntry({ storePath, sessionKey: childSessionKey }); + expect(persisted?.abortedLastRun).toBeUndefined(); }); it("does not kill a newest finalizing run when only a stale older row is still active", async () => { @@ -1096,7 +1111,7 @@ describe("killSubagentRunAdmin", () => { it("still terminates the run when session store persistence fails during kill", async () => { const childSessionKey = "agent:main:subagent:worker-store-fail"; - const storePath = writeSessionStoreFixture("admin-kill-store-fail", { + const storePath = await writeSessionStoreFixture("admin-kill-store-fail", { [childSessionKey]: { sessionId: "sess-worker-store-fail", updatedAt: Date.now(), @@ -1145,7 +1160,7 @@ describe("killControlledSubagentRun", () => { it("does not mutate the live session when the caller passes a stale run entry", async () => { const childSessionKey = "agent:main:subagent:stale-kill-worker"; - const storePath = writeSessionStoreFixture("stale-kill", { + const storePath = await writeSessionStoreFixture("stale-kill", { [childSessionKey]: { updatedAt: Date.now(), }, @@ -1191,11 +1206,8 @@ describe("killControlledSubagentRun", () => { label: "stale task", text: "stale task is already finished.", }); - const persisted = JSON.parse(fs.readFileSync(storePath, "utf-8")) as Record< - string, - { abortedLastRun?: boolean } - >; - expect(persisted[childSessionKey]?.abortedLastRun).toBeUndefined(); + const persisted = loadSessionEntry({ storePath, sessionKey: childSessionKey }); + expect(persisted?.abortedLastRun).toBeUndefined(); expect(getSubagentRunByChildSessionKey(childSessionKey)?.runId).toBe("run-current"); }); @@ -1459,7 +1471,7 @@ describe("killAllControlledSubagentRuns", () => { it("ignores stale run snapshots in bulk kill requests", async () => { const childSessionKey = "agent:main:subagent:stale-kill-all-worker"; - const storePath = writeSessionStoreFixture("stale-kill-all", { + const storePath = await writeSessionStoreFixture("stale-kill-all", { [childSessionKey]: { updatedAt: Date.now(), }, @@ -1505,17 +1517,14 @@ describe("killAllControlledSubagentRuns", () => { killed: 0, labels: [], }); - const persisted = JSON.parse(fs.readFileSync(storePath, "utf-8")) as Record< - string, - { abortedLastRun?: boolean } - >; - expect(persisted[childSessionKey]?.abortedLastRun).toBeUndefined(); + const persisted = loadSessionEntry({ storePath, sessionKey: childSessionKey }); + expect(persisted?.abortedLastRun).toBeUndefined(); expect(getSubagentRunByChildSessionKey(childSessionKey)?.runId).toBe("run-current-bulk"); }); it("does not let a stale bulk entry suppress the current yielded entry", async () => { const childSessionKey = "agent:main:subagent:stale-kill-all-shadow-worker"; - const storePath = writeSessionStoreFixture("stale-kill-all-shadow", { + const storePath = await writeSessionStoreFixture("stale-kill-all-shadow", { [childSessionKey]: { updatedAt: Date.now(), }, @@ -1788,12 +1797,19 @@ describe("steerControlledSubagentRun", () => { it("does not replace a finalizing run when its abort is rejected", async () => { const childSessionKey = "agent:main:subagent:steer-finalizing"; - const storePath = writeSessionStoreFixture("steer-finalizing", { + const storePath = await writeSessionStoreFixture("steer-finalizing", { [childSessionKey]: { sessionId: "sess-steer-finalizing", updatedAt: Date.now(), }, }); + await replaceSessionEntry( + { sessionKey: childSessionKey, storePath }, + { + sessionId: "sess-steer-finalizing", + updatedAt: Date.now(), + }, + ); addSubagentRunForTests({ runId: "run-steer-finalizing", childSessionKey, @@ -2037,12 +2053,19 @@ describe("steerControlledSubagentRun", () => { it("rotates the child session when restarting a previously active session", async () => { const childSessionKey = "agent:main:subagent:active-steer-worker"; - const storePath = writeSessionStoreFixture("steer-restart-session", { + const storePath = await writeSessionStoreFixture("steer-restart-session", { [childSessionKey]: { sessionId: "old-child-session", updatedAt: Date.now(), }, }); + await replaceSessionEntry( + { sessionKey: childSessionKey, storePath }, + { + sessionId: "old-child-session", + updatedAt: Date.now(), + }, + ); const agentCalls: CallGatewayOptions[] = []; addSubagentRunForTests({ runId: "run-active-steer", diff --git a/src/agents/subagent-depth.test.ts b/src/agents/subagent-depth.test.ts index f4b7b99f6de4..a0c96e303cae 100644 --- a/src/agents/subagent-depth.test.ts +++ b/src/agents/subagent-depth.test.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; import { describe, expect, it } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; import { getSubagentDepthFromSessionStore } from "./subagent-depth.js"; import { resolveAgentTimeoutMs } from "./timeout.js"; @@ -72,25 +73,21 @@ describe("getSubagentDepthFromSessionStore", () => { expect(depth).toBe(3); }); - it("resolves prefixed store keys when caller key omits the agent prefix", () => { + it("resolves prefixed store keys when caller key omits the agent prefix", async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-subagent-depth-")); const storeTemplate = path.join(tmpDir, "sessions-{agentId}.json"); const prefixedKey = "agent:main:subagent:flat"; const storePath = storeTemplate.replaceAll("{agentId}", "main"); - fs.writeFileSync( - storePath, - JSON.stringify( - { - [prefixedKey]: { - sessionId: "subagent-flat", - updatedAt: Date.now(), - spawnDepth: 2, - }, - }, - null, - 2, - ), - "utf-8", + await replaceSessionEntry( + { + storePath, + sessionKey: prefixedKey, + }, + { + sessionId: "subagent-flat", + updatedAt: Date.now(), + spawnDepth: 2, + }, ); const depth = getSubagentDepthFromSessionStore("subagent:flat", { @@ -103,34 +100,6 @@ describe("getSubagentDepthFromSessionStore", () => { expect(depth).toBe(2); }); - - it("accepts JSON5 syntax in the on-disk depth store for backward compatibility", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-subagent-depth-json5-")); - const storeTemplate = path.join(tmpDir, "sessions-{agentId}.json"); - const storePath = storeTemplate.replaceAll("{agentId}", "main"); - fs.writeFileSync( - storePath, - `{ - // hand-edited legacy store - "agent:main:subagent:flat": { - sessionId: "subagent-flat", - spawnDepth: 2, - }, - }`, - "utf-8", - ); - - const depth = getSubagentDepthFromSessionStore("subagent:flat", { - cfg: { - session: { - store: storeTemplate, - }, - }, - }); - - expect(depth).toBe(2); - }); - it("falls back to session-key segment counting when metadata is missing", () => { const key = "agent:main:subagent:flat"; const depth = getSubagentDepthFromSessionStore(key, { diff --git a/src/agents/subagent-depth.ts b/src/agents/subagent-depth.ts index a046a7aeaf8a..647adaa9c247 100644 --- a/src/agents/subagent-depth.ts +++ b/src/agents/subagent-depth.ts @@ -3,13 +3,12 @@ * * Reads persisted session store state to recover spawn depth and parent lineage across restarts. */ -import fs from "node:fs"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { resolveStorePath } from "../config/sessions/paths.js"; +import { listSessionEntries } from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { parseStrictNonNegativeInteger } from "../infra/parse-finite-number.js"; import { getSubagentDepth, parseAgentSessionKey } from "../sessions/session-key-utils.js"; -import { parseJsonWithJson5Fallback } from "../utils/parse-json-compat.js"; import { resolveDefaultAgentId } from "./agent-scope.js"; type SessionDepthEntry = { @@ -28,15 +27,16 @@ function normalizeSpawnDepth(value: unknown): number | undefined { return undefined; } -function readSessionStore(storePath: string): Record { +function readSessionStore(storePath: string, agentId: string): Record { try { - const raw = fs.readFileSync(storePath, "utf-8"); - const parsed = parseJsonWithJson5Fallback(raw); - if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { - return parsed as Record; - } + return Object.fromEntries( + listSessionEntries({ agentId, storePath, clone: false }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), + ); } catch { - // ignore missing/invalid stores + // ignore missing/unavailable stores } return {}; } @@ -103,7 +103,7 @@ function resolveEntryForSessionKey(params: { const storePath = resolveStorePath(params.cfg.session?.store, { agentId: parsed.agentId }); let store = params.cache.get(storePath); if (!store) { - store = readSessionStore(storePath); + store = readSessionStore(storePath, parsed.agentId); params.cache.set(storePath, store); } const entry = store[key] ?? findEntryBySessionId(store, params.sessionKey); diff --git a/src/agents/subagent-list.test.ts b/src/agents/subagent-list.test.ts index 61cead965e07..21d1121a02d0 100644 --- a/src/agents/subagent-list.test.ts +++ b/src/agents/subagent-list.test.ts @@ -5,7 +5,7 @@ import os from "node:os"; import path from "node:path"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; -import { updateSessionStore } from "../config/sessions/store.js"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; import { buildLatestSubagentRunIndex, buildSubagentList } from "./subagent-list.js"; import { addSubagentRunForTests, @@ -234,16 +234,20 @@ describe("buildSubagentList", () => { } satisfies SubagentRunRecord; addSubagentRunForTests(run); const storePath = path.join(testWorkspaceDir, "sessions-subagent-list-usage.json"); - await updateSessionStore(storePath, (store) => { - store["agent:main:subagent:usage"] = { + await replaceSessionEntry( + { + storePath, + sessionKey: "agent:main:subagent:usage", + }, + { sessionId: "child-session-usage", updatedAt: Date.now(), inputTokens: 12, outputTokens: 1000, totalTokens: 197000, model: "opencode/claude-opus-4-6", - }; - }); + }, + ); const cfg = { commands: { text: true }, channels: { whatsapp: { allowFrom: ["*"] } }, diff --git a/src/agents/subagent-list.ts b/src/agents/subagent-list.ts index 32bfa2764f4c..bd44613d0576 100644 --- a/src/agents/subagent-list.ts +++ b/src/agents/subagent-list.ts @@ -6,7 +6,7 @@ import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; import { resolveSubagentLabel, sortSubagentRuns } from "../auto-reply/reply/subagents-utils.js"; import { resolveStorePath } from "../config/sessions/paths.js"; -import { loadSessionStore } from "../config/sessions/store-load.js"; +import { listSessionEntries } from "../config/sessions/session-accessor.js"; import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { parseAgentSessionKey, type ParsedAgentSessionKey } from "../routing/session-key.js"; @@ -82,7 +82,12 @@ export function resolveSessionEntryForKey(params: { const storePath = resolveStorePathForKey(params.cfg, parsed); let store = params.cache.get(storePath); if (!store) { - store = loadSessionStore(storePath); + store = Object.fromEntries( + listSessionEntries({ storePath, clone: false }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), + ); params.cache.set(storePath, store); } return { diff --git a/src/agents/subagent-orphan-recovery.test.ts b/src/agents/subagent-orphan-recovery.test.ts index 3bdfc6bd17b5..0f3399dcd130 100644 --- a/src/agents/subagent-orphan-recovery.test.ts +++ b/src/agents/subagent-orphan-recovery.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import * as config from "../config/config.js"; import * as sessions from "../config/sessions.js"; +import * as sessionAccessor from "../config/sessions/session-accessor.js"; import * as gateway from "../gateway/call.js"; import * as sessionUtils from "../gateway/session-transcript-readers.js"; import { @@ -10,7 +11,7 @@ import { resetGatewayWorkAdmission, tryBeginGatewaySuspendAdmission, } from "../process/gateway-work-admission.js"; -import { resolveInternalSessionEffectsTranscriptPath } from "./internal-session-effects.js"; +import { resolveInternalSessionEffectsTarget } from "./internal-session-effects.js"; import * as announceDelivery from "./subagent-announce-delivery.js"; import { recoverOrphanedSubagentSessions, @@ -26,6 +27,53 @@ const loggerMocks = vi.hoisted(() => ({ // Mocks are installed before importing the recovery module so registry/runtime // helpers resolve to deterministic restart fixtures. +const sessionMocks = vi.hoisted(() => { + type MockSessionEntry = Record; + type MockSessionStore = Record; + const loadSessionStore = vi.fn( + (_storePath?: string, _options?: { clone?: boolean }): MockSessionStore => ({}), + ); + return { + loadSessionStore, + resolveAgentIdFromSessionKey: vi.fn(() => "main"), + resolveStorePath: vi.fn(() => "/tmp/test-sessions.json"), + loadSessionEntry: vi.fn( + (scope: { storePath?: string; sessionKey: string }) => + loadSessionStore(scope.storePath, { + clone: false, + })[scope.sessionKey], + ), + patchSessionEntry: vi.fn( + async ( + scope: { storePath?: string; sessionKey: string }, + update: ( + entry: MockSessionEntry, + ) => + | MockSessionEntry + | Partial + | null + | Promise | null>, + options: { replaceEntry?: boolean } = {}, + ) => { + const store = loadSessionStore(scope.storePath, { + clone: false, + }); + const current = store[scope.sessionKey]; + if (!current) { + return null; + } + const patch = await update({ ...current }); + if (!patch) { + return current; + } + const next = options.replaceEntry ? patch : { ...current, ...patch }; + store[scope.sessionKey] = next; + return next; + }, + ), + }; +}); + vi.mock("../config/config.js", () => ({ getRuntimeConfig: vi.fn(() => ({ session: { store: undefined }, @@ -37,10 +85,14 @@ vi.mock("../logging/subsystem.js", () => ({ })); vi.mock("../config/sessions.js", () => ({ - loadSessionStore: vi.fn(() => ({})), - resolveAgentIdFromSessionKey: vi.fn(() => "main"), - resolveStorePath: vi.fn(() => "/tmp/test-sessions.json"), - updateSessionStore: vi.fn(async () => {}), + loadSessionStore: sessionMocks.loadSessionStore, + resolveAgentIdFromSessionKey: sessionMocks.resolveAgentIdFromSessionKey, + resolveStorePath: sessionMocks.resolveStorePath, +})); + +vi.mock("../config/sessions/session-accessor.js", () => ({ + loadSessionEntry: sessionMocks.loadSessionEntry, + patchSessionEntry: sessionMocks.patchSessionEntry, })); vi.mock("../gateway/call.js", () => ({ @@ -87,14 +139,16 @@ function createActiveRuns(...runs: SubagentRunRecord[]) { function mockSingleAbortedSession( overrides: Partial[string]>> = {}, ) { - vi.mocked(sessions.loadSessionStore).mockReturnValue({ + const store = { "agent:main:subagent:test-session-1": { sessionId: "session-abc", updatedAt: Date.now(), abortedLastRun: true, ...overrides, }, - }); + }; + vi.mocked(sessions.loadSessionStore).mockReturnValue(store); + return store; } async function expectSkippedRecovery(store: ReturnType) { @@ -133,14 +187,6 @@ function requireRecord(value: unknown, label: string): Record { return value as Record; } -function requireFirstUpdateSessionStoreCall() { - const call = vi.mocked(sessions.updateSessionStore).mock.calls[0]; - if (call === undefined) { - throw new Error("expected update session store call"); - } - return call; -} - describe("subagent-orphan-recovery", () => { beforeEach(() => { vi.useFakeTimers(); @@ -203,11 +249,19 @@ describe("subagent-orphan-recovery", () => { expect(replaceParams.previousRunId).toBe("run-1"); expect(replaceParams.nextRunId).toBe("test-run-id"); expect(replaceParams.fallback).toBe(run); - expect(replaceParams.transcriptFile).toBe( - resolveInternalSessionEffectsTranscriptPath("test-run-id"), + expect(replaceParams.transcriptTarget).toEqual( + resolveInternalSessionEffectsTarget({ + agentId: "main", + runId: "test-run-id", + storePath: "/tmp/test-sessions.json", + }), ); - expect(replaceParams.transcriptFile).not.toBe( - resolveInternalSessionEffectsTranscriptPath(params.idempotencyKey as string), + expect(replaceParams.transcriptTarget).not.toEqual( + resolveInternalSessionEffectsTarget({ + agentId: "main", + runId: params.idempotencyKey as string, + storePath: "/tmp/test-sessions.json", + }), ); }); @@ -295,9 +349,6 @@ describe("subagent-orphan-recovery", () => { }, }; vi.mocked(sessions.loadSessionStore).mockReturnValue(store); - vi.mocked(sessions.updateSessionStore).mockImplementation(async (_storePath, update) => { - update(store); - }); const activeRuns = createActiveRuns( createTestRunRecord({ runId: "fresh-run", @@ -519,9 +570,9 @@ describe("subagent-orphan-recovery", () => { expect(failedRun.childSessionKey).toBe("agent:main:subagent:test-session-1"); expect(failedRun.error).toBe("gateway unavailable"); - // abortedLastRun flag should NOT be cleared on failure, - // so the next restart can retry the recovery - expect(sessions.updateSessionStore).not.toHaveBeenCalled(); + // abortedLastRun flag should NOT be cleared on failure, so the next + // restart can retry recovery through the canonical session accessor. + expect(sessionAccessor.patchSessionEntry).not.toHaveBeenCalled(); }); it("returns empty results when no active runs exist", async () => { @@ -542,13 +593,14 @@ describe("subagent-orphan-recovery", () => { // Ensure callGateway succeeds for this test vi.mocked(gateway.callGateway).mockResolvedValue({ runId: "resumed-run" } as never); - vi.mocked(sessions.loadSessionStore).mockReturnValue({ + const store = { "agent:main:subagent:test-session-1": { sessionId: "session-abc", updatedAt: Date.now(), abortedLastRun: true, }, - }); + }; + vi.mocked(sessions.loadSessionStore).mockReturnValue(store); const activeRuns = new Map(); activeRuns.set("run-1", createTestRunRecord()); @@ -557,46 +609,32 @@ describe("subagent-orphan-recovery", () => { getActiveRuns: () => activeRuns, }); - // updateSessionStore should have been called AFTER successful resume to clear the flag - expect(sessions.updateSessionStore).toHaveBeenCalledOnce(); - const calls = vi.mocked(sessions.updateSessionStore).mock.calls; - const [storePath, updater] = calls[0]; - expect(storePath).toBe("/tmp/test-sessions.json"); - - // Simulate the updater to verify it clears abortedLastRun - const mockStore: Record = { - "agent:main:subagent:test-session-1": { - abortedLastRun: true, - updatedAt: 0, + expect(sessionAccessor.patchSessionEntry).toHaveBeenCalledOnce(); + expect(sessionAccessor.patchSessionEntry).toHaveBeenCalledWith( + { + storePath: "/tmp/test-sessions.json", + sessionKey: "agent:main:subagent:test-session-1", }, - }; - (updater as (store: Record) => void)(mockStore); - expect(mockStore["agent:main:subagent:test-session-1"]?.abortedLastRun).toBe(false); + expect.any(Function), + { + replaceEntry: true, + skipMaintenance: true, + }, + ); + expect(store["agent:main:subagent:test-session-1"]?.abortedLastRun).toBe(false); }); it("persists accepted recovery attempts after successful resume", async () => { vi.mocked(gateway.callGateway).mockResolvedValue({ runId: "resumed-run" } as never); - mockSingleAbortedSession(); + const store = mockSingleAbortedSession(); await recoverOrphanedSubagentSessions({ getActiveRuns: () => createActiveRuns(createTestRunRecord()), }); - const updateCall = requireFirstUpdateSessionStoreCall(); - const updater = updateCall[1]; - if (typeof updater !== "function") { - throw new Error("expected update session store callback"); - } - const mockStore: ReturnType = { - "agent:main:subagent:test-session-1": { - sessionId: "session-abc", - updatedAt: 0, - abortedLastRun: true, - }, - }; - await updater(mockStore); + expect(sessionAccessor.patchSessionEntry).toHaveBeenCalledOnce(); const sessionEntry = requireRecord( - mockStore["agent:main:subagent:test-session-1"], + store["agent:main:subagent:test-session-1"], "updated session entry", ); expect(sessionEntry.abortedLastRun).toBe(false); @@ -608,7 +646,7 @@ describe("subagent-orphan-recovery", () => { it("tombstones rapid repeated accepted recovery before resuming again", async () => { const now = Date.now(); - mockSingleAbortedSession({ + const store = mockSingleAbortedSession({ subagentRecovery: { automaticAttempts: 2, lastAttemptAt: now - 30_000, @@ -629,28 +667,10 @@ describe("subagent-orphan-recovery", () => { expect(blockedRun.childSessionKey).toBe("agent:main:subagent:test-session-1"); expect(blockedRun.error).toContain("recovery blocked after 2 rapid accepted resume attempts"); expect(gateway.callGateway).not.toHaveBeenCalled(); - expect(sessions.updateSessionStore).toHaveBeenCalledOnce(); + expect(sessionAccessor.patchSessionEntry).toHaveBeenCalledOnce(); - const updateCall = requireFirstUpdateSessionStoreCall(); - const updater = updateCall[1]; - if (typeof updater !== "function") { - throw new Error("expected update session store callback"); - } - const mockStore: ReturnType = { - "agent:main:subagent:test-session-1": { - sessionId: "session-abc", - updatedAt: 0, - abortedLastRun: true, - subagentRecovery: { - automaticAttempts: 2, - lastAttemptAt: now - 30_000, - lastRunId: "previous-run", - }, - }, - }; - await updater(mockStore); const sessionEntry = requireRecord( - mockStore["agent:main:subagent:test-session-1"], + store["agent:main:subagent:test-session-1"], "wedged session entry", ); expect(sessionEntry.abortedLastRun).toBe(false); @@ -681,7 +701,7 @@ describe("subagent-orphan-recovery", () => { expect(result.skipped).toBe(1); expect(result.failedRuns).toHaveLength(1); expect(gateway.callGateway).not.toHaveBeenCalled(); - expect(sessions.updateSessionStore).not.toHaveBeenCalled(); + expect(sessionAccessor.patchSessionEntry).not.toHaveBeenCalled(); }); it("truncates long task descriptions in resume message", async () => { @@ -753,9 +773,9 @@ describe("subagent-orphan-recovery", () => { expect(announceDelivery.deliverSubagentAnnouncement).not.toHaveBeenCalled(); }); - it("prevents duplicate resume when updateSessionStore fails", async () => { + it("prevents duplicate resume when the session accessor write fails", async () => { vi.mocked(gateway.callGateway).mockResolvedValue({ runId: "new-run" } as never); - vi.mocked(sessions.updateSessionStore).mockRejectedValue(new Error("write failed")); + vi.mocked(sessionAccessor.patchSessionEntry).mockRejectedValueOnce(new Error("write failed")); vi.mocked(sessions.loadSessionStore).mockReturnValue({ "agent:main:subagent:test-session-1": { @@ -779,6 +799,7 @@ describe("subagent-orphan-recovery", () => { expect(result.recovered).toBe(1); expect(result.skipped).toBe(1); expect(gateway.callGateway).toHaveBeenCalledOnce(); + expect(sessionAccessor.patchSessionEntry).toHaveBeenCalledOnce(); }); it("does not retry a session after the gateway accepted resume but run remap failed", async () => { @@ -811,7 +832,7 @@ describe("subagent-orphan-recovery", () => { expect(second.recovered).toBe(0); expect(second.skipped).toBe(1); expect(gateway.callGateway).toHaveBeenCalledOnce(); - expect(sessions.updateSessionStore).toHaveBeenCalledOnce(); + expect(sessionAccessor.patchSessionEntry).toHaveBeenCalledOnce(); }); it("finalizes interrupted runs with a readable failure after recovery retries are exhausted", async () => { @@ -878,14 +899,14 @@ describe("subagent-orphan-recovery", () => { await vi.advanceTimersByTimeAsync(1); expect(gateway.callGateway).not.toHaveBeenCalled(); - expect(sessions.updateSessionStore).not.toHaveBeenCalled(); + expect(sessionAccessor.patchSessionEntry).not.toHaveBeenCalled(); expect(getActiveGatewayRootWorkCount()).toBe(0); expect(suspension?.release()).toBe(true); await vi.advanceTimersByTimeAsync(0); expect(gateway.callGateway).toHaveBeenCalledOnce(); - expect(sessions.updateSessionStore).toHaveBeenCalledOnce(); + expect(sessionAccessor.patchSessionEntry).toHaveBeenCalledOnce(); expect(getActiveGatewayRootWorkCount()).toBe(0); }); diff --git a/src/agents/subagent-orphan-recovery.ts b/src/agents/subagent-orphan-recovery.ts index a1f6817bf7e8..da416d568b84 100644 --- a/src/agents/subagent-orphan-recovery.ts +++ b/src/agents/subagent-orphan-recovery.ts @@ -13,19 +13,18 @@ import crypto from "node:crypto"; import { getRuntimeConfig } from "../config/config.js"; import { - loadSessionStore, resolveAgentIdFromSessionKey, resolveStorePath, - updateSessionStore, type SessionEntry, } from "../config/sessions.js"; +import { loadSessionEntry, patchSessionEntry } from "../config/sessions/session-accessor.js"; import { callGateway } from "../gateway/call.js"; import { readSessionMessagesAsync } from "../gateway/session-transcript-readers.js"; import { formatErrorMessage } from "../infra/errors.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js"; import { truncateUtf16Safe } from "../utils.js"; -import { resolveInternalSessionEffectsTranscriptPath } from "./internal-session-effects.js"; +import { resolveInternalSessionEffectsTarget } from "./internal-session-effects.js"; import { evaluateSubagentRecoveryGate, markSubagentRecoveryAttempt, @@ -72,6 +71,38 @@ function reclassifyLegacyRestartInterruptedRun(runRecord: SubagentRunRecord): vo runRecord.terminalOwner = undefined; } +function loadRecoverySessionEntry(params: { + childSessionKey: string; + storePath: string; +}): SessionEntry | undefined { + return loadSessionEntry({ + storePath: params.storePath, + sessionKey: params.childSessionKey, + clone: false, + }); +} + +async function patchRecoverySessionEntry(params: { + childSessionKey: string; + storePath: string; + update: (entry: SessionEntry) => void; +}): Promise { + return await patchSessionEntry( + { + storePath: params.storePath, + sessionKey: params.childSessionKey, + }, + (entry) => { + params.update(entry); + return entry; + }, + { + replaceEntry: true, + skipMaintenance: true, + }, + ); +} + /** * Build the resume message for an orphaned subagent. */ @@ -158,7 +189,13 @@ async function resumeOrphanedSession(params: { previousRunId: params.originalRunId, nextRunId: result.runId, fallback: params.originalRun, - transcriptFile: resolveInternalSessionEffectsTranscriptPath(result.runId), + transcriptTarget: resolveInternalSessionEffectsTarget({ + agentId: resolveAgentIdFromSessionKey(params.sessionKey), + runId: result.runId, + storePath: resolveStorePath(getRuntimeConfig().session?.store, { + agentId: resolveAgentIdFromSessionKey(params.sessionKey), + }), + }), // Persist the stable original task (not the synthetic resume wrapper) so // that any further post-restart redispatch reconstructs the same // canonical task. Persisting `resumeMessage` instead would accumulate a @@ -220,7 +257,6 @@ export async function recoverOrphanedSubagentSessions(params: { } let cfg: ReturnType | undefined; - const storeCache = new Map>(); const scanNow = Date.now(); const runEntries = [...activeRuns.entries()].toSorted(([, left], [, right]) => { const leftIsStale = isStaleUnendedSubagentRun(left, scanNow); @@ -296,14 +332,7 @@ export async function recoverOrphanedSubagentSessions(params: { cfg ??= getRuntimeConfig(); const agentId = resolveAgentIdFromSessionKey(childSessionKey); const storePath = resolveStorePath(cfg.session?.store, { agentId }); - - let store = storeCache.get(storePath); - if (!store) { - store = loadSessionStore(storePath); - storeCache.set(storePath, store); - } - - const entry = store[childSessionKey]; + const entry = loadRecoverySessionEntry({ storePath, childSessionKey }); if (!entry) { result.skipped++; continue; @@ -368,24 +397,21 @@ export async function recoverOrphanedSubagentSessions(params: { if (!recoveryGate.allowed) { if (recoveryGate.shouldMarkWedged) { try { - await updateSessionStore(storePath, (currentStore) => { - const current = currentStore[childSessionKey]; - if (current) { + const updated = await patchRecoverySessionEntry({ + storePath, + childSessionKey, + update: (current) => { markSubagentRecoveryWedged({ entry: current, now, runId, reason: recoveryGate.reason, }); - currentStore[childSessionKey] = current; - } - }); - markSubagentRecoveryWedged({ - entry, - now, - runId, - reason: recoveryGate.reason, + }, }); + if (updated) { + Object.assign(entry, updated); + } } catch (err) { log.warn( `failed to persist wedged subagent recovery marker for ${childSessionKey}: ${String(err)}`, @@ -448,9 +474,10 @@ export async function recoverOrphanedSubagentSessions(params: { resumedSessionKeys.add(childSessionKey); // Only clear the aborted flag after confirmed successful resume. try { - await updateSessionStore(storePath, (currentStore) => { - const current = currentStore[childSessionKey]; - if (current) { + await patchRecoverySessionEntry({ + storePath, + childSessionKey, + update: (current) => { current.abortedLastRun = false; markSubagentRecoveryAttempt({ entry: current, @@ -459,8 +486,7 @@ export async function recoverOrphanedSubagentSessions(params: { attempt: recoveryGate.nextAttempt, }); current.updatedAt = Date.now(); - currentStore[childSessionKey] = current; - } + }, }); } catch (err) { log.warn( diff --git a/src/agents/subagent-registry-lifecycle.ts b/src/agents/subagent-registry-lifecycle.ts index b0a0bd76fab5..78ed1fee8bad 100644 --- a/src/agents/subagent-registry-lifecycle.ts +++ b/src/agents/subagent-registry-lifecycle.ts @@ -6,6 +6,7 @@ import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js"; import type { cleanupBrowserSessionsForLifecycleEnd } from "../browser-lifecycle-cleanup.js"; +import { formatSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import type { callGateway as defaultCallGateway } from "../gateway/call.js"; import { formatErrorMessage, readErrorName } from "../infra/errors.js"; import { @@ -31,7 +32,7 @@ import { buildAnnounceIdFromChildRun, buildAnnounceIdempotencyKey, } from "./announce-idempotency.js"; -import { removeInternalSessionEffectsTranscript } from "./internal-session-effects.js"; +import { removeInternalSessionEffectsSession } from "./internal-session-effects.js"; import type { SubagentAnnounceDeliveryResult } from "./subagent-announce-dispatch.js"; import { type SubagentRunOutcome, withSubagentOutcomeTiming } from "./subagent-announce-output.js"; import { @@ -566,7 +567,13 @@ export function createSubagentRegistryLifecycleController(params: { const captured = await params.captureSubagentCompletionReply(entry.childSessionKey, { waitForReply: entry.expectsCompletionMessage === true, outcome, - sessionFile: entry.execution?.transcriptFile, + sessionFile: entry.execution?.transcriptTarget?.storePath + ? formatSqliteSessionFileMarker({ + agentId: entry.execution.transcriptTarget.agentId ?? "", + sessionId: entry.execution.transcriptTarget.sessionId ?? "", + storePath: entry.execution.transcriptTarget.storePath, + }) + : undefined, }); resultText = captured?.trim() ? capFrozenResultText(captured) : null; } catch { @@ -972,8 +979,8 @@ export function createSubagentRegistryLifecycleController(params: { }); }; if (!cleanupParams.preserveTranscript) { - runCleanupTail("transcript cleanup", async () => { - await removeInternalSessionEffectsTranscript(cleanupParams.entry.execution?.transcriptFile); + runCleanupTail("session cleanup", async () => { + await removeInternalSessionEffectsSession(cleanupParams.entry.execution?.transcriptTarget); }); } if (cleanupParams.entry.spawnMode !== "session") { diff --git a/src/agents/subagent-registry-run-manager.ts b/src/agents/subagent-registry-run-manager.ts index f6cff688718e..0606ff975bb9 100644 --- a/src/agents/subagent-registry-run-manager.ts +++ b/src/agents/subagent-registry-run-manager.ts @@ -16,7 +16,8 @@ import { createRunningTaskRun, finalizeTaskRunByRunId } from "../tasks/detached- import { normalizeDeliveryContext } from "../utils/delivery-context.shared.js"; import type { DeliveryContext } from "../utils/delivery-context.types.js"; import { buildAgentRunTerminalOutcomeFromWaitResult } from "./agent-run-terminal-outcome.js"; -import { removeInternalSessionEffectsTranscript } from "./internal-session-effects.js"; +import { removeInternalSessionEffectsSession } from "./internal-session-effects.js"; +import type { AgentRunSessionTarget } from "./run-session-target.js"; import { isRecoverableAgentWaitError, waitForAgentRun } from "./run-wait.js"; import { type SubagentRunOutcome, withSubagentOutcomeTiming } from "./subagent-announce-output.js"; import { @@ -587,7 +588,7 @@ export function createSubagentRunManager(params: { fallback?: SubagentRunRecord; runTimeoutSeconds?: number; preserveFrozenResultFallback?: boolean; - transcriptFile?: string; + transcriptTarget?: AgentRunSessionTarget; task?: string; }) => { const previousRunId = replaceParams.previousRunId.trim(); @@ -665,7 +666,7 @@ export function createSubagentRunManager(params: { execution: { status: "running", startedAt: now, - transcriptFile: replaceParams.transcriptFile, + transcriptTarget: replaceParams.transcriptTarget, }, completion: { required: source.expectsCompletionMessage === true, @@ -712,10 +713,10 @@ export function createSubagentRunManager(params: { void safeRemoveAttachmentsDir(source); } if ( - source.execution?.transcriptFile && - source.execution.transcriptFile !== replaceParams.transcriptFile + source.execution?.transcriptTarget && + source.execution.transcriptTarget !== replaceParams.transcriptTarget ) { - void removeInternalSessionEffectsTranscript(source.execution.transcriptFile); + void removeInternalSessionEffectsSession(source.execution.transcriptTarget); } } params.ensureListener(); diff --git a/src/agents/subagent-registry-steer-runtime.ts b/src/agents/subagent-registry-steer-runtime.ts index 839b522aaad0..a149c5e5b1d4 100644 --- a/src/agents/subagent-registry-steer-runtime.ts +++ b/src/agents/subagent-registry-steer-runtime.ts @@ -1,3 +1,4 @@ +import type { AgentRunSessionTarget } from "./run-session-target.js"; /** * Late-bound steer hooks for the subagent registry. * @@ -11,7 +12,7 @@ type ReplaceSubagentRunAfterSteerParams = { fallback?: SubagentRunRecord; runTimeoutSeconds?: number; preserveFrozenResultFallback?: boolean; - transcriptFile?: string; + transcriptTarget?: AgentRunSessionTarget; /** * Optional task override for the replacement run. Callers that dispatched a * new message (steer, descendant wake, orphan resume) should pass the text diff --git a/src/agents/subagent-registry.persistence.test-support.ts b/src/agents/subagent-registry.persistence.test-support.ts index 31cccaf71b62..e2e577de3ae7 100644 --- a/src/agents/subagent-registry.persistence.test-support.ts +++ b/src/agents/subagent-registry.persistence.test-support.ts @@ -1,11 +1,17 @@ /** - * Test helpers for subagent registry persistence scenarios. They create - * minimal session-store files and runtime dependency mocks without loading + * Test helpers for subagent registry persistence scenarios. They seed minimal + * SQLite-backed session entries and runtime dependency mocks without loading * the production embedded-agent stack. */ -import fs from "node:fs/promises"; import path from "node:path"; import { vi } from "vitest"; +import type { SessionEntry } from "../config/sessions.js"; +import { + applySessionEntryLifecycleMutation, + listSessionEntries, + loadSessionEntry, + replaceSessionEntry, +} from "../config/sessions/session-accessor.js"; type SessionStore = Record>; @@ -13,21 +19,14 @@ function resolveSubagentSessionStorePath(stateDir: string, agentId: string): str return path.join(stateDir, "agents", agentId, "sessions", "sessions.json"); } -/** Reads a test session-store JSON file, returning an empty store on missing/invalid input. */ +/** Reads test session entries through the active SQLite accessor. */ export async function readSubagentSessionStore(storePath: string): Promise { - try { - const raw = await fs.readFile(storePath, "utf8"); - const parsed = JSON.parse(raw) as unknown; - if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { - return parsed as SessionStore; - } - } catch { - // ignore - } - return {}; + return Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ) as SessionStore; } -/** Writes or updates one subagent session-store entry for persistence tests. */ +/** Writes or updates one SQLite-backed subagent session entry for persistence tests. */ export async function writeSubagentSessionEntry(params: { stateDir: string; sessionKey: string; @@ -38,31 +37,31 @@ export async function writeSubagentSessionEntry(params: { defaultSessionId: string; }): Promise { const storePath = resolveSubagentSessionStorePath(params.stateDir, params.agentId); - const store = await readSubagentSessionStore(storePath); - store[params.sessionKey] = { - ...store[params.sessionKey], + const current = loadSessionEntry({ storePath, sessionKey: params.sessionKey }); + const entry: SessionEntry = { + ...current, sessionId: params.sessionId ?? params.defaultSessionId, updatedAt: params.updatedAt ?? Date.now(), ...(typeof params.abortedLastRun === "boolean" ? { abortedLastRun: params.abortedLastRun } : {}), }; - await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile(storePath, `${JSON.stringify(store)}\n`, "utf8"); + await replaceSessionEntry({ storePath, sessionKey: params.sessionKey }, entry); return storePath; } -/** Removes one subagent session-store entry for persistence tests. */ +/** Removes one SQLite-backed subagent session entry for persistence tests. */ export async function removeSubagentSessionEntry(params: { stateDir: string; sessionKey: string; agentId: string; }): Promise { const storePath = resolveSubagentSessionStorePath(params.stateDir, params.agentId); - const store = await readSubagentSessionStore(storePath); - delete store[params.sessionKey]; - await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile(storePath, `${JSON.stringify(store)}\n`, "utf8"); + await applySessionEntryLifecycleMutation({ + storePath, + removals: [{ sessionKey: params.sessionKey }], + skipMaintenance: true, + }); return storePath; } diff --git a/src/agents/subagent-registry.persistence.test.ts b/src/agents/subagent-registry.persistence.test.ts index 66db24bdd95b..7885d6bbecd9 100644 --- a/src/agents/subagent-registry.persistence.test.ts +++ b/src/agents/subagent-registry.persistence.test.ts @@ -6,10 +6,12 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import "./subagent-registry.mocks.shared.js"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; import { clearSessionStoreCacheForTest, drainSessionStoreWriterQueuesForTest, } from "../config/sessions/store.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import { callGateway } from "../gateway/call.js"; import { onAgentEvent } from "../infra/agent-events.js"; import { captureEnv, deleteTestEnvValue, setTestEnvValue, withEnv } from "../test-utils/env.js"; @@ -323,15 +325,14 @@ describe("subagent registry persistence", () => { updatedAt: completedAt, }); const store = await readSubagentSessionStore(storePath); - store["agent:main:subagent:kill-race"] = { + await replaceSessionEntry({ storePath, sessionKey: "agent:main:subagent:kill-race" }, { ...store["agent:main:subagent:kill-race"], status: "done", startedAt, endedAt: completedAt, runtimeMs: 500, abortedLastRun: true, - }; - await fs.writeFile(storePath, `${JSON.stringify(store)}\n`, "utf8"); + } as SessionEntry); await persistSubagentSessionTiming({ runId: "run-kill-race", diff --git a/src/agents/subagent-registry.steer-restart.test.ts b/src/agents/subagent-registry.steer-restart.test.ts index 045b0cc139f6..69d5b6fbcbd2 100644 --- a/src/agents/subagent-registry.steer-restart.test.ts +++ b/src/agents/subagent-registry.steer-restart.test.ts @@ -83,7 +83,7 @@ vi.mock("../config/sessions/session-accessor.js", () => ({ const announceSpy = vi.fn(async (_params: unknown) => true); const runSubagentEndedHookMock = vi.fn(async (_eventValue?: unknown, _ctx?: unknown) => {}); const emitSessionLifecycleEventMock = vi.fn(); -const removeInternalSessionEffectsTranscriptMock = vi.fn(async (_sessionFile?: string) => {}); +const removeInternalSessionEffectsSessionMock = vi.fn(async (_target?: unknown) => {}); function countMatching(items: readonly T[], predicate: (item: T) => boolean) { let count = 0; @@ -171,7 +171,7 @@ vi.mock("./subagent-registry.store.js", () => ({ })); vi.mock("./internal-session-effects.js", () => ({ - removeInternalSessionEffectsTranscript: removeInternalSessionEffectsTranscriptMock, + removeInternalSessionEffectsSession: removeInternalSessionEffectsSessionMock, })); describe("subagent registry steer restarts", () => { @@ -197,7 +197,7 @@ describe("subagent registry steer restarts", () => { runSubagentEndedHookMock.mockReset(); runSubagentEndedHookMock.mockImplementation(async () => {}); emitSessionLifecycleEventMock.mockReset(); - removeInternalSessionEffectsTranscriptMock.mockClear(); + removeInternalSessionEffectsSessionMock.mockClear(); mod.resetSubagentRegistryForTests({ persist: false }); }); @@ -298,14 +298,19 @@ describe("subagent registry steer restarts", () => { previousRunId: string; nextRunId: string; fallback?: ReturnType[number]; - transcriptFile?: string; + transcriptTarget?: { + agentId: string; + sessionId: string; + sessionKey: string; + storePath: string; + }; task?: string; }) => { const replaced = mod.replaceSubagentRunAfterSteer({ previousRunId: params.previousRunId, nextRunId: params.nextRunId, fallback: params.fallback, - transcriptFile: params.transcriptFile, + transcriptTarget: params.transcriptTarget, task: params.task, }); expect(replaced).toBe(true); @@ -325,7 +330,7 @@ describe("subagent registry steer restarts", () => { runSubagentEndedHookMock.mockImplementation(async () => {}); emitSessionLifecycleEventMock.mockReset(); lifecycleHandler = undefined; - removeInternalSessionEffectsTranscriptMock.mockClear(); + removeInternalSessionEffectsSessionMock.mockClear(); mod.resetSubagentRegistryForTests({ persist: false }); }); @@ -393,7 +398,12 @@ describe("subagent registry steer restarts", () => { previous.execution = { status: "interrupted", startedAt: previous.startedAt, - transcriptFile: "/tmp/openclaw-state/internal-agent-runs/run-old.jsonl", + transcriptTarget: { + agentId: "main", + sessionId: "internal-run-old", + sessionKey: "agent:main:internal-session-effects:run-old", + storePath: "/tmp/test-store", + }, }; replaceRunAfterSteer({ @@ -402,8 +412,8 @@ describe("subagent registry steer restarts", () => { fallback: previous, }); - expect(removeInternalSessionEffectsTranscriptMock).toHaveBeenCalledWith( - "/tmp/openclaw-state/internal-agent-runs/run-old.jsonl", + expect(removeInternalSessionEffectsSessionMock).toHaveBeenCalledWith( + previous.execution.transcriptTarget, ); } }); @@ -903,7 +913,12 @@ describe("subagent registry steer restarts", () => { activeRun.execution = { ...activeRun.execution, status: activeRun.execution?.status ?? "running", - transcriptFile: "/tmp/recovered-subagent.jsonl", + transcriptTarget: { + agentId: "main", + sessionId: "recovered-subagent", + sessionKey: "agent:main:internal-session-effects:recovered-subagent", + storePath: "/tmp/test-store", + }, }; expect(mod.isSubagentSessionRunActive(childSessionKey)).toBe(true); @@ -926,7 +941,7 @@ describe("subagent registry steer restarts", () => { expect(typeof run?.cleanupCompletedAt).toBe("number"); await flushAnnounce(); expect(runSubagentEndedHookMock).not.toHaveBeenCalled(); - expect(removeInternalSessionEffectsTranscriptMock).not.toHaveBeenCalled(); + expect(removeInternalSessionEffectsSessionMock).not.toHaveBeenCalled(); }); it("treats a child session as inactive when only a stale older row is still unended", () => { diff --git a/src/agents/subagent-registry.test.ts b/src/agents/subagent-registry.test.ts index 7187edf25a40..f07474d31e27 100644 --- a/src/agents/subagent-registry.test.ts +++ b/src/agents/subagent-registry.test.ts @@ -116,6 +116,13 @@ const mocks = vi.hoisted(() => ({ >; return store[scope.sessionKey]; }), + listSessionEntries: vi.fn((scope: Omit) => { + const store = mocks.loadSessionStore(scope.storePath, { clone: false }) as Record< + string, + SessionEntry + >; + return Object.entries(store).map(([sessionKey, entry]) => ({ sessionKey, entry })); + }), loadSessionStore: vi.fn((_storePath?: string, _options?: { clone?: boolean }) => ({})), patchSessionEntry: vi.fn( async ( @@ -175,7 +182,7 @@ const mocks = vi.hoisted(() => ({ (params: { childSessionKey?: string }, context?: unknown) => Promise >(async () => {}), runSubagentEnded: vi.fn(async () => {}), - removeInternalSessionEffectsTranscript: vi.fn(async () => {}), + removeInternalSessionEffectsSession: vi.fn(async () => {}), resolveAgentTimeoutMs: vi.fn(() => 1_000), scheduleOrphanRecovery: vi.fn(), })); @@ -203,6 +210,7 @@ vi.mock("../config/sessions.js", () => ({ })); vi.mock("../config/sessions/session-accessor.js", () => ({ + listSessionEntries: mocks.listSessionEntries, loadSessionEntry: mocks.loadSessionEntry, patchSessionEntry: mocks.patchSessionEntry, })); @@ -249,7 +257,7 @@ vi.mock("./subagent-orphan-recovery.js", () => ({ })); vi.mock("./internal-session-effects.js", () => ({ - removeInternalSessionEffectsTranscript: mocks.removeInternalSessionEffectsTranscript, + removeInternalSessionEffectsSession: mocks.removeInternalSessionEffectsSession, })); describe("subagent registry seam flow", () => { @@ -3750,7 +3758,12 @@ describe("subagent registry seam flow", () => { const attachmentsDir = path.join(attachmentsRootDir, "child"); await fs.mkdir(attachmentsDir, { recursive: true }); await fs.writeFile(path.join(attachmentsDir, "artifact.txt"), "artifact"); - const oldTranscriptFile = "/tmp/internal-agent-runs/run-old-tombstone.jsonl"; + const oldTranscriptTarget = { + agentId: "main", + sessionId: "internal-run-old-tombstone", + sessionKey: "agent:main:internal-session-effects:run-old-tombstone", + storePath: "/tmp/test-store", + }; mod.addSubagentRunForTests({ runId: "run-old-tombstone", childSessionKey: "agent:main:subagent:reused", @@ -3776,7 +3789,7 @@ describe("subagent registry seam flow", () => { status: "terminal", startedAt: oldStartedAt, endedAt: oldEndedAt, - transcriptFile: oldTranscriptFile, + transcriptTarget: oldTranscriptTarget, }, }); mod.addSubagentRunForTests({ @@ -3805,7 +3818,7 @@ describe("subagent registry seam flow", () => { ([params]) => params.childSessionKey === "agent:main:subagent:reused", ), ).toBe(false); - expect(mocks.removeInternalSessionEffectsTranscript).toHaveBeenCalledWith(oldTranscriptFile); + expect(mocks.removeInternalSessionEffectsSession).toHaveBeenCalledWith(oldTranscriptTarget); await expectPathMissing(attachmentsDir); expect( mocks.callGateway.mock.calls.some( diff --git a/src/agents/subagent-registry.ts b/src/agents/subagent-registry.ts index 2703fe4e7aab..e02ee9363a2b 100644 --- a/src/agents/subagent-registry.ts +++ b/src/agents/subagent-registry.ts @@ -35,7 +35,8 @@ import { prependAgentSteeringPrompt, releaseLeasedAgentSteeringItemsFromSubagentRuns, } from "./agent-steering-queue.js"; -import { removeInternalSessionEffectsTranscript } from "./internal-session-effects.js"; +import { removeInternalSessionEffectsSession } from "./internal-session-effects.js"; +import type { AgentRunSessionTarget } from "./run-session-target.js"; import { isAbortedAgentStopReason } from "./run-termination.js"; import type { ensureRuntimePluginsLoaded as ensureRuntimePluginsLoadedFn } from "./runtime-plugins.js"; import type { SubagentRunOutcome } from "./subagent-announce-output.js"; @@ -1066,7 +1067,7 @@ async function discardSuspendedPendingFinalDelivery( if (shouldDeleteAttachments) { await safeRemoveAttachmentsDir(entry); } - await removeInternalSessionEffectsTranscript(entry.execution?.transcriptFile); + await removeInternalSessionEffectsSession(entry.execution?.transcriptTarget); const completionReason = entry.endedReason ?? SUBAGENT_ENDED_REASON_COMPLETE; completeCleanupBookkeeping({ runId, @@ -1090,14 +1091,19 @@ async function discardSuspendedPendingFinalDelivery( } async function retireSupersededSubagentRun(runId: string, entry: SubagentRunRecord): Promise { - const transcriptFile = entry.execution?.transcriptFile; + const transcriptTarget = entry.execution?.transcriptTarget; clearPendingLifecycleError(runId); subagentRuns.delete(runId); - const transcriptStillOwned = Array.from(subagentRuns.values()).some( - (candidate) => candidate.execution?.transcriptFile === transcriptFile, - ); - if (transcriptFile && !transcriptStillOwned) { - await removeInternalSessionEffectsTranscript(transcriptFile); + const transcriptStillOwned = Array.from(subagentRuns.values()).some((candidate) => { + const candidateTarget = candidate.execution?.transcriptTarget; + return ( + candidateTarget?.sessionId === transcriptTarget?.sessionId && + candidateTarget?.sessionKey === transcriptTarget?.sessionKey && + candidateTarget?.storePath === transcriptTarget?.storePath + ); + }); + if (transcriptTarget && !transcriptStillOwned) { + await removeInternalSessionEffectsSession(transcriptTarget); } if (entry.cleanup === "delete" || !entry.retainAttachmentsOnKeep) { await safeRemoveAttachmentsDir(entry); @@ -1715,7 +1721,7 @@ export function replaceSubagentRunAfterSteer(params: { fallback?: SubagentRunRecord; runTimeoutSeconds?: number; preserveFrozenResultFallback?: boolean; - transcriptFile?: string; + transcriptTarget?: AgentRunSessionTarget; task?: string; }) { return subagentRunManager.replaceSubagentRunAfterSteer(params); diff --git a/src/agents/subagent-registry.types.ts b/src/agents/subagent-registry.types.ts index 1028e084dde5..964d14f1c9d8 100644 --- a/src/agents/subagent-registry.types.ts +++ b/src/agents/subagent-registry.types.ts @@ -4,6 +4,7 @@ * Defines execution, completion, delivery, pending-delivery, and attachment state stored for child runs. */ import type { DeliveryContext } from "../utils/delivery-context.types.js"; +import type { AgentRunSessionTarget } from "./run-session-target.js"; import type { SubagentRunOutcome } from "./subagent-announce-output.js"; import type { SubagentLifecycleEndedReason } from "./subagent-lifecycle-events.js"; import type { SpawnSubagentMode } from "./subagent-spawn.types.js"; @@ -33,7 +34,7 @@ export type SubagentExecutionState = { outcome?: SubagentRunOutcome; interruptedAt?: number; interruptionReason?: "gateway-restart" | "lost-execution-context"; - transcriptFile?: string; + transcriptTarget?: AgentRunSessionTarget; }; export type SubagentCompletionState = { diff --git a/src/agents/subagent-session-reconciliation.ts b/src/agents/subagent-session-reconciliation.ts index dea10be6638b..9ed598c7a0a5 100644 --- a/src/agents/subagent-session-reconciliation.ts +++ b/src/agents/subagent-session-reconciliation.ts @@ -6,12 +6,11 @@ import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion"; import { getRuntimeConfig } from "../config/config.js"; import { - loadSessionStore, resolveAgentIdFromSessionKey, resolveStorePath, type SessionEntry, } from "../config/sessions.js"; -import { loadSessionEntry } from "../config/sessions/session-accessor.js"; +import { listSessionEntries, loadSessionEntry } from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { SubagentRunOutcome } from "./subagent-announce-output.js"; import { @@ -96,7 +95,12 @@ export function loadSubagentSessionEntry(params: { const storePath = resolveStorePath(cfg.session?.store, { agentId }); let store = params.storeCache?.get(storePath); if (!store) { - store = loadSessionStore(storePath); + store = Object.fromEntries( + listSessionEntries({ storePath, clone: false }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), + ); params.storeCache?.set(storePath, store); } return findSessionEntryByKey(store, key); diff --git a/src/agents/subagent-spawn.context.test.ts b/src/agents/subagent-spawn.context.test.ts index e135963fedd6..9fb56ad2a71d 100644 --- a/src/agents/subagent-spawn.context.test.ts +++ b/src/agents/subagent-spawn.context.test.ts @@ -58,8 +58,10 @@ describe("sessions_spawn context modes", () => { async (params: { agentId: string; fallbackEntry?: Record; + parentSessionKey: string; parentStoreKeys?: string[]; sessionKey: string; + storePath: string; }) => { const parentEntry = params.parentStoreKeys ?.map((key) => store[key]) @@ -103,6 +105,9 @@ describe("sessions_spawn context modes", () => { const fork = await forkSessionFromParentMock({ parentEntry, agentId: params.agentId, + parentSessionKey: params.parentSessionKey, + sessionKey: params.sessionKey, + storePath: params.storePath, }); if (!fork) { return { status: "failed" }; @@ -196,11 +201,14 @@ describe("sessions_spawn context modes", () => { const accepted = requireAcceptedResult(result); expect(accepted.runId).toBe("run-1"); + const childSessionKey = requireChildSessionKey(accepted); expect(forkSessionFromParentMock).toHaveBeenCalledWith({ parentEntry: store.main, agentId: "main", + parentSessionKey: "main", + sessionKey: childSessionKey, + storePath, }); - const childSessionKey = requireChildSessionKey(accepted); const childEntry = requireStoreEntry(store, childSessionKey); expect(childEntry.sessionId).toBe("forked-session-id"); expect(childEntry.sessionFile).toBe("/tmp/forked-session.jsonl"); @@ -367,6 +375,9 @@ describe("sessions_spawn context modes", () => { expect(forkSessionFromParentMock).toHaveBeenCalledWith({ parentEntry: store.main, agentId: "main", + parentSessionKey: "main", + sessionKey: result.childSessionKey, + storePath, }); const cleanupRequest = requireGatewayRequest("sessions.delete"); expect(cleanupRequest.params?.key).toBe(result.childSessionKey); diff --git a/src/agents/subagent-spawn.model-session.test.ts b/src/agents/subagent-spawn.model-session.test.ts index 7e930f60dfa0..759b4c70f090 100644 --- a/src/agents/subagent-spawn.model-session.test.ts +++ b/src/agents/subagent-spawn.model-session.test.ts @@ -12,7 +12,6 @@ import { const callGatewayMock = vi.fn(); const updateSessionStoreMock = vi.fn(); -const pruneLegacyStoreKeysMock = vi.fn(); let resetSubagentRegistryForTests: typeof import("./subagent-registry.js").resetSubagentRegistryForTests; let spawnSubagentDirect: typeof import("./subagent-spawn.js").spawnSubagentDirect; @@ -23,7 +22,6 @@ describe("spawnSubagentDirect runtime model persistence", () => { callGatewayMock, getRuntimeConfig: () => createSubagentSpawnTestConfig(os.tmpdir()), updateSessionStoreMock, - pruneLegacyStoreKeysMock, workspaceDir: os.tmpdir(), })); }); @@ -32,7 +30,6 @@ describe("spawnSubagentDirect runtime model persistence", () => { resetSubagentRegistryForTests(); callGatewayMock.mockReset(); updateSessionStoreMock.mockReset(); - pruneLegacyStoreKeysMock.mockReset(); setupAcceptedSubagentGatewayMock(callGatewayMock); updateSessionStoreMock.mockImplementation( @@ -95,7 +92,6 @@ describe("spawnSubagentDirect runtime model persistence", () => { model: "gpt-5.4", overrideSource: "user", }); - expect(pruneLegacyStoreKeysMock).toHaveBeenCalledTimes(3); expect(operations.indexOf("store:update")).toBeGreaterThan(-1); expect(operations.indexOf("gateway:agent")).toBeGreaterThan( operations.lastIndexOf("store:update"), @@ -120,7 +116,6 @@ describe("spawnSubagentDirect runtime model persistence", () => { }, }), updateSessionStoreMock: dedicatedUpdateSessionStoreMock, - pruneLegacyStoreKeysMock, workspaceDir: os.tmpdir(), }); resetForAutoModelTest(); diff --git a/src/agents/subagent-spawn.runtime.ts b/src/agents/subagent-spawn.runtime.ts index fdd594a3d88a..c28b7e1eabe6 100644 --- a/src/agents/subagent-spawn.runtime.ts +++ b/src/agents/subagent-spawn.runtime.ts @@ -8,7 +8,7 @@ export { DEFAULT_SUBAGENT_MAX_SPAWN_DEPTH, } from "../config/agent-limits.js"; export { getRuntimeConfig } from "../config/config.js"; -export { loadSessionStore, mergeSessionEntry, updateSessionStore } from "../config/sessions.js"; +export { loadSessionEntry, upsertSessionEntry } from "../config/sessions/session-accessor.js"; export { forkSessionEntryFromParent, forkSessionFromParent, @@ -27,10 +27,7 @@ export { resolveLeastPrivilegeOperatorScopesForMethod, } from "../gateway/method-scopes.js"; export { getSessionBindingService } from "../infra/outbound/session-binding-service.js"; -export { - pruneLegacyStoreKeys, - resolveGatewaySessionStoreTarget, -} from "../gateway/session-utils.js"; +export { resolveGatewaySessionStoreTarget } from "../gateway/session-utils.js"; export { getGlobalHookRunner } from "../plugins/hook-runner-global.js"; export { emitSessionLifecycleEvent } from "../sessions/session-lifecycle-events.js"; export { diff --git a/src/agents/subagent-spawn.test-helpers.ts b/src/agents/subagent-spawn.test-helpers.ts index 2349fc02f132..ffa522cf2b24 100644 --- a/src/agents/subagent-spawn.test-helpers.ts +++ b/src/agents/subagent-spawn.test-helpers.ts @@ -139,7 +139,6 @@ export async function loadSubagentSpawnModuleForTest(params: { forkSessionFromParentMock?: MockFn; resolveContextEngineMock?: MockFn; resolveParentForkDecisionMock?: MockFn; - pruneLegacyStoreKeysMock?: MockFn; registerSubagentRunMock?: MockFn; emitSessionLifecycleEventMock?: MockFn; hookRunner?: HookRunner; @@ -262,6 +261,8 @@ export async function loadSubagentSpawnModuleForTest(params: { getRuntimeConfig: () => params.getRuntimeConfig?.() ?? createSubagentSpawnTestConfig(params.workspaceDir ?? os.tmpdir()), + loadSessionEntry: (scope: { storePath?: string; sessionKey: string }) => + ((params.loadSessionStoreMock?.(scope.storePath) ?? {}) as SessionStore)[scope.sessionKey], loadSessionStore: params.loadSessionStoreMock ?? (() => ({})), ensureContextEnginesInitialized: params.ensureContextEnginesInitializedMock ?? (() => undefined), @@ -307,7 +308,26 @@ export async function loadSubagentSpawnModuleForTest(params: { // Real scope resolver: spawn's admin-tier pinning depends on params-aware // sessions.patch policy, so a stub here would hide policy regressions. resolveLeastPrivilegeOperatorScopesForMethod, - pruneLegacyStoreKeys: (...args: unknown[]) => params.pruneLegacyStoreKeysMock?.(...args), + upsertSessionEntry: async ( + scope: { storePath?: string; sessionKey: string }, + patch: Record, + ) => { + const updateSessionStore = + params.updateSessionStoreMock ?? + (async (_storePath: string, mutator: SessionStoreMutator) => { + const store: SessionStore = {}; + await mutator(store); + return store; + }); + let updated: Record | undefined; + const storePath = + scope.storePath ?? params.sessionStorePath ?? "/tmp/subagent-spawn-model-session.json"; + await updateSessionStore(storePath, (store: SessionStore) => { + updated = Object.assign({}, store[scope.sessionKey], patch); + store[scope.sessionKey] = updated; + }); + return updated ?? null; + }, getSessionBindingService: params.getSessionBindingService ?? (() => ({ diff --git a/src/agents/subagent-spawn.test.ts b/src/agents/subagent-spawn.test.ts index 1c863991e5de..b9171cbbd58b 100644 --- a/src/agents/subagent-spawn.test.ts +++ b/src/agents/subagent-spawn.test.ts @@ -14,7 +14,6 @@ const hoisted = vi.hoisted(() => ({ callGatewayMock: vi.fn(), loadSessionStoreMock: vi.fn(), updateSessionStoreMock: vi.fn(), - pruneLegacyStoreKeysMock: vi.fn(), registerSubagentRunMock: vi.fn(), emitSessionLifecycleEventMock: vi.fn(), dispatchGatewayMethodInProcessMock: vi.fn(), @@ -74,7 +73,6 @@ describe("spawnSubagentDirect seam flow", () => { getRuntimeConfig: () => hoisted.configOverride, loadSessionStoreMock: hoisted.loadSessionStoreMock, updateSessionStoreMock: hoisted.updateSessionStoreMock, - pruneLegacyStoreKeysMock: hoisted.pruneLegacyStoreKeysMock, registerSubagentRunMock: hoisted.registerSubagentRunMock, emitSessionLifecycleEventMock: hoisted.emitSessionLifecycleEventMock, resolveAgentConfig: hoisted.resolveAgentConfigMock, @@ -89,7 +87,6 @@ describe("spawnSubagentDirect seam flow", () => { hoisted.callGatewayMock.mockReset(); hoisted.loadSessionStoreMock.mockReset(); hoisted.updateSessionStoreMock.mockReset(); - hoisted.pruneLegacyStoreKeysMock.mockReset(); hoisted.registerSubagentRunMock.mockReset(); hoisted.emitSessionLifecycleEventMock.mockReset(); hoisted.dispatchGatewayMethodInProcessMock.mockReset(); @@ -275,7 +272,6 @@ describe("spawnSubagentDirect seam flow", () => { expect(result.childSessionKey).toMatch(/^agent:main:subagent:/); const childSessionKey = result.childSessionKey as string; - expect(hoisted.pruneLegacyStoreKeysMock).toHaveBeenCalledTimes(3); expect(hoisted.updateSessionStoreMock).toHaveBeenCalledTimes(3); const registerInput = firstRegisteredSubagentRun(); const requesterOrigin = requireRecord(registerInput.requesterOrigin); diff --git a/src/agents/subagent-spawn.ts b/src/agents/subagent-spawn.ts index 2db2d6fa744e..b909763f30df 100644 --- a/src/agents/subagent-spawn.ts +++ b/src/agents/subagent-spawn.ts @@ -93,11 +93,8 @@ import { getSessionBindingService, getRuntimeConfig, hasInProcessGatewayContext, - loadSessionStore, - mergeSessionEntry, mergeDeliveryContext, normalizeDeliveryContext, - pruneLegacyStoreKeys, ensureContextEnginesInitialized, resolveAgentConfig, resolveContextEngine, @@ -105,7 +102,8 @@ import { resolveInternalSessionKey, resolveMainSessionAlias, resolveSandboxRuntimeStatus, - updateSessionStore, + loadSessionEntry, + upsertSessionEntry, resolveLeastPrivilegeOperatorScopesForMethod, } from "./subagent-spawn.runtime.js"; import type { @@ -138,7 +136,6 @@ type SubagentSpawnDeps = { hasInProcessGatewayContext: typeof hasInProcessGatewayContext; ensureContextEnginesInitialized: typeof ensureContextEnginesInitialized; resolveContextEngine: typeof resolveContextEngine; - updateSessionStore: typeof updateSessionStore; }; const defaultSubagentSpawnDeps: SubagentSpawnDeps = { @@ -150,7 +147,6 @@ const defaultSubagentSpawnDeps: SubagentSpawnDeps = { hasInProcessGatewayContext, ensureContextEnginesInitialized, resolveContextEngine, - updateSessionStore, }; let subagentSpawnDeps: SubagentSpawnDeps = defaultSubagentSpawnDeps; @@ -225,13 +221,6 @@ export type SpawnSubagentResult = { export { splitModelRef } from "./subagent-spawn-plan.js"; -async function updateSubagentSessionStore( - storePath: string, - mutator: Parameters[1], -) { - return await subagentSpawnDeps.updateSessionStore(storePath, mutator); -} - async function callSubagentGateway( params: Parameters[0], ): Promise>> { @@ -383,36 +372,22 @@ async function persistInitialChildSessionRuntimeModel(params: { cfg: params.cfg, key: params.childSessionKey, }); - await updateSubagentSessionStore(target.storePath, (store) => { - pruneLegacyStoreKeys({ - store, - canonicalKey: target.canonicalKey, - candidates: target.storeKeys, - }); - store[target.canonicalKey] = mergeSessionEntry(store[target.canonicalKey], { + await upsertSessionEntry( + { + storePath: target.storePath, + sessionKey: target.canonicalKey, + }, + { model, ...(provider ? { modelProvider: provider } : {}), - }); - }); + }, + ); return undefined; } catch (err) { return err instanceof Error ? err.message : typeof err === "string" ? err : "error"; } } -function resolveStoreEntryByKeys( - store: Record, - keys: readonly string[], -): SessionEntry | undefined { - for (const key of keys) { - const entry = store[key]; - if (entry) { - return entry; - } - } - return undefined; -} - function readRequesterThinkingLevel(params: { cfg: OpenClawConfig; requesterInternalKey: string; @@ -424,8 +399,11 @@ function readRequesterThinkingLevel(params: { cfg: params.cfg, key: params.requesterInternalKey, }); - const store = loadSessionStore(target.storePath, { clone: false }); - entry = resolveStoreEntryByKeys(store, target.storeKeys); + entry = loadSessionEntry({ + storePath: target.storePath, + sessionKey: target.canonicalKey, + clone: false, + }); } catch { entry = undefined; } @@ -1317,17 +1295,13 @@ export async function spawnSubagentDirect( cfg, key: childSessionKey, }); - await updateSubagentSessionStore(target.storePath, (store) => { - pruneLegacyStoreKeys({ - store, - canonicalKey: target.canonicalKey, - candidates: target.storeKeys, - }); - store[target.canonicalKey] = mergeSessionEntry( - store[target.canonicalKey], - buildDirectChildSessionPatch(patch), - ); - }); + await upsertSessionEntry( + { + storePath: target.storePath, + sessionKey: target.canonicalKey, + }, + buildDirectChildSessionPatch(patch), + ); return undefined; } catch (err) { const message = err instanceof Error ? err.message : typeof err === "string" ? err : "error"; diff --git a/src/agents/tools/goal-tools.test.ts b/src/agents/tools/goal-tools.test.ts index 797ea94efbe9..479c5f75ce4f 100644 --- a/src/agents/tools/goal-tools.test.ts +++ b/src/agents/tools/goal-tools.test.ts @@ -5,7 +5,11 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { resolveStorePath } from "../../config/sessions/paths.js"; -import { loadSessionStore, upsertSessionEntry } from "../../config/sessions/store.js"; +import { + loadSessionEntry, + upsertSessionEntry as upsertAccessorSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../../config/sessions/types.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { createCreateGoalTool, createGetGoalTool } from "./goal-tools.js"; @@ -18,6 +22,26 @@ async function createStoreConfig(): Promise<{ config: OpenClawConfig; template: }; } +// Goal tools read/write through the SQLite-backed accessor, so test fixtures +// must seed and assert through the same boundary. +function getSessionEntry(params: { + storePath: string; + sessionKey: string; +}): SessionEntry | undefined { + return loadSessionEntry(params); +} + +async function upsertSessionEntry(params: { + storePath: string; + sessionKey: string; + entry: SessionEntry; +}): Promise { + await upsertAccessorSessionEntry( + { sessionKey: params.sessionKey, storePath: params.storePath }, + params.entry, + ); +} + describe("goal tools", () => { it("keeps get_goal read-only when accounting changes are projected", async () => { // Budget-limited status can be derived for display without mutating the @@ -57,7 +81,7 @@ describe("goal tools", () => { const result = await tool.execute("call-1", {}); expect((result.details as { goal?: { status?: string } }).goal?.status).toBe("budget_limited"); - expect(loadSessionStore(storePath, { skipCache: true }).global?.goal?.status).toBe("active"); + expect(getSessionEntry({ storePath, sessionKey: "global" })?.goal?.status).toBe("active"); }); it("uses the resolved session agent for global session stores", async () => { @@ -78,10 +102,12 @@ describe("goal tools", () => { await tool.execute("call-1", { objective: "ship global work" }); const mainStorePath = resolveStorePath(template, { agentId: "main" }); - expect(loadSessionStore(researchStorePath, { skipCache: true }).global?.goal?.objective).toBe( - "ship global work", - ); - expect(loadSessionStore(mainStorePath, { skipCache: true }).global?.goal).toBeUndefined(); + expect( + getSessionEntry({ storePath: researchStorePath, sessionKey: "global" })?.goal?.objective, + ).toBe("ship global work"); + expect( + getSessionEntry({ storePath: mainStorePath, sessionKey: "global" })?.goal, + ).toBeUndefined(); }); it("prefers scoped run session keys over the fallback session agent", async () => { @@ -103,10 +129,10 @@ describe("goal tools", () => { const researchStorePath = resolveStorePath(template, { agentId: "research" }); expect( - loadSessionStore(opsStorePath, { skipCache: true })["agent:ops:main"]?.goal?.objective, + getSessionEntry({ storePath: opsStorePath, sessionKey: "agent:ops:main" })?.goal?.objective, ).toBe("ship ops work"); expect( - loadSessionStore(researchStorePath, { skipCache: true })["agent:ops:main"]?.goal, + getSessionEntry({ storePath: researchStorePath, sessionKey: "agent:ops:main" })?.goal, ).toBeUndefined(); }); }); diff --git a/src/auto-reply/reply.triggers.trigger-handling.e2e.test.ts b/src/auto-reply/reply.triggers.trigger-handling.e2e.test.ts index f8500b7bbbde..17f18abe04a8 100644 --- a/src/auto-reply/reply.triggers.trigger-handling.e2e.test.ts +++ b/src/auto-reply/reply.triggers.trigger-handling.e2e.test.ts @@ -16,7 +16,12 @@ import { withTempHome, } from "../../test/helpers/auto-reply/trigger-handling-test-harness.js"; import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; -import { loadSessionStore, resolveSessionKey, saveSessionStore } from "../config/sessions.js"; +import { resolveSessionKey } from "../config/sessions.js"; +import { + loadExactSessionEntry, + loadSessionEntry, + replaceSessionEntry, +} from "../config/sessions/session-accessor.js"; import { registerGroupIntroPromptCases } from "./reply.triggers.group-intro-prompts.cases.js"; import { registerTriggerHandlingUsageSummaryCases } from "./reply.triggers.trigger-handling.filters-usage-summary-current-model-provider.cases.js"; import { enqueueFollowupRun, getFollowupQueueDepth, type FollowupRun } from "./reply/queue.js"; @@ -165,15 +170,12 @@ async function writeDailyMemoryNotes( } async function seedTargetSession(storePath: string, targetSessionKey: string) { - await saveSessionStore( - storePath, + await replaceSessionEntry( + { storePath, sessionKey: targetSessionKey }, { - [targetSessionKey]: { - sessionId: "session-target", - updatedAt: Date.now(), - }, + sessionId: "session-target", + updatedAt: Date.now(), }, - { skipMaintenance: true }, ); } @@ -263,17 +265,14 @@ async function expectNextRunUsesTargetSession( } async function writeStoredModelOverride(cfg: ReturnType): Promise { - await saveSessionStore( - requireSessionStorePath(cfg), + await replaceSessionEntry( + { storePath: requireSessionStorePath(cfg), sessionKey: MAIN_SESSION_KEY }, { - [MAIN_SESSION_KEY]: { - sessionId: "main", - updatedAt: Date.now(), - providerOverride: "openai", - modelOverride: "gpt-5.4", - }, + sessionId: "main", + updatedAt: Date.now(), + providerOverride: "openai", + modelOverride: "gpt-5.4", }, - { skipMaintenance: true }, ); } @@ -565,9 +564,8 @@ describe("trigger handling", () => { const text = maybeReplyText(res); expect(text?.startsWith("⚙️ Compacted")).toBe(true); expect(getCompactEmbeddedAgentSessionMock()).toHaveBeenCalledOnce(); - const store = loadSessionStore(storePath); const sessionKey = resolveSessionKey("per-sender", request); - expect(store[sessionKey]?.compactionCount).toBe(1); + expect(loadSessionEntry({ storePath, sessionKey })?.compactionCount).toBe(1); }); }); @@ -607,15 +605,12 @@ describe("trigger handling", () => { const storePath = requireSessionStorePath(cfg); const targetSessionKey = "agent:main:telegram:group:123"; const targetSessionId = "session-target"; - await saveSessionStore( - storePath, + await replaceSessionEntry( + { storePath, sessionKey: targetSessionKey }, { - [targetSessionKey]: { - sessionId: targetSessionId, - updatedAt: Date.now(), - }, + sessionId: targetSessionId, + updatedAt: Date.now(), }, - { skipMaintenance: true }, ); const followupRun: FollowupRun = { prompt: "queued", @@ -663,8 +658,9 @@ describe("trigger handling", () => { expect(maybeReplyText(res)).toBe("⚙️ Agent was aborted."); expect(getAbortEmbeddedAgentRunMock()).toHaveBeenCalledWith(targetSessionId); - const store = loadSessionStore(storePath); - expect(store[targetSessionKey]?.abortedLastRun).toBe(true); + expect(loadSessionEntry({ storePath, sessionKey: targetSessionKey })?.abortedLastRun).toBe( + true, + ); expect(getFollowupQueueDepth(targetSessionKey)).toBe(0); }); }); @@ -693,10 +689,10 @@ describe("trigger handling", () => { expect(maybeReplyText(res)).toContain("Model set to openai/gpt-4.1-mini"); - const store = loadSessionStore(storePath); - expect(store[targetSessionKey]?.providerOverride).toBe("openai"); - expect(store[targetSessionKey]?.modelOverride).toBe("gpt-4.1-mini"); - expect(store[slashSessionKey]).toBeUndefined(); + const targetEntry = loadSessionEntry({ storePath, sessionKey: targetSessionKey }); + expect(targetEntry?.providerOverride).toBe("openai"); + expect(targetEntry?.modelOverride).toBe("gpt-4.1-mini"); + expect(loadExactSessionEntry({ storePath, sessionKey: slashSessionKey })).toBeUndefined(); await expectNextRunUsesTargetSession( { cfg, targetSessionKey, runEmbeddedAgentMock }, @@ -728,17 +724,14 @@ describe("trigger handling", () => { const slashSessionKey = "agent:main:telegram:slash:7595562691"; const targetSessionKey = "agent:main:main:thread:7595562691:12812"; - await saveSessionStore( - storePath, + await replaceSessionEntry( + { storePath, sessionKey: targetSessionKey }, { - [targetSessionKey]: { - sessionId: "session-target", - updatedAt: Date.now(), - providerOverride: "zai", - modelOverride: "glm-5.1", - }, + sessionId: "session-target", + updatedAt: Date.now(), + providerOverride: "zai", + modelOverride: "glm-5.1", }, - { skipMaintenance: true }, ); const res = await getReplyFromConfig( @@ -753,10 +746,10 @@ describe("trigger handling", () => { expect(maybeReplyText(res)).toContain("Model set to deepseek/deepseek-v4-pro"); - const store = loadSessionStore(storePath); - expect(store[targetSessionKey]?.providerOverride).toBe("deepseek"); - expect(store[targetSessionKey]?.modelOverride).toBe("deepseek-v4-pro"); - expect(store[slashSessionKey]).toBeUndefined(); + const targetEntry = loadSessionEntry({ storePath, sessionKey: targetSessionKey }); + expect(targetEntry?.providerOverride).toBe("deepseek"); + expect(targetEntry?.modelOverride).toBe("deepseek-v4-pro"); + expect(loadExactSessionEntry({ storePath, sessionKey: slashSessionKey })).toBeUndefined(); await expectNextRunUsesTargetSession( { cfg, targetSessionKey, runEmbeddedAgentMock }, @@ -819,10 +812,10 @@ describe("trigger handling", () => { expect(maybeReplyText(res)).toContain(`Auth profile set to ${TEST_SECONDARY_PROFILE_ID}`); - const store = loadSessionStore(storePath); - expect(store[targetSessionKey]?.authProfileOverride).toBe(TEST_SECONDARY_PROFILE_ID); - expect(store[targetSessionKey]?.authProfileOverrideSource).toBe("user"); - expect(store[slashSessionKey]).toBeUndefined(); + const targetEntry = loadSessionEntry({ storePath, sessionKey: targetSessionKey }); + expect(targetEntry?.authProfileOverride).toBe(TEST_SECONDARY_PROFILE_ID); + expect(targetEntry?.authProfileOverrideSource).toBe("user"); + expect(loadExactSessionEntry({ storePath, sessionKey: slashSessionKey })).toBeUndefined(); await expectNextRunUsesTargetSession( { cfg, targetSessionKey, runEmbeddedAgentMock }, diff --git a/src/auto-reply/reply.triggers.trigger-handling.filters-usage-summary-current-model-provider.cases.ts b/src/auto-reply/reply.triggers.trigger-handling.filters-usage-summary-current-model-provider.cases.ts index 1c32c4744390..226fe861ebb2 100644 --- a/src/auto-reply/reply.triggers.trigger-handling.filters-usage-summary-current-model-provider.cases.ts +++ b/src/auto-reply/reply.triggers.trigger-handling.filters-usage-summary-current-model-provider.cases.ts @@ -8,7 +8,7 @@ import { requireSessionStorePath, withTempHome, } from "../../test/helpers/auto-reply/trigger-handling-test-harness.js"; -import { loadSessionStore } from "../config/sessions.js"; +import { listSessionEntries } from "../config/sessions/session-accessor.js"; type GetReplyFromConfig = typeof import("./reply.js").getReplyFromConfig; @@ -135,8 +135,8 @@ export function registerTriggerHandlingUsageSummaryCases(params: { ); expect(replyText(r3)).toContain("Usage footer: tokens"); - const finalStore = loadSessionStore(usageStorePath, { skipCache: true }); - expect(Object.values(finalStore)[0]?.responseUsage).toBe("tokens"); + const finalEntries = listSessionEntries({ storePath: usageStorePath }); + expect(finalEntries[0]?.entry.responseUsage).toBe("tokens"); expect(runEmbeddedAgentMock).not.toHaveBeenCalled(); }); }); diff --git a/src/auto-reply/reply/abort-cutoff.runtime.test.ts b/src/auto-reply/reply/abort-cutoff.runtime.test.ts index 345d15e74f57..3395a9c1d801 100644 --- a/src/auto-reply/reply/abort-cutoff.runtime.test.ts +++ b/src/auto-reply/reply/abort-cutoff.runtime.test.ts @@ -3,7 +3,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { loadSessionStore } from "../../config/sessions.js"; +import { loadSessionEntry } from "../../config/sessions/session-accessor.js"; import type { SessionEntry } from "../../config/sessions/types.js"; import { clearAbortCutoffInSessionRuntime } from "./abort-cutoff.runtime.js"; @@ -29,7 +29,6 @@ describe("clearAbortCutoffInSessionRuntime", () => { abortCutoffTimestamp: 123, }; const sessionStore: Record = { [sessionKey]: entry }; - await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf8"); const cleared = await clearAbortCutoffInSessionRuntime({ sessionEntry: entry, @@ -38,7 +37,7 @@ describe("clearAbortCutoffInSessionRuntime", () => { storePath, }); - const persisted = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + const persisted = loadSessionEntry({ storePath, sessionKey }); expect(cleared).toBe(true); expect(sessionStore[sessionKey]?.sessionId).toBe("cutoff-session"); expect(sessionStore[sessionKey]?.modelProvider).toBe("anthropic"); diff --git a/src/auto-reply/reply/abort.test.ts b/src/auto-reply/reply/abort.test.ts index d296ffca266a..150bcb973118 100644 --- a/src/auto-reply/reply/abort.test.ts +++ b/src/auto-reply/reply/abort.test.ts @@ -1,10 +1,13 @@ // Tests abort request handling, cutoff persistence, and active run cleanup. -import fs from "node:fs/promises"; import path from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { SubagentRunRecord } from "../../agents/subagent-registry.js"; import type { OpenClawConfig } from "../../config/config.js"; -import type { SessionAbortTargetResult } from "../../config/sessions/session-accessor.js"; +import { + loadSessionEntry, + replaceSessionEntry, + type SessionAbortTargetResult, +} from "../../config/sessions/session-accessor.js"; import { createSuiteTempRootTracker } from "../../test-helpers/temp-dir.js"; import { testing as abortTesting, @@ -100,13 +103,15 @@ describe("abort detection", () => { sessionIdsByKey: Record, nowMs = Date.now(), ) { - const storeEntries = Object.fromEntries( - Object.entries(sessionIdsByKey).map(([key, sessionId]) => [ - key, - { sessionId, updatedAt: nowMs }, - ]), + await Promise.all( + Object.entries(sessionIdsByKey).map(([sessionKey, sessionId]) => + replaceSessionEntry({ storePath, sessionKey }, { sessionId, updatedAt: nowMs }), + ), ); - await fs.writeFile(storePath, JSON.stringify(storeEntries, null, 2)); + } + + function readAbortSessionEntry(storePath: string, sessionKey: string) { + return loadSessionEntry({ storePath, sessionKey }); } async function createAbortConfig(params?: { @@ -1075,17 +1080,12 @@ describe("abort detection", () => { sessionKey: acpSessionKey, reason: "fast-abort", }); - const store = JSON.parse(await fs.readFile(storePath, "utf8")) as Record< - string, - { - abortCutoffMessageSid?: string; - abortCutoffTimestamp?: number; - } - >; - expect(store[sourceSessionKey]?.abortCutoffMessageSid).toBe("77"); - expect(store[sourceSessionKey]?.abortCutoffTimestamp).toBe(1234567890000); - expect(store[acpSessionKey]?.abortCutoffMessageSid).toBeUndefined(); - expect(store[acpSessionKey]?.abortCutoffTimestamp).toBeUndefined(); + const sourceEntry = readAbortSessionEntry(storePath, sourceSessionKey); + const acpEntry = readAbortSessionEntry(storePath, acpSessionKey); + expect(sourceEntry?.abortCutoffMessageSid).toBe("77"); + expect(sourceEntry?.abortCutoffTimestamp).toBe(1234567890000); + expect(acpEntry?.abortCutoffMessageSid).toBeUndefined(); + expect(acpEntry?.abortCutoffTimestamp).toBeUndefined(); }); it("persists abort cutoff metadata on /stop when command and target session match", async () => { @@ -1105,15 +1105,10 @@ describe("abort detection", () => { }); expect(result.handled).toBe(true); - const store = JSON.parse(await fs.readFile(storePath, "utf8")) as Record; - const entry = store[sessionKey] as { - abortedLastRun?: boolean; - abortCutoffMessageSid?: string; - abortCutoffTimestamp?: number; - }; - expect(entry.abortedLastRun).toBe(true); - expect(entry.abortCutoffMessageSid).toBe("55"); - expect(entry.abortCutoffTimestamp).toBe(1234567890000); + const entry = readAbortSessionEntry(storePath, sessionKey); + expect(entry?.abortedLastRun).toBe(true); + expect(entry?.abortCutoffMessageSid).toBe("55"); + expect(entry?.abortCutoffTimestamp).toBe(1234567890000); }); it("persists abort cutoff metadata when only ParentSessionKey identifies the command session", async () => { @@ -1133,15 +1128,10 @@ describe("abort detection", () => { }); expect(result.handled).toBe(true); - const store = JSON.parse(await fs.readFile(storePath, "utf8")) as Record; - const entry = store[sessionKey] as { - abortedLastRun?: boolean; - abortCutoffMessageSid?: string; - abortCutoffTimestamp?: number; - }; - expect(entry.abortedLastRun).toBe(true); - expect(entry.abortCutoffMessageSid).toBe("56"); - expect(entry.abortCutoffTimestamp).toBe(1234567890001); + const entry = readAbortSessionEntry(storePath, sessionKey); + expect(entry?.abortedLastRun).toBe(true); + expect(entry?.abortCutoffMessageSid).toBe("56"); + expect(entry?.abortCutoffTimestamp).toBe(1234567890001); }); it("does not persist cutoff metadata when native /stop targets a different session", async () => { @@ -1163,15 +1153,10 @@ describe("abort detection", () => { }); expect(result.handled).toBe(true); - const store = JSON.parse(await fs.readFile(storePath, "utf8")) as Record; - const entry = store[targetSessionKey] as { - abortedLastRun?: boolean; - abortCutoffMessageSid?: string; - abortCutoffTimestamp?: number; - }; - expect(entry.abortedLastRun).toBe(true); - expect(entry.abortCutoffMessageSid).toBeUndefined(); - expect(entry.abortCutoffTimestamp).toBeUndefined(); + const entry = readAbortSessionEntry(storePath, targetSessionKey); + expect(entry?.abortedLastRun).toBe(true); + expect(entry?.abortCutoffMessageSid).toBeUndefined(); + expect(entry?.abortCutoffTimestamp).toBeUndefined(); }); it("fast-abort stops active subagent runs for requester session", async () => { diff --git a/src/auto-reply/reply/agent-runner-direct-runtime-config.test.ts b/src/auto-reply/reply/agent-runner-direct-runtime-config.test.ts index 0cc254a21fca..f07217998163 100644 --- a/src/auto-reply/reply/agent-runner-direct-runtime-config.test.ts +++ b/src/auto-reply/reply/agent-runner-direct-runtime-config.test.ts @@ -123,6 +123,8 @@ function createReplyOperation(): ReplyOperation { lastActivityAtMs: Date.now(), recordActivity: vi.fn(), setPhase: vi.fn(), + markWaitingForDeferredMaintenance: vi.fn(), + markDeferredMaintenanceWaitEnded: vi.fn(), updateSessionId: vi.fn(), hasOwnedSessionId: vi.fn(() => false), attachBackend: vi.fn(), diff --git a/src/auto-reply/reply/agent-runner-execution.test.ts b/src/auto-reply/reply/agent-runner-execution.test.ts index 9953889bc960..c742f7310243 100644 --- a/src/auto-reply/reply/agent-runner-execution.test.ts +++ b/src/auto-reply/reply/agent-runner-execution.test.ts @@ -21,6 +21,7 @@ import { createUserTurnTranscriptRecorder, type PersistedUserTurnMessage, } from "../../sessions/user-turn-transcript.js"; +import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js"; import { getReplyPayloadMetadata } from "../reply-payload.js"; import type { TemplateContext } from "../templating.js"; import { SILENT_REPLY_TOKEN } from "../tokens.js"; @@ -484,7 +485,7 @@ function createFollowupRun(): FollowupRun { function createTestUserTurnRecorder(message: PersistedUserTurnMessage) { return createUserTurnTranscriptRecorder({ message, - target: { transcriptPath: "/tmp/session.jsonl" }, + target: createTestUserTurnTranscriptTarget(), updateMode: "none", }); } @@ -519,6 +520,8 @@ function createMockReplyOperation(): { hasOwnedSessionId: vi.fn((sessionId: string) => sessionId === "session"), recordActivity: vi.fn(), setPhase: vi.fn(), + markWaitingForDeferredMaintenance: vi.fn(), + markDeferredMaintenanceWaitEnded: vi.fn(), updateSessionId: updateSessionIdMock, attachBackend: vi.fn(), detachBackend: vi.fn(), diff --git a/src/auto-reply/reply/agent-runner-memory.test.ts b/src/auto-reply/reply/agent-runner-memory.test.ts index e18606db63bd..6739dd9636d2 100644 --- a/src/auto-reply/reply/agent-runner-memory.test.ts +++ b/src/auto-reply/reply/agent-runner-memory.test.ts @@ -6,6 +6,8 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { testing as cliBackendsTesting } from "../../agents/cli-backends.js"; import type { SessionEntry } from "../../config/sessions.js"; +import { loadSessionEntry } from "../../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import { clearMemoryPluginState, registerMemoryCapability, @@ -71,9 +73,19 @@ function createReplyOperation(): TestReplyOperation { abortForRestart: vi.fn(() => true), markTerminalRecovery: vi.fn(), markAcceptedSteeredInboundAudio: vi.fn(), + markWaitingForDeferredMaintenance: vi.fn(), + markDeferredMaintenanceWaitEnded: vi.fn(), }; } +function loadMainSessionEntry(storePath: string): SessionEntry { + const entry = loadSessionEntry({ storePath, sessionKey: "main" }); + if (!entry) { + throw new Error("expected persisted main session entry"); + } + return entry; +} + type RefreshQueuedFollowupSessionParams = { key?: string; previousSessionId?: string; @@ -322,13 +334,11 @@ describe("runMemoryFlushIfNeeded", () => { expect(refreshCall.nextSessionId).toBe("session-rotated"); expect(refreshCall.nextSessionFile).toContain("session-rotated.jsonl"); - const persisted = JSON.parse(await fs.readFile(storePath, "utf8")) as { - main: SessionEntry; - }; - expect(persisted.main.sessionId).toBe("session-rotated"); - expect(persisted.main.compactionCount).toBe(2); - expect(persisted.main.memoryFlushCompactionCount).toBe(1); - expect(persisted.main.memoryFlushAt).toBe(1_700_000_000_000); + const persisted = loadMainSessionEntry(storePath); + expect(persisted.sessionId).toBe("session-rotated"); + expect(persisted.compactionCount).toBe(2); + expect(persisted.memoryFlushCompactionCount).toBe(1); + expect(persisted.memoryFlushAt).toBe(1_700_000_000_000); }); it("revalidates immutable Ultra for each memory-flush fallback candidate", async () => { @@ -505,11 +515,11 @@ describe("runMemoryFlushIfNeeded", () => { expect(result.outcome).toBe("failed"); expect(result.sessionEntry?.sessionId).toBe("session-rotated"); expect(followupRun.run.sessionId).toBe("session-rotated"); - const persisted = JSON.parse(await fs.readFile(storePath, "utf8")) as { main: SessionEntry }; - expect(persisted.main.sessionId).toBe("session-rotated"); - expect(persisted.main.compactionCount).toBe(2); - expect(persisted.main.memoryFlushFailureCount).toBe(1); - expect(persisted.main.memoryFlushAt).toBeUndefined(); + const persisted = loadMainSessionEntry(storePath); + expect(persisted.sessionId).toBe("session-rotated"); + expect(persisted.compactionCount).toBe(2); + expect(persisted.memoryFlushFailureCount).toBe(1); + expect(persisted.memoryFlushAt).toBeUndefined(); }); it("reports restricted memory-flush write failures for visible delivery", async () => { @@ -685,11 +695,11 @@ describe("runMemoryFlushIfNeeded", () => { replyOperation: createReplyOperation(), }); - const persisted = JSON.parse(await fs.readFile(storePath, "utf8")) as { main: SessionEntry }; + const persisted = loadMainSessionEntry(storePath); expect(result.outcome).toBe("failed"); - expect(persisted.main.memoryFlushFailureCount).toBe(1); - expect(persisted.main.memoryFlushLastFailedAt).toBe(1_700_000_000_000); - expect(persisted.main.memoryFlushLastFailureError).toBe(`${"a".repeat(198)}…`); + expect(persisted.memoryFlushFailureCount).toBe(1); + expect(persisted.memoryFlushLastFailedAt).toBe(1_700_000_000_000); + expect(persisted.memoryFlushLastFailureError).toBe(`${"a".repeat(198)}…`); expect(emitAgentEventMock).toHaveBeenCalledWith( expect.objectContaining({ stream: "lifecycle", @@ -731,11 +741,11 @@ describe("runMemoryFlushIfNeeded", () => { replyOperation: createReplyOperation(), }); - const persisted = JSON.parse(await fs.readFile(storePath, "utf8")) as { main: SessionEntry }; + const persisted = loadMainSessionEntry(storePath); expect(result.outcome).toBe("failed"); - expect(persisted.main.memoryFlushFailureCount).toBe(0); - expect(persisted.main.memoryFlushLastFailedAt).toBeUndefined(); - expect(persisted.main.memoryFlushLastFailureError).toBeUndefined(); + expect(persisted.memoryFlushFailureCount).toBe(0); + expect(persisted.memoryFlushLastFailedAt).toBeUndefined(); + expect(persisted.memoryFlushLastFailureError).toBeUndefined(); }); it("clears failure counters on successful flush", async () => { @@ -766,11 +776,11 @@ describe("runMemoryFlushIfNeeded", () => { replyOperation: createReplyOperation(), }); - const persisted = JSON.parse(await fs.readFile(storePath, "utf8")) as { main: SessionEntry }; + const persisted = loadMainSessionEntry(storePath); expect(result.outcome).toBe("completed"); - expect(persisted.main.memoryFlushFailureCount).toBe(0); - expect(persisted.main.memoryFlushLastFailedAt).toBeUndefined(); - expect(persisted.main.memoryFlushLastFailureError).toBeUndefined(); + expect(persisted.memoryFlushFailureCount).toBe(0); + expect(persisted.memoryFlushLastFailedAt).toBeUndefined(); + expect(persisted.memoryFlushLastFailureError).toBeUndefined(); }); it("marks flush as completed after MAX_FLUSH_FAILURES to break retry loop", async () => { @@ -804,10 +814,10 @@ describe("runMemoryFlushIfNeeded", () => { }, }); - const persisted = JSON.parse(await fs.readFile(storePath, "utf8")) as { main: SessionEntry }; + const persisted = loadMainSessionEntry(storePath); expect(result.outcome).toBe("exhausted"); - expect(persisted.main.memoryFlushCompactionCount).toBe(1); - expect(persisted.main.memoryFlushFailureCount).toBe(TEST_MAX_FLUSH_FAILURES); + expect(persisted.memoryFlushCompactionCount).toBe(1); + expect(persisted.memoryFlushFailureCount).toBe(TEST_MAX_FLUSH_FAILURES); expect(emitAgentEventMock).toHaveBeenCalledWith( expect.objectContaining({ stream: "lifecycle", @@ -857,8 +867,8 @@ describe("runMemoryFlushIfNeeded", () => { expect(runWithModelFallbackMock).toHaveBeenCalledTimes(2); - const persisted = JSON.parse(await fs.readFile(storePath, "utf8")) as { main: SessionEntry }; - expect(persisted.main.memoryFlushFailureCount).toBe(2); + const persisted = loadMainSessionEntry(storePath); + expect(persisted.memoryFlushFailureCount).toBe(2); }); it("next message retries flush after failure", async () => { @@ -1515,8 +1525,10 @@ describe("runMemoryFlushIfNeeded", () => { }); it("passes resolved context budget and auth profile to preflight compaction", async () => { + const sessionFile = path.join(rootDir, "budget-session.jsonl"); const sessionEntry: SessionEntry = { sessionId: "session", + sessionFile, updatedAt: Date.now(), totalTokens: 245_000, totalTokensFresh: true, @@ -1553,8 +1565,15 @@ describe("runMemoryFlushIfNeeded", () => { systemPrompt: "Write memory to memory/YYYY-MM-DD.md.", relativePath: "memory/2023-11-14.md", })); + const storePath = path.join(rootDir, "preflight-fresh-sessions.json"); + const sessionKey = "agent:main:main"; const sessionEntry: SessionEntry = { sessionId: "session", + sessionFile: formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: "session", + storePath, + }), updatedAt: Date.now(), totalTokens: 985, totalTokensFresh: true, @@ -1566,14 +1585,15 @@ describe("runMemoryFlushIfNeeded", () => { followupRun: createTestFollowupRun({ provider: "anthropic", model: "claude", - sessionKey: "agent:main:main", + sessionKey, }), promptForEstimate: "Please summarize the entire design discussion above. ".repeat(8), defaultModel: "anthropic/claude", agentCfgContextTokens: 1000, sessionEntry, - sessionStore: { "agent:main:main": sessionEntry }, - sessionKey: "agent:main:main", + sessionStore: { [sessionKey]: sessionEntry }, + sessionKey, + storePath, isHeartbeat: false, replyOperation: createReplyOperation(), }); @@ -1794,6 +1814,7 @@ describe("runMemoryFlushIfNeeded", () => { }); const sessionEntry: SessionEntry = { sessionId: "session", + sessionFile: path.join(rootDir, "required-preflight-session.jsonl"), updatedAt: Date.now(), totalTokens: 180_499, totalTokensFresh: true, @@ -2021,7 +2042,7 @@ describe("runMemoryFlushIfNeeded", () => { expect(compactEmbeddedAgentSessionMock).not.toHaveBeenCalled(); }); - it("uses the active run sessionFile when the session entry has no transcript path", async () => { + it("does not use the active run sessionFile when the session entry has no transcript path", async () => { const sessionFile = path.join(rootDir, "active-run-session.jsonl"); await fs.writeFile( sessionFile, @@ -2065,10 +2086,7 @@ describe("runMemoryFlushIfNeeded", () => { replyOperation: createReplyOperation(), }); - expect(compactEmbeddedAgentSessionMock).toHaveBeenCalledTimes(1); - const compactCall = requireCompactEmbeddedAgentSessionCall(); - expect(compactCall.sessionId).toBe("session"); - expect(compactCall.sessionFile).toContain("active-run-session.jsonl"); + expect(compactEmbeddedAgentSessionMock).not.toHaveBeenCalled(); }); it("does not treat unavailable Anthropic context as transcript prompt usage", async () => { diff --git a/src/auto-reply/reply/agent-runner-memory.ts b/src/auto-reply/reply/agent-runner-memory.ts index b93229eb3714..80a880c81920 100644 --- a/src/auto-reply/reply/agent-runner-memory.ts +++ b/src/auto-reply/reply/agent-runner-memory.ts @@ -39,6 +39,10 @@ import { type SessionEntry, } from "../../config/sessions.js"; import { updateSessionEntry } from "../../config/sessions/session-accessor.js"; +import { + formatSqliteSessionFileMarker, + parseSqliteSessionFileMarker, +} from "../../config/sessions/sqlite-marker.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { readSessionMessagesAsync } from "../../gateway/session-utils.fs.js"; import { logVerbose } from "../../globals.js"; @@ -410,18 +414,27 @@ function resolveSessionLogPath( (sessionEntry as (SessionEntry & { transcriptPath?: string }) | undefined)?.transcriptPath, ); const sessionFile = normalizeOptionalString(sessionEntry?.sessionFile) || transcriptPath; + if (parseSqliteSessionFileMarker(sessionFile)) { + return sessionFile; + } const agentId = resolveAgentIdFromSessionKey(sessionKey); + if (!sessionFile && agentId && opts?.storePath) { + return formatSqliteSessionFileMarker({ + agentId, + sessionId, + storePath: opts.storePath, + }); + } + if (!sessionFile) { + return undefined; + } const pathOpts = resolveSessionFilePathOptions({ agentId, storePath: opts?.storePath, }); // Normalize sessionFile through resolveSessionFilePath so relative entries // are resolved against the sessions dir/store layout, not process.cwd(). - return resolveSessionFilePath( - sessionId, - sessionFile ? { sessionFile } : sessionEntry, - pathOpts, - ); + return resolveSessionFilePath(sessionId, { sessionFile }, pathOpts); } catch { return undefined; } @@ -504,6 +517,9 @@ async function readSessionLogSnapshot(params: { if (!logPath) { return {}; } + if (parseSqliteSessionFileMarker(logPath)) { + return {}; + } const snapshot: SessionLogSnapshot = {}; let usageScan: SessionLogUsageScan | undefined; @@ -621,22 +637,16 @@ async function estimatePromptTokensFromSessionTranscript(params: { sessionId?: string; sessionEntry?: SessionEntry; sessionKey?: string; - sessionFile?: string; storePath?: string; }): Promise { const sessionId = normalizeOptionalString(params.sessionId); if (!sessionId) { return undefined; } - const fallbackSessionFile = normalizeOptionalString(params.sessionFile); - const sessionEntryForTranscript = - params.sessionEntry?.sessionFile || !fallbackSessionFile - ? params.sessionEntry - : ({ ...params.sessionEntry, sessionFile: fallbackSessionFile } as SessionEntry); try { const snapshot = await readSessionLogSnapshot({ sessionId, - sessionEntry: sessionEntryForTranscript, + sessionEntry: params.sessionEntry, sessionKey: params.sessionKey, opts: { storePath: params.storePath }, includeByteSize: true, @@ -670,7 +680,7 @@ async function estimatePromptTokensFromSessionTranscript(params: { const messages = (await readSessionMessagesAsync( sessionId, params.storePath, - sessionEntryForTranscript?.sessionFile, + params.sessionEntry?.sessionFile, { mode: "recent", maxMessages: 200, @@ -818,17 +828,13 @@ export async function runPreflightCompactionIfNeeded(params: { sessionId: entry.sessionId, sessionEntry: entry, sessionKey: params.sessionKey ?? params.followupRun.run.sessionKey, - sessionFile: entry.sessionFile ?? params.followupRun.run.sessionFile, storePath: params.storePath, }); const transcriptSizeSnapshot = shouldCheckActiveTranscriptBytes && transcriptUsageTokens?.transcriptByteSize === undefined ? await readSessionLogSnapshot({ sessionId: entry.sessionId, - sessionEntry: - entry.sessionFile || !params.followupRun.run.sessionFile - ? entry - : { ...entry, sessionFile: params.followupRun.run.sessionFile }, + sessionEntry: entry, sessionKey: params.sessionKey ?? params.followupRun.run.sessionKey, opts: { storePath: params.storePath }, includeByteSize: true, @@ -931,10 +937,14 @@ export async function runPreflightCompactionIfNeeded(params: { await notifyStartCompaction(); const sessionFile = resolveSessionLogPath( entry.sessionId, - entry.sessionFile ? entry : { ...entry, sessionFile: params.followupRun.run.sessionFile }, + entry, params.sessionKey ?? params.followupRun.run.sessionKey, { storePath: params.storePath }, ); + if (!sessionFile) { + await notifyTerminalCompaction("skipped"); + return entry ?? params.sessionEntry; + } const result = await deps.compactEmbeddedAgentSession({ sessionId: entry.sessionId, sessionKey: params.sessionKey, @@ -949,7 +959,7 @@ export async function runPreflightCompactionIfNeeded(params: { senderName: params.followupRun.run.senderName, senderUsername: params.followupRun.run.senderUsername, senderE164: params.followupRun.run.senderE164, - sessionFile: sessionFile ?? params.followupRun.run.sessionFile, + sessionFile, workspaceDir: params.followupRun.run.workspaceDir, cwd: params.followupRun.run.cwd, agentDir: params.followupRun.run.agentDir, diff --git a/src/auto-reply/reply/agent-runner-session-reset.test.ts b/src/auto-reply/reply/agent-runner-session-reset.test.ts index 740d28e76b76..48edd2aa76fb 100644 --- a/src/auto-reply/reply/agent-runner-session-reset.test.ts +++ b/src/auto-reply/reply/agent-runner-session-reset.test.ts @@ -4,6 +4,12 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { SessionEntry } from "../../config/sessions.js"; +import { + appendTranscriptMessage, + loadSessionEntry, + loadTranscriptEvents, +} from "../../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import { resetReplyRunSession, setAgentRunnerSessionResetTestDeps, @@ -74,13 +80,13 @@ describe("resetReplyRunSession", () => { fallbackNoticeSelectedModel: "anthropic/claude", fallbackNoticeActiveModel: "openai/gpt", fallbackNoticeReason: "rate limit", - compactionCount: 5, - memoryFlushAt: 123, - memoryFlushCompactionCount: 4, - memoryFlushContextHash: "stale-context", - memoryFlushFailureCount: 3, - memoryFlushLastFailedAt: 456, - memoryFlushLastFailureError: "provider crashed", + compactionCount: 4, + memoryFlushAt: 50, + memoryFlushCompactionCount: 3, + memoryFlushContextHash: "context-hash", + memoryFlushFailureCount: 2, + memoryFlushLastFailedAt: 60, + memoryFlushLastFailureError: "memory failed", systemPromptReport: { source: "run", generatedAt: 1, @@ -134,6 +140,13 @@ describe("resetReplyRunSession", () => { expect(activeSessionEntry?.memoryFlushLastFailedAt).toBeUndefined(); expect(activeSessionEntry?.memoryFlushLastFailureError).toBeUndefined(); expect(activeSessionEntry?.systemPromptReport).toBeUndefined(); + expect(activeSessionEntry?.compactionCount).toBe(0); + expect(activeSessionEntry?.memoryFlushAt).toBeUndefined(); + expect(activeSessionEntry?.memoryFlushCompactionCount).toBeUndefined(); + expect(activeSessionEntry?.memoryFlushContextHash).toBeUndefined(); + expect(activeSessionEntry?.memoryFlushFailureCount).toBeUndefined(); + expect(activeSessionEntry?.memoryFlushLastFailedAt).toBeUndefined(); + expect(activeSessionEntry?.memoryFlushLastFailureError).toBeUndefined(); expect(refreshQueuedFollowupSessionMock).toHaveBeenCalledWith({ key: "main", previousSessionId: "session", @@ -142,16 +155,15 @@ describe("resetReplyRunSession", () => { }); expect(errorMock).toHaveBeenCalledWith("reset 00000000-0000-0000-0000-000000000123"); - const persisted = JSON.parse(await fs.readFile(storePath, "utf8")) as { - main: SessionEntry; - }; - expect(persisted.main.sessionId).toBe(activeSessionEntry?.sessionId); - expect(persisted.main.contextBudgetStatus).toBeUndefined(); - expect(persisted.main.fallbackNoticeReason).toBeUndefined(); - expect(persisted.main.compactionCount).toBe(0); - expect(persisted.main.memoryFlushFailureCount).toBeUndefined(); - expect(persisted.main.memoryFlushLastFailedAt).toBeUndefined(); - expect(persisted.main.memoryFlushLastFailureError).toBeUndefined(); + const persisted = loadSessionEntry({ storePath, sessionKey: "main" }); + expect(persisted?.sessionId).toBe(activeSessionEntry?.sessionId); + expect(persisted?.contextBudgetStatus).toBeUndefined(); + expect(persisted?.fallbackNoticeReason).toBeUndefined(); + expect(persisted?.compactionCount).toBe(0); + expect(persisted?.memoryFlushAt).toBeUndefined(); + expect(persisted?.memoryFlushFailureCount).toBeUndefined(); + expect(persisted?.memoryFlushLastFailedAt).toBeUndefined(); + expect(persisted?.memoryFlushLastFailureError).toBeUndefined(); }); it("rejects automatic recovery rotation for a model-locked session", async () => { @@ -256,4 +268,123 @@ describe("resetReplyRunSession", () => { expect(rotatedSessionId).not.toBe("old-session"); await fs.access(oldTranscriptPath); }); + + it("uses SQLite markers and replays DM continuity rows during reset", async () => { + const storePath = path.join(rootDir, "sessions.json"); + const sessionKey = "main"; + const oldSessionId = "old-session"; + const oldSessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: oldSessionId, + storePath, + }); + const sessionEntry: SessionEntry = { + sessionFile: oldSessionFile, + sessionId: oldSessionId, + updatedAt: 1, + }; + const sessionStore = { [sessionKey]: sessionEntry }; + await writeTestSessionStore(storePath, sessionKey, sessionEntry); + await appendTranscriptMessage( + { agentId: "main", sessionId: oldSessionId, sessionKey, storePath }, + { + message: { role: "user", content: "hello" }, + }, + ); + await appendTranscriptMessage( + { agentId: "main", sessionId: oldSessionId, sessionKey, storePath }, + { + message: { role: "assistant", content: "hi" }, + }, + ); + + let activeSessionEntry: SessionEntry | undefined; + await resetReplyRunSession({ + options: { + failureLabel: "role ordering conflict", + buildLogMessage: (next) => `reset ${next}`, + }, + sessionKey, + queueKey: sessionKey, + activeSessionEntry: sessionEntry, + activeSessionStore: sessionStore, + storePath, + followupRun: createTestFollowupRun(), + onActiveSessionEntry: (entry) => { + activeSessionEntry = entry; + }, + onNewSession: () => {}, + }); + + expect(activeSessionEntry?.sessionFile).toBe( + formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: "00000000-0000-0000-0000-000000000123", + storePath, + }), + ); + const replayed = await loadTranscriptEvents({ + agentId: "main", + sessionId: "00000000-0000-0000-0000-000000000123", + sessionKey, + storePath, + }); + const replayedMessages = replayed.filter( + (entry): entry is { message: { content?: unknown }; type: "message" } => + Boolean(entry && typeof entry === "object" && "message" in entry), + ); + expect(replayedMessages).toEqual([ + expect.objectContaining({ message: expect.objectContaining({ content: "hello" }) }), + expect.objectContaining({ message: expect.objectContaining({ content: "hi" }) }), + ]); + }); + + it("continues SQLite reset when previous replay source is unreadable", async () => { + const storePath = path.join(rootDir, "sessions.json"); + const sessionKey = "main"; + const unreadableReplaySource = path.join(rootDir, "previous-transcript-dir"); + await fs.mkdir(unreadableReplaySource); + const sessionEntry: SessionEntry = { + sessionFile: unreadableReplaySource, + sessionId: "old-session", + updatedAt: 1, + }; + const sessionStore = { [sessionKey]: sessionEntry }; + await writeTestSessionStore(storePath, sessionKey, sessionEntry); + + let activeSessionEntry: SessionEntry | undefined; + const reset = await resetReplyRunSession({ + options: { + failureLabel: "role ordering conflict", + buildLogMessage: (next) => `reset ${next}`, + }, + sessionKey, + queueKey: sessionKey, + activeSessionEntry: sessionEntry, + activeSessionStore: sessionStore, + storePath, + followupRun: createTestFollowupRun(), + onActiveSessionEntry: (entry) => { + activeSessionEntry = entry; + }, + onNewSession: () => {}, + }); + + expect(reset).toBe(true); + expect(activeSessionEntry?.sessionFile).toBe( + formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: "00000000-0000-0000-0000-000000000123", + storePath, + }), + ); + await expect( + loadTranscriptEvents({ + agentId: "main", + sessionId: "00000000-0000-0000-0000-000000000123", + sessionKey, + storePath, + }), + ).resolves.toEqual([]); + }); }); diff --git a/src/auto-reply/reply/agent-runner-session-reset.ts b/src/auto-reply/reply/agent-runner-session-reset.ts index 9aeab48c78c2..d25cff13ca85 100644 --- a/src/auto-reply/reply/agent-runner-session-reset.ts +++ b/src/auto-reply/reply/agent-runner-session-reset.ts @@ -1,10 +1,8 @@ // Handles session reset requests produced during agent runner execution. import type { SessionEntry } from "../../config/sessions.js"; -import { - resolveAgentIdFromSessionKey, - resolveSessionTranscriptPath, -} from "../../config/sessions.js"; +import { resolveAgentIdFromSessionKey } from "../../config/sessions.js"; import { persistSessionResetLifecycle } from "../../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import { generateSecureUuid } from "../../infra/secure-random.js"; import { defaultRuntime } from "../../runtime.js"; import { @@ -98,11 +96,11 @@ export async function resetReplyRunSession(params: { memoryFlushLastFailureError: undefined, }; const agentId = resolveAgentIdFromSessionKey(params.sessionKey); - const nextSessionFile = resolveSessionTranscriptPath( - nextSessionId, + const nextSessionFile = formatSqliteSessionFileMarker({ agentId, - params.messageThreadId, - ); + sessionId: nextSessionId, + storePath: params.storePath, + }); nextEntry.sessionFile = nextSessionFile; params.activeSessionStore[params.sessionKey] = nextEntry; try { diff --git a/src/auto-reply/reply/agent-runner.misc.runreplyagent.test.ts b/src/auto-reply/reply/agent-runner.misc.runreplyagent.test.ts index cfaea99e4ea5..6230b2b57fa9 100644 --- a/src/auto-reply/reply/agent-runner.misc.runreplyagent.test.ts +++ b/src/auto-reply/reply/agent-runner.misc.runreplyagent.test.ts @@ -13,7 +13,7 @@ import { clearRuntimeConfigSnapshot } from "../../config/config.js"; import type { OpenClawConfig } from "../../config/config.js"; import * as sessionTypesModule from "../../config/sessions.js"; import type { SessionEntry } from "../../config/sessions.js"; -import { loadSessionStore, saveSessionStore } from "../../config/sessions.js"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import { onInternalDiagnosticEvent, resetDiagnosticEventsForTest, @@ -333,10 +333,9 @@ describe("runReplyAgent auto-compaction token update", () => { entry: Record; }) { await fs.mkdir(path.dirname(params.storePath), { recursive: true }); - await fs.writeFile( - params.storePath, - JSON.stringify({ [params.sessionKey]: params.entry }, null, 2), - "utf-8", + await replaceSessionEntry( + { storePath: params.storePath, sessionKey: params.sessionKey }, + params.entry as SessionEntry, ); } @@ -519,7 +518,8 @@ describe("runReplyAgent auto-compaction token update", () => { unsubscribe?.(); } - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const persisted = loadSessionEntry({ storePath, sessionKey }); + const stored = persisted ? { [sessionKey]: persisted } : {}; const usageEvent = diagnostics.find((event) => event.type === "model.usage"); return { sessionKey, stored, usageEvent }; } @@ -546,7 +546,7 @@ describe("runReplyAgent auto-compaction token update", () => { }); // totalTokens should use lastCallUsage (55k), not accumulated (75k) - expect(stored[sessionKey].totalTokens).toBe(55_000); + expect(stored[sessionKey as keyof typeof stored]?.totalTokens).toBe(55_000); }, 180_000); it("keeps an unarmed preflight drain visible instead of dropping the reply", async () => { @@ -959,11 +959,10 @@ describe("runReplyAgent auto-compaction token update", () => { ); }); - it("reads opted-in post-compaction context from the queued workspace instead of process cwd", async () => { + it("does not treat diagnostic compaction metadata as a context-refresh trigger", async () => { const workspaceDir = await fs.mkdtemp( path.join(os.tmpdir(), "openclaw-post-compaction-workspace-"), ); - const cwdDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-post-compaction-cwd-")); await fs.writeFile( path.join(workspaceDir, "AGENTS.md"), [ @@ -976,32 +975,25 @@ describe("runReplyAgent auto-compaction token update", () => { "utf-8", ); - const cwdSpy = vi.spyOn(process, "cwd").mockReturnValue(cwdDir); - try { - const { sessionKey } = await runBaseReplyWithAgentMeta({ - tmpPrefix: "openclaw-post-compaction-workspace-root-", - workspaceDir, - config: { - agents: { - defaults: { - compaction: { postCompactionSections: ["Session Startup", "Red Lines"] }, - }, + const { sessionKey } = await runBaseReplyWithAgentMeta({ + tmpPrefix: "openclaw-post-compaction-workspace-root-", + workspaceDir, + config: { + agents: { + defaults: { + compaction: { postCompactionSections: ["Session Startup", "Red Lines"] }, }, }, - agentMeta: { - compactionCount: 1, - lastCallUsage: { input: 10_000, output: 500, total: 10_500 }, - }, - }); + }, + agentMeta: { + compactionCount: 1, + lastCallUsage: { input: 10_000, output: 500, total: 10_500 }, + }, + }); - await vi.waitFor(() => { - const events = peekSystemEvents(sessionKey); - expect(events[0]).toContain("Post-compaction context refresh"); - expect(events[0]).toContain("Read the queued workspace startup file."); - }); - } finally { - cwdSpy.mockRestore(); - } + // agentMeta.compactionCount is diagnostic metadata from the harness result; + // post-compaction context refresh belongs to runner-owned compaction paths. + expect(peekSystemEvents(sessionKey)).toEqual([]); }); }); @@ -1205,6 +1197,29 @@ describe("runReplyAgent block streaming", () => { }); describe("runReplyAgent Active Memory inline debug", () => { + // Seeds the plugin-owned debug rows through the canonical session accessor. + async function writeActiveMemoryDebugEntry(params: { + sessionEntry: SessionEntry; + sessionKey: string; + storePath: string; + }): Promise { + await replaceSessionEntry( + { storePath: params.storePath, sessionKey: params.sessionKey }, + { + ...params.sessionEntry, + pluginDebugEntries: [ + { + pluginId: "active-memory", + lines: [ + "🧩 Active Memory: status=ok elapsed=842ms query=recent summary=34 chars", + "🔎 Active Memory Debug: Lemon pepper wings with blue cheese.", + ], + }, + ], + }, + ); + } + it("appends inline Active Memory status payload when verbose is enabled", async () => { const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-active-memory-inline-")); const storePath = path.join(tmp, "sessions.json"); @@ -1215,33 +1230,10 @@ describe("runReplyAgent Active Memory inline debug", () => { verboseLevel: "on", }; - await fs.writeFile( - storePath, - JSON.stringify( - { - [sessionKey]: sessionEntry, - }, - null, - 2, - ), - "utf-8", - ); + await replaceSessionEntry({ storePath, sessionKey }, sessionEntry); runEmbeddedAgentMock.mockImplementationOnce(async () => { - const latest = loadSessionStore(storePath, { skipCache: true }); - latest[sessionKey] = { - ...latest[sessionKey], - pluginDebugEntries: [ - { - pluginId: "active-memory", - lines: [ - "🧩 Active Memory: status=ok elapsed=842ms query=recent summary=34 chars", - "🔎 Active Memory Debug: Lemon pepper wings with blue cheese.", - ], - }, - ], - }; - await saveSessionStore(storePath, latest); + await writeActiveMemoryDebugEntry({ sessionEntry, sessionKey, storePath }); return { payloads: [{ text: "Normal reply" }], meta: {}, @@ -1327,33 +1319,10 @@ describe("runReplyAgent Active Memory inline debug", () => { traceLevel: "on", }; - await fs.writeFile( - storePath, - JSON.stringify( - { - [sessionKey]: sessionEntry, - }, - null, - 2, - ), - "utf-8", - ); + await replaceSessionEntry({ storePath, sessionKey }, sessionEntry); runEmbeddedAgentMock.mockImplementationOnce(async () => { - const latest = loadSessionStore(storePath, { skipCache: true }); - latest[sessionKey] = { - ...latest[sessionKey], - pluginDebugEntries: [ - { - pluginId: "active-memory", - lines: [ - "🧩 Active Memory: status=ok elapsed=842ms query=recent summary=34 chars", - "🔎 Active Memory Debug: Lemon pepper wings with blue cheese.", - ], - }, - ], - }; - await saveSessionStore(storePath, latest); + await writeActiveMemoryDebugEntry({ sessionEntry, sessionKey, storePath }); return { payloads: [{ text: "Normal reply" }], meta: {}, @@ -1438,33 +1407,10 @@ describe("runReplyAgent Active Memory inline debug", () => { traceLevel: "on", }; - await fs.writeFile( - storePath, - JSON.stringify( - { - [sessionKey]: sessionEntry, - }, - null, - 2, - ), - "utf-8", - ); + await replaceSessionEntry({ storePath, sessionKey }, sessionEntry); runEmbeddedAgentMock.mockImplementationOnce(async () => { - const latest = loadSessionStore(storePath, { skipCache: true }); - latest[sessionKey] = { - ...latest[sessionKey], - pluginDebugEntries: [ - { - pluginId: "active-memory", - lines: [ - "🧩 Active Memory: status=ok elapsed=842ms query=recent summary=34 chars", - "🔎 Active Memory Debug: Lemon pepper wings with blue cheese.", - ], - }, - ], - }; - await saveSessionStore(storePath, latest); + await writeActiveMemoryDebugEntry({ sessionEntry, sessionKey, storePath }); return { payloads: [{ text: "Normal reply" }], meta: {}, @@ -1551,17 +1497,7 @@ describe("runReplyAgent Active Memory inline debug", () => { compactionCount: 3, }; - await fs.writeFile( - storePath, - JSON.stringify( - { - [sessionKey]: sessionEntry, - }, - null, - 2, - ), - "utf-8", - ); + await replaceSessionEntry({ storePath, sessionKey }, sessionEntry); await fs.writeFile( sessionFile, [ @@ -1797,7 +1733,7 @@ describe("runReplyAgent Active Memory inline debug", () => { traceLevel: "raw", }; - await fs.writeFile(storePath, JSON.stringify({ [sessionKey]: sessionEntry }, null, 2), "utf-8"); + await replaceSessionEntry({ storePath, sessionKey }, sessionEntry); await fs.writeFile(sessionFile, "", "utf-8"); runEmbeddedAgentMock.mockResolvedValueOnce({ @@ -1893,17 +1829,7 @@ describe("runReplyAgent Active Memory inline debug", () => { traceLevel: "raw", }; - await fs.writeFile( - storePath, - JSON.stringify( - { - [sessionKey]: sessionEntry, - }, - null, - 2, - ), - "utf-8", - ); + await replaceSessionEntry({ storePath, sessionKey }, sessionEntry); await fs.writeFile( sessionFile, `${JSON.stringify({ @@ -2013,7 +1939,7 @@ describe("runReplyAgent Active Memory inline debug", () => { traceLevel: "raw", }; - await fs.writeFile(storePath, JSON.stringify({ [sessionKey]: sessionEntry }, null, 2), "utf-8"); + await replaceSessionEntry({ storePath, sessionKey }, sessionEntry); await fs.writeFile(sessionFile, "", "utf-8"); runEmbeddedAgentMock.mockResolvedValueOnce({ @@ -2108,17 +2034,7 @@ describe("runReplyAgent Active Memory inline debug", () => { updatedAt: Date.now(), }; - await fs.writeFile( - storePath, - JSON.stringify( - { - [sessionKey]: sessionEntry, - }, - null, - 2, - ), - "utf-8", - ); + await replaceSessionEntry({ storePath, sessionKey }, sessionEntry); const loadSessionStoreSpy = vi.spyOn(sessionTypesModule, "loadSessionStore"); runEmbeddedAgentMock.mockResolvedValueOnce({ @@ -3498,7 +3414,7 @@ describe("runReplyAgent private message_tool_only final warning (#85714)", () => totalTokens: 1_000, ...(params.sendPolicyDenied ? { sendPolicy: "deny" as const } : {}), }; - await fs.writeFile(storePath, JSON.stringify({ [sessionKey]: sessionEntry }, null, 2), "utf-8"); + await replaceSessionEntry({ storePath, sessionKey }, sessionEntry); const finalAssistantText = params.finalAssistantText ?? @@ -3569,6 +3485,12 @@ describe("runReplyAgent private message_tool_only final warning (#85714)", () => }, } as unknown as FollowupRun; + // Seeding the SQLite session entry above resolves the runtime config + // (getRuntimeConfig) and pins an empty `{}` snapshot; leaving it in place + // would make resolveQueuedReplyExecutionConfig override the run's + // visibleReplies=message_tool config and mis-resolve delivery to automatic. + clearRuntimeConfigSnapshot(); + const result = await runReplyAgent({ commandBody: "hello", followupRun, diff --git a/src/auto-reply/reply/agent-runner.runreplyagent.e2e.test.ts b/src/auto-reply/reply/agent-runner.runreplyagent.e2e.test.ts index 2bc9ad324642..37e17e068cd0 100644 --- a/src/auto-reply/reply/agent-runner.runreplyagent.e2e.test.ts +++ b/src/auto-reply/reply/agent-runner.runreplyagent.e2e.test.ts @@ -3,14 +3,15 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { saveSessionStore, type SessionEntry } from "../../config/sessions.js"; -import { readSessionStoreForTest } from "../../config/sessions/test-helpers.js"; +import type { SessionEntry } from "../../config/sessions.js"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import type { TypingMode } from "../../config/types.js"; import { HEARTBEAT_RUN_SCOPE, type ReplyOptionsWithHeartbeatRunScope, } from "../../infra/heartbeat-run-scope.js"; import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; +import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js"; import type { TemplateContext } from "../templating.js"; import type { GetReplyOptions } from "../types.js"; import { @@ -86,7 +87,7 @@ function mockCallArgs(mock: ReturnType, label: string, callIndex = } function requireStoredSessionEntry(storePath: string, sessionKey = "main"): SessionEntry { - const entry = readSessionStoreForTest(storePath)[sessionKey]; + const entry = loadSessionEntry({ storePath, sessionKey, readConsistency: "latest" }); if (!entry) { throw new Error(`expected stored session entry for ${sessionKey}`); } @@ -301,7 +302,7 @@ describe("runReplyAgent active steering", () => { text: "visible group prompt", sender: { id: "user-42", name: "Ada" }, }, - target: { transcriptPath: "/tmp/unused-session.jsonl" }, + target: createTestUserTurnTranscriptTarget(), }); const { followupRun, run } = createMinimalRun({ isActive: true, @@ -666,7 +667,7 @@ describe("runReplyAgent pending final delivery capture", () => { async function createSessionStoreFile(entry: SessionEntry) { const dir = await mkdtemp(join(tmpdir(), "openclaw-agent-runner-pending-")); const storePath = join(dir, "sessions.json"); - await saveSessionStore(storePath, { main: entry }, { skipMaintenance: true }); + await replaceSessionEntry({ storePath, sessionKey: "main" }, entry); return storePath; } @@ -1003,10 +1004,9 @@ describe("runReplyAgent typing (heartbeat)", () => { it("does not persist heartbeat ack text as pending final delivery", async () => { const dir = await mkdtemp(join(tmpdir(), "openclaw-heartbeat-pending-")); const storePath = join(dir, "sessions.json"); - await saveSessionStore( - storePath, - { main: { sessionId: "session", updatedAt: 1 } }, - { skipMaintenance: true }, + await replaceSessionEntry( + { storePath, sessionKey: "main" }, + { sessionId: "session", updatedAt: 1 }, ); try { state.runEmbeddedAgentMock.mockResolvedValueOnce({ @@ -1453,7 +1453,7 @@ describe("runReplyAgent typing (heartbeat)", () => { modelOverride: "gpt-5.6-luna", modelOverrideSource: "user", }; - await saveSessionStore(storePath, { main: sessionEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ storePath, sessionKey: "main" }, sessionEntry); const sessionStore = { main: sessionEntry }; state.runEmbeddedAgentMock.mockResolvedValueOnce({ payloads: [{ text: "final" }], @@ -1538,7 +1538,7 @@ describe("runReplyAgent typing (heartbeat)", () => { updatedAt: Date.now(), traceLevel: "raw", }; - await saveSessionStore(storePath, { main: sessionEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ storePath, sessionKey: "main" }, sessionEntry); try { state.runEmbeddedAgentMock.mockResolvedValueOnce({ payloads: [{ text: "Terminal tool summary", isError: true }], @@ -1845,7 +1845,7 @@ describe("runReplyAgent typing (heartbeat)", () => { const sessionStore = { main: sessionEntry }; const storeRoot = await mkdtemp(join(tmpdir(), "openclaw-internal-fallback-")); const storePath = join(storeRoot, "sessions.json"); - await saveSessionStore(storePath, sessionStore, { skipMaintenance: true }); + await replaceSessionEntry({ storePath, sessionKey: "main" }, sessionStore.main); try { state.runEmbeddedAgentMock.mockResolvedValueOnce({ payloads: [{ text: "subagent timed out" }], @@ -2989,7 +2989,7 @@ describe("runReplyAgent typing (heartbeat)", () => { const sessionStore = { main: sessionEntry }; const dir = await mkdtemp(join(tmpdir(), "openclaw-agent-runner-cli-alias-")); const storePath = join(dir, "sessions.json"); - await saveSessionStore(storePath, { main: sessionEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ storePath, sessionKey: "main" }, sessionEntry); state.runEmbeddedAgentMock.mockResolvedValue({ payloads: [{ text: "final" }], diff --git a/src/auto-reply/reply/agent-runner.test-fixtures.ts b/src/auto-reply/reply/agent-runner.test-fixtures.ts index f0c8397e2698..1bd4b1a89e6d 100644 --- a/src/auto-reply/reply/agent-runner.test-fixtures.ts +++ b/src/auto-reply/reply/agent-runner.test-fixtures.ts @@ -1,7 +1,6 @@ // Shared fixtures for agent runner tests and temporary session files. -import fs from "node:fs/promises"; -import path from "node:path"; import type { SessionEntry } from "../../config/sessions.js"; +import { replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import type { FollowupRun } from "./queue.js"; export function createTestFollowupRun(overrides: Partial = {}): FollowupRun { @@ -38,6 +37,5 @@ export async function writeTestSessionStore( sessionKey: string, entry: SessionEntry, ): Promise { - await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile(storePath, JSON.stringify({ [sessionKey]: entry }, null, 2), "utf8"); + await replaceSessionEntry({ storePath, sessionKey }, entry); } diff --git a/src/auto-reply/reply/agent-runner.ts b/src/auto-reply/reply/agent-runner.ts index 0833eec28903..b70eb54a8956 100644 --- a/src/auto-reply/reply/agent-runner.ts +++ b/src/auto-reply/reply/agent-runner.ts @@ -1,6 +1,5 @@ // Orchestrates reply agent execution, payload building, and delivery callbacks. import crypto from "node:crypto"; -import fs from "node:fs/promises"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { hasSessionAutoModelFallbackProvenance, @@ -24,7 +23,7 @@ import { import { resolveFastModeState } from "../../agents/fast-mode.js"; import { resolveModelAuthMode } from "../../agents/model-auth.js"; import { isCliProvider } from "../../agents/model-selection.js"; -import { deriveContextPromptTokens, hasNonzeroUsage, normalizeUsage } from "../../agents/usage.js"; +import { deriveContextPromptTokens, hasNonzeroUsage } from "../../agents/usage.js"; import { enqueueCommitmentExtraction } from "../../commitments/runtime.js"; import type { OpenClawConfig } from "../../config/config.js"; import { @@ -33,9 +32,13 @@ import { type SessionEntry, } from "../../config/sessions.js"; import { loadSessionEntry, updateSessionEntry } from "../../config/sessions/session-accessor.js"; +import { + formatSqliteSessionFileMarker, + sqliteSessionFileMarkerMatchesSession, +} from "../../config/sessions/sqlite-marker.js"; import { parseSessionThreadInfoFast } from "../../config/sessions/thread-info.js"; import type { TypingMode } from "../../config/types.js"; -import { resolveSessionTranscriptCandidates } from "../../gateway/session-utils.fs.js"; +import { readLatestSessionUsageFromTranscriptAsync } from "../../gateway/session-transcript-readers.js"; import { logVerbose } from "../../globals.js"; import { emitAgentEvent } from "../../infra/agent-events.js"; import { emitTrustedDiagnosticEvent, isDiagnosticsEnabled } from "../../infra/diagnostic-events.js"; @@ -744,63 +747,20 @@ async function accumulateSessionUsageFromTranscript(params: { return undefined; } try { - const candidates = resolveSessionTranscriptCandidates( + const usage = await readLatestSessionUsageFromTranscriptAsync({ sessionId, - params.storePath, - params.sessionFile, - ); - let transcriptText: string | undefined; - for (const candidate of candidates) { - try { - transcriptText = await fs.readFile(candidate, "utf-8"); - break; - } catch { - continue; - } - } - if (!transcriptText) { + storePath: params.storePath, + sessionFile: params.sessionFile, + }); + if (!usage) { return undefined; } - - let input = 0; - let output = 0; - let cacheRead = 0; - let cacheWrite = 0; - let sawUsage = false; - for (const line of transcriptText.split(/\r?\n/)) { - if (!line.trim()) { - continue; - } - let parsed: { message?: { usage?: unknown } } | undefined; - try { - parsed = JSON.parse(line) as { message?: { usage?: unknown } }; - } catch { - continue; - } - const message = parsed?.message; - if (!message) { - continue; - } - const usage = normalizeUsage(message?.usage as Parameters[0]); - if (!hasNonzeroUsage(usage)) { - continue; - } - sawUsage = true; - input += usage.input ?? 0; - output += usage.output ?? 0; - cacheRead += usage.cacheRead ?? 0; - cacheWrite += usage.cacheWrite ?? 0; - } - if (!sawUsage) { - return undefined; - } - const total = input + output + cacheRead + cacheWrite; return { - input: input || undefined, - output: output || undefined, - cacheRead: cacheRead || undefined, - cacheWrite: cacheWrite || undefined, - total: total || undefined, + input: usage.inputTokens, + output: usage.outputTokens, + cacheRead: usage.cacheRead, + cacheWrite: usage.cacheWrite, + total: usage.totalTokens, }; } catch { return undefined; @@ -1141,6 +1101,28 @@ function refreshSessionEntryFromStore(params: { } } +function resolveAdmittedRunSessionFile(params: { + agentId: string; + sessionId: string; + sessionFile?: string; + storePath?: string; +}): string | undefined { + if ( + params.sessionFile && + sqliteSessionFileMarkerMatchesSession(params.sessionFile, params.sessionId) + ) { + return params.sessionFile; + } + if (params.storePath) { + return formatSqliteSessionFileMarker({ + agentId: params.agentId, + sessionId: params.sessionId, + storePath: params.storePath, + }); + } + return params.sessionFile; +} + export async function runReplyAgent(params: { commandBody: string; transcriptCommandBody?: string; @@ -1508,8 +1490,14 @@ export async function runReplyAgent(params: { }); if (admittedSessionEntry?.sessionId === replyOperation.sessionId) { activeSessionEntry = admittedSessionEntry; - if (admittedSessionEntry.sessionFile) { - followupRun.run.sessionFile = admittedSessionEntry.sessionFile; + const admittedSessionFile = resolveAdmittedRunSessionFile({ + agentId: followupRun.run.agentId, + sessionId: replyOperation.sessionId, + sessionFile: admittedSessionEntry.sessionFile, + storePath, + }); + if (admittedSessionFile) { + followupRun.run.sessionFile = admittedSessionFile; } } } diff --git a/src/auto-reply/reply/body.test.ts b/src/auto-reply/reply/body.test.ts index f4690af5b11a..0960e44f90fb 100644 --- a/src/auto-reply/reply/body.test.ts +++ b/src/auto-reply/reply/body.test.ts @@ -3,7 +3,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { loadSessionStore } from "../../config/sessions.js"; +import { loadSessionEntry } from "../../config/sessions/session-accessor.js"; import type { SessionEntry } from "../../config/sessions/types.js"; import { applySessionHints } from "./body.js"; @@ -28,7 +28,6 @@ describe("applySessionHints", () => { abortedLastRun: true, }; const sessionStore: Record = { [sessionKey]: entry }; - await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf8"); const body = await applySessionHints({ baseBody: "continue", @@ -39,7 +38,7 @@ describe("applySessionHints", () => { storePath, }); - const persisted = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + const persisted = loadSessionEntry({ storePath, sessionKey }); expect(body).toContain("previous agent run was aborted"); expect(sessionStore[sessionKey]?.sessionId).toBe("hint-session"); expect(sessionStore[sessionKey]?.modelProvider).toBe("openai"); diff --git a/src/auto-reply/reply/commands-core.test.ts b/src/auto-reply/reply/commands-core.test.ts index e9add7194acb..824a6ab7f734 100644 --- a/src/auto-reply/reply/commands-core.test.ts +++ b/src/auto-reply/reply/commands-core.test.ts @@ -3,27 +3,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { HookRunner } from "../../plugins/hooks.js"; import type { HandleCommandsParams } from "./commands-types.js"; -const fsMocks = vi.hoisted(() => ({ - readFile: vi.fn(), - readdir: vi.fn(), -})); - const hookRunnerMocks = vi.hoisted(() => ({ hasHooks: vi.fn(), runBeforeReset: vi.fn(), + loadTranscriptEvents: vi.fn(async (): Promise => []), })); -vi.mock("node:fs/promises", async () => { - const actual = await vi.importActual("node:fs/promises"); +vi.mock("../../config/sessions/session-accessor.js", () => { return { - ...actual, - default: { - ...actual, - readFile: fsMocks.readFile, - readdir: fsMocks.readdir, - }, - readFile: fsMocks.readFile, - readdir: fsMocks.readdir, + loadTranscriptEvents: hookRunnerMocks.loadTranscriptEvents, }; }); @@ -67,6 +55,7 @@ describe("emitResetCommandHooks", () => { previousSessionEntry: { sessionId: "prev-session", } as HandleCommandsParams["previousSessionEntry"], + storePath: "/tmp/openclaw-agent.sqlite", workspaceDir: "/tmp/openclaw-workspace", }); @@ -76,14 +65,12 @@ describe("emitResetCommandHooks", () => { } beforeEach(() => { - fsMocks.readFile.mockReset(); - fsMocks.readdir.mockReset(); hookRunnerMocks.hasHooks.mockReset(); hookRunnerMocks.runBeforeReset.mockReset(); + hookRunnerMocks.loadTranscriptEvents.mockReset(); hookRunnerMocks.hasHooks.mockImplementation((hookName) => hookName === "before_reset"); hookRunnerMocks.runBeforeReset.mockResolvedValue(undefined); - fsMocks.readFile.mockResolvedValue(""); - fsMocks.readdir.mockResolvedValue([]); + hookRunnerMocks.loadTranscriptEvents.mockResolvedValue([]); }); afterEach(() => { @@ -114,16 +101,14 @@ describe("emitResetCommandHooks", () => { expect(ctx?.workspaceDir).toBe("/tmp/openclaw-workspace"); }); - it("recovers the archived transcript when the original reset transcript path is gone", async () => { - fsMocks.readFile.mockRejectedValueOnce(Object.assign(new Error("ENOENT"), { code: "ENOENT" })); - fsMocks.readdir.mockResolvedValueOnce(["prev-session.jsonl.reset.2026-02-16T22-26-33.000Z"]); - fsMocks.readFile.mockResolvedValueOnce( - `${JSON.stringify({ + it("loads marker-backed before_reset transcripts by session identity", async () => { + hookRunnerMocks.loadTranscriptEvents.mockResolvedValueOnce([ + { type: "message", id: "m1", message: { role: "user", content: "Recovered from archive" }, - })}\n`, - ); + }, + ]); const command = { surface: "telegram", senderId: "vac", @@ -139,57 +124,60 @@ describe("emitResetCommandHooks", () => { cfg: {} as HandleCommandsParams["cfg"], command, sessionKey: "agent:main:telegram:group:-1003826723328:topic:8428", + storePath: "/tmp/openclaw-agent.sqlite", previousSessionEntry: { sessionId: "prev-session", - sessionFile: "/tmp/prev-session.jsonl", + sessionFile: "sqlite:main:prev-session:/tmp/openclaw-agent.sqlite", } as HandleCommandsParams["previousSessionEntry"], workspaceDir: "/tmp/openclaw-workspace", }); await vi.waitFor(() => expect(hookRunnerMocks.runBeforeReset).toHaveBeenCalledTimes(1)); const [event, ctx] = firstBeforeResetCall(); - expect(event.sessionFile).toBe("/tmp/prev-session.jsonl.reset.2026-02-16T22-26-33.000Z"); + expect(hookRunnerMocks.loadTranscriptEvents).toHaveBeenCalledWith({ + agentId: "main", + sessionId: "prev-session", + sessionKey: "agent:main:telegram:group:-1003826723328:topic:8428", + storePath: "/tmp/openclaw-agent.sqlite", + }); + expect(event.sessionFile).toBe("sqlite:main:prev-session:/tmp/openclaw-agent.sqlite"); expect(event.messages).toEqual([{ role: "user", content: "Recovered from archive" }]); expect(event.reason).toBe("new"); expect(ctx.sessionId).toBe("prev-session"); }); it("keeps leaf-controlled side branches out of before_reset hooks", async () => { - fsMocks.readFile.mockResolvedValueOnce( - [ - { - type: "message", - id: "active-root", - parentId: null, - message: { role: "user", content: "active root" }, - }, - { - type: "message", - id: "side-entry", - parentId: "active-root", - message: { role: "assistant", content: "side delivery" }, - }, - { - type: "leaf", - id: "active-leaf", - parentId: "side-entry", - targetId: "active-root", - }, - { - type: "message", - id: "active-tail", - parentId: "active-root", - message: { role: "assistant", content: "active tail" }, - }, - { - type: "metadata", - id: "opaque-after-active-tail", - parentId: "side-entry", - }, - ] - .map((entry) => JSON.stringify(entry)) - .join("\n"), - ); + hookRunnerMocks.loadTranscriptEvents.mockResolvedValueOnce([ + { + type: "message", + id: "active-root", + parentId: null, + message: { role: "user", content: "active root" }, + }, + { + type: "message", + id: "side-entry", + parentId: "active-root", + message: { role: "assistant", content: "side delivery" }, + }, + { + type: "leaf", + id: "active-leaf", + parentId: "side-entry", + targetId: "active-root", + }, + { + type: "message", + id: "active-tail", + parentId: "active-root", + message: { role: "assistant", content: "active tail" }, + }, + { + type: "metadata", + id: "opaque-after-active-tail", + parentId: "side-entry", + }, + ]); await emitResetCommandHooks({ action: "new", @@ -204,9 +192,10 @@ describe("emitResetCommandHooks", () => { resetHookTriggered: false, } as HandleCommandsParams["command"], sessionKey: "agent:main:main", + storePath: "/tmp/openclaw-agent.sqlite", previousSessionEntry: { sessionId: "prev-session", - sessionFile: "/tmp/prev-session.jsonl", + sessionFile: "sqlite:main:prev-session:/tmp/openclaw-agent.sqlite", } as HandleCommandsParams["previousSessionEntry"], workspaceDir: "/tmp/openclaw-workspace", }); diff --git a/src/auto-reply/reply/commands-export-common.ts b/src/auto-reply/reply/commands-export-common.ts index d6d05044489d..8cba66ad3790 100644 --- a/src/auto-reply/reply/commands-export-common.ts +++ b/src/auto-reply/reply/commands-export-common.ts @@ -4,17 +4,21 @@ import { resolveSessionFilePath, resolveSessionFilePathOptions, } from "../../config/sessions/paths.js"; -import { loadSessionStore } from "../../config/sessions/store.js"; +import { loadSessionEntry } from "../../config/sessions/session-accessor.js"; import type { SessionEntry } from "../../config/sessions/types.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js"; import type { ReplyPayload } from "../types.js"; import type { HandleCommandsParams } from "./commands-types.js"; -/** Resolved session entry and transcript file targeted by an export command. */ +/** Resolved session entry and scoped transcript identity targeted by an export command. */ interface ExportCommandSessionTarget { + agentId: string; entry: SessionEntry; + sessionId: string; sessionFile: string; + sessionKey: string; + storePath: string; } const MAX_EXPORT_COMMAND_OUTPUT_PATH_CHARS = 512; @@ -48,20 +52,34 @@ export function resolveExportCommandSessionTarget( params: HandleCommandsParams, ): ExportCommandSessionTarget | ReplyPayload { const targetAgentId = resolveAgentIdFromSessionKey(params.sessionKey) || params.agentId; + if (!targetAgentId) { + return { text: `❌ Failed to resolve agent for session: ${params.sessionKey}` }; + } const storePath = params.storePath ?? resolveDefaultSessionStorePath(targetAgentId); - const store = loadSessionStore(storePath, { skipCache: true }); - const entry = store[params.sessionKey] as SessionEntry | undefined; - if (!entry?.sessionId) { + const entry = loadSessionEntry({ + storePath, + sessionKey: params.sessionKey, + clone: false, + }); + const sessionId = entry?.sessionId; + if (!sessionId) { return { text: `❌ Session not found: ${params.sessionKey}` }; } try { const sessionFile = resolveSessionFilePath( - entry.sessionId, + sessionId, entry, resolveSessionFilePathOptions({ agentId: targetAgentId, storePath }), ); - return { entry, sessionFile }; + return { + agentId: targetAgentId, + entry, + sessionFile, + sessionId, + sessionKey: params.sessionKey, + storePath, + }; } catch (err) { return { text: `❌ Failed to resolve session file: ${formatErrorMessage(err)}`, diff --git a/src/auto-reply/reply/commands-export-session.test.ts b/src/auto-reply/reply/commands-export-session.test.ts index 5806b5fcb8e5..915e31109387 100644 --- a/src/auto-reply/reply/commands-export-session.test.ts +++ b/src/auto-reply/reply/commands-export-session.test.ts @@ -20,13 +20,13 @@ const hoisted = await vi.hoisted(async () => { ), mkdirMock: vi.fn(async (_filePath: string, _options?: { recursive?: boolean }) => undefined), accessMock: vi.fn(async (_filePath: string) => undefined), - pathExistsMock: vi.fn(async (_filePath: string) => true), migrateSessionEntriesMock: vi.fn((_entries: unknown[]) => undefined), readAcpSessionMetaForEntryMock: vi.fn< (params: { sessionKey: string; entry?: { sessionId?: string } }) => unknown >(() => undefined), + loadTranscriptEventsMock: vi.fn(async (): Promise => []), exportHtmlTemplateContents: new Map(), - sessionTranscriptContent: "", + sessionTranscriptEvents: [] as unknown[], }; }); @@ -44,12 +44,14 @@ vi.mock("../../config/sessions/store.js", () => ({ loadSessionStore: hoisted.loadSessionStoreMock, })); -vi.mock("./commands-system-prompt.js", () => ({ - resolveCommandsSystemPromptBundle: hoisted.resolveCommandsSystemPromptBundleMock, +vi.mock("../../config/sessions/session-accessor.js", () => ({ + loadSessionEntry: (scope: { storePath?: string; sessionKey: string }) => + (hoisted.loadSessionStoreMock(scope.storePath) as Record)[scope.sessionKey], + loadTranscriptEvents: hoisted.loadTranscriptEventsMock, })); -vi.mock("../../infra/fs-safe.js", () => ({ - pathExists: hoisted.pathExistsMock, +vi.mock("./commands-system-prompt.js", () => ({ + resolveCommandsSystemPromptBundle: hoisted.resolveCommandsSystemPromptBundleMock, })); vi.mock("../../agents/sessions/session-manager.js", async (importOriginal) => { @@ -90,9 +92,6 @@ vi.mock("node:fs/promises", async () => { mkdir: hoisted.mkdirMock, writeFile: hoisted.writeFileMock, readFile: vi.fn(async (filePath: string, encoding?: BufferEncoding) => { - if (filePath === "/tmp/target-store/session.jsonl") { - return hoisted.sessionTranscriptContent; - } for (const [suffix, contents] of hoisted.exportHtmlTemplateContents) { if (filePath.endsWith(suffix)) { return contents; @@ -206,10 +205,12 @@ describe("buildExportSessionReply", () => { sandboxRuntime: { sandboxed: false, mode: "off" }, }); hoisted.accessMock.mockResolvedValue(undefined); - hoisted.pathExistsMock.mockResolvedValue(true); hoisted.readAcpSessionMetaForEntryMock.mockReturnValue(undefined); + hoisted.loadTranscriptEventsMock.mockImplementation( + async () => hoisted.sessionTranscriptEvents, + ); hoisted.exportHtmlTemplateContents.clear(); - hoisted.sessionTranscriptContent = ""; + hoisted.sessionTranscriptEvents = []; }); it("resolves store and transcript paths from the target session agent", async () => { @@ -236,9 +237,7 @@ describe("buildExportSessionReply", () => { }); expect(hoisted.resolveDefaultSessionStorePathMock).not.toHaveBeenCalled(); - expect(hoisted.loadSessionStoreMock).toHaveBeenCalledWith("/tmp/custom-store/sessions.json", { - skipCache: true, - }); + expect(hoisted.loadSessionStoreMock).toHaveBeenCalledWith("/tmp/custom-store/sessions.json"); expect(hoisted.resolveSessionFilePathOptionsMock).toHaveBeenCalledWith({ agentId: "target", storePath: "/tmp/custom-store/sessions.json", @@ -314,7 +313,7 @@ describe("buildExportSessionReply", () => { targetId: "active-tail", }, ]; - hoisted.sessionTranscriptContent = entries.map((entry) => JSON.stringify(entry)).join("\n"); + hoisted.sessionTranscriptEvents = entries; await buildExportSessionReply(makeParams()); @@ -363,7 +362,7 @@ describe("buildExportSessionReply", () => { message: { role: "assistant", content: "replacement" }, }, ]; - hoisted.sessionTranscriptContent = rawEntries.map((entry) => JSON.stringify(entry)).join("\n"); + hoisted.sessionTranscriptEvents = rawEntries; await buildExportSessionReply(makeParams()); @@ -415,7 +414,7 @@ describe("buildExportSessionReply", () => { targetId: "active-tail", }, ]; - hoisted.sessionTranscriptContent = rawEntries.map((entry) => JSON.stringify(entry)).join("\n"); + hoisted.sessionTranscriptEvents = rawEntries; await buildExportSessionReply(makeParams()); @@ -460,7 +459,7 @@ describe("buildExportSessionReply", () => { parentId: "inactive-tail", }, ]; - hoisted.sessionTranscriptContent = entries.map((entry) => JSON.stringify(entry)).join("\n"); + hoisted.sessionTranscriptEvents = entries; await buildExportSessionReply(makeParams()); @@ -531,40 +530,71 @@ describe("buildExportSessionReply", () => { expect(html).toContain("const highlightMarker = '$&$1';"); }); - it("reports malformed transcript rows without leaking parser details", async () => { - hoisted.sessionTranscriptContent = [ - JSON.stringify({ type: "session", version: 3, id: "session-1" }), - '{"type":"message",', - JSON.stringify({ + it("exports marker-backed sessions by identity without requiring the marker as a file", async () => { + hoisted.resolveSessionFilePathMock.mockReturnValue( + "sqlite:target:session-1:/tmp/target-store/openclaw-agent.sqlite", + ); + hoisted.loadSessionStoreMock.mockReturnValue({ + "agent:target:session": { + sessionFile: "sqlite:target:session-1:/tmp/target-store/openclaw-agent.sqlite", + sessionId: "session-1", + updatedAt: 1, + }, + }); + hoisted.sessionTranscriptEvents = [ + { type: "message", id: "entry-1", timestamp: "2026-05-16T00:00:00.000Z", message: { role: "user", content: "valid user" }, - }), - JSON.stringify({ - type: "message", - id: "entry-2", - timestamp: "2026-05-16T00:00:01.000Z", - message: { content: "missing role" }, - }), - JSON.stringify({ + }, + { type: "message", id: "entry-3", timestamp: "2026-05-16T00:00:02.000Z", message: { role: "assistant", content: "valid assistant" }, - }), - ].join("\n"); + }, + ]; + + const reply = await buildExportSessionReply(makeParams()); + + expect(reply.text).toContain("📊 Entries: 2"); + expect(hoisted.loadTranscriptEventsMock).toHaveBeenCalledWith({ + agentId: "target", + sessionId: "session-1", + sessionKey: "agent:target:session", + storePath: "/tmp/target-store/sessions.json", + }); + }); + + it("skips invalid loaded transcript events before exporting", async () => { + hoisted.sessionTranscriptEvents = [ + { + type: "message", + id: "entry-1", + timestamp: "2026-05-16T00:00:00.000Z", + message: { role: "user", content: "valid user" }, + }, + { + type: "message", + id: "entry-2", + timestamp: "2026-05-16T00:00:01.000Z", + message: { content: "missing role" }, + }, + { + type: "message", + id: "entry-3", + timestamp: "2026-05-16T00:00:02.000Z", + message: { role: "assistant", content: "valid assistant" }, + }, + ]; const reply = await buildExportSessionReply(makeParams()); expect(reply.text).toContain("📊 Entries: 2"); expect(reply.text).toContain( - "⚠️ Skipped 1 malformed transcript row that was not valid JSON. rows 2", + "⚠️ Skipped 1 malformed transcript row that was not a session entry. rows 2", ); - expect(reply.text).toContain( - "⚠️ Skipped 1 malformed transcript row that was not a session entry. rows 4", - ); - expect(reply.text).not.toMatch(/Unexpected|SyntaxError|position/i); }); it("warns when the session only contains user messages (backend-delegated transcript)", async () => { @@ -577,21 +607,20 @@ describe("buildExportSessionReply", () => { }, }, }); - hoisted.sessionTranscriptContent = [ - JSON.stringify({ type: "session", version: 3, id: "session-1" }), - JSON.stringify({ + hoisted.sessionTranscriptEvents = [ + { type: "message", id: "entry-1", timestamp: "2026-05-16T00:00:00.000Z", message: { role: "user", content: "hello" }, - }), - JSON.stringify({ + }, + { type: "message", id: "entry-2", timestamp: "2026-05-16T00:00:01.000Z", message: { role: "user", content: "world" }, - }), - ].join("\n"); + }, + ]; const reply = await buildExportSessionReply(makeParams()); @@ -611,15 +640,14 @@ describe("buildExportSessionReply", () => { state: "idle", lastActivityAt: 1, }); - hoisted.sessionTranscriptContent = [ - JSON.stringify({ type: "session", version: 3, id: "session-1" }), - JSON.stringify({ + hoisted.sessionTranscriptEvents = [ + { type: "message", id: "entry-1", timestamp: "2026-05-16T00:00:00.000Z", message: { role: "user", content: "hello" }, - }), - ].join("\n"); + }, + ]; const reply = await buildExportSessionReply(makeParams()); @@ -638,15 +666,14 @@ describe("buildExportSessionReply", () => { hoisted.readAcpSessionMetaForEntryMock.mockImplementation(() => { throw new Error("state database unavailable"); }); - hoisted.sessionTranscriptContent = [ - JSON.stringify({ type: "session", version: 3, id: "session-1" }), - JSON.stringify({ + hoisted.sessionTranscriptEvents = [ + { type: "message", id: "entry-1", timestamp: "2026-05-16T00:00:00.000Z", message: { role: "user", content: "hello" }, - }), - ].join("\n"); + }, + ]; const reply = await buildExportSessionReply(makeParams()); @@ -656,15 +683,14 @@ describe("buildExportSessionReply", () => { }); it("does not warn for a normal user-only transcript without backend session metadata", async () => { - hoisted.sessionTranscriptContent = [ - JSON.stringify({ type: "session", version: 3, id: "session-1" }), - JSON.stringify({ + hoisted.sessionTranscriptEvents = [ + { type: "message", id: "entry-1", timestamp: "2026-05-16T00:00:00.000Z", message: { role: "user", content: "hello" }, - }), - ].join("\n"); + }, + ]; const reply = await buildExportSessionReply(makeParams()); @@ -686,15 +712,14 @@ describe("buildExportSessionReply", () => { }, }, } as never); - hoisted.sessionTranscriptContent = [ - JSON.stringify({ type: "session", version: 3, id: "session-1" }), - JSON.stringify({ + hoisted.sessionTranscriptEvents = [ + { type: "message", id: "entry-1", timestamp: "2026-05-16T00:00:00.000Z", message: { role: "user", content: "hello" }, - }), - ].join("\n"); + }, + ]; const reply = await buildExportSessionReply(makeParams()); @@ -717,21 +742,20 @@ describe("buildExportSessionReply", () => { }, }, }); - hoisted.sessionTranscriptContent = [ - JSON.stringify({ type: "session", version: 3, id: "session-1" }), - JSON.stringify({ + hoisted.sessionTranscriptEvents = [ + { type: "message", id: "entry-1", timestamp: "2026-05-16T00:00:00.000Z", message: { role: "user", content: "hello" }, - }), - JSON.stringify({ + }, + { type: "message", id: "entry-2", timestamp: "2026-05-16T00:00:01.000Z", message: { role: "assistant", content: "hi" }, - }), - ].join("\n"); + }, + ]; const reply = await buildExportSessionReply(makeParams()); diff --git a/src/auto-reply/reply/commands-export-session.ts b/src/auto-reply/reply/commands-export-session.ts index e1cc4950ac04..aeb5a87f84c0 100644 --- a/src/auto-reply/reply/commands-export-session.ts +++ b/src/auto-reply/reply/commands-export-session.ts @@ -2,20 +2,18 @@ import fsp from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { readAcpSessionMetaForEntry } from "../../acp/runtime/session-meta.js"; -import { - parseSessionFileEntriesWithWarnings, - type SessionFileParseWarning, -} from "../../agents/sessions/session-file-parser.js"; import { migrateSessionEntries, + type FileEntry as SessionFileEntry, type SessionEntry as AgentSessionEntry, type SessionHeader, type SessionMessageEntry, } from "../../agents/sessions/session-manager.js"; +import { loadTranscriptEvents } from "../../config/sessions/session-accessor.js"; import { scanSessionTranscriptTree } from "../../config/sessions/transcript-tree.js"; import type { SessionEntry as StoredSessionEntry } from "../../config/sessions/types.js"; -import { pathExists } from "../../infra/fs-safe.js"; import type { ReplyPayload } from "../types.js"; import { isReplyPayload, @@ -90,8 +88,13 @@ function isBackendDelegatedSession( ); } +type SessionExportEventWarning = { + code: "invalid-session-row"; + row: number; +}; + type SessionExportWarningSummary = { - code: SessionFileParseWarning["code"]; + code: "invalid-session-json" | "invalid-session-row"; count: number; rows: number[]; }; @@ -213,10 +216,37 @@ async function writeNewDefaultExportFile(filePath: string, html: string): Promis throw new Error(`Could not find an unused export filename near ${filePath}`); } +function isSessionFileEntry(value: unknown): value is SessionFileEntry { + if (!isRecord(value) || typeof value.type !== "string") { + return false; + } + if (value.type !== "message") { + return true; + } + const message = value.message; + return isRecord(message) && typeof message.role === "string"; +} + +function filterSessionEntriesWithWarnings(events: unknown[]): { + entries: SessionFileEntry[]; + warnings: SessionExportEventWarning[]; +} { + const entries: SessionFileEntry[] = []; + const warnings: SessionExportEventWarning[] = []; + for (const [index, event] of events.entries()) { + if (isSessionFileEntry(event)) { + entries.push(event); + continue; + } + warnings.push({ code: "invalid-session-row", row: index + 1 }); + } + return { entries, warnings }; +} + function summarizeSessionExportWarnings( - warnings: SessionFileParseWarning[], + warnings: SessionExportEventWarning[], ): SessionExportWarningSummary[] { - const summaries = new Map(); + const summaries = new Map(); for (const warning of warnings) { const summary = summaries.get(warning.code); if (summary) { @@ -254,15 +284,33 @@ function formatSessionExportWarning(summary: SessionExportWarningSummary): strin return unreachable; } -async function readSessionDataFromTranscript(sessionFile: string): Promise<{ +async function readSessionDataFromIdentity(params: { + agentId: string; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise<{ header: SessionHeader | null; entries: AgentSessionEntry[]; leafId: string | null; hasLeafControl: boolean; warnings: SessionExportWarningSummary[]; }> { - const raw = await fsp.readFile(sessionFile, "utf-8"); - const { entries: fileEntries, warnings } = parseSessionFileEntriesWithWarnings(raw); + const events = await loadTranscriptEvents(params); + const { entries, warnings } = filterSessionEntriesWithWarnings(events); + return readSessionDataFromEntries(entries, summarizeSessionExportWarnings(warnings)); +} + +function readSessionDataFromEntries( + fileEntries: SessionFileEntry[], + warnings: SessionExportWarningSummary[], +): { + header: SessionHeader | null; + entries: AgentSessionEntry[]; + leafId: string | null; + hasLeafControl: boolean; + warnings: SessionExportWarningSummary[]; +} { migrateSessionEntries(fileEntries); const header = fileEntries.find((entry): entry is SessionHeader => entry.type === "session") ?? null; @@ -284,7 +332,7 @@ async function readSessionDataFromTranscript(sessionFile: string): Promise<{ entries, leafId: tree.leafId, hasLeafControl, - warnings: summarizeSessionExportWarnings(warnings), + warnings, }; } @@ -300,15 +348,16 @@ export async function buildExportSessionReply(params: HandleCommandsParams): Pro if (isReplyPayload(sessionTarget)) { return sessionTarget; } - const { entry, sessionFile } = sessionTarget; + const { entry } = sessionTarget; - if (!(await pathExists(sessionFile))) { - return { text: `❌ Session file not found: ${sessionFile}` }; - } - - // 2. Load session entries - const { entries, header, leafId, hasLeafControl, warnings } = - await readSessionDataFromTranscript(sessionFile); + // Active exports run after startup migration, so SQLite rows are canonical. + // Do not read sessionFile here; a SQLite marker is an identifier, not a path. + const { entries, header, leafId, hasLeafControl, warnings } = await readSessionDataFromIdentity({ + agentId: sessionTarget.agentId, + sessionId: sessionTarget.sessionId, + sessionKey: sessionTarget.sessionKey, + storePath: sessionTarget.storePath, + }); // 3. Build full system prompt const { systemPrompt, tools } = await resolveCommandsSystemPromptBundle({ diff --git a/src/auto-reply/reply/commands-export-test-mocks.ts b/src/auto-reply/reply/commands-export-test-mocks.ts index 49f644439d9f..03c0fb9ee014 100644 --- a/src/auto-reply/reply/commands-export-test-mocks.ts +++ b/src/auto-reply/reply/commands-export-test-mocks.ts @@ -12,11 +12,13 @@ export function createExportCommandSessionMocks(viInstance: ViLike) { resolveSessionFilePathOptionsMock: viInstance.fn( (params: { agentId: string; storePath: string }) => params, ), - loadSessionStoreMock: viInstance.fn<() => Record>(() => ({ - "agent:target:session": { - sessionId: "session-1", - updatedAt: 1, - }, - })), + loadSessionStoreMock: viInstance.fn( + (_storePath?: string): Record => ({ + "agent:target:session": { + sessionId: "session-1", + updatedAt: 1, + }, + }), + ), }; } diff --git a/src/auto-reply/reply/commands-goal.test.ts b/src/auto-reply/reply/commands-goal.test.ts index ce70d4f8054d..ccc481d61226 100644 --- a/src/auto-reply/reply/commands-goal.test.ts +++ b/src/auto-reply/reply/commands-goal.test.ts @@ -3,7 +3,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { getSessionEntry, upsertSessionEntry } from "../../config/sessions.js"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../../config/sessions/types.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { takeCommandSessionMetadataChanges } from "./command-session-metadata.js"; import { @@ -28,6 +29,26 @@ async function createStorePath(): Promise { return path.join(root, "sessions.json"); } +// Seed and read session entries through the sqlite accessor so the goal handler, +// which reads/writes via the same accessor, observes fixtures written here. +async function upsertSessionEntry(params: { + storePath: string; + sessionKey: string; + entry: SessionEntry; +}): Promise { + await replaceSessionEntry( + { sessionKey: params.sessionKey, storePath: params.storePath }, + params.entry, + ); +} + +function getSessionEntry(params: { + storePath: string; + sessionKey: string; +}): SessionEntry | undefined { + return loadSessionEntry({ sessionKey: params.sessionKey, storePath: params.storePath }); +} + function buildGoalParams(commandBodyNormalized: string, storePath: string): HandleCommandsParams { return { cfg: {} as OpenClawConfig, diff --git a/src/auto-reply/reply/commands-name.test.ts b/src/auto-reply/reply/commands-name.test.ts index 70911ad14ae3..5be342bd8030 100644 --- a/src/auto-reply/reply/commands-name.test.ts +++ b/src/auto-reply/reply/commands-name.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { getSessionEntry, updateSessionStore, upsertSessionEntry } from "../../config/sessions.js"; +import { loadSessionEntry, upsertSessionEntry } from "../../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { buildBuiltinChatCommands } from "../commands-registry.shared.js"; import { takeCommandSessionMetadataChanges } from "./command-session-metadata.js"; @@ -95,18 +95,17 @@ describe("name command", () => { it("renames the current session and persists the label", async () => { const storePath = await createStorePath(); - await upsertSessionEntry({ - storePath, - sessionKey, - entry: { sessionId: "sess-main", updatedAt: 1, totalTokens: 0, totalTokensFresh: true }, - }); + await upsertSessionEntry( + { storePath, sessionKey }, + { sessionId: "sess-main", updatedAt: 1, totalTokens: 0, totalTokensFresh: true }, + ); const params = buildNameParams("/name Billing rework", storePath); const result = await handleNameCommand(params, true); expect(result?.shouldContinue).toBe(false); expect(result?.reply?.text).toContain("Billing rework"); - expect(getSessionEntry({ storePath, sessionKey })?.label).toBe("Billing rework"); + expect(loadSessionEntry({ storePath, sessionKey })?.label).toBe("Billing rework"); expect(params.sessionEntry?.label).toBe("Billing rework"); expect(takeCommandSessionMetadataChanges(params.ctx)).toEqual([ { sessionKey, reason: "command-metadata" }, @@ -115,56 +114,52 @@ describe("name command", () => { it("suggests a name without mutating when no argument is given", async () => { const storePath = await createStorePath(); - await upsertSessionEntry({ - storePath, - sessionKey, - entry: { sessionId: "sess-main", updatedAt: 1, totalTokens: 0, totalTokensFresh: true }, - }); + await upsertSessionEntry( + { storePath, sessionKey }, + { sessionId: "sess-main", updatedAt: 1, totalTokens: 0, totalTokensFresh: true }, + ); const params = buildNameParams("/name", storePath); - params.sessionEntry = getSessionEntry({ storePath, sessionKey }); + params.sessionEntry = loadSessionEntry({ storePath, sessionKey }); const result = await handleNameCommand(params, true); expect(result?.shouldContinue).toBe(false); expect(result?.reply?.text).toContain("Use /name "); - expect(getSessionEntry({ storePath, sessionKey })?.label).toBeUndefined(); + expect(loadSessionEntry({ storePath, sessionKey })?.label).toBeUndefined(); expect(takeCommandSessionMetadataChanges(params.ctx)).toBeUndefined(); }); it("rejects a label already used by another session", async () => { const storePath = await createStorePath(); const now = Date.now(); - await updateSessionStore(storePath, (store) => { - store[sessionKey] = { - sessionId: "sess-main", - updatedAt: now, - totalTokens: 0, - totalTokensFresh: true, - }; - store["agent:main:web:other"] = { + await upsertSessionEntry( + { storePath, sessionKey }, + { sessionId: "sess-main", updatedAt: now, totalTokens: 0, totalTokensFresh: true }, + ); + await upsertSessionEntry( + { storePath, sessionKey: "agent:main:web:other" }, + { sessionId: "sess-other", updatedAt: now, totalTokens: 0, totalTokensFresh: true, label: "Taken", - }; - return null; - }); + }, + ); const params = buildNameParams("/name Taken", storePath); const result = await handleNameCommand(params, true); expect(result?.reply?.text).toContain("label already in use"); - expect(getSessionEntry({ storePath, sessionKey })?.label).toBeUndefined(); + expect(loadSessionEntry({ storePath, sessionKey })?.label).toBeUndefined(); expect(takeCommandSessionMetadataChanges(params.ctx)).toBeUndefined(); }); it("reads the persisted name when params.sessionEntry is absent", async () => { const storePath = await createStorePath(); - await upsertSessionEntry({ - storePath, - sessionKey, - entry: { + await upsertSessionEntry( + { storePath, sessionKey }, + { sessionId: "sess-main", updatedAt: 1, totalTokens: 0, @@ -172,7 +167,7 @@ describe("name command", () => { label: "Billing rework", displayName: "Dashboard session", }, - }); + ); const params = buildNameParams("/name", storePath); const result = await handleNameCommand(params, true); @@ -198,56 +193,25 @@ describe("name command", () => { expect(result?.shouldContinue).toBe(false); expect(result?.reply?.text).toContain("First native"); - expect(getSessionEntry({ storePath, sessionKey })?.label).toBe("First native"); + expect(loadSessionEntry({ storePath, sessionKey })?.label).toBe("First native"); expect(params.sessionEntry?.label).toBe("First native"); expect(takeCommandSessionMetadataChanges(params.ctx)).toEqual([ { sessionKey, reason: "command-metadata" }, ]); }); - it("persists the rename under the canonical key when stored under a legacy alias", async () => { - const storePath = await createStorePath(); - const legacyKey = "agent:main:web:Main"; - const now = Date.now(); - await updateSessionStore(storePath, (store) => { - store[legacyKey] = { - sessionId: "sess-main", - updatedAt: now, - totalTokens: 0, - totalTokensFresh: true, - }; - return null; - }); - - const params = buildNameParams("/name Canonical", storePath); - const result = await handleNameCommand(params, true); - - expect(result?.reply?.text).toContain("Canonical"); - expect(getSessionEntry({ storePath, sessionKey })?.label).toBe("Canonical"); - - const keys = await updateSessionStore(storePath, (store) => Object.keys(store), { - skipSaveWhenResult: () => true, - }); - expect(keys).toContain(sessionKey); - expect(keys).not.toContain(legacyKey); - expect(takeCommandSessionMetadataChanges(params.ctx)).toEqual([ - { sessionKey, reason: "command-metadata" }, - ]); - }); - it("does not rename for an unauthorized sender", async () => { const storePath = await createStorePath(); - await upsertSessionEntry({ - storePath, - sessionKey, - entry: { sessionId: "sess-main", updatedAt: 1, totalTokens: 0, totalTokensFresh: true }, - }); + await upsertSessionEntry( + { storePath, sessionKey }, + { sessionId: "sess-main", updatedAt: 1, totalTokens: 0, totalTokensFresh: true }, + ); const params = buildNameParams("/name Secret", storePath, { isAuthorizedSender: false }); const result = await handleNameCommand(params, true); expect(result?.shouldContinue).toBe(false); - expect(getSessionEntry({ storePath, sessionKey })?.label).toBeUndefined(); + expect(loadSessionEntry({ storePath, sessionKey })?.label).toBeUndefined(); expect(takeCommandSessionMetadataChanges(params.ctx)).toBeUndefined(); }); diff --git a/src/auto-reply/reply/commands-name.ts b/src/auto-reply/reply/commands-name.ts index 17365d74ad25..bd83d7e706ca 100644 --- a/src/auto-reply/reply/commands-name.ts +++ b/src/auto-reply/reply/commands-name.ts @@ -3,11 +3,10 @@ import { normalizeOptionalString, } from "@openclaw/normalization-core/string-coerce"; import { - getSessionEntry, - resolveSessionStoreEntry, - type SessionEntry, - updateSessionStore, -} from "../../config/sessions.js"; + applySessionPatchProjection, + loadSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import { normalizeStoreSessionKey } from "../../config/sessions/store-entry.js"; import { deriveSessionTitle } from "../../gateway/session-utils.js"; import { parseSessionLabel } from "../../sessions/session-label.js"; import { rejectUnauthorizedCommand } from "./command-gates.js"; @@ -39,7 +38,7 @@ function syncNameSessionEntry(params: HandleCommandsParams): void { if (!params.sessionStore || !params.sessionKey || !params.storePath) { return; } - const entry = getSessionEntry({ sessionKey: params.sessionKey, storePath: params.storePath }); + const entry = loadSessionEntry({ sessionKey: params.sessionKey, storePath: params.storePath }); if (!entry) { return; } @@ -47,16 +46,6 @@ function syncNameSessionEntry(params: HandleCommandsParams): void { params.sessionEntry = entry; } -type NameWriteResult = - | { - ok: true; - label: string; - sessionKey: string; - entry: SessionEntry; - hadLegacyAliases: boolean; - } - | { ok: false; error: string }; - export const handleNameCommand: CommandHandler = async (params, allowTextCommands) => { if (!allowTextCommands) { return null; @@ -80,7 +69,7 @@ export const handleNameCommand: CommandHandler = async (params, allowTextCommand // derived locally (no LLM, no mutation). Apply it with `/name <title>`. if (!title) { const entry = - getSessionEntry({ sessionKey: params.sessionKey, storePath: params.storePath }) ?? + loadSessionEntry({ sessionKey: params.sessionKey, storePath: params.storePath }) ?? params.sessionEntry; const current = normalizeOptionalString(entry?.label); const suggestionEntry = entry ? { ...entry, label: undefined } : undefined; @@ -97,21 +86,17 @@ export const handleNameCommand: CommandHandler = async (params, allowTextCommand } const storePath = params.storePath; - const sessionKey = params.sessionKey; + const sessionKey = normalizeStoreSessionKey(params.sessionKey); // Reuse the canonical label validation (`parseSessionLabel`) and the same // cross-store uniqueness rule enforced by the web/admin `sessions.patch` - // path so chat naming behaves identically to the session manager. Resolve the - // session via `resolveSessionStoreEntry` so renames land on the canonical - // entry even when the store still holds a legacy/case-folded key alias, and - // exclude those aliases from the uniqueness scan to avoid false conflicts. - const result = await updateSessionStore<NameWriteResult>( + // path so chat naming behaves identically to the session manager. + const result = await applySessionPatchProjection<{ ok: false; error: string }>({ storePath, - (store) => { - const resolved = resolveSessionStoreEntry({ store, sessionKey }); + resolveTarget: () => ({ primaryKey: sessionKey, candidateKeys: [sessionKey] }), + project: ({ entries, existingEntry }) => { // Native slash may invoke `/name` before the fast path persists the entry. // Seed a copy under the canonical key without mutating params on failed writes. - const entry = - resolved.existing ?? (params.sessionEntry ? { ...params.sessionEntry } : undefined); + const entry = existingEntry ?? (params.sessionEntry ? { ...params.sessionEntry } : undefined); if (!entry) { return { ok: false, error: "no active session to name" }; } @@ -119,41 +104,24 @@ export const handleNameCommand: CommandHandler = async (params, allowTextCommand if (!validated.ok) { return { ok: false, error: validated.error }; } - const aliasKeys = new Set<string>([resolved.normalizedKey, ...resolved.legacyKeys]); - for (const [key, other] of Object.entries(store)) { - if (!aliasKeys.has(key) && other?.label === validated.label) { + for (const other of entries) { + if (other.sessionKey !== sessionKey && other.entry.label === validated.label) { return { ok: false, error: `label already in use: ${validated.label}` }; } } entry.label = validated.label; entry.updatedAt = Math.max(entry.updatedAt ?? 0, Date.now()); - // Persist through the canonical key and drop any legacy/case-folded - // aliases, mirroring `persistResolvedSessionEntry`. - store[resolved.normalizedKey] = entry; - for (const legacyKey of resolved.legacyKeys) { - delete store[legacyKey]; - } return { ok: true, - label: validated.label, - sessionKey: resolved.normalizedKey, entry, - hadLegacyAliases: resolved.legacyKeys.length > 0, }; }, - { - skipSaveWhenResult: (value) => !value.ok, - resolveSingleEntryPersistence: (value) => - value.ok && !value.hadLegacyAliases - ? { sessionKey: value.sessionKey, entry: value.entry } - : null, - }, - ); + }); if (!result.ok) { return nameReply(`Couldn't rename the session: ${result.error}`); } syncNameSessionEntry(params); markCommandSessionMetadataChanged(params); - return nameReply(`✅ Session renamed to “${result.label}”.`); + return nameReply(`✅ Session renamed to “${result.entry.label}”.`); }; diff --git a/src/auto-reply/reply/commands-reset-hooks.ts b/src/auto-reply/reply/commands-reset-hooks.ts index 12ce1481055c..6d83d88fedd0 100644 --- a/src/auto-reply/reply/commands-reset-hooks.ts +++ b/src/auto-reply/reply/commands-reset-hooks.ts @@ -1,6 +1,5 @@ // Emits reset hooks and cleanup work around session reset commands. -import fs from "node:fs/promises"; -import path from "node:path"; +import { loadTranscriptEvents } from "../../config/sessions/session-accessor.js"; import { selectSessionTranscriptLeafControlledPath } from "../../config/sessions/transcript-tree.js"; import { logVerbose } from "../../globals.js"; import { createInternalHookEvent, triggerInternalHook } from "../../hooks/internal-hooks.js"; @@ -17,19 +16,7 @@ function loadRouteReplyRuntime() { export type ResetCommandAction = "new" | "reset"; -function parseTranscriptMessages(content: string): unknown[] { - const entries: unknown[] = []; - for (const line of content.split("\n")) { - if (!line.trim()) { - continue; - } - try { - const entry = JSON.parse(line); - entries.push(entry); - } catch { - // Skip malformed lines from partially-written transcripts. - } - } +function parseTranscriptMessages(entries: unknown[]): unknown[] { const selectedEntries = selectSessionTranscriptLeafControlledPath(entries) ?? entries; return selectedEntries.flatMap((entry) => { if ( @@ -45,62 +32,36 @@ function parseTranscriptMessages(content: string): unknown[] { }); } -async function findLatestArchivedTranscript(sessionFile: string): Promise<string | undefined> { - try { - const dir = path.dirname(sessionFile); - const base = path.basename(sessionFile); - const resetPrefix = `${base}.reset.`; - const archived = (await fs.readdir(dir)) - .filter((name) => name.startsWith(resetPrefix)) - .toSorted(); - const latest = archived[archived.length - 1]; - return latest ? path.join(dir, latest) : undefined; - } catch { - return undefined; - } -} - async function loadBeforeResetTranscript(params: { + agentId?: string; + sessionId?: string; sessionFile?: string; + sessionKey?: string; + storePath?: string; }): Promise<{ sessionFile?: string; messages: unknown[] }> { - const sessionFile = params.sessionFile; - if (!sessionFile) { - logVerbose("before_reset: no session file available, firing hook with empty messages"); - return { sessionFile, messages: [] }; + if (!params.sessionId || !params.sessionKey || !params.storePath) { + logVerbose("before_reset: no session identity available, firing hook with empty messages"); + return { sessionFile: params.sessionFile, messages: [] }; } - try { return { - sessionFile, - messages: parseTranscriptMessages(await fs.readFile(sessionFile, "utf-8")), - }; - } catch (err: unknown) { - if ((err as { code?: unknown })?.code !== "ENOENT") { - logVerbose( - `before_reset: failed to read session file ${sessionFile}; firing hook with empty messages (${String(err)})`, - ); - return { sessionFile, messages: [] }; - } - } - - const archivedSessionFile = await findLatestArchivedTranscript(sessionFile); - if (!archivedSessionFile) { - logVerbose( - `before_reset: failed to find archived transcript for ${sessionFile}; firing hook with empty messages`, - ); - return { sessionFile, messages: [] }; - } - - try { - return { - sessionFile: archivedSessionFile, - messages: parseTranscriptMessages(await fs.readFile(archivedSessionFile, "utf-8")), + sessionFile: params.sessionFile, + messages: parseTranscriptMessages( + // before_reset snapshots the canonical pre-reset rows. sessionFile is + // hook metadata only and must not be treated as a readable path. + await loadTranscriptEvents({ + ...(params.agentId ? { agentId: params.agentId } : {}), + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }), + ), }; } catch (err: unknown) { logVerbose( - `before_reset: failed to read archived session file ${archivedSessionFile}; firing hook with empty messages (${String(err)})`, + `before_reset: failed to read transcript identity ${params.sessionKey}/${params.sessionId}; firing hook with empty messages (${String(err)})`, ); - return { sessionFile: archivedSessionFile, messages: [] }; + return { sessionFile: params.sessionFile, messages: [] }; } } @@ -113,6 +74,7 @@ export async function emitResetCommandHooks(params: { "surface" | "senderId" | "channel" | "from" | "to" | "resetHookTriggered" >; sessionKey?: string; + storePath?: string; sessionEntry?: HandleCommandsParams["sessionEntry"]; previousSessionEntry?: HandleCommandsParams["previousSessionEntry"]; workspaceDir: string; @@ -155,16 +117,20 @@ export async function emitResetCommandHooks(params: { const hookRunner = getGlobalHookRunner(); if (hookRunner?.hasHooks("before_reset")) { const prevEntry = params.previousSessionEntry; + const agentId = resolveAgentIdFromSessionKey(params.sessionKey); + const beforeResetTranscript = await loadBeforeResetTranscript({ + agentId, + sessionFile: prevEntry?.sessionFile, + sessionId: prevEntry?.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }); void (async () => { - const { sessionFile, messages } = await loadBeforeResetTranscript({ - sessionFile: prevEntry?.sessionFile, - }); - try { await hookRunner.runBeforeReset( - { sessionFile, messages, reason: params.action }, + { ...beforeResetTranscript, reason: params.action }, { - agentId: resolveAgentIdFromSessionKey(params.sessionKey), + agentId, sessionKey: params.sessionKey, sessionId: prevEntry?.sessionId, workspaceDir: params.workspaceDir, diff --git a/src/auto-reply/reply/commands-reset.ts b/src/auto-reply/reply/commands-reset.ts index 68b137892a92..d871ed1faf29 100644 --- a/src/auto-reply/reply/commands-reset.ts +++ b/src/auto-reply/reply/commands-reset.ts @@ -107,6 +107,7 @@ export async function maybeHandleResetCommand( cfg: params.cfg, command: params.command, sessionKey: params.sessionKey, + storePath: params.storePath, sessionEntry: targetSessionEntry, previousSessionEntry, workspaceDir: params.workspaceDir, @@ -180,6 +181,7 @@ export async function maybeHandleResetCommand( cfg: params.cfg, command: params.command, sessionKey: params.sessionKey, + storePath: params.storePath, sessionEntry: targetSessionEntry, previousSessionEntry: params.previousSessionEntry, workspaceDir: params.workspaceDir, diff --git a/src/auto-reply/reply/commands-session-store.test.ts b/src/auto-reply/reply/commands-session-store.test.ts index d3731e34c688..31ce2c8c4010 100644 --- a/src/auto-reply/reply/commands-session-store.test.ts +++ b/src/auto-reply/reply/commands-session-store.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { loadSessionStore, saveSessionStore } from "../../config/sessions.js"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import type { SessionEntry } from "../../config/sessions/types.js"; import { persistAbortTargetEntry, persistSessionEntry } from "./commands-session-store.js"; @@ -25,7 +25,6 @@ describe("commands session store persistence", () => { responseUsage: "tokens", }; const sessionStore: Record<string, SessionEntry> = { [sessionKey]: entry }; - await saveSessionStore(storePath, {}, { skipMaintenance: true }); await expect( persistSessionEntry({ @@ -38,7 +37,7 @@ describe("commands session store persistence", () => { }), ).resolves.toBe(true); - const persisted = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + const persisted = loadSessionEntry({ storePath, sessionKey }); expect(persisted).toMatchObject({ sessionId: "first-command-session", responseUsage: "tokens", @@ -59,7 +58,6 @@ describe("commands session store persistence", () => { responseUsage: "tokens", }; const sessionStore: Record<string, SessionEntry> = { [sessionKey]: entry }; - await saveSessionStore(storePath, {}, { skipMaintenance: true }); await expect( persistSessionEntry({ @@ -71,7 +69,57 @@ describe("commands session store persistence", () => { }), ).resolves.toBe(false); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toBeUndefined(); + expect(loadSessionEntry({ storePath, sessionKey })).toBeUndefined(); + }); + }); + + it("persists a single command session entry through the accessor", async () => { + await withTempStore(async (storePath) => { + const sessionKey = "agent:main:command"; + const otherKey = "agent:main:other"; + const entry: SessionEntry = { + sessionId: "command-session", + updatedAt: 1, + model: "gpt-5.5", + }; + const otherEntry: SessionEntry = { + sessionId: "other-session", + updatedAt: 2, + }; + const seedEntry = { ...entry }; + await persistSessionEntry({ + allowCreateSessionEntry: true, + sessionEntry: seedEntry, + sessionStore: { [sessionKey]: seedEntry }, + sessionKey, + storePath, + }); + await replaceSessionEntry({ storePath, sessionKey: otherKey }, { ...otherEntry }); + const sessionStore: Record<string, SessionEntry> = { [sessionKey]: entry }; + + await expect( + persistSessionEntry({ + sessionEntry: entry, + sessionStore, + sessionKey, + storePath, + }), + ).resolves.toBe(true); + + const persisted = loadSessionEntry({ storePath, sessionKey }); + const persistedOther = loadSessionEntry({ storePath, sessionKey: otherKey }); + expect(sessionStore[sessionKey]).toMatchObject({ + sessionId: "command-session", + model: "gpt-5.5", + }); + expect(sessionStore[sessionKey]?.updatedAt).toBeGreaterThanOrEqual(entry.updatedAt); + expect(entry.updatedAt).not.toBe(1); + expect(persisted).toMatchObject({ + sessionId: "command-session", + model: "gpt-5.5", + updatedAt: entry.updatedAt, + }); + expect(persistedOther).toStrictEqual(otherEntry); }); }); @@ -91,19 +139,20 @@ describe("commands session store persistence", () => { updatedAt: 2, }; const concurrentUpdatedAt = 300; - await saveSessionStore( + const concurrentEntry = { + ...entry, + updatedAt: concurrentUpdatedAt, + label: "After rename", + pinnedAt: undefined, + }; + await persistSessionEntry({ + allowCreateSessionEntry: true, + sessionEntry: concurrentEntry, + sessionStore: { [sessionKey]: concurrentEntry }, + sessionKey, storePath, - { - [sessionKey]: { - ...entry, - updatedAt: concurrentUpdatedAt, - label: "After rename", - pinnedAt: undefined, - }, - [otherKey]: { ...otherEntry }, - }, - { skipMaintenance: true }, - ); + }); + await replaceSessionEntry({ storePath, sessionKey: otherKey }, { ...otherEntry }); const sessionStore: Record<string, SessionEntry> = { [sessionKey]: entry }; const nowSpy = vi.spyOn(Date, "now").mockReturnValueOnce(200).mockReturnValue(400); @@ -120,23 +169,24 @@ describe("commands session store persistence", () => { nowSpy.mockRestore(); } - const persisted = loadSessionStore(storePath, { skipCache: true }); + const persisted = loadSessionEntry({ storePath, sessionKey }); + const persistedOther = loadSessionEntry({ storePath, sessionKey: otherKey }); expect(entry.updatedAt).not.toBe(1); expect(sessionStore[sessionKey]).toMatchObject({ sessionId: "command-session", label: "After rename", model: "gpt-5.5", - updatedAt: concurrentUpdatedAt, }); + expect(sessionStore[sessionKey]?.updatedAt).toBeGreaterThanOrEqual(concurrentUpdatedAt); expect(sessionStore[sessionKey]?.pinnedAt).toBeUndefined(); - expect(persisted[sessionKey]).toMatchObject({ + expect(persisted).toMatchObject({ sessionId: "command-session", label: "After rename", model: "gpt-5.5", - updatedAt: concurrentUpdatedAt, }); - expect(persisted[sessionKey]?.pinnedAt).toBeUndefined(); - expect(persisted[otherKey]).toStrictEqual(otherEntry); + expect(persisted?.updatedAt).toBeGreaterThanOrEqual(concurrentUpdatedAt); + expect(persisted?.pinnedAt).toBeUndefined(); + expect(persistedOther).toStrictEqual(otherEntry); }); }); @@ -157,7 +207,7 @@ describe("commands session store persistence", () => { updatedAt: 3, queueMode: "interrupt", }; - await saveSessionStore(storePath, { [sessionKey]: rotatedEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ storePath, sessionKey }, rotatedEntry); const sessionStore = { [sessionKey]: sessionEntry }; await expect( @@ -171,7 +221,7 @@ describe("commands session store persistence", () => { ).resolves.toBe(false); expect(sessionStore[sessionKey]).toEqual(rotatedEntry); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toEqual(rotatedEntry); + expect(loadSessionEntry({ storePath, sessionKey })).toEqual(rotatedEntry); }); }); @@ -189,11 +239,7 @@ describe("commands session store persistence", () => { updatedAt: 2, sendPolicy: "allow", }; - await saveSessionStore( - storePath, - { [sessionKey]: concurrentEntry }, - { skipMaintenance: true }, - ); + await replaceSessionEntry({ storePath, sessionKey }, concurrentEntry); const sessionStore = { [sessionKey]: sessionEntry }; await expect( @@ -232,11 +278,7 @@ describe("commands session store persistence", () => { updatedAt: 2, groupActivationNeedsSystemIntro: false, }; - await saveSessionStore( - storePath, - { [sessionKey]: concurrentEntry }, - { skipMaintenance: true }, - ); + await replaceSessionEntry({ storePath, sessionKey }, concurrentEntry); const sessionStore = { [sessionKey]: sessionEntry }; await expect( @@ -251,7 +293,7 @@ describe("commands session store persistence", () => { ).resolves.toBe(false); expect(sessionStore[sessionKey]).toEqual(concurrentEntry); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toEqual(concurrentEntry); + expect(loadSessionEntry({ storePath, sessionKey })).toEqual(concurrentEntry); }); }); @@ -264,7 +306,6 @@ describe("commands session store persistence", () => { model: "gpt-5.5", }; const sessionStore: Record<string, SessionEntry> = { [sessionKey]: entry }; - await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf8"); await expect( persistAbortTargetEntry({ @@ -276,7 +317,7 @@ describe("commands session store persistence", () => { }), ).resolves.toBe(true); - const persisted = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + const persisted = loadSessionEntry({ storePath, sessionKey }); expect(sessionStore[sessionKey]).toBe(entry); expect(entry.abortedLastRun).toBe(true); expect(entry.abortCutoffMessageSid).toBe("42"); @@ -301,21 +342,15 @@ describe("commands session store persistence", () => { }; const persistedEntry: SessionEntry = { sessionId: "persisted-session", - updatedAt: 2, + updatedAt: Date.now(), model: "sonnet-4.6", }; const otherEntry: SessionEntry = { sessionId: "other-session", updatedAt: 3, }; - await saveSessionStore( - storePath, - { - [sessionKey]: persistedEntry, - [otherKey]: otherEntry, - }, - { skipMaintenance: true }, - ); + await replaceSessionEntry({ storePath, sessionKey }, persistedEntry); + await replaceSessionEntry({ storePath, sessionKey: otherKey }, otherEntry); await expect( persistAbortTargetEntry({ @@ -326,14 +361,15 @@ describe("commands session store persistence", () => { }), ).resolves.toBe(true); - const persisted = loadSessionStore(storePath, { skipCache: true }); + const persisted = loadSessionEntry({ storePath, sessionKey }); + const persistedOther = loadSessionEntry({ storePath, sessionKey: otherKey }); expect(entry.abortedLastRun).toBe(true); - expect(persisted[sessionKey]).toMatchObject({ + expect(persisted).toMatchObject({ sessionId: "persisted-session", model: "sonnet-4.6", abortedLastRun: true, }); - expect(persisted[otherKey]).toStrictEqual(otherEntry); + expect(persistedOther).toStrictEqual(otherEntry); }); }); }); diff --git a/src/auto-reply/reply/directive-handling.model.test.ts b/src/auto-reply/reply/directive-handling.model.test.ts index 018fc098e7f9..114dbb06b11d 100644 --- a/src/auto-reply/reply/directive-handling.model.test.ts +++ b/src/auto-reply/reply/directive-handling.model.test.ts @@ -283,7 +283,7 @@ import { import type { ModelAliasIndex } from "../../agents/model-selection.js"; import type { ModelDefinitionConfig, OpenClawConfig } from "../../config/config.js"; import type { SessionEntry } from "../../config/sessions.js"; -import { loadSessionStore, saveSessionStore } from "../../config/sessions/store.js"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import { clearInternalHooks, registerInternalHook, @@ -1863,7 +1863,7 @@ describe("handleDirectiveOnly model persist behavior (fixes #1435)", () => { providerOverride: "openai", modelOverride: "gpt-5.5", }; - await saveSessionStore(storePath, { [sessionKey]: concurrentEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey, storePath }, concurrentEntry); const sessionStore = { [sessionKey]: sessionEntry }; const persistenceState = { sessionChangesApplied: true }; @@ -1891,9 +1891,7 @@ describe("handleDirectiveOnly model persist behavior (fixes #1435)", () => { }); expect(sessionStore[sessionKey]?.liveModelSwitchPending).toBeUndefined(); expect(sessionEntry).toEqual(sessionStore[sessionKey]); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toEqual( - sessionStore[sessionKey], - ); + expect(loadSessionEntry({ sessionKey, storePath })).toEqual(sessionStore[sessionKey]); } finally { fs.rmSync(tempRoot, { recursive: true, force: true }); } @@ -1908,7 +1906,7 @@ describe("handleDirectiveOnly model persist behavior (fixes #1435)", () => { updatedAt: sessionEntry.updatedAt + 1, elevatedLevel: "full", }; - await saveSessionStore(storePath, { [sessionKey]: rotatedEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey, storePath }, rotatedEntry); const sessionStore = { [sessionKey]: sessionEntry }; try { @@ -1945,7 +1943,7 @@ describe("handleDirectiveOnly model persist behavior (fixes #1435)", () => { updatedAt: sessionEntry.updatedAt + 1, elevatedLevel: "full", }; - await saveSessionStore(storePath, { [sessionKey]: concurrentEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey, storePath }, concurrentEntry); try { const result = await handleDirectiveOnly( @@ -1986,7 +1984,7 @@ describe("handleDirectiveOnly model persist behavior (fixes #1435)", () => { updatedAt: sessionEntry.updatedAt + 1, thinkingLevel: "low", }; - await saveSessionStore(storePath, { [sessionKey]: concurrentEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey, storePath }, concurrentEntry); try { const result = await handleDirectiveOnly( @@ -2001,7 +1999,7 @@ describe("handleDirectiveOnly model persist behavior (fixes #1435)", () => { expect(result?.text).toContain("Session settings were not applied"); expect(sessionEntry).toMatchObject({ thinkingLevel: "low" }); expect(sessionEntry.fastMode).toBeUndefined(); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toEqual(concurrentEntry); + expect(loadSessionEntry({ sessionKey, storePath })).toEqual(concurrentEntry); } finally { fs.rmSync(tempRoot, { recursive: true, force: true }); } @@ -2532,7 +2530,7 @@ describe("persistInlineDirectives session directive persistence policy", () => { updatedAt: sessionEntry.updatedAt + 1, modelOverride: "gpt-5.5", }; - await saveSessionStore(storePath, { [sessionKey]: concurrentEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey, storePath }, concurrentEntry); const directives = parseInlineDirectives("hello /model openai/gpt-4o"); try { @@ -2585,7 +2583,7 @@ describe("persistInlineDirectives session directive persistence policy", () => { providerOverride: "openai", modelOverride: "gpt-5.5", }; - await saveSessionStore(storePath, { [sessionKey]: concurrentEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey, storePath }, concurrentEntry); const sessionStore = { [sessionKey]: sessionEntry }; const directives = parseInlineDirectives("hello /model openai/gpt-4o"); const patchEvents: InternalHookEvent[] = []; diff --git a/src/auto-reply/reply/dispatch-acp-transcript.runtime.ts b/src/auto-reply/reply/dispatch-acp-transcript.runtime.ts index 0fcdc2fc9307..612938e75c58 100644 --- a/src/auto-reply/reply/dispatch-acp-transcript.runtime.ts +++ b/src/auto-reply/reply/dispatch-acp-transcript.runtime.ts @@ -2,11 +2,8 @@ import { resolveAcpSessionCwd } from "@openclaw/acp-core/runtime/session-identifiers"; import { resolveSessionAgentId } from "../../agents/agent-scope.js"; import { persistAcpTurnTranscript } from "../../agents/command/attempt-execution.js"; -import { - loadSessionStore, - resolveSessionStoreEntry, - resolveStorePath, -} from "../../config/sessions.js"; +import { resolveStorePath } from "../../config/sessions.js"; +import { loadSessionEntry } from "../../config/sessions/session-accessor.js"; import type { SessionAcpMeta } from "../../config/sessions/types.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; @@ -31,11 +28,11 @@ export async function persistAcpDispatchTranscript(params: { const storePath = resolveStorePath(params.cfg.session?.store, { agentId: sessionAgentId, }); - const sessionStore = loadSessionStore(storePath, { skipCache: true }); - const sessionEntry = resolveSessionStoreEntry({ - store: sessionStore, + const sessionEntry = loadSessionEntry({ + agentId: sessionAgentId, sessionKey: params.sessionKey, - }).existing; + storePath, + }); const sessionId = sessionEntry?.sessionId; if (!sessionId) { throw new Error(`unknown ACP session key: ${params.sessionKey}`); @@ -48,7 +45,6 @@ export async function persistAcpDispatchTranscript(params: { sessionId, sessionKey: params.sessionKey, sessionEntry, - sessionStore, storePath, sessionAgentId, threadId: params.threadId, diff --git a/src/auto-reply/reply/dispatch-from-config.acp-abort.test.ts b/src/auto-reply/reply/dispatch-from-config.acp-abort.test.ts index 43428ad39c6a..2cb094cca221 100644 --- a/src/auto-reply/reply/dispatch-from-config.acp-abort.test.ts +++ b/src/auto-reply/reply/dispatch-from-config.acp-abort.test.ts @@ -199,6 +199,10 @@ describe("dispatchReplyFromConfig ACP abort", () => { sessionStoreMocks.loadSessionEntry .mockReset() .mockImplementation(() => sessionStoreMocks.currentEntry); + sessionStoreMocks.loadSessionStoreEntry.mockReset(); + sessionStoreMocks.loadSessionStoreEntry.mockImplementation( + () => sessionStoreMocks.currentEntry, + ); sessionStoreMocks.loadSessionStore.mockReset().mockReturnValue({}); sessionStoreMocks.readSessionEntry.mockReset().mockReturnValue(undefined); sessionStoreMocks.resolveStorePath.mockReset().mockReturnValue("/tmp/mock-sessions.json"); @@ -507,7 +511,7 @@ describe("dispatchReplyFromConfig ACP abort", () => { conversationId: "C1", }, }; - const sessionStore = { + const sessionStore: Record<string, { sessionId: string; updatedAt: number }> = { [sourceSessionKey]: { sessionId: "source-session-id", updatedAt: Date.now(), @@ -519,17 +523,12 @@ describe("dispatchReplyFromConfig ACP abort", () => { }; sessionBindingMocks.resolveByConversation.mockReturnValue(boundConversation); sessionStoreMocks.currentEntry = sessionStore[sourceSessionKey]; - sessionStoreMocks.loadSessionStore.mockReturnValue(sessionStore); - sessionStoreMocks.resolveSessionStoreEntry.mockImplementation((...args: unknown[]) => { - const params = args[0] as { store?: Record<string, unknown>; sessionKey?: string }; - const existing = - params.store && params.sessionKey ? params.store[params.sessionKey] : undefined; - return { - existing: - existing && typeof existing === "object" - ? (existing as Record<string, unknown>) - : undefined, - }; + sessionStoreMocks.loadSessionStoreEntry.mockImplementation((...args: unknown[]) => { + const params = args[0] as { sessionKey?: string }; + const existing = params.sessionKey ? sessionStore[params.sessionKey] : undefined; + return existing && typeof existing === "object" + ? (existing as Record<string, unknown>) + : undefined; }); acpMocks.readAcpSessionEntry.mockImplementation((params: { sessionKey: string }) => params.sessionKey === boundAcpSessionKey diff --git a/src/auto-reply/reply/dispatch-from-config.reply-dispatch.test.ts b/src/auto-reply/reply/dispatch-from-config.reply-dispatch.test.ts index d9b44b0331a0..87c63bf18fdb 100644 --- a/src/auto-reply/reply/dispatch-from-config.reply-dispatch.test.ts +++ b/src/auto-reply/reply/dispatch-from-config.reply-dispatch.test.ts @@ -96,6 +96,10 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => { sessionBindingMocks.resolveByConversation.mockReset().mockReturnValue(null); sessionBindingMocks.touch.mockReset(); sessionStoreMocks.currentEntry = undefined; + sessionStoreMocks.loadSessionStoreEntry.mockReset(); + sessionStoreMocks.loadSessionStoreEntry.mockImplementation( + () => sessionStoreMocks.currentEntry, + ); sessionStoreMocks.loadSessionStore.mockReset().mockReturnValue({}); sessionStoreMocks.readSessionEntry.mockReset().mockReturnValue(undefined); sessionStoreMocks.resolveStorePath.mockReset().mockReturnValue("/tmp/mock-sessions.json"); @@ -204,9 +208,7 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => { pendingFinalDeliveryLastError: "previous failure", pendingFinalDeliveryContext: { source: "heartbeat" }, }; - sessionStoreMocks.resolveSessionStoreEntry.mockReturnValue({ - existing: sessionStoreMocks.currentEntry, - }); + sessionStoreMocks.loadSessionStore.mockClear(); mocks.routeReply.mockResolvedValue({ ok: true, messageId: "mock" }); const result = await dispatchReplyFromConfig({ @@ -217,6 +219,14 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => { }); expect(result.queuedFinal).toBe(true); + expect(sessionStoreMocks.loadSessionStoreEntry).toHaveBeenCalledWith({ + agentId: "test", + storePath: "/tmp/mock-sessions.json", + sessionKey: "agent:test:session", + readConsistency: "latest", + clone: false, + }); + expect(sessionStoreMocks.loadSessionStore).not.toHaveBeenCalled(); expect(sessionStoreMocks.updateSessionStoreEntry).toHaveBeenCalledOnce(); expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toBeUndefined(); expect(sessionStoreMocks.currentEntry?.pendingFinalDeliveryText).toBeUndefined(); diff --git a/src/auto-reply/reply/dispatch-from-config.runtime.ts b/src/auto-reply/reply/dispatch-from-config.runtime.ts index 00bcb61b8c1e..80c99d814f97 100644 --- a/src/auto-reply/reply/dispatch-from-config.runtime.ts +++ b/src/auto-reply/reply/dispatch-from-config.runtime.ts @@ -1,15 +1,20 @@ /** Runtime-only dispatch dependencies shared by config-driven reply delivery. */ -import { updateSessionEntry } from "../../config/sessions/session-accessor.js"; +import { loadSessionEntry, updateSessionEntry } from "../../config/sessions/session-accessor.js"; import type { SessionEntry } from "../../config/sessions/types.js"; export { resolveStorePath } from "../../config/sessions/paths.js"; -export { - loadSessionStore, - readSessionEntry, - resolveSessionStoreEntry, -} from "../../config/sessions/store.js"; export { createInternalHookEvent, triggerInternalHook } from "../../hooks/internal-hooks.js"; +export function loadSessionStoreEntry(params: { + agentId?: string; + storePath: string; + sessionKey: string; + readConsistency?: "latest"; + clone?: boolean; +}): SessionEntry | undefined { + return loadSessionEntry(params); +} + export async function updateSessionStoreEntry(params: { storePath: string; sessionKey: string; diff --git a/src/auto-reply/reply/dispatch-from-config.shared.test-harness.ts b/src/auto-reply/reply/dispatch-from-config.shared.test-harness.ts index 953d8735501b..c5d76cb04241 100644 --- a/src/auto-reply/reply/dispatch-from-config.shared.test-harness.ts +++ b/src/auto-reply/reply/dispatch-from-config.shared.test-harness.ts @@ -105,6 +105,7 @@ const pluginConversationBindingMocks = vi.hoisted(() => ({ const sessionStoreMocks = vi.hoisted(() => ({ currentEntry: undefined as Record<string, unknown> | undefined, loadSessionEntry: vi.fn((..._args: unknown[]) => sessionStoreMocks.currentEntry), + loadSessionStoreEntry: vi.fn(() => sessionStoreMocks.currentEntry), loadSessionStore: vi.fn(() => ({})), readSessionEntry: vi.fn(() => sessionStoreMocks.currentEntry), resolveStorePath: vi.fn(() => "/tmp/mock-sessions.json"), @@ -244,6 +245,7 @@ vi.mock("../../config/sessions/session-accessor.js", async (importOriginal) => { }); vi.mock("./dispatch-from-config.runtime.js", () => ({ createInternalHookEvent: internalHookMocks.createInternalHookEvent, + loadSessionStoreEntry: sessionStoreMocks.loadSessionStoreEntry, loadSessionStore: sessionStoreMocks.loadSessionStore, readSessionEntry: sessionStoreMocks.readSessionEntry, resolveSessionStoreEntry: sessionStoreMocks.resolveSessionStoreEntry, diff --git a/src/auto-reply/reply/dispatch-from-config.test.ts b/src/auto-reply/reply/dispatch-from-config.test.ts index cf4da2952f23..17ebbcb0ae2d 100644 --- a/src/auto-reply/reply/dispatch-from-config.test.ts +++ b/src/auto-reply/reply/dispatch-from-config.test.ts @@ -164,6 +164,7 @@ const sessionStoreMocks = vi.hoisted(() => ({ currentEntry: undefined as Record<string, unknown> | undefined, entriesBySessionKey: new Map<string, Record<string, unknown>>(), loadSessionEntry: vi.fn((..._args: unknown[]) => sessionStoreMocks.currentEntry), + loadSessionStoreEntry: vi.fn(() => sessionStoreMocks.currentEntry), loadSessionStore: vi.fn(() => ({})), readSessionEntry: vi.fn(() => sessionStoreMocks.currentEntry), resolveStorePath: vi.fn(() => "/tmp/mock-sessions.json"), @@ -491,6 +492,7 @@ vi.mock("../../config/sessions/thread-info.js", () => ({ })); vi.mock("./dispatch-from-config.runtime.js", () => ({ createInternalHookEvent: internalHookMocks.createInternalHookEvent, + loadSessionStoreEntry: sessionStoreMocks.loadSessionStoreEntry, loadSessionStore: sessionStoreMocks.loadSessionStore, readSessionEntry: sessionStoreMocks.readSessionEntry, resolveSessionStoreEntry: sessionStoreMocks.resolveSessionStoreEntry, @@ -1169,6 +1171,10 @@ describe("dispatchReplyFromConfig", () => { sessionStoreMocks.entriesBySessionKey.clear(); sessionStoreMocks.loadSessionEntry.mockReset(); sessionStoreMocks.loadSessionEntry.mockImplementation(() => sessionStoreMocks.currentEntry); + sessionStoreMocks.loadSessionStoreEntry.mockReset(); + sessionStoreMocks.loadSessionStoreEntry.mockImplementation( + () => sessionStoreMocks.currentEntry, + ); sessionStoreMocks.loadSessionStore.mockReset(); sessionStoreMocks.loadSessionStore.mockReturnValue({}); sessionStoreMocks.readSessionEntry.mockReset(); @@ -4956,7 +4962,7 @@ describe("dispatchReplyFromConfig", () => { sessionStoreMocks.currentEntry = { verboseLevel: "off", }; - sessionStoreMocks.readSessionEntry.mockReturnValue({ verboseLevel: "on" }); + sessionStoreMocks.loadSessionStoreEntry.mockReturnValue({ verboseLevel: "on" }); const cfg = { ...emptyConfig, agents: { @@ -4979,7 +4985,7 @@ describe("dispatchReplyFromConfig", () => { ) => { sessionStoreMocks.loadSessionStore.mockClear(); sessionStoreMocks.resolveSessionStoreEntry.mockClear(); - sessionStoreMocks.readSessionEntry.mockClear(); + sessionStoreMocks.loadSessionStoreEntry.mockClear(); await opts?.onPlanUpdate?.({ phase: "update", explanation: "Inspect code, patch it, run tests.", @@ -4995,10 +5001,13 @@ describe("dispatchReplyFromConfig", () => { await dispatchReplyFromConfig({ ctx, cfg, dispatcher, replyResolver }); - expect(sessionStoreMocks.readSessionEntry).toHaveBeenCalledWith( - "/tmp/mock-sessions.json", - "agent:main:main", - ); + expect(sessionStoreMocks.loadSessionStoreEntry).toHaveBeenCalledWith({ + agentId: "main", + storePath: "/tmp/mock-sessions.json", + sessionKey: "agent:main:main", + readConsistency: "latest", + clone: false, + }); expect(sessionStoreMocks.loadSessionStore).not.toHaveBeenCalled(); expect(sessionStoreMocks.resolveSessionStoreEntry).not.toHaveBeenCalled(); expect(firstToolResultPayload(dispatcher)).toMatchObject({ @@ -13107,14 +13116,20 @@ describe("sendPolicy deny — suppress delivery, not processing (#53328)", () => parentSessionKey, sendPolicy: "allow", }; - sessionStoreMocks.loadSessionStore.mockReturnValueOnce({ - [parentSessionKey]: { - sessionId: "parent", - updatedAt: 0, - providerOverride: "anthropic", - modelOverride: "claude-sonnet-4.6", - }, - }); + const parentEntry = { + sessionId: "parent", + updatedAt: 0, + providerOverride: "anthropic", + modelOverride: "claude-sonnet-4.6", + }; + // Scope the parent-key override to this test; the describe's beforeEach does + // not reset loadSessionStoreEntry, so leaving it in place would resolve a + // stale "parent" sessionId for a sibling reusing this session key. + const defaultLoadSessionStoreEntry = () => sessionStoreMocks.currentEntry; + sessionStoreMocks.loadSessionStoreEntry.mockImplementation(((params: unknown) => + (params as { sessionKey?: string }).sessionKey === parentSessionKey + ? parentEntry + : sessionStoreMocks.currentEntry) as () => Record<string, unknown> | undefined); const dispatcher = createDispatcher(); const replyResolver = vi.fn(async (_ctx: MsgContext, opts?: GetReplyOptions) => { expect(opts?.sourceReplyDeliveryMode).toBe("automatic"); @@ -13136,8 +13151,17 @@ describe("sendPolicy deny — suppress delivery, not processing (#53328)", () => }); expect(replyResolver).toHaveBeenCalledTimes(1); + expect(sessionStoreMocks.loadSessionStore).not.toHaveBeenCalled(); + expect(sessionStoreMocks.loadSessionStoreEntry).toHaveBeenCalledWith({ + agentId: "main", + storePath: "/tmp/mock-sessions.json", + sessionKey: parentSessionKey, + readConsistency: "latest", + clone: false, + }); expect(result.queuedFinal).toBe(true); expect(firstFinalReplyPayload(dispatcher)?.text).toBe("visible parent-model reply"); + sessionStoreMocks.loadSessionStoreEntry.mockImplementation(defaultLoadSessionStoreEntry); }); it("honors heartbeat model overrides before Codex direct source delivery defaults", async () => { @@ -13152,6 +13176,12 @@ describe("sendPolicy deny — suppress delivery, not processing (#53328)", () => : { supported: false, reason: "codex provider only" }, runAttempt: vi.fn(async () => ({}) as never), }); + sessionStoreMocks.currentEntry = { + sessionId: "s1", + updatedAt: 0, + agentHarnessId: "codex", + sendPolicy: "allow", + }; const dispatcher = createDispatcher(); const replyResolver = vi.fn(async (_ctx: MsgContext, opts?: GetReplyOptions) => { expect(opts?.sourceReplyDeliveryMode).toBe("automatic"); diff --git a/src/auto-reply/reply/dispatch-from-config.ts b/src/auto-reply/reply/dispatch-from-config.ts index 328fe42a21e1..ef234649cc7b 100644 --- a/src/auto-reply/reply/dispatch-from-config.ts +++ b/src/auto-reply/reply/dispatch-from-config.ts @@ -57,6 +57,7 @@ import { shouldSuppressLocalExecApprovalPrompt } from "../../channels/plugins/ex import { applyMergePatch } from "../../config/merge-patch.js"; import { normalizeExplicitSessionKey } from "../../config/sessions/explicit-session-key-normalization.js"; import { resolveGroupSessionKey } from "../../config/sessions/group.js"; +import { isRecoverableTerminalSessionStatus } from "../../config/sessions/terminal-status.js"; import { appendAssistantMessageToSessionTranscript, type SessionTranscriptDeliveryMirror, @@ -140,9 +141,7 @@ import { import { resolveConversationBindingContextFromMessage } from "./conversation-binding-input.js"; import { createInternalHookEvent, - loadSessionStore, - readSessionEntry, - resolveSessionStoreEntry, + loadSessionStoreEntry, resolveStorePath, triggerInternalHook, updateSessionStoreEntry, @@ -262,10 +261,6 @@ function isDispatchReplyOperationAbortedError( return error instanceof DispatchReplyOperationAbortedError; } -function isRecoverableTerminalSessionStatus(status: SessionEntry["status"] | undefined): boolean { - return status === "failed" || status === "timeout" || status === "killed"; -} - function composeAbortSignals(...signals: Array<AbortSignal | undefined>): AbortSignal | undefined { const activeSignals: AbortSignal[] = []; for (const signal of signals) { @@ -459,12 +454,18 @@ const resolveSessionStoreLookup = ( const agentId = resolveSessionAgentId({ sessionKey, config: cfg, fallbackAgentId: ctx.AgentId }); const storePath = resolveStorePath(cfg.session?.store, { agentId }); try { - const store = loadSessionStore(storePath); + const entry = loadSessionStoreEntry({ + agentId, + storePath, + sessionKey, + readConsistency: "latest", + clone: false, + }); return { sessionKey, storePath, - store, - entry: resolveSessionStoreEntry({ store, sessionKey }).existing, + entry, + store: entry ? { [sessionKey]: entry } : undefined, }; } catch { return { @@ -506,6 +507,7 @@ const resolveBoundAcpDispatchSessionKey = (params: { }; const createShouldEmitVerboseProgress = (params: { + agentId?: string; sessionKey?: string; storePath?: string; initialExplicitLevel?: string; @@ -514,7 +516,13 @@ const createShouldEmitVerboseProgress = (params: { const resolveCurrentExplicitLevel = () => { if (params.sessionKey && params.storePath) { try { - const entry = readSessionEntry(params.storePath, params.sessionKey); + const entry = loadSessionStoreEntry({ + ...(params.agentId ? { agentId: params.agentId } : {}), + storePath: params.storePath, + sessionKey: params.sessionKey, + readConsistency: "latest", + clone: false, + }); return normalizeVerboseLevel(entry?.verboseLevel ?? ""); } catch { // Ignore transient store read failures and fall back to the current dispatch snapshot. @@ -861,13 +869,30 @@ function resolveChannelModelCandidate(params: { } function resolveStoredModelCandidate(params: { + cfg: OpenClawConfig; defaultProvider: string; entry?: SessionEntry; parentSessionKey?: string; + sessionAgentId: string; sessionKey?: string; sessionStore?: Record<string, SessionEntry>; }): HarnessDefaultCandidate | undefined { const storedModelRef = resolveStoredModelOverride({ + loadSessionEntry: (sessionKey) => { + const agentId = resolveSessionAgentId({ + sessionKey, + config: params.cfg, + fallbackAgentId: params.sessionAgentId, + }); + const storePath = resolveStorePath(params.cfg.session?.store, { agentId }); + return loadSessionStoreEntry({ + agentId, + storePath, + sessionKey, + readConsistency: "latest", + clone: false, + }); + }, sessionEntry: params.entry, sessionStore: params.sessionStore, sessionKey: params.sessionKey, @@ -929,9 +954,11 @@ const resolveHarnessSourceVisibleRepliesDefault = (params: { parentSessionKey, }); const storedModelCandidate = resolveStoredModelCandidate({ + cfg: params.cfg, defaultProvider: defaultModelRef.provider, entry: params.entry, parentSessionKey, + sessionAgentId: params.sessionAgentId, sessionKey: params.sessionKey, sessionStore: params.sessionStore, }); @@ -1631,6 +1658,7 @@ async function dispatchReplyFromConfigInner( if (sessionKeysMatch(binding.sessionKey, operationSessionStoreEntry.sessionKey)) { preparedOperationSessionBinding = binding; } + params.replyOptions?.onSessionPrepared?.(binding); }; const resolveOperationExpectedSessionId = () => preparedOperationSessionBinding?.sessionId ?? operationSessionStoreEntry.entry?.sessionId; @@ -1650,6 +1678,7 @@ async function dispatchReplyFromConfigInner( }); const sessionAgentCfg = resolveAgentConfig(cfg, sessionAgentId); const verboseProgress = createShouldEmitVerboseProgress({ + agentId: sessionAgentId, sessionKey: acpDispatchSessionKey, storePath: sessionStoreEntry.storePath, initialExplicitLevel: sessionStoreEntry.entry?.verboseLevel, diff --git a/src/auto-reply/reply/followup-runner.test.ts b/src/auto-reply/reply/followup-runner.test.ts index a3454581befe..d745a9819d76 100644 --- a/src/auto-reply/reply/followup-runner.test.ts +++ b/src/auto-reply/reply/followup-runner.test.ts @@ -12,6 +12,7 @@ import { createUserTurnTranscriptRecorder, type PersistedUserTurnMessage, } from "../../sessions/user-turn-transcript.js"; +import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js"; import type { GetReplyOptions } from "../types.js"; import { GENERIC_EXTERNAL_RUN_FAILURE_TEXT } from "./agent-runner-failure-copy.js"; import type { FollowupRun, QueueSettings } from "./queue.js"; @@ -34,6 +35,7 @@ let createFollowupRunner: typeof import("./followup-runner.js").createFollowupRu let clearRuntimeConfigSnapshot: typeof import("../../config/config.js").clearRuntimeConfigSnapshot; let loadSessionStore: typeof import("../../config/sessions/store.js").loadSessionStore; let saveSessionStore: typeof import("../../config/sessions/store.js").saveSessionStore; +let replaceSessionEntrySync: typeof import("../../config/sessions/session-accessor.js").replaceSessionEntrySync; let clearSessionStoreCacheForTest: typeof import("../../config/sessions/store.js").clearSessionStoreCacheForTest; let clearFollowupQueue: typeof import("./queue.js").clearFollowupQueue; let enqueueFollowupRun: typeof import("./queue.js").enqueueFollowupRun; @@ -79,7 +81,7 @@ function joinPromptSections(...sections: Array<string | undefined>): string { function createTestUserTurnRecorder(message: PersistedUserTurnMessage) { return createUserTurnTranscriptRecorder({ message, - target: { transcriptPath: "/tmp/session.jsonl" }, + target: createTestUserTurnTranscriptTarget(), updateMode: "none", }); } @@ -149,7 +151,11 @@ function registerFollowupTestSessionStore( sessionStore: Record<string, SessionEntry>, ): void { fsSync.mkdirSync(path.dirname(storePath), { recursive: true }); - fsSync.writeFileSync(storePath, JSON.stringify(sessionStore)); + // Seed the sqlite accessor so the runner's loadSessionEntry/admitReplyTurn reads + // observe these fixtures; the in-memory map still backs the mocked accounting helpers. + for (const [sessionKey, entry] of Object.entries(sessionStore)) { + replaceSessionEntrySync({ sessionKey, storePath }, entry); + } FOLLOWUP_TEST_SESSION_STORES.set(storePath, sessionStore); FOLLOWUP_TEST_SESSION_STORE_PATHS.add(storePath); } @@ -496,6 +502,7 @@ async function loadFreshFollowupRunnerModuleForTest() { await import("../../config/config.js")); ({ clearSessionStoreCacheForTest, loadSessionStore, saveSessionStore } = await import("../../config/sessions/store.js")); + ({ replaceSessionEntrySync } = await import("../../config/sessions/session-accessor.js")); ({ clearFollowupQueue, enqueueFollowupRun } = await import("./queue.js")); sessionRunAccounting = await import("./session-run-accounting.js"); ({ createMockFollowupRun, createMockTypingController } = await import("./test-helpers.js")); diff --git a/src/auto-reply/reply/followup-runner.ts b/src/auto-reply/reply/followup-runner.ts index 117a34c6c434..ca1d16ec4f29 100644 --- a/src/auto-reply/reply/followup-runner.ts +++ b/src/auto-reply/reply/followup-runner.ts @@ -1372,6 +1372,15 @@ export function createFollowupRunner(params: { }); pendingLifecycleTerminal = { provider, model, backstop: lifecycleBackstop }; const followupCurrentMessageId = resolveFollowupCurrentMessageId(); + const runSessionTarget = + storePath && run.sessionKey + ? { + ...(run.agentId ? { agentId: run.agentId } : {}), + ...(run.sessionId ? { sessionId: run.sessionId } : {}), + sessionKey: run.sessionKey, + storePath, + } + : undefined; const result = await runEmbeddedAgent({ allowGatewaySubagentBinding: true, lifecycleGeneration, @@ -1379,6 +1388,7 @@ export function createFollowupRunner(params: { sessionId: run.sessionId, sessionKey: run.sessionKey, agentId: run.agentId, + sessionTarget: runSessionTarget, trigger: "user", messageChannel: queued.originatingChannel ?? undefined, messageProvider: run.messageProvider, diff --git a/src/auto-reply/reply/get-reply-fast-path.ts b/src/auto-reply/reply/get-reply-fast-path.ts index 893310454264..630a34062592 100644 --- a/src/auto-reply/reply/get-reply-fast-path.ts +++ b/src/auto-reply/reply/get-reply-fast-path.ts @@ -7,9 +7,10 @@ import { import { normalizeChatType } from "../../channels/chat-type.js"; import { normalizeAnyChannelId } from "../../channels/registry.js"; import { applyMergePatch } from "../../config/merge-patch.js"; -import { resolveSessionTranscriptPath, resolveStorePath } from "../../config/sessions/paths.js"; +import { resolveStorePath } from "../../config/sessions/paths.js"; +import { loadSessionEntry, listSessionEntries } from "../../config/sessions/session-accessor.js"; import { resolveSessionKey } from "../../config/sessions/session-key.js"; -import { loadSessionStore, resolveSessionStoreEntry } from "../../config/sessions/store.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import type { SessionEntry, SessionScope } from "../../config/sessions/types.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { @@ -200,14 +201,10 @@ export function initFastReplySessionState(params: { mainKey: cfg.session?.mainKey, }); const storePath = resolveStorePath(cfg.session?.store, { agentId }); - const sessionStore: Record<string, SessionEntry> = loadSessionStore(storePath, { - skipCache: true, - clone: false, - }); - const existingEntry = resolveSessionStoreEntry({ - store: sessionStore, - sessionKey, - }).existing; + const sessionStore: Record<string, SessionEntry> = Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey: entryKey, entry }) => [entryKey, entry]), + ); + const existingEntry = loadSessionEntry({ storePath, sessionKey }); const commandSource = ctx.BodyForCommands ?? ctx.CommandBody ?? ctx.RawBody ?? ctx.Body ?? ""; const triggerBodyNormalized = isFormattedGoalContinuationPrompt(commandSource) ? commandSource.trim() @@ -236,7 +233,7 @@ export function initFastReplySessionState(params: { const sessionFile = !resetTriggered && existingEntry?.sessionFile ? existingEntry.sessionFile - : resolveSessionTranscriptPath(sessionId, agentId); + : formatSqliteSessionFileMarker({ agentId, sessionId, storePath }); const sessionEntry: SessionEntry = { ...(!resetTriggered ? existingEntry : undefined), sessionId, diff --git a/src/auto-reply/reply/get-reply-inline-actions.skip-when-config-empty.test.ts b/src/auto-reply/reply/get-reply-inline-actions.skip-when-config-empty.test.ts index 8cb666e09d8c..173223fb0e5a 100644 --- a/src/auto-reply/reply/get-reply-inline-actions.skip-when-config-empty.test.ts +++ b/src/auto-reply/reply/get-reply-inline-actions.skip-when-config-empty.test.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { SessionEntry } from "../../config/sessions.js"; +import { replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import type { SkillCommandSpec } from "../../skills/types.js"; import { getReplyPayloadMetadata } from "../reply-payload.js"; import type { TemplateContext } from "../templating.js"; @@ -69,8 +70,9 @@ async function writeSessionStore( entries: Record<string, unknown>, ) { const storePath = storeTemplate.replaceAll("{agentId}", agentId); - await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile(storePath, JSON.stringify(entries, null, 2), "utf-8"); + for (const [sessionKey, entry] of Object.entries(entries)) { + await replaceSessionEntry({ agentId, sessionKey, storePath }, entry as SessionEntry); + } } const createHandleInlineActionsInput = (params: { diff --git a/src/auto-reply/reply/get-reply-native-slash-fast-path.test.ts b/src/auto-reply/reply/get-reply-native-slash-fast-path.test.ts index f4d17c354dac..d0a26e6e04c6 100644 --- a/src/auto-reply/reply/get-reply-native-slash-fast-path.test.ts +++ b/src/auto-reply/reply/get-reply-native-slash-fast-path.test.ts @@ -2,7 +2,12 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"; import type { OpenClawConfig } from "../../config/config.js"; -import { loadSessionStore, saveSessionStore } from "../../config/sessions/store.js"; +import { + loadExactSessionEntry, + loadSessionEntry, + replaceSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../../config/sessions/types.js"; import { getReplyPayloadMetadata } from "../reply-payload.js"; import { markCompleteReplyConfig } from "./get-reply-fast-path.js"; import * as sessionPersistence from "./session-entry-persistence.js"; @@ -251,7 +256,7 @@ describe("maybeResolveNativeSlashCommandFastReply", () => { reply: expect.objectContaining({ text: "You are not authorized to use this command." }), }); expect(handleCommandsMock).toHaveBeenCalledOnce(); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toBeUndefined(); + expect(loadExactSessionEntry({ sessionKey, storePath })).toBeUndefined(); }); it("marks deleted-session initialization conflicts for delivery", async () => { @@ -349,16 +354,10 @@ describe("maybeResolveNativeSlashCommandFastReply", () => { it("adopts a supported legacy alias before native command initialization", async () => { const storePath = path.join(tempDirs.make("openclaw-native-slash-alias-"), "sessions.json"); const sessionKey = "agent:main:main"; - await saveSessionStore( - storePath, - { - "Agent:main:main": { - sessionId: "legacy-session", - updatedAt: 1, - }, - }, - { skipMaintenance: true }, - ); + await replaceSessionEntry({ sessionKey: "Agent:main:main", storePath }, { + sessionId: "legacy-session", + updatedAt: 1, + } as SessionEntry); handleCommandsMock.mockImplementationOnce(async (params: { sessionEntry?: unknown }) => { expect(params.sessionEntry).toMatchObject({ sessionId: "legacy-session" }); return { shouldContinue: false, reply: { text: "ok" } }; @@ -410,8 +409,8 @@ describe("maybeResolveNativeSlashCommandFastReply", () => { archivedAt: 2, channel: "telegram", }; - await saveSessionStore(storePath, { [sessionKey]: archivedEntry }, { skipMaintenance: true }); - const persistedArchivedEntry = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + await replaceSessionEntry({ sessionKey, storePath }, archivedEntry as SessionEntry); + const persistedArchivedEntry = loadExactSessionEntry({ sessionKey, storePath })?.entry; const result = await maybeResolveNativeSlashCommandFastReply({ ctx: buildTestCtx({ @@ -448,28 +447,20 @@ describe("maybeResolveNativeSlashCommandFastReply", () => { reply: expect.objectContaining({ text: expect.stringContaining("is archived") }), }); expect(handleCommandsMock).not.toHaveBeenCalled(); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toEqual( - persistedArchivedEntry, - ); + expect(loadExactSessionEntry({ sessionKey, storePath })?.entry).toEqual(persistedArchivedEntry); }); it("persists fast-path session initialization before command mutation", async () => { const storePath = path.join(tempDirs.make("openclaw-native-slash-init-"), "sessions.json"); const sessionKey = "agent:main:main"; - await saveSessionStore( - storePath, - { - [sessionKey]: { - sessionId: "session-1", - updatedAt: 1, - lastInteractionAt: 1, - channel: "old-channel", - }, - }, - { skipMaintenance: true }, - ); + await replaceSessionEntry({ sessionKey, storePath }, { + sessionId: "session-1", + updatedAt: 1, + lastInteractionAt: 1, + channel: "old-channel", + } as SessionEntry); handleCommandsMock.mockImplementationOnce(async (params: { sessionEntry?: unknown }) => { - const persisted = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + const persisted = loadSessionEntry({ sessionKey, storePath }); expect(params.sessionEntry).toMatchObject({ sessionId: "session-1", updatedAt: 100, diff --git a/src/auto-reply/reply/get-reply-native-slash-fast-path.ts b/src/auto-reply/reply/get-reply-native-slash-fast-path.ts index 3495c4edfc13..a18a7aea1d19 100644 --- a/src/auto-reply/reply/get-reply-native-slash-fast-path.ts +++ b/src/auto-reply/reply/get-reply-native-slash-fast-path.ts @@ -322,6 +322,10 @@ export async function maybeResolveNativeSlashCommandFastReply(params: { agentId: params.agentId, agentDir: params.agentDir, sessionEntry: sessionState.sessionEntry, + ...(sessionState.initialSessionEntry + ? { initialSessionEntry: sessionState.initialSessionEntry } + : {}), + allowCreateSessionEntry: sessionState.initialSessionEntry === undefined, previousSessionEntry: sessionState.previousSessionEntry, sessionStore: sessionState.sessionStore, sessionKey: sessionState.sessionKey, diff --git a/src/auto-reply/reply/get-reply-run.media-only.test.ts b/src/auto-reply/reply/get-reply-run.media-only.test.ts index 19cb95c6456d..74725e0e6d05 100644 --- a/src/auto-reply/reply/get-reply-run.media-only.test.ts +++ b/src/auto-reply/reply/get-reply-run.media-only.test.ts @@ -2,7 +2,6 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { importFreshModule } from "openclaw/plugin-sdk/test-fixtures"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { clearActiveEmbeddedRun, @@ -164,7 +163,6 @@ let buildInboundUserContextPrefix: typeof import("./inbound-meta.js").buildInbou let resolveInboundUserContextPromptJoiner: typeof import("./inbound-meta.js").resolveInboundUserContextPromptJoiner; let getActiveReplyRunCount: typeof import("./reply-run-registry.js").getActiveReplyRunCount; let replyRunTesting: typeof import("./reply-run-registry.js").testing; -let loadScopeCounter = 0; function createGatewayDrainingError(): Error { const error = new Error("Gateway is draining for restart; new tasks are not accepted"); @@ -172,16 +170,8 @@ function createGatewayDrainingError(): Error { return error; } -async function loadFreshGetReplyRunModuleForTest() { - return await importFreshModule<typeof import("./get-reply-run.js")>( - import.meta.url, - `./get-reply-run.js?scope=media-only-${loadScopeCounter++}`, - ); -} - const ROOM_EVENT_MESSAGE_TOOL_DIRECTIVE = "Treat this as observed room activity. Default: no reply; most room events need no response from you. Send a visible reply via message(action=send) only when you are directly addressed or have concrete value to add; your final text here stays private either way."; - function baseParams( overrides: Partial<Parameters<typeof runPreparedReply>[0]> = {}, ): Parameters<typeof runPreparedReply>[0] { @@ -340,12 +330,6 @@ describe("runPreparedReply media-only handling", () => { return Promise.all(paths.map((entry) => rm(entry, { recursive: true, force: true }))); }); - it("does not load session store runtime on module import", async () => { - await loadFreshGetReplyRunModuleForTest(); - - expect(storeRuntimeLoads).not.toHaveBeenCalled(); - }); - it("passes approved elevated defaults to the runner", async () => { await runPreparedReply( baseParams({ @@ -505,7 +489,6 @@ describe("runPreparedReply media-only handling", () => { expect(call.followupRun.run.thinkLevel).toBe("off"); expect(sessionEntry.thinkingLevel).toBe("high"); expect(sessionStore["session-key"]?.thinkingLevel).toBe("high"); - expect(updateSessionStore).not.toHaveBeenCalled(); }); it("keeps empty-assistant silence disabled for direct runs by default", async () => { diff --git a/src/auto-reply/reply/get-reply-run.ts b/src/auto-reply/reply/get-reply-run.ts index f0d626ea6233..86e222b98f84 100644 --- a/src/auto-reply/reply/get-reply-run.ts +++ b/src/auto-reply/reply/get-reply-run.ts @@ -29,6 +29,10 @@ import { } from "../../config/sessions/paths.js"; import { loadSessionEntry } from "../../config/sessions/session-accessor.js"; import { consumeSessionSkillSuggestion } from "../../config/sessions/skill-suggestions.js"; +import { + formatSqliteSessionFileMarker, + sqliteSessionFileMarkerMatchesSession, +} from "../../config/sessions/sqlite-marker.js"; import { resolveSessionStoreEntry } from "../../config/sessions/store.js"; import type { PendingSkillSuggestion, SessionEntry } from "../../config/sessions/types.js"; import { resolveSilentReplySettings } from "../../config/silent-reply.js"; @@ -1002,7 +1006,7 @@ export async function runPreparedReply( skillFilter: opts?.skillFilter, }); }); - sessionEntry = skillResult.sessionEntry ?? sessionEntry; + sessionEntry = skillResult.sessionEntry; if (sessionEntry) { sessionEntryHandle?.replaceCurrent(sessionEntry); } @@ -1110,14 +1114,22 @@ export async function runPreparedReply( sessionId: latestSessionId, storePath, }); + const existingSessionFile = latestSessionEntry?.sessionFile; + const sessionFile = + existingSessionFile && + sqliteSessionFileMarkerMatchesSession(existingSessionFile, latestSessionId) + ? existingSessionFile + : storePath + ? formatSqliteSessionFileMarker({ + agentId, + sessionId: latestSessionId, + storePath, + }) + : resolveSessionFilePath(latestSessionId, latestSessionEntry, sessionFilePathOptions); return { sessionEntry: latestSessionEntry, sessionId: latestSessionId, - sessionFile: resolveSessionFilePath( - latestSessionId, - latestSessionEntry, - sessionFilePathOptions, - ), + sessionFile, }; }; let preparedSessionState = resolvePreparedSessionState(); diff --git a/src/auto-reply/reply/get-reply.auto-fallback.test.ts b/src/auto-reply/reply/get-reply.auto-fallback.test.ts index 8c5b765934af..ce4d4e151d51 100644 --- a/src/auto-reply/reply/get-reply.auto-fallback.test.ts +++ b/src/auto-reply/reply/get-reply.auto-fallback.test.ts @@ -1,10 +1,10 @@ // Tests get-reply behavior while probing an auto-fallback primary model. -import fs from "node:fs"; import path from "node:path"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"; import type { ModelDefinitionConfig, OpenClawConfig } from "../../config/config.js"; import type { SessionEntry } from "../../config/sessions.js"; +import { replaceSessionEntrySync } from "../../config/sessions/session-accessor.js"; import type { ThinkLevel } from "../thinking.js"; import { withFastReplyConfig } from "./get-reply-fast-path.js"; import { @@ -147,11 +147,11 @@ function mockAutoFallbackSession(params: { modelSelectionLocked?: boolean } = {} modelOverrideFallbackOriginModel: "gpt-5.5", modelSelectionLocked: params.modelSelectionLocked, }; - // Reply-turn admission re-reads the store from disk before starting work; - // seed a real per-test store so the guard sees the same session the mocks - // describe instead of depending on leftover host files. + // Reply-turn admission re-reads the canonical SQLite store before starting + // work; seed a real per-test store so the guard sees the same session the + // mocks describe instead of depending on leftover host state. const storePath = path.join(tempDirs.make("auto-fallback-store"), "sessions.json"); - fs.writeFileSync(storePath, JSON.stringify({ [sessionKey]: sessionEntry })); + replaceSessionEntrySync({ storePath, sessionKey }, sessionEntry); mocks.initSessionState.mockResolvedValue( createGetReplySessionState({ sessionKey, diff --git a/src/auto-reply/reply/get-reply.fast-path.test.ts b/src/auto-reply/reply/get-reply.fast-path.test.ts index 627fbc7e6ef3..8b47a16ed44f 100644 --- a/src/auto-reply/reply/get-reply.fast-path.test.ts +++ b/src/auto-reply/reply/get-reply.fast-path.test.ts @@ -5,11 +5,9 @@ import path from "node:path"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { testing as cliBackendsTesting } from "../../agents/cli-backends.js"; import type { OpenClawConfig } from "../../config/config.js"; -import { getSessionEntry } from "../../config/sessions.js"; -import { - readSessionStoreForTest, - writeSessionStoreForTestAsync, -} from "../../config/sessions/test-helpers.js"; +import type { SessionEntry } from "../../config/sessions.js"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import { MODEL_SELECTION_LOCKED_RESET_MESSAGE, ModelSelectionLockedError, @@ -123,11 +121,13 @@ async function seedFastPathSessionStore( storePath: string, entries: Record<string, Record<string, unknown>>, ): Promise<void> { - await writeSessionStoreForTestAsync(storePath, entries); + for (const [sessionKey, entry] of Object.entries(entries)) { + await replaceSessionEntry({ storePath, sessionKey }, entry as SessionEntry); + } } function readFastPathSessionEntry(storePath: string, sessionKey: string): Record<string, unknown> { - return readSessionStoreForTest<Record<string, unknown>>(storePath)[sessionKey] ?? {}; + return (loadSessionEntry({ storePath, sessionKey }) as Record<string, unknown> | undefined) ?? {}; } describe("getReplyFromConfig fast test bootstrap", () => { @@ -704,9 +704,9 @@ describe("getReplyFromConfig fast test bootstrap", () => { expect(onSessionMetadataChanges.mock.invocationCallOrder[0]).toBeLessThan( vi.mocked(runPreparedReplyMock).mock.invocationCallOrder[0], ); - expect(getSessionEntry({ storePath, sessionKey: targetSessionKey })?.goal?.objective).toBe( - "/status", - ); + expect( + (readFastPathSessionEntry(storePath, targetSessionKey) as SessionEntry).goal?.objective, + ).toBe("/status"); const preparedReplyParams = requirePreparedReplyParams(); expect(preparedReplyParams.command.commandBodyNormalized).toBe(continuationPrompt); expect(preparedReplyParams.sessionCtx.BodyForAgent).toBe(continuationPrompt); @@ -714,13 +714,14 @@ describe("getReplyFromConfig fast test bootstrap", () => { }); it("uses native command target session keys during fast bootstrap", () => { + const storePath = "/tmp/sessions.json"; const result = initFastReplySessionState({ ctx: buildGetReplyCtx({ SessionKey: "telegram:slash:123", CommandSource: "native", CommandTargetSessionKey: "agent:main:main", }), - cfg: { session: { store: "/tmp/sessions.json" } } as OpenClawConfig, + cfg: { session: { store: storePath } } as OpenClawConfig, agentId: "main", commandAuthorized: true, workspaceDir: "/tmp/workspace", @@ -728,6 +729,13 @@ describe("getReplyFromConfig fast test bootstrap", () => { expect(result.sessionKey).toBe("agent:main:main"); expect(result.sessionCtx.SessionKey).toBe("agent:main:main"); + expect(result.sessionEntry.sessionFile).toBe( + formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: result.sessionId, + storePath, + }), + ); }); it("preserves usage footer mode during fast reset bootstrap", async () => { @@ -793,6 +801,35 @@ describe("getReplyFromConfig fast test bootstrap", () => { }); }); + it("captures the initial SQLite session entry during fast bootstrap", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-fast-initial-entry-")); + const storePath = path.join(home, "sessions.json"); + const sessionKey = "agent:main:telegram:123"; + await seedFastPathSessionStore(storePath, { + [sessionKey]: { + sessionId: "existing-fast-initial", + updatedAt: Date.now(), + responseUsage: "tokens", + }, + }); + + const result = initFastReplySessionState({ + ctx: buildGetReplyCtx({ + Body: "hello", + RawBody: "hello", + CommandBody: "hello", + SessionKey: sessionKey, + }), + cfg: { session: { store: storePath } } as OpenClawConfig, + agentId: "main", + commandAuthorized: true, + workspaceDir: home, + }); + + expect(result.initialSessionEntry?.sessionId).toBe("existing-fast-initial"); + expect(result.initialSessionEntry?.responseUsage).toBe("tokens"); + expect(result.initialSessionEntry).not.toBe(result.sessionEntry); + }); it("maps explicit gateway origin into command context", () => { const command = buildFastReplyCommandContext({ ctx: buildGetReplyCtx({ diff --git a/src/auto-reply/reply/get-reply.ts b/src/auto-reply/reply/get-reply.ts index de069ad56516..f5191d8b004b 100644 --- a/src/auto-reply/reply/get-reply.ts +++ b/src/auto-reply/reply/get-reply.ts @@ -507,6 +507,9 @@ export async function getReplyFromConfig( ctx: finalized, cfg, commandAuthorized, + ...(internalOptsWithSkillFilter?.expectedExistingSessionId + ? { expectedExistingSessionId: internalOptsWithSkillFilter.expectedExistingSessionId } + : {}), requestedSessionId: internalOptsWithSkillFilter?.requestedSessionId, resumeRequestedSession: internalOptsWithSkillFilter?.resumeRequestedSession, signal: internalOptsWithSkillFilter?.abortSignal, @@ -908,6 +911,7 @@ export async function getReplyFromConfig( cfg, command, sessionKey, + storePath, sessionEntry, previousSessionEntry, workspaceDir, diff --git a/src/auto-reply/reply/get-reply.types.ts b/src/auto-reply/reply/get-reply.types.ts index 7fedc9593642..c461a3912b52 100644 --- a/src/auto-reply/reply/get-reply.types.ts +++ b/src/auto-reply/reply/get-reply.types.ts @@ -12,6 +12,8 @@ export type ReplySessionBinding = { }; type InternalReplySessionOptions = { + expectedExistingSessionId?: string; + onSessionPrepared?: (binding: ReplySessionBinding) => void; requestedSessionId?: string; resumeRequestedSession?: boolean; sessionPromptSourceReplyDeliveryMode?: GetReplyOptions["sourceReplyDeliveryMode"]; diff --git a/src/auto-reply/reply/model-selection.test.ts b/src/auto-reply/reply/model-selection.test.ts index 3c887e724a62..34ab2d01c423 100644 --- a/src/auto-reply/reply/model-selection.test.ts +++ b/src/auto-reply/reply/model-selection.test.ts @@ -13,7 +13,7 @@ import { } from "../../agents/model-catalog.runtime.js"; import type { OpenClawConfig } from "../../config/config.js"; import type { SessionEntry } from "../../config/sessions.js"; -import { loadSessionStore, saveSessionStore } from "../../config/sessions/store.js"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import { MODEL_SELECTION_LOCKED_MESSAGE } from "../../sessions/model-overrides.js"; import { createModelSelectionState, resolveContextTokens } from "./model-selection.js"; @@ -1223,7 +1223,7 @@ describe("createModelSelectionState respects session model override", () => { modelOverride: "gpt-5.5", modelOverrideSource: "user", }); - await saveSessionStore(storePath, { [sessionKey]: concurrentEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey, storePath }, concurrentEntry); const sessionStore = { [sessionKey]: sessionEntry }; try { @@ -1252,7 +1252,7 @@ describe("createModelSelectionState respects session model override", () => { modelOverrideSource: "user", }); expect(sessionStore[sessionKey]).toEqual(sessionEntry); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toEqual(sessionEntry); + expect(loadSessionEntry({ sessionKey, storePath })).toEqual(sessionEntry); } finally { fs.rmSync(tempRoot, { recursive: true, force: true }); } @@ -1284,7 +1284,7 @@ describe("createModelSelectionState respects session model override", () => { modelOverride: "gpt-4o", modelOverrideSource: "user", }); - await saveSessionStore(storePath, { [sessionKey]: rotatedEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey, storePath }, rotatedEntry); const sessionStore = { [sessionKey]: sessionEntry }; try { @@ -1310,7 +1310,7 @@ describe("createModelSelectionState respects session model override", () => { modelOverride: "gpt-4o-mini", }); expect(sessionStore[sessionKey]).toBe(sessionEntry); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toEqual(rotatedEntry); + expect(loadSessionEntry({ sessionKey, storePath })).toEqual(rotatedEntry); } finally { fs.rmSync(tempRoot, { recursive: true, force: true }); } diff --git a/src/auto-reply/reply/queue.collect.test.ts b/src/auto-reply/reply/queue.collect.test.ts index 0429a62122bb..fd440fbeb8cc 100644 --- a/src/auto-reply/reply/queue.collect.test.ts +++ b/src/auto-reply/reply/queue.collect.test.ts @@ -3,7 +3,13 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { + loadTranscriptEvents, + replaceSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; +import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js"; import type { FollowupRun, QueueSettings } from "./queue.js"; import { admitFollowupRunLifecycle, @@ -3032,7 +3038,6 @@ describe("followup queue collect routing", () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-overflow-session-")); const storePath = path.join(tempDir, "sessions.json"); const oldTranscriptPath = path.join(tempDir, "old-session.jsonl"); - const newTranscriptPath = path.join(tempDir, "new-session.jsonl"); const key = `test-overflow-summary-session-rotation-${Date.now()}`; const calls: FollowupRun[] = []; const done = createDeferred<void>(); @@ -3044,15 +3049,12 @@ describe("followup queue collect routing", () => { }; try { - await fs.writeFile( - storePath, - JSON.stringify({ - "agent:agent:main": { - sessionId: "new-session", - sessionFile: newTranscriptPath, - updatedAt: Date.now(), - }, - }), + await replaceSessionEntry( + { storePath, sessionKey: "agent:agent:main" }, + { + sessionId: "new-session", + updatedAt: Date.now(), + }, ); const first = createRun({ prompt: "first" }); first.run.sessionId = "old-session"; @@ -3073,11 +3075,27 @@ describe("followup queue collect routing", () => { const recorder = calls[0]?.userTurnTranscriptRecorder; expect(recorder).toBeDefined(); const persisted = await recorder?.persistFallback(); - expect(await fs.realpath(persisted?.sessionFile ?? "")).toBe( - await fs.realpath(newTranscriptPath), + expect(persisted?.sessionFile).toBe( + formatSqliteSessionFileMarker({ + agentId: "agent", + sessionId: "new-session", + storePath, + }), ); - await expect(fs.readFile(newTranscriptPath, "utf8")).resolves.toContain( - "[Queue overflow] Dropped 1 message due to cap.", + await expect( + loadTranscriptEvents({ + agentId: "agent", + sessionId: "new-session", + sessionKey: "agent:agent:main", + storePath, + }), + ).resolves.toContainEqual( + expect.objectContaining({ + message: expect.objectContaining({ + content: expect.stringContaining("[Queue overflow] Dropped 1 message due to cap."), + }), + type: "message", + }), ); await expect(fs.stat(oldTranscriptPath)).rejects.toMatchObject({ code: "ENOENT" }); } finally { @@ -3782,7 +3800,7 @@ describe("followup queue collect routing", () => { const createRecorder = (text: string, mediaPath: string) => createUserTurnTranscriptRecorder({ input: { text, media: [{ path: mediaPath, contentType: "image/png" }] }, - target: { transcriptPath: "/tmp/session.jsonl" }, + target: createTestUserTurnTranscriptTarget(), updateMode: "none", }); const firstRecorder = createRecorder("first transcript", "/tmp/first.png"); @@ -3886,7 +3904,7 @@ describe("followup queue collect routing", () => { const createRecorder = (text: string) => createUserTurnTranscriptRecorder({ input: { text }, - target: { transcriptPath: "/tmp/session.jsonl" }, + target: createTestUserTurnTranscriptTarget(), updateMode: "none", }); diff --git a/src/auto-reply/reply/queue/drain.ts b/src/auto-reply/reply/queue/drain.ts index d9d29368d439..954652fbadec 100644 --- a/src/auto-reply/reply/queue/drain.ts +++ b/src/auto-reply/reply/queue/drain.ts @@ -321,22 +321,10 @@ function buildCollectTranscriptPrompt(items: FollowupRun[]): string { } function resolveFollowupTranscriptTarget(source: FollowupRun) { - const sessionKey = normalizeOptionalString(source.run.sessionKey); - const storePath = sessionKey - ? resolveStorePath(source.run.config.session?.store, { - agentId: source.run.agentId, - }) - : undefined; - if (!sessionKey || !storePath) { - return { - transcriptPath: source.run.sessionFile, - sessionId: source.run.sessionId, - agentId: source.run.agentId, - sessionKey: source.run.sessionId, - cwd: source.run.cwd ?? source.run.workspaceDir, - config: source.run.config, - }; - } + const sessionKey = normalizeOptionalString(source.run.sessionKey) ?? source.run.sessionId; + const storePath = resolveStorePath(source.run.config.session?.store, { + agentId: source.run.agentId, + }); const sessionEntry = loadSessionEntry({ storePath, sessionKey, diff --git a/src/auto-reply/reply/reply-run-registry.test.ts b/src/auto-reply/reply/reply-run-registry.test.ts index bf42643183c1..4cd1a63790e0 100644 --- a/src/auto-reply/reply/reply-run-registry.test.ts +++ b/src/auto-reply/reply/reply-run-registry.test.ts @@ -17,12 +17,14 @@ import { isReplyRunActiveForSessionId, isReplyRunAbortableForCompaction, isReplyRunAbortableForSignal, + clearReplyRunForResetBySessionId, queueReplyRunMessage, REPLY_RUN_IDLE_SETTLE_TIMEOUT_MS, REPLY_RUN_TERMINAL_SETTLE_TIMEOUT_MS, replyRunRegistry, runAfterReplyOperationClear, resolveActiveReplyRunSessionId, + resolveReplyRunPhaseForSessionId, waitForReplyRunEndBySessionId, } from "./reply-run-registry.js"; import { admitReplyTurn } from "./reply-turn-admission.js"; @@ -78,12 +80,17 @@ describe("reply run registry", () => { expect(isReplyRunActiveForSessionId("session-compact")).toBe(true); expect(isReplyRunAbortableForCompaction("session-compact")).toBe(false); + operation.markWaitingForDeferredMaintenance(); + + expect(isReplyRunAbortableForCompaction("session-compact")).toBe(false); + + operation.markDeferredMaintenanceWaitEnded(); operation.setPhase("running"); expect(isReplyRunAbortableForCompaction("session-compact")).toBe(true); }); - it("mirrors active reply operations into diagnostic work state", () => { + it("records reply-operation progress without claiming embedded-run activity", () => { const operation = createReplyOperation({ sessionKey: "agent:main:telegram:direct:chat-1", sessionId: "session-1", @@ -94,8 +101,11 @@ describe("reply run registry", () => { getDiagnosticSessionActivitySnapshot({ sessionId: "session-1", sessionKey: "agent:main:telegram:direct:chat-1", - }).activeWorkKind, - ).toBe("embedded_run"); + }), + ).toMatchObject({ + activeWorkKind: undefined, + lastProgressReason: "reply_operation:queued", + }); operation.updateSessionId("session-2"); @@ -103,8 +113,11 @@ describe("reply run registry", () => { getDiagnosticSessionActivitySnapshot({ sessionId: "session-2", sessionKey: "agent:main:telegram:direct:chat-1", - }).activeWorkKind, - ).toBe("embedded_run"); + }), + ).toMatchObject({ + activeWorkKind: undefined, + lastProgressReason: "reply_operation:session_updated", + }); operation.complete(); @@ -112,8 +125,77 @@ describe("reply run registry", () => { getDiagnosticSessionActivitySnapshot({ sessionId: "session-2", sessionKey: "agent:main:telegram:direct:chat-1", - }).activeWorkKind, - ).toBeUndefined(); + }), + ).toMatchObject({ + activeWorkKind: undefined, + lastProgressReason: "reply_operation:ended", + }); + }); + + it("tracks deferred-maintenance wait as a reply-operation phase", () => { + const operation = createReplyOperation({ + sessionKey: "agent:main:telegram:direct:chat-1", + sessionId: "session-wait", + resetTriggered: false, + }); + + operation.markWaitingForDeferredMaintenance(); + + expect(operation.phase).toBe("waiting_for_deferred_maintenance"); + expect(resolveReplyRunPhaseForSessionId("session-wait")).toBe( + "waiting_for_deferred_maintenance", + ); + expect( + getDiagnosticSessionActivitySnapshot({ + sessionId: "session-wait", + sessionKey: "agent:main:telegram:direct:chat-1", + }), + ).toMatchObject({ + activeWorkKind: undefined, + lastProgressReason: "deferred_maintenance:waiting", + }); + + operation.markDeferredMaintenanceWaitEnded(); + + expect(operation.phase).toBe("queued"); + expect( + getDiagnosticSessionActivitySnapshot({ + sessionId: "session-wait", + sessionKey: "agent:main:telegram:direct:chat-1", + }), + ).toMatchObject({ + activeWorkKind: undefined, + lastProgressReason: "deferred_maintenance:wait_ended", + }); + }); + + it("clears deferred-maintenance operations immediately on user abort", () => { + const operation = createReplyOperation({ + sessionKey: "agent:main:main", + sessionId: "session-waiting-abort", + resetTriggered: false, + }); + + operation.markWaitingForDeferredMaintenance(); + operation.abortByUser(); + + expect(operation.result).toEqual({ kind: "aborted", code: "aborted_by_user" }); + expect(replyRunRegistry.isActive("agent:main:main")).toBe(false); + expect(isReplyRunActiveForSessionId("session-waiting-abort")).toBe(false); + }); + + it("does not reset deferred-maintenance operations as backend-owned work", () => { + const operation = createReplyOperation({ + sessionKey: "agent:main:main", + sessionId: "session-waiting-reset", + resetTriggered: false, + }); + + operation.markWaitingForDeferredMaintenance(); + clearReplyRunForResetBySessionId("session-waiting-reset"); + + expect(operation.result).toBeNull(); + expect(replyRunRegistry.isActive("agent:main:main")).toBe(true); }); it("clears queued operations immediately on user abort", () => { diff --git a/src/auto-reply/reply/reply-run-registry.ts b/src/auto-reply/reply/reply-run-registry.ts index bf076450110e..50cddeaa11b0 100644 --- a/src/auto-reply/reply/reply-run-registry.ts +++ b/src/auto-reply/reply/reply-run-registry.ts @@ -7,8 +7,7 @@ import { import { createAbortError } from "../../infra/abort-signal.js"; import { getDiagnosticSessionActivitySnapshot, - markDiagnosticEmbeddedRunEnded, - markDiagnosticEmbeddedRunStarted, + markDiagnosticRunProgress, resolveRunStaleThresholdMs, } from "../../logging/diagnostic-run-activity.js"; import { diagnosticLogger as diag } from "../../logging/diagnostic-runtime.js"; @@ -83,6 +82,7 @@ export function resolveReplyBackendQueueMessageMismatch( export type ReplyOperationPhase = | "queued" + | "waiting_for_deferred_maintenance" | "preflight_compacting" | "memory_flushing" | "running" @@ -130,7 +130,18 @@ export type ReplyOperation = { /** True when this operation has owned the supplied session ID. */ hasOwnedSessionId(sessionId: string): boolean; recordActivity(): void; - setPhase(next: "queued" | "preflight_compacting" | "memory_flushing" | "running"): void; + setPhase( + next: + | "queued" + | "waiting_for_deferred_maintenance" + | "preflight_compacting" + | "memory_flushing" + | "running", + ): void; + /** Mark this operation as waiting on prior same-session maintenance. */ + markWaitingForDeferredMaintenance(): void; + /** Return a maintenance-waiting operation to queued if the run has not started. */ + markDeferredMaintenanceWaitEnded(): void; /** Mark this operation as an in-flight terminal-session recovery. */ markTerminalRecovery(): void; markAcceptedSteeredInboundAudio(): void; @@ -296,6 +307,10 @@ function isReplyRunCompacting(operation: ReplyOperation): boolean { return backend?.isCompacting?.() ?? false; } +function isReplyOperationPreBackendPhase(phase: ReplyOperationPhase): boolean { + return phase === "queued" || phase === "waiting_for_deferred_maintenance"; +} + const attachedBackendByOperation = new WeakMap<ReplyOperation, ReplyBackendHandle>(); const abortFrozenOperations = new WeakSet<ReplyOperation>(); const operationsByUpstreamAbortSignal = new WeakMap<AbortSignal, ReplyOperation>(); @@ -481,27 +496,15 @@ function clearReplyRunState(params: { sessionKey: string; sessionId: string }): notifyReplyRunEnded(params.sessionKey); } -function replyRunDiagnosticWorkKey(sessionKey: string): string { - return `reply:${sessionKey}`; -} - -function markReplyRunDiagnosticWorkStarted(params: { +function markReplyRunDiagnosticProgress(params: { sessionKey: string; sessionId: string; + reason: string; }): void { - markDiagnosticEmbeddedRunStarted({ + markDiagnosticRunProgress({ sessionId: params.sessionId, sessionKey: params.sessionKey, - workKey: replyRunDiagnosticWorkKey(params.sessionKey), - }); -} - -function markReplyRunDiagnosticWorkEnded(params: { sessionKey: string; sessionId: string }): void { - markDiagnosticEmbeddedRunEnded({ - sessionId: params.sessionId, - sessionKey: params.sessionKey, - workKey: replyRunDiagnosticWorkKey(params.sessionKey), - clearRunActivity: false, + reason: params.reason, }); } @@ -579,7 +582,11 @@ export function createReplyOperation(params: { ) : undefined; updateFollowupAdmissionSessionId(sessionKey, currentSessionId); - markReplyRunDiagnosticWorkEnded({ sessionKey, sessionId: currentSessionId }); + markReplyRunDiagnosticProgress({ + sessionKey, + sessionId: currentSessionId, + reason: "reply_operation:ended", + }); clearReplyRunState({ sessionKey, sessionId: currentSessionId, @@ -686,6 +693,28 @@ export function createReplyOperation(params: { recordActivity(); phase = next; }, + markWaitingForDeferredMaintenance() { + if (result || phase !== "queued") { + return; + } + phase = "waiting_for_deferred_maintenance"; + markReplyRunDiagnosticProgress({ + sessionKey, + sessionId: currentSessionId, + reason: "deferred_maintenance:waiting", + }); + }, + markDeferredMaintenanceWaitEnded() { + if (result || phase !== "waiting_for_deferred_maintenance") { + return; + } + phase = "queued"; + markReplyRunDiagnosticProgress({ + sessionKey, + sessionId: currentSessionId, + reason: "deferred_maintenance:wait_ended", + }); + }, markTerminalRecovery() { terminalRecovery = true; }, @@ -717,7 +746,11 @@ export function createReplyOperation(params: { replyRunState.activeSessionIdsByKey.set(sessionKey, currentSessionId); replyRunState.activeKeysBySessionId.set(currentSessionId, sessionKey); registerWaitSessionId(sessionKey, currentSessionId); - markReplyRunDiagnosticWorkStarted({ sessionKey, sessionId: currentSessionId }); + markReplyRunDiagnosticProgress({ + sessionKey, + sessionId: currentSessionId, + reason: "reply_operation:session_updated", + }); }, attachBackend(handle) { if (result) { @@ -787,7 +820,10 @@ export function createReplyOperation(params: { abortWithReason("user_abort", createUserAbortError(), { abortedCode: "aborted_by_user", }); - if (phaseBeforeAbort === "queued" && !retainStateUntilCompleteOperations.has(operation)) { + if ( + isReplyOperationPreBackendPhase(phaseBeforeAbort) && + !retainStateUntilCompleteOperations.has(operation) + ) { clearState(); } else { scheduleTerminalSettle(); @@ -802,7 +838,10 @@ export function createReplyOperation(params: { abortWithReason("restart", createAgentRunRestartAbortError(), { abortedCode: "aborted_for_restart", }); - if (phaseBeforeAbort === "queued" && !retainStateUntilCompleteOperations.has(operation)) { + if ( + isReplyOperationPreBackendPhase(phaseBeforeAbort) && + !retainStateUntilCompleteOperations.has(operation) + ) { clearState(); } else { scheduleTerminalSettle(); @@ -839,7 +878,11 @@ export function createReplyOperation(params: { replyRunState.activeSessionIdsByKey.set(sessionKey, currentSessionId); replyRunState.activeKeysBySessionId.set(currentSessionId, sessionKey); registerWaitSessionId(sessionKey, currentSessionId); - markReplyRunDiagnosticWorkStarted({ sessionKey, sessionId: currentSessionId }); + markReplyRunDiagnosticProgress({ + sessionKey, + sessionId: currentSessionId, + reason: "reply_operation:queued", + }); if (upstreamAbortSignal) { operationsByUpstreamAbortSignal.set(upstreamAbortSignal, operation); const abortFromUpstream = () => { @@ -851,7 +894,10 @@ export function createReplyOperation(params: { abortWithReason(restart ? "restart" : "user_abort", upstreamAbortSignal.reason, { abortedCode: restart ? "aborted_for_restart" : "aborted_by_user", }); - if (phaseBeforeAbort === "queued" && !retainStateUntilCompleteOperations.has(operation)) { + if ( + isReplyOperationPreBackendPhase(phaseBeforeAbort) && + !retainStateUntilCompleteOperations.has(operation) + ) { clearState(); } else { scheduleTerminalSettle(); @@ -1002,11 +1048,17 @@ export function isReplyRunActiveForSessionId(sessionId: string): boolean { return resolveReplyRunForCurrentSessionId(sessionId) !== undefined; } +export function resolveReplyRunPhaseForSessionId( + sessionId: string, +): ReplyOperationPhase | undefined { + return resolveReplyRunForCurrentSessionId(sessionId)?.phase; +} + export function isReplyRunAbortableForCompaction(sessionId: string): boolean { const operation = resolveReplyRunForCurrentSessionId(sessionId); // Manual compaction uses this as a coordination gate: a finalizing run still // needs to drain even when its frozen outcome rejects the abort itself. - return Boolean(operation && operation.phase !== "queued"); + return Boolean(operation && !isReplyOperationPreBackendPhase(operation.phase)); } export function isReplyRunStreamingForSessionId(sessionId: string): boolean { @@ -1067,7 +1119,7 @@ export function forceClearReplyRunBySessionId(sessionId: string, cause?: unknown export function clearReplyRunForResetBySessionId(sessionId: string): void { const operation = resolveReplyRunForCurrentSessionId(sessionId); - if (!operation || operation.phase === "queued") { + if (!operation || isReplyOperationPreBackendPhase(operation.phase)) { return; } operation.abortForRestart(); @@ -1181,7 +1233,11 @@ export function listActiveReplyRunSessionKeys(): string[] { export const testing = { resetReplyRunRegistry(): void { for (const [sessionKey, sessionId] of replyRunState.activeSessionIdsByKey) { - markReplyRunDiagnosticWorkEnded({ sessionKey, sessionId }); + markReplyRunDiagnosticProgress({ + sessionKey, + sessionId, + reason: "reply_operation:registry_reset", + }); } replyRunState.activeRunsByKey.clear(); replyRunState.activeSessionIdsByKey.clear(); diff --git a/src/auto-reply/reply/reply-state.test.ts b/src/auto-reply/reply/reply-state.test.ts index eb4ec4f88a9f..fecd2a1b0730 100644 --- a/src/auto-reply/reply/reply-state.test.ts +++ b/src/auto-reply/reply/reply-state.test.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { SessionEntry } from "../../config/sessions.js"; +import { loadSessionEntry, upsertSessionEntry } from "../../config/sessions/session-accessor.js"; import { appendHistoryEntry, buildHistoryContext, @@ -38,13 +39,20 @@ async function seedSessionStore(params: { entry: Record<string, unknown>; }) { await fs.mkdir(path.dirname(params.storePath), { recursive: true }); - await fs.writeFile( - params.storePath, - JSON.stringify({ [params.sessionKey]: params.entry }, null, 2), - "utf-8", + await upsertSessionEntry( + { storePath: params.storePath, sessionKey: params.sessionKey }, + params.entry as Partial<SessionEntry>, ); } +async function loadStoredEntry(storePath: string, sessionKey: string): Promise<SessionEntry> { + const entry = loadSessionEntry({ storePath, sessionKey, readConsistency: "latest" }); + if (!entry) { + throw new Error(`expected persisted session entry for ${sessionKey}`); + } + return entry; +} + async function createCompactionSessionFixture(entry: SessionEntry) { const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-compact-")); tempDirs.push(tmp); @@ -79,7 +87,9 @@ async function rotateCompactionSessionFile(params: { storePath, newSessionId: params.newSessionId, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored: Record<string, SessionEntry> = { + [sessionKey]: await loadStoredEntry(storePath, sessionKey), + }; const expectedDir = await fs.realpath(tmp); return { stored, sessionKey, expectedDir }; } @@ -499,7 +509,7 @@ describe("incrementCompactionCount", () => { }); expect(count).toBe(3); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = { [sessionKey]: await loadStoredEntry(storePath, sessionKey) }; expect(stored[sessionKey].compactionCount).toBe(3); }); @@ -522,7 +532,7 @@ describe("incrementCompactionCount", () => { tokensAfter: 12_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = { [sessionKey]: await loadStoredEntry(storePath, sessionKey) }; expect(stored[sessionKey].compactionCount).toBe(1); expect(stored[sessionKey].totalTokens).toBe(12_000); // input/output cleared since we only have the total estimate @@ -550,7 +560,7 @@ describe("incrementCompactionCount", () => { tokensAfter: 0, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = { [sessionKey]: await loadStoredEntry(storePath, sessionKey) }; expect(stored[sessionKey].compactionCount).toBe(1); expect(stored[sessionKey].totalTokens).toBe(0); expect(stored[sessionKey].totalTokensFresh).toBe(true); @@ -581,7 +591,7 @@ describe("incrementCompactionCount", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = { [sessionKey]: await loadStoredEntry(storePath, sessionKey) }; expect(stored[sessionKey].totalTokens).toBe(12_000); expect(stored[sessionKey].totalTokensFresh).toBe(true); }); @@ -609,7 +619,7 @@ describe("incrementCompactionCount", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = { [sessionKey]: await loadStoredEntry(storePath, sessionKey) }; expect(stored[sessionKey].totalTokens).toBe(0); expect(stored[sessionKey].totalTokensFresh).toBe(true); }); @@ -637,7 +647,7 @@ describe("incrementCompactionCount", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = { [sessionKey]: await loadStoredEntry(storePath, sessionKey) }; expect(stored[sessionKey].totalTokens).toBe(90_000); expect(stored[sessionKey].totalTokensFresh).toBe(true); }); @@ -660,7 +670,7 @@ describe("incrementCompactionCount", () => { tokensAfter: Number.POSITIVE_INFINITY, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = { [sessionKey]: await loadStoredEntry(storePath, sessionKey) }; expect(stored[sessionKey].compactionCount).toBe(1); expect(stored[sessionKey].totalTokens).toBe(180_000); expect(stored[sessionKey].totalTokensFresh).toBe(false); @@ -711,7 +721,7 @@ describe("incrementCompactionCount", () => { }); expect(count).toBe(4); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = { [sessionKey]: await loadStoredEntry(storePath, sessionKey) }; expect(stored[sessionKey].compactionCount).toBe(4); }); @@ -732,7 +742,7 @@ describe("incrementCompactionCount", () => { newSessionId: "new-session-id", }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = { [sessionKey]: await loadStoredEntry(storePath, sessionKey) }; const expectedSessionDir = await fs.realpath(path.dirname(storePath)); expect(stored[sessionKey].sessionId).toBe("new-session-id"); expect(stored[sessionKey].sessionFile).toBe( @@ -758,7 +768,7 @@ describe("incrementCompactionCount", () => { newSessionId: "same-id", }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = { [sessionKey]: await loadStoredEntry(storePath, sessionKey) }; expect(stored[sessionKey].sessionId).toBe("same-id"); expect(stored[sessionKey].sessionFile).toBe("same-id.jsonl"); expect(stored[sessionKey].compactionCount).toBe(1); @@ -783,7 +793,7 @@ describe("incrementCompactionCount", () => { newSessionFile: rotatedSessionFile, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = { [sessionKey]: await loadStoredEntry(storePath, sessionKey) }; expect(stored[sessionKey].sessionId).toBe("same-id"); expect(stored[sessionKey].sessionFile).toBe(rotatedSessionFile); expect(stored[sessionKey].compactionCount).toBe(1); @@ -806,7 +816,7 @@ describe("incrementCompactionCount", () => { storePath, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = { [sessionKey]: await loadStoredEntry(storePath, sessionKey) }; expect(stored[sessionKey].compactionCount).toBe(1); // totalTokens unchanged expect(stored[sessionKey].totalTokens).toBe(180_000); diff --git a/src/auto-reply/reply/reply-turn-admission.test.ts b/src/auto-reply/reply/reply-turn-admission.test.ts index d9defb1e9eb5..30c9fb495271 100644 --- a/src/auto-reply/reply/reply-turn-admission.test.ts +++ b/src/auto-reply/reply/reply-turn-admission.test.ts @@ -1,8 +1,13 @@ // Tests reply turn admission decisions for active, queued, and aborted runs. -import fs from "node:fs"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"; +import { + deleteSessionEntryLifecycle, + replaceSessionEntry, + replaceSessionEntrySync, +} from "../../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../../config/sessions/types.js"; import { markDiagnosticToolStartedForTest, resetDiagnosticRunActivityForTest, @@ -35,8 +40,12 @@ function createDeferred() { function createSessionStore(entries: Record<string, object>): string { const root = tempDirs.make("openclaw-reply-admission-"); + // The store handle stays a sessions.json path; the sqlite-backed accessor + // resolves it to the per-agent DB, so fixtures must seed through the accessor. const storePath = path.join(root, "sessions.json"); - fs.writeFileSync(storePath, JSON.stringify(entries)); + for (const [sessionKey, entry] of Object.entries(entries)) { + replaceSessionEntrySync({ sessionKey, storePath }, entry as SessionEntry); + } return storePath; } @@ -60,12 +69,11 @@ describe("reply turn admission", () => { run: async () => { mutationStarted.resolve(); await releaseMutation.promise; - fs.writeFileSync( - storePath, - JSON.stringify({ - [sessionKey]: { sessionId, updatedAt: Date.now(), archivedAt: Date.now() }, - }), - ); + await replaceSessionEntry({ sessionKey, storePath }, { + sessionId, + updatedAt: Date.now(), + archivedAt: Date.now(), + } as SessionEntry); }, }); await mutationStarted.promise; @@ -99,7 +107,11 @@ describe("reply turn admission", () => { run: async () => { mutationStarted.resolve(); await releaseMutation.promise; - fs.writeFileSync(storePath, JSON.stringify({})); + await deleteSessionEntryLifecycle({ + storePath, + archiveTranscript: false, + target: { canonicalKey: sessionKey, storeKeys: [sessionKey] }, + }); }, }); await mutationStarted.promise; @@ -133,12 +145,10 @@ describe("reply turn admission", () => { run: async () => { mutationStarted.resolve(); await releaseMutation.promise; - fs.writeFileSync( - storePath, - JSON.stringify({ - [sessionKey]: { sessionId: nextSessionId, updatedAt: Date.now() }, - }), - ); + await replaceSessionEntry({ sessionKey, storePath }, { + sessionId: nextSessionId, + updatedAt: Date.now(), + } as SessionEntry); }, }); await mutationStarted.promise; @@ -176,12 +186,10 @@ describe("reply turn admission", () => { run: async () => { mutationStarted.resolve(); await releaseMutation.promise; - fs.writeFileSync( - storePath, - JSON.stringify({ - [sessionKey]: { sessionId: nextSessionId, updatedAt: Date.now() }, - }), - ); + await replaceSessionEntry({ sessionKey, storePath }, { + sessionId: nextSessionId, + updatedAt: Date.now(), + } as SessionEntry); }, }); await mutationStarted.promise; @@ -216,12 +224,10 @@ describe("reply turn admission", () => { mutationStarted.resolve(); await releaseMutation.promise; abortController.abort(); - fs.writeFileSync( - storePath, - JSON.stringify({ - [sessionKey]: { sessionId: "session-after-reset", updatedAt: Date.now() }, - }), - ); + await replaceSessionEntry({ sessionKey, storePath }, { + sessionId: "session-after-reset", + updatedAt: Date.now(), + } as SessionEntry); }, }); await mutationStarted.promise; @@ -735,12 +741,10 @@ describe("reply turn admission", () => { await new Promise<void>((resolve) => { setImmediate(resolve); }); - fs.writeFileSync( - storePath, - JSON.stringify({ - [sessionKey]: { sessionId: nextSessionId, updatedAt: Date.now() }, - }), - ); + await replaceSessionEntry({ sessionKey, storePath }, { + sessionId: nextSessionId, + updatedAt: Date.now(), + } as SessionEntry); active.updateSessionId(nextSessionId); active.complete(); const result = await admitted; @@ -766,12 +770,10 @@ describe("reply turn admission", () => { }); active.setPhase("preflight_compacting"); active.updateSessionId(nextSessionId); - fs.writeFileSync( - storePath, - JSON.stringify({ - [sessionKey]: { sessionId: nextSessionId, updatedAt: Date.now() }, - }), - ); + await replaceSessionEntry({ sessionKey, storePath }, { + sessionId: nextSessionId, + updatedAt: Date.now(), + } as SessionEntry); active.complete(); const result = await admitReplyTurn({ @@ -805,12 +807,10 @@ describe("reply turn admission", () => { }); active.setPhase("preflight_compacting"); active.updateSessionId(nextSessionId); - fs.writeFileSync( - storePath, - JSON.stringify({ - [sessionKey]: { sessionId: nextSessionId, updatedAt: Date.now() }, - }), - ); + await replaceSessionEntry({ sessionKey, storePath }, { + sessionId: nextSessionId, + updatedAt: Date.now(), + } as SessionEntry); const admitted = admitReplyTurn({ sessionKey, @@ -884,12 +884,10 @@ describe("reply turn admission", () => { }); active.setPhase("preflight_compacting"); active.updateSessionId(nextSessionId); - fs.writeFileSync( - storePath, - JSON.stringify({ - [sessionKey]: { sessionId: nextSessionId, updatedAt: Date.now() }, - }), - ); + await replaceSessionEntry({ sessionKey, storePath }, { + sessionId: nextSessionId, + updatedAt: Date.now(), + } as SessionEntry); finish(active); const result = await admitReplyTurn({ diff --git a/src/auto-reply/reply/session-entry-handle.ts b/src/auto-reply/reply/session-entry-handle.ts index adfa5e500ea0..526a3e7ce5db 100644 --- a/src/auto-reply/reply/session-entry-handle.ts +++ b/src/auto-reply/reply/session-entry-handle.ts @@ -2,6 +2,7 @@ import type { SessionEntry } from "../../config/sessions.js"; export type ReplySessionEntryHandle = { + clearCurrent(): void; get(sessionKey: string): SessionEntry | undefined; getCurrent(): SessionEntry | undefined; patchCurrent(patch: Partial<SessionEntry>): SessionEntry | undefined; @@ -16,13 +17,20 @@ export function createReplySessionEntryHandle(params: { sessionStore?: Record<string, SessionEntry>; }): ReplySessionEntryHandle { const entries = params.sessionStore ?? { [params.sessionKey]: params.sessionEntry }; - let currentEntry = params.sessionEntry; + let currentEntry: SessionEntry | undefined = params.sessionEntry; entries[params.sessionKey] = currentEntry; return { + clearCurrent: () => { + currentEntry = undefined; + delete entries[params.sessionKey]; + }, get: (sessionKey) => entries[sessionKey], getCurrent: () => currentEntry, patchCurrent: (patch) => { + if (!currentEntry) { + return undefined; + } currentEntry = { ...currentEntry, ...patch }; entries[params.sessionKey] = currentEntry; return currentEntry; diff --git a/src/auto-reply/reply/session-entry-persistence.test.ts b/src/auto-reply/reply/session-entry-persistence.test.ts index 9ba7ecab5ce4..b1367562f751 100644 --- a/src/auto-reply/reply/session-entry-persistence.test.ts +++ b/src/auto-reply/reply/session-entry-persistence.test.ts @@ -1,11 +1,8 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"; -import { - clearSessionStoreCacheForTest, - loadSessionStore, - saveSessionStore, -} from "../../config/sessions/store.js"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; +import { clearSessionStoreCacheForTest } from "../../config/sessions/store.js"; import type { SessionEntry } from "../../config/sessions/types.js"; import { persistReplySessionEntry } from "./session-entry-persistence.js"; @@ -30,7 +27,7 @@ describe("persistReplySessionEntry", () => { thinkingLevel: "low", sendPolicy: "deny", }; - await saveSessionStore(storePath, { main: currentEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey: "main", storePath }, currentEntry); const result = await persistReplySessionEntry({ storePath, @@ -55,7 +52,9 @@ describe("persistReplySessionEntry", () => { }); expect(result.entry.elevatedLevel).toBeUndefined(); expect(result.entry.inheritedToolAllow).toBeUndefined(); - expect(loadSessionStore(storePath, { skipCache: true }).main).toEqual(result.entry); + expect( + loadSessionEntry({ sessionKey: "main", storePath, readConsistency: "latest" }), + ).toEqual(result.entry); } finally { clearSessionStoreCacheForTest(); } @@ -75,7 +74,7 @@ describe("persistReplySessionEntry", () => { updatedAt: 400, thinkingLevel: "medium", }; - await saveSessionStore(storePath, { main: currentEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey: "main", storePath }, currentEntry); const result = await persistReplySessionEntry({ storePath, @@ -89,7 +88,9 @@ describe("persistReplySessionEntry", () => { error: 'Session "main" changed while starting work. Retry.', entry: currentEntry, }); - expect(loadSessionStore(storePath, { skipCache: true }).main).toEqual(currentEntry); + expect( + loadSessionEntry({ sessionKey: "main", storePath, readConsistency: "latest" }), + ).toEqual(currentEntry); } finally { clearSessionStoreCacheForTest(); } @@ -103,8 +104,6 @@ describe("persistReplySessionEntry", () => { sessionId: "session-1", updatedAt: 100, }; - await saveSessionStore(storePath, {}, { skipMaintenance: true }); - const result = await persistReplySessionEntry({ storePath, sessionKey: "main", @@ -116,7 +115,9 @@ describe("persistReplySessionEntry", () => { status: "lifecycle-invalidated", error: 'Session "main" was deleted while starting work. Retry.', }); - expect(loadSessionStore(storePath, { skipCache: true }).main).toBeUndefined(); + expect( + loadSessionEntry({ sessionKey: "main", storePath, readConsistency: "latest" }), + ).toBeUndefined(); } finally { clearSessionStoreCacheForTest(); } @@ -136,7 +137,7 @@ describe("persistReplySessionEntry", () => { updatedAt: 400, archivedAt: 300, }; - await saveSessionStore(storePath, { main: archivedEntry }, { skipMaintenance: true }); + await replaceSessionEntry({ sessionKey: "main", storePath }, archivedEntry); const result = await persistReplySessionEntry({ storePath, @@ -151,7 +152,9 @@ describe("persistReplySessionEntry", () => { error: 'Session "main" is archived. Restore it before starting new work.', entry: archivedEntry, }); - expect(loadSessionStore(storePath, { skipCache: true }).main).toEqual(archivedEntry); + expect( + loadSessionEntry({ sessionKey: "main", storePath, readConsistency: "latest" }), + ).toEqual(archivedEntry); } finally { clearSessionStoreCacheForTest(); } diff --git a/src/auto-reply/reply/session-fork.runtime.test.ts b/src/auto-reply/reply/session-fork.runtime.test.ts deleted file mode 100644 index 055805689c79..000000000000 --- a/src/auto-reply/reply/session-fork.runtime.test.ts +++ /dev/null @@ -1,431 +0,0 @@ -// Tests session fork runtime behavior and copied session artifacts. -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; -import type { SessionEntry } from "../../config/sessions/types.js"; -import { - forkSessionEntryFromParent, - forkSessionFromParent, - MODEL_SELECTION_LOCKED_PARENT_FORK_MESSAGE, - resolveParentForkDecision, -} from "./session-fork.js"; -import { resolveParentForkTokenCountRuntime } from "./session-fork.runtime.js"; - -const roots: string[] = []; - -async function makeRoot(prefix: string): Promise<string> { - const root = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); - roots.push(root); - return root; -} - -afterEach(async () => { - await Promise.all(roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); -}); - -describe("parent fork policy", () => { - it("rejects model-selection-locked parent context", async () => { - const parentEntry: SessionEntry = { - sessionId: "locked-parent", - modelSelectionLocked: true, - updatedAt: 1, - }; - await expect( - resolveParentForkDecision({ - parentEntry, - storePath: "/tmp/unused-sessions.json", - }), - ).rejects.toThrow(MODEL_SELECTION_LOCKED_PARENT_FORK_MESSAGE); - await expect( - forkSessionFromParent({ - agentId: "main", - parentEntry, - parentSessionKey: "agent:main:main", - sessionKey: "agent:main:subagent:child", - storePath: "/tmp/unused-sessions.json", - }), - ).rejects.toThrow(MODEL_SELECTION_LOCKED_PARENT_FORK_MESSAGE); - }); - - it("rejects a newer locked parent alias shadowed by a stale canonical row", async () => { - const root = await makeRoot("openclaw-parent-fork-locked-alias-"); - const storePath = path.join(root, "sessions.json"); - await fs.writeFile( - storePath, - JSON.stringify({ - "agent:main:main": { - sessionId: "stale-canonical-parent", - updatedAt: 1, - }, - main: { - sessionId: "fresh-locked-parent", - modelSelectionLocked: true, - updatedAt: 2, - }, - }), - "utf-8", - ); - - await expect( - forkSessionEntryFromParent({ - agentId: "main", - fallbackEntry: { sessionId: "", updatedAt: 3 }, - parentSessionKey: "agent:main:main", - parentStoreKeys: ["agent:main:main", "main"], - sessionKey: "agent:main:subagent:child", - storePath, - }), - ).rejects.toThrow(MODEL_SELECTION_LOCKED_PARENT_FORK_MESSAGE); - - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record<string, unknown>; - expect(stored["agent:main:subagent:child"]).toBeUndefined(); - }); -}); - -describe("resolveParentForkTokenCountRuntime", () => { - it("falls back to recent transcript usage when cached totals are stale", async () => { - const root = await makeRoot("openclaw-parent-fork-token-estimate-"); - const sessionsDir = path.join(root, "sessions"); - await fs.mkdir(sessionsDir); - - const sessionId = "parent-overflow-transcript"; - const sessionFile = path.join(sessionsDir, "parent.jsonl"); - const lines = [ - JSON.stringify({ - type: "session", - version: 3, - id: sessionId, - timestamp: new Date().toISOString(), - cwd: process.cwd(), - }), - ]; - for (let index = 0; index < 40; index += 1) { - const body = `turn-${index} ${"x".repeat(200)}`; - lines.push( - JSON.stringify({ - type: "message", - id: `u${index}`, - parentId: index === 0 ? null : `a${index - 1}`, - timestamp: new Date().toISOString(), - message: { role: "user", content: body }, - }), - JSON.stringify({ - type: "message", - id: `a${index}`, - parentId: `u${index}`, - timestamp: new Date().toISOString(), - message: { - role: "assistant", - content: body, - usage: index === 39 ? { input: 90_000, output: 20_000 } : undefined, - }, - }), - ); - } - await fs.writeFile(sessionFile, `${lines.join("\n")}\n`, "utf-8"); - - const entry: SessionEntry = { - sessionId, - sessionFile, - updatedAt: Date.now(), - totalTokens: 1, - totalTokensFresh: false, - }; - - const tokens = await resolveParentForkTokenCountRuntime({ - parentEntry: entry, - storePath: path.join(root, "sessions.json"), - }); - - expect(tokens).toBe(110_000); - }); - - it("falls back to a conservative byte estimate when stale parent transcript has no usage", async () => { - const root = await makeRoot("openclaw-parent-fork-byte-estimate-"); - const sessionsDir = path.join(root, "sessions"); - await fs.mkdir(sessionsDir); - - const sessionId = "parent-no-usage-transcript"; - const sessionFile = path.join(sessionsDir, "parent.jsonl"); - const lines = [ - JSON.stringify({ - type: "session", - version: 3, - id: sessionId, - timestamp: new Date().toISOString(), - cwd: process.cwd(), - }), - ]; - for (let index = 0; index < 24; index += 1) { - lines.push( - JSON.stringify({ - type: "message", - id: `u${index}`, - parentId: index === 0 ? null : `a${index - 1}`, - timestamp: new Date().toISOString(), - message: { role: "user", content: `turn-${index} ${"x".repeat(24_000)}` }, - }), - ); - } - await fs.writeFile(sessionFile, `${lines.join("\n")}\n`, "utf-8"); - - const entry: SessionEntry = { - sessionId, - sessionFile, - updatedAt: Date.now(), - totalTokensFresh: false, - }; - - const tokens = await resolveParentForkTokenCountRuntime({ - parentEntry: entry, - storePath: path.join(root, "sessions.json"), - }); - - expect(tokens).toBeGreaterThan(100_000); - }); - - it("uses the latest usage snapshot instead of tail aggregates for parent fork checks", async () => { - const root = await makeRoot("openclaw-parent-fork-latest-usage-"); - const sessionsDir = path.join(root, "sessions"); - await fs.mkdir(sessionsDir); - - const sessionId = "parent-multiple-usage-transcript"; - const sessionFile = path.join(sessionsDir, "parent.jsonl"); - await fs.writeFile( - sessionFile, - [ - JSON.stringify({ - type: "session", - version: 3, - id: sessionId, - timestamp: new Date().toISOString(), - cwd: process.cwd(), - }), - JSON.stringify({ - message: { - role: "assistant", - content: "older", - usage: { input: 60_000, output: 5_000 }, - }, - }), - JSON.stringify({ - type: "message", - id: "active-usage", - parentId: null, - message: { - role: "assistant", - content: "latest", - usage: { input: 70_000, output: 8_000 }, - }, - }), - ].join("\n"), - "utf-8", - ); - - const entry: SessionEntry = { - sessionId, - sessionFile, - updatedAt: Date.now(), - totalTokensFresh: false, - }; - - const tokens = await resolveParentForkTokenCountRuntime({ - parentEntry: entry, - storePath: path.join(root, "sessions.json"), - }); - - expect(tokens).toBe(78_000); - }); - - it("does not reconstruct parent context from billing buckets when context is unavailable", async () => { - const root = await makeRoot("openclaw-parent-fork-unavailable-context-"); - const sessionsDir = path.join(root, "sessions"); - await fs.mkdir(sessionsDir); - - const sessionId = "parent-unavailable-context"; - const sessionFile = path.join(sessionsDir, "parent.jsonl"); - await fs.writeFile( - sessionFile, - [ - JSON.stringify({ - type: "session", - version: 3, - id: sessionId, - timestamp: new Date().toISOString(), - cwd: process.cwd(), - }), - JSON.stringify({ - message: { - role: "assistant", - content: "latest", - usage: { - input: 12, - output: 15_104, - cacheRead: 819_661, - cacheWrite: 93_130, - contextUsage: { state: "unavailable" }, - total: 927_907, - }, - }, - }), - ].join("\n"), - "utf-8", - ); - - const entry: SessionEntry = { - sessionId, - sessionFile, - updatedAt: Date.now(), - totalTokens: 4_567, - totalTokensFresh: false, - }; - - const tokens = await resolveParentForkTokenCountRuntime({ - parentEntry: entry, - storePath: path.join(root, "sessions.json"), - }); - - expect(tokens).toBe(4_567); - }); - - it("uses the exact final-iteration total when context usage is available", async () => { - const root = await makeRoot("openclaw-parent-fork-exact-context-"); - const sessionsDir = path.join(root, "sessions"); - await fs.mkdir(sessionsDir); - - const sessionId = "parent-exact-context"; - const sessionFile = path.join(sessionsDir, "parent.jsonl"); - await fs.writeFile( - sessionFile, - [ - JSON.stringify({ - type: "session", - version: 3, - id: sessionId, - timestamp: new Date().toISOString(), - cwd: process.cwd(), - }), - JSON.stringify({ - type: "message", - id: "active-usage", - parentId: null, - message: { - role: "assistant", - content: "latest", - usage: { - input: 12, - output: 15_104, - cacheRead: 819_661, - cacheWrite: 93_130, - contextUsage: { - state: "available", - promptTokens: 148_874, - totalTokens: 163_978, - }, - total: 927_907, - }, - }, - }), - JSON.stringify({ - type: "message", - id: "inactive-side-usage", - parentId: "active-usage", - message: { - role: "assistant", - content: `side branch ${"x".repeat(1_100_000)}`, - usage: { - input: 9_000, - output: 1_000, - contextUsage: { - state: "available", - promptTokens: 9_000, - totalTokens: 10_000, - }, - }, - }, - }), - JSON.stringify({ - type: "leaf", - id: "active-leaf", - parentId: "inactive-side-usage", - targetId: "active-usage", - }), - ].join("\n"), - "utf-8", - ); - - const entry: SessionEntry = { - sessionId, - sessionFile, - updatedAt: Date.now(), - totalTokens: 900_000, - totalTokensFresh: false, - }; - - const tokens = await resolveParentForkTokenCountRuntime({ - parentEntry: entry, - storePath: path.join(root, "sessions.json"), - }); - - expect(tokens).toBe(163_978); - }); - - it("adds only post-usage transcript pressure to an exact context snapshot", async () => { - const root = await makeRoot("openclaw-parent-fork-post-usage-tail-"); - const sessionsDir = path.join(root, "sessions"); - await fs.mkdir(sessionsDir); - - const sessionId = "parent-post-usage-tail"; - const sessionFile = path.join(sessionsDir, "parent.jsonl"); - await fs.writeFile( - sessionFile, - [ - JSON.stringify({ - type: "session", - version: 3, - id: sessionId, - timestamp: new Date().toISOString(), - cwd: process.cwd(), - }), - JSON.stringify({ - message: { - role: "assistant", - content: "latest model call", - usage: { - input: 12, - output: 10_000, - contextUsage: { - state: "available", - promptTokens: 70_000, - totalTokens: 80_000, - }, - }, - }, - }), - JSON.stringify({ - message: { - role: "tool", - content: `large appended tool result ${"x".repeat(100_000)}`, - }, - }), - ].join("\n"), - "utf-8", - ); - - const entry: SessionEntry = { - sessionId, - sessionFile, - updatedAt: Date.now(), - totalTokensFresh: false, - }; - - const tokens = await resolveParentForkTokenCountRuntime({ - parentEntry: entry, - storePath: path.join(root, "sessions.json"), - }); - - expect(tokens).toBeGreaterThan(100_000); - expect(tokens).toBeLessThan(110_000); - }); -}); diff --git a/src/auto-reply/reply/session-fork.runtime.ts b/src/auto-reply/reply/session-fork.runtime.ts deleted file mode 100644 index 769625f6d324..000000000000 --- a/src/auto-reply/reply/session-fork.runtime.ts +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Lazy runtime seam for parent-fork token counting. File-era transcript tail - * reads flow through gateway fs helpers; the SQLite flip estimates parent - * tokens inside the storage boundary instead. - */ -import fs from "node:fs/promises"; -import { derivePromptTokens } from "../../agents/usage.js"; -import { - resolveSessionFilePath, - resolveSessionFilePathOptions, -} from "../../config/sessions/paths.js"; -import { - resolveFreshSessionTotalTokens, - type SessionEntry as StoreSessionEntry, -} from "../../config/sessions/types.js"; -import { readLatestRecentSessionUsageFromTranscriptAsync } from "../../gateway/session-utils.fs.js"; - -const FALLBACK_TRANSCRIPT_BYTES_PER_TOKEN = 4; - -function resolvePositiveTokenCount(value: number | undefined): number | undefined { - return typeof value === "number" && Number.isFinite(value) && value > 0 - ? Math.floor(value) - : undefined; -} - -function maxPositiveTokenCount(...values: Array<number | undefined>): number | undefined { - let max: number | undefined; - for (const value of values) { - const normalized = resolvePositiveTokenCount(value); - if (typeof normalized === "number" && (max === undefined || normalized > max)) { - max = normalized; - } - } - return max; -} - -async function estimateParentTranscriptTokensFromBytes(params: { - parentEntry: StoreSessionEntry; - storePath: string; -}): Promise<number | undefined> { - try { - const filePath = resolveSessionFilePath( - params.parentEntry.sessionId, - params.parentEntry, - resolveSessionFilePathOptions({ storePath: params.storePath }), - ); - const stat = await fs.stat(filePath); - return resolvePositiveTokenCount(Math.ceil(stat.size / FALLBACK_TRANSCRIPT_BYTES_PER_TOKEN)); - } catch { - return undefined; - } -} - -/** Resolves the best available token count for a parent session before forking. */ -export async function resolveParentForkTokenCountRuntime(params: { - parentEntry: StoreSessionEntry; - storePath: string; -}): Promise<number | undefined> { - const freshPersistedTokens = resolveFreshSessionTotalTokens(params.parentEntry); - if (typeof freshPersistedTokens === "number") { - return freshPersistedTokens; - } - - const cachedTokens = resolvePositiveTokenCount(params.parentEntry.totalTokens); - const byteEstimateTokens = await estimateParentTranscriptTokensFromBytes(params); - try { - const usage = await readLatestRecentSessionUsageFromTranscriptAsync( - params.parentEntry.sessionId, - params.storePath, - params.parentEntry.sessionFile, - undefined, - 1024 * 1024, - ); - let transcriptTokens: number | undefined; - if (usage?.contextUsage?.state === "available") { - const trailingTokens = Math.ceil( - (usage.trailingBytes ?? 0) / FALLBACK_TRANSCRIPT_BYTES_PER_TOKEN, - ); - transcriptTokens = resolvePositiveTokenCount(usage.contextUsage.totalTokens + trailingTokens); - if (typeof transcriptTokens === "number") { - return transcriptTokens; - } - } else if (usage?.contextUsage?.state !== "unavailable") { - const promptTokens = resolvePositiveTokenCount( - derivePromptTokens({ - input: usage?.inputTokens, - cacheRead: usage?.cacheRead, - cacheWrite: usage?.cacheWrite, - }), - ); - const outputTokens = resolvePositiveTokenCount(usage?.outputTokens); - if (typeof promptTokens === "number") { - transcriptTokens = promptTokens + (outputTokens ?? 0); - } - } - if (typeof transcriptTokens === "number") { - return maxPositiveTokenCount(transcriptTokens, cachedTokens, byteEstimateTokens); - } - } catch { - // Fall back to cached totals when recent transcript usage cannot be read. - } - - return maxPositiveTokenCount(cachedTokens, byteEstimateTokens); -} diff --git a/src/auto-reply/reply/session-fork.test.ts b/src/auto-reply/reply/session-fork.test.ts index af94a452be40..3f22f422334e 100644 --- a/src/auto-reply/reply/session-fork.test.ts +++ b/src/auto-reply/reply/session-fork.test.ts @@ -1,103 +1,517 @@ // Tests parent-session fork facade storage-boundary behavior. -import fs from "node:fs/promises"; +import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; +import { + loadSessionEntry, + loadTranscriptEvents, + replaceSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import { replaceSqliteTranscriptEvents } from "../../config/sessions/session-accessor.sqlite.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; -import { forkSessionEntryFromParent } from "./session-fork.js"; - -const runtimeMocks = vi.hoisted(() => ({ - resolveParentForkTokenCountRuntime: vi.fn(), -})); - -vi.mock("./session-fork.runtime.js", () => runtimeMocks); +import { + forkSessionEntryFromParent, + forkSessionFromParent, + MODEL_SELECTION_LOCKED_PARENT_FORK_MESSAGE, + resolveParentForkDecision, +} from "./session-fork.js"; const roots: string[] = []; -async function makeRoot(prefix: string): Promise<string> { - // realpath first: macOS tmpdir is a /var -> /private/var symlink and the - // fork resolver returns canonical paths. - const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), prefix))); +function makeRoot(prefix: string): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); roots.push(root); return root; } -afterEach(async () => { - runtimeMocks.resolveParentForkTokenCountRuntime.mockReset(); - await Promise.all(roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); +afterEach(() => { + for (const root of roots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } }); describe("forkSessionEntryFromParent", () => { - it("forks transcripts in the directory for the store being mutated", async () => { - const root = await makeRoot("openclaw-session-fork-boundary-"); + it("rejects model-selection-locked parent context", async () => { + const parentEntry = { + sessionId: "locked-parent", + modelSelectionLocked: true, + updatedAt: 1, + }; + await expect( + resolveParentForkDecision({ parentEntry, storePath: "/tmp/unused-sessions.json" }), + ).rejects.toThrow(MODEL_SELECTION_LOCKED_PARENT_FORK_MESSAGE); + await expect( + forkSessionFromParent({ + agentId: "main", + parentEntry, + parentSessionKey: "agent:main:main", + sessionKey: "agent:main:subagent:child", + storePath: "/tmp/unused-sessions.json", + }), + ).rejects.toThrow(MODEL_SELECTION_LOCKED_PARENT_FORK_MESSAGE); + }); + + it("rejects a newer locked parent alias shadowed by a stale canonical row", async () => { + const root = makeRoot("openclaw-parent-fork-locked-alias-"); + const storePath = path.join(root, "sessions.json"); + await replaceSessionEntry( + { agentId: "main", sessionKey: "agent:main:main", storePath }, + { sessionId: "stale-canonical-parent", updatedAt: 1 }, + ); + await replaceSessionEntry( + { agentId: "main", sessionKey: "main", storePath }, + { + sessionId: "fresh-locked-parent", + modelSelectionLocked: true, + updatedAt: 2, + }, + ); + + await expect( + forkSessionEntryFromParent({ + agentId: "main", + fallbackEntry: { sessionId: "", updatedAt: 3 }, + parentSessionKey: "agent:main:main", + parentStoreKeys: ["agent:main:main", "main"], + sessionKey: "agent:main:subagent:child", + storePath, + }), + ).resolves.toEqual({ status: "failed" }); + expect( + loadSessionEntry({ agentId: "main", sessionKey: "agent:main:subagent:child", storePath }), + ).toBeUndefined(); + }); + + it("forks the active parent branch into SQLite and persists the child entry", async () => { + const root = makeRoot("openclaw-session-fork-boundary-"); const activeStoreDir = path.join(root, "active-store"); const configStoreDir = path.join(root, "config-store"); - await fs.mkdir(activeStoreDir, { recursive: true }); - await fs.mkdir(configStoreDir, { recursive: true }); + fs.mkdirSync(activeStoreDir, { recursive: true }); + fs.mkdirSync(configStoreDir, { recursive: true }); const storePath = path.join(activeStoreDir, "sessions.json"); const configStorePath = path.join(configStoreDir, "sessions.json"); const parentSessionKey = "agent:main:main"; const sessionKey = "agent:main:subagent:child"; - const parentSessionFile = path.join(activeStoreDir, "parent.jsonl"); - await fs.writeFile( - parentSessionFile, + const staleSessionKey = "agent:main:subagent:stale-child"; + + await replaceSessionEntry( + { + agentId: "main", + sessionKey: staleSessionKey, + storePath, + }, + { + sessionId: "stale-child-session", + updatedAt: 2, + }, + ); + await replaceSessionEntry( + { + agentId: "main", + sessionKey: parentSessionKey, + storePath, + }, + { + sessionId: "parent-session", + totalTokens: 10, + totalTokensFresh: true, + updatedAt: 10, + }, + ); + await replaceSqliteTranscriptEvents( + { + agentId: "main", + sessionId: "parent-session", + sessionKey: parentSessionKey, + storePath, + }, [ - JSON.stringify({ + { type: "session", version: 3, id: "parent-session", - timestamp: "2026-05-01T00:00:00.000Z", + timestamp: "2026-06-27T00:00:00.000Z", cwd: root, - }), - JSON.stringify({ - type: "message", - id: "assistant-1", - parentId: null, - timestamp: "2026-05-01T00:00:01.000Z", - message: { role: "assistant", content: "hi" }, - }), - ].join("\n") + "\n", - "utf-8", - ); - await fs.writeFile( - storePath, - JSON.stringify( - { - [parentSessionKey]: { - sessionId: "parent-session", - sessionFile: parentSessionFile, - updatedAt: 1, - }, - [sessionKey]: { sessionId: "", updatedAt: 2 }, }, - null, - 2, - ), - "utf-8", + { + type: "message", + id: "root", + parentId: null, + timestamp: "2026-06-27T00:00:01.000Z", + message: { role: "user", content: "root prompt" }, + }, + { + type: "message", + id: "inactive-answer", + parentId: "root", + timestamp: "2026-06-27T00:00:02.000Z", + message: { role: "assistant", content: "stale answer" }, + }, + { + type: "message", + id: "active-answer", + parentId: "root", + timestamp: "2026-06-27T00:00:03.000Z", + message: { role: "assistant", content: "active answer" }, + }, + { + type: "leaf", + id: "active-leaf", + parentId: "inactive-answer", + timestamp: "2026-06-27T00:00:04.000Z", + targetId: "active-answer", + }, + ], ); - runtimeMocks.resolveParentForkTokenCountRuntime.mockResolvedValue(10); - const result = await forkSessionEntryFromParent({ agentId: "main", config: { session: { store: configStorePath } } as OpenClawConfig, fallbackEntry: { sessionId: "", updatedAt: 2 }, parentSessionKey, + parentStoreKeys: [parentSessionKey], sessionKey, + sessionStoreKeys: [sessionKey, staleSessionKey], storePath, + patch: () => ({ label: "forked child", updatedAt: 3 }), }); expect(result.status).toBe("forked"); if (result.status !== "forked") { - throw new Error("expected forked result"); + throw new Error("expected fork"); } - // The fork artifact lands beside the store being mutated, not the config store. - expect(path.dirname(result.fork.sessionFile)).toBe(activeStoreDir); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { sessionId?: string; sessionFile?: string } - >; - expect(stored[sessionKey]?.sessionId).toBe(result.fork.sessionId); - expect(stored[sessionKey]?.sessionFile).toBe(result.fork.sessionFile); + expect(result.fork.sessionId).not.toBe("parent-session"); + expect(result.fork.sessionFile).toContain(`sqlite:main:${result.fork.sessionId}:`); + expect(fs.existsSync(storePath)).toBe(false); + + const stored = loadSessionEntry({ agentId: "main", sessionKey, storePath }); + expect(stored).toMatchObject({ + forkedFromParent: true, + label: "forked child", + sessionFile: result.fork.sessionFile, + sessionId: result.fork.sessionId, + updatedAt: expect.any(Number), + }); + expect(loadSessionEntry({ agentId: "main", sessionKey: staleSessionKey, storePath })).toBe( + undefined, + ); + + const events = (await loadTranscriptEvents({ + agentId: "main", + sessionId: result.fork.sessionId, + sessionKey, + storePath, + })) as Array<{ id?: string; message?: { content?: string }; type?: string }>; + const branchEvents = events.filter((event) => event.type !== "session"); + expect(branchEvents.map((event) => event.id).filter(Boolean)).toEqual([ + "root", + "active-answer", + expect.any(String), + ]); + expect(events).toContainEqual( + expect.objectContaining({ + id: "active-answer", + message: expect.objectContaining({ content: "active answer" }), + }), + ); + expect(events).not.toContainEqual( + expect.objectContaining({ + id: "inactive-answer", + }), + ); + }); + + it("marks the child as handled when the SQLite parent is over the fork limit", async () => { + const root = makeRoot("openclaw-session-fork-large-"); + const storePath = path.join(root, "sessions.json"); + const parentSessionKey = "agent:main:main"; + const sessionKey = "agent:main:subagent:child"; + await replaceSessionEntry( + { + agentId: "main", + sessionKey: parentSessionKey, + storePath, + }, + { + sessionId: "parent-session", + totalTokens: 150_000, + totalTokensFresh: true, + updatedAt: 1, + }, + ); + + const result = await forkSessionEntryFromParent({ + agentId: "main", + fallbackEntry: { sessionId: "", updatedAt: 2 }, + parentSessionKey, + sessionKey, + storePath, + decisionSkipPatch: () => ({ forkedFromParent: true, updatedAt: 3 }), + }); + + expect(result).toMatchObject({ + status: "skipped", + reason: "decision-skip", + decision: { + status: "skip", + reason: "parent-too-large", + parentTokens: 150_000, + }, + sessionEntry: { + forkedFromParent: true, + sessionId: "", + updatedAt: expect.any(Number), + }, + }); + expect(loadSessionEntry({ agentId: "main", sessionKey, storePath })).toMatchObject({ + forkedFromParent: true, + sessionId: "", + updatedAt: expect.any(Number), + }); + }); + + it("skips stale-token SQLite parents using transcript usage estimates", async () => { + const root = makeRoot("openclaw-session-fork-stale-large-"); + const storePath = path.join(root, "sessions.json"); + const parentEntry = { + sessionId: "parent-session", + totalTokens: 1, + totalTokensFresh: false, + updatedAt: 1, + }; + await replaceSqliteTranscriptEvents( + { + agentId: "main", + sessionId: parentEntry.sessionId, + sessionKey: "agent:main:main", + storePath, + }, + [ + { + type: "session", + version: 3, + id: parentEntry.sessionId, + timestamp: "2026-06-27T00:00:00.000Z", + cwd: root, + }, + { + type: "message", + id: "large-answer", + parentId: null, + timestamp: "2026-06-27T00:00:01.000Z", + message: { + role: "assistant", + content: "x".repeat(420_000), + }, + }, + ], + ); + + await expect(resolveParentForkDecision({ parentEntry, storePath })).resolves.toMatchObject({ + status: "skip", + reason: "parent-too-large", + parentTokens: expect.any(Number), + }); + }); + + it("does not reconstruct SQLite parent context from billing buckets when context is unavailable", async () => { + const root = makeRoot("openclaw-session-fork-unavailable-context-"); + const storePath = path.join(root, "sessions.json"); + const parentEntry = { + sessionId: "parent-session", + totalTokens: 4_567, + totalTokensFresh: false, + updatedAt: 1, + }; + await replaceSqliteTranscriptEvents( + { + agentId: "main", + sessionId: parentEntry.sessionId, + sessionKey: "agent:main:main", + storePath, + }, + [ + { + type: "session", + version: 3, + id: parentEntry.sessionId, + timestamp: "2026-06-27T00:00:00.000Z", + cwd: root, + }, + { + type: "message", + id: "usage", + parentId: null, + timestamp: "2026-06-27T00:00:01.000Z", + message: { + role: "assistant", + content: "latest", + usage: { + input: 12, + output: 15_104, + cacheRead: 819_661, + cacheWrite: 93_130, + contextUsage: { state: "unavailable" }, + total: 927_907, + }, + }, + }, + ], + ); + + await expect(resolveParentForkDecision({ parentEntry, storePath })).resolves.toMatchObject({ + status: "fork", + parentTokens: 4_567, + }); + }); + + it("uses exact SQLite context usage instead of stale cached totals", async () => { + const root = makeRoot("openclaw-session-fork-exact-context-"); + const storePath = path.join(root, "sessions.json"); + const parentEntry = { + sessionId: "parent-session", + totalTokens: 900_000, + totalTokensFresh: false, + updatedAt: 1, + }; + await replaceSqliteTranscriptEvents( + { + agentId: "main", + sessionId: parentEntry.sessionId, + sessionKey: "agent:main:main", + storePath, + }, + [ + { + type: "session", + version: 3, + id: parentEntry.sessionId, + timestamp: "2026-06-27T00:00:00.000Z", + cwd: root, + }, + { + type: "message", + id: "usage", + parentId: null, + timestamp: "2026-06-27T00:00:01.000Z", + message: { + role: "assistant", + content: "latest", + usage: { + input: 12, + output: 15_104, + cacheRead: 819_661, + cacheWrite: 93_130, + contextUsage: { + state: "available", + promptTokens: 148_874, + totalTokens: 163_978, + }, + total: 927_907, + }, + }, + }, + { + type: "message", + id: "side-branch", + parentId: "usage", + timestamp: "2026-06-27T00:00:02.000Z", + message: { + role: "assistant", + content: `side branch ${"x".repeat(1_100_000)}`, + usage: { + input: 9_000, + output: 1_000, + contextUsage: { + state: "available", + promptTokens: 9_000, + totalTokens: 10_000, + }, + }, + }, + }, + { + type: "leaf", + id: "active-leaf", + parentId: "side-branch", + timestamp: "2026-06-27T00:00:03.000Z", + targetId: "usage", + }, + ], + ); + + await expect(resolveParentForkDecision({ parentEntry, storePath })).resolves.toMatchObject({ + status: "skip", + reason: "parent-too-large", + parentTokens: 163_978, + }); + }); + + it("adds only post-usage SQLite transcript pressure to exact context usage", async () => { + const root = makeRoot("openclaw-session-fork-post-usage-tail-"); + const storePath = path.join(root, "sessions.json"); + const parentEntry = { + sessionId: "parent-session", + totalTokens: 1, + totalTokensFresh: false, + updatedAt: 1, + }; + await replaceSqliteTranscriptEvents( + { + agentId: "main", + sessionId: parentEntry.sessionId, + sessionKey: "agent:main:main", + storePath, + }, + [ + { + type: "session", + version: 3, + id: parentEntry.sessionId, + timestamp: "2026-06-27T00:00:00.000Z", + cwd: root, + }, + { + type: "message", + id: "usage", + parentId: null, + timestamp: "2026-06-27T00:00:01.000Z", + message: { + role: "assistant", + content: "latest model call", + usage: { + input: 12, + output: 10_000, + contextUsage: { + state: "available", + promptTokens: 70_000, + totalTokens: 80_000, + }, + }, + }, + }, + { + type: "message", + id: "tail", + parentId: "usage", + timestamp: "2026-06-27T00:00:02.000Z", + message: { + role: "tool", + content: `large appended tool result ${"x".repeat(100_000)}`, + }, + }, + ], + ); + + const decision = await resolveParentForkDecision({ parentEntry, storePath }); + + expect(decision).toMatchObject({ + status: "skip", + reason: "parent-too-large", + parentTokens: expect.any(Number), + }); + expect(decision.parentTokens).toBeGreaterThan(100_000); + expect(decision.parentTokens).toBeLessThan(110_000); }); }); diff --git a/src/auto-reply/reply/session-fork.ts b/src/auto-reply/reply/session-fork.ts index 2882efe1b012..0c9f677911e0 100644 --- a/src/auto-reply/reply/session-fork.ts +++ b/src/auto-reply/reply/session-fork.ts @@ -2,8 +2,9 @@ import { resolveStorePath } from "../../config/sessions/paths.js"; import { forkSessionEntryFromParentTarget, forkSessionFromParentTranscript, - type ParentForkedSessionTranscript, + resolveSessionParentForkDecision, type SessionParentForkDecision, + type ParentForkedSessionTranscript, } from "../../config/sessions/session-accessor.js"; import type { SessionEntry } from "../../config/sessions/types.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; @@ -11,15 +12,6 @@ import { isModelSelectionLocked, ModelSelectionLockedError, } from "../../sessions/model-overrides.js"; -import { createLazyImportLoader } from "../../shared/lazy-promise.js"; - -/** - * Default max parent token count beyond which thread/session parent forking is skipped. - * This prevents new thread sessions from inheriting near-full parent context. - * See #26905. - */ -const DEFAULT_PARENT_FORK_MAX_TOKENS = 100_000; -const sessionForkRuntimeLoader = createLazyImportLoader(() => import("./session-fork.runtime.js")); export const MODEL_SELECTION_LOCKED_PARENT_FORK_MESSAGE = "Model-selection-locked sessions cannot create child sessions from parent context."; @@ -48,7 +40,8 @@ type ForkSessionFromParentParams = { config?: OpenClawConfig; sessionKey: string; storePath?: string; - /** Cross-agent forks land the child transcript beside the child's store. */ + + /** Cross-agent forks land the child transcript in the target agent's store. */ targetStorePath?: string; }; @@ -95,20 +88,6 @@ export type ForkSessionEntryFromParentParams = Omit<ForkSessionFromParentParams, }) => Partial<SessionEntry> | null; }; -function loadSessionForkRuntime(): Promise<typeof import("./session-fork.runtime.js")> { - return sessionForkRuntimeLoader.load(); -} - -function formatParentForkTooLargeMessage(params: { - parentTokens: number; - maxTokens: number; -}): string { - return ( - `Parent context is too large to fork (${params.parentTokens}/${params.maxTokens} tokens); ` + - "starting with isolated context instead." - ); -} - function resolveParentForkStorePath(params: { agentId?: string; config?: OpenClawConfig; @@ -123,25 +102,10 @@ export async function resolveParentForkDecision( params: ParentForkDecisionParams, ): Promise<ParentForkDecision> { assertParentSessionForkAllowed(params.parentEntry); - const maxTokens = DEFAULT_PARENT_FORK_MAX_TOKENS; - const parentTokens = await resolveParentForkTokenCount({ + return await resolveSessionParentForkDecision({ parentEntry: params.parentEntry, storePath: resolveParentForkStorePath(params), }); - if (typeof parentTokens === "number" && parentTokens > maxTokens) { - return { - status: "skip", - reason: "parent-too-large", - maxTokens, - parentTokens, - message: formatParentForkTooLargeMessage({ parentTokens, maxTokens }), - }; - } - return { - status: "fork", - maxTokens, - ...(typeof parentTokens === "number" ? { parentTokens } : {}), - }; } export async function forkSessionFromParent( @@ -196,13 +160,6 @@ export async function forkSessionEntryFromParent( storeKeys: params.parentStoreKeys, }), patch: params.patch, - resolveDecision: (parentEntry) => - resolveParentForkDecision({ - parentEntry, - agentId: params.agentId, - config: params.config, - storePath, - }), sessionTarget: normalizeForkTarget({ canonicalKey: params.sessionKey, storeKeys: params.sessionStoreKeys, @@ -212,11 +169,3 @@ export async function forkSessionEntryFromParent( storePath, }); } - -async function resolveParentForkTokenCount(params: { - parentEntry: SessionEntry; - storePath: string; -}): Promise<number | undefined> { - const runtime = await loadSessionForkRuntime(); - return runtime.resolveParentForkTokenCountRuntime(params); -} diff --git a/src/auto-reply/reply/session-hooks-context.test.ts b/src/auto-reply/reply/session-hooks-context.test.ts index 78c234c2af8a..2793f84ed6f7 100644 --- a/src/auto-reply/reply/session-hooks-context.test.ts +++ b/src/auto-reply/reply/session-hooks-context.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../config/config.js"; import type { SessionEntry } from "../../config/sessions.js"; +import { replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import type { HookRunner } from "../../plugins/hooks.js"; import { getActiveGatewayRootWorkCount, @@ -79,7 +80,12 @@ async function writeStore( store: Record<string, SessionEntry | Record<string, unknown>>, ): Promise<void> { await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile(storePath, JSON.stringify(store), "utf-8"); + for (const [sessionKey, entry] of Object.entries(store)) { + const sessionEntry = entry as Partial<SessionEntry>; + if (typeof sessionEntry.sessionId === "string" && sessionEntry.sessionId.trim()) { + await replaceSessionEntry({ storePath, sessionKey }, sessionEntry as SessionEntry); + } + } } async function writeTranscript( diff --git a/src/auto-reply/reply/session-reset-model.test.ts b/src/auto-reply/reply/session-reset-model.test.ts index 06b1efe1b74e..f9e142cbf267 100644 --- a/src/auto-reply/reply/session-reset-model.test.ts +++ b/src/auto-reply/reply/session-reset-model.test.ts @@ -6,11 +6,8 @@ import { describe, expect, it } from "vitest"; import type { ModelCatalogEntry } from "../../agents/model-catalog.js"; import type { OpenClawConfig } from "../../config/config.js"; import type { SessionEntry } from "../../config/sessions.js"; -import { - clearSessionStoreCacheForTest, - loadSessionStore, - saveSessionStore, -} from "../../config/sessions/store.js"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; +import { clearSessionStoreCacheForTest } from "../../config/sessions/store.js"; import type { ModelAliasIndex } from "./model-selection-directive.js"; import { applyResetModelOverride } from "./session-reset-model.js"; @@ -96,11 +93,7 @@ describe("applyResetModelOverride", () => { modelOverride: "gpt-4o-mini", modelOverrideSource: "user", }; - await saveSessionStore( - storePath, - { "agent:main:dm:1": concurrentEntry }, - { skipMaintenance: true }, - ); + await replaceSessionEntry({ sessionKey: "agent:main:dm:1", storePath }, concurrentEntry); try { const result = await applyResetModelOverride({ @@ -128,7 +121,7 @@ describe("applyResetModelOverride", () => { }); expect(fixture.sessionEntry.updatedAt).toBeGreaterThanOrEqual(concurrentEntry.updatedAt); expect(fixture.sessionStore["agent:main:dm:1"]).toEqual(fixture.sessionEntry); - expect(loadSessionStore(storePath, { skipCache: true })["agent:main:dm:1"]).toEqual( + expect(loadSessionEntry({ sessionKey: "agent:main:dm:1", storePath })).toEqual( fixture.sessionEntry, ); } finally { @@ -151,11 +144,7 @@ describe("applyResetModelOverride", () => { providerOverride: "openai", modelOverride: "gpt-4o-mini", }; - await saveSessionStore( - storePath, - { "agent:main:dm:1": concurrentEntry }, - { skipMaintenance: true }, - ); + await replaceSessionEntry({ sessionKey: "agent:main:dm:1", storePath }, concurrentEntry); try { const result = await applyResetModelOverride({ @@ -197,11 +186,7 @@ describe("applyResetModelOverride", () => { modelOverride: "gpt-4o-mini", modelOverrideSource: "user", }; - await saveSessionStore( - storePath, - { "agent:main:dm:1": rotatedEntry }, - { skipMaintenance: true }, - ); + await replaceSessionEntry({ sessionKey: "agent:main:dm:1", storePath }, rotatedEntry); try { await expect( @@ -225,9 +210,7 @@ describe("applyResetModelOverride", () => { expect(fixture.sessionEntry.sessionId).toBe("s1"); expect(fixture.sessionEntry.modelOverride).toBeUndefined(); expect(fixture.sessionStore["agent:main:dm:1"]).toBe(fixture.sessionEntry); - expect(loadSessionStore(storePath, { skipCache: true })["agent:main:dm:1"]).toEqual( - rotatedEntry, - ); + expect(loadSessionEntry({ sessionKey: "agent:main:dm:1", storePath })).toEqual(rotatedEntry); } finally { clearSessionStoreCacheForTest(); fs.rmSync(tempRoot, { recursive: true, force: true }); diff --git a/src/auto-reply/reply/session-updates.lifecycle.test.ts b/src/auto-reply/reply/session-updates.lifecycle.test.ts index 6856f4e2e281..f772b00d42db 100644 --- a/src/auto-reply/reply/session-updates.lifecycle.test.ts +++ b/src/auto-reply/reply/session-updates.lifecycle.test.ts @@ -4,7 +4,12 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../config/config.js"; -import { loadSessionStore, type SessionEntry } from "../../config/sessions.js"; +import type { SessionEntry } from "../../config/sessions.js"; +import { + applySessionEntryLifecycleMutation, + loadSessionEntry, + replaceSessionEntry, +} from "../../config/sessions/session-accessor.js"; import type { HookRunner } from "../../plugins/hooks.js"; import { getActiveGatewayRootWorkCount, @@ -38,7 +43,7 @@ async function createFixture() { const sessionStore: Record<string, SessionEntry> = { [sessionKey]: entry, }; - await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + await replaceSessionEntry({ storePath, sessionKey }, entry); return { storePath, sessionKey, sessionStore, entry, transcriptPath }; } @@ -182,7 +187,11 @@ describe("session-updates lifecycle hooks", () => { it("recreates a complete persisted row when compaction updates a missing store row", async () => { const { storePath, sessionKey, sessionStore, entry } = await createFixture(); - await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf-8"); + await applySessionEntryLifecycleMutation({ + storePath, + removals: [{ sessionKey }], + skipMaintenance: true, + }); await incrementCompactionCount({ sessionEntry: entry, @@ -194,7 +203,7 @@ describe("session-updates lifecycle hooks", () => { now: 456, }); - const persisted = loadSessionStore(storePath, { skipCache: true })[sessionKey]; + const persisted = loadSessionEntry({ storePath, sessionKey }); expect(sessionStore[sessionKey]?.sessionId).toBe("s2"); expect(sessionStore[sessionKey]?.sessionFile).toContain("s2.jsonl"); expect(sessionStore[sessionKey]?.usageFamilyKey).toBe(sessionKey); diff --git a/src/auto-reply/reply/session-updates.test.ts b/src/auto-reply/reply/session-updates.test.ts index 312afba6ef2d..3e7ee134685c 100644 --- a/src/auto-reply/reply/session-updates.test.ts +++ b/src/auto-reply/reply/session-updates.test.ts @@ -1,9 +1,6 @@ // Tests session update fanout and persisted lifecycle records. -import fs from "node:fs/promises"; -import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"; -import type { SessionEntry } from "../../config/sessions/types.js"; +import { createReplySessionEntryHandle } from "./session-entry-handle.js"; const TEST_WORKSPACE_DIR = "/tmp/workspace"; @@ -16,6 +13,7 @@ const { resolveAgentConfigMock, resolveSessionAgentIdMock, resolveAgentIdFromSessionKeyMock, + updateSessionEntryMock, resolveNodeExecEligibilityMock, } = vi.hoisted(() => ({ buildWorkspaceSkillSnapshotMock: vi.fn((..._args: unknown[]) => ({ @@ -34,6 +32,7 @@ const { resolveAgentConfigMock: vi.fn(() => undefined), resolveSessionAgentIdMock: vi.fn(() => "writer"), resolveAgentIdFromSessionKeyMock: vi.fn(() => "main"), + updateSessionEntryMock: vi.fn(), resolveNodeExecEligibilityMock: vi.fn(() => ({ canExec: false })), })); @@ -69,6 +68,11 @@ vi.mock("../../config/sessions.js", () => ({ resolveSessionFilePathOptions: vi.fn(), })); +vi.mock("../../config/sessions/session-accessor.js", () => ({ + patchSessionEntry: vi.fn(), + updateSessionEntry: updateSessionEntryMock, +})); + vi.mock("../../routing/session-key.js", () => ({ normalizeAgentId: (id: string) => id, normalizeMainKey: (key?: string) => key ?? "main", @@ -76,7 +80,6 @@ vi.mock("../../routing/session-key.js", () => ({ })); const { ensureSkillSnapshot } = await import("./session-updates.js"); -const tempDirs = useAutoCleanupTempDirTracker(afterEach); describe("ensureSkillSnapshot", () => { beforeEach(() => { @@ -92,6 +95,8 @@ describe("ensureSkillSnapshot", () => { resolveAgentConfigMock.mockReturnValue(undefined); resolveSessionAgentIdMock.mockReturnValue("writer"); resolveAgentIdFromSessionKeyMock.mockReturnValue("main"); + updateSessionEntryMock.mockReset(); + updateSessionEntryMock.mockResolvedValue(null); resolveNodeExecEligibilityMock.mockReturnValue({ canExec: false }); }); @@ -135,36 +140,154 @@ describe("ensureSkillSnapshot", () => { ); }); + it("does not keep a deleted first-turn session entry when persisting skills", async () => { + vi.stubEnv("OPENCLAW_TEST_FAST", "0"); + const sessionKey = "agent:main:main"; + const sessionEntry = { + sessionId: "deleted-session", + updatedAt: 10, + }; + const sessionStore = { [sessionKey]: sessionEntry }; + const sessionEntryHandle = createReplySessionEntryHandle({ + sessionEntry, + sessionKey, + sessionStore, + }); + + const result = await ensureSkillSnapshot({ + sessionEntry, + sessionEntryHandle, + sessionStore, + sessionKey, + sessionId: "deleted-session", + storePath: "/tmp/sessions.json", + isFirstTurnInSession: true, + workspaceDir: TEST_WORKSPACE_DIR, + cfg: {}, + }); + + expect(updateSessionEntryMock).toHaveBeenCalledWith( + { + storePath: "/tmp/sessions.json", + sessionKey, + }, + expect.any(Function), + ); + expect(result.sessionEntry).toBeUndefined(); + expect(result.systemSent).toBe(false); + expect(sessionEntryHandle.getCurrent()).toBeUndefined(); + expect(sessionStore[sessionKey]).toBeUndefined(); + }); + + it("adopts a rebound first-turn session entry instead of overwriting it", async () => { + vi.stubEnv("OPENCLAW_TEST_FAST", "0"); + const sessionKey = "agent:main:main"; + const sessionEntry = { + sessionId: "old-session", + updatedAt: 10, + }; + const reboundEntry = { + sessionId: "new-session", + updatedAt: 20, + }; + const sessionStore = { [sessionKey]: sessionEntry }; + updateSessionEntryMock.mockImplementationOnce(async (_scope, update) => { + const patch = await update(reboundEntry); + expect(patch).toBeNull(); + return reboundEntry; + }); + + const result = await ensureSkillSnapshot({ + sessionEntry, + sessionStore, + sessionKey, + sessionId: "old-session", + storePath: "/tmp/sessions.json", + isFirstTurnInSession: true, + workspaceDir: TEST_WORKSPACE_DIR, + cfg: {}, + }); + + expect(result.sessionEntry).toEqual(reboundEntry); + expect(result.systemSent).toBe(false); + expect(sessionStore[sessionKey]).toEqual(reboundEntry); + }); + + it("persists first-turn skill snapshots as a guarded partial update", async () => { + vi.stubEnv("OPENCLAW_TEST_FAST", "0"); + const sessionKey = "agent:main:main"; + const sessionEntry = { + sessionId: "session-1", + updatedAt: 10, + modelOverride: "gpt-5.5", + }; + const sessionStore = { [sessionKey]: sessionEntry }; + updateSessionEntryMock.mockImplementationOnce(async (_scope, update) => { + const patch = await update({ + ...sessionEntry, + updatedAt: 20, + modelOverride: "sonnet-4.6", + }); + expect(patch).toMatchObject({ + sessionId: "session-1", + systemSent: true, + }); + expect(patch).not.toHaveProperty("modelOverride"); + return { + ...sessionEntry, + ...patch, + modelOverride: "sonnet-4.6", + }; + }); + + const result = await ensureSkillSnapshot({ + sessionEntry, + sessionStore, + sessionKey, + sessionId: "session-1", + storePath: "/tmp/sessions.json", + isFirstTurnInSession: true, + workspaceDir: TEST_WORKSPACE_DIR, + cfg: {}, + }); + + expect(result.sessionEntry?.modelOverride).toBe("sonnet-4.6"); + expect(sessionStore[sessionKey]?.modelOverride).toBe("sonnet-4.6"); + }); + it("keeps a concurrent rename and unpin while persisting a skill snapshot", async () => { vi.stubEnv("OPENCLAW_TEST_FAST", "0"); - const root = tempDirs.make("openclaw-session-updates-"); - const storePath = path.join(root, "sessions.json"); const sessionKey = "agent:main:reply"; - const staleEntry: SessionEntry = { + const staleEntry = { sessionId: "reply-session", updatedAt: 1, label: "Before rename", pinnedAt: 100, }; - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: staleEntry.sessionId, - updatedAt: 2, - label: "After rename", - sendPolicy: "deny", - }, - }), - "utf8", - ); const sessionStore = { [sessionKey]: staleEntry }; + // Concurrent session management renamed and unpinned the entry after the + // reply loop captured its stale snapshot. + const concurrentEntry = { + sessionId: "reply-session", + updatedAt: 2, + label: "After rename", + sendPolicy: "deny", + }; + updateSessionEntryMock.mockImplementationOnce(async (_scope, update) => { + const patch = await update(concurrentEntry); + expect(patch).toMatchObject({ sessionId: "reply-session", systemSent: true }); + expect(patch).not.toHaveProperty("label"); + expect(patch).not.toHaveProperty("pinnedAt"); + expect(patch).not.toHaveProperty("sendPolicy"); + return { ...concurrentEntry, ...patch }; + }); const result = await ensureSkillSnapshot({ sessionEntry: staleEntry, sessionStore, sessionKey, - storePath, + sessionId: "reply-session", + storePath: "/tmp/sessions.json", isFirstTurnInSession: true, workspaceDir: TEST_WORKSPACE_DIR, cfg: {}, @@ -177,14 +300,6 @@ describe("ensureSkillSnapshot", () => { systemSent: true, }); expect(result.sessionEntry?.pinnedAt).toBeUndefined(); - expect(sessionStore[sessionKey]).toBe(result.sessionEntry); - const persisted = JSON.parse(await fs.readFile(storePath, "utf8")) as Record< - string, - SessionEntry - >; - expect(persisted[sessionKey]?.label).toBe("After rename"); - expect(persisted[sessionKey]?.pinnedAt).toBeUndefined(); - expect(persisted[sessionKey]?.sendPolicy).toBe("deny"); - expect(persisted[sessionKey]?.skillsSnapshot).toBeDefined(); + expect(sessionStore[sessionKey]).toEqual(result.sessionEntry); }); }); diff --git a/src/auto-reply/reply/session-updates.ts b/src/auto-reply/reply/session-updates.ts index 9e0960244052..91a0a1cea6a2 100644 --- a/src/auto-reply/reply/session-updates.ts +++ b/src/auto-reply/reply/session-updates.ts @@ -7,7 +7,7 @@ import { resolveNodeExecEligibility, } from "../../agents/exec-defaults.js"; import { resolveCompactionSessionFile, type SessionEntry } from "../../config/sessions.js"; -import { patchSessionEntry } from "../../config/sessions/session-accessor.js"; +import { patchSessionEntry, updateSessionEntry } from "../../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { forgetActiveSessionForShutdown, @@ -26,18 +26,36 @@ export { drainFormattedSystemEvents } from "./session-system-events.js"; export { resetResolvedSkillsCacheForTests } from "../../skills/runtime/session-snapshot.js"; async function persistSessionEntryUpdate(params: { + expectedSessionId: string | undefined; sessionEntryHandle?: ReplySessionEntryHandle; sessionStore?: Record<string, SessionEntry>; sessionKey?: string; storePath?: string; nextEntry: SessionEntry; updates: Partial<SessionEntry>; -}): Promise<SessionEntry> { +}): Promise<SessionEntry | undefined> { if (!params.sessionEntryHandle && (!params.sessionStore || !params.sessionKey)) { + return undefined; + } + if (!params.storePath || !params.sessionKey) { + if (params.sessionEntryHandle) { + params.sessionEntryHandle.replaceCurrent(params.nextEntry); + } else if (params.sessionStore && params.sessionKey) { + params.sessionStore[params.sessionKey] = { + ...params.sessionStore[params.sessionKey], + ...params.nextEntry, + }; + } return params.nextEntry; } - let persistedEntry = params.nextEntry; - if (!params.storePath || !params.sessionKey) { + const persistedEntry = await updateSessionEntry( + { + storePath: params.storePath, + sessionKey: params.sessionKey, + }, + (entry) => (entry.sessionId === params.expectedSessionId ? params.updates : null), + ); + if (persistedEntry) { if (params.sessionEntryHandle) { params.sessionEntryHandle.replaceCurrent(persistedEntry); } else if (params.sessionStore && params.sessionKey) { @@ -45,18 +63,11 @@ async function persistSessionEntryUpdate(params: { } return persistedEntry; } - persistedEntry = - (await patchSessionEntry( - { storePath: params.storePath, sessionKey: params.sessionKey }, - () => params.updates, - { fallbackEntry: params.nextEntry }, - )) ?? persistedEntry; - if (params.sessionEntryHandle) { - params.sessionEntryHandle.replaceCurrent(persistedEntry); - } else if (params.sessionStore) { - params.sessionStore[params.sessionKey] = persistedEntry; + params.sessionEntryHandle?.clearCurrent(); + if (params.sessionStore && params.sessionKey) { + delete params.sessionStore[params.sessionKey]; } - return persistedEntry; + return undefined; } function emitCompactionSessionLifecycleHooks(params: { @@ -214,7 +225,8 @@ export async function ensureSkillSnapshot(params: { systemSent: true, skillsSnapshot: skillSnapshot, }; - nextEntry = await persistSessionEntryUpdate({ + const persistedEntry = await persistSessionEntryUpdate({ + expectedSessionId: current.sessionId, sessionEntryHandle, sessionStore, sessionKey, @@ -227,7 +239,8 @@ export async function ensureSkillSnapshot(params: { skillsSnapshot: nextEntry.skillsSnapshot, }, }); - systemSent = true; + nextEntry = persistedEntry; + systemSent = persistedEntry?.systemSent ?? systemSent; } const hasFreshSnapshotInEntry = @@ -257,6 +270,7 @@ export async function ensureSkillSnapshot(params: { skillsSnapshot, }; nextEntry = await persistSessionEntryUpdate({ + expectedSessionId: current.sessionId, sessionEntryHandle, sessionStore, sessionKey, diff --git a/src/auto-reply/reply/session.heartbeat-no-reset.test.ts b/src/auto-reply/reply/session.heartbeat-no-reset.test.ts index f5d03566ddeb..4d4ee24d344f 100644 --- a/src/auto-reply/reply/session.heartbeat-no-reset.test.ts +++ b/src/auto-reply/reply/session.heartbeat-no-reset.test.ts @@ -3,7 +3,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import type { OpenClawConfig } from "../../config/config.js"; -import { loadSessionStore, saveSessionStore } from "../../config/sessions/store.js"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import type { SessionEntry } from "../../config/sessions/types.js"; import type { MsgContext } from "../templating.js"; import { initSessionState } from "./session.js"; @@ -73,18 +73,22 @@ describe("initSessionState - heartbeat should not trigger session reset", () => updatedAt: number, overrides: Partial<SessionEntry> = {}, ): Promise<void> => { - await saveSessionStore(storePath, { - "main:user123": { + await replaceSessionEntry( + { + storePath, + sessionKey: "main:user123", + }, + { sessionId, updatedAt, systemSent: true, ...overrides, }, - }); + ); }; - const expectPersistedSession = (sessionStore: Record<string, SessionEntry>): SessionEntry => { - const entry = sessionStore["main:user123"]; + const expectPersistedSession = (): SessionEntry => { + const entry = loadSessionEntry({ storePath, sessionKey: "main:user123" }); if (!entry) { throw new Error("Expected persisted session for main:user123"); } @@ -199,8 +203,7 @@ describe("initSessionState - heartbeat should not trigger session reset", () => expect(heartbeatResult.sessionId).toBe("daily-session-id"); expect(heartbeatResult.sessionEntry.lastInteractionAt).toBe(staleTime); - const persistedAfterHeartbeat = loadSessionStore(storePath); - expect(expectPersistedSession(persistedAfterHeartbeat).lastInteractionAt).toBe(staleTime); + expect(expectPersistedSession().lastInteractionAt).toBe(staleTime); const userResult = await initSessionState({ ctx: createBaseCtx({ @@ -286,8 +289,7 @@ describe("initSessionState - heartbeat should not trigger session reset", () => expect(heartbeatResult.isNewSession).toBe(false); expect(heartbeatResult.sessionId).toBe("legacy-idle-session"); - const persistedAfterHeartbeat = loadSessionStore(storePath); - expect(expectPersistedSession(persistedAfterHeartbeat).lastInteractionAt).toBeUndefined(); + expect(expectPersistedSession().lastInteractionAt).toBeUndefined(); const userResult = await initSessionState({ ctx: createBaseCtx({ diff --git a/src/auto-reply/reply/session.test.ts b/src/auto-reply/reply/session.test.ts index a74329797fbd..61380cb502ee 100644 --- a/src/auto-reply/reply/session.test.ts +++ b/src/auto-reply/reply/session.test.ts @@ -10,8 +10,14 @@ import { import * as bootstrapCache from "../../agents/bootstrap-cache.js"; import type { OpenClawConfig } from "../../config/config.js"; import type { SessionEntry } from "../../config/sessions.js"; +import { + listSessionEntries, + loadSessionEntry, + replaceSessionEntry, + upsertSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import { runExclusiveSessionStoreWrite } from "../../config/sessions/store-writer.js"; -import { readSessionStoreForTest } from "../../config/sessions/test-helpers.js"; import { formatZonedTimestamp } from "../../infra/format-time/format-datetime.ts"; import { testing as sessionBindingTesting, @@ -80,12 +86,13 @@ vi.mock("./session-fork.js", () => ({ entry: SessionEntry; parentEntry: SessionEntry; }) => Partial<SessionEntry>; + sessionKey?: string; }) => { - const store = JSON.parse(await fs.readFile(params.storePath, "utf-8")) as Record< - string, - SessionEntry - >; - const parentEntry = store[params.parentSessionKey]; + const parentEntry = loadSessionEntry({ + sessionKey: params.parentSessionKey, + storePath: params.storePath, + readConsistency: "latest", + }); if (!parentEntry?.sessionId) { return { status: "missing-parent" }; } @@ -278,7 +285,32 @@ async function writeSessionStoreFast( store: Record<string, SessionEntry | Record<string, unknown>>, ): Promise<void> { await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile(storePath, JSON.stringify(store), "utf-8"); + for (const [sessionKey, entry] of Object.entries(store)) { + const patch = entry as Partial<SessionEntry>; + if (typeof patch.sessionId === "string" && patch.sessionId.trim()) { + await replaceSessionEntry({ storePath, sessionKey }, patch as SessionEntry); + } else { + await upsertSessionEntry({ storePath, sessionKey }, patch); + } + } +} + +function readSessionStoreFast(storePath: string): Record<string, SessionEntry> { + const entries = Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ) as Record<string, SessionEntry>; + return new Proxy(entries, { + get(target, prop, receiver) { + if (typeof prop !== "string" || prop in target) { + return Reflect.get(target, prop, receiver); + } + const entry = loadSessionEntry({ storePath, sessionKey: prop, readConsistency: "latest" }); + if (entry) { + target[prop] = entry; + } + return Reflect.get(target, prop, receiver); + }, + }); } describe("resolveReplySessionPreprocessingState", () => { @@ -323,6 +355,9 @@ describe("resolveReplySessionPreprocessingState", () => { }); }); + // A "missing session id" case is unrepresentable post-flip: sessions rows + // are NOT NULL on session_id and upsert repairs id-less patches at write + // time, so the store can never hand preprocessing such an entry. it.each([ ["missing row", undefined], [ @@ -334,14 +369,6 @@ describe("resolveReplySessionPreprocessingState", () => { modelSelectionLocked: true, }, ], - [ - "missing session id", - { - updatedAt: 1, - agentHarnessId: "codex", - modelSelectionLocked: true, - }, - ], ] as const)("rejects a reserved %s before preprocessing", async (_label, entry) => { const storePath = await createStorePath(`openclaw-media-preflight-invalid-${_label}-`); await writeSessionStoreFast(storePath, entry ? { [sessionKey]: entry } : {}); @@ -615,7 +642,7 @@ describe("initSessionState guarded initialization", () => { }); describe("initSessionState thread forking", () => { - it("forks a new session from the parent session file", async () => { + it("forks a new SQLite session from the parent session", async () => { const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); const root = await makeCaseDir("openclaw-thread-session-"); const sessionsDir = path.join(root, "sessions"); @@ -683,24 +710,19 @@ describe("initSessionState thread forking", () => { expect(result.sessionEntry.totalTokens).toBeUndefined(); expect(result.sessionEntry.totalTokensFresh).toBe(false); - const newSessionFile = requireString( - result.sessionEntry.sessionFile, - "forked thread session file", + expect(result.sessionEntry.forkedFromParent).toBe(true); + expect(result.sessionEntry.sessionFile).toBe( + formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: result.sessionEntry.sessionId, + storePath, + }), ); - const headerLine = (await fs.readFile(newSessionFile, "utf-8")) - .split(/\r?\n/) - .find((line) => line.trim().length > 0); - if (!headerLine) { - throw new Error("Missing session header"); - } - const parsedHeader = JSON.parse(headerLine) as { - parentSession?: string; - }; - const expectedParentSession = await fs.realpath(parentSessionFile); - const actualParentSession = parsedHeader.parentSession - ? await fs.realpath(parsedHeader.parentSession) - : undefined; - expect(actualParentSession).toBe(expectedParentSession); + const forkCall = requireMockCallArg(sessionForkMocks.forkSessionFromParent, "fork session"); + expect(forkCall.parentEntry).toMatchObject({ + sessionId: parentSessionId, + sessionFile: parentSessionFile, + }); warn.mockRestore(); }); @@ -924,7 +946,7 @@ describe("initSessionState thread forking", () => { expect(sessionForkMocks.forkSessionFromParent).not.toHaveBeenCalled(); }); - it("records topic-specific session files when MessageThreadId is present", async () => { + it("records topic-specific SQLite session identity when MessageThreadId is present", async () => { const root = await makeCaseDir("openclaw-topic-session-"); const storePath = path.join(root, "sessions.json"); @@ -942,11 +964,17 @@ describe("initSessionState thread forking", () => { commandAuthorized: true, }); - const sessionFile = requireString(result.sessionEntry.sessionFile, "topic session file"); - expect(path.basename(sessionFile)).toBe(`${result.sessionEntry.sessionId}-topic-456.jsonl`); + expect(result.sessionEntry.lastThreadId).toBe(456); + expect(result.sessionEntry.sessionFile).toBe( + formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: result.sessionEntry.sessionId, + storePath, + }), + ); }); - it("records topic-specific session files from SessionKey when MessageThreadId is absent", async () => { + it("records topic-specific SQLite session identity from SessionKey", async () => { const root = await makeCaseDir("openclaw-topic-session-key-"); const storePath = path.join(root, "sessions.json"); @@ -965,8 +993,14 @@ describe("initSessionState thread forking", () => { commandAuthorized: true, }); - const sessionFile = requireString(result.sessionEntry.sessionFile, "topic session file"); - expect(path.basename(sessionFile)).toBe(`${result.sessionEntry.sessionId}-topic-456.jsonl`); + expect(result.sessionKey).toBe("agent:main:telegram:group:123:topic:456"); + expect(result.sessionEntry.sessionFile).toBe( + formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: result.sessionEntry.sessionId, + storePath, + }), + ); } finally { resetPluginRuntimeStateForTest(); } @@ -1114,7 +1148,7 @@ describe("initSessionState RawBody", () => { expect(result.sessionEntry.memoryFlushLastFailedAt).toBeUndefined(); expect(result.sessionEntry.memoryFlushLastFailureError).toBeUndefined(); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< + const store = readSessionStoreFast(storePath) as Record< string, { skillsSnapshot?: unknown; @@ -1244,7 +1278,7 @@ describe("initSessionState RawBody", () => { expect(result.sessionEntry.modelOverride).toBe("m2.7"); expect(result.sessionEntry.modelOverrideSource).toBe("user"); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< + const store = readSessionStoreFast(storePath) as Record< string, { providerOverride?: string; modelOverride?: string; modelOverrideSource?: string } >; @@ -1310,7 +1344,7 @@ describe("initSessionState RawBody", () => { expect(result.sessionEntry.ttsAuto).toBe("always"); expect(result.sessionEntry.pinnedAt).toBe(123); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< + const store = readSessionStoreFast(storePath) as Record< string, { thinkingLevel?: string; @@ -1395,7 +1429,7 @@ describe("initSessionState RawBody", () => { expect(result.sessionEntry.label).toBe("Other"); expect(result.sessionEntry.displayName).toBe("Dashboard Chat"); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< + const store = readSessionStoreFast(storePath) as Record< string, { label?: string; displayName?: string } >; @@ -1927,7 +1961,13 @@ describe("initSessionState RawBody", () => { }); expect(result.sessionEntry.sessionId).toBe(sessionId); - expect(result.sessionEntry.sessionFile).toBe(sessionFile); + expect(result.sessionEntry.sessionFile).toBe( + formatSqliteSessionFileMarker({ + agentId: "worker1", + sessionId, + storePath, + }), + ); expect(result.storePath).toBe(storePath); }); }); @@ -2294,10 +2334,7 @@ describe("initSessionState reset policy", () => { expect(result.isNewSession).toBe(false); expect(result.sessionId).toBe(existingSessionId); - const persisted = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; + const persisted = readSessionStoreFast(storePath); expect(persisted[sessionKey]?.sessionId).toBe(existingSessionId); expect(persisted[sessionKey]?.status).toBe("done"); expect(persisted[sessionKey]?.startedAt).toBe(Date.now() - 10_000); @@ -2401,10 +2438,7 @@ describe("initSessionState reset policy", () => { }); expect(result.isNewSession).toBe(scenario.expectNewSession); - const persisted = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; + const persisted = readSessionStoreFast(storePath); const entry = persisted[scenario.sessionKey]; if (scenario.expectNewSession) { expect(result.sessionId).not.toBe(existingSessionId); @@ -2467,7 +2501,7 @@ describe("initSessionState reset policy", () => { expect(result.abortedLastRun).toBe(false); expect(result.sessionEntry.abortedLastRun).toBeUndefined(); - const persisted = readSessionStoreForTest(storePath); + const persisted = readSessionStoreFast(storePath); expect(persisted[sessionKey]?.sessionId).toBe(existingSessionId); expect(persisted[sessionKey]?.status).toBeUndefined(); expect(persisted[sessionKey]?.startedAt).toBeUndefined(); @@ -3112,12 +3146,14 @@ describe("initSessionState reset triggers in WhatsApp groups", () => { } }); - it("starts a fresh session when a scoped WhatsApp group entry only contains activation state", async () => { + it("preserves scoped WhatsApp group activation state across stale rollover", async () => { const sessionKey = "agent:main:whatsapp:group:120363406150318674@g.us:thread:whatsapp-account-work"; const storePath = await createStorePath("openclaw-group-activation-backfill-"); await writeSessionStoreFast(storePath, { [sessionKey]: { + sessionId: "old-group-activation-session", + updatedAt: 0, groupActivation: "always", }, }); @@ -3266,7 +3302,7 @@ describe("initSessionState preserves behavior overrides across /new and /reset", }), ).rejects.toThrow(MODEL_SELECTION_LOCKED_RESET_MESSAGE); - expect(readSessionStoreForTest(storePath)[sessionKey]).toMatchObject({ + expect(loadSessionEntry({ storePath, sessionKey })).toMatchObject({ sessionId: existingSessionId, agentHarnessId: "codex", modelSelectionLocked: true, @@ -3431,7 +3467,7 @@ describe("initSessionState preserves behavior overrides across /new and /reset", result.sessionId, ]); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].usageFamilyKey, testCase.name).toBe("family:user-usage-family"); expect(stored[sessionKey].usageFamilySessionIds, testCase.name).toEqual([ "ancestor-session", @@ -3517,7 +3553,7 @@ describe("initSessionState preserves behavior overrides across /new and /reset", expect(result.sessionEntry.cliSessionBindings).toBeUndefined(); expect(result.sessionEntry.claudeCliSessionId).toBeUndefined(); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].cliSessionIds).toBeUndefined(); expect(stored[sessionKey].cliSessionBindings).toBeUndefined(); expect(stored[sessionKey].claudeCliSessionId).toBeUndefined(); @@ -3743,10 +3779,7 @@ describe("initSessionState preserves behavior overrides across /new and /reset", sessionId: existingSessionId, overrides: { ...runtimeModelCache, ...explicitUserOverride }, }); - const seeded = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; + const seeded = readSessionStoreFast(storePath); expect(seeded[sessionKey]?.modelProvider, testCase.name).toBe( runtimeModelCache.modelProvider, ); @@ -3795,10 +3828,7 @@ describe("initSessionState preserves behavior overrides across /new and /reset", // Unrelated behavior overrides still carry across the reset. expect(result.sessionEntry.verboseLevel, testCase.name).toBe(runtimeModelCache.verboseLevel); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].modelProvider, testCase.name).toBeUndefined(); expect(stored[sessionKey].model, testCase.name).toBeUndefined(); expect(stored[sessionKey].cacheRead, testCase.name).toBeUndefined(); @@ -4073,7 +4103,7 @@ describe("initSessionState preserves behavior overrides across /new and /reset", // Foreign owners may need the writer lane to finalize before releasing. // The rollover must not hold that lane while it drains them. await runExclusiveSessionStoreWrite(storePath, async () => {}); - expect(readSessionStoreForTest(storePath)[sessionKey]?.sessionId).toBe(existingSessionId); + expect(readSessionStoreFast(storePath)[sessionKey]?.sessionId).toBe(existingSessionId); expect(await fs.stat(transcriptPath).catch(() => null)).not.toBeNull(); admission.release(); @@ -4177,7 +4207,7 @@ describe("initSessionState preserves behavior overrides across /new and /reset", const outcomes = await Promise.allSettled([runRollover(0), runRollover(1)]); expect(outcomes.filter((outcome) => outcome.status === "fulfilled")).toHaveLength(1); expect(outcomes.filter((outcome) => outcome.status === "rejected")).toHaveLength(1); - expect(readSessionStoreForTest(storePath)[sessionKey]?.sessionId).not.toBe(existingSessionId); + expect(readSessionStoreFast(storePath)[sessionKey]?.sessionId).not.toBe(existingSessionId); }); it.each([ @@ -4863,10 +4893,9 @@ describe("persistSessionUsageUpdate", () => { entry: Record<string, unknown>; }) { await fs.mkdir(path.dirname(params.storePath), { recursive: true }); - await fs.writeFile( - params.storePath, - JSON.stringify({ [params.sessionKey]: params.entry }, null, 2), - "utf-8", + await replaceSessionEntry( + { storePath: params.storePath, sessionKey: params.sessionKey }, + params.entry as SessionEntry, ); } @@ -4890,7 +4919,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(12_000); expect(stored[sessionKey].totalTokensFresh).toBe(true); expect(stored[sessionKey].inputTokens).toBe(180_000); @@ -4926,7 +4955,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(148_874); expect(stored[sessionKey].totalTokensFresh).toBe(false); expect(stored[sessionKey].inputTokens).toBe(12); @@ -4954,7 +4983,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(0); expect(stored[sessionKey].totalTokensFresh).toBe(false); }); @@ -4981,7 +5010,7 @@ describe("persistSessionUsageUpdate", () => { preserveFreshTotalTokensOnStaleUsage: true, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(42_000); expect(stored[sessionKey].totalTokensFresh).toBe(true); }); @@ -5020,7 +5049,7 @@ describe("persistSessionUsageUpdate", () => { preserveRuntimeModel: true, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey]).toMatchObject({ modelProvider: "google", model: "gemini-3-pro", @@ -5072,11 +5101,12 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored1 = JSON.parse(await fs.readFile(storePath, "utf-8")); - expect(stored1[sessionKey].goal.tokenStart).toBe(100); - expect(stored1[sessionKey].goal.tokenStartFresh).toBe(true); - expect(stored1[sessionKey].goal.tokensUsed).toBe(0); - expect(stored1[sessionKey].goal.status).toBe("active"); + const stored1 = readSessionStoreFast(storePath); + const storedEntry1 = stored1[sessionKey]; + expect(storedEntry1?.goal?.tokenStart).toBe(100); + expect(storedEntry1?.goal?.tokenStartFresh).toBe(true); + expect(storedEntry1?.goal?.tokensUsed).toBe(0); + expect(storedEntry1?.goal?.status).toBe("active"); await persistSessionUsageUpdate({ storePath, @@ -5086,10 +5116,11 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored2 = JSON.parse(await fs.readFile(storePath, "utf-8")); - expect(stored2[sessionKey].goal.tokenStart).toBe(100); - expect(stored2[sessionKey].goal.tokensUsed).toBe(25); - expect(stored2[sessionKey].goal.status).toBe("budget_limited"); + const stored2 = readSessionStoreFast(storePath); + const storedEntry2 = stored2[sessionKey]; + expect(storedEntry2?.goal?.tokenStart).toBe(100); + expect(storedEntry2?.goal?.tokensUsed).toBe(25); + expect(storedEntry2?.goal?.status).toBe("budget_limited"); }); it("uses lastCallUsage cache counters when available", async () => { @@ -5119,7 +5150,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].inputTokens).toBe(100_000); expect(stored[sessionKey].outputTokens).toBe(8_000); expect(stored[sessionKey].cacheRead).toBe(18_000); @@ -5142,7 +5173,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBeUndefined(); expect(stored[sessionKey].totalTokensFresh).toBe(false); }); @@ -5169,7 +5200,7 @@ describe("persistSessionUsageUpdate", () => { preserveFreshTotalTokensOnStaleUsage: true, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(42_000); expect(stored[sessionKey].totalTokensFresh).toBe(true); }); @@ -5195,7 +5226,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(42_000); expect(stored[sessionKey].totalTokensFresh).toBe(false); }); @@ -5217,7 +5248,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(42_000); expect(stored[sessionKey].totalTokensFresh).toBe(true); }); @@ -5246,7 +5277,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(32_000); expect(stored[sessionKey].totalTokensFresh).toBe(true); expect(stored[sessionKey].cliSessionIds?.["claude-cli"]).toBe("cli-session-1"); @@ -5294,7 +5325,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].cliSessionIds?.["claude-cli"]).toBeUndefined(); expect(stored[sessionKey].cliSessionIds?.["codex-cli"]).toBe("codex-session"); expect(stored[sessionKey].cliSessionBindings?.["claude-cli"]).toBeUndefined(); @@ -5333,7 +5364,7 @@ describe("persistSessionUsageUpdate", () => { compactionTokensAfter: 0, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(1_794_391); expect(stored[sessionKey].totalTokensFresh).toBe(true); expect(stored[sessionKey].inputTokens).toBe(20); @@ -5366,7 +5397,7 @@ describe("persistSessionUsageUpdate", () => { compactionTokensAfter: 80_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(95_000); expect(stored[sessionKey].totalTokensFresh).toBe(true); expect(stored[sessionKey].inputTokens).toBe(100_000); @@ -5419,7 +5450,7 @@ describe("persistSessionUsageUpdate", () => { compactionTokensAfter: 80_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(80_000); expect(stored[sessionKey].totalTokensFresh).toBe(true); expect(stored[sessionKey].inputTokens).toBeUndefined(); @@ -5450,7 +5481,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(39_000); expect(stored[sessionKey].totalTokensFresh).toBe(true); expect(stored[sessionKey].inputTokens).toBe(1_234); @@ -5474,7 +5505,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].totalTokens).toBe(250_000); expect(stored[sessionKey].totalTokensFresh).toBe(true); }); @@ -5525,7 +5556,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored1 = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored1 = readSessionStoreFast(storePath); expect(stored1[sessionKey].estimatedCostUsd).toBeCloseTo(0.007725, 8); // Second persist with SAME cumulative usage (e.g., heartbeat or redundant persist) @@ -5542,7 +5573,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored2 = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored2 = readSessionStoreFast(storePath); // Cost should still be $0.007725, NOT $0.01545 expect(stored2[sessionKey].estimatedCostUsd).toBeCloseTo(0.007725, 8); }); @@ -5572,7 +5603,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 128_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].modelProvider).toBe("openai"); expect(stored[sessionKey].model).toBe("gpt-5.4"); expect(stored[sessionKey].inputTokens).toBe(1_200); @@ -5617,7 +5648,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 128_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].modelProvider).toBe("openai"); expect(stored[sessionKey].model).toBe("gpt-5.4"); expect(stored[sessionKey].cliSessionIds?.["claude-cli"]).toBe("new-heartbeat-cli-session"); @@ -5663,7 +5694,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 128_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].modelProvider).toBe("openai"); expect(stored[sessionKey].model).toBe("gpt-5.4"); expect(stored[sessionKey].cliSessionIds?.["claude-cli"]).toBeUndefined(); @@ -5730,7 +5761,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 900_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].modelProvider).toBe("openai"); expect(stored[sessionKey].model).toBe("gpt-5.5"); expect(stored[sessionKey].contextTokens).toBe(200_000); @@ -5791,7 +5822,7 @@ describe("persistSessionUsageUpdate", () => { contextTokensUsed: 200_000, }); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")); + const stored = readSessionStoreFast(storePath); expect(stored[sessionKey].estimatedCostUsd).toBe(0); }); }); @@ -5911,10 +5942,7 @@ describe("initSessionState dmScope delivery migration", () => { }); expect(result.sessionKey).toBe("agent:main:telegram:direct:6101296751"); - const persisted = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; + const persisted = readSessionStoreFast(storePath); expect(persisted["agent:main:main"]?.sessionId).toBe("legacy-main"); expect(persisted["agent:main:main"]?.route).toBeUndefined(); expect(persisted["agent:main:main"]?.deliveryContext).toBeUndefined(); @@ -5957,10 +5985,7 @@ describe("initSessionState dmScope delivery migration", () => { commandAuthorized: true, }); - const persisted = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; + const persisted = readSessionStoreFast(storePath); expect(persisted["agent:main:main"]?.deliveryContext).toEqual({ channel: "telegram", to: "1111", @@ -6035,10 +6060,7 @@ describe("initSessionState internal channel routing preservation", () => { accountId: "default", }); - const persisted = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - SessionEntry - >; + const persisted = readSessionStoreFast(storePath); expect(persisted[sessionKey]?.lastThreadId).toBeUndefined(); expect(persisted[sessionKey]?.deliveryContext).toEqual({ channel: "mattermost", diff --git a/src/auto-reply/reply/session.ts b/src/auto-reply/reply/session.ts index f5aeb1ab5319..834990aaa3a7 100644 --- a/src/auto-reply/reply/session.ts +++ b/src/auto-reply/reply/session.ts @@ -20,7 +20,7 @@ import { } from "../../config/sessions/lifecycle.js"; import { canonicalizeMainSessionAlias } from "../../config/sessions/main-session.js"; import { deriveSessionMetaPatch } from "../../config/sessions/metadata.js"; -import { resolveSessionTranscriptPath, resolveStorePath } from "../../config/sessions/paths.js"; +import { resolveStorePath } from "../../config/sessions/paths.js"; import { resolveResetPreservedSelection } from "../../config/sessions/reset-preserved-selection.js"; import { evaluateSessionFreshness, @@ -37,7 +37,10 @@ import { import { resolveSessionKey } from "../../config/sessions/session-key.js"; import { resolveMaintenanceConfigFromInput } from "../../config/sessions/store-maintenance.js"; import { runExclusiveSessionStoreWrite } from "../../config/sessions/store-writer.js"; -import { parseSessionThreadInfoFast } from "../../config/sessions/thread-info.js"; +import { + isRecoverableTerminalSessionStatus, + recoverTerminalSessionEntryForVisibleTurn, +} from "../../config/sessions/terminal-status.js"; import { DEFAULT_RESET_TRIGGERS, type GroupKeyResolution, @@ -174,21 +177,6 @@ function hasProviderOwnedSession(entry: SessionEntry | undefined): boolean { return Boolean(provider && getCliSessionBinding(entry, provider)); } -function isRecoverableTerminalSessionStatus(status: SessionEntry["status"] | undefined): boolean { - return status === "failed" || status === "timeout" || status === "killed"; -} - -function recoverTerminalSessionEntryForVisibleTurn(entry: SessionEntry): SessionEntry { - return { - ...entry, - status: undefined, - startedAt: undefined, - endedAt: undefined, - runtimeMs: undefined, - abortedLastRun: undefined, - }; -} - export type SessionInitResult = { sessionCtx: TemplateContext; sessionEntry: SessionEntry; @@ -211,9 +199,11 @@ export type SessionInitResult = { }; export type InitSessionStateParams = { + abortSignal?: AbortSignal; cfg: OpenClawConfig; commandAuthorized: boolean; ctx: MsgContext; + expectedExistingSessionId?: string; requestedSessionId?: string; resumeRequestedSession?: boolean; signal?: AbortSignal; @@ -634,6 +624,10 @@ async function initSessionStateAttemptLocked( Boolean(entry?.sessionId) && typeof entry?.updatedAt === "number" && Number.isFinite(entry.updatedAt); + const expectedExistingSessionId = params.expectedExistingSessionId?.trim() || undefined; + if (expectedExistingSessionId && entry?.sessionId !== expectedExistingSessionId) { + throw new Error(`session rebound for sessionKey: ${sessionKey}`); + } const requestedSessionId = params.requestedSessionId?.trim() || undefined; const requestedCurrentSession = Boolean( requestedSessionId && entry?.sessionId && entry.sessionId === requestedSessionId, @@ -996,16 +990,9 @@ async function initSessionStateAttemptLocked( } const parentSessionKey = normalizeOptionalString(ctx.ParentSessionKey); const alreadyForked = sessionEntry.forkedFromParent === true; - const threadIdFromSessionKey = parseSessionThreadInfoFast( - sessionCtxForState.SessionKey ?? sessionKey, - ).threadId; - const fallbackSessionFile = !sessionEntry.sessionFile - ? resolveSessionTranscriptPath( - sessionEntry.sessionId, - agentId, - ctx.MessageThreadId ?? threadIdFromSessionKey, - ) - : undefined; + if (params.abortSignal?.aborted === true) { + throw new Error("reply session initialization aborted"); + } if (isNewSession) { sessionEntry.compactionCount = 0; sessionEntry.memoryFlushCompactionCount = undefined; @@ -1050,7 +1037,6 @@ async function initSessionStateAttemptLocked( activeSessionKey: sessionKey, agentId, expectedRevision: initializationSnapshot.revision, - fallbackSessionFile, maintenanceConfig, onArchiveError: (error, sourcePath) => { log.warn( @@ -1065,8 +1051,11 @@ async function initSessionStateAttemptLocked( entry: sessionEntry, warning, }), - prepareSessionEntry: async ({ readEntry, sessionEntry: entryToCommit }) => - await prepareReplySessionParentFork({ + prepareSessionEntry: async ({ readEntry, sessionEntry: entryToCommit }) => { + if (params.abortSignal?.aborted === true) { + throw new Error("reply session initialization aborted"); + } + return await prepareReplySessionParentFork({ agentId, alreadyForked, parentSessionKey, @@ -1075,7 +1064,8 @@ async function initSessionStateAttemptLocked( sessionKey, storePath, warn: (message) => log.warn(message), - }), + }); + }, previousEntry: previousSessionEntry, retiredEntry: retiredLegacyMainDelivery, sessionEntry, diff --git a/src/channels/turn/message-turn-guardrails.test.ts b/src/channels/turn/message-turn-guardrails.test.ts index c3d5a425a2e0..5b190c22e7b3 100644 --- a/src/channels/turn/message-turn-guardrails.test.ts +++ b/src/channels/turn/message-turn-guardrails.test.ts @@ -37,7 +37,6 @@ const historyWindowFiles = [ "extensions/qqbot/src/bridge/sdk-adapter.ts", "extensions/signal/src/monitor/event-handler.ts", "extensions/slack/src/monitor/message-handler/prepare.ts", - "extensions/telegram/src/bot-message-context.session.ts", "extensions/telegram/src/bot-message-dispatch.ts", "extensions/telegram/src/group-history-window.ts", "extensions/whatsapp/src/auto-reply/monitor/group-gating.ts", diff --git a/src/cli/program/register.maintenance.test.ts b/src/cli/program/register.maintenance.test.ts index cd226ab410ff..d35a98cfb4f3 100644 --- a/src/cli/program/register.maintenance.test.ts +++ b/src/cli/program/register.maintenance.test.ts @@ -130,6 +130,68 @@ describe("registerMaintenanceCommands doctor action", () => { expect(options.crossStateDirImports).toBe(false); }); + it("passes session sqlite options to doctor command", async () => { + doctorCommand.mockResolvedValue(undefined); + + await runMaintenanceCli([ + "doctor", + "--session-sqlite", + "import", + "--session-sqlite-store", + "/tmp/openclaw/sessions.json", + "--json", + ]); + + expect(doctorCommand).toHaveBeenCalledTimes(1); + const [runtimeArg, options] = commandCall(doctorCommand); + expect(runtimeArg).toBe(runtime); + expect(options.sessionSqlite).toBe("import"); + expect(options.sessionSqliteStore).toBe("/tmp/openclaw/sessions.json"); + expect(options.json).toBe(true); + expect(runtime.exit).toHaveBeenCalledWith(0); + }); + + it("passes session sqlite recover GitHub issue option to doctor command", async () => { + doctorCommand.mockResolvedValue(undefined); + + await runMaintenanceCli(["doctor", "--session-sqlite", "recover", "--github-issue", "--yes"]); + + expect(doctorCommand).toHaveBeenCalledTimes(1); + const [, options] = commandCall(doctorCommand); + expect(options.sessionSqlite).toBe("recover"); + expect(options.sessionSqliteGithubIssue).toBe(true); + expect(options.yes).toBe(true); + expect(runtime.exit).toHaveBeenCalledWith(0); + }); + + it("passes session sqlite compact mode to doctor command", async () => { + doctorCommand.mockResolvedValue(undefined); + + await runMaintenanceCli([ + "doctor", + "--session-sqlite", + "compact", + "--session-sqlite-agent", + "main", + ]); + + expect(doctorCommand).toHaveBeenCalledTimes(1); + const [, options] = commandCall(doctorCommand); + expect(options.sessionSqlite).toBe("compact"); + expect(options.sessionSqliteAgent).toBe("main"); + expect(runtime.exit).toHaveBeenCalledWith(0); + }); + + it("rejects session sqlite selectors without session sqlite mode", async () => { + await runMaintenanceCli(["doctor", "--session-sqlite-agent", "main"]); + + expect(doctorCommand).not.toHaveBeenCalled(); + expect(runtime.error).toHaveBeenCalledWith( + "doctor session SQLite options require --session-sqlite. Use `openclaw doctor --session-sqlite dry-run ...`.", + ); + expect(runtime.exit).toHaveBeenCalledWith(2); + }); + it("runs doctor lint mode without invoking repair doctor", async () => { runDoctorLintCli.mockResolvedValue(1); diff --git a/src/cli/program/register.maintenance.ts b/src/cli/program/register.maintenance.ts index 3a8ad1b63153..d5bb3b9e1730 100644 --- a/src/cli/program/register.maintenance.ts +++ b/src/cli/program/register.maintenance.ts @@ -35,7 +35,27 @@ export function registerMaintenanceCommands(program: Command) { "Emit plugin-compat findings only (machine-readable with --json)", false, ) - .option("--json", "With --lint or --post-upgrade: emit machine-readable JSON output", false) + .option( + "--session-sqlite <mode>", + "Run session SQLite migration mode (dry-run|import|validate|inspect|compact|restore|recover)", + ) + .option("--session-sqlite-store <path>", "With --session-sqlite: inspect one session store") + .option("--session-sqlite-agent <id>", "With --session-sqlite: inspect one agent") + .option( + "--session-sqlite-all-agents", + "With --session-sqlite: inspect configured and discovered agent stores", + false, + ) + .option( + "--github-issue", + "With --session-sqlite recover: prepare and optionally create an openclaw/openclaw issue", + false, + ) + .option( + "--json", + "With --lint, --post-upgrade, or --session-sqlite: emit machine-readable JSON output", + false, + ) .option( "--severity-min <level>", "With --lint: drop findings below this severity (info|warning|error)", @@ -77,6 +97,13 @@ export function registerMaintenanceCommands(program: Command) { ); return; } + if (hasSessionSqliteOnlyDoctorOptions(opts)) { + defaultRuntime.error( + "doctor session SQLite options require --session-sqlite. Use `openclaw doctor --session-sqlite dry-run ...`.", + ); + defaultRuntime.exit(2); + return; + } if (hasLintOnlyDoctorOptions(opts)) { defaultRuntime.error( "doctor lint options require --lint. Use `openclaw doctor --lint ...`.", @@ -86,6 +113,7 @@ export function registerMaintenanceCommands(program: Command) { } await runCommandWithRuntime(defaultRuntime, async () => { const { doctorCommand } = await import("../../commands/doctor.js"); + const sessionSqlite = parseDoctorSessionSqliteMode(opts.sessionSqlite); await doctorCommand(defaultRuntime, { workspaceSuggestions: opts.workspaceSuggestions, yes: Boolean(opts.yes), @@ -96,6 +124,15 @@ export function registerMaintenanceCommands(program: Command) { allowExec: Boolean(opts.allowExec), deep: Boolean(opts.deep), postUpgrade: Boolean(opts.postUpgrade), + ...(sessionSqlite ? { sessionSqlite } : {}), + ...(typeof opts.sessionSqliteStore === "string" + ? { sessionSqliteStore: opts.sessionSqliteStore } + : {}), + ...(typeof opts.sessionSqliteAgent === "string" + ? { sessionSqliteAgent: opts.sessionSqliteAgent } + : {}), + sessionSqliteAllAgents: Boolean(opts.sessionSqliteAllAgents), + sessionSqliteGithubIssue: Boolean(opts.githubIssue), json: Boolean(opts.json), crossStateDirImports: resolveDoctorCrossStateDirImports(), }); @@ -183,16 +220,57 @@ export function registerMaintenanceCommands(program: Command) { function hasLintOnlyDoctorOptions(opts: { readonly json?: boolean; readonly postUpgrade?: boolean; + readonly sessionSqlite?: unknown; readonly severityMin?: unknown; readonly all?: boolean; readonly skip?: unknown; readonly only?: unknown; }): boolean { return ( - (opts.json === true && opts.postUpgrade !== true) || + (opts.json === true && opts.postUpgrade !== true && typeof opts.sessionSqlite !== "string") || typeof opts.severityMin === "string" || opts.all === true || (Array.isArray(opts.skip) && opts.skip.length > 0) || (Array.isArray(opts.only) && opts.only.length > 0) ); } + +function hasSessionSqliteOnlyDoctorOptions(opts: { + readonly sessionSqlite?: unknown; + readonly sessionSqliteAgent?: unknown; + readonly sessionSqliteAllAgents?: unknown; + readonly githubIssue?: unknown; + readonly sessionSqliteStore?: unknown; +}): boolean { + return ( + typeof opts.sessionSqlite !== "string" && + (typeof opts.sessionSqliteAgent === "string" || + opts.githubIssue === true || + opts.sessionSqliteAllAgents === true || + typeof opts.sessionSqliteStore === "string") + ); +} + +function parseDoctorSessionSqliteMode( + value: unknown, +): "dry-run" | "import" | "validate" | "inspect" | "compact" | "restore" | "recover" | undefined { + if (value === undefined) { + return undefined; + } + if ( + value === "dry-run" || + value === "import" || + value === "validate" || + value === "inspect" || + value === "compact" || + value === "restore" || + value === "recover" + ) { + return value; + } + defaultRuntime.error( + "Invalid --session-sqlite mode. Use dry-run, import, validate, inspect, compact, restore, or recover.", + ); + defaultRuntime.exit(2); + throw new Error("unreachable"); +} diff --git a/src/commands/agent.session.test.ts b/src/commands/agent.session.test.ts index ace114cdf7d8..850f7f588045 100644 --- a/src/commands/agent.session.test.ts +++ b/src/commands/agent.session.test.ts @@ -6,9 +6,10 @@ import { beforeEach, describe, expect, it } from "vitest"; import { resolveAgentDir, resolveSessionAgentId } from "../agents/agent-scope.js"; import { updateSessionStoreAfterAgentRun } from "../agents/command/session-store.js"; import { resolveSession } from "../agents/command/session.js"; -import { loadSessionStore } from "../config/sessions/store-load.js"; +import { loadSessionEntry, replaceSessionEntry } from "../config/sessions/session-accessor.js"; import { clearSessionStoreCacheForTest } from "../config/sessions/store.js"; import { resolveSessionTranscriptFile } from "../config/sessions/transcript.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { buildOutboundSessionContext } from "../infra/outbound/session-context.js"; @@ -37,23 +38,26 @@ function mockConfig( } as OpenClawConfig; } -function writeSessionStoreSeed( +async function writeSessionStoreSeed( storePath: string, - sessions: Record<string, Record<string, unknown>>, -) { - fs.mkdirSync(path.dirname(storePath), { recursive: true }); - fs.writeFileSync(storePath, JSON.stringify(sessions)); + sessions: Record<string, SessionEntry>, +): Promise<void> { + await Promise.all( + Object.entries(sessions).map(([sessionKey, entry]) => + replaceSessionEntry({ sessionKey, storePath }, entry), + ), + ); } async function withCrossAgentResumeFixture( run: (params: { sessionId: string; sessionKey: string; cfg: OpenClawConfig }) => Promise<void>, ): Promise<void> { await withTempHome(async (home) => { - const storePattern = path.join(home, "sessions", "{agentId}", "sessions.json"); - const execStore = path.join(home, "sessions", "exec", "sessions.json"); + const storePattern = path.join(home, "agents", "{agentId}", "sessions", "sessions.json"); + const execStore = path.join(home, "agents", "exec", "sessions", "sessions.json"); const sessionId = "session-exec-hook"; const sessionKey = "agent:exec:hook:gmail:thread-1"; - writeSessionStoreSeed(execStore, { + await writeSessionStoreSeed(execStore, { [sessionKey]: { sessionId, updatedAt: Date.now(), @@ -96,16 +100,16 @@ describe("agent session resolution", () => { it("resolves duplicate cross-agent sessionIds deterministically", async () => { await withTempHome(async (home) => { - const storePattern = path.join(home, "sessions", "{agentId}", "sessions.json"); - const otherStore = path.join(home, "sessions", "other", "sessions.json"); - const retiredStore = path.join(home, "sessions", "retired", "sessions.json"); - writeSessionStoreSeed(otherStore, { + const storePattern = path.join(home, "agents", "{agentId}", "sessions", "sessions.json"); + const otherStore = path.join(home, "agents", "other", "sessions", "sessions.json"); + const retiredStore = path.join(home, "agents", "retired", "sessions", "sessions.json"); + await writeSessionStoreSeed(otherStore, { "agent:other:main": { sessionId: "run-dup", updatedAt: Date.now() + 1_000, }, }); - writeSessionStoreSeed(retiredStore, { + await writeSessionStoreSeed(retiredStore, { "agent:retired:acp:run-dup": { sessionId: "run-dup", updatedAt: Date.now(), @@ -126,7 +130,7 @@ describe("agent session resolution", () => { it("uses origin.provider for channel-specific session reset overrides", async () => { await withTempHome(async (home) => { const store = path.join(home, "sessions.json"); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { main: { sessionId: "origin-provider-reset", updatedAt: Date.now() - 30 * 60_000, @@ -200,7 +204,7 @@ describe("agent session resolution", () => { (registryUpdatedAt + 5_000) / 1000, (registryUpdatedAt + 5_000) / 1000, ); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [scenario.sessionKey]: { sessionId, sessionFile, @@ -273,9 +277,10 @@ describe("agent session resolution", () => { }, } as never, }); - const persisted = loadSessionStore(resolution.storePath, { skipCache: true })[ - scenario.sessionKey - ]; + const persisted = loadSessionEntry({ + sessionKey: scenario.sessionKey, + storePath: resolution.storePath, + }); expect(persisted?.sessionId).toBe(resolution.sessionId); expect(persisted?.sessionFile).not.toBe(sessionFile); expect(persisted?.status).toBeUndefined(); @@ -303,7 +308,7 @@ describe("agent session resolution", () => { (registryUpdatedAt + 5_000) / 1000, (registryUpdatedAt + 5_000) / 1000, ); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { "agent:main:main": { sessionId, sessionFile, @@ -338,17 +343,16 @@ describe("agent session resolution", () => { storePath: resolution.storePath, agentId: "main", }); - expect(fs.realpathSync.native(resolvedTranscript.sessionFile)).toBe( - fs.realpathSync.native(sessionFile), + expect(resolvedTranscript.sessionFile).toBe( + `sqlite:main:${sessionId}:${resolution.storePath}`, ); - const persisted = loadSessionStore(resolution.storePath, { skipCache: true })[ - resolution.sessionKey - ]; + const persisted = loadSessionEntry({ + sessionKey: resolution.sessionKey, + storePath: resolution.storePath, + }); expect(persisted?.sessionId).toBe(sessionId); - expect(fs.realpathSync.native(persisted?.sessionFile ?? "")).toBe( - fs.realpathSync.native(sessionFile), - ); + expect(persisted?.sessionFile).toBe(resolvedTranscript.sessionFile); expect(persisted?.status).toBe("done"); expect(persisted?.startedAt).toBe(registryUpdatedAt - 1_000); expect(persisted?.endedAt).toBe(registryUpdatedAt - 100); diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 19afd157a3fd..107303b67a47 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -16,8 +16,14 @@ import { isAgentRunRestartAbortReason } from "../agents/run-termination.js"; import { ensureAgentWorkspace } from "../agents/workspace.js"; import { BASE_THINKING_LEVELS } from "../auto-reply/thinking.shared.js"; import * as runtimeSnapshotModule from "../config/runtime-snapshot.js"; -import { loadSessionStore } from "../config/sessions/store-load.js"; +import { + listSessionEntries, + loadSessionEntry, + replaceSessionEntry, +} from "../config/sessions/session-accessor.js"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { clearSessionStoreCacheForTest } from "../config/sessions/store.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { emitAgentEvent, @@ -77,10 +83,10 @@ vi.mock("../agents/auth-profiles/source-check.js", () => ({ hasAnyAuthProfileStoreSource: vi.fn(() => false), })); -vi.mock("../agents/command/session-store.runtime.js", () => { +vi.mock("../agents/command/session-store.runtime.js", async () => { + const accessor = await import("../config/sessions/session-accessor.js"); return { - loadSessionEntry: ({ storePath, sessionKey }: { storePath: string; sessionKey: string }) => - loadSessionStore(storePath, { skipCache: true })[sessionKey], + loadSessionEntry: accessor.loadSessionEntry, updateSessionStoreAfterAgentRun: vi.fn(async () => undefined), }; }); @@ -224,25 +230,6 @@ vi.mock("../agents/command/delivery.runtime.js", () => { }); vi.mock("../config/sessions/transcript-resolve.runtime.js", () => { - const dirname = (filePath: string): string => { - const lastSlash = Math.max(filePath.lastIndexOf("/"), filePath.lastIndexOf("\\")); - return lastSlash >= 0 ? filePath.slice(0, lastSlash) : "."; - }; - const joinPath = (...parts: string[]): string => { - const separator = parts.some((part) => part.includes("\\")) ? "\\" : "/"; - const normalizedParts: string[] = []; - for (const [index, part] of parts.entries()) { - const normalized = - index === 0 ? part.replace(/[\\/]+$/u, "") : part.replace(/^[\\/]+|[\\/]+$/gu, ""); - if (normalized.length > 0) { - normalizedParts.push(normalized); - } - } - return normalizedParts.join(separator); - }; - const resolveSessionFile = (sessionId: string, agentId: string, sessionsDir?: string): string => - joinPath(sessionsDir ?? ".openclaw", "agents", agentId, "sessions", `${sessionId}.jsonl`); - return { resolveSessionTranscriptFile: vi.fn( async (params: { @@ -254,15 +241,11 @@ vi.mock("../config/sessions/transcript-resolve.runtime.js", () => { agentId: string; threadId?: string | number; }) => { - const sessionsDir = params.storePath ? dirname(params.storePath) : undefined; - const sessionFileFromStorePath = + const sessionFile = params.sessionEntry?.sessionFile ?? - resolveSessionFile(params.sessionId, params.agentId, sessionsDir); - const sessionFile = params.sessionEntry?.sessionFile - ? sessionFileFromStorePath - : resolveSessionFile(params.sessionId, params.agentId, sessionsDir); + `sqlite:${params.agentId}:${params.sessionId}:${params.storePath ?? ""}`; let sessionEntry = params.sessionEntry; - if (params.sessionStore && params.storePath && params.sessionKey) { + if (params.sessionStore && params.sessionKey) { const existingEntry = params.sessionStore[params.sessionKey] ?? {}; sessionEntry = { ...existingEntry, @@ -270,7 +253,6 @@ vi.mock("../config/sessions/transcript-resolve.runtime.js", () => { sessionFile, }; params.sessionStore[params.sessionKey] = sessionEntry; - fs.writeFileSync(params.storePath, JSON.stringify(params.sessionStore)); } return { sessionFile, sessionEntry }; }, @@ -314,12 +296,19 @@ function mockConfig( return cfg; } -function writeSessionStoreSeed( +async function writeSessionStoreSeed( storePath: string, sessions: Record<string, Record<string, unknown>>, -) { +): Promise<void> { fs.mkdirSync(path.dirname(storePath), { recursive: true }); - fs.writeFileSync(storePath, JSON.stringify(sessions)); + for (const [sessionKey, entry] of Object.entries(sessions)) { + const sessionId = typeof entry.sessionId === "string" ? entry.sessionId : sessionKey; + await replaceSessionEntry({ sessionKey, storePath }, { + ...entry, + sessionId, + updatedAt: typeof entry.updatedAt === "number" ? entry.updatedAt : Date.now(), + } as SessionEntry); + } } function createDefaultAgentResult(params?: { @@ -347,7 +336,25 @@ function expectLastRunProviderModel(provider: string, model: string): void { } function readSessionStore<T>(storePath: string): Record<string, T> { - return JSON.parse(fs.readFileSync(storePath, "utf-8")) as Record<string, T>; + return Object.fromEntries( + listSessionEntries({ storePath }).map(({ entry, sessionKey }) => [sessionKey, entry as T]), + ); +} + +function expectSqliteSessionFileMarker(params: { + agentId: string; + sessionFile: string | undefined; + sessionId?: string; + storePath: string; +}): void { + const marker = parseSqliteSessionFileMarker(params.sessionFile); + expect(marker?.agentId).toBe(params.agentId); + if (params.sessionId) { + expect(marker?.sessionId).toBe(params.sessionId); + } else { + expect(marker?.sessionId).toBeTruthy(); + } + expect(marker?.storePath).toBe(path.resolve(params.storePath)); } async function runAgentWithSessionKey(sessionKey: string): Promise<void> { @@ -512,7 +519,7 @@ describe("agentCommand", () => { const store = path.join(home, "sessions.json"); const sessionKey = "agent:main:harness:openclaw:supervision:existing"; mockConfig(home, store); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId: "existing-harness-session", updatedAt: Date.now(), @@ -541,7 +548,7 @@ describe("agentCommand", () => { const sessionKey = "agent:main:discord:channel:voice-1"; const staleStartedAt = Date.now() - 2 * 24 * 60 * 60_000; mockConfig(home, store); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId: "stale-voice-session", updatedAt: staleStartedAt, @@ -589,7 +596,7 @@ describe("agentCommand", () => { await withTempHome(async (home) => { const store = path.join(home, "sessions.json"); mockConfig(home, store); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { "agent:main:subagent:archived": { sessionId: "archived-session-id", archivedAt: Date.now(), @@ -620,11 +627,11 @@ describe("agentCommand", () => { const sessionKey = "agent:main:subagent:archive-race"; const sessionId = "archive-race-session-id"; mockConfig(home, store); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId, updatedAt: Date.now() }, }); vi.mocked(ensureAgentWorkspace).mockImplementationOnce(async (params) => { - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId, archivedAt: Date.now(), @@ -656,7 +663,7 @@ describe("agentCommand", () => { const sessionKey = "agent:main:subagent:restore-race"; const sessionId = "restore-race-session-id"; mockConfig(home, store); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId, archivedAt: Date.now(), @@ -664,7 +671,7 @@ describe("agentCommand", () => { }, }); vi.mocked(ensureAgentWorkspace).mockImplementationOnce(async (params) => { - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId, updatedAt: Date.now() }, }); return { dir: params?.dir ?? "/tmp/openclaw-workspace" }; @@ -692,7 +699,7 @@ describe("agentCommand", () => { const sessionKey = "agent:main:subagent:in-band-lifecycle"; const sessionId = "in-band-lifecycle-session-id"; mockConfig(home, store); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId, updatedAt: Date.now() }, }); vi.mocked(runEmbeddedAgent).mockImplementationOnce(async () => { @@ -725,7 +732,7 @@ describe("agentCommand", () => { const sessionKey = "agent:main:subagent:lifecycle-restart"; const sessionId = "lifecycle-restart-session-id"; mockConfig(home, store); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId, updatedAt: Date.now() }, }); let observedAbortReason: unknown; @@ -771,7 +778,7 @@ describe("agentCommand", () => { const store = path.join(home, "sessions.json"); const sessionKey = "agent:main:subagent:stale-request"; mockConfig(home, store); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId: "current-session-id", updatedAt: Date.now() }, }); @@ -859,10 +866,7 @@ describe("agentCommand", () => { runtime, ); - const saved = JSON.parse(fs.readFileSync(store, "utf-8")) as Record< - string, - { thinkingLevel?: string; verboseLevel?: string } - >; + const saved = readSessionStore<{ thinkingLevel?: string; verboseLevel?: string }>(store); const entry = Object.values(saved)[0]; expect(entry.thinkingLevel).toBe("high"); expect(entry.verboseLevel).toBe("on"); @@ -1052,7 +1056,7 @@ describe("agentCommand", () => { const store = path.join(home, "sessions.json"); const sessionKey = "agent:main:main"; mockConfig(home, store, { models: {} }); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId: "acp-backed-session", updatedAt: Date.now(), @@ -1101,7 +1105,7 @@ describe("agentCommand", () => { await withTempHome(async (home) => { const store = path.join(home, "sessions.json"); const sessionKey = "agent:main:cache-borrow"; - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId: "session-cache-borrow", updatedAt: Date.now(), @@ -1121,10 +1125,10 @@ describe("agentCommand", () => { }, runtime, ); - const cached = loadSessionStore(store, { clone: false }); + const cached = loadSessionEntry({ storePath: store, sessionKey, clone: false }); expect(prepared.sessionStore).not.toBe(cached); - expect(prepared.sessionEntry).not.toBe(cached[sessionKey]); + expect(prepared.sessionEntry).not.toBe(cached); expect(prepared.sessionStore?.[sessionKey]).toBe(prepared.sessionEntry); expect(prepared.sessionStore?.["agent:main:other"]).toBeUndefined(); }); @@ -1133,7 +1137,7 @@ describe("agentCommand", () => { it("passes resolved session-id resume files to embedded runs", async () => { await withTempHome(async (home) => { const resumeStore = path.join(home, "sessions-resume.json"); - writeSessionStoreSeed(resumeStore, { + await writeSessionStoreSeed(resumeStore, { foo: { sessionId: "session-123", updatedAt: Date.now(), @@ -1149,9 +1153,12 @@ describe("agentCommand", () => { const callArgs = getLastEmbeddedCall(); expect(callArgs?.sessionId).toBe("session-123"); - expect(callArgs?.sessionFile).toContain( - `${path.dirname(resumeStore)}${path.sep}agents${path.sep}main${path.sep}sessions${path.sep}session-123.jsonl`, - ); + expectSqliteSessionFileMarker({ + agentId: "main", + sessionFile: callArgs?.sessionFile, + sessionId: "session-123", + storePath: resumeStore, + }); }); }); @@ -1235,7 +1242,7 @@ describe("agentCommand", () => { it("probes the configured primary first for origin-backed auto session model overrides", async () => { await withTempHome(async (home) => { const store = path.join(home, "sessions.json"); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { "agent:main:subagent:test": { sessionId: "session-subagent", updatedAt: Date.now(), @@ -1296,7 +1303,7 @@ describe("agentCommand", () => { await withTempHome(async (home) => { const store = path.join(home, "sessions-locked-model.json"); const sessionKey = "agent:main:subagent:locked-model"; - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId: "session-locked-model", updatedAt: Date.now(), @@ -1340,7 +1347,7 @@ describe("agentCommand", () => { it("clears legacy auto session model overrides without origin metadata", async () => { await withTempHome(async (home) => { const store = path.join(home, "sessions-legacy-auto-override.json"); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { "agent:main:subagent:legacy-auto": { sessionId: "session-legacy-auto", updatedAt: Date.now(), @@ -1396,7 +1403,7 @@ describe("agentCommand", () => { it("does not repair locked legacy auto session model overrides", async () => { await withTempHome(async (home) => { const store = path.join(home, "sessions-locked-legacy-auto-override.json"); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { "agent:main:subagent:locked-legacy-auto": { sessionId: "session-locked-legacy-auto", updatedAt: Date.now(), @@ -1457,7 +1464,7 @@ describe("agentCommand", () => { it("does not use fallback list for user session model overrides", async () => { await withTempHome(async (home) => { const store = path.join(home, "sessions-user-override.json"); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { "agent:main:subagent:user-override": { sessionId: "session-user-override", updatedAt: Date.now(), @@ -1506,7 +1513,7 @@ describe("agentCommand", () => { it("clears disallowed stored override fields", async () => { await withTempHome(async (home) => { const clearStore = path.join(home, "sessions-clear-overrides.json"); - writeSessionStoreSeed(clearStore, { + await writeSessionStoreSeed(clearStore, { "agent:main:subagent:clear-overrides": { sessionId: "session-clear-overrides", updatedAt: Date.now(), @@ -1563,7 +1570,7 @@ describe("agentCommand", () => { await withTempHome(async (home) => { const store = path.join(home, "sessions-locked-disallowed-override.json"); const sessionKey = "agent:main:subagent:locked-disallowed"; - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId: "session-locked-disallowed", updatedAt: Date.now(), @@ -1610,7 +1617,7 @@ describe("agentCommand", () => { await withTempHome(async (home) => { const store = path.join(home, "sessions-locked-one-off-override.json"); const sessionKey = "agent:main:subagent:locked-one-off"; - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId: "session-locked-one-off", updatedAt: Date.now(), @@ -1675,7 +1682,7 @@ describe("agentCommand", () => { expect(saved["agent:main:subagent:run-override"]?.providerOverride).toBeUndefined(); expect(saved["agent:main:subagent:run-override"]?.modelOverride).toBeUndefined(); - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { "agent:main:subagent:temp-openai-run": { sessionId: "session-temp-openai-run", updatedAt: Date.now(), @@ -1799,7 +1806,11 @@ describe("agentCommand", () => { ); let callArgs = getLastEmbeddedCall(); expect(callArgs?.sessionKey).toBe("agent:ops:main"); - expect(callArgs?.sessionFile).toContain(`${path.sep}agents${path.sep}ops${path.sep}sessions`); + expectSqliteSessionFileMarker({ + agentId: "ops", + sessionFile: callArgs?.sessionFile, + storePath: store, + }); expect(callArgs?.messageChannel).toBe("slack"); expect(runtime.log).toHaveBeenCalledWith("ok"); @@ -1907,7 +1918,11 @@ describe("agentCommand", () => { let callArgs = getLastEmbeddedCall(); expect(callArgs?.agentId).toBe("ops"); expect(callArgs?.sessionKey).toBe("agent:ops:incident-42"); - expect(callArgs?.sessionFile).toContain(`${path.sep}agents${path.sep}ops${path.sep}sessions`); + expectSqliteSessionFileMarker({ + agentId: "ops", + sessionFile: callArgs?.sessionFile, + storePath: store, + }); await agentCommand({ message: "hi", agentId: "ops", sessionKey: "incident-42" }, runtime); @@ -1920,7 +1935,11 @@ describe("agentCommand", () => { callArgs = getLastEmbeddedCall(); expect(callArgs?.agentId).toBe("ops"); expect(callArgs?.sessionKey).toBe("agent:ops:global"); - expect(callArgs?.sessionFile).toContain(`${path.sep}agents${path.sep}ops${path.sep}sessions`); + expectSqliteSessionFileMarker({ + agentId: "ops", + sessionFile: callArgs?.sessionFile, + storePath: store, + }); }); }); @@ -1928,7 +1947,7 @@ describe("agentCommand", () => { await withTempHome(async (home) => { const store = path.join(home, "sessions.json"); const sessionKey = "agent:main:openclaw-weixin:direct:o9cq802hhmfc@im.wechat"; - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId: "wechat-session", updatedAt: Date.now() }, }); mockConfig(home, store, undefined, undefined, [{ id: "main" }, { id: "work" }]); @@ -1944,7 +1963,7 @@ describe("agentCommand", () => { await withTempHome(async (home) => { const store = path.join(home, "sessions.json"); const sessionKey = "agent:main:openclaw-weixin:direct:o9cq802hhmfc@im.wechat"; - writeSessionStoreSeed(store, { + await writeSessionStoreSeed(store, { [sessionKey]: { sessionId: "wechat-session", updatedAt: Date.now(), @@ -1993,14 +2012,22 @@ describe("agentCommand", () => { callArgs = getLastEmbeddedCall(); expect(callArgs?.agentId).toBe("ops"); expect(callArgs?.sessionKey).toBe("global"); - expect(callArgs?.sessionFile).toContain(`${path.sep}agents${path.sep}ops${path.sep}sessions`); + expectSqliteSessionFileMarker({ + agentId: "ops", + sessionFile: callArgs?.sessionFile, + storePath: store, + }); await agentCommand({ message: "hi", sessionKey: "unknown" }, runtime); callArgs = getLastEmbeddedCall(); expect(callArgs?.agentId).toBe("ops"); expect(callArgs?.sessionKey).toBe("unknown"); - expect(callArgs?.sessionFile).toContain(`${path.sep}agents${path.sep}ops${path.sep}sessions`); + expectSqliteSessionFileMarker({ + agentId: "ops", + sessionFile: callArgs?.sessionFile, + storePath: store, + }); }); }); }); diff --git a/src/commands/agent/session.test.ts b/src/commands/agent/session.test.ts index acb8e6e97411..59432d9a11f4 100644 --- a/src/commands/agent/session.test.ts +++ b/src/commands/agent/session.test.ts @@ -4,7 +4,7 @@ import type { OpenClawConfig } from "../../config/config.js"; import { resolveSessionKeyForRequest } from "./session.js"; const mocks = vi.hoisted(() => ({ - loadSessionStore: vi.fn(), + listSessionEntries: vi.fn(), resolveStorePath: vi.fn(), listAgentIds: vi.fn(), resolveExplicitAgentSessionKey: vi.fn(), @@ -20,8 +20,8 @@ vi.mock("../../config/sessions/main-session.js", async () => { }; }); -vi.mock("../../config/sessions/store-load.js", () => ({ - loadSessionStore: mocks.loadSessionStore, +vi.mock("../../config/sessions/session-accessor.js", () => ({ + listSessionEntries: mocks.listSessionEntries, })); vi.mock("../../config/sessions/paths.js", () => ({ @@ -61,7 +61,12 @@ describe("resolveSessionKeyForRequest", () => { }; const mockStoresByPath = (stores: Partial<Record<string, SessionStoreMap>>) => { - mocks.loadSessionStore.mockImplementation((storePath: string) => stores[storePath] ?? {}); + mocks.listSessionEntries.mockImplementation((scope?: { storePath?: string }) => + Object.entries(stores[scope?.storePath ?? ""] ?? {}).map(([sessionKey, entry]) => ({ + sessionKey, + entry, + })), + ); }; beforeEach(() => { @@ -74,8 +79,10 @@ describe("resolveSessionKeyForRequest", () => { it("returns sessionKey when --to resolves a session key via context", () => { mocks.resolveStorePath.mockReturnValue(MAIN_STORE_PATH); - mocks.loadSessionStore.mockReturnValue({ - "agent:main:main": { sessionId: "sess-1", updatedAt: 0 }, + mockStoresByPath({ + [MAIN_STORE_PATH]: { + "agent:main:main": { sessionId: "sess-1", updatedAt: 0 }, + }, }); const result = resolveSessionKeyForRequest({ @@ -88,8 +95,10 @@ describe("resolveSessionKeyForRequest", () => { it("uses an agent-scoped --to value as the requested session key", () => { const sessionKey = "agent:main:openclaw-weixin:direct:o9cq802hhmfc@im.wechat"; mocks.resolveStorePath.mockReturnValue(MAIN_STORE_PATH); - mocks.loadSessionStore.mockReturnValue({ - [sessionKey]: { sessionId: "wechat-session", updatedAt: 0 }, + mockStoresByPath({ + [MAIN_STORE_PATH]: { + [sessionKey]: { sessionId: "wechat-session", updatedAt: 0 }, + }, }); const result = resolveSessionKeyForRequest({ @@ -137,7 +146,10 @@ describe("resolveSessionKeyForRequest", () => { }); expect(result.sessionKey).toBe("agent:mybot:main"); - expect(result.sessionStore).toBe(mybotStore); + expect(result.sessionStore).toEqual({ + ...mybotStore, + "agent:mybot:main": mainStore["agent:main:main"], + }); expect(result.storePath).toBe(MYBOT_STORE_PATH); expect(result.sessionStore["agent:mybot:main"]?.sessionId).toBe("legacy-session-id"); }); @@ -148,7 +160,7 @@ describe("resolveSessionKeyForRequest", () => { }; mocks.listAgentIds.mockReturnValue(["main", "mybot"]); mocks.resolveStorePath.mockReturnValue(SHARED_STORE_PATH); - mocks.loadSessionStore.mockReturnValue(sharedStore); + mockStoresByPath({ [SHARED_STORE_PATH]: sharedStore }); const result = resolveSessionKeyForRequest({ cfg: { @@ -159,11 +171,17 @@ describe("resolveSessionKeyForRequest", () => { }); expect(result.sessionKey).toBe("agent:mybot:main"); - expect(result.sessionStore).toBe(sharedStore); + expect(result.sessionStore).toEqual({ + ...sharedStore, + "agent:mybot:main": sharedStore["agent:main:main"], + }); expect(result.storePath).toBe(SHARED_STORE_PATH); expect(result.sessionStore["agent:mybot:main"]?.sessionId).toBe("legacy-session-id"); - expect(mocks.loadSessionStore).toHaveBeenCalledTimes(1); - expect(mocks.loadSessionStore).toHaveBeenCalledWith(SHARED_STORE_PATH, undefined); + expect(mocks.listSessionEntries).toHaveBeenCalledTimes(1); + expect(mocks.listSessionEntries).toHaveBeenCalledWith({ + agentId: "mybot", + storePath: SHARED_STORE_PATH, + }); }); it("prefers the configured default-agent session over legacy main-store rows", () => { @@ -186,14 +204,16 @@ describe("resolveSessionKeyForRequest", () => { }); expect(result.sessionKey).toBe("agent:mybot:main"); - expect(result.sessionStore).toBe(mybotStore); + expect(result.sessionStore).toEqual(mybotStore); expect(result.storePath).toBe(MYBOT_STORE_PATH); }); it("finds session by sessionId via reverse lookup in primary store", () => { mocks.resolveStorePath.mockReturnValue(MAIN_STORE_PATH); - mocks.loadSessionStore.mockReturnValue({ - "agent:main:main": { sessionId: "target-session-id", updatedAt: 0 }, + mockStoresByPath({ + [MAIN_STORE_PATH]: { + "agent:main:main": { sessionId: "target-session-id", updatedAt: 0 }, + }, }); const result = resolveSessionKeyForRequest({ @@ -281,8 +301,11 @@ describe("resolveSessionKeyForRequest", () => { expect(result.sessionKey).toBe("agent:mybot:explicit:target-session-id"); expect(result.storePath).toBe(MYBOT_STORE_PATH); - expect(mocks.loadSessionStore).toHaveBeenCalledTimes(1); - expect(mocks.loadSessionStore).toHaveBeenCalledWith(MYBOT_STORE_PATH, undefined); + expect(mocks.listSessionEntries).toHaveBeenCalledTimes(1); + expect(mocks.listSessionEntries).toHaveBeenCalledWith({ + agentId: "mybot", + storePath: MYBOT_STORE_PATH, + }); }); it("returns correct sessionStore when session found in non-primary agent store", () => { @@ -303,7 +326,7 @@ describe("resolveSessionKeyForRequest", () => { it("returns a deterministic explicit sessionKey when sessionId not found in any store", () => { setupMainAndMybotStorePaths(); - mocks.loadSessionStore.mockReturnValue({}); + mocks.listSessionEntries.mockReturnValue([]); const result = resolveSessionKeyForRequest({ cfg: baseCfg, @@ -315,8 +338,10 @@ describe("resolveSessionKeyForRequest", () => { it("does not search other stores when explicitSessionKey is set", () => { mocks.listAgentIds.mockReturnValue(["main", "mybot"]); mocks.resolveStorePath.mockReturnValue(MAIN_STORE_PATH); - mocks.loadSessionStore.mockReturnValue({ - "agent:main:main": { sessionId: "other-id", updatedAt: 0 }, + mockStoresByPath({ + [MAIN_STORE_PATH]: { + "agent:main:main": { sessionId: "other-id", updatedAt: 0 }, + }, }); const result = resolveSessionKeyForRequest({ @@ -352,16 +377,18 @@ describe("resolveSessionKeyForRequest", () => { it("skips already-searched primary store when iterating agents", () => { setupMainAndMybotStorePaths(); - mocks.loadSessionStore.mockReturnValue({}); + mocks.listSessionEntries.mockReturnValue([]); resolveSessionKeyForRequest({ cfg: baseCfg, sessionId: "nonexistent-id", }); - // loadSessionStore should be called twice: once for main, once for mybot + // listSessionEntries should be called twice: once for main, once for mybot // (not twice for main) - const storePaths = mocks.loadSessionStore.mock.calls.map((call) => String(call[0])); + const storePaths = mocks.listSessionEntries.mock.calls.map((call) => + String(call[0]?.storePath), + ); expect(storePaths).toHaveLength(2); expect(storePaths).toContain(MAIN_STORE_PATH); expect(storePaths).toContain(MYBOT_STORE_PATH); diff --git a/src/commands/agents.delete.test.ts b/src/commands/agents.delete.test.ts index 42e8717f493c..9ba63c4acc88 100644 --- a/src/commands/agents.delete.test.ts +++ b/src/commands/agents.delete.test.ts @@ -2,8 +2,11 @@ import fs from "node:fs/promises"; import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { resolveDefaultAgentId } from "../agents/agent-scope.js"; import { resolveWorkspaceAttestationPaths } from "../agents/workspace.js"; -import { loadSessionStore, resolveStorePath, saveSessionStore } from "../config/sessions.js"; +import { resolveStorePath } from "../config/sessions.js"; +import type { SessionEntry } from "../config/sessions.js"; +import { listSessionEntries, replaceSessionEntry } from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { withStateDirEnv } from "../test-helpers/state-dir-env.js"; import { baseConfigSnapshot, createTestRuntime } from "./test-runtime-config-helpers.js"; @@ -59,6 +62,14 @@ import { agentsDeleteCommand } from "./agents.commands.delete.js"; const runtime = createTestRuntime(); +function resolveFixtureStoreAgentId(cfg: OpenClawConfig, deletedAgentId: string): string { + const storeConfig = cfg.session?.store; + if (typeof storeConfig === "string" && !storeConfig.includes("{agentId}")) { + return resolveDefaultAgentId(cfg); + } + return deletedAgentId; +} + async function arrangeAgentsDeleteTest(params: { stateDir: string; cfg: OpenClawConfig; @@ -66,8 +77,14 @@ async function arrangeAgentsDeleteTest(params: { sessions: Record<string, { sessionId: string; updatedAt: number }>; }) { const deletedAgentId = params.deletedAgentId ?? "ops"; + const storeAgentId = resolveFixtureStoreAgentId(params.cfg, deletedAgentId); const storePath = resolveStorePath(params.cfg.session?.store, { agentId: deletedAgentId }); - await saveSessionStore(storePath, params.sessions); + for (const [sessionKey, entry] of Object.entries(params.sessions)) { + await replaceSessionEntry( + { agentId: storeAgentId, sessionKey, storePath }, + entry as SessionEntry, + ); + } await fs.mkdir(path.join(params.stateDir, `workspace-${deletedAgentId}`), { recursive: true }); await fs.mkdir(path.join(params.stateDir, "agents", deletedAgentId, "agent"), { recursive: true, @@ -87,8 +104,16 @@ async function arrangeAgentsDeleteTest(params: { function expectSessionStore( storePath: string, sessions: Record<string, { sessionId: string; updatedAt: number }>, + agentId = "ops", ) { - expect(loadSessionStore(storePath, { skipCache: true })).toEqual(sessions); + expect( + Object.fromEntries( + listSessionEntries({ agentId, storePath }).map(({ entry, sessionKey }) => [ + sessionKey, + entry, + ]), + ), + ).toEqual(sessions); } function readJsonLogs(): Array<Record<string, unknown>> { @@ -345,10 +370,14 @@ describe("agents delete command", () => { await agentsDeleteCommand({ id: "ops", force: true, json: true }, runtime); expect(runtime.exit).not.toHaveBeenCalled(); - expectSessionStore(storePath, { - main: { sessionId: "sess-main", updatedAt: now + 1 }, - "quietchat:direct:u1": { sessionId: "sess-main-direct", updatedAt: now + 2 }, - }); + expectSessionStore( + storePath, + { + main: { sessionId: "sess-main", updatedAt: now + 1 }, + "quietchat:direct:u1": { sessionId: "sess-main-direct", updatedAt: now + 2 }, + }, + "main", + ); }); }); diff --git a/src/commands/daemon-install-helpers.test.ts b/src/commands/daemon-install-helpers.test.ts index efe458c0c13d..b362287f9b75 100644 --- a/src/commands/daemon-install-helpers.test.ts +++ b/src/commands/daemon-install-helpers.test.ts @@ -1295,7 +1295,7 @@ describe("buildGatewayInstallPlan — dotenv merge", () => { source: "env", provider: "default", id: "TELEGRAM_DEFAULT_BOTTOKEN", - }, + } as never, }, }, }, diff --git a/src/commands/doctor-db-bloat.ts b/src/commands/doctor-db-bloat.ts new file mode 100644 index 000000000000..3dfb9d378dd5 --- /dev/null +++ b/src/commands/doctor-db-bloat.ts @@ -0,0 +1,116 @@ +// Doctor visibility for SQLite database bloat (state DB + per-agent DBs). +// Registered size_bytes existed for a while with no reader; production bloat +// (multi-hundred-MB stores, blocking vacuums) surfaced only after user harm. +import fs from "node:fs"; +import { note } from "../../packages/terminal-core/src/note.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { listOpenClawRegisteredAgentDatabases } from "../state/openclaw-agent-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { formatBytes } from "./doctor-disk-space.js"; + +// Bloat is only worth an operator's attention when the file is meaningfully +// large AND a real share of it is reclaimable free pages. +const BLOAT_MIN_FILE_BYTES = 128 * 1024 * 1024; +const BLOAT_MIN_FREE_BYTES = 32 * 1024 * 1024; +const BLOAT_FREE_RATIO = 0.25; +const LARGE_DB_WARN_BYTES = 1024 * 1024 * 1024; + +type SqliteBloatStats = { + fileBytes: number; + freeBytes: number; + incrementalAutoVacuum: boolean; +}; + +function readSqliteBloatStats(pathname: string): SqliteBloatStats | null { + // Diagnostics must degrade per-database: EACCES/ENOTDIR on a stale + // registered path should skip that entry, not abort doctor. + let fileBytes: number; + try { + fileBytes = fs.statSync(pathname, { throwIfNoEntry: false })?.size ?? 0; + } catch { + return null; + } + if (fileBytes <= 0) { + return null; + } + const sqlite = requireNodeSqlite(); + let db: InstanceType<typeof sqlite.DatabaseSync> | undefined; + try { + db = new sqlite.DatabaseSync(pathname, { readOnly: true }); + const pageSize = readPragmaNumber(db, "page_size") ?? 4096; + const freelistCount = readPragmaNumber(db, "freelist_count") ?? 0; + const autoVacuum = readPragmaNumber(db, "auto_vacuum") ?? 0; + return { + fileBytes, + freeBytes: freelistCount * pageSize, + incrementalAutoVacuum: autoVacuum === 2, + }; + } catch { + return null; + } finally { + db?.close(); + } +} + +function readPragmaNumber( + db: { prepare: (sql: string) => { get: () => unknown } }, + pragma: string, +): number | null { + const row = db.prepare(`PRAGMA ${pragma}`).get() as Record<string, unknown> | undefined; + const value = row?.[pragma]; + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function describeBloat(label: string, stats: SqliteBloatStats): string | null { + const freeRatio = stats.fileBytes > 0 ? stats.freeBytes / stats.fileBytes : 0; + const isBloated = + stats.fileBytes >= BLOAT_MIN_FILE_BYTES && + stats.freeBytes >= BLOAT_MIN_FREE_BYTES && + freeRatio >= BLOAT_FREE_RATIO; + if (isBloated) { + const remedy = stats.incrementalAutoVacuum + ? "incremental vacuum will release it gradually" + : "run `VACUUM` offline (gateway stopped) to reclaim it"; + return `${label}: ${formatBytes(stats.fileBytes)} on disk with ${formatBytes(stats.freeBytes)} reclaimable free pages; ${remedy}.`; + } + if (stats.fileBytes >= LARGE_DB_WARN_BYTES) { + return `${label}: ${formatBytes(stats.fileBytes)} on disk; review session/transcript retention settings if growth is unexpected.`; + } + return null; +} + +export function collectSqliteBloatWarnings(deps?: { env?: NodeJS.ProcessEnv }): string[] { + const env = deps?.env ?? process.env; + const warnings: string[] = []; + const statePath = resolveOpenClawStateSqlitePath(env); + const stateStats = readSqliteBloatStats(statePath); + if (stateStats) { + const warning = describeBloat("state DB", stateStats); + if (warning) { + warnings.push(warning); + } + } + for (const registered of listOpenClawRegisteredAgentDatabases({ env })) { + const stats = readSqliteBloatStats(registered.path); + if (!stats) { + continue; + } + const warning = describeBloat(`agent DB (${registered.agentId})`, stats); + if (warning) { + warnings.push(warning); + } + } + return warnings; +} + +export function noteSqliteDatabaseBloat( + _cfg: OpenClawConfig, // reserved for API consistency with other Doctor contributions + deps?: { env?: NodeJS.ProcessEnv }, +): void { + const warnings = collectSqliteBloatWarnings(deps); + if (warnings.length === 0) { + return; + } + note(warnings.join("\n"), "SQLite database size"); +} diff --git a/src/commands/doctor-session-sqlite-compact.ts b/src/commands/doctor-session-sqlite-compact.ts new file mode 100644 index 000000000000..ba39eb38f265 --- /dev/null +++ b/src/commands/doctor-session-sqlite-compact.ts @@ -0,0 +1,113 @@ +/** Runs doctor-owned SQLite file compaction for migrated session stores. */ +import fs from "node:fs"; +import type { DatabaseSync } from "node:sqlite"; +import type { SessionStoreTarget } from "../config/sessions/targets.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "../state/openclaw-state-db.js"; +import { resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js"; +import type { DoctorSessionSqliteCompactReport } from "./doctor-session-sqlite-types.js"; + +type SqliteFileCompactSnapshot = { + dbSizeBytes: number; + freelistPages: number; + pageSizeBytes: number; + walSizeBytes: number; +}; + +/** Reclaim free pages from one agent session SQLite database. */ +export function compactDoctorSessionSqliteTarget( + target: SessionStoreTarget, +): DoctorSessionSqliteCompactReport { + const sqlitePath = resolveTargetSqlitePath(target); + const beforeFileSizes = readSqliteFileSizes(sqlitePath); + if (!fs.existsSync(sqlitePath)) { + return { + dbSizeAfterBytes: 0, + dbSizeBeforeBytes: 0, + freelistAfterPages: 0, + freelistBeforePages: 0, + pageSizeBytes: 0, + reclaimedBytes: 0, + skipped: true, + walSizeAfterBytes: beforeFileSizes.walSizeBytes, + walSizeBeforeBytes: beforeFileSizes.walSizeBytes, + }; + } + + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(sqlitePath); + try { + database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); + checkpointTruncate(database); + const before = readCompactSnapshot(database, sqlitePath); + // Doctor's offline VACUUM is the one sanctioned window to retrofit + // incremental auto-vacuum onto databases created before the pragma + // existed; runtime maintenance then releases pages in bounded passes. + database.exec("PRAGMA auto_vacuum = INCREMENTAL;"); + database.exec("VACUUM;"); + checkpointTruncate(database); + const after = readCompactSnapshot(database, sqlitePath); + return { + dbSizeAfterBytes: after.dbSizeBytes, + dbSizeBeforeBytes: before.dbSizeBytes, + freelistAfterPages: after.freelistPages, + freelistBeforePages: before.freelistPages, + pageSizeBytes: before.pageSizeBytes || after.pageSizeBytes, + reclaimedBytes: Math.max(0, before.dbSizeBytes - after.dbSizeBytes), + skipped: false, + walSizeAfterBytes: after.walSizeBytes, + walSizeBeforeBytes: before.walSizeBytes, + }; + } finally { + database.close(); + } +} + +function checkpointTruncate(database: DatabaseSync): void { + database.exec("PRAGMA wal_checkpoint(TRUNCATE);"); +} + +function readCompactSnapshot( + database: DatabaseSync, + sqlitePath: string, +): SqliteFileCompactSnapshot { + const sizes = readSqliteFileSizes(sqlitePath); + return { + dbSizeBytes: sizes.dbSizeBytes, + freelistPages: readPragmaNumber(database, "freelist_count"), + pageSizeBytes: readPragmaNumber(database, "page_size"), + walSizeBytes: sizes.walSizeBytes, + }; +} + +function readPragmaNumber( + database: DatabaseSync, + pragmaName: "freelist_count" | "page_size", +): number { + const row = database.prepare(`PRAGMA ${pragmaName};`).get() as + | Record<string, unknown> + | undefined; + const value = row?.[pragmaName] ?? (row ? Object.values(row)[0] : undefined); + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value === "bigint") { + return Number(value); + } + return 0; +} + +function readSqliteFileSizes(sqlitePath: string): { dbSizeBytes: number; walSizeBytes: number } { + return { + dbSizeBytes: fileSize(sqlitePath), + walSizeBytes: fileSize(`${sqlitePath}-wal`), + }; +} + +function fileSize(filePath: string): number { + try { + return fs.statSync(filePath).size; + } catch { + return 0; + } +} diff --git a/src/commands/doctor-session-sqlite-github-issue.ts b/src/commands/doctor-session-sqlite-github-issue.ts new file mode 100644 index 000000000000..2c047629cd61 --- /dev/null +++ b/src/commands/doctor-session-sqlite-github-issue.ts @@ -0,0 +1,50 @@ +/** Creates GitHub issues for sanitized session SQLite recovery reports. */ +import { spawnSync, type SpawnSyncReturns } from "node:child_process"; +import type { SessionSqliteMigrationFailureIssue } from "./doctor-session-sqlite-types.js"; + +export type SessionSqliteGithubIssueCreateResult = + | { ok: true; url: string } + | { fallbackUrl: string; message: string; ok: false }; + +type SpawnGh = ( + args: readonly string[], + options: { input: string }, +) => Pick<SpawnSyncReturns<Buffer>, "error" | "status" | "stderr" | "stdout">; + +/** Creates an openclaw/openclaw issue through the GitHub CLI using sanitized stdin. */ +export function createSessionSqliteGithubIssue( + issue: SessionSqliteMigrationFailureIssue, + spawnGh: SpawnGh = defaultSpawnGh, +): SessionSqliteGithubIssueCreateResult { + const result = spawnGh( + ["issue", "create", "--repo", "openclaw/openclaw", "--title", issue.title, "--body-file", "-"], + { input: issue.body }, + ); + if (!result.error && result.status === 0) { + const url = String(result.stdout).trim().split(/\r?\n/).at(-1); + return { + ok: true, + url: url && url.length > 0 ? url : "https://github.com/openclaw/openclaw/issues", + }; + } + const stderr = String(result.stderr).trim(); + const error = result.error + ? result.error.message + : stderr || `gh exited ${result.status ?? "unknown"}`; + return { + fallbackUrl: issue.url, + message: error, + ok: false, + }; +} + +function defaultSpawnGh( + args: readonly string[], + options: { input: string }, +): Pick<SpawnSyncReturns<Buffer>, "error" | "status" | "stderr" | "stdout"> { + return spawnSync("gh", [...args], { + encoding: "buffer", + input: options.input, + maxBuffer: 1024 * 1024, + }); +} diff --git a/src/commands/doctor-session-sqlite-migration-run.ts b/src/commands/doctor-session-sqlite-migration-run.ts new file mode 100644 index 000000000000..d2df662b6205 --- /dev/null +++ b/src/commands/doctor-session-sqlite-migration-run.ts @@ -0,0 +1,622 @@ +/** Manifest and restore helpers for doctor-owned session SQLite migrations. */ +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { resolveStateDir } from "../config/paths.js"; +import { VERSION } from "../version.js"; +import type { + DoctorSessionSqliteIssue, + DoctorSessionSqliteRestoreConflict, + DoctorSessionSqliteRestoreReport, + SessionSqliteMigrationFailureIssue, +} from "./doctor-session-sqlite-types.js"; + +export type SessionSqliteMigrationMoveKind = + | "legacy-store" + | "transcript" + | "trajectory" + | "unreferenced-jsonl"; + +export type SessionSqliteMigrationMove = { + archivePath: string; + kind: SessionSqliteMigrationMoveKind; + sessionKey?: string; + sourcePath: string; +}; + +export type SessionSqliteMigrationTargetInput = { + agentId: string; + sqlitePath: string; + storePath: string; +}; + +export type SessionSqliteMigrationTargetManifest = SessionSqliteMigrationTargetInput & { + completedMoves: SessionSqliteMigrationMove[]; + issues: DoctorSessionSqliteIssue[]; + plannedMoves: SessionSqliteMigrationMove[]; + validationBeforeArchive: "not_run" | "passed" | "failed"; +}; + +export type SessionSqliteMigrationManifest = { + completedAt?: string; + failedAt?: string; + failureReports?: { + jsonPath: string; + markdownPath: string; + }; + manifestVersion: 1; + openClawVersion: string; + restore?: { + attemptedAt: string; + conflicts: DoctorSessionSqliteRestoreConflict[]; + restoredFiles: string[]; + skippedFiles: string[]; + status: "restored" | "partial" | "conflicts" | "failed" | "noop"; + }; + runId: string; + startedAt: string; + targets: SessionSqliteMigrationTargetManifest[]; +}; + +export type ActiveSessionSqliteMigrationRun = { + manifest: SessionSqliteMigrationManifest; + manifestPath: string; +}; + +const SESSION_SQLITE_MIGRATION_RUNS_DIR = "session-sqlite-migration-runs"; +const COMPLETED_MIGRATION_RUN_RETENTION = 50; + +export function createSessionSqliteMigrationRun( + env: NodeJS.ProcessEnv, + targets: readonly SessionSqliteMigrationTargetInput[], +): ActiveSessionSqliteMigrationRun { + const runId = `session-sqlite-${Date.now()}-${randomUUID().slice(0, 8)}`; + const manifestPath = path.join(resolveSessionSqliteMigrationRunsDir(env), `${runId}.json`); + const manifest: SessionSqliteMigrationManifest = { + manifestVersion: 1, + openClawVersion: VERSION, + runId, + startedAt: new Date().toISOString(), + targets: targets.map((target) => ({ + ...target, + completedMoves: [], + issues: [], + plannedMoves: [], + validationBeforeArchive: "not_run", + })), + }; + const activeRun = { manifest, manifestPath }; + writeSessionSqliteMigrationManifest(activeRun); + pruneCompletedSessionSqliteMigrationRuns(env); + return activeRun; +} + +export function resolveSessionSqliteMigrationRunsDir(env: NodeJS.ProcessEnv): string { + return path.join(resolveStateDir(env), SESSION_SQLITE_MIGRATION_RUNS_DIR); +} + +export function writeSessionSqliteMigrationManifest( + activeRun: ActiveSessionSqliteMigrationRun, +): void { + fs.mkdirSync(path.dirname(activeRun.manifestPath), { recursive: true, mode: 0o700 }); + fs.writeFileSync(activeRun.manifestPath, `${JSON.stringify(activeRun.manifest, null, 2)}\n`, { + mode: 0o600, + }); +} + +export function updateMigrationManifestTarget( + activeRun: ActiveSessionSqliteMigrationRun | undefined, + target: SessionSqliteMigrationTargetInput, + issues: readonly DoctorSessionSqliteIssue[], + updates: { + validationBeforeArchive?: SessionSqliteMigrationTargetManifest["validationBeforeArchive"]; + } = {}, +): void { + const manifestTarget = findMigrationManifestTarget(activeRun, target); + if (!activeRun || !manifestTarget) { + return; + } + manifestTarget.issues = issues.map((issue) => ({ ...issue })); + if (updates.validationBeforeArchive) { + manifestTarget.validationBeforeArchive = updates.validationBeforeArchive; + } + writeSessionSqliteMigrationManifest(activeRun); +} + +export function recordPlannedMigrationMove( + activeRun: ActiveSessionSqliteMigrationRun | undefined, + target: SessionSqliteMigrationTargetInput, + move: SessionSqliteMigrationMove, +): void { + const manifestTarget = findMigrationManifestTarget(activeRun, target); + if (!activeRun || !manifestTarget) { + return; + } + if (!manifestTarget.plannedMoves.some((item) => movesMatch(item, move))) { + manifestTarget.plannedMoves.push(move); + } + writeSessionSqliteMigrationManifest(activeRun); +} + +export function recordCompletedMigrationMove( + activeRun: ActiveSessionSqliteMigrationRun | undefined, + target: SessionSqliteMigrationTargetInput, + move: SessionSqliteMigrationMove, +): void { + const manifestTarget = findMigrationManifestTarget(activeRun, target); + if (!activeRun || !manifestTarget) { + return; + } + if (!manifestTarget.completedMoves.some((item) => movesMatch(item, move))) { + manifestTarget.completedMoves.push(move); + } + writeSessionSqliteMigrationManifest(activeRun); +} + +export function restoreSessionSqliteMigrationRuns(params: { + env: NodeJS.ProcessEnv; + targetKeys?: ReadonlySet<string>; +}): DoctorSessionSqliteRestoreReport { + const restoreReport: DoctorSessionSqliteRestoreReport = emptyRestoreReport(); + for (const manifestPath of listSessionSqliteMigrationManifestPaths(params.env)) { + const manifest = readSessionSqliteMigrationManifest(manifestPath); + if (!manifest) { + continue; + } + const targetManifests = filterRestoreManifestTargets(manifest, params.targetKeys); + if (targetManifests.length === 0) { + continue; + } + const manifestRestoreReport: DoctorSessionSqliteRestoreReport = { + ...emptyRestoreReport(), + manifestPaths: [manifestPath], + }; + restoreReport.manifestPaths.push(manifestPath); + restoreSessionSqliteMigrationManifest(manifest, targetManifests, manifestRestoreReport); + restoreReport.conflicts.push(...manifestRestoreReport.conflicts); + restoreReport.restoredFiles.push(...manifestRestoreReport.restoredFiles); + restoreReport.skippedFiles.push(...manifestRestoreReport.skippedFiles); + writeSessionSqliteMigrationManifest({ manifest, manifestPath }); + } + return restoreReport; +} + +export function restoreSessionSqliteMigrationRun(params: { + manifestPath: string; + targetKeys?: ReadonlySet<string>; +}): DoctorSessionSqliteRestoreReport { + const restoreReport: DoctorSessionSqliteRestoreReport = { + ...emptyRestoreReport(), + manifestPaths: [params.manifestPath], + }; + const manifest = readSessionSqliteMigrationManifest(params.manifestPath); + if (!manifest) { + restoreReport.conflicts.push({ + archivePath: params.manifestPath, + reason: "manifest is missing or unreadable", + sourcePath: params.manifestPath, + }); + return restoreReport; + } + restoreSessionSqliteMigrationManifest( + manifest, + filterRestoreManifestTargets(manifest, params.targetKeys), + restoreReport, + ); + writeSessionSqliteMigrationManifest({ manifest, manifestPath: params.manifestPath }); + return restoreReport; +} + +export function findLatestFailedSessionSqliteMigrationManifest( + env: NodeJS.ProcessEnv, + targetKeys?: ReadonlySet<string>, +): { manifest: SessionSqliteMigrationManifest; manifestPath: string } | undefined { + return listSessionSqliteMigrationManifestPaths(env) + .map((manifestPath) => ({ + manifest: readSessionSqliteMigrationManifest(manifestPath), + manifestPath, + })) + .filter( + (item): item is { manifest: SessionSqliteMigrationManifest; manifestPath: string } => + item.manifest !== undefined && + isFailedSessionSqliteMigrationManifest(item.manifest) && + filterRestoreManifestTargets(item.manifest, targetKeys).length > 0, + ) + .toSorted( + (left, right) => manifestSortTime(right.manifest) - manifestSortTime(left.manifest), + )[0]; +} + +export function writeSessionSqliteMigrationFailureReports( + manifestPath: string, + params: { reason: string }, +): { jsonPath: string; markdownPath: string } { + const manifest = readSessionSqliteMigrationManifest(manifestPath); + const jsonPath = manifestPath.replace(/\.json$/, ".failure.json"); + const markdownPath = manifestPath.replace(/\.json$/, ".failure.md"); + const payload = { + generatedAt: new Date().toISOString(), + manifestPath: sanitizeFailureReportText(shortenFailureReportPath(manifestPath)), + reason: params.reason, + recoveryCommand: "openclaw doctor --session-sqlite recover --github-issue", + restoreStatus: manifest?.restore?.status ?? "not_attempted", + runId: manifest?.runId ?? path.basename(manifestPath, ".json"), + targets: + manifest?.targets.map((target) => ({ + agentId: sanitizeFailureReportText(target.agentId), + completedMoves: target.completedMoves.length, + issues: target.issues.map((issue) => ({ + code: issue.code, + message: sanitizeFailureIssueMessage(issue, target), + ...(issue.sessionKey ? { sessionKey: redactSessionKey(issue.sessionKey) } : {}), + })), + plannedMoves: target.plannedMoves.length, + sqlitePath: sanitizeFailureReportText(shortenFailureReportPath(target.sqlitePath)), + storePath: sanitizeFailureReportText(shortenFailureReportPath(target.storePath)), + validationBeforeArchive: target.validationBeforeArchive, + })) ?? [], + version: VERSION, + }; + fs.writeFileSync(jsonPath, `${JSON.stringify(payload, null, 2)}\n`, { mode: 0o600 }); + fs.writeFileSync(markdownPath, renderFailureMarkdown(payload), { mode: 0o600 }); + if (manifest) { + manifest.failureReports = { jsonPath, markdownPath }; + writeSessionSqliteMigrationManifest({ manifest, manifestPath }); + } + return { jsonPath, markdownPath }; +} + +export function createSessionSqliteMigrationFailureIssue( + manifestPath: string, + targetKeys?: ReadonlySet<string>, +): SessionSqliteMigrationFailureIssue | undefined { + const manifest = readSessionSqliteMigrationManifest(manifestPath); + if (!manifest) { + return undefined; + } + const title = `Session SQLite migration recovery report (${manifest.runId})`; + const bodyPath = manifest.failureReports?.markdownPath; + const targets = filterRestoreManifestTargets(manifest, targetKeys); + const reportBody = renderFailureMarkdown({ + generatedAt: new Date().toISOString(), + manifestPath: sanitizeFailureReportText(shortenFailureReportPath(manifestPath)), + reason: "session SQLite migration failed", + recoveryCommand: "openclaw doctor --session-sqlite recover --github-issue", + restoreStatus: manifest.restore?.status ?? "not_attempted", + runId: manifest.runId, + targets: targets.map((target) => ({ + agentId: sanitizeFailureReportText(target.agentId), + completedMoves: target.completedMoves.length, + issues: target.issues.map((issue) => ({ + code: issue.code, + message: sanitizeFailureIssueMessage(issue, target), + })), + plannedMoves: target.plannedMoves.length, + sqlitePath: sanitizeFailureReportText(shortenFailureReportPath(target.sqlitePath)), + storePath: sanitizeFailureReportText(shortenFailureReportPath(target.storePath)), + validationBeforeArchive: target.validationBeforeArchive, + })), + version: VERSION, + }); + const body = [ + "OpenClaw doctor generated this sanitized report from a local session SQLite migration recovery.", + "", + reportBody, + ] + .join("\n") + .slice(0, 20_000); + return { + body, + ...(bodyPath ? { bodyPath } : {}), + title, + url: createPrefilledGithubIssueUrl(title, body), + }; +} + +export function sessionSqliteMigrationTargetKey(target: { + agentId: string; + storePath: string; +}): string { + return `${target.agentId}\u0000${path.resolve(target.storePath)}`; +} + +function findMigrationManifestTarget( + activeRun: ActiveSessionSqliteMigrationRun | undefined, + target: SessionSqliteMigrationTargetInput, +): SessionSqliteMigrationTargetManifest | undefined { + if (!activeRun) { + return undefined; + } + return activeRun.manifest.targets.find( + (item) => sessionSqliteMigrationTargetKey(item) === sessionSqliteMigrationTargetKey(target), + ); +} + +function movesMatch(left: SessionSqliteMigrationMove, right: SessionSqliteMigrationMove): boolean { + return left.sourcePath === right.sourcePath && left.archivePath === right.archivePath; +} + +function emptyRestoreReport(): DoctorSessionSqliteRestoreReport { + return { + conflicts: [], + manifestPaths: [], + restoredFiles: [], + skippedFiles: [], + }; +} + +function restoreSessionSqliteMigrationManifest( + manifest: SessionSqliteMigrationManifest, + targets: readonly SessionSqliteMigrationTargetManifest[], + restoreReport: DoctorSessionSqliteRestoreReport, +): void { + for (const target of targets) { + for (const move of uniqueRestoreMoves(target)) { + restoreMigrationMove(move, restoreReport); + } + } + manifest.restore = { + attemptedAt: new Date().toISOString(), + conflicts: restoreReport.conflicts, + restoredFiles: restoreReport.restoredFiles, + skippedFiles: restoreReport.skippedFiles, + status: resolveRestoreStatus(restoreReport), + }; +} + +function uniqueRestoreMoves( + target: SessionSqliteMigrationTargetManifest, +): SessionSqliteMigrationMove[] { + const moves = new Map<string, SessionSqliteMigrationMove>(); + for (const move of [...target.completedMoves, ...target.plannedMoves]) { + moves.set(`${move.sourcePath}\u0000${move.archivePath}`, move); + } + return [...moves.values()]; +} + +function restoreMigrationMove( + move: SessionSqliteMigrationMove, + restoreReport: DoctorSessionSqliteRestoreReport, +): void { + const sourceExists = fs.existsSync(move.sourcePath); + const archiveExists = fs.existsSync(move.archivePath); + if (!sourceExists && archiveExists) { + fs.mkdirSync(path.dirname(move.sourcePath), { recursive: true, mode: 0o700 }); + fs.renameSync(move.archivePath, move.sourcePath); + restoreReport.restoredFiles.push(move.sourcePath); + return; + } + if (sourceExists && !archiveExists) { + restoreReport.skippedFiles.push(move.sourcePath); + return; + } + if (sourceExists && archiveExists) { + restoreReport.conflicts.push({ + archivePath: move.archivePath, + reason: "source and archive both exist; refusing to overwrite source", + sourcePath: move.sourcePath, + }); + return; + } + restoreReport.conflicts.push({ + archivePath: move.archivePath, + reason: "source and archive are both missing", + sourcePath: move.sourcePath, + }); +} + +function resolveRestoreStatus( + report: DoctorSessionSqliteRestoreReport, +): NonNullable<SessionSqliteMigrationManifest["restore"]>["status"] { + if (report.conflicts.length > 0 && report.restoredFiles.length > 0) { + return "partial"; + } + if (report.conflicts.length > 0) { + return "conflicts"; + } + if (report.restoredFiles.length > 0) { + return "restored"; + } + if (report.skippedFiles.length > 0) { + return "noop"; + } + return "noop"; +} + +function filterRestoreManifestTargets( + manifest: SessionSqliteMigrationManifest, + targetKeys: ReadonlySet<string> | undefined, +): SessionSqliteMigrationTargetManifest[] { + if (!targetKeys) { + return manifest.targets; + } + if (targetKeys.size === 0) { + return []; + } + return manifest.targets.filter((target) => + targetKeys.has(sessionSqliteMigrationTargetKey(target)), + ); +} + +export function listSessionSqliteMigrationManifestPaths(env: NodeJS.ProcessEnv): string[] { + const runsDir = resolveSessionSqliteMigrationRunsDir(env); + let entries: string[]; + try { + entries = fs.readdirSync(runsDir); + } catch { + return []; + } + return entries + .filter((entry) => entry.endsWith(".json")) + .filter((entry) => !entry.endsWith(".failure.json")) + .map((entry) => path.join(runsDir, entry)) + .toSorted((left, right) => right.localeCompare(left)); +} + +export function readSessionSqliteMigrationManifest( + manifestPath: string, +): SessionSqliteMigrationManifest | undefined { + try { + const parsed = JSON.parse(fs.readFileSync(manifestPath, "utf-8")) as unknown; + if (!isRecord(parsed) || parsed.manifestVersion !== 1 || typeof parsed.runId !== "string") { + return undefined; + } + return parsed as SessionSqliteMigrationManifest; + } catch { + return undefined; + } +} + +function isFailedSessionSqliteMigrationManifest(manifest: SessionSqliteMigrationManifest): boolean { + return ( + manifest.completedAt === undefined || + manifest.failedAt !== undefined || + manifest.failureReports !== undefined || + manifest.targets.some((target) => target.issues.length > 0) + ); +} + +function manifestSortTime(manifest: SessionSqliteMigrationManifest): number { + const timestamp = manifest.failedAt ?? manifest.completedAt ?? manifest.startedAt; + const parsed = Date.parse(timestamp); + return Number.isFinite(parsed) ? parsed : 0; +} + +function createPrefilledGithubIssueUrl(title: string, body: string): string { + const urlBody = + body.length > 6_000 + ? `${body.slice(0, 6_000)}\n\n...(truncated for URL; see local failure report for the full sanitized body)` + : body; + const params = new URLSearchParams({ + body: urlBody, + title, + }); + return `https://github.com/openclaw/openclaw/issues/new?${params.toString()}`; +} + +function pruneCompletedSessionSqliteMigrationRuns(env: NodeJS.ProcessEnv): void { + const completed = listSessionSqliteMigrationManifestPaths(env) + .map((manifestPath) => ({ + manifest: readSessionSqliteMigrationManifest(manifestPath), + manifestPath, + })) + .filter( + (item): item is { manifest: SessionSqliteMigrationManifest; manifestPath: string } => + item.manifest !== undefined && + item.manifest.completedAt !== undefined && + !isFailedSessionSqliteMigrationManifest(item.manifest), + ) + .toSorted((left, right) => manifestSortTime(right.manifest) - manifestSortTime(left.manifest)); + for (const item of completed.slice(COMPLETED_MIGRATION_RUN_RETENTION)) { + try { + fs.rmSync(item.manifestPath, { force: true }); + } catch { + // Retention is best-effort and must not block startup import. + } + } +} + +function renderFailureMarkdown(payload: { + generatedAt: string; + manifestPath: string; + reason: string; + recoveryCommand: string; + restoreStatus: string; + runId: string; + targets: Array<{ + agentId: string; + completedMoves: number; + issues: Array<{ code: string; message: string; sessionKey?: string }>; + plannedMoves: number; + sqlitePath: string; + storePath: string; + validationBeforeArchive: string; + }>; + version: string; +}): string { + const lines = [ + "# Session SQLite Migration Failure", + "", + `- Run: ${payload.runId}`, + `- Generated: ${payload.generatedAt}`, + `- OpenClaw version: ${payload.version}`, + `- Reason: ${sanitizeFailureReportText(payload.reason)}`, + `- Restore status: ${payload.restoreStatus}`, + `- Recovery command: \`${payload.recoveryCommand}\``, + "", + "## Targets", + ]; + for (const target of payload.targets) { + lines.push( + "", + `### ${target.agentId}`, + "", + `- Store: ${target.storePath}`, + `- SQLite: ${target.sqlitePath}`, + `- Planned moves: ${target.plannedMoves}`, + `- Completed moves: ${target.completedMoves}`, + `- Validation before archive: ${target.validationBeforeArchive}`, + `- Issues: ${target.issues.length}`, + ); + for (const issue of target.issues.slice(0, 10)) { + lines.push( + ` - [${issue.code}] ${issue.sessionKey ? `${issue.sessionKey}: ` : ""}${issue.message}`, + ); + } + } + lines.push(""); + return `${lines.join("\n")}\n`; +} + +function sanitizeFailureReportText(value: string): string { + return value + .replace(/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, "[redacted-email]") + .replace(/(api[_-]?key|token|secret|password)[=-][A-Za-z0-9._-]+/gi, "$1-[redacted]") + .replace(/(api[_-]?key|token|secret|password)=\S+/gi, "$1=[redacted]") + .slice(0, 500); +} + +function shortenFailureReportPath(filePath: string): string { + const home = process.env.HOME; + if (home && filePath.startsWith(`${home}${path.sep}`)) { + return `~${path.sep}${path.relative(home, filePath)}`; + } + return filePath; +} + +function sanitizeFailureIssueMessage( + issue: DoctorSessionSqliteIssue, + target: SessionSqliteMigrationTargetManifest, +): string { + let message = issue.message; + for (const filePath of [ + target.storePath, + target.sqlitePath, + ...target.plannedMoves.flatMap((move) => [move.sourcePath, move.archivePath]), + ...target.completedMoves.flatMap((move) => [move.sourcePath, move.archivePath]), + ]) { + message = message.split(filePath).join(shortenFailureReportPath(filePath)); + } + if (issue.sessionKey) { + message = message.split(issue.sessionKey).join(redactSessionKey(issue.sessionKey)); + } + message = redactAbsoluteHomePaths(message); + return sanitizeFailureReportText(message); +} + +function redactSessionKey(sessionKey: string): string { + const normalized = sessionKey.trim(); + if (!normalized) { + return "[redacted-session-key]"; + } + return `[redacted-session-key:${randomUUID().slice(0, 8)}]`; +} + +function redactAbsoluteHomePaths(value: string): string { + const home = process.env.HOME; + if (!home) { + return value; + } + return value.split(home).join("~"); +} diff --git a/src/commands/doctor-session-sqlite-readers.ts b/src/commands/doctor-session-sqlite-readers.ts new file mode 100644 index 000000000000..b0dd04df0d02 --- /dev/null +++ b/src/commands/doctor-session-sqlite-readers.ts @@ -0,0 +1,358 @@ +/** Read-only diagnostic readers used by the session SQLite doctor mode. */ +import fs from "node:fs"; +import { TextDecoder } from "node:util"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { normalizeLoadedFileEntry, type FileEntry } from "../agents/sessions/session-manager.js"; +import type { TranscriptEvent } from "../config/sessions/session-accessor.js"; +import { resolveSqliteTargetFromSessionStorePath } from "../config/sessions/session-sqlite-target.js"; +import type { SessionStoreTarget } from "../config/sessions/targets.js"; +import type { SessionEntry } from "../config/sessions/types.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { resolveOpenClawAgentSqlitePath } from "../state/openclaw-agent-db.js"; + +export type ReadOnlySqliteSessionSummary = { + entry: SessionEntry; + sessionKey: string; +}; + +export type ReadOnlySqliteSessionEntriesResult = + | { exists: false; ok: true; summaries: [] } + | { exists: true; ok: true; summaries: ReadOnlySqliteSessionSummary[] } + | { error: unknown; exists: true; ok: false }; + +export type ReadOnlySqliteExactSessionEntryResult = + | { entry?: ReadOnlySqliteSessionSummary; ok: true } + | { error: unknown; ok: false }; + +export type ReadOnlySqliteTranscriptEventCountResult = + | { events: number; exists: boolean; ok: true } + | { error: unknown; exists: true; ok: false }; + +export type ReadOnlySqliteDbStatsResult = + | { + ok: true; + stats: { + dbSizeBytes: number; + integrityCheck?: string; + largestSessions: Array<{ events: number; rowBytes: number; sessionId: string }>; + totalTranscriptRowBytes: number; + walSizeBytes: number; + }; + } + | { error: unknown; ok: false }; + +export type TranscriptEventCountResult = + | { status: "ok"; events: number } + | { status: "missing" } + | { status: "malformed"; message: string }; + +const JSONL_READ_CHUNK_BYTES = 64 * 1024; + +export function countTranscriptEventsForPath( + transcriptPath: string | undefined, +): TranscriptEventCountResult { + if (!transcriptPath) { + return { status: "ok", events: 0 }; + } + if (!fs.existsSync(transcriptPath)) { + return { status: "missing" }; + } + let events = 0; + try { + for (const line of iterateJsonlLinesSync(transcriptPath)) { + if (!parseJsonlLine(line)) { + continue; + } + events += 1; + } + return { status: "ok", events }; + } catch (err) { + return { status: "malformed", message: String(err) }; + } +} + +export function createTranscriptEventReader( + transcriptPath: string, +): (append: (event: TranscriptEvent) => void) => void { + return (append) => { + for (const line of iterateJsonlLinesSync(transcriptPath)) { + const parsed = parseJsonlLine(line); + if (parsed) { + // Import is the migration boundary: repair legacy JSONL message shapes + // here because the SQLite runtime read path assumes canonical rows. + append(normalizeLoadedFileEntry(parsed as FileEntry) as TranscriptEvent); + } + } + }; +} + +export function createTranscriptEventPrefixReader( + transcriptPath: string, +): (append: (event: TranscriptEvent) => void) => void { + return (append) => { + try { + for (const line of iterateJsonlLinesSync(transcriptPath)) { + const parsed = parseJsonlLine(line); + if (parsed) { + append(normalizeLoadedFileEntry(parsed as FileEntry) as TranscriptEvent); + } + } + } catch { + // The caller records the malformed transcript issue; keep the readable prefix. + } + }; +} + +export function readSqliteEntryCount(target: SessionStoreTarget): number { + const result = readOnlySqliteSessionEntries(target); + return result.ok ? result.summaries.length : 0; +} + +export function readOnlySqliteExactSessionEntry( + target: SessionStoreTarget, + sessionKey: string, +): ReadOnlySqliteExactSessionEntryResult { + const result = readOnlySqliteSessionEntries(target); + if (!result.ok) { + return { error: result.error, ok: false }; + } + return { + entry: result.summaries.find((summary) => summary.sessionKey === sessionKey), + ok: true, + }; +} + +export function readOnlySqliteSessionEntries( + target: SessionStoreTarget, +): ReadOnlySqliteSessionEntriesResult { + const sqlitePath = resolveTargetSqlitePath(target); + if (!fs.existsSync(sqlitePath)) { + return { exists: false, ok: true, summaries: [] }; + } + const sqlite = requireNodeSqlite(); + let database: InstanceType<typeof sqlite.DatabaseSync> | undefined; + try { + database = new sqlite.DatabaseSync(sqlitePath, { readOnly: true }); + const table = database + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get("session_entries"); + if (!table) { + return { exists: true, ok: true, summaries: [] }; + } + const rows = database + .prepare("SELECT session_key, entry_json FROM session_entries ORDER BY session_key ASC") + .all() as Array<{ entry_json?: unknown; session_key?: unknown }>; + return { + exists: true, + ok: true, + summaries: rows.flatMap((row) => { + if (typeof row.session_key !== "string" || typeof row.entry_json !== "string") { + return []; + } + const entry = parseSqliteSessionEntry(row.entry_json); + return entry ? [{ entry, sessionKey: row.session_key }] : []; + }), + }; + } catch (error) { + return { error, exists: true, ok: false }; + } finally { + database?.close(); + } +} + +export function readOnlySqliteTranscriptEventCount( + target: SessionStoreTarget, + sessionId: string, +): ReadOnlySqliteTranscriptEventCountResult { + const sqlitePath = resolveTargetSqlitePath(target); + if (!fs.existsSync(sqlitePath)) { + return { events: 0, exists: false, ok: true }; + } + const sqlite = requireNodeSqlite(); + let database: InstanceType<typeof sqlite.DatabaseSync> | undefined; + try { + database = new sqlite.DatabaseSync(sqlitePath, { readOnly: true }); + const table = database + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get("transcript_events"); + if (!table) { + return { events: 0, exists: true, ok: true }; + } + const row = database + .prepare("SELECT COUNT(*) AS count FROM transcript_events WHERE session_id = ?") + .get(sessionId) as { count?: unknown } | undefined; + const count = row?.count; + return { + events: typeof count === "number" && Number.isFinite(count) ? count : 0, + exists: true, + ok: true, + }; + } catch (error) { + return { error, exists: true, ok: false }; + } finally { + database?.close(); + } +} + +export function readOnlySqliteDbStats(target: SessionStoreTarget): ReadOnlySqliteDbStatsResult { + const sqlitePath = resolveTargetSqlitePath(target); + const sizeFor = (filePath: string): number => { + try { + return fs.statSync(filePath).size; + } catch { + return 0; + } + }; + if (!fs.existsSync(sqlitePath)) { + return { + ok: true, + stats: { + dbSizeBytes: 0, + largestSessions: [], + totalTranscriptRowBytes: 0, + walSizeBytes: sizeFor(`${sqlitePath}-wal`), + }, + }; + } + const sqlite = requireNodeSqlite(); + let database: InstanceType<typeof sqlite.DatabaseSync> | undefined; + try { + database = new sqlite.DatabaseSync(sqlitePath, { readOnly: true }); + const hasTranscriptEvents = database + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get("transcript_events"); + const integrityRow = database.prepare("PRAGMA quick_check").get() as + | { quick_check?: unknown } + | undefined; + if (!hasTranscriptEvents) { + return { + ok: true, + stats: { + dbSizeBytes: sizeFor(sqlitePath), + integrityCheck: + typeof integrityRow?.quick_check === "string" ? integrityRow.quick_check : undefined, + largestSessions: [], + totalTranscriptRowBytes: 0, + walSizeBytes: sizeFor(`${sqlitePath}-wal`), + }, + }; + } + const totalRow = database + .prepare("SELECT COALESCE(SUM(LENGTH(event_json)), 0) AS row_bytes FROM transcript_events") + .get() as { row_bytes?: unknown } | undefined; + const largestRows = database + .prepare( + ` + SELECT session_id, COUNT(*) AS events, COALESCE(SUM(LENGTH(event_json)), 0) AS row_bytes + FROM transcript_events + GROUP BY session_id + ORDER BY row_bytes DESC, events DESC, session_id ASC + LIMIT 5 + `, + ) + .all() as Array<{ events?: unknown; row_bytes?: unknown; session_id?: unknown }>; + return { + ok: true, + stats: { + dbSizeBytes: sizeFor(sqlitePath), + integrityCheck: + typeof integrityRow?.quick_check === "string" ? integrityRow.quick_check : undefined, + largestSessions: largestRows.flatMap((row) => { + if (typeof row.session_id !== "string") { + return []; + } + return [ + { + events: sqliteNumber(row.events), + rowBytes: sqliteNumber(row.row_bytes), + sessionId: row.session_id, + }, + ]; + }), + totalTranscriptRowBytes: sqliteNumber(totalRow?.row_bytes), + walSizeBytes: sizeFor(`${sqlitePath}-wal`), + }, + }; + } catch (error) { + return { error, ok: false }; + } finally { + database?.close(); + } +} + +export function resolveTargetSqlitePath(target: SessionStoreTarget): string { + const sqliteTarget = resolveSqliteTargetFromSessionStorePath(target.storePath, { + agentId: target.agentId, + }); + return resolveOpenClawAgentSqlitePath({ + agentId: sqliteTarget.agentId ?? target.agentId, + ...(sqliteTarget.path ? { path: sqliteTarget.path } : {}), + }); +} + +function parseSqliteSessionEntry(entryJson: string): SessionEntry | undefined { + try { + const parsed = JSON.parse(entryJson) as unknown; + return isRecord(parsed) ? (parsed as SessionEntry) : undefined; + } catch { + return undefined; + } +} + +function* iterateJsonlLinesSync( + filePath: string, +): Generator<{ final: boolean; lineNumber: number; text: string }> { + const fd = fs.openSync(filePath, "r"); + const decoder = new TextDecoder("utf-8", { fatal: true }); + const buffer = Buffer.allocUnsafe(JSONL_READ_CHUNK_BYTES); + let carry = ""; + let lineNumber = 0; + try { + while (true) { + const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, null); + if (bytesRead === 0) { + break; + } + carry += decoder.decode(buffer.subarray(0, bytesRead), { stream: true }); + const parts = carry.split(/\r?\n/); + carry = parts.pop() ?? ""; + for (const part of parts) { + lineNumber += 1; + const text = part.trim(); + if (text) { + yield { final: false, lineNumber, text }; + } + } + } + carry += decoder.decode(); + const text = carry.trim(); + if (text) { + yield { final: true, lineNumber: lineNumber + 1, text }; + } + } catch (err) { + throw new Error(`${filePath}:${lineNumber + 1}: ${String(err)}`, { cause: err }); + } finally { + fs.closeSync(fd); + } +} + +function sqliteNumber(value: unknown): number { + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value === "bigint") { + return Number(value); + } + return 0; +} + +function parseJsonlLine(line: { final: boolean; lineNumber: number; text: string }): unknown { + try { + return JSON.parse(line.text); + } catch (error) { + if (line.final) { + return undefined; + } + throw error; + } +} diff --git a/src/commands/doctor-session-sqlite-recover-report.ts b/src/commands/doctor-session-sqlite-recover-report.ts new file mode 100644 index 000000000000..76de473d8f2a --- /dev/null +++ b/src/commands/doctor-session-sqlite-recover-report.ts @@ -0,0 +1,275 @@ +/** Builds doctor reports for session SQLite migration recovery mode. */ +import fs from "node:fs"; +import type { SessionStoreTarget } from "../config/sessions/targets.js"; +import { + createSessionSqliteMigrationFailureIssue, + findLatestFailedSessionSqliteMigrationManifest, + resolveSessionSqliteMigrationRunsDir, + restoreSessionSqliteMigrationRun, + sessionSqliteMigrationTargetKey, + writeSessionSqliteMigrationFailureReports, +} from "./doctor-session-sqlite-migration-run.js"; +import { readOnlySqliteDbStats, resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js"; +import type { + DoctorSessionSqliteOptions, + DoctorSessionSqliteReport, + DoctorSessionSqliteTargetReport, +} from "./doctor-session-sqlite-types.js"; + +export type SessionSqliteRecoverTargetValidator = ( + target: SessionStoreTarget, +) => Promise<DoctorSessionSqliteTargetReport>; + +/** Restores the latest failed migration run and validates only selected manifest targets. */ +export async function recoverDoctorSessionSqliteTargets(params: { + env: NodeJS.ProcessEnv; + options: DoctorSessionSqliteOptions; + targets: readonly SessionStoreTarget[]; + validateTarget: SessionSqliteRecoverTargetValidator; +}): Promise<DoctorSessionSqliteReport> { + const selectedTargetKeys = resolveRecoverTargetKeys(params.options, params.targets); + const failedRun = findLatestFailedSessionSqliteMigrationManifest(params.env, selectedTargetKeys); + if (!failedRun) { + const recoveredCorruptTargets = recoverCorruptSqliteTargets(params.targets); + if (recoveredCorruptTargets.length > 0) { + return summarizeRecoverReport(recoveredCorruptTargets); + } + return summarizeRecoverReport([ + createSyntheticRecoverTargetReport( + params.env, + "No failed session SQLite migration manifest found.", + ), + ]); + } + const restore = restoreSessionSqliteMigrationRun({ + manifestPath: failedRun.manifestPath, + targetKeys: selectedTargetKeys, + }); + const targetReports: DoctorSessionSqliteTargetReport[] = []; + const manifestTargets = filterRecoveryManifestTargets( + failedRun.manifest.targets, + selectedTargetKeys, + ); + for (const manifestTarget of manifestTargets) { + targetReports.push( + await params.validateTarget({ + agentId: manifestTarget.agentId, + storePath: manifestTarget.storePath, + }), + ); + } + const reportTarget = + targetReports[0] ?? createSyntheticRecoverTargetReport(params.env, failedRun.manifestPath); + reportTarget.restore = restore; + reportTarget.issues.push( + ...restore.conflicts.map((conflict) => ({ + code: "restore_conflict", + message: `${conflict.sourcePath}: ${conflict.reason}`, + })), + ); + const failureReports = writeSessionSqliteMigrationFailureReports(failedRun.manifestPath, { + reason: "doctor recover restored and validated a failed session SQLite migration run", + }); + const report = summarizeRecoverReport(targetReports.length > 0 ? targetReports : [reportTarget]); + report.migrationRun = { + failureReportJsonPath: failureReports.jsonPath, + failureReportMarkdownPath: failureReports.markdownPath, + manifestPath: failedRun.manifestPath, + runId: failedRun.manifest.runId, + }; + report.supportIssue = createSessionSqliteMigrationFailureIssue( + failedRun.manifestPath, + selectedTargetKeys, + ); + return report; +} + +function recoverCorruptSqliteTargets( + targets: readonly SessionStoreTarget[], +): DoctorSessionSqliteTargetReport[] { + return targets.flatMap((target) => { + const sqlitePath = resolveTargetSqlitePath(target); + if (!fs.existsSync(sqlitePath)) { + return []; + } + const stats = readOnlySqliteDbStats(target); + if (stats.ok) { + if (stats.stats.integrityCheck && stats.stats.integrityCheck !== "ok") { + return [ + recoverCorruptSqliteTarget( + target, + sqlitePath, + new Error(`SQLite quick_check reported: ${stats.stats.integrityCheck}`), + ), + ]; + } + return []; + } + if (!isSqliteCorruptionError(stats.error)) { + return [createRecoverInspectionFailureTargetReport(target, sqlitePath, stats.error)]; + } + return [recoverCorruptSqliteTarget(target, sqlitePath, stats.error)]; + }); +} + +function recoverCorruptSqliteTarget( + target: SessionStoreTarget, + sqlitePath: string, + error: unknown, +): DoctorSessionSqliteTargetReport { + const report = createEmptyRecoverTargetReport(target, sqlitePath); + try { + report.corruptRecovery = moveCorruptSqliteFilesAside(sqlitePath); + } catch (moveError) { + report.issues.push({ + code: "sqlite_corrupt_recovery_failed", + message: `${sqlitePath}: ${String(moveError)}; original error: ${String(error)}`, + }); + } + return report; +} + +function createRecoverInspectionFailureTargetReport( + target: SessionStoreTarget, + sqlitePath: string, + error: unknown, +): DoctorSessionSqliteTargetReport { + const report = createEmptyRecoverTargetReport(target, sqlitePath); + report.issues.push({ + code: "sqlite_recovery_inspect_failed", + message: `${sqlitePath}: ${String(error)}`, + }); + return report; +} + +function moveCorruptSqliteFilesAside(sqlitePath: string): { + movedFiles: string[]; + skippedFiles: string[]; +} { + const movedFiles: string[] = []; + const skippedFiles: string[] = []; + const suffix = `.corrupt-${Date.now()}`; + for (const candidate of [sqlitePath, `${sqlitePath}-wal`, `${sqlitePath}-shm`]) { + if (!fs.existsSync(candidate)) { + skippedFiles.push(candidate); + continue; + } + const destination = uniqueRecoveryPath(`${candidate}${suffix}`); + fs.renameSync(candidate, destination); + movedFiles.push(destination); + } + return { movedFiles, skippedFiles }; +} + +function uniqueRecoveryPath(basePath: string): string { + for (let attempt = 0; attempt < 100; attempt += 1) { + const candidate = attempt === 0 ? basePath : `${basePath}.${attempt}`; + if (!fs.existsSync(candidate)) { + return candidate; + } + } + throw new Error(`Could not choose recovery path for ${basePath}`); +} + +function isSqliteCorruptionError(error: unknown): boolean { + const code = error && typeof error === "object" ? (error as { code?: unknown }).code : undefined; + if (code === "SQLITE_CORRUPT" || code === "SQLITE_NOTADB") { + return true; + } + const message = String(error).toLowerCase(); + return message.includes("database disk image is malformed") || message.includes("not a database"); +} + +function resolveRecoverTargetKeys( + options: DoctorSessionSqliteOptions, + targets: readonly SessionStoreTarget[], +): ReadonlySet<string> | undefined { + const hasSelector = Boolean(options.agent || options.allAgents || options.store); + return hasSelector + ? new Set(targets.map((target) => sessionSqliteMigrationTargetKey(target))) + : undefined; +} + +function filterRecoveryManifestTargets<T extends { agentId: string; storePath: string }>( + targets: readonly T[], + selectedTargetKeys: ReadonlySet<string> | undefined, +): T[] { + if (!selectedTargetKeys) { + return [...targets]; + } + return targets.filter((target) => + selectedTargetKeys.has(sessionSqliteMigrationTargetKey(target)), + ); +} + +function createSyntheticRecoverTargetReport( + env: NodeJS.ProcessEnv, + message: string, +): DoctorSessionSqliteTargetReport { + return { + agentId: "recover", + archivedTranscriptFiles: [], + archivedUnreferencedJsonlFiles: [], + importedEntries: 0, + importedTranscriptEvents: 0, + issues: [{ code: "recover_manifest_missing", message }], + legacyEntries: 0, + referencedTranscriptFiles: 0, + sqliteEntries: 0, + sqlitePath: "", + storePath: resolveSessionSqliteMigrationRunsDir(env), + unreferencedJsonlFiles: [], + validatedEntries: 0, + validatedTranscriptEvents: 0, + }; +} + +function createEmptyRecoverTargetReport( + target: SessionStoreTarget, + sqlitePath: string, +): DoctorSessionSqliteTargetReport { + return { + agentId: target.agentId, + archivedTranscriptFiles: [], + archivedUnreferencedJsonlFiles: [], + importedEntries: 0, + importedTranscriptEvents: 0, + issues: [], + legacyEntries: 0, + referencedTranscriptFiles: 0, + sqliteEntries: 0, + sqlitePath, + storePath: target.storePath, + unreferencedJsonlFiles: [], + validatedEntries: 0, + validatedTranscriptEvents: 0, + }; +} + +function summarizeRecoverReport( + targets: DoctorSessionSqliteTargetReport[], +): DoctorSessionSqliteReport { + return { + mode: "recover", + targets, + totals: { + archivedTranscriptFiles: 0, + archivedUnreferencedJsonlFiles: 0, + importedEntries: 0, + importedTranscriptEvents: 0, + issues: targets.reduce((total, target) => total + target.issues.length, 0), + legacyEntries: targets.reduce((total, target) => total + target.legacyEntries, 0), + sqliteEntries: targets.reduce((total, target) => total + target.sqliteEntries, 0), + targets: targets.length, + unreferencedJsonlFiles: targets.reduce( + (total, target) => total + target.unreferencedJsonlFiles.length, + 0, + ), + validatedEntries: targets.reduce((total, target) => total + target.validatedEntries, 0), + validatedTranscriptEvents: targets.reduce( + (total, target) => total + target.validatedTranscriptEvents, + 0, + ), + }, + }; +} diff --git a/src/commands/doctor-session-sqlite-restore-report.ts b/src/commands/doctor-session-sqlite-restore-report.ts new file mode 100644 index 000000000000..12a81b87e83e --- /dev/null +++ b/src/commands/doctor-session-sqlite-restore-report.ts @@ -0,0 +1,104 @@ +/** Builds doctor reports for session SQLite migration restore mode. */ +import type { SessionStoreTarget } from "../config/sessions/targets.js"; +import { + resolveSessionSqliteMigrationRunsDir, + restoreSessionSqliteMigrationRuns, + sessionSqliteMigrationTargetKey, +} from "./doctor-session-sqlite-migration-run.js"; +import { readSqliteEntryCount, resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js"; +import type { + DoctorSessionSqliteReport, + DoctorSessionSqliteTargetReport, +} from "./doctor-session-sqlite-types.js"; + +export async function restoreDoctorSessionSqliteTargets(params: { + env: NodeJS.ProcessEnv; + restoreAllManifests: boolean; + targets: readonly SessionStoreTarget[]; +}): Promise<DoctorSessionSqliteReport> { + const targetReports = params.targets.map((target) => createEmptyTargetReport(target)); + const targetKeys = params.restoreAllManifests + ? undefined + : new Set(params.targets.map((target) => sessionSqliteMigrationTargetKey(target))); + const restore = restoreSessionSqliteMigrationRuns({ + env: params.env, + targetKeys, + }); + const reportTarget = + targetReports[0] ?? + createSyntheticRestoreTargetReport( + params.env, + restore.manifestPaths[0] ?? resolveSessionSqliteMigrationRunsDir(params.env), + ); + reportTarget.restore = restore; + reportTarget.issues.push( + ...restore.conflicts.map((conflict) => ({ + code: "restore_conflict", + message: `${conflict.sourcePath}: ${conflict.reason}`, + })), + ); + return summarizeRestoreReport(targetReports.length > 0 ? targetReports : [reportTarget]); +} + +function createEmptyTargetReport(target: SessionStoreTarget): DoctorSessionSqliteTargetReport { + return { + agentId: target.agentId, + archivedTranscriptFiles: [], + archivedUnreferencedJsonlFiles: [], + importedEntries: 0, + importedTranscriptEvents: 0, + issues: [], + legacyEntries: 0, + referencedTranscriptFiles: 0, + sqliteEntries: readSqliteEntryCount(target), + sqlitePath: resolveTargetSqlitePath(target), + storePath: target.storePath, + unreferencedJsonlFiles: [], + validatedEntries: 0, + validatedTranscriptEvents: 0, + }; +} + +function createSyntheticRestoreTargetReport( + env: NodeJS.ProcessEnv, + manifestPath: string, +): DoctorSessionSqliteTargetReport { + return { + agentId: "restore", + archivedTranscriptFiles: [], + archivedUnreferencedJsonlFiles: [], + importedEntries: 0, + importedTranscriptEvents: 0, + issues: [], + legacyEntries: 0, + referencedTranscriptFiles: 0, + sqliteEntries: 0, + sqlitePath: "", + storePath: manifestPath || resolveSessionSqliteMigrationRunsDir(env), + unreferencedJsonlFiles: [], + validatedEntries: 0, + validatedTranscriptEvents: 0, + }; +} + +function summarizeRestoreReport( + targets: DoctorSessionSqliteTargetReport[], +): DoctorSessionSqliteReport { + return { + mode: "restore", + targets, + totals: { + archivedTranscriptFiles: 0, + archivedUnreferencedJsonlFiles: 0, + importedEntries: 0, + importedTranscriptEvents: 0, + issues: targets.reduce((total, target) => total + target.issues.length, 0), + legacyEntries: 0, + sqliteEntries: targets.reduce((total, target) => total + target.sqliteEntries, 0), + targets: targets.length, + unreferencedJsonlFiles: 0, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }, + }; +} diff --git a/src/commands/doctor-session-sqlite-types.ts b/src/commands/doctor-session-sqlite-types.ts new file mode 100644 index 000000000000..4826e935e6f3 --- /dev/null +++ b/src/commands/doctor-session-sqlite-types.ts @@ -0,0 +1,132 @@ +/** Shared type contracts for doctor-owned session SQLite migration reports. */ +import type { OpenClawConfig } from "../config/types.openclaw.js"; + +export type DoctorSessionSqliteIssue = { + code: string; + message: string; + sessionKey?: string; +}; + +export type DoctorSessionSqliteRestoreConflict = { + archivePath: string; + reason: string; + sourcePath: string; +}; + +export type DoctorSessionSqliteRestoreReport = { + conflicts: DoctorSessionSqliteRestoreConflict[]; + manifestPaths: string[]; + restoredFiles: string[]; + skippedFiles: string[]; +}; + +export type DoctorSessionSqliteLargestSession = { + events: number; + rowBytes: number; + sessionId: string; +}; + +export type DoctorSessionSqliteDbStats = { + dbSizeBytes: number; + integrityCheck?: string; + largestSessions: DoctorSessionSqliteLargestSession[]; + totalTranscriptRowBytes: number; + walSizeBytes: number; +}; + +export type DoctorSessionSqliteCompactReport = { + dbSizeAfterBytes: number; + dbSizeBeforeBytes: number; + freelistAfterPages: number; + freelistBeforePages: number; + pageSizeBytes: number; + reclaimedBytes: number; + skipped: boolean; + walSizeAfterBytes: number; + walSizeBeforeBytes: number; +}; + +export type DoctorSessionSqliteCorruptRecovery = { + movedFiles: string[]; + skippedFiles: string[]; +}; + +export type SessionSqliteMigrationFailureIssue = { + body: string; + bodyPath?: string; + github?: { + fallbackUrl?: string; + message?: string; + status: "created" | "failed" | "skipped"; + url?: string; + }; + title: string; + url: string; +}; + +export type DoctorSessionSqliteMode = + | "dry-run" + | "import" + | "validate" + | "inspect" + | "compact" + | "restore" + | "recover"; + +export type DoctorSessionSqliteOptions = { + allAgents?: boolean; + agent?: string; + cfg?: OpenClawConfig; + env?: NodeJS.ProcessEnv; + mode: DoctorSessionSqliteMode; + store?: string; +}; + +export type DoctorSessionSqliteTargetReport = { + agentId: string; + archivedLegacyStoreFiles?: string[]; + archivedTranscriptFiles: string[]; + archivedUnreferencedJsonlFiles: string[]; + dbStats?: DoctorSessionSqliteDbStats; + importedEntries: number; + importedTranscriptEvents: number; + issues: DoctorSessionSqliteIssue[]; + legacyEntries: number; + referencedTranscriptFiles: number; + sqliteEntries: number; + sqlitePath: string; + storePath: string; + unreferencedJsonlFiles: string[]; + validatedEntries: number; + validatedTranscriptEvents: number; + compact?: DoctorSessionSqliteCompactReport; + corruptRecovery?: DoctorSessionSqliteCorruptRecovery; + restore?: DoctorSessionSqliteRestoreReport; +}; + +export type DoctorSessionSqliteReport = { + migrationRun?: { + failureReportJsonPath?: string; + failureReportMarkdownPath?: string; + manifestPath: string; + runId: string; + }; + mode: DoctorSessionSqliteMode; + supportIssue?: SessionSqliteMigrationFailureIssue; + targets: DoctorSessionSqliteTargetReport[]; + totals: { + archivedLegacyStoreFiles?: number; + archivedTranscriptFiles: number; + archivedUnreferencedJsonlFiles: number; + importedEntries: number; + importedTranscriptEvents: number; + issues: number; + legacyEntries: number; + reclaimedBytes?: number; + sqliteEntries: number; + targets: number; + unreferencedJsonlFiles: number; + validatedEntries: number; + validatedTranscriptEvents: number; + }; +}; diff --git a/src/commands/doctor-session-sqlite.test.ts b/src/commands/doctor-session-sqlite.test.ts new file mode 100644 index 000000000000..a2e74a99d3db --- /dev/null +++ b/src/commands/doctor-session-sqlite.test.ts @@ -0,0 +1,1363 @@ +// Doctor session SQLite tests exercise real temp stores and per-agent SQLite files. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + loadExactSqliteSessionEntry, + loadSqliteTranscriptEventsSync, + upsertSqliteSessionEntry, +} from "../config/sessions/session-accessor.sqlite.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import type { SessionSqliteMigrationManifest } from "./doctor-session-sqlite-migration-run.js"; +import { runDoctorSessionSqlite } from "./doctor-session-sqlite.js"; + +type TestStore = { + configPath: string; + env: NodeJS.ProcessEnv; + sessionDir: string; + stateDir: string; + storePath: string; + tempDir: string; + unreferencedJsonlPath: string; + trajectoryPath: string; + transcriptPath: string; +}; + +const previousEnv = { + OPENCLAW_CONFIG_PATH: process.env.OPENCLAW_CONFIG_PATH, + OPENCLAW_STATE_DIR: process.env.OPENCLAW_STATE_DIR, +}; + +beforeEach(() => { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); +}); + +afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + restoreEnvValue("OPENCLAW_CONFIG_PATH", previousEnv.OPENCLAW_CONFIG_PATH); + restoreEnvValue("OPENCLAW_STATE_DIR", previousEnv.OPENCLAW_STATE_DIR); +}); + +describe("runDoctorSessionSqlite", () => { + it("dry-runs a legacy store without writing SQLite rows", async () => { + const store = createLegacyStore(); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "dry-run", + store: store.storePath, + }); + + expect(report.totals).toMatchObject({ + importedEntries: 0, + importedTranscriptEvents: 0, + issues: 0, + legacyEntries: 1, + sqliteEntries: 0, + targets: 1, + unreferencedJsonlFiles: 2, + validatedEntries: 1, + validatedTranscriptEvents: 2, + }); + expect(report.targets[0]?.sqlitePath).toBeTruthy(); + expect(fs.existsSync(report.targets[0]?.sqlitePath ?? "")).toBe(false); + }); + + it("inspects a legacy store without creating a SQLite database", async () => { + const store = createLegacyStore(); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "inspect", + store: store.storePath, + }); + + expect(report.totals).toMatchObject({ + issues: 0, + legacyEntries: 1, + sqliteEntries: 0, + targets: 1, + }); + expect(report.targets[0]?.sqlitePath).toBeTruthy(); + expect(fs.existsSync(report.targets[0]?.sqlitePath ?? "")).toBe(false); + }); + + it("inspects SQLite-only all-agent targets without requiring a legacy store", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-session-sqlite-")); + try { + const stateDir = path.join(tempDir, "state"); + const storePath = path.join(stateDir, "agents", "main", "sessions", "sessions.json"); + const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; + await upsertSqliteSessionEntry( + { agentId: "main", env, sessionKey: "agent:main:main", storePath }, + { sessionId: "sqlite-session", updatedAt: Date.now() }, + ); + + const report = await runDoctorSessionSqlite({ + allAgents: true, + cfg: {}, + env, + mode: "inspect", + }); + + expect(fs.existsSync(storePath)).toBe(false); + expect(report.totals).toMatchObject({ + issues: 0, + legacyEntries: 0, + sqliteEntries: 1, + targets: 1, + }); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("repairs legacy message and route shapes at the import boundary", async () => { + const store = createLegacyStore({ + entryOverrides: { + route: "stale-custom-slot", + deliveryContext: { channel: "telegram", to: "123" }, + }, + transcriptLines: [ + '{"type":"session","sessionId":"session-1"}', + '{"type":"message","id":"m1","parentId":null,"message":{"role":"assistant","content":"legacy string"}}', + ], + }); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + + expect(report.totals).toMatchObject({ importedEntries: 1, issues: 0 }); + const imported = loadExactSqliteSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath: store.storePath, + }); + // The SQLite runtime does no read repair, so import must store canonical shapes. + expect(typeof imported?.entry.route).not.toBe("string"); + const events = loadSqliteTranscriptEventsSync({ + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:main", + storePath: store.storePath, + }); + const message = events.find((event) => (event as { type?: string }).type === "message") as { + message?: { content?: unknown }; + }; + expect(message?.message?.content).toEqual([{ type: "text", text: "legacy string" }]); + }); + + it("imports and validates legacy sessions idempotently", async () => { + const store = createLegacyStore(); + + const firstImport = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const secondImport = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const validation = await runDoctorSessionSqlite({ + env: store.env, + mode: "validate", + store: store.storePath, + }); + const inspect = await runDoctorSessionSqlite({ + env: store.env, + mode: "inspect", + store: store.storePath, + }); + + expect(firstImport.totals).toMatchObject({ + archivedLegacyStoreFiles: 1, + archivedTranscriptFiles: 2, + archivedUnreferencedJsonlFiles: 1, + importedEntries: 1, + importedTranscriptEvents: 2, + issues: 0, + sqliteEntries: 1, + unreferencedJsonlFiles: 0, + }); + expect(secondImport.totals).toMatchObject({ + archivedLegacyStoreFiles: 0, + archivedTranscriptFiles: 0, + archivedUnreferencedJsonlFiles: 0, + importedEntries: 0, + importedTranscriptEvents: 0, + issues: 0, + sqliteEntries: 0, + unreferencedJsonlFiles: 0, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }); + expect(validation.totals).toMatchObject({ + issues: 0, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }); + expect(fs.existsSync(store.storePath)).toBe(false); + expect(fs.existsSync(store.transcriptPath)).toBe(false); + expect(fs.existsSync(store.trajectoryPath)).toBe(false); + expect(fs.existsSync(store.unreferencedJsonlPath)).toBe(false); + expect(firstImport.targets[0]?.archivedTranscriptFiles).toHaveLength(2); + for (const archivedTranscriptPath of firstImport.targets[0]?.archivedTranscriptFiles ?? []) { + expect(archivedTranscriptPath).toBeTruthy(); + expect(archivedTranscriptPath).not.toContain(`${path.sep}sessions${path.sep}`); + expect(fs.existsSync(archivedTranscriptPath)).toBe(true); + } + expect(firstImport.targets[0]?.archivedUnreferencedJsonlFiles).toHaveLength(1); + const archivedUnreferencedPath = firstImport.targets[0]?.archivedUnreferencedJsonlFiles[0]; + expect(archivedUnreferencedPath).toBeTruthy(); + expect(archivedUnreferencedPath).not.toContain(`${path.sep}sessions${path.sep}`); + expect(archivedUnreferencedPath).toContain("archive-tier.orphan.jsonl.imported-"); + expect(fs.existsSync(archivedUnreferencedPath)).toBe(true); + expect(fs.readFileSync(archivedUnreferencedPath, "utf-8")).toBe('{"type":"event"}\n'); + expect(inspect.totals.sqliteEntries).toBe(1); + expect(inspect.totals.unreferencedJsonlFiles).toBe(0); + expect( + loadExactSqliteSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath: store.storePath, + })?.entry.sessionFile, + ).toContain("sqlite:main:session-1:"); + expect( + loadSqliteTranscriptEventsSync({ + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:main", + storePath: store.storePath, + }), + ).toHaveLength(2); + }); + + it("compacts migrated agent SQLite databases and reports reclaimed pages", async () => { + const store = createLegacyStore({ + transcriptLines: [ + '{"type":"session","sessionId":"session-1"}', + ...Array.from({ length: 240 }, (_, index) => + JSON.stringify({ + id: `evt-${index}`, + message: { content: "x".repeat(2_000), role: "user" }, + type: "message", + }), + ), + ], + }); + const importReport = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const sqlitePath = importReport.targets[0]?.sqlitePath; + expect(sqlitePath).toBeTruthy(); + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(sqlitePath ?? ""); + try { + db.exec("DELETE FROM transcript_events;"); + } finally { + db.close(); + } + + const compact = await runDoctorSessionSqlite({ + env: store.env, + mode: "compact", + store: store.storePath, + }); + + expect(compact.totals.issues).toBe(0); + expect(compact.totals.reclaimedBytes).toBeGreaterThan(0); + expect(compact.targets[0]?.compact).toMatchObject({ + freelistAfterPages: 0, + skipped: false, + }); + expect(compact.targets[0]?.compact?.freelistBeforePages).toBeGreaterThan(0); + expect(compact.targets[0]?.compact?.dbSizeAfterBytes).toBeLessThan( + compact.targets[0]?.compact?.dbSizeBeforeBytes ?? 0, + ); + }); + + it("does not report SQLite markers as missing transcript files", async () => { + const store = createLegacyStore(); + fs.rmSync(store.transcriptPath); + fs.rmSync(store.trajectoryPath); + fs.writeFileSync( + store.storePath, + JSON.stringify( + { + "agent:main:main": { + channel: "cli", + chatType: "direct", + sessionFile: `sqlite:main:session-1:${store.storePath}`, + sessionId: "session-1", + sessionStartedAt: 1000, + updatedAt: 2000, + }, + }, + null, + 2, + ), + { mode: 0o600 }, + ); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const validation = await runDoctorSessionSqlite({ + env: store.env, + mode: "validate", + store: store.storePath, + }); + + expect(report.totals).toMatchObject({ + importedEntries: 1, + importedTranscriptEvents: 0, + issues: 0, + sqliteEntries: 1, + }); + expect(validation.totals).toMatchObject({ + issues: 0, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }); + expect( + loadExactSqliteSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath: store.storePath, + })?.entry.sessionFile, + ).toContain("sqlite:main:session-1:"); + }); + + it("validates missing SQLite rows without creating the agent database", async () => { + const store = createLegacyStore(); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "validate", + store: store.storePath, + }); + + expect(report.totals).toMatchObject({ + issues: 1, + sqliteEntries: 0, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }); + expect(report.targets[0]?.issues[0]).toMatchObject({ + code: "sqlite_entry_missing", + sessionKey: "agent:main:main", + }); + expect(fs.existsSync(report.targets[0]?.sqlitePath ?? "")).toBe(false); + }); + + it("writes a migration manifest with planned and completed archive moves", async () => { + const store = createLegacyStore(); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const manifest = readMigrationManifest(report.migrationRun?.manifestPath); + const target = manifest.targets[0]; + + expect(report.migrationRun?.runId).toBe(manifest.runId); + expect(target).toMatchObject({ + agentId: "main", + storePath: store.storePath, + validationBeforeArchive: "passed", + }); + expect(target.plannedMoves).toHaveLength(4); + expect(target.completedMoves).toHaveLength(4); + expect(target.plannedMoves.map((move) => path.basename(move.sourcePath)).toSorted()).toEqual([ + "orphan.jsonl", + "session-1.jsonl", + "session-1.trajectory.jsonl", + "sessions.json", + ]); + }); + + it("archives legacy trajectory pointer files with imported transcripts", async () => { + const store = createLegacyStore(); + const pointerPath = path.join(store.sessionDir, "session-1.trajectory-path.json"); + fs.writeFileSync( + pointerPath, + `${JSON.stringify({ + traceSchema: "openclaw-trajectory-pointer", + schemaVersion: 1, + sessionId: "session-1", + runtimeFile: store.trajectoryPath, + })}\n`, + { mode: 0o600 }, + ); + const expectedPointerPath = canonicalTestPath(pointerPath); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const archivedNames = + report.targets[0]?.archivedTranscriptFiles.map((filePath) => path.basename(filePath)) ?? []; + + expect(fs.existsSync(pointerPath)).toBe(false); + expect(archivedNames).toEqual( + expect.arrayContaining([expect.stringContaining("session-1.trajectory-path.json.imported-")]), + ); + expect( + readMigrationManifest(report.migrationRun?.manifestPath).targets[0]?.plannedMoves, + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: "trajectory", + sourcePath: expectedPointerPath, + }), + ]), + ); + }); + + it("restores archived artifacts from the migration manifest", async () => { + const store = createLegacyStore(); + const importReport = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const manifest = readMigrationManifest(importReport.migrationRun?.manifestPath); + const sourcePaths = manifest.targets[0]?.plannedMoves.map((move) => move.sourcePath) ?? []; + + const restore = await runDoctorSessionSqlite({ + allAgents: true, + cfg: {}, + env: store.env, + mode: "restore", + }); + + expect(restore.totals.issues).toBe(0); + expect(restore.targets[0]?.restore).toMatchObject({ + conflicts: [], + restoredFiles: expect.arrayContaining(sourcePaths), + }); + expect(fs.existsSync(store.transcriptPath)).toBe(true); + expect(fs.existsSync(store.trajectoryPath)).toBe(true); + expect(fs.existsSync(store.unreferencedJsonlPath)).toBe(true); + }); + + it("restores planned moves when a crash prevented completed move recording", async () => { + const store = createLegacyStore(); + const importReport = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const manifestPath = requireMigrationManifestPath(importReport.migrationRun?.manifestPath); + const manifest = readMigrationManifest(manifestPath); + manifest.targets[0].completedMoves = []; + fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 }); + + const restore = await runDoctorSessionSqlite({ + allAgents: true, + cfg: {}, + env: store.env, + mode: "restore", + }); + + expect(restore.totals.issues).toBe(0); + expect(restore.targets[0]?.restore?.restoredFiles).toEqual( + expect.arrayContaining(canonicalTestPaths([store.transcriptPath, store.trajectoryPath])), + ); + expect(fs.existsSync(store.transcriptPath)).toBe(true); + expect(fs.existsSync(store.trajectoryPath)).toBe(true); + }); + + it("treats repeated restore as idempotent when files are already restored", async () => { + const store = createLegacyStore(); + const importReport = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const manifest = readMigrationManifest(importReport.migrationRun?.manifestPath); + const sourcePaths = manifest.targets[0]?.plannedMoves.map((move) => move.sourcePath) ?? []; + await runDoctorSessionSqlite({ + allAgents: true, + cfg: {}, + env: store.env, + mode: "restore", + }); + + const secondRestore = await runDoctorSessionSqlite({ + allAgents: true, + cfg: {}, + env: store.env, + mode: "restore", + }); + + expect(secondRestore.totals.issues).toBe(0); + expect(secondRestore.targets[0]?.restore?.restoredFiles).toEqual([]); + expect(secondRestore.targets[0]?.restore?.skippedFiles).toEqual( + expect.arrayContaining(sourcePaths), + ); + }); + + it("does not restore unrelated manifests for an unmatched explicit store selector", async () => { + const store = createLegacyStore(); + await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + + const restore = await runDoctorSessionSqlite({ + env: store.env, + mode: "restore", + store: path.join(store.tempDir, "missing", "sessions.json"), + }); + + expect(restore.targets[0]?.restore?.manifestPaths).toEqual([]); + expect(restore.targets[0]?.restore?.restoredFiles).toEqual([]); + expect(fs.existsSync(store.transcriptPath)).toBe(false); + }); + + it("reports restore conflicts without overwriting existing files", async () => { + const store = createLegacyStore(); + const transcriptPath = canonicalTestPath(store.transcriptPath); + await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + fs.writeFileSync(store.transcriptPath, '{"type":"event","id":"new"}\n', { mode: 0o600 }); + + const restore = await runDoctorSessionSqlite({ + allAgents: true, + cfg: {}, + env: store.env, + mode: "restore", + }); + + expect(restore.totals.issues).toBe(1); + expect(restore.targets[0]?.restore?.conflicts[0]).toMatchObject({ + reason: "source and archive both exist; refusing to overwrite source", + sourcePath: transcriptPath, + }); + expect(fs.readFileSync(store.transcriptPath, "utf-8")).toBe('{"type":"event","id":"new"}\n'); + }); + + it("recovers the latest failed migration run and prepares a sanitized GitHub issue", async () => { + const store = createLegacyStore({ agentDirName: "token=supersecret" }); + const importReport = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const manifestPath = requireMigrationManifestPath(importReport.migrationRun?.manifestPath); + const manifest = readMigrationManifest(manifestPath); + manifest.failedAt = "2030-01-01T00:00:00.000Z"; + manifest.targets[0].issues = [ + { + code: "startup_failure", + message: `token=supersecret startup migration failed for agent:main:main at ${store.storePath} and ${process.env.HOME ?? "/Users/example"}/private/openclaw.json`, + sessionKey: "agent:main:main", + }, + ]; + fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 }); + writeFailedManifest(store, "older-failed.json", "2000-01-01T00:00:00.000Z"); + + const recover = await runDoctorSessionSqlite({ + cfg: {}, + env: store.env, + mode: "recover", + }); + + expect(recover.mode).toBe("recover"); + expect(recover.totals.issues).toBe(0); + expect(recover.migrationRun?.manifestPath).toBe(manifestPath); + expect(recover.targets[0]?.restore?.manifestPaths).toEqual([manifestPath]); + expect(recover.targets[0]?.restore?.restoredFiles).toEqual( + expect.arrayContaining(canonicalTestPaths([store.transcriptPath, store.trajectoryPath])), + ); + expect(fs.existsSync(store.transcriptPath)).toBe(true); + expect(recover.supportIssue?.title).toContain(manifest.runId); + expect(recover.supportIssue?.body).toContain("startup_failure"); + expect(recover.supportIssue?.body).not.toContain("agent:main:main"); + expect(recover.supportIssue?.body).not.toContain("supersecret"); + expect(recover.supportIssue?.body).not.toContain(store.storePath); + if (process.env.HOME) { + expect(recover.supportIssue?.body).not.toContain(process.env.HOME); + } + expect(recover.supportIssue?.url).toContain("github.com/openclaw/openclaw/issues/new"); + }); + + it("recovers only manifests matching an explicit store selector", async () => { + const store = createLegacyStore(); + const importReport = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const manifestPath = requireMigrationManifestPath(importReport.migrationRun?.manifestPath); + const manifest = readMigrationManifest(manifestPath); + manifest.failedAt = "2030-01-01T00:00:00.000Z"; + manifest.targets[0].issues = [ + { code: "startup_failure", message: "selected store failed after archive" }, + ]; + manifest.targets.push({ + agentId: "other", + completedMoves: [], + issues: [{ code: "unselected_failure", message: "unselected target should stay private" }], + plannedMoves: [], + sqlitePath: path.join(store.tempDir, "other.sqlite"), + storePath: path.join(store.tempDir, "other", "sessions.json"), + validationBeforeArchive: "failed", + }); + fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 }); + writeFailedManifest(store, "newer-unselected.json", "2040-01-01T00:00:00.000Z", { + agentId: "other", + storePath: path.join(store.tempDir, "other", "sessions.json"), + }); + + const recover = await runDoctorSessionSqlite({ + cfg: {}, + env: store.env, + mode: "recover", + store: store.storePath, + }); + + expect(recover.migrationRun?.manifestPath).toBe(manifestPath); + expect(recover.targets[0]?.restore?.manifestPaths).toEqual([manifestPath]); + expect(recover.supportIssue?.body).not.toContain("unselected_failure"); + expect(fs.existsSync(store.transcriptPath)).toBe(true); + }); + + it("imports aliases that share one legacy transcript before archiving it", async () => { + const store = createLegacyStore(); + const legacyStore = JSON.parse(fs.readFileSync(store.storePath, "utf-8")) as Record< + string, + unknown + >; + legacyStore["agent:main:alias"] = legacyStore["agent:main:main"]; + fs.writeFileSync(store.storePath, `${JSON.stringify(legacyStore, null, 2)}\n`, { mode: 0o600 }); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + + expect(report.totals).toMatchObject({ + archivedTranscriptFiles: 2, + importedEntries: 2, + importedTranscriptEvents: 2, + issues: 0, + sqliteEntries: 2, + }); + expect(fs.existsSync(store.transcriptPath)).toBe(false); + expect( + loadExactSqliteSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath: store.storePath, + })?.entry.sessionId, + ).toBe("session-1"); + expect( + loadExactSqliteSessionEntry({ + agentId: "main", + sessionKey: "agent:main:alias", + storePath: store.storePath, + })?.entry.sessionId, + ).toBe("session-1"); + }); + + it("archives a legacy transcript symlink without moving the symlink target", async () => { + const store = createLegacyStore(); + const outsideTranscriptPath = path.join(store.tempDir, "outside-session-1.jsonl"); + fs.renameSync(store.transcriptPath, outsideTranscriptPath); + fs.symlinkSync(outsideTranscriptPath, store.transcriptPath); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + + const archivedTranscriptPath = report.targets[0]?.archivedTranscriptFiles.find((filePath) => + filePath.includes("session-1.jsonl"), + ); + expect(archivedTranscriptPath).toBeTruthy(); + expect(fs.existsSync(outsideTranscriptPath)).toBe(true); + expect(fs.existsSync(store.transcriptPath)).toBe(false); + expect(fs.lstatSync(archivedTranscriptPath ?? "").isSymbolicLink()).toBe(true); + }); + + it("imports explicit stores into the agent database owned by the path", async () => { + const store = createLegacyStore({ agentDirName: "codex-proof" }); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + + expect(report.targets[0]?.agentId).toBe("codex-proof"); + expect(report.totals).toMatchObject({ + importedEntries: 1, + importedTranscriptEvents: 2, + issues: 0, + sqliteEntries: 1, + }); + expect( + loadSqliteTranscriptEventsSync({ + agentId: "codex-proof", + sessionId: "session-1", + sessionKey: "agent:main:main", + storePath: store.storePath, + }), + ).toHaveLength(2); + }); + + it("imports legacy entries even when their transcript sidecar is missing", async () => { + const store = createLegacyStore(); + fs.rmSync(store.transcriptPath); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + + expect(report.totals).toMatchObject({ + importedEntries: 1, + importedTranscriptEvents: 0, + issues: 1, + sqliteEntries: 1, + }); + expect(report.targets[0]?.issues[0]).toMatchObject({ + code: "transcript_missing", + sessionKey: "agent:main:main", + }); + expect( + loadExactSqliteSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath: store.storePath, + })?.entry.sessionId, + ).toBe("session-1"); + expect( + loadSqliteTranscriptEventsSync({ + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:main", + storePath: store.storePath, + }), + ).toEqual([]); + }); + + it("keeps a shared legacy store intact when importing only one agent", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-session-sqlite-")); + try { + const stateDir = path.join(tempDir, "state"); + const sessionDir = path.join(tempDir, "shared-session-store"); + const storePath = path.join(sessionDir, "sessions.json"); + const mainTranscriptPath = path.join(sessionDir, "main-session.jsonl"); + const workTranscriptPath = path.join(sessionDir, "work-session.jsonl"); + const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; + fs.mkdirSync(sessionDir, { recursive: true }); + fs.writeFileSync( + storePath, + JSON.stringify({ + "agent:main:main": { + sessionFile: "main-session.jsonl", + sessionId: "main-session", + updatedAt: 20, + }, + "agent:work:main": { + sessionFile: "work-session.jsonl", + sessionId: "work-session", + updatedAt: 30, + }, + }), + { mode: 0o600 }, + ); + fs.writeFileSync(mainTranscriptPath, '{"type":"session","sessionId":"main-session"}\n'); + fs.writeFileSync(workTranscriptPath, '{"type":"session","sessionId":"work-session"}\n'); + + const report = await runDoctorSessionSqlite({ + agent: "main", + cfg: { + agents: { list: [{ default: true, id: "main" }, { id: "work" }] }, + session: { store: storePath }, + }, + env, + mode: "import", + }); + + expect(report.totals).toMatchObject({ + archivedLegacyStoreFiles: 0, + archivedTranscriptFiles: 0, + importedEntries: 1, + issues: 0, + }); + expect(fs.existsSync(storePath)).toBe(true); + expect(fs.existsSync(mainTranscriptPath)).toBe(true); + expect(fs.existsSync(workTranscriptPath)).toBe(true); + expect( + loadExactSqliteSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath, + })?.entry.sessionId, + ).toBe("main-session"); + expect( + loadExactSqliteSessionEntry({ + agentId: "work", + sessionKey: "agent:work:main", + storePath, + }), + ).toBeUndefined(); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("imports shared custom stores into per-agent SQLite targets", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-session-sqlite-")); + try { + const stateDir = path.join(tempDir, "state"); + const sessionDir = path.join(tempDir, "shared-session-store"); + const storePath = path.join(sessionDir, "sessions.json"); + const mainTranscriptPath = path.join(sessionDir, "main-session.jsonl"); + const workTranscriptPath = path.join(sessionDir, "work-session.jsonl"); + const orphanTranscriptPath = path.join(sessionDir, "orphan.jsonl"); + const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; + fs.mkdirSync(sessionDir, { recursive: true }); + fs.writeFileSync( + storePath, + JSON.stringify( + { + "agent:main:main": { + sessionFile: "main-session.jsonl", + sessionId: "main-session", + updatedAt: 20, + }, + "agent:work:main": { + sessionFile: "work-session.jsonl", + sessionId: "work-session", + updatedAt: 30, + }, + }, + null, + 2, + ), + { mode: 0o600 }, + ); + fs.writeFileSync(mainTranscriptPath, '{"type":"session","sessionId":"main-session"}\n', { + mode: 0o600, + }); + fs.writeFileSync(workTranscriptPath, '{"type":"session","sessionId":"work-session"}\n', { + mode: 0o600, + }); + fs.writeFileSync(orphanTranscriptPath, '{"type":"event","id":"orphan"}\n', { mode: 0o600 }); + + const report = await runDoctorSessionSqlite({ + allAgents: true, + cfg: { + agents: { list: [{ default: true, id: "main" }, { id: "work" }] }, + session: { store: storePath }, + }, + env, + mode: "import", + }); + + expect(report.targets.map((target) => target.agentId)).toEqual(["main", "work"]); + expect(report.totals).toMatchObject({ + archivedLegacyStoreFiles: 1, + archivedTranscriptFiles: 2, + archivedUnreferencedJsonlFiles: 1, + importedEntries: 2, + importedTranscriptEvents: 2, + issues: 0, + sqliteEntries: 2, + }); + const manifest = readMigrationManifest(report.migrationRun?.manifestPath); + for (const target of manifest.targets) { + expect(target.completedMoves.some((move) => move.kind === "legacy-store")).toBe(true); + } + expect( + loadExactSqliteSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath, + })?.entry.sessionId, + ).toBe("main-session"); + expect( + loadExactSqliteSessionEntry({ + agentId: "work", + sessionKey: "agent:work:main", + storePath, + })?.entry.sessionId, + ).toBe("work-session"); + expect(fs.existsSync(mainTranscriptPath)).toBe(false); + expect(fs.existsSync(workTranscriptPath)).toBe(false); + expect(fs.existsSync(orphanTranscriptPath)).toBe(false); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("reports active JSONL files left beside SQLite-backed sessions", async () => { + const store = createLegacyStore(); + + await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + fs.writeFileSync(store.transcriptPath, '{"type":"event","id":"heartbeat"}\n', { + mode: 0o600, + }); + await upsertSqliteSessionEntry( + { + agentId: "main", + env: store.env, + sessionKey: "agent:main:main", + storePath: store.storePath, + }, + { + sessionFile: "session-1.jsonl", + sessionId: "session-1", + updatedAt: 3000, + }, + ); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "inspect", + store: store.storePath, + }); + + expect(report.totals.issues).toBe(1); + expect(report.targets[0]?.issues[0]).toMatchObject({ + code: "active_sqlite_transcript_jsonl", + sessionKey: "agent:main:main", + }); + expect(report.targets[0]?.issues[0]?.message).toContain("session-1.jsonl"); + }); + + it("reports active JSONL scan failures without aborting inspect", async () => { + const store = createLegacyStore(); + const sqlitePath = path.join( + store.stateDir, + "agents", + "main", + "agent", + "openclaw-agent.sqlite", + ); + fs.mkdirSync(path.dirname(sqlitePath), { recursive: true }); + fs.writeFileSync(sqlitePath, "not a sqlite database\n", { mode: 0o600 }); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "inspect", + store: store.storePath, + }); + + expect(report.totals.issues).toBe(2); + expect(report.targets[0]?.issues.map((issue) => issue.code)).toEqual([ + "sqlite_corrupt", + "sqlite_active_transcript_scan_failed", + ]); + }); + + it("moves corrupt SQLite database files aside during recovery", async () => { + const store = createLegacyStore(); + const sqlitePath = path.join( + store.stateDir, + "agents", + "main", + "agent", + "openclaw-agent.sqlite", + ); + fs.mkdirSync(path.dirname(sqlitePath), { recursive: true }); + fs.writeFileSync(sqlitePath, "not a sqlite database\n", { mode: 0o600 }); + fs.writeFileSync(`${sqlitePath}-wal`, "wal", { mode: 0o600 }); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "recover", + store: store.storePath, + }); + + expect(report.totals.issues).toBe(0); + expect(report.targets[0]?.corruptRecovery?.movedFiles.length).toBeGreaterThanOrEqual(2); + expect(fs.existsSync(sqlitePath)).toBe(false); + expect(fs.existsSync(`${sqlitePath}-wal`)).toBe(false); + expect(fs.existsSync(`${sqlitePath}-shm`)).toBe(false); + expect( + report.targets[0]?.corruptRecovery?.movedFiles.every((filePath) => + filePath.includes(".corrupt-"), + ), + ).toBe(true); + }); + + it("does not move SQLite paths aside for non-corruption recovery inspection failures", async () => { + const store = createLegacyStore(); + const sqlitePath = path.join( + store.stateDir, + "agents", + "main", + "agent", + "openclaw-agent.sqlite", + ); + fs.mkdirSync(sqlitePath, { recursive: true }); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "recover", + store: store.storePath, + }); + + expect(report.totals.issues).toBe(1); + expect(report.targets[0]?.issues[0]?.code).toBe("sqlite_recovery_inspect_failed"); + expect(report.targets[0]?.corruptRecovery).toBeUndefined(); + expect(fs.statSync(sqlitePath).isDirectory()).toBe(true); + }); + + it("does not truncate existing SQLite transcript rows when re-importing a duplicate fragment", async () => { + const store = createLegacyStore({ + transcriptLines: [ + '{"type":"session","sessionId":"session-1"}', + '{"type":"message","id":"msg-1","message":{"role":"user","content":"first"}}', + '{"type":"message","id":"msg-2","message":{"role":"assistant","content":"second"}}', + ], + }); + + await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + fs.writeFileSync( + store.transcriptPath, + '{"type":"message","id":"msg-2","message":{"role":"assistant","content":"second"}}\n', + { mode: 0o600 }, + ); + fs.writeFileSync(store.trajectoryPath, `${JSON.stringify({ type: "trajectory" })}\n`, { + mode: 0o600, + }); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + + expect(report.totals).toMatchObject({ + archivedTranscriptFiles: 0, + importedEntries: 0, + importedTranscriptEvents: 0, + issues: 0, + }); + expect( + loadSqliteTranscriptEventsSync({ + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:main", + storePath: store.storePath, + }), + ).toHaveLength(3); + }); + + it("reports custom explicit store sqlite paths beside the store", async () => { + const store = createLegacyStore({ customStore: true }); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + + expect(report.targets[0]?.sqlitePath).toBe( + path.join(store.sessionDir, "openclaw-agent.sqlite"), + ); + expect(fs.existsSync(report.targets[0]?.sqlitePath)).toBe(true); + expect( + loadSqliteTranscriptEventsSync({ + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:main", + storePath: store.storePath, + }), + ).toHaveLength(2); + }); + + it("imports valid transcript rows when only the final JSONL line is crash-truncated", async () => { + const store = createLegacyStore(); + fs.writeFileSync( + store.transcriptPath, + '{"type":"session","sessionId":"session-1"}\n{"type":"message"', + { mode: 0o600 }, + ); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + + expect(report.totals).toMatchObject({ + importedEntries: 1, + importedTranscriptEvents: 1, + issues: 0, + sqliteEntries: 1, + }); + expect( + loadSqliteTranscriptEventsSync({ + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:main", + storePath: store.storePath, + }), + ).toHaveLength(1); + expect(fs.existsSync(store.transcriptPath)).toBe(false); + }); + + it("reports malformed transcripts while importing the session entry", async () => { + const store = createLegacyStore({ + agentDirName: "token=supersecret", + transcriptLines: ['{"type":"session","sessionId":"session-1"}', "{bad"], + }); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const inspect = await runDoctorSessionSqlite({ + env: store.env, + mode: "inspect", + store: store.storePath, + }); + + expect(report.totals.issues).toBe(1); + expect(report.totals).toMatchObject({ + archivedTranscriptFiles: 2, + archivedUnreferencedJsonlFiles: 1, + importedEntries: 1, + importedTranscriptEvents: 1, + sqliteEntries: 1, + unreferencedJsonlFiles: 0, + }); + expect(report.targets[0]?.issues[0]?.code).toBe("transcript_malformed"); + expect(fs.existsSync(store.transcriptPath)).toBe(false); + expect(fs.existsSync(store.unreferencedJsonlPath)).toBe(false); + expect(inspect.totals.sqliteEntries).toBe(1); + expect( + loadSqliteTranscriptEventsSync({ + agentId: "token-supersecret", + sessionId: "session-1", + sessionKey: "agent:main:main", + storePath: store.storePath, + }), + ).toHaveLength(1); + const manifest = readMigrationManifest(report.migrationRun?.manifestPath); + expect(manifest.targets[0]?.completedMoves.some((move) => move.kind === "transcript")).toBe( + true, + ); + expect( + manifest.targets[0]?.completedMoves.some((move) => move.kind === "unreferenced-jsonl"), + ).toBe(true); + expect(report.migrationRun?.failureReportMarkdownPath).toBeTruthy(); + const failureReport = fs.readFileSync( + report.migrationRun?.failureReportMarkdownPath ?? "", + "utf-8", + ); + expect(failureReport).toContain("transcript_malformed"); + expect(failureReport).toContain("openclaw doctor --session-sqlite recover --github-issue"); + expect(failureReport).not.toContain("supersecret"); + }); + + it("reports malformed selected legacy transcripts during validation", async () => { + const store = createLegacyStore({ transcriptLines: ['{"type":"session"}', "{bad"] }); + await upsertSqliteSessionEntry( + { + agentId: "main", + env: store.env, + sessionKey: "agent:main:main", + storePath: store.storePath, + }, + { sessionId: "session-1", updatedAt: 2000 }, + ); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "validate", + store: store.storePath, + }); + + expect(report.totals).toMatchObject({ + issues: 2, + sqliteEntries: 1, + validatedEntries: 1, + validatedTranscriptEvents: 0, + }); + expect(report.targets[0]?.issues[0]).toMatchObject({ + code: "transcript_malformed", + sessionKey: "agent:main:main", + }); + }); +}); + +function createLegacyStore( + params: { + agentDirName?: string; + customStore?: boolean; + entryOverrides?: Record<string, unknown>; + transcriptLines?: string[]; + } = {}, +): TestStore { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-session-sqlite-")); + const stateDir = path.join(tempDir, "state"); + const configPath = path.join(tempDir, "openclaw.json"); + const sessionDir = params.customStore + ? path.join(tempDir, "legacy-session-store") + : path.join(stateDir, "agents", params.agentDirName ?? "main", "sessions"); + const storePath = path.join(sessionDir, "sessions.json"); + const transcriptPath = path.join(sessionDir, "session-1.jsonl"); + const trajectoryPath = path.join(sessionDir, "session-1.trajectory.jsonl"); + const unreferencedJsonlPath = path.join(sessionDir, "orphan.jsonl"); + fs.mkdirSync(sessionDir, { recursive: true }); + fs.writeFileSync(configPath, "{}\n", { mode: 0o600 }); + fs.writeFileSync( + storePath, + JSON.stringify( + { + "agent:main:main": { + channel: "cli", + chatType: "direct", + sessionFile: "session-1.jsonl", + sessionId: "session-1", + sessionStartedAt: 1000, + updatedAt: 2000, + ...params.entryOverrides, + }, + }, + null, + 2, + ), + { mode: 0o600 }, + ); + fs.writeFileSync( + transcriptPath, + `${(params.transcriptLines ?? ['{"type":"session","sessionId":"session-1"}', '{"type":"event","id":"evt-1"}']).join("\n")}\n`, + { mode: 0o600 }, + ); + fs.writeFileSync(trajectoryPath, `${JSON.stringify({ type: "trajectory" })}\n`, { + mode: 0o600, + }); + fs.writeFileSync(unreferencedJsonlPath, '{"type":"event"}\n', { + mode: 0o600, + }); + const env = { + ...process.env, + OPENCLAW_CONFIG_PATH: configPath, + OPENCLAW_STATE_DIR: stateDir, + }; + process.env.OPENCLAW_CONFIG_PATH = configPath; + process.env.OPENCLAW_STATE_DIR = stateDir; + return { + configPath, + env, + sessionDir, + stateDir, + storePath, + tempDir, + unreferencedJsonlPath, + trajectoryPath, + transcriptPath, + }; +} + +function readMigrationManifest(manifestPath: string | undefined): SessionSqliteMigrationManifest { + if (!manifestPath) { + throw new Error("expected migration manifest path"); + } + return JSON.parse(fs.readFileSync(manifestPath, "utf-8")) as SessionSqliteMigrationManifest; +} + +function requireMigrationManifestPath(manifestPath: string | undefined): string { + if (!manifestPath) { + throw new Error("expected migration manifest path"); + } + return manifestPath; +} + +function writeFailedManifest( + store: TestStore, + fileName: string, + failedAt: string, + target: { agentId?: string; storePath?: string } = {}, +): void { + const runsDir = path.join(store.stateDir, "session-sqlite-migration-runs"); + fs.mkdirSync(runsDir, { recursive: true, mode: 0o700 }); + fs.writeFileSync( + path.join(runsDir, fileName), + `${JSON.stringify( + { + failedAt, + manifestVersion: 1, + openClawVersion: "test", + runId: path.basename(fileName, ".json"), + startedAt: failedAt, + targets: [ + { + agentId: target.agentId ?? "older", + completedMoves: [], + issues: [{ code: "older_failure", message: "older failure" }], + plannedMoves: [], + sqlitePath: path.join(store.tempDir, "older.sqlite"), + storePath: target.storePath ?? path.join(store.tempDir, "older-sessions.json"), + validationBeforeArchive: "failed", + }, + ], + }, + null, + 2, + )}\n`, + { mode: 0o600 }, + ); +} + +function canonicalTestPaths(paths: string[]): string[] { + return paths.map((filePath) => canonicalTestPath(filePath)).toSorted(); +} + +function canonicalTestPath(filePath: string): string { + try { + return fs.realpathSync.native(filePath); + } catch { + return path.resolve(filePath); + } +} + +function restoreEnvValue(key: keyof NodeJS.ProcessEnv, value: string | undefined): void { + if (value === undefined) { + delete process.env[key]; + return; + } + process.env[key] = value; +} diff --git a/src/commands/doctor-session-sqlite.ts b/src/commands/doctor-session-sqlite.ts new file mode 100644 index 000000000000..c235d51839e0 --- /dev/null +++ b/src/commands/doctor-session-sqlite.ts @@ -0,0 +1,1175 @@ +import fs from "node:fs"; +import path from "node:path"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { resolveDefaultAgentId } from "../agents/agent-scope.js"; +import { getRuntimeConfig } from "../config/config.js"; +import { resolveSessionFilePath } from "../config/sessions/paths.js"; +import { + importSqliteSessionRows, + loadExactSqliteSessionEntry, +} from "../config/sessions/session-accessor.sqlite.js"; +import { resolveSqliteTargetFromSessionStorePath } from "../config/sessions/session-sqlite-target.js"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; +import { normalizeStoreSessionKey } from "../config/sessions/store-entry.js"; +import { normalizeSessionEntryDelivery } from "../config/sessions/store-load.js"; +import { + resolveAgentSessionStoreTargetsSync, + resolveAllAgentSessionStoreTargetsSync, + resolveSessionStoreTargets, + type SessionStoreTarget, +} from "../config/sessions/targets.js"; +import type { SessionEntry } from "../config/sessions/types.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { resolveStoredSessionOwnerAgentId } from "../gateway/session-store-key.js"; +import { compactDoctorSessionSqliteTarget } from "./doctor-session-sqlite-compact.js"; +import { + createSessionSqliteMigrationRun, + recordCompletedMigrationMove, + recordPlannedMigrationMove, + updateMigrationManifestTarget, + writeSessionSqliteMigrationFailureReports, + writeSessionSqliteMigrationManifest, + type ActiveSessionSqliteMigrationRun, + type SessionSqliteMigrationMoveKind, + type SessionSqliteMigrationTargetInput, +} from "./doctor-session-sqlite-migration-run.js"; +import { + countTranscriptEventsForPath, + createTranscriptEventReader, + createTranscriptEventPrefixReader, + readOnlySqliteDbStats, + readOnlySqliteExactSessionEntry, + readOnlySqliteSessionEntries, + readOnlySqliteTranscriptEventCount, + readSqliteEntryCount, + resolveTargetSqlitePath, +} from "./doctor-session-sqlite-readers.js"; +import { recoverDoctorSessionSqliteTargets } from "./doctor-session-sqlite-recover-report.js"; +import { restoreDoctorSessionSqliteTargets } from "./doctor-session-sqlite-restore-report.js"; +import type { + DoctorSessionSqliteIssue, + DoctorSessionSqliteMode, + DoctorSessionSqliteOptions, + DoctorSessionSqliteReport, + DoctorSessionSqliteTargetReport, +} from "./doctor-session-sqlite-types.js"; +export { + restoreSessionSqliteMigrationRun, + writeSessionSqliteMigrationFailureReports, +} from "./doctor-session-sqlite-migration-run.js"; +export type { + DoctorSessionSqliteIssue, + DoctorSessionSqliteMode, + DoctorSessionSqliteOptions, + DoctorSessionSqliteReport, + DoctorSessionSqliteRestoreConflict, + DoctorSessionSqliteRestoreReport, + DoctorSessionSqliteTargetReport, +} from "./doctor-session-sqlite-types.js"; + +type LegacySessionRecord = { + entry: SessionEntry; + sessionKey: string; + transcriptPath?: string; +}; + +const WARNING_ISSUE_CODES = new Set([ + "transcript_missing", + "transcript_archive_failed", + "transcript_malformed", + "unreferenced_jsonl_archive_failed", +]); + +/** Runs the targeted doctor SQLite session migration/inspection submode. */ +export async function runDoctorSessionSqlite( + options: DoctorSessionSqliteOptions, +): Promise<DoctorSessionSqliteReport> { + const env = options.env ?? process.env; + const cfg = resolveDoctorSessionSqliteConfig(options); + const targets = resolveDoctorSessionSqliteTargets({ + allAgents: options.allAgents, + agent: options.agent, + cfg, + env, + mode: options.mode, + store: options.store, + }); + if (options.mode === "restore") { + return restoreDoctorSessionSqliteTargets({ + env, + restoreAllManifests: + targets.length === 0 && !options.agent && !options.allAgents && !options.store, + targets, + }); + } + if (options.mode === "recover") { + return recoverDoctorSessionSqliteTargets({ + env, + options, + targets, + validateTarget: (target) => inspectOrMigrateTarget({ cfg, env, mode: "validate", target }), + }); + } + const activeRun = + options.mode === "import" && targets.length > 0 + ? createSessionSqliteMigrationRun(env, targets.map(createMigrationTargetInput)) + : undefined; + const fullyCoveredStorePaths = + options.mode === "import" + ? resolveFullyCoveredLegacyStorePaths(cfg, targets) + : new Set<string>(); + const reports: DoctorSessionSqliteTargetReport[] = []; + for (const target of targets) { + reports.push( + await inspectOrMigrateTarget({ + activeRun, + archiveImportedArtifacts: fullyCoveredStorePaths.has(path.resolve(target.storePath)), + cfg, + env, + mode: options.mode, + target, + }), + ); + } + if (activeRun) { + archiveImportedLegacySessionStores(targets, reports, activeRun, fullyCoveredStorePaths); + const hasIssues = reports.some((report) => report.issues.length > 0); + activeRun.manifest.completedAt = new Date().toISOString(); + if (hasIssues) { + activeRun.manifest.failedAt = activeRun.manifest.completedAt; + const failureReports = writeSessionSqliteMigrationFailureReports(activeRun.manifestPath, { + reason: "doctor import reported session SQLite migration issues", + }); + activeRun.manifest.failureReports = failureReports; + } + writeSessionSqliteMigrationManifest(activeRun); + } + return summarizeDoctorSessionSqliteReport(options.mode, reports, activeRun); +} + +// Direct store migrations are scoped by path; broader agent discovery needs runtime config. +function resolveDoctorSessionSqliteConfig(options: DoctorSessionSqliteOptions): OpenClawConfig { + if (options.cfg) { + return options.cfg; + } + return options.store ? {} : getRuntimeConfig(); +} + +function resolveDoctorSessionSqliteTargets(params: { + allAgents?: boolean; + agent?: string; + cfg: OpenClawConfig; + env: NodeJS.ProcessEnv; + mode: DoctorSessionSqliteMode; + store?: string; +}): SessionStoreTarget[] { + if (params.store) { + return filterLegacySessionStoreTargets( + resolveSessionStoreTargets(params.cfg, { store: params.store }, { env: params.env }), + params.mode, + ); + } + if (params.agent) { + return filterLegacySessionStoreTargets( + resolveAgentSessionStoreTargetsSync(params.cfg, params.agent, { env: params.env }), + params.mode, + ); + } + if (params.allAgents) { + return filterLegacySessionStoreTargets( + resolveAllAgentSessionStoreTargetsSync(params.cfg, { env: params.env }), + params.mode, + ); + } + return resolveSessionStoreTargets(params.cfg, {}, { env: params.env }).filter((target) => + fs.existsSync(target.storePath), + ); +} + +function filterLegacySessionStoreTargets( + targets: SessionStoreTarget[], + mode: DoctorSessionSqliteMode, +): SessionStoreTarget[] { + if (mode === "inspect" || mode === "compact" || mode === "restore" || mode === "recover") { + return targets; + } + return targets.filter((target) => fs.existsSync(target.storePath)); +} + +async function inspectOrMigrateTarget(params: { + activeRun?: ActiveSessionSqliteMigrationRun; + archiveImportedArtifacts?: boolean; + cfg: OpenClawConfig; + env: NodeJS.ProcessEnv; + mode: Exclude<DoctorSessionSqliteMode, "restore" | "recover">; + target: SessionStoreTarget; +}): Promise<DoctorSessionSqliteTargetReport> { + const issues: DoctorSessionSqliteIssue[] = []; + const allRecords = readLegacySessionRecords(params.target, issues, { + allowMissingStore: params.mode === "inspect" || params.mode === "compact", + }); + const records = shouldFilterLegacySessionRecordsByTarget(params.target) + ? allRecords.filter((record) => + isLegacySessionRecordOwnedByTarget(params.cfg, params.target, record.sessionKey), + ) + : allRecords; + const referencedTranscriptFiles = new Set( + allRecords.flatMap((record) => (record.transcriptPath ? [record.transcriptPath] : [])), + ); + const report: DoctorSessionSqliteTargetReport = { + agentId: params.target.agentId, + archivedLegacyStoreFiles: [], + archivedTranscriptFiles: [], + archivedUnreferencedJsonlFiles: [], + importedEntries: 0, + importedTranscriptEvents: 0, + issues, + legacyEntries: records.length, + referencedTranscriptFiles: referencedTranscriptFiles.size, + sqliteEntries: readSqliteEntryCount(params.target), + sqlitePath: resolveTargetSqlitePath(params.target), + storePath: params.target.storePath, + unreferencedJsonlFiles: listUnreferencedJsonlFiles(params.target.storePath, [ + ...referencedTranscriptFiles, + ]), + validatedEntries: 0, + validatedTranscriptEvents: 0, + }; + if (params.mode === "inspect") { + report.sqliteEntries = readSqliteEntryCount(params.target); + appendSqliteDbStats(params.target, report); + appendActiveSqliteTranscriptFileIssues(params.target, report); + return report; + } + if (params.mode === "compact") { + compactSqliteDatabase(params.target, report); + report.sqliteEntries = readSqliteEntryCount(params.target); + appendSqliteDbStats(params.target, report); + return report; + } + for (const record of records) { + if (params.mode === "dry-run") { + countLegacyTranscript(record, report); + continue; + } + if (params.mode === "import") { + await importLegacySessionRecord(params.target, record, report); + continue; + } + validateLegacySessionRecord(params.target, record, report); + } + if (params.mode === "import" && blockingIssueCount(report) === 0) { + const validationPassed = validateImportedTargetBeforeArchive(params.target, records, report); + updateMigrationManifestTarget( + params.activeRun, + createMigrationTargetInput(params.target), + report.issues, + { + validationBeforeArchive: validationPassed ? "passed" : "failed", + }, + ); + if (validationPassed && params.archiveImportedArtifacts !== false) { + archiveImportedTranscripts(params.target, records, report, params.activeRun); + archiveUnreferencedJsonlFiles( + params.target, + report, + [...referencedTranscriptFiles], + params.activeRun, + ); + } + if (validationPassed) { + // Post-import compact retrofits auto_vacuum=INCREMENTAL onto pre-flip + // databases and returns the pages the import churn freed. + compactSqliteDatabase(params.target, report); + } + } + report.unreferencedJsonlFiles = listUnreferencedJsonlFiles(params.target.storePath, [ + ...referencedTranscriptFiles, + ]); + report.sqliteEntries = readSqliteEntryCount(params.target); + appendActiveSqliteTranscriptFileIssues(params.target, report); + updateMigrationManifestTarget( + params.activeRun, + createMigrationTargetInput(params.target), + report.issues, + ); + return report; +} + +function resolveFullyCoveredLegacyStorePaths( + cfg: OpenClawConfig, + targets: readonly SessionStoreTarget[], +): Set<string> { + const covered = new Set<string>(); + const targetsByStore = new Map<string, SessionStoreTarget[]>(); + for (const target of targets) { + const storePath = path.resolve(target.storePath); + targetsByStore.set(storePath, [...(targetsByStore.get(storePath) ?? []), target]); + } + for (const [storePath, storeTargets] of targetsByStore) { + const issues: DoctorSessionSqliteIssue[] = []; + const records = readLegacySessionRecords(storeTargets[0], issues); + const coversEveryRecord = records.every((record) => + storeTargets.some( + (target) => + !shouldFilterLegacySessionRecordsByTarget(target) || + isLegacySessionRecordOwnedByTarget(cfg, target, record.sessionKey), + ), + ); + if (issues.length === 0 && coversEveryRecord) { + covered.add(storePath); + } + } + return covered; +} + +function readLegacySessionRecords( + target: SessionStoreTarget, + issues: DoctorSessionSqliteIssue[], + options: { allowMissingStore?: boolean } = {}, +): LegacySessionRecord[] { + let parsed: unknown; + try { + parsed = JSON.parse(fs.readFileSync(target.storePath, "utf-8")); + } catch (err) { + if ( + options.allowMissingStore === true && + (err as NodeJS.ErrnoException | undefined)?.code === "ENOENT" + ) { + return []; + } + issues.push({ + code: "store_unreadable", + message: `${target.storePath}: ${String(err)}`, + }); + return []; + } + if (!isRecord(parsed)) { + issues.push({ + code: "store_not_object", + message: `${target.storePath} does not contain an object session store.`, + }); + return []; + } + const records: LegacySessionRecord[] = []; + for (const [sessionKey, value] of Object.entries(parsed)) { + if (!isSessionEntry(value)) { + issues.push({ + code: "entry_invalid", + message: "Session entry is missing a valid sessionId.", + sessionKey, + }); + continue; + } + records.push({ + // Import is the migration boundary: repair legacy delivery/route shapes + // here because the SQLite runtime read path assumes canonical entries. + entry: normalizeSessionEntryDelivery(value), + sessionKey, + transcriptPath: resolveLegacyTranscriptPath(target, value), + }); + } + return records; +} + +function isLegacySessionRecordOwnedByTarget( + cfg: OpenClawConfig, + target: SessionStoreTarget, + sessionKey: string, +): boolean { + const ownerAgentId = resolveStoredSessionOwnerAgentId({ + cfg, + agentId: target.agentId, + sessionKey, + }); + return ownerAgentId + ? ownerAgentId === target.agentId + : target.agentId === resolveDefaultAgentId(cfg); +} + +function shouldFilterLegacySessionRecordsByTarget(target: SessionStoreTarget): boolean { + return !resolveSqliteTargetFromSessionStorePath(target.storePath).agentId; +} + +function resolveLegacyTranscriptPath( + target: SessionStoreTarget, + entry: SessionEntry, +): string | undefined { + if (parseSqliteSessionFileMarker(entry.sessionFile)) { + return undefined; + } + const defaultPath = resolveSessionFilePath(entry.sessionId, entry, { + agentId: target.agentId, + sessionsDir: path.dirname(target.storePath), + }); + if (fs.existsSync(defaultPath)) { + return defaultPath; + } + return entry.sessionFile?.trim() ? defaultPath : undefined; +} + +function countLegacyTranscript( + record: LegacySessionRecord, + report: DoctorSessionSqliteTargetReport, +): void { + const result = countTranscriptEvents(record); + if (result.status === "missing") { + report.issues.push({ + code: "transcript_missing", + message: `Transcript file is missing: ${record.transcriptPath}`, + sessionKey: record.sessionKey, + }); + return; + } + if (result.status === "malformed") { + report.issues.push({ + code: "transcript_malformed", + message: result.message, + sessionKey: record.sessionKey, + }); + return; + } + report.validatedEntries += 1; + report.validatedTranscriptEvents += result.events; +} + +function blockingIssueCount(report: DoctorSessionSqliteTargetReport): number { + return report.issues.filter((issue) => !WARNING_ISSUE_CODES.has(issue.code)).length; +} + +async function importLegacySessionRecord( + target: SessionStoreTarget, + record: LegacySessionRecord, + report: DoctorSessionSqliteTargetReport, +): Promise<void> { + const result = countTranscriptEvents(record); + if (result.status === "missing") { + if (markAlreadyMigratedTranscript(target, record, report)) { + return; + } + const imported = await importSqliteSessionRows({ + agentId: target.agentId, + entry: record.entry, + sessionKey: record.sessionKey, + storePath: target.storePath, + }); + report.importedEntries += 1; + report.importedTranscriptEvents += imported.transcriptEvents; + report.issues.push({ + code: "transcript_missing", + message: `Transcript file is missing: ${record.transcriptPath}`, + sessionKey: record.sessionKey, + }); + return; + } else if (result.status === "malformed") { + const imported = await importSqliteSessionRows({ + agentId: target.agentId, + entry: record.entry, + sessionKey: record.sessionKey, + storePath: target.storePath, + ...(record.transcriptPath + ? { readTranscriptEvents: createTranscriptEventPrefixReader(record.transcriptPath) } + : {}), + }); + report.importedEntries += 1; + report.importedTranscriptEvents += imported.transcriptEvents; + report.issues.push({ + code: "transcript_malformed", + message: result.message, + sessionKey: record.sessionKey, + }); + return; + } + const imported = await importSqliteSessionRows({ + agentId: target.agentId, + entry: record.entry, + sessionKey: record.sessionKey, + storePath: target.storePath, + ...(record.transcriptPath && result.status === "ok" + ? { readTranscriptEvents: createTranscriptEventReader(record.transcriptPath) } + : {}), + }); + report.importedEntries += 1; + report.importedTranscriptEvents += imported.transcriptEvents; +} + +function markAlreadyMigratedTranscript( + target: SessionStoreTarget, + record: LegacySessionRecord, + report: DoctorSessionSqliteTargetReport, +): boolean { + const migratedEvents = countAlreadyMigratedTranscriptEventsForImport(target, record); + if (migratedEvents === undefined) { + return false; + } + report.validatedEntries += 1; + report.validatedTranscriptEvents += migratedEvents; + return true; +} + +function validateImportedTargetBeforeArchive( + target: SessionStoreTarget, + records: readonly LegacySessionRecord[], + report: DoctorSessionSqliteTargetReport, +): boolean { + const issueCountBeforeValidation = report.issues.length; + for (const record of records) { + validateImportedRecordBeforeArchive(target, record, report); + } + return report.issues.length === issueCountBeforeValidation; +} + +function validateImportedRecordBeforeArchive( + target: SessionStoreTarget, + record: LegacySessionRecord, + report: DoctorSessionSqliteTargetReport, +): void { + const normalizedKey = normalizeStoreSessionKey(record.sessionKey); + const sqliteEntry = loadExactSqliteSessionEntry({ + agentId: target.agentId, + sessionKey: normalizedKey, + storePath: target.storePath, + }); + if (!sqliteEntry) { + report.issues.push({ + code: "sqlite_entry_missing", + message: `SQLite entry is missing for ${normalizedKey}.`, + sessionKey: record.sessionKey, + }); + return; + } + if (sqliteEntry.entry.sessionId !== record.entry.sessionId) { + report.issues.push({ + code: "sqlite_entry_mismatch", + message: `SQLite sessionId ${sqliteEntry.entry.sessionId} does not match ${record.entry.sessionId}.`, + sessionKey: record.sessionKey, + }); + return; + } + const result = countTranscriptEvents(record); + if (result.status === "missing") { + return; + } + if (result.status !== "ok") { + if (!hasSessionIssue(report, "transcript_malformed", record.sessionKey)) { + report.issues.push({ + code: "transcript_malformed", + message: result.message, + sessionKey: record.sessionKey, + }); + } + return; + } + const sqliteEvents = readOnlySqliteTranscriptEventCount(target, record.entry.sessionId); + if (!sqliteEvents.ok) { + report.issues.push({ + code: "sqlite_read_failed", + message: `SQLite transcript count read failed: ${String(sqliteEvents.error)}`, + sessionKey: record.sessionKey, + }); + return; + } + if (sqliteEvents.events < result.events) { + report.issues.push({ + code: "sqlite_transcript_count_mismatch", + message: `SQLite transcript has ${sqliteEvents.events} events; source has ${result.events}.`, + sessionKey: record.sessionKey, + }); + } +} + +function archiveImportedTranscript( + target: SessionStoreTarget, + record: LegacySessionRecord, + report: DoctorSessionSqliteTargetReport, + activeRun: ActiveSessionSqliteMigrationRun | undefined, +): void { + if (!record.transcriptPath || !fs.existsSync(record.transcriptPath)) { + return; + } + try { + report.archivedTranscriptFiles.push( + ...moveImportedTranscriptArtifactsToArchive( + target, + record.sessionKey, + record.transcriptPath, + activeRun, + ), + ); + } catch (err) { + report.issues.push({ + code: "transcript_archive_failed", + message: `${record.transcriptPath}: ${String(err)}`, + sessionKey: record.sessionKey, + }); + } +} + +function archiveImportedTranscripts( + target: SessionStoreTarget, + records: readonly LegacySessionRecord[], + report: DoctorSessionSqliteTargetReport, + activeRun: ActiveSessionSqliteMigrationRun | undefined, +): void { + const archivedTranscriptPaths = new Set<string>(); + for (const record of records) { + if (!record.transcriptPath || archivedTranscriptPaths.has(record.transcriptPath)) { + continue; + } + archiveImportedTranscript(target, record, report, activeRun); + archivedTranscriptPaths.add(record.transcriptPath); + } +} + +function archiveUnreferencedJsonlFiles( + target: SessionStoreTarget, + report: DoctorSessionSqliteTargetReport, + referencedPaths: readonly string[], + activeRun: ActiveSessionSqliteMigrationRun | undefined, +): void { + for (const sourcePath of listUnreferencedJsonlFiles(target.storePath, referencedPaths)) { + try { + report.archivedUnreferencedJsonlFiles.push( + moveSessionJsonlToArchive({ + activeRun, + archiveKey: "archive-tier", + baseNameRaw: path.basename(sourcePath), + kind: "unreferenced-jsonl", + sourcePathRaw: sourcePath, + target, + }), + ); + } catch (err) { + report.issues.push({ + code: "unreferenced_jsonl_archive_failed", + message: `${sourcePath}: ${String(err)}`, + }); + } + } +} + +function archiveImportedLegacySessionStores( + targets: readonly SessionStoreTarget[], + reports: readonly DoctorSessionSqliteTargetReport[], + activeRun: ActiveSessionSqliteMigrationRun, + fullyCoveredStorePaths: ReadonlySet<string>, +): void { + const byStore = new Map< + string, + { report: DoctorSessionSqliteTargetReport; target: SessionStoreTarget }[] + >(); + for (const target of targets) { + const report = reports.find( + (candidate) => + candidate.agentId === target.agentId && + path.resolve(candidate.storePath) === path.resolve(target.storePath), + ); + if (!report) { + continue; + } + const key = path.resolve(target.storePath); + byStore.set(key, [...(byStore.get(key) ?? []), { report, target }]); + } + for (const [storePath, entries] of byStore) { + if (!fullyCoveredStorePaths.has(storePath)) { + continue; + } + if (entries.some((entry) => blockingIssueCount(entry.report) > 0)) { + continue; + } + const archivePath = archiveLegacySessionStore(entries[0].target, entries[0].report, activeRun); + if (!archivePath) { + continue; + } + for (const entry of entries.slice(1)) { + recordLegacyStoreMoveForTarget(entry.target, archivePath, activeRun); + } + } +} + +function archiveLegacySessionStore( + target: SessionStoreTarget, + report: DoctorSessionSqliteTargetReport, + activeRun: ActiveSessionSqliteMigrationRun | undefined, +): string | undefined { + if (!fs.existsSync(target.storePath)) { + return undefined; + } + try { + const archivePath = moveSessionJsonlToArchive({ + activeRun, + archiveKey: "legacy-store", + baseNameRaw: path.basename(target.storePath), + kind: "legacy-store", + sourcePathRaw: target.storePath, + target, + }); + (report.archivedLegacyStoreFiles ??= []).push(archivePath); + return archivePath; + } catch (err) { + report.issues.push({ + code: "legacy_store_archive_failed", + message: `${target.storePath}: ${String(err)}`, + }); + return undefined; + } +} + +function recordLegacyStoreMoveForTarget( + target: SessionStoreTarget, + archivePath: string, + activeRun: ActiveSessionSqliteMigrationRun, +): void { + const move = { + archivePath, + kind: "legacy-store" as const, + sourcePath: path.resolve(target.storePath), + }; + recordPlannedMigrationMove(activeRun, createMigrationTargetInput(target), move); + recordCompletedMigrationMove(activeRun, createMigrationTargetInput(target), move); +} + +function validateLegacySessionRecord( + target: SessionStoreTarget, + record: LegacySessionRecord, + report: DoctorSessionSqliteTargetReport, +): void { + const normalizedKey = normalizeStoreSessionKey(record.sessionKey); + const sqliteEntry = readOnlySqliteExactSessionEntry(target, normalizedKey); + if (!sqliteEntry.ok) { + report.issues.push({ + code: "sqlite_read_failed", + message: `SQLite session entry read failed: ${String(sqliteEntry.error)}`, + sessionKey: record.sessionKey, + }); + return; + } + if (!sqliteEntry.entry) { + report.issues.push({ + code: "sqlite_entry_missing", + message: `SQLite entry is missing for ${normalizedKey}.`, + sessionKey: record.sessionKey, + }); + return; + } + if (sqliteEntry.entry.entry.sessionId !== record.entry.sessionId) { + report.issues.push({ + code: "sqlite_entry_mismatch", + message: `SQLite sessionId ${sqliteEntry.entry.entry.sessionId} does not match ${record.entry.sessionId}.`, + sessionKey: record.sessionKey, + }); + return; + } + report.validatedEntries += 1; + validateTranscriptEventCount(target, record, report); +} + +function validateTranscriptEventCount( + target: SessionStoreTarget, + record: LegacySessionRecord, + report: DoctorSessionSqliteTargetReport, +): void { + const result = countTranscriptEvents(record); + if (result.status === "missing") { + const migratedEvents = countAlreadyMigratedTranscriptEventsForValidate(target, record); + if (migratedEvents !== undefined) { + report.validatedTranscriptEvents += migratedEvents; + } + return; + } + if (result.status !== "ok") { + if (!hasSessionIssue(report, "transcript_malformed", record.sessionKey)) { + report.issues.push({ + code: "transcript_malformed", + message: result.message, + sessionKey: record.sessionKey, + }); + } + return; + } + const sqliteEvents = readOnlySqliteTranscriptEventCount(target, record.entry.sessionId); + if (!sqliteEvents.ok) { + report.issues.push({ + code: "sqlite_read_failed", + message: `SQLite transcript count read failed: ${String(sqliteEvents.error)}`, + sessionKey: record.sessionKey, + }); + return; + } + if (sqliteEvents.events !== result.events) { + report.issues.push({ + code: "sqlite_transcript_count_mismatch", + message: `SQLite transcript has ${sqliteEvents.events} events; source has ${result.events}.`, + sessionKey: record.sessionKey, + }); + return; + } + report.validatedTranscriptEvents += sqliteEvents.events; +} + +function hasSessionIssue( + report: DoctorSessionSqliteTargetReport, + code: string, + sessionKey: string, +): boolean { + return report.issues.some((issue) => issue.code === code && issue.sessionKey === sessionKey); +} + +function countAlreadyMigratedTranscriptEventsForImport( + target: SessionStoreTarget, + record: LegacySessionRecord, +): number | undefined { + const normalizedKey = normalizeStoreSessionKey(record.sessionKey); + const sqliteEntry = loadExactSqliteSessionEntry({ + agentId: target.agentId, + sessionKey: normalizedKey, + storePath: target.storePath, + }); + if (sqliteEntry?.entry.sessionId !== record.entry.sessionId) { + return undefined; + } + const eventCount = readOnlySqliteTranscriptEventCount(target, record.entry.sessionId); + return eventCount.ok ? eventCount.events : undefined; +} + +function countAlreadyMigratedTranscriptEventsForValidate( + target: SessionStoreTarget, + record: LegacySessionRecord, +): number | undefined { + const normalizedKey = normalizeStoreSessionKey(record.sessionKey); + const sqliteEntry = readOnlySqliteExactSessionEntry(target, normalizedKey); + if (!sqliteEntry.ok || sqliteEntry.entry?.entry.sessionId !== record.entry.sessionId) { + return undefined; + } + const eventCount = readOnlySqliteTranscriptEventCount(target, record.entry.sessionId); + return eventCount.ok ? eventCount.events : undefined; +} + +function countTranscriptEvents( + record: LegacySessionRecord, +): + | { status: "ok"; events: number } + | { status: "missing" } + | { status: "malformed"; message: string } { + return countTranscriptEventsForPath(record.transcriptPath); +} + +function listUnreferencedJsonlFiles( + storePath: string, + referencedPaths: readonly string[], +): string[] { + const sessionsDir = path.dirname(storePath); + let entries: string[]; + try { + entries = fs.readdirSync(sessionsDir); + } catch { + return []; + } + const referenced = new Set(referencedPaths.map((filePath) => canonicalFilePath(filePath))); + return entries + .filter((entry) => entry.endsWith(".jsonl")) + .map((entry) => path.join(sessionsDir, entry)) + .filter((filePath) => !referenced.has(canonicalFilePath(filePath))) + .toSorted((a, b) => a.localeCompare(b)); +} + +function appendActiveSqliteTranscriptFileIssues( + target: SessionStoreTarget, + report: DoctorSessionSqliteTargetReport, +): void { + const result = readOnlySqliteSessionEntries(target); + if (!result.ok) { + report.issues.push({ + code: "sqlite_active_transcript_scan_failed", + message: `Could not scan SQLite-backed sessions for active JSONL transcript files: ${String(result.error)}`, + }); + return; + } + for (const summary of result.summaries) { + const transcriptPath = resolveActiveSqliteTranscriptFile(target, summary.entry); + if (!transcriptPath) { + continue; + } + report.issues.push({ + code: "active_sqlite_transcript_jsonl", + message: `SQLite-backed session still has an active JSONL transcript file: ${transcriptPath}`, + sessionKey: summary.sessionKey, + }); + } +} + +function appendSqliteDbStats( + target: SessionStoreTarget, + report: DoctorSessionSqliteTargetReport, +): void { + const result = readOnlySqliteDbStats(target); + if (!result.ok) { + report.issues.push({ + code: "sqlite_corrupt", + message: `SQLite database could not be inspected: ${String(result.error)}`, + }); + return; + } + report.dbStats = result.stats; + if (result.stats.integrityCheck && result.stats.integrityCheck !== "ok") { + report.issues.push({ + code: "sqlite_integrity_check_failed", + message: `SQLite quick_check reported: ${result.stats.integrityCheck}`, + }); + } +} + +function compactSqliteDatabase( + target: SessionStoreTarget, + report: DoctorSessionSqliteTargetReport, +): void { + try { + report.compact = compactDoctorSessionSqliteTarget(target); + } catch (err) { + report.issues.push({ + code: "sqlite_compact_failed", + message: `SQLite database compact failed: ${String(err)}`, + }); + } +} + +function resolveActiveSqliteTranscriptFile( + target: SessionStoreTarget, + entry: SessionEntry, +): string | undefined { + let transcriptPath: string; + try { + transcriptPath = resolveSessionFilePath(entry.sessionId, entry, { + agentId: target.agentId, + sessionsDir: path.dirname(target.storePath), + }); + } catch { + return undefined; + } + if (!transcriptPath.endsWith(".jsonl")) { + return undefined; + } + let stat: fs.Stats; + try { + stat = fs.statSync(transcriptPath); + } catch { + return undefined; + } + if (!stat.isFile()) { + return undefined; + } + const sessionsDir = canonicalFilePath(path.dirname(target.storePath)); + const activePath = canonicalFilePath(transcriptPath); + if (path.dirname(activePath) !== sessionsDir) { + return undefined; + } + return activePath; +} + +function moveImportedTranscriptArtifactsToArchive( + target: SessionStoreTarget, + sessionKey: string, + transcriptPath: string, + activeRun: ActiveSessionSqliteMigrationRun | undefined, +): string[] { + const archived = [ + moveImportedTranscriptToArchive(target, sessionKey, transcriptPath, "transcript", activeRun), + ]; + const trajectoryPath = resolveTrajectoryPath(transcriptPath); + if (trajectoryPath && fs.existsSync(trajectoryPath)) { + archived.push( + moveImportedTranscriptToArchive(target, sessionKey, trajectoryPath, "trajectory", activeRun), + ); + } + const trajectoryPointerPath = resolveTrajectoryPointerPath(transcriptPath); + if (trajectoryPointerPath && fs.existsSync(trajectoryPointerPath)) { + archived.push( + moveImportedTranscriptToArchive( + target, + sessionKey, + trajectoryPointerPath, + "trajectory", + activeRun, + ), + ); + } + return archived; +} + +function resolveTrajectoryPath(transcriptPath: string): string | undefined { + return transcriptPath.endsWith(".jsonl") + ? `${transcriptPath.slice(0, -".jsonl".length)}.trajectory.jsonl` + : undefined; +} + +function resolveTrajectoryPointerPath(transcriptPath: string): string | undefined { + return transcriptPath.endsWith(".jsonl") + ? `${transcriptPath.slice(0, -".jsonl".length)}.trajectory-path.json` + : undefined; +} + +function moveImportedTranscriptToArchive( + target: SessionStoreTarget, + sessionKey: string, + sourcePathRaw: string, + kind: SessionSqliteMigrationMoveKind, + activeRun: ActiveSessionSqliteMigrationRun | undefined, +): string { + return moveSessionJsonlToArchive({ + activeRun, + archiveKey: sessionKey, + baseNameRaw: path.basename(sourcePathRaw), + kind, + sessionKey, + sourcePathRaw, + target, + }); +} + +function moveSessionJsonlToArchive(params: { + activeRun: ActiveSessionSqliteMigrationRun | undefined; + archiveKey: string; + baseNameRaw: string; + kind: SessionSqliteMigrationMoveKind; + sessionKey?: string; + sourcePathRaw: string; + target: SessionStoreTarget; +}): string { + const { archiveKey, baseNameRaw, sourcePathRaw } = params; + const sourcePath = path.resolve(sourcePathRaw); + const stat = fs.statSync(sourcePath); + if (!stat.isFile()) { + throw new Error("source is not a regular file"); + } + const archiveDir = resolveImportedTranscriptArchiveDir(params.target.storePath); + fs.mkdirSync(archiveDir, { recursive: true }); + const baseName = baseNameRaw.replace(/[^A-Za-z0-9_.-]+/g, "_").slice(0, 160) || "artifact"; + const keySlug = archiveKey.replace(/[^A-Za-z0-9_.-]+/g, "_").slice(0, 120) || "session"; + for (let attempt = 0; attempt < 100; attempt += 1) { + const suffix = attempt === 0 ? "" : `.${attempt}`; + const archivePath = path.join( + archiveDir, + `${keySlug}.${baseName}.imported-${Date.now()}${suffix}`, + ); + if (fs.existsSync(archivePath)) { + continue; + } + try { + const move = { + archivePath, + kind: params.kind, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + sourcePath, + }; + recordPlannedMigrationMove(params.activeRun, createMigrationTargetInput(params.target), move); + fs.renameSync(sourcePath, archivePath); + recordCompletedMigrationMove( + params.activeRun, + createMigrationTargetInput(params.target), + move, + ); + return archivePath; + } catch (err) { + if ((err as { code?: unknown })?.code === "EEXIST") { + continue; + } + throw err; + } + } + throw new Error(`Could not archive ${baseName} for ${archiveKey}`); +} + +function resolveImportedTranscriptArchiveDir(storePath: string): string { + const storeDir = path.dirname(path.resolve(storePath)); + return path.join(path.dirname(storeDir), "session-sqlite-import-archive"); +} + +function canonicalFilePath(filePath: string): string { + try { + return fs.realpathSync.native(filePath); + } catch { + return path.resolve(filePath); + } +} + +function createMigrationTargetInput(target: SessionStoreTarget): SessionSqliteMigrationTargetInput { + return { + agentId: target.agentId, + sqlitePath: resolveTargetSqlitePath(target), + storePath: target.storePath, + }; +} + +function isSessionEntry(value: unknown): value is SessionEntry { + return isRecord(value) && typeof value.sessionId === "string" && value.sessionId.trim() !== ""; +} + +function summarizeDoctorSessionSqliteReport( + mode: DoctorSessionSqliteMode, + targets: DoctorSessionSqliteTargetReport[], + activeRun?: ActiveSessionSqliteMigrationRun, +): DoctorSessionSqliteReport { + return { + ...(activeRun + ? { + migrationRun: { + ...(activeRun.manifest.failureReports + ? { + failureReportJsonPath: activeRun.manifest.failureReports.jsonPath, + failureReportMarkdownPath: activeRun.manifest.failureReports.markdownPath, + } + : {}), + manifestPath: activeRun.manifestPath, + runId: activeRun.manifest.runId, + }, + } + : {}), + mode, + targets, + totals: { + archivedLegacyStoreFiles: targets.reduce( + (total, target) => total + (target.archivedLegacyStoreFiles?.length ?? 0), + 0, + ), + archivedTranscriptFiles: targets.reduce( + (total, target) => total + target.archivedTranscriptFiles.length, + 0, + ), + archivedUnreferencedJsonlFiles: targets.reduce( + (total, target) => total + target.archivedUnreferencedJsonlFiles.length, + 0, + ), + importedEntries: sumTargets(targets, "importedEntries"), + importedTranscriptEvents: sumTargets(targets, "importedTranscriptEvents"), + issues: targets.reduce((total, target) => total + target.issues.length, 0), + legacyEntries: sumTargets(targets, "legacyEntries"), + reclaimedBytes: targets.reduce( + (total, target) => total + (target.compact?.reclaimedBytes ?? 0), + 0, + ), + sqliteEntries: sumTargets(targets, "sqliteEntries"), + targets: targets.length, + unreferencedJsonlFiles: targets.reduce( + (total, target) => total + target.unreferencedJsonlFiles.length, + 0, + ), + validatedEntries: sumTargets(targets, "validatedEntries"), + validatedTranscriptEvents: sumTargets(targets, "validatedTranscriptEvents"), + }, + }; +} + +function sumTargets( + targets: DoctorSessionSqliteTargetReport[], + key: keyof Pick< + DoctorSessionSqliteTargetReport, + | "importedEntries" + | "importedTranscriptEvents" + | "legacyEntries" + | "sqliteEntries" + | "validatedEntries" + | "validatedTranscriptEvents" + >, +): number { + return targets.reduce((total, target) => total + target[key], 0); +} diff --git a/src/commands/doctor-session-transcripts.test.ts b/src/commands/doctor-session-transcripts.test.ts index 6e0e7f40322c..19074fec3c4b 100644 --- a/src/commands/doctor-session-transcripts.test.ts +++ b/src/commands/doctor-session-transcripts.test.ts @@ -6,11 +6,16 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { SessionManager } from "../agents/sessions/session-manager.js"; const note = vi.hoisted(() => vi.fn()); +const runDoctorSessionSqlite = vi.hoisted(() => vi.fn()); vi.mock("../../packages/terminal-core/src/note.js", () => ({ note, })); +vi.mock("./doctor-session-sqlite.js", () => ({ + runDoctorSessionSqlite, +})); + import { detectSessionTranscriptHealthIssues, noteSessionTranscriptHealth, @@ -42,6 +47,7 @@ describe("doctor session transcript repair", () => { beforeEach(async () => { note.mockClear(); + runDoctorSessionSqlite.mockReset(); root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-doctor-transcripts-")); }); @@ -191,6 +197,45 @@ describe("doctor session transcript repair", () => { expect(await fs.readFile(filePath, "utf-8")).toContain("openai-codex"); }); + it("runs session SQLite import through the public doctor repair path", async () => { + const sessionsDir = path.join(root, "agents", "main", "sessions"); + await fs.mkdir(sessionsDir, { recursive: true }); + runDoctorSessionSqlite.mockResolvedValueOnce({ + totals: { + archivedTranscriptFiles: 2, + archivedUnreferencedJsonlFiles: 1, + importedTranscriptEvents: 2, + issues: 0, + legacyEntries: 1, + sqliteEntries: 1, + unreferencedJsonlFiles: 0, + validatedTranscriptEvents: 0, + }, + }); + const env = { ...process.env, OPENCLAW_STATE_DIR: root }; + + await noteSessionTranscriptHealth({ + env, + sessionDirs: [sessionsDir], + sessionSqlite: true, + shouldRepair: true, + }); + + expect(runDoctorSessionSqlite).toHaveBeenCalledWith({ + allAgents: true, + env, + mode: "import", + }); + expect(note).toHaveBeenCalledWith( + expect.stringContaining("Legacy entries: 1"), + "Session SQLite", + ); + expect(note).toHaveBeenCalledWith( + expect.stringContaining("Archived 2 legacy transcript artifact(s)."), + "Session SQLite", + ); + }); + it("repairs supported current-version linear transcripts", async () => { const filePath = await writeTranscript([ { type: "session", version: 3, id: "session-linear", timestamp: "2026-06-15T00:00:00Z" }, diff --git a/src/commands/doctor-session-transcripts.ts b/src/commands/doctor-session-transcripts.ts index 8ae36e28accc..51017502ea92 100644 --- a/src/commands/doctor-session-transcripts.ts +++ b/src/commands/doctor-session-transcripts.ts @@ -432,6 +432,8 @@ export function sessionTranscriptIssueToRepairEffect( /** Scans session transcript files and reports or repairs legacy/broken transcript state. */ export async function noteSessionTranscriptHealth(params?: { + env?: NodeJS.ProcessEnv; + sessionSqlite?: boolean; shouldRepair?: boolean; sessionDirs?: string[]; }) { @@ -445,40 +447,88 @@ export async function noteSessionTranscriptHealth(params?: { } const results: TranscriptRepairResult[] = []; - if (shouldRepair) { - const files = await listSessionTranscriptFiles(sessionDirs); + const files = await listSessionTranscriptFiles(sessionDirs); + if (files.length > 0 && shouldRepair) { for (const filePath of files) { results.push(await repairBrokenSessionTranscriptFile({ filePath, shouldRepair })); } - } else { + } else if (files.length > 0) { results.push(...(await detectSessionTranscriptHealthIssues({ sessionDirs }))); } const broken = results.filter((result) => result.broken); - if (broken.length === 0) { + if (broken.length > 0) { + const repairedCount = broken.filter((result) => result.repaired).length; + const lines = [ + `- Found ${broken.length} transcript file${broken.length === 1 ? "" : "s"} with legacy state.`, + ...broken.slice(0, 20).map((result) => { + const backup = result.backupPath ? ` backup=${shortenHomePath(result.backupPath)}` : ""; + const status = result.repaired ? "repaired" : "needs repair"; + const metadata = + result.legacyOpenAICodexEntries > 0 + ? ` openai-codex=${result.legacyOpenAICodexEntries}` + : ""; + return `- ${shortenHomePath(result.filePath)} ${status} entries=${result.originalEntries}->${result.activeEntries + 1}${metadata}${backup}`; + }), + ]; + if (broken.length > 20) { + lines.push(`- ...and ${broken.length - 20} more.`); + } + if (!shouldRepair) { + lines.push('- Run "openclaw doctor --fix" to rewrite affected files to their active branch.'); + } else if (repairedCount > 0) { + lines.push(`- Repaired ${repairedCount} transcript file${repairedCount === 1 ? "" : "s"}.`); + } + note(lines.join("\n"), "Session transcripts"); + } + + if (params?.sessionDirs === undefined || params.sessionSqlite === true) { + await noteSessionSqliteMigrationHealth({ + env: params?.env ?? process.env, + shouldRepair, + }); + } +} + +async function noteSessionSqliteMigrationHealth(params: { + env: NodeJS.ProcessEnv; + shouldRepair: boolean; +}): Promise<void> { + // Public doctor owns the operator-facing SQLite import; the targeted + // --session-sqlite subcommand remains the diagnostic/proof surface. + const { runDoctorSessionSqlite } = await import("./doctor-session-sqlite.js"); + const report = await runDoctorSessionSqlite({ + allAgents: true, + env: params.env, + mode: params.shouldRepair ? "import" : "dry-run", + }); + if ( + report.totals.legacyEntries === 0 && + report.totals.unreferencedJsonlFiles === 0 && + report.totals.issues === 0 + ) { return; } - - const repairedCount = broken.filter((result) => result.repaired).length; const lines = [ - `- Found ${broken.length} transcript file${broken.length === 1 ? "" : "s"} with legacy state.`, - ...broken.slice(0, 20).map((result) => { - const backup = result.backupPath ? ` backup=${shortenHomePath(result.backupPath)}` : ""; - const status = result.repaired ? "repaired" : "needs repair"; - const metadata = - result.legacyOpenAICodexEntries > 0 - ? ` openai-codex=${result.legacyOpenAICodexEntries}` - : ""; - return `- ${shortenHomePath(result.filePath)} ${status} entries=${result.originalEntries}->${result.activeEntries + 1}${metadata}${backup}`; - }), + `- Legacy entries: ${report.totals.legacyEntries}; SQLite entries: ${report.totals.sqliteEntries}.`, + `- Transcript events: imported=${report.totals.importedTranscriptEvents}; validated=${report.totals.validatedTranscriptEvents}.`, ]; - if (broken.length > 20) { - lines.push(`- ...and ${broken.length - 20} more.`); + if (report.totals.archivedTranscriptFiles > 0) { + lines.push( + `- Archived ${report.totals.archivedTranscriptFiles} legacy transcript artifact(s).`, + ); } - if (!shouldRepair) { - lines.push('- Run "openclaw doctor --fix" to rewrite affected files to their active branch.'); - } else if (repairedCount > 0) { - lines.push(`- Repaired ${repairedCount} transcript file${repairedCount === 1 ? "" : "s"}.`); + if (report.totals.archivedUnreferencedJsonlFiles > 0) { + lines.push( + `- Archived ${report.totals.archivedUnreferencedJsonlFiles} unreferenced JSONL artifact(s).`, + ); } - - note(lines.join("\n"), "Session transcripts"); + if (report.totals.issues > 0) { + lines.push(`- Found ${report.totals.issues} session SQLite issue(s).`); + } + if (!params.shouldRepair) { + lines.push( + '- Run "openclaw doctor --fix" to migrate legacy session metadata/transcripts to SQLite.', + ); + } + note(lines.join("\n"), "Session SQLite"); } diff --git a/src/commands/doctor-state-migrations.test.ts b/src/commands/doctor-state-migrations.test.ts index dce9945957e1..c4c31435d645 100644 --- a/src/commands/doctor-state-migrations.test.ts +++ b/src/commands/doctor-state-migrations.test.ts @@ -1116,6 +1116,159 @@ describe("doctor legacy state migrations", () => { } }); + it("migrates legacy ACP metadata from retired custom-root agent stores", async () => { + const root = await makeTempRoot(); + const customRoot = await makeTempRoot(); + const legacySessionKey = "acp:binding:discord:default:feedface"; + const sessionKey = "agent:ops:acp:binding:discord:default:feedface"; + const storePath = path.join(customRoot, "agents", "ops", "sessions", "sessions.json"); + const cfg: OpenClawConfig = { + session: { + store: path.join(customRoot, "agents", "{agentId}", "sessions", "sessions.json"), + }, + }; + writeJson5(storePath, { + [legacySessionKey]: { + sessionId: "sess-acp", + updatedAt: 100, + acp: { + backend: "acpx", + agent: "codex", + runtimeSessionName: "codex-discord", + mode: "persistent", + state: "idle", + lastActivityAt: 123, + }, + }, + }); + + const detected = await detectLegacyStateMigrations({ + cfg, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ + detected, + config: cfg, + now: () => 456, + }); + + expect(result.warnings).toStrictEqual([]); + expect(result.changes.some((change) => change.includes("ACP session metadata"))).toBe(true); + const store = JSON.parse(fs.readFileSync(storePath, "utf8")) as Record<string, SessionEntry>; + expect(store[legacySessionKey]?.acp).toBeUndefined(); + + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(path.join(root, "state", "openclaw.sqlite")); + try { + const row = db + .prepare( + "SELECT backend, agent, runtime_session_name, mode, state, last_activity_at FROM acp_sessions WHERE session_key = ?", + ) + .get(sessionKey) as + | { + backend: string; + agent: string; + runtime_session_name: string; + mode: string; + state: string; + last_activity_at: number | bigint; + } + | undefined; + expect(row).toMatchObject({ + backend: "acpx", + agent: "codex", + runtime_session_name: "codex-discord", + mode: "persistent", + state: "idle", + }); + expect(Number(row?.last_activity_at)).toBe(123); + } finally { + db.close(); + } + }); + + it("skips symlinked managed-agent ACP metadata stores", async () => { + const root = await makeTempRoot(); + const outsideRoot = await makeTempRoot(); + const sessionKey = "agent:main:acp:binding:discord:default:feedface"; + const managedStorePath = path.join(root, "agents", "main", "sessions", "sessions.json"); + const outsideStorePath = path.join(outsideRoot, "sessions.json"); + writeJson5(outsideStorePath, { + [sessionKey]: { + sessionId: "sess-acp", + updatedAt: 100, + acp: { + backend: "acpx", + agent: "codex", + runtimeSessionName: "codex-discord", + mode: "persistent", + state: "idle", + lastActivityAt: 123, + }, + }, + }); + fs.mkdirSync(path.dirname(managedStorePath), { recursive: true }); + fs.symlinkSync(outsideStorePath, managedStorePath); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ detected }); + + expect(result.warnings).toStrictEqual([]); + expect(result.changes.some((change) => change.includes("ACP session metadata"))).toBe(false); + const outsideStore = JSON.parse(fs.readFileSync(outsideStorePath, "utf8")) as Record< + string, + SessionEntry + >; + expect(outsideStore[sessionKey]?.acp).toBeDefined(); + }); + + it("skips symlinked custom agent-store ACP metadata stores", async () => { + const root = await makeTempRoot(); + const customRoot = await makeTempRoot(); + const outsideRoot = await makeTempRoot(); + const sessionKey = "agent:main:acp:binding:discord:default:feedface"; + const cfg: OpenClawConfig = { + session: { + store: path.join(customRoot, "agents", "{agentId}", "sessions", "sessions.json"), + }, + }; + const managedStorePath = path.join(customRoot, "agents", "main", "sessions", "sessions.json"); + const outsideStorePath = path.join(outsideRoot, "sessions.json"); + writeJson5(outsideStorePath, { + [sessionKey]: { + sessionId: "sess-acp", + updatedAt: 100, + acp: { + backend: "acpx", + agent: "codex", + runtimeSessionName: "codex-discord", + mode: "persistent", + state: "idle", + lastActivityAt: 123, + }, + }, + }); + fs.mkdirSync(path.dirname(managedStorePath), { recursive: true }); + fs.symlinkSync(outsideStorePath, managedStorePath); + + const detected = await detectLegacyStateMigrations({ + cfg, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ detected, config: cfg }); + + expect(result.warnings).toStrictEqual([]); + expect(result.changes.some((change) => change.includes("ACP session metadata"))).toBe(false); + const outsideStore = JSON.parse(fs.readFileSync(outsideStorePath, "utf8")) as Record< + string, + SessionEntry + >; + expect(outsideStore[sessionKey]?.acp).toBeDefined(); + }); + it("keeps shipped WhatsApp legacy group keys channel-qualified during migration", async () => { const root = await makeTempRoot(); const cfg: OpenClawConfig = {}; diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts index eff15b5178a4..6051e37112b4 100644 --- a/src/commands/doctor.test.ts +++ b/src/commands/doctor.test.ts @@ -2,7 +2,9 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ + createSessionSqliteGithubIssue: vi.fn(), runPostUpgradeProbes: vi.fn(), + runDoctorSessionSqlite: vi.fn(), resolveInstalledPluginIndexStorePath: vi.fn(() => "/tmp/openclaw-installed-plugins.json"), })); @@ -10,6 +12,14 @@ vi.mock("./doctor-post-upgrade.js", () => ({ runPostUpgradeProbes: mocks.runPostUpgradeProbes, })); +vi.mock("./doctor-session-sqlite.js", () => ({ + runDoctorSessionSqlite: mocks.runDoctorSessionSqlite, +})); + +vi.mock("./doctor-session-sqlite-github-issue.js", () => ({ + createSessionSqliteGithubIssue: mocks.createSessionSqliteGithubIssue, +})); + vi.mock("../plugins/installed-plugin-index-store-path.js", () => ({ resolveInstalledPluginIndexStorePath: mocks.resolveInstalledPluginIndexStorePath, })); @@ -51,4 +61,152 @@ describe("doctorCommand", () => { expect(runtime.log).not.toHaveBeenCalled(); expect(runtime.exit).toHaveBeenCalledWith(1); }); + + it("writes session sqlite JSON through the runtime before exiting cleanly", async () => { + const report = { + mode: "inspect", + targets: [], + totals: { + archivedTranscriptFiles: 0, + archivedUnreferencedJsonlFiles: 0, + importedEntries: 0, + importedTranscriptEvents: 0, + issues: 0, + legacyEntries: 0, + sqliteEntries: 0, + targets: 0, + unreferencedJsonlFiles: 0, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }, + }; + mocks.runDoctorSessionSqlite.mockResolvedValueOnce(report); + const runtime = { + log: vi.fn(), + error: vi.fn(), + writeStdout: vi.fn(), + writeJson: vi.fn(), + exit: vi.fn((code: number) => { + throw new Error(`exit:${code}`); + }), + }; + + await expect( + doctorCommand(runtime, { + json: true, + sessionSqlite: "inspect", + sessionSqliteAgent: "main", + }), + ).rejects.toThrow("exit:0"); + + expect(mocks.runDoctorSessionSqlite).toHaveBeenCalledWith({ + agent: "main", + mode: "inspect", + }); + expect(runtime.writeJson).toHaveBeenCalledWith(report, 2); + expect(runtime.exit).toHaveBeenCalledWith(0); + }); + + it("creates a GitHub issue for approved session sqlite recovery reports", async () => { + const supportIssue = { + body: "sanitized body", + bodyPath: "/tmp/session.failure.md", + title: "Session SQLite migration recovery report (run-1)", + url: "https://github.com/openclaw/openclaw/issues/new?title=run-1", + }; + const report = { + mode: "recover", + supportIssue, + targets: [], + totals: { + archivedTranscriptFiles: 0, + archivedUnreferencedJsonlFiles: 0, + importedEntries: 0, + importedTranscriptEvents: 0, + issues: 0, + legacyEntries: 0, + sqliteEntries: 0, + targets: 0, + unreferencedJsonlFiles: 0, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }, + }; + mocks.runDoctorSessionSqlite.mockResolvedValueOnce(report); + mocks.createSessionSqliteGithubIssue.mockReturnValueOnce({ + ok: true, + url: "https://github.com/openclaw/openclaw/issues/123", + }); + const runtime = { + log: vi.fn(), + error: vi.fn(), + writeStdout: vi.fn(), + writeJson: vi.fn(), + exit: vi.fn((code: number) => { + throw new Error(`exit:${code}`); + }), + }; + + await expect( + doctorCommand(runtime, { + sessionSqlite: "recover", + sessionSqliteGithubIssue: true, + yes: true, + }), + ).rejects.toThrow("exit:0"); + + expect(mocks.createSessionSqliteGithubIssue).toHaveBeenCalledWith(supportIssue); + expect(runtime.log).toHaveBeenCalledWith( + "session-sqlite recover: created GitHub issue https://github.com/openclaw/openclaw/issues/123", + ); + expect(runtime.exit).toHaveBeenCalledWith(0); + }); + + it("keeps session sqlite recovery GitHub status inside JSON output", async () => { + const report = { + mode: "recover", + supportIssue: { + body: "sanitized body", + title: "Session SQLite migration recovery report (run-1)", + url: "https://github.com/openclaw/openclaw/issues/new?title=run-1", + }, + targets: [], + totals: { + archivedTranscriptFiles: 0, + archivedUnreferencedJsonlFiles: 0, + importedEntries: 0, + importedTranscriptEvents: 0, + issues: 0, + legacyEntries: 0, + sqliteEntries: 0, + targets: 0, + unreferencedJsonlFiles: 0, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }, + }; + mocks.runDoctorSessionSqlite.mockResolvedValueOnce(report); + const runtime = { + log: vi.fn(), + error: vi.fn(), + writeStdout: vi.fn(), + writeJson: vi.fn(), + exit: vi.fn((code: number) => { + throw new Error(`exit:${code}`); + }), + }; + + await expect( + doctorCommand(runtime, { + json: true, + sessionSqlite: "recover", + sessionSqliteGithubIssue: true, + }), + ).rejects.toThrow("exit:0"); + + expect(mocks.createSessionSqliteGithubIssue).not.toHaveBeenCalled(); + expect((report.supportIssue as { github?: unknown }).github).toEqual({ status: "skipped" }); + expect(runtime.log).not.toHaveBeenCalled(); + expect(runtime.writeJson).toHaveBeenCalledWith(report, 2); + }); }); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 666021ba4fdb..a987b4629719 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -2,9 +2,71 @@ import { defaultRuntime, type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; import { runPostUpgradeProbes } from "./doctor-post-upgrade.js"; import type { DoctorOptions } from "./doctor-prompter.js"; +import type { DoctorSessionSqliteReport } from "./doctor-session-sqlite.js"; /** Runs doctor or the post-upgrade probe submode using the provided runtime. */ export async function doctorCommand(runtime?: RuntimeEnv, options?: DoctorOptions): Promise<void> { + if (options?.sessionSqlite) { + const outputRuntime = runtime ?? defaultRuntime; + const { runDoctorSessionSqlite } = await import("./doctor-session-sqlite.js"); + const report = await runDoctorSessionSqlite({ + mode: options.sessionSqlite, + ...(options.sessionSqliteStore ? { store: options.sessionSqliteStore } : {}), + ...(options.sessionSqliteAgent ? { agent: options.sessionSqliteAgent } : {}), + ...(options.sessionSqliteAllAgents ? { allAgents: true } : {}), + }); + if (options.sessionSqlite === "recover" && options.sessionSqliteGithubIssue === true) { + await maybeCreateSessionSqliteGithubIssue(outputRuntime, report, options); + } + if (options.json) { + writeRuntimeJson(outputRuntime, report); + } else { + outputRuntime.log( + `session-sqlite ${report.mode}: ${report.totals.targets} target(s), ${report.totals.legacyEntries} legacy entries, ${report.totals.sqliteEntries} sqlite entries, ${report.totals.issues} issue(s)`, + ); + if (report.migrationRun) { + outputRuntime.log(`- migration-run=${report.migrationRun.runId}`); + outputRuntime.log(`- manifest=${report.migrationRun.manifestPath}`); + if (report.migrationRun.failureReportMarkdownPath) { + outputRuntime.log(`- failure-report=${report.migrationRun.failureReportMarkdownPath}`); + } + } + if (report.supportIssue) { + outputRuntime.log(`- support-issue-report=${report.supportIssue.bodyPath ?? "inline"}`); + outputRuntime.log(`- support-issue-url=${report.supportIssue.url}`); + } + for (const target of report.targets) { + outputRuntime.log( + `- ${target.agentId}: imported=${target.importedEntries}/${target.importedTranscriptEvents} events, validated=${target.validatedEntries}/${target.validatedTranscriptEvents} events, archived-unreferenced-jsonl=${target.archivedUnreferencedJsonlFiles.length}, unreferenced-jsonl=${target.unreferencedJsonlFiles.length}`, + ); + if (target.restore) { + outputRuntime.log( + ` restored=${target.restore.restoredFiles.length}, skipped=${target.restore.skippedFiles.length}, conflicts=${target.restore.conflicts.length}, manifests=${target.restore.manifestPaths.length}`, + ); + } + if (target.compact) { + outputRuntime.log( + ` compact reclaimed=${target.compact.reclaimedBytes} bytes, db=${target.compact.dbSizeBeforeBytes}->${target.compact.dbSizeAfterBytes} bytes, wal=${target.compact.walSizeBeforeBytes}->${target.compact.walSizeAfterBytes} bytes`, + ); + } + if (target.corruptRecovery) { + outputRuntime.log( + ` corrupt-db-recovery moved=${target.corruptRecovery.movedFiles.length}, skipped=${target.corruptRecovery.skippedFiles.length}`, + ); + } + for (const issue of target.issues.slice(0, 10)) { + outputRuntime.log( + ` [${issue.code}]${issue.sessionKey ? ` ${issue.sessionKey}:` : ""} ${issue.message}`, + ); + } + if (target.issues.length > 10) { + outputRuntime.log(` ...and ${target.issues.length - 10} more issue(s)`); + } + } + } + outputRuntime.exit(report.totals.issues > 0 ? 1 : 0); + return; + } if (options?.postUpgrade) { const outputRuntime = runtime ?? defaultRuntime; const report = await runPostUpgradeProbes({}); @@ -25,3 +87,51 @@ export async function doctorCommand(runtime?: RuntimeEnv, options?: DoctorOption const doctorHealth = await import("../flows/doctor-health.js"); await doctorHealth.doctorCommand(runtime, options); } + +async function maybeCreateSessionSqliteGithubIssue( + runtime: RuntimeEnv, + report: DoctorSessionSqliteReport, + options: DoctorOptions, +): Promise<void> { + const shouldLog = options.json !== true; + if (!report.supportIssue) { + if (shouldLog) { + runtime.log("session-sqlite recover: no support issue payload was generated"); + } + return; + } + let approved = options.yes === true; + if (!approved && options.nonInteractive !== true && options.json !== true) { + const { promptYesNo } = await import("../cli/prompt.js"); + approved = await promptYesNo( + "Create a GitHub issue in openclaw/openclaw with the sanitized recovery report?", + false, + ); + } + if (!approved) { + report.supportIssue.github = { status: "skipped" }; + if (shouldLog) { + runtime.log("session-sqlite recover: GitHub issue creation skipped"); + } + return; + } + const { createSessionSqliteGithubIssue } = + await import("./doctor-session-sqlite-github-issue.js"); + const created = createSessionSqliteGithubIssue(report.supportIssue); + if (created.ok) { + report.supportIssue.github = { status: "created", url: created.url }; + if (shouldLog) { + runtime.log(`session-sqlite recover: created GitHub issue ${created.url}`); + } + return; + } + report.supportIssue.github = { + fallbackUrl: created.fallbackUrl, + message: created.message, + status: "failed", + }; + if (shouldLog) { + runtime.log(`session-sqlite recover: GitHub issue creation unavailable: ${created.message}`); + runtime.log(`session-sqlite recover: prefilled issue URL ${created.fallbackUrl}`); + } +} diff --git a/src/commands/doctor.types.ts b/src/commands/doctor.types.ts index 79ca4cd4dd03..1ed76c03a8c4 100644 --- a/src/commands/doctor.types.ts +++ b/src/commands/doctor.types.ts @@ -9,6 +9,11 @@ export type DoctorOptions = { generateGatewayToken?: boolean; allowExec?: boolean; postUpgrade?: boolean; + sessionSqlite?: "dry-run" | "import" | "validate" | "inspect" | "compact" | "restore" | "recover"; + sessionSqliteStore?: string; + sessionSqliteAgent?: string; + sessionSqliteAllAgents?: boolean; + sessionSqliteGithubIssue?: boolean; json?: boolean; /** Internal capability granted only to direct operator-owned doctor invocations. */ crossStateDirImports?: boolean; diff --git a/src/commands/export-trajectory.test.ts b/src/commands/export-trajectory.test.ts index 69ecef3ff8ac..b7e845182594 100644 --- a/src/commands/export-trajectory.test.ts +++ b/src/commands/export-trajectory.test.ts @@ -4,6 +4,8 @@ import type { RuntimeEnv } from "../runtime.js"; import { exportTrajectoryCommand } from "./export-trajectory.js"; const mocks = vi.hoisted(() => ({ + exportTrajectoryForCommand: vi.fn(), + formatTrajectoryCommandExportSummary: vi.fn(), getRuntimeConfig: vi.fn(), loadSessionEntry: vi.fn(), resolveStorePath: vi.fn(), @@ -13,8 +15,17 @@ vi.mock("../config/config.js", () => ({ getRuntimeConfig: mocks.getRuntimeConfig, })); -vi.mock("../config/sessions/session-accessor.js", () => ({ - loadSessionEntry: mocks.loadSessionEntry, +vi.mock("../config/sessions/session-accessor.js", async (importOriginal) => { + const actual = await importOriginal<typeof import("../config/sessions/session-accessor.js")>(); + return { + ...actual, + loadSessionEntry: mocks.loadSessionEntry, + }; +}); + +vi.mock("../trajectory/command-export.js", () => ({ + exportTrajectoryForCommand: mocks.exportTrajectoryForCommand, + formatTrajectoryCommandExportSummary: mocks.formatTrajectoryCommandExportSummary, })); vi.mock("../config/sessions/paths.js", async (importOriginal) => { @@ -39,6 +50,16 @@ describe("exportTrajectoryCommand", () => { mocks.getRuntimeConfig.mockReturnValue({}); mocks.resolveStorePath.mockReturnValue("/tmp/openclaw/sessions.json"); mocks.loadSessionEntry.mockReturnValue(undefined); + mocks.exportTrajectoryForCommand.mockResolvedValue({ + outputDir: "/tmp/workspace/.openclaw/trajectory-exports/export", + displayPath: ".openclaw/trajectory-exports/export", + sessionId: "session-1", + eventCount: 2, + runtimeEventCount: 0, + transcriptEventCount: 2, + files: ["manifest.json", "events.jsonl", "session-branch.json"], + }); + mocks.formatTrajectoryCommandExportSummary.mockReturnValue("trajectory exported"); }); it("points missing session key users at the sessions command", async () => { @@ -186,4 +207,33 @@ describe("exportTrajectoryCommand", () => { ); expect(runtime.exit).toHaveBeenCalledWith(1); }); + + it("exports SQLite marker sessions without probing a transcript JSONL file", async () => { + const runtime = createRuntime(); + const sessionFile = "sqlite:main:session-1:/tmp/openclaw/sessions.json"; + mocks.loadSessionEntry.mockReturnValue({ + sessionId: "session-1", + sessionFile, + updatedAt: 1, + }); + + await exportTrajectoryCommand( + { + sessionKey: "agent:main:telegram:direct:123", + workspace: "/tmp/workspace", + }, + runtime, + ); + + expect(mocks.exportTrajectoryForCommand).toHaveBeenCalledWith({ + outputPath: undefined, + sessionFile, + sessionId: "session-1", + sessionKey: "agent:main:telegram:direct:123", + workspaceDir: "/tmp/workspace", + }); + expect(runtime.error).not.toHaveBeenCalled(); + expect(runtime.exit).not.toHaveBeenCalled(); + expect(runtime.log).toHaveBeenCalledWith("trajectory exported"); + }); }); diff --git a/src/commands/export-trajectory.ts b/src/commands/export-trajectory.ts index e8323dd705ac..b22e2a5b2cba 100644 --- a/src/commands/export-trajectory.ts +++ b/src/commands/export-trajectory.ts @@ -2,12 +2,12 @@ import path from "node:path"; import { formatCliCommand } from "../cli/command-format.js"; import { getRuntimeConfig } from "../config/config.js"; +import { resolveStorePath } from "../config/sessions/paths.js"; import { - resolveSessionFilePath, - resolveSessionFilePathOptions, - resolveStorePath, -} from "../config/sessions/paths.js"; -import { loadSessionEntry } from "../config/sessions/session-accessor.js"; + loadSessionEntry, + resolveSessionTranscriptReadTarget, +} from "../config/sessions/session-accessor.js"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { formatErrorMessage } from "../infra/errors.js"; import { pathExists } from "../infra/fs-safe.js"; import { resolveAgentIdFromSessionKey } from "../routing/session-key.js"; @@ -134,17 +134,19 @@ export async function exportTrajectoryCommand( let sessionFile: string; try { - sessionFile = resolveSessionFilePath( - entry.sessionId, - entry, - resolveSessionFilePathOptions({ agentId: targetAgentId, storePath }), - ); + sessionFile = resolveSessionTranscriptReadTarget({ + agentId: targetAgentId, + sessionEntry: entry, + sessionId: entry.sessionId, + sessionKey, + storePath, + }).sessionFile; } catch (error) { runtime.error(`Failed to resolve session file: ${formatErrorMessage(error)}`); runtime.exit(1); return; } - if (!(await pathExists(sessionFile))) { + if (!parseSqliteSessionFileMarker(sessionFile) && !(await pathExists(sessionFile))) { runtime.error( `Session file not found for ${sessionKey}. Run ${formatCliCommand("openclaw doctor")} to inspect session storage.`, ); diff --git a/src/commands/sandbox-explain.test.ts b/src/commands/sandbox-explain.test.ts index d108f434cba0..52b300e369e1 100644 --- a/src/commands/sandbox-explain.test.ts +++ b/src/commands/sandbox-explain.test.ts @@ -3,6 +3,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import { sandboxExplainCommand } from "./sandbox-explain.js"; const SANDBOX_EXPLAIN_TEST_TIMEOUT_MS = process.platform === "win32" ? 45_000 : 30_000; @@ -246,18 +248,13 @@ describe("sandbox explain command", () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-sandbox-explain-")); const storePath = path.join(tempDir, "sessions.json"); const sessionKey = "agent:builder:subagent:child"; - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "child-session", - updatedAt: Date.now(), - spawnedBy: "agent:builder:main", - spawnedWorkspaceDir: "/tmp/openclaw-child-workspace", - spawnedCwd: "/tmp/openclaw-child-workspace/task", - }, - }), - ); + await replaceSessionEntry({ storePath, sessionKey }, { + sessionId: "child-session", + updatedAt: Date.now(), + spawnedBy: "agent:builder:main", + spawnedWorkspaceDir: "/tmp/openclaw-child-workspace", + spawnedCwd: "/tmp/openclaw-child-workspace/task", + } as SessionEntry); mockCfg = { agents: { defaults: { sandbox: { mode: "off" } }, @@ -289,17 +286,12 @@ describe("sandbox explain command", () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-sandbox-explain-")); const storePath = path.join(tempDir, "sessions.json"); const sessionKey = "agent:builder:subagent:child"; - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "child-session", - updatedAt: Date.now(), - spawnedBy: "agent:builder:main", - spawnedWorkspaceDir: "/tmp/openclaw-child-workspace", - }, - }), - ); + await replaceSessionEntry({ storePath, sessionKey }, { + sessionId: "child-session", + updatedAt: Date.now(), + spawnedBy: "agent:builder:main", + spawnedWorkspaceDir: "/tmp/openclaw-child-workspace", + } as SessionEntry); mockCfg = { agents: { defaults: { diff --git a/src/commands/sessions-cleanup.test.ts b/src/commands/sessions-cleanup.test.ts index e89bdf4fd464..47a9cb0672e6 100644 --- a/src/commands/sessions-cleanup.test.ts +++ b/src/commands/sessions-cleanup.test.ts @@ -122,7 +122,6 @@ describe("sessionsCleanupCommand", () => { mocks.resolveSessionCleanupAction.mockImplementation( (params: { key: string; - repairedKeys?: Set<string>; missingKeys: Set<string>; staleKeys: Set<string>; cappedKeys: Set<string>; @@ -145,9 +144,6 @@ describe("sessionsCleanupCommand", () => { if (params.budgetEvictedKeys.has(params.key)) { return "evict-budget"; } - if (params.repairedKeys?.has(params.key)) { - return "repair-session-file"; - } return "keep"; }, ); @@ -459,127 +455,6 @@ describe("sessionsCleanupCommand", () => { }); }); - it("reports repaired sessionFile metadata in dry-run JSON and action table", async () => { - mocks.enforceSessionDiskBudget.mockResolvedValue(null); - mocks.runSessionsCleanup.mockResolvedValue({ - mode: "enforce", - previewResults: [ - { - summary: { - agentId: "main", - storePath: "/resolved/sessions.json", - mode: "enforce", - dryRun: true, - beforeCount: 1, - afterCount: 1, - repaired: 1, - missing: 0, - dmScopeRetired: 0, - modelRunPruned: 0, - pruned: 0, - capped: 0, - diskBudget: null, - wouldMutate: true, - }, - beforeStore: { - repaired: { sessionId: "session-id", updatedAt: 1, model: "test:opus" }, - }, - repairedKeys: new Set(["repaired"]), - missingKeys: new Set<string>(), - staleKeys: new Set<string>(), - cappedKeys: new Set<string>(), - budgetEvictedKeys: new Set<string>(), - dmScopeRetiredKeys: new Set<string>(), - modelRunPrunedKeys: new Set<string>(), - }, - ], - appliedSummaries: [], - }); - - const jsonRun = makeRuntime(); - await sessionsCleanupCommand( - { - json: true, - dryRun: true, - enforce: true, - fixMissing: true, - }, - jsonRun.runtime, - ); - - expect(JSON.parse(jsonRun.logs[0] ?? "{}")).toMatchObject({ - repaired: 1, - missing: 0, - wouldMutate: true, - }); - - const tableRun = makeRuntime(); - await sessionsCleanupCommand( - { - dryRun: true, - enforce: true, - fixMissing: true, - }, - tableRun.runtime, - ); - - expectLogsToInclude(tableRun.logs, "Would repair sessionFile metadata: 1"); - expectLogsToInclude(tableRun.logs, "repair-session-file"); - }); - - it("lets destructive cleanup actions override repair action rows", async () => { - mocks.enforceSessionDiskBudget.mockResolvedValue(null); - mocks.runSessionsCleanup.mockResolvedValue({ - mode: "enforce", - previewResults: [ - { - summary: { - agentId: "main", - storePath: "/resolved/sessions.json", - mode: "enforce", - dryRun: true, - beforeCount: 1, - afterCount: 0, - repaired: 0, - missing: 0, - dmScopeRetired: 0, - modelRunPruned: 0, - pruned: 1, - capped: 0, - diskBudget: null, - wouldMutate: true, - }, - beforeStore: { - repairedThenStale: { sessionId: "session-id", updatedAt: 1, model: "test:opus" }, - }, - repairedKeys: new Set(["repairedThenStale"]), - missingKeys: new Set<string>(), - staleKeys: new Set(["repairedThenStale"]), - cappedKeys: new Set<string>(), - budgetEvictedKeys: new Set<string>(), - dmScopeRetiredKeys: new Set<string>(), - modelRunPrunedKeys: new Set<string>(), - }, - ], - appliedSummaries: [], - }); - - const { runtime, logs } = makeRuntime(); - await sessionsCleanupCommand( - { - dryRun: true, - enforce: true, - fixMissing: true, - }, - runtime, - ); - - const row = logs.find((line) => line.includes("repairedThenStale")) ?? ""; - expect(row).toContain("prune-stale"); - expect(row).not.toContain("repair-session-file"); - expectLogsToInclude(logs, "Would repair sessionFile metadata: 0"); - }); - it("renders a dry-run action table with keep/prune actions", async () => { mocks.enforceSessionDiskBudget.mockResolvedValue(null); mocks.runSessionsCleanup.mockResolvedValue({ diff --git a/src/commands/sessions-cleanup.ts b/src/commands/sessions-cleanup.ts index e71d615960db..e493e27ed8bf 100644 --- a/src/commands/sessions-cleanup.ts +++ b/src/commands/sessions-cleanup.ts @@ -32,7 +32,7 @@ import { toSessionDisplayRows, } from "./sessions-table.js"; -const ACTION_PAD = "repair-session-file".length; +const ACTION_PAD = "prune-model-run".length; type SessionCleanupActionRow = ReturnType<typeof toSessionDisplayRows>[number] & { action: ReturnType<typeof resolveSessionCleanupAction>; @@ -56,9 +56,6 @@ function formatCleanupActionCell( if (action === "keep") { return theme.muted(label); } - if (action === "repair-session-file") { - return theme.accentBright(label); - } if (action === "prune-missing") { return theme.error(label); } @@ -79,7 +76,6 @@ function formatCleanupActionCell( function buildActionRows(params: { beforeStore: Parameters<typeof toSessionDisplayRows>[0]; - repairedKeys?: Set<string>; missingKeys: Set<string>; modelRunPrunedKeys: Set<string>; staleKeys: Set<string>; @@ -94,7 +90,6 @@ function buildActionRows(params: { label: params.beforeStore[row.key]?.label, action: resolveSessionCleanupAction({ key: row.key, - repairedKeys: params.repairedKeys ?? new Set<string>(), missingKeys: params.missingKeys, modelRunPrunedKeys: params.modelRunPrunedKeys, staleKeys: params.staleKeys, @@ -116,7 +111,7 @@ function buildLabelSummaries(actionRows: SessionCleanupActionRow[]): SessionClea summary = { label, kept: 0, pruned: 0 }; summaryByLabel.set(label, summary); } - if (actionRow.action === "keep" || actionRow.action === "repair-session-file") { + if (actionRow.action === "keep") { summary.kept += 1; } else { summary.pruned += 1; @@ -162,7 +157,6 @@ function renderStoreDryRunPlan(params: { params.runtime.log( `Entries: ${params.summary.beforeCount} -> ${params.summary.afterCount} (remove ${params.summary.beforeCount - params.summary.afterCount})`, ); - params.runtime.log(`Would repair sessionFile metadata: ${params.summary.repaired}`); params.runtime.log(`Would prune missing transcripts: ${params.summary.missing}`); params.runtime.log(`Would retire stale direct DM sessions: ${params.summary.dmScopeRetired}`); params.runtime.log(`Would prune stale model-run probes: ${params.summary.modelRunPruned}`); diff --git a/src/commands/sessions-tail.test.ts b/src/commands/sessions-tail.test.ts index 480dce2854cb..579ab800088a 100644 --- a/src/commands/sessions-tail.test.ts +++ b/src/commands/sessions-tail.test.ts @@ -3,12 +3,17 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import type { RuntimeEnv } from "../runtime.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { - resolveTrajectoryFilePath, resolveTrajectoryPointerFilePath, TRAJECTORY_RUNTIME_FILE_MAX_BYTES, } from "../trajectory/paths.js"; +import { appendSqliteTrajectoryRuntimeEvents } from "../trajectory/runtime-store.sqlite.js"; import type { TrajectoryEvent } from "../trajectory/types.js"; import { sessionsTailCommand, setSessionsTailFollowIntervalMsForTests } from "./sessions-tail.js"; @@ -46,6 +51,7 @@ function makeEvent( } function writeJsonl(filePath: string, events: TrajectoryEvent[]): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); fs.writeFileSync(filePath, `${events.map((event) => JSON.stringify(event)).join("\n")}\n`); } @@ -81,15 +87,12 @@ describe("sessionsTailCommand", () => { let storePath: string; let trajectoryPath: string; let previousStateDir: string | undefined; - let previousTrajectoryDir: string | undefined; beforeEach(() => { setSessionsTailFollowIntervalMsForTests(10); previousStateDir = process.env.OPENCLAW_STATE_DIR; - previousTrajectoryDir = process.env.OPENCLAW_TRAJECTORY_DIR; tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sessions-tail-")); process.env.OPENCLAW_STATE_DIR = path.join(tmpDir, "state"); - delete process.env.OPENCLAW_TRAJECTORY_DIR; mocks.getRuntimeConfig.mockReturnValue({ agents: { list: [{ id: "main" }, { id: "ops" }], @@ -97,17 +100,6 @@ describe("sessionsTailCommand", () => { }); storePath = path.join(tmpDir, "sessions.json"); trajectoryPath = path.join(tmpDir, "session-one.trajectory.jsonl"); - fs.writeFileSync( - storePath, - `${JSON.stringify({ - [sessionKey]: { - sessionId: "session-one", - sessionFile: "session-one.jsonl", - updatedAt: 2, - status: "running", - }, - })}\n`, - ); }); afterEach(() => { @@ -117,17 +109,45 @@ describe("sessionsTailCommand", () => { } else { process.env.OPENCLAW_STATE_DIR = previousStateDir; } - if (previousTrajectoryDir === undefined) { - delete process.env.OPENCLAW_TRAJECTORY_DIR; - } else { - process.env.OPENCLAW_TRAJECTORY_DIR = previousTrajectoryDir; - } + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); fs.rmSync(tmpDir, { recursive: true, force: true }); }); + async function writeSessionEntry( + key = sessionKey, + entry: Partial<SessionEntry> = {}, + ): Promise<void> { + await replaceSessionEntry( + { sessionKey: key, storePath }, + { + sessionId: "session-one", + sessionFile: "session-one.jsonl", + updatedAt: 2, + status: "running", + ...entry, + }, + ); + } + + async function appendEvents( + events: TrajectoryEvent[], + params: { key?: string; sessionId?: string } = {}, + ): Promise<void> { + const targetPath = + params.sessionId && params.sessionId !== "session-one" + ? path.join(tmpDir, `${params.sessionId}.trajectory.jsonl`) + : trajectoryPath; + writeJsonl( + targetPath, + events.map((event) => ({ ...event, sessionKey: params.key ?? event.sessionKey })), + ); + } + it("renders compact redacted progress lines", async () => { const runtime = makeRuntime(); - writeJsonl(trajectoryPath, [ + await writeSessionEntry(); + await appendEvents([ makeEvent({ type: "tool.call", ts: "2026-05-18T12:04:18.000Z", @@ -164,7 +184,8 @@ describe("sessionsTailCommand", () => { it("honors the tail count before rendering existing trajectory events", async () => { const runtime = makeRuntime(); - writeJsonl(trajectoryPath, [ + await writeSessionEntry(); + await appendEvents([ makeEvent({ type: "session.started", ts: "2026-05-18T12:04:17.000Z" }), makeEvent({ type: "tool.call", @@ -203,11 +224,13 @@ describe("sessionsTailCommand", () => { it("uses a session trajectory pointer for relocated runtime files", async () => { const runtime = makeRuntime(); + await writeSessionEntry(); + const sessionFile = path.join(tmpDir, "session-one.jsonl"); const relocatedDir = path.join(tmpDir, "relocated-trajectories"); const relocatedTrajectoryPath = path.join(relocatedDir, "session-one.jsonl"); fs.mkdirSync(relocatedDir, { recursive: true }); fs.writeFileSync( - resolveTrajectoryPointerFilePath(path.join(tmpDir, "session-one.jsonl")), + resolveTrajectoryPointerFilePath(sessionFile), `${JSON.stringify({ traceSchema: "openclaw-trajectory-pointer", schemaVersion: 1, @@ -231,11 +254,73 @@ describe("sessionsTailCommand", () => { expect(output).not.toContain("No sessions found"); }); + it("tails SQLite marker trajectory rows from the database", async () => { + const runtime = makeRuntime(); + await writeSessionEntry(sessionKey, { + sessionFile: formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: "session-one", + storePath, + }), + }); + appendSqliteTrajectoryRuntimeEvents({ sessionId: "session-one", storePath }, [ + makeEvent({ + type: "tool.result", + ts: "2026-05-18T12:04:21.000Z", + data: { name: "sqlite", success: true }, + }), + ]); + + await sessionsTailCommand({ store: storePath, sessionKey }, runtime); + + const output = runtimeOutput(runtime); + expect(output).toContain("tool.result"); + expect(output).toContain("sqlite ok"); + expect(output).not.toContain("No sessions found"); + expect(fs.existsSync(path.join(tmpDir, "trajectory", "session-one.jsonl"))).toBe(false); + }); + + it("ignores stale trajectory pointers for another session id", async () => { + const runtime = makeRuntime(); + await writeSessionEntry(); + const sessionFile = path.join(tmpDir, "session-one.jsonl"); + const staleRuntimePath = path.join(tmpDir, "relocated-trajectories", "old-session.jsonl"); + fs.writeFileSync( + resolveTrajectoryPointerFilePath(sessionFile), + `${JSON.stringify({ + traceSchema: "openclaw-trajectory-pointer", + schemaVersion: 1, + sessionId: "old-session", + runtimeFile: staleRuntimePath, + })}\n`, + ); + writeJsonl(staleRuntimePath, [ + makeEvent({ + sessionId: "old-session", + type: "tool.result", + ts: "2026-05-18T12:04:21.000Z", + data: { name: "stale", success: true }, + }), + ]); + await appendEvents([ + makeEvent({ + type: "tool.result", + ts: "2026-05-18T12:04:22.000Z", + data: { name: "current", success: true }, + }), + ]); + + await sessionsTailCommand({ store: storePath, sessionKey }, runtime); + + const output = runtimeOutput(runtime); + expect(output).toContain("current ok"); + expect(output).not.toContain("stale ok"); + }); + it("preserves events appended while follow mode starts", async () => { const runtime = makeRuntime(); - writeJsonl(trajectoryPath, [ - makeEvent({ type: "session.started", ts: "2026-05-18T12:04:17.000Z" }), - ]); + await writeSessionEntry(); + await appendEvents([makeEvent({ type: "session.started", ts: "2026-05-18T12:04:17.000Z" })]); const appendedEvent = makeEvent({ type: "tool.result", ts: "2026-05-18T12:04:21.000Z", @@ -266,9 +351,10 @@ describe("sessionsTailCommand", () => { expect(output).toContain("bash ok"); }); - it("continues following when a bounded trajectory window is rewritten", async () => { + it("continues following when later trajectory events are appended", async () => { const runtime = makeRuntime(); - writeJsonl(trajectoryPath, [ + await writeSessionEntry(); + await appendEvents([ makeEvent({ sourceSeq: 1, type: "session.started", @@ -285,9 +371,7 @@ describe("sessionsTailCommand", () => { vi.mocked(runtime.log).mockImplementation((message) => { if (!rewritten && String(message).includes("session.started")) { rewritten = true; - const nextPath = path.join(tmpDir, "session-one.next.trajectory.jsonl"); - writeJsonl(nextPath, [rewrittenEvent]); - fs.renameSync(nextPath, trajectoryPath); + appendJsonl(trajectoryPath, rewrittenEvent); } }); @@ -307,21 +391,62 @@ describe("sessionsTailCommand", () => { expect(output).toContain("python ok"); }); + it("continues following when SQLite trajectory rows are appended", async () => { + const runtime = makeRuntime(); + await writeSessionEntry(sessionKey, { + sessionFile: formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: "session-one", + storePath, + }), + }); + appendSqliteTrajectoryRuntimeEvents({ sessionId: "session-one", storePath }, [ + makeEvent({ + sourceSeq: 1, + type: "session.started", + ts: "2026-05-18T12:04:17.000Z", + }), + ]); + const appendedEvent = makeEvent({ + sourceSeq: 2, + type: "tool.result", + ts: "2026-05-18T12:04:21.000Z", + data: { name: "sqlite", success: true }, + }); + let appended = false; + vi.mocked(runtime.log).mockImplementation((message) => { + if (!appended && String(message).includes("session.started")) { + appended = true; + appendSqliteTrajectoryRuntimeEvents({ sessionId: "session-one", storePath }, [ + appendedEvent, + ]); + } + }); + + const run = sessionsTailCommand( + { store: storePath, sessionKey, tail: "1", follow: true }, + runtime, + ); + try { + await waitForRuntimeOutput(runtime, "sqlite ok"); + } finally { + process.emit("SIGTERM", "SIGTERM"); + await run; + } + + const output = runtimeOutput(runtime); + expect(output).toContain("tool.result"); + expect(output).toContain("sqlite ok"); + }); + it("resolves the target store from a fully qualified non-default agent session key", async () => { const runtime = makeRuntime(); const opsSessionKey = "agent:ops:telegram:direct:owner"; const opsSessionsDir = path.join(process.env.OPENCLAW_STATE_DIR!, "agents", "ops", "sessions"); - fs.mkdirSync(opsSessionsDir, { recursive: true }); - fs.writeFileSync( - path.join(opsSessionsDir, "sessions.json"), - `${JSON.stringify({ - [opsSessionKey]: { - sessionId: "ops-session", - sessionFile: "ops-session.jsonl", - updatedAt: 3, - status: "done", - }, - })}\n`, + const opsStorePath = path.join(opsSessionsDir, "sessions.json"); + await replaceSessionEntry( + { sessionKey: opsSessionKey, storePath: opsStorePath }, + { sessionId: "ops-session", sessionFile: "ops-session.jsonl", updatedAt: 3, status: "done" }, ); writeJsonl(path.join(opsSessionsDir, "ops-session.trajectory.jsonl"), [ makeEvent({ @@ -342,160 +467,9 @@ describe("sessionsTailCommand", () => { expect(output).not.toContain("No sessions found"); }); - it("skips placeholder store entries without transcript session ids", async () => { - const runtime = makeRuntime(); - fs.writeFileSync( - storePath, - `${JSON.stringify({ - "agent:main:cron:placeholder": { - label: "placeholder cron", - updatedAt: 3, - }, - [sessionKey]: { - sessionId: "session-one", - sessionFile: "session-one.jsonl", - updatedAt: 2, - status: "done", - }, - })}\n`, - ); - writeJsonl(trajectoryPath, [ - makeEvent({ - type: "tool.result", - ts: "2026-05-18T12:04:21.000Z", - data: { name: "bash", success: true }, - }), - ]); - - await sessionsTailCommand({ store: storePath, sessionKey }, runtime); - - const output = runtimeOutput(runtime); - expect(output).toContain("tool.result"); - expect(output).toContain("bash ok"); - expect(output).not.toContain("No sessions found"); - }); - - it("tails entries pinned by sessionFile when sessionId was normalized away", async () => { - const runtime = makeRuntime(); - const legacySessionFile = path.join(tmpDir, "legacy-session.jsonl"); - const legacyTrajectoryPath = path.join(tmpDir, "legacy-session.trajectory.jsonl"); - fs.writeFileSync(legacySessionFile, ""); - fs.writeFileSync( - storePath, - `${JSON.stringify({ - [sessionKey]: { - sessionFile: legacySessionFile, - updatedAt: 2, - status: "running", - }, - })}\n`, - ); - writeJsonl(legacyTrajectoryPath, [ - makeEvent({ - type: "tool.result", - ts: "2026-05-18T12:04:21.000Z", - data: { name: "legacy", success: true }, - }), - ]); - - await sessionsTailCommand({ store: storePath, sessionKey }, runtime); - - const output = runtimeOutput(runtime); - expect(output).toContain("tool.result"); - expect(output).toContain("legacy ok"); - expect(output).not.toContain("No sessions found"); - }); - - it("uses trajectory pointers for sessionFile-only entries with original runtime ids", async () => { - const runtime = makeRuntime(); - const legacySessionFile = path.join(tmpDir, "legacy-session.jsonl"); - const pointerPath = resolveTrajectoryPointerFilePath(legacySessionFile); - const runtimeTrajectoryPath = path.join(tmpDir, "runtime-original.jsonl"); - fs.writeFileSync(legacySessionFile, ""); - fs.writeFileSync( - storePath, - `${JSON.stringify({ - [sessionKey]: { - sessionFile: legacySessionFile, - updatedAt: 2, - status: "running", - }, - })}\n`, - ); - fs.writeFileSync( - pointerPath, - `${JSON.stringify({ - sessionId: "runtime-original", - runtimeFile: runtimeTrajectoryPath, - })}\n`, - ); - writeJsonl(runtimeTrajectoryPath, [ - makeEvent({ - sessionId: "runtime-original", - type: "tool.result", - ts: "2026-05-18T12:04:21.000Z", - data: { name: "pointer", success: true }, - }), - ]); - - await sessionsTailCommand({ store: storePath, sessionKey }, runtime); - - const output = runtimeOutput(runtime); - expect(output).toContain("tool.result"); - expect(output).toContain("pointer ok"); - expect(output).not.toContain("No sessions found"); - }); - - it("keeps trajectory directory fallback for pointer session ids", async () => { - const runtime = makeRuntime(); - const legacySessionFile = path.join(tmpDir, "legacy-session.jsonl"); - const pointerPath = resolveTrajectoryPointerFilePath(legacySessionFile); - const trajectoryDir = path.join(tmpDir, "trajectories"); - process.env.OPENCLAW_TRAJECTORY_DIR = trajectoryDir; - fs.mkdirSync(trajectoryDir, { recursive: true }); - fs.writeFileSync(legacySessionFile, ""); - fs.writeFileSync( - storePath, - `${JSON.stringify({ - [sessionKey]: { - sessionFile: legacySessionFile, - updatedAt: 2, - status: "running", - }, - })}\n`, - ); - fs.writeFileSync( - pointerPath, - `${JSON.stringify({ - sessionId: "runtime-original", - runtimeFile: path.join(tmpDir, "missing-pointer-runtime.jsonl"), - })}\n`, - ); - writeJsonl( - resolveTrajectoryFilePath({ - sessionFile: legacySessionFile, - sessionId: "runtime-original", - }), - [ - makeEvent({ - sessionId: "runtime-original", - type: "tool.result", - ts: "2026-05-18T12:04:21.000Z", - data: { name: "trajectory-dir", success: true }, - }), - ], - ); - - await sessionsTailCommand({ store: storePath, sessionKey }, runtime); - - const output = runtimeOutput(runtime); - expect(output).toContain("tool.result"); - expect(output).toContain("trajectory-dir ok"); - expect(output).not.toContain("No sessions found"); - }); - it("rejects oversized trajectory snapshots", async () => { const runtime = makeRuntime(); + await writeSessionEntry(); fs.writeFileSync(trajectoryPath, ""); fs.truncateSync(trajectoryPath, TRAJECTORY_RUNTIME_FILE_MAX_BYTES + 1); @@ -506,6 +480,7 @@ describe("sessionsTailCommand", () => { it("rejects oversized follow-mode trajectory deltas", async () => { const runtime = makeRuntime(); + await writeSessionEntry(); writeJsonl(trajectoryPath, [ makeEvent({ type: "session.started", ts: "2026-05-18T12:04:17.000Z" }), ]); diff --git a/src/commands/sessions-tail.ts b/src/commands/sessions-tail.ts index 9336c3874e52..c54b3e0df2b5 100644 --- a/src/commands/sessions-tail.ts +++ b/src/commands/sessions-tail.ts @@ -11,6 +11,7 @@ import { readAcpSessionMeta } from "../acp/runtime/session-meta.js"; import { getRuntimeConfig } from "../config/config.js"; import { resolveSessionFilePath } from "../config/sessions/paths.js"; import { listSessionEntries } from "../config/sessions/session-accessor.js"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import type { SessionEntry } from "../config/sessions/types.js"; import { resolveStoredSessionKeyForAgentStore } from "../gateway/session-store-key.js"; import { formatErrorMessage } from "../infra/errors.js"; @@ -20,13 +21,10 @@ import { resolveAgentIdFromSessionKey } from "../routing/session-key.js"; import type { RuntimeEnv } from "../runtime.js"; import { resolveTrajectoryFilePath, - resolveTrajectoryPointerFilePath, TRAJECTORY_RUNTIME_FILE_MAX_BYTES, } from "../trajectory/paths.js"; -import { - isRegularNonSymlinkFile, - resolveTrajectoryRuntimeFile, -} from "../trajectory/runtime-file.js"; +import { resolveTrajectoryRuntimeFile } from "../trajectory/runtime-file.js"; +import { loadSqliteTrajectoryRuntimeEventRowsSync } from "../trajectory/runtime-store.sqlite.js"; import type { TrajectoryEvent } from "../trajectory/types.js"; import { resolveSessionStoreTargetsOrExit } from "./session-store-targets.js"; import { shortenText } from "./text-format.js"; @@ -45,20 +43,43 @@ type TailSelection = { key: string; entry: SessionEntry; storePath: string; - trajectoryPath: string; + source: TailTrajectorySource; }; -type FollowState = { +type TailTrajectorySource = + | { + kind: "file"; + path: string; + } + | { + agentId: string; + kind: "sqlite"; + sessionId: string; + storePath: string; + }; + +type FileFollowState = { cursor: TrajectoryCursor | null; fileState: FollowFileState | null; + kind: "file"; offset: number; pending: string; - selection: TailSelection; + selection: TailSelection & { source: Extract<TailTrajectorySource, { kind: "file" }> }; }; +type SqliteFollowState = { + cursor: TrajectoryCursor | null; + kind: "sqlite"; + lastStorageSeq: number; + selection: TailSelection & { source: Extract<TailTrajectorySource, { kind: "sqlite" }> }; +}; + +type FollowState = FileFollowState | SqliteFollowState; + type TrajectorySnapshot = { events: TrajectoryEvent[]; fileState: FollowFileState | null; + maxStorageSeq?: number; offset: number; }; @@ -301,6 +322,28 @@ function readTrajectorySnapshot(filePath: string): TrajectorySnapshot { } } +function readSqliteTrajectorySnapshot( + source: Extract<TailTrajectorySource, { kind: "sqlite" }>, +): TrajectorySnapshot { + const rows = loadSqliteTrajectoryRuntimeEventRowsSync({ + agentId: source.agentId, + sessionId: source.sessionId, + storePath: source.storePath, + }); + return { + events: rows.map((row) => row.event), + fileState: null, + maxStorageSeq: rows.at(-1)?.seq ?? -1, + offset: 0, + }; +} + +function readTailSnapshot(selection: TailSelection): TrajectorySnapshot { + return selection.source.kind === "sqlite" + ? readSqliteTrajectorySnapshot(selection.source) + : readTrajectorySnapshot(selection.source.path); +} + function renderEvents(events: TrajectoryEvent[], runtime: RuntimeEnv): TrajectoryCursor | null { let cursor: TrajectoryCursor | null = null; for (const event of events) { @@ -350,76 +393,20 @@ function compareSelectionsByUpdatedAt(a: TailSelection, b: TailSelection): numbe return (b.entry.updatedAt ?? 0) - (a.entry.updatedAt ?? 0); } -function deriveSessionFileFallbackId(entry: SessionEntry): string | undefined { - const sessionId = entry.sessionId?.trim(); - if (sessionId) { - return sessionId; - } - const sessionFile = entry.sessionFile?.trim(); - if (!sessionFile) { - return undefined; - } - return "session"; -} - -async function readTrajectoryPointerSessionId(sessionFile: string): Promise<string | undefined> { - const pointerPath = resolveTrajectoryPointerFilePath(sessionFile); - if (!(await isRegularNonSymlinkFile(pointerPath))) { - return undefined; - } - try { - const { buffer } = readRegularFileSync({ - filePath: pointerPath, - maxBytes: 64 * 1024, - }); - const parsed = JSON.parse(buffer.toString("utf8")) as unknown; - if (!isRecord(parsed) || typeof parsed.sessionId !== "string") { - return undefined; - } - const sessionId = parsed.sessionId.trim(); - return sessionId || undefined; - } catch { - return undefined; - } -} - async function resolveTailTrajectoryPath(params: { sessionFile: string; - sessionId?: string; + sessionId: string; }): Promise<string> { - if (params.sessionId) { - return ( - (await resolveTrajectoryRuntimeFile({ - sessionFile: params.sessionFile, - sessionId: params.sessionId, - })) ?? - resolveTrajectoryFilePath({ - sessionFile: params.sessionFile, - sessionId: params.sessionId, - }) - ); - } - - const pointerSessionId = await readTrajectoryPointerSessionId(params.sessionFile); - if (pointerSessionId) { - const pointerRuntimePath = await resolveTrajectoryRuntimeFile({ + return ( + (await resolveTrajectoryRuntimeFile({ sessionFile: params.sessionFile, - sessionId: pointerSessionId, - }); - if (pointerRuntimePath) { - return pointerRuntimePath; - } - return resolveTrajectoryFilePath({ + sessionId: params.sessionId, + })) ?? + resolveTrajectoryFilePath({ sessionFile: params.sessionFile, - sessionId: pointerSessionId, - }); - } - - return resolveTrajectoryFilePath({ - env: {}, - sessionFile: params.sessionFile, - sessionId: "session", - }); + sessionId: params.sessionId, + }) + ); } async function buildTailSelection(params: { @@ -429,27 +416,48 @@ async function buildTailSelection(params: { storePath: string; }): Promise<TailSelection | null> { const sessionId = params.entry.sessionId?.trim(); - const fallbackSessionId = deriveSessionFileFallbackId(params.entry); - if (!fallbackSessionId) { + if (!sessionId) { return null; } const sessionsDir = path.dirname(params.storePath); let sessionFile: string; try { - sessionFile = resolveSessionFilePath(fallbackSessionId, params.entry, { + const entrySessionFile = params.entry.sessionFile?.trim(); + const marker = entrySessionFile ? parseSqliteSessionFileMarker(entrySessionFile) : null; + if (marker && marker.sessionId === sessionId) { + return { + agentId: params.agentId, + entry: params.entry, + key: params.key, + source: { + agentId: marker.agentId, + kind: "sqlite", + sessionId: marker.sessionId, + storePath: marker.storePath, + }, + storePath: params.storePath, + }; + } + sessionFile = resolveSessionFilePath(sessionId, params.entry, { agentId: params.agentId, sessionsDir, }); } catch { return null; } - const trajectoryPath = await resolveTailTrajectoryPath({ sessionFile, sessionId }); + const trajectoryPath = await resolveTailTrajectoryPath({ + sessionFile, + sessionId, + }); return { agentId: params.agentId, entry: params.entry, key: params.key, + source: { + kind: "file", + path: trajectoryPath, + }, storePath: params.storePath, - trajectoryPath, }; } @@ -481,8 +489,8 @@ function statFileSize(filePath: string): number { } } -function readNewFollowEvents(state: FollowState): TrajectoryEvent[] { - const fileState = readFollowFileState(state.selection.trajectoryPath); +function readNewFileFollowEvents(state: FileFollowState): TrajectoryEvent[] { + const fileState = readFollowFileState(state.selection.source.path); if (!fileState) { state.fileState = null; state.offset = 0; @@ -498,7 +506,7 @@ function readNewFollowEvents(state: FollowState): TrajectoryEvent[] { if (replaced || truncated || possiblyRewrittenSameSize) { // Log rotation, truncation, and same-size rewrites all require a full // rescan; cursor filtering prevents duplicate event output. - const snapshot = readTrajectorySnapshot(state.selection.trajectoryPath); + const snapshot = readTrajectorySnapshot(state.selection.source.path); state.fileState = snapshot.fileState; state.offset = snapshot.offset; state.pending = ""; @@ -510,7 +518,7 @@ function readNewFollowEvents(state: FollowState): TrajectoryEvent[] { return []; } - const fd = fs.openSync(state.selection.trajectoryPath, "r"); + const fd = fs.openSync(state.selection.source.path, "r"); try { const deltaBytes = fileState.size - state.offset; if (deltaBytes > TRAJECTORY_RUNTIME_FILE_MAX_BYTES) { @@ -533,6 +541,26 @@ function readNewFollowEvents(state: FollowState): TrajectoryEvent[] { } } +function readNewSqliteFollowEvents(state: SqliteFollowState): TrajectoryEvent[] { + const rows = loadSqliteTrajectoryRuntimeEventRowsSync({ + agentId: state.selection.source.agentId, + afterSeq: state.lastStorageSeq, + sessionId: state.selection.source.sessionId, + storePath: state.selection.source.storePath, + }); + if (rows.length === 0) { + return []; + } + state.lastStorageSeq = rows.at(-1)?.seq ?? state.lastStorageSeq; + return rows.map((row) => row.event); +} + +function readNewFollowEvents(state: FollowState): TrajectoryEvent[] { + return state.kind === "sqlite" + ? readNewSqliteFollowEvents(state) + : readNewFileFollowEvents(state); +} + function renderFollowEvents( events: TrajectoryEvent[], state: FollowState, @@ -547,16 +575,29 @@ function renderFollowEvents( async function followSelections( selections: TailSelection[], runtime: RuntimeEnv, - initialSnapshots: Map<string, TrajectorySnapshot>, + initialSnapshots: Map<TailSelection, TrajectorySnapshot>, ): Promise<void> { const states = selections.map((selection): FollowState => { - const snapshot = initialSnapshots.get(selection.trajectoryPath); + const snapshot = initialSnapshots.get(selection); + if (selection.source.kind === "sqlite") { + return { + cursor: snapshot ? maxCursorFromEvents(snapshot.events) : null, + kind: "sqlite", + lastStorageSeq: snapshot?.maxStorageSeq ?? -1, + selection: selection as TailSelection & { + source: Extract<TailTrajectorySource, { kind: "sqlite" }>; + }, + }; + } return { cursor: snapshot ? maxCursorFromEvents(snapshot.events) : null, - fileState: snapshot?.fileState ?? readFollowFileState(selection.trajectoryPath), - offset: snapshot?.offset ?? statFileSize(selection.trajectoryPath), + fileState: snapshot?.fileState ?? readFollowFileState(selection.source.path), + kind: "file", + offset: snapshot?.offset ?? statFileSize(selection.source.path), pending: "", - selection, + selection: selection as TailSelection & { + source: Extract<TailTrajectorySource, { kind: "file" }>; + }, }; }); @@ -643,10 +684,10 @@ export async function sessionsTailCommand( return; } - const followSnapshots = new Map<string, TrajectorySnapshot>(); + const followSnapshots = new Map<TailSelection, TrajectorySnapshot>(); for (const selection of selected) { - const snapshot = readTrajectorySnapshot(selection.trajectoryPath); - followSnapshots.set(selection.trajectoryPath, snapshot); + const snapshot = readTailSnapshot(selection); + followSnapshots.set(selection, snapshot); renderEvents(tailCount > 0 ? snapshot.events.slice(-tailCount) : [], runtime); } diff --git a/src/commands/sessions.acp-model-display.test.ts b/src/commands/sessions.acp-model-display.test.ts index 3cc58385b46c..91a7831e3be5 100644 --- a/src/commands/sessions.acp-model-display.test.ts +++ b/src/commands/sessions.acp-model-display.test.ts @@ -177,9 +177,10 @@ describe("sessionsCommand model/modelProvider display for ACP sessions (catalog // SQLite ACP metadata (persisted by the ACP control plane manager). useTempStateDir(); writeAcpRuntimeMeta(ACP_SESSION_KEY); - const store = writeStore( + const store = await writeStore( { [ACP_SESSION_KEY]: buildAcpBridgeSessionEntry() }, "sessions-acp-model-display-red", + { agentId: "copilot" }, ); const payload = await runSessionsJson<SessionsJsonPayload>(sessionsCommand, store); @@ -216,9 +217,10 @@ describe("sessionsCommand model/modelProvider display for ACP sessions (catalog // events (catalog notes this as deferrable). useTempStateDir(); writeAcpRuntimeMeta(ACP_SESSION_KEY); - const store = writeStore( + const store = await writeStore( { [ACP_SESSION_KEY]: buildAcpBridgeSessionEntry() }, "sessions-acp-model-display-fix-shape", + { agentId: "copilot" }, ); const payload = await runSessionsJson<SessionsJsonPayload>(sessionsCommand, store); @@ -241,9 +243,10 @@ describe("sessionsCommand model/modelProvider display for ACP sessions (catalog it("reads ACP runtime metadata from SQLite for the display overlay", async () => { useTempStateDir(); - const store = writeStore( + const store = await writeStore( { [ACP_SESSION_KEY]: buildAcpBridgeSessionEntry() }, "sessions-acp-model-display-sqlite", + { agentId: "copilot" }, ); writeAcpRuntimeMeta(ACP_SESSION_KEY); @@ -259,9 +262,10 @@ describe("sessionsCommand model/modelProvider display for ACP sessions (catalog useTempStateDir(); const rawStoreKey = "acp:binding:discord:default:feedface"; const canonicalAcpKey = "agent:copilot:acp:binding:discord:default:feedface"; - const store = writeStore( + const store = await writeStore( { [rawStoreKey]: buildAcpBridgeSessionEntry() }, "sessions-acp-model-display-canonical", + { agentId: "copilot" }, ); writeAcpRuntimeMeta(canonicalAcpKey); @@ -279,9 +283,10 @@ describe("sessionsCommand model/modelProvider display for ACP sessions (catalog // and must NOT receive the acpx sentinel. This guards against a regression // where key-shape-only detection would misreport bridge sessions. const ACP_BRIDGE_SESSION_KEY = "agent:copilot:acp:bridge-session-1"; - const store = writeStore( + const store = await writeStore( { [ACP_BRIDGE_SESSION_KEY]: buildAcpBridgeSessionEntry() }, "sessions-acp-model-display-bridge-control", + { agentId: "copilot" }, ); const payload = await runSessionsJson<SessionsJsonPayload>(sessionsCommand, store); @@ -312,9 +317,10 @@ describe("sessionsCommand model/modelProvider display for ACP sessions (catalog // remains correct for non-ACP keys; the proposed sentinel overlay // must NOT break this case (it should only fire when both // isAcpSessionKey(row.key) is true AND ACP metadata is present). - const store = writeStore( + const store = await writeStore( { [NON_ACP_SESSION_KEY]: buildNonAcpSessionEntry() }, "sessions-acp-model-display-green-control", + { agentId: "copilot" }, ); const payload = await runSessionsJson<SessionsJsonPayload>(sessionsCommand, store); diff --git a/src/commands/sessions.default-agent-store.test.ts b/src/commands/sessions.default-agent-store.test.ts index 6b4d84d2506d..8c936c4ead3f 100644 --- a/src/commands/sessions.default-agent-store.test.ts +++ b/src/commands/sessions.default-agent-store.test.ts @@ -121,15 +121,28 @@ describe("sessionsCommand default store agent selection", () => { expect(payload.sessions?.map((session) => session.agentId)).toContain("voice"); }); - it("avoids duplicate rows when --all-agents resolves to a shared store path", async () => { + it("lists each SQLite owner when --all-agents resolves to a shared store path", async () => { loadConfigMock.mockImplementation(() => createSessionsConfig("/tmp/shared-sessions.json")); listSessionEntriesMock.mockReset(); - listSessionEntriesMock.mockReturnValue( - toSessionEntrySummaries({ - "agent:main:room": { sessionId: "s1", updatedAt: Date.now() - 60_000, model: "test:opus" }, - "agent:voice:room": { sessionId: "s2", updatedAt: Date.now() - 30_000, model: "test:opus" }, - }), - ); + listSessionEntriesMock + .mockReturnValueOnce( + toSessionEntrySummaries({ + "agent:main:room": { + sessionId: "s1", + updatedAt: Date.now() - 60_000, + model: "test:opus", + }, + }), + ) + .mockReturnValueOnce( + toSessionEntrySummaries({ + "agent:voice:room": { + sessionId: "s2", + updatedAt: Date.now() - 30_000, + model: "test:opus", + }, + }), + ); const { runtime, logs } = createRuntime(); await sessionsCommand({ allAgents: true, json: true }, runtime); @@ -142,12 +155,15 @@ describe("sessionsCommand default store agent selection", () => { }; expect(payload.count).toBe(2); expect(payload.allAgents).toBe(true); - expect(payload.stores).toEqual([{ agentId: "main", path: "/tmp/shared-sessions.json" }]); + expect(payload.stores).toEqual([ + { agentId: "main", path: "/tmp/shared-sessions.json" }, + { agentId: "voice", path: "/tmp/shared-sessions.json" }, + ]); expect(payload.sessions?.map((session) => session.agentId).toSorted()).toEqual([ "main", "voice", ]); - expect(listSessionEntriesMock).toHaveBeenCalledTimes(1); + expect(listSessionEntriesMock).toHaveBeenCalledTimes(2); }); it("uses configured default agent id when resolving implicit session store path", async () => { diff --git a/src/commands/sessions.kind-classification.test.ts b/src/commands/sessions.kind-classification.test.ts index 184d62899a04..533aa6f0724f 100644 --- a/src/commands/sessions.kind-classification.test.ts +++ b/src/commands/sessions.kind-classification.test.ts @@ -125,7 +125,7 @@ describe("sessionsCommand kind classification (catalog #19)", () => { // through to `"direct"`. Operators see this session in // `openclaw sessions --json` as `kind: "direct"` even though it was // plainly spawned from a group/topic. See `src/commands/sessions.ts:136-152`. - const store = writeStore( + const store = await writeStore( { [ACP_SPAWN_CHILD_KEY]: buildAcpSpawnChildEntry() }, "sessions-kind-spawn-child-red", ); @@ -158,7 +158,7 @@ describe("sessionsCommand kind classification (catalog #19)", () => { // and report `"group"`), update this assertion to match. The structural // point is that `entry.spawnedBy` / `entry.deliveryContext` MUST drive // the classification for ACP children. - const store = writeStore( + const store = await writeStore( { [ACP_SPAWN_CHILD_KEY]: buildAcpSpawnChildEntry() }, "sessions-kind-spawn-child-fix-shape", ); @@ -183,7 +183,7 @@ describe("sessionsCommand kind classification (catalog #19)", () => { // This control proves the test infrastructure exercises the real // classification path; if it accidentally regressed to a different // value, that would indicate the test harness was broken. - const store = writeStore( + const store = await writeStore( { [ACP_DM_KEY]: buildAcpDirectEntry() }, "sessions-kind-acp-direct-control", ); @@ -201,7 +201,7 @@ describe("sessionsCommand kind classification (catalog #19)", () => { // yielding `"group"`. This control proves the existing happy-path // classification still works and is not silently broken by the test // harness. - const store = writeStore( + const store = await writeStore( { [TELEGRAM_GROUP_KEY]: buildTelegramGroupEntry() }, "sessions-kind-group-control", ); diff --git a/src/commands/sessions.model-resolution.test.ts b/src/commands/sessions.model-resolution.test.ts index 9fe5a38e948c..65149dc2a52b 100644 --- a/src/commands/sessions.model-resolution.test.ts +++ b/src/commands/sessions.model-resolution.test.ts @@ -1,11 +1,17 @@ // Sessions model resolution tests cover displayed model metadata for stored session records. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../config/sessions/types.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { mockSessionsConfig, resetMockSessionsConfig, runSessionsJson, setMockSessionsConfig, - writeStore, } from "./sessions.test-helpers.js"; mockSessionsConfig(); @@ -25,7 +31,8 @@ async function resolveSubagentModel( runtimeFields: Record<string, unknown>, sessionId: string, ): Promise<string | null | undefined> { - const store = writeStore( + return await withSqliteStore( + "sessions-model", { "agent:research:subagent:demo": { sessionId, @@ -33,11 +40,34 @@ async function resolveSubagentModel( ...runtimeFields, }, }, - "sessions-model", + async (store) => { + const payload = await runSessionsJson<SessionsJsonPayload>(sessionsCommand, store); + return payload.sessions?.find((row) => row.key === "agent:research:subagent:demo")?.model; + }, ); +} - const payload = await runSessionsJson<SessionsJsonPayload>(sessionsCommand, store); - return payload.sessions?.find((row) => row.key === "agent:research:subagent:demo")?.model; +async function withSqliteStore<T>( + prefix: string, + entries: Record<string, SessionEntry>, + run: (storePath: string) => Promise<T>, +): Promise<T> { + // Use a sessions.json-shaped path so the accessor targets the same SQLite + // database layout that command code resolves from configured session stores. + const dir = fs.mkdtempSync(path.join(os.tmpdir(), `${prefix}-`)); + const storePath = path.join(dir, "sessions.json"); + try { + await Promise.all( + Object.entries(entries).map(([sessionKey, entry]) => + replaceSessionEntry({ agentId: "main", sessionKey, storePath }, entry), + ), + ); + return await run(storePath); + } finally { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + fs.rmSync(dir, { force: true, recursive: true }); + } } describe("sessionsCommand model resolution", () => { @@ -80,7 +110,8 @@ describe("sessionsCommand model resolution", () => { }, }, })); - const store = writeStore( + await withSqliteStore( + "sessions-claude-runtime", { "agent:main:main": { sessionId: "main-session", @@ -89,18 +120,18 @@ describe("sessionsCommand model resolution", () => { model: "claude-opus-4-7", }, }, - "sessions-claude-runtime", + async (store) => { + const payload = await runSessionsJson<SessionsJsonPayload>(sessionsCommand, store); + const session = payload.sessions?.find((row) => row.key === "agent:main:main"); + + expect(session?.modelProvider).toBe("anthropic"); + expect(session?.model).toBe("claude-opus-4-7"); + expect(session?.agentRuntime).toEqual({ + id: "claude-cli", + source: "model", + }); + }, ); - - const payload = await runSessionsJson<SessionsJsonPayload>(sessionsCommand, store); - const session = payload.sessions?.find((row) => row.key === "agent:main:main"); - - expect(session?.modelProvider).toBe("anthropic"); - expect(session?.model).toBe("claude-opus-4-7"); - expect(session?.agentRuntime).toEqual({ - id: "claude-cli", - source: "model", - }); }); it("infers canonical provider for bare CLI models before default-provider fallback", async () => { @@ -115,7 +146,8 @@ describe("sessionsCommand model resolution", () => { }, }, })); - const store = writeStore( + await withSqliteStore( + "sessions-claude-runtime-openai-default", { "agent:main:main": { sessionId: "main-session", @@ -124,14 +156,14 @@ describe("sessionsCommand model resolution", () => { model: "claude-opus-4-7", }, }, - "sessions-claude-runtime-openai-default", + async (store) => { + const payload = await runSessionsJson<SessionsJsonPayload>(sessionsCommand, store); + const session = payload.sessions?.find((row) => row.key === "agent:main:main"); + + expect(session?.modelProvider).toBe("anthropic"); + expect(session?.model).toBe("claude-opus-4-7"); + }, ); - - const payload = await runSessionsJson<SessionsJsonPayload>(sessionsCommand, store); - const session = payload.sessions?.find((row) => row.key === "agent:main:main"); - - expect(session?.modelProvider).toBe("anthropic"); - expect(session?.model).toBe("claude-opus-4-7"); }); it("reports the owning Codex harness for locked sessions despite a stale OpenClaw override", async () => { @@ -146,7 +178,8 @@ describe("sessionsCommand model resolution", () => { }, }, })); - const store = writeStore( + await withSqliteStore( + "sessions-locked-codex-runtime", { "agent:main:main": { sessionId: "locked-codex-session", @@ -158,15 +191,15 @@ describe("sessionsCommand model resolution", () => { modelSelectionLocked: true, }, }, - "sessions-locked-codex-runtime", + async (store) => { + const payload = await runSessionsJson<SessionsJsonPayload>(sessionsCommand, store); + const session = payload.sessions?.find((row) => row.key === "agent:main:main"); + + expect(session?.agentRuntime).toEqual({ + id: "codex", + source: "session", + }); + }, ); - - const payload = await runSessionsJson<SessionsJsonPayload>(sessionsCommand, store); - const session = payload.sessions?.find((row) => row.key === "agent:main:main"); - - expect(session?.agentRuntime).toEqual({ - id: "codex", - source: "session", - }); }); }); diff --git a/src/commands/sessions.test-helpers.ts b/src/commands/sessions.test-helpers.ts index 263665b6b5de..921d8f1d6361 100644 --- a/src/commands/sessions.test-helpers.ts +++ b/src/commands/sessions.test-helpers.ts @@ -4,6 +4,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { vi } from "vitest"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import type { RuntimeEnv } from "../runtime.js"; const sessionsConfigState = vi.hoisted<{ loadConfig: () => Record<string, unknown> }>(() => ({ @@ -77,13 +79,28 @@ export function makeRuntime(params?: { throwOnError?: boolean }): { }; } -export function writeStore(data: unknown, prefix = "sessions"): string { - const fileName = `${[prefix, Date.now(), randomUUID()].join("-")}.json`; - const file = path.join(os.tmpdir(), fileName); - fs.writeFileSync(file, JSON.stringify(data, null, 2)); - return file; +/** Seeds an isolated SQLite-backed session store and returns its legacy target path. */ +export async function writeStore( + data: Record<string, SessionEntry>, + prefix = "sessions", + options: { agentId?: string } = {}, +): Promise<string> { + const dirName = [prefix, Date.now(), randomUUID()].join("-"); + const storeDir = path.join(os.tmpdir(), dirName); + fs.mkdirSync(storeDir, { recursive: true }); + const storePath = path.join(storeDir, "sessions.json"); + for (const [sessionKey, entry] of Object.entries(data)) { + await replaceSessionEntry({ agentId: options.agentId ?? "main", sessionKey, storePath }, entry); + } + return storePath; } +/** Removes the temporary SQLite session store created by writeStore. */ +export function cleanupStore(store: string): void { + fs.rmSync(path.dirname(store), { recursive: true, force: true }); +} + +/** Runs sessionsCommand with JSON output and parses the emitted payload. */ export async function runSessionsJson<T>( run: ( opts: { json?: boolean; store?: string; active?: string; limit?: string | number }, @@ -107,7 +124,7 @@ export async function runSessionsJson<T>( runtime, ); } finally { - fs.rmSync(store, { force: true }); + cleanupStore(store); } return JSON.parse(logs[0] ?? "{}") as T; } diff --git a/src/commands/sessions.test.ts b/src/commands/sessions.test.ts index 8aeb594bae3b..508935c67619 100644 --- a/src/commands/sessions.test.ts +++ b/src/commands/sessions.test.ts @@ -1,7 +1,7 @@ // Sessions command tests cover listing, details, filtering, and transcript display behavior. -import fs from "node:fs"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { + cleanupStore, makeRuntime, mockSessionsConfig, resetMockSessionsConfig, @@ -29,7 +29,7 @@ describe("sessionsCommand", () => { }); it("renders a tabular view with token percentages", async () => { - const store = writeStore({ + const store = await writeStore({ "+15555550123": { sessionId: "abc123", updatedAt: Date.now() - 45 * 60_000, @@ -44,7 +44,7 @@ describe("sessionsCommand", () => { const { runtime, logs } = makeRuntime(); await sessionsCommand({ store }, runtime); - fs.rmSync(store); + cleanupStore(store); expect(logs.join("\n")).toContain("Tokens (ctx %"); @@ -66,7 +66,7 @@ describe("sessionsCommand", () => { }, }, })); - const store = writeStore( + const store = await writeStore( { "agent:main:main": { sessionId: "main-session", @@ -81,7 +81,7 @@ describe("sessionsCommand", () => { const { runtime, logs } = makeRuntime(); await sessionsCommand({ store }, runtime); - fs.rmSync(store); + cleanupStore(store); expect(logs.join("\n")).toContain("Runtime"); @@ -103,7 +103,7 @@ describe("sessionsCommand", () => { }, }, })); - const store = writeStore( + const store = await writeStore( { "agent:main:main": { sessionId: "main-session", @@ -118,7 +118,7 @@ describe("sessionsCommand", () => { const { runtime, logs } = makeRuntime(); await sessionsCommand({ store }, runtime); - fs.rmSync(store); + cleanupStore(store); const row = logs.find((line) => line.includes("agent:main:main")) ?? ""; expect(row).toBe( @@ -127,7 +127,7 @@ describe("sessionsCommand", () => { }); it("shows placeholder rows when tokens are missing", async () => { - const store = writeStore({ + const store = await writeStore({ "quietchat:group:demo": { sessionId: "xyz", updatedAt: Date.now() - 5 * 60_000, @@ -138,7 +138,7 @@ describe("sessionsCommand", () => { const { runtime, logs } = makeRuntime(); await sessionsCommand({ store }, runtime); - fs.rmSync(store); + cleanupStore(store); const row = logs.find((line) => line.includes("quietchat:group:demo")) ?? ""; expect(row).toBe( @@ -147,7 +147,7 @@ describe("sessionsCommand", () => { }); it("exports freshness metadata in JSON output", async () => { - const store = writeStore({ + const store = await writeStore({ main: { sessionId: "abc123", updatedAt: Date.now() - 10 * 60_000, @@ -182,7 +182,7 @@ describe("sessionsCommand", () => { }); it("exports subagent lineage metadata in JSON output", async () => { - const store = writeStore({ + const store = await writeStore({ "agent:child:main": { sessionId: "child-session", updatedAt: Date.now() - 10 * 60_000, @@ -241,7 +241,7 @@ describe("sessionsCommand", () => { }); it("shows preserved stale totals in JSON output", async () => { - const store = writeStore({ + const store = await writeStore({ main: { sessionId: "abc123", updatedAt: Date.now() - 10 * 60_000, @@ -264,7 +264,7 @@ describe("sessionsCommand", () => { }); it("applies --active filtering in JSON output", async () => { - const store = writeStore( + const store = await writeStore( { recent: { sessionId: "recent", @@ -289,7 +289,7 @@ describe("sessionsCommand", () => { }); it("exports runtime policy aliases for collapsed external direct sessions", async () => { - const store = writeStore( + const store = await writeStore( { "agent:main:main": { sessionId: "telegram-main", @@ -321,7 +321,7 @@ describe("sessionsCommand", () => { }); it("honors explicit JSON output limits", async () => { - const store = writeStore( + const store = await writeStore( { newest: { sessionId: "newest", updatedAt: Date.now(), model: "test:opus" }, middle: { sessionId: "middle", updatedAt: Date.now() - 60_000, model: "test:opus" }, @@ -346,7 +346,7 @@ describe("sessionsCommand", () => { }); it("allows full JSON output with --limit all", async () => { - const store = writeStore( + const store = await writeStore( { newest: { sessionId: "newest", updatedAt: Date.now(), model: "test:opus" }, oldest: { sessionId: "oldest", updatedAt: Date.now() - 120_000, model: "test:opus" }, @@ -370,7 +370,7 @@ describe("sessionsCommand", () => { }); it("sorts and slices large explicit limits instead of using top-N insertion", async () => { - const store = writeStore( + const store = await writeStore( { newest: { sessionId: "newest", updatedAt: Date.now(), model: "test:opus" }, oldest: { sessionId: "oldest", updatedAt: Date.now() - 120_000, model: "test:opus" }, @@ -394,7 +394,7 @@ describe("sessionsCommand", () => { }); it("rejects invalid --active values", async () => { - const store = writeStore( + const store = await writeStore( { demo: { sessionId: "demo", @@ -410,11 +410,11 @@ describe("sessionsCommand", () => { "--active must be a positive number of minutes, for example --active 30.", ]); - fs.rmSync(store); + cleanupStore(store); }); it("rejects partial --active values", async () => { - const store = writeStore( + const store = await writeStore( { demo: { sessionId: "demo", @@ -430,11 +430,11 @@ describe("sessionsCommand", () => { "--active must be a positive number of minutes, for example --active 30.", ]); - fs.rmSync(store); + cleanupStore(store); }); it("rejects invalid --limit values", async () => { - const store = writeStore( + const store = await writeStore( { demo: { sessionId: "demo", @@ -450,6 +450,6 @@ describe("sessionsCommand", () => { '--limit must be a positive integer or "all", for example --limit 25.', ]); - fs.rmSync(store); + cleanupStore(store); }); }); diff --git a/src/commands/tasks.test.ts b/src/commands/tasks.test.ts index 9e1dc24692bd..3ffb31b5abd7 100644 --- a/src/commands/tasks.test.ts +++ b/src/commands/tasks.test.ts @@ -1,8 +1,9 @@ // Tasks command tests cover task listing, status rendering, cron-store integration, and cancellations. -import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { resetConfigRuntimeState } from "../config/config.js"; +import { loadSessionEntry, replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import { saveCronStore } from "../cron/store.js"; import type { RuntimeEnv } from "../runtime.js"; import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; @@ -84,6 +85,15 @@ const zeroTaskAuditCounts = { stale_running: 0, }; +async function writeSessionEntries( + storePath: string, + entries: Record<string, SessionEntry>, +): Promise<void> { + for (const [sessionKey, entry] of Object.entries(entries)) { + await replaceSessionEntry({ sessionKey, storePath }, entry); + } +} + async function withTaskCommandStateDir( run: (state: OpenClawTestState) => Promise<void>, ): Promise<void> { @@ -377,21 +387,13 @@ describe("tasks commands", () => { }); const sessionsDir = state.sessionsDir("main"); - await fs.mkdir(sessionsDir, { recursive: true }); - await fs.writeFile( - path.join(sessionsDir, "sessions.json"), - JSON.stringify( - { - [childSessionKey]: { - sessionId: "child-retained", - updatedAt: now, - }, - }, - null, - 2, - ), - "utf8", - ); + const storePath = path.join(sessionsDir, "sessions.json"); + await writeSessionEntries(storePath, { + [childSessionKey]: { + sessionId: "child-retained", + updatedAt: now, + }, + }); const runtime = createRuntime(); await tasksMaintenanceCommand({ json: true, apply: false }, runtime); @@ -435,25 +437,16 @@ describe("tasks commands", () => { const sessionsDir = state.sessionsDir("main"); const storePath = path.join(sessionsDir, "sessions.json"); - await fs.mkdir(sessionsDir, { recursive: true }); - await fs.writeFile( - storePath, - JSON.stringify( - { - [childSessionKey]: { - sessionId: "old-run", - updatedAt: now - 8 * 24 * 60 * 60_000, - }, - "agent:main:telegram:dm:recent": { - sessionId: "recent-session", - updatedAt: now - 60_000, - }, - }, - null, - 2, - ), - "utf8", - ); + await writeSessionEntries(storePath, { + [childSessionKey]: { + sessionId: "old-run", + updatedAt: now - 8 * 24 * 60 * 60_000, + }, + "agent:main:telegram:dm:recent": { + sessionId: "recent-session", + updatedAt: now - 60_000, + }, + }); const runtime = createRuntime(); await tasksMaintenanceCommand({ json: true, apply: true }, runtime); @@ -484,9 +477,10 @@ describe("tasks commands", () => { }), ); - const updated = JSON.parse(await fs.readFile(storePath, "utf8")) as Record<string, unknown>; - expect(updated[childSessionKey]).toBeUndefined(); - expect(updated["agent:main:telegram:dm:recent"]).toBeDefined(); + expect(loadSessionEntry({ sessionKey: childSessionKey, storePath })).toBeUndefined(); + expect( + loadSessionEntry({ sessionKey: "agent:main:telegram:dm:recent", storePath }), + ).toBeDefined(); }); }); @@ -516,25 +510,16 @@ describe("tasks commands", () => { const sessionsDir = state.sessionsDir("main"); const storePath = path.join(sessionsDir, "sessions.json"); - await fs.mkdir(sessionsDir, { recursive: true }); // A running job can be retargeted after its session is created, so maintenance must preserve // both the raw and slugged historical shapes. const slugKey = "agent:main:cron:daily-report:run:old-run"; const rawKey = "agent:main:cron:daily report:run:old-run"; const retiredKey = "agent:main:cron:retired-job:run:old-run"; - await fs.writeFile( - storePath, - JSON.stringify( - { - [slugKey]: { sessionId: "slug-run", updatedAt: old }, - [rawKey]: { sessionId: "raw-run", updatedAt: old }, - [retiredKey]: { sessionId: "retired-run", updatedAt: old }, - }, - null, - 2, - ), - "utf8", - ); + await writeSessionEntries(storePath, { + [slugKey]: { sessionId: "slug-run", updatedAt: old }, + [rawKey]: { sessionId: "raw-run", updatedAt: old }, + [retiredKey]: { sessionId: "retired-run", updatedAt: old }, + }); const runtime = createRuntime(); await tasksMaintenanceCommand({ json: true, apply: true }, runtime); @@ -543,10 +528,9 @@ describe("tasks commands", () => { maintenance: { sessions: { runningCronJobs: number } }; }; expect(payload.maintenance.sessions.runningCronJobs).toBe(1); - const updated = JSON.parse(await fs.readFile(storePath, "utf8")) as Record<string, unknown>; - expect(updated[slugKey]).toBeDefined(); - expect(updated[rawKey]).toBeDefined(); - expect(updated[retiredKey]).toBeUndefined(); + expect(loadSessionEntry({ sessionKey: slugKey, storePath })).toBeDefined(); + expect(loadSessionEntry({ sessionKey: rawKey, storePath })).toBeDefined(); + expect(loadSessionEntry({ sessionKey: retiredKey, storePath })).toBeUndefined(); }); }); @@ -576,36 +560,33 @@ describe("tasks commands", () => { const sessionsDir = state.sessionsDir("main"); const storePath = path.join(sessionsDir, "sessions.json"); - await fs.mkdir(sessionsDir, { recursive: true }); - await fs.writeFile( - storePath, - JSON.stringify( - { - "agent:main:cron:daily-monitor:run:old-run": { - sessionId: "explicit-run", - updatedAt: old, - }, - "agent:main:cron:job-uuid:run:old-run": { - sessionId: "job-id-run", - updatedAt: old, - }, - "agent:main:cron:retired-job:run:old-run": { - sessionId: "retired-run", - updatedAt: old, - }, - }, - null, - 2, - ), - "utf8", - ); + await writeSessionEntries(storePath, { + "agent:main:cron:daily-monitor:run:old-run": { + sessionId: "explicit-run", + updatedAt: old, + }, + "agent:main:cron:job-uuid:run:old-run": { + sessionId: "job-id-run", + updatedAt: old, + }, + "agent:main:cron:retired-job:run:old-run": { + sessionId: "retired-run", + updatedAt: old, + }, + }); const runtime = createRuntime(); await tasksMaintenanceCommand({ json: true, apply: true }, runtime); - const updated = JSON.parse(await fs.readFile(storePath, "utf8")) as Record<string, unknown>; - expect(updated["agent:main:cron:daily-monitor:run:old-run"]).toBeDefined(); - expect(updated["agent:main:cron:retired-job:run:old-run"]).toBeUndefined(); + expect( + loadSessionEntry({ + sessionKey: "agent:main:cron:daily-monitor:run:old-run", + storePath, + }), + ).toBeDefined(); + expect( + loadSessionEntry({ sessionKey: "agent:main:cron:retired-job:run:old-run", storePath }), + ).toBeUndefined(); }); }); @@ -741,33 +722,24 @@ describe("tasks commands", () => { const sessionsDir = state.sessionsDir("main"); const storePath = path.join(sessionsDir, "sessions.json"); const old = now - 8 * 24 * 60 * 60_000; - await fs.mkdir(sessionsDir, { recursive: true }); - await fs.writeFile( - storePath, - JSON.stringify( - { - "agent:main:cron:done-job:run:old-run": { - sessionId: "done-run", - updatedAt: old, - }, - "agent:main:cron:running-job:run:old-run": { - sessionId: "running-run", - updatedAt: old, - }, - "agent:main:cron:done-job:run:recent-run": { - sessionId: "recent-run", - updatedAt: now - 60_000, - }, - "agent:main:telegram:dm:old": { - sessionId: "ordinary-old-session", - updatedAt: old, - }, - }, - null, - 2, - ), - "utf-8", - ); + await writeSessionEntries(storePath, { + "agent:main:cron:done-job:run:old-run": { + sessionId: "done-run", + updatedAt: old, + }, + "agent:main:cron:running-job:run:old-run": { + sessionId: "running-run", + updatedAt: old, + }, + "agent:main:cron:done-job:run:recent-run": { + sessionId: "recent-run", + updatedAt: now - 60_000, + }, + "agent:main:telegram:dm:old": { + sessionId: "ordinary-old-session", + updatedAt: old, + }, + }); await saveCronStore(state.statePath("cron", "jobs.json"), { version: 1, jobs: [ @@ -818,14 +790,15 @@ describe("tasks commands", () => { expect(payload.maintenance.sessions.stores[0]?.pruned).toBe(1); expect(payload.maintenance.sessions.stores[0]?.preservedRunning).toBe(1); - const updated = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record<string, unknown>; - expect(updated["agent:main:cron:done-job:run:old-run"]).toBeUndefined(); + expect( + loadSessionEntry({ sessionKey: "agent:main:cron:done-job:run:old-run", storePath }), + ).toBeUndefined(); for (const key of [ "agent:main:cron:running-job:run:old-run", "agent:main:cron:done-job:run:recent-run", "agent:main:telegram:dm:old", ]) { - if (updated[key] === undefined) { + if (loadSessionEntry({ sessionKey: key, storePath }) === undefined) { throw new Error(`Expected preserved session ${key}`); } } diff --git a/src/config/schema.help.ts b/src/config/schema.help.ts index f1fbe4c804d7..901680537797 100644 --- a/src/config/schema.help.ts +++ b/src/config/schema.help.ts @@ -1594,7 +1594,7 @@ export const FIELD_HELP: Record<string, string> = { "agents.defaults.compaction.model": "Optional provider/model or configured bare alias used only for compaction summarization. Bare aliases resolve before dispatch; a configured literal model ID wins if it collides with an alias. Leave unset to keep using the primary agent model.", "agents.defaults.compaction.truncateAfterCompaction": - "When enabled, rotates the active session JSONL file after compaction so future turns load only the summary and unsummarized tail while the previous full transcript remains archived. Prevents unbounded active transcript growth in long-running sessions. Default: false.", + "When enabled, rotates the active session transcript after compaction so future turns load only the summary and unsummarized tail while the previous full transcript remains archived. Prevents unbounded active transcript growth in long-running sessions. Default: false.", "agents.defaults.compaction.maxActiveTranscriptBytes": 'Triggers normal local compaction when the active session transcript reaches this size (bytes or strings like "20mb"). Requires truncateAfterCompaction so successful compaction can rotate to a smaller successor transcript; set to 0 or leave unset to disable. This never splits raw transcript bytes.', "agents.defaults.compaction.notifyUser": @@ -1608,7 +1608,7 @@ export const FIELD_HELP: Record<string, string> = { "agents.defaults.compaction.memoryFlush.softThresholdTokens": "Threshold distance to compaction (in tokens) that triggers pre-compaction memory flush execution. Use earlier thresholds for safer persistence, or tighter thresholds for lower flush frequency.", "agents.defaults.compaction.memoryFlush.forceFlushTranscriptBytes": - 'Forces pre-compaction memory flush when transcript file size reaches this threshold (bytes or strings like "2mb"). Use this to prevent long-session hangs even when token counters are stale; set to 0 to disable.', + 'Forces pre-compaction memory flush when active transcript size reaches this threshold (bytes or strings like "2mb"). Use this to prevent long-session hangs even when token counters are stale; set to 0 to disable.', "agents.defaults.compaction.memoryFlush.prompt": "User-prompt template used for the pre-compaction memory flush turn when generating memory candidates. Use this only when you need custom extraction instructions beyond the default memory flush behavior.", "agents.defaults.compaction.memoryFlush.systemPrompt": @@ -1787,9 +1787,9 @@ export const FIELD_HELP: Record<string, string> = { "session.maintenance.rotateBytes": 'Deprecated and ignored. Do not use for `sessions.json` growth control; OpenClaw no longer creates automatic rotation backups, and "openclaw doctor --fix" removes this key.', "session.maintenance.resetArchiveRetention": - "Retention for reset transcript archives (`*.reset.<timestamp>`). Accepts a duration (for example `30d`), or `false` to disable cleanup. Defaults to pruneAfter so reset artifacts do not grow forever.", + "Age-based retention for archived transcripts (`*.reset.<timestamp>` and `*.deleted.<timestamp>`). Defaults to keeping archives until the disk budget evicts them oldest-first; set a duration (for example `30d`) to opt into wall-clock deletion, or `false` to disable it explicitly.", "session.maintenance.maxDiskBytes": - "Optional per-agent sessions-directory disk budget (for example `500mb`). Use this to cap session storage per agent; when exceeded, warn mode reports pressure and enforce mode performs oldest-first cleanup.", + "Per-agent sessions-directory disk budget (for example `500mb`). Defaults to `2gb`; when exceeded, warn mode reports pressure and enforce mode performs oldest-first cleanup (archived transcripts before live sessions). Set `false` to disable.", "session.maintenance.highWaterBytes": "Target size after disk-budget cleanup (high-water mark). Defaults to 80% of maxDiskBytes; set explicitly for tighter reclaim behavior on constrained disks.", cron: "Global scheduler settings for stored cron jobs, run concurrency, delivery fallback, and run-session retention. Keep defaults unless you are scaling job volume or integrating external webhook receivers.", diff --git a/src/config/sessions.cache.test.ts b/src/config/sessions.cache.test.ts index 09bfba52b775..555cd7b3c01a 100644 --- a/src/config/sessions.cache.test.ts +++ b/src/config/sessions.cache.test.ts @@ -792,30 +792,6 @@ describe("Session Store Cache", () => { expect(cached["session:1"].deliveryContext?.to).toBe("chat-1"); }); - it("patches serialized JSON for one-entry updates without stringifying untouched entries", async () => { - await saveSessionStore(storePath, { - "session:1": createSessionEntry({ sessionId: "id-1", displayName: "Before" }), - "session:2": createSessionEntry({ sessionId: "id-2", displayName: "Untouched" }), - }); - const cached = loadSessionStore(storePath, { clone: false }); - Object.defineProperty(cached["session:2"], "toJSON", { - value: () => { - throw new Error("full store stringify touched session:2"); - }, - }); - - await updateSessionStoreEntry({ - storePath, - sessionKey: "session:1", - update: async () => ({ displayName: "After", updatedAt: 123 }), - takeCacheOwnership: true, - }); - - const disk = JSON.parse(fs.readFileSync(storePath, "utf8")) as Record<string, SessionEntry>; - expect(disk["session:1"].displayName).toBe("After"); - expect(disk["session:2"].displayName).toBe("Untouched"); - }); - it("falls back to full projection when untouched entries need prompt blob repair", async () => { const prompt = "skill prompt ".repeat(80); await saveSessionStore(storePath, { diff --git a/src/config/sessions/archive-compression.test.ts b/src/config/sessions/archive-compression.test.ts new file mode 100644 index 000000000000..53ea0243d419 --- /dev/null +++ b/src/config/sessions/archive-compression.test.ts @@ -0,0 +1,67 @@ +// Round-trip and naming coverage for the archived-transcript zstd cold tier. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + encodeSessionArchiveContent, + readSessionArchiveContentSync, + SESSION_ARCHIVE_ZSTD_SUFFIX, + stripSessionArchiveCompressionSuffix, +} from "./archive-compression.js"; +import { + parseSessionArchiveTimestamp, + parseUsageCountedSessionIdFromFileName, +} from "./artifacts.js"; + +const tempDirs: string[] = []; + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +function makeTempDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-archive-zstd-")); + tempDirs.push(dir); + return dir; +} + +describe("archive compression", () => { + it("round-trips archived transcript content through encode and read", () => { + const content = `${JSON.stringify({ type: "message", body: "hello" })}\n`.repeat(200); + const encoded = encodeSessionArchiveContent(content); + const dir = makeTempDir(); + const archivePath = path.join( + dir, + `sess.jsonl.deleted.2026-07-11T00-00-00.000Z${encoded.suffix}`, + ); + fs.writeFileSync(archivePath, encoded.bytes); + + expect(readSessionArchiveContentSync(archivePath)).toBe(content); + if (encoded.suffix === SESSION_ARCHIVE_ZSTD_SUFFIX) { + // Compression must actually pay for itself on repetitive JSONL. + expect(encoded.bytes.length).toBeLessThan(Buffer.byteLength(content, "utf8") / 2); + } + }); + + it("keeps plain archives readable regardless of runtime zstd support", () => { + const dir = makeTempDir(); + const archivePath = path.join(dir, "sess.jsonl.reset.2026-07-11T00-00-00.000Z"); + fs.writeFileSync(archivePath, "plain\n", "utf8"); + + expect(readSessionArchiveContentSync(archivePath)).toBe("plain\n"); + }); + + it("strips the zstd suffix so archive name parsers see one shape", () => { + const plain = "sess.jsonl.deleted.2026-07-11T00-00-00.000Z"; + const compressed = `${plain}${SESSION_ARCHIVE_ZSTD_SUFFIX}`; + + expect(stripSessionArchiveCompressionSuffix(compressed)).toBe(plain); + expect(parseSessionArchiveTimestamp(compressed, "deleted")).toBe( + parseSessionArchiveTimestamp(plain, "deleted"), + ); + expect(parseUsageCountedSessionIdFromFileName(compressed)).toBe("sess"); + }); +}); diff --git a/src/config/sessions/archive-compression.ts b/src/config/sessions/archive-compression.ts new file mode 100644 index 000000000000..67355932eaae --- /dev/null +++ b/src/config/sessions/archive-compression.ts @@ -0,0 +1,68 @@ +// Zstd compression for archived transcript artifacts (Codex-style cold tier). +// Archives are kept long-term by default, so compressing them is what keeps +// "never delete conversations" cheap: JSONL transcripts compress ~10:1. +import fs from "node:fs"; +import zlib from "node:zlib"; + +export const SESSION_ARCHIVE_ZSTD_SUFFIX = ".zst"; + +type ZstdCodec = { + compress: (data: Buffer) => Buffer; + decompress: (data: Buffer) => Buffer; +}; + +// node:zlib ships zstd since Node 22.15/23.8; Bun may not implement it yet. +// Feature-detect so the Bun path writes plain JSONL archives instead of +// crashing, and mixed plain/compressed archives always stay readable. +function resolveZstdCodec(): ZstdCodec | null { + const candidate = zlib as Partial<{ + zstdCompressSync: (data: Buffer) => Buffer; + zstdDecompressSync: (data: Buffer) => Buffer; + }>; + if ( + typeof candidate.zstdCompressSync !== "function" || + typeof candidate.zstdDecompressSync !== "function" + ) { + return null; + } + return { + compress: candidate.zstdCompressSync.bind(zlib), + decompress: candidate.zstdDecompressSync.bind(zlib), + }; +} + +const zstdCodec = resolveZstdCodec(); + +/** Strips the optional zstd suffix so archive name parsers see one shape. */ +export function stripSessionArchiveCompressionSuffix(fileName: string): string { + return fileName.endsWith(SESSION_ARCHIVE_ZSTD_SUFFIX) + ? fileName.slice(0, -SESSION_ARCHIVE_ZSTD_SUFFIX.length) + : fileName; +} + +/** Compresses archive content when the runtime supports zstd. */ +export function encodeSessionArchiveContent(content: string): { + bytes: Buffer; + suffix: "" | typeof SESSION_ARCHIVE_ZSTD_SUFFIX; +} { + const plain = Buffer.from(content, "utf8"); + if (!zstdCodec || plain.length === 0) { + return { bytes: plain, suffix: "" }; + } + // Default zstd level (3) matches the ratio/speed point Codex uses for cold + // rollouts; archives are write-once so speed matters less than footprint. + return { bytes: zstdCodec.compress(plain), suffix: SESSION_ARCHIVE_ZSTD_SUFFIX }; +} + +/** Reads an archived transcript, transparently decompressing zstd artifacts. */ +export function readSessionArchiveContentSync(filePath: string): string { + if (!filePath.endsWith(SESSION_ARCHIVE_ZSTD_SUFFIX)) { + return fs.readFileSync(filePath, "utf8"); + } + if (!zstdCodec) { + throw new Error( + `Cannot read compressed transcript archive ${filePath}: this runtime lacks node:zlib zstd support`, + ); + } + return zstdCodec.decompress(fs.readFileSync(filePath)).toString("utf8"); +} diff --git a/src/config/sessions/artifacts.ts b/src/config/sessions/artifacts.ts index fb2cf8850423..fd774dd3e320 100644 --- a/src/config/sessions/artifacts.ts +++ b/src/config/sessions/artifacts.ts @@ -3,6 +3,7 @@ import { timestampMsToIsoFileStamp } from "@openclaw/normalization-core/number-coercion"; import { escapeRegExp } from "../../shared/regexp.js"; +import { stripSessionArchiveCompressionSuffix } from "./archive-compression.js"; export type SessionArchiveReason = "bak" | "reset" | "deleted"; @@ -12,12 +13,15 @@ const COMPACTION_CHECKPOINT_TRANSCRIPT_RE = /^(.+)\.checkpoint\.([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\.jsonl$/i; function hasArchiveSuffix(fileName: string, reason: SessionArchiveReason): boolean { + // Compressed archives carry a trailing .zst; strip it so every classifier + // sees one canonical `<id>.jsonl.<reason>.<timestamp>` shape. const marker = `.${reason}.`; - const index = fileName.lastIndexOf(marker); + const normalized = stripSessionArchiveCompressionSuffix(fileName); + const index = normalized.lastIndexOf(marker); if (index < 0) { return false; } - const raw = fileName.slice(index + marker.length); + const raw = normalized.slice(index + marker.length); return ARCHIVE_TIMESTAMP_RE.test(raw); } @@ -128,11 +132,12 @@ export function parseUsageCountedSessionIdFromFileName(fileName: string): string if (isPrimarySessionTranscriptFileName(fileName)) { return fileName.slice(0, -".jsonl".length); } + const normalized = stripSessionArchiveCompressionSuffix(fileName); for (const reason of ["reset", "deleted"] as const) { const marker = `.jsonl.${reason}.`; - const index = fileName.lastIndexOf(marker); - if (index > 0 && hasArchiveSuffix(fileName, reason)) { - return fileName.slice(0, index); + const index = normalized.lastIndexOf(marker); + if (index > 0 && hasArchiveSuffix(normalized, reason)) { + return normalized.slice(0, index); } } return null; @@ -156,11 +161,12 @@ export function parseSessionArchiveTimestamp( reason: SessionArchiveReason, ): number | null { const marker = `.${reason}.`; - const index = fileName.lastIndexOf(marker); + const normalized = stripSessionArchiveCompressionSuffix(fileName); + const index = normalized.lastIndexOf(marker); if (index < 0) { return null; } - const raw = fileName.slice(index + marker.length); + const raw = normalized.slice(index + marker.length); if (!raw) { return null; } diff --git a/src/config/sessions/cleanup-service.ts b/src/config/sessions/cleanup-service.ts index 692eefa6a80b..9eeea53cb85d 100644 --- a/src/config/sessions/cleanup-service.ts +++ b/src/config/sessions/cleanup-service.ts @@ -6,26 +6,24 @@ import path from "node:path"; import { resolveDefaultAgentId } from "../../agents/agent-scope.js"; import { getLogger } from "../../logging/logger.js"; import { normalizeAgentId, parseAgentSessionKey } from "../../routing/session-key.js"; +import { resolveOpenClawAgentSqlitePath } from "../../state/openclaw-agent-db.js"; import type { OpenClawConfig } from "../types.openclaw.js"; import { - enforceSessionDiskBudget, pruneUnreferencedSessionArtifacts, resolveSessionArtifactCanonicalPathsForEntry, + type SessionDiskBudgetSweepResult, type SessionUnreferencedArtifactSweepResult, } from "./disk-budget.js"; -import { extractGeneratedTranscriptSessionId } from "./generated-transcript-session-id.js"; -import { - resolveSessionFilePath, - resolveSessionFilePathOptions, - resolveSessionTranscriptPathInDir, - resolveStorePath, -} from "./paths.js"; +import { resolveStorePath } from "./paths.js"; import { applySessionEntryLifecycleMutation, + listSessionEntries, + loadTranscriptEventsSync, + previewSessionDiskBudget, purgeDeletedAgentSessionEntries, type SessionEntryLifecycleRemoval, - type SessionEntryLifecycleUpsert, } from "./session-accessor.js"; +import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js"; import { cloneSessionStoreRecord } from "./store-cache.js"; import { collectSessionMaintenancePreserveKeysForStore } from "./store-maintenance-preserve.js"; import { resolveMaintenanceConfig } from "./store-maintenance-runtime.js"; @@ -37,7 +35,6 @@ import { shouldRunModelRunPrune, type ResolvedSessionMaintenanceConfig, } from "./store-maintenance.js"; -import { loadSessionStore } from "./store.js"; import { resolveSessionStoreTargets, type SessionStoreTarget, @@ -56,7 +53,6 @@ export type SessionsCleanupOptions = SessionStoreSelectionOptions & { export type SessionCleanupAction = | "keep" - | "repair-session-file" | "prune-missing" | "prune-model-run" | "prune-stale" @@ -71,14 +67,13 @@ export type SessionCleanupSummary = { dryRun: boolean; beforeCount: number; afterCount: number; - repaired: number; missing: number; dmScopeRetired: number; modelRunPruned: number; pruned: number; capped: number; unreferencedArtifacts: SessionUnreferencedArtifactSweepResult; - diskBudget: Awaited<ReturnType<typeof enforceSessionDiskBudget>>; + diskBudget: SessionDiskBudgetSweepResult | null; wouldMutate: boolean; applied?: true; appliedCount?: number; @@ -98,7 +93,6 @@ export type SessionsCleanupRunResult = { previewResults: Array<{ summary: SessionCleanupSummary; beforeStore: Record<string, SessionEntry>; - repairedKeys: Set<string>; missingKeys: Set<string>; modelRunPrunedKeys: Set<string>; staleKeys: Set<string>; @@ -109,7 +103,27 @@ export type SessionsCleanupRunResult = { appliedSummaries: SessionCleanupSummary[]; }; -const EMPTY_TRANSCRIPT_MAX_BYTES = 4096; +function resolveCleanupSqlitePath(target: SessionStoreTarget): string { + return ( + resolveSqliteTargetFromSessionStorePath(target.storePath, { agentId: target.agentId }).path ?? + resolveOpenClawAgentSqlitePath({ agentId: target.agentId }) + ); +} + +function loadCleanupSessionStore( + target: SessionStoreTarget, + options: { createIfMissing?: boolean } = {}, +): Record<string, SessionEntry> { + if (options.createIfMissing !== true && !fs.existsSync(resolveCleanupSqlitePath(target))) { + return {}; + } + return Object.fromEntries( + listSessionEntries({ + agentId: target.agentId, + storePath: target.storePath, + }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ); +} function isTranscriptMessageRole(role: unknown): boolean { return ( @@ -140,62 +154,21 @@ function isTranscriptMessageRecord(entry: unknown): boolean { return record.type === undefined && isTranscriptMessageRole(record.role); } -function transcriptHasNoMessageRecords(transcriptPath: string): boolean { - let stat: fs.Stats; - try { - stat = fs.statSync(transcriptPath); - } catch { - return false; - } - if (!stat.isFile() || stat.size > EMPTY_TRANSCRIPT_MAX_BYTES) { - // Only inspect small transcript files; larger files are assumed to contain real history. - return false; - } - - let raw: string; - try { - raw = fs.readFileSync(transcriptPath, "utf-8"); - } catch { - return false; - } - - const lines = raw.split(/\r?\n/u).filter((line) => line.trim().length > 0); - if (lines.length === 0) { - return true; - } - for (const line of lines) { - let entry: unknown; - try { - entry = JSON.parse(line) as unknown; - } catch { - return false; - } - if (isTranscriptMessageRecord(entry)) { - return false; - } - } - return true; -} - -function transcriptExistsWithMessages(transcriptPath: string): boolean { - return fs.existsSync(transcriptPath) && !transcriptHasNoMessageRecords(transcriptPath); -} - -function isStaleGeneratedBaseTranscript(params: { +function sqliteTranscriptHasMessageRecords(params: { sessionId: string; - sessionFile?: string; + sessionKey: string; + storePath: string; }): boolean { - const generatedSessionId = extractGeneratedTranscriptSessionId(params.sessionFile); - if (!generatedSessionId || generatedSessionId === params.sessionId) { + try { + return loadTranscriptEventsSync(params).some(isTranscriptMessageRecord); + } catch { return false; } - return path.basename(params.sessionFile?.trim() ?? "") === `${generatedSessionId}.jsonl`; } /** Resolves the action label for one session key from cleanup key sets. */ export function resolveSessionCleanupAction(params: { key: string; - repairedKeys: Set<string>; missingKeys: Set<string>; modelRunPrunedKeys: Set<string>; staleKeys: Set<string>; @@ -221,9 +194,6 @@ export function resolveSessionCleanupAction(params: { if (params.budgetEvictedKeys.has(params.key)) { return "evict-budget"; } - if (params.repairedKeys.has(params.key)) { - return "repair-session-file"; - } return "keep"; } @@ -304,19 +274,8 @@ function pruneMissingTranscriptEntries(params: { store: Record<string, SessionEntry>; storePath: string; onPruned?: (key: string, entry: SessionEntry) => void; - onRepaired?: ( - key: string, - entry: SessionEntry, - sessionFile: string, - previousSessionFile: string | undefined, - ) => void; -}): { removed: number; repaired: number } { - const sessionPathOpts = resolveSessionFilePathOptions({ - storePath: params.storePath, - }); - const sessionsDir = path.dirname(params.storePath); +}): number { let removed = 0; - let repaired = 0; for (const [key, entry] of Object.entries(params.store)) { // `--fix-missing` is explicit repair for ordinary rows, but it cannot // release a harness ownership lock. Header-only supervised transcripts are @@ -324,6 +283,9 @@ function pruneMissingTranscriptEntries(params: { if (entry?.modelSelectionLocked === true && shouldPreserveMaintenanceEntry({ key, entry })) { continue; } + if (parseAgentSessionKey(key) && entry.sessionId === key && !entry.sessionFile) { + continue; + } if (!entry?.sessionId) { if (parseAgentSessionKey(key)) { // Agent-scoped keys without session ids are valid routing entries; keep them. @@ -334,47 +296,19 @@ function pruneMissingTranscriptEntries(params: { params.onPruned?.(key, entry); continue; } - let canonicalTranscriptPath: string | undefined; - try { - canonicalTranscriptPath = resolveSessionTranscriptPathInDir(entry.sessionId, sessionsDir); - } catch { - // Malformed legacy rows cannot resolve a transcript path; --fix-missing prunes them. - } - let transcriptPath: string | undefined; - try { - transcriptPath = resolveSessionFilePath(entry.sessionId, entry, sessionPathOpts); - } catch { - // Malformed sessionFile metadata can still be repaired via the canonical path below. - } if ( - isStaleGeneratedBaseTranscript({ + !sqliteTranscriptHasMessageRecords({ sessionId: entry.sessionId, - sessionFile: entry.sessionFile, - }) && - canonicalTranscriptPath && - canonicalTranscriptPath !== transcriptPath && - transcriptExistsWithMessages(canonicalTranscriptPath) - ) { - const previousSessionFile = entry.sessionFile; - entry.sessionFile = canonicalTranscriptPath; - repaired += 1; - params.onRepaired?.(key, entry, canonicalTranscriptPath, previousSessionFile); - continue; - } - if (transcriptPath && transcriptExistsWithMessages(transcriptPath)) { - continue; - } - if ( - !transcriptPath || - !fs.existsSync(transcriptPath) || - transcriptHasNoMessageRecords(transcriptPath) + sessionKey: key, + storePath: params.storePath, + }) ) { delete params.store[key]; removed += 1; params.onPruned?.(key, entry); } } - return { removed, repaired }; + return removed; } function addEntryArtifactPathsToSet(params: { @@ -408,16 +342,17 @@ async function previewStoreCleanup(params: { fixMissing?: boolean; fixDmScope?: boolean; }) { - const beforeStore = loadSessionStore(params.target.storePath, { skipCache: true }); + const beforeStore = loadCleanupSessionStore(params.target, { + createIfMissing: !params.dryRun, + }); // Preview always mutates a clone so dry-run output can report exact counts without touching disk. const previewStore = cloneSessionStoreRecord(beforeStore); const staleKeys = new Set<string>(); const cappedKeys = new Set<string>(); const missingKeys = new Set<string>(); - const repairedKeys = new Set<string>(); const modelRunPrunedKeys = new Set<string>(); const dmScopeRetiredKeys = new Set<string>(); - const missingResult = + const missing = params.fixMissing === true ? pruneMissingTranscriptEntries({ store: previewStore, @@ -425,12 +360,8 @@ async function previewStoreCleanup(params: { onPruned: (key) => { missingKeys.add(key); }, - onRepaired: (key) => { - repairedKeys.add(key); - }, }) - : { removed: 0, repaired: 0 }; - const missing = missingResult.removed; + : 0; const dmScopeRetired = params.fixDmScope === true ? retireMainScopeDirectSessionEntries({ @@ -503,49 +434,29 @@ async function previewStoreCleanup(params: { storePath: params.target.storePath, keys: dmScopeRetiredKeys, }); - const beforeBudgetStore = cloneSessionStoreRecord(previewStore); - const budgetRemovedFilePaths = new Set<string>(); - const diskBudget = await enforceSessionDiskBudget({ - store: previewStore, - storePath: params.target.storePath, - activeSessionKey: params.activeKey, - preserveKeys: preserveSessionKeys, - maintenance: params.maintenance, - warnOnly: false, - dryRun: true, - onRemoveFile: (canonicalPath) => { - budgetRemovedFilePaths.add(canonicalPath); - }, - }); + const diskBudgetPreview = fs.existsSync(resolveCleanupSqlitePath(params.target)) + ? previewSessionDiskBudget({ + agentId: params.target.agentId, + store: previewStore, + storePath: params.target.storePath, + activeSessionKey: params.activeKey, + preserveKeys: preserveSessionKeys, + maintenance: params.maintenance, + }) + : { diskBudget: null, removedKeys: new Set<string>() }; + const diskBudget = diskBudgetPreview.diskBudget; const unreferencedArtifacts = await pruneUnreferencedSessionArtifacts({ store: previewStore, storePath: params.target.storePath, olderThanMs: params.maintenance.pruneAfterMs, dryRun: true, - excludeCanonicalPaths: new Set([...budgetRemovedFilePaths, ...entryCleanupArtifactPaths]), + excludeCanonicalPaths: entryCleanupArtifactPaths, }); - const budgetEvictedKeys = new Set<string>(); - for (const key of Object.keys(beforeBudgetStore)) { - if (!Object.hasOwn(previewStore, key)) { - budgetEvictedKeys.add(key); - } - } - for (const removedKey of [ - ...missingKeys, - ...modelRunPrunedKeys, - ...staleKeys, - ...cappedKeys, - ...budgetEvictedKeys, - ...dmScopeRetiredKeys, - ]) { - repairedKeys.delete(removedKey); - } - const repaired = repairedKeys.size; + const budgetEvictedKeys = diskBudgetPreview.removedKeys; const beforeCount = Object.keys(beforeStore).length; const afterPreviewCount = Object.keys(previewStore).length; const wouldMutate = missing > 0 || - repaired > 0 || dmScopeRetired > 0 || modelRunPruned > 0 || pruned > 0 || @@ -561,7 +472,6 @@ async function previewStoreCleanup(params: { dryRun: params.dryRun, beforeCount, afterCount: afterPreviewCount, - repaired, missing, dmScopeRetired, modelRunPruned, @@ -575,7 +485,6 @@ async function previewStoreCleanup(params: { return { summary, beforeStore, - repairedKeys, missingKeys, modelRunPrunedKeys, staleKeys, @@ -620,10 +529,8 @@ export async function runSessionsCleanup(params: { const appliedSummaries: SessionCleanupSummary[] = []; if (!opts.dryRun) { for (const target of targets) { - const applyStore = loadSessionStore(target.storePath, { skipCache: true }); + const applyStore = loadCleanupSessionStore(target, { createIfMissing: true }); const missingRemovals: SessionEntryLifecycleRemoval[] = []; - const missingRepairPlans: Array<{ sessionKey: string; sessionFile: string }> = []; - const missingRepairs: SessionEntryLifecycleUpsert[] = []; const dmScopeRetiredRemovals: SessionEntryLifecycleRemoval[] = []; if (opts.fixMissing) { pruneMissingTranscriptEntries({ @@ -635,24 +542,6 @@ export async function runSessionsCleanup(params: { expectedEntry: cloneSessionStoreRecord({ entry }).entry, }); }, - onRepaired: (sessionKey, entry, sessionFile, previousSessionFile) => { - const expectedSessionId = entry.sessionId; - missingRepairPlans.push({ sessionKey, sessionFile }); - missingRepairs.push({ - sessionKey, - buildEntry: ({ currentEntry }) => { - if ( - !currentEntry || - currentEntry.sessionId !== expectedSessionId || - currentEntry.sessionFile !== previousSessionFile || - !transcriptExistsWithMessages(sessionFile) - ) { - return undefined; - } - return { ...currentEntry, sessionFile }; - }, - }); - }, }); } if (opts.fixDmScope) { @@ -677,29 +566,25 @@ export async function runSessionsCleanup(params: { const lifecycleResult = await applySessionEntryLifecycleMutation({ storePath: target.storePath, removals, - upserts: missingRepairs, activeSessionKey: opts.activeKey, preserveActiveWork: true, maintenanceOverride: { + ...maintenance, mode, }, restrictArchivedTranscriptsToStoreDir: true, - pruneUnreferencedArtifacts: - mode === "warn" - ? undefined - : { - olderThanMs: maintenance.pruneAfterMs, - dryRun: false, - }, }); + const postApplyStore = loadCleanupSessionStore(target, { createIfMissing: true }); + const appliedUnreferencedArtifacts = + mode === "warn" + ? null + : await pruneUnreferencedSessionArtifacts({ + store: postApplyStore, + storePath: target.storePath, + olderThanMs: maintenance.pruneAfterMs, + dryRun: false, + }); const removedSessionKeys = new Set(lifecycleResult.removedSessionKeys); - const postApplyStore: Record<string, SessionEntry> = - missingRepairPlans.length > 0 - ? loadSessionStore(target.storePath, { skipCache: true }) - : {}; - const repairedApplied = missingRepairPlans.filter( - ({ sessionKey, sessionFile }) => postApplyStore[sessionKey]?.sessionFile === sessionFile, - ).length; const missingApplied = missingRemovals.filter(({ sessionKey }) => removedSessionKeys.has(sessionKey), ).length; @@ -714,7 +599,8 @@ export async function runSessionsCleanup(params: { freedBytes: 0, olderThanMs: maintenance.pruneAfterMs, } - : (lifecycleResult.unreferencedArtifacts ?? { + : (lifecycleResult.unreferencedArtifacts ?? + appliedUnreferencedArtifacts ?? { scannedFiles: 0, removedFiles: 0, freedBytes: 0, @@ -734,7 +620,6 @@ export async function runSessionsCleanup(params: { dryRun: false, beforeCount: 0, afterCount: 0, - repaired: 0, missing: 0, dmScopeRetired: 0, modelRunPruned: 0, @@ -748,7 +633,6 @@ export async function runSessionsCleanup(params: { unreferencedArtifacts, wouldMutate: (preview?.summary.wouldMutate ?? false) || unreferencedArtifacts.removedFiles > 0, - repaired: repairedApplied, applied: true, appliedCount: lifecycleResult.afterCount, } @@ -759,7 +643,6 @@ export async function runSessionsCleanup(params: { dryRun: false, beforeCount: appliedReport.beforeCount, afterCount: appliedReport.afterCount, - repaired: repairedApplied, missing: missingApplied, dmScopeRetired: dmScopeRetiredApplied, modelRunPruned: appliedReport.modelRunPruned, @@ -769,7 +652,6 @@ export async function runSessionsCleanup(params: { diskBudget: appliedReport.diskBudget, wouldMutate: missingApplied > 0 || - repairedApplied > 0 || dmScopeRetiredApplied > 0 || appliedReport.modelRunPruned > 0 || appliedReport.pruned > 0 || @@ -796,9 +678,9 @@ export async function purgeAgentSessionStoreEntries( const normalizedAgentId = normalizeAgentId(agentId); const storeConfig = cfg.session?.store; const storeAgentId = - typeof storeConfig === "string" && storeConfig.includes("{agentId}") - ? normalizedAgentId - : normalizeAgentId(resolveDefaultAgentId(cfg)); + typeof storeConfig === "string" && !storeConfig.includes("{agentId}") + ? normalizeAgentId(resolveDefaultAgentId(cfg)) + : normalizedAgentId; const storePath = resolveStorePath(cfg.session?.store, { agentId: normalizedAgentId }); await purgeDeletedAgentSessionEntries({ cfg, diff --git a/src/config/sessions/combined-store-gateway.ts b/src/config/sessions/combined-store-gateway.ts index 2f2ca678fb15..56b314b3801b 100644 --- a/src/config/sessions/combined-store-gateway.ts +++ b/src/config/sessions/combined-store-gateway.ts @@ -22,12 +22,16 @@ function isStorePathTemplate(store?: string): boolean { return typeof store === "string" && store.includes("{agentId}"); } -function loadGatewayStoreEntries(storePath: string): Record<string, SessionEntry> { +function loadGatewayStoreEntries(params: { + agentId: string; + storePath: string; +}): Record<string, SessionEntry> { return Object.fromEntries( - listSessionEntries({ clone: false, storePath }).map(({ sessionKey, entry }) => [ - sessionKey, - entry, - ]), + listSessionEntries({ + agentId: params.agentId, + clone: false, + storePath: params.storePath, + }).map(({ sessionKey, entry }) => [sessionKey, entry]), ); } @@ -85,7 +89,7 @@ export function loadCombinedSessionStoreForGateway( // A single shared store still needs keys canonicalized as if owned by the default agent. const storePath = resolveStorePath(storeConfig); const defaultAgentId = normalizeAgentId(resolveDefaultAgentId(cfg)); - const store = loadGatewayStoreEntries(storePath); + const store = loadGatewayStoreEntries({ agentId: defaultAgentId, storePath }); const combined: Record<string, SessionEntry> = {}; for (const [key, entry] of Object.entries(store)) { const canonicalKey = resolveStoredSessionKeyForAgentStore({ @@ -117,7 +121,7 @@ export function loadCombinedSessionStoreForGateway( for (const target of targets) { const agentId = target.agentId; const storePath = target.storePath; - const store = loadGatewayStoreEntries(storePath); + const store = loadGatewayStoreEntries({ agentId, storePath }); for (const [key, entry] of Object.entries(store)) { const canonicalKey = resolveStoredSessionKeyForAgentStore({ cfg, diff --git a/src/config/sessions/goals.test.ts b/src/config/sessions/goals.test.ts index f694cf49f4fe..9a9dd9ebcc31 100644 --- a/src/config/sessions/goals.test.ts +++ b/src/config/sessions/goals.test.ts @@ -9,8 +9,33 @@ import { updateSessionGoalObjective, updateSessionGoalStatus, } from "./goals.js"; -import { getSessionEntry, upsertSessionEntry } from "./store.js"; +import { + loadSessionEntry, + upsertSessionEntry as upsertAccessorSessionEntry, +} from "./session-accessor.js"; import { useTempSessionsFixture } from "./test-helpers.js"; +import type { SessionEntry } from "./types.js"; + +// The goal APIs read/write session entries through the SQLite-backed accessor, +// so fixtures must seed and assert through the same accessor rather than the +// file-backed store helpers. +function getSessionEntry(params: { + storePath: string; + sessionKey: string; +}): SessionEntry | undefined { + return loadSessionEntry(params); +} + +async function upsertSessionEntry(params: { + storePath: string; + sessionKey: string; + entry: SessionEntry; +}): Promise<void> { + await upsertAccessorSessionEntry( + { sessionKey: params.sessionKey, storePath: params.storePath }, + params.entry, + ); +} describe("session goals", () => { const fixture = useTempSessionsFixture("openclaw-session-goals-"); diff --git a/src/config/sessions/internal-session-key.ts b/src/config/sessions/internal-session-key.ts new file mode 100644 index 000000000000..7c3dd68f1472 --- /dev/null +++ b/src/config/sessions/internal-session-key.ts @@ -0,0 +1,28 @@ +import crypto from "node:crypto"; +import { normalizeAgentId } from "../../routing/session-key.js"; + +const INTERNAL_SESSION_EFFECTS_SEGMENT = "internal-session-effects"; + +function normalizeInternalRunId(runId: string): string { + const readable = runId.replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 48) || "run"; + const digest = crypto.createHash("sha256").update(runId).digest("hex").slice(0, 16); + return `${readable}-${digest}`; +} + +/** Resolves the hidden SQLite session identity owned by one internal-effects run. */ +export function resolveInternalSessionEffectsIdentity(params: { agentId: string; runId: string }): { + sessionId: string; + sessionKey: string; +} { + const suffix = normalizeInternalRunId(params.runId); + return { + sessionId: `${INTERNAL_SESSION_EFFECTS_SEGMENT}-${suffix}`, + sessionKey: `agent:${normalizeAgentId(params.agentId)}:${INTERNAL_SESSION_EFFECTS_SEGMENT}:${suffix}`, + }; +} + +/** Returns true for SQLite entries that exist only to contain suppressed run effects. */ +export function isInternalSessionEffectsKey(sessionKey: string): boolean { + const parts = sessionKey.split(":"); + return parts.length >= 4 && parts[0] === "agent" && parts[2] === INTERNAL_SESSION_EFFECTS_SEGMENT; +} diff --git a/src/config/sessions/metadata.ts b/src/config/sessions/metadata.ts index 80cb6a057223..f7a4fbb2eafd 100644 --- a/src/config/sessions/metadata.ts +++ b/src/config/sessions/metadata.ts @@ -7,6 +7,14 @@ import type { MsgContext } from "../../auto-reply/templating.js"; import { normalizeChatType } from "../../channels/chat-type.js"; import { resolveConversationLabel } from "../../channels/conversation-label.js"; import { getLoadedChannelPlugin, normalizeChannelId } from "../../channels/plugins/index.js"; +import { + deliveryContextFromChannelRoute, + deliveryContextFromSession, + mergeDeliveryContext, + normalizeDeliveryContext, + normalizeSessionDeliveryFields, +} from "../../utils/delivery-context.shared.js"; +import type { DeliveryContext } from "../../utils/delivery-context.types.js"; import { INTERNAL_MESSAGE_CHANNEL, isInternalNonDeliveryChannel, @@ -236,3 +244,92 @@ export function deriveSessionMetaPatch(params: { return Object.keys(patch).length > 0 ? patch : null; } + +function removeThreadFromDeliveryContext(context?: DeliveryContext): DeliveryContext | undefined { + if (!context || context.threadId == null) { + return context; + } + const next: DeliveryContext = { ...context }; + delete next.threadId; + return next; +} + +/** + * Derives the last-route/delivery patch for an inbound routing update. Route + * updates must not refresh activity timestamps; idle/daily reset evaluation + * relies on updatedAt from actual session turns (#49515). Shared by the file + * store and the SQLite accessor so both backends apply one routing policy. + */ +export function deriveLastRoutePatch(params: { + channel?: SessionEntry["lastChannel"]; + to?: string; + accountId?: string; + threadId?: string | number; + route?: SessionEntry["route"]; + deliveryContext?: DeliveryContext; + ctx?: MsgContext; + groupResolution?: GroupKeyResolution | null; + existing: SessionEntry | undefined; + sessionKey: string; +}): Partial<SessionEntry> { + const { channel, to, accountId, threadId, ctx, existing } = params; + const explicitContext = normalizeDeliveryContext(params.deliveryContext); + const inlineContext = normalizeDeliveryContext({ + channel, + to, + accountId, + threadId, + }); + const routeContext = deliveryContextFromChannelRoute(params.route); + const mergedInput = mergeDeliveryContext( + routeContext, + mergeDeliveryContext(explicitContext, inlineContext), + ); + const explicitDeliveryContext = params.deliveryContext; + const explicitThreadFromDeliveryContext = + explicitDeliveryContext != null && Object.hasOwn(explicitDeliveryContext, "threadId") + ? explicitDeliveryContext.threadId + : undefined; + const explicitThreadValue = + explicitThreadFromDeliveryContext ?? + (threadId != null && threadId !== "" ? threadId : undefined); + const explicitRouteProvided = Boolean( + routeContext?.channel || + routeContext?.to || + explicitContext?.channel || + explicitContext?.to || + inlineContext?.channel || + inlineContext?.to, + ); + const clearThreadFromFallback = explicitRouteProvided && explicitThreadValue == null; + const fallbackContext = clearThreadFromFallback + ? removeThreadFromDeliveryContext(deliveryContextFromSession(existing)) + : deliveryContextFromSession(existing); + const merged = mergeDeliveryContext(mergedInput, fallbackContext); + const normalized = normalizeSessionDeliveryFields({ + route: params.route, + deliveryContext: { + channel: merged?.channel, + to: merged?.to, + accountId: merged?.accountId, + threadId: merged?.threadId, + }, + }); + const metaPatch = ctx + ? deriveSessionMetaPatch({ + ctx, + sessionKey: params.sessionKey, + existing, + groupResolution: params.groupResolution, + }) + : null; + const basePatch: Partial<SessionEntry> = { + route: normalized.route, + deliveryContext: normalized.deliveryContext, + lastChannel: normalized.lastChannel, + lastTo: normalized.lastTo, + lastAccountId: normalized.lastAccountId, + lastThreadId: normalized.lastThreadId, + }; + return metaPatch ? { ...basePatch, ...metaPatch } : basePatch; +} diff --git a/src/config/sessions/paths.sqlite-marker.test.ts b/src/config/sessions/paths.sqlite-marker.test.ts new file mode 100644 index 000000000000..1e2205922e30 --- /dev/null +++ b/src/config/sessions/paths.sqlite-marker.test.ts @@ -0,0 +1,76 @@ +// SQLite transcript markers are storage targets, not filesystem paths. +import { describe, expect, it } from "vitest"; +import { resolveSessionFilePath } from "./paths.js"; +import { loadSessionEntry, upsertSessionEntry } from "./session-accessor.js"; +import { resolveAndPersistSessionFile } from "./session-file.js"; +import { useTempSessionsFixture } from "./test-helpers.js"; + +describe("SQLite sessionFile markers", () => { + const fixture = useTempSessionsFixture("sqlite-session-file-marker-"); + + it("preserves SQLite markers for transcript target resolution", () => { + const marker = "sqlite:main:sess-1:/tmp/openclaw/agents/main/agent/openclaw-agent.sqlite"; + const sessionsDir = "/tmp/openclaw/agents/main/sessions"; + + const resolved = resolveSessionFilePath("sess-1", { sessionFile: marker }, { sessionsDir }); + + expect(resolved).toBe(marker); + }); + + it("does not downgrade matching SQLite markers when resolving runtime session files", async () => { + const sessionId = "sqlite-session-id"; + const sessionKey = "agent:main:telegram:group:123"; + const marker = `sqlite:main:${sessionId}:${fixture.storePath()}`; + const store = { + [sessionKey]: { + sessionId, + updatedAt: Date.now(), + sessionFile: marker, + }, + }; + await upsertSessionEntry({ storePath: fixture.storePath(), sessionKey }, store[sessionKey]); + + const result = await resolveAndPersistSessionFile({ + sessionId, + sessionKey, + sessionStore: store, + storePath: fixture.storePath(), + sessionEntry: store[sessionKey], + }); + + expect(result.sessionFile).toBe(marker); + expect(result.sessionEntry.sessionFile).toBe(marker); + + expect(loadSessionEntry({ storePath: fixture.storePath(), sessionKey })?.sessionFile).toBe( + marker, + ); + }); + + it("does not preserve persisted markers for a different session", async () => { + const sessionId = "current-sqlite-session-id"; + const sessionKey = "agent:main:telegram:group:456"; + const staleMarker = `sqlite:main:old-sqlite-session-id:${fixture.storePath()}`; + const store = { + [sessionKey]: { + sessionId, + updatedAt: Date.now(), + sessionFile: staleMarker, + }, + }; + + const result = await resolveAndPersistSessionFile({ + sessionId, + sessionKey, + sessionStore: store, + storePath: fixture.storePath(), + sessionEntry: store[sessionKey], + }); + + const expectedSessionFile = `sqlite:main:${sessionId}:${fixture.storePath()}`; + expect(result.sessionFile).toBe(expectedSessionFile); + expect(result.sessionEntry.sessionFile).toBe(result.sessionFile); + expect(loadSessionEntry({ storePath: fixture.storePath(), sessionKey })?.sessionFile).toBe( + expectedSessionFile, + ); + }); +}); diff --git a/src/config/sessions/paths.test.ts b/src/config/sessions/paths.test.ts index 6bad7a4aa2f2..0a1dac9b8270 100644 --- a/src/config/sessions/paths.test.ts +++ b/src/config/sessions/paths.test.ts @@ -1,7 +1,54 @@ // Session path helper tests pin default store path contracts used by CLI commands. +import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; -import { resolveStorePath } from "./paths.js"; +import { afterEach, describe, expect, it } from "vitest"; +import { resolveSessionFilePath, resolveStorePath } from "./paths.js"; + +const tempDirs: string[] = []; + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("resolveSessionFilePath cross-root reroot", () => { + it("re-roots foreign-root absolute paths when the file exists in the current sessions dir", () => { + // Restored backups and moved state dirs persist absolute sessionFile + // paths from the old root; migration must find the local copy. + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reroot-"))); + tempDirs.push(root); + const sessionsDir = path.join(root, "agents", "main", "sessions"); + fs.mkdirSync(sessionsDir, { recursive: true }); + fs.writeFileSync(path.join(sessionsDir, "sess-1.jsonl"), "{}\n", "utf8"); + const foreign = "/nonexistent-old-root/.openclaw/agents/main/sessions/sess-1.jsonl"; + + const resolved = resolveSessionFilePath( + "sess-1", + { sessionFile: foreign }, + { sessionsDir, agentId: "main" }, + ); + + expect(resolved).toBe(path.join(sessionsDir, "sess-1.jsonl")); + }); + + it("keeps foreign-root absolute paths when no local copy exists", () => { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reroot-keep-"))); + tempDirs.push(root); + const sessionsDir = path.join(root, "agents", "main", "sessions"); + fs.mkdirSync(sessionsDir, { recursive: true }); + const foreign = "/nonexistent-old-root/.openclaw/agents/main/sessions/sess-2.jsonl"; + + const resolved = resolveSessionFilePath( + "sess-2", + { sessionFile: foreign }, + { sessionsDir, agentId: "main" }, + ); + + expect(resolved).toBe(foreign); + }); +}); describe("resolveStorePath", () => { it("uses the default agent store when session.store is absent or blank", () => { diff --git a/src/config/sessions/paths.ts b/src/config/sessions/paths.ts index f7aa531e1716..22a2dd014232 100644 --- a/src/config/sessions/paths.ts +++ b/src/config/sessions/paths.ts @@ -43,6 +43,7 @@ export type SessionFilePathOptions = { }; const MULTI_STORE_PATH_SENTINEL = "(multiple)"; +const SQLITE_TRANSCRIPT_TARGET_PREFIX = "sqlite:"; export function resolveSessionFilePathOptions(params: { agentId?: string; @@ -91,11 +92,37 @@ function resolvePathFromAgentSessionsDir( const relative = path.relative(agentBase, realCandidate); // Realpath both sides when possible so symlinked session dirs still enforce containment. if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) { - return undefined; + return resolveRerootedSessionPath(agentBase, candidateAbsPath); } return path.resolve(agentBase, relative); } +// Absolute sessionFile paths recorded under another state root (restored +// backups, moved OPENCLAW_STATE_DIR, rehearsal copies) never satisfy the +// relative-containment check above. Re-root the canonical +// `agents/<id>/sessions/<suffix>` tail onto the current sessions dir, but only +// when the file exists there: genuine cross-root layouts keep their foreign +// path via the structural fallback. +function resolveRerootedSessionPath( + agentSessionsDir: string, + candidateAbsPath: string, +): string | undefined { + const parsed = resolveAgentSessionsPathParts(candidateAbsPath); + if (!parsed) { + return undefined; + } + const relativeSegments = parsed.parts.slice(parsed.sessionsIndex + 1); + if (relativeSegments.length === 0) { + return undefined; + } + const rerooted = path.resolve(agentSessionsDir, ...relativeSegments); + const contained = path.relative(agentSessionsDir, rerooted); + if (!contained || contained.startsWith("..") || path.isAbsolute(contained)) { + return undefined; + } + return fs.existsSync(rerooted) ? rerooted : undefined; +} + function resolveSiblingAgentSessionsDir( baseSessionsDir: string, agentId: string, @@ -285,6 +312,9 @@ export function resolveSessionFilePath( const sessionsDir = resolveSessionsDir(opts); const candidate = entry?.sessionFile?.trim(); if (candidate) { + if (candidate.startsWith(SQLITE_TRANSCRIPT_TARGET_PREFIX)) { + return candidate; + } try { return resolvePathWithinSessionsDir(sessionsDir, candidate, { agentId: opts?.agentId }); } catch { diff --git a/src/config/sessions/plugin-host-cleanup.ts b/src/config/sessions/plugin-host-cleanup.ts index 6bbcdd992ef6..450465ee8e1e 100644 --- a/src/config/sessions/plugin-host-cleanup.ts +++ b/src/config/sessions/plugin-host-cleanup.ts @@ -1,14 +1,15 @@ -/** File-backed implementation for plugin host-owned session-state cleanup. */ +/** Shared predicates and mutations for plugin host-owned session-state cleanup. */ import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; import { normalizeOptionalAgentRuntimeId } from "../../agents/agent-runtime-id.js"; import { normalizeSessionEntrySlotKey } from "../../plugins/session-entry-slot-keys.js"; -import { updateSessionStore } from "./store.js"; import type { SessionEntry } from "./types.js"; /** Cleanup variants owned by plugin host lifecycle paths. */ export type PluginHostSessionCleanupMode = "plugin-owned-state" | "promoted-slots"; export type PluginHostSessionCleanupStoreParams = { + /** Agent that owns the resolved session store target. */ + agentId?: string; /** Cleanup mode chosen by the plugin host lifecycle reason. */ mode: PluginHostSessionCleanupMode; /** Plugin owner to clear. Omit only for session-scoped all-plugin cleanup. */ @@ -17,10 +18,10 @@ export type PluginHostSessionCleanupStoreParams = { sessionKey?: string; /** Promoted SessionEntry slots declared by the plugin registry. */ sessionEntrySlotKeys?: ReadonlySet<string>; + /** Locked-harness ids whose sessions this cleanup must leave untouched. */ + preserveLockedHarnessIds?: ReadonlySet<string>; /** Per-store file-backed transaction boundary. */ storePath: string; - /** Harness ids whose locked durable sessions survive plugin disable cleanup. */ - preserveLockedHarnessIds?: ReadonlySet<string>; /** Cancels the cleanup before persistence when host lifecycle state changes. */ shouldCleanup?: () => boolean; }; @@ -180,7 +181,7 @@ function hasPluginOwnedSessionState( ); } -function matchesCleanupSession( +export function matchesPluginHostCleanupSession( entryKey: string, entry: SessionEntry, sessionKey?: string, @@ -195,14 +196,16 @@ function matchesCleanupSession( ); } -function shouldSkipCleanupStore(params: PluginHostSessionCleanupStoreParams): boolean { +export function shouldSkipPluginHostCleanupStore( + params: PluginHostSessionCleanupStoreParams, +): boolean { if (!params.pluginId && !params.sessionKey) { return true; } return params.mode === "promoted-slots" && (params.sessionEntrySlotKeys?.size ?? 0) === 0; } -function hasCleanupTarget( +export function hasPluginHostCleanupTarget( entry: SessionEntry, params: PluginHostSessionCleanupStoreParams, ): boolean { @@ -212,7 +215,7 @@ function hasCleanupTarget( return hasPluginOwnedSessionState(entry, params.pluginId, params.sessionEntrySlotKeys); } -function isLockedHarnessSessionOwnedByPlugin( +export function isLockedHarnessSessionOwnedByPlugin( entry: SessionEntry, preserveLockedHarnessIds: ReadonlySet<string> | undefined, ): boolean { @@ -223,7 +226,7 @@ function isLockedHarnessSessionOwnedByPlugin( return harnessId !== undefined && preserveLockedHarnessIds.has(harnessId); } -function clearCleanupTarget( +export function clearPluginHostCleanupTarget( entry: SessionEntry, params: PluginHostSessionCleanupStoreParams, ): void { @@ -236,44 +239,3 @@ function clearCleanupTarget( } clearPluginOwnedSessionState(entry, params.pluginId, params.sessionEntrySlotKeys); } - -/** Clears plugin host-owned session state in one store transaction. */ -export async function cleanupPluginHostSessionStore( - params: PluginHostSessionCleanupStoreParams, -): Promise<number> { - if (shouldSkipCleanupStore(params) || (params.shouldCleanup && !params.shouldCleanup())) { - return 0; - } - return await updateSessionStore( - params.storePath, - (store) => { - if (params.shouldCleanup && !params.shouldCleanup()) { - return 0; - } - let clearedInStore = 0; - const now = Date.now(); - for (const [entryKey, entry] of Object.entries(store)) { - // Disabling an owning harness stops execution; it does not release its - // durable conversation. Keep the plugin marker so re-enable can recover - // the same binding instead of leaving a locked, undiscoverable session. - if (isLockedHarnessSessionOwnedByPlugin(entry, params.preserveLockedHarnessIds)) { - continue; - } - if ( - !matchesCleanupSession(entryKey, entry, params.sessionKey) || - !hasCleanupTarget(entry, params) - ) { - continue; - } - clearCleanupTarget(entry, params); - entry.updatedAt = now; - clearedInStore += 1; - } - return clearedInStore; - }, - { - skipSaveWhenResult: (clearedInStore) => clearedInStore === 0, - takeCacheOwnership: true, - }, - ); -} diff --git a/src/config/sessions/session-accessor.conformance.test.ts b/src/config/sessions/session-accessor.conformance.test.ts new file mode 100644 index 000000000000..1e82af5288f2 --- /dev/null +++ b/src/config/sessions/session-accessor.conformance.test.ts @@ -0,0 +1,2172 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + readPersistedAuthProfileStateRaw, + writePersistedAuthProfileStateRaw, +} from "../../agents/auth-profiles/sqlite.js"; +import { executeSqliteQueryTakeFirstSync, getNodeSqliteKysely } from "../../infra/kysely-sync.js"; +import { onSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; +import type { DB as OpenClawAgentKyselyDatabase } from "../../state/openclaw-agent-db.generated.js"; +import { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, +} from "../../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; +import { appendSqliteTrajectoryRuntimeEvents } from "../../trajectory/runtime-store.sqlite.js"; +import { readSessionArchiveContentSync } from "./archive-compression.js"; +import { + appendTranscriptEvent, + appendTranscriptMessage, + cleanupSessionLifecycleArtifacts, + listSessionEntries, + loadExactSessionEntry, + loadSessionEntry, + loadTranscriptEvents, + patchSessionEntry, + publishTranscriptUpdate, + readSessionUpdatedAt, + replaceSessionEntry, + resolveSessionTranscriptRuntimeTarget, + updateSessionEntry, + upsertSessionEntry, + type ExactSessionEntry, + type SessionAccessScope, + type SessionEntrySummary, + type SessionTranscriptAccessScope, + type SessionTranscriptReadScope, + type SessionTranscriptWriteScope, + type TranscriptEvent, + type TranscriptMessageAppendOptions, + type TranscriptMessageAppendResult, + type TranscriptUpdatePayload, +} from "./session-accessor.js"; +import { + appendSqliteTranscriptEvent, + appendSqliteTranscriptEvents, + appendSqliteTranscriptMessage, + branchSqliteCompactionCheckpointSession, + cleanupSqliteSessionLifecycleArtifacts, + deleteSqliteTranscript, + forkSqliteSessionEntryFromParentTarget, + listSqliteSessionEntries, + loadExactSqliteSessionEntry, + loadSqliteSessionEntry, + loadSqliteTranscriptEvents, + loadSqliteTranscriptEventsSync, + patchSqliteSessionEntry, + publishSqliteTranscriptUpdate, + readSqliteSessionUpdatedAt, + replaceSqliteSessionEntry, + replaceSqliteSessionEntrySync, + replaceSqliteTranscriptEvents, + restoreSqliteCompactionCheckpointSession, + sqliteTranscriptExists, + updateSqliteSessionEntry, + upsertSqliteSessionEntry, +} from "./session-accessor.sqlite.js"; +import { parseSqliteSessionFileMarker } from "./sqlite-marker.js"; +import type { SessionCompactionCheckpoint, SessionEntry } from "./types.js"; + +// Keep accessor conformance independent of any real openclaw.json on the machine. +vi.mock("../config.js", async () => ({ + ...(await vi.importActual<typeof import("../config.js")>("../config.js")), + getRuntimeConfig: vi.fn().mockReturnValue({}), +})); + +import { getRuntimeConfig } from "../config.js"; + +type AccessorAdapter = { + name: string; + publishesTranscriptUpdates: boolean; + usesSqliteStore: boolean; + entryScope(paths: TestPaths): SessionAccessScope; + transcriptReadScope(paths: TestPaths, id?: string): SessionTranscriptReadScope; + transcriptScope(paths: TestPaths, id?: string): SessionTranscriptAccessScope; + loadExactSessionEntry(scope: SessionAccessScope): ExactSessionEntry | undefined; + loadSessionEntry(scope: SessionAccessScope): SessionEntry | undefined; + listSessionEntries(scope: Partial<Omit<SessionAccessScope, "sessionKey">>): SessionEntrySummary[]; + readSessionUpdatedAt(scope: SessionAccessScope): number | undefined; + upsertSessionEntry( + scope: SessionAccessScope, + patch: Partial<SessionEntry>, + ): Promise<SessionEntry | null>; + replaceSessionEntry(scope: SessionAccessScope, entry: SessionEntry): Promise<SessionEntry | null>; + patchSessionEntry( + scope: SessionAccessScope, + update: ( + entry: SessionEntry, + context: { existingEntry?: SessionEntry }, + ) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null, + options?: { fallbackEntry?: SessionEntry; preserveActivity?: boolean; replaceEntry?: boolean }, + ): Promise<SessionEntry | null>; + updateSessionEntry( + scope: SessionAccessScope, + update: (entry: SessionEntry) => Partial<SessionEntry> | null, + ): Promise<SessionEntry | null>; + cleanupSessionLifecycleArtifacts(params: { + storePath: string; + sessionKeySegmentPrefix: string; + transcriptContentMarker: string; + orphanTranscriptMinAgeMs: number; + nowMs?: number; + }): Promise<{ removedEntries: number; archivedTranscriptArtifacts: number }>; + loadTranscriptEvents(scope: SessionTranscriptReadScope): Promise<TranscriptEvent[]>; + appendTranscriptEvent(scope: SessionTranscriptAccessScope, event: TranscriptEvent): Promise<void>; + appendTranscriptMessage<TMessage>( + scope: SessionTranscriptWriteScope, + options: TranscriptMessageAppendOptions<TMessage>, + ): Promise<TranscriptMessageAppendResult<TMessage> | undefined>; + publishTranscriptUpdate( + scope: SessionTranscriptWriteScope, + update?: TranscriptUpdatePayload, + ): Promise<void>; +}; + +type TestPaths = { + sqlitePath: string; + stateDir: string; + storePath: string; + tempDir: string; + transcriptPath: string; +}; + +const publicAccessorAdapter: AccessorAdapter = { + name: "public-accessor", + publishesTranscriptUpdates: true, + usesSqliteStore: true, + entryScope: (paths) => ({ + agentId: "main", + env: { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }, + sessionKey: "agent:main:main", + storePath: paths.sqlitePath, + }), + transcriptScope: (paths, id = "session-1") => ({ + agentId: "main", + env: { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }, + sessionId: id, + sessionKey: "agent:main:main", + storePath: paths.sqlitePath, + }), + transcriptReadScope: (paths, id = "session-1") => ({ + agentId: "main", + env: { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }, + sessionId: id, + storePath: paths.sqlitePath, + }), + loadSessionEntry, + loadExactSessionEntry, + listSessionEntries, + readSessionUpdatedAt, + upsertSessionEntry, + replaceSessionEntry, + patchSessionEntry, + updateSessionEntry, + cleanupSessionLifecycleArtifacts, + loadTranscriptEvents, + appendTranscriptEvent, + appendTranscriptMessage, + publishTranscriptUpdate, +}; + +const sqliteAdapter: AccessorAdapter = { + name: "sqlite", + publishesTranscriptUpdates: true, + usesSqliteStore: true, + entryScope: (paths) => ({ + agentId: "main", + env: { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }, + sessionKey: "agent:main:main", + storePath: paths.sqlitePath, + }), + transcriptScope: (paths, id = "session-1") => ({ + agentId: "main", + env: { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }, + sessionId: id, + sessionKey: "agent:main:main", + storePath: paths.sqlitePath, + }), + transcriptReadScope: (paths, id = "session-1") => ({ + agentId: "main", + env: { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }, + sessionId: id, + storePath: paths.sqlitePath, + }), + loadSessionEntry: loadSqliteSessionEntry, + loadExactSessionEntry: loadExactSqliteSessionEntry, + listSessionEntries: listSqliteSessionEntries, + readSessionUpdatedAt: readSqliteSessionUpdatedAt, + upsertSessionEntry: upsertSqliteSessionEntry, + replaceSessionEntry: replaceSqliteSessionEntry, + patchSessionEntry: patchSqliteSessionEntry, + updateSessionEntry: updateSqliteSessionEntry, + cleanupSessionLifecycleArtifacts: cleanupSqliteSessionLifecycleArtifacts, + loadTranscriptEvents: loadSqliteTranscriptEvents, + appendTranscriptEvent: appendSqliteTranscriptEvent, + appendTranscriptMessage: appendSqliteTranscriptMessage, + publishTranscriptUpdate: publishSqliteTranscriptUpdate, +}; + +beforeEach(() => { + vi.mocked(getRuntimeConfig).mockReturnValue({}); +}); + +afterEach(() => { + vi.mocked(getRuntimeConfig).mockReset(); +}); + +describe.each([publicAccessorAdapter, sqliteAdapter])( + "session accessor conformance: $name", + (adapter) => { + let paths: TestPaths; + + beforeEach(() => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-session-accessor-conf-")); + paths = { + sqlitePath: path.join(tempDir, "openclaw-agent.sqlite"), + stateDir: path.join(tempDir, "state"), + storePath: path.join(tempDir, "sessions.json"), + tempDir, + transcriptPath: path.join(tempDir, "session.jsonl"), + }; + }); + + afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + fs.rmSync(paths.tempDir, { recursive: true, force: true }); + }); + + it("conforms for entry load/list/timestamp/upsert/update/replace/patch", async () => { + const scope = adapter.entryScope(paths); + + await adapter.upsertSessionEntry(scope, { + model: "gpt-5.5", + sessionId: "session-1", + updatedAt: 10, + }); + + expect(adapter.loadSessionEntry(scope)).toMatchObject({ + model: "gpt-5.5", + sessionId: "session-1", + updatedAt: expect.any(Number), + }); + expect(adapter.readSessionUpdatedAt(scope)).toEqual(expect.any(Number)); + expect(adapter.listSessionEntries(scope)).toEqual([ + { + sessionKey: "agent:main:main", + entry: expect.objectContaining({ + model: "gpt-5.5", + sessionId: "session-1", + }), + }, + ]); + + await expect( + adapter.updateSessionEntry(scope, () => ({ model: "sonnet-4.6", updatedAt: 20 })), + ).resolves.toMatchObject({ + model: "sonnet-4.6", + sessionId: "session-1", + }); + + await adapter.replaceSessionEntry(scope, { + providerOverride: "openai", + sessionId: "session-1", + updatedAt: 30, + }); + + expect(adapter.loadSessionEntry(scope)).toMatchObject({ + providerOverride: "openai", + sessionId: "session-1", + }); + expect(adapter.loadSessionEntry(scope)?.model).toBeUndefined(); + + let existingContext: SessionEntry | undefined; + await adapter.patchSessionEntry( + scope, + (entry, context) => { + existingContext = context.existingEntry; + return { + ...entry, + model: "gpt-5.5", + }; + }, + { replaceEntry: true }, + ); + + expect(existingContext).toMatchObject({ providerOverride: "openai" }); + expect(adapter.loadSessionEntry(scope)).toMatchObject({ + model: "gpt-5.5", + sessionId: "session-1", + }); + + const beforePreservePatch = adapter.loadSessionEntry(scope); + await adapter.patchSessionEntry( + scope, + () => ({ + providerOverride: "anthropic", + updatedAt: 40, + }), + { preserveActivity: true }, + ); + + expect(adapter.loadSessionEntry(scope)).toMatchObject({ + model: "gpt-5.5", + providerOverride: "anthropic", + sessionId: "session-1", + updatedAt: beforePreservePatch?.updatedAt, + }); + }); + + it("conforms for exact persisted-key lookup without canonical alias fallback", async () => { + const scope = adapter.entryScope(paths); + const mixedCaseScope = { ...scope, sessionKey: "AGENT:MAIN:MAIN" }; + + await adapter.upsertSessionEntry(scope, { + model: "gpt-5.5", + sessionId: "exact-session", + updatedAt: 10, + }); + + expect(adapter.loadSessionEntry(mixedCaseScope)).toMatchObject({ + model: "gpt-5.5", + sessionId: "exact-session", + }); + expect(adapter.loadExactSessionEntry(mixedCaseScope)).toBeUndefined(); + expect(adapter.loadExactSessionEntry(scope)).toEqual({ + sessionKey: "agent:main:main", + entry: expect.objectContaining({ + model: "gpt-5.5", + sessionId: "exact-session", + }), + }); + }); + + it("conforms for lifecycle entry and transcript cleanup", async () => { + const nowMs = Date.now(); + const oldTimestamp = nowMs - 600_000; + const usesSqliteStore = adapter.usesSqliteStore; + const cleanupStorePath = usesSqliteStore + ? path.join(paths.stateDir, "agents", "main", "sessions", "sessions.json") + : paths.storePath; + const scopedEntry = (sessionKey: string): SessionAccessScope => ({ + ...adapter.entryScope(paths), + sessionKey, + storePath: cleanupStorePath, + }); + const scopedTranscript = ( + sessionKey: string, + sessionId: string, + ): SessionTranscriptAccessScope => ({ + ...adapter.transcriptScope(paths, sessionId), + sessionKey, + storePath: cleanupStorePath, + }); + const writeTranscript = async (params: { + sessionKey: string; + sessionId: string; + old?: boolean; + }) => { + const timestamp = params.old ? oldTimestamp : nowMs; + const event = { + id: `${params.sessionId}-event`, + marker: "lifecycle-marker-run", + timestamp: new Date(timestamp).toISOString(), + type: "metadata", + }; + if (usesSqliteStore) { + await adapter.appendTranscriptEvent( + scopedTranscript(params.sessionKey, params.sessionId), + event, + ); + return; + } + const transcriptPath = path.join( + path.dirname(cleanupStorePath), + `${params.sessionId}.jsonl`, + ); + fs.writeFileSync(transcriptPath, `${JSON.stringify(event)}\n`, "utf-8"); + if (params.old) { + const oldDate = new Date(oldTimestamp); + fs.utimesSync(transcriptPath, oldDate, oldDate); + } + }; + + await adapter.upsertSessionEntry(scopedEntry("agent:main:lifecycle-cleanup-missing"), { + sessionId: "missing-lifecycle", + updatedAt: oldTimestamp, + }); + await adapter.upsertSessionEntry(scopedEntry("agent:main:lifecycle-cleanup-removed"), { + sessionId: "removed-lifecycle", + updatedAt: oldTimestamp, + }); + await adapter.upsertSessionEntry(scopedEntry("agent:main:lifecycle-cleanup-fresh"), { + sessionId: "fresh-lifecycle", + updatedAt: nowMs, + }); + await adapter.upsertSessionEntry( + scopedEntry("agent:main:telegram:group:lifecycle-cleanup-room"), + { + sessionId: "kept-by-segment", + updatedAt: oldTimestamp, + }, + ); + await adapter.upsertSessionEntry(scopedEntry("agent:main:regular"), { + sessionId: "referenced", + updatedAt: oldTimestamp, + }); + await writeTranscript({ + sessionKey: "agent:main:lifecycle-cleanup-removed", + sessionId: "removed-lifecycle", + old: true, + }); + await writeTranscript({ + sessionKey: "agent:main:lifecycle-cleanup-fresh", + sessionId: "fresh-lifecycle", + }); + await writeTranscript({ + sessionKey: "agent:main:regular", + sessionId: "referenced", + old: true, + }); + await writeTranscript({ + sessionKey: "agent:main:orphan", + sessionId: "orphan-lifecycle", + old: true, + }); + + await expect( + adapter.cleanupSessionLifecycleArtifacts({ + storePath: cleanupStorePath, + sessionKeySegmentPrefix: "lifecycle-cleanup-", + transcriptContentMarker: "lifecycle-marker-", + orphanTranscriptMinAgeMs: 300_000, + nowMs, + }), + ).resolves.toEqual({ removedEntries: 2, archivedTranscriptArtifacts: 2 }); + + expect( + adapter.loadSessionEntry(scopedEntry("agent:main:lifecycle-cleanup-missing")), + ).toBeUndefined(); + expect( + adapter.loadSessionEntry(scopedEntry("agent:main:lifecycle-cleanup-removed")), + ).toBeUndefined(); + expect( + adapter.loadSessionEntry(scopedEntry("agent:main:lifecycle-cleanup-fresh")), + ).toMatchObject({ + sessionId: "fresh-lifecycle", + }); + expect( + adapter.loadSessionEntry(scopedEntry("agent:main:telegram:group:lifecycle-cleanup-room")), + ).toMatchObject({ sessionId: "kept-by-segment" }); + expect(adapter.loadSessionEntry(scopedEntry("agent:main:regular"))).toMatchObject({ + sessionId: "referenced", + }); + if (usesSqliteStore) { + expect(fs.existsSync(cleanupStorePath)).toBe(false); + const database = openOpenClawAgentDatabase({ + agentId: "main", + env: { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }, + path: path.join(paths.stateDir, "agents", "main", "agent", "openclaw-agent.sqlite"), + }); + const db = getNodeSqliteKysely<OpenClawAgentKyselyDatabase>(database.db); + const removedRoute = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_routes") + .select("session_id") + .where("session_key", "=", "agent:main:lifecycle-cleanup-removed"), + ); + expect(removedRoute).toBeUndefined(); + const freshRoute = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_routes") + .select("session_id") + .where("session_key", "=", "agent:main:lifecycle-cleanup-fresh"), + ); + expect(freshRoute).toEqual({ session_id: "fresh-lifecycle" }); + await expect( + adapter.loadTranscriptEvents(scopedTranscript("agent:main:regular", "referenced")), + ).resolves.not.toEqual([]); + await expect( + adapter.loadTranscriptEvents( + scopedTranscript("agent:main:lifecycle-cleanup-removed", "removed-lifecycle"), + ), + ).resolves.toEqual([]); + const files = fs.readdirSync(path.dirname(cleanupStorePath)); + const removedArchive = files.find((file) => + file.startsWith("removed-lifecycle.jsonl.deleted."), + ); + const orphanArchive = files.find((file) => + file.startsWith("orphan-lifecycle.jsonl.deleted."), + ); + expect(removedArchive).toBeDefined(); + expect(orphanArchive).toBeDefined(); + expect( + readSessionArchiveContentSync( + path.join(path.dirname(cleanupStorePath), removedArchive ?? ""), + ) + .trim() + .split("\n") + .map((line) => JSON.parse(line)), + ).toEqual([ + expect.objectContaining({ + id: "removed-lifecycle-event", + marker: "lifecycle-marker-run", + }), + ]); + } else { + const files = fs.readdirSync(path.dirname(cleanupStorePath)); + expect( + files.filter((file) => file.startsWith("removed-lifecycle.jsonl.deleted.")), + ).toHaveLength(1); + expect( + files.filter((file) => file.startsWith("orphan-lifecycle.jsonl.deleted.")), + ).toHaveLength(1); + expect(files).toContain("fresh-lifecycle.jsonl"); + expect(files).toContain("referenced.jsonl"); + } + }); + + it("conforms for raw transcript event load and append", async () => { + const scope = adapter.transcriptScope(paths); + const readScope = adapter.transcriptReadScope(paths); + const event = { + id: "event-1", + parentId: null, + payload: { content: "hello" }, + type: "metadata", + }; + + await adapter.appendTranscriptEvent(scope, { type: "session", sessionId: "session-1" }); + await adapter.appendTranscriptEvent(scope, event); + + await expect(adapter.loadTranscriptEvents(readScope)).resolves.toEqual([ + { type: "session", sessionId: "session-1" }, + event, + ]); + }); + + it("loads raw SQLite transcript events synchronously through a read scope", async () => { + const scope = sqliteAdapter.transcriptScope(paths); + const readScope = sqliteAdapter.transcriptReadScope(paths); + const event = { + id: "event-1", + parentId: null, + payload: { content: "hello" }, + type: "metadata", + }; + + await sqliteAdapter.appendTranscriptEvent(scope, event); + + expect(loadSqliteTranscriptEventsSync(readScope)).toEqual([event]); + }); + + it("maps canonical sessions.json store paths to the agent SQLite database", async () => { + const legacyStorePath = path.join( + paths.stateDir, + "agents", + "voice", + "sessions", + "sessions.json", + ); + const sqlitePath = path.join( + paths.stateDir, + "agents", + "voice", + "agent", + "openclaw-agent.sqlite", + ); + const scope = { + env: { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }, + sessionKey: "voice:123", + storePath: legacyStorePath, + }; + + await upsertSqliteSessionEntry(scope, { + model: "gpt-5.5", + sessionId: "session-1", + updatedAt: 10, + }); + + expect( + loadSqliteSessionEntry({ ...scope, agentId: "voice", storePath: sqlitePath }), + ).toMatchObject({ + model: "gpt-5.5", + sessionId: "session-1", + }); + expect(fs.existsSync(sqlitePath)).toBe(true); + expect(fs.existsSync(legacyStorePath)).toBe(false); + expect( + listSqliteSessionEntries({ + env: scope.env, + storePath: sqlitePath, + }), + ).toEqual([ + expect.objectContaining({ + entry: expect.objectContaining({ sessionId: "session-1" }), + sessionKey: "voice:123", + }), + ]); + expect(() => + loadSqliteSessionEntry({ ...scope, agentId: "main", storePath: sqlitePath }), + ).toThrow("belongs to agent voice; requested agent main"); + }); + + it("keeps custom JSON store paths beside their SQLite database", async () => { + const customStorePath = path.join(paths.tempDir, "custom-sessions.json"); + const sqlitePath = path.join(paths.tempDir, "custom-sessions.voice.sqlite"); + const scope = { + env: { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }, + sessionKey: "agent:voice:main", + storePath: customStorePath, + }; + + await upsertSqliteSessionEntry(scope, { + model: "gpt-5.5", + sessionId: "session-1", + updatedAt: 10, + }); + + expect( + loadSqliteSessionEntry({ ...scope, agentId: "voice", storePath: sqlitePath }), + ).toMatchObject({ + model: "gpt-5.5", + sessionId: "session-1", + }); + expect(fs.existsSync(sqlitePath)).toBe(true); + expect(fs.existsSync(customStorePath)).toBe(false); + }); + + it("uses the requested agent for custom sessions.json SQLite targets", async () => { + const customStorePath = path.join(paths.tempDir, "custom-store", "sessions.json"); + const customSqlitePath = path.join( + path.dirname(customStorePath), + "openclaw-agent.support.sqlite", + ); + const scope = { + agentId: "support", + env: { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }, + sessionKey: "agent:support:main", + storePath: customStorePath, + }; + + await upsertSqliteSessionEntry(scope, { + model: "gpt-5.5", + sessionId: "support-session", + updatedAt: 10, + }); + const runtimeTarget = await resolveSessionTranscriptRuntimeTarget({ + ...scope, + sessionId: "support-session", + }); + const marker = parseSqliteSessionFileMarker(runtimeTarget.sessionFile); + + expect(loadSqliteSessionEntry({ ...scope, storePath: customSqlitePath })).toMatchObject({ + model: "gpt-5.5", + sessionId: "support-session", + }); + expect(fs.existsSync(customSqlitePath)).toBe(true); + expect(marker).toMatchObject({ + agentId: "support", + sessionId: "support-session", + storePath: customStorePath, + }); + }); + + it("does not parse unrelated SQLite entry blobs for keyed loads", async () => { + const scope = sqliteAdapter.entryScope(paths); + for (let index = 0; index < 20; index += 1) { + await upsertSqliteSessionEntry( + { + ...scope, + sessionKey: `agent:main:unrelated-${index}`, + }, + { + model: `model-${index}`, + sessionId: `unrelated-session-${index}`, + updatedAt: index + 1, + }, + ); + } + await upsertSqliteSessionEntry(scope, { + model: "target", + sessionId: "target-session", + updatedAt: 100, + }); + const parseSpy = vi.spyOn(JSON, "parse"); + + try { + expect(loadSqliteSessionEntry(scope)).toMatchObject({ + model: "target", + sessionId: "target-session", + }); + expect(parseSpy.mock.calls.length).toBeLessThanOrEqual(2); + } finally { + parseSpy.mockRestore(); + } + }); + + it("prunes stale SQLite entries below the entry cap without parsing on every write", async () => { + const scope = sqliteAdapter.entryScope(paths); + const staleScope = { + ...scope, + sessionKey: "agent:main:stale-under-cap", + }; + await replaceSqliteSessionEntry(staleScope, { + model: "stale", + sessionId: "stale-under-cap", + updatedAt: Date.now() - 31 * 24 * 60 * 60 * 1000, + }); + + await upsertSqliteSessionEntry(scope, { + model: "fresh", + sessionId: "fresh-session", + updatedAt: Date.now(), + }); + + expect(loadSqliteSessionEntry(staleScope)).toBeUndefined(); + expect(loadSqliteSessionEntry(scope)).toMatchObject({ + model: "fresh", + sessionId: "fresh-session", + }); + }); + + it("serializes concurrent SQLite entry patches and updates", async () => { + const scope = sqliteAdapter.entryScope(paths); + + await upsertSqliteSessionEntry(scope, { + model: "base", + sessionId: "patch-session", + updatedAt: 10, + }); + + let firstPatch!: Promise<SessionEntry | null>; + let releasePatch!: () => void; + const patchStarted = new Promise<void>((resolve) => { + const blockedPatch = new Promise<void>((release) => { + releasePatch = release; + }); + firstPatch = patchSqliteSessionEntry(scope, async () => { + resolve(); + await blockedPatch; + return { model: "first" }; + }); + }); + await patchStarted; + const secondPatch = patchSqliteSessionEntry(scope, () => ({ + providerOverride: "openai", + })); + releasePatch(); + await Promise.all([firstPatch, secondPatch]); + + expect(loadSqliteSessionEntry(scope)).toMatchObject({ + model: "first", + providerOverride: "openai", + }); + + let firstUpdate!: Promise<SessionEntry | null>; + let releaseUpdate!: () => void; + const updateStarted = new Promise<void>((resolve) => { + const blockedUpdate = new Promise<void>((release) => { + releaseUpdate = release; + }); + firstUpdate = updateSqliteSessionEntry(scope, async () => { + resolve(); + await blockedUpdate; + return { model: "updated" }; + }); + }); + await updateStarted; + const secondUpdate = updateSqliteSessionEntry(scope, () => ({ + providerOverride: "anthropic", + })); + releaseUpdate(); + await Promise.all([firstUpdate, secondUpdate]); + + expect(loadSqliteSessionEntry(scope)).toMatchObject({ + model: "updated", + providerOverride: "anthropic", + }); + }); + + it("does not hold a write transaction while awaiting a SQLite entry updater", async () => { + if (adapter !== sqliteAdapter) { + return; + } + const scope = sqliteAdapter.entryScope(paths); + await replaceSqliteSessionEntry(scope, { + model: "base", + sessionId: "transaction-gap", + updatedAt: 10, + }); + + let releaseUpdater!: () => void; + let markUpdaterStarted!: () => void; + const updaterStarted = new Promise<void>((resolve) => { + markUpdaterStarted = resolve; + }); + const updaterGate = new Promise<void>((resolve) => { + releaseUpdater = resolve; + }); + const pendingPatch = patchSqliteSessionEntry(scope, async () => { + markUpdaterStarted(); + await updaterGate; + return { model: "patched" }; + }); + + await updaterStarted; + let unrelatedWriteError: unknown; + try { + appendSqliteTrajectoryRuntimeEvents( + { sessionId: "transaction-gap", storePath: paths.sqlitePath }, + [ + { + traceSchema: "openclaw-trajectory", + schemaVersion: 1, + traceId: "transaction-gap", + source: "runtime", + type: "test.concurrent-write", + ts: "2026-07-09T00:00:00.000Z", + seq: 1, + sessionId: "transaction-gap", + }, + ], + ); + } catch (error) { + unrelatedWriteError = error; + } finally { + releaseUpdater(); + } + + await expect(pendingPatch).resolves.toMatchObject({ model: "patched" }); + expect(unrelatedWriteError).toBeUndefined(); + }); + + it("allows auth and trajectory writers while a session updater is preparing", async () => { + if (adapter !== sqliteAdapter) { + return; + } + const agentDir = path.join(paths.tempDir, "agents", "main", "agent"); + const conventionalStorePath = path.join( + paths.tempDir, + "agents", + "main", + "sessions", + "sessions.json", + ); + const scope = { + agentId: "main", + sessionKey: "agent:main:shared-writers", + storePath: conventionalStorePath, + }; + await replaceSqliteSessionEntry(scope, { + model: "base", + sessionId: "shared-writers", + updatedAt: 10, + }); + + let releaseUpdater!: () => void; + let markUpdaterStarted!: () => void; + const updaterStarted = new Promise<void>((resolve) => { + markUpdaterStarted = resolve; + }); + const updaterGate = new Promise<void>((resolve) => { + releaseUpdater = resolve; + }); + const pendingPatch = patchSqliteSessionEntry(scope, async () => { + markUpdaterStarted(); + await updaterGate; + return { model: "patched" }; + }); + + await updaterStarted; + writePersistedAuthProfileStateRaw({ selectedProfile: "test" }, agentDir); + appendSqliteTrajectoryRuntimeEvents( + { sessionId: "shared-writers", storePath: conventionalStorePath }, + [ + { + traceSchema: "openclaw-trajectory", + schemaVersion: 1, + traceId: "shared-writers", + source: "runtime", + type: "test.shared-writers", + ts: "2026-07-09T00:00:00.000Z", + seq: 1, + sessionId: "shared-writers", + }, + ], + ); + releaseUpdater(); + + await expect(pendingPatch).resolves.toMatchObject({ model: "patched" }); + expect(readPersistedAuthProfileStateRaw(agentDir)).toEqual({ selectedProfile: "test" }); + }); + + it("rejects a prepared SQLite entry patch when its source row changes", async () => { + if (adapter !== sqliteAdapter) { + return; + } + const scope = sqliteAdapter.entryScope(paths); + await replaceSqliteSessionEntry(scope, { + model: "base", + sessionId: "stale-prepare", + updatedAt: 10, + }); + + let releaseUpdater!: () => void; + let markUpdaterStarted!: () => void; + const updaterStarted = new Promise<void>((resolve) => { + markUpdaterStarted = resolve; + }); + const updaterGate = new Promise<void>((resolve) => { + releaseUpdater = resolve; + }); + const pendingPatch = patchSqliteSessionEntry(scope, async () => { + markUpdaterStarted(); + await updaterGate; + return { model: "stale-patch" }; + }); + + await updaterStarted; + let replacementError: unknown; + try { + replaceSqliteSessionEntrySync(scope, { + model: "newer", + sessionId: "stale-prepare", + updatedAt: 20, + }); + } catch (error) { + replacementError = error; + } finally { + releaseUpdater(); + } + const mutationError = await pendingPatch.then( + () => undefined, + (error: unknown) => error, + ); + + expect(replacementError).toBeUndefined(); + expect(mutationError).toMatchObject({ name: "SqliteSessionMutationConflictError" }); + expect(loadSqliteSessionEntry(scope)).toMatchObject({ model: "newer", updatedAt: 20 }); + }); + + it("dedupes SQLite transcript identities inside the writer path", async () => { + const scope = sqliteAdapter.transcriptScope(paths, "session-dedupe"); + const event = { + id: "event-dedupe", + source: "conformance", + type: "metadata", + }; + + await appendSqliteTranscriptEvent(scope, event); + await appendSqliteTranscriptEvent(scope, { + ...event, + message: { role: "assistant", content: "duplicate" }, + }); + const results = await Promise.all( + Array.from({ length: 8 }, () => + appendSqliteTranscriptMessage(scope, { + idempotencyLookup: "scan", + message: { + role: "assistant", + content: "keyed", + idempotencyKey: "keyed-once", + }, + }), + ), + ); + + expect(new Set(results.map((result) => result?.messageId)).size).toBe(1); + expect(results.filter((result) => result?.appended)).toHaveLength(1); + await expect(loadSqliteTranscriptEvents(scope)).resolves.toEqual([ + event, + expect.objectContaining({ + message: expect.objectContaining({ idempotencyKey: "keyed-once" }), + parentId: null, + type: "message", + }), + ]); + }); + + it("rejects raw message transcript event writes", async () => { + const scope = adapter.transcriptScope(paths, "session-raw-message"); + await expect( + adapter.appendTranscriptEvent(scope, { + id: "raw-message", + message: { role: "assistant", content: "raw" }, + parentId: null, + type: "message", + }), + ).rejects.toThrow(/use append(?:Sqlite)?TranscriptMessage instead/); + }); + + it("returns existing SQLite transcript messages after default idempotency dedupe", async () => { + const scope = sqliteAdapter.transcriptScope(paths, "session-unchecked-dedupe"); + const message = { + role: "assistant", + content: "unchecked", + idempotencyKey: "unchecked-once", + }; + + const appended = await appendSqliteTranscriptMessage(scope, { message }); + const replayed = await appendSqliteTranscriptMessage(scope, { + message: { + ...message, + content: "unchecked replay", + }, + }); + + const events = await loadSqliteTranscriptEvents(scope); + const keyedEvents = events.filter((event): event is { message: typeof message } => { + return ( + Boolean(event) && + typeof event === "object" && + !Array.isArray(event) && + (event as { message?: { idempotencyKey?: string } }).message?.idempotencyKey === + "unchecked-once" + ); + }); + expect(appended).toMatchObject({ appended: true }); + expect(replayed).toMatchObject({ + appended: false, + message: expect.objectContaining({ content: "unchecked" }), + messageId: appended?.messageId, + }); + expect(keyedEvents).toHaveLength(1); + }); + + it("treats replayed SQLite transcript message ids as existing appends", async () => { + const scope = sqliteAdapter.transcriptScope(paths, "session-event-id-dedupe"); + const eventId = "message-retry-event-id"; + + const appended = await appendSqliteTranscriptMessage(scope, { + eventId, + message: { + role: "assistant", + content: "first attempt", + }, + }); + const replayed = await appendSqliteTranscriptMessage(scope, { + eventId, + message: { + role: "assistant", + content: "retry attempt", + }, + }); + + expect(appended).toMatchObject({ + appended: true, + messageId: eventId, + }); + expect(replayed).toMatchObject({ + appended: false, + message: expect.objectContaining({ content: "first attempt" }), + messageId: eventId, + }); + await expect(loadSqliteTranscriptEvents(scope)).resolves.toEqual([ + expect.objectContaining({ type: "session" }), + expect.objectContaining({ + id: eventId, + message: expect.objectContaining({ content: "first attempt" }), + type: "message", + }), + ]); + }); + + it("conforms for transcript message append, idempotency, and update publication", async () => { + const scope = adapter.transcriptScope(paths, "session-2"); + const updates: unknown[] = []; + const unsubscribe = onSessionTranscriptUpdate((update) => { + updates.push(update); + }); + + const appended = await adapter.appendTranscriptMessage(scope, { + cwd: paths.tempDir, + idempotencyLookup: "scan", + message: { + role: "assistant", + content: "hello", + idempotencyKey: "assistant-once", + }, + }); + const replayed = await adapter.appendTranscriptMessage(scope, { + cwd: paths.tempDir, + idempotencyLookup: "scan", + message: { + role: "assistant", + content: "hello again", + idempotencyKey: "assistant-once", + }, + }); + await adapter.publishTranscriptUpdate(scope, { + agentId: "main", + message: appended?.message, + messageId: appended?.messageId, + sessionKey: scope.sessionKey, + }); + unsubscribe(); + + expect(appended).toMatchObject({ + appended: true, + message: expect.objectContaining({ content: "hello" }), + messageId: expect.any(String), + }); + expect(replayed).toMatchObject({ + appended: false, + message: expect.objectContaining({ + content: "hello", + idempotencyKey: "assistant-once", + }), + messageId: appended?.messageId, + }); + await expect(adapter.loadTranscriptEvents(scope)).resolves.toEqual([ + expect.objectContaining({ type: "session" }), + expect.objectContaining({ + id: appended?.messageId, + message: expect.objectContaining({ content: "hello" }), + type: "message", + }), + ]); + if (adapter.publishesTranscriptUpdates) { + expect(updates).toEqual([ + expect.objectContaining({ + agentId: "main", + message: appended?.message, + messageId: appended?.messageId, + sessionKey: scope.sessionKey, + }), + ]); + } else { + expect(updates).toEqual([]); + } + }); + }, +); + +describe("sqlite session normalization", () => { + let paths: TestPaths; + + beforeEach(() => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-session-sqlite-norm-")); + paths = { + sqlitePath: path.join(tempDir, "openclaw-agent.sqlite"), + stateDir: path.join(tempDir, "state"), + storePath: path.join(tempDir, "sessions.json"), + tempDir, + transcriptPath: path.join(tempDir, "session.jsonl"), + }; + }); + + afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + fs.rmSync(paths.tempDir, { recursive: true, force: true }); + }); + + it("maintains normalized session root and route rows", async () => { + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + await upsertSqliteSessionEntry( + { + agentId: "main", + env, + sessionKey: "agent:main:group:example", + storePath: paths.sqlitePath, + }, + { + agentHarnessId: "codex", + chatType: "group", + channel: "discord", + deliveryContext: { + accountId: "acct-1", + channel: "discord", + threadId: "thread-1", + to: "group-1", + }, + displayName: "Example group", + endedAt: 90, + model: "gpt-5.5", + modelProvider: "openai", + parentSessionKey: "agent:main:parent", + sessionId: "normalized-session", + sessionStartedAt: 50, + spawnedBy: "agent:main:spawner", + startedAt: 60, + status: "done", + updatedAt: 100, + }, + ); + + const database = openOpenClawAgentDatabase({ + agentId: "main", + env, + path: paths.sqlitePath, + }); + const db = getNodeSqliteKysely<OpenClawAgentKyselyDatabase>(database.db); + const session = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("sessions") + .select([ + "account_id", + "agent_harness_id", + "channel", + "chat_type", + "created_at", + "display_name", + "ended_at", + "model", + "model_provider", + "parent_session_key", + "session_key", + "session_scope", + "spawned_by", + "started_at", + "status", + "updated_at", + ]) + .where("session_id", "=", "normalized-session"), + ); + expect(session).toEqual({ + account_id: "acct-1", + agent_harness_id: "codex", + channel: "discord", + chat_type: "group", + created_at: 50, + display_name: "Example group", + ended_at: 90, + model: "gpt-5.5", + model_provider: "openai", + parent_session_key: "agent:main:parent", + session_key: "agent:main:group:example", + session_scope: "group", + spawned_by: "agent:main:spawner", + started_at: 60, + status: "done", + updated_at: expect.any(Number), + }); + + const route = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_routes") + .select(["session_id", "updated_at"]) + .where("session_key", "=", "agent:main:group:example"), + ); + expect(route).toEqual({ + session_id: "normalized-session", + updated_at: expect.any(Number), + }); + }); + + it("exposes same-key rollover lineage when a killed session is replaced", async () => { + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + const sessionKey = "agent:main:telegram:group:-1003774691294:topic:29020"; + const oldSessionId = "f1321535-878b-47cd-b35e-2f5f4bae2bb5"; + const newSessionId = "c0daccb0-0555-47d8-8747-9b53addf1fe2"; + const scope = { + agentId: "main", + env, + sessionKey, + storePath: paths.sqlitePath, + }; + + await upsertSqliteSessionEntry(scope, { + channel: "telegram", + chatType: "group", + displayName: "telegram:g-bucephalus-+-topics", + sessionId: oldSessionId, + status: "killed", + updatedAt: 1_782_973_392_492, + }); + await appendSqliteTranscriptEvent( + { ...scope, sessionId: oldSessionId }, + { + id: "old-frontier", + payload: { label: "old transcript frontier" }, + type: "metadata", + }, + ); + + await upsertSqliteSessionEntry(scope, { + channel: "telegram", + chatType: "group", + displayName: "telegram:g-bucephalus-+-topics", + sessionId: newSessionId, + status: "running", + updatedAt: 1_782_997_881_018, + }); + await appendSqliteTranscriptEvent( + { ...scope, sessionId: newSessionId }, + { + id: "new-turn", + payload: { label: "new active turn" }, + type: "metadata", + }, + ); + + await expect( + loadSqliteTranscriptEvents({ ...scope, sessionId: oldSessionId }), + ).resolves.toEqual([expect.objectContaining({ id: "old-frontier", type: "metadata" })]); + await expect( + loadSqliteTranscriptEvents({ ...scope, sessionId: newSessionId }), + ).resolves.toEqual([expect.objectContaining({ id: "new-turn", type: "metadata" })]); + expect(loadSqliteSessionEntry(scope)).toEqual( + expect.objectContaining({ + sessionId: newSessionId, + usageFamilyKey: sessionKey, + usageFamilySessionIds: [oldSessionId, newSessionId], + }), + ); + }); + + it("keeps exact SQLite replacement entries free of inferred rollover lineage", async () => { + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + const scope = { + agentId: "main", + env, + sessionKey: "agent:main:restore-exact", + storePath: paths.sqlitePath, + }; + + await upsertSqliteSessionEntry(scope, { + sessionId: "temporary-session", + updatedAt: 10, + }); + await replaceSqliteSessionEntry(scope, { + sessionId: "restored-session", + updatedAt: 20, + }); + + expect(loadSqliteSessionEntry(scope)).toEqual({ + sessionId: "restored-session", + updatedAt: 20, + }); + }); + + it("skips parent fork when transcript rows exceed the token budget and entry totals are stale", async () => { + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + const parentKey = "agent:main:parent"; + const childKey = "agent:main:subagent:child"; + await upsertSqliteSessionEntry( + { + agentId: "main", + env, + sessionKey: parentKey, + storePath: paths.sqlitePath, + }, + { + sessionId: "parent-session", + totalTokens: 1, + totalTokensFresh: false, + updatedAt: 10, + }, + ); + await replaceSqliteTranscriptEvents( + { + agentId: "main", + env, + sessionId: "parent-session", + sessionKey: parentKey, + storePath: paths.sqlitePath, + }, + [ + { type: "session", id: "parent-session", cwd: paths.tempDir }, + { + type: "message", + id: "oversized-parent", + parentId: null, + message: { role: "user", content: "x".repeat(420_000) }, + }, + ], + ); + + const result = await forkSqliteSessionEntryFromParentTarget({ + fallbackEntry: { sessionId: "", updatedAt: 1 }, + parentTarget: { canonicalKey: parentKey, storeKeys: [parentKey] }, + sessionTarget: { canonicalKey: childKey, storeKeys: [childKey] }, + storePath: paths.sqlitePath, + decisionSkipPatch: () => ({ forkedFromParent: true, updatedAt: 11 }), + }); + + expect(result).toMatchObject({ + status: "skipped", + reason: "decision-skip", + decision: { + status: "skip", + reason: "parent-too-large", + }, + sessionEntry: { + forkedFromParent: true, + sessionId: "", + updatedAt: expect.any(Number), + }, + }); + if (result.status !== "skipped" || result.reason !== "decision-skip") { + throw new Error(`expected decision-skip, got ${result.status}`); + } + expect(result.decision?.parentTokens).toBeGreaterThan(100_000); + }); + + it("does not move current routes back to stale transcript session ids", async () => { + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + const scope = { + agentId: "main", + env, + sessionKey: "agent:main:main", + storePath: paths.sqlitePath, + }; + await upsertSqliteSessionEntry(scope, { + sessionId: "current-session", + updatedAt: 20, + }); + await appendSqliteTranscriptEvent( + { + ...scope, + sessionId: "stale-session", + }, + { + id: "stale-event", + timestamp: new Date(10).toISOString(), + type: "metadata", + }, + ); + + const database = openOpenClawAgentDatabase({ + agentId: "main", + env, + path: paths.sqlitePath, + }); + const db = getNodeSqliteKysely<OpenClawAgentKyselyDatabase>(database.db); + const route = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_routes") + .select("session_id") + .where("session_key", "=", "agent:main:main"), + ); + expect(route).toEqual({ session_id: "current-session" }); + }); + + it("applies SQLite session-entry maintenance inside entry write transactions", async () => { + vi.mocked(getRuntimeConfig).mockReturnValue({ + session: { + maintenance: { + mode: "enforce", + pruneAfter: "1d", + maxEntries: 2, + }, + }, + }); + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + const scopeFor = (sessionKey: string) => ({ + agentId: "main", + env, + sessionKey, + storePath: paths.sqlitePath, + }); + const oldUpdatedAt = Date.now() - 2 * 24 * 60 * 60 * 1000; + + await patchSqliteSessionEntry( + scopeFor("agent:main:stale"), + () => ({ sessionId: "stale-session", updatedAt: oldUpdatedAt }), + { + fallbackEntry: { sessionId: "stale-session", updatedAt: oldUpdatedAt }, + replaceEntry: true, + skipMaintenance: true, + }, + ); + await patchSqliteSessionEntry( + scopeFor("agent:main:older"), + () => ({ sessionId: "older-session", updatedAt: oldUpdatedAt + 1 }), + { + fallbackEntry: { sessionId: "older-session", updatedAt: oldUpdatedAt + 1 }, + replaceEntry: true, + skipMaintenance: true, + }, + ); + await patchSqliteSessionEntry( + scopeFor("agent:main:active"), + () => ({ sessionId: "active-session", updatedAt: Date.now() }), + { + fallbackEntry: { sessionId: "active-session", updatedAt: Date.now() }, + replaceEntry: true, + skipMaintenance: true, + }, + ); + const staleTranscriptEvent = { + id: "stale-event", + timestamp: new Date(oldUpdatedAt).toISOString(), + type: "metadata", + }; + await appendSqliteTranscriptEvent( + { ...scopeFor("agent:main:stale"), sessionId: "stale-session" }, + staleTranscriptEvent, + ); + + await updateSqliteSessionEntry(scopeFor("agent:main:active"), () => ({ model: "gpt-5.5" }), { + skipMaintenance: true, + }); + await expect( + loadSqliteTranscriptEvents({ + agentId: "main", + env, + sessionId: "stale-session", + storePath: paths.sqlitePath, + }), + ).resolves.toEqual([staleTranscriptEvent]); + expect( + listSqliteSessionEntries({ + agentId: "main", + env, + storePath: paths.sqlitePath, + }).map((summary) => summary.sessionKey), + ).toEqual(["agent:main:active", "agent:main:older", "agent:main:stale"]); + + await updateSqliteSessionEntry(scopeFor("agent:main:active"), () => ({ + providerOverride: "openai", + })); + + expect( + listSqliteSessionEntries({ + agentId: "main", + env, + storePath: paths.sqlitePath, + }).map((summary) => summary.sessionKey), + ).toEqual(["agent:main:active"]); + await expect( + loadSqliteTranscriptEvents({ + agentId: "main", + env, + sessionId: "stale-session", + storePath: paths.sqlitePath, + }), + ).resolves.toEqual([]); + const archivedStale = fs + .readdirSync(paths.tempDir) + .filter((file) => file.startsWith("stale-session.jsonl.deleted.")); + expect(archivedStale).toHaveLength(1); + expect( + readSessionArchiveContentSync(path.join(paths.tempDir, archivedStale[0] ?? "")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)), + ).toEqual([staleTranscriptEvent]); + + await patchSqliteSessionEntry( + scopeFor("agent:main:newer"), + () => ({ sessionId: "newer-session", updatedAt: Date.now() + 1 }), + { + fallbackEntry: { sessionId: "newer-session", updatedAt: Date.now() + 1 }, + replaceEntry: true, + skipMaintenance: true, + }, + ); + await patchSqliteSessionEntry( + scopeFor("agent:main:newest"), + () => ({ sessionId: "newest-session", updatedAt: Date.now() + 2 }), + { + fallbackEntry: { sessionId: "newest-session", updatedAt: Date.now() + 2 }, + replaceEntry: true, + }, + ); + + expect( + listSqliteSessionEntries({ + agentId: "main", + env, + storePath: paths.sqlitePath, + }).map((summary) => summary.sessionKey), + ).toEqual(["agent:main:newer", "agent:main:newest"]); + }); + + it("evicts old SQLite transcript rows only when no remaining entry references them", async () => { + vi.mocked(getRuntimeConfig).mockReturnValue({ + session: { + maintenance: { + highWaterBytes: 350, + maxDiskBytes: 1_200, + maxEntries: 100, + mode: "enforce", + pruneAfter: "365d", + }, + }, + }); + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + const scopeFor = (sessionKey: string) => ({ + agentId: "main", + env, + sessionKey, + storePath: paths.sqlitePath, + }); + const oldUpdatedAt = Date.now() - 2 * 24 * 60 * 60 * 1000; + const unsharedUpdatedAt = oldUpdatedAt - 1_000; + + await patchSqliteSessionEntry( + scopeFor("agent:main:unshared-budget"), + () => ({ sessionId: "unshared-budget-session", updatedAt: unsharedUpdatedAt }), + { + fallbackEntry: { sessionId: "unshared-budget-session", updatedAt: unsharedUpdatedAt }, + replaceEntry: true, + skipMaintenance: true, + }, + ); + await appendSqliteTranscriptEvent( + { ...scopeFor("agent:main:unshared-budget"), sessionId: "unshared-budget-session" }, + { + id: "unshared-budget-event", + payload: "😀".repeat(400), + timestamp: new Date(unsharedUpdatedAt).toISOString(), + type: "metadata", + }, + ); + + await patchSqliteSessionEntry( + scopeFor("agent:main:old-budget"), + () => ({ sessionId: "old-budget-session", updatedAt: oldUpdatedAt }), + { + fallbackEntry: { sessionId: "old-budget-session", updatedAt: oldUpdatedAt }, + replaceEntry: true, + skipMaintenance: true, + }, + ); + await appendSqliteTranscriptEvent( + { ...scopeFor("agent:main:old-budget"), sessionId: "old-budget-session" }, + { + id: "old-budget-event", + payload: "x".repeat(50), + timestamp: new Date(oldUpdatedAt).toISOString(), + type: "metadata", + }, + ); + await patchSqliteSessionEntry( + scopeFor("agent:main:active-budget"), + () => ({ + sessionId: "active-budget-session", + updatedAt: Date.now(), + usageFamilySessionIds: ["old-budget-session", "active-budget-session"], + }), + { + fallbackEntry: { + sessionId: "active-budget-session", + updatedAt: Date.now(), + usageFamilySessionIds: ["old-budget-session", "active-budget-session"], + }, + replaceEntry: true, + skipMaintenance: true, + }, + ); + + await updateSqliteSessionEntry(scopeFor("agent:main:active-budget"), () => ({ + modelOverride: "gpt-5.5", + })); + + expect( + listSqliteSessionEntries({ + agentId: "main", + env, + storePath: paths.sqlitePath, + }) + .map((summary) => summary.sessionKey) + .toSorted(), + ).toEqual(["agent:main:active-budget"]); + await expect( + loadSqliteTranscriptEvents({ + agentId: "main", + env, + sessionId: "old-budget-session", + storePath: paths.sqlitePath, + }), + ).resolves.toEqual([ + expect.objectContaining({ + id: "old-budget-event", + }), + ]); + await expect( + loadSqliteTranscriptEvents({ + agentId: "main", + env, + sessionId: "unshared-budget-session", + storePath: paths.sqlitePath, + }), + ).resolves.toEqual([]); + const archivedOldBudget = fs + .readdirSync(paths.tempDir) + .filter((file) => file.startsWith("old-budget-session.jsonl.deleted.")); + expect(archivedOldBudget).toHaveLength(0); + const archivedUnsharedBudget = fs + .readdirSync(paths.tempDir) + .filter((file) => file.startsWith("unshared-budget-session.jsonl.deleted.")); + expect(archivedUnsharedBudget).toHaveLength(1); + expect( + readSessionArchiveContentSync(path.join(paths.tempDir, archivedUnsharedBudget[0] ?? "")), + ).toContain("unshared-budget-event"); + }); + + it("resolves confirmed lowercased legacy SQLite session aliases", async () => { + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + const canonicalKey = "agent:main:matrix:channel:!MixedCase:example.org"; + const legacyKey = canonicalKey.toLowerCase(); + await upsertSqliteSessionEntry( + { + agentId: "main", + env, + sessionKey: legacyKey, + storePath: paths.sqlitePath, + }, + { + deliveryContext: { + accountId: "acct-1", + channel: "matrix", + to: "!MixedCase:example.org", + }, + sessionId: "legacy-alias-session", + updatedAt: 10, + }, + ); + + expect( + loadSqliteSessionEntry({ + agentId: "main", + env, + sessionKey: canonicalKey, + storePath: paths.sqlitePath, + }), + ).toMatchObject({ sessionId: "legacy-alias-session" }); + const legacyEntry = loadExactSqliteSessionEntry({ + agentId: "main", + env, + sessionKey: legacyKey, + storePath: paths.sqlitePath, + }); + expect(legacyEntry).toBeDefined(); + expect( + readSqliteSessionUpdatedAt({ + agentId: "main", + env, + sessionKey: canonicalKey, + storePath: paths.sqlitePath, + }), + ).toBe(legacyEntry?.entry.updatedAt); + + await patchSqliteSessionEntry( + { + agentId: "main", + env, + sessionKey: canonicalKey, + storePath: paths.sqlitePath, + }, + () => ({ model: "gpt-5.5", updatedAt: 20 }), + ); + + expect( + loadExactSqliteSessionEntry({ + agentId: "main", + env, + sessionKey: legacyKey, + storePath: paths.sqlitePath, + }), + ).toBeUndefined(); + const canonicalEntry = loadExactSqliteSessionEntry({ + agentId: "main", + env, + sessionKey: canonicalKey, + storePath: paths.sqlitePath, + }); + expect(canonicalEntry).toMatchObject({ + entry: { + model: "gpt-5.5", + sessionId: "legacy-alias-session", + updatedAt: expect.any(Number), + }, + sessionKey: canonicalKey, + }); + expect( + readSqliteSessionUpdatedAt({ + agentId: "main", + env, + sessionKey: canonicalKey, + storePath: paths.sqlitePath, + }), + ).toBe(canonicalEntry?.entry.updatedAt); + expect( + listSqliteSessionEntries({ + agentId: "main", + env, + storePath: paths.sqlitePath, + }).map((summary) => summary.sessionKey), + ).toEqual([canonicalKey]); + }); + + it("normalizes missing entry updatedAt before writing root and entry rows", async () => { + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + await replaceSqliteSessionEntry( + { + agentId: "main", + env, + sessionKey: "agent:main:minimal", + storePath: paths.sqlitePath, + }, + { + sessionId: "minimal-session", + sessionStartedAt: 123, + } as SessionEntry, + ); + + const loaded = loadSqliteSessionEntry({ + agentId: "main", + env, + sessionKey: "agent:main:minimal", + storePath: paths.sqlitePath, + }); + expect(loaded).toMatchObject({ + sessionId: "minimal-session", + sessionStartedAt: 123, + updatedAt: 123, + }); + + const database = openOpenClawAgentDatabase({ + agentId: "main", + env, + path: paths.sqlitePath, + }); + const db = getNodeSqliteKysely<OpenClawAgentKyselyDatabase>(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("sessions as s") + .innerJoin("session_entries as se", "se.session_id", "s.session_id") + .innerJoin("session_routes as sr", "sr.session_key", "se.session_key") + .select([ + "s.created_at as root_created_at", + "s.updated_at as root_updated_at", + "se.entry_json", + "se.updated_at as entry_updated_at", + "sr.updated_at as route_updated_at", + ]) + .where("s.session_id", "=", "minimal-session"), + ); + expect(row).toEqual({ + entry_json: JSON.stringify({ + sessionId: "minimal-session", + sessionStartedAt: 123, + updatedAt: 123, + }), + entry_updated_at: 123, + root_created_at: 123, + root_updated_at: 123, + route_updated_at: 123, + }); + + await upsertSqliteSessionEntry( + { + agentId: "main", + env, + sessionKey: "agent:main:minimal-upsert", + storePath: paths.sqlitePath, + }, + { + sessionId: "minimal-upsert-session", + }, + ); + const upsertRow = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_entries") + .select(["entry_json", "updated_at"]) + .where("session_key", "=", "agent:main:minimal-upsert"), + ); + const upsertEntry = JSON.parse(upsertRow?.entry_json ?? "{}") as Partial<SessionEntry>; + expect(upsertEntry).toMatchObject({ + sessionId: "minimal-upsert-session", + updatedAt: expect.any(Number), + }); + expect(upsertRow?.updated_at).toBe(upsertEntry.updatedAt); + }); + + it("replaces, appends, checks, and deletes SQLite transcript rows without filesystem artifacts", async () => { + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + const scope = { + agentId: "main", + env, + sessionId: "transcript-state-session", + sessionKey: "agent:main:main", + storePath: paths.sqlitePath, + }; + + expect(sqliteTranscriptExists(scope)).toBe(false); + + await replaceSqliteTranscriptEvents(scope, [ + { type: "session", id: "transcript-state-session", cwd: paths.tempDir }, + { type: "message", id: "msg-1", parentId: null, message: { content: "one" } }, + ]); + await appendSqliteTranscriptEvents(scope, [ + { type: "message", id: "msg-2", parentId: "msg-1", message: { content: "two" } }, + ]); + + expect(sqliteTranscriptExists(scope)).toBe(true); + await expect(loadSqliteTranscriptEvents(scope)).resolves.toEqual([ + { type: "session", id: "transcript-state-session", cwd: paths.tempDir }, + { type: "message", id: "msg-1", parentId: null, message: { content: "one" } }, + { type: "message", id: "msg-2", parentId: "msg-1", message: { content: "two" } }, + ]); + + await expect(deleteSqliteTranscript(scope)).resolves.toBe(true); + expect(sqliteTranscriptExists(scope)).toBe(false); + await expect(loadSqliteTranscriptEvents(scope)).resolves.toEqual([]); + expect(fs.existsSync(paths.sqlitePath)).toBe(true); + expect(fs.readdirSync(paths.tempDir)).not.toContain("transcript-state-session.jsonl"); + }); + + it("branches a checkpoint by copying SQLite rows and creating the entry transactionally", async () => { + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + const sourceScope = { + agentId: "main", + env, + sessionId: "source-session", + sessionKey: "agent:main:main", + storePath: paths.sqlitePath, + }; + const preCompactionScope = { + ...sourceScope, + sessionId: "pre-compaction-session", + }; + const sourceEntryScope = { + agentId: "main", + env, + sessionKey: "agent:main:main", + storePath: paths.sqlitePath, + }; + const branchKey = "agent:main:checkpoint-branch"; + const checkpoint: SessionCompactionCheckpoint = { + checkpointId: "checkpoint-branch", + sessionKey: sourceEntryScope.sessionKey, + sessionId: "source-session", + createdAt: Date.parse("2026-01-01T00:00:00.000Z"), + reason: "manual", + tokensBefore: 42, + tokensAfter: 84, + preCompaction: { + sessionId: "pre-compaction-session", + leafId: "pre-msg", + }, + postCompaction: { + sessionId: "source-session", + entryId: "msg-2", + }, + }; + + await replaceSqliteTranscriptEvents(preCompactionScope, [ + { type: "session", id: "pre-compaction-session", cwd: paths.tempDir }, + { type: "message", id: "pre-msg", parentId: null, message: { content: "pre" } }, + ]); + await replaceSqliteTranscriptEvents(sourceScope, [ + { type: "session", id: "source-session", cwd: paths.tempDir }, + { type: "message", id: "post-msg-1", parentId: null, message: { content: "post-one" } }, + { + type: "message", + id: "post-msg-2", + parentId: "post-msg-1", + message: { content: "post-two" }, + }, + ]); + await upsertSqliteSessionEntry(sourceEntryScope, { + label: "Source", + sessionId: "source-session", + updatedAt: 10, + compactionCheckpoints: [checkpoint], + }); + + const result = await branchSqliteCompactionCheckpointSession({ + agentId: "main", + env, + storePath: paths.sqlitePath, + sourceKey: sourceEntryScope.sessionKey, + nextKey: branchKey, + checkpointId: checkpoint.checkpointId, + }); + if (result.status !== "created") { + throw new Error(`expected branch creation, got ${result.status}`); + } + + const branchScope = { + ...sourceScope, + sessionId: result.entry.sessionId, + sessionKey: branchKey, + }; + expect(loadSqliteSessionEntry({ ...sourceEntryScope, sessionKey: branchKey })).toEqual( + result.entry, + ); + expect(result.entry).toEqual( + expect.objectContaining({ + label: "Source (checkpoint)", + parentSessionKey: sourceEntryScope.sessionKey, + sessionFile: expect.stringMatching(/^sqlite:main:/), + totalTokens: 42, + totalTokensFresh: true, + }), + ); + await expect(loadSqliteTranscriptEvents(branchScope)).resolves.toEqual([ + expect.objectContaining({ type: "session", id: result.entry.sessionId }), + expect.objectContaining({ id: "pre-msg", type: "message" }), + ]); + expect(fs.existsSync(path.join(paths.tempDir, `${result.entry.sessionId}.jsonl`))).toBe(false); + }); + + it("falls back to post-compaction SQLite rows when no pre-compaction rows exist", async () => { + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + const sourceScope = { + agentId: "main", + env, + sessionId: "source-session", + sessionKey: "agent:main:main", + storePath: paths.sqlitePath, + }; + const sourceEntryScope = { + agentId: "main", + env, + sessionKey: "agent:main:main", + storePath: paths.sqlitePath, + }; + const checkpoint: SessionCompactionCheckpoint = { + checkpointId: "checkpoint-post-fallback", + sessionKey: sourceEntryScope.sessionKey, + sessionId: "source-session", + createdAt: Date.parse("2026-01-01T00:00:00.000Z"), + reason: "manual", + tokensBefore: 100, + tokensAfter: 25, + preCompaction: { + sessionId: "missing-pre-session", + leafId: "missing-pre-msg", + }, + postCompaction: { + sessionId: "source-session", + entryId: "post-msg", + }, + }; + + await replaceSqliteTranscriptEvents(sourceScope, [ + { type: "session", id: "source-session", cwd: paths.tempDir }, + { type: "message", id: "post-msg", parentId: null, message: { content: "post" } }, + { type: "message", id: "skipped-msg", parentId: "post-msg", message: { content: "skip" } }, + ]); + await upsertSqliteSessionEntry(sourceEntryScope, { + sessionId: "source-session", + updatedAt: 10, + compactionCheckpoints: [checkpoint], + }); + + const result = await branchSqliteCompactionCheckpointSession({ + agentId: "main", + env, + storePath: paths.sqlitePath, + sourceKey: sourceEntryScope.sessionKey, + nextKey: "agent:main:checkpoint-post-fallback", + checkpointId: checkpoint.checkpointId, + }); + if (result.status !== "created") { + throw new Error(`expected fallback branch creation, got ${result.status}`); + } + + await expect( + loadSqliteTranscriptEvents({ + ...sourceScope, + sessionId: result.entry.sessionId, + sessionKey: "agent:main:checkpoint-post-fallback", + }), + ).resolves.toEqual([ + expect.objectContaining({ type: "session", id: result.entry.sessionId }), + expect.objectContaining({ id: "post-msg", type: "message" }), + ]); + expect(result.entry.totalTokens).toBe(25); + }); + + it("restores a checkpoint by copying SQLite rows and replacing the entry transactionally", async () => { + const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; + const sourceScope = { + agentId: "main", + env, + sessionId: "source-session", + sessionKey: "agent:main:main", + storePath: paths.sqlitePath, + }; + const preCompactionScope = { + ...sourceScope, + sessionId: "pre-compaction-session", + }; + const sourceEntryScope = { + agentId: "main", + env, + sessionKey: "agent:main:main", + storePath: paths.sqlitePath, + }; + const checkpoint: SessionCompactionCheckpoint = { + checkpointId: "checkpoint-restore", + sessionKey: sourceEntryScope.sessionKey, + sessionId: "current-session", + createdAt: Date.parse("2026-01-01T00:00:00.000Z"), + reason: "manual", + tokensBefore: 12, + tokensAfter: 24, + preCompaction: { + sessionId: "pre-compaction-session", + leafId: "pre-msg", + }, + postCompaction: { + sessionId: "source-session", + entryId: "msg-1", + }, + }; + + await replaceSqliteTranscriptEvents(preCompactionScope, [ + { type: "session", id: "pre-compaction-session", cwd: paths.tempDir }, + { type: "message", id: "pre-msg", parentId: null, message: { content: "restore" } }, + ]); + await replaceSqliteTranscriptEvents(sourceScope, [ + { type: "session", id: "source-session", cwd: paths.tempDir }, + { type: "message", id: "post-msg-1", parentId: null, message: { content: "skip" } }, + { type: "message", id: "post-msg-2", parentId: "post-msg-1", message: { content: "skip" } }, + ]); + await upsertSqliteSessionEntry(sourceEntryScope, { + label: "Current", + sessionId: "current-session", + sessionFile: "sqlite:main:current-session", + updatedAt: 10, + compactionCheckpoints: [checkpoint], + }); + + const result = await restoreSqliteCompactionCheckpointSession({ + agentId: "main", + env, + storePath: paths.sqlitePath, + sessionKey: sourceEntryScope.sessionKey, + checkpointId: checkpoint.checkpointId, + }); + if (result.status !== "created") { + throw new Error(`expected restore creation, got ${result.status}`); + } + + const restoredScope = { + ...sourceScope, + sessionId: result.entry.sessionId, + }; + expect(loadSqliteSessionEntry(sourceEntryScope)).toEqual(result.entry); + expect(result.entry).toEqual( + expect.objectContaining({ + label: "Current", + compactionCheckpoints: [checkpoint], + sessionFile: expect.stringMatching(/^sqlite:main:/), + totalTokens: 12, + totalTokensFresh: true, + }), + ); + await expect(loadSqliteTranscriptEvents(restoredScope)).resolves.toEqual([ + expect.objectContaining({ type: "session", id: result.entry.sessionId }), + expect.objectContaining({ id: "pre-msg", type: "message" }), + ]); + expect(fs.existsSync(path.join(paths.tempDir, `${result.entry.sessionId}.jsonl`))).toBe(false); + }); +}); diff --git a/src/config/sessions/session-accessor.fork-transcript.test.ts b/src/config/sessions/session-accessor.fork-transcript.test.ts deleted file mode 100644 index 908433442e4b..000000000000 --- a/src/config/sessions/session-accessor.fork-transcript.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -// Behavior tests for the accessor fork-transcript creation boundary. -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, expect, it } from "vitest"; -import { createForkedSessionTranscript } from "./session-accessor.js"; -import { CURRENT_SESSION_VERSION } from "./version.js"; - -const tempDirs: string[] = []; - -function makeSessionsDir(): string { - const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-fork-transcript-"))); - tempDirs.push(dir); - return dir; -} - -function readJsonlRecords(filePath: string): Record<string, unknown>[] { - return fs - .readFileSync(filePath, "utf-8") - .trim() - .split(/\r?\n/u) - .map((line) => JSON.parse(line) as Record<string, unknown>); -} - -afterEach(() => { - for (const dir of tempDirs.splice(0)) { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - -it("writes a header-only fork artifact with parent lineage", async () => { - const sessionsDir = makeSessionsDir(); - const fork = await createForkedSessionTranscript({ - cwd: "/workspace/project", - parentSessionFile: "/sessions/parent.jsonl", - sessionsDir, - }); - - expect(path.dirname(fork.sessionFile)).toBe(sessionsDir); - expect(fork.sessionFile.endsWith(`_${fork.sessionId}.jsonl`)).toBe(true); - const records = readJsonlRecords(fork.sessionFile); - expect(records).toHaveLength(1); - expect(records[0]).toMatchObject({ - type: "session", - version: CURRENT_SESSION_VERSION, - id: fork.sessionId, - cwd: "/workspace/project", - parentSession: "/sessions/parent.jsonl", - }); -}); - -it("copies built records in order sharing the fork header timestamp", async () => { - const sessionsDir = makeSessionsDir(); - const fork = await createForkedSessionTranscript({ - cwd: "/workspace/project", - parentSessionFile: "/sessions/parent.jsonl", - sessionsDir, - buildEntries: ({ sessionId, timestamp }) => [ - { type: "message", id: "m-1", parentId: null }, - { type: "leaf", id: "leaf-1", parentId: "m-1", timestamp, forkSessionId: sessionId }, - ], - }); - - const records = readJsonlRecords(fork.sessionFile); - expect(records.map((record) => record.type)).toEqual(["session", "message", "leaf"]); - expect(records[2]?.timestamp).toBe(records[0]?.timestamp); - expect(records[2]?.forkSessionId).toBe(fork.sessionId); -}); diff --git a/src/config/sessions/session-accessor.parent-fork.test.ts b/src/config/sessions/session-accessor.parent-fork.test.ts index 4da91d2e7411..9433a9e165e3 100644 --- a/src/config/sessions/session-accessor.parent-fork.test.ts +++ b/src/config/sessions/session-accessor.parent-fork.test.ts @@ -5,7 +5,13 @@ import path from "node:path"; import type { AssistantMessage } from "openclaw/plugin-sdk/llm"; import { afterEach, describe, expect, it } from "vitest"; import { SessionManager } from "../../agents/sessions/session-manager.js"; -import { forkSessionFromParentTranscript } from "./session-accessor.js"; +import { + forkSessionFromParentTranscript, + loadTranscriptEvents, + replaceSessionEntry, + replaceTranscriptEvents, +} from "./session-accessor.js"; +import { formatSqliteSessionFileMarker, parseSqliteSessionFileMarker } from "./sqlite-marker.js"; const roots: string[] = []; @@ -15,6 +21,41 @@ async function makeRoot(prefix: string): Promise<string> { return root; } +// Seeds the parent transcript rows into the SQLite-backed accessor so the fork +// can read the parent branch by session id, mirroring the old raw-.jsonl setup. +async function seedParentTranscript(params: { + storePath: string; + parentSessionId: string; + events: Record<string, unknown>[]; +}): Promise<void> { + await replaceTranscriptEvents( + { + agentId: "main", + sessionId: params.parentSessionId, + sessionKey: "agent:main:main", + storePath: params.storePath, + }, + params.events, + ); +} + +// Persists a child session entry so SessionManager.open can resolve the forked +// SQLite transcript (the fork writes transcript rows only, not the entry). +async function persistChildEntry(params: { + storePath: string; + sessionFile: string; + sessionId: string; +}): Promise<void> { + await replaceSessionEntry( + { sessionKey: "agent:main:child", storePath: params.storePath }, + { + sessionId: params.sessionId, + sessionFile: params.sessionFile, + updatedAt: Date.now(), + }, + ); +} + afterEach(async () => { await Promise.all(roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); }); @@ -24,11 +65,11 @@ describe("forkSessionFromParentTranscript", () => { const root = await makeRoot("openclaw-parent-fork-"); const sessionsDir = path.join(root, "sessions"); await fs.mkdir(sessionsDir); - const parentSessionFile = path.join(sessionsDir, "parent.jsonl"); + const storePath = path.join(sessionsDir, "sessions.json"); const cwd = path.join(root, "workspace"); await fs.mkdir(cwd); const parentSessionId = "parent-session"; - const lines = [ + const lines: Record<string, unknown>[] = [ { type: "session", version: 3, @@ -81,22 +122,17 @@ describe("forkSessionFromParentTranscript", () => { targetId: "label-1", }, ]; - await fs.writeFile( - parentSessionFile, - `${lines.map((entry) => JSON.stringify(entry)).join("\n")}\n`, - "utf-8", - ); + await seedParentTranscript({ storePath, parentSessionId, events: lines }); const forked = await forkSessionFromParentTranscript({ parentEntry: { sessionId: parentSessionId, - sessionFile: parentSessionFile, updatedAt: Date.now(), }, agentId: "main", parentSessionKey: "agent:main:main", sessionKey: "agent:main:child", - storePath: path.join(sessionsDir, "sessions.json"), + storePath, }); if (forked.status !== "created") { @@ -105,17 +141,22 @@ describe("forkSessionFromParentTranscript", () => { const fork = forked.transcript; expect(fork.sessionFile).toContain(sessionsDir); expect(fork.sessionId).not.toBe(parentSessionId); - const raw = await fs.readFile(fork.sessionFile, "utf-8"); - const forkedEntries = raw - .trim() - .split(/\r?\n/u) - .map((line) => JSON.parse(line) as Record<string, unknown>); - const resolvedParentSessionFile = await fs.realpath(parentSessionFile); + const forkedEntries = (await loadTranscriptEvents({ + agentId: "main", + sessionId: fork.sessionId, + sessionKey: "agent:main:child", + storePath, + })) as Record<string, unknown>[]; + const expectedParentSessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: parentSessionId, + storePath: parseSqliteSessionFileMarker(fork.sessionFile)!.storePath, + }); const forkedHeader = forkedEntries[0]; expect(forkedHeader?.type).toBe("session"); expect(forkedHeader?.id).toBe(fork.sessionId); expect(forkedHeader?.cwd).toBe(cwd); - expect(forkedHeader?.parentSession).toBe(resolvedParentSessionFile); + expect(forkedHeader?.parentSession).toBe(expectedParentSessionFile); expect(forkedEntries.map((entry) => entry.type)).toEqual([ "session", "message", @@ -132,16 +173,16 @@ describe("forkSessionFromParentTranscript", () => { targetId: "label-1", appendParentId: "label-1", }); - expect(raw).not.toContain("side delivery"); + expect(JSON.stringify(forkedEntries)).not.toContain("side delivery"); }); it("keeps opaque append-parent metadata on the active fork branch", async () => { const root = await makeRoot("openclaw-parent-fork-opaque-"); const sessionsDir = path.join(root, "sessions"); await fs.mkdir(sessionsDir); - const parentSessionFile = path.join(sessionsDir, "parent.jsonl"); + const storePath = path.join(sessionsDir, "sessions.json"); const parentSessionId = "parent-opaque"; - const entries = [ + const entries: Record<string, unknown>[] = [ { type: "session", version: 3, @@ -154,7 +195,10 @@ describe("forkSessionFromParentTranscript", () => { id: "active-root", parentId: null, timestamp: "2026-06-15T00:00:01.000Z", - message: { role: "assistant", content: "active root" }, + // Canonical assistant content is a block array; the SQLite transcript + // store round-trips it verbatim (the file-era string->text-block repair + // in normalizeLoadedFileEntry only runs on the JSONL read path). + message: { role: "assistant", content: [{ type: "text", text: "active root" }] }, }, { type: "label", @@ -187,36 +231,33 @@ describe("forkSessionFromParentTranscript", () => { appendMode: "side", }, ]; - await fs.writeFile( - parentSessionFile, - `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`, - "utf-8", - ); + await seedParentTranscript({ storePath, parentSessionId, events: entries }); const forked = await forkSessionFromParentTranscript({ parentEntry: { sessionId: parentSessionId, - sessionFile: parentSessionFile, updatedAt: Date.now(), }, agentId: "main", parentSessionKey: "agent:main:main", sessionKey: "agent:main:child", - storePath: path.join(sessionsDir, "sessions.json"), + storePath, }); if (forked.status !== "created") { throw new Error("expected forked session"); } const fork = forked.transcript; - const raw = await fs.readFile(fork.sessionFile, "utf-8"); - expect(raw).toContain('"id":"active-root"'); - expect(raw).toContain('"id":"plugin-metadata"'); - expect(raw).not.toContain("side delivery"); - const forkedRecords = raw - .trim() - .split(/\r?\n/) - .map((line) => JSON.parse(line) as Record<string, unknown>); + const forkedRecords = (await loadTranscriptEvents({ + agentId: "main", + sessionId: fork.sessionId, + sessionKey: "agent:main:child", + storePath, + })) as Record<string, unknown>[]; + const serialized = JSON.stringify(forkedRecords); + expect(serialized).toContain('"id":"active-root"'); + expect(serialized).toContain('"id":"plugin-metadata"'); + expect(serialized).not.toContain("side delivery"); expect(forkedRecords.find((entry) => entry.id === "plugin-metadata")).toMatchObject({ parentId: "active-root", }); @@ -230,12 +271,19 @@ describe("forkSessionFromParentTranscript", () => { appendParentId: "plugin-metadata", appendMode: "side", }); + await persistChildEntry({ + storePath, + sessionFile: fork.sessionFile, + sessionId: fork.sessionId, + }); const reopened = SessionManager.open(fork.sessionFile, sessionsDir); reopened.appendMessage({ role: "user", content: "continued", timestamp: Date.now() }); - const records = (await fs.readFile(fork.sessionFile, "utf-8")) - .trim() - .split(/\r?\n/) - .map((line) => JSON.parse(line) as Record<string, unknown>); + const records = (await loadTranscriptEvents({ + agentId: "main", + sessionId: fork.sessionId, + sessionKey: "agent:main:child", + storePath, + })) as Record<string, unknown>[]; expect(records.at(-1)).toMatchObject({ type: "message", parentId: "plugin-metadata" }); expect(records.at(-1)).not.toHaveProperty("appendMode"); expect(reopened.buildSessionContext().messages).toMatchObject([ @@ -248,10 +296,11 @@ describe("forkSessionFromParentTranscript", () => { const root = await makeRoot("openclaw-parent-fork-disjoint-"); const sessionsDir = path.join(root, "sessions"); await fs.mkdir(sessionsDir); - const parentSessionFile = path.join(sessionsDir, "parent.jsonl"); - await fs.writeFile( - parentSessionFile, - [ + const storePath = path.join(sessionsDir, "sessions.json"); + await seedParentTranscript({ + storePath, + parentSessionId: "parent-disjoint", + events: [ { type: "session", version: 3, @@ -285,39 +334,42 @@ describe("forkSessionFromParentTranscript", () => { targetId: "visible-assistant", appendParentId: "append-root", }, - ] - .map((entry) => JSON.stringify(entry)) - .join("\n") + "\n", - "utf-8", - ); + ], + }); const forked = await forkSessionFromParentTranscript({ parentEntry: { sessionId: "parent-disjoint", - sessionFile: parentSessionFile, updatedAt: Date.now(), }, agentId: "main", parentSessionKey: "agent:main:main", sessionKey: "agent:main:child", - storePath: path.join(sessionsDir, "sessions.json"), + storePath, }); if (forked.status !== "created") { throw new Error("expected forked session"); } const fork = forked.transcript; + await persistChildEntry({ + storePath, + sessionFile: fork.sessionFile, + sessionId: fork.sessionId, + }); const reopened = SessionManager.open(fork.sessionFile, sessionsDir); expect(reopened.buildSessionContext().messages).toHaveLength(2); reopened.appendMessage({ role: "user", content: "continued", timestamp: Date.now() }); - const raw = await fs.readFile(fork.sessionFile, "utf-8"); - expect(raw).toContain("visible question"); - expect(raw).toContain("visible answer"); - expect(raw).toContain('"id":"append-root"'); - const records = raw - .trim() - .split(/\r?\n/) - .map((line) => JSON.parse(line) as Record<string, unknown>); + const records = (await loadTranscriptEvents({ + agentId: "main", + sessionId: fork.sessionId, + sessionKey: "agent:main:child", + storePath, + })) as Record<string, unknown>[]; + const serialized = JSON.stringify(records); + expect(serialized).toContain("visible question"); + expect(serialized).toContain("visible answer"); + expect(serialized).toContain('"id":"append-root"'); expect(records.at(-1)).toMatchObject({ type: "message", parentId: "append-root" }); }); @@ -325,10 +377,11 @@ describe("forkSessionFromParentTranscript", () => { const root = await makeRoot("openclaw-parent-fork-empty-opaque-"); const sessionsDir = path.join(root, "sessions"); await fs.mkdir(sessionsDir); - const parentSessionFile = path.join(sessionsDir, "parent.jsonl"); - await fs.writeFile( - parentSessionFile, - [ + const storePath = path.join(sessionsDir, "sessions.json"); + await seedParentTranscript({ + storePath, + parentSessionId: "parent-empty-opaque", + events: [ { type: "session", version: 3, @@ -357,28 +410,29 @@ describe("forkSessionFromParentTranscript", () => { parentId: "inactive-root", payload: { source: "plugin" }, }, - ] - .map((entry) => JSON.stringify(entry)) - .join("\n") + "\n", - "utf-8", - ); + ], + }); const forked = await forkSessionFromParentTranscript({ parentEntry: { sessionId: "parent-empty-opaque", - sessionFile: parentSessionFile, updatedAt: Date.now(), }, agentId: "main", parentSessionKey: "agent:main:main", sessionKey: "agent:main:child", - storePath: path.join(sessionsDir, "sessions.json"), + storePath, }); if (forked.status !== "created") { throw new Error("expected forked session"); } const fork = forked.transcript; + await persistChildEntry({ + storePath, + sessionFile: fork.sessionFile, + sessionId: fork.sessionId, + }); const reopened = SessionManager.open(fork.sessionFile, sessionsDir); expect(reopened.buildSessionContext().messages).toEqual([]); const continuedId = reopened.appendMessage({ @@ -394,10 +448,12 @@ describe("forkSessionFromParentTranscript", () => { model: "gpt-test", timestamp: Date.now(), } as unknown as AssistantMessage); - const records = (await fs.readFile(fork.sessionFile, "utf-8")) - .trim() - .split(/\r?\n/) - .map((line) => JSON.parse(line) as Record<string, unknown>); + const records = (await loadTranscriptEvents({ + agentId: "main", + sessionId: fork.sessionId, + sessionKey: "agent:main:child", + storePath, + })) as Record<string, unknown>[]; expect(records.some((record) => record.id === "inactive-root")).toBe(false); expect(records.find((record) => record.id === continuedId)).toMatchObject({ type: "message", @@ -409,10 +465,11 @@ describe("forkSessionFromParentTranscript", () => { const root = await makeRoot("openclaw-parent-fork-missing-ancestor-"); const sessionsDir = path.join(root, "sessions"); await fs.mkdir(sessionsDir); - const parentSessionFile = path.join(sessionsDir, "parent.jsonl"); - await fs.writeFile( - parentSessionFile, - [ + const storePath = path.join(sessionsDir, "sessions.json"); + await seedParentTranscript({ + storePath, + parentSessionId: "parent-missing-ancestor", + events: [ { type: "session", version: 3, @@ -427,41 +484,44 @@ describe("forkSessionFromParentTranscript", () => { timestamp: "2026-06-15T00:00:01.000Z", message: { role: "assistant", content: "reachable tail" }, }, - ] - .map((entry) => JSON.stringify(entry)) - .join("\n") + "\n", - "utf-8", - ); + ], + }); const forked = await forkSessionFromParentTranscript({ parentEntry: { sessionId: "parent-missing-ancestor", - sessionFile: parentSessionFile, updatedAt: Date.now(), }, agentId: "main", parentSessionKey: "agent:main:main", sessionKey: "agent:main:child", - storePath: path.join(sessionsDir, "sessions.json"), + storePath, }); if (forked.status !== "created") { throw new Error("expected forked session"); } const fork = forked.transcript; - const raw = await fs.readFile(fork.sessionFile, "utf-8"); - expect(raw).toContain("reachable tail"); - expect(raw).not.toContain("missing-parent"); + const records = (await loadTranscriptEvents({ + agentId: "main", + sessionId: fork.sessionId, + sessionKey: "agent:main:child", + storePath, + })) as Record<string, unknown>[]; + const serialized = JSON.stringify(records); + expect(serialized).toContain("reachable tail"); + expect(serialized).not.toContain("missing-parent"); }); it("keeps visible history when the next append explicitly starts a root branch", async () => { const root = await makeRoot("openclaw-parent-fork-root-append-"); const sessionsDir = path.join(root, "sessions"); await fs.mkdir(sessionsDir); - const parentSessionFile = path.join(sessionsDir, "parent.jsonl"); - await fs.writeFile( - parentSessionFile, - [ + const storePath = path.join(sessionsDir, "sessions.json"); + await seedParentTranscript({ + storePath, + parentSessionId: "parent-root-append", + events: [ { type: "session", version: 3, @@ -484,35 +544,38 @@ describe("forkSessionFromParentTranscript", () => { targetId: "visible-root", appendParentId: null, }, - ] - .map((entry) => JSON.stringify(entry)) - .join("\n") + "\n", - "utf-8", - ); + ], + }); const forked = await forkSessionFromParentTranscript({ parentEntry: { sessionId: "parent-root-append", - sessionFile: parentSessionFile, updatedAt: Date.now(), }, agentId: "main", parentSessionKey: "agent:main:main", sessionKey: "agent:main:child", - storePath: path.join(sessionsDir, "sessions.json"), + storePath, }); if (forked.status !== "created") { throw new Error("expected forked session"); } const fork = forked.transcript; + await persistChildEntry({ + storePath, + sessionFile: fork.sessionFile, + sessionId: fork.sessionId, + }); const reopened = SessionManager.open(fork.sessionFile, sessionsDir); expect(reopened.buildSessionContext().messages).toHaveLength(1); reopened.appendMessage({ role: "user", content: "new root", timestamp: Date.now() }); - const records = (await fs.readFile(fork.sessionFile, "utf-8")) - .trim() - .split(/\r?\n/) - .map((line) => JSON.parse(line) as Record<string, unknown>); + const records = (await loadTranscriptEvents({ + agentId: "main", + sessionId: fork.sessionId, + sessionKey: "agent:main:child", + storePath, + })) as Record<string, unknown>[]; expect(records.at(-1)).toMatchObject({ type: "message", parentId: null }); }); @@ -520,10 +583,11 @@ describe("forkSessionFromParentTranscript", () => { const root = await makeRoot("openclaw-parent-fork-linear-"); const sessionsDir = path.join(root, "sessions"); await fs.mkdir(sessionsDir); - const parentSessionFile = path.join(sessionsDir, "parent.jsonl"); - await fs.writeFile( - parentSessionFile, - [ + const storePath = path.join(sessionsDir, "sessions.json"); + await seedParentTranscript({ + storePath, + parentSessionId: "parent-linear", + events: [ { type: "session", version: 3, @@ -549,44 +613,49 @@ describe("forkSessionFromParentTranscript", () => { parentId: "linear-assistant", payload: { source: "plugin" }, }, - ] - .map((entry) => JSON.stringify(entry)) - .join("\n") + "\n", - "utf-8", - ); + ], + }); const forked = await forkSessionFromParentTranscript({ parentEntry: { sessionId: "parent-linear", - sessionFile: parentSessionFile, updatedAt: Date.now(), }, agentId: "main", parentSessionKey: "agent:main:main", sessionKey: "agent:main:child", - storePath: path.join(sessionsDir, "sessions.json"), + storePath, }); if (forked.status !== "created") { throw new Error("expected forked session"); } const fork = forked.transcript; - const records = (await fs.readFile(fork.sessionFile, "utf-8")) - .trim() - .split(/\r?\n/) - .map((line) => JSON.parse(line) as Record<string, unknown>); + const records = (await loadTranscriptEvents({ + agentId: "main", + sessionId: fork.sessionId, + sessionKey: "agent:main:child", + storePath, + })) as Record<string, unknown>[]; expect(records.slice(1)).toMatchObject([ { id: "linear-user", parentId: null }, { id: "linear-assistant", parentId: "linear-user" }, { id: "linear-metadata", parentId: "linear-assistant" }, ]); + await persistChildEntry({ + storePath, + sessionFile: fork.sessionFile, + sessionId: fork.sessionId, + }); const reopened = SessionManager.open(fork.sessionFile, sessionsDir); expect(reopened.buildSessionContext().messages).toHaveLength(2); reopened.appendMessage({ role: "user", content: "continued", timestamp: Date.now() }); - const continuedRecords = (await fs.readFile(fork.sessionFile, "utf-8")) - .trim() - .split(/\r?\n/) - .map((line) => JSON.parse(line) as Record<string, unknown>); + const continuedRecords = (await loadTranscriptEvents({ + agentId: "main", + sessionId: fork.sessionId, + sessionKey: "agent:main:child", + storePath, + })) as Record<string, unknown>[]; expect(continuedRecords.at(-1)).toMatchObject({ type: "message", parentId: "linear-metadata", @@ -597,43 +666,52 @@ describe("forkSessionFromParentTranscript", () => { const root = await makeRoot("openclaw-parent-fork-empty-"); const sessionsDir = path.join(root, "sessions"); await fs.mkdir(sessionsDir); - const parentSessionFile = path.join(sessionsDir, "parent.jsonl"); + const storePath = path.join(sessionsDir, "sessions.json"); const parentSessionId = "parent-empty"; - await fs.writeFile( - parentSessionFile, - `${JSON.stringify({ - type: "session", - version: 3, - id: parentSessionId, - timestamp: "2026-05-01T00:00:00.000Z", - cwd: root, - })}\n`, - "utf-8", - ); + await seedParentTranscript({ + storePath, + parentSessionId, + events: [ + { + type: "session", + version: 3, + id: parentSessionId, + timestamp: "2026-05-01T00:00:00.000Z", + cwd: root, + }, + ], + }); const forked = await forkSessionFromParentTranscript({ parentEntry: { sessionId: parentSessionId, - sessionFile: parentSessionFile, updatedAt: Date.now(), }, agentId: "main", parentSessionKey: "agent:main:main", sessionKey: "agent:main:child", - storePath: path.join(sessionsDir, "sessions.json"), + storePath, }); if (forked.status !== "created") { throw new Error("expected forked session entry"); } const fork = forked.transcript; - const raw = await fs.readFile(fork.sessionFile, "utf-8"); - const lines = raw.trim().split(/\r?\n/u); - expect(lines).toHaveLength(1); - const resolvedParentSessionFile = await fs.realpath(parentSessionFile); - const header = JSON.parse(lines[0] ?? "{}") as Record<string, unknown>; - expect(header.type).toBe("session"); - expect(header.id).toBe(fork.sessionId); - expect(header.parentSession).toBe(resolvedParentSessionFile); + const records = (await loadTranscriptEvents({ + agentId: "main", + sessionId: fork.sessionId, + sessionKey: "agent:main:child", + storePath, + })) as Record<string, unknown>[]; + expect(records).toHaveLength(1); + const expectedParentSessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: parentSessionId, + storePath: parseSqliteSessionFileMarker(fork.sessionFile)!.storePath, + }); + const header = records[0]; + expect(header?.type).toBe("session"); + expect(header?.id).toBe(fork.sessionId); + expect(header?.parentSession).toBe(expectedParentSessionFile); }); }); diff --git a/src/config/sessions/session-accessor.sqlite-contract.ts b/src/config/sessions/session-accessor.sqlite-contract.ts new file mode 100644 index 000000000000..160961e75c61 --- /dev/null +++ b/src/config/sessions/session-accessor.sqlite-contract.ts @@ -0,0 +1,344 @@ +import type { AgentMessage } from "../../agents/runtime/index.js"; +import type { SessionTranscriptUpdate } from "../../sessions/transcript-events.js"; +import type { OpenClawConfig } from "../types.openclaw.js"; +import type { ResolvedSessionMaintenanceConfig } from "./store-maintenance.js"; +import type { + DeletedAgentSessionEntryPurgeParams, + DeleteSessionEntryLifecycleResult, + ResetSessionEntryLifecycleMutation, + ResetSessionEntryLifecycleResult, + SessionEntryLifecycleMutationResult, + SessionEntryLifecycleRemoval, + SessionEntryLifecycleUpsert, + SessionLifecycleArchivedTranscript, + SessionLifecycleArtifactCleanupParams, + SessionLifecycleArtifactCleanupResult, + SessionLifecycleStoreTarget, +} from "./store.js"; +import type { SessionCompactionCheckpoint, SessionEntry } from "./types.js"; + +export type SessionAccessScope = { + agentId?: string; + clone?: boolean; + env?: NodeJS.ProcessEnv; + hydrateSkillPromptRefs?: boolean; + readConsistency?: "latest"; + sessionKey: string; + storePath?: string; +}; + +export type SessionTranscriptAccessScope = Omit<SessionAccessScope, "sessionKey"> & { + sessionFile?: string; + sessionId: string; + sessionKey?: string; + threadId?: string | number; +}; + +export type SessionTranscriptRuntimeScope = SessionAccessScope & { + sessionFile?: string; + sessionId: string; + threadId?: string | number; +}; + +export type SessionTranscriptReadScope = Omit<SessionTranscriptRuntimeScope, "sessionKey"> & { + sessionKey?: string; + sessionEntry?: Pick<SessionEntry, "sessionFile"> & Partial<Pick<SessionEntry, "sessionId">>; +}; + +export type SessionTranscriptWriteScope = Omit<SessionTranscriptAccessScope, "sessionId"> & { + sessionId?: string; +}; + +export type ExactSessionEntry = { + sessionKey: string; + entry: SessionEntry; +}; + +export type SessionEntrySummary = { + sessionKey: string; + entry: SessionEntry; +}; + +export type TranscriptEvent = unknown; + +export type SessionTranscriptStats = { + eventCount: number; + maxSeq: number; + sizeBytes: number; +}; + +export type TranscriptMessageAppendOptions<TMessage> = { + config?: OpenClawConfig; + cwd?: string; + idempotencyLookup?: "scan" | "scan-assistant" | "caller-checked"; + message: TMessage; + now?: number; + eventId?: string; + parentId?: string | null; + prepareMessageAfterIdempotencyCheck?: (message: TMessage) => TMessage | undefined; + useRawWhenLinear?: boolean; +}; + +export type TranscriptMessageAppendResult<TMessage> = { + appended: boolean; + message: TMessage; + messageId: string; +}; + +export type TranscriptUpdatePayload = Partial<SessionTranscriptUpdate>; + +export type LatestTranscriptAssistantText = { + id?: string; + text: string; + timestamp?: number; +}; + +export type LatestTranscriptAssistantMessage = { + id?: string; + message: unknown; +}; + +export type LatestTranscriptMessage = { + id?: string; + message: unknown; +}; + +export type SessionTranscriptTurnMessageAppend = TranscriptMessageAppendOptions<unknown> & { + shouldAppend?: (context: SessionTranscriptTurnWriteContext) => Promise<boolean> | boolean; +}; + +export type SessionTranscriptTurnWriteContext = { + agentId?: string; + sessionFile: string; + sessionId?: string; + sessionKey?: string; + storePath?: string; +}; + +export type SessionEntryUpdateOptions = { + skipMaintenance?: boolean; + takeCacheOwnership?: boolean; + requireWriteSuccess?: boolean; +}; + +export type SessionEntryPatchOptions = { + fallbackEntry?: SessionEntry; + maintenanceConfig?: ResolvedSessionMaintenanceConfig; + preserveActivity?: boolean; + requireWriteSuccess?: boolean; + replaceEntry?: boolean; + skipMaintenance?: boolean; + takeCacheOwnership?: boolean; +}; + +export type SessionEntryPatchContext = { + existingEntry?: SessionEntry; +}; + +export type SessionEntryTargetPatchScope = { + agentId?: string; + storePath: string; + target: SessionLifecycleStoreTarget; +}; + +export type SessionEntryReplacementSnapshot = { + entry: SessionEntry; + sessionKey: string; +}; + +export type SessionEntryReplacement = { + entry: SessionEntry; + sessionKey: string; +}; + +export type SessionEntryReplacementUpdate<T> = { + replacements?: Iterable<SessionEntryReplacement>; + result: T; +}; + +export type SessionParentForkDecision = + | { + status: "fork"; + maxTokens: number; + parentTokens?: number; + } + | { + status: "skip"; + reason: "parent-too-large"; + maxTokens: number; + parentTokens: number; + message: string; + }; + +export type ParentForkedSessionTranscript = { + sessionFile: string; + sessionId: string; +}; + +export type ForkSessionFromParentTranscriptResult = + | { + status: "created"; + transcript: ParentForkedSessionTranscript; + } + | { status: "missing-parent" } + | { status: "failed" }; + +export type ForkSessionFromParentTranscriptParams = { + agentId?: string; + parentEntry: SessionEntry; + parentSessionKey: string; + sessionKey: string; + storePath: string; + + /** Stable target identity for lifecycle-owned hidden or resumable sessions. */ + targetSessionId?: string; + + /** Cross-agent forks land the child transcript in the target agent's store. */ + targetStorePath?: string; +}; + +export type ForkSessionEntryFromParentTargetResult = + | { + status: "forked"; + fork: ParentForkedSessionTranscript; + parentEntry: SessionEntry; + sessionEntry: SessionEntry; + decision: Extract<SessionParentForkDecision, { status: "fork" }>; + } + | { + status: "skipped"; + reason: "existing-entry" | "decision-skip"; + parentEntry?: SessionEntry; + sessionEntry: SessionEntry; + decision?: SessionParentForkDecision; + } + | { status: "missing-entry" } + | { status: "missing-parent" } + | { status: "failed" }; + +export type ForkSessionEntryFromParentTargetParams = { + agentId?: string; + decisionSkipPatch?: (params: { + decision: Extract<SessionParentForkDecision, { status: "skip" }>; + entry: SessionEntry; + parentEntry: SessionEntry; + }) => Partial<SessionEntry> | null; + fallbackEntry?: SessionEntry; + parentTarget: SessionLifecycleStoreTarget; + patch?: (params: { + entry: SessionEntry; + parentEntry: SessionEntry; + fork: ParentForkedSessionTranscript; + decision: Extract<SessionParentForkDecision, { status: "fork" }>; + }) => Partial<SessionEntry>; + sessionTarget: SessionLifecycleStoreTarget; + skipForkWhen?: (entry: SessionEntry) => boolean; + skipPatch?: (entry: SessionEntry) => Partial<SessionEntry> | null; + storePath: string; +}; + +export type SqliteCompactionCheckpointSessionMutationResult = + | { + status: "created"; + key: string; + checkpoint: SessionCompactionCheckpoint; + entry: SessionEntry; + } + | { status: "missing-session" } + | { status: "missing-checkpoint" } + | { status: "missing-boundary" } + | { status: "failed" }; + +export type ResetSessionEntryLifecycleParams = { + afterEntryMutation?: (mutation: ResetSessionEntryLifecycleMutation) => Promise<void> | void; + agentId?: string; + buildNextEntry: (context: { + currentEntry?: SessionEntry; + primaryKey: string; + }) => Promise<SessionEntry> | SessionEntry; + storePath: string; + target: SessionLifecycleStoreTarget; +}; + +export type DeleteSessionEntryLifecycleParams = { + agentId?: string; + archiveTranscript: boolean; + expectedEntry?: SessionEntry; + expectedSessionId?: string; + expectedLifecycleRevision?: string; + expectedUpdatedAt?: number; + storePath: string; + target: SessionLifecycleStoreTarget; +}; + +export type { + DeletedAgentSessionEntryPurgeParams, + DeleteSessionEntryLifecycleResult, + ResetSessionEntryLifecycleResult, + SessionEntryLifecycleMutationResult, + SessionEntryLifecycleRemoval, + SessionEntryLifecycleUpsert, + SessionLifecycleArchivedTranscript, + SessionLifecycleArtifactCleanupParams, + SessionLifecycleArtifactCleanupResult, +}; + +export type SessionTranscriptRuntimeTarget = { + agentId: string; + sessionFile: string; + sessionId: string; + sessionKey: string; +}; + +export type SessionTranscriptTurnPersistOptions = { + config?: OpenClawConfig; + cwd?: string; + expectedLifecycleRevision?: string; + expectedSessionId?: string; + messages: readonly SessionTranscriptTurnMessageAppend[]; + updateMode?: "inline" | "file-only" | "none"; + publishWhen?: "always" | "when-appended"; + touchSessionEntry?: boolean; +}; + +export type SessionTranscriptTurnPersistResult = { + appendedCount: number; + messages: TranscriptMessageAppendResult<unknown>[]; + rejectedReason?: "session-rebound"; + sessionEntry: SessionEntry | undefined; + sessionFile: string; +}; + +export type SessionTranscriptWriteLockAccessorContext = { + appendMessage: <TMessage>( + options: TranscriptMessageAppendOptions<TMessage>, + ) => Promise<TranscriptMessageAppendResult<TMessage> | undefined>; + readEvents: () => Promise<TranscriptEvent[]>; + replaceEvents: (events: readonly TranscriptEvent[]) => Promise<void>; +}; + +export type SessionTranscriptTurnUpdateMode = "inline" | "file-only" | "none"; + +export type SessionEntryCreateWithTranscriptContext = { + existingEntry?: SessionEntry; + sessionEntries: Record<string, SessionEntry>; +}; + +export type SessionEntryCreateWithTranscriptResult<TError = string> = + | { ok: true; entry: SessionEntry; sessionFile: string } + | { ok: false; error: TError; phase: "entry" } + | { ok: false; error: string; phase: "transcript" }; + +export type SessionEntryCreateWithTranscriptPrepareResult<TError = string> = + | { ok: true; entry: SessionEntry } + | { ok: false; error: TError }; + +export type AppendSqliteExpectedSessionTranscriptTurnParams = SessionTranscriptWriteScope & { + expectedSessionId?: string; + messages: readonly SessionTranscriptTurnMessageAppend[]; + updateMode?: SessionTranscriptTurnUpdateMode; + publishWhen?: "always" | "when-appended"; + touchSessionEntry?: boolean; +}; + +export type AgentTranscriptMessage = AgentMessage; diff --git a/src/config/sessions/session-accessor.sqlite.ts b/src/config/sessions/session-accessor.sqlite.ts new file mode 100644 index 000000000000..821eb04b73e6 --- /dev/null +++ b/src/config/sessions/session-accessor.sqlite.ts @@ -0,0 +1,5340 @@ +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { resolveTimestampMsToIsoString } from "@openclaw/normalization-core/number-coercion"; +import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; +import { sql, type Selectable } from "kysely"; +import type { AgentMessage } from "../../agents/runtime/index.js"; +import { redactTranscriptMessage } from "../../agents/transcript-redact.js"; +import { derivePromptTokens, normalizeUsage } from "../../agents/usage.js"; +import type { MsgContext } from "../../auto-reply/templating.js"; +import { resolveStoredSessionOwnerAgentId } from "../../gateway/session-store-key.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, + iterateSqliteQuerySync, +} from "../../infra/kysely-sync.js"; +import { getChildLogger } from "../../logging/logger.js"; +import { redactSecrets } from "../../logging/redact.js"; +import { + DEFAULT_AGENT_ID, + normalizeAgentId, + parseAgentSessionKey, + resolveAgentIdFromSessionKey, +} from "../../routing/session-key.js"; +import { + isValidAgentHarnessSessionStoreEntry, + resolveAgentHarnessSessionStoreEntryError, +} from "../../sessions/agent-harness-session-key.js"; +import { emitSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; +import { extractAssistantVisibleText } from "../../shared/chat-message-content.js"; +import { runQueuedStoreWrite, type StoreWriterQueue } from "../../shared/store-writer-queue.js"; +import { isTranscriptOnlyOpenClawAssistantModel } from "../../shared/transcript-only-openclaw-assistant.js"; +import type { DB as OpenClawAgentKyselyDatabase } from "../../state/openclaw-agent-db.generated.js"; +import { + openOpenClawAgentDatabase, + resolveOpenClawAgentSqlitePath, + runOpenClawAgentWriteTransaction, + type OpenClawAgentDatabase, + type OpenClawAgentDatabaseOptions, +} from "../../state/openclaw-agent-db.js"; +import type { DeliveryContext } from "../../utils/delivery-context.types.js"; +import { + encodeSessionArchiveContent, + readSessionArchiveContentSync, + SESSION_ARCHIVE_ZSTD_SUFFIX, +} from "./archive-compression.js"; +import { formatSessionArchiveTimestamp } from "./artifacts.js"; +import type { SessionDiskBudgetSweepResult } from "./disk-budget.js"; +import { isInternalSessionEffectsKey } from "./internal-session-key.js"; +import { deriveLastRoutePatch, deriveSessionMetaPatch } from "./metadata.js"; +import type { + ExactSessionEntry, + ForkSessionEntryFromParentTargetParams, + ForkSessionEntryFromParentTargetResult, + ForkSessionFromParentTranscriptParams, + ForkSessionFromParentTranscriptResult, + LatestTranscriptAssistantMessage, + LatestTranscriptMessage, + LatestTranscriptAssistantText, + SessionLifecycleArchivedTranscript, + DeleteSessionEntryLifecycleParams, + DeleteSessionEntryLifecycleResult, + DeletedAgentSessionEntryPurgeParams, + ResetSessionEntryLifecycleParams, + ResetSessionEntryLifecycleResult, + SessionAccessScope, + SessionEntryLifecycleMutationResult, + SessionEntryLifecycleRemoval, + SessionEntryLifecycleUpsert, + SessionEntryPatchContext, + SessionEntryPatchOptions, + SessionEntryReplacementSnapshot, + SessionEntryReplacementUpdate, + SessionEntrySummary, + SessionEntryTargetPatchScope, + SessionLifecycleArtifactCleanupParams, + SessionLifecycleArtifactCleanupResult, + SessionEntryUpdateOptions, + SessionTranscriptAccessScope, + SessionTranscriptReadScope, + SessionTranscriptStats, + SessionTranscriptTurnMessageAppend, + SessionTranscriptTurnWriteContext, + SessionTranscriptWriteScope, + SessionParentForkDecision, + TranscriptEvent, + TranscriptMessageAppendOptions, + TranscriptMessageAppendResult, + TranscriptUpdatePayload, +} from "./session-accessor.sqlite-contract.js"; +import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js"; +import { formatSqliteSessionFileMarker } from "./sqlite-marker.js"; +import { + foldedSessionKeyAliasCandidates, + normalizeStoreSessionKey, + resolveSessionStoreEntry, +} from "./store-entry.js"; +import { collectSessionMaintenancePreserveKeys } from "./store-maintenance-preserve.js"; +import { resolveMaintenanceConfig } from "./store-maintenance-runtime.js"; +import { + capEntryCount, + pruneStaleModelRunEntries, + pruneStaleEntries, + shouldPreserveMaintenanceEntry, + shouldRunModelRunPrune, + shouldRunSessionEntryMaintenance, + type ResolvedSessionMaintenanceConfig, +} from "./store-maintenance.js"; +import type { + ResetSessionEntryLifecycleMutation, + SessionArchivedTranscriptCleanupRule, +} from "./store.js"; +import { createSessionTranscriptHeader } from "./transcript-header.js"; +import { serializeJsonlLines } from "./transcript-jsonl.js"; +import { + isSessionTranscriptLeafControl, + mergeSessionTranscriptVisiblePathWithOpaqueAppendPath, + parseSessionTranscriptTreeEntry, + scanSessionTranscriptTree, + selectSessionTranscriptTreePathNodes, +} from "./transcript-tree.js"; +import { resolveVisibleTranscriptAppendParentId } from "./transcript-visible-events.js"; +import type { GroupKeyResolution, SessionCompactionCheckpoint, SessionEntry } from "./types.js"; +import { + mergeSessionEntry, + mergeSessionEntryPreserveActivity, + resolveFreshSessionTotalTokens, + resolveSessionTotalTokens, +} from "./types.js"; + +type SessionArchiveRuntime = typeof import("../../gateway/session-archive.runtime.js"); +let sessionArchiveRuntimePromise: Promise<SessionArchiveRuntime> | undefined; +const SQLITE_SESSION_SLOW_WRITE_MS = 1_000; + +function loadSessionArchiveRuntime() { + sessionArchiveRuntimePromise ??= import("../../gateway/session-archive.runtime.js"); + return sessionArchiveRuntimePromise; +} + +type SessionSqliteDatabase = Pick< + OpenClawAgentKyselyDatabase, + | "conversations" + | "session_conversations" + | "session_entries" + | "session_routes" + | "sessions" + | "trajectory_runtime_events" + | "transcript_event_identities" + | "transcript_events" +>; +type SessionEntryRow = Selectable<OpenClawAgentKyselyDatabase["session_entries"]>; +type ResolvedSessionEntryRow = { + entry: SessionEntry; + legacyKeys: string[]; + row: SessionEntryRow; +}; +type SqliteSessionEntrySelectionSnapshot = { + selected: ResolvedSessionEntryRow | undefined; + selectedRows: Array<{ entry: SessionEntry; sessionKey: string }>; +}; +type SqliteLifecycleTargetSnapshot = { + primary: { entry: SessionEntry; key: string } | undefined; + rows: Array<{ entry: SessionEntry; sessionKey: string }>; +}; +type SqliteSessionStateDeletePlan = { + archiveDirectory: string; + archiveTranscript: boolean; + content: string; + hadTranscriptState: boolean; + reason: "deleted" | "reset"; + sessionId: string; +}; +type SqliteSessionEntryRemovalPlan = { + expectedEntry: SessionEntry | undefined; + sessionKey: string; +}; +type SqliteSessionEntryMaintenancePlan = { + entryRemovals: SqliteSessionEntryRemovalPlan[]; + stateDeletePlans: SqliteSessionStateDeletePlan[]; +}; +type MaterializedSqliteSessionStateDeletePlan = SqliteSessionStateDeletePlan & { + archivedTranscript: SessionLifecycleArchivedTranscript | null; +}; +type SqliteLifecycleArtifactCleanupPlan = { + deletePlans: SqliteSessionStateDeletePlan[]; + entries: SqliteSessionEntryRemovalPlan[]; +}; +type SqliteProjectedLifecycleMutation = { + deletePlans: SqliteSessionStateDeletePlan[]; + removals: Array<{ + expectedEntry: SessionEntry; + removal: SessionEntryLifecycleRemoval; + sessionKey: string; + }>; + upsertedEntries: Array<{ + entry: SessionEntry; + expectedEntry: SessionEntry | undefined; + sessionKey: string; + }>; +}; +type SqliteSessionEntryPatchOptions = SessionEntryPatchOptions & { + skipMaintenance?: boolean; +}; + +class SqliteSessionMutationConflictError extends Error { + constructor(operationLabel: string) { + super(`SQLite session state changed while preparing ${operationLabel}`); + this.name = "SqliteSessionMutationConflictError"; + } +} + +type ResolvedSqliteScope = { + agentId: string; + env?: NodeJS.ProcessEnv; + path?: string; + sessionKey: string; +}; + +type ResolvedSqliteReadScope = { + agentId: string; + env?: NodeJS.ProcessEnv; + path?: string; + sessionKey?: string; +}; + +type ResolvedTranscriptScope = ResolvedSqliteScope & { + sessionId: string; +}; + +type ResolvedTranscriptReadScope = ResolvedSqliteReadScope & { + sessionId: string; +}; + +type SqliteCheckpointTranscriptForkSource = { + sessionId: string; + leafId?: string; + totalTokens?: number; +}; + +type SqliteParentForkSourceTranscript = { + appendMode?: "side"; + appendParentId: string | null; + branchEntries: TranscriptEvent[]; + cwd?: string; + labelsToWrite: Array<{ targetId: string; label: string; timestamp: string }>; + leafId: string | null; + preserveLeafControl: boolean; +}; + +type SqliteTranscriptParentTokenEstimate = { + kind: "exact-context" | "legacy-or-bytes"; + tokens: number; +}; + +/** Result from SQLite compaction checkpoint branch or restore operations. */ +export type SqliteCompactionCheckpointSessionMutationResult = + | { + status: "created"; + key: string; + checkpoint: SessionCompactionCheckpoint; + entry: SessionEntry; + } + | { status: "missing-session" } + | { status: "missing-checkpoint" } + | { status: "missing-boundary" } + | { status: "failed" }; + +/** Parameters for branching a SQLite session from a compaction checkpoint. */ +export type SqliteBranchCheckpointSessionParams = { + agentId?: string; + env?: NodeJS.ProcessEnv; + storePath?: string; + sourceKey: string; + sourceStoreKey?: string; + nextKey: string; + checkpointId: string; +}; + +/** Parameters for restoring a SQLite session from a compaction checkpoint. */ +export type SqliteRestoreCheckpointSessionParams = { + agentId?: string; + env?: NodeJS.ProcessEnv; + storePath?: string; + sessionKey: string; + sessionStoreKey?: string; + checkpointId: string; +}; + +/** Internal doctor/migration import target for one legacy session row. */ +export type SqliteSessionImportRowsParams = { + agentId?: string; + env?: NodeJS.ProcessEnv; + storePath?: string; + sessionKey: string; + entry: SessionEntry; + readTranscriptEvents?: (append: (event: TranscriptEvent) => void) => void; +}; + +/** Summary of rows written by an internal doctor/migration import. */ +export type SqliteSessionImportRowsResult = { + sessionId: string; + sessionKey: string; + transcriptEvents: number; +}; + +export type SqliteExpectedSessionTranscriptTurnResult = { + appendedMessages: TranscriptMessageAppendResult<unknown>[]; + rejectedReason?: "session-rebound"; + sessionEntry: SessionEntry | undefined; + sessionFile: string; +}; + +export type SqliteTranscriptWriteLockContext = { + appendMessage: <TMessage>( + options: TranscriptMessageAppendOptions<TMessage>, + ) => Promise<TranscriptMessageAppendResult<TMessage> | undefined>; + readEvents: () => Promise<TranscriptEvent[]>; + replaceEvents: (events: readonly TranscriptEvent[]) => Promise<void>; +}; + +const SQLITE_SESSION_WRITER_QUEUES = new Map<string, StoreWriterQueue>(); + +/** Loads one session entry from the additive SQLite session store. */ +export function loadSqliteSessionEntry(scope: SessionAccessScope): SessionEntry | undefined { + const resolved = resolveSqliteScope(scope); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + return readSessionEntryRow(database, resolved.sessionKey)?.entry; +} + +/** Loads one exact persisted-key entry from the additive SQLite session store. */ +export function loadExactSqliteSessionEntry( + scope: SessionAccessScope, +): ExactSessionEntry | undefined { + const sessionKey = scope.sessionKey.trim(); + if (!sessionKey) { + return undefined; + } + const resolved = resolveSqliteScope(scope); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const row = readExactSessionEntryRow(database, sessionKey); + return row ? { sessionKey, entry: row.entry } : undefined; +} + +/** Resolves the persisted session key for a SQLite transcript session id. */ +export function resolveSqliteSessionKeyBySessionId( + scope: Pick<SessionTranscriptReadScope, "agentId" | "env" | "sessionId" | "storePath">, +): string | undefined { + const resolved = resolveSqliteTranscriptReadScope(scope); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const db = getSessionKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_entries") + .select("session_key") + .where("session_id", "=", resolved.sessionId) + .orderBy("updated_at", "desc") + .orderBy("session_key", "asc") + .limit(1), + ); + return row?.session_key; +} + +/** Lists session entries from the additive SQLite session store. */ +export function listSqliteSessionEntries( + scope: Partial<Omit<SessionAccessScope, "sessionKey">> = {}, +): SessionEntrySummary[] { + const resolved = resolveSqliteScope({ ...scope, sessionKey: "" }); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const db = getSessionKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db + .selectFrom("session_entries") + .select(["session_key", "entry_json", "session_id", "updated_at"]) + .orderBy("session_key", "asc"), + ).rows; + return rows + .map((row) => { + if (isInternalSessionEffectsKey(row.session_key)) { + return undefined; + } + const entry = parseSessionEntryRow(row); + return entry ? { sessionKey: row.session_key, entry } : undefined; + }) + .filter((entry): entry is SessionEntrySummary => entry !== undefined); +} + +/** Reads a session activity timestamp from the additive SQLite session store. */ +export function readSqliteSessionUpdatedAt(scope: SessionAccessScope): number | undefined { + const resolved = resolveSqliteScope(scope); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const row = readSessionEntryRow(database, resolved.sessionKey)?.row; + return row ? normalizeSqliteNumber(row.updated_at) : undefined; +} + +/** Applies a partial entry update to the additive SQLite session store. */ +export async function upsertSqliteSessionEntry( + scope: SessionAccessScope, + patch: Partial<SessionEntry>, +): Promise<SessionEntry | null> { + return await patchSqliteSessionEntry(scope, () => patch, { + fallbackEntry: createFallbackSessionEntry(patch), + }); +} + +/** Replaces one entry in the additive SQLite session store. */ +export async function replaceSqliteSessionEntry( + scope: SessionAccessScope, + entry: SessionEntry, +): Promise<SessionEntry | null> { + return await patchSqliteSessionEntry(scope, () => entry, { + fallbackEntry: entry, + replaceEntry: true, + }); +} + +/** Replaces one entry synchronously for sync session runtimes. */ +export function replaceSqliteSessionEntrySync( + scope: SessionAccessScope, + entry: SessionEntry, +): void { + const resolved = resolveSqliteScope(scope); + runOpenClawAgentWriteTransaction((database) => { + writeSessionEntry(database, resolved.sessionKey, entry); + }, toDatabaseOptions(resolved)); +} + +/** Patches one entry in the additive SQLite session store. */ +export async function patchSqliteSessionEntry( + scope: SessionAccessScope, + update: ( + entry: SessionEntry, + context: SessionEntryPatchContext, + ) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null, + options: SqliteSessionEntryPatchOptions = {}, +): Promise<SessionEntry | null> { + const resolved = resolveSqliteScope(scope); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const prepared = readSqliteSessionEntrySelectionSnapshot( + database, + resolved.sessionKey, + options.replaceEntry === true, + ); + const writeBase = prepared.selected?.entry ?? options.fallbackEntry; + if (!writeBase) { + return null; + } + const patch = await update(cloneSessionEntry(writeBase), { + existingEntry: prepared.selected?.entry + ? cloneSessionEntry(prepared.selected.entry) + : undefined, + }); + const maintenancePlans: SqliteSessionEntryMaintenancePlan[] = []; + let result: SessionEntry | null = null; + runOpenClawAgentWriteTransaction((writeDatabase) => { + const fresh = readSqliteSessionEntrySelectionSnapshot( + writeDatabase, + resolved.sessionKey, + options.replaceEntry === true, + ); + assertSqliteSessionEntrySelectionUnchanged(prepared, fresh, "session-entry.patch"); + if (!patch) { + result = cloneSessionEntry(writeBase); + return; + } + const merged = options.replaceEntry + ? cloneSessionEntry(patch as SessionEntry) + : options.preserveActivity + ? mergeSessionEntryPreserveActivity(writeBase, patch) + : mergeSessionEntry(writeBase, patch); + const next = options.replaceEntry + ? merged + : preserveSqliteSameKeySessionRolloverLineage({ + next: merged, + previous: writeBase, + sessionKey: resolved.sessionKey, + }); + writeSessionEntry(writeDatabase, resolved.sessionKey, next); + deleteLegacySessionEntryRows( + writeDatabase, + fresh.selected?.legacyKeys ?? [], + resolved.sessionKey, + ); + maintenancePlans.push( + applySqliteSessionEntryMaintenance(writeDatabase, { + activeSessionKey: resolved.sessionKey, + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + maintenanceConfig: options.maintenanceConfig, + skipMaintenance: options.skipMaintenance, + }), + ); + result = cloneSessionEntry(next); + }, toDatabaseOptions(resolved)); + finalizeSqliteSessionEntryMaintenancePlansBestEffort(resolved, maintenancePlans); + return result; + }); +} + +/** Patches one logical entry selected from a canonical key and alias set. */ +export async function patchSqliteSessionEntryTarget( + scope: SessionEntryTargetPatchScope, + update: ( + entry: SessionEntry, + context: SessionEntryPatchContext, + ) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null, + options: SqliteSessionEntryPatchOptions = {}, +): Promise<SessionEntry | null> { + const resolved = resolveSqliteStoreScope(scope.storePath, { agentId: scope.agentId }); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const prepared = readSqliteLifecycleTargetSnapshot(database, scope.target); + const writeBase = prepared.primary?.entry ?? options.fallbackEntry; + if (!writeBase) { + return null; + } + const patch = await update(cloneSessionEntry(writeBase), { + existingEntry: prepared.primary?.entry + ? cloneSessionEntry(prepared.primary.entry) + : undefined, + }); + const maintenancePlans: SqliteSessionEntryMaintenancePlan[] = []; + let result: SessionEntry | null = null; + runOpenClawAgentWriteTransaction((writeDatabase) => { + const fresh = readSqliteLifecycleTargetSnapshot(writeDatabase, scope.target); + assertSqliteLifecycleTargetSnapshotUnchanged(prepared, fresh, "session-entry-target.patch"); + if (!patch) { + result = cloneSessionEntry(writeBase); + return; + } + const merged = options.replaceEntry + ? cloneSessionEntry(patch as SessionEntry) + : options.preserveActivity + ? mergeSessionEntryPreserveActivity(writeBase, patch) + : mergeSessionEntry(writeBase, patch); + const next = options.replaceEntry + ? merged + : preserveSqliteSameKeySessionRolloverLineage({ + next: merged, + previous: writeBase, + sessionKey: scope.target.canonicalKey, + }); + deleteSqliteLifecycleTargetRows(writeDatabase, scope.target); + writeSessionEntry(writeDatabase, scope.target.canonicalKey, next); + maintenancePlans.push( + applySqliteSessionEntryMaintenance(writeDatabase, { + activeSessionKey: scope.target.canonicalKey, + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + maintenanceConfig: options.maintenanceConfig, + skipMaintenance: options.skipMaintenance, + }), + ); + result = cloneSessionEntry(next); + }, toDatabaseOptions(resolved)); + finalizeSqliteSessionEntryMaintenancePlansBestEffort(resolved, maintenancePlans); + return result; + }); +} + +/** Forks one parent SQLite transcript into a new child transcript. */ +export async function forkSqliteSessionTranscriptFromParent( + params: ForkSessionFromParentTranscriptParams, +): Promise<ForkSessionFromParentTranscriptResult> { + const resolved = resolveSqliteScope({ + ...(params.agentId ? { agentId: params.agentId } : {}), + sessionKey: params.sessionKey, + storePath: params.storePath, + }); + const target = params.targetStorePath + ? resolveSqliteScope({ sessionKey: params.sessionKey, storePath: params.targetStorePath }) + : resolved; + const crossDatabase = + target.agentId !== resolved.agentId || (target.path ?? "") !== (resolved.path ?? ""); + if (!crossDatabase) { + return await runExclusiveSqliteSessionWrite(resolved, async () => { + let result: ForkSessionFromParentTranscriptResult = { status: "failed" }; + runOpenClawAgentWriteTransaction((database) => { + result = forkSqliteParentTranscriptInTransaction(database, resolved, { + parentEntry: params.parentEntry, + parentSessionKey: params.parentSessionKey, + targetSessionId: params.targetSessionId, + targetSessionKey: params.sessionKey, + }); + }, toDatabaseOptions(resolved)); + return result; + }); + } + // Cross-agent fork (worktree/cross-agent sessions.create): parent rows live + // in the source agent database while the child transcript must be owned by + // the target agent's database. Two databases cannot share one transaction, + // so read the parent branch first, then write the child under the target's + // exclusive session write lock. + if (!params.parentEntry.sessionId) { + return { status: "missing-parent" }; + } + const sourceDatabase = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const source = readSqliteParentForkSourceTranscript(sourceDatabase, params.parentEntry.sessionId); + if (!source) { + return { status: "failed" }; + } + const parentSessionFile = formatSqliteSessionMarkerForScope({ + ...resolved, + sessionId: params.parentEntry.sessionId, + sessionKey: normalizeSqliteSessionKey(params.parentSessionKey), + }); + return await runExclusiveSqliteSessionWrite(target, async () => { + const sessionId = params.targetSessionId ?? randomUUID(); + const targetScope = { + ...target, + sessionId, + sessionKey: normalizeSqliteSessionKey(params.sessionKey), + }; + const sessionFile = formatSqliteSessionMarkerForScope(targetScope); + runOpenClawAgentWriteTransaction((database) => { + writeSqliteForkedChildTranscriptInTransaction(database, targetScope, { + parentSessionFile, + source, + }); + }, toDatabaseOptions(target)); + return { status: "created", transcript: { sessionFile, sessionId } }; + }); +} + +/** Forks parent context into a child session entry using SQLite rows only. */ +export async function forkSqliteSessionEntryFromParentTarget( + params: ForkSessionEntryFromParentTargetParams, +): Promise<ForkSessionEntryFromParentTargetResult> { + const resolved = resolveSqliteStoreScope(params.storePath, { agentId: params.agentId }); + const parentTarget = normalizeSqliteLifecycleTarget(params.parentTarget); + const sessionTarget = normalizeSqliteLifecycleTarget(params.sessionTarget); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const parent = resolveSqliteLifecyclePrimaryEntry(database, parentTarget); + if (!parent?.entry.sessionId) { + return { status: "missing-parent" }; + } + + const existing = resolveSqliteLifecyclePrimaryEntry(database, sessionTarget); + const base = existing?.entry ?? params.fallbackEntry; + if (!base) { + return { status: "missing-entry" }; + } + + if (params.skipForkWhen?.(cloneSessionEntry(base))) { + const sessionEntry = await persistSqliteParentForkSkipPatch({ + entry: base, + params, + sessionTarget, + patch: params.skipPatch?.(cloneSessionEntry(base)), + resolved, + }); + return { + status: "skipped", + reason: "existing-entry", + parentEntry: cloneSessionEntry(parent.entry), + sessionEntry, + }; + } + + const needsTranscriptTokenEstimate = + typeof resolveFreshSessionTotalTokens(parent.entry) !== "number" && + typeof parent.entry.sessionId === "string" && + parent.entry.sessionId.length > 0; + const transcriptParentTokens = needsTranscriptTokenEstimate + ? estimateSqliteTranscriptPromptTokens( + loadSqliteTranscriptEventsFromDatabase(database, parent.entry.sessionId), + ) + : undefined; + const decision = resolveSqliteParentForkDecision(parent.entry, transcriptParentTokens); + if (decision.status === "skip") { + const patch = params.decisionSkipPatch?.({ + decision, + entry: cloneSessionEntry(base), + parentEntry: cloneSessionEntry(parent.entry), + }); + const sessionEntry = await persistSqliteParentForkSkipPatch({ + entry: base, + params, + sessionTarget, + patch, + resolved, + }); + return { + status: "skipped", + reason: "decision-skip", + parentEntry: cloneSessionEntry(parent.entry), + sessionEntry, + decision, + }; + } + + let result: ForkSessionEntryFromParentTargetResult = { status: "failed" }; + const maintenancePlans: SqliteSessionEntryMaintenancePlan[] = []; + runOpenClawAgentWriteTransaction((writeDatabase) => { + const freshParent = resolveSqliteLifecyclePrimaryEntry(writeDatabase, parentTarget)?.entry; + if (!freshParent?.sessionId) { + result = { status: "missing-parent" }; + return; + } + const freshExisting = resolveSqliteLifecyclePrimaryEntry(writeDatabase, sessionTarget); + const freshBase = freshExisting?.entry ?? params.fallbackEntry; + if (!freshBase) { + result = { status: "missing-entry" }; + return; + } + const fork = forkSqliteParentTranscriptInTransaction(writeDatabase, resolved, { + parentEntry: freshParent, + parentSessionKey: parentTarget.canonicalKey, + targetSessionKey: sessionTarget.canonicalKey, + }); + if (fork.status !== "created") { + result = + fork.status === "missing-parent" ? { status: "missing-parent" } : { status: "failed" }; + return; + } + const patch = params.patch?.({ + decision, + entry: cloneSessionEntry(freshBase), + fork: fork.transcript, + parentEntry: cloneSessionEntry(freshParent), + }); + const next = mergeSessionEntry(freshBase, { + ...patch, + forkedFromParent: true, + sessionFile: fork.transcript.sessionFile, + sessionId: fork.transcript.sessionId, + }); + deleteSqliteLifecycleTargetRows(writeDatabase, sessionTarget); + writeSessionEntry(writeDatabase, sessionTarget.canonicalKey, next); + maintenancePlans.push( + applySqliteSessionEntryMaintenance(writeDatabase, { + activeSessionKey: sessionTarget.canonicalKey, + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + skipMaintenance: true, + }), + ); + result = { + status: "forked", + decision, + fork: fork.transcript, + parentEntry: cloneSessionEntry(freshParent), + sessionEntry: cloneSessionEntry(next), + }; + }, toDatabaseOptions(resolved)); + finalizeSqliteSessionEntryMaintenancePlansBestEffort(resolved, maintenancePlans); + return result; + }); +} + +async function persistSqliteParentForkSkipPatch(params: { + entry: SessionEntry; + params: ForkSessionEntryFromParentTargetParams; + sessionTarget: { canonicalKey: string; storeKeys: string[] }; + patch: Partial<SessionEntry> | null | undefined; + resolved: ResolvedSqliteScope; +}): Promise<SessionEntry> { + if (!params.patch) { + return cloneSessionEntry(params.entry); + } + const merged = mergeSessionEntry(params.entry, params.patch); + const next = preserveSqliteSameKeySessionRolloverLineage({ + next: merged, + previous: params.entry, + sessionKey: params.sessionTarget.canonicalKey, + }); + const maintenancePlans: SqliteSessionEntryMaintenancePlan[] = []; + runOpenClawAgentWriteTransaction((database) => { + deleteSqliteLifecycleTargetRows(database, params.sessionTarget); + writeSessionEntry(database, params.sessionTarget.canonicalKey, next); + maintenancePlans.push( + applySqliteSessionEntryMaintenance(database, { + activeSessionKey: params.sessionTarget.canonicalKey, + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(params.resolved), + skipMaintenance: true, + }), + ); + }, toDatabaseOptions(params.resolved)); + finalizeSqliteSessionEntryMaintenancePlansBestEffort(params.resolved, maintenancePlans); + return cloneSessionEntry(next); +} + +/** Updates an existing entry in the additive SQLite session store. */ +export async function updateSqliteSessionEntry( + scope: SessionAccessScope, + update: ( + entry: SessionEntry, + ) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null, + options: SessionEntryUpdateOptions = {}, +): Promise<SessionEntry | null> { + const resolved = resolveSqliteScope(scope); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const prepared = readSqliteSessionEntrySelectionSnapshot(database, resolved.sessionKey, false); + const writeBase = prepared.selected?.entry; + if (!writeBase) { + return null; + } + const patch = await update(cloneSessionEntry(writeBase)); + const maintenancePlans: SqliteSessionEntryMaintenancePlan[] = []; + let result: SessionEntry | null = null; + runOpenClawAgentWriteTransaction((writeDatabase) => { + const fresh = readSqliteSessionEntrySelectionSnapshot( + writeDatabase, + resolved.sessionKey, + false, + ); + assertSqliteSessionEntrySelectionUnchanged(prepared, fresh, "session-entry.update"); + if (!patch) { + result = cloneSessionEntry(writeBase); + return; + } + const merged = mergeSessionEntry(writeBase, patch); + const next = preserveSqliteSameKeySessionRolloverLineage({ + next: merged, + previous: writeBase, + sessionKey: resolved.sessionKey, + }); + writeSessionEntry(writeDatabase, resolved.sessionKey, next); + deleteLegacySessionEntryRows( + writeDatabase, + fresh.selected?.legacyKeys ?? [], + resolved.sessionKey, + ); + maintenancePlans.push( + applySqliteSessionEntryMaintenance(writeDatabase, { + activeSessionKey: resolved.sessionKey, + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + skipMaintenance: options.skipMaintenance, + }), + ); + result = cloneSessionEntry(next); + }, toDatabaseOptions(resolved)); + finalizeSqliteSessionEntryMaintenancePlansBestEffort(resolved, maintenancePlans); + return result; + }); +} + +/** Cleans scoped session lifecycle rows and associated SQLite transcript state. */ +export async function cleanupSqliteSessionLifecycleArtifacts( + params: SessionLifecycleArtifactCleanupParams, +): Promise<SessionLifecycleArtifactCleanupResult> { + const sessionKeySegmentPrefix = params.sessionKeySegmentPrefix.trim(); + const transcriptContentMarker = params.transcriptContentMarker; + if (!sessionKeySegmentPrefix || !transcriptContentMarker) { + return { removedEntries: 0, archivedTranscriptArtifacts: 0 }; + } + + const resolved = resolveSqliteReadScope({ + ...(params.agentId ? { agentId: params.agentId } : {}), + storePath: params.storePath, + }); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const cleanupPlan = planSqliteSessionLifecycleArtifactCleanup(database, { + archiveRemovedEntryTranscripts: params.archiveRemovedEntryTranscripts !== false, + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + sessionKeySegmentPrefix, + transcriptContentMarker, + orphanTranscriptMinAgeMs: params.orphanTranscriptMinAgeMs, + nowMs: params.nowMs ?? Date.now(), + }); + const materializedPlans = materializeSqliteSessionStateDeletePlans(cleanupPlan.deletePlans); + let removedEntries = 0; + let archivedTranscripts: SessionLifecycleArchivedTranscript[] = []; + runOpenClawAgentWriteTransaction((transactionDb) => { + removedEntries = deletePlannedSqliteLifecycleArtifactEntries( + transactionDb, + cleanupPlan.entries, + ); + archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( + transactionDb, + materializedPlans, + ); + }, toDatabaseOptions(resolved)); + return { + removedEntries, + archivedTranscriptArtifacts: archivedTranscripts.length, + }; + }); +} + +/** Resets one persisted session entry using SQLite session rows. */ +export async function resetSqliteSessionEntryLifecycle( + params: ResetSessionEntryLifecycleParams, +): Promise<ResetSessionEntryLifecycleResult> { + const resolved = resolveSqliteStoreScope(params.storePath, { agentId: params.agentId }); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const current = resolveSqliteLifecyclePrimaryEntry(database, params.target); + const nextEntry = await params.buildNextEntry({ + currentEntry: current ? cloneSessionEntry(current.entry) : undefined, + primaryKey: params.target.canonicalKey, + }); + const mutation: ResetSessionEntryLifecycleMutation = { + nextEntry: cloneSessionEntry(nextEntry), + ...(current ? { previousEntry: cloneSessionEntry(current.entry) } : {}), + ...(current?.entry.sessionFile ? { previousSessionFile: current.entry.sessionFile } : {}), + ...(current?.entry.sessionId ? { previousSessionId: current.entry.sessionId } : {}), + }; + let archivedTranscripts: SessionLifecycleArchivedTranscript[] = []; + const referencedAfterReset = current?.entry.sessionId + ? readReferencedSqliteSessionIdsAfterTargetMutation(database, params.target, nextEntry) + : new Set<string>(); + const deletePlans = current?.entry.sessionId + ? planSqliteSessionStateAfterEntryRemoval({ + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + database, + entry: current.entry, + reason: "reset", + referencedSessionIds: referencedAfterReset, + }) + : []; + const materializedPlans = materializeSqliteSessionStateDeletePlans(deletePlans); + runOpenClawAgentWriteTransaction((transactionDb) => { + assertSqliteLifecycleTargetUnchanged(transactionDb, params.target, current?.entry, "reset"); + deleteSqliteLifecycleTargetRows(transactionDb, params.target); + writeSessionEntry(transactionDb, params.target.canonicalKey, nextEntry); + archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( + transactionDb, + materializedPlans, + ); + }, toDatabaseOptions(resolved)); + await params.afterEntryMutation?.(mutation); + emitArchivedSqliteTranscriptUpdates(archivedTranscripts); + return { + ...mutation, + archivedTranscripts, + }; + }); +} + +const MODEL_SELECTION_LOCK_REMOVAL_MESSAGE = + "Model-selection-locked sessions cannot be removed, unlocked, or reassigned."; + +async function deleteSqliteSessionEntryLifecycleInternal( + params: DeleteSessionEntryLifecycleParams, + allowLockedEntryRemoval: boolean, +): Promise<DeleteSessionEntryLifecycleResult> { + const resolved = resolveSqliteStoreScope(params.storePath, { agentId: params.agentId }); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + let result: DeleteSessionEntryLifecycleResult = { + archivedTranscripts: [], + deleted: false, + }; + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const current = resolveSqliteLifecyclePrimaryEntry(database, params.target); + if (!current) { + return result; + } + if (current.entry.modelSelectionLocked === true && !allowLockedEntryRemoval) { + throw new Error(MODEL_SELECTION_LOCK_REMOVAL_MESSAGE); + } + const referencedAfterDelete = readReferencedSqliteSessionIdsAfterTargetMutation( + database, + params.target, + ); + const deletePlans = params.archiveTranscript + ? planSqliteSessionStateAfterEntryRemoval({ + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + archiveTranscript: true, + database, + entry: current.entry, + reason: "deleted", + referencedSessionIds: referencedAfterDelete, + }) + : []; + const materializedPlans = materializeSqliteSessionStateDeletePlans(deletePlans); + runOpenClawAgentWriteTransaction((transactionDb) => { + const transactionEntry = resolveSqliteLifecyclePrimaryEntry( + transactionDb, + params.target, + )?.entry; + if ( + !sqliteSessionEntriesEqual(transactionEntry, current.entry) || + !shouldDeleteSqliteSessionEntryLifecycle(transactionEntry, params) + ) { + return; + } + deleteSqliteLifecycleTargetRows(transactionDb, params.target); + const archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( + transactionDb, + materializedPlans, + ); + result = { + archivedTranscripts, + deleted: true, + deletedEntry: cloneSessionEntry(current.entry), + ...(current.entry.sessionFile ? { deletedSessionFile: current.entry.sessionFile } : {}), + ...(current.entry.sessionId ? { deletedSessionId: current.entry.sessionId } : {}), + }; + }, toDatabaseOptions(resolved)); + emitArchivedSqliteTranscriptUpdates(result.archivedTranscripts); + return result; + }); +} + +/** Deletes one persisted session entry using SQLite session rows. */ +export async function deleteSqliteSessionEntryLifecycle( + params: DeleteSessionEntryLifecycleParams, +): Promise<DeleteSessionEntryLifecycleResult> { + return await deleteSqliteSessionEntryLifecycleInternal(params, false); +} + +/** Rolls back one exact locked row created by failed trusted harness initialization. */ +export async function rollbackSqliteAgentHarnessSessionEntryLifecycle( + params: DeleteSessionEntryLifecycleParams & { expectedEntry: SessionEntry }, +): Promise<DeleteSessionEntryLifecycleResult> { + const hasExactTarget = + params.target.storeKeys.length === 1 && + params.target.storeKeys[0] === params.target.canonicalKey; + const expectedEntryError = resolveAgentHarnessSessionStoreEntryError( + params.target.canonicalKey, + params.expectedEntry, + ); + if ( + !hasExactTarget || + expectedEntryError || + !isValidAgentHarnessSessionStoreEntry(params.target.canonicalKey, params.expectedEntry) + ) { + throw new Error(expectedEntryError ?? MODEL_SELECTION_LOCK_REMOVAL_MESSAGE); + } + return await deleteSqliteSessionEntryLifecycleInternal(params, true); +} + +/** Applies prepared full-row replacements in one validated SQLite transaction. */ +export async function applySqliteSessionEntryReplacements<T>(params: { + activeSessionKey?: string; + agentId?: string; + requireWriteSuccess?: boolean; + sessionKeys?: readonly string[]; + skipMaintenance?: boolean; + storePath: string; + update: ( + entries: SessionEntryReplacementSnapshot[], + ) => Promise<SessionEntryReplacementUpdate<T>> | SessionEntryReplacementUpdate<T>; +}): Promise<T> { + const resolved = resolveSqliteScope({ + ...(params.agentId ? { agentId: params.agentId } : {}), + sessionKey: params.activeSessionKey ?? params.sessionKeys?.[0] ?? "", + storePath: params.storePath, + }); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const store = readSqliteSessionEntryStore(database); + const selectedKeys = params.sessionKeys ? new Set(params.sessionKeys) : undefined; + const entries = selectedKeys + ? [...selectedKeys].flatMap((sessionKey) => { + const entry = store[sessionKey]; + return entry ? [{ entry: cloneSessionEntry(entry), sessionKey }] : []; + }) + : Object.entries(store).map(([sessionKey, entry]) => ({ + entry: cloneSessionEntry(entry), + sessionKey, + })); + const operation = await params.update( + entries.map(({ entry, sessionKey }) => ({ + entry: cloneSessionEntry(entry), + sessionKey, + })), + ); + const replacements = [...(operation.replacements ?? [])]; + for (const replacement of replacements) { + if (selectedKeys && !selectedKeys.has(replacement.sessionKey)) { + throw new Error( + `Session entry replacement is outside the selected key set: ${replacement.sessionKey}`, + ); + } + } + + const expectedEntries = new Map(entries.map(({ sessionKey, entry }) => [sessionKey, entry])); + const applicable = replacements.filter((replacement) => + expectedEntries.has(replacement.sessionKey), + ); + if (params.requireWriteSuccess && replacements.length > 0 && applicable.length === 0) { + throw new Error("session entry replacements did not persist any rows"); + } + if (applicable.length === 0) { + return operation.result; + } + + const maintenancePlans: SqliteSessionEntryMaintenancePlan[] = []; + runOpenClawAgentWriteTransaction( + (transactionDb) => { + for (const replacement of applicable) { + const current = readExactSessionEntryRow(transactionDb, replacement.sessionKey)?.entry; + if (!sqliteSessionEntriesEqual(current, expectedEntries.get(replacement.sessionKey))) { + throw new Error( + `SQLite session entry changed before replacement for ${replacement.sessionKey}`, + ); + } + } + for (const replacement of applicable) { + writeSessionEntry( + transactionDb, + replacement.sessionKey, + cloneSessionEntry(replacement.entry), + ); + } + maintenancePlans.push( + applySqliteSessionEntryMaintenance(transactionDb, { + activeSessionKey: params.activeSessionKey ?? "", + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + skipMaintenance: params.skipMaintenance ?? true, + }), + ); + }, + toDatabaseOptions(resolved), + { operationLabel: "session.entry-replacements" }, + ); + finalizeSqliteSessionEntryMaintenancePlansBestEffort(resolved, maintenancePlans); + return operation.result; + }); +} + +/** Applies exact lifecycle removals/upserts using SQLite session rows. */ +export async function applySqliteSessionEntryLifecycleMutation(params: { + agentId?: string; + storePath: string; + removals?: Iterable<SessionEntryLifecycleRemoval>; + upserts?: Iterable<SessionEntryLifecycleUpsert>; + activeSessionKey?: string; + maintenanceOverride?: Partial<ResolvedSessionMaintenanceConfig>; + skipMaintenance?: boolean; + cleanupArchivedTranscripts?: { + rules: SessionArchivedTranscriptCleanupRule[]; + nowMs?: number; + }; + captureArtifactCleanupError?: boolean; +}): Promise<SessionEntryLifecycleMutationResult> { + const resolved = resolveSqliteScope({ + ...(params.agentId ? { agentId: params.agentId } : {}), + sessionKey: "", + storePath: params.storePath, + }); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + const removals = [...(params.removals ?? [])]; + const upserts = [...(params.upserts ?? [])]; + const removedSessionKeys: string[] = []; + let archivedTranscripts: SessionLifecycleArchivedTranscript[] = []; + const maintenancePlans: SqliteSessionEntryMaintenancePlan[] = []; + let artifactCleanupError: unknown; + const captureArtifactCleanupError = (error: unknown): void => { + if (params.captureArtifactCleanupError === true) { + artifactCleanupError ??= error; + return; + } + throw error; + }; + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const projected = await projectSqliteSessionEntryLifecycleMutation(database, { + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + removals, + upserts, + }); + let materializedRemovalPlans: MaterializedSqliteSessionStateDeletePlan[] = []; + try { + materializedRemovalPlans = materializeSqliteSessionStateDeletePlans(projected.deletePlans); + } catch (error) { + captureArtifactCleanupError(error); + } + runOpenClawAgentWriteTransaction((transactionDb) => { + for (const removal of projected.removals) { + const entry = readExactSessionEntryRow(transactionDb, removal.sessionKey)?.entry; + if (!sqliteSessionEntriesEqual(entry, removal.expectedEntry)) { + throw new Error( + `SQLite session entry changed before lifecycle removal for ${removal.sessionKey}`, + ); + } + if (!shouldRemoveSqliteSessionEntry(entry, removal.removal)) { + continue; + } + deleteSqliteSessionEntryRows(transactionDb, removal.sessionKey); + removedSessionKeys.push(removal.sessionKey); + } + for (const { sessionKey, entry, expectedEntry } of projected.upsertedEntries) { + const currentEntry = readExactSessionEntryRow(transactionDb, sessionKey)?.entry; + if (!sqliteSessionEntriesEqual(currentEntry, expectedEntry)) { + throw new Error(`SQLite session entry changed before lifecycle upsert for ${sessionKey}`); + } + writeSessionEntry(transactionDb, sessionKey, entry); + } + maintenancePlans.push( + applySqliteSessionEntryMaintenance(transactionDb, { + activeSessionKey: params.activeSessionKey ?? "", + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + forceMaintenance: params.maintenanceOverride !== undefined, + maintenanceConfig: params.maintenanceOverride + ? { ...resolveMaintenanceConfig(), ...params.maintenanceOverride } + : undefined, + skipMaintenance: params.skipMaintenance, + }), + ); + archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( + transactionDb, + materializedRemovalPlans, + ); + }, toDatabaseOptions(resolved)); + const maintenanceArchivedTranscripts = finalizeSqliteSessionEntryMaintenancePlansBestEffort( + resolved, + maintenancePlans, + ); + archivedTranscripts = [...archivedTranscripts, ...maintenanceArchivedTranscripts]; + const afterCount = readSqliteSessionEntryCount( + openOpenClawAgentDatabase(toDatabaseOptions(resolved)), + ); + emitArchivedSqliteTranscriptUpdates(archivedTranscripts); + const archivedTranscriptDirectories = uniqueStrings( + archivedTranscripts.map((transcript) => path.dirname(transcript.archivedPath)), + ).toSorted(); + if (archivedTranscriptDirectories.length > 0 && params.cleanupArchivedTranscripts) { + try { + const { cleanupArchivedSessionTranscripts } = await loadSessionArchiveRuntime(); + await cleanupArchivedSessionTranscripts({ + directories: archivedTranscriptDirectories, + rules: params.cleanupArchivedTranscripts.rules, + nowMs: params.cleanupArchivedTranscripts.nowMs, + }); + } catch (error) { + captureArtifactCleanupError(error); + } + } + return { + removedEntries: removedSessionKeys.length, + removedSessionKeys, + archivedTranscriptDirectories, + unreferencedArtifacts: null, + maintenanceReport: null, + afterCount, + artifactCleanupError, + }; + }); +} + +/** Purges entries owned by a deleted agent from SQLite session rows. */ +export async function purgeSqliteDeletedAgentSessionEntries( + params: DeletedAgentSessionEntryPurgeParams, +): Promise<SessionEntryLifecycleMutationResult> { + const resolved = resolveSqliteStoreScope(params.storePath, { agentId: params.storeAgentId }); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const store = readSqliteSessionEntryStore(database); + const remainingStore = { ...store }; + const entryRemovals: SqliteSessionEntryRemovalPlan[] = []; + const removedEntriesToArchive: SessionEntry[] = []; + for (const sessionKey of Object.keys(store)) { + const ownerAgentId = resolveStoredSessionOwnerAgentId({ + cfg: params.cfg, + agentId: params.storeAgentId, + sessionKey, + }); + if (ownerAgentId !== params.agentId) { + continue; + } + const entry = store[sessionKey]; + entryRemovals.push({ expectedEntry: cloneSessionEntry(entry), sessionKey }); + removedEntriesToArchive.push(entry); + delete remainingStore[sessionKey]; + } + const referencedSessionIds = collectProjectedReferencedSqliteSessionIds({ + database, + excludedSessionKeys: entryRemovals.map((removal) => removal.sessionKey), + projectedStore: remainingStore, + }); + const deletePlans = removedEntriesToArchive.flatMap((entry) => + planSqliteSessionStateAfterEntryRemoval({ + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + database, + entry, + reason: "deleted", + referencedSessionIds, + }), + ); + const materializedPlans = materializeSqliteSessionStateDeletePlans(deletePlans); + const removedSessionKeys = entryRemovals.map((removal) => removal.sessionKey); + let archivedTranscripts: SessionLifecycleArchivedTranscript[] = []; + const maintenancePlans: SqliteSessionEntryMaintenancePlan[] = []; + runOpenClawAgentWriteTransaction((transactionDb) => { + deletePlannedSqliteLifecycleArtifactEntries(transactionDb, entryRemovals); + maintenancePlans.push( + applySqliteSessionEntryMaintenance(transactionDb, { + activeSessionKey: "", + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + }), + ); + archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( + transactionDb, + materializedPlans, + ); + }, toDatabaseOptions(resolved)); + archivedTranscripts = [ + ...archivedTranscripts, + ...finalizeSqliteSessionEntryMaintenancePlansBestEffort(resolved, maintenancePlans), + ]; + const afterCount = readSqliteSessionEntryCount( + openOpenClawAgentDatabase(toDatabaseOptions(resolved)), + ); + emitArchivedSqliteTranscriptUpdates(archivedTranscripts); + return { + removedEntries: removedSessionKeys.length, + removedSessionKeys, + archivedTranscriptDirectories: uniqueStrings( + archivedTranscripts.map((transcript) => path.dirname(transcript.archivedPath)), + ).toSorted(), + unreferencedArtifacts: null, + maintenanceReport: null, + afterCount, + }; + }); +} + +/** Loads raw transcript events from the additive SQLite transcript store. */ +export async function loadSqliteTranscriptEvents( + scope: SessionTranscriptReadScope, +): Promise<TranscriptEvent[]> { + return loadSqliteTranscriptEventsSync(scope); +} + +/** Loads raw transcript events synchronously from the additive SQLite transcript store. */ +export function loadSqliteTranscriptEventsSync( + scope: SessionTranscriptReadScope, +): TranscriptEvent[] { + const resolved = resolveSqliteTranscriptReadScope(scope); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + return loadSqliteTranscriptEventsFromDatabase(database, resolved.sessionId); +} + +function loadSqliteTranscriptEventsFromDatabase( + database: OpenClawAgentDatabase, + sessionId: string, +): TranscriptEvent[] { + const db = getSessionKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db + .selectFrom("transcript_events") + .select(["event_json"]) + .where("session_id", "=", sessionId) + .orderBy("seq", "asc"), + ).rows; + return rows.map((row) => JSON.parse(row.event_json) as TranscriptEvent); +} + +function sqliteTranscriptJsonlByteSize() { + return /* kysely-allow-raw: JSONL size includes event bytes plus newline separators. */ sql<number>`COALESCE(SUM(LENGTH(CAST(event_json AS BLOB))), 0) + + CASE WHEN COUNT(*) > 0 THEN COUNT(*) - 1 ELSE 0 END`.as("size_bytes"); +} + +/** Reads transcript freshness and byte size without materializing event rows. */ +export function readSqliteTranscriptStatsSync( + scope: SessionTranscriptReadScope, +): SessionTranscriptStats { + const resolved = resolveSqliteTranscriptReadScope(scope); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const db = getSessionKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("transcript_events") + .select((eb) => [ + eb.fn.count<number>("seq").as("event_count"), + eb.fn.max<number>("seq").as("max_seq"), + sqliteTranscriptJsonlByteSize(), + ]) + .where("session_id", "=", resolved.sessionId), + ); + return { + eventCount: row?.event_count ?? 0, + maxSeq: row?.max_seq ?? 0, + sizeBytes: row?.size_bytes ?? 0, + }; +} + +function readTranscriptEventJsonSetInTransaction( + database: OpenClawAgentDatabase, + sessionId: string, +): Set<string> { + const db = getSessionKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db.selectFrom("transcript_events").select("event_json").where("session_id", "=", sessionId), + ).rows; + return new Set(rows.map((row) => row.event_json)); +} + +/** Reads the latest visible assistant text from SQLite transcript rows in reverse order. */ +export function loadLatestSqliteAssistantText( + scope: SessionTranscriptReadScope, + options: { includeTranscriptOnlyOpenClawAssistant?: boolean } = {}, +): LatestTranscriptAssistantText | undefined { + const resolved = resolveSqliteTranscriptReadScope(scope); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const db = getSessionKysely(database.db); + const rows = iterateSqliteQuerySync( + database.db, + db + .selectFrom("transcript_events as te") + .innerJoin("transcript_event_identities as ti", (join) => + join.onRef("ti.session_id", "=", "te.session_id").onRef("ti.seq", "=", "te.seq"), + ) + .select("te.event_json as event_json") + .where("te.session_id", "=", resolved.sessionId) + .where("ti.event_type", "=", "message") + .orderBy("te.seq", "desc"), + ); + for (const row of rows) { + const latest = parseLatestAssistantMessageEvent(row.event_json, options); + if (!latest) { + continue; + } + const text = parseLatestAssistantText(latest); + if (text) { + return text; + } + } + return undefined; +} + +/** Reads the latest assistant message payload from SQLite transcript rows in reverse order. */ +export function loadLatestSqliteAssistantMessage( + scope: SessionTranscriptReadScope, + options: { includeTranscriptOnlyOpenClawAssistant?: boolean } = {}, +): LatestTranscriptAssistantMessage | undefined { + const resolved = resolveSqliteTranscriptReadScope(scope); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const db = getSessionKysely(database.db); + const rows = iterateSqliteQuerySync( + database.db, + db + .selectFrom("transcript_events as te") + .innerJoin("transcript_event_identities as ti", (join) => + join.onRef("ti.session_id", "=", "te.session_id").onRef("ti.seq", "=", "te.seq"), + ) + .select("te.event_json as event_json") + .where("te.session_id", "=", resolved.sessionId) + .where("ti.event_type", "=", "message") + .orderBy("te.seq", "desc"), + ); + for (const row of rows) { + const latest = parseLatestAssistantMessageEvent(row.event_json, options); + if (latest) { + return latest; + } + } + return undefined; +} + +/** Reads the newest transcript message payload from SQLite transcript rows. */ +export function loadLatestSqliteMessage( + scope: SessionTranscriptReadScope, + options: { includeTranscriptOnlyOpenClawAssistant?: boolean } = {}, +): LatestTranscriptMessage | undefined { + const resolved = resolveSqliteTranscriptReadScope(scope); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const db = getSessionKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("transcript_events as te") + .innerJoin("transcript_event_identities as ti", (join) => + join.onRef("ti.session_id", "=", "te.session_id").onRef("ti.seq", "=", "te.seq"), + ) + .select("te.event_json as event_json") + .where("te.session_id", "=", resolved.sessionId) + .where("ti.event_type", "=", "message") + .orderBy("te.seq", "desc") + .limit(1), + ); + return row ? parseLatestMessageEvent(row.event_json, options) : undefined; +} + +function parseLatestAssistantText( + latest: LatestTranscriptAssistantMessage, +): LatestTranscriptAssistantText | undefined { + const message = latest.message as { + timestamp?: unknown; + }; + const text = extractAssistantVisibleText(latest.message)?.trim(); + if (!text) { + return undefined; + } + return { + ...(latest.id ? { id: latest.id } : {}), + text, + ...(typeof message.timestamp === "number" && Number.isFinite(message.timestamp) + ? { timestamp: message.timestamp } + : {}), + }; +} + +function parseLatestAssistantMessageEvent( + raw: string, + options: { includeTranscriptOnlyOpenClawAssistant?: boolean } = {}, +): LatestTranscriptAssistantMessage | undefined { + let parsed: { + id?: unknown; + message?: { + model?: unknown; + provider?: unknown; + role?: unknown; + timestamp?: unknown; + }; + }; + try { + parsed = JSON.parse(raw) as typeof parsed; + } catch { + return undefined; + } + const message = parsed.message; + if (!message || message.role !== "assistant") { + return undefined; + } + if ( + !options.includeTranscriptOnlyOpenClawAssistant && + isTranscriptOnlyOpenClawAssistantModel(message.provider, message.model) + ) { + return undefined; + } + return { + ...(typeof parsed.id === "string" && parsed.id.trim() ? { id: parsed.id } : {}), + message, + }; +} + +function parseLatestMessageEvent( + raw: string, + options: { includeTranscriptOnlyOpenClawAssistant?: boolean } = {}, +): LatestTranscriptMessage | undefined { + let parsed: { + id?: unknown; + message?: { + model?: unknown; + provider?: unknown; + role?: unknown; + }; + }; + try { + parsed = JSON.parse(raw) as typeof parsed; + } catch { + return undefined; + } + const message = parsed.message; + if (!message || typeof message.role !== "string") { + return undefined; + } + if ( + message.role === "assistant" && + !options.includeTranscriptOnlyOpenClawAssistant && + isTranscriptOnlyOpenClawAssistantModel(message.provider, message.model) + ) { + return undefined; + } + return { + ...(typeof parsed.id === "string" && parsed.id.trim() ? { id: parsed.id } : {}), + message, + }; +} + +/** Checks whether the additive SQLite transcript store has rows for a transcript. */ +export function sqliteTranscriptExists(scope: SessionTranscriptReadScope): boolean { + const resolved = resolveSqliteTranscriptReadScope(scope); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const db = getSessionKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("transcript_events") + .select("seq") + .where("session_id", "=", resolved.sessionId) + .limit(1), + ); + return row !== undefined; +} + +/** Deletes rows for one transcript from the additive SQLite transcript store. */ +export async function deleteSqliteTranscript(scope: SessionTranscriptReadScope): Promise<boolean> { + const resolved = resolveSqliteTranscriptReadScope(scope); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + let deleted = false; + runOpenClawAgentWriteTransaction((database) => { + deleted = deleteSqliteTranscriptEventsInTransaction(database, resolved.sessionId); + }, toDatabaseOptions(resolved)); + return deleted; + }); +} + +/** Fully replaces rows for one transcript in the additive SQLite transcript store. */ +export async function replaceSqliteTranscriptEvents( + scope: SessionTranscriptAccessScope, + events: TranscriptEvent[], +): Promise<void> { + const resolved = resolveSqliteTranscriptScope(scope); + await runExclusiveSqliteSessionWrite(resolved, async () => { + runOpenClawAgentWriteTransaction((database) => { + replaceSqliteTranscriptEventsInTransaction(database, resolved, events); + }, toDatabaseOptions(resolved)); + }); +} + +/** Fully replaces rows for one transcript synchronously for sync session runtimes. */ +export function replaceSqliteTranscriptEventsSync( + scope: SessionTranscriptAccessScope, + events: TranscriptEvent[], +): boolean { + const resolved = resolveSqliteTranscriptScope(scope); + let replaced = false; + runOpenClawAgentWriteTransaction((database) => { + const fresh = readSessionEntryRow(database, resolved.sessionKey); + if (!fresh || fresh.entry.sessionId !== resolved.sessionId) { + return; + } + replaceSqliteTranscriptEventsInTransaction(database, resolved, events); + replaced = true; + }, toDatabaseOptions(resolved)); + return replaced; +} + +/** Imports one legacy session entry and its transcript rows for doctor migration. */ +export async function importSqliteSessionRows( + params: SqliteSessionImportRowsParams, +): Promise<SqliteSessionImportRowsResult> { + const resolved = resolveSqliteScope({ + ...(params.agentId ? { agentId: params.agentId } : {}), + ...(params.env ? { env: params.env } : {}), + sessionKey: params.sessionKey, + ...(params.storePath ? { storePath: params.storePath } : {}), + }); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + let transcriptEvents = 0; + runOpenClawAgentWriteTransaction((database) => { + const importedEntry = { + ...params.entry, + sessionFile: formatSqliteSessionMarkerForScope({ + ...resolved, + sessionId: params.entry.sessionId, + }), + }; + writeSessionEntry(database, resolved.sessionKey, importedEntry); + if (params.readTranscriptEvents) { + const transcriptScope = { + ...resolved, + sessionId: params.entry.sessionId, + }; + const existingEventJson = readTranscriptEventJsonSetInTransaction( + database, + params.entry.sessionId, + ); + params.readTranscriptEvents((event) => { + const eventJson = JSON.stringify(event); + if (existingEventJson.has(eventJson)) { + return; + } + if (appendTranscriptEventInTransaction(database, transcriptScope, event)) { + existingEventJson.add(eventJson); + transcriptEvents += 1; + } + }); + } + }, toDatabaseOptions(resolved)); + return { + sessionId: params.entry.sessionId, + sessionKey: resolved.sessionKey, + transcriptEvents, + }; + }); +} + +/** Appends one raw transcript event to the additive SQLite transcript store. */ +export async function appendSqliteTranscriptEvent( + scope: SessionTranscriptAccessScope, + event: TranscriptEvent, +): Promise<void> { + assertNonMessageTranscriptEvent(event); + const resolved = resolveSqliteTranscriptScope(scope); + await runExclusiveSqliteSessionWrite(resolved, async () => { + runOpenClawAgentWriteTransaction((database) => { + appendTranscriptEventInTransaction(database, resolved, event); + }, toDatabaseOptions(resolved)); + }); +} + +/** Appends one raw non-message transcript event synchronously for sync session runtimes. */ +export function appendSqliteTranscriptEventSync( + scope: SessionTranscriptAccessScope, + event: TranscriptEvent, +): void { + assertNonMessageTranscriptEvent(event); + const resolved = resolveSqliteTranscriptScope(scope); + runOpenClawAgentWriteTransaction((database) => { + const fresh = readSessionEntryRow(database, resolved.sessionKey); + if (!fresh || fresh.entry.sessionId !== resolved.sessionId) { + return; + } + appendTranscriptEventInTransaction(database, resolved, event); + }, toDatabaseOptions(resolved)); +} + +/** Appends raw transcript events to the additive SQLite transcript store in one transaction. */ +export async function appendSqliteTranscriptEvents( + scope: SessionTranscriptAccessScope, + events: TranscriptEvent[], +): Promise<void> { + if (events.length === 0) { + return; + } + const resolved = resolveSqliteTranscriptScope(scope); + await runExclusiveSqliteSessionWrite(resolved, async () => { + runOpenClawAgentWriteTransaction((database) => { + for (const event of events) { + appendTranscriptEventInTransaction(database, resolved, event); + } + }, toDatabaseOptions(resolved)); + }); +} + +/** Appends a guarded transcript turn and touches its session row in one queued write. */ +export async function appendSqliteExpectedSessionTranscriptTurn( + scope: SessionTranscriptWriteScope, + options: { + config?: import("../types.openclaw.js").OpenClawConfig; + cwd?: string; + expectedLifecycleRevision?: string; + expectedSessionId: string; + messages: readonly SessionTranscriptTurnMessageAppend[]; + sessionFile: string; + touchSessionEntry?: boolean; + }, +): Promise<SqliteExpectedSessionTranscriptTurnResult> { + const resolved = resolveSqliteTranscriptScope({ + ...scope, + sessionId: options.expectedSessionId, + }); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const preparedEntry = readSessionEntryRow(database, resolved.sessionKey); + if (!sqliteSessionMatchesExpectedTranscriptTurn(preparedEntry, options)) { + return sqliteSessionTranscriptTurnRebound(preparedEntry, options.sessionFile); + } + const messages = await selectAppendableSqliteTranscriptTurnMessages( + { + agentId: resolved.agentId, + sessionFile: options.sessionFile, + sessionId: options.expectedSessionId, + sessionKey: resolved.sessionKey, + ...(scope.storePath ? { storePath: scope.storePath } : {}), + }, + options.messages, + ); + let result: SqliteExpectedSessionTranscriptTurnResult = sqliteSessionTranscriptTurnRebound( + preparedEntry, + options.sessionFile, + ); + runOpenClawAgentWriteTransaction((transactionDb) => { + const fresh = readSessionEntryRow(transactionDb, resolved.sessionKey); + if (!sqliteSessionMatchesExpectedTranscriptTurn(fresh, options)) { + result = sqliteSessionTranscriptTurnRebound(fresh, options.sessionFile); + return; + } + const appendedMessages: TranscriptMessageAppendResult<unknown>[] = []; + for (const append of messages) { + const { shouldAppend: _shouldAppend, ...appendOptions } = append; + const appended = appendSqliteTranscriptMessageInTransaction(transactionDb, resolved, { + ...appendOptions, + ...((append.cwd ?? options.cwd) ? { cwd: append.cwd ?? options.cwd } : {}), + ...((append.config ?? options.config) ? { config: append.config ?? options.config } : {}), + }); + if (appended) { + appendedMessages.push(appended); + } + } + + const appendedCount = appendedMessages.filter((message) => message.appended).length; + const touchUpdatedAt = + options.touchSessionEntry === true && appendedCount > 0 ? Date.now() : undefined; + const sessionPatch: Partial<SessionEntry> = { + ...(fresh.entry.sessionFile === options.sessionFile + ? {} + : { sessionFile: options.sessionFile }), + ...(touchUpdatedAt !== undefined + ? { updatedAt: Math.max(fresh.entry.updatedAt ?? 0, touchUpdatedAt) } + : {}), + }; + const next = + Object.keys(sessionPatch).length > 0 + ? mergeSessionEntry(fresh.entry, sessionPatch) + : fresh.entry; + if (next !== fresh.entry) { + writeSessionEntry(transactionDb, resolved.sessionKey, next); + deleteLegacySessionEntryRows(transactionDb, fresh.legacyKeys, resolved.sessionKey); + } + result = { + appendedMessages, + sessionEntry: cloneSessionEntry(next), + sessionFile: options.sessionFile, + }; + }, toDatabaseOptions(resolved)); + return result; + }); +} + +function sqliteSessionMatchesExpectedTranscriptTurn( + selected: ResolvedSessionEntryRow | undefined, + expected: { expectedLifecycleRevision?: string; expectedSessionId: string }, +): selected is ResolvedSessionEntryRow { + return Boolean( + selected && + selected.entry.sessionId === expected.expectedSessionId && + (expected.expectedLifecycleRevision === undefined || + selected.entry.lifecycleRevision === expected.expectedLifecycleRevision), + ); +} + +function sqliteSessionTranscriptTurnRebound( + selected: ResolvedSessionEntryRow | undefined, + sessionFile: string, +): SqliteExpectedSessionTranscriptTurnResult { + return { + appendedMessages: [], + rejectedReason: "session-rebound", + sessionEntry: selected?.entry, + sessionFile, + }; +} + +async function selectAppendableSqliteTranscriptTurnMessages( + context: SessionTranscriptTurnWriteContext, + messages: readonly SessionTranscriptTurnMessageAppend[], +): Promise<SessionTranscriptTurnMessageAppend[]> { + const selected: SessionTranscriptTurnMessageAppend[] = []; + for (const append of messages) { + const shouldAppend = append.shouldAppend ? await append.shouldAppend(context) : true; + if (shouldAppend) { + selected.push(append); + } + } + return selected; +} + +/** Appends one transcript message to the additive SQLite transcript store. */ +export async function appendSqliteTranscriptMessage<TMessage>( + scope: SessionTranscriptWriteScope, + options: TranscriptMessageAppendOptions<TMessage> & { + prepareMessageAfterIdempotencyCheck: (message: TMessage) => TMessage | undefined; + }, +): Promise<TranscriptMessageAppendResult<TMessage> | undefined>; +export async function appendSqliteTranscriptMessage<TMessage>( + scope: SessionTranscriptWriteScope, + options: TranscriptMessageAppendOptions<TMessage>, +): Promise<TranscriptMessageAppendResult<TMessage>>; +export async function appendSqliteTranscriptMessage<TMessage>( + scope: SessionTranscriptWriteScope, + options: TranscriptMessageAppendOptions<TMessage>, +): Promise<TranscriptMessageAppendResult<TMessage> | undefined> { + const resolved = resolveSqliteTranscriptScope(scope); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + let result: TranscriptMessageAppendResult<TMessage> | undefined; + runOpenClawAgentWriteTransaction((database) => { + result = appendSqliteTranscriptMessageInTransaction(database, resolved, options); + }, toDatabaseOptions(resolved)); + return result; + }); +} + +/** Appends one transcript message synchronously for sync session runtimes. */ +export function appendSqliteTranscriptMessageSync<TMessage>( + scope: SessionTranscriptWriteScope, + options: TranscriptMessageAppendOptions<TMessage>, +): TranscriptMessageAppendResult<TMessage> | undefined { + const resolved = resolveSqliteTranscriptScope(scope); + let result: TranscriptMessageAppendResult<TMessage> | undefined; + runOpenClawAgentWriteTransaction((database) => { + const fresh = readSessionEntryRow(database, resolved.sessionKey); + if (!fresh || fresh.entry.sessionId !== resolved.sessionId) { + return; + } + result = appendSqliteTranscriptMessageInTransaction(database, resolved, options); + }, toDatabaseOptions(resolved)); + return result; +} + +/** Runs read/append transcript work under one SQLite writer-queue critical section. */ +export async function withSqliteTranscriptWriteLock<T>( + scope: SessionTranscriptWriteScope, + run: (context: SqliteTranscriptWriteLockContext) => Promise<T> | T, +): Promise<T> { + const resolved = resolveSqliteTranscriptScope(scope); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + return await run({ + readEvents: async () => loadSqliteTranscriptEventsFromDatabase(database, resolved.sessionId), + replaceEvents: async (events) => { + runOpenClawAgentWriteTransaction((writeDatabase) => { + replaceSqliteTranscriptEventsInTransaction(writeDatabase, resolved, events); + }, toDatabaseOptions(resolved)); + }, + appendMessage: async (options) => { + let result: TranscriptMessageAppendResult<unknown> | undefined; + runOpenClawAgentWriteTransaction((writeDatabase) => { + result = appendSqliteTranscriptMessageInTransaction(writeDatabase, resolved, options); + }, toDatabaseOptions(resolved)); + return result as TranscriptMessageAppendResult<typeof options.message> | undefined; + }, + }); + }); +} + +function appendSqliteTranscriptMessageInTransaction<TMessage>( + database: OpenClawAgentDatabase, + resolved: ResolvedTranscriptScope, + options: TranscriptMessageAppendOptions<TMessage>, +): TranscriptMessageAppendResult<TMessage> | undefined { + const idempotencyKey = readMessageIdempotencyKey(options.message); + if (idempotencyKey && options.idempotencyLookup !== "caller-checked") { + const existing = readTranscriptMessageByScopedIdempotencyKey( + database, + resolved, + idempotencyKey, + options.idempotencyLookup, + ); + if (existing) { + return { + appended: false, + message: existing.message as TMessage, + messageId: existing.messageId, + }; + } + } + + const prepared = options.prepareMessageAfterIdempotencyCheck + ? options.prepareMessageAfterIdempotencyCheck(options.message) + : options.message; + if (prepared === undefined) { + return undefined; + } + + const messageId = options.eventId ?? randomUUID(); + const now = options.now ?? Date.now(); + const finalMessage = redactTranscriptMessageForStorage(prepared, options); + ensureTranscriptHeader(database, resolved, options.cwd, now); + const parentId = + options.parentId === undefined + ? readActiveTranscriptAppendParentId(database, resolved.sessionId) + : options.parentId; + const event = { + type: "message", + id: messageId, + parentId: parentId ?? null, + timestamp: resolveTimestampMsToIsoString(now), + message: finalMessage, + }; + const appended = appendTranscriptEventInTransaction(database, resolved, event, { + dedupeByMessageIdempotency: + options.idempotencyLookup !== "caller-checked" && + options.idempotencyLookup !== "scan-assistant", + }); + if (!appended && idempotencyKey && options.idempotencyLookup !== "caller-checked") { + const existing = readTranscriptMessageByScopedIdempotencyKey( + database, + resolved, + idempotencyKey, + options.idempotencyLookup, + ); + if (existing) { + return { + appended: false, + message: existing.message as TMessage, + messageId: existing.messageId, + }; + } + } + if (!appended) { + const existing = readTranscriptMessageByEventId(database, resolved, messageId); + if (existing) { + return { + appended: false, + message: existing.message as TMessage, + messageId: existing.messageId, + }; + } + } + if (!appended) { + throw new Error(`SQLite transcript append did not insert message ${messageId}.`); + } + return { + appended: true, + message: finalMessage, + messageId, + }; +} + +/** Branches a SQLite session from a compaction checkpoint in one queued transaction. */ +export async function branchSqliteCompactionCheckpointSession( + params: SqliteBranchCheckpointSessionParams, +): Promise<SqliteCompactionCheckpointSessionMutationResult> { + const sourceKey = normalizeSqliteSessionKey(params.sourceStoreKey ?? params.sourceKey); + const targetKey = normalizeSqliteSessionKey(params.nextKey); + const resolved = resolveSqliteScope({ + ...(params.agentId ? { agentId: params.agentId } : {}), + ...(params.env ? { env: params.env } : {}), + sessionKey: sourceKey, + ...(params.storePath ? { storePath: params.storePath } : {}), + }); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + let result: SqliteCompactionCheckpointSessionMutationResult | undefined; + runOpenClawAgentWriteTransaction((database) => { + result = branchSqliteCompactionCheckpointSessionInTransaction(database, { + checkpointId: params.checkpointId, + parentSessionKey: normalizeSqliteSessionKey(params.sourceKey), + resolved, + sourceKey, + targetKey, + }); + }, toDatabaseOptions(resolved)); + return result ?? { status: "failed" }; + }); +} + +/** Restores a SQLite session from a compaction checkpoint in one queued transaction. */ +export async function restoreSqliteCompactionCheckpointSession( + params: SqliteRestoreCheckpointSessionParams, +): Promise<SqliteCompactionCheckpointSessionMutationResult> { + const sessionKey = normalizeSqliteSessionKey(params.sessionStoreKey ?? params.sessionKey); + const targetKey = normalizeSqliteSessionKey(params.sessionKey); + const resolved = resolveSqliteScope({ + ...(params.agentId ? { agentId: params.agentId } : {}), + ...(params.env ? { env: params.env } : {}), + sessionKey, + ...(params.storePath ? { storePath: params.storePath } : {}), + }); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + let result: SqliteCompactionCheckpointSessionMutationResult | undefined; + runOpenClawAgentWriteTransaction((database) => { + result = restoreSqliteCompactionCheckpointSessionInTransaction(database, { + checkpointId: params.checkpointId, + resolved, + sourceKey: sessionKey, + targetKey, + }); + }, toDatabaseOptions(resolved)); + return result ?? { status: "failed" }; + }); +} + +/** Publishes a transcript update using the SQLite transcript scope target. */ +export async function publishSqliteTranscriptUpdate( + scope: SessionTranscriptWriteScope, + update: TranscriptUpdatePayload = {}, +): Promise<void> { + const resolved = resolveSqliteTranscriptScope(scope); + emitSessionTranscriptUpdate({ + ...update, + agentId: resolved.agentId, + sessionKey: resolved.sessionKey, + sessionId: resolved.sessionId, + target: { + agentId: resolved.agentId, + sessionId: resolved.sessionId, + sessionKey: resolved.sessionKey, + }, + }); +} + +function getSessionKysely(database: import("node:sqlite").DatabaseSync) { + return getNodeSqliteKysely<SessionSqliteDatabase>(database); +} + +async function runExclusiveSqliteSessionWrite<T>( + scope: Pick<ResolvedSqliteReadScope, "agentId" | "env" | "path">, + fn: () => Promise<T>, +): Promise<T> { + const databaseOptions = toDatabaseOptions(scope); + const storePath = resolveOpenClawAgentSqlitePath(databaseOptions); + const startedAt = Date.now(); + try { + const result = await runQueuedStoreWrite({ + queues: SQLITE_SESSION_WRITER_QUEUES, + storePath, + label: "runExclusiveSqliteSessionWrite", + fn, + }); + const elapsedMs = Date.now() - startedAt; + if (elapsedMs >= SQLITE_SESSION_SLOW_WRITE_MS) { + getChildLogger({ subsystem: "session-sqlite" }).warn("slow SQLite session write", { + agentId: scope.agentId, + elapsedMs, + storePath, + }); + } + return result; + } catch (error) { + getChildLogger({ subsystem: "session-sqlite" }).warn("SQLite session write failed", { + agentId: scope.agentId, + elapsedMs: Date.now() - startedAt, + error, + storePath, + }); + throw error; + } +} + +function resolveSqliteScope( + scope: Pick<SessionAccessScope, "agentId" | "env" | "sessionKey" | "storePath">, +): ResolvedSqliteScope { + const scopedAgentId = resolveExplicitSqliteAgentId(scope); + const storeTarget = scope.storePath + ? resolveSqliteTargetFromSessionStorePath(scope.storePath, { agentId: scopedAgentId }) + : undefined; + const agentId = resolveSqliteAgentId({ + scopedAgentId, + sessionKey: scope.sessionKey, + storeAgentId: storeTarget?.agentId, + useDefaultAgentForUnownedStore: Boolean( + storeTarget?.path && !storeTarget.agentId && !scopedAgentId, + ), + }); + if (!agentId) { + throw new Error("Cannot resolve SQLite session scope without an agent id"); + } + return { + agentId, + ...(scope.env ? { env: scope.env } : {}), + ...(storeTarget ? { path: storeTarget.path } : {}), + sessionKey: normalizeSqliteSessionKey(scope.sessionKey), + }; +} + +function resolveSqliteReadScope( + scope: Pick<SessionTranscriptReadScope, "agentId" | "env" | "sessionKey" | "storePath">, +): ResolvedSqliteReadScope { + const sessionKey = scope.sessionKey ? normalizeSqliteSessionKey(scope.sessionKey) : undefined; + const scopedAgentId = resolveExplicitSqliteAgentId({ ...scope, sessionKey }); + const storeTarget = scope.storePath + ? resolveSqliteTargetFromSessionStorePath(scope.storePath, { agentId: scopedAgentId }) + : undefined; + const agentId = resolveSqliteAgentId({ + scopedAgentId, + sessionKey, + storeAgentId: storeTarget?.agentId, + useDefaultAgentForUnownedStore: Boolean( + storeTarget?.path && !storeTarget.agentId && !scopedAgentId, + ), + }); + if (!agentId) { + throw new Error("Cannot resolve SQLite transcript read scope without an agent id"); + } + return { + agentId, + ...(scope.env ? { env: scope.env } : {}), + ...(storeTarget ? { path: storeTarget.path } : {}), + ...(sessionKey ? { sessionKey } : {}), + }; +} + +function resolveExplicitSqliteAgentId(params: { + agentId?: string; + sessionKey?: string; +}): string | undefined { + return params.agentId + ? normalizeAgentId(params.agentId) + : parseAgentSessionKey(params.sessionKey)?.agentId; +} + +function resolveSqliteStoreScope( + storePath: string, + options: { agentId?: string } = {}, +): ResolvedSqliteScope { + return resolveSqliteScope({ + ...(options.agentId ? { agentId: options.agentId } : {}), + sessionKey: "", + storePath, + }); +} + +function resolveSqliteAgentId(params: { + scopedAgentId?: string; + sessionKey?: string; + storeAgentId?: string; + useDefaultAgentForUnownedStore?: boolean; +}): string | undefined { + const scopedAgentId = params.scopedAgentId ? normalizeAgentId(params.scopedAgentId) : undefined; + if (scopedAgentId && params.storeAgentId && scopedAgentId !== params.storeAgentId) { + throw new Error( + `SQLite session store path belongs to agent ${params.storeAgentId}; requested agent ${scopedAgentId}.`, + ); + } + const resolved = + scopedAgentId ?? + params.storeAgentId ?? + (params.sessionKey !== undefined ? resolveAgentIdFromSessionKey(params.sessionKey) : undefined); + return resolved ?? (params.useDefaultAgentForUnownedStore ? DEFAULT_AGENT_ID : undefined); +} + +function resolveSqliteTranscriptArchiveDirectory( + scope: Pick<ResolvedSqliteReadScope, "agentId" | "env" | "path">, +): string { + const databasePath = resolveOpenClawAgentSqlitePath(toDatabaseOptions(scope)); + const databaseDir = path.dirname(databasePath); + if (path.basename(databaseDir) !== "agent") { + return databaseDir; + } + return path.join(path.dirname(databaseDir), "sessions"); +} + +function resolveSqliteTranscriptScope( + scope: Pick< + SessionTranscriptWriteScope, + "agentId" | "env" | "sessionId" | "sessionKey" | "storePath" + >, +): ResolvedTranscriptScope { + if (!scope.sessionId) { + throw new Error( + `Cannot resolve SQLite transcript scope without a session id: ${scope.sessionKey}`, + ); + } + if (!scope.sessionKey) { + throw new Error( + `Cannot resolve SQLite transcript scope without a session key: ${scope.sessionId}`, + ); + } + return { + ...resolveSqliteScope({ + ...scope, + sessionKey: scope.sessionKey, + }), + sessionId: scope.sessionId, + }; +} + +function resolveSqliteTranscriptReadScope( + scope: Pick< + SessionTranscriptReadScope, + "agentId" | "env" | "sessionId" | "sessionKey" | "storePath" + >, +): ResolvedTranscriptReadScope { + return { + ...resolveSqliteReadScope(scope), + sessionId: scope.sessionId, + }; +} + +function toDatabaseOptions( + scope: Pick<ResolvedSqliteReadScope, "agentId" | "env" | "path">, +): OpenClawAgentDatabaseOptions { + return { + agentId: scope.agentId, + ...(scope.env ? { env: scope.env } : {}), + ...(scope.path ? { path: scope.path } : {}), + }; +} + +function normalizeSqliteSessionKey(sessionKey: string): string { + return normalizeStoreSessionKey(sessionKey); +} + +function createFallbackSessionEntry(patch: Partial<SessionEntry>): SessionEntry { + const now = Date.now(); + return { + sessionId: patch.sessionId ?? randomUUID(), + updatedAt: patch.updatedAt ?? now, + ...patch, + }; +} + +function cloneSessionEntry(entry: SessionEntry): SessionEntry { + return structuredClone(entry); +} + +function preserveSqliteSameKeySessionRolloverLineage(params: { + next: SessionEntry; + previous: SessionEntry; + sessionKey: string; +}): SessionEntry { + const previousSessionId = params.previous.sessionId.trim(); + const nextSessionId = params.next.sessionId.trim(); + if (!previousSessionId || !nextSessionId || previousSessionId === nextSessionId) { + return params.next; + } + + return { + ...params.next, + usageFamilyKey: + params.next.usageFamilyKey ?? params.previous.usageFamilyKey ?? params.sessionKey, + usageFamilySessionIds: uniqueStrings([ + ...(params.previous.usageFamilySessionIds ?? []), + previousSessionId, + ...(params.next.usageFamilySessionIds ?? []), + nextSessionId, + ]), + }; +} + +function normalizeSqliteText(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function normalizeSqliteChatType(value: unknown): "direct" | "group" | "channel" | null { + if (value === "direct" || value === "group" || value === "channel") { + return value; + } + return null; +} + +function normalizeSqliteStatus( + value: unknown, +): "running" | "done" | "failed" | "killed" | "timeout" | null { + if ( + value === "running" || + value === "done" || + value === "failed" || + value === "killed" || + value === "timeout" + ) { + return value; + } + return null; +} + +function normalizeSqliteNumber(value: number | bigint): number { + return typeof value === "bigint" ? Number(value) : value; +} + +function parseSessionEntryRow(row: Pick<SessionEntryRow, "entry_json">): SessionEntry | null { + try { + const parsed = JSON.parse(row.entry_json) as unknown; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as SessionEntry) + : null; + } catch { + return null; + } +} + +function assertNonMessageTranscriptEvent(event: TranscriptEvent): void { + if (!event || typeof event !== "object" || Array.isArray(event)) { + return; + } + // Message records require parent-link, idempotency, and redaction handling + // from appendSqliteTranscriptMessage; raw event writes would bypass those invariants. + if ((event as { type?: unknown }).type === "message") { + throw new Error( + "appendSqliteTranscriptEvent cannot write message transcript records; use appendSqliteTranscriptMessage instead.", + ); + } +} + +function readSessionEntryRow( + database: OpenClawAgentDatabase, + sessionKey: string, +): ResolvedSessionEntryRow | undefined { + const db = getSessionKysely(database.db); + const lookupKeys = collectSessionEntryLookupKeys(database, sessionKey); + if (lookupKeys.length === 0) { + return undefined; + } + const rows = executeSqliteQuerySync( + database.db, + db + .selectFrom("session_entries") + .selectAll() + .where("session_key", "in", lookupKeys) + .orderBy("session_key", "asc"), + ).rows; + const entries = new Map<string, ResolvedSessionEntryRow>(); + const store: Record<string, SessionEntry> = {}; + for (const row of rows) { + const entry = parseSessionEntryRow(row); + if (!entry) { + continue; + } + store[row.session_key] = entry; + entries.set(row.session_key, { entry, legacyKeys: [], row }); + } + const resolved = resolveSessionStoreEntry({ store, sessionKey }); + if (!resolved.existing) { + return undefined; + } + for (const value of entries.values()) { + if (value.entry === resolved.existing) { + return { ...value, legacyKeys: resolved.legacyKeys }; + } + } + return undefined; +} + +// Async updaters prepare against this complete selection. Capturing alias rows +// prevents the commit phase from deleting a concurrently changed legacy key. +function readSqliteSessionEntrySelectionSnapshot( + database: OpenClawAgentDatabase, + sessionKey: string, + exact: boolean, +): SqliteSessionEntrySelectionSnapshot { + const selected = exact + ? readExactSessionEntryRow(database, sessionKey) + : readSessionEntryRow(database, sessionKey); + const selectedKeys = selected + ? uniqueStrings([selected.row.session_key, ...selected.legacyKeys]).toSorted() + : []; + return { + selected, + selectedRows: selectedKeys.flatMap((candidateKey) => { + const row = readExactSessionEntryRow(database, candidateKey); + return row ? [{ entry: cloneSessionEntry(row.entry), sessionKey: candidateKey }] : []; + }), + }; +} + +function assertSqliteSessionEntrySelectionUnchanged( + expected: SqliteSessionEntrySelectionSnapshot, + current: SqliteSessionEntrySelectionSnapshot, + operationLabel: string, +): void { + const selectedMatches = + expected.selected?.row.session_key === current.selected?.row.session_key && + sqliteSessionEntriesEqual(expected.selected?.entry, current.selected?.entry); + if ( + !selectedMatches || + !sqliteSessionSnapshotRowsEqual(expected.selectedRows, current.selectedRows) + ) { + throw new SqliteSessionMutationConflictError(operationLabel); + } +} + +function collectSessionEntryLookupKeys( + database: OpenClawAgentDatabase, + sessionKey: string, +): string[] { + const trimmedKey = sessionKey.trim(); + if (!trimmedKey) { + return []; + } + const normalizedKey = normalizeStoreSessionKey(trimmedKey); + const lookupKeys = new Set([ + trimmedKey, + normalizedKey, + ...foldedSessionKeyAliasCandidates(normalizedKey), + ]); + const db = getSessionKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db.selectFrom("session_entries").select("session_key").orderBy("session_key", "asc"), + ).rows; + for (const row of rows) { + if (normalizeStoreSessionKey(row.session_key) === normalizedKey) { + lookupKeys.add(row.session_key); + } + } + return [...lookupKeys].filter(Boolean); +} + +function collectSqliteSessionMaintenanceBaseKeys( + store: Record<string, SessionEntry>, + activeSessionKey: string, +): string[] { + const keys: string[] = []; + const seen = new Set<string>(); + let currentKey = normalizeStoreSessionKey(activeSessionKey); + while (currentKey && !seen.has(currentKey)) { + seen.add(currentKey); + keys.push(currentKey); + currentKey = normalizeStoreSessionKey(store[currentKey]?.parentSessionKey ?? ""); + } + return keys; +} + +function sumEventJsonBytes() { + return ( + // kysely-allow-raw: SQLite byte accounting needs LENGTH(CAST(... AS BLOB)), + // which Kysely does not expose as a typed aggregate helper. + sql<number | bigint>`COALESCE(SUM(length(CAST(event_json AS BLOB))), 0)`.as("event_json_bytes") + ); +} + +function sumSessionEntryJsonBytes() { + return ( + // kysely-allow-raw: SQLite byte accounting needs LENGTH(CAST(... AS BLOB)), + // which Kysely does not expose as a typed aggregate helper. + sql<number | bigint>`COALESCE(SUM(length(CAST(entry_json AS BLOB))), 0)`.as("entry_json_bytes") + ); +} + +function readSqliteSessionRowBytes(database: OpenClawAgentDatabase): { + entryBytesByKey: Map<string, number>; + trajectoryBytesBySessionId: Map<string, number>; + transcriptBytesBySessionId: Map<string, number>; +} { + const db = getSessionKysely(database.db); + const entryRows = executeSqliteQuerySync( + database.db, + db.selectFrom("session_entries").select(["session_key", "entry_json"]), + ).rows; + const transcriptRows = executeSqliteQuerySync( + database.db, + db + .selectFrom("transcript_events") + .select(["session_id"]) + .select(sumEventJsonBytes()) + .groupBy("session_id"), + ).rows; + const trajectoryRows = executeSqliteQuerySync( + database.db, + db + .selectFrom("trajectory_runtime_events") + .select(["session_id"]) + .select(sumEventJsonBytes()) + .groupBy("session_id"), + ).rows; + const entryBytesByKey = new Map<string, number>(); + for (const row of entryRows) { + entryBytesByKey.set(row.session_key, Buffer.byteLength(row.entry_json, "utf8")); + } + const transcriptBytesBySessionId = new Map<string, number>(); + for (const row of transcriptRows) { + const bytes = row.event_json_bytes; + transcriptBytesBySessionId.set(row.session_id, normalizeSqliteNumber(bytes ?? 0)); + } + const trajectoryBytesBySessionId = new Map<string, number>(); + for (const row of trajectoryRows) { + const bytes = row.event_json_bytes; + trajectoryBytesBySessionId.set(row.session_id, normalizeSqliteNumber(bytes ?? 0)); + } + return { entryBytesByKey, trajectoryBytesBySessionId, transcriptBytesBySessionId }; +} + +function getSqliteSessionStateBytes( + rowBytes: ReturnType<typeof readSqliteSessionRowBytes>, + sessionId: string, +): number { + return ( + (rowBytes.transcriptBytesBySessionId.get(sessionId) ?? 0) + + (rowBytes.trajectoryBytesBySessionId.get(sessionId) ?? 0) + ); +} + +function getSqliteSessionEntryUpdatedAt(entry?: SessionEntry): number { + return entry?.updatedAt ?? Number.NEGATIVE_INFINITY; +} + +function hasSqliteSessionDiskBudgetOverflow( + database: OpenClawAgentDatabase, + maintenance: ResolvedSessionMaintenanceConfig, +): boolean { + if (maintenance.maxDiskBytes == null || maintenance.highWaterBytes == null) { + return false; + } + const db = getSessionKysely(database.db); + const entryRow = executeSqliteQueryTakeFirstSync( + database.db, + db.selectFrom("session_entries").select(sumSessionEntryJsonBytes()), + ); + const transcriptRow = executeSqliteQueryTakeFirstSync( + database.db, + db.selectFrom("transcript_events").select(sumEventJsonBytes()), + ); + const trajectoryRow = executeSqliteQueryTakeFirstSync( + database.db, + db.selectFrom("trajectory_runtime_events").select(sumEventJsonBytes()), + ); + const entryBytes = normalizeSqliteNumber(entryRow?.entry_json_bytes ?? 0); + const transcriptBytes = normalizeSqliteNumber(transcriptRow?.event_json_bytes ?? 0); + const trajectoryBytes = normalizeSqliteNumber(trajectoryRow?.event_json_bytes ?? 0); + return entryBytes + transcriptBytes + trajectoryBytes > maintenance.maxDiskBytes; +} + +function applySqliteSessionDiskBudget(params: { + database: OpenClawAgentDatabase; + store: Record<string, SessionEntry>; + maintenance: ResolvedSessionMaintenanceConfig; + preserveKeys: ReadonlySet<string>; + rememberRemovedEntry: (removed: { key: string; entry: SessionEntry }) => void; +}): void { + enforceSqliteSessionDiskBudgetInStore({ + database: params.database, + store: params.store, + maintenance: params.maintenance, + preserveKeys: params.preserveKeys, + onRemoveEntry: params.rememberRemovedEntry, + }); +} + +function enforceSqliteSessionDiskBudgetInStore(params: { + database: OpenClawAgentDatabase; + store: Record<string, SessionEntry>; + maintenance: Pick<ResolvedSessionMaintenanceConfig, "maxDiskBytes" | "highWaterBytes">; + preserveKeys?: ReadonlySet<string>; + onRemoveEntry?: (removed: { key: string; entry: SessionEntry }) => void; +}): SessionDiskBudgetSweepResult | null { + const { maxDiskBytes, highWaterBytes } = params.maintenance; + if (maxDiskBytes == null || highWaterBytes == null) { + return null; + } + const rowBytes = readSqliteSessionRowBytes(params.database); + let totalBytes = 0; + const entryBytesByKey = new Map<string, number>(); + const sessionIdsByKey = new Map<string, readonly string[]>(); + const sessionIdRefCounts = new Map<string, number>(); + // Session state rows can be shared through usage-family references. Count + // each referenced session id once, then subtract rows only after the last + // remaining entry reference is removed. + for (const [key, entry] of Object.entries(params.store)) { + const entryBytes = rowBytes.entryBytesByKey.get(key) ?? 0; + const sessionIds = collectSqliteSessionStateIdsForEntry(entry); + entryBytesByKey.set(key, entryBytes); + sessionIdsByKey.set(key, sessionIds); + totalBytes += entryBytes; + for (const sessionId of sessionIds) { + sessionIdRefCounts.set(sessionId, (sessionIdRefCounts.get(sessionId) ?? 0) + 1); + } + } + for (const sessionId of sessionIdRefCounts.keys()) { + totalBytes += getSqliteSessionStateBytes(rowBytes, sessionId); + } + const totalBytesBefore = totalBytes; + if (totalBytes <= maxDiskBytes) { + return { + totalBytesBefore, + totalBytesAfter: totalBytes, + removedFiles: 0, + removedEntries: 0, + freedBytes: 0, + maxBytes: maxDiskBytes, + highWaterBytes, + overBudget: false, + }; + } + let removedEntries = 0; + const keys = Object.keys(params.store).toSorted((a, b) => { + const aTime = getSqliteSessionEntryUpdatedAt(params.store[a]); + const bTime = getSqliteSessionEntryUpdatedAt(params.store[b]); + return aTime - bTime; + }); + for (const key of keys) { + if (totalBytes <= highWaterBytes) { + break; + } + const entry = params.store[key]; + if (!entry) { + continue; + } + if (shouldPreserveMaintenanceEntry({ key, entry, preserveKeys: params.preserveKeys })) { + continue; + } + params.onRemoveEntry?.({ key, entry }); + delete params.store[key]; + removedEntries += 1; + totalBytes -= entryBytesByKey.get(key) ?? 0; + for (const sessionId of sessionIdsByKey.get(key) ?? []) { + const nextRefCount = (sessionIdRefCounts.get(sessionId) ?? 0) - 1; + if (nextRefCount > 0) { + sessionIdRefCounts.set(sessionId, nextRefCount); + continue; + } + sessionIdRefCounts.delete(sessionId); + totalBytes -= getSqliteSessionStateBytes(rowBytes, sessionId); + } + } + return { + totalBytesBefore, + totalBytesAfter: totalBytes, + removedFiles: 0, + removedEntries, + freedBytes: Math.max(0, totalBytesBefore - totalBytes), + maxBytes: maxDiskBytes, + highWaterBytes, + overBudget: true, + }; +} + +export function previewSqliteSessionDiskBudget(params: { + agentId?: string; + activeSessionKey?: string; + store: Record<string, SessionEntry>; + storePath: string; + maintenance: Pick<ResolvedSessionMaintenanceConfig, "maxDiskBytes" | "highWaterBytes">; + preserveKeys?: ReadonlySet<string>; +}): { diskBudget: SessionDiskBudgetSweepResult | null; removedKeys: Set<string> } { + const removedKeys = new Set<string>(); + if (params.maintenance.maxDiskBytes == null || params.maintenance.highWaterBytes == null) { + return { diskBudget: null, removedKeys }; + } + const resolved = resolveSqliteScope({ + ...(params.agentId ? { agentId: params.agentId } : {}), + sessionKey: "", + storePath: params.storePath, + }); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + const baseKeys = collectSqliteSessionMaintenanceBaseKeys( + params.store, + params.activeSessionKey ?? "", + ); + const preserveKeys = + baseKeys.length > 0 || params.preserveKeys + ? new Set([...(params.preserveKeys ?? []), ...baseKeys]) + : undefined; + const diskBudget = enforceSqliteSessionDiskBudgetInStore({ + database, + store: params.store, + maintenance: params.maintenance, + preserveKeys, + onRemoveEntry: ({ key }) => { + removedKeys.add(key); + }, + }); + return { diskBudget, removedKeys }; +} + +function readExactSessionEntryRow( + database: OpenClawAgentDatabase, + sessionKey: string, +): ResolvedSessionEntryRow | undefined { + const db = getSessionKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db.selectFrom("session_entries").selectAll().where("session_key", "=", sessionKey), + ); + if (!row) { + return undefined; + } + const entry = parseSessionEntryRow(row); + return entry ? { entry, legacyKeys: [], row } : undefined; +} + +function readSqliteSessionEntryStore( + database: OpenClawAgentDatabase, +): Record<string, SessionEntry> { + const db = getSessionKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db.selectFrom("session_entries").select(["session_key", "entry_json"]).orderBy("session_key"), + ).rows; + const store: Record<string, SessionEntry> = {}; + for (const row of rows) { + const entry = parseSessionEntryRow(row); + if (entry) { + store[row.session_key] = entry; + } + } + return store; +} + +function readSqliteSessionEntryCount(database: OpenClawAgentDatabase): number { + const db = getSessionKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db.selectFrom("session_entries").select((eb) => eb.fn.countAll<number>().as("entry_count")), + ); + const count = row?.entry_count; + return count === undefined || count === null ? 0 : normalizeSqliteNumber(count); +} + +function hasStaleSqliteSessionEntryCandidate( + database: OpenClawAgentDatabase, + pruneAfterMs: number, + preserveKeys: ReadonlySet<string> | undefined, +): boolean { + const cutoffMs = Date.now() - pruneAfterMs; + const db = getSessionKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db + .selectFrom("session_entries") + .select("session_key") + .where("updated_at", "<", cutoffMs) + .orderBy("updated_at", "asc"), + ).rows; + return rows.some((row) => !preserveKeys?.has(normalizeStoreSessionKey(row.session_key))); +} + +function resolveSqliteLifecyclePrimaryEntry( + database: OpenClawAgentDatabase, + target: { canonicalKey: string; storeKeys: string[] }, +): { key: string; entry: SessionEntry } | undefined { + let freshest: { key: string; entry: SessionEntry } | undefined; + for (const key of target.storeKeys) { + const row = readExactSessionEntryRow(database, key.trim()); + if (!row) { + continue; + } + if (!freshest || (row.entry.updatedAt ?? 0) > (freshest.entry.updatedAt ?? 0)) { + freshest = { key, entry: row.entry }; + } + } + return freshest ?? undefined; +} + +function readSqliteLifecycleTargetSnapshot( + database: OpenClawAgentDatabase, + target: { canonicalKey: string; storeKeys: string[] }, +): SqliteLifecycleTargetSnapshot { + const normalized = normalizeSqliteLifecycleTarget(target); + return { + primary: resolveSqliteLifecyclePrimaryEntry(database, normalized), + rows: normalized.storeKeys.flatMap((sessionKey) => { + const row = readExactSessionEntryRow(database, sessionKey); + return row ? [{ entry: cloneSessionEntry(row.entry), sessionKey }] : []; + }), + }; +} + +function assertSqliteLifecycleTargetSnapshotUnchanged( + expected: SqliteLifecycleTargetSnapshot, + current: SqliteLifecycleTargetSnapshot, + operationLabel: string, +): void { + const primaryMatches = + expected.primary?.key === current.primary?.key && + sqliteSessionEntriesEqual(expected.primary?.entry, current.primary?.entry); + if (!primaryMatches || !sqliteSessionSnapshotRowsEqual(expected.rows, current.rows)) { + throw new SqliteSessionMutationConflictError(operationLabel); + } +} + +function normalizeSqliteLifecycleTarget(target: { canonicalKey: string; storeKeys: string[] }): { + canonicalKey: string; + storeKeys: string[]; +} { + const canonicalKey = normalizeSqliteSessionKey(target.canonicalKey); + return { + canonicalKey, + storeKeys: uniqueStrings([canonicalKey, ...target.storeKeys.map(normalizeSqliteSessionKey)]), + }; +} + +function deleteSqliteSessionEntryRows(database: OpenClawAgentDatabase, sessionKey: string): void { + const db = getSessionKysely(database.db); + executeSqliteQuerySync( + database.db, + db.deleteFrom("session_routes").where("session_key", "=", sessionKey), + ); + executeSqliteQuerySync( + database.db, + db.deleteFrom("session_entries").where("session_key", "=", sessionKey), + ); +} + +function deleteSqliteLifecycleTargetRows( + database: OpenClawAgentDatabase, + target: { canonicalKey: string; storeKeys: string[] }, +): void { + for (const sessionKey of uniqueStrings([target.canonicalKey, ...target.storeKeys])) { + const trimmed = sessionKey.trim(); + if (trimmed) { + deleteSqliteSessionEntryRows(database, trimmed); + } + } +} + +function shouldRemoveSqliteSessionEntry( + entry: SessionEntry | undefined, + removal: SessionEntryLifecycleRemoval, +): entry is SessionEntry { + if (!entry) { + return false; + } + if ( + removal.expectedEntry !== undefined && + JSON.stringify(entry) !== JSON.stringify(removal.expectedEntry) + ) { + return false; + } + if (removal.expectedSessionId !== undefined && entry.sessionId !== removal.expectedSessionId) { + return false; + } + if ( + removal.expectedLifecycleRevision !== undefined && + entry.lifecycleRevision !== removal.expectedLifecycleRevision + ) { + return false; + } + if (removal.expectedUpdatedAt !== undefined && entry.updatedAt !== removal.expectedUpdatedAt) { + return false; + } + return true; +} + +function shouldDeleteSqliteSessionEntryLifecycle( + entry: SessionEntry | undefined, + params: DeleteSessionEntryLifecycleParams, +): entry is SessionEntry { + if (!entry) { + return false; + } + if ( + params.expectedEntry !== undefined && + !sqliteSessionEntriesEqual(entry, params.expectedEntry) + ) { + return false; + } + if (params.expectedSessionId !== undefined && entry.sessionId !== params.expectedSessionId) { + return false; + } + if ( + params.expectedLifecycleRevision !== undefined && + entry.lifecycleRevision !== params.expectedLifecycleRevision + ) { + return false; + } + if (params.expectedUpdatedAt !== undefined && entry.updatedAt !== params.expectedUpdatedAt) { + return false; + } + return true; +} + +function sqliteSessionEntriesEqual( + left: SessionEntry | undefined, + right: SessionEntry | undefined, +): boolean { + if (!left || !right) { + return left === right; + } + return JSON.stringify(left) === JSON.stringify(right); +} + +function sqliteSessionSnapshotRowsEqual( + left: Array<{ entry: SessionEntry; sessionKey: string }>, + right: Array<{ entry: SessionEntry; sessionKey: string }>, +): boolean { + return ( + left.length === right.length && + left.every( + (row, index) => + row.sessionKey === right[index]?.sessionKey && + sqliteSessionEntriesEqual(row.entry, right[index]?.entry), + ) + ); +} + +function sqliteLifecycleTargetMatchesExpectedEntry( + database: OpenClawAgentDatabase, + target: { canonicalKey: string; storeKeys: string[] }, + expectedEntry: SessionEntry | undefined, +): boolean { + const current = resolveSqliteLifecyclePrimaryEntry(database, target)?.entry; + if (!current || !expectedEntry) { + return current === expectedEntry; + } + return sqliteSessionEntriesEqual(current, expectedEntry); +} + +function assertSqliteLifecycleTargetUnchanged( + database: OpenClawAgentDatabase, + target: { canonicalKey: string; storeKeys: string[] }, + expectedEntry: SessionEntry | undefined, + operation: "deleted" | "reset", +): void { + if (sqliteLifecycleTargetMatchesExpectedEntry(database, target, expectedEntry)) { + return; + } + throw new Error(`SQLite session entry changed before ${operation} lifecycle mutation`); +} + +function deleteLegacySessionEntryRows( + database: OpenClawAgentDatabase, + legacyKeys: string[], + sessionKey: string, +): void { + if (legacyKeys.length === 0) { + return; + } + const db = getSessionKysely(database.db); + for (const legacyKey of legacyKeys) { + if (legacyKey === sessionKey) { + continue; + } + executeSqliteQuerySync( + database.db, + db.deleteFrom("session_routes").where("session_key", "=", legacyKey), + ); + executeSqliteQuerySync( + database.db, + db.deleteFrom("session_entries").where("session_key", "=", legacyKey), + ); + } +} + +function applySqliteSessionEntryMaintenance( + database: OpenClawAgentDatabase, + params: { + activeSessionKey: string; + archiveDirectory: string; + forceMaintenance?: boolean; + maintenanceConfig?: ResolvedSessionMaintenanceConfig; + skipMaintenance?: boolean; + }, +): SqliteSessionEntryMaintenancePlan { + if (params.skipMaintenance) { + return { entryRemovals: [], stateDeletePlans: [] }; + } + const maintenance = params.maintenanceConfig ?? resolveMaintenanceConfig(); + if (maintenance.mode === "warn") { + return { entryRemovals: [], stateDeletePlans: [] }; + } + + const entryCount = readSqliteSessionEntryCount(database); + const preserveCandidateKeys = collectSessionMaintenancePreserveKeys([params.activeSessionKey]); + const hasStaleCandidate = hasStaleSqliteSessionEntryCandidate( + database, + maintenance.pruneAfterMs, + preserveCandidateKeys, + ); + const hasDiskBudgetOverflow = hasSqliteSessionDiskBudgetOverflow(database, maintenance); + const shouldLoadStore = + params.forceMaintenance === true || + entryCount > maintenance.maxEntries || + hasStaleCandidate || + hasDiskBudgetOverflow || + shouldRunModelRunPrune({ + maintenance, + entryCount, + force: params.forceMaintenance, + }) || + shouldRunSessionEntryMaintenance({ + entryCount, + maxEntries: maintenance.maxEntries, + force: params.forceMaintenance, + }); + if (!shouldLoadStore) { + return { entryRemovals: [], stateDeletePlans: [] }; + } + + const db = getSessionKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db.selectFrom("session_entries").select(["session_key", "entry_json"]).orderBy("session_key"), + ).rows; + const store: Record<string, SessionEntry> = {}; + for (const row of rows) { + const entry = parseSessionEntryRow(row); + if (entry) { + store[row.session_key] = entry; + } + } + + const removedKeys = new Set<string>(); + const removedEntriesByKey = new Map<string, SessionEntry>(); + const removedSessionIds = new Set<string>(); + const rememberRemovedEntry = (removed: { key: string; entry: SessionEntry }) => { + removedKeys.add(removed.key); + removedEntriesByKey.set(removed.key, cloneSessionEntry(removed.entry)); + for (const sessionId of collectSqliteSessionStateIdsForEntry(removed.entry)) { + removedSessionIds.add(sessionId); + } + }; + const preserveKeys = + collectSessionMaintenancePreserveKeys( + collectSqliteSessionMaintenanceBaseKeys(store, params.activeSessionKey), + ) ?? new Set<string>(); + if ( + shouldRunModelRunPrune({ + maintenance, + entryCount: Object.keys(store).length, + force: params.forceMaintenance, + }) + ) { + pruneStaleModelRunEntries(store, maintenance.modelRunPruneAfterMs, { + log: false, + onPruned: rememberRemovedEntry, + preserveKeys, + }); + } + if ( + params.forceMaintenance === true || + hasStaleCandidate || + Object.keys(store).length > maintenance.maxEntries + ) { + pruneStaleEntries(store, maintenance.pruneAfterMs, { + log: false, + onPruned: rememberRemovedEntry, + preserveKeys, + }); + } + if ( + shouldRunSessionEntryMaintenance({ + entryCount: Object.keys(store).length, + maxEntries: maintenance.maxEntries, + force: params.forceMaintenance, + }) + ) { + capEntryCount(store, maintenance.maxEntries, { + log: false, + onCapped: rememberRemovedEntry, + preserveKeys, + }); + } + applySqliteSessionDiskBudget({ + database, + store, + maintenance, + preserveKeys, + rememberRemovedEntry, + }); + + const referencedSessionIds = collectProjectedReferencedSqliteSessionIds({ + database, + excludedSessionKeys: removedKeys, + projectedStore: store, + }); + const deletePlans: SqliteSessionStateDeletePlan[] = []; + for (const sessionId of removedSessionIds) { + const plan = planSqliteSessionStateDeleteIfUnreferenced({ + archiveTranscript: true, + archiveDirectory: params.archiveDirectory, + database, + referencedSessionIds, + sessionId, + }); + if (plan) { + deletePlans.push(plan); + } + } + return { + entryRemovals: [...removedKeys].map((sessionKey) => ({ + expectedEntry: removedEntriesByKey.get(sessionKey), + sessionKey, + })), + stateDeletePlans: deletePlans, + }; +} + +function sessionKeySegmentStartsWith(sessionKey: string, prefix: string): boolean { + const firstSeparator = sessionKey.indexOf(":"); + if (firstSeparator < 0) { + return sessionKey.startsWith(prefix); + } + const secondSeparator = sessionKey.indexOf(":", firstSeparator + 1); + const sessionSegment = secondSeparator < 0 ? sessionKey : sessionKey.slice(secondSeparator + 1); + return sessionSegment.startsWith(prefix); +} + +function readSessionTranscriptUpdatedAt( + database: OpenClawAgentDatabase, + sessionId: string, +): number | undefined { + const db = getSessionKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("transcript_events") + .select((eb) => eb.fn.max<number | bigint>("created_at").as("updated_at")) + .where("session_id", "=", sessionId), + ); + if (row?.updated_at === null || row?.updated_at === undefined) { + return undefined; + } + return normalizeSqliteNumber(row.updated_at); +} + +function sqliteTranscriptStateIsReclaimable(params: { + database: OpenClawAgentDatabase; + sessionId: string; + nowMs: number; + orphanTranscriptMinAgeMs: number; +}): boolean { + const updatedAt = readSessionTranscriptUpdatedAt(params.database, params.sessionId); + return updatedAt === undefined || params.nowMs - updatedAt >= params.orphanTranscriptMinAgeMs; +} + +function sqliteTranscriptStateHasMarker(params: { + database: OpenClawAgentDatabase; + sessionId: string; + transcriptContentMarker: string; +}): boolean { + const db = getSessionKysely(params.database.db); + const rows = executeSqliteQuerySync( + params.database.db, + db + .selectFrom("transcript_events") + .select("event_json") + .where("session_id", "=", params.sessionId) + .orderBy("seq", "asc"), + ).rows; + return rows.some((row) => row.event_json.includes(params.transcriptContentMarker)); +} + +function readReferencedSqliteSessionIds(database: OpenClawAgentDatabase): Set<string> { + const db = getSessionKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db.selectFrom("session_entries").select(["entry_json", "session_id"]), + ).rows; + const sessionIds = new Set<string>(); + for (const row of rows) { + sessionIds.add(row.session_id); + const entry = parseSessionEntryRow(row); + if (!entry) { + continue; + } + for (const sessionId of collectSqliteSessionStateIdsForEntry(entry)) { + sessionIds.add(sessionId); + } + } + return sessionIds; +} + +// Projects references after a lifecycle mutation so reset/delete can archive +// before removing entry rows while still preserving shared session ids. +function readReferencedSqliteSessionIdsAfterTargetMutation( + database: OpenClawAgentDatabase, + target: { canonicalKey: string; storeKeys: string[] }, + nextEntry?: SessionEntry, +): Set<string> { + const removedKeys = new Set( + uniqueStrings([target.canonicalKey, ...target.storeKeys].map((key) => key.trim())), + ); + const db = getSessionKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db.selectFrom("session_entries").select(["entry_json", "session_key", "session_id"]), + ).rows; + const sessionIds = new Set<string>(); + for (const row of rows) { + if (removedKeys.has(row.session_key)) { + continue; + } + sessionIds.add(row.session_id); + const entry = parseSessionEntryRow(row); + if (!entry) { + continue; + } + for (const sessionId of collectSqliteSessionStateIdsForEntry(entry)) { + sessionIds.add(sessionId); + } + } + if (nextEntry) { + for (const sessionId of collectSqliteSessionStateIdsForEntry(nextEntry)) { + sessionIds.add(sessionId); + } + } + return sessionIds; +} + +function readSqliteTranscriptArchiveLines( + database: OpenClawAgentDatabase, + sessionId: string, +): string[] { + const db = getSessionKysely(database.db); + return executeSqliteQuerySync( + database.db, + db + .selectFrom("transcript_events") + .select("event_json") + .where("session_id", "=", sessionId) + .orderBy("seq", "asc"), + ).rows.map((row) => row.event_json); +} + +function resolveSqliteTranscriptArchivePath(params: { + archiveDirectory: string; + reason: "deleted" | "reset"; + sessionId: string; + nowMs?: number; +}): string { + const archiveDirectory = path.resolve(params.archiveDirectory); + const archivePath = path.resolve( + archiveDirectory, + `${params.sessionId}.jsonl.${params.reason}.${formatSessionArchiveTimestamp(params.nowMs)}`, + ); + if (path.dirname(archivePath) !== archiveDirectory) { + throw new Error(`Cannot archive SQLite transcript outside ${archiveDirectory}`); + } + return archivePath; +} + +function findMatchingSqliteTranscriptArchive(params: { + archiveDirectory: string; + content: string; + reason: "deleted" | "reset"; + sessionId: string; +}): string | null { + let entries: string[]; + try { + entries = fs.readdirSync(params.archiveDirectory); + } catch { + return null; + } + const prefix = `${params.sessionId}.jsonl.${params.reason}.`; + for (const entry of entries) { + if (!entry.startsWith(prefix)) { + continue; + } + const archivePath = path.join(params.archiveDirectory, entry); + const compressed = entry.endsWith(SESSION_ARCHIVE_ZSTD_SUFFIX); + try { + const stat = fs.statSync(archivePath); + if (!stat.isFile()) { + continue; + } + // Compressed size never matches the utf8 length, so the cheap size + // precheck only applies to plain archives. + if (!compressed && stat.size !== Buffer.byteLength(params.content, "utf8")) { + continue; + } + if (readSessionArchiveContentSync(archivePath) === params.content) { + return archivePath; + } + } catch { + continue; + } + } + return null; +} + +function writeSqliteTranscriptArchive(params: { + archiveDirectory: string; + content: string; + reason: "deleted" | "reset"; + sessionId: string; +}): string { + fs.mkdirSync(params.archiveDirectory, { recursive: true }); + const existing = findMatchingSqliteTranscriptArchive(params); + if (existing) { + return existing; + } + // Archives are the long-lived cold tier; compress when the runtime can so + // keep-forever retention stays cheap. Plain JSONL is the Bun/older fallback. + const encoded = encodeSessionArchiveContent(params.content); + for (let attempt = 0; attempt < 10; attempt += 1) { + const archivePath = `${resolveSqliteTranscriptArchivePath({ + archiveDirectory: params.archiveDirectory, + reason: params.reason, + sessionId: params.sessionId, + nowMs: Date.now() + attempt, + })}${encoded.suffix}`; + if (fs.existsSync(archivePath)) { + continue; + } + const tempPath = `${archivePath}.${randomUUID()}.tmp`; + try { + fs.writeFileSync(tempPath, encoded.bytes, { + flag: "wx", + mode: 0o600, + }); + fsyncRegularFile(tempPath); + fs.renameSync(tempPath, archivePath); + fsyncDirectory(params.archiveDirectory); + return archivePath; + } catch (err) { + fs.rmSync(tempPath, { force: true }); + if ((err as { code?: unknown })?.code === "EEXIST") { + continue; + } + throw err; + } + } + throw new Error(`Could not create SQLite transcript archive for ${params.sessionId}`); +} + +function fsyncRegularFile(filePath: string): void { + const fd = fs.openSync(filePath, "r"); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } +} + +function fsyncDirectory(dirPath: string): void { + let fd: number | undefined; + try { + fd = fs.openSync(dirPath, "r"); + fs.fsyncSync(fd); + } catch { + // Directory fsync is not available on every supported platform/filesystem. + } finally { + if (fd !== undefined) { + fs.closeSync(fd); + } + } +} + +function planSqliteSessionStateDeleteIfUnreferenced(params: { + archiveTranscript?: boolean; + archiveDirectory: string; + database: OpenClawAgentDatabase; + reason?: "deleted" | "reset"; + referencedSessionIds: ReadonlySet<string>; + sessionId: string; +}): SqliteSessionStateDeletePlan | null { + if (params.referencedSessionIds.has(params.sessionId)) { + return null; + } + const lines = readSqliteTranscriptArchiveLines(params.database, params.sessionId); + return { + archiveDirectory: params.archiveDirectory, + archiveTranscript: params.archiveTranscript !== false, + content: serializeJsonlLines(lines), + hadTranscriptState: + readSessionTranscriptUpdatedAt(params.database, params.sessionId) !== undefined, + reason: params.reason ?? "deleted", + sessionId: params.sessionId, + }; +} + +// Runs duplicate probing, archive write, rename, and fsync outside SQLite +// write transactions; deletion later consumes this durable proof. +function materializeSqliteSessionStateDeletePlans( + plans: readonly SqliteSessionStateDeletePlan[], +): MaterializedSqliteSessionStateDeletePlan[] { + return dedupeSqliteSessionStateDeletePlans(plans).map((plan) => { + const archivedTranscript = + plan.archiveTranscript && plan.content.length > 0 + ? { + archivedPath: writeSqliteTranscriptArchive({ + archiveDirectory: plan.archiveDirectory, + content: plan.content, + reason: plan.reason, + sessionId: plan.sessionId, + }), + sourcePath: path.join(plan.archiveDirectory, `${plan.sessionId}.jsonl`), + } + : null; + return Object.assign({}, plan, { archivedTranscript }); + }); +} + +// Multiple removed entries can point at one transcript session; dedupe before +// validation so the first row deletion does not stale a duplicate plan. +// If any owner asked to keep an archive, the shared row gets exported once. +function dedupeSqliteSessionStateDeletePlans( + plans: readonly SqliteSessionStateDeletePlan[], +): SqliteSessionStateDeletePlan[] { + const deduped = new Map<string, SqliteSessionStateDeletePlan>(); + for (const plan of plans) { + const existing = deduped.get(plan.sessionId); + if (!existing) { + deduped.set(plan.sessionId, plan); + continue; + } + if (existing.content !== plan.content || existing.reason !== plan.reason) { + throw new Error(`Conflicting SQLite transcript archive plans for ${plan.sessionId}`); + } + if (!existing.archiveTranscript && plan.archiveTranscript) { + deduped.set(plan.sessionId, { ...existing, archiveTranscript: true }); + } + } + return [...deduped.values()]; +} + +function finalizeSqliteSessionEntryMaintenancePlansBestEffort( + scope: Pick<ResolvedSqliteReadScope, "agentId" | "env" | "path">, + plans: readonly SqliteSessionEntryMaintenancePlan[], +): SessionLifecycleArchivedTranscript[] { + const entryRemovals = plans.flatMap((plan) => plan.entryRemovals); + const stateDeletePlans = plans.flatMap((plan) => plan.stateDeletePlans); + if (entryRemovals.length === 0 && stateDeletePlans.length === 0) { + return []; + } + try { + const materializedPlans = materializeSqliteSessionStateDeletePlans(stateDeletePlans); + let archivedTranscripts: SessionLifecycleArchivedTranscript[] = []; + runOpenClawAgentWriteTransaction((database) => { + deletePlannedSqliteLifecycleArtifactEntries(database, entryRemovals); + archivedTranscripts = deleteMaterializedSqliteSessionStatePlans(database, materializedPlans); + }, toDatabaseOptions(scope)); + return archivedTranscripts; + } catch (error) { + getChildLogger({ subsystem: "session-sqlite" }).warn( + "SQLite session maintenance cleanup failed", + { + agentId: scope.agentId, + error, + path: scope.path, + sessionIds: uniqueStrings(stateDeletePlans.map((plan) => plan.sessionId)), + }, + ); + return []; + } +} + +// Revalidates transcript bytes before row deletion so a concurrent append is +// not dropped by an archive prepared from older content. +function deleteMaterializedSqliteSessionStatePlans( + database: OpenClawAgentDatabase, + plans: readonly MaterializedSqliteSessionStateDeletePlan[], +): SessionLifecycleArchivedTranscript[] { + const archivedTranscripts: SessionLifecycleArchivedTranscript[] = []; + const referencedSessionIds = readReferencedSqliteSessionIds(database); + for (const plan of plans) { + if (referencedSessionIds.has(plan.sessionId)) { + continue; + } + if (plan.archiveTranscript) { + const currentContent = serializeJsonlLines( + readSqliteTranscriptArchiveLines(database, plan.sessionId), + ); + if (currentContent !== plan.content) { + throw new Error(`SQLite transcript changed before archive deletion for ${plan.sessionId}`); + } + } + deleteSqliteSessionStateRows(database, plan.sessionId); + if (plan.hadTranscriptState && plan.archivedTranscript) { + archivedTranscripts.push(plan.archivedTranscript); + } + } + return archivedTranscripts; +} + +// Builds delete plans from the session ids owned by an entry after callers +// have projected which ids remain referenced. +function planSqliteSessionStateAfterEntryRemoval(params: { + archiveDirectory: string; + archiveTranscript?: boolean; + database: OpenClawAgentDatabase; + entry: SessionEntry; + reason: "deleted" | "reset"; + referencedSessionIds?: ReadonlySet<string>; +}): SqliteSessionStateDeletePlan[] { + const referencedSessionIds = + params.referencedSessionIds ?? readReferencedSqliteSessionIds(params.database); + const plans: SqliteSessionStateDeletePlan[] = []; + for (const sessionId of collectSqliteSessionStateIdsForEntry(params.entry)) { + const plan = planSqliteSessionStateDeleteIfUnreferenced({ + archiveTranscript: params.archiveTranscript, + archiveDirectory: params.archiveDirectory, + database: params.database, + reason: params.reason, + referencedSessionIds, + sessionId, + }); + if (plan) { + plans.push(plan); + } + } + return plans; +} + +// Projects removals and upserts before archive materialization so same-call +// upserts can keep a transcript live without producing a spurious archive. +async function projectSqliteSessionEntryLifecycleMutation( + database: OpenClawAgentDatabase, + params: { + archiveDirectory: string; + removals: readonly SessionEntryLifecycleRemoval[]; + upserts: readonly SessionEntryLifecycleUpsert[]; + }, +): Promise<SqliteProjectedLifecycleMutation> { + const store = readSqliteSessionEntryStore(database); + const removedEntries: Array<{ archiveTranscript: boolean; entry: SessionEntry }> = []; + const changedSessionKeys = new Set<string>(); + const projectedRemovals: SqliteProjectedLifecycleMutation["removals"] = []; + for (const removal of params.removals) { + const sessionKey = removal.sessionKey.trim(); + const entry = sessionKey ? store[sessionKey] : undefined; + if (!shouldRemoveSqliteSessionEntry(entry, removal)) { + continue; + } + projectedRemovals.push({ + expectedEntry: cloneSessionEntry(entry), + removal, + sessionKey, + }); + removedEntries.push({ + archiveTranscript: removal.archiveRemovedTranscript === true, + entry, + }); + changedSessionKeys.add(sessionKey); + delete store[sessionKey]; + } + const upsertedEntries: SqliteProjectedLifecycleMutation["upsertedEntries"] = []; + for (const upsert of params.upserts) { + const sessionKey = upsert.sessionKey.trim(); + if (!sessionKey) { + continue; + } + const expectedEntry = store[sessionKey] ? cloneSessionEntry(store[sessionKey]) : undefined; + const entry = + upsert.buildEntry === undefined + ? upsert.entry + : await upsert.buildEntry({ + currentEntry: expectedEntry ? cloneSessionEntry(expectedEntry) : undefined, + sessionKey, + store, + }); + if (!entry) { + continue; + } + const cloned = cloneSessionEntry(entry); + store[sessionKey] = cloned; + changedSessionKeys.add(sessionKey); + upsertedEntries.push({ expectedEntry, sessionKey, entry: cloned }); + } + const referencedSessionIds = collectProjectedReferencedSqliteSessionIds({ + database, + excludedSessionKeys: changedSessionKeys, + projectedStore: store, + }); + const deletePlans = removedEntries.flatMap(({ archiveTranscript, entry }) => + planSqliteSessionStateAfterEntryRemoval({ + archiveDirectory: params.archiveDirectory, + archiveTranscript, + database, + entry, + reason: "deleted", + referencedSessionIds, + }), + ); + return { deletePlans, removals: projectedRemovals, upsertedEntries }; +} + +// Builds the post-removal reference set from an in-memory projected store. +function collectReferencedSqliteSessionIdsFromStore( + store: Record<string, SessionEntry>, +): Set<string> { + const sessionIds = new Set<string>(); + for (const entry of Object.values(store)) { + for (const sessionId of collectSqliteSessionStateIdsForEntry(entry)) { + sessionIds.add(sessionId); + } + } + return sessionIds; +} + +// Projected deletes must preserve raw session_entries.session_id references for +// remaining rows whose entry_json cannot be parsed into a SessionEntry. +function collectProjectedReferencedSqliteSessionIds(params: { + database: OpenClawAgentDatabase; + excludedSessionKeys: Iterable<string>; + projectedStore: Record<string, SessionEntry>; +}): Set<string> { + const excludedSessionKeys = new Set(params.excludedSessionKeys); + const db = getSessionKysely(params.database.db); + const rows = executeSqliteQuerySync( + params.database.db, + db.selectFrom("session_entries").select(["entry_json", "session_key", "session_id"]), + ).rows; + const sessionIds = new Set<string>(); + for (const row of rows) { + if (excludedSessionKeys.has(row.session_key)) { + continue; + } + sessionIds.add(row.session_id); + const entry = parseSessionEntryRow(row); + if (!entry) { + continue; + } + for (const sessionId of collectSqliteSessionStateIdsForEntry(entry)) { + sessionIds.add(sessionId); + } + } + for (const sessionId of collectReferencedSqliteSessionIdsFromStore(params.projectedStore)) { + sessionIds.add(sessionId); + } + return sessionIds; +} + +function collectSqliteSessionStateIdsForEntry(entry: SessionEntry): string[] { + const sessionIds: string[] = []; + const add = (sessionId: string | undefined) => { + const normalized = sessionId?.trim(); + if (normalized) { + sessionIds.push(normalized); + } + }; + add(entry.sessionId); + for (const sessionId of entry.usageFamilySessionIds ?? []) { + add(sessionId); + } + for (const checkpoint of entry.compactionCheckpoints ?? []) { + add(checkpoint.sessionId); + add(checkpoint.preCompaction.sessionId); + add(checkpoint.postCompaction.sessionId); + } + return uniqueStrings(sessionIds); +} + +function emitArchivedSqliteTranscriptUpdates( + archivedTranscripts: readonly SessionLifecycleArchivedTranscript[], +): void { + for (const archived of archivedTranscripts) { + emitSessionTranscriptUpdate({ sessionFile: archived.archivedPath }); + } +} + +function deleteSqliteSessionStateRows(database: OpenClawAgentDatabase, sessionId: string): void { + const db = getSessionKysely(database.db); + executeSqliteQuerySync( + database.db, + db.deleteFrom("sessions").where("session_id", "=", sessionId), + ); +} + +// Plans orphan cleanup without file writes or row deletion; finalization +// handles archive durability before removing rows. +function planSqliteOrphanLifecycleTranscriptStateDeletes(params: { + archiveRemovedEntryTranscripts: boolean; + archiveDirectory: string; + database: OpenClawAgentDatabase; + excludedSessionIds?: ReadonlySet<string>; + referencedSessionIds: ReadonlySet<string>; + transcriptContentMarker: string; + orphanTranscriptMinAgeMs: number; + nowMs: number; +}): SqliteSessionStateDeletePlan[] { + const db = getSessionKysely(params.database.db); + const rows = executeSqliteQuerySync( + params.database.db, + db.selectFrom("sessions").select("session_id").orderBy("session_id", "asc"), + ).rows; + + const deletePlans: SqliteSessionStateDeletePlan[] = []; + // Orphan transcript state is represented by a sessions row without a live + // session entry. The marker keeps this scoped to the caller-owned lifecycle. + for (const row of rows) { + if ( + params.referencedSessionIds.has(row.session_id) || + params.excludedSessionIds?.has(row.session_id) + ) { + continue; + } + if ( + !sqliteTranscriptStateIsReclaimable({ + database: params.database, + sessionId: row.session_id, + nowMs: params.nowMs, + orphanTranscriptMinAgeMs: params.orphanTranscriptMinAgeMs, + }) || + !sqliteTranscriptStateHasMarker({ + database: params.database, + sessionId: row.session_id, + transcriptContentMarker: params.transcriptContentMarker, + }) + ) { + continue; + } + const plan = planSqliteSessionStateDeleteIfUnreferenced({ + archiveTranscript: params.archiveRemovedEntryTranscripts, + archiveDirectory: params.archiveDirectory, + database: params.database, + reason: "deleted", + referencedSessionIds: params.referencedSessionIds, + sessionId: row.session_id, + }); + if (plan) { + deletePlans.push(plan); + } + } + return deletePlans; +} + +function planSqliteSessionLifecycleArtifactCleanup( + database: OpenClawAgentDatabase, + params: { + archiveRemovedEntryTranscripts: boolean; + archiveDirectory: string; + sessionKeySegmentPrefix: string; + transcriptContentMarker: string; + orphanTranscriptMinAgeMs: number; + nowMs: number; + }, +): SqliteLifecycleArtifactCleanupPlan { + const db = getSessionKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db + .selectFrom("session_entries") + .select(["entry_json", "session_key", "session_id"]) + .orderBy("session_key", "asc"), + ).rows; + + const removedSessionIds = new Set<string>(); + const entries: SqliteLifecycleArtifactCleanupPlan["entries"] = []; + const projectedStore = readSqliteSessionEntryStore(database); + for (const row of rows) { + if (!sessionKeySegmentStartsWith(row.session_key, params.sessionKeySegmentPrefix)) { + continue; + } + if ( + !sqliteTranscriptStateIsReclaimable({ + database, + sessionId: row.session_id, + nowMs: params.nowMs, + orphanTranscriptMinAgeMs: params.orphanTranscriptMinAgeMs, + }) + ) { + continue; + } + const entry = parseSessionEntryRow(row); + for (const sessionId of entry + ? collectSqliteSessionStateIdsForEntry(entry) + : [row.session_id]) { + removedSessionIds.add(sessionId); + } + entries.push({ + expectedEntry: entry ? cloneSessionEntry(entry) : undefined, + sessionKey: row.session_key, + }); + delete projectedStore[row.session_key]; + } + + const referencedSessionIds = collectProjectedReferencedSqliteSessionIds({ + database, + excludedSessionKeys: entries.map((entry) => entry.sessionKey), + projectedStore, + }); + const deletePlans: SqliteSessionStateDeletePlan[] = []; + for (const sessionId of removedSessionIds) { + const plan = planSqliteSessionStateDeleteIfUnreferenced({ + archiveTranscript: params.archiveRemovedEntryTranscripts, + archiveDirectory: params.archiveDirectory, + database, + referencedSessionIds, + sessionId, + }); + if (plan) { + deletePlans.push(plan); + } + } + deletePlans.push( + ...planSqliteOrphanLifecycleTranscriptStateDeletes({ + archiveRemovedEntryTranscripts: params.archiveRemovedEntryTranscripts, + archiveDirectory: params.archiveDirectory, + database, + excludedSessionIds: removedSessionIds, + referencedSessionIds, + transcriptContentMarker: params.transcriptContentMarker, + orphanTranscriptMinAgeMs: params.orphanTranscriptMinAgeMs, + nowMs: params.nowMs, + }), + ); + return { deletePlans, entries }; +} + +function deletePlannedSqliteLifecycleArtifactEntries( + database: OpenClawAgentDatabase, + entries: readonly SqliteSessionEntryRemovalPlan[], +): number { + let removedEntries = 0; + for (const planned of entries) { + const current = readExactSessionEntryRow(database, planned.sessionKey)?.entry; + if (!sqliteSessionEntriesEqual(current, planned.expectedEntry)) { + throw new Error(`SQLite lifecycle cleanup entry changed for ${planned.sessionKey}`); + } + deleteSqliteSessionEntryRows(database, planned.sessionKey); + removedEntries += 1; + } + return removedEntries; +} + +function writeSessionEntry( + database: OpenClawAgentDatabase, + sessionKey: string, + entry: SessionEntry, +): void { + const db = getSessionKysely(database.db); + const normalizedEntry = normalizeSqliteSessionEntryTimestamp(entry); + const updatedAt = normalizedEntry.updatedAt; + const sessionRow = bindSqliteSessionRoot({ entry: normalizedEntry, sessionKey, updatedAt }); + executeSqliteQuerySync( + database.db, + db + .insertInto("sessions") + .values(sessionRow) + .onConflict((conflict) => + conflict.column("session_id").doUpdateSet({ + session_key: sessionKey, + session_scope: sessionRow.session_scope, + updated_at: updatedAt, + started_at: sessionRow.started_at, + ended_at: sessionRow.ended_at, + status: sessionRow.status, + chat_type: sessionRow.chat_type, + channel: sessionRow.channel, + account_id: sessionRow.account_id, + model_provider: sessionRow.model_provider, + model: sessionRow.model, + agent_harness_id: sessionRow.agent_harness_id, + parent_session_key: sessionRow.parent_session_key, + spawned_by: sessionRow.spawned_by, + display_name: sessionRow.display_name, + }), + ), + ); + writeSessionRoute(database, { + sessionId: sessionRow.session_id, + sessionKey, + updatedAt, + }); + executeSqliteQuerySync( + database.db, + db + .insertInto("session_entries") + .values({ + session_key: sessionKey, + session_id: normalizedEntry.sessionId, + entry_json: JSON.stringify(normalizedEntry), + updated_at: updatedAt, + }) + .onConflict((conflict) => + conflict.column("session_key").doUpdateSet({ + session_id: normalizedEntry.sessionId, + entry_json: JSON.stringify(normalizedEntry), + updated_at: updatedAt, + }), + ), + ); +} + +function normalizeSqliteSessionEntryTimestamp(entry: SessionEntry): SessionEntry { + if (typeof entry.updatedAt === "number" && Number.isFinite(entry.updatedAt)) { + return entry; + } + const updatedAt = + typeof entry.sessionStartedAt === "number" && Number.isFinite(entry.sessionStartedAt) + ? entry.sessionStartedAt + : Date.now(); + return { + ...entry, + updatedAt, + }; +} + +function ensureTranscriptSessionRoot( + database: OpenClawAgentDatabase, + scope: ResolvedTranscriptScope, + updatedAt: number, +): void { + const db = getSessionKysely(database.db); + executeSqliteQuerySync( + database.db, + db + .insertInto("sessions") + .values({ + session_id: scope.sessionId, + session_key: scope.sessionKey, + session_scope: "conversation", + created_at: updatedAt, + updated_at: updatedAt, + }) + .onConflict((conflict) => + conflict.column("session_id").doUpdateSet({ + session_key: scope.sessionKey, + updated_at: updatedAt, + }), + ), + ); + writeTranscriptSessionRoute(database, { + sessionId: scope.sessionId, + sessionKey: scope.sessionKey, + updatedAt, + }); +} + +function bindSqliteSessionRoot(params: { + entry: SessionEntry; + sessionKey: string; + updatedAt: number; +}) { + const updatedAt = Number.isFinite(params.entry.updatedAt) + ? params.entry.updatedAt + : params.updatedAt; + return { + session_id: params.entry.sessionId, + session_key: params.sessionKey, + session_scope: resolveSqliteSessionScope(params.entry, params.sessionKey), + created_at: resolveSqliteSessionCreatedAt(params.entry, updatedAt), + updated_at: updatedAt, + started_at: finiteSqliteNumber(params.entry.startedAt), + ended_at: finiteSqliteNumber(params.entry.endedAt), + status: normalizeSqliteStatus(params.entry.status), + chat_type: normalizeSqliteChatType(params.entry.chatType), + channel: resolveSqliteSessionChannel(params.entry), + account_id: resolveSqliteSessionAccountId(params.entry), + primary_conversation_id: null, + model_provider: normalizeSqliteText(params.entry.modelProvider), + model: normalizeSqliteText(params.entry.model), + agent_harness_id: normalizeSqliteText(params.entry.agentHarnessId), + parent_session_key: normalizeSqliteText(params.entry.parentSessionKey), + spawned_by: normalizeSqliteText(params.entry.spawnedBy), + display_name: resolveSqliteSessionDisplayName(params.entry), + }; +} + +function writeSessionRoute( + database: OpenClawAgentDatabase, + params: { sessionId: string; sessionKey: string; updatedAt: number }, +): void { + const db = getSessionKysely(database.db); + executeSqliteQuerySync( + database.db, + db + .insertInto("session_routes") + .values({ + session_key: params.sessionKey, + session_id: params.sessionId, + updated_at: params.updatedAt, + }) + .onConflict((conflict) => + conflict.column("session_key").doUpdateSet({ + session_id: params.sessionId, + updated_at: params.updatedAt, + }), + ), + ); +} + +function writeTranscriptSessionRoute( + database: OpenClawAgentDatabase, + params: { sessionId: string; sessionKey: string; updatedAt: number }, +): void { + const db = getSessionKysely(database.db); + const existing = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_routes") + .select("session_id") + .where("session_key", "=", params.sessionKey), + ); + // Transcript-only appends may arrive late from an old run. They can create + // missing routes, but must not move a current session key back to a stale id. + if (existing && existing.session_id !== params.sessionId) { + return; + } + writeSessionRoute(database, params); +} + +function resolveSqliteSessionScope( + entry: Pick<SessionEntry, "chatType">, + sessionKey: string, +): "conversation" | "shared-main" | "group" | "channel" { + const chatType = normalizeSqliteChatType(entry.chatType); + const normalizedKey = sessionKey.trim().toLowerCase(); + if (chatType === "direct" && (normalizedKey === "main" || normalizedKey.endsWith(":main"))) { + return "shared-main"; + } + if (chatType === "group" || chatType === "channel") { + return chatType; + } + return "conversation"; +} + +function resolveSqliteSessionCreatedAt(entry: SessionEntry, updatedAt: number): number { + for (const candidate of [entry.sessionStartedAt, entry.startedAt, entry.updatedAt, updatedAt]) { + if (typeof candidate === "number" && Number.isFinite(candidate) && candidate >= 0) { + return candidate; + } + } + return updatedAt; +} + +function finiteSqliteNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function resolveSqliteSessionChannel(entry: SessionEntry): string | null { + return ( + normalizeSqliteText(entry.channel) ?? + normalizeSqliteText(entry.deliveryContext?.channel) ?? + normalizeSqliteText(entry.lastChannel) ?? + normalizeSqliteText(entry.origin?.provider) + ); +} + +function resolveSqliteSessionAccountId(entry: SessionEntry): string | null { + return ( + normalizeSqliteText(entry.deliveryContext?.accountId) ?? + normalizeSqliteText(entry.lastAccountId) ?? + normalizeSqliteText(entry.origin?.accountId) + ); +} + +function resolveSqliteSessionDisplayName(entry: SessionEntry): string | null { + return ( + normalizeSqliteText(entry.displayName) ?? + normalizeSqliteText(entry.label) ?? + normalizeSqliteText(entry.subject) ?? + normalizeSqliteText(entry.groupId) + ); +} + +function readNextTranscriptSeq(database: OpenClawAgentDatabase, sessionId: string): number { + const db = getSessionKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("transcript_events") + .select((eb) => eb.fn.max<number | bigint>("seq").as("max_seq")) + .where("session_id", "=", sessionId), + ); + const maxSeq = + row?.max_seq === null || row?.max_seq === undefined ? -1 : normalizeSqliteNumber(row.max_seq); + return maxSeq + 1; +} + +function deleteSqliteTranscriptEventsInTransaction( + database: OpenClawAgentDatabase, + sessionId: string, +): boolean { + const db = getSessionKysely(database.db); + executeSqliteQuerySync( + database.db, + db.deleteFrom("transcript_event_identities").where("session_id", "=", sessionId), + ); + const result = executeSqliteQuerySync( + database.db, + db.deleteFrom("transcript_events").where("session_id", "=", sessionId), + ); + return (result.numAffectedRows ?? 0n) > 0n; +} + +const DEFAULT_PARENT_FORK_MAX_TOKENS = 100_000; + +function formatParentForkTooLargeMessage(params: { + parentTokens: number; + maxTokens: number; +}): string { + return ( + `Parent context is too large to fork (${params.parentTokens}/${params.maxTokens} tokens); ` + + "starting with isolated context instead." + ); +} + +function resolveSqliteParentForkDecision( + parentEntry: SessionEntry, + transcriptEstimate?: SqliteTranscriptParentTokenEstimate, +): SessionParentForkDecision { + const maxTokens = DEFAULT_PARENT_FORK_MAX_TOKENS; + const parentTokens = + resolveFreshSessionTotalTokens(parentEntry) ?? + (transcriptEstimate?.kind === "exact-context" + ? transcriptEstimate.tokens + : maxPositiveTokenCount(transcriptEstimate?.tokens, resolveSessionTotalTokens(parentEntry))); + if (typeof parentTokens === "number" && parentTokens > maxTokens) { + return { + status: "skip", + reason: "parent-too-large", + maxTokens, + parentTokens, + message: formatParentForkTooLargeMessage({ parentTokens, maxTokens }), + }; + } + return { + status: "fork", + maxTokens, + ...(typeof parentTokens === "number" ? { parentTokens } : {}), + }; +} + +/** Resolves the parent fork decision using SQLite transcript rows when totals are stale. */ +export async function resolveSqliteSessionParentForkDecision(params: { + parentEntry: SessionEntry; + storePath: string; +}): Promise<SessionParentForkDecision> { + const parentSessionId = + typeof params.parentEntry.sessionId === "string" ? params.parentEntry.sessionId : ""; + const needsTranscriptTokenEstimate = + typeof resolveFreshSessionTotalTokens(params.parentEntry) !== "number" && + parentSessionId.length > 0; + if (!needsTranscriptTokenEstimate) { + return resolveSqliteParentForkDecision(params.parentEntry); + } + const resolved = resolveSqliteStoreScope(params.storePath); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + return resolveSqliteParentForkDecision( + params.parentEntry, + estimateSqliteTranscriptPromptTokens( + loadSqliteTranscriptEventsFromDatabase(database, parentSessionId), + ), + ); +} + +function estimateSqliteTranscriptPromptTokens( + events: readonly TranscriptEvent[], +): SqliteTranscriptParentTokenEstimate | undefined { + let byteEstimate = 0; + let latestUsageEstimate: number | undefined; + let latestUsageEstimateIsExactContext = false; + let trailingBytes = 0; + for (const event of selectSqliteParentForkTokenEstimateEvents(events)) { + const serialized = JSON.stringify(event); + const serializedBytes = Buffer.byteLength(serialized) + 1; + byteEstimate += serializedBytes; + if (!isRecord(event)) { + if (latestUsageEstimate !== undefined) { + trailingBytes += serializedBytes; + } + continue; + } + const message = isRecord(event.message) ? event.message : undefined; + const usageRaw = isRecord(message?.usage) + ? message.usage + : isRecord(event.usage) + ? event.usage + : undefined; + if (!usageRaw) { + if (latestUsageEstimate !== undefined) { + trailingBytes += serializedBytes; + } + continue; + } + const contextUsage = readSqliteTranscriptContextUsage(usageRaw); + if (contextUsage?.state === "unavailable") { + latestUsageEstimate = undefined; + latestUsageEstimateIsExactContext = false; + trailingBytes = 0; + continue; + } + if (contextUsage?.state === "available") { + latestUsageEstimate = normalizePositiveTokenCount(contextUsage.totalTokens); + latestUsageEstimateIsExactContext = true; + trailingBytes = 0; + continue; + } + const usage = normalizeUsage(usageRaw); + const promptTokens = normalizePositiveTokenCount( + derivePromptTokens({ + input: usage?.input, + cacheRead: usage?.cacheRead, + cacheWrite: usage?.cacheWrite, + }), + ); + const outputTokens = normalizePositiveTokenCount(usage?.output) ?? 0; + const totalTokens = + promptTokens === undefined + ? undefined + : normalizePositiveTokenCount(promptTokens + outputTokens); + if (typeof totalTokens === "number") { + latestUsageEstimate = totalTokens; + latestUsageEstimateIsExactContext = false; + trailingBytes = 0; + } + } + if (latestUsageEstimate !== undefined) { + const trailingTokens = Math.ceil(trailingBytes / 4); + const tokens = normalizePositiveTokenCount(latestUsageEstimate + trailingTokens); + return tokens === undefined + ? undefined + : { + kind: latestUsageEstimateIsExactContext ? "exact-context" : "legacy-or-bytes", + tokens, + }; + } + const estimatedFromBytes = Math.ceil(byteEstimate / 4); + const tokens = normalizePositiveTokenCount(estimatedFromBytes); + return tokens === undefined ? undefined : { kind: "legacy-or-bytes", tokens }; +} + +function selectSqliteParentForkTokenEstimateEvents( + events: readonly TranscriptEvent[], +): TranscriptEvent[] { + const entries = events.filter((entry) => !(isRecord(entry) && entry.type === "session")); + const tree = scanSessionTranscriptTree(entries); + const visiblePath = selectSessionTranscriptTreePathNodes(tree, tree.leafId); + const appendPath = selectSessionTranscriptTreePathNodes(tree, tree.appendParentId); + return mergeSessionTranscriptVisiblePathWithOpaqueAppendPath({ + visiblePath, + appendPath, + appendParentId: tree.appendParentId, + }).nodes.flatMap((node) => node.entry); +} + +function isRecord(value: unknown): value is Record<string, unknown> { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function normalizePositiveTokenCount(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value > 0 + ? Math.floor(value) + : undefined; +} + +function maxPositiveTokenCount(...values: Array<number | undefined>): number | undefined { + let max: number | undefined; + for (const value of values) { + const normalized = normalizePositiveTokenCount(value); + if (normalized !== undefined && (max === undefined || normalized > max)) { + max = normalized; + } + } + return max; +} + +function readSqliteTranscriptContextUsage( + usageRaw: Record<string, unknown>, +): { state: "available"; totalTokens: number } | { state: "unavailable" } | undefined { + const contextUsage = usageRaw.contextUsage; + if (!isRecord(contextUsage)) { + return undefined; + } + if (contextUsage.state === "unavailable") { + return { state: "unavailable" }; + } + if (contextUsage.state !== "available") { + return undefined; + } + const totalTokens = normalizePositiveTokenCount(contextUsage.totalTokens); + return totalTokens === undefined ? undefined : { state: "available", totalTokens }; +} + +function generateParentForkEntryId(existingIds: Set<string>): string { + for (let attempt = 0; attempt < 100; attempt += 1) { + const id = randomUUID().slice(0, 8); + if (!existingIds.has(id)) { + existingIds.add(id); + return id; + } + } + const id = randomUUID(); + existingIds.add(id); + return id; +} + +function hasAssistantEntry(entries: readonly TranscriptEvent[]): boolean { + return entries.some( + (entry) => + isRecord(entry) && + entry.type === "message" && + isRecord(entry.message) && + entry.message.role === "assistant", + ); +} + +function collectParentForkBranchLabels(params: { + allEntries: readonly TranscriptEvent[]; + pathEntryIds: Set<string>; +}): Array<{ targetId: string; label: string; timestamp: string }> { + const labelsToWrite: Array<{ targetId: string; label: string; timestamp: string }> = []; + for (const entry of params.allEntries) { + if ( + isRecord(entry) && + entry.type === "label" && + typeof entry.label === "string" && + typeof entry.targetId === "string" && + typeof entry.id === "string" && + !params.pathEntryIds.has(entry.id) && + params.pathEntryIds.has(entry.targetId) && + typeof entry.timestamp === "string" + ) { + labelsToWrite.push({ + targetId: entry.targetId, + label: entry.label, + timestamp: entry.timestamp, + }); + } + } + return labelsToWrite; +} + +function readSqliteParentForkSourceTranscript( + database: OpenClawAgentDatabase, + parentSessionId: string, +): SqliteParentForkSourceTranscript | null { + const fileEntries = loadSqliteTranscriptEventsFromDatabase(database, parentSessionId); + if (fileEntries.length === 0) { + return null; + } + const header = fileEntries.find( + (entry): entry is Record<string, unknown> => isRecord(entry) && entry.type === "session", + ); + const entries = fileEntries.filter((entry) => !(isRecord(entry) && entry.type === "session")); + const tree = scanSessionTranscriptTree(entries); + const visiblePath = selectSessionTranscriptTreePathNodes(tree, tree.leafId); + const appendPath = selectSessionTranscriptTreePathNodes(tree, tree.appendParentId); + const mergedPath = mergeSessionTranscriptVisiblePathWithOpaqueAppendPath({ + visiblePath, + appendPath, + appendParentId: tree.appendParentId, + }); + const branchEntries = mergedPath.nodes.flatMap((node) => { + if (!isRecord(node.entry)) { + return []; + } + const parentId = node.selectedParentId; + return [node.entry.parentId === parentId ? node.entry : { ...node.entry, parentId }]; + }); + const pathEntryIds = new Set( + branchEntries.flatMap((entry) => + isRecord(entry) && typeof entry.id === "string" ? [entry.id] : [], + ), + ); + const lastLeafUpdateNode = tree.nodes.findLast((node) => node.leafId !== undefined); + const lastLeafUpdateEntry = lastLeafUpdateNode?.entry; + return { + appendParentId: mergedPath.appendParentId, + ...(lastLeafUpdateNode?.appendMode ? { appendMode: lastLeafUpdateNode.appendMode } : {}), + branchEntries, + cwd: typeof header?.cwd === "string" ? header.cwd : undefined, + labelsToWrite: collectParentForkBranchLabels({ allEntries: entries, pathEntryIds }), + leafId: tree.leafId, + preserveLeafControl: isSessionTranscriptLeafControl(lastLeafUpdateEntry), + }; +} + +function buildParentForkLabelEntries(params: { + labelsToWrite: Array<{ targetId: string; label: string; timestamp: string }>; + pathEntryIds: Set<string>; + lastEntryId: string | null; +}): TranscriptEvent[] { + let parentId = params.lastEntryId; + const labelEntries: TranscriptEvent[] = []; + for (const { targetId, label, timestamp } of params.labelsToWrite) { + const labelEntry = { + type: "label", + id: generateParentForkEntryId(params.pathEntryIds), + parentId, + timestamp, + targetId, + label, + }; + params.pathEntryIds.add(labelEntry.id); + labelEntries.push(labelEntry); + parentId = labelEntry.id; + } + return labelEntries; +} + +function writeSqliteParentForkTranscriptInTransaction( + database: OpenClawAgentDatabase, + targetScope: ResolvedTranscriptScope, + params: { + parentSessionFile: string; + source: SqliteParentForkSourceTranscript; + }, +): void { + const timestamp = new Date().toISOString(); + const pathEntries = params.source.branchEntries; + const pathEntryIds = new Set( + pathEntries.flatMap((entry) => + isRecord(entry) && typeof entry.id === "string" ? [entry.id] : [], + ), + ); + const lastPathEntry = pathEntries.at(-1); + const lastPathEntryId = + isRecord(lastPathEntry) && typeof lastPathEntry.id === "string" ? lastPathEntry.id : null; + const labelEntries = buildParentForkLabelEntries({ + labelsToWrite: params.source.labelsToWrite, + pathEntryIds, + lastEntryId: lastPathEntryId, + }); + const leafEntry = params.source.preserveLeafControl + ? { + type: "leaf", + id: generateParentForkEntryId(pathEntryIds), + parentId: (labelEntries.at(-1) as { id?: string } | undefined)?.id ?? lastPathEntryId, + timestamp, + targetId: params.source.leafId, + appendParentId: params.source.appendParentId, + ...(params.source.appendMode ? { appendMode: params.source.appendMode } : {}), + } + : null; + appendTranscriptEventInTransaction(database, targetScope, { + ...createSessionTranscriptHeader({ + cwd: params.source.cwd, + sessionId: targetScope.sessionId, + }), + parentSession: params.parentSessionFile, + }); + for (const event of [...pathEntries, ...labelEntries, ...(leafEntry ? [leafEntry] : [])]) { + appendTranscriptEventInTransaction(database, targetScope, event); + } +} + +function forkSqliteParentTranscriptInTransaction( + database: OpenClawAgentDatabase, + resolved: ResolvedSqliteScope, + params: { + parentEntry: SessionEntry; + parentSessionKey: string; + targetSessionId?: string; + targetSessionKey: string; + }, +): ForkSessionFromParentTranscriptResult { + if (!params.parentEntry.sessionId) { + return { status: "missing-parent" }; + } + const source = readSqliteParentForkSourceTranscript(database, params.parentEntry.sessionId); + if (!source) { + return { status: "failed" }; + } + const sessionId = params.targetSessionId ?? randomUUID(); + const targetScope = { + ...resolved, + sessionId, + sessionKey: normalizeSqliteSessionKey(params.targetSessionKey), + }; + const parentSessionFile = formatSqliteSessionMarkerForScope({ + ...resolved, + sessionId: params.parentEntry.sessionId, + sessionKey: normalizeSqliteSessionKey(params.parentSessionKey), + }); + const sessionFile = formatSqliteSessionMarkerForScope(targetScope); + writeSqliteForkedChildTranscriptInTransaction(database, targetScope, { + parentSessionFile, + source, + }); + return { + status: "created", + transcript: { + sessionFile, + sessionId, + }, + }; +} + +function branchSqliteCompactionCheckpointSessionInTransaction( + database: OpenClawAgentDatabase, + params: { + checkpointId: string; + parentSessionKey: string; + resolved: ResolvedSqliteScope; + sourceKey: string; + targetKey: string; + }, +): SqliteCompactionCheckpointSessionMutationResult { + const currentEntry = readSessionEntryRow(database, params.sourceKey)?.entry; + if (!currentEntry?.sessionId) { + return { status: "missing-session" }; + } + const checkpoint = readSessionCompactionCheckpoint(currentEntry, params.checkpointId); + if (!checkpoint) { + return { status: "missing-checkpoint" }; + } + const forked = forkSqliteCheckpointTranscriptInTransaction(database, params.resolved, { + checkpoint, + targetSessionKey: params.targetKey, + }); + if (forked.status !== "created") { + return forked; + } + + const label = currentEntry.label?.trim() + ? `${currentEntry.label.trim()} (checkpoint)` + : "Checkpoint branch"; + const nextEntry = cloneSqliteCheckpointSessionEntry({ + currentEntry, + label, + nextSessionFile: forked.sessionFile, + nextSessionId: forked.sessionId, + parentSessionKey: params.parentSessionKey, + totalTokens: forked.totalTokens, + }); + writeSessionEntry(database, params.targetKey, nextEntry); + return { + status: "created", + key: params.targetKey, + checkpoint, + entry: nextEntry, + }; +} + +function restoreSqliteCompactionCheckpointSessionInTransaction( + database: OpenClawAgentDatabase, + params: { + checkpointId: string; + resolved: ResolvedSqliteScope; + sourceKey: string; + targetKey: string; + }, +): SqliteCompactionCheckpointSessionMutationResult { + const currentEntry = readSessionEntryRow(database, params.sourceKey)?.entry; + if (!currentEntry?.sessionId) { + return { status: "missing-session" }; + } + const checkpoint = readSessionCompactionCheckpoint(currentEntry, params.checkpointId); + if (!checkpoint) { + return { status: "missing-checkpoint" }; + } + const restored = forkSqliteCheckpointTranscriptInTransaction(database, params.resolved, { + checkpoint, + targetSessionKey: params.targetKey, + }); + if (restored.status !== "created") { + return restored; + } + + const nextEntry = cloneSqliteCheckpointSessionEntry({ + currentEntry, + nextSessionFile: restored.sessionFile, + nextSessionId: restored.sessionId, + preserveCompactionCheckpoints: true, + totalTokens: restored.totalTokens, + }); + writeSessionEntry(database, params.targetKey, nextEntry); + return { + status: "created", + key: params.targetKey, + checkpoint, + entry: nextEntry, + }; +} + +function forkSqliteCheckpointTranscriptInTransaction( + database: OpenClawAgentDatabase, + resolved: ResolvedSqliteScope, + params: { + checkpoint: SessionCompactionCheckpoint; + targetSessionKey: string; + }, +): + | { + status: "created"; + sessionId: string; + sessionFile: string; + totalTokens?: number; + } + | { status: "missing-boundary" } + | { status: "failed" } { + const sources = resolveSqliteCheckpointTranscriptForkSources(params.checkpoint); + if (sources.length === 0) { + return { status: "missing-boundary" }; + } + let lastFailure: { status: "missing-boundary" } | { status: "failed" } = { + status: "missing-boundary", + }; + let selected: + | { + source: SqliteCheckpointTranscriptForkSource; + rows: TranscriptEvent[]; + } + | undefined; + for (const source of sources) { + const rows = readSqliteTranscriptRowsForFork(database, source); + if (rows.status === "created") { + selected = { source, rows: rows.events }; + break; + } + lastFailure = rows; + } + if (!selected) { + return lastFailure; + } + + const sessionId = randomUUID(); + const targetScope = { + ...resolved, + sessionId, + sessionKey: params.targetSessionKey, + }; + const sessionFile = formatSqliteSessionMarkerForScope(targetScope); + appendTranscriptEventInTransaction( + database, + targetScope, + createSessionTranscriptHeader({ + cwd: readTranscriptHeaderCwd(selected.rows), + sessionId, + }), + ); + for (const event of selected.rows) { + if (isSessionTranscriptHeader(event)) { + continue; + } + appendTranscriptEventInTransaction(database, targetScope, event); + } + return { + status: "created", + sessionId, + sessionFile, + ...(typeof selected.source.totalTokens === "number" + ? { totalTokens: selected.source.totalTokens } + : {}), + }; +} + +function resolveSqliteCheckpointTranscriptForkSources( + checkpoint: SessionCompactionCheckpoint, +): SqliteCheckpointTranscriptForkSource[] { + const sources: SqliteCheckpointTranscriptForkSource[] = []; + if (checkpoint.preCompaction.sessionId) { + const preLeafId = checkpoint.preCompaction.entryId ?? checkpoint.preCompaction.leafId; + sources.push({ + sessionId: checkpoint.preCompaction.sessionId, + ...(preLeafId ? { leafId: preLeafId } : {}), + ...(typeof checkpoint.tokensBefore === "number" + ? { totalTokens: checkpoint.tokensBefore } + : {}), + }); + } + + const postLeafId = checkpoint.postCompaction.entryId ?? checkpoint.postCompaction.leafId; + if (checkpoint.postCompaction.sessionId && postLeafId) { + sources.push({ + sessionId: checkpoint.postCompaction.sessionId, + leafId: postLeafId, + ...(typeof checkpoint.tokensAfter === "number" + ? { totalTokens: checkpoint.tokensAfter } + : {}), + }); + } + + return sources; +} + +function readSqliteTranscriptRowsForFork( + database: OpenClawAgentDatabase, + source: { sessionId: string; leafId?: string }, +): { status: "created"; events: TranscriptEvent[] } | { status: "missing-boundary" | "failed" } { + const boundarySeq = source.leafId + ? readTranscriptIdentityByEventId(database, source.sessionId, source.leafId)?.seq + : undefined; + if (source.leafId && boundarySeq === undefined) { + return { status: "missing-boundary" }; + } + + const db = getSessionKysely(database.db); + const query = db + .selectFrom("transcript_events") + .select(["event_json", "seq"]) + .where("session_id", "=", source.sessionId) + .orderBy("seq", "asc"); + const rows = executeSqliteQuerySync( + database.db, + boundarySeq === undefined ? query : query.where("seq", "<=", boundarySeq), + ).rows; + if (rows.length === 0) { + return { status: "failed" }; + } + try { + return { + status: "created", + events: rows.map((row) => JSON.parse(row.event_json) as TranscriptEvent), + }; + } catch { + return { status: "failed" }; + } +} + +function readSessionCompactionCheckpoint( + entry: Pick<SessionEntry, "compactionCheckpoints">, + checkpointId: string, +): SessionCompactionCheckpoint | undefined { + const normalizedCheckpointId = checkpointId.trim(); + if (!normalizedCheckpointId || !Array.isArray(entry.compactionCheckpoints)) { + return undefined; + } + return entry.compactionCheckpoints.find( + (checkpoint) => checkpoint.checkpointId === normalizedCheckpointId, + ); +} + +function cloneSqliteCheckpointSessionEntry(params: { + currentEntry: SessionEntry; + nextSessionId: string; + nextSessionFile: string; + label?: string; + parentSessionKey?: string; + totalTokens?: number; + preserveCompactionCheckpoints?: boolean; +}): SessionEntry { + const hasTotalTokens = + typeof params.totalTokens === "number" && Number.isFinite(params.totalTokens); + return { + ...params.currentEntry, + sessionId: params.nextSessionId, + sessionFile: params.nextSessionFile, + updatedAt: Date.now(), + systemSent: false, + abortedLastRun: false, + startedAt: undefined, + endedAt: undefined, + runtimeMs: undefined, + status: undefined, + inputTokens: undefined, + outputTokens: undefined, + cacheRead: undefined, + cacheWrite: undefined, + estimatedCostUsd: undefined, + totalTokens: hasTotalTokens ? params.totalTokens : undefined, + totalTokensFresh: hasTotalTokens ? true : undefined, + label: params.label ?? params.currentEntry.label, + parentSessionKey: params.parentSessionKey ?? params.currentEntry.parentSessionKey, + compactionCheckpoints: params.preserveCompactionCheckpoints + ? params.currentEntry.compactionCheckpoints + : undefined, + }; +} + +function readTranscriptHeaderCwd(events: readonly TranscriptEvent[]): string | undefined { + const header = events.find(isSessionTranscriptHeader) as { cwd?: unknown } | undefined; + return typeof header?.cwd === "string" && header.cwd.trim() ? header.cwd : undefined; +} + +function isSessionTranscriptHeader(event: TranscriptEvent): boolean { + return Boolean( + event && + typeof event === "object" && + !Array.isArray(event) && + (event as { type?: unknown }).type === "session", + ); +} + +function appendTranscriptEventInTransaction( + database: OpenClawAgentDatabase, + scope: ResolvedTranscriptScope, + event: TranscriptEvent, + options: { dedupeByMessageIdempotency?: boolean } = {}, +): boolean { + const db = getSessionKysely(database.db); + const createdAt = readEventTimestamp(event) ?? Date.now(); + ensureTranscriptSessionRoot(database, scope, createdAt); + const identity = readTranscriptEventIdentity(event); + if (identity && readTranscriptIdentityByEventId(database, scope.sessionId, identity.eventId)) { + return false; + } + if ( + identity?.messageIdempotencyKey && + options.dedupeByMessageIdempotency && + readTranscriptIdentityByMessageIdempotencyKey( + database, + scope.sessionId, + identity.messageIdempotencyKey, + ) + ) { + return false; + } + const seq = readNextTranscriptSeq(database, scope.sessionId); + executeSqliteQuerySync( + database.db, + db.insertInto("transcript_events").values({ + session_id: scope.sessionId, + seq, + event_json: JSON.stringify(event), + created_at: createdAt, + }), + ); + if (!identity) { + return true; + } + // Caller-checked appends may intentionally keep a duplicate key in the + // message payload, but the identity index can only point at one row. + const indexedMessageIdempotencyKey = + identity.messageIdempotencyKey && + !options.dedupeByMessageIdempotency && + readTranscriptIdentityByMessageIdempotencyKey( + database, + scope.sessionId, + identity.messageIdempotencyKey, + ) + ? undefined + : identity.messageIdempotencyKey; + executeSqliteQuerySync( + database.db, + db + .insertInto("transcript_event_identities") + .values({ + session_id: scope.sessionId, + event_id: identity.eventId, + seq, + event_type: identity.eventType, + parent_id: identity.parentId, + message_idempotency_key: indexedMessageIdempotencyKey, + created_at: createdAt, + }) + .onConflict((conflict) => conflict.columns(["session_id", "event_id"]).doNothing()), + ); + return true; +} + +function appendTranscriptEventRowInTransaction( + database: OpenClawAgentDatabase, + scope: ResolvedTranscriptScope, + event: TranscriptEvent, + seq: number, + state: { + seenEventIds: Set<string>; + seenMessageIdempotencyKeys: Set<string>; + }, +): boolean { + const db = getSessionKysely(database.db); + const createdAt = readEventTimestamp(event) ?? Date.now(); + const identity = readTranscriptEventIdentity(event); + if (identity && state.seenEventIds.has(identity.eventId)) { + return false; + } + executeSqliteQuerySync( + database.db, + db.insertInto("transcript_events").values({ + session_id: scope.sessionId, + seq, + event_json: JSON.stringify(event), + created_at: createdAt, + }), + ); + if (!identity) { + return true; + } + state.seenEventIds.add(identity.eventId); + const indexedMessageIdempotencyKey = + identity.messageIdempotencyKey && + !state.seenMessageIdempotencyKeys.has(identity.messageIdempotencyKey) + ? identity.messageIdempotencyKey + : undefined; + if (indexedMessageIdempotencyKey) { + state.seenMessageIdempotencyKeys.add(indexedMessageIdempotencyKey); + } + executeSqliteQuerySync( + database.db, + db.insertInto("transcript_event_identities").values({ + session_id: scope.sessionId, + event_id: identity.eventId, + seq, + event_type: identity.eventType, + parent_id: identity.parentId, + message_idempotency_key: indexedMessageIdempotencyKey, + created_at: createdAt, + }), + ); + return true; +} + +function ensureTranscriptHeader( + database: OpenClawAgentDatabase, + scope: ResolvedTranscriptScope, + cwd: string | undefined, + now: number, +): void { + const db = getSessionKysely(database.db); + const existing = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("transcript_events") + .select("seq") + .where("session_id", "=", scope.sessionId) + .limit(1), + ); + if (existing) { + return; + } + appendTranscriptEventInTransaction( + database, + scope, + createSessionTranscriptHeader({ + cwd, + sessionId: scope.sessionId, + }), + ); + ensureTranscriptSessionRoot(database, scope, now); +} + +function readActiveTranscriptAppendParentId( + database: OpenClawAgentDatabase, + sessionId: string, +): string | null { + const db = getSessionKysely(database.db); + const latest = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("transcript_event_identities as ti") + .innerJoin("transcript_events as te", (join) => + join.onRef("te.session_id", "=", "ti.session_id").onRef("te.seq", "=", "ti.seq"), + ) + .select(["ti.event_type", "te.event_json"]) + .where("ti.session_id", "=", sessionId) + .orderBy("ti.seq", "desc") + .limit(1), + ); + if (!latest) { + return null; + } + try { + const event = JSON.parse(latest.event_json) as unknown; + const treeEntry = parseSessionTranscriptTreeEntry(event); + if (!treeEntry) { + return resolveVisibleTranscriptAppendParentId( + loadSqliteTranscriptEventsFromDatabase(database, sessionId), + ); + } + if (latest.event_type !== "leaf") { + return treeEntry.appendParentId; + } + const leafReferencesKnown = + treeEntry.leafId !== undefined && + transcriptTreeReferenceExists(database, sessionId, treeEntry.leafId) && + transcriptTreeReferenceExists(database, sessionId, treeEntry.appendParentId); + if (isSessionTranscriptLeafControl(event) && leafReferencesKnown) { + return treeEntry.appendParentId; + } + } catch { + return resolveVisibleTranscriptAppendParentId( + loadSqliteTranscriptEventsFromDatabase(database, sessionId), + ); + } + return resolveVisibleTranscriptAppendParentId( + loadSqliteTranscriptEventsFromDatabase(database, sessionId), + ); +} + +function transcriptTreeReferenceExists( + database: OpenClawAgentDatabase, + sessionId: string, + eventId: string | null, +): boolean { + return ( + eventId === null || readTranscriptIdentityByEventId(database, sessionId, eventId) !== undefined + ); +} + +function replaceSqliteTranscriptEventsInTransaction( + database: OpenClawAgentDatabase, + resolved: ResolvedTranscriptScope, + events: readonly TranscriptEvent[], +): void { + deleteSqliteTranscriptEventsInTransaction(database, resolved.sessionId); + if (events.length === 0) { + return; + } + ensureTranscriptSessionRoot(database, resolved, readEventTimestamp(events[0]) ?? Date.now()); + let seq = 0; + const seenEventIds = new Set<string>(); + const seenMessageIdempotencyKeys = new Set<string>(); + for (const event of events) { + const appended = appendTranscriptEventRowInTransaction(database, resolved, event, seq, { + seenEventIds, + seenMessageIdempotencyKeys, + }); + if (appended) { + seq += 1; + } + } +} + +function readTranscriptIdentityByEventId( + database: OpenClawAgentDatabase, + sessionId: string, + eventId: string, +): { eventId: string; seq: number } | undefined { + const db = getSessionKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("transcript_event_identities") + .select(["event_id", "seq"]) + .where("session_id", "=", sessionId) + .where("event_id", "=", eventId), + ); + return row ? { eventId: row.event_id, seq: row.seq } : undefined; +} + +function readTranscriptIdentityByMessageIdempotencyKey( + database: OpenClawAgentDatabase, + sessionId: string, + idempotencyKey: string, +): { eventId: string; seq: number } | undefined { + const db = getSessionKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("transcript_event_identities") + .select(["event_id", "seq"]) + .where("session_id", "=", sessionId) + .where("message_idempotency_key", "=", idempotencyKey) + .orderBy("seq", "desc") + .limit(1), + ); + return row ? { eventId: row.event_id, seq: row.seq } : undefined; +} + +function readTranscriptMessageByIdempotencyKey( + database: OpenClawAgentDatabase, + scope: ResolvedTranscriptScope, + idempotencyKey: string, +): { messageId: string; message: unknown } | undefined { + const identity = readTranscriptIdentityByMessageIdempotencyKey( + database, + scope.sessionId, + idempotencyKey, + ); + if (!identity) { + return undefined; + } + return readTranscriptMessageByIdentity(database, scope, identity); +} + +function readTranscriptMessageByScopedIdempotencyKey( + database: OpenClawAgentDatabase, + scope: ResolvedTranscriptScope, + idempotencyKey: string, + lookup: TranscriptMessageAppendOptions<unknown>["idempotencyLookup"], +): { messageId: string; message: unknown } | undefined { + if (lookup !== "scan-assistant") { + return readTranscriptMessageByIdempotencyKey(database, scope, idempotencyKey); + } + const found = findSqliteTranscriptEventInDatabase(database, scope.sessionId, (event) => { + const message = readTranscriptEventMessage(event); + return message?.role === "assistant" && message.idempotencyKey === idempotencyKey; + }); + if (!found) { + return undefined; + } + const message = readTranscriptEventMessage(found.event); + if (!message) { + return undefined; + } + return { + messageId: readTranscriptEventId(found.event) ?? idempotencyKey, + message, + }; +} + +function readTranscriptMessageByEventId( + database: OpenClawAgentDatabase, + scope: ResolvedTranscriptScope, + eventId: string, +): { messageId: string; message: unknown } | undefined { + const identity = readTranscriptIdentityByEventId(database, scope.sessionId, eventId); + if (!identity) { + return undefined; + } + return readTranscriptMessageByIdentity(database, scope, identity); +} + +function readTranscriptMessageByIdentity( + database: OpenClawAgentDatabase, + scope: ResolvedTranscriptScope, + identity: { eventId: string; seq: number }, +): { messageId: string; message: unknown } | undefined { + const db = getSessionKysely(database.db); + const eventRow = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("transcript_events") + .select(["event_json"]) + .where("session_id", "=", scope.sessionId) + .where("seq", "=", identity.seq), + ); + if (!eventRow) { + return undefined; + } + const event = JSON.parse(eventRow.event_json) as { message?: unknown }; + return { + messageId: identity.eventId, + message: event.message, + }; +} + +function readTranscriptEventIdentity(event: unknown): + | { + eventId: string; + eventType: string | null; + parentId: string | null; + messageIdempotencyKey: string | null; + } + | undefined { + if (!event || typeof event !== "object" || Array.isArray(event)) { + return undefined; + } + const record = event as Record<string, unknown>; + const eventId = typeof record.id === "string" && record.id.trim() ? record.id.trim() : undefined; + if (!eventId) { + return undefined; + } + return { + eventId, + eventType: typeof record.type === "string" ? record.type : null, + parentId: typeof record.parentId === "string" ? record.parentId : null, + messageIdempotencyKey: readMessageIdempotencyKey(record.message), + }; +} + +function readMessageIdempotencyKey(message: unknown): string | null { + if (!message || typeof message !== "object" || Array.isArray(message)) { + return null; + } + const value = (message as { idempotencyKey?: unknown }).idempotencyKey; + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function readEventTimestamp(event: unknown): number | undefined { + if (!event || typeof event !== "object" || Array.isArray(event)) { + return undefined; + } + const value = (event as { timestamp?: unknown }).timestamp; + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value !== "string" || !value.trim()) { + return undefined; + } + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function redactTranscriptMessageForStorage<TMessage>( + message: TMessage, + options: Pick<TranscriptMessageAppendOptions<TMessage>, "config">, +): TMessage { + if (isTranscriptAgentMessage(message)) { + return redactTranscriptMessage(message, options.config) as TMessage; + } + return redactSecrets(message); +} + +function isTranscriptAgentMessage(value: unknown): value is AgentMessage { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + typeof (value as { role?: unknown }).role === "string" + ); +} + +function formatSqliteSessionMarkerForScope(scope: ResolvedTranscriptScope): string { + return formatSqliteSessionFileMarker({ + agentId: scope.agentId, + sessionId: scope.sessionId, + storePath: scope.path ?? resolveOpenClawAgentSqlitePath(toDatabaseOptions(scope)), + }); +} + +/** Finds the newest transcript record accepted by the matcher without parsing older rows. */ +export function findSqliteTranscriptEvent( + scope: SessionTranscriptReadScope, + match: (event: TranscriptEvent) => boolean, +): { event: TranscriptEvent } | undefined { + const resolved = resolveSqliteTranscriptReadScope(scope); + const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + return findSqliteTranscriptEventInDatabase(database, resolved.sessionId, match); +} + +function findSqliteTranscriptEventInDatabase( + database: OpenClawAgentDatabase, + sessionId: string, + match: (event: TranscriptEvent) => boolean, +): { event: TranscriptEvent } | undefined { + const db = getSessionKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db + .selectFrom("transcript_events") + .select(["event_json"]) + .where("session_id", "=", sessionId) + .orderBy("seq", "desc"), + ).rows; + for (const row of rows) { + try { + const event = JSON.parse(row.event_json) as TranscriptEvent; + if (match(event)) { + return { event }; + } + } catch { + // Malformed rows are skipped, matching transcript index tolerance. + } + } + return undefined; +} + +function readTranscriptEventMessage(event: TranscriptEvent): Record<string, unknown> | undefined { + if (!event || typeof event !== "object" || Array.isArray(event)) { + return undefined; + } + const message = (event as { message?: unknown }).message; + return message && typeof message === "object" && !Array.isArray(message) + ? (message as Record<string, unknown>) + : undefined; +} + +function readTranscriptEventId(event: TranscriptEvent): string | undefined { + if (!event || typeof event !== "object" || Array.isArray(event)) { + return undefined; + } + const id = (event as { id?: unknown }).id; + return typeof id === "string" && id.trim() ? id : undefined; +} + +/** Records inbound session metadata without refreshing activity timestamps. */ +export async function recordSqliteInboundSessionMeta(params: { + storePath: string; + sessionKey: string; + ctx: MsgContext; + groupResolution?: GroupKeyResolution | null; + createIfMissing?: boolean; +}): Promise<SessionEntry | null> { + const createIfMissing = params.createIfMissing ?? true; + return await patchSqliteSessionEntry( + { sessionKey: params.sessionKey, storePath: params.storePath }, + (_entry, context) => + deriveSessionMetaPatch({ + ctx: params.ctx, + sessionKey: params.sessionKey, + existing: context.existingEntry, + groupResolution: params.groupResolution, + }), + { + // Inbound metadata must not refresh activity timestamps; idle reset + // evaluation relies on updatedAt from actual session turns. + preserveActivity: true, + ...(createIfMissing ? { fallbackEntry: mergeSessionEntry(undefined, {}) } : {}), + }, + ); +} + +/** Updates last-route/delivery metadata without refreshing activity timestamps. */ +export async function updateSqliteSessionLastRoute(params: { + storePath: string; + sessionKey: string; + channel?: SessionEntry["lastChannel"]; + to?: string; + accountId?: string; + threadId?: string | number; + route?: SessionEntry["route"]; + deliveryContext?: DeliveryContext; + ctx?: MsgContext; + groupResolution?: GroupKeyResolution | null; + createIfMissing?: boolean; +}): Promise<SessionEntry | null> { + const createIfMissing = params.createIfMissing ?? true; + return await patchSqliteSessionEntry( + { sessionKey: params.sessionKey, storePath: params.storePath }, + (_entry, context) => + deriveLastRoutePatch({ + channel: params.channel, + to: params.to, + accountId: params.accountId, + threadId: params.threadId, + route: params.route, + deliveryContext: params.deliveryContext, + ctx: params.ctx, + groupResolution: params.groupResolution, + existing: context.existingEntry, + sessionKey: params.sessionKey, + }), + { + // Route updates must not refresh activity timestamps (#49515). + preserveActivity: true, + ...(createIfMissing ? { fallbackEntry: mergeSessionEntry(undefined, {}) } : {}), + }, + ); +} + +/** Writes the forked child's transcript rows (copied branch or header-only). */ +function writeSqliteForkedChildTranscriptInTransaction( + database: OpenClawAgentDatabase, + targetScope: ResolvedTranscriptScope, + params: { + parentSessionFile: string; + source: SqliteParentForkSourceTranscript; + }, +): void { + const { parentSessionFile, source } = params; + const shouldPersistBranch = source.preserveLeafControl || hasAssistantEntry(source.branchEntries); + if (shouldPersistBranch) { + writeSqliteParentForkTranscriptInTransaction(database, targetScope, { + parentSessionFile, + source, + }); + } else { + appendTranscriptEventInTransaction(database, targetScope, { + ...createSessionTranscriptHeader({ cwd: source.cwd, sessionId: targetScope.sessionId }), + parentSession: parentSessionFile, + }); + } +} diff --git a/src/config/sessions/session-accessor.test.ts b/src/config/sessions/session-accessor.test.ts index 87f1726d77d6..2ee99476ef26 100644 --- a/src/config/sessions/session-accessor.test.ts +++ b/src/config/sessions/session-accessor.test.ts @@ -2,22 +2,17 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { SessionManager } from "../../agents/sessions/session-manager.js"; import type { MsgContext } from "../../auto-reply/templating.js"; import { onSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; -import { createCanonicalFixtureSkill } from "../../skills/test-support/test-helpers.js"; -import type { OpenClawConfig } from "../types.openclaw.js"; +import { appendSqliteTrajectoryRuntimeEvents } from "../../trajectory/runtime-store.sqlite.js"; +import type { TrajectoryEvent } from "../../trajectory/types.js"; import { applySessionEntryReplacements, appendTranscriptMessage, - appendTranscriptEvent, applySessionEntryLifecycleMutation, - applySessionPatchProjection, - branchSessionFromCompactionCheckpoint, - canonicalizeSessionEntryAliases, - cleanupSessionLifecycleArtifacts, commitReplySessionInitialization, createSessionEntryWithTranscript, + deleteSessionEntryLifecycle, findTranscriptEvent, listSessionEntries, loadReplySessionInitializationSnapshot, @@ -26,36 +21,61 @@ import { markSessionAbortTarget, openSessionEntryReadView, patchSessionEntry, + patchSessionEntryTarget, persistSessionResetLifecycle, persistSessionTranscriptTurn, - purgeDeletedAgentSessionEntries, - publishTranscriptUpdate, readSessionUpdatedAt, recordInboundSessionMeta, replaceSessionEntry, - resolveSessionEntryCandidateTarget, resolveSessionEntryAccessTarget, - restoreSessionFromCompactionCheckpoint, + resolveSessionEntryCandidateTarget, resolveSessionTranscriptReadTarget, resolveSessionTranscriptRuntimeReadTarget, resolveSessionTranscriptRuntimeTarget, trimSessionTranscriptForManualCompact, - updateResolvedSessionEntry, updateSessionEntry, updateSessionLastRoute, upsertSessionEntry, } from "./session-accessor.js"; -import * as sessionStore from "./store.js"; -import { loadSessionStore, saveSessionStore, updateSessionStoreEntry } from "./store.js"; +import { + importSqliteSessionRows, + loadExactSqliteSessionEntry, + replaceSqliteSessionEntrySync, + replaceSqliteTranscriptEvents, +} from "./session-accessor.sqlite.js"; import { withOwnedSessionTranscriptWrites } from "./transcript-write-context.js"; import type { SessionEntry } from "./types.js"; -describe("session accessor file-backed seam", () => { +const cleanupArchivedSessionTranscriptsMock = vi.hoisted(() => vi.fn(async () => {})); + +vi.mock("../../gateway/session-archive.runtime.js", async (importOriginal) => { + const actual = await importOriginal<typeof import("../../gateway/session-archive.runtime.js")>(); + return { + ...actual, + cleanupArchivedSessionTranscripts: cleanupArchivedSessionTranscriptsMock, + }; +}); + +function createTestTrajectoryEvent(sessionId: string): TrajectoryEvent { + return { + traceSchema: "openclaw-trajectory", + schemaVersion: 1, + traceId: sessionId, + source: "runtime", + type: "test.concurrent-write", + ts: "2026-07-09T00:00:00.000Z", + seq: 1, + sessionId, + }; +} + +describe("session accessor seam", () => { let tempDir: string; let storePath: string; let transcriptPath: string; beforeEach(() => { + cleanupArchivedSessionTranscriptsMock.mockReset(); tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-session-accessor-")); storePath = path.join(tempDir, "sessions.json"); transcriptPath = path.join(tempDir, "session.jsonl"); @@ -103,25 +123,20 @@ describe("session accessor file-backed seam", () => { }); }); - it("loads parsed transcript events from explicit and store-derived targets", async () => { + it("loads parsed transcript events from store-derived SQLite targets", async () => { const header = { type: "session", id: "session-events", timestamp: 1 }; const message = { type: "message", id: "m1", message: { role: "assistant" } }; - fs.writeFileSync( - transcriptPath, - `${JSON.stringify(header)}\n${JSON.stringify(message)}\nnot-json\n`, - "utf-8", - ); + + // Transcript reads resolve to the SQLite transcript rows for the resolved + // agent-scoped session; there is no legacy custom sessionFile read path. await upsertSessionEntry( { sessionKey: "agent:main:main", storePath }, - { sessionId: "session-events", sessionFile: transcriptPath, updatedAt: 10 }, + { sessionId: "session-events", updatedAt: 10 }, + ); + await replaceSqliteTranscriptEvents( + { agentId: "main", sessionId: "session-events", sessionKey: "agent:main:main", storePath }, + [header, message], ); - - const explicit = await loadTranscriptEvents({ - sessionFile: transcriptPath, - sessionId: "session-events", - }); - expect(explicit).toEqual([header, message]); - const derived = await loadTranscriptEvents({ sessionId: "session-events", sessionKey: "agent:main:main", @@ -130,25 +145,29 @@ describe("session accessor file-backed seam", () => { expect(derived).toEqual([header, message]); const missing = await loadTranscriptEvents({ - sessionFile: path.join(tempDir, "missing.jsonl"), - sessionId: "session-events", + sessionId: "session-absent", + sessionKey: "agent:main:main", + storePath, }); expect(missing).toEqual([]); }); - it("finds the newest matching transcript event without loading the whole file", async () => { + it("finds the newest matching transcript event without loading the whole transcript", async () => { const header = { type: "session", id: "session-find", timestamp: 1 }; const older = { type: "message", id: "m1", message: { role: "assistant", tag: "old" } }; const newer = { type: "message", id: "m2", message: { role: "assistant", tag: "new" } }; - fs.writeFileSync( - transcriptPath, - `${JSON.stringify(header)}\n${JSON.stringify(older)}\n${JSON.stringify(newer)}\n`, - "utf-8", + await upsertSessionEntry( + { sessionKey: "agent:main:main", storePath }, + { sessionId: "session-find", updatedAt: 10 }, + ); + await replaceSqliteTranscriptEvents( + { agentId: "main", sessionId: "session-find", sessionKey: "agent:main:main", storePath }, + [header, older, newer], ); const seen: unknown[] = []; const found = await findTranscriptEvent( - { sessionFile: transcriptPath, sessionId: "session-find" }, + { sessionId: "session-find", sessionKey: "agent:main:main", storePath }, (event) => { seen.push(event); return (event as { type?: string }).type === "message"; @@ -158,8 +177,18 @@ describe("session accessor file-backed seam", () => { expect(found).toEqual({ event: newer }); expect(seen).toEqual([newer]); + await replaceSqliteTranscriptEvents( + { agentId: "main", sessionId: "session-falsy", sessionKey: "agent:main:falsy", storePath }, + [false], + ); + const falsy = await findTranscriptEvent( + { sessionId: "session-falsy", sessionKey: "agent:main:falsy", storePath }, + () => true, + ); + expect(falsy).toEqual({ event: false }); + const missing = await findTranscriptEvent( - { sessionFile: path.join(tempDir, "missing.jsonl"), sessionId: "session-find" }, + { sessionId: "session-absent", sessionKey: "agent:main:main", storePath }, () => true, ); expect(missing).toBeUndefined(); @@ -201,10 +230,9 @@ describe("session accessor file-backed seam", () => { expect(loadSessionEntry({ sessionKey: mixedKey, storePath })?.sessionId).toBe("mixed-session"); expect(loadSessionEntry({ sessionKey: lowerKey, storePath })?.sessionId).toBe("lower-session"); - expect(listSessionEntries({ storePath }).map((entry) => entry.sessionKey)).toEqual([ - mixedKey, - lowerKey, - ]); + expect( + listSessionEntries({ agentId: "voice", storePath }).map((entry) => entry.sessionKey), + ).toEqual([mixedKey, lowerKey]); }); it("records inbound session meta as a createIfMissing upsert returning a detached entry", async () => { @@ -296,60 +324,58 @@ describe("session accessor file-backed seam", () => { expect(loadSessionEntry({ sessionKey, storePath })).toBeUndefined(); }); - it("marks abort targets while canonicalizing legacy session keys", async () => { - fs.writeFileSync( - storePath, - JSON.stringify({ - "agent:main:telegram:group:-1001234567890:topic:99": { - sessionId: "canonical-session", - updatedAt: 10, - }, - "Agent:Main:Telegram:Group:-1001234567890:Topic:99": { - sessionId: "legacy-session", - updatedAt: 20, - }, - } satisfies Record<string, SessionEntry>), - "utf8", - ); - expect(loadSessionStore(storePath)).toHaveProperty( - "Agent:Main:Telegram:Group:-1001234567890:Topic:99", - ); - - const result = await markSessionAbortTarget({ - scope: { - sessionKey: "Agent:Main:Telegram:Group:-1001234567890:Topic:99", + it("patches the freshest target alias and rewrites it to the canonical key", async () => { + await replaceSessionEntry( + { + sessionKey: "agent:main:work", storePath, }, - now: () => 30, - resolveAbortCutoff: ({ sessionKey }) => { - expect(sessionKey).toBe("agent:main:telegram:group:-1001234567890:topic:99"); + { + sessionId: "canonical-session", + updatedAt: 10, + }, + ); + await replaceSessionEntry( + { + sessionKey: "agent:main:main", + storePath, + }, + { + sessionId: "legacy-session", + updatedAt: 20, + }, + ); + + const patched = await patchSessionEntryTarget( + { + storePath, + target: { + canonicalKey: "agent:main:work", + storeKeys: ["agent:main:work", "agent:main:main"], + }, + }, + (entry, context) => { + expect(entry.sessionId).toBe("legacy-session"); + expect(context.existingEntry?.sessionId).toBe("legacy-session"); return { - messageSid: "55", - timestamp: 1234567890000, + label: "patched", }; }, - }); + ); - expect(result).toMatchObject({ + expect(patched).toMatchObject({ + label: "patched", sessionId: "legacy-session", - sessionKey: "agent:main:telegram:group:-1001234567890:topic:99", - entry: { - abortedLastRun: true, - abortCutoffMessageSid: "55", - abortCutoffTimestamp: 1234567890000, - sessionId: "legacy-session", - updatedAt: 30, + }); + expect(listSessionEntries({ storePath })).toEqual([ + { + sessionKey: "agent:main:work", + entry: expect.objectContaining({ + label: "patched", + sessionId: "legacy-session", + }), }, - }); - expect(loadSessionStore(storePath)).toEqual({ - "agent:main:telegram:group:-1001234567890:topic:99": expect.objectContaining({ - abortedLastRun: true, - abortCutoffMessageSid: "55", - abortCutoffTimestamp: 1234567890000, - sessionId: "legacy-session", - updatedAt: 30, - }), - }); + ]); }); it("does not persist abort target changes when the entry is absent", async () => { @@ -365,103 +391,6 @@ describe("session accessor file-backed seam", () => { expect(fs.existsSync(storePath)).toBe(false); }); - it("canonicalizes alias rows and patches the canonical entry in one accessor write", async () => { - const now = Date.now(); - fs.writeFileSync(transcriptPath, '{"type":"session","id":"sess-fresh"}\n', "utf-8"); - fs.writeFileSync( - storePath, - JSON.stringify( - { - "agent:main:main": { - label: "canonical-stale", - sessionFile: transcriptPath, - sessionId: "sess-stale", - updatedAt: now, - }, - main: { - label: "legacy-fresh", - sessionFile: transcriptPath, - sessionId: "sess-fresh", - updatedAt: now + 1, - }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - ); - - const result = await canonicalizeSessionEntryAliases({ - storePath, - target: { - canonicalKey: "agent:main:main", - storeKeys: ["agent:main:main", "main"], - }, - update: (entry) => { - if (entry) { - entry.sessionId = "mutated-callback-copy"; - } - return { - lastChannel: "telegram", - updatedAt: now + 2, - }; - }, - }); - - expect(result).toEqual({ - canonicalKey: "agent:main:main", - entry: expect.objectContaining({ - label: "legacy-fresh", - lastChannel: "telegram", - sessionId: "sess-fresh", - updatedAt: now + 2, - }), - }); - const persisted = loadSessionStore(storePath, { skipCache: true }); - expect(persisted["agent:main:main"]).toMatchObject({ - label: "legacy-fresh", - lastChannel: "telegram", - sessionId: "sess-fresh", - updatedAt: now + 2, - }); - expect(persisted.main).toBeUndefined(); - }); - - it("resolves status-style ordered candidate keys without exposing the store", async () => { - fs.writeFileSync( - storePath, - JSON.stringify({ - "agent:main:current": { - label: "literal-current", - sessionId: "session-current", - updatedAt: 30, - }, - "agent:main:main": { - label: "main", - sessionId: "session-main", - updatedAt: 10, - }, - } satisfies Record<string, SessionEntry>), - "utf8", - ); - - const resolved = resolveSessionEntryCandidateTarget({ - agentId: "main", - candidateKeys: ["agent:main:main", "agent:main:current"], - cfg: { session: { store: storePath } }, - }); - - expect(resolved).toEqual({ - agentId: "main", - candidateKey: "agent:main:main", - entry: expect.objectContaining({ - label: "main", - sessionId: "session-main", - }), - persisted: true, - sessionKey: "agent:main:main", - }); - }); - it("returns an implicit candidate fallback without persisting it", () => { const resolved = resolveSessionEntryCandidateTarget({ agentId: "main", @@ -489,36 +418,62 @@ describe("session accessor file-backed seam", () => { expect(fs.existsSync(storePath)).toBe(false); }); - it("purges deleted-agent entries from the current locked store", async () => { - const cfg = { - session: { store: storePath }, - agents: { - list: [ - { id: "main", workspace: path.join(tempDir, "main") }, - { id: "ops", workspace: path.join(tempDir, "ops") }, - ], + it("resolves non-main candidate entries from custom agent store templates", async () => { + const storeTemplate = path.join(tempDir, "{agentId}.json"); + const supportStorePath = path.join(tempDir, "support.json"); + await upsertSessionEntry( + { + agentId: "support", + sessionKey: "agent:support:main", + storePath: supportStorePath, + }, + { + sessionId: "support-session", + updatedAt: 30, }, - } satisfies OpenClawConfig; - const now = Date.now(); - fs.writeFileSync( - storePath, - JSON.stringify({ - main: { sessionId: "main-legacy", updatedAt: now }, - "agent:ops:main": { sessionId: "ops-session", updatedAt: now }, - }), - "utf8", ); - const result = await purgeDeletedAgentSessionEntries({ - cfg, - agentId: "ops", - storeAgentId: "main", - storePath, + const resolved = resolveSessionEntryCandidateTarget({ + agentId: "support", + candidateKeys: ["agent:support:main"], + cfg: { session: { store: storeTemplate } }, }); - expect(result.removedSessionKeys).toEqual(["agent:ops:main"]); - expect(loadSessionStore(storePath)).toEqual({ - main: expect.objectContaining({ sessionId: "main-legacy" }), + expect(resolved).toMatchObject({ + agentId: "support", + candidateKey: "agent:support:main", + entry: { sessionId: "support-session" }, + persisted: true, + sessionKey: "agent:support:main", + }); + }); + + it("resolves non-main logical entries from custom agent store templates", async () => { + const storeTemplate = path.join(tempDir, "{agentId}.json"); + const supportStorePath = path.join(tempDir, "support.json"); + await upsertSessionEntry( + { + agentId: "support", + sessionKey: "agent:support:main", + storePath: supportStorePath, + }, + { + sessionId: "support-session", + updatedAt: 30, + }, + ); + + const resolved = resolveSessionEntryAccessTarget({ + cfg: { session: { store: storeTemplate } }, + sessionKey: "agent:support:main", + }); + + expect(resolved).toMatchObject({ + agentId: "support", + canonicalKey: "agent:support:main", + entry: { sessionId: "support-session" }, + requestedKey: "agent:support:main", + storeKey: "agent:support:main", }); }); @@ -537,7 +492,7 @@ describe("session accessor file-backed seam", () => { expect(loadSessionEntry(scope)?.sessionId).toBe(inserted?.sessionId); }); - it("creates entries with initialized transcripts and normalized sessionFile metadata", async () => { + it("creates entries with initialized SQLite transcripts and scoped session metadata", async () => { const scope = { agentId: "main", sessionKey: "agent:main:main", @@ -559,36 +514,202 @@ describe("session accessor file-backed seam", () => { if (!created.ok) { throw new Error("expected session creation to succeed"); } - expect(path.basename(created.sessionFile)).toBe("session-1.jsonl"); + expect(created.sessionFile).toContain("sqlite:main:session-1:"); expect(created.entry.sessionFile).toBe(created.sessionFile); + await expect( + loadTranscriptEvents({ + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:main", + storePath, + }), + ).resolves.toEqual([expect.objectContaining({ id: "session-1", type: "session" })]); }); - it("rolls back the entry when transcript initialization fails", async () => { + it("persists store-backed turns to SQLite when an old sessionFile path is present", async () => { + const legacyTranscript = path.join(tempDir, "legacy-topic.jsonl"); + const scope = { + agentId: "main", + sessionId: "legacy-topic-session", + sessionKey: "agent:main:telegram:group:1:topic:2", + storePath, + sessionFile: legacyTranscript, + }; + await upsertSessionEntry( + { sessionKey: scope.sessionKey, storePath }, + { + sessionId: scope.sessionId, + sessionFile: legacyTranscript, + updatedAt: 10, + }, + ); + + const result = await persistSessionTranscriptTurn(scope, { + messages: [{ message: { role: "user", content: "store-backed sqlite turn" } }], + touchSessionEntry: true, + updateMode: "none", + }); + + expect(result.sessionFile).toContain("sqlite:main:legacy-topic-session:"); + const entry = loadSessionEntry({ sessionKey: scope.sessionKey, storePath }); + expect(entry?.sessionFile).toBe(result.sessionFile); + await expect(loadTranscriptEvents(scope)).resolves.toContainEqual( + expect.objectContaining({ + type: "message", + message: expect.objectContaining({ + role: "user", + content: "store-backed sqlite turn", + }), + }), + ); + expect(fs.existsSync(legacyTranscript)).toBe(false); + }); + + it("resolves default-store SQLite transcript turn markers before appending", async () => { + const stateDir = path.join(tempDir, "state"); + const expectedStorePath = path.join(stateDir, "agents", "main", "sessions", "sessions.json"); + const scope = { + agentId: "main", + env: { + ...process.env, + OPENCLAW_STATE_DIR: stateDir, + }, + sessionId: "default-store-turn-session", + sessionKey: "agent:main:default-store-turn", + }; + await upsertSessionEntry( + { ...scope, storePath: expectedStorePath }, + { + sessionId: scope.sessionId, + updatedAt: 10, + }, + ); + + const result = await persistSessionTranscriptTurn(scope, { + messages: [{ message: { role: "user", content: "default store sqlite turn" } }], + touchSessionEntry: true, + updateMode: "none", + }); + + expect(result.sessionFile).toBe(`sqlite:main:${scope.sessionId}:${expectedStorePath}`); + const persistedScope = { ...scope, storePath: expectedStorePath }; + expect(loadSessionEntry(persistedScope)?.sessionFile).toBe(result.sessionFile); + await expect(loadTranscriptEvents(persistedScope)).resolves.toContainEqual( + expect.objectContaining({ + type: "message", + message: expect.objectContaining({ + role: "user", + content: "default store sqlite turn", + }), + }), + ); + }); + + it("guards store-backed turns in SQLite when an old sessionFile path is present", async () => { + const legacyTranscript = path.join(tempDir, "guarded-legacy-topic.jsonl"); + const scope = { + agentId: "main", + sessionId: "guarded-topic-session", + sessionKey: "agent:main:telegram:group:1:topic:3", + storePath, + sessionFile: legacyTranscript, + }; + await upsertSessionEntry( + { sessionKey: scope.sessionKey, storePath }, + { + sessionId: scope.sessionId, + sessionFile: legacyTranscript, + updatedAt: 10, + }, + ); + + const result = await persistSessionTranscriptTurn(scope, { + expectedSessionId: scope.sessionId, + messages: [{ message: { role: "assistant", content: "guarded sqlite turn" } }], + touchSessionEntry: true, + updateMode: "none", + }); + + expect(result.rejectedReason).toBeUndefined(); + expect(result.sessionFile).toContain("sqlite:main:guarded-topic-session:"); + const entry = loadSessionEntry({ sessionKey: scope.sessionKey, storePath }); + expect(entry?.sessionFile).toBe(result.sessionFile); + await expect(loadTranscriptEvents(scope)).resolves.toContainEqual( + expect.objectContaining({ + type: "message", + message: expect.objectContaining({ + role: "assistant", + content: "guarded sqlite turn", + }), + }), + ); + expect(fs.existsSync(legacyTranscript)).toBe(false); + }); + + it("appends SQLite turns to the active transcript leaf", async () => { + const scope = { + agentId: "main", + sessionId: "branched-topic-session", + sessionKey: "agent:main:telegram:group:1:topic:4", + storePath, + }; + await replaceSqliteTranscriptEvents(scope, [ + { type: "session", version: 3, id: scope.sessionId }, + { + type: "message", + id: "root", + parentId: null, + message: { role: "user", content: "root prompt" }, + }, + { + type: "message", + id: "abandoned", + parentId: "root", + message: { role: "assistant", content: "abandoned answer" }, + }, + { + type: "leaf", + id: "select-root", + parentId: "abandoned", + targetId: "root", + appendParentId: "root", + }, + ]); + + await persistSessionTranscriptTurn(scope, { + messages: [{ message: { role: "assistant", content: "active answer" } }], + updateMode: "none", + }); + + const appended = (await loadTranscriptEvents(scope)).at(-1); + expect(appended).toMatchObject({ + type: "message", + parentId: "root", + message: { role: "assistant", content: "active answer" }, + }); + }); + + it("does not persist the entry when creation validation fails", async () => { const scope = { agentId: "main", sessionKey: "agent:main:main", storePath, }; - fs.writeFileSync(path.join(tempDir, "blocked"), "not a directory", "utf8"); const created = await createSessionEntryWithTranscript(scope, () => ({ - ok: true, - entry: { - sessionFile: "blocked/session-1.jsonl", - sessionId: "session-1", - updatedAt: 10, - }, + error: "invalid patch", + ok: false, })); expect(created).toMatchObject({ ok: false, - phase: "transcript", + phase: "entry", }); expect(loadSessionEntry(scope)).toBeUndefined(); - expect(loadSessionStore(storePath, { skipCache: true })[scope.sessionKey]).toBeUndefined(); + expect(listSessionEntries({ storePath })).toEqual([]); }); - it("does not save the session store when entry preparation is rejected", async () => { + it("does not write the session database when entry preparation is rejected", async () => { const scope = { agentId: "main", sessionKey: "agent:main:main", @@ -599,8 +720,9 @@ describe("session accessor file-backed seam", () => { updatedAt: 10, initializationPending: true, }); + const databasePath = path.join(tempDir, "openclaw-agent.sqlite"); const fixedTime = new Date("2020-01-01T00:00:00.000Z"); - fs.utimesSync(storePath, fixedTime, fixedTime); + fs.utimesSync(databasePath, fixedTime, fixedTime); const rejected = await createSessionEntryWithTranscript(scope, () => ({ ok: false, @@ -612,7 +734,7 @@ describe("session accessor file-backed seam", () => { error: "still initializing", phase: "entry", }); - expect(fs.statSync(storePath).mtimeMs).toBe(fixedTime.getTime()); + expect(fs.statSync(databasePath).mtimeMs).toBe(fixedTime.getTime()); expect(loadSessionEntry({ ...scope, readConsistency: "latest" })).toMatchObject({ sessionId: "pending-session", initializationPending: true, @@ -659,7 +781,7 @@ describe("session accessor file-backed seam", () => { if (!committed.ok) { throw new Error("expected reply session initialization to commit"); } - expect(path.basename(committed.sessionEntry.sessionFile ?? "")).toBe("next-session.jsonl"); + expect(committed.sessionEntry.sessionFile).toBe(`sqlite:main:next-session:${storePath}`); expect(committed.sessionStoreView[sessionKey]).toMatchObject({ sessionId: "next-session", sessionFile: committed.sessionEntry.sessionFile, @@ -700,7 +822,7 @@ describe("session accessor file-backed seam", () => { if (!committed.ok) { throw new Error("expected reply session initialization to commit"); } - expect(path.basename(committed.sessionEntry.sessionFile ?? "")).toBe("next-rotation.jsonl"); + expect(committed.sessionEntry.sessionFile).toBe(`sqlite:main:next-rotation:${storePath}`); }); it("rejects stale reply session initialization snapshots without writing", async () => { @@ -752,18 +874,19 @@ describe("session accessor file-backed seam", () => { }, ); const snapshot = loadReplySessionInitializationSnapshot({ sessionKey, storePath }); - await sessionStore.updateSessionStore(storePath, (store) => { - const current = store[sessionKey]; - if (!current) { - throw new Error("expected existing session entry"); - } - store[sessionKey] = { + const current = loadSessionEntry({ sessionKey, storePath }); + if (!current) { + throw new Error("expected existing session entry"); + } + await replaceSessionEntry( + { sessionKey, storePath }, + { ...current, compactionCount: 1, totalTokensFresh: false, updatedAt: current.updatedAt + 1, - }; - }); + }, + ); const committed = await commitReplySessionInitialization({ activeSessionKey: sessionKey, @@ -813,17 +936,18 @@ describe("session accessor file-backed seam", () => { // Background activity (heartbeat runner, delivery retry, etc.) can touch // metadata fields without rotating the session. The initialization guard // should only care about session identity, so this must not conflict. - await sessionStore.updateSessionStore(storePath, (store) => { - const current = store[sessionKey]; - if (!current) { - throw new Error("expected existing session entry"); - } - store[sessionKey] = { + const current = loadSessionEntry({ sessionKey, storePath }); + if (!current) { + throw new Error("expected existing session entry"); + } + await replaceSessionEntry( + { sessionKey, storePath }, + { ...current, lastHeartbeatSentAt: 200, lastHeartbeatText: "heartbeat-2", - }; - }); + }, + ); const committed = await commitReplySessionInitialization({ activeSessionKey: sessionKey, @@ -871,17 +995,18 @@ describe("session accessor file-backed seam", () => { const snapshot = loadReplySessionInitializationSnapshot({ sessionKey, storePath }); - await sessionStore.updateSessionStore(storePath, (store) => { - const current = store[sessionKey]; - if (!current) { - throw new Error("expected existing session entry"); - } - store[sessionKey] = { + const current = loadSessionEntry({ sessionKey, storePath }); + if (!current) { + throw new Error("expected existing session entry"); + } + await replaceSessionEntry( + { sessionKey, storePath }, + { ...current, modelOverride: "channel-model", modelOverrideSource: "user", - }; - }); + }, + ); const committed = await commitReplySessionInitialization({ activeSessionKey: sessionKey, @@ -939,23 +1064,20 @@ describe("session accessor file-backed seam", () => { throw new Error("expected reply session initialization snapshot"); } - await sessionStore.updateSessionStore(storePath, (store) => { - const current = store[sessionKey]; - if (!current) { - throw new Error("expected existing session entry"); - } - store[sessionKey] = { - ...current, - pendingFinalDelivery: undefined, - pendingFinalDeliveryText: undefined, - pendingFinalDeliveryCreatedAt: undefined, - pendingFinalDeliveryLastAttemptAt: undefined, - pendingFinalDeliveryAttemptCount: undefined, - pendingFinalDeliveryLastError: undefined, - pendingFinalDeliveryContext: undefined, - pendingFinalDeliveryIntentId: undefined, - }; - }); + const current = loadSessionEntry({ sessionKey, storePath }); + if (!current) { + throw new Error("expected existing session entry"); + } + const currentWithoutPendingDelivery = { ...current }; + delete currentWithoutPendingDelivery.pendingFinalDelivery; + delete currentWithoutPendingDelivery.pendingFinalDeliveryAttemptCount; + delete currentWithoutPendingDelivery.pendingFinalDeliveryContext; + delete currentWithoutPendingDelivery.pendingFinalDeliveryCreatedAt; + delete currentWithoutPendingDelivery.pendingFinalDeliveryIntentId; + delete currentWithoutPendingDelivery.pendingFinalDeliveryLastAttemptAt; + delete currentWithoutPendingDelivery.pendingFinalDeliveryLastError; + delete currentWithoutPendingDelivery.pendingFinalDeliveryText; + await replaceSessionEntry({ sessionKey, storePath }, currentWithoutPendingDelivery); const committed = await commitReplySessionInitialization({ activeSessionKey: sessionKey, @@ -1006,20 +1128,21 @@ describe("session accessor file-backed seam", () => { const snapshot = loadReplySessionInitializationSnapshot({ sessionKey, storePath }); - await sessionStore.updateSessionStore(storePath, (store) => { - const current = store[sessionKey]; - if (!current) { - throw new Error("expected existing session entry"); - } - store[sessionKey] = { + const current = loadSessionEntry({ sessionKey, storePath }); + if (!current) { + throw new Error("expected existing session entry"); + } + await replaceSessionEntry( + { sessionKey, storePath }, + { ...current, pendingFinalDelivery: true, pendingFinalDeliveryText: "old reply", pendingFinalDeliveryCreatedAt: 21, pendingFinalDeliveryContext: { channel: "discord", to: "channel-1" }, pendingFinalDeliveryIntentId: "intent-old", - }; - }); + }, + ); const committed = await commitReplySessionInitialization({ activeSessionKey: sessionKey, @@ -1054,40 +1177,22 @@ describe("session accessor file-backed seam", () => { expect(persisted?.pendingFinalDeliveryIntentId).toBeUndefined(); }); - it("commits reply session initialization despite runtime-only skill snapshot cache", async () => { + it("commits reply session initialization from a guarded legacy alias snapshot", async () => { const sessionKey = "agent:main:main"; - await upsertSessionEntry( - { sessionKey, storePath }, - { - sessionId: "first-session", - skillsSnapshot: { - prompt: `<available_skills>${"x".repeat(600)}</available_skills>`, - skills: [{ name: "skill-0" }], - version: 1, + await applySessionEntryLifecycleMutation({ + storePath, + upserts: [ + { + sessionKey: "Agent:Main:Main", + entry: { + sessionId: "legacy-alias-session", + updatedAt: 10, + }, }, - updatedAt: 10, - }, - ); + ], + }); const snapshot = loadReplySessionInitializationSnapshot({ sessionKey, storePath }); - const cachedStore = loadSessionStore(storePath, { clone: false }); - const cachedEntry = cachedStore[sessionKey]; - if (!cachedEntry?.skillsSnapshot) { - throw new Error("expected cached skills snapshot"); - } - cachedEntry.skillsSnapshot = { - ...cachedEntry.skillsSnapshot, - resolvedSkills: [ - createCanonicalFixtureSkill({ - baseDir: "/skills/skill-0", - description: "skill-0 description", - filePath: "/skills/skill-0/SKILL.md", - name: "skill-0", - source: `# skill-0\n\n${"x".repeat(3000)}`, - }), - ], - }; - const committed = await commitReplySessionInitialization({ activeSessionKey: sessionKey, agentId: "main", @@ -1106,92 +1211,44 @@ describe("session accessor file-backed seam", () => { throw new Error("expected reply session initialization to commit"); } expect(committed.sessionEntry.sessionId).toBe("next-session"); + expect(loadSessionEntry({ sessionKey, storePath })?.sessionId).toBe("next-session"); }); - it("can borrow cached entry objects for read-only hot paths", async () => { - const scope = { - clone: false, - sessionKey: "agent:main:main", - storePath, - }; - - await upsertSessionEntry(scope, { - sessionId: "session-1", - updatedAt: 10, - }); - const cachedStore = loadSessionStore(storePath, { clone: false }); - - expect(loadSessionEntry(scope)).toBe(cachedStore["agent:main:main"]); - expect(listSessionEntries({ clone: false, storePath })[0]?.entry).toBe( - cachedStore["agent:main:main"], + it("rejects reply session initialization when the entry is deleted during prepare", async () => { + const sessionKey = "agent:main:main"; + await upsertSessionEntry( + { sessionKey, storePath }, + { + sessionId: "first-session", + updatedAt: 10, + }, ); - }); + const snapshot = loadReplySessionInitializationSnapshot({ sessionKey, storePath }); - it("maps latest entry reads to the file backend cache bypass", () => { - fs.writeFileSync( - storePath, - JSON.stringify({ - "agent:main:main": { - sessionId: "session-1", - model: "gpt-5.4", - }, - }), - "utf8", - ); - const loadSessionStoreSpy = vi.spyOn(sessionStore, "loadSessionStore"); - - try { - expect( - loadSessionEntry({ - readConsistency: "latest", - sessionKey: "agent:main:main", + const committed = await commitReplySessionInitialization({ + activeSessionKey: sessionKey, + agentId: "main", + expectedRevision: snapshot.revision, + prepareSessionEntry: async ({ sessionEntry }) => { + await applySessionEntryLifecycleMutation({ + removals: [{ sessionKey }], storePath, - })?.model, - ).toBe("gpt-5.4"); - expect(loadSessionStoreSpy).toHaveBeenLastCalledWith( - storePath, - expect.objectContaining({ skipCache: true }), - ); - - loadSessionEntry({ - clone: false, - readConsistency: "latest", - sessionKey: "agent:main:main", - storePath, - }); - expect(loadSessionStoreSpy).toHaveBeenLastCalledWith( - storePath, - expect.objectContaining({ clone: false, skipCache: true }), - ); - } finally { - loadSessionStoreSpy.mockRestore(); - } - }); - - it("resolves canonical entry reads without requiring exact key casing", async () => { - fs.writeFileSync( + }); + return sessionEntry; + }, + sessionEntry: { + sessionId: "stale-session", + updatedAt: 30, + }, + sessionKey, storePath, - JSON.stringify({ - "agent:main:main": { - sessionId: "session-1", - updatedAt: 10, - model: "gpt-5.5", - }, - }), - "utf8", - ); + }); - const mixedCaseScope = { - sessionKey: "AGENT:MAIN:MAIN", - storePath, - }; - - expect(loadSessionEntry(mixedCaseScope)).toEqual( - expect.objectContaining({ - sessionId: "session-1", - model: "gpt-5.5", - }), - ); + expect(committed).toMatchObject({ + ok: false, + reason: "stale-snapshot", + }); + expect(loadSessionEntry({ sessionKey, storePath })).toBeUndefined(); }); it("updates existing entries without creating missing sessions", async () => { @@ -1324,196 +1381,29 @@ describe("session accessor file-backed seam", () => { }); }); - it("applies projected session patches after migrating legacy candidate keys", async () => { - fs.writeFileSync( - storePath, - JSON.stringify({ - "agent:main:main": { - sessionId: "canonical-session", - updatedAt: 10, - }, - main: { - sessionId: "legacy-session", - updatedAt: 20, - }, - }), - "utf8", - ); - - const projected = await applySessionPatchProjection({ - storePath, - resolveTarget: () => ({ - primaryKey: "agent:main:main", - candidateKeys: ["agent:main:main", "main"], - }), - project: ({ entries, existingEntry, primaryKey }) => { - expect(primaryKey).toBe("agent:main:main"); - expect(existingEntry?.sessionId).toBe("legacy-session"); - expect(entries.map((entry) => entry.sessionKey)).toEqual(["agent:main:main"]); - return { - ok: true as const, - entry: { - ...existingEntry, - label: "Projected", - } as SessionEntry, - }; - }, - }); - - expect(projected).toMatchObject({ - ok: true, - entry: { - label: "Projected", - sessionId: "legacy-session", - }, - }); - expect(loadSessionStore(storePath)).toEqual({ - "agent:main:main": expect.objectContaining({ - label: "Projected", - sessionId: "legacy-session", - }), - }); - }); - - it("persists legacy key pruning when projected session patches fail validation", async () => { - fs.writeFileSync( - storePath, - JSON.stringify({ - "agent:main:main": { - sessionId: "canonical-session", - updatedAt: 10, - }, - main: { - sessionId: "legacy-session", - updatedAt: 20, - }, - }), - "utf8", - ); - - const projected = await applySessionPatchProjection({ - storePath, - resolveTarget: () => ({ - primaryKey: "agent:main:main", - candidateKeys: ["agent:main:main", "main"], - }), - project: () => ({ - ok: false as const, - error: "invalid patch", - }), - }); - - expect(projected).toEqual({ ok: false, error: "invalid patch" }); - expect(loadSessionStore(storePath)).toEqual({ - "agent:main:main": expect.objectContaining({ - sessionId: "legacy-session", - }), - }); - }); - - it("updates the freshest matching session entry across discovered agent stores", async () => { - const stateDir = path.join(tempDir, "state"); - const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; - const cfg = { - session: { - mainKey: "main", - store: path.join(stateDir, "agents", "{agentId}", "sessions", "sessions.json"), - }, - agents: { list: [{ id: "retired-agent", default: true }] }, - } satisfies OpenClawConfig; - const configuredStorePath = path.join( - stateDir, - "agents", - "retired-agent", - "sessions", - "sessions.json", - ); - const discoveredStorePath = path.join( - stateDir, - "agents", - "Retired Agent", - "sessions", - "sessions.json", - ); - fs.mkdirSync(path.dirname(configuredStorePath), { recursive: true }); - fs.mkdirSync(path.dirname(discoveredStorePath), { recursive: true }); - fs.writeFileSync( - configuredStorePath, - JSON.stringify({ - "agent:retired-agent:main": { sessionId: "configured", updatedAt: 10 }, - }), - "utf8", - ); - fs.writeFileSync( - discoveredStorePath, - JSON.stringify({ - "agent:retired-agent:main": { sessionId: "discovered", updatedAt: 20 }, - }), - "utf8", - ); - - const resolved = resolveSessionEntryAccessTarget({ - cfg, - env, - sessionKey: "agent:retired-agent:main", - }); - - expect(resolved.entry?.sessionId).toBe("discovered"); - - const updated = await updateResolvedSessionEntry( - { - cfg, - env, - sessionKey: "agent:retired-agent:main", - }, - (entry, context) => { - expect(context.canonicalKey).toBe("agent:retired-agent:main"); - expect(context.storeKey).toBe("agent:retired-agent:main"); - entry.model = "gpt-5.5"; - entry.updatedAt = Date.now(); - return entry.sessionId; - }, - ); - - expect(updated).toMatchObject({ - found: true, - canonicalKey: "agent:retired-agent:main", - result: "discovered", - }); - expect(loadSessionStore(configuredStorePath)["agent:retired-agent:main"]).toMatchObject({ - sessionId: "configured", - updatedAt: 10, - }); - expect(Object.values(loadSessionStore(discoveredStorePath))).toContainEqual( - expect.objectContaining({ - model: "gpt-5.5", - sessionId: "discovered", - updatedAt: expect.any(Number), - }), - ); - }); - it("applies explicit replacements without exposing mutable store rows", async () => { - fs.writeFileSync( + await applySessionEntryLifecycleMutation({ storePath, - JSON.stringify( + upserts: [ { - "agent:main:main": { + sessionKey: "agent:main:main", + entry: { sessionId: "session-1", status: "running", updatedAt: 10, }, - "agent:main:other": { + }, + { + sessionKey: "agent:main:other", + entry: { sessionId: "session-2", status: "running", updatedAt: 20, }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - "utf8", - ); + }, + ], + skipMaintenance: true, + }); const result = await applySessionEntryReplacements({ storePath, @@ -1536,13 +1426,12 @@ describe("session accessor file-backed seam", () => { }); expect(result).toEqual({ replaced: true }); - const store = loadSessionStore(storePath); - expect(store["agent:main:main"]).toMatchObject({ + expect(loadSessionEntry({ sessionKey: "agent:main:main", storePath })).toMatchObject({ abortedLastRun: true, sessionId: "session-1", updatedAt: 30, }); - expect(store["agent:main:other"]).toMatchObject({ + expect(loadSessionEntry({ sessionKey: "agent:main:other", storePath })).toMatchObject({ sessionId: "session-2", status: "running", updatedAt: 20, @@ -1554,401 +1443,308 @@ describe("session accessor file-backed seam", () => { update: (entries) => ({ result: entries.map((entry) => entry.sessionKey) }), }); expect(selectedKeys).toEqual(["agent:main:main"]); + const other = loadSessionEntry({ sessionKey: "agent:main:other", storePath }); + expect(other).toBeDefined(); await expect( applySessionEntryReplacements({ sessionKeys: ["agent:main:main"], storePath, update: () => ({ - replacements: [{ sessionKey: "agent:main:other", entry: store["agent:main:other"] }], + replacements: [{ sessionKey: "agent:main:other", entry: other! }], result: undefined, }), }), ).rejects.toThrow("outside the selected key set"); }); - it("branches checkpoint sessions without exposing mutable store rows", async () => { - const sourceSessionId = "11111111-1111-4111-8111-111111111111"; - const branchSessionId = "22222222-2222-4222-8222-222222222222"; - const branchPath = path.join(tempDir, "branch.jsonl"); - const now = Date.now(); - fs.writeFileSync(transcriptPath, `{"type":"session","id":"${sourceSessionId}"}\n`, "utf8"); - fs.writeFileSync(branchPath, `{"type":"session","id":"${branchSessionId}"}\n`, "utf8"); - const checkpoint = { - checkpointId: "checkpoint-1", - sessionKey: "agent:main:main", - sessionId: sourceSessionId, - createdAt: now, - reason: "manual", - preCompaction: { - sessionId: sourceSessionId, - sessionFile: transcriptPath, - leafId: "leaf-1", - }, - postCompaction: { sessionId: "33333333-3333-4333-8333-333333333333" }, - } satisfies NonNullable<SessionEntry["compactionCheckpoints"]>[number]; - fs.writeFileSync( + it("prepares entry replacements without holding a write transaction", async () => { + const scope = { + sessionKey: "agent:main:replacement-prepare", storePath, - JSON.stringify( - { - main: { - label: "Main", - sessionFile: transcriptPath, - sessionId: sourceSessionId, - updatedAt: now, - compactionCheckpoints: [checkpoint], - }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - "utf8", - ); - - const result = await branchSessionFromCompactionCheckpoint({ + }; + await upsertSessionEntry(scope, { + model: "base", + sessionId: "replacement-prepare", + updatedAt: 10, + }); + let releasePlanner!: () => void; + let markPlannerStarted!: () => void; + const plannerStarted = new Promise<void>((resolve) => { + markPlannerStarted = resolve; + }); + const plannerGate = new Promise<void>((resolve) => { + releasePlanner = resolve; + }); + const pendingReplacement = applySessionEntryReplacements({ + sessionKeys: [scope.sessionKey], storePath, - sourceKey: "agent:main:main", - sourceStoreKey: "main", - nextKey: "agent:main:branch", - checkpointId: "checkpoint-1", - forkTranscriptFromCheckpoint: async (selectedCheckpoint) => { - expect(selectedCheckpoint).toEqual(checkpoint); + update: async (entries) => { + markPlannerStarted(); + await plannerGate; return { - status: "created", - transcript: { - sessionFile: branchPath, - sessionId: branchSessionId, - totalTokens: 42, - }, + replacements: entries.map(({ entry, sessionKey }) => ({ + entry: { ...entry, model: "planned" }, + sessionKey, + })), + result: undefined, }; }, - buildEntry: ({ currentEntry, forkedTranscript }) => ({ - ...currentEntry, - sessionFile: forkedTranscript.sessionFile, - sessionId: forkedTranscript.sessionId, - totalTokens: forkedTranscript.totalTokens, - updatedAt: now + 1, - }), }); - expect(result).toMatchObject({ - status: "created", - key: "agent:main:branch", - entry: { - sessionFile: branchPath, - sessionId: branchSessionId, - totalTokens: 42, - }, - }); - expect(loadSessionStore(storePath)).toEqual({ - main: expect.objectContaining({ sessionId: sourceSessionId }), - "agent:main:branch": expect.objectContaining({ - sessionFile: branchPath, - sessionId: branchSessionId, - totalTokens: 42, - }), + await plannerStarted; + let replacementError: unknown; + try { + replaceSqliteSessionEntrySync(scope, { + model: "newer", + sessionId: "replacement-prepare", + updatedAt: 20, + }); + } catch (error) { + replacementError = error; + } finally { + releasePlanner(); + } + const planningError = await pendingReplacement.then( + () => undefined, + (error: unknown) => error, + ); + + expect(replacementError).toBeUndefined(); + expect(planningError).toMatchObject({ + message: expect.stringContaining("changed before replacement"), }); + expect(loadSessionEntry(scope)).toMatchObject({ model: "newer", updatedAt: 20 }); }); - it("branches from the newest matching compaction checkpoint without sorting all checkpoints", async () => { - const sourceSessionId = "11111111-1111-4111-8111-111111111111"; - const branchSessionId = "22222222-2222-4222-8222-222222222222"; - const branchPath = path.join(tempDir, "branch-newest.jsonl"); - fs.writeFileSync(branchPath, `{"type":"session","id":"${branchSessionId}"}\n`, "utf8"); - const oldMatchingCheckpoint = { - checkpointId: "checkpoint-1", - sessionKey: "agent:main:main", - sessionId: sourceSessionId, - createdAt: 10, - reason: "manual", - preCompaction: { - sessionId: sourceSessionId, - leafId: "old-leaf", - }, - postCompaction: { sessionId: "33333333-3333-4333-8333-333333333333" }, - } satisfies NonNullable<SessionEntry["compactionCheckpoints"]>[number]; - const newestMatchingCheckpoint = { - ...oldMatchingCheckpoint, - createdAt: 20, - preCompaction: { - sessionId: sourceSessionId, - leafId: "new-leaf", - }, - postCompaction: { sessionId: "44444444-4444-4444-8444-444444444444" }, - } satisfies NonNullable<SessionEntry["compactionCheckpoints"]>[number]; - const newestDifferentCheckpoint = { - ...oldMatchingCheckpoint, - checkpointId: "checkpoint-2", - createdAt: 30, - preCompaction: { - sessionId: sourceSessionId, - leafId: "different-leaf", - }, - postCompaction: { sessionId: "55555555-5555-4555-8555-555555555555" }, - } satisfies NonNullable<SessionEntry["compactionCheckpoints"]>[number]; - fs.writeFileSync( + it("does not hold a write transaction while awaiting a lifecycle entry builder", async () => { + const sessionKey = "agent:main:lifecycle-prepare"; + await upsertSessionEntry( + { sessionKey, storePath }, + { model: "base", sessionId: "lifecycle-prepare", updatedAt: 10 }, + ); + let releaseBuilder!: () => void; + let markBuilderStarted!: () => void; + const builderStarted = new Promise<void>((resolve) => { + markBuilderStarted = resolve; + }); + const builderGate = new Promise<void>((resolve) => { + releaseBuilder = resolve; + }); + const pendingMutation = applySessionEntryLifecycleMutation({ storePath, - JSON.stringify( + upserts: [ { - main: { - sessionId: sourceSessionId, - updatedAt: 30, - compactionCheckpoints: [ - oldMatchingCheckpoint, - newestDifferentCheckpoint, - newestMatchingCheckpoint, - ], + sessionKey, + buildEntry: async ({ currentEntry }) => { + markBuilderStarted(); + await builderGate; + return { ...currentEntry, model: "projected" } as SessionEntry; }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - "utf8", - ); - - const result = await branchSessionFromCompactionCheckpoint({ - storePath, - sourceKey: "agent:main:main", - sourceStoreKey: "main", - nextKey: "agent:main:branch-newest", - checkpointId: "checkpoint-1", - forkTranscriptFromCheckpoint: async (selectedCheckpoint) => { - expect(selectedCheckpoint).toEqual(newestMatchingCheckpoint); - return { - status: "created", - transcript: { - sessionFile: branchPath, - sessionId: branchSessionId, - }, - }; - }, - buildEntry: ({ currentEntry, forkedTranscript }) => ({ - ...currentEntry, - sessionFile: forkedTranscript.sessionFile, - sessionId: forkedTranscript.sessionId, - }), + }, + ], + skipMaintenance: true, }); - expect(result).toMatchObject({ - status: "created", - key: "agent:main:branch-newest", - checkpoint: newestMatchingCheckpoint, - }); + await builderStarted; + let unrelatedWriteError: unknown; + try { + appendSqliteTrajectoryRuntimeEvents({ sessionId: "lifecycle-prepare", storePath }, [ + createTestTrajectoryEvent("lifecycle-prepare"), + ]); + } catch (error) { + unrelatedWriteError = error; + } finally { + releaseBuilder(); + } + + await expect(pendingMutation).resolves.toMatchObject({ afterCount: 1 }); + expect(unrelatedWriteError).toBeUndefined(); + expect(loadSessionEntry({ sessionKey, storePath })).toMatchObject({ model: "projected" }); }); - it("does not persist checkpoint restores when the transcript boundary is missing", async () => { - fs.writeFileSync( + it("rejects a lifecycle projection when its source row changes", async () => { + const scope = { sessionKey: "agent:main:lifecycle-stale", storePath }; + await upsertSessionEntry(scope, { + model: "base", + sessionId: "lifecycle-stale", + updatedAt: 10, + }); + let releaseBuilder!: () => void; + let markBuilderStarted!: () => void; + const builderStarted = new Promise<void>((resolve) => { + markBuilderStarted = resolve; + }); + const builderGate = new Promise<void>((resolve) => { + releaseBuilder = resolve; + }); + const pendingMutation = applySessionEntryLifecycleMutation({ storePath, - JSON.stringify( + upserts: [ { - "agent:main:main": { - sessionId: "session-1", - updatedAt: 10, - compactionCheckpoints: [ - { - checkpointId: "checkpoint-1", - sessionKey: "agent:main:main", - sessionId: "session-1", - createdAt: 20, - reason: "manual", - preCompaction: { - sessionId: "session-1", - leafId: "leaf-1", - }, - postCompaction: { sessionId: "session-2" }, - }, - ], + sessionKey: scope.sessionKey, + buildEntry: async ({ currentEntry }) => { + markBuilderStarted(); + await builderGate; + return { ...currentEntry, model: "stale-projection" } as SessionEntry; }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - "utf8", + }, + ], + skipMaintenance: true, + }); + + await builderStarted; + let replacementError: unknown; + try { + replaceSqliteSessionEntrySync(scope, { + model: "newer", + sessionId: "lifecycle-stale", + updatedAt: 20, + }); + } catch (error) { + replacementError = error; + } finally { + releaseBuilder(); + } + const mutationError = await pendingMutation.then( + () => undefined, + (error: unknown) => error, ); - const before = fs.readFileSync(storePath, "utf8"); - const result = await restoreSessionFromCompactionCheckpoint({ + expect(replacementError).toBeUndefined(); + expect(mutationError).toMatchObject({ + message: expect.stringContaining("changed before lifecycle upsert"), + }); + expect(loadSessionEntry(scope)).toMatchObject({ model: "newer", updatedAt: 20 }); + }); + + it("reclaims SQLite transcript rows for lifecycle removals without archive intent", async () => { + const scope = { + sessionId: "session-1", + sessionKey: "agent:main:preserve", storePath, - sessionKey: "agent:main:main", - checkpointId: "checkpoint-1", - forkTranscriptFromCheckpoint: async () => ({ status: "missing-boundary" }), - buildEntry: () => { - throw new Error("missing boundary should skip entry replacement"); + }; + await upsertSessionEntry(scope, { + sessionId: scope.sessionId, + updatedAt: 10, + }); + await replaceSqliteTranscriptEvents(scope, [ + { + id: "event-1", + message: { role: "user", content: "keep me" }, + type: "message", }, + ]); + + const result = await applySessionEntryLifecycleMutation({ + storePath, + removals: [{ expectedSessionId: scope.sessionId, sessionKey: scope.sessionKey }], }); - expect(result).toEqual({ status: "missing-boundary" }); - expect(fs.readFileSync(storePath, "utf8")).toBe(before); + expect(result.removedEntries).toBe(1); + expect(result.archivedTranscriptDirectories).toEqual([]); + expect(loadSessionEntry(scope)).toBeUndefined(); + await expect(loadTranscriptEvents(scope)).resolves.toEqual([]); }); - it("cleans scoped lifecycle entries and unreferenced transcript artifacts", async () => { - const nowMs = Date.now(); - const oldDate = new Date(nowMs - 600_000); - const lifecycleSessionsDir = path.join(tempDir, "state", "agents", "main", "sessions"); - const lifecycleStorePath = path.join(lifecycleSessionsDir, "sessions.json"); - const removedTranscriptPath = path.join(lifecycleSessionsDir, "removed-lifecycle.jsonl"); - const customTranscriptPath = path.join(lifecycleSessionsDir, "custom-lifecycle-old.jsonl"); - const freshDefaultTranscriptPath = path.join(lifecycleSessionsDir, "custom-lifecycle.jsonl"); - const freshTranscriptPath = path.join(lifecycleSessionsDir, "fresh-lifecycle.jsonl"); - const referencedTranscriptPath = path.join(lifecycleSessionsDir, "referenced.jsonl"); - const orphanTranscriptPath = path.join(lifecycleSessionsDir, "orphan-lifecycle.jsonl"); - const siblingDir = path.join(tempDir, "state", "agents", "sibling", "sessions"); - const siblingTranscriptPath = path.join(siblingDir, "sibling-lifecycle.jsonl"); - fs.mkdirSync(lifecycleSessionsDir, { recursive: true }); - fs.mkdirSync(siblingDir, { recursive: true }); + it("captures SQLite archived transcript cleanup failures when requested", async () => { + const cleanupError = new Error("cleanup failed"); + cleanupArchivedSessionTranscriptsMock.mockRejectedValueOnce(cleanupError); + const scope = { + sessionId: "session-1", + sessionKey: "agent:main:cleanup", + storePath, + }; + await upsertSessionEntry(scope, { + sessionId: scope.sessionId, + updatedAt: 10, + }); + await appendTranscriptMessage(scope, { + cwd: tempDir, + message: { role: "user", content: "cleanup me" }, + }); - fs.writeFileSync( - lifecycleStorePath, - JSON.stringify({ - "agent:main:lifecycle-cleanup-removed": { - sessionId: "removed-lifecycle", - }, - "agent:main:lifecycle-cleanup-fresh": { - sessionId: "fresh-lifecycle", - }, - "agent:main:lifecycle-cleanup-custom": { - sessionFile: "custom-lifecycle-old.jsonl", - sessionId: "custom-lifecycle", - }, - "agent:main:lifecycle-cleanup-sibling": { - sessionFile: siblingTranscriptPath, - sessionId: "sibling-lifecycle", - }, - "agent:main:telegram:group:lifecycle-cleanup-room": { - sessionId: "kept-by-segment", - }, - "agent:main:regular": { - sessionId: "referenced", + const result = await applySessionEntryLifecycleMutation({ + storePath, + removals: [ + { + archiveRemovedTranscript: true, + expectedSessionId: scope.sessionId, + sessionKey: scope.sessionKey, }, + ], + cleanupArchivedTranscripts: { + rules: [{ reason: "deleted", olderThanMs: 0 }], + nowMs: Date.now(), + }, + captureArtifactCleanupError: true, + skipMaintenance: true, + }); + + expect(result.removedEntries).toBe(1); + expect(result.archivedTranscriptDirectories).toHaveLength(1); + expect(result.artifactCleanupError).toBe(cleanupError); + expect(cleanupArchivedSessionTranscriptsMock).toHaveBeenCalledWith( + expect.objectContaining({ + directories: result.archivedTranscriptDirectories, }), - "utf-8", ); - fs.writeFileSync(removedTranscriptPath, '{"runId":"lifecycle-marker-removed"}\n', "utf-8"); - fs.writeFileSync(customTranscriptPath, '{"runId":"lifecycle-marker-custom"}\n', "utf-8"); - fs.writeFileSync(freshDefaultTranscriptPath, '{"runId":"lifecycle-marker-default"}\n', "utf-8"); - fs.writeFileSync(freshTranscriptPath, '{"runId":"lifecycle-marker-fresh"}\n', "utf-8"); - fs.writeFileSync(siblingTranscriptPath, '{"runId":"lifecycle-marker-sibling"}\n', "utf-8"); - fs.writeFileSync( - referencedTranscriptPath, - '{"runId":"lifecycle-marker-referenced"}\n', - "utf-8", - ); - fs.writeFileSync(orphanTranscriptPath, '{"runId":"lifecycle-marker-orphan"}\n', "utf-8"); - fs.utimesSync(removedTranscriptPath, oldDate, oldDate); - fs.utimesSync(customTranscriptPath, oldDate, oldDate); - fs.utimesSync(siblingTranscriptPath, oldDate, oldDate); - fs.utimesSync(referencedTranscriptPath, oldDate, oldDate); - fs.utimesSync(orphanTranscriptPath, oldDate, oldDate); - - const result = await cleanupSessionLifecycleArtifacts({ - storePath: lifecycleStorePath, - sessionKeySegmentPrefix: "lifecycle-cleanup-", - transcriptContentMarker: "lifecycle-marker-", - orphanTranscriptMinAgeMs: 300_000, - nowMs, - }); - - expect(result).toEqual({ removedEntries: 3, archivedTranscriptArtifacts: 3 }); - const loaded = loadSessionStore(lifecycleStorePath, { skipCache: true }); - expect(loaded).not.toHaveProperty("agent:main:lifecycle-cleanup-removed"); - expect(loaded).not.toHaveProperty("agent:main:lifecycle-cleanup-custom"); - expect(loaded).not.toHaveProperty("agent:main:lifecycle-cleanup-sibling"); - expect(loaded).toHaveProperty("agent:main:lifecycle-cleanup-fresh"); - expect(loaded).toHaveProperty("agent:main:telegram:group:lifecycle-cleanup-room"); - expect(loaded).toHaveProperty("agent:main:regular"); - const files = fs.readdirSync(lifecycleSessionsDir); - expect( - files.filter((file) => file.startsWith("removed-lifecycle.jsonl.deleted.")), - ).toHaveLength(1); - expect(files.filter((file) => file.startsWith("orphan-lifecycle.jsonl.deleted."))).toHaveLength( - 1, - ); - expect( - files.filter((file) => file.startsWith("custom-lifecycle-old.jsonl.deleted.")), - ).toHaveLength(1); - expect(files).toContain("custom-lifecycle.jsonl"); - expect(files).toContain("fresh-lifecycle.jsonl"); - expect(files).toContain("referenced.jsonl"); - expect(fs.existsSync(siblingTranscriptPath)).toBe(true); - expect(fs.readdirSync(siblingDir)).toEqual(["sibling-lifecycle.jsonl"]); }); - it("preserves fresh lifecycle entries that only have explicit sessionFile metadata", async () => { - const nowMs = Date.now(); - const lifecycleSessionsDir = path.join(tempDir, "state", "agents", "main", "sessions"); - const lifecycleStorePath = path.join(lifecycleSessionsDir, "sessions.json"); - const freshTranscriptPath = path.join(lifecycleSessionsDir, "session-file-only.jsonl"); - fs.mkdirSync(lifecycleSessionsDir, { recursive: true }); - await saveSessionStore( - lifecycleStorePath, - { - "agent:main:lifecycle-cleanup-file-only": { - sessionFile: freshTranscriptPath, - updatedAt: nowMs, - } as SessionEntry, - }, - { skipMaintenance: true }, - ); - fs.writeFileSync(freshTranscriptPath, '{"runId":"lifecycle-marker-file-only"}\n', "utf-8"); - - const result = await cleanupSessionLifecycleArtifacts({ - storePath: lifecycleStorePath, - sessionKeySegmentPrefix: "lifecycle-cleanup-", - transcriptContentMarker: "lifecycle-marker-", - orphanTranscriptMinAgeMs: 300_000, - nowMs, - }); - - expect(result).toEqual({ removedEntries: 0, archivedTranscriptArtifacts: 0 }); - expect(loadSessionStore(lifecycleStorePath, { skipCache: true })).toHaveProperty( - "agent:main:lifecycle-cleanup-file-only", - ); - expect(fs.existsSync(freshTranscriptPath)).toBe(true); - }); - - it("prefers current generated lifecycle transcripts over stale generated sessionFile metadata", async () => { - const nowMs = Date.now(); - const oldDate = new Date(nowMs - 600_000); - const currentSessionId = "11111111-1111-4111-8111-111111111111"; - const staleSessionId = "22222222-2222-4222-8222-222222222222"; - const lifecycleSessionsDir = path.join(tempDir, "state", "agents", "main", "sessions"); - const lifecycleStorePath = path.join(lifecycleSessionsDir, "sessions.json"); - const currentTranscriptPath = path.join(lifecycleSessionsDir, `${currentSessionId}.jsonl`); - const staleTranscriptPath = path.join(lifecycleSessionsDir, `${staleSessionId}.jsonl`); - fs.mkdirSync(lifecycleSessionsDir, { recursive: true }); - await saveSessionStore( - lifecycleStorePath, - { - "agent:main:lifecycle-cleanup-current": { - sessionFile: staleTranscriptPath, - sessionId: currentSessionId, - updatedAt: nowMs, + it.each([ + { + name: "exact entry", + params: { + expectedEntry: { + lifecycleRevision: "original-revision", + sessionId: "session-1", + updatedAt: 999, }, }, - { skipMaintenance: true }, - ); - fs.writeFileSync(currentTranscriptPath, '{"runId":"lifecycle-marker-current"}\n', "utf-8"); - fs.writeFileSync(staleTranscriptPath, '{"runId":"lifecycle-marker-stale"}\n', "utf-8"); - fs.utimesSync(staleTranscriptPath, oldDate, oldDate); + }, + { + name: "session id", + params: { expectedSessionId: "session-2" }, + }, + { + name: "lifecycle revision", + params: { expectedLifecycleRevision: "replacement-revision" }, + }, + { + name: "updatedAt", + params: { expectedUpdatedAt: 20 }, + }, + ])( + "does not delete SQLite lifecycle entries when the $name guard mismatches", + async ({ params }) => { + const scope = { + sessionId: "session-1", + sessionKey: "agent:main:guarded-delete", + storePath, + }; + await upsertSessionEntry(scope, { + lifecycleRevision: "original-revision", + sessionId: scope.sessionId, + updatedAt: 10, + }); - const result = await cleanupSessionLifecycleArtifacts({ - storePath: lifecycleStorePath, - sessionKeySegmentPrefix: "lifecycle-cleanup-", - transcriptContentMarker: "lifecycle-marker-", - orphanTranscriptMinAgeMs: 300_000, - nowMs, - }); + const result = await deleteSessionEntryLifecycle({ + archiveTranscript: false, + storePath, + target: { + canonicalKey: scope.sessionKey, + storeKeys: [scope.sessionKey], + }, + ...params, + }); - expect(result).toEqual({ removedEntries: 0, archivedTranscriptArtifacts: 1 }); - expect(loadSessionStore(lifecycleStorePath, { skipCache: true })).toHaveProperty( - "agent:main:lifecycle-cleanup-current", - ); - expect(fs.existsSync(currentTranscriptPath)).toBe(true); - expect( - fs - .readdirSync(lifecycleSessionsDir) - .filter((file) => file.startsWith(`${staleSessionId}.jsonl.deleted.`)), - ).toHaveLength(1); - }); + expect(result.deleted).toBe(false); + expect(loadSessionEntry(scope)).toMatchObject({ + lifecycleRevision: "original-revision", + sessionId: scope.sessionId, + updatedAt: expect.any(Number), + }); + }, + ); it("persists reset lifecycle entry changes with transcript replay and archive", async () => { const now = Date.now(); @@ -2015,187 +1811,6 @@ describe("session accessor file-backed seam", () => { expect(fs.readFileSync(nextTranscript, "utf-8")).toContain('"content":"hello"'); }); - it("appends transcript events through a session scope", async () => { - const scope = { - sessionFile: transcriptPath, - sessionId: "session-1", - sessionKey: "agent:main:main", - storePath, - }; - const event = { - payload: { value: "hello" }, - type: "metadata", - }; - - await appendTranscriptEvent(scope, { type: "session", sessionId: "session-1" }); - await appendTranscriptEvent(scope, event); - - expect(fs.statSync(transcriptPath).mode & 0o777).toBe(0o600); - }); - - it("applies keyed lifecycle removals and artifact cleanup from the final store", async () => { - const removedTranscriptPath = path.join(tempDir, "removed-session.jsonl"); - const sharedTranscriptPath = path.join(tempDir, "shared-session.jsonl"); - const orphanTranscriptPath = path.join(tempDir, "orphan-session.jsonl"); - fs.writeFileSync( - storePath, - JSON.stringify( - { - "agent:main:removed": { - sessionId: "removed-session", - }, - "agent:main:shared-remove": { - sessionId: "shared-session", - }, - "agent:main:shared-keep": { - sessionId: "shared-session", - }, - }, - null, - 2, - ), - "utf-8", - ); - fs.writeFileSync(removedTranscriptPath, '{"type":"session"}\n', "utf-8"); - fs.writeFileSync(sharedTranscriptPath, '{"type":"session"}\n', "utf-8"); - fs.writeFileSync(orphanTranscriptPath, "orphan", "utf-8"); - const oldDate = new Date(Date.now() - 60_000); - fs.utimesSync(orphanTranscriptPath, oldDate, oldDate); - - const result = await applySessionEntryLifecycleMutation({ - storePath, - removals: [ - { sessionKey: "agent:main:removed", archiveRemovedTranscript: true }, - { sessionKey: "agent:main:shared-remove", archiveRemovedTranscript: true }, - ], - upserts: [ - { - sessionKey: "agent:main:new", - entry: { sessionId: "new-session", updatedAt: 123 }, - }, - ], - skipMaintenance: true, - restrictArchivedTranscriptsToStoreDir: true, - pruneUnreferencedArtifacts: { olderThanMs: 1 }, - }); - - expect(result.removedEntries).toBe(2); - expect(result.unreferencedArtifacts?.removedFiles).toBe(1); - expect(loadSessionStore(storePath, { skipCache: true })).toEqual({ - "agent:main:shared-keep": { - sessionId: "shared-session", - }, - "agent:main:new": { - sessionId: "new-session", - updatedAt: 123, - }, - }); - expect(fs.existsSync(removedTranscriptPath)).toBe(false); - expect(fs.existsSync(sharedTranscriptPath)).toBe(true); - expect(fs.existsSync(orphanTranscriptPath)).toBe(false); - expect( - fs.readdirSync(tempDir).filter((file) => file.startsWith("removed-session.jsonl.deleted.")), - ).toHaveLength(1); - }); - - it("does not apply stale lifecycle removal plans to changed entries", async () => { - const stalePlanEntry: SessionEntry = { - sessionId: "planned-session", - updatedAt: 1, - }; - fs.writeFileSync( - storePath, - JSON.stringify( - { - "agent:main:planned": { - sessionId: "planned-session", - updatedAt: 2, - }, - }, - null, - 2, - ), - "utf-8", - ); - - const result = await applySessionEntryLifecycleMutation({ - storePath, - removals: [ - { - sessionKey: "agent:main:planned", - expectedEntry: stalePlanEntry, - archiveRemovedTranscript: true, - }, - ], - skipMaintenance: true, - }); - - expect(result.removedEntries).toBe(0); - expect(loadSessionStore(storePath, { skipCache: true })).toEqual({ - "agent:main:planned": { - sessionId: "planned-session", - updatedAt: 2, - }, - }); - }); - - it("builds lifecycle upsert entries from the locked store snapshot", async () => { - fs.writeFileSync( - storePath, - JSON.stringify( - { - "agent:main:existing": { - sessionId: "current-session", - updatedAt: 10, - }, - }, - null, - 2, - ), - "utf-8", - ); - - const result = await applySessionEntryLifecycleMutation({ - storePath, - upserts: [ - { - sessionKey: "agent:main:existing", - buildEntry: ({ currentEntry }) => ({ - ...currentEntry, - sessionId: currentEntry?.sessionId ?? "missing", - updatedAt: 20, - }), - }, - ], - skipMaintenance: true, - }); - - expect(result.afterCount).toBe(1); - expect(loadSessionStore(storePath, { skipCache: true })).toEqual({ - "agent:main:existing": { - sessionId: "current-session", - updatedAt: 20, - }, - }); - }); - - it("appends to an explicit transcript artifact without a session key", async () => { - const scope = { - sessionFile: transcriptPath, - sessionId: "session-1", - storePath, - }; - const event = { - payload: { value: "keyless" }, - type: "metadata", - }; - - await appendTranscriptEvent(scope, event); - - // Explicit-artifact writes never touch entry metadata: no entry appears. - expect(listSessionEntries({ storePath })).toEqual([]); - }); - it("trims a manual compact transcript and clears stale token metadata", async () => { const sessionId = "11111111-1111-4111-8111-111111111111"; const manualTranscriptPath = path.join(tempDir, `${sessionId}.jsonl`); @@ -2250,8 +1865,10 @@ describe("session accessor file-backed seam", () => { message: { role: "user", content: `message ${index}`, timestamp: index }, })), ]; - const originalTranscript = `${transcriptRecords.map((record) => JSON.stringify(record)).join("\n")}\n`; - fs.writeFileSync(manualTranscriptPath, originalTranscript, { encoding: "utf-8", mode: 0o640 }); + await replaceSqliteTranscriptEvents( + scope, + transcriptRecords as Parameters<typeof replaceSqliteTranscriptEvents>[1], + ); const updates: unknown[] = []; const unsubscribe = onSessionTranscriptUpdate((update) => updates.push(update)); @@ -2263,22 +1880,13 @@ describe("session accessor file-backed seam", () => { unsubscribe(); expect(result).toMatchObject({ compacted: true, kept: 3 }); const archived = result.compacted ? result.archived : ""; - expect(path.basename(archived)).toMatch(new RegExp(`^${sessionId}\\.jsonl\\.bak\\.`)); - expect(fs.readFileSync(archived, "utf-8")).toBe(originalTranscript); - const trimmedRecords = fs - .readFileSync(manualTranscriptPath, "utf-8") - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record<string, unknown>); + expect(archived).toContain(`sqlite:main:${sessionId}:`); + const trimmedRecords = (await loadTranscriptEvents(scope)) as Array<Record<string, unknown>>; expect(trimmedRecords).toMatchObject([ { type: "session", id: sessionId }, { type: "message", id: "entry-3", parentId: null }, { type: "message", id: "entry-4", parentId: "entry-3" }, ]); - expect(fs.statSync(manualTranscriptPath).mode & 0o777).toBe(0o600); - const reopened = SessionManager.open(manualTranscriptPath, tempDir, tempDir); - expect(reopened.getEntries().map((entry) => entry.id)).toEqual(["entry-3", "entry-4"]); - expect(reopened.buildSessionContext().messages).toHaveLength(2); const updatedEntry = loadSessionEntry(scope); expect(updatedEntry).toMatchObject({ sessionFile: manualTranscriptPath, @@ -2290,201 +1898,7 @@ describe("session accessor file-backed seam", () => { expect(updatedEntry?.outputTokens).toBeUndefined(); expect(updatedEntry?.totalTokens).toBeUndefined(); expect(updatedEntry?.totalTokensFresh).toBeUndefined(); - expect(updates).toEqual([ - { sessionFile: archived }, - { sessionFile: fs.realpathSync(manualTranscriptPath) }, - ]); - }); - - it("keeps retained messages reachable through an out-of-window label", async () => { - const sessionId = "22222222-2222-4222-8222-222222222222"; - const sessionFile = path.join(tempDir, `${sessionId}.jsonl`); - const scope = { - agentId: "main", - sessionId, - sessionKey: "agent:main:main", - storePath, - }; - const records = [ - { - type: "session", - version: 3, - id: sessionId, - timestamp: "2026-06-19T12:00:00.000Z", - cwd: tempDir, - }, - { - type: "message", - id: "old", - parentId: null, - timestamp: "2026-06-19T12:00:01.000Z", - message: { role: "user", content: "old", timestamp: 1 }, - }, - { - type: "message", - id: "kept-1", - parentId: "old", - timestamp: "2026-06-19T12:00:02.000Z", - message: { role: "user", content: "kept one", timestamp: 2 }, - }, - { - type: "label", - id: "label-1", - parentId: "kept-1", - targetId: "old", - label: "trimmed target", - timestamp: "2026-06-19T12:00:03.000Z", - }, - { - type: "message", - id: "kept-2", - parentId: "label-1", - timestamp: "2026-06-19T12:00:04.000Z", - message: { role: "user", content: "kept two", timestamp: 4 }, - }, - ]; - await upsertSessionEntry(scope, { sessionFile, sessionId, updatedAt: 1 }); - fs.writeFileSync( - sessionFile, - `${records.map((record) => JSON.stringify(record)).join("\n")}\n`, - "utf-8", - ); - - await expect( - trimSessionTranscriptForManualCompact(scope, { maxLines: 4 }), - ).resolves.toMatchObject({ compacted: true, kept: 4 }); - - const context = SessionManager.open(sessionFile, tempDir, tempDir).buildSessionContext(); - expect(JSON.stringify(context.messages)).toContain("kept one"); - expect(JSON.stringify(context.messages)).toContain("kept two"); - }); - - it("does not reactivate an abandoned branch when a leaf target was trimmed", async () => { - const sessionId = "44444444-4444-4444-8444-444444444444"; - const sessionFile = path.join(tempDir, `${sessionId}.jsonl`); - const scope = { - agentId: "main", - sessionId, - sessionKey: "agent:main:main", - storePath, - }; - const records = [ - { - type: "session", - version: 3, - id: sessionId, - timestamp: "2026-06-19T12:00:00.000Z", - cwd: tempDir, - }, - { - type: "message", - id: "selected-before-window", - parentId: null, - timestamp: "2026-06-19T12:00:01.000Z", - message: { role: "user", content: "selected", timestamp: 1 }, - }, - { - type: "message", - id: "abandoned-side-row", - parentId: "selected-before-window", - appendMode: "side", - timestamp: "2026-06-19T12:00:02.000Z", - message: { role: "user", content: "must stay hidden", timestamp: 2 }, - }, - { - type: "leaf", - id: "leaf-1", - parentId: "abandoned-side-row", - targetId: "selected-before-window", - appendParentId: "selected-before-window", - timestamp: "2026-06-19T12:00:03.000Z", - }, - ]; - await upsertSessionEntry(scope, { sessionFile, sessionId, updatedAt: 1 }); - fs.writeFileSync( - sessionFile, - `${records.map((record) => JSON.stringify(record)).join("\n")}\n`, - "utf-8", - ); - - await expect( - trimSessionTranscriptForManualCompact(scope, { maxLines: 3 }), - ).resolves.toMatchObject({ compacted: true, kept: 3 }); - - const persisted = fs - .readFileSync(sessionFile, "utf-8") - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record<string, unknown>); - expect(persisted.find((entry) => entry.type === "leaf")).toMatchObject({ - targetId: null, - appendParentId: null, - }); - expect( - SessionManager.open(sessionFile, tempDir, tempDir).buildSessionContext().messages, - ).toEqual([]); - }); - - it("keeps malformed leaf controls transparent while re-rooting retained descendants", async () => { - const sessionId = "55555555-5555-4555-8555-555555555555"; - const sessionFile = path.join(tempDir, `${sessionId}.jsonl`); - const scope = { - agentId: "main", - sessionId, - sessionKey: "agent:main:main", - storePath, - }; - const records = [ - { - type: "session", - version: 3, - id: sessionId, - timestamp: "2026-06-19T12:00:00.000Z", - cwd: tempDir, - }, - { - type: "message", - id: "trimmed", - parentId: null, - message: { role: "user", content: "trimmed", timestamp: 1 }, - timestamp: "2026-06-19T12:00:01.000Z", - }, - { - type: "message", - id: "retained-root", - parentId: "trimmed", - message: { role: "user", content: "retained root", timestamp: 2 }, - timestamp: "2026-06-19T12:00:02.000Z", - }, - { - type: "leaf", - id: "malformed-leaf", - parentId: "retained-root", - timestamp: "2026-06-19T12:00:03.000Z", - }, - { - type: "message", - id: "retained-child", - parentId: "malformed-leaf", - message: { role: "user", content: "retained child", timestamp: 4 }, - timestamp: "2026-06-19T12:00:04.000Z", - }, - ]; - await upsertSessionEntry(scope, { sessionFile, sessionId, updatedAt: 1 }); - fs.writeFileSync( - sessionFile, - `${records.map((record) => JSON.stringify(record)).join("\n")}\n`, - ); - - await expect( - trimSessionTranscriptForManualCompact(scope, { maxLines: 4 }), - ).resolves.toMatchObject({ compacted: true, kept: 4 }); - - const serializedContext = JSON.stringify( - SessionManager.open(sessionFile, tempDir, tempDir).buildSessionContext().messages, - ); - expect(serializedContext).toContain("retained root"); - expect(serializedContext).toContain("retained child"); + expect(updates).toEqual([]); }); it("repairs a retained compaction boundary when its first kept entry was trimmed", async () => { @@ -2545,179 +1959,29 @@ describe("session accessor file-backed seam", () => { }, ]; await upsertSessionEntry(scope, { sessionFile, sessionId, updatedAt: 1 }); - fs.writeFileSync( - sessionFile, - `${records.map((record) => JSON.stringify(record)).join("\n")}\n`, - "utf-8", + await replaceSqliteTranscriptEvents( + scope, + records as Parameters<typeof replaceSqliteTranscriptEvents>[1], ); await expect( trimSessionTranscriptForManualCompact(scope, { maxLines: 5 }), ).resolves.toMatchObject({ compacted: true, kept: 5 }); - const reopened = SessionManager.open(sessionFile, tempDir, tempDir); + const reopened = (await loadTranscriptEvents(scope)) as Array<Record<string, unknown>>; expect( - reopened - .getEntries() - .find((entry) => entry.type === "compaction" && entry.id === "compaction-1"), + reopened.find((entry) => entry.type === "compaction" && entry.id === "compaction-1"), ).toMatchObject({ firstKeptEntryId: "kept-before-compaction", }); expect( - reopened - .getEntries() - .find((entry) => entry.type === "compaction" && entry.id === "compaction-2"), + reopened.find((entry) => entry.type === "compaction" && entry.id === "compaction-2"), ).toMatchObject({ firstKeptEntryId: "compaction-2" }); - const serializedContext = JSON.stringify(reopened.buildSessionContext().messages); - expect(serializedContext).not.toContain("kept before"); + const serializedContext = JSON.stringify(reopened); + expect(serializedContext).toContain("kept before"); expect(serializedContext).toContain("kept after"); }); - it("prefers the current generated transcript over a stale generated sessionFile", async () => { - const currentSessionId = "11111111-1111-4111-8111-111111111111"; - const staleSessionId = "22222222-2222-4222-8222-222222222222"; - const currentTranscriptPath = path.join(tempDir, `${currentSessionId}.jsonl`); - const staleTranscriptPath = path.join(tempDir, `${staleSessionId}.jsonl`); - const scope = { - agentId: "main", - sessionId: currentSessionId, - sessionKey: "agent:main:main", - storePath, - }; - await upsertSessionEntry(scope, { - sessionFile: staleTranscriptPath, - sessionId: currentSessionId, - updatedAt: 100, - }); - const currentHeader = { - type: "session", - version: 3, - id: currentSessionId, - timestamp: "2026-06-19T12:00:00.000Z", - cwd: tempDir, - }; - const currentOne = { - type: "message", - id: "current-one", - parentId: null, - timestamp: "2026-06-19T12:00:01.000Z", - message: { role: "user", content: "current one", timestamp: 1 }, - }; - const currentTwo = { - type: "message", - id: "current-two", - parentId: "current-one", - timestamp: "2026-06-19T12:00:02.000Z", - message: { role: "user", content: "current two", timestamp: 2 }, - }; - fs.writeFileSync( - currentTranscriptPath, - `${[currentHeader, currentOne, currentTwo].map((record) => JSON.stringify(record)).join("\n")}\n`, - "utf-8", - ); - fs.writeFileSync(staleTranscriptPath, "stale one\nstale two\n", "utf-8"); - - const result = await trimSessionTranscriptForManualCompact(scope, { - maxLines: 2, - sessionFile: staleTranscriptPath, - }); - - expect(result).toMatchObject({ compacted: true, kept: 2 }); - expect(fs.readFileSync(currentTranscriptPath, "utf-8")).toBe( - `${JSON.stringify(currentHeader)}\n${JSON.stringify({ ...currentTwo, parentId: null })}\n`, - ); - expect(fs.readFileSync(staleTranscriptPath, "utf-8")).toBe("stale one\nstale two\n"); - }); - - it("rejects transcript writes without a session key or explicit file", async () => { - await expect( - appendTranscriptEvent({ sessionId: "session-1", storePath }, { type: "metadata" }), - ).rejects.toThrow(/session key or explicit session file/); - }); - - it("rejects raw message transcript events", async () => { - const scope = { - sessionFile: transcriptPath, - sessionId: "session-1", - sessionKey: "agent:main:main", - storePath, - }; - - await expect( - appendTranscriptEvent(scope, { - id: "msg-1", - message: { role: "user", content: "hello" }, - parentId: null, - type: "message", - }), - ).rejects.toThrow(/appendTranscriptMessage/); - expect(fs.existsSync(transcriptPath)).toBe(false); - }); - - it("appends messages and publishes updates through a session scope", async () => { - const scope = { - agentId: "main", - sessionFile: transcriptPath, - sessionId: "session-1", - sessionKey: "agent:main:main", - storePath, - }; - const updates: unknown[] = []; - const unsubscribe = onSessionTranscriptUpdate((update) => { - updates.push(update); - }); - - const appended = await appendTranscriptMessage(scope, { - cwd: tempDir, - idempotencyLookup: "scan", - message: { - role: "assistant", - content: "hello", - idempotencyKey: "assistant-once", - }, - }); - const replayed = await appendTranscriptMessage(scope, { - cwd: tempDir, - idempotencyLookup: "scan", - message: { - role: "assistant", - content: "hello again", - idempotencyKey: "assistant-once", - }, - }); - await publishTranscriptUpdate(scope, { - agentId: "main", - message: appended.message, - messageId: appended.messageId, - sessionKey: scope.sessionKey, - }); - unsubscribe(); - - expect(replayed).toMatchObject({ - appended: false, - messageId: appended.messageId, - message: expect.objectContaining({ - content: "hello", - idempotencyKey: "assistant-once", - }), - }); - expect(updates).toEqual([ - { - agentId: "main", - message: appended.message, - messageId: appended.messageId, - sessionId: scope.sessionId, - sessionFile: transcriptPath, - sessionKey: scope.sessionKey, - target: { - agentId: "main", - sessionId: scope.sessionId, - sessionKey: scope.sessionKey, - }, - }, - ]); - }); - it("persists a transcript turn, touches metadata, and publishes after the write", async () => { const scope = { agentId: "main", @@ -2730,15 +1994,12 @@ describe("session accessor file-backed seam", () => { updatedAt: 10, }); const updates: Array<{ - lineCount: number; - sessionFile: string | undefined; + target: unknown; updatedAt: number | undefined; }> = []; const unsubscribe = onSessionTranscriptUpdate((update) => { - const lines = fs.readFileSync(update.sessionFile, "utf8").trim().split("\n"); updates.push({ - lineCount: lines.length, - sessionFile: loadSessionEntry(scope)?.sessionFile, + target: update.target, updatedAt: loadSessionEntry(scope)?.updatedAt, }); }); @@ -2768,6 +2029,7 @@ describe("session accessor file-backed seam", () => { unsubscribe(); expect(result.appendedCount).toBe(2); + await expect(loadTranscriptEvents(scope)).resolves.toHaveLength(3); expect(loadSessionEntry(scope)).toMatchObject({ sessionFile: result.sessionFile, sessionId: scope.sessionId, @@ -2776,14 +2038,17 @@ describe("session accessor file-backed seam", () => { expect(loadSessionEntry(scope)?.updatedAt).toBeGreaterThanOrEqual(10); expect(updates).toEqual([ { - lineCount: 3, - sessionFile: result.sessionFile, + target: { + agentId: "main", + sessionId: "session-lock-order", + sessionKey: "agent:main:lock-order", + }, updatedAt: expect.any(Number), }, ]); }); - it("queues transcript turn appends before taking the file write lock", async () => { + it("allows concurrent SQLite transcript turn and direct appends", async () => { const scope = { agentId: "main", sessionId: "session-1", @@ -2824,6 +2089,14 @@ describe("session accessor file-backed seam", () => { }); await shouldAppendEntered; + let unrelatedWriteError: unknown; + try { + appendSqliteTrajectoryRuntimeEvents({ sessionId: scope.sessionId, storePath }, [ + createTestTrajectoryEvent(scope.sessionId), + ]); + } catch (error) { + unrelatedWriteError = error; + } const queuedAppendPromise = appendTranscriptMessage(scope, { cwd: tempDir, message: { @@ -2843,9 +2116,52 @@ describe("session accessor file-backed seam", () => { ]); expect(completed).toBe(true); await results; + expect(unrelatedWriteError).toBeUndefined(); }); - it("rejects expected-session transcript turns after a queued session rebind", async () => { + it("persists expected-session SQLite transcript turns without reentering the writer queue", async () => { + const scope = { + agentId: "main", + sessionId: "session-expected", + sessionKey: "agent:main:expected", + storePath, + }; + await upsertSessionEntry(scope, { + sessionId: scope.sessionId, + updatedAt: 10, + }); + + const turnPromise = persistSessionTranscriptTurn(scope, { + cwd: tempDir, + expectedSessionId: scope.sessionId, + messages: [ + { + message: { + role: "assistant", + content: "expected reply", + timestamp: 100, + }, + }, + ], + publishWhen: "always", + touchSessionEntry: true, + updateMode: "file-only", + }); + + const completed = await Promise.race([ + turnPromise.then(() => true), + new Promise<boolean>((resolve) => { + setTimeout(() => resolve(false), 1_000); + }), + ]); + expect(completed).toBe(true); + const result = await turnPromise; + + expect(result.appendedCount).toBe(1); + await expect(loadTranscriptEvents(scope)).resolves.toHaveLength(2); + }); + + it("rejects expected-session transcript turns after a session rebind", async () => { const scope = { agentId: "main", sessionId: "session-original", @@ -2856,30 +2172,19 @@ describe("session accessor file-backed seam", () => { sessionId: scope.sessionId, updatedAt: 10, }); - let releaseReset = () => {}; - const resetGate = new Promise<void>((resolve) => { - releaseReset = resolve; - }); - let markResetStarted = () => {}; - const resetStarted = new Promise<void>((resolve) => { - markResetStarted = resolve; - }); - const replacementSessionFile = path.join(tempDir, "session-replacement.jsonl"); - const reset = updateSessionStoreEntry({ - storePath, - sessionKey: scope.sessionKey, - update: async () => { - markResetStarted(); - await resetGate; - return { - sessionFile: replacementSessionFile, - sessionId: "session-replacement", - }; + await updateSessionEntry( + { + sessionKey: scope.sessionKey, + storePath, }, - }); - await resetStarted; + () => ({ + sessionFile: "sqlite:main:session-replacement", + sessionId: "session-replacement", + }), + { skipMaintenance: true }, + ); - const turn = persistSessionTranscriptTurn(scope, { + const result = await persistSessionTranscriptTurn(scope, { expectedSessionId: scope.sessionId, messages: [ { @@ -2894,23 +2199,119 @@ describe("session accessor file-backed seam", () => { touchSessionEntry: true, updateMode: "file-only", }); - await new Promise<void>((resolve) => { - setImmediate(resolve); - }); - releaseReset(); - - await reset; - const result = await turn; expect(result).toMatchObject({ appendedCount: 0, rejectedReason: "session-rebound", }); - expect(fs.existsSync(path.join(tempDir, "session-original.jsonl"))).toBe(false); - expect(fs.existsSync(replacementSessionFile)).toBe(false); + await expect(loadTranscriptEvents(scope)).resolves.toEqual([]); }); - it("publishes transcript turn appends through an active owned write lock", async () => { + it("rejects an expected-session transcript turn rebound during predicate preparation", async () => { + const scope = { + agentId: "main", + sessionId: "session-predicate-original", + sessionKey: "agent:main:predicate-rebind", + storePath, + }; + await upsertSessionEntry(scope, { + lifecycleRevision: "predicate-revision", + sessionId: scope.sessionId, + updatedAt: 10, + }); + let releasePredicate!: () => void; + let markPredicateStarted!: () => void; + const predicateStarted = new Promise<void>((resolve) => { + markPredicateStarted = resolve; + }); + const predicateGate = new Promise<void>((resolve) => { + releasePredicate = resolve; + }); + const pendingTurn = persistSessionTranscriptTurn(scope, { + expectedLifecycleRevision: "predicate-revision", + expectedSessionId: scope.sessionId, + messages: [ + { + message: { role: "assistant", content: "late reply", timestamp: 100 }, + shouldAppend: async () => { + markPredicateStarted(); + await predicateGate; + return true; + }, + }, + ], + touchSessionEntry: true, + updateMode: "file-only", + }); + + await predicateStarted; + let replacementError: unknown; + try { + replaceSqliteSessionEntrySync(scope, { + lifecycleRevision: "replacement-revision", + sessionId: "session-predicate-replacement", + updatedAt: 20, + }); + } catch (error) { + replacementError = error; + } finally { + releasePredicate(); + } + const result = await pendingTurn; + + expect(replacementError).toBeUndefined(); + expect(result).toMatchObject({ appendedCount: 0, rejectedReason: "session-rebound" }); + await expect(loadTranscriptEvents(scope)).resolves.toEqual([]); + }); + + it("rejects expected-session transcript turns after lifecycle ownership changes", async () => { + const scope = { + agentId: "main", + sessionId: "session-original", + sessionKey: "agent:main:main", + storePath, + }; + await upsertSessionEntry(scope, { + lifecycleRevision: "original-revision", + sessionId: scope.sessionId, + updatedAt: 10, + }); + await updateSessionEntry( + { + sessionKey: scope.sessionKey, + storePath, + }, + () => ({ + lifecycleRevision: "replacement-revision", + }), + { skipMaintenance: true }, + ); + + const result = await persistSessionTranscriptTurn(scope, { + expectedLifecycleRevision: "original-revision", + expectedSessionId: scope.sessionId, + messages: [ + { + message: { + role: "assistant", + content: "late reply", + timestamp: 100, + }, + }, + ], + publishWhen: "always", + touchSessionEntry: true, + updateMode: "file-only", + }); + + expect(result).toMatchObject({ + appendedCount: 0, + rejectedReason: "session-rebound", + }); + await expect(loadTranscriptEvents(scope)).resolves.toEqual([]); + }); + + it("routes SQLite transcript turn appends through an active owned file lock", async () => { const scope = { agentId: "main", sessionFile: transcriptPath, @@ -2950,110 +2351,77 @@ describe("session accessor file-backed seam", () => { }), ); - expect(publishOptions).toEqual([true]); - expect(publishedEntryBatches).toHaveLength(1); - expect(publishedEntryBatches[0]).toEqual([ - expect.objectContaining({ kind: "header" }), - expect.objectContaining({ kind: "id" }), - ]); + expect(publishOptions).toEqual([undefined]); + expect(publishedEntryBatches).toEqual([[]]); + await expect(loadTranscriptEvents(scope)).resolves.toHaveLength(2); }); - it("honors thread fallback paths when resolving transcript scope from the store", async () => { - const scope = { - agentId: "main", - sessionId: "session-1", - sessionKey: "agent:main:demo-channel:1234:thread:456", - storePath, - }; - const event = { - payload: { value: "hello" }, - type: "metadata", - }; - - await upsertSessionEntry(scope, { - sessionId: scope.sessionId, - updatedAt: 10, - }); - await appendTranscriptEvent(scope, event); - - const expectedTranscriptPath = path.join(tempDir, "session-1-topic-456.jsonl"); - expect(fs.existsSync(expectedTranscriptPath)).toBe(true); - expect(fs.existsSync(path.join(tempDir, "session-1.jsonl"))).toBe(false); - expect(fs.realpathSync(loadSessionEntry(scope)?.sessionFile ?? "")).toBe( - fs.realpathSync(expectedTranscriptPath), - ); - }); - - it("resolves runtime transcript targets from scope without caller-owned paths", async () => { + it("resolves store-backed runtime transcript targets with stale file paths to markers", async () => { + const staleSessionFile = path.join(tempDir, "session-1.jsonl"); const scope = { agentId: "main", sessionId: "session-1", sessionKey: "agent:main:main", storePath, }; + const marker = `sqlite:main:${scope.sessionId}:${storePath}`; await upsertSessionEntry(scope, { sessionId: scope.sessionId, - updatedAt: 10, - }); - - const target = await resolveSessionTranscriptRuntimeTarget(scope); - - expect(target).toMatchObject({ - agentId: "main", - sessionId: "session-1", - sessionKey: "agent:main:main", - }); - expect(fs.realpathSync(path.dirname(target.sessionFile))).toBe(fs.realpathSync(tempDir)); - expect(path.basename(target.sessionFile)).toBe("session-1.jsonl"); - expect(loadSessionEntry(scope)?.sessionFile).toBe(target.sessionFile); - }); - - it("preserves an explicitly resolved runtime transcript file target", async () => { - const explicitSessionFile = path.join(tempDir, "explicit-session.jsonl"); - const scope = { - agentId: "main", - sessionFile: explicitSessionFile, - sessionId: "session-1", - sessionKey: "agent:main:main", - storePath, - }; - - await upsertSessionEntry(scope, { - sessionId: scope.sessionId, + sessionFile: staleSessionFile, updatedAt: 10, }); const readTarget = await resolveSessionTranscriptRuntimeReadTarget(scope); const writeTarget = await resolveSessionTranscriptRuntimeTarget(scope); - expect(readTarget.sessionFile).toBe(explicitSessionFile); - expect(writeTarget.sessionFile).toBe(explicitSessionFile); - expect(loadSessionEntry(scope)?.sessionFile).toBeUndefined(); + expect(readTarget.sessionFile).toBe(marker); + expect(writeTarget.sessionFile).toBe(marker); + expect(loadSessionEntry(scope)?.sessionFile).toBe(staleSessionFile); }); - it("uses a supplied read session entry without loading the store", () => { - const explicitSessionFile = path.join(tempDir, "entry-session.jsonl"); - fs.writeFileSync(explicitSessionFile, "", "utf8"); - fs.writeFileSync(storePath, "{not-json", "utf8"); - - const target = resolveSessionTranscriptReadTarget({ + it("resolves SQLite-backed runtime transcript targets to markers", async () => { + const scope = { agentId: "main", - sessionEntry: { - sessionFile: explicitSessionFile, - sessionId: "session-1", - }, sessionId: "session-1", sessionKey: "agent:main:main", storePath, + }; + const marker = `sqlite:main:${scope.sessionId}:${storePath}`; + + await upsertSessionEntry(scope, { + sessionId: scope.sessionId, + sessionFile: marker, + updatedAt: 10, }); - expect(target).toMatchObject({ + const readTarget = await resolveSessionTranscriptRuntimeReadTarget(scope); + const writeTarget = await resolveSessionTranscriptRuntimeTarget(scope); + + expect(readTarget.sessionFile).toBe(marker); + expect(writeTarget.sessionFile).toBe(marker); + }); + + it("normalizes imported legacy session transcript paths to SQLite markers", async () => { + const sessionKey = "agent:main:main"; + await importSqliteSessionRows({ agentId: "main", - sessionFile: fs.realpathSync(explicitSessionFile), - sessionId: "session-1", - sessionKey: "agent:main:main", + entry: { + sessionFile: path.join(tempDir, "legacy-transcript.jsonl"), + sessionId: "session-1", + updatedAt: 10, + }, + sessionKey, + storePath, }); + + expect( + loadExactSqliteSessionEntry({ + agentId: "main", + sessionKey, + storePath, + })?.entry.sessionFile, + ).toBe(`sqlite:main:session-1:${path.join(tempDir, "openclaw-agent.sqlite")}`); }); it("resolves an explicit read transcript file without agent identity", () => { @@ -3070,60 +2438,25 @@ describe("session accessor file-backed seam", () => { }); }); - it("keeps read and write runtime targets aligned for new topic sessions", async () => { - const scope = { - agentId: "main", - sessionId: "session-2", - sessionKey: "agent:main:main:topic:456", - storePath, - threadId: "456", - }; - fs.writeFileSync( - path.join(tempDir, "session-1-topic-456.jsonl"), - '{"type":"session","id":"session-1"}\n', - "utf8", - ); + it("does not expose legacy custom transcript paths as read fallbacks after SQLite migration", async () => { + const legacyTranscript = path.join(tempDir, "custom-topic-transcript.jsonl"); + const sessionKey = "agent:main:telegram:group:1:topic:9"; await upsertSessionEntry( - { sessionKey: scope.sessionKey, storePath }, + { sessionKey, storePath }, { - sessionFile: "session-1-topic-456.jsonl", - sessionId: "session-1", + sessionId: "custom-topic-session", + sessionFile: legacyTranscript, updatedAt: 10, }, ); - const readTarget = await resolveSessionTranscriptRuntimeReadTarget(scope); - const writeTarget = await resolveSessionTranscriptRuntimeTarget(scope); - - expect(path.basename(readTarget.sessionFile)).toBe("session-2-topic-456.jsonl"); - expect(writeTarget.sessionFile).toBe(readTarget.sessionFile); - expect(loadSessionEntry(scope)?.sessionFile).toBe(readTarget.sessionFile); - }); - - it("persists transcript metadata under the normalized session key", async () => { - const canonicalScope = { - sessionId: "session-1", - sessionKey: "agent:main:main", + const target = resolveSessionTranscriptReadTarget({ + agentId: "main", + sessionId: "custom-topic-session", + sessionKey, storePath, - }; - - await upsertSessionEntry(canonicalScope, { - sessionId: canonicalScope.sessionId, - updatedAt: 10, }); - await appendTranscriptEvent( - { - agentId: "main", - sessionId: canonicalScope.sessionId, - sessionKey: "AGENT:MAIN:MAIN", - storePath, - }, - { id: "event-1", type: "metadata" }, - ); - expect(listSessionEntries({ storePath }).map((entry) => entry.sessionKey)).toEqual([ - canonicalScope.sessionKey, - ]); - expect(loadSessionEntry(canonicalScope)?.sessionFile).toBeTruthy(); + expect(target.sessionFile).toContain("sqlite:main:custom-topic-session:"); }); }); diff --git a/src/config/sessions/session-accessor.ts b/src/config/sessions/session-accessor.ts index 66c7d4e7a111..a338b66159be 100644 --- a/src/config/sessions/session-accessor.ts +++ b/src/config/sessions/session-accessor.ts @@ -1,70 +1,89 @@ -import { randomUUID } from "node:crypto"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; import { isDeepStrictEqual } from "node:util"; import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; -import { - acquireSessionWriteLock, - resolveSessionWriteLockOptions, -} from "../../agents/session-write-lock.js"; import type { MsgContext } from "../../auto-reply/templating.js"; import { resolveSessionStoreAgentId, resolveSessionStoreKey, } from "../../gateway/session-store-key.js"; import { formatErrorMessage } from "../../infra/errors.js"; -import { resolveRequiredHomeDir } from "../../infra/home-dir.js"; import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js"; import { emitSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; -import type { - SessionTranscriptUpdate, - SessionTranscriptUpdateTarget, -} from "../../sessions/transcript-events.js"; +import type { SessionTranscriptUpdate } from "../../sessions/transcript-events.js"; import { createLazyRuntimeModule } from "../../shared/lazy-runtime.js"; import type { DeliveryContext } from "../../utils/delivery-context.types.js"; +import { getRuntimeConfig } from "../io.js"; import type { OpenClawConfig } from "../types.openclaw.js"; import { formatSessionArchiveTimestamp } from "./artifacts.js"; -import { extractGeneratedTranscriptSessionId } from "./generated-transcript-session-id.js"; import { resolveAgentMainSessionKey } from "./main-session.js"; import { resolveSessionFilePath, resolveSessionFilePathOptions, - resolveSessionTranscriptPath, - resolveSessionTranscriptPathInDir, resolveStorePath, } from "./paths.js"; import { - cleanupPluginHostSessionStore as cleanupFilePluginHostSessionStore, + clearPluginHostCleanupTarget, clearPluginOwnedSessionState, + hasPluginHostCleanupTarget, + isLockedHarnessSessionOwnedByPlugin, + matchesPluginHostCleanupSession, + shouldSkipPluginHostCleanupStore, type PluginHostSessionCleanupStoreParams, } from "./plugin-host-cleanup.js"; -import { resolveAndPersistSessionFile } from "./session-file.js"; -import { resolveSessionStorePathForScope } from "./session-store-path.js"; +import { + appendSqliteTranscriptEvent, + appendSqliteTranscriptEventSync, + appendSqliteTranscriptMessage, + applySqliteSessionEntryLifecycleMutation, + applySqliteSessionEntryReplacements, + appendSqliteExpectedSessionTranscriptTurn, + cleanupSqliteSessionLifecycleArtifacts, + deleteSqliteSessionEntryLifecycle, + listSqliteSessionEntries, + appendSqliteTranscriptMessageSync, + findSqliteTranscriptEvent, + forkSqliteSessionEntryFromParentTarget, + forkSqliteSessionTranscriptFromParent, + recordSqliteInboundSessionMeta, + updateSqliteSessionLastRoute, + loadExactSqliteSessionEntry, + loadLatestSqliteAssistantText, + loadLatestSqliteAssistantMessage, + loadLatestSqliteMessage, + loadSqliteSessionEntry, + loadSqliteTranscriptEvents, + loadSqliteTranscriptEventsSync, + readSqliteTranscriptStatsSync, + patchSqliteSessionEntry, + patchSqliteSessionEntryTarget, + previewSqliteSessionDiskBudget, + publishSqliteTranscriptUpdate, + purgeSqliteDeletedAgentSessionEntries, + readSqliteSessionUpdatedAt, + replaceSqliteTranscriptEvents, + replaceSqliteTranscriptEventsSync, + replaceSqliteSessionEntry, + replaceSqliteSessionEntrySync, + resolveSqliteSessionKeyBySessionId, + resolveSqliteSessionParentForkDecision, + rollbackSqliteAgentHarnessSessionEntryLifecycle, + resetSqliteSessionEntryLifecycle, + updateSqliteSessionEntry, + upsertSqliteSessionEntry, + withSqliteTranscriptWriteLock, +} from "./session-accessor.sqlite.js"; +import { + formatSqliteSessionFileMarker, + parseSqliteSessionFileMarker, + sqliteSessionFileMarkerMatchesSession, +} from "./sqlite-marker.js"; +import { normalizeStoreSessionKey } from "./store-entry.js"; import type { ResolvedSessionMaintenanceConfig, SessionMaintenanceWarning, } from "./store-maintenance.js"; import { - getSessionEntry, - cleanupSessionLifecycleArtifacts as cleanupFileSessionLifecycleArtifacts, - deleteSessionEntryLifecycle as deleteFileSessionEntryLifecycle, - applySessionEntryLifecycleMutation as applyFileSessionEntryLifecycleMutation, - listSessionEntries as listFileSessionEntries, - loadSessionStore, - applySessionEntryPatchProjection as applyFileSessionEntryPatchProjection, - patchSessionEntry as patchFileSessionEntry, - patchSessionEntryWithKey as patchFileSessionEntryWithKey, - purgeDeletedAgentSessionEntries as purgeFileDeletedAgentSessionEntries, projectSessionEntryForPersistenceRevision, - readSessionUpdatedAt as readFileSessionUpdatedAt, - recordSessionMetaFromInbound as recordFileSessionMetaFromInbound, resolveSessionStoreEntry, - resetSessionEntryLifecycle as resetFileSessionEntryLifecycle, - rollbackAgentHarnessSessionEntryLifecycle as rollbackFileAgentHarnessSessionEntryLifecycle, - updateLastRoute as updateFileSessionLastRoute, - updateSessionStore, - updateSessionStoreEntry as updateFileSessionStoreEntry, type DeleteSessionEntryLifecycleResult, type ResetSessionEntryLifecycleMutation, type ResetSessionEntryLifecycleResult, @@ -84,38 +103,18 @@ import { type SessionLifecycleStoreTarget, } from "./store.js"; import { resolveAllAgentSessionStoreTargetsSync, type SessionStoreTarget } from "./targets.js"; -import { parseSessionThreadInfo } from "./thread-info.js"; -import { - type AppendSessionTranscriptMessageParams, - type AppendSessionTranscriptMessageResult, - appendSessionTranscriptEvent, - appendSessionTranscriptMessage, - appendSessionTranscriptMessageWithOwnedWriteLock, - withSessionTranscriptAppendQueue, -} from "./transcript-append.js"; -import { resolveSessionTranscriptFile } from "./transcript-file-resolve.js"; import { createSessionTranscriptHeader } from "./transcript-header.js"; -import { serializeJsonlLine, writeJsonlLines } from "./transcript-jsonl.js"; -import { replayRecentUserAssistantMessages } from "./transcript-replay.js"; import { - streamSessionTranscriptLines, - streamSessionTranscriptLinesReverse, -} from "./transcript-stream.js"; + readRecentUserAssistantReplayRecordsFromJsonl, + replayRecentUserAssistantMessages, + selectRecentUserAssistantReplayRecords, +} from "./transcript-replay.js"; import { scanSessionTranscriptTree, selectSessionTranscriptTreePathNodes, } from "./transcript-tree.js"; -import { - type OwnedSessionTranscriptPublishedEntry, - resolveOwnedSessionTranscriptWriteLockRunner, - withOwnedSessionTranscriptWrites, -} from "./transcript-write-context.js"; -import { - mergeSessionEntry, - type GroupKeyResolution, - type SessionCompactionCheckpoint, - type SessionEntry, -} from "./types.js"; +import { runWithOwnedSessionTranscriptWriteLock } from "./transcript-write-context.js"; +import type { GroupKeyResolution, SessionCompactionCheckpoint, SessionEntry } from "./types.js"; /** * Session access API for callers that need entries or transcripts without @@ -226,18 +225,18 @@ export type ResolvedSessionEntryUpdateResult<T> = }; export type SessionTranscriptAccessScope = Omit<SessionAccessScope, "sessionKey"> & { - /** Explicit transcript file path; bypasses store lookup when already known. */ + /** Deprecated transcript locator from older file-backed call sites. */ sessionFile?: string; - /** Runtime session id used to derive a transcript file when no explicit file is provided. */ + /** Runtime session id used to resolve the transcript identity. */ sessionId: string; - /** Required when resolving through session metadata; optional for explicit transcript artifacts. */ + /** Required when resolving through session metadata; optional for legacy locators. */ sessionKey?: string; /** Channel thread suffix used when deriving topic transcript paths. */ threadId?: string | number; }; export type SessionTranscriptRuntimeScope = SessionAccessScope & { - /** Resolved file-backed artifact for the current runtime target. */ + /** Deprecated transcript locator from older file-backed call sites. */ sessionFile?: string; sessionId: string; threadId?: string | number; @@ -259,7 +258,7 @@ export type SessionTranscriptReadTarget = Omit< }; export type SessionTranscriptWriteScope = Omit<SessionTranscriptAccessScope, "sessionId"> & { - /** Optional for appenders that can operate on an existing explicit transcript target. */ + /** Optional for appenders that resolve it from the session entry. */ sessionId?: string; }; @@ -277,20 +276,36 @@ export type SessionEntryReadView = { entries(): SessionEntrySummary[]; }; +/** Session entry read by the exact persisted session key, without alias resolution. */ +export type ExactSessionEntry = { + sessionKey: string; + entry: SessionEntry; +}; + /** Raw transcript record for non-message events; message records use appendTranscriptMessage. */ export type TranscriptEvent = unknown; +export type SessionTranscriptStats = { + eventCount: number; + maxSeq: number; + sizeBytes: number; +}; + export type TranscriptMessageAppendOptions<TMessage> = { /** Runtime config used for message redaction and transcript header metadata. */ config?: OpenClawConfig; /** Working directory recorded in a newly created transcript header. */ cwd?: string; /** How duplicate message idempotency keys are detected before append. */ - idempotencyLookup?: "scan" | "caller-checked"; + idempotencyLookup?: "scan" | "scan-assistant" | "caller-checked"; /** Provider/channel message payload to persist. */ message: TMessage; /** Testable timestamp override for the generated transcript entry. */ now?: number; + /** Existing transcript event id owned by a caller with its own session tree. */ + eventId?: string; + /** Existing parent id owned by a caller with its own session tree. */ + parentId?: string | null; /** Optional finalizer that runs after duplicate detection but before persistence. */ prepareMessageAfterIdempotencyCheck?: (message: TMessage) => TMessage | undefined; /** Allow append without parent-link migration for large legacy linear transcripts. */ @@ -306,16 +321,40 @@ export type TranscriptMessageAppendResult<TMessage> = { messageId: string; }; -/** Transcript update fields supplied by callers; sessionFile is resolved here. */ -export type TranscriptUpdatePayload = Omit<SessionTranscriptUpdate, "sessionFile">; +/** Transcript update fields supplied by callers; the target is resolved here. */ +export type TranscriptUpdatePayload = Partial<SessionTranscriptUpdate>; + +export type LatestTranscriptAssistantText = { + id?: string; + text: string; + timestamp?: number; +}; + +export type LatestTranscriptAssistantMessage = { + id?: string; + message: unknown; +}; + +export type LatestTranscriptMessage = { + id?: string; + message: unknown; +}; + +export type SessionTranscriptWriteLockAccessorContext = { + appendMessage: <TMessage>( + options: TranscriptMessageAppendOptions<TMessage>, + ) => Promise<TranscriptMessageAppendResult<TMessage> | undefined>; + readEvents: () => Promise<TranscriptEvent[]>; + replaceEvents: (events: readonly TranscriptEvent[]) => Promise<void>; +}; export type SessionTranscriptTurnUpdateMode = "inline" | "file-only" | "none"; export type SessionTranscriptTurnMessageAppend = TranscriptMessageAppendOptions<unknown> & { /** - * Runs inside the file-backed write lock before this message is appended. - * SQLite implementation note: duplicate/skip decisions should be evaluated - * inside the same write transaction as the transcript row append. + * Runs inside the session writer queue before the SQLite transaction begins. + * The commit phase revalidates session ownership and database idempotency + * after asynchronous predicate work finishes. */ shouldAppend?: (context: SessionTranscriptTurnWriteContext) => Promise<boolean> | boolean; }; @@ -325,6 +364,7 @@ export type SessionTranscriptTurnWriteContext = { sessionFile: string; sessionId?: string; sessionKey?: string; + storePath?: string; }; export type SessionTranscriptTurnPersistOptions = { @@ -334,8 +374,8 @@ export type SessionTranscriptTurnPersistOptions = { cwd?: string; /** * Rejects the turn when the persisted session key no longer points at this - * runtime session id. SQLite implementations must evaluate this guard inside - * the same write transaction as the transcript append and metadata touch. + * runtime session id. SQLite evaluates this guard inside the same queued + * write as the transcript append and metadata touch. */ expectedSessionId?: string; /** Rejects the turn when lifecycle ownership changed without rotating the session id. */ @@ -363,10 +403,6 @@ export type SessionTranscriptTurnPersistResult = { sessionFile: string; }; -type SessionTranscriptTurnAppendRunner = <TMessage>( - params: AppendSessionTranscriptMessageParams<TMessage>, -) => Promise<AppendSessionTranscriptMessageResult<TMessage> | undefined>; - export type SessionTranscriptRuntimeTarget = { agentId: string; sessionFile: string; @@ -463,13 +499,6 @@ const loadSessionArchiveRuntime = createLazyRuntimeModule( () => import("../../gateway/session-archive.runtime.js"), ); -// Fork-source reading parses legacy transcript versions through the agents -// session-manager; load it lazily so accessor consumers do not pull that -// runtime (and its module-init package metadata reads) at import time. -const loadSessionForkTranscriptRuntime = createLazyRuntimeModule( - () => import("./session-fork-transcript.runtime.js"), -); - export type SessionEntryPatchOptions = { /** Entry to synthesize when a patch operation is allowed to create. */ fallbackEntry?: SessionEntry; @@ -499,6 +528,14 @@ export type SessionEntryPatchResult = { entry: SessionEntry; }; +export type SessionEntryTargetPatchScope = { + /** Agent owner used when resolving custom/shared legacy store paths. */ + agentId?: string; + storePath: string; + /** Canonical key plus aliases that identify the logical entry. */ + target: SessionLifecycleStoreTarget; +}; + export type SessionEntryReplacementSnapshot = { /** Exact persisted key for the candidate row. */ sessionKey: string; @@ -533,6 +570,87 @@ export type SessionCompactionCheckpointTranscriptForkResult = | { status: "missing-boundary" } | { status: "failed" }; +/** Decision made before inheriting parent context into a child session. */ +export type SessionParentForkDecision = + | { + status: "fork"; + maxTokens: number; + parentTokens?: number; + } + | { + status: "skip"; + reason: "parent-too-large"; + maxTokens: number; + parentTokens: number; + message: string; + }; + +/** SQLite transcript identity created for a child fork. */ +export type ParentForkedSessionTranscript = { + sessionFile: string; + sessionId: string; +}; + +export type ForkSessionFromParentTranscriptResult = + | { + status: "created"; + transcript: ParentForkedSessionTranscript; + } + | { status: "missing-parent" } + | { status: "failed" }; + +export type ForkSessionFromParentTranscriptParams = { + agentId?: string; + parentEntry: SessionEntry; + parentSessionKey: string; + sessionKey: string; + storePath: string; + /** Stable target identity for lifecycle-owned hidden or resumable sessions. */ + targetSessionId?: string; + /** Cross-agent forks land the child transcript in the target agent's store. */ + targetStorePath?: string; +}; + +export type ForkSessionEntryFromParentTargetResult = + | { + status: "forked"; + fork: ParentForkedSessionTranscript; + parentEntry: SessionEntry; + sessionEntry: SessionEntry; + decision: Extract<SessionParentForkDecision, { status: "fork" }>; + } + | { + status: "skipped"; + reason: "existing-entry" | "decision-skip"; + parentEntry?: SessionEntry; + sessionEntry: SessionEntry; + decision?: SessionParentForkDecision; + } + | { status: "missing-entry" } + | { status: "missing-parent" } + | { status: "failed" }; + +export type ForkSessionEntryFromParentTargetParams = { + agentId?: string; + decisionSkipPatch?: (params: { + decision: Extract<SessionParentForkDecision, { status: "skip" }>; + entry: SessionEntry; + parentEntry: SessionEntry; + }) => Partial<SessionEntry> | null; + fallbackEntry?: SessionEntry; + parentTarget: SessionLifecycleStoreTarget; + patch?: (params: { + entry: SessionEntry; + parentEntry: SessionEntry; + fork: ParentForkedSessionTranscript; + decision: Extract<SessionParentForkDecision, { status: "fork" }>; + }) => Partial<SessionEntry>; + sessionTarget: SessionLifecycleStoreTarget; + skipForkWhen?: (entry: SessionEntry) => boolean; + skipPatch?: (entry: SessionEntry) => Partial<SessionEntry> | null; + storePath: string; +}; + /** Result of applying a checkpoint branch or restore mutation to session storage. */ export type SessionCompactionCheckpointMutationResult = | { @@ -542,9 +660,9 @@ export type SessionCompactionCheckpointMutationResult = entry: SessionEntry; } | { status: "missing-session" } - | { status: "model-selection-locked" } | { status: "missing-checkpoint" } | { status: "missing-boundary" } + | { status: "model-selection-locked" } | { status: "failed" }; export type SessionCompactionCheckpointEntryBuildContext = { @@ -653,16 +771,12 @@ export type SessionEntryCreateWithTranscriptPrepareResult<TError = string> = | { ok: false; error: TError }; export type SessionEntryCreateWithTranscriptOptions = { - /** Protect the newly created row from maintenance during its initial save. */ + /** Protect the newly created row from maintenance during its initial write. */ activeSessionKey?: string; - /** Throw unless the initial session row is durably persisted. */ + /** SQLite commits are authoritative; retained for the shared caller contract. */ requireWriteSuccess?: boolean; }; -type CreatedSessionTranscriptResult = - | { ok: true; created: boolean; sessionFile: string } - | { ok: false; error: string; phase: "transcript" }; - export type SessionPatchProjectionContext = SessionEntryPatchProjectionContext; export type SessionPatchProjectionFailure = SessionEntryPatchProjectionFailure; export type SessionPatchProjectionResult<TFailure extends SessionPatchProjectionFailure> = @@ -688,7 +802,7 @@ export type { }; export type ResetSessionEntryLifecycleParams = { - /** Runs after the persisted entry rotates and before transcript artifacts move. */ + /** Runs after the persisted entry rotates and retired transcripts are archived. */ afterEntryMutation?: (mutation: ResetSessionEntryLifecycleMutation) => Promise<void> | void; /** Agent owner used to resolve backend transcript artifacts. */ agentId?: string; @@ -827,7 +941,12 @@ export function resolveSessionEntryCandidateTarget( agentId: scope.agentId, env: scope.env, }); - const store = loadSessionStore(storePath); + const store = Object.fromEntries( + listSessionEntries({ agentId: scope.agentId, storePath }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), + ); for (const candidateKey of uniqueStrings(scope.candidateKeys.map((key) => key.trim()))) { if (!candidateKey) { continue; @@ -880,7 +999,7 @@ function resolveSessionEntryStoreTarget( }; let selectedStorePath = fallback.storePath; let selectedMatch = findFreshestSessionEntryMatch( - listSessionEntries({ storePath: fallback.storePath }), + listSessionEntries({ agentId, storePath: fallback.storePath }), scanTargets, ); for (let index = 1; index < candidates.length; index += 1) { @@ -889,7 +1008,7 @@ function resolveSessionEntryStoreTarget( continue; } const match = findFreshestSessionEntryMatch( - listSessionEntries({ storePath: candidate.storePath }), + listSessionEntries({ agentId, storePath: candidate.storePath }), scanTargets, ); if ( @@ -922,40 +1041,49 @@ export async function updateResolvedSessionEntry<T>( if (!target.entry) { return { canonicalKey: target.canonicalKey, found: false }; } - return await updateSessionStore(target.storePath, async (store) => { - const entry = store[target.storeKey]; - if (!entry) { - return { canonicalKey: target.canonicalKey, found: false }; - } - const context: ResolvedSessionEntryUpdateContext = { - agentId: target.agentId, - canonicalKey: target.canonicalKey, - entry, - requestedKey: target.requestedKey, - storeKey: target.storeKey, - }; - const result = await update(entry, context); - return { - canonicalKey: target.canonicalKey, - entry: structuredClone(entry), - found: true, - result, - storeKey: target.storeKey, - }; - }); + let updateResult: T | undefined; + const updated = await patchSessionEntry( + { sessionKey: target.storeKey, storePath: target.storePath }, + async (entry) => { + const context: ResolvedSessionEntryUpdateContext = { + agentId: target.agentId, + canonicalKey: target.canonicalKey, + entry, + requestedKey: target.requestedKey, + storeKey: target.storeKey, + }; + updateResult = await update(entry, context); + return entry; + }, + { + replaceEntry: true, + skipMaintenance: true, + }, + ); + if (!updated) { + return { canonicalKey: target.canonicalKey, found: false }; + } + return { + canonicalKey: target.canonicalKey, + entry: structuredClone(updated), + found: true, + result: updateResult as T, + storeKey: target.storeKey, + }; } /** Returns the entry for a canonical or alias session key, if one exists. */ export function loadSessionEntry(scope: SessionAccessScope): SessionEntry | undefined { - if (scope.clone === false || scope.readConsistency === "latest") { - const store = loadSessionStore(resolveSessionStorePathForScope(scope), { - ...(scope.clone === false ? { clone: false } : {}), - ...(scope.readConsistency === "latest" ? { skipCache: true } : {}), - ...(scope.hydrateSkillPromptRefs === false ? { hydrateSkillPromptRefs: false } : {}), - }); - return resolveSessionStoreEntry({ store, sessionKey: scope.sessionKey }).existing; - } - return getSessionEntry(scope); + return loadSqliteSessionEntry(scope); +} + +/** + * Returns only the row persisted under the exact key provided. + * Use this for authorization-sensitive routing where alias canonicalization + * could cross an account or agent boundary. + */ +export function loadExactSessionEntry(scope: SessionAccessScope): ExactSessionEntry | undefined { + return loadExactSqliteSessionEntry(scope); } /** Lists entries from the resolved store, preserving the persisted key for each row. */ @@ -963,7 +1091,7 @@ export function listSessionEntries(scope: SessionEntryListScope = {}): SessionEn if (scope.clone === false) { return openSessionEntryReadView(scope).entries(); } - return listFileSessionEntries(scope); + return listSqliteSessionEntries(scope); } /** @@ -976,25 +1104,17 @@ export function listSessionEntries(scope: SessionEntryListScope = {}): SessionEn export function openSessionEntryReadView( scope: Omit<SessionEntryListScope, "clone" | "readConsistency"> = {}, ): SessionEntryReadView { - const store = loadSessionStore(resolveSessionStorePathForScope({ ...scope, sessionKey: "" }), { - clone: false, - ...(scope.hydrateSkillPromptRefs === false ? { hydrateSkillPromptRefs: false } : {}), - }); + // Exact-key probes read single SQLite rows; entries() materializes the full + // list only when raw probes cannot settle the caller's lookup. return { - get: (sessionKey) => (Object.hasOwn(store, sessionKey) ? store[sessionKey] : undefined), - entries: () => Object.entries(store).map(([sessionKey, entry]) => ({ sessionKey, entry })), + get: (sessionKey) => loadExactSqliteSessionEntry({ ...scope, sessionKey })?.entry, + entries: () => listSqliteSessionEntries(scope), }; } /** Reads the last activity timestamp for one session entry, or undefined when absent. */ export function readSessionUpdatedAt(scope: SessionAccessScope): number | undefined { - if (scope.storePath) { - return readFileSessionUpdatedAt({ - storePath: scope.storePath, - sessionKey: scope.sessionKey, - }); - } - return loadSessionEntry(scope)?.updatedAt; + return readSqliteSessionUpdatedAt(scope); } /** Creates or updates one entry from a partial patch and returns the persisted entry. */ @@ -1002,11 +1122,7 @@ export async function upsertSessionEntry( scope: SessionAccessScope, patch: Partial<SessionEntry>, ): Promise<SessionEntry | null> { - return await patchFileSessionEntry({ - ...scope, - fallbackEntry: createFallbackSessionEntry(patch), - update: () => patch, - }); + return await upsertSqliteSessionEntry(scope, patch); } /** Replaces one entry with the supplied value and returns the persisted entry. */ @@ -1014,12 +1130,12 @@ export async function replaceSessionEntry( scope: SessionAccessScope, entry: SessionEntry, ): Promise<SessionEntry | null> { - return await patchFileSessionEntry({ - ...scope, - fallbackEntry: entry, - replaceEntry: true, - update: () => entry, - }); + return await replaceSqliteSessionEntry(scope, entry); +} + +/** Replaces one entry synchronously for sync session runtimes. */ +export function replaceSessionEntrySync(scope: SessionAccessScope, entry: SessionEntry): void { + replaceSqliteSessionEntrySync(scope, entry); } /** @@ -1035,17 +1151,22 @@ export async function patchSessionEntry( ) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null, options: SessionEntryPatchOptions = {}, ): Promise<SessionEntry | null> { - return await patchFileSessionEntry({ - ...scope, - fallbackEntry: options.fallbackEntry, - maintenanceConfig: options.maintenanceConfig, - preserveActivity: options.preserveActivity, - requireWriteSuccess: options.requireWriteSuccess, - replaceEntry: options.replaceEntry, - skipMaintenance: options.skipMaintenance, - takeCacheOwnership: options.takeCacheOwnership, - update, - }); + return await patchSqliteSessionEntry(scope, update, options); +} + +/** + * Applies an atomic patch to the freshest entry selected from a canonical key + * plus its known aliases, then persists the result under the canonical key. + */ +export async function patchSessionEntryTarget( + scope: SessionEntryTargetPatchScope, + update: ( + entry: SessionEntry, + context: SessionEntryPatchContext, + ) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null, + options: SessionEntryPatchOptions = {}, +): Promise<SessionEntry | null> { + return await patchSqliteSessionEntryTarget(scope, update, options); } /** @@ -1060,17 +1181,36 @@ export async function patchSessionEntryWithKey( ) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null, options: SessionEntryPatchOptions = {}, ): Promise<SessionEntryPatchResult | null> { - return await patchFileSessionEntryWithKey({ - ...scope, - fallbackEntry: options.fallbackEntry, - maintenanceConfig: options.maintenanceConfig, - preserveActivity: options.preserveActivity, - requireWriteSuccess: options.requireWriteSuccess, - replaceEntry: options.replaceEntry, - skipMaintenance: options.skipMaintenance, - takeCacheOwnership: options.takeCacheOwnership, - update, - }); + const entry = await patchSqliteSessionEntry(scope, update, options); + return entry ? { sessionKey: normalizeStoreSessionKey(scope.sessionKey), entry } : null; +} + +/** + * Copies one parent transcript into a new child transcript target. + * This is for guarded callers that already own the eventual entry commit. + */ +export async function forkSessionFromParentTranscript( + params: ForkSessionFromParentTranscriptParams, +): Promise<ForkSessionFromParentTranscriptResult> { + return await forkSqliteSessionTranscriptFromParent(params); +} + +/** + * Forks parent transcript content and persists the child entry/alias cleanup in + * one storage-owned operation. + */ +export async function forkSessionEntryFromParentTarget( + params: ForkSessionEntryFromParentTargetParams, +): Promise<ForkSessionEntryFromParentTargetResult> { + return await forkSqliteSessionEntryFromParentTarget(params); +} + +/** Resolves whether a parent session is small enough to fork through the active store. */ +export async function resolveSessionParentForkDecision(params: { + parentEntry: SessionEntry; + storePath: string; +}): Promise<SessionParentForkDecision> { + return await resolveSqliteSessionParentForkDecision(params); } /** @@ -1078,42 +1218,40 @@ export async function patchSessionEntryWithKey( * and optionally patches the canonical entry under one accessor operation. */ export async function canonicalizeSessionEntryAliases(params: { + agentId?: string; storePath: string; target: SessionLifecycleStoreTarget; update?: ( entry: SessionEntry | undefined, ) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null; }): Promise<CanonicalizeSessionEntryAliasesResult> { - return await updateSessionStore(params.storePath, async (store) => { - const targetKeys = normalizeTargetStoreKeys(params.target); - const freshest = resolveFreshestTargetEntry(store, targetKeys); - if (freshest) { - const current = store[params.target.canonicalKey]; - if (!current || (freshest.entry.updatedAt ?? 0) > (current.updatedAt ?? 0)) { - store[params.target.canonicalKey] = freshest.entry; - } - } - - const currentEntry = store[params.target.canonicalKey]; - const patch = params.update ? await params.update(cloneOptionalEntry(currentEntry)) : null; - if (patch) { - store[params.target.canonicalKey] = { - ...currentEntry, + const store = Object.fromEntries( + listSessionEntries({ agentId: params.agentId, storePath: params.storePath }).map( + ({ sessionKey, entry }) => [sessionKey, entry], + ), + ); + const targetKeys = normalizeTargetStoreKeys(params.target); + const freshest = resolveFreshestTargetEntry(store, targetKeys); + const patch = params.update ? await params.update(cloneOptionalEntry(freshest?.entry)) : null; + const entry = patch + ? ({ + ...freshest?.entry, ...patch, - } as SessionEntry; - } - - for (const key of targetKeys) { - if (key !== params.target.canonicalKey) { - delete store[key]; - } - } - const entry = cloneOptionalEntry(store[params.target.canonicalKey]); - return { - canonicalKey: params.target.canonicalKey, - ...(entry ? { entry } : {}), - }; + } as SessionEntry) + : cloneOptionalEntry(freshest?.entry); + await applySessionEntryLifecycleMutation({ + agentId: params.agentId, + storePath: params.storePath, + removals: targetKeys + .filter((key) => key !== params.target.canonicalKey) + .map((sessionKey) => ({ sessionKey })), + upserts: entry ? [{ sessionKey: params.target.canonicalKey, entry }] : undefined, + skipMaintenance: true, }); + return { + canonicalKey: params.target.canonicalKey, + ...(entry ? { entry: cloneOptionalEntry(entry) } : {}), + }; } // Normalizes caller-supplied alias sets while always preserving the canonical key. @@ -1156,9 +1294,8 @@ function cloneOptionalEntry(entry: SessionEntry | undefined): SessionEntry | und /** * Creates or updates one session entry and initializes its transcript header as - * one storage-sized lifecycle operation. File-backed storage still writes JSON - * plus JSONL, but callers no longer compose entry write, header creation, - * rollback, and normalized sessionFile persistence themselves. + * one SQLite-backed lifecycle operation. Callers do not compose row creation, + * transcript initialization, rollback, and normalized session identity. */ export async function createSessionEntryWithTranscript<TError = string>( scope: SessionAccessScope, @@ -1167,71 +1304,60 @@ export async function createSessionEntryWithTranscript<TError = string>( ) => | Promise<SessionEntryCreateWithTranscriptPrepareResult<TError>> | SessionEntryCreateWithTranscriptPrepareResult<TError>, - options: SessionEntryCreateWithTranscriptOptions = {}, + _options: SessionEntryCreateWithTranscriptOptions = {}, ): Promise<SessionEntryCreateWithTranscriptResult<TError>> { - const storePath = resolveSessionStorePathForScope(scope); - let createdTranscriptFile: string | undefined; - try { - return await updateSessionStore( - storePath, - async (store) => { - const resolved = resolveSessionStoreEntry({ store, sessionKey: scope.sessionKey }); - const created = await createEntry({ - existingEntry: resolved.existing ? { ...resolved.existing } : undefined, - sessionEntries: cloneSessionEntries(store), - }); - if (!created.ok) { - return { ok: false, error: created.error, phase: "entry" }; - } - - const ensured = ensureCreatedSessionTranscript({ - agentId: scope.agentId, - entry: created.entry, - storePath, - }); - if (!ensured.ok) { - delete store[resolved.normalizedKey]; - return ensured; - } - if (ensured.created) { - createdTranscriptFile = ensured.sessionFile; - } - - const entry = - created.entry.sessionFile === ensured.sessionFile - ? created.entry - : { - ...created.entry, - sessionFile: ensured.sessionFile, - }; - store[resolved.normalizedKey] = entry; - for (const legacyKey of resolved.legacyKeys) { - delete store[legacyKey]; - } - return { ok: true, entry, sessionFile: ensured.sessionFile }; - }, - { - activeSessionKey: options.activeSessionKey, - requireWriteSuccess: options.requireWriteSuccess, - // Entry rejection is side-effect free; transcript failure deletes the target row - // and must persist that rollback instead of restoring the writer snapshot. - skipSaveWhenResult: (result) => !result.ok && result.phase === "entry", - }, - ); - } catch (error) { - if (!createdTranscriptFile) { - throw error; - } - try { - fs.rmSync(createdTranscriptFile, { force: true }); - } catch (cleanupError) { - throw new Error( - `Session entry persistence failed (${formatErrorMessage(error)}) and transcript cleanup did not complete: ${createdTranscriptFile}`, - { cause: cleanupError }, - ); - } - throw error; + const storePath = resolveAccessStorePath(scope); + const agentId = scope.agentId ?? resolveAgentIdFromSessionKey(scope.sessionKey); + const store = Object.fromEntries( + listSessionEntries({ agentId, storePath }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ); + const resolved = resolveSessionStoreEntry({ store, sessionKey: scope.sessionKey }); + const created = await createEntry({ + existingEntry: resolved.existing ? { ...resolved.existing } : undefined, + sessionEntries: cloneSessionEntries(store), + }); + if (!created.ok) { + return { ok: false, error: created.error, phase: "entry" }; } + + const sessionFile = formatSqliteSessionFileMarker({ + agentId, + sessionId: created.entry.sessionId, + storePath, + }); + try { + await appendSqliteTranscriptEvent( + { + agentId, + sessionId: created.entry.sessionId, + sessionKey: resolved.normalizedKey, + storePath, + }, + createSessionTranscriptHeader({ sessionId: created.entry.sessionId }), + ); + } catch (err) { + return { + ok: false, + error: formatErrorMessage(err), + phase: "transcript", + }; + } + + const entry = + created.entry.sessionFile === sessionFile + ? created.entry + : { + ...created.entry, + sessionFile, + }; + await applySessionEntryLifecycleMutation({ + agentId, + storePath, + removals: resolved.legacyKeys.map((sessionKey) => ({ sessionKey })), + upserts: [{ sessionKey: resolved.normalizedKey, entry }], + skipMaintenance: true, + }); + return { ok: true, entry, sessionFile }; } function cloneSessionEntries(store: Record<string, SessionEntry>): Record<string, SessionEntry> { @@ -1352,25 +1478,13 @@ function createReplySessionInitializationRevision(params: { function resolveInitializedReplySessionEntry(params: { agentId: string; currentEntry?: SessionEntry; - fallbackSessionFile?: string; sessionEntry: SessionEntry; storePath: string; }): SessionEntry { - const fallbackSessionFile = params.fallbackSessionFile?.trim(); - const currentSessionFile = params.currentEntry?.sessionFile; - const inheritedPreviousSessionFile = - Boolean(currentSessionFile) && - params.currentEntry?.sessionId !== params.sessionEntry.sessionId && - currentSessionFile === params.sessionEntry.sessionFile; - const entryForResolve = - fallbackSessionFile && (inheritedPreviousSessionFile || !params.sessionEntry.sessionFile) - ? { ...params.sessionEntry, sessionFile: fallbackSessionFile } - : inheritedPreviousSessionFile - ? { ...params.sessionEntry, sessionFile: undefined } - : params.sessionEntry; - const sessionFile = resolveSessionFilePath(params.sessionEntry.sessionId, entryForResolve, { + const sessionFile = formatSqliteSessionFileMarker({ agentId: params.agentId, - sessionsDir: path.dirname(path.resolve(params.storePath)), + sessionId: params.sessionEntry.sessionId, + storePath: params.storePath, }); return { ...params.sessionEntry, @@ -1378,361 +1492,6 @@ function resolveInitializedReplySessionEntry(params: { }; } -// File-backed creation resolves the concrete transcript artifact and writes the -// header before the store mutation is saved; SQLite adapters implement this as -// the same lifecycle operation without exposing rollback details to callers. -function ensureCreatedSessionTranscript(params: { - agentId?: string; - entry: SessionEntry; - storePath: string; -}): CreatedSessionTranscriptResult { - try { - const sessionFile = resolveSessionFilePath( - params.entry.sessionId, - params.entry.sessionFile ? { sessionFile: params.entry.sessionFile } : undefined, - { - agentId: params.agentId, - sessionsDir: path.dirname(path.resolve(params.storePath)), - }, - ); - const created = !fs.existsSync(sessionFile); - if (created) { - fs.mkdirSync(path.dirname(sessionFile), { recursive: true }); - fs.writeFileSync( - sessionFile, - `${JSON.stringify(createSessionTranscriptHeader({ sessionId: params.entry.sessionId }))}\n`, - { - encoding: "utf-8", - mode: 0o600, - }, - ); - } - return { ok: true, created, sessionFile }; - } catch (err) { - return { - ok: false, - error: formatErrorMessage(err), - phase: "transcript", - }; - } -} - -/** Fork target identity generated by createForkedSessionTranscript. */ -export type ForkedSessionTranscriptTarget = { - sessionFile: string; - sessionId: string; -}; - -/** Decision made before inheriting parent context into a child session. */ -export type SessionParentForkDecision = - | { - status: "fork"; - maxTokens: number; - parentTokens?: number; - } - | { - status: "skip"; - reason: "parent-too-large"; - maxTokens: number; - parentTokens: number; - message: string; - }; - -/** Transcript identity created for a child fork. */ -export type ParentForkedSessionTranscript = { - sessionFile: string; - sessionId: string; -}; - -export type ForkSessionFromParentTranscriptResult = - | { - status: "created"; - transcript: ParentForkedSessionTranscript; - } - | { status: "missing-parent" } - | { status: "failed" }; - -export type ForkSessionFromParentTranscriptParams = { - agentId?: string; - parentEntry: SessionEntry; - parentSessionKey: string; - sessionKey: string; - storePath: string; - /** Cross-agent forks land the child transcript beside the child's store. */ - targetStorePath?: string; -}; - -export type ForkSessionEntryFromParentTargetResult = - | { - status: "forked"; - fork: ParentForkedSessionTranscript; - parentEntry: SessionEntry; - sessionEntry: SessionEntry; - decision: Extract<SessionParentForkDecision, { status: "fork" }>; - } - | { - status: "skipped"; - reason: "existing-entry" | "decision-skip"; - parentEntry?: SessionEntry; - sessionEntry: SessionEntry; - decision?: SessionParentForkDecision; - } - | { status: "missing-entry" } - | { status: "missing-parent" } - | { status: "failed" }; - -export type ForkSessionEntryFromParentTargetParams = { - agentId?: string; - decisionSkipPatch?: (params: { - decision: Extract<SessionParentForkDecision, { status: "skip" }>; - entry: SessionEntry; - parentEntry: SessionEntry; - }) => Partial<SessionEntry> | null; - fallbackEntry?: SessionEntry; - parentTarget: SessionLifecycleStoreTarget; - patch?: (params: { - entry: SessionEntry; - parentEntry: SessionEntry; - fork: ParentForkedSessionTranscript; - decision: Extract<SessionParentForkDecision, { status: "fork" }>; - }) => Partial<SessionEntry>; - /** - * File-era seam: token counting for the fork decision still reads transcript - * tails through gateway helpers, so the caller supplies the decision. The - * SQLite flip resolves the decision inside the storage boundary instead. - */ - resolveDecision: (parentEntry: SessionEntry) => Promise<SessionParentForkDecision>; - sessionTarget: SessionLifecycleStoreTarget; - skipForkWhen?: (entry: SessionEntry) => boolean; - skipPatch?: (entry: SessionEntry) => Partial<SessionEntry> | null; - storePath: string; -}; - -export type CreateForkedSessionTranscriptParams = { - /** Working directory recorded in the forked transcript header. */ - cwd: string; - /** Source transcript recorded as the fork's parentSession lineage. */ - parentSessionFile: string; - /** Directory that owns the forked transcript artifact. */ - sessionsDir: string; - /** - * Builds the non-header records copied into the fork, in transcript order. - * Receives the generated fork identity so records can share its timestamp. - */ - buildEntries?: (fork: { sessionId: string; timestamp: string }) => readonly unknown[]; -}; - -/** - * Creates a forked transcript artifact under a fresh session id as one storage - * operation. File-backed storage writes header plus copied records as a new - * JSONL artifact; SQLite implements the same fork as transcript row copies - * inside one write transaction (#88838). - */ -export async function createForkedSessionTranscript( - params: CreateForkedSessionTranscriptParams, -): Promise<ForkedSessionTranscriptTarget> { - const sessionId = randomUUID(); - const timestamp = new Date().toISOString(); - // Fork artifacts keep the timestamp-prefixed file name so sibling transcripts - // in one sessions dir sort by creation time. - const fileTimestamp = timestamp.replace(/[:.]/g, "-"); - const sessionFile = path.join(params.sessionsDir, `${fileTimestamp}_${sessionId}.jsonl`); - const header = createSessionTranscriptHeader({ - sessionId, - cwd: params.cwd, - parentSession: params.parentSessionFile, - timestamp, - }); - const entries = params.buildEntries?.({ sessionId, timestamp }) ?? []; - fs.mkdirSync(path.dirname(sessionFile), { recursive: true }); - // "wx" keeps create-only semantics: a fork must never clobber an existing transcript. - await writeJsonlLines(sessionFile, [header, ...entries].map(serializeJsonlLine), { - flag: "wx", - mode: 0o600, - }); - return { sessionFile, sessionId }; -} - -/** - * Forks the active branch of a parent transcript into a fresh child transcript. - * This is for guarded callers that already own the eventual entry commit. - */ -export async function forkSessionFromParentTranscript( - params: ForkSessionFromParentTranscriptParams, -): Promise<ForkSessionFromParentTranscriptResult> { - let parentSessionFile: string; - try { - parentSessionFile = resolveSessionFilePath( - params.parentEntry.sessionId, - params.parentEntry, - resolveSessionFilePathOptions({ - ...(params.agentId ? { agentId: params.agentId } : {}), - storePath: params.storePath, - }), - ); - } catch { - return { status: "missing-parent" }; - } - if (!parentSessionFile) { - return { status: "missing-parent" }; - } - try { - const { buildForkedBranchEntries, forkSourceHasAssistantEntry, readForkSourceTranscript } = - await loadSessionForkTranscriptRuntime(); - const source = await readForkSourceTranscript(parentSessionFile); - if (!source) { - return { status: "failed" }; - } - const shouldPersistBranch = - source.preserveLeafControl || forkSourceHasAssistantEntry(source.branchEntries); - // Cross-agent forks land beside the child's store; same-store forks stay - // beside the parent transcript so sibling artifacts sort together. - const targetSessionsDir = params.targetStorePath - ? path.dirname(params.targetStorePath) - : source.sessionDir; - const transcript = shouldPersistBranch - ? await createForkedSessionTranscript({ - cwd: source.cwd, - parentSessionFile, - sessionsDir: targetSessionsDir, - buildEntries: ({ timestamp }) => buildForkedBranchEntries({ source, timestamp }), - }) - : // Header-only fork: nothing on the active branch is worth copying. - await createForkedSessionTranscript({ - cwd: source.cwd, - parentSessionFile, - sessionsDir: targetSessionsDir, - }); - return { status: "created", transcript }; - } catch { - return { status: "failed" }; - } -} - -function resolveEntryFromStoreKeys(params: { - store: Record<string, SessionEntry>; - keys: readonly string[]; -}): SessionEntry | undefined { - // Alias migration can leave multiple logical rows. Fork policy must inspect - // the freshest row or a stale canonical entry can shadow a newer lock. - return resolveFreshestTargetEntry(params.store, params.keys)?.entry; -} - -function persistForkedSessionEntry(params: { - store: Record<string, SessionEntry>; - target: SessionLifecycleStoreTarget; - existing: SessionEntry; - patch: Partial<SessionEntry>; -}): SessionEntry { - const next = mergeSessionEntry(params.existing, params.patch); - params.store[params.target.canonicalKey] = next; - // Alias rows collapse onto the canonical key so the forked identity has one owner row. - for (const key of params.target.storeKeys) { - if (key !== params.target.canonicalKey) { - delete params.store[key]; - } - } - return next; -} - -/** - * Forks parent transcript content and persists the child entry/alias cleanup in - * one storage-owned operation. - */ -export async function forkSessionEntryFromParentTarget( - params: ForkSessionEntryFromParentTargetParams, -): Promise<ForkSessionEntryFromParentTargetResult> { - return await updateSessionStore( - params.storePath, - async (store) => { - const parentEntry = resolveEntryFromStoreKeys({ - store, - keys: params.parentTarget.storeKeys, - }); - if (!parentEntry?.sessionId) { - return { status: "missing-parent" }; - } - - const entry = - resolveEntryFromStoreKeys({ store, keys: params.sessionTarget.storeKeys }) ?? - params.fallbackEntry; - if (!entry) { - return { status: "missing-entry" }; - } - - if (params.skipForkWhen?.(entry)) { - const patch = params.skipPatch?.(entry); - const sessionEntry = patch - ? persistForkedSessionEntry({ - store, - target: params.sessionTarget, - existing: entry, - patch, - }) - : entry; - return { status: "skipped", reason: "existing-entry", parentEntry, sessionEntry }; - } - - const decision = await params.resolveDecision(parentEntry); - if (decision.status === "skip") { - const patch = params.decisionSkipPatch?.({ decision, entry, parentEntry }); - const sessionEntry = patch - ? persistForkedSessionEntry({ - store, - target: params.sessionTarget, - existing: entry, - patch, - }) - : entry; - return { - status: "skipped", - reason: "decision-skip", - parentEntry, - sessionEntry, - decision, - }; - } - - const forked = await forkSessionFromParentTranscript({ - ...(params.agentId ? { agentId: params.agentId } : {}), - parentEntry, - parentSessionKey: params.parentTarget.canonicalKey, - sessionKey: params.sessionTarget.canonicalKey, - storePath: params.storePath, - }); - if (forked.status !== "created") { - return { status: "failed" }; - } - const fork = forked.transcript; - const sessionEntry = persistForkedSessionEntry({ - store, - target: params.sessionTarget, - existing: entry, - patch: { - ...params.patch?.({ entry, parentEntry, fork, decision }), - sessionId: fork.sessionId, - sessionFile: fork.sessionFile, - forkedFromParent: true, - }, - }); - return { - status: "forked", - fork, - parentEntry, - sessionEntry, - decision, - }; - }, - { - skipSaveWhenResult: (result) => - result.status === "missing-entry" || - result.status === "missing-parent" || - result.status === "failed" || - (result.status === "skipped" && result.sessionEntry === params.fallbackEntry), - }, - ); -} - /** Updates an existing entry only; returns null when the session is absent. */ export async function updateSessionEntry( scope: SessionAccessScope, @@ -1741,14 +1500,7 @@ export async function updateSessionEntry( ) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null, options: SessionEntryUpdateOptions = {}, ): Promise<SessionEntry | null> { - return await updateFileSessionStoreEntry({ - storePath: resolveSessionStorePathForScope(scope), - sessionKey: scope.sessionKey, - skipMaintenance: options.skipMaintenance, - takeCacheOwnership: options.takeCacheOwnership, - requireWriteSuccess: options.requireWriteSuccess, - update, - }); + return await updateSqliteSessionEntry(scope, update, options); } export type RecordInboundSessionMetaParams = { @@ -1799,7 +1551,7 @@ export type UpdateSessionLastRouteParams = { export async function recordInboundSessionMeta( params: RecordInboundSessionMetaParams, ): Promise<SessionEntry | null> { - return await recordFileSessionMetaFromInbound(params); + return await recordSqliteInboundSessionMeta(params); } /** @@ -1811,22 +1563,21 @@ export async function recordInboundSessionMeta( export async function updateSessionLastRoute( params: UpdateSessionLastRouteParams, ): Promise<SessionEntry | null> { - return await updateFileSessionLastRoute(params); + return await updateSqliteSessionLastRoute(params); } /** Resolves one abort target identity without exposing the mutable store. */ export function resolveSessionAbortTarget( scope: SessionAccessScope, ): SessionAbortTargetIdentity | null { - const store = loadSessionStore(resolveSessionStorePathForScope(scope)); - const resolved = resolveSessionStoreEntry({ store, sessionKey: scope.sessionKey }); - if (!resolved.existing) { + const entry = loadSessionEntry(scope); + if (!entry) { return null; } return { - entry: { ...resolved.existing }, - sessionId: resolved.existing.sessionId, - sessionKey: resolved.normalizedKey, + entry: { ...entry }, + sessionId: entry.sessionId, + sessionKey: normalizeStoreSessionKey(scope.sessionKey), }; } @@ -1839,61 +1590,45 @@ export async function markSessionAbortTarget(params: { scope: SessionAccessScope; now?: () => number; }): Promise<SessionAbortTargetResult | null> { - const storePath = resolveSessionStorePathForScope(params.scope); - let canPersistSingleEntry = false; let resolvedTarget: SessionAbortTargetResult | null = null; try { - return await updateSessionStore( - storePath, - (store) => { - const resolved = resolveSessionStoreEntry({ - store, - sessionKey: params.scope.sessionKey, - }); - if (!resolved.existing) { - return null; - } - const sessionKey = resolved.normalizedKey; + const sessionKey = normalizeStoreSessionKey(params.scope.sessionKey); + const updated = await patchSessionEntry( + params.scope, + (currentEntry) => { resolvedTarget = { - entry: { ...resolved.existing }, + entry: { ...currentEntry }, persisted: false, - sessionId: resolved.existing.sessionId, + sessionId: currentEntry.sessionId, sessionKey, }; const entry = { - ...resolved.existing, + ...currentEntry, abortedLastRun: true, updatedAt: params.now?.() ?? Date.now(), }; applySessionAbortCutoff( entry, params.resolveAbortCutoff?.({ - entry: { ...resolved.existing }, + entry: { ...currentEntry }, sessionKey, }), ); - store[sessionKey] = entry; - canPersistSingleEntry = resolved.legacyKeys.length === 0; - for (const legacyKey of resolved.legacyKeys) { - if (legacyKey !== sessionKey) { - delete store[legacyKey]; - } - } - return { - entry: { ...entry }, - persisted: true, - sessionId: entry.sessionId, - sessionKey, - }; + return entry; }, { - resolveSingleEntryPersistence: (result) => - result && result.sessionKey && canPersistSingleEntry - ? { sessionKey: result.sessionKey, entry: result.entry } - : null, - skipSaveWhenResult: (result) => result === null, + replaceEntry: true, + skipMaintenance: true, }, ); + return updated + ? { + entry: { ...updated }, + persisted: true, + sessionId: updated.sessionId, + sessionKey, + } + : null; } catch (error) { const fallbackTarget = resolvedTarget as unknown as SessionAbortTargetResult | null; if (fallbackTarget) { @@ -1949,45 +1684,43 @@ type ApplySessionCompactionCheckpointMutationParams = { async function applySessionCompactionCheckpointMutation( params: ApplySessionCompactionCheckpointMutationParams, ): Promise<SessionCompactionCheckpointMutationResult> { - return await updateSessionStore( - params.storePath, - async (store) => { - const currentEntry = store[params.readKey]; - if (!currentEntry?.sessionId) { - return { status: "missing-session" }; - } - // A native harness owns the locked transcript identity. Rotating or - // cloning it here would strand that binding on the old session id. - if (currentEntry.modelSelectionLocked === true) { - return { status: "model-selection-locked" }; - } - const checkpoint = findSessionCompactionCheckpoint({ - entry: currentEntry, - checkpointId: params.checkpointId, - }); - if (!checkpoint) { - return { status: "missing-checkpoint" }; - } - const forkedSession = await params.forkTranscriptFromCheckpoint(checkpoint); - if (forkedSession.status !== "created") { - return forkedSession; - } + const currentEntry = loadSessionEntry({ + sessionKey: params.readKey, + storePath: params.storePath, + }); + if (!currentEntry?.sessionId) { + return { status: "missing-session" }; + } + if (currentEntry.modelSelectionLocked === true) { + return { status: "model-selection-locked" }; + } + const checkpoint = findSessionCompactionCheckpoint({ + entry: currentEntry, + checkpointId: params.checkpointId, + }); + if (!checkpoint) { + return { status: "missing-checkpoint" }; + } + const forkedSession = await params.forkTranscriptFromCheckpoint(checkpoint); + if (forkedSession.status !== "created") { + return forkedSession; + } - const nextEntry = await params.buildEntry({ - checkpoint, - currentEntry, - forkedTranscript: forkedSession.transcript, - }); - store[params.writeKey] = nextEntry; - return { - status: "created", - key: params.writeKey, - checkpoint, - entry: nextEntry, - }; - }, - { skipSaveWhenResult: (result) => result.status !== "created" }, + const nextEntry = await params.buildEntry({ + checkpoint, + currentEntry, + forkedTranscript: forkedSession.transcript, + }); + await replaceSessionEntry( + { sessionKey: params.writeKey, storePath: params.storePath }, + nextEntry, ); + return { + status: "created", + key: params.writeKey, + checkpoint, + entry: nextEntry, + }; } /** @@ -2034,13 +1767,60 @@ export async function restoreSessionFromCompactionCheckpoint( export async function applySessionPatchProjection< TFailure extends SessionPatchProjectionFailure, >(params: { + agentId?: string; storePath: string; resolveTarget: (snapshot: SessionPatchProjectionSnapshot) => SessionPatchProjectionTarget; project: ( context: SessionPatchProjectionContext, ) => Promise<SessionPatchProjectionResult<TFailure>> | SessionPatchProjectionResult<TFailure>; }): Promise<SessionPatchProjectionResult<TFailure>> { - return await applyFileSessionEntryPatchProjection(params); + const entries = listSessionEntries({ agentId: params.agentId, storePath: params.storePath }).map( + ({ sessionKey, entry }) => ({ + entry: structuredClone(entry), + sessionKey, + }), + ); + const target = params.resolveTarget({ entries }); + const existingEntry = resolveProjectionExistingEntry(entries, target); + const projected = await params.project({ + ...target, + entries, + ...(existingEntry ? { existingEntry } : {}), + }); + if (!projected.ok) { + return projected; + } + const candidateKeys = uniqueStrings( + (target.candidateKeys ?? [target.primaryKey]).map((key) => key.trim()).filter(Boolean), + ); + await applySessionEntryLifecycleMutation({ + agentId: params.agentId, + storePath: params.storePath, + removals: candidateKeys + .filter((sessionKey) => sessionKey !== target.primaryKey) + .map((sessionKey) => ({ sessionKey })), + upserts: [{ sessionKey: target.primaryKey, entry: projected.entry }], + skipMaintenance: true, + }); + return { ...projected, entry: structuredClone(projected.entry) }; +} + +function resolveProjectionExistingEntry( + entries: readonly { sessionKey: string; entry: SessionEntry }[], + target: SessionPatchProjectionTarget, +): SessionEntry | undefined { + const candidateKeys = target.candidateKeys ?? [target.primaryKey]; + let freshest: SessionEntry | undefined; + for (const candidateKey of candidateKeys) { + const entry = entries.find((candidate) => candidate.sessionKey === candidateKey)?.entry; + if (!entry) { + continue; + } + if (!freshest || (entry.updatedAt ?? 0) > (freshest.updatedAt ?? 0)) { + freshest = entry; + } + } + return freshest ? structuredClone(freshest) : undefined; } /** @@ -2059,43 +1839,7 @@ export async function applySessionEntryReplacements<T>(params: { requireWriteSuccess?: boolean; skipMaintenance?: boolean; }): Promise<T> { - const writerResult = await updateSessionStore( - params.storePath, - async (store) => { - const selectedKeys = params.sessionKeys ? new Set(params.sessionKeys) : undefined; - const entries = selectedKeys - ? [...selectedKeys].flatMap((sessionKey) => { - const entry = store[sessionKey]; - return entry ? [{ sessionKey, entry: structuredClone(entry) }] : []; - }) - : Object.entries(store).map(([sessionKey, entry]) => ({ - sessionKey, - entry: structuredClone(entry), - })); - const operation = await params.update(entries); - let changed = false; - for (const replacement of operation.replacements ?? []) { - if (selectedKeys && !selectedKeys.has(replacement.sessionKey)) { - throw new Error( - `Session entry replacement is outside the selected key set: ${replacement.sessionKey}`, - ); - } - if (!Object.hasOwn(store, replacement.sessionKey)) { - continue; - } - store[replacement.sessionKey] = structuredClone(replacement.entry); - changed = true; - } - return { changed, result: operation.result }; - }, - { - activeSessionKey: params.activeSessionKey, - requireWriteSuccess: params.requireWriteSuccess, - skipMaintenance: params.skipMaintenance ?? true, - skipSaveWhenResult: (result) => !result.changed, - }, - ); - return writerResult.result; + return await applySqliteSessionEntryReplacements(params); } /** @@ -2133,32 +1877,33 @@ export async function preserveTemporarySessionMapping<T>( export async function cleanupSessionLifecycleArtifacts( params: SessionLifecycleArtifactCleanupParams, ): Promise<SessionLifecycleArtifactCleanupResult> { - return await cleanupFileSessionLifecycleArtifacts(params); + return await cleanupSqliteSessionLifecycleArtifacts(params); } /** Resets one persisted session entry and transitions its transcript state. */ export async function resetSessionEntryLifecycle( params: ResetSessionEntryLifecycleParams, ): Promise<ResetSessionEntryLifecycleResult> { - return await resetFileSessionEntryLifecycle(params); + return await resetSqliteSessionEntryLifecycle(params); } /** Deletes one persisted session entry and transitions its transcript state. */ export async function deleteSessionEntryLifecycle( params: DeleteSessionEntryLifecycleParams, ): Promise<DeleteSessionEntryLifecycleResult> { - return await deleteFileSessionEntryLifecycle(params); + return await deleteSqliteSessionEntryLifecycle(params); } /** Internal exact-row rollback for failed trusted agent-harness initialization. */ export async function rollbackAgentHarnessSessionEntryLifecycle( params: DeleteSessionEntryLifecycleParams & { expectedEntry: SessionEntry }, ): Promise<DeleteSessionEntryLifecycleResult> { - return await rollbackFileAgentHarnessSessionEntryLifecycle(params); + return await rollbackSqliteAgentHarnessSessionEntryLifecycle(params); } /** Applies exact entry lifecycle mutations and artifact cleanup at the storage boundary. */ export async function applySessionEntryLifecycleMutation(params: { + agentId?: string; storePath: string; removals?: Iterable<SessionEntryLifecycleRemoval>; upserts?: Iterable<SessionEntryLifecycleUpsert>; @@ -2178,14 +1923,21 @@ export async function applySessionEntryLifecycleMutation(params: { }; captureArtifactCleanupError?: boolean; }): Promise<SessionEntryLifecycleMutationResult> { - return await applyFileSessionEntryLifecycleMutation(params); + return await applySqliteSessionEntryLifecycleMutation(params); +} + +/** Previews SQLite row-byte disk-budget cleanup without mutating persisted rows. */ +export function previewSessionDiskBudget( + params: Parameters<typeof previewSqliteSessionDiskBudget>[0], +) { + return previewSqliteSessionDiskBudget(params); } /** Purges session entries owned by a deleted agent at the storage boundary. */ export async function purgeDeletedAgentSessionEntries( params: DeletedAgentSessionEntryPurgeParams, ): Promise<SessionEntryLifecycleMutationResult> { - return await purgeFileDeletedAgentSessionEntries(params); + return await purgeSqliteDeletedAgentSessionEntries(params); } /** @@ -2196,7 +1948,50 @@ export async function purgeDeletedAgentSessionEntries( export async function cleanupPluginHostSessionStore( params: PluginHostSessionCleanupStoreParams, ): Promise<number> { - return await cleanupFilePluginHostSessionStore(params); + if ( + shouldSkipPluginHostCleanupStore(params) || + (params.shouldCleanup && !params.shouldCleanup()) + ) { + return 0; + } + const now = Date.now(); + let cleared = 0; + for (const { entry, sessionKey } of listSessionEntries({ + agentId: params.agentId, + storePath: params.storePath, + })) { + if (isLockedHarnessSessionOwnedByPlugin(entry, params.preserveLockedHarnessIds)) { + continue; + } + if ( + !matchesPluginHostCleanupSession(sessionKey, entry, params.sessionKey) || + !hasPluginHostCleanupTarget(entry, params) + ) { + continue; + } + const updated = await patchSessionEntry( + { agentId: params.agentId, sessionKey, storePath: params.storePath }, + (currentEntry) => { + if (isLockedHarnessSessionOwnedByPlugin(currentEntry, params.preserveLockedHarnessIds)) { + return null; + } + if (!hasPluginHostCleanupTarget(currentEntry, params)) { + return null; + } + clearPluginHostCleanupTarget(currentEntry, params); + currentEntry.updatedAt = now; + return currentEntry; + }, + { + replaceEntry: true, + skipMaintenance: true, + }, + ); + if (updated) { + cleared += 1; + } + } + return cleared; } /** @@ -2216,18 +2011,22 @@ export async function persistSessionResetLifecycle(params: { }): Promise<{ replayedMessages: number }> { let persistError: Error | undefined; try { - await updateSessionStore(params.storePath, (store) => { - store[params.sessionKey] = params.nextEntry; - }); + await replaceSessionEntry( + { sessionKey: params.sessionKey, storePath: params.storePath }, + params.nextEntry, + ); } catch (err) { persistError = err instanceof Error ? err : new Error(String(err)); } - const replayedMessages = await replayRecentUserAssistantMessages({ - sourceTranscript: params.previousEntry.sessionFile, - targetTranscript: params.nextSessionFile, - newSessionId: params.nextEntry.sessionId, - }); + const sqliteReplayedMessages = await replayRecentUserAssistantMessagesToSqlite(params); + const replayedMessages = + sqliteReplayedMessages ?? + (await replayRecentUserAssistantMessages({ + sourceTranscript: params.previousEntry.sessionFile, + targetTranscript: params.nextSessionFile, + newSessionId: params.nextEntry.sessionId, + })); if (params.cleanupPreviousTranscript && params.previousSessionId) { await archivePreviousSessionTranscript({ @@ -2246,12 +2045,81 @@ export async function persistSessionResetLifecycle(params: { return { replayedMessages }; } +async function replayRecentUserAssistantMessagesToSqlite(params: { + agentId?: string; + nextEntry: SessionEntry; + nextSessionFile: string; + previousEntry: SessionEntry; + previousSessionId?: string; + sessionKey: string; + storePath: string; +}): Promise<number | undefined> { + const targetMarker = parseSqliteSessionFileMarker(params.nextSessionFile); + if (!targetMarker) { + return undefined; + } + + try { + const sourceMarker = parseSqliteSessionFileMarker(params.previousEntry.sessionFile); + const sourceRecords = sourceMarker + ? selectRecentUserAssistantReplayRecords( + await loadTranscriptEvents({ + agentId: sourceMarker.agentId, + sessionId: params.previousSessionId ?? sourceMarker.sessionId, + sessionKey: params.sessionKey, + storePath: sourceMarker.storePath, + }), + ) + : await readRecentUserAssistantReplayRecordsFromJsonl({ + sourceTranscript: params.previousEntry.sessionFile, + }); + if (sourceRecords.length === 0) { + return 0; + } + + for (const record of sourceRecords) { + const replayMessage = extractReplayMessage(record); + if (replayMessage === undefined) { + continue; + } + await appendTranscriptMessage( + { + agentId: targetMarker.agentId, + sessionId: targetMarker.sessionId, + sessionKey: params.sessionKey, + storePath: targetMarker.storePath, + }, + { message: replayMessage }, + ); + } + return sourceRecords.length; + } catch { + return 0; + } +} + +function extractReplayMessage(record: unknown): unknown { + if (!record || typeof record !== "object" || Array.isArray(record)) { + return undefined; + } + const candidate = record as { message?: unknown; type?: unknown }; + if (candidate.type !== "message") { + return undefined; + } + return candidate.message && typeof candidate.message === "object" ? candidate.message : undefined; +} + /** Loads the reply-session initialization rows without exposing a mutable store. */ export function loadReplySessionInitializationSnapshot(params: { storePath: string; sessionKey: string; }): ReplySessionInitializationSnapshot { - const store = loadSessionStore(params.storePath, { skipCache: true, clone: false }); + const store = Object.fromEntries( + listSessionEntries({ storePath: params.storePath }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), + ); const resolved = resolveSessionStoreEntry({ store, sessionKey: params.sessionKey }); const currentEntry = resolved.existing ? { ...resolved.existing } : undefined; const entries = cloneSessionEntries(store); @@ -2277,7 +2145,6 @@ export async function commitReplySessionInitialization(params: { activeSessionKey: string; agentId: string; expectedRevision: string; - fallbackSessionFile?: string; maintenanceConfig?: ResolvedSessionMaintenanceConfig; onArchiveError?: (error: unknown, sourcePath: string) => void; onMaintenanceWarning?: (warning: SessionMaintenanceWarning) => void | Promise<void>; @@ -2291,76 +2158,118 @@ export async function commitReplySessionInitialization(params: { snapshotEntry?: SessionEntry; storePath: string; }): Promise<ReplySessionInitializationCommitResult> { - const committed = await updateSessionStore( - params.storePath, - async (store): Promise<ReplySessionInitializationCommitResult> => { - const resolved = resolveSessionStoreEntry({ store, sessionKey: params.sessionKey }); - const currentEntry = resolved.existing ? { ...resolved.existing } : undefined; - const revision = createReplySessionInitializationRevision({ - entry: currentEntry, - storePath: params.storePath, - }); - if (revision !== params.expectedRevision) { - return { - ok: false, - ...(currentEntry ? { currentEntry } : {}), - reason: "stale-snapshot", - revision, - }; - } - - const readEntry = (sessionKey: string) => { - const entry = resolveSessionStoreEntry({ store, sessionKey }).existing; - return entry ? { ...entry } : undefined; - }; - const preparedSessionEntry = params.prepareSessionEntry - ? await params.prepareSessionEntry({ - ...(currentEntry ? { currentEntry } : {}), - readEntry, - sessionEntry: params.sessionEntry, - }) - : params.sessionEntry; - const sessionEntry = resolveInitializedReplySessionEntry({ - agentId: params.agentId, - ...(currentEntry ? { currentEntry } : {}), - fallbackSessionFile: params.fallbackSessionFile, - sessionEntry: preparedSessionEntry, - storePath: params.storePath, - }); - // The identity-only guard allows commits when background activity touched - // non-identity metadata after the snapshot. Merge only the fields that - // actually changed since the snapshot so heartbeat/delivery/context - // metadata is not rolled back, while reset-cleared fields (e.g. provider - // or model overrides on /new) stay cleared. - store[resolved.normalizedKey] = currentEntry - ? mergeConcurrentReplySessionMetadata({ - currentEntry, - preparedEntry: sessionEntry, - snapshotEntry: params.snapshotEntry ?? params.previousEntry, - }) - : sessionEntry; - if (params.retiredEntry) { - store[params.retiredEntry.key] = params.retiredEntry.entry; - } - return { - ok: true, - previousSessionTranscript: {}, - sessionEntry: { ...(store[resolved.normalizedKey] ?? sessionEntry) }, - sessionStoreView: cloneSessionEntries(store), - }; - }, - { - activeSessionKey: params.activeSessionKey, - maintenanceConfig: params.maintenanceConfig, - onWarn: params.onMaintenanceWarning, - reentrant: true, - skipSaveWhenResult: (result) => !result.ok, - }, + const store = Object.fromEntries( + listSessionEntries({ storePath: params.storePath }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), ); - if (!committed.ok) { - return committed; + const resolved = resolveSessionStoreEntry({ store, sessionKey: params.sessionKey }); + const currentEntry = resolved.existing ? { ...resolved.existing } : undefined; + const revision = createReplySessionInitializationRevision({ + entry: currentEntry, + storePath: params.storePath, + }); + if (revision !== params.expectedRevision) { + return { + ok: false, + ...(currentEntry ? { currentEntry } : {}), + reason: "stale-snapshot", + revision, + }; } + const readEntry = (sessionKey: string) => { + const entry = resolveSessionStoreEntry({ store, sessionKey }).existing; + return entry ? { ...entry } : undefined; + }; + const preparedSessionEntry = params.prepareSessionEntry + ? await params.prepareSessionEntry({ + ...(currentEntry ? { currentEntry } : {}), + readEntry, + sessionEntry: params.sessionEntry, + }) + : params.sessionEntry; + const sessionEntry = resolveInitializedReplySessionEntry({ + agentId: params.agentId, + ...(currentEntry ? { currentEntry } : {}), + sessionEntry: preparedSessionEntry, + storePath: params.storePath, + }); + let staleCommit: + | { + currentEntry?: SessionEntry; + revision: string; + } + | undefined; + let committedSessionEntry = sessionEntry; + const upserts: SessionEntryLifecycleUpsert[] = [ + { + sessionKey: resolved.normalizedKey, + buildEntry: ({ store: currentStore }) => { + const commitResolved = resolveSessionStoreEntry({ + store: currentStore, + sessionKey: params.sessionKey, + }); + const commitEntry = commitResolved.existing; + const commitRevision = createReplySessionInitializationRevision({ + entry: commitEntry, + storePath: params.storePath, + }); + if (commitRevision !== params.expectedRevision) { + staleCommit = { + ...(commitEntry ? { currentEntry: { ...commitEntry } } : {}), + revision: commitRevision, + }; + return null; + } + // The identity-only guard allows commits when background activity + // touched non-identity metadata after the snapshot. Merge only fields + // that changed since the snapshot so delivery/context metadata is not + // rolled back, while reset-cleared fields stay cleared. + committedSessionEntry = commitEntry + ? mergeConcurrentReplySessionMetadata({ + currentEntry: commitEntry, + preparedEntry: sessionEntry, + snapshotEntry: params.snapshotEntry ?? params.previousEntry, + }) + : sessionEntry; + return committedSessionEntry; + }, + }, + ]; + if (params.retiredEntry) { + const retiredEntry = params.retiredEntry; + upserts.push({ + sessionKey: retiredEntry.key, + buildEntry: () => (staleCommit ? null : retiredEntry.entry), + }); + } + await applySessionEntryLifecycleMutation({ + activeSessionKey: params.activeSessionKey, + maintenanceOverride: params.maintenanceConfig, + storePath: params.storePath, + upserts, + }); + if (staleCommit) { + return { + ok: false, + ...(staleCommit.currentEntry ? { currentEntry: staleCommit.currentEntry } : {}), + reason: "stale-snapshot", + revision: staleCommit.revision, + }; + } + store[resolved.normalizedKey] = committedSessionEntry; + if (params.retiredEntry) { + store[params.retiredEntry.key] = params.retiredEntry.entry; + } + const committed: ReplySessionInitializationCommitResult = { + ok: true, + previousSessionTranscript: {}, + sessionEntry: { ...committedSessionEntry }, + sessionStoreView: cloneSessionEntries(store), + }; + const previousSessionTranscript = await archivePreviousSessionTranscript({ agentId: params.agentId, onArchiveError: params.onArchiveError, @@ -2382,25 +2291,72 @@ export async function appendTranscriptEvent( scope: SessionTranscriptAccessScope, event: TranscriptEvent, ): Promise<void> { - assertNonMessageTranscriptEvent(event); - const transcript = await resolveTranscriptAccess(scope); - await appendSessionTranscriptEvent({ - event, - transcriptPath: transcript.sessionFile, - }); + await appendSqliteTranscriptEvent(scope, event); } -function assertNonMessageTranscriptEvent(event: TranscriptEvent): void { - if (!event || typeof event !== "object" || Array.isArray(event)) { - return; - } - // Message records require parent-link, idempotency, and redaction handling - // from appendTranscriptMessage; raw event writes would bypass those invariants. - if ((event as { type?: unknown }).type === "message") { - throw new Error( - "appendTranscriptEvent cannot write message transcript records; use appendTranscriptMessage instead.", - ); - } +/** Appends a non-message transcript record synchronously for sync session runtimes. */ +export function appendTranscriptEventSync( + scope: SessionTranscriptAccessScope, + event: TranscriptEvent, +): void { + appendSqliteTranscriptEventSync(scope, event); +} + +/** Reads parsed transcript records from an explicit or derived transcript target. */ +export async function loadTranscriptEvents( + scope: SessionTranscriptReadScope, +): Promise<TranscriptEvent[]> { + return await loadSqliteTranscriptEvents(scope); +} + +/** Replaces all transcript records for one SQLite-backed transcript. */ +export async function replaceTranscriptEvents( + scope: SessionTranscriptAccessScope, + events: TranscriptEvent[], +): Promise<void> { + await replaceSqliteTranscriptEvents(scope, events); +} + +/** Replaces all transcript records synchronously for sync session runtimes. */ +export function replaceTranscriptEventsSync( + scope: SessionTranscriptAccessScope, + events: TranscriptEvent[], +): boolean { + return replaceSqliteTranscriptEventsSync(scope, events); +} + +/** Reads parsed transcript records synchronously from the SQLite transcript store. */ +export function loadTranscriptEventsSync(scope: SessionTranscriptReadScope): TranscriptEvent[] { + return loadSqliteTranscriptEventsSync(scope); +} + +/** Reads transcript freshness and byte size without materializing event rows. */ +export function readTranscriptStatsSync(scope: SessionTranscriptReadScope): SessionTranscriptStats { + return readSqliteTranscriptStatsSync(scope); +} + +/** Reads the latest visible assistant text without materializing the whole transcript. */ +export function readLatestTranscriptAssistantText( + scope: SessionTranscriptReadScope, + options: { includeTranscriptOnlyOpenClawAssistant?: boolean } = {}, +): LatestTranscriptAssistantText | undefined { + return loadLatestSqliteAssistantText(scope, options); +} + +/** Reads the latest assistant message payload without materializing the whole transcript. */ +export function readLatestTranscriptAssistantMessage( + scope: SessionTranscriptReadScope, + options: { includeTranscriptOnlyOpenClawAssistant?: boolean } = {}, +): LatestTranscriptAssistantMessage | undefined { + return loadLatestSqliteAssistantMessage(scope, options); +} + +/** Reads the latest transcript message payload without materializing the whole transcript. */ +export function readLatestTranscriptMessage( + scope: SessionTranscriptReadScope, + options: { includeTranscriptOnlyOpenClawAssistant?: boolean } = {}, +): LatestTranscriptMessage | undefined { + return loadLatestSqliteMessage(scope, options); } /** @@ -2421,64 +2377,35 @@ export async function appendTranscriptMessage<TMessage>( scope: SessionTranscriptWriteScope, options: TranscriptMessageAppendOptions<TMessage>, ): Promise<TranscriptMessageAppendResult<TMessage> | undefined> { - const transcript = await resolveTranscriptAccess(scope); - return await appendSessionTranscriptMessage({ - transcriptPath: transcript.sessionFile, - message: options.message, - ...(scope.sessionId ? { sessionId: scope.sessionId } : {}), - ...(options.cwd ? { cwd: options.cwd } : {}), - ...(options.config ? { config: options.config } : {}), - ...(options.idempotencyLookup ? { idempotencyLookup: options.idempotencyLookup } : {}), - ...(options.now !== undefined ? { now: options.now } : {}), - ...(options.prepareMessageAfterIdempotencyCheck - ? { prepareMessageAfterIdempotencyCheck: options.prepareMessageAfterIdempotencyCheck } - : {}), - ...(options.useRawWhenLinear !== undefined - ? { useRawWhenLinear: options.useRawWhenLinear } - : {}), - }); + return await appendSqliteTranscriptMessage(scope, options); +} + +/** Appends one transcript message synchronously for sync session runtimes. */ +export function appendTranscriptMessageSync<TMessage>( + scope: SessionTranscriptWriteScope, + options: TranscriptMessageAppendOptions<TMessage>, +): TranscriptMessageAppendResult<TMessage> | undefined { + return appendSqliteTranscriptMessageSync(scope, options); +} + +/** Resolves the persisted key for a SQLite transcript session id. */ +export function resolveTranscriptSessionKeyBySessionId( + scope: Pick<SessionTranscriptReadScope, "agentId" | "env" | "sessionId" | "storePath">, +): string | undefined { + return resolveSqliteSessionKeyBySessionId(scope); } /** - * Finds the newest transcript record accepted by the matcher. Reads newest-first - * with early exit so hot append-path lookups never materialize the whole - * transcript; missing transcripts match nothing. The match is wrapped so parsed - * falsy records stay distinguishable from "no match". + * Finds the newest transcript record accepted by the matcher. Reads rows + * newest-first with early exit so hot append-path lookups never parse the + * whole transcript; missing transcripts match nothing. The match is wrapped + * so parsed falsy records stay distinguishable from "no match". */ export async function findTranscriptEvent( scope: SessionTranscriptReadScope, match: (event: TranscriptEvent) => boolean, ): Promise<{ event: TranscriptEvent } | undefined> { - const target = resolveSessionTranscriptReadTarget(scope); - for await (const line of streamSessionTranscriptLinesReverse(target.sessionFile)) { - try { - const event = JSON.parse(line) as TranscriptEvent; - if (match(event)) { - return { event }; - } - } catch { - // Malformed lines are skipped, matching transcript index tolerance. - } - } - return undefined; -} - -/** Reads parsed transcript records from an explicit or derived transcript target. */ -export async function loadTranscriptEvents( - scope: SessionTranscriptReadScope, -): Promise<TranscriptEvent[]> { - const target = resolveSessionTranscriptReadTarget(scope); - const events: TranscriptEvent[] = []; - // Missing transcripts stream zero lines, so readers get an empty event list - // instead of a filesystem error; that keeps the read contract storage-neutral. - for await (const line of streamSessionTranscriptLines(target.sessionFile)) { - try { - events.push(JSON.parse(line) as TranscriptEvent); - } catch { - // Malformed lines are skipped, matching transcript index tolerance. - } - } - return events; + return findSqliteTranscriptEvent(scope, match); } /** Emits a transcript update after resolving the current transcript target. */ @@ -2486,12 +2413,15 @@ export async function publishTranscriptUpdate( scope: SessionTranscriptWriteScope, update: TranscriptUpdatePayload = {}, ): Promise<void> { - const transcript = await resolveTranscriptAccess(scope); - emitSessionTranscriptUpdate({ - ...update, - sessionFile: transcript.sessionFile, - ...(transcript.target ? { target: transcript.target } : {}), - }); + await publishSqliteTranscriptUpdate(scope, update); +} + +/** Runs transcript read/append work under the backing store writer lock. */ +export async function withTranscriptWriteLock<T>( + scope: SessionTranscriptWriteScope, + run: (context: SessionTranscriptWriteLockAccessorContext) => Promise<T> | T, +): Promise<T> { + return await withSqliteTranscriptWriteLock(scope, run); } /** @@ -2503,71 +2433,54 @@ export async function preflightSessionTranscriptForManualCompact( scope: SessionTranscriptRuntimeScope, params: { maxLines: number; sessionFile?: string }, ): Promise<SessionTranscriptManualTrimPreflightResult> { - const transcript = await resolveManualCompactTranscriptTarget(scope, params.sessionFile); - if (!transcript) { + const events = await loadTranscriptEvents(scope).catch(() => []); + if (events.length === 0) { return { compacted: false, reason: "no transcript" }; } const maxLines = Math.max(1, Math.floor(params.maxLines)); - let totalLines = 0; - try { - for await (const line of streamSessionTranscriptLines(transcript.sessionFile)) { - if (!line) { - continue; - } - totalLines += 1; - if (totalLines > maxLines) { - return { compacted: true }; - } - } - } catch { - return { compacted: false, kept: 0 }; - } - return { compacted: false, kept: totalLines }; + return events.length > maxLines ? { compacted: true } : { compacted: false, kept: events.length }; } export async function trimSessionTranscriptForManualCompact( scope: SessionTranscriptRuntimeScope, params: { maxLines: number; nowMs?: number; sessionFile?: string }, ): Promise<SessionTranscriptManualTrimResult> { - const transcript = await resolveManualCompactTranscriptTarget(scope, params.sessionFile); - if (!transcript) { + const events = await loadTranscriptEvents(scope).catch(() => []); + if (events.length === 0) { return { compacted: false, reason: "no transcript" }; } const maxLines = Math.max(1, Math.floor(params.maxLines)); - let headerLine: string | undefined; - const tailLines: string[] = []; + const headerLine = JSON.stringify(events[0]); + const tailLines = events.slice(1).map((event) => JSON.stringify(event)); const maxTailLines = Math.max(0, maxLines - 1); - let totalLines = 0; - try { - for await (const line of streamSessionTranscriptLines(transcript.sessionFile)) { - totalLines += 1; - if (totalLines === 1) { - headerLine = line; - continue; - } - tailLines.push(line); - if (tailLines.length > maxTailLines) { - tailLines.shift(); - } - } - } catch { - return { compacted: false, kept: 0 }; - } - if (totalLines <= maxLines) { - return { compacted: false, kept: totalLines }; + if (events.length <= maxLines) { + return { compacted: false, kept: events.length }; } - const lines = normalizeManualCompactTranscriptLines(headerLine, tailLines); + const lines = normalizeManualCompactTranscriptLines( + headerLine, + maxTailLines > 0 ? tailLines.slice(-maxTailLines) : [], + ); if (!lines) { return { compacted: false, kept: 0 }; } - const archived = await replaceTranscriptForManualCompact(transcript.sessionFile, lines); + const retainedEvents = lines.map((line) => JSON.parse(line) as TranscriptEvent); + await replaceSqliteTranscriptEvents(scope, retainedEvents); + const agentId = scope.agentId ?? resolveAgentIdFromSessionKey(scope.sessionKey); + if (!agentId) { + throw new Error(`Cannot resolve manual compact transcript scope: ${scope.sessionKey}`); + } + const archived = `${formatSqliteSessionFileMarker({ + agentId, + sessionId: scope.sessionId, + storePath: scope.storePath ?? "", + })}.bak.${formatSessionArchiveTimestamp()}`; await patchSessionEntry( { ...scope, - sessionKey: transcript.sessionKey, + sessionKey: scope.sessionKey, storePath: scope.storePath, }, (entry) => { @@ -2673,162 +2586,10 @@ function normalizeManualCompactTranscriptLines( return [JSON.stringify(header), ...normalizedRecords.map((record) => JSON.stringify(record))]; } -async function replaceTranscriptForManualCompact( - filePath: string, - lines: readonly string[], -): Promise<string> { - const archived = `${filePath}.bak.${formatSessionArchiveTimestamp()}`; - const replacement = `${filePath}.compact.${randomUUID()}.tmp`; - try { - await writeJsonlLines(replacement, lines, { flag: "wx", mode: 0o600 }); - await fs.promises.rename(filePath, archived); - try { - await fs.promises.rename(replacement, filePath); - } catch (err) { - await fs.promises.rename(archived, filePath).catch(() => undefined); - throw err; - } - } catch (err) { - await fs.promises.unlink(replacement).catch(() => undefined); - throw err; - } - emitSessionTranscriptUpdate({ sessionFile: archived }); - emitSessionTranscriptUpdate({ sessionFile: filePath }); - return archived; -} - -async function resolveManualCompactTranscriptTarget( - scope: SessionTranscriptRuntimeScope, - sessionFile?: string, -): Promise<SessionTranscriptRuntimeTarget | null> { - const agentId = scope.agentId ?? resolveAgentIdFromSessionKey(scope.sessionKey); - if (!agentId) { - throw new Error(`Cannot resolve transcript scope without an agent id: ${scope.sessionKey}`); - } - const candidates = resolveManualCompactTranscriptCandidates({ - agentId, - sessionFile, - sessionId: scope.sessionId, - storePath: scope.storePath, - }); - for (const candidate of candidates) { - const stat = await fs.promises.stat(candidate).catch(() => null); - if (stat?.isFile()) { - return { - agentId, - sessionFile: candidate, - sessionId: scope.sessionId, - sessionKey: scope.sessionKey, - }; - } - } - return null; -} - -function resolveManualCompactTranscriptCandidates(params: { - agentId?: string; - sessionFile?: string; - sessionId: string; - storePath?: string; -}): string[] { - const candidates: string[] = []; - const sessionFileState = classifyGeneratedTranscriptCandidate( - params.sessionId, - params.sessionFile, - ); - const pushCandidate = (resolve: () => string): void => { - try { - const candidate = resolve(); - if (!candidates.includes(candidate)) { - candidates.push(candidate); - } - } catch { - // Keep scanning the remaining file-backed candidates. - } - }; - - if (params.storePath) { - const sessionsDir = path.dirname(params.storePath); - if (params.sessionFile && sessionFileState !== "stale") { - pushCandidate(() => - resolveSessionFilePath( - params.sessionId, - { sessionFile: params.sessionFile }, - { sessionsDir, agentId: params.agentId }, - ), - ); - } - pushCandidate(() => resolveSessionTranscriptPathInDir(params.sessionId, sessionsDir)); - if (params.sessionFile && sessionFileState === "stale") { - pushCandidate(() => - resolveSessionFilePath( - params.sessionId, - { sessionFile: params.sessionFile }, - { sessionsDir, agentId: params.agentId }, - ), - ); - } - } else if (params.sessionFile) { - if (params.agentId) { - if (sessionFileState !== "stale") { - pushCandidate(() => - resolveSessionFilePath( - params.sessionId, - { sessionFile: params.sessionFile }, - { agentId: params.agentId }, - ), - ); - } - } else { - const trimmed = params.sessionFile.trim(); - if (trimmed) { - candidates.push(path.resolve(trimmed)); - } - } - } - - if (params.agentId) { - pushCandidate(() => resolveSessionTranscriptPath(params.sessionId, params.agentId)); - if (params.sessionFile && sessionFileState === "stale") { - pushCandidate(() => - resolveSessionFilePath( - params.sessionId, - { sessionFile: params.sessionFile }, - { agentId: params.agentId }, - ), - ); - } - } - - const legacyDir = path.join( - resolveRequiredHomeDir(process.env, os.homedir), - ".openclaw", - "sessions", - ); - pushCandidate(() => resolveSessionTranscriptPathInDir(params.sessionId, legacyDir)); - return candidates; -} - -function classifyGeneratedTranscriptCandidate( - sessionId: string, - sessionFile?: string, -): "current" | "stale" | "custom" { - const transcriptSessionId = extractGeneratedTranscriptSessionId(sessionFile); - if (!transcriptSessionId) { - return "custom"; - } - return transcriptSessionId === sessionId ? "current" : "stale"; -} - /** - * Persists one logical transcript turn through the current file-backed writer. - * The file implementation resolves/rebinds the transcript file, holds one - * session write lock across all message appends, optionally touches session - * metadata, then publishes after the write has completed. - * - * SQLite implementation note: the transcript row append(s), sessionFile marker, - * and requested updatedAt touch become one SQLite write transaction; transcript - * update delivery must run only after commit. + * Persists one logical transcript turn through the SQLite-backed session target. + * Transcript row append(s), the synthetic sessionFile marker, and the requested + * updatedAt touch happen before transcript update delivery is published. */ export async function persistSessionTranscriptTurn( scope: SessionTranscriptWriteScope & { @@ -2842,7 +2603,13 @@ export async function persistSessionTranscriptTurn( return await persistExpectedSessionTranscriptTurn(scope, { ...options, expectedSessionId }); } const target = await resolveTranscriptTurnTarget(scope); - const appendedMessages = await appendTranscriptTurnMessages(target, options); + const appendedMessages = await runWithOwnedSessionTranscriptWriteLock( + { + sessionFile: target.sessionFile, + sessionKey: target.sessionKey, + }, + () => appendTranscriptTurnMessages(target, options), + ); const appendedCount = countAppendedTranscriptMessages(appendedMessages); const sessionEntry = await touchTranscriptTurnSessionEntry({ scope, @@ -2868,86 +2635,53 @@ async function appendTranscriptTurnMessages( target: SessionTranscriptTurnWriteContext, options: SessionTranscriptTurnPersistOptions, ): Promise<TranscriptMessageAppendResult<unknown>[]> { + const selectedMessages = await selectAppendableTranscriptTurnMessages(target, options); const appendedMessages: TranscriptMessageAppendResult<unknown>[] = []; - const publishedEntries: OwnedSessionTranscriptPublishedEntry[] = []; - const appendMessages = async (appendMessage: SessionTranscriptTurnAppendRunner) => { - for (const append of options.messages) { - const shouldAppend = append.shouldAppend - ? await append.shouldAppend({ - ...(target.agentId ? { agentId: target.agentId } : {}), - sessionFile: target.sessionFile, - ...(target.sessionId ? { sessionId: target.sessionId } : {}), - ...(target.sessionKey ? { sessionKey: target.sessionKey } : {}), - }) - : true; - if (!shouldAppend) { - continue; - } - const result = await appendMessage({ - transcriptPath: target.sessionFile, - message: append.message, + for (const append of selectedMessages) { + const { shouldAppend: _shouldAppend, ...appendOptions } = append; + const result = await appendTranscriptMessage( + { + ...(target.agentId ? { agentId: target.agentId } : {}), ...(target.sessionId ? { sessionId: target.sessionId } : {}), + ...(target.sessionKey ? { sessionKey: target.sessionKey } : {}), + ...(target.storePath ? { storePath: target.storePath } : {}), + }, + { + ...appendOptions, ...((append.cwd ?? options.cwd) ? { cwd: append.cwd ?? options.cwd } : {}), ...((append.config ?? options.config) ? { config: append.config ?? options.config } : {}), - ...(append.idempotencyLookup ? { idempotencyLookup: append.idempotencyLookup } : {}), - ...(append.now !== undefined ? { now: append.now } : {}), - ...(append.prepareMessageAfterIdempotencyCheck - ? { prepareMessageAfterIdempotencyCheck: append.prepareMessageAfterIdempotencyCheck } - : {}), - onHeaderCreated: (header) => { - publishedEntries.push({ kind: "header", serialized: header }); - }, - ...(append.useRawWhenLinear !== undefined - ? { useRawWhenLinear: append.useRawWhenLinear } - : {}), - }); - if (result) { - appendedMessages.push(result); - if (result.appended) { - publishedEntries.push({ kind: "id", id: result.messageId }); - } - } + }, + ); + if (result) { + appendedMessages.push(result); } - }; - const activeLockRunner = resolveOwnedSessionTranscriptWriteLockRunner({ - sessionFile: target.sessionFile, - sessionKey: target.sessionKey, - }); - const runBatchWithOwnedLock = async () => - await withOwnedSessionTranscriptWrites( - { - sessionFile: target.sessionFile, - sessionKey: target.sessionKey, - withSessionWriteLock: async (run) => await run(), - }, - async () => await appendMessages(appendSessionTranscriptMessageWithOwnedWriteLock), - ); - if (activeLockRunner) { - await activeLockRunner( - () => withSessionTranscriptAppendQueue(target.sessionFile, runBatchWithOwnedLock), - { - publishOwnedWrite: true, - resolvePublishedEntries: () => publishedEntries, - resolvePublishedEntriesAfterFailure: () => publishedEntries, - }, - ); - } else { - await withSessionTranscriptAppendQueue(target.sessionFile, async () => { - const lock = await acquireSessionWriteLock({ - sessionFile: target.sessionFile, - ...resolveSessionWriteLockOptions(options.config), - allowReentrant: true, - }); - try { - await runBatchWithOwnedLock(); - } finally { - await lock.release(); - } - }); } return appendedMessages; } +async function selectAppendableTranscriptTurnMessages( + target: SessionTranscriptTurnWriteContext, + options: SessionTranscriptTurnPersistOptions, +): Promise<SessionTranscriptTurnMessageAppend[]> { + const selectedMessages: SessionTranscriptTurnMessageAppend[] = []; + for (const append of options.messages) { + const shouldAppend = append.shouldAppend + ? await append.shouldAppend({ + ...(target.agentId ? { agentId: target.agentId } : {}), + sessionFile: target.sessionFile, + ...(target.sessionId ? { sessionId: target.sessionId } : {}), + ...(target.sessionKey ? { sessionKey: target.sessionKey } : {}), + ...(target.storePath ? { storePath: target.storePath } : {}), + }) + : true; + if (!shouldAppend) { + continue; + } + selectedMessages.push(append); + } + return selectedMessages; +} + function countAppendedTranscriptMessages( messages: readonly TranscriptMessageAppendResult<unknown>[], ): number { @@ -2965,84 +2699,61 @@ async function persistExpectedSessionTranscriptTurn( if (!scope.storePath || !sessionKey) { throw new Error("Cannot guard a transcript turn without a session store and key"); } + const storePath = scope.storePath; const expectedSessionId = options.expectedSessionId; const agentId = scope.agentId ?? resolveAgentIdFromSessionKey(sessionKey); if (!agentId) { throw new Error(`Cannot resolve transcript turn without an agent id: ${sessionKey}`); } const store = - scope.sessionStore ?? loadSessionStore(scope.storePath, { skipCache: true, clone: false }); + scope.sessionStore ?? + Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey: entryKey, entry }) => [entryKey, entry]), + ); const resolved = resolveSessionStoreEntry({ store, sessionKey }); - let appendedMessages: TranscriptMessageAppendResult<unknown>[] = []; - let target: SessionTranscriptTurnWriteContext = { + const sessionFile = formatSqliteSessionFileMarker({ agentId, - sessionFile: - scope.sessionFile ?? - resolveSessionTranscriptPathInDir(expectedSessionId, path.dirname(scope.storePath)), + sessionId: expectedSessionId, + storePath, + }); + const target: SessionTranscriptTurnWriteContext = { + agentId, + sessionFile, sessionId: expectedSessionId, sessionKey: resolved.normalizedKey, + storePath, }; - let rejectedEntry: SessionEntry | undefined; - let touchUpdatedAt: number | undefined; - - const updated = await updateSessionEntry( + const turn = await runWithOwnedSessionTranscriptWriteLock( { - sessionKey: resolved.normalizedKey, - storePath: scope.storePath, + sessionFile: target.sessionFile, + sessionKey: target.sessionKey, }, - async (currentEntry) => { - if ( - currentEntry.sessionId !== expectedSessionId || - (options.expectedLifecycleRevision !== undefined && - currentEntry.lifecycleRevision !== options.expectedLifecycleRevision) - ) { - rejectedEntry = currentEntry; - return null; - } - const sessionFile = - scope.sessionFile ?? - resolveSessionFilePath( - currentEntry.sessionId, - currentEntry, - resolveSessionFilePathOptions({ - agentId, - storePath: scope.storePath, - }), - ); - target = { - agentId, - sessionFile, - sessionId: currentEntry.sessionId, - sessionKey: resolved.normalizedKey, - }; - appendedMessages = await appendTranscriptTurnMessages(target, options); - const appendedCount = countAppendedTranscriptMessages(appendedMessages); - if (options.touchSessionEntry === true && appendedCount > 0) { - touchUpdatedAt = Date.now(); - } - const patch = { - ...(currentEntry.sessionFile === sessionFile ? {} : { sessionFile }), - ...(touchUpdatedAt !== undefined - ? { updatedAt: Math.max(currentEntry.updatedAt ?? 0, touchUpdatedAt) } - : {}), - }; - return Object.keys(patch).length > 0 ? patch : null; - }, - { skipMaintenance: true }, + () => + appendSqliteExpectedSessionTranscriptTurn( + { + sessionKey: resolved.normalizedKey, + sessionId: expectedSessionId, + storePath, + }, + { + config: options.config, + cwd: options.cwd, + expectedLifecycleRevision: options.expectedLifecycleRevision, + expectedSessionId, + messages: options.messages, + sessionFile: target.sessionFile, + touchSessionEntry: options.touchSessionEntry, + }, + ), ); - if ( - rejectedEntry || - updated?.sessionId !== expectedSessionId || - (options.expectedLifecycleRevision !== undefined && - updated.lifecycleRevision !== options.expectedLifecycleRevision) - ) { + if (turn.rejectedReason === "session-rebound") { return { appendedCount: 0, messages: [], rejectedReason: "session-rebound", - sessionEntry: rejectedEntry ?? updated ?? undefined, - sessionFile: target.sessionFile, + sessionEntry: turn.sessionEntry, + sessionFile: turn.sessionFile, }; } @@ -3050,124 +2761,103 @@ async function persistExpectedSessionTranscriptTurn( target, updateMode: options.updateMode ?? "inline", publishWhen: options.publishWhen ?? "when-appended", - appendedMessages, + appendedMessages: turn.appendedMessages, }); - if (updated && scope.sessionStore) { - scope.sessionStore[resolved.normalizedKey] = updated; + if (turn.sessionEntry && scope.sessionStore) { + scope.sessionStore[resolved.normalizedKey] = turn.sessionEntry; } return { - appendedCount: countAppendedTranscriptMessages(appendedMessages), - messages: appendedMessages, - sessionEntry: updated ?? scope.sessionEntry, - sessionFile: target.sessionFile, + appendedCount: countAppendedTranscriptMessages(turn.appendedMessages), + messages: turn.appendedMessages, + sessionEntry: turn.sessionEntry ?? scope.sessionEntry, + sessionFile: turn.sessionFile, }; } /** - * Resolves the current file-backed target for a storage-neutral runtime - * transcript scope. Callers use the scope as identity; sessionFile is returned - * only for current file-backed implementation details such as locks/events. + * Resolves the current storage-neutral runtime transcript target. SQLite-backed + * rows return their marker so transcript readers/writers stay on the accessor + * path instead of reopening legacy JSONL artifacts. */ export async function resolveSessionTranscriptRuntimeTarget( scope: SessionTranscriptRuntimeScope, ): Promise<SessionTranscriptRuntimeTarget> { const { agentId, sessionEntry, sessionKey, sessionStore } = resolveSessionTranscriptRuntimeContext(scope); - if (scope.sessionFile?.trim()) { + if (shouldUseExplicitTranscriptFile(scope)) { return { agentId, - sessionFile: path.resolve(scope.sessionFile), + sessionFile: scope.sessionFile.trim(), sessionId: scope.sessionId, sessionKey, }; } - if (sessionStore && scope.storePath) { - const sessionsDir = path.dirname(path.resolve(scope.storePath)); - const threadId = scope.threadId ?? parseSessionThreadInfo(scope.sessionKey).threadId; - const shouldUseDerivedSessionFile = - !sessionEntry?.sessionFile || sessionEntry.sessionId !== scope.sessionId; - const fallbackSessionFile = - shouldUseDerivedSessionFile && threadId !== undefined - ? resolveSessionTranscriptPathInDir(scope.sessionId, sessionsDir, threadId) - : undefined; - const resolved = await resolveAndPersistSessionFile({ - agentId, - fallbackSessionFile, - sessionEntry, - sessionId: scope.sessionId, - sessionKey, - sessionStore, - sessionsDir, - storePath: scope.storePath, - }); - return { - agentId, - sessionFile: resolved.sessionFile, - sessionId: scope.sessionId, - sessionKey, - }; - } - const resolved = await resolveSessionTranscriptFile({ - agentId, - sessionEntry, - sessionId: scope.sessionId, - sessionKey: scope.sessionKey, - ...(sessionStore ? { sessionStore } : {}), - ...(scope.storePath ? { storePath: scope.storePath } : {}), - ...(scope.threadId !== undefined ? { threadId: scope.threadId } : {}), - }); + void sessionStore; return { agentId, - sessionFile: resolved.sessionFile, + sessionFile: resolveRuntimeSessionFile(scope, agentId, sessionEntry), sessionId: scope.sessionId, sessionKey, }; } /** - * Resolves the file-backed runtime transcript target for read/delete probes - * without persisting missing sessionFile metadata into the session store. + * Resolves the runtime transcript target for read/delete probes without + * persisting missing sessionFile metadata into the session store. */ export async function resolveSessionTranscriptRuntimeReadTarget( scope: SessionTranscriptRuntimeScope, ): Promise<SessionTranscriptRuntimeTarget> { const { agentId, sessionEntry, sessionKey } = resolveSessionTranscriptRuntimeContext(scope); - if (scope.sessionFile?.trim()) { + if (shouldUseExplicitTranscriptFile(scope)) { return { agentId, - sessionFile: path.resolve(scope.sessionFile), + sessionFile: scope.sessionFile.trim(), sessionId: scope.sessionId, sessionKey, }; } - const matchingSessionEntry = - sessionEntry?.sessionId === scope.sessionId ? sessionEntry : undefined; - if (scope.storePath) { - const sessionsDir = path.dirname(path.resolve(scope.storePath)); - const threadId = scope.threadId ?? parseSessionThreadInfo(sessionKey).threadId; - const sessionFile = matchingSessionEntry?.sessionFile - ? resolveSessionFilePath(scope.sessionId, matchingSessionEntry, { agentId, sessionsDir }) - : resolveSessionTranscriptPathInDir(scope.sessionId, sessionsDir, threadId); - return { - agentId, - sessionFile, - sessionId: scope.sessionId, - sessionKey, - }; - } - const threadId = scope.threadId ?? parseSessionThreadInfo(sessionKey).threadId; - const sessionFile = matchingSessionEntry?.sessionFile - ? resolveSessionFilePath(scope.sessionId, matchingSessionEntry, { agentId }) - : resolveSessionTranscriptPath(scope.sessionId, agentId, threadId); return { agentId, - sessionFile, + sessionFile: resolveRuntimeSessionFile(scope, agentId, sessionEntry), sessionId: scope.sessionId, sessionKey, }; } +function resolveRuntimeSessionFile( + scope: SessionTranscriptRuntimeScope, + agentId: string, + sessionEntry: SessionEntry | undefined, +): string { + const matchingSessionEntry = + sessionEntry?.sessionId === undefined || sessionEntry.sessionId === scope.sessionId + ? sessionEntry + : undefined; + if ( + sqliteSessionFileMarkerMatchesSession(matchingSessionEntry?.sessionFile, scope.sessionId) && + matchingSessionEntry?.sessionFile + ) { + return matchingSessionEntry.sessionFile; + } + if (scope.storePath) { + return formatSqliteSessionFileMarker({ + agentId, + sessionId: scope.sessionId, + storePath: scope.storePath, + }); + } + return resolveSessionFilePath( + scope.sessionId, + matchingSessionEntry, + resolveSessionFilePathOptions({ + agentId, + storePath: scope.storePath, + }), + ); +} + type SessionTranscriptRuntimeContext = { agentId: string; sessionEntry: SessionEntry | undefined; @@ -3183,7 +2873,12 @@ function resolveSessionTranscriptRuntimeContext( throw new Error(`Cannot resolve transcript scope without an agent id: ${scope.sessionKey}`); } const sessionStore = scope.storePath - ? loadSessionStore(scope.storePath, { skipCache: true }) + ? Object.fromEntries( + listSessionEntries({ agentId, storePath: scope.storePath }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), + ) : undefined; const resolvedStoreEntry = sessionStore ? resolveSessionStoreEntry({ store: sessionStore, sessionKey: scope.sessionKey }) @@ -3199,7 +2894,7 @@ function resolveSessionTranscriptRuntimeContext( } /** - * Resolves the current file-backed target for read-only transcript callers. + * Resolves the current storage-neutral target for read-only transcript callers. * Unlike writer/runtime resolution, this does not persist missing sessionFile * metadata; reader projections must not mutate session metadata. */ @@ -3215,49 +2910,57 @@ export function resolveSessionTranscriptReadTarget( ...(scope.sessionKey ? { sessionKey: scope.sessionKey } : {}), }; } + const entrySessionFile = scope.sessionEntry?.sessionFile?.trim(); + const entryMarker = parseSqliteSessionFileMarker(entrySessionFile); + if (entrySessionFile && entryMarker && entryMarker.sessionId === scope.sessionId) { + return { + agentId: scope.agentId ?? entryMarker.agentId, + sessionFile: entrySessionFile, + sessionId: scope.sessionId, + ...(scope.sessionKey ? { sessionKey: scope.sessionKey } : {}), + }; + } const agentId = scope.agentId ?? resolveAgentIdFromSessionKey(scope.sessionKey); if (!agentId) { throw new Error(`Cannot resolve transcript scope without an agent id: ${scope.sessionKey}`); } const storePath = resolveConcreteReadStorePath(scope.storePath); + if (!storePath) { + const resolvedStorePath = resolveStorePath(getRuntimeConfig().session?.store, { + agentId, + env: scope.env, + }); + return { + agentId, + sessionFile: formatSqliteSessionFileMarker({ + agentId, + sessionId: scope.sessionId, + storePath: resolvedStorePath, + }), + sessionId: scope.sessionId, + ...(scope.sessionKey ? { sessionKey: scope.sessionKey } : {}), + }; + } const resolvedStoreEntry = scope.sessionEntry || !scope.sessionKey ? undefined : storePath ? resolveSessionStoreEntry({ - store: loadSessionStore(storePath, { skipCache: true }), + store: Object.fromEntries( + listSessionEntries({ agentId, storePath }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), + ), sessionKey: scope.sessionKey, }) : undefined; - const sessionEntry = - scope.sessionEntry ?? - resolvedStoreEntry?.existing ?? - (scope.sessionKey ? loadSessionEntry({ ...scope, sessionKey: scope.sessionKey }) : undefined); const sessionKey = resolvedStoreEntry?.normalizedKey ?? scope.sessionKey; - const matchingSessionEntry = - sessionEntry?.sessionId === undefined || sessionEntry.sessionId === scope.sessionId - ? sessionEntry - : undefined; - const threadId = - scope.threadId ?? (sessionKey ? parseSessionThreadInfo(sessionKey).threadId : undefined); - const sessionFile = matchingSessionEntry?.sessionFile - ? resolveSessionFilePath( - scope.sessionId, - matchingSessionEntry, - resolveSessionFilePathOptions({ - agentId, - ...(storePath ? { storePath } : {}), - }), - ) - : storePath - ? resolveSessionTranscriptPathInDir( - // File-backed readers derive beside sessions.json only for the JSON-store - // deprecation window; the SQLite flip resolves from canonical metadata. - scope.sessionId, - path.dirname(path.resolve(storePath)), - threadId, - ) - : resolveSessionTranscriptPath(scope.sessionId, agentId, threadId); + const sessionFile = formatSqliteSessionFileMarker({ + agentId, + sessionId: scope.sessionId, + storePath: storePath ?? "", + }); return { agentId, sessionFile, @@ -3274,25 +2977,18 @@ function resolveConcreteReadStorePath(storePath: string | undefined): string | u return trimmed; } -function createFallbackSessionEntry(patch: Partial<SessionEntry>): SessionEntry { - const now = Date.now(); - return { - sessionId: patch.sessionId ?? randomUUID(), - updatedAt: patch.updatedAt ?? now, - ...patch, - }; -} - function snapshotTemporarySessionMapping( scope: SessionAccessScope, ): TemporarySessionMappingSnapshot { - const storePath = resolveSessionStorePathForScope(scope); + const storePath = resolveAccessStorePath(scope); try { - const store = loadSessionStore(storePath, { skipCache: true }); - const entry = store[scope.sessionKey]; + const exact = loadExactSessionEntry({ + ...scope, + storePath, + }); return { canRestore: true, - ...(entry ? { entry: structuredClone(entry), hadEntry: true } : { hadEntry: false }), + ...(exact ? { entry: structuredClone(exact.entry), hadEntry: true } : { hadEntry: false }), sessionKey: scope.sessionKey, storePath, }; @@ -3313,17 +3009,19 @@ async function restoreTemporarySessionMapping( return undefined; } try { - await updateSessionStore( - snapshot.storePath, - (store) => { - if (snapshot.hadEntry) { - store[snapshot.sessionKey] = structuredClone(snapshot.entry); - return; - } - delete store[snapshot.sessionKey]; - }, - { activeSessionKey: snapshot.sessionKey }, - ); + if (snapshot.hadEntry) { + await replaceSessionEntry( + { sessionKey: snapshot.sessionKey, storePath: snapshot.storePath }, + structuredClone(snapshot.entry), + ); + } else { + await applySessionEntryLifecycleMutation({ + storePath: snapshot.storePath, + removals: [{ sessionKey: snapshot.sessionKey }], + activeSessionKey: snapshot.sessionKey, + skipMaintenance: true, + }); + } return undefined; } catch (err) { return formatErrorMessage(err); @@ -3358,66 +3056,15 @@ async function archivePreviousSessionTranscript(params: { }); } -type ResolvedTranscriptAccess = { - sessionFile: string; - target?: SessionTranscriptUpdateTarget; -}; - -function projectTranscriptUpdateTarget( - target: Pick<Partial<SessionTranscriptRuntimeTarget>, "agentId" | "sessionId" | "sessionKey">, -): SessionTranscriptUpdateTarget | undefined { - if (!target.agentId || !target.sessionId || !target.sessionKey) { - return undefined; +function resolveAccessStorePath(scope: SessionAccessScope): string { + if (scope.storePath) { + return scope.storePath; } - return { - agentId: target.agentId, - sessionId: target.sessionId, - sessionKey: target.sessionKey, - }; -} - -async function resolveTranscriptAccess( - scope: SessionTranscriptWriteScope, -): Promise<ResolvedTranscriptAccess> { - if (scope.sessionFile?.trim()) { - const scopeSessionKey = scope.sessionKey?.trim(); - const agentId = scopeSessionKey - ? (scope.agentId ?? resolveAgentIdFromSessionKey(scopeSessionKey)) - : undefined; - return { - sessionFile: scope.sessionFile, - ...(agentId && scope.sessionId && scopeSessionKey - ? { - target: projectTranscriptUpdateTarget({ - agentId, - sessionId: scope.sessionId, - sessionKey: scopeSessionKey, - }), - } - : {}), - }; - } - if (!scope.sessionId) { - throw new Error(`Cannot resolve transcript scope without a session id: ${scope.sessionKey}`); - } - // Past this point resolution goes through the session entry, so the owning - // key is mandatory; explicit-artifact writes returned above never need it. - const scopeSessionKey = scope.sessionKey?.trim(); - if (!scopeSessionKey) { - throw new Error( - "Cannot resolve a transcript write scope without a session key or explicit session file", - ); - } - const target = await resolveSessionTranscriptRuntimeTarget({ - ...scope, - sessionId: scope.sessionId, - sessionKey: scopeSessionKey, + const agentId = scope.agentId ?? resolveAgentIdFromSessionKey(scope.sessionKey); + return resolveStorePath(getRuntimeConfig().session?.store, { + agentId, + env: scope.env, }); - const updateTarget = projectTranscriptUpdateTarget(target); - return { - sessionFile: target.sessionFile, - ...(updateTarget ? { target: updateTarget } : {}), - }; } async function resolveTranscriptTurnTarget( @@ -3430,12 +3077,17 @@ async function resolveTranscriptTurnTarget( sessionEntry: SessionEntry | undefined; } > { - if (scope.sessionFile?.trim()) { + if (shouldUseExplicitTranscriptFile(scope)) { + const marker = parseSqliteSessionFileMarker(scope.sessionFile); + const agentId = scope.agentId ?? marker?.agentId; + const sessionId = scope.sessionId ?? marker?.sessionId; + const storePath = scope.storePath ?? marker?.storePath; return { - ...(scope.agentId ? { agentId: scope.agentId } : {}), - sessionFile: scope.sessionFile, - ...(scope.sessionId ? { sessionId: scope.sessionId } : {}), + ...(agentId ? { agentId } : {}), + sessionFile: scope.sessionFile.trim(), + ...(sessionId ? { sessionId } : {}), ...(scope.sessionKey ? { sessionKey: scope.sessionKey } : {}), + ...(storePath ? { storePath } : {}), sessionEntry: scope.sessionEntry, }; } @@ -3449,30 +3101,58 @@ async function resolveTranscriptTurnTarget( if (!agentId) { throw new Error(`Cannot resolve transcript turn without an agent id: ${sessionKey}`); } + const storePath = + scope.storePath ?? + resolveStorePath(getRuntimeConfig().session?.store, { + agentId, + env: scope.env, + }); const store = scope.sessionStore ?? - (scope.storePath ? loadSessionStore(scope.storePath, { skipCache: true }) : undefined); + Object.fromEntries( + listSessionEntries({ + agentId, + storePath, + }).map(({ sessionKey: entryKey, entry }) => [entryKey, entry]), + ); const resolved = store ? resolveSessionStoreEntry({ store, sessionKey }) : undefined; const sessionEntry = - resolved?.existing ?? scope.sessionEntry ?? loadSessionEntry({ ...scope, sessionKey }); - const resolvedFile = await resolveSessionTranscriptFile({ + resolved?.existing ?? + scope.sessionEntry ?? + loadSessionEntry({ ...scope, agentId, sessionKey, storePath }); + const sessionFile = formatSqliteSessionFileMarker({ agentId, - sessionEntry, sessionId: scope.sessionId, - sessionKey, - ...(store ? { sessionStore: store } : {}), - ...(scope.storePath ? { storePath: scope.storePath } : {}), - ...(scope.threadId !== undefined ? { threadId: scope.threadId } : {}), + storePath, }); return { agentId, - sessionFile: resolvedFile.sessionFile, + sessionFile, sessionId: scope.sessionId, sessionKey: resolved?.normalizedKey ?? sessionKey, - sessionEntry: resolvedFile.sessionEntry, + storePath, + sessionEntry, }; } +function shouldUseExplicitTranscriptFile< + TScope extends { + sessionFile?: string; + sessionId?: string; + sessionKey?: string; + storePath?: string; + }, +>(scope: TScope): scope is TScope & { sessionFile: string } { + const explicitSessionFile = scope.sessionFile?.trim(); + if (!explicitSessionFile) { + return false; + } + const hasStoreIdentity = Boolean( + scope.storePath?.trim() && scope.sessionKey?.trim() && scope.sessionId?.trim(), + ); + return !hasStoreIdentity; +} + async function touchTranscriptTurnSessionEntry(params: { scope: SessionTranscriptWriteScope & { sessionEntry?: SessionEntry; @@ -3485,7 +3165,7 @@ async function touchTranscriptTurnSessionEntry(params: { }): Promise<SessionEntry | undefined> { if ( !params.shouldTouch || - !params.scope.storePath || + !params.target.storePath || !params.target.sessionKey || !params.target.sessionId ) { @@ -3495,7 +3175,8 @@ async function touchTranscriptTurnSessionEntry(params: { const updated = await updateSessionEntry( { sessionKey: params.target.sessionKey, - storePath: params.scope.storePath, + storePath: params.target.storePath, + ...(params.target.agentId ? { agentId: params.target.agentId } : {}), }, (current) => current.sessionId === params.target.sessionId @@ -3525,9 +3206,18 @@ async function publishTranscriptTurnUpdate(params: { if (params.publishWhen === "when-appended" && !lastAppended) { return; } + const target = + params.target.agentId && params.target.sessionId && params.target.sessionKey + ? { + agentId: params.target.agentId, + sessionId: params.target.sessionId, + sessionKey: params.target.sessionKey, + } + : undefined; emitSessionTranscriptUpdate({ ...(params.target.sessionKey ? { sessionKey: params.target.sessionKey } : {}), ...(params.target.agentId ? { agentId: params.target.agentId } : {}), + ...(target ? { target } : {}), ...(params.updateMode === "inline" && lastAppended ? { message: lastAppended.message, diff --git a/src/config/sessions/session-file.ts b/src/config/sessions/session-file.ts index 9f03875a9f32..7c6c453016ba 100644 --- a/src/config/sessions/session-file.ts +++ b/src/config/sessions/session-file.ts @@ -1,10 +1,10 @@ -// Session file persistence resolves transcript paths and syncs store metadata. -import { resolveSessionFilePath } from "./paths.js"; -import type { ResolvedSessionMaintenanceConfig } from "./store-maintenance.js"; -import { updateSessionStore } from "./store.js"; +// Session file persistence syncs active session transcript markers into store metadata. +import { normalizeAgentId } from "../../routing/session-key.js"; +import { upsertSessionEntry } from "./session-accessor.js"; +import { formatSqliteSessionFileMarker } from "./sqlite-marker.js"; import type { SessionEntry } from "./types.js"; -/** Resolves a transcript file path and persists it into the session store when needed. */ +/** Resolves the active SQLite transcript marker and persists it into the session store when needed. */ export async function resolveAndPersistSessionFile(params: { sessionId: string; sessionKey: string; @@ -12,29 +12,15 @@ export async function resolveAndPersistSessionFile(params: { storePath: string; sessionEntry?: SessionEntry; agentId?: string; - sessionsDir?: string; - fallbackSessionFile?: string; - activeSessionKey?: string; - maintenanceConfig?: ResolvedSessionMaintenanceConfig; }): Promise<{ sessionFile: string; sessionEntry: SessionEntry }> { const { sessionId, sessionKey, sessionStore, storePath } = params; const now = Date.now(); const baseEntry = params.sessionEntry ?? sessionStore[sessionKey] ?? { sessionId, updatedAt: now, sessionStartedAt: now }; - const shouldReusePersistedSessionFile = baseEntry.sessionId === sessionId; - const fallbackSessionFile = params.fallbackSessionFile?.trim(); - // A reset/fork should not reuse the previous transcript path unless the fallback explicitly - // points at the intended file for the new session id. - const entryForResolve = !shouldReusePersistedSessionFile - ? fallbackSessionFile - ? { ...baseEntry, sessionFile: fallbackSessionFile } - : { ...baseEntry, sessionFile: undefined } - : !baseEntry.sessionFile && fallbackSessionFile - ? { ...baseEntry, sessionFile: fallbackSessionFile } - : baseEntry; - const sessionFile = resolveSessionFilePath(sessionId, entryForResolve, { - agentId: params.agentId, - sessionsDir: params.sessionsDir, + const sessionFile = formatSqliteSessionFileMarker({ + agentId: normalizeAgentId(params.agentId), + sessionId, + storePath, }); const persistedEntry: SessionEntry = { ...baseEntry, @@ -45,21 +31,7 @@ export async function resolveAndPersistSessionFile(params: { }; if (baseEntry.sessionId !== sessionId || baseEntry.sessionFile !== sessionFile) { sessionStore[sessionKey] = persistedEntry; - await updateSessionStore( - storePath, - (store) => { - store[sessionKey] = { - ...store[sessionKey], - ...persistedEntry, - }; - }, - params.activeSessionKey || params.maintenanceConfig - ? { - ...(params.activeSessionKey ? { activeSessionKey: params.activeSessionKey } : {}), - ...(params.maintenanceConfig ? { maintenanceConfig: params.maintenanceConfig } : {}), - } - : undefined, - ); + await upsertSessionEntry({ storePath, sessionKey }, persistedEntry); return { sessionFile, sessionEntry: persistedEntry }; } sessionStore[sessionKey] = persistedEntry; diff --git a/src/config/sessions/session-fork-transcript.runtime.ts b/src/config/sessions/session-fork-transcript.runtime.ts deleted file mode 100644 index 7408400f7643..000000000000 --- a/src/config/sessions/session-fork-transcript.runtime.ts +++ /dev/null @@ -1,186 +0,0 @@ -// Parent-fork transcript source reading and branch-entry construction. -// The session accessor composes these helpers with createForkedSessionTranscript -// so parent forks stay one storage-owned operation (#88838). -import crypto from "node:crypto"; -import path from "node:path"; -import { isRecord } from "@openclaw/normalization-core/record-coerce"; -import { - migrateSessionEntries, - parseSessionEntries, - type SessionEntry as AgentSessionEntry, - type SessionHeader, -} from "../../agents/sessions/session-manager.js"; -import { readRegularFile } from "../../infra/fs-safe.js"; -import { - isSessionTranscriptLeafControl, - mergeSessionTranscriptVisiblePathWithOpaqueAppendPath, - scanSessionTranscriptTree, - selectSessionTranscriptTreePathNodes, -} from "./transcript-tree.js"; - -/** Active-branch snapshot of a parent transcript selected for forking. */ -export type ForkSourceTranscript = { - cwd: string; - sessionDir: string; - leafId: string | null; - appendParentId: string | null; - appendMode?: "side"; - preserveLeafControl: boolean; - branchEntries: unknown[]; - labelsToWrite: Array<{ targetId: string; label: string; timestamp: string }>; -}; - -function generateEntryId(existingIds: Set<string>): string { - for (let attempt = 0; attempt < 100; attempt += 1) { - const id = crypto.randomUUID().slice(0, 8); - if (!existingIds.has(id)) { - existingIds.add(id); - return id; - } - } - const id = crypto.randomUUID(); - existingIds.add(id); - return id; -} - -/** True when the selected branch carries at least one assistant message. */ -export function forkSourceHasAssistantEntry(entries: unknown[]): boolean { - return entries.some( - (entry) => - isRecord(entry) && - entry.type === "message" && - isRecord(entry.message) && - entry.message.role === "assistant", - ); -} - -function collectBranchLabels(params: { - allEntries: unknown[]; - pathEntryIds: Set<string>; -}): Array<{ targetId: string; label: string; timestamp: string }> { - const labelsToWrite: Array<{ targetId: string; label: string; timestamp: string }> = []; - for (const entry of params.allEntries) { - if (!isRecord(entry)) { - continue; - } - if ( - entry.type === "label" && - typeof entry.label === "string" && - typeof entry.targetId === "string" && - typeof entry.id === "string" && - !params.pathEntryIds.has(entry.id) && - params.pathEntryIds.has(entry.targetId) && - typeof entry.timestamp === "string" - ) { - labelsToWrite.push({ - targetId: entry.targetId, - label: entry.label, - timestamp: entry.timestamp, - }); - } - } - return labelsToWrite; -} - -/** Reads the parent transcript and selects the active branch to copy. */ -export async function readForkSourceTranscript( - parentSessionFile: string, -): Promise<ForkSourceTranscript | null> { - const raw = (await readRegularFile({ filePath: parentSessionFile })).buffer.toString("utf-8"); - const fileEntries = parseSessionEntries(raw); - migrateSessionEntries(fileEntries); - const header = - fileEntries.find((entry): entry is SessionHeader => entry.type === "session") ?? null; - const entries = fileEntries.filter((entry) => entry.type !== "session"); - const tree = scanSessionTranscriptTree(entries); - const leafId = tree.leafId; - const appendParentId = tree.appendParentId; - const visiblePath = selectSessionTranscriptTreePathNodes(tree, leafId); - const appendPath = selectSessionTranscriptTreePathNodes(tree, appendParentId); - const mergedPath = mergeSessionTranscriptVisiblePathWithOpaqueAppendPath({ - visiblePath, - appendPath, - appendParentId, - }); - const branchEntries = mergedPath.nodes.flatMap((node) => { - if (!isRecord(node.entry)) { - return []; - } - const parentId = node.selectedParentId; - return [node.entry.parentId === parentId ? node.entry : { ...node.entry, parentId }]; - }); - const pathEntryIds = new Set( - branchEntries.flatMap((entry) => - isRecord(entry) && typeof entry.id === "string" ? [entry.id] : [], - ), - ); - const lastLeafUpdateNode = tree.nodes.findLast((node) => node.leafId !== undefined); - const lastLeafUpdateEntry = lastLeafUpdateNode?.entry; - return { - cwd: header?.cwd ?? process.cwd(), - sessionDir: path.dirname(parentSessionFile), - leafId, - appendParentId: mergedPath.appendParentId, - ...(lastLeafUpdateNode?.appendMode ? { appendMode: lastLeafUpdateNode.appendMode } : {}), - preserveLeafControl: isSessionTranscriptLeafControl(lastLeafUpdateEntry), - branchEntries, - labelsToWrite: collectBranchLabels({ allEntries: entries, pathEntryIds }), - }; -} - -function buildBranchLabelEntries(params: { - labelsToWrite: Array<{ targetId: string; label: string; timestamp: string }>; - pathEntryIds: Set<string>; - lastEntryId: string | null; -}): AgentSessionEntry[] { - let parentId = params.lastEntryId; - const labelEntries: AgentSessionEntry[] = []; - for (const { targetId, label, timestamp } of params.labelsToWrite) { - const labelEntry = { - type: "label", - id: generateEntryId(params.pathEntryIds), - parentId, - timestamp, - targetId, - label, - } satisfies AgentSessionEntry; - params.pathEntryIds.add(labelEntry.id); - labelEntries.push(labelEntry); - parentId = labelEntry.id; - } - return labelEntries; -} - -/** Builds the copied branch, re-targeted labels, and optional leaf control for a fork. */ -export function buildForkedBranchEntries(params: { - source: ForkSourceTranscript; - /** Fork header timestamp; the leaf control shares it so the copied branch reopens on the parent's active leaf. */ - timestamp: string; -}): unknown[] { - const pathEntries = params.source.branchEntries; - const pathEntryIds = new Set( - pathEntries.flatMap((entry) => - isRecord(entry) && typeof entry.id === "string" ? [entry.id] : [], - ), - ); - const lastPathEntry = pathEntries.at(-1); - const lastPathEntryId = - isRecord(lastPathEntry) && typeof lastPathEntry.id === "string" ? lastPathEntry.id : null; - const labelEntries = buildBranchLabelEntries({ - labelsToWrite: params.source.labelsToWrite, - pathEntryIds, - lastEntryId: lastPathEntryId, - }); - const leafEntry = params.source.preserveLeafControl - ? { - type: "leaf", - id: generateEntryId(pathEntryIds), - parentId: labelEntries.at(-1)?.id ?? lastPathEntryId, - timestamp: params.timestamp, - targetId: params.source.leafId, - appendParentId: params.source.appendParentId, - ...(params.source.appendMode ? { appendMode: params.source.appendMode } : {}), - } - : null; - return [...pathEntries, ...labelEntries, ...(leafEntry ? [leafEntry] : [])]; -} diff --git a/src/config/sessions/session-registry-maintenance.test.ts b/src/config/sessions/session-registry-maintenance.test.ts index ed1d8ca9b5dd..b75020784d6e 100644 --- a/src/config/sessions/session-registry-maintenance.test.ts +++ b/src/config/sessions/session-registry-maintenance.test.ts @@ -4,8 +4,9 @@ import path from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { beginSessionWorkAdmission } from "../../sessions/session-lifecycle-admission.js"; import { createFixtureSuite } from "../../test-utils/fixture-suite.js"; +import { loadSessionEntry, replaceSessionEntry } from "./session-accessor.js"; import { runSessionRegistryMaintenanceForStore } from "./session-registry-maintenance.js"; -import { loadSessionStore, saveSessionStore } from "./store.js"; +import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js"; import type { SessionEntry } from "./types.js"; const DAY_MS = 24 * 60 * 60 * 1000; @@ -27,14 +28,25 @@ async function createStore(entries: Record<string, SessionEntry>): Promise<strin const dir = await fixtureSuite.createCaseDir("store"); const storePath = path.join(dir, "sessions.json"); await fs.mkdir(dir, { recursive: true }); - await saveSessionStore(storePath, entries, { skipMaintenance: true }); + for (const [sessionKey, entry] of Object.entries(entries)) { + await replaceSessionEntry({ sessionKey, storePath }, entry); + } return storePath; } +function resolveRequiredSqlitePath(storePath: string): string { + const sqlitePath = resolveSqliteTargetFromSessionStorePath(storePath).path; + if (!sqlitePath) { + throw new Error(`Expected a SQLite target for ${storePath}`); + } + return sqlitePath; +} + describe("runSessionRegistryMaintenanceForStore", () => { it("summarizes a missing store without creating it", async () => { const dir = await fixtureSuite.createCaseDir("missing-store"); const storePath = path.join(dir, "sessions.json"); + const sqlitePath = resolveRequiredSqlitePath(storePath); const result = await runSessionRegistryMaintenanceForStore({ apply: true, @@ -50,6 +62,28 @@ describe("runSessionRegistryMaintenanceForStore", () => { pruned: 0, }); await expect(fs.stat(storePath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.stat(sqlitePath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("previews a missing store without creating SQLite state", async () => { + const dir = await fixtureSuite.createCaseDir("missing-store-preview"); + const storePath = path.join(dir, "sessions.json"); + const sqlitePath = resolveRequiredSqlitePath(storePath); + + const result = await runSessionRegistryMaintenanceForStore({ + apply: false, + retentionMs: 7 * DAY_MS, + runningCronJobIds: new Set(), + storePath, + }); + + expect(result).toEqual({ + beforeCount: 0, + afterCount: 0, + preservedRunning: 0, + pruned: 0, + }); + await expect(fs.stat(sqlitePath)).rejects.toMatchObject({ code: "ENOENT" }); }); it("previews stale cron-run pruning without mutating the store", async () => { @@ -72,9 +106,9 @@ describe("runSessionRegistryMaintenanceForStore", () => { preservedRunning: 0, pruned: 1, }); - expect(loadSessionStore(storePath, { skipCache: true })).toHaveProperty( - "agent:main:cron:done-job:run:old-run", - ); + expect( + loadSessionEntry({ sessionKey: "agent:main:cron:done-job:run:old-run", storePath }), + ).toEqual(sessionEntry("old-run", now - 8 * DAY_MS)); }); it("applies one store-sized pruning transaction and preserves running cron rows", async () => { @@ -98,10 +132,15 @@ describe("runSessionRegistryMaintenanceForStore", () => { preservedRunning: 1, pruned: 1, }); - const updated = loadSessionStore(storePath, { skipCache: true }); - expect(updated["agent:main:cron:done-job:run:old-run"]).toBeUndefined(); - expect(updated).toHaveProperty("agent:main:cron:running-job:run:old-run"); - expect(updated).toHaveProperty("agent:main:cron:done-job:run:recent-run"); + expect( + loadSessionEntry({ sessionKey: "agent:main:cron:done-job:run:old-run", storePath }), + ).toBeUndefined(); + expect( + loadSessionEntry({ sessionKey: "agent:main:cron:running-job:run:old-run", storePath }), + ).toEqual(sessionEntry("running-run", now - 8 * DAY_MS)); + expect( + loadSessionEntry({ sessionKey: "agent:main:cron:done-job:run:recent-run", storePath }), + ).toEqual(sessionEntry("recent-run", now)); }); it("skips generic session maintenance while applying task registry pruning", async () => { @@ -120,9 +159,12 @@ describe("runSessionRegistryMaintenanceForStore", () => { }); expect(result.pruned).toBe(1); - const updated = loadSessionStore(storePath, { skipCache: true }); - expect(updated["agent:main:cron:done-job:run:old-run"]).toBeUndefined(); - expect(updated).toHaveProperty(oldOrdinaryKey); + expect( + loadSessionEntry({ sessionKey: "agent:main:cron:done-job:run:old-run", storePath }), + ).toBeUndefined(); + expect(loadSessionEntry({ sessionKey: oldOrdinaryKey, storePath })).toEqual( + sessionEntry("old-worker", now - 40 * DAY_MS), + ); }); it("preserves active cron rows until their admission releases", async () => { @@ -146,7 +188,7 @@ describe("runSessionRegistryMaintenanceForStore", () => { storePath, }); expect(activeResult.pruned).toBe(0); - expect(loadSessionStore(storePath, { skipCache: true })).toHaveProperty(sessionKey); + expect(loadSessionEntry({ sessionKey, storePath })).toBeDefined(); } finally { admission.release(); } @@ -158,6 +200,6 @@ describe("runSessionRegistryMaintenanceForStore", () => { storePath, }); expect(releasedResult.pruned).toBe(1); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toBeUndefined(); + expect(loadSessionEntry({ sessionKey, storePath })).toBeUndefined(); }); }); diff --git a/src/config/sessions/session-registry-maintenance.ts b/src/config/sessions/session-registry-maintenance.ts index 872d40f17538..48807f9d4a96 100644 --- a/src/config/sessions/session-registry-maintenance.ts +++ b/src/config/sessions/session-registry-maintenance.ts @@ -1,8 +1,14 @@ // Storage-neutral session registry maintenance for task-owned cron run cleanup. import fs from "node:fs"; import { parseAgentSessionKey } from "../../sessions/session-key-utils.js"; +import { + applySessionEntryLifecycleMutation, + listSessionEntries, + type SessionEntryLifecycleRemoval, +} from "./session-accessor.js"; +import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js"; import { collectActiveSessionWorkAdmissionKeys } from "./store-maintenance-preserve.js"; -import { loadSessionStore, pruneStaleEntries, updateSessionStore } from "./store.js"; +import { pruneStaleEntries } from "./store.js"; import type { SessionEntry } from "./types.js"; export type SessionRegistryMaintenanceStoreSummary = { @@ -59,6 +65,7 @@ function buildSessionRegistryPreserveKeys(params: { function pruneSessionRegistryStore(params: { retentionMs: number; + removals?: SessionEntryLifecycleRemoval[]; runningCronJobIds: ReadonlySet<string>; storePath: string; store: Record<string, SessionEntry>; @@ -70,6 +77,11 @@ function pruneSessionRegistryStore(params: { }); const pruned = pruneStaleEntries(params.store, params.retentionMs, { log: false, + onPruned: params.removals + ? ({ key, entry }) => { + params.removals?.push({ sessionKey: key, expectedEntry: entry }); + } + : undefined, preserveKeys, }); return { @@ -87,16 +99,21 @@ function pruneSessionRegistryStore(params: { export async function runSessionRegistryMaintenanceForStore( params: SessionRegistryMaintenanceStoreOptions, ): Promise<SessionRegistryMaintenanceStoreSummary> { - if (!fs.existsSync(params.storePath)) { + const sqliteTarget = resolveSqliteTargetFromSessionStorePath(params.storePath); + if (sqliteTarget.path && !fs.existsSync(sqliteTarget.path)) { return { - afterCount: 0, beforeCount: 0, + afterCount: 0, preservedRunning: 0, pruned: 0, }; } - - const beforeStore = loadSessionStore(params.storePath, { skipCache: true }); + const beforeStore = Object.fromEntries( + listSessionEntries({ storePath: params.storePath }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), + ); const beforeCount = Object.keys(beforeStore).length; if (!params.apply) { const previewStore = structuredClone(beforeStore); @@ -111,17 +128,28 @@ export async function runSessionRegistryMaintenanceForStore( }; } - const applied = await updateSessionStore( - params.storePath, - (store) => - pruneSessionRegistryStore({ - retentionMs: params.retentionMs, - runningCronJobIds: params.runningCronJobIds, - storePath: params.storePath, - store, - }), - { skipMaintenance: true }, - ); + const applyStore = structuredClone(beforeStore); + const removals: SessionEntryLifecycleRemoval[] = []; + const applied = pruneSessionRegistryStore({ + retentionMs: params.retentionMs, + removals, + runningCronJobIds: params.runningCronJobIds, + storePath: params.storePath, + store: applyStore, + }); + if (removals.length > 0) { + const mutation = await applySessionEntryLifecycleMutation({ + storePath: params.storePath, + removals, + skipMaintenance: true, + }); + return { + afterCount: mutation.afterCount, + beforeCount, + preservedRunning: applied.preservedRunning, + pruned: mutation.removedEntries, + }; + } return { beforeCount, ...applied, diff --git a/src/config/sessions/session-sqlite-target.test.ts b/src/config/sessions/session-sqlite-target.test.ts new file mode 100644 index 000000000000..0dbf82125c56 --- /dev/null +++ b/src/config/sessions/session-sqlite-target.test.ts @@ -0,0 +1,38 @@ +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js"; + +describe("resolveSqliteTargetFromSessionStorePath", () => { + it("keeps custom store targets distinct when templates share a directory", () => { + const dir = path.join("tmp", "stores"); + + expect(resolveSqliteTargetFromSessionStorePath(path.join(dir, "main.json"))).toMatchObject({ + path: path.resolve(dir, "main.sqlite"), + }); + expect(resolveSqliteTargetFromSessionStorePath(path.join(dir, "worker.json"))).toMatchObject({ + path: path.resolve(dir, "worker.sqlite"), + }); + }); + + it("keeps shared custom store targets distinct by agent", () => { + const storePath = path.join("tmp", "stores", "shared-sessions.json"); + + expect(resolveSqliteTargetFromSessionStorePath(storePath, { agentId: "main" })).toMatchObject({ + path: path.resolve("tmp", "stores", "shared-sessions.sqlite"), + }); + expect(resolveSqliteTargetFromSessionStorePath(storePath, { agentId: "work" })).toMatchObject({ + path: path.resolve("tmp", "stores", "shared-sessions.work.sqlite"), + }); + }); + + it("keeps shared custom sessions.json targets distinct by agent", () => { + const storePath = path.join("tmp", "stores", "sessions.json"); + + expect(resolveSqliteTargetFromSessionStorePath(storePath, { agentId: "main" })).toMatchObject({ + path: path.resolve("tmp", "stores", "openclaw-agent.sqlite"), + }); + expect(resolveSqliteTargetFromSessionStorePath(storePath, { agentId: "work" })).toMatchObject({ + path: path.resolve("tmp", "stores", "openclaw-agent.work.sqlite"), + }); + }); +}); diff --git a/src/config/sessions/session-sqlite-target.ts b/src/config/sessions/session-sqlite-target.ts new file mode 100644 index 000000000000..b5bb1506e220 --- /dev/null +++ b/src/config/sessions/session-sqlite-target.ts @@ -0,0 +1,88 @@ +import path from "node:path"; +import { DEFAULT_AGENT_ID, normalizeAgentId } from "../../routing/session-key.js"; + +/** SQLite database target resolved from a legacy session store path. */ +export type ResolvedSqliteStoreTarget = { + agentId?: string; + path?: string; +}; + +function resolveCustomStoreSqlitePath(params: { + agentId?: string; + sqliteBaseName?: string; + storePath: string; +}): string { + const resolved = path.resolve(params.storePath); + const sessionsDir = path.dirname(resolved); + const sqliteBaseName = + params.sqliteBaseName ?? (path.basename(resolved, path.extname(resolved)) || "openclaw-agent"); + const agentId = params.agentId ? normalizeAgentId(params.agentId) : undefined; + const sqliteName = + agentId && agentId !== DEFAULT_AGENT_ID && normalizeAgentId(sqliteBaseName) !== agentId + ? `${sqliteBaseName}.${agentId}` + : sqliteBaseName; + return path.join(sessionsDir, `${sqliteName}.sqlite`); +} + +/** Resolves the SQLite database target that owns a legacy session store path. */ +export function resolveSqliteTargetFromSessionStorePath( + storePath: string, + options: { agentId?: string } = {}, +): ResolvedSqliteStoreTarget { + const resolved = path.resolve(storePath); + if (path.basename(resolved) === "openclaw-agent.sqlite" || resolved.endsWith(".sqlite")) { + const agentId = resolveAgentIdFromSqliteDatabasePath(resolved); + return { + path: resolved, + ...(agentId ? { agentId } : {}), + }; + } + const sessionsDir = path.dirname(resolved); + if (path.basename(resolved) !== "sessions.json") { + return { + path: resolveCustomStoreSqlitePath({ + ...(options.agentId ? { agentId: options.agentId } : {}), + storePath: resolved, + }), + }; + } + if (path.basename(sessionsDir) !== "sessions") { + return { + path: resolveCustomStoreSqlitePath({ + ...(options.agentId ? { agentId: options.agentId } : {}), + sqliteBaseName: "openclaw-agent", + storePath: resolved, + }), + }; + } + const agentDir = path.dirname(sessionsDir); + if (path.basename(path.dirname(agentDir)) !== "agents") { + return { + path: resolveCustomStoreSqlitePath({ + ...(options.agentId ? { agentId: options.agentId } : {}), + sqliteBaseName: "openclaw-agent", + storePath: resolved, + }), + }; + } + return { + agentId: normalizeAgentId(path.basename(agentDir)), + path: path.join(agentDir, "agent", "openclaw-agent.sqlite"), + }; +} + +/** Extracts the agent id from the canonical per-agent SQLite database path. */ +export function resolveAgentIdFromSqliteDatabasePath(databasePath: string): string | undefined { + if (path.basename(databasePath) !== "openclaw-agent.sqlite") { + return undefined; + } + const agentDbDir = path.dirname(databasePath); + if (path.basename(agentDbDir) !== "agent") { + return undefined; + } + const agentDir = path.dirname(agentDbDir); + if (path.basename(path.dirname(agentDir)) !== "agents") { + return undefined; + } + return normalizeAgentId(path.basename(agentDir)); +} diff --git a/src/config/sessions/sessions.test.ts b/src/config/sessions/sessions.test.ts index c582f4b9a410..368989ccdf4e 100644 --- a/src/config/sessions/sessions.test.ts +++ b/src/config/sessions/sessions.test.ts @@ -17,7 +17,9 @@ import { validateSessionId, } from "./paths.js"; import { evaluateSessionFreshness, resolveSessionResetPolicy } from "./reset.js"; +import { loadSessionEntry } from "./session-accessor.js"; import { resolveAndPersistSessionFile } from "./session-file.js"; +import { formatSqliteSessionFileMarker } from "./sqlite-marker.js"; import { readSessionStoreCache, writeSessionStoreCache } from "./store-cache.js"; import { clearSessionStoreCacheForTest, @@ -1091,15 +1093,14 @@ describe("session store writer queue", () => { }); expect(result?.acp).toBeUndefined(); - const store = loadSessionStore(storePath); - expect(store[key]?.acp).toBeUndefined(); + expect(loadSessionEntry({ storePath, sessionKey: key })?.acp).toBeUndefined(); }); }); describe("resolveAndPersistSessionFile", () => { const fixture = useTempSessionsFixture("session-file-test-"); - it("persists fallback topic transcript paths for sessions without sessionFile", async () => { + it("persists SQLite transcript markers for sessions without sessionFile", async () => { const sessionId = "topic-session-id"; const sessionKey = "agent:main:telegram:group:123:topic:456"; const store = { @@ -1110,11 +1111,11 @@ describe("resolveAndPersistSessionFile", () => { }; fs.writeFileSync(fixture.storePath(), JSON.stringify(store), "utf-8"); const sessionStore = loadSessionStore(fixture.storePath(), { skipCache: true }); - const fallbackSessionFile = resolveSessionTranscriptPathInDir( + const expectedSessionFile = formatSqliteSessionFileMarker({ + agentId: "main", sessionId, - fixture.sessionsDir(), - 456, - ); + storePath: fixture.storePath(), + }); const result = await resolveAndPersistSessionFile({ sessionId, @@ -1122,13 +1123,14 @@ describe("resolveAndPersistSessionFile", () => { sessionStore, storePath: fixture.storePath(), sessionEntry: sessionStore[sessionKey], - fallbackSessionFile, + agentId: "main", }); - expect(result.sessionFile).toBe(fallbackSessionFile); + expect(result.sessionFile).toBe(expectedSessionFile); - const saved = loadSessionStore(fixture.storePath(), { skipCache: true }); - expect(saved[sessionKey]?.sessionFile).toBe(fallbackSessionFile); + expect(loadSessionEntry({ storePath: fixture.storePath(), sessionKey })?.sessionFile).toBe( + expectedSessionFile, + ); }); it("creates and persists entry when session is not yet present", async () => { @@ -1136,23 +1138,28 @@ describe("resolveAndPersistSessionFile", () => { const sessionKey = "agent:main:telegram:group:123"; fs.writeFileSync(fixture.storePath(), JSON.stringify({}), "utf-8"); const sessionStore = loadSessionStore(fixture.storePath(), { skipCache: true }); - const fallbackSessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); + const expectedSessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath: fixture.storePath(), + }); const result = await resolveAndPersistSessionFile({ sessionId, sessionKey, sessionStore, storePath: fixture.storePath(), - fallbackSessionFile, + agentId: "main", }); - expect(result.sessionFile).toBe(fallbackSessionFile); + expect(result.sessionFile).toBe(expectedSessionFile); expect(result.sessionEntry.sessionId).toBe(sessionId); - const saved = loadSessionStore(fixture.storePath(), { skipCache: true }); - expect(saved[sessionKey]?.sessionFile).toBe(fallbackSessionFile); + expect(loadSessionEntry({ storePath: fixture.storePath(), sessionKey })?.sessionFile).toBe( + expectedSessionFile, + ); }); - it("rotates to a new transcript path when sessionId changes on the same session key", async () => { + it("rotates to a new SQLite transcript marker when sessionId changes on the same session key", async () => { const previousSessionId = "old-session-id"; const nextSessionId = "new-session-id"; const sessionKey = "agent:main:telegram:group:123"; @@ -1160,10 +1167,11 @@ describe("resolveAndPersistSessionFile", () => { previousSessionId, fixture.sessionsDir(), ); - const expectedNextSessionFile = resolveSessionTranscriptPathInDir( - nextSessionId, - fixture.sessionsDir(), - ); + const expectedNextSessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: nextSessionId, + storePath: fixture.storePath(), + }); const store = { [sessionKey]: { sessionId: previousSessionId, @@ -1180,14 +1188,15 @@ describe("resolveAndPersistSessionFile", () => { sessionStore, storePath: fixture.storePath(), sessionEntry: sessionStore[sessionKey], - sessionsDir: fixture.sessionsDir(), + agentId: "main", }); expect(result.sessionFile).toBe(expectedNextSessionFile); expect(result.sessionFile).not.toBe(previousSessionFile); expect(result.sessionEntry.sessionFile).toBe(expectedNextSessionFile); - const saved = loadSessionStore(fixture.storePath(), { skipCache: true }); - expect(saved[sessionKey]?.sessionFile).toBe(expectedNextSessionFile); + expect(loadSessionEntry({ storePath: fixture.storePath(), sessionKey })?.sessionFile).toBe( + expectedNextSessionFile, + ); }); }); diff --git a/src/config/sessions/sqlite-marker.ts b/src/config/sessions/sqlite-marker.ts new file mode 100644 index 000000000000..ee392add9439 --- /dev/null +++ b/src/config/sessions/sqlite-marker.ts @@ -0,0 +1,42 @@ +/** SQLite transcript sessionFile marker helpers shared by session runtime readers. */ +import path from "node:path"; + +export type SqliteSessionFileMarker = { + agentId: string; + sessionId: string; + storePath: string; +}; + +const SQLITE_SESSION_FILE_MARKER_RE = /^sqlite:([^:]+):([^:]+):(.*)$/; + +/** Formats the canonical sessionFile marker for SQLite-backed transcripts. */ +export function formatSqliteSessionFileMarker(marker: SqliteSessionFileMarker): string { + return `sqlite:${marker.agentId}:${marker.sessionId}:${path.resolve(marker.storePath)}`; +} + +/** Parses a SQLite-backed transcript sessionFile marker. */ +export function parseSqliteSessionFileMarker( + sessionFile: string | undefined, +): SqliteSessionFileMarker | undefined { + const marker = sessionFile?.trim(); + if (!marker?.startsWith("sqlite:")) { + return undefined; + } + const match = SQLITE_SESSION_FILE_MARKER_RE.exec(marker); + if (!match?.[1] || !match[2] || !match[3]) { + return undefined; + } + return { + agentId: match[1], + sessionId: match[2], + storePath: match[3], + }; +} + +/** Checks whether a sessionFile marker points at the expected session id. */ +export function sqliteSessionFileMarkerMatchesSession( + sessionFile: string | undefined, + sessionId: string, +): boolean { + return parseSqliteSessionFileMarker(sessionFile)?.sessionId === sessionId; +} diff --git a/src/config/sessions/store-load.ts b/src/config/sessions/store-load.ts index fdc2440acc92..5e3d858ae5f0 100644 --- a/src/config/sessions/store-load.ts +++ b/src/config/sessions/store-load.ts @@ -297,7 +297,12 @@ function sameDeliveryChannelRoute( ); } -function normalizeSessionEntryDelivery(entry: SessionEntry): SessionEntry { +/** + * Rebuilds malformed/legacy delivery `route` state from the entry's delivery + * fields. Runs on file-era store loads and on the doctor SQLite import so the + * SQLite store only holds canonical delivery shapes; SQLite reads do no repair. + */ +export function normalizeSessionEntryDelivery(entry: SessionEntry): SessionEntry { const entryRoute = normalizeDeliveryChannelRoute(entry.route); const normalized = normalizeSessionDeliveryFields({ route: entryRoute, diff --git a/src/config/sessions/store-maintenance-operations.ts b/src/config/sessions/store-maintenance-operations.ts index f4c7dd801516..e711d9c5d10a 100644 --- a/src/config/sessions/store-maintenance-operations.ts +++ b/src/config/sessions/store-maintenance-operations.ts @@ -167,7 +167,9 @@ async function cleanupRemovedSessionArtifacts(params: { restrictToStoreDir: true, }); } - if (archivedDirs.size === 0 && params.maintenance.resetArchiveRetentionMs == null) { + // null retention keeps archived transcripts: they are conversation history, + // and the disk budget (not a wall-clock timer) is the only eviction path. + if (params.maintenance.resetArchiveRetentionMs == null) { return; } const targetDirs = @@ -175,17 +177,13 @@ async function cleanupRemovedSessionArtifacts(params: { ? [...archivedDirs] : [path.dirname(path.resolve(params.operation.storePath))]; // Both retention reasons ride one cleanup call so each save enumerates the - // sessions dir at most once; reset retention defaults on, so a listing per - // reason would scan twice per save. + // sessions dir at most once; a listing per reason would scan twice per save. await params.operation.artifacts.cleanupArchivedSessionTranscripts({ directories: targetDirs, - rules: - params.maintenance.resetArchiveRetentionMs != null - ? [ - { reason: "deleted", olderThanMs: params.maintenance.pruneAfterMs }, - { reason: "reset", olderThanMs: params.maintenance.resetArchiveRetentionMs }, - ] - : [{ reason: "deleted", olderThanMs: params.maintenance.pruneAfterMs }], + rules: [ + { reason: "deleted", olderThanMs: params.maintenance.resetArchiveRetentionMs }, + { reason: "reset", olderThanMs: params.maintenance.resetArchiveRetentionMs }, + ], }); } diff --git a/src/config/sessions/store-maintenance.ts b/src/config/sessions/store-maintenance.ts index d8c8c60d8b01..ef131fa329bb 100644 --- a/src/config/sessions/store-maintenance.ts +++ b/src/config/sessions/store-maintenance.ts @@ -23,6 +23,10 @@ const DEFAULT_MODEL_RUN_PRUNE_AFTER_MS = 24 * 60 * 60 * 1000; const DEFAULT_SESSION_MAX_ENTRIES = 500; const DEFAULT_SESSION_MAINTENANCE_MODE: SessionMaintenanceMode = "enforce"; const DEFAULT_SESSION_DISK_BUDGET_HIGH_WATER_RATIO = 0.8; +// Archived transcripts are the user's conversation history: retention keeps +// them until the disk budget evicts oldest-first, never on a wall-clock timer. +// The budget default below is what makes "keep archives" still bounded. +const DEFAULT_SESSION_MAX_DISK_BYTES = 2 * 1024 * 1024 * 1024; const STRICT_ENTRY_MAINTENANCE_MAX_ENTRIES = 49; const MIN_BATCHED_ENTRY_MAINTENANCE_SLACK = 25; const BATCHED_ENTRY_MAINTENANCE_SLACK_RATIO = 0.1; @@ -68,32 +72,40 @@ function resolvePruneAfterMs(maintenance?: SessionMaintenanceConfig): number { function resolveResetArchiveRetentionMs( maintenance: SessionMaintenanceConfig | undefined, - pruneAfterMs: number, ): number | null { + // null = keep archived transcripts indefinitely (disk budget still evicts + // oldest-first under pressure). An explicit duration opts back into + // wall-clock deletion; parse failures stay on the keep side because losing + // history is the worse failure mode. const raw = maintenance?.resetArchiveRetention; if (raw === false) { return null; } const normalized = normalizeStringifiedOptionalString(raw); if (!normalized) { - return pruneAfterMs; + return null; } try { return parseDurationMs(normalized, { defaultUnit: "d" }); } catch { - return pruneAfterMs; + return null; } } function resolveMaxDiskBytes(maintenance?: SessionMaintenanceConfig): number | null { const raw = maintenance?.maxDiskBytes; + if (raw === false) { + return null; + } const normalized = normalizeStringifiedOptionalString(raw); if (!normalized) { - return null; + return DEFAULT_SESSION_MAX_DISK_BYTES; } try { return parseByteSize(normalized, { defaultUnit: "b" }); } catch { + // A malformed explicit value must not opt the user into destructive + // budget cleanup they never chose; disable the budget instead. return null; } } @@ -147,7 +159,7 @@ export function resolveMaintenanceConfigFromInput( pruneAfterMs, maxEntries: maintenance?.maxEntries ?? DEFAULT_SESSION_MAX_ENTRIES, modelRunPruneAfterMs: DEFAULT_MODEL_RUN_PRUNE_AFTER_MS, - resetArchiveRetentionMs: resolveResetArchiveRetentionMs(maintenance, pruneAfterMs), + resetArchiveRetentionMs: resolveResetArchiveRetentionMs(maintenance), maxDiskBytes, highWaterBytes: resolveHighWaterBytes(maintenance, maxDiskBytes), }; diff --git a/src/config/sessions/store.agent-harness-invariant.test.ts b/src/config/sessions/store.agent-harness-invariant.test.ts index 56f2240fbd84..bc13fa712c02 100644 --- a/src/config/sessions/store.agent-harness-invariant.test.ts +++ b/src/config/sessions/store.agent-harness-invariant.test.ts @@ -2,7 +2,6 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { loadSessionStore as loadPluginSessionStore } from "../../plugin-sdk/session-store-runtime.js"; import { AGENT_HARNESS_SESSION_KEY_RESERVED_MESSAGE } from "../../sessions/agent-harness-session-key.js"; import { applySessionEntryLifecycleMutation, @@ -498,32 +497,6 @@ describe("agent harness session store invariant", () => { }); }); - it("does not expose the writer-owned cache through the plugin SDK", async () => { - const sessionKey = "agent:main:harness:codex:supervision:native-thread"; - const entry: SessionEntry = { - agentHarnessId: "codex", - modelSelectionLocked: true, - sessionId: "native-session", - updatedAt: 1, - }; - await saveSessionStore(storePath, { [sessionKey]: entry }, { skipMaintenance: true }); - - const exposed = loadPluginSessionStore(storePath, { clone: false }); - delete exposed[sessionKey]; - await updateSessionStore( - storePath, - (store) => { - store["agent:main:ordinary"] = { sessionId: "ordinary-session", updatedAt: 2 }; - }, - { skipMaintenance: true }, - ); - - expect(loadSessionStore(storePath, { skipCache: true })).toEqual({ - [sessionKey]: entry, - "agent:main:ordinary": { sessionId: "ordinary-session", updatedAt: 2 }, - }); - }); - it("rejects locked lifecycle cleanup before transcript side effects", async () => { const sessionKey = "agent:main:harness:codex:supervision:native-thread"; const transcriptPath = path.join(tempDir, "native-session.jsonl"); diff --git a/src/config/sessions/store.pruning.integration.test.ts b/src/config/sessions/store.pruning.integration.test.ts index a29030cd4525..646b80b3b65b 100644 --- a/src/config/sessions/store.pruning.integration.test.ts +++ b/src/config/sessions/store.pruning.integration.test.ts @@ -3,11 +3,17 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; import { createSuiteTempRootTracker } from "../../test-helpers/temp-dir.js"; import { resolveTrajectoryFilePath, resolveTrajectoryPointerFilePath, } from "../../trajectory/paths.js"; +import { + appendSqliteTrajectoryRuntimeEvents, + loadSqliteTrajectoryRuntimeEvents, +} from "../../trajectory/runtime-store.sqlite.js"; +import type { TrajectoryEvent } from "../../trajectory/types.js"; import type { SessionEntry } from "./types.js"; // Keep integration tests deterministic: never read a real openclaw.json. @@ -18,6 +24,14 @@ vi.mock("../config.js", async () => ({ import { getRuntimeConfig } from "../config.js"; import { runSessionsCleanup } from "./cleanup-service.js"; +import { + appendTranscriptMessage, + listSessionEntries, + loadSessionEntry, + loadTranscriptEventsSync, + patchSessionEntry, + replaceSessionEntry, +} from "./session-accessor.js"; import { registerSessionMaintenancePreserveKeysProvider } from "./store-maintenance-preserve.js"; import { clearSessionStoreCacheForTest, @@ -101,6 +115,73 @@ function createStaleAndFreshStore(now = Date.now()): Record<string, SessionEntry }; } +async function seedSqliteSessionStore( + targetStorePath: string, + store: Record<string, SessionEntry>, +): Promise<void> { + for (const [sessionKey, entry] of Object.entries(store)) { + await patchSessionEntry({ storePath: targetStorePath, sessionKey }, () => entry, { + fallbackEntry: entry, + replaceEntry: true, + skipMaintenance: true, + }); + } +} + +function loadSqliteSessionStore(targetStorePath: string): Record<string, SessionEntry> { + return Object.fromEntries( + listSessionEntries({ storePath: targetStorePath }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), + ); +} + +async function seedSqliteTranscriptMessage(params: { + content: string; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise<void> { + await appendTranscriptMessage( + { + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }, + { + cwd: path.dirname(params.storePath), + message: { role: "user", content: params.content }, + }, + ); +} + +async function seedSqliteTrajectoryEvent(params: { + content: string; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise<void> { + const event: TrajectoryEvent = { + traceSchema: "openclaw-trajectory", + schemaVersion: 1, + traceId: params.sessionId, + source: "runtime", + type: "budget.test", + ts: "2026-07-03T00:00:00.000Z", + seq: 1, + sourceSeq: 1, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + runId: "budget-test", + data: { content: params.content }, + }; + appendSqliteTrajectoryRuntimeEvents( + { sessionId: params.sessionId, storePath: params.storePath }, + [event], + ); +} + describe("Integration: saveSessionStore with pruning", () => { let testDir: string; let storePath: string; @@ -127,6 +208,7 @@ describe("Integration: saveSessionStore with pruning", () => { afterEach(() => { mockLoadConfig.mockReset(); clearSessionStoreCacheForTest(); + closeOpenClawAgentDatabasesForTest(); if (savedCacheTtl === undefined) { delete process.env.OPENCLAW_SESSION_CACHE_TTL_MS; } else { @@ -167,7 +249,7 @@ describe("Integration: saveSessionStore with pruning", () => { [staleModelRun]: makeEntry(now - 2 * DAY_MS), [recentModelRun]: makeEntry(now), }; - await fs.writeFile(storePath, JSON.stringify(store), "utf-8"); + await seedSqliteSessionStore(storePath, store); const cfg = { session: { store: storePath } }; mockLoadConfig.mockReturnValue({ @@ -186,7 +268,7 @@ describe("Integration: saveSessionStore with pruning", () => { }); expect(defaultDryRun.previewResults[0]?.summary.modelRunPruned).toBe(0); - expect(loadSessionStore(storePath, { skipCache: true })).toHaveProperty(staleModelRun); + expect(loadSessionEntry({ storePath, sessionKey: staleModelRun })).toBeDefined(); mockLoadConfig.mockReturnValue({ session: { @@ -204,7 +286,7 @@ describe("Integration: saveSessionStore with pruning", () => { }); expect(dryRun.previewResults[0]?.summary.modelRunPruned).toBe(1); - expect(loadSessionStore(storePath, { skipCache: true })).toHaveProperty(staleModelRun); + expect(loadSessionEntry({ storePath, sessionKey: staleModelRun })).toBeDefined(); const applied = await runSessionsCleanup({ cfg, @@ -213,11 +295,28 @@ describe("Integration: saveSessionStore with pruning", () => { }); expect(applied.appliedSummaries[0]?.modelRunPruned).toBe(1); - const loaded = loadSessionStore(storePath, { skipCache: true }); + const loaded = loadSqliteSessionStore(storePath); expect(loaded[staleModelRun]).toBeUndefined(); expect(loaded).toHaveProperty(recentModelRun); }); + it("sessions cleanup dry-run does not create a missing SQLite store", async () => { + applyEnforcedMaintenanceConfig(mockLoadConfig); + const sqlitePath = path.join(testDir, "openclaw-agent.sqlite"); + + await expectPathMissing(sqlitePath); + + const dryRun = await runSessionsCleanup({ + cfg: { session: { store: storePath } }, + opts: { dryRun: true, enforce: true, store: storePath }, + targets: [{ agentId: "main", storePath }], + }); + + expect(dryRun.previewResults[0]?.summary.beforeCount).toBe(0); + expect(dryRun.previewResults[0]?.summary.afterCount).toBe(0); + await expectPathMissing(sqlitePath); + }); + it("saveSessionStore pressure-gates unset default model-run pruning", async () => { const now = Date.now(); const staleModelRun = "agent:main:explicit:model-run-123e4567-e89b-12d3-a456-426614174020"; @@ -398,7 +497,7 @@ describe("Integration: saveSessionStore with pruning", () => { testDir, "orphan-session.checkpoint.11111111-1111-4111-8111-111111111111.jsonl", ); - await fs.writeFile(storePath, JSON.stringify(store, null, 2), "utf-8"); + await seedSqliteSessionStore(storePath, store); await fs.writeFile(referencedTranscript, "referenced", "utf-8"); await fs.writeFile(referencedCheckpointPath, "referenced checkpoint", "utf-8"); await fs.writeFile(referencedPostCompactionPath, "referenced post-compaction", "utf-8"); @@ -452,79 +551,68 @@ describe("Integration: saveSessionStore with pruning", () => { applyEnforcedMaintenanceConfig(mockLoadConfig); const now = Date.now(); - const validTranscript = path.join(testDir, "valid-present.jsonl"); - const legacyPresentTranscript = path.join(testDir, "legacy-present.jsonl"); - const emptyPresentTranscript = path.join(testDir, "empty-present.jsonl"); - const headerOnlyPresentTranscript = path.join(testDir, "header-only-present.jsonl"); - const userOnlyPresentTranscript = path.join(testDir, "user-only-present.jsonl"); - const legacyRolePresentTranscript = path.join(testDir, "legacy-role-present.jsonl"); - const legacyNestedRolePresentTranscript = path.join( - testDir, - "legacy-nested-role-present.jsonl", - ); - await fs.writeFile( + await seedSqliteSessionStore(storePath, { + "invalid-no-file": { sessionId: "invalid-no-file", updatedAt: now }, + "invalid-bad-file": { + sessionId: "invalid-bad-file", + sessionFile: "../outside.jsonl", + updatedAt: now, + }, + "invalid-missing-relative-file": { + sessionId: "invalid-missing-relative-file", + sessionFile: "missing.jsonl", + updatedAt: now, + }, + "agent:main:metadata": { + sessionId: "agent:main:metadata", + updatedAt: now, + groupActivation: "always", + }, + "legacy-present-invalid-id": { + sessionId: "agent:main:main", + sessionFile: "legacy-present.jsonl", + updatedAt: now, + }, + "valid-present": { sessionId: "valid-present", updatedAt: now }, + "empty-present": { sessionId: "empty-present", updatedAt: now }, + "header-only-present": { sessionId: "header-only-present", updatedAt: now }, + "user-only-present": { sessionId: "user-only-present", updatedAt: now }, + "legacy-role-present": { sessionId: "legacy-role-present", updatedAt: now }, + "legacy-nested-role-present": { + sessionId: "legacy-nested-role-present", + updatedAt: now, + }, + } satisfies Record<string, SessionEntry>); + await seedSqliteTranscriptMessage({ + content: "valid", + sessionId: "valid-present", + sessionKey: "valid-present", storePath, - JSON.stringify( - { - "invalid-no-file": { sessionId: "agent:main:main", updatedAt: now }, - "invalid-bad-file": { - sessionId: "agent:main:main", - sessionFile: "../outside.jsonl", - updatedAt: now, - }, - "invalid-missing-relative-file": { - sessionId: "agent:main:main", - sessionFile: "missing.jsonl", - updatedAt: now, - }, - "agent:main:metadata": { - sessionId: "agent:main:metadata", - updatedAt: now, - groupActivation: "always", - }, - "legacy-present-invalid-id": { - sessionId: "agent:main:main", - sessionFile: "legacy-present.jsonl", - updatedAt: now, - }, - "valid-present": { sessionId: "valid-present", updatedAt: now }, - "empty-present": { sessionId: "empty-present", updatedAt: now }, - "header-only-present": { sessionId: "header-only-present", updatedAt: now }, - "user-only-present": { sessionId: "user-only-present", updatedAt: now }, - "legacy-role-present": { sessionId: "legacy-role-present", updatedAt: now }, - "legacy-nested-role-present": { - sessionId: "legacy-nested-role-present", - updatedAt: now, - }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - "utf-8", - ); - await fs.writeFile(validTranscript, "valid", "utf-8"); - await fs.writeFile(legacyPresentTranscript, "legacy", "utf-8"); - await fs.writeFile(emptyPresentTranscript, "", "utf-8"); - await fs.writeFile( - headerOnlyPresentTranscript, - '{"type":"session","id":"header-only-present","cwd":"/tmp"}\n', - "utf-8", - ); - await fs.writeFile( - userOnlyPresentTranscript, - '{"type":"session","id":"user-only-present"}\n{"type":"message","message":{"role":"user","content":"hello"}}\n', - "utf-8", - ); - await fs.writeFile( - legacyRolePresentTranscript, - '{"role":"user","content":"legacy transcript row"}\n', - "utf-8", - ); - await fs.writeFile( - legacyNestedRolePresentTranscript, - '{"message":{"role":"user","content":"legacy nested transcript row"}}\n', - "utf-8", - ); + }); + await seedSqliteTranscriptMessage({ + content: "legacy", + sessionId: "agent:main:main", + sessionKey: "legacy-present-invalid-id", + storePath, + }); + await seedSqliteTranscriptMessage({ + content: "hello", + sessionId: "user-only-present", + sessionKey: "user-only-present", + storePath, + }); + await seedSqliteTranscriptMessage({ + content: "legacy transcript row", + sessionId: "legacy-role-present", + sessionKey: "legacy-role-present", + storePath, + }); + await seedSqliteTranscriptMessage({ + content: "legacy nested transcript row", + sessionId: "legacy-nested-role-present", + sessionKey: "legacy-nested-role-present", + storePath, + }); const dryRun = await runSessionsCleanup({ cfg: {}, @@ -545,11 +633,7 @@ describe("Integration: saveSessionStore with pruning", () => { expect(preview?.missingKeys.has("user-only-present")).toBe(false); expect(preview?.missingKeys.has("legacy-role-present")).toBe(false); expect(preview?.missingKeys.has("legacy-nested-role-present")).toBe(false); - const rawAfterDryRun = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - unknown - >; - expect(rawAfterDryRun).toHaveProperty("invalid-no-file"); + expect(loadSessionEntry({ storePath, sessionKey: "invalid-no-file" })).toBeDefined(); const applied = await runSessionsCleanup({ cfg: {}, @@ -559,76 +643,73 @@ describe("Integration: saveSessionStore with pruning", () => { expect(applied.appliedSummaries[0]?.missing).toBe(5); expect(applied.appliedSummaries[0]?.afterCount).toBe(6); - const persisted = loadSessionStore(storePath, { skipCache: true }); - expect(Object.keys(persisted)).toEqual([ - "agent:main:metadata", - "legacy-present-invalid-id", - "valid-present", - "user-only-present", - "legacy-role-present", - "legacy-nested-role-present", - ]); + const persisted = loadSqliteSessionStore(storePath); + expect(Object.keys(persisted).toSorted()).toEqual( + [ + "agent:main:metadata", + "legacy-nested-role-present", + "legacy-present-invalid-id", + "legacy-role-present", + "user-only-present", + "valid-present", + ].toSorted(), + ); expect(persisted["agent:main:metadata"]).toMatchObject({ groupActivation: "always" }); - expect(persisted["agent:main:metadata"]?.sessionId).toBeUndefined(); + expect(persisted["agent:main:metadata"]?.sessionId).toBe("agent:main:metadata"); expect(persisted["legacy-present-invalid-id"]?.sessionId).toBe("agent:main:main"); - await expectPathExists(validTranscript); - await expectPathExists(legacyPresentTranscript); - await expectPathExists(legacyRolePresentTranscript); - await expectPathExists(legacyNestedRolePresentTranscript); - await expectPathExists(userOnlyPresentTranscript); + expect( + loadTranscriptEventsSync({ + sessionId: "valid-present", + sessionKey: "valid-present", + storePath, + }).length, + ).toBeGreaterThan(0); + expect( + loadTranscriptEventsSync({ + sessionId: "agent:main:main", + sessionKey: "legacy-present-invalid-id", + storePath, + }).length, + ).toBeGreaterThan(0); + expect( + loadTranscriptEventsSync({ + sessionId: "legacy-role-present", + sessionKey: "legacy-role-present", + storePath, + }).length, + ).toBeGreaterThan(0); + expect( + loadTranscriptEventsSync({ + sessionId: "legacy-nested-role-present", + sessionKey: "legacy-nested-role-present", + storePath, + }).length, + ).toBeGreaterThan(0); + expect( + loadTranscriptEventsSync({ + sessionId: "user-only-present", + sessionKey: "user-only-present", + storePath, + }).length, + ).toBeGreaterThan(0); }); - it("sessions cleanup fix-missing preserves locked header-only harness sessions", async () => { + it("sessions cleanup fix-missing preserves locked harness sessions", async () => { applyEnforcedMaintenanceConfig(mockLoadConfig); - - const now = Date.now(); const lockedKey = "agent:main:harness-owned:locked"; - const lockedTranscript = path.join(testDir, "locked-header-only.jsonl"); - const removableTranscript = path.join(testDir, "removable-header-only.jsonl"); - await fs.writeFile( - storePath, - JSON.stringify( - { - [lockedKey]: { - sessionId: "locked-header-only", - updatedAt: now, - agentHarnessId: "fixture-harness", - modelSelectionLocked: true, - }, - "removable-header-only": { - sessionId: "removable-header-only", - updatedAt: now, - }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - "utf-8", + await replaceSessionEntry( + { agentId: "main", sessionKey: lockedKey, storePath }, + { + sessionId: "locked-header-only", + updatedAt: Date.now(), + agentHarnessId: "fixture-harness", + modelSelectionLocked: true, + }, ); - await fs.writeFile( - lockedTranscript, - '{"type":"session","id":"locked-header-only","cwd":"/tmp"}\n', - "utf-8", + await replaceSessionEntry( + { agentId: "main", sessionKey: "removable-header-only", storePath }, + { sessionId: "removable-header-only", updatedAt: Date.now() }, ); - await fs.writeFile( - removableTranscript, - '{"type":"session","id":"removable-header-only","cwd":"/tmp"}\n', - "utf-8", - ); - - const dryRun = await runSessionsCleanup({ - cfg: {}, - opts: { store: storePath, dryRun: true, enforce: true, fixMissing: true }, - targets: [{ agentId: "main", storePath }], - }); - - const preview = dryRun.previewResults[0]; - expect(preview?.summary.missing).toBe(1); - expect(preview?.summary.beforeCount).toBe(2); - expect(preview?.summary.afterCount).toBe(1); - expect(preview?.missingKeys.has(lockedKey)).toBe(false); - expect(preview?.missingKeys.has("removable-header-only")).toBe(true); - expect(loadSessionStore(storePath, { skipCache: true })).toHaveProperty(lockedKey); const applied = await runSessionsCleanup({ cfg: {}, @@ -637,264 +718,47 @@ describe("Integration: saveSessionStore with pruning", () => { }); expect(applied.appliedSummaries[0]?.missing).toBe(1); - expect(applied.appliedSummaries[0]?.afterCount).toBe(1); - const persisted = loadSessionStore(storePath, { skipCache: true }); - expect(persisted[lockedKey]).toMatchObject({ - sessionId: "locked-header-only", + expect(loadSessionEntry({ agentId: "main", sessionKey: lockedKey, storePath })).toMatchObject({ agentHarnessId: "fixture-harness", modelSelectionLocked: true, + sessionId: "locked-header-only", }); - expect(persisted["removable-header-only"]).toBeUndefined(); - await expectPathExists(lockedTranscript); - }); - - it("sessions cleanup repairs stale generated sessionFile metadata before pruning", async () => { - applyEnforcedMaintenanceConfig(mockLoadConfig); - - const now = Date.now(); - const sessionId = "11111111-1111-4111-8111-111111111111"; - const staleTranscript = path.join(testDir, "22222222-2222-4222-8222-222222222222.jsonl"); - const canonicalTranscript = path.join(testDir, `${sessionId}.jsonl`); - await fs.writeFile( - storePath, - JSON.stringify( - { - "agent:main:cron:job:run:fresh": { - sessionId, - updatedAt: now, - sessionFile: staleTranscript, - }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - "utf-8", - ); - await fs.writeFile( - canonicalTranscript, - '{"type":"session","id":"11111111-1111-4111-8111-111111111111","cwd":"/tmp"}\n{"type":"message","message":{"role":"user","content":"still valid"}}\n', - "utf-8", - ); - - const dryRun = await runSessionsCleanup({ - cfg: {}, - opts: { store: storePath, dryRun: true, enforce: true, fixMissing: true }, - targets: [{ agentId: "main", storePath }], - }); - - const preview = dryRun.previewResults[0]; - expect(preview?.summary.repaired).toBe(1); - expect(preview?.summary.missing).toBe(0); - expect(preview?.summary.beforeCount).toBe(1); - expect(preview?.summary.afterCount).toBe(1); - expect(preview?.summary.wouldMutate).toBe(true); - expect(preview?.repairedKeys.has("agent:main:cron:job:run:fresh")).toBe(true); - expect(preview?.missingKeys.size).toBe(0); expect( - loadSessionStore(storePath, { skipCache: true })["agent:main:cron:job:run:fresh"] - ?.sessionFile, - ).toBe(staleTranscript); - - const applied = await runSessionsCleanup({ - cfg: {}, - opts: { store: storePath, enforce: true, fixMissing: true }, - targets: [{ agentId: "main", storePath }], - }); - - expect(applied.appliedSummaries[0]?.repaired).toBe(1); - expect(applied.appliedSummaries[0]?.missing).toBe(0); - expect(applied.appliedSummaries[0]?.afterCount).toBe(1); - expect(applied.appliedSummaries[0]?.wouldMutate).toBe(true); - const persisted = loadSessionStore(storePath, { skipCache: true }); - expect(persisted["agent:main:cron:job:run:fresh"]?.sessionFile).toBe(canonicalTranscript); - await expectPathExists(canonicalTranscript); - }); - - it("sessions cleanup repairs stale generated metadata even when the stale transcript exists", async () => { - applyEnforcedMaintenanceConfig(mockLoadConfig); - - const now = Date.now(); - const oldDate = new Date(now - 10 * DAY_MS); - const sessionKey = "agent:main:cron:job:run:fresh"; - const sessionId = "55555555-5555-4555-8555-555555555555"; - const staleTranscript = path.join(testDir, "66666666-6666-4666-8666-666666666666.jsonl"); - const canonicalTranscript = path.join(testDir, `${sessionId}.jsonl`); - await fs.writeFile( - storePath, - JSON.stringify( - { - [sessionKey]: { - sessionId, - updatedAt: now, - sessionFile: staleTranscript, - }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - "utf-8", - ); - await fs.writeFile( - staleTranscript, - '{"type":"session","id":"66666666-6666-4666-8666-666666666666","cwd":"/tmp"}\n{"type":"message","message":{"role":"user","content":"old transcript"}}\n', - "utf-8", - ); - await fs.writeFile( - canonicalTranscript, - '{"type":"session","id":"55555555-5555-4555-8555-555555555555","cwd":"/tmp"}\n{"type":"message","message":{"role":"user","content":"current transcript"}}\n', - "utf-8", - ); - await fs.utimes(staleTranscript, oldDate, oldDate); - await fs.utimes(canonicalTranscript, oldDate, oldDate); - - const applied = await runSessionsCleanup({ - cfg: {}, - opts: { store: storePath, enforce: true, fixMissing: true }, - targets: [{ agentId: "main", storePath }], - }); - - expect(applied.appliedSummaries[0]?.repaired).toBe(1); - expect(applied.appliedSummaries[0]?.missing).toBe(0); - const persisted = loadSessionStore(storePath, { skipCache: true }); - expect(persisted[sessionKey]?.sessionFile).toBe(canonicalTranscript); - await expectPathExists(canonicalTranscript); - }); - - it("sessions cleanup does not clobber concurrent sessionFile updates while repairing", async () => { - applyEnforcedMaintenanceConfig(mockLoadConfig); - - const now = Date.now(); - const sessionKey = "agent:main:cron:job:run:fresh"; - const sessionId = "33333333-3333-4333-8333-333333333333"; - const staleTranscript = path.join(testDir, "44444444-4444-4444-8444-444444444444.jsonl"); - const canonicalTranscript = path.join(testDir, `${sessionId}.jsonl`); - const concurrentTranscript = path.join(testDir, "concurrent-cron-session.jsonl"); - await fs.writeFile( - storePath, - JSON.stringify( - { - [sessionKey]: { - sessionId, - updatedAt: now, - sessionFile: staleTranscript, - }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - "utf-8", - ); - await fs.writeFile( - canonicalTranscript, - '{"type":"session","id":"33333333-3333-4333-8333-333333333333","cwd":"/tmp"}\n{"type":"message","message":{"role":"user","content":"still valid"}}\n', - "utf-8", - ); - await fs.writeFile( - concurrentTranscript, - '{"type":"session","id":"33333333-3333-4333-8333-333333333333","cwd":"/tmp"}\n{"type":"message","message":{"role":"assistant","content":"newer write"}}\n', - "utf-8", - ); - - const staleEntry = loadSessionStore(storePath, { skipCache: true })[sessionKey]; - expect(staleEntry?.sessionFile).toBe(staleTranscript); - await updateSessionStore(storePath, (store) => { - const current = store[sessionKey]; - if (!current) { - throw new Error("expected session entry"); - } - store[sessionKey] = { - ...current, - sessionFile: concurrentTranscript, - updatedAt: now + 1, - }; - }); - - const applied = await runSessionsCleanup({ - cfg: {}, - opts: { store: storePath, enforce: true, fixMissing: true }, - targets: [{ agentId: "main", storePath }], - }); - - expect(applied.appliedSummaries[0]?.repaired).toBe(0); - expect(applied.appliedSummaries[0]?.missing).toBe(0); - const persisted = loadSessionStore(storePath, { skipCache: true }); - expect(persisted[sessionKey]?.sessionFile).toBe(concurrentTranscript); - expect(persisted[sessionKey]?.updatedAt).toBe(now + 1); - }); - - it("sessions cleanup does not repair missing topic transcripts to the base transcript", async () => { - applyEnforcedMaintenanceConfig(mockLoadConfig); - - const now = Date.now(); - const sessionKey = "agent:main:telegram:thread:topic"; - const sessionId = "aaaaaaaa-1111-4111-8111-aaaaaaaaaaaa"; - const missingTopicTranscript = path.join(testDir, `${sessionId}-topic-456.jsonl`); - const baseTranscript = path.join(testDir, `${sessionId}.jsonl`); - await fs.writeFile( - storePath, - JSON.stringify( - { - [sessionKey]: { - sessionId, - updatedAt: now, - sessionFile: missingTopicTranscript, - }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - "utf-8", - ); - await fs.writeFile( - baseTranscript, - '{"type":"session","id":"aaaaaaaa-1111-4111-8111-aaaaaaaaaaaa","cwd":"/tmp"}\n{"type":"message","message":{"role":"user","content":"base conversation"}}\n', - "utf-8", - ); - - const applied = await runSessionsCleanup({ - cfg: {}, - opts: { store: storePath, enforce: true, fixMissing: true }, - targets: [{ agentId: "main", storePath }], - }); - - expect(applied.appliedSummaries[0]?.missing).toBe(1); - expect(applied.appliedSummaries[0]?.afterCount).toBe(0); - const persisted = loadSessionStore(storePath, { skipCache: true }); - expect(persisted[sessionKey]).toBeUndefined(); - await expectPathExists(baseTranscript); + loadSessionEntry({ + agentId: "main", + sessionKey: "removable-header-only", + storePath, + }), + ).toBeUndefined(); }); it("sessions cleanup previews stale direct DM rows after dmScope returns to main", async () => { applyEnforcedMaintenanceConfig(mockLoadConfig); const now = Date.now(); - const directTranscript = path.join(testDir, "direct-session.jsonl"); - await fs.writeFile( - storePath, - JSON.stringify( - { - "agent:main:main": { - sessionId: "main-session", - updatedAt: now, - }, - "agent:main:telegram:direct:6101296751": { - sessionId: "direct-session", - updatedAt: now, - lastChannel: "telegram", - lastTo: "6101296751", - }, - "agent:main:telegram::direct:malformed": { - sessionId: "malformed-session", - updatedAt: now, - }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - "utf-8", - ); + await seedSqliteSessionStore(storePath, { + "agent:main:main": { + sessionId: "main-session", + updatedAt: now, + }, + "agent:main:telegram:direct:6101296751": { + sessionId: "direct-session", + updatedAt: now, + lastChannel: "telegram", + lastTo: "6101296751", + }, + "agent:main:telegram::direct:malformed": { + sessionId: "malformed-session", + updatedAt: now, + }, + } satisfies Record<string, SessionEntry>); await fs.writeFile(path.join(testDir, "main-session.jsonl"), "main", "utf-8"); - await fs.writeFile(directTranscript, "direct", "utf-8"); + await seedSqliteTranscriptMessage({ + content: "direct", + sessionId: "direct-session", + sessionKey: "agent:main:telegram:direct:6101296751", + storePath, + }); const dryRun = await runSessionsCleanup({ cfg: { session: { dmScope: "main" } }, @@ -908,7 +772,13 @@ describe("Integration: saveSessionStore with pruning", () => { expect(preview?.dmScopeRetiredKeys.has("agent:main:telegram:direct:6101296751")).toBe(true); expect(preview?.dmScopeRetiredKeys.has("agent:main:telegram::direct:malformed")).toBe(false); expect(preview?.summary.unreferencedArtifacts.removedFiles).toBe(0); - await expectPathExists(directTranscript); + expect( + loadTranscriptEventsSync({ + sessionId: "direct-session", + sessionKey: "agent:main:telegram:direct:6101296751", + storePath, + }).length, + ).toBeGreaterThan(0); }); it("sessions cleanup retires stale direct DM rows and archives their transcripts", async () => { @@ -917,34 +787,31 @@ describe("Integration: saveSessionStore with pruning", () => { const now = Date.now(); const directTranscript = path.join(testDir, "direct-session.jsonl"); const nestedTranscript = path.join(testDir, "nested-agent-session.jsonl"); - await fs.writeFile( - storePath, - JSON.stringify( - { - "agent:main:main": { - sessionId: "main-session", - updatedAt: now, - }, - "agent:main:telegram:direct:6101296751": { - sessionId: "direct-session", - updatedAt: now, - sessionFile: directTranscript, - lastChannel: "telegram", - lastTo: "6101296751", - }, - "agent:main:agent:direct:customer": { - sessionId: "nested-agent-session", - updatedAt: now, - sessionFile: nestedTranscript, - }, - } satisfies Record<string, SessionEntry>, - null, - 2, - ), - "utf-8", - ); + await seedSqliteSessionStore(storePath, { + "agent:main:main": { + sessionId: "main-session", + updatedAt: now, + }, + "agent:main:telegram:direct:6101296751": { + sessionId: "direct-session", + updatedAt: now, + sessionFile: directTranscript, + lastChannel: "telegram", + lastTo: "6101296751", + }, + "agent:main:agent:direct:customer": { + sessionId: "nested-agent-session", + updatedAt: now, + sessionFile: nestedTranscript, + }, + } satisfies Record<string, SessionEntry>); await fs.writeFile(path.join(testDir, "main-session.jsonl"), "main", "utf-8"); - await fs.writeFile(directTranscript, "direct", "utf-8"); + await seedSqliteTranscriptMessage({ + content: "direct", + sessionId: "direct-session", + sessionKey: "agent:main:telegram:direct:6101296751", + storePath, + }); await fs.writeFile(nestedTranscript, "nested", "utf-8"); const applied = await runSessionsCleanup({ @@ -954,10 +821,17 @@ describe("Integration: saveSessionStore with pruning", () => { }); expect(applied.appliedSummaries[0]?.dmScopeRetired).toBe(1); - const persisted = loadSessionStore(storePath, { skipCache: true }); + const persisted = loadSqliteSessionStore(storePath); expect(persisted).toHaveProperty("agent:main:main"); expect(persisted).toHaveProperty("agent:main:agent:direct:customer"); expect(persisted["agent:main:telegram:direct:6101296751"]).toBeUndefined(); + expect( + loadTranscriptEventsSync({ + sessionId: "direct-session", + sessionKey: "agent:main:telegram:direct:6101296751", + storePath, + }), + ).toEqual([]); await expectPathMissing(directTranscript); await expectPathExists(nestedTranscript); const files = await fs.readdir(testDir); @@ -967,7 +841,7 @@ describe("Integration: saveSessionStore with pruning", () => { expect(archivedDirectTranscripts.length).toBeGreaterThan(0); }); - it("sessions cleanup dry-run does not double-count artifacts already covered by disk budget", async () => { + it("sessions cleanup dry-run reports unreferenced artifacts outside SQLite row budget", async () => { mockLoadConfig.mockReturnValue({ session: { maintenance: { @@ -984,7 +858,7 @@ describe("Integration: saveSessionStore with pruning", () => { fresh: { sessionId: "fresh-session", updatedAt: Date.now() }, }; const oldOrphanTranscript = path.join(testDir, "orphan-session.jsonl"); - await fs.writeFile(storePath, JSON.stringify(store, null, 2), "utf-8"); + await seedSqliteSessionStore(storePath, store); await fs.writeFile(oldOrphanTranscript, "x".repeat(2000), "utf-8"); const oldDate = new Date(Date.now() - 10 * DAY_MS); await fs.utimes(oldOrphanTranscript, oldDate, oldDate); @@ -999,11 +873,247 @@ describe("Integration: saveSessionStore with pruning", () => { if (diskBudgetSummary === null || diskBudgetSummary === undefined) { throw new Error("expected disk budget cleanup summary"); } - expect(diskBudgetSummary.removedFiles).toBe(1); - expect(dryRun.previewResults[0]?.summary.unreferencedArtifacts.removedFiles).toBe(0); + expect(diskBudgetSummary.removedFiles).toBe(0); + expect(dryRun.previewResults[0]?.summary.unreferencedArtifacts.removedFiles).toBe(1); await expectPathExists(oldOrphanTranscript); }); + it("sessions cleanup dry-run reports SQLite transcript row bytes for disk budget", async () => { + mockLoadConfig.mockReturnValue({ + session: { + maintenance: { + mode: "enforce", + pruneAfter: "365d", + maxEntries: 500, + maxDiskBytes: 1_600, + highWaterBytes: 900, + }, + }, + }); + + const now = Date.now(); + const oldKey = "agent:main:explicit:old-budget"; + const freshKey = "agent:main:explicit:fresh-budget"; + await seedSqliteSessionStore(storePath, { + [oldKey]: { sessionId: "old-budget-session", updatedAt: now - DAY_MS }, + [freshKey]: { sessionId: "fresh-budget-session", updatedAt: now }, + }); + await seedSqliteTranscriptMessage({ + content: "old-" + "x".repeat(1_200), + sessionId: "old-budget-session", + sessionKey: oldKey, + storePath, + }); + await seedSqliteTranscriptMessage({ + content: "fresh-" + "y".repeat(1_200), + sessionId: "fresh-budget-session", + sessionKey: freshKey, + storePath, + }); + + const dryRun = await runSessionsCleanup({ + cfg: {}, + opts: { store: storePath, dryRun: true, enforce: true, activeKey: freshKey }, + targets: [{ agentId: "main", storePath }], + }); + + const preview = dryRun.previewResults[0]; + const diskBudgetSummary = preview?.summary.diskBudget; + if (diskBudgetSummary === null || diskBudgetSummary === undefined) { + throw new Error("expected SQLite row-byte disk budget summary"); + } + expect(diskBudgetSummary.totalBytesBefore).toBeGreaterThan(2_400); + expect(diskBudgetSummary.totalBytesAfter).toBeLessThan(diskBudgetSummary.totalBytesBefore); + expect(diskBudgetSummary.removedEntries).toBe(1); + expect(diskBudgetSummary.removedFiles).toBe(0); + expect(preview?.summary.afterCount).toBe(1); + expect(preview?.budgetEvictedKeys.has(oldKey)).toBe(true); + expect(preview?.budgetEvictedKeys.has(freshKey)).toBe(false); + expect(loadSessionEntry({ storePath, sessionKey: oldKey })).toBeDefined(); + expect( + loadTranscriptEventsSync({ + sessionId: "old-budget-session", + sessionKey: oldKey, + storePath, + }).length, + ).toBeGreaterThan(0); + }); + + it("sessions cleanup apply reports SQLite disk-budget row eviction", async () => { + mockLoadConfig.mockReturnValue({ + session: { + maintenance: { + mode: "enforce", + pruneAfter: "365d", + maxEntries: 500, + maxDiskBytes: 1_600, + highWaterBytes: 900, + }, + }, + }); + + const now = Date.now(); + const oldKey = "agent:main:explicit:old-apply-budget"; + const freshKey = "agent:main:explicit:fresh-apply-budget"; + await seedSqliteSessionStore(storePath, { + [oldKey]: { sessionId: "old-apply-budget-session", updatedAt: now - DAY_MS }, + [freshKey]: { sessionId: "fresh-apply-budget-session", updatedAt: now }, + }); + await seedSqliteTranscriptMessage({ + content: "old-" + "x".repeat(1_200), + sessionId: "old-apply-budget-session", + sessionKey: oldKey, + storePath, + }); + await seedSqliteTranscriptMessage({ + content: "fresh-" + "y".repeat(1_200), + sessionId: "fresh-apply-budget-session", + sessionKey: freshKey, + storePath, + }); + + const applied = await runSessionsCleanup({ + cfg: {}, + opts: { store: storePath, enforce: true, activeKey: freshKey }, + targets: [{ agentId: "main", storePath }], + }); + + const summary = applied.appliedSummaries[0]; + const diskBudgetSummary = summary?.diskBudget; + if (diskBudgetSummary === null || diskBudgetSummary === undefined) { + throw new Error("expected applied SQLite row-byte disk budget summary"); + } + expect(diskBudgetSummary.removedEntries).toBe(1); + expect(diskBudgetSummary.removedFiles).toBe(0); + expect(summary?.appliedCount).toBe(1); + expect(loadSessionEntry({ storePath, sessionKey: oldKey })).toBeUndefined(); + expect(loadSessionEntry({ storePath, sessionKey: freshKey })).toBeDefined(); + expect( + loadTranscriptEventsSync({ + sessionId: "old-apply-budget-session", + sessionKey: oldKey, + storePath, + }), + ).toEqual([]); + expect( + loadTranscriptEventsSync({ + sessionId: "fresh-apply-budget-session", + sessionKey: freshKey, + storePath, + }).length, + ).toBeGreaterThan(0); + }); + + it("sessions cleanup dry-run accounts SQLite trajectory rows in disk budget", async () => { + mockLoadConfig.mockReturnValue({ + session: { + maintenance: { + mode: "enforce", + pruneAfter: "365d", + maxEntries: 500, + maxDiskBytes: 1_200, + highWaterBytes: 700, + }, + }, + }); + + const now = Date.now(); + const oldKey = "agent:main:explicit:old-trajectory-budget"; + const freshKey = "agent:main:explicit:fresh-trajectory-budget"; + await seedSqliteSessionStore(storePath, { + [oldKey]: { sessionId: "old-trajectory-budget-session", updatedAt: now - DAY_MS }, + [freshKey]: { sessionId: "fresh-trajectory-budget-session", updatedAt: now }, + }); + await seedSqliteTrajectoryEvent({ + content: "old-" + "x".repeat(1_400), + sessionId: "old-trajectory-budget-session", + sessionKey: oldKey, + storePath, + }); + + const dryRun = await runSessionsCleanup({ + cfg: {}, + opts: { store: storePath, dryRun: true, enforce: true, activeKey: freshKey }, + targets: [{ agentId: "main", storePath }], + }); + + const diskBudgetSummary = dryRun.previewResults[0]?.summary.diskBudget; + if (diskBudgetSummary === null || diskBudgetSummary === undefined) { + throw new Error("expected SQLite trajectory row-byte disk budget summary"); + } + expect(diskBudgetSummary.totalBytesBefore).toBeGreaterThan(1_200); + expect(diskBudgetSummary.totalBytesAfter).toBeLessThan(diskBudgetSummary.totalBytesBefore); + expect(diskBudgetSummary.removedEntries).toBe(1); + expect(dryRun.previewResults[0]?.budgetEvictedKeys.has(oldKey)).toBe(true); + expect(loadSessionEntry({ storePath, sessionKey: oldKey })).toBeDefined(); + await expect( + loadSqliteTrajectoryRuntimeEvents({ + sessionId: "old-trajectory-budget-session", + storePath, + }), + ).resolves.toHaveLength(1); + }); + + it("sessions cleanup apply removes budget-evicted SQLite trajectory rows", async () => { + mockLoadConfig.mockReturnValue({ + session: { + maintenance: { + mode: "enforce", + pruneAfter: "365d", + maxEntries: 500, + maxDiskBytes: 1_200, + highWaterBytes: 700, + }, + }, + }); + + const now = Date.now(); + const oldKey = "agent:main:explicit:old-trajectory-apply-budget"; + const freshKey = "agent:main:explicit:fresh-trajectory-apply-budget"; + await seedSqliteSessionStore(storePath, { + [oldKey]: { sessionId: "old-trajectory-apply-budget-session", updatedAt: now - DAY_MS }, + [freshKey]: { sessionId: "fresh-trajectory-apply-budget-session", updatedAt: now }, + }); + await seedSqliteTrajectoryEvent({ + content: "old-" + "x".repeat(1_400), + sessionId: "old-trajectory-apply-budget-session", + sessionKey: oldKey, + storePath, + }); + await seedSqliteTrajectoryEvent({ + content: "fresh", + sessionId: "fresh-trajectory-apply-budget-session", + sessionKey: freshKey, + storePath, + }); + + const applied = await runSessionsCleanup({ + cfg: {}, + opts: { store: storePath, enforce: true, activeKey: freshKey }, + targets: [{ agentId: "main", storePath }], + }); + + const diskBudgetSummary = applied.appliedSummaries[0]?.diskBudget; + if (diskBudgetSummary === null || diskBudgetSummary === undefined) { + throw new Error("expected applied SQLite trajectory row-byte disk budget summary"); + } + expect(diskBudgetSummary.removedEntries).toBe(1); + expect(loadSessionEntry({ storePath, sessionKey: oldKey })).toBeUndefined(); + expect(loadSessionEntry({ storePath, sessionKey: freshKey })).toBeDefined(); + await expect( + loadSqliteTrajectoryRuntimeEvents({ + sessionId: "old-trajectory-apply-budget-session", + storePath, + }), + ).resolves.toEqual([]); + await expect( + loadSqliteTrajectoryRuntimeEvents({ + sessionId: "fresh-trajectory-apply-budget-session", + storePath, + }), + ).resolves.toHaveLength(1); + }); + it("sessions cleanup dry-run excludes stale and capped entry transcripts from orphan counts", async () => { mockLoadConfig.mockReturnValue({ session: { @@ -1017,14 +1127,14 @@ describe("Integration: saveSessionStore with pruning", () => { const now = Date.now(); const store: Record<string, SessionEntry> = { - stale: { sessionId: "stale-session", updatedAt: now - 30 * DAY_MS }, - capped: { sessionId: "capped-session", updatedAt: now - DAY_MS }, - fresh: { sessionId: "fresh-session", updatedAt: now }, + "agent:main:explicit:stale": { sessionId: "stale-session", updatedAt: now - 30 * DAY_MS }, + "agent:main:explicit:capped": { sessionId: "capped-session", updatedAt: now - DAY_MS }, + "agent:main:explicit:fresh": { sessionId: "fresh-session", updatedAt: now }, }; const staleTranscript = path.join(testDir, "stale-session.jsonl"); const cappedTranscript = path.join(testDir, "capped-session.jsonl"); const freshTranscript = path.join(testDir, "fresh-session.jsonl"); - await fs.writeFile(storePath, JSON.stringify(store, null, 2), "utf-8"); + await seedSqliteSessionStore(storePath, store); await fs.writeFile(staleTranscript, "stale", "utf-8"); await fs.writeFile(cappedTranscript, "capped", "utf-8"); await fs.writeFile(freshTranscript, "fresh", "utf-8"); @@ -1046,7 +1156,7 @@ describe("Integration: saveSessionStore with pruning", () => { await expectPathExists(freshTranscript); }); - it("cleans up archived transcripts older than the prune window", async () => { + it("keeps archived transcripts by default", async () => { applyEnforcedMaintenanceConfig(mockLoadConfig); const now = Date.now(); @@ -1059,6 +1169,41 @@ describe("Integration: saveSessionStore with pruning", () => { const staleTranscript = path.join(testDir, `${staleSessionId}.jsonl`); await fs.writeFile(staleTranscript, '{"type":"session"}\n', "utf-8"); + const oldArchived = path.join( + testDir, + `old-session.jsonl.deleted.${archiveTimestamp(now - 9 * DAY_MS)}`, + ); + const bakArchived = path.join( + testDir, + `bak-session.jsonl.bak.${archiveTimestamp(now - 20 * DAY_MS)}`, + ); + await fs.writeFile(oldArchived, "old", "utf-8"); + await fs.writeFile(bakArchived, "bak", "utf-8"); + + await saveSessionStore(storePath, store); + + // Archives are conversation history: without an explicit retention + // duration only the disk budget may evict them. + await expectPathExists(oldArchived); + await expectPathExists(bakArchived); + }); + + it("cleans up deleted archives when resetArchiveRetention is configured", async () => { + mockLoadConfig.mockReturnValue({ + session: { + maintenance: { + mode: "enforce", + pruneAfter: "7d", + resetArchiveRetention: "7d", + maxEntries: 500, + }, + }, + }); + + const now = Date.now(); + const store: Record<string, SessionEntry> = { + fresh: { sessionId: "fresh-session", updatedAt: now }, + }; const oldArchived = path.join( testDir, `old-session.jsonl.deleted.${archiveTimestamp(now - 9 * DAY_MS)}`, diff --git a/src/config/sessions/store.pruning.test.ts b/src/config/sessions/store.pruning.test.ts index 61849d021dfb..f1ce5bbd373c 100644 --- a/src/config/sessions/store.pruning.test.ts +++ b/src/config/sessions/store.pruning.test.ts @@ -807,6 +807,32 @@ describe("resolveMaintenanceConfigFromInput", () => { expect(resolveMaintenanceConfigFromInput().modelRunPruneAfterMs).toBe(DAY_MS); }); + it("keeps archived transcripts by default and bounds growth with a disk budget", () => { + const maintenance = resolveMaintenanceConfigFromInput(); + + expect(maintenance.resetArchiveRetentionMs).toBeNull(); + expect(maintenance.maxDiskBytes).toBe(2 * 1024 * 1024 * 1024); + expect(maintenance.highWaterBytes).toBe(Math.floor(2 * 1024 * 1024 * 1024 * 0.8)); + }); + + it("honors explicit archive retention and disk budget opt-outs", () => { + const maintenance = resolveMaintenanceConfigFromInput({ + resetArchiveRetention: "7d", + maxDiskBytes: false, + }); + + expect(maintenance.resetArchiveRetentionMs).toBe(7 * DAY_MS); + expect(maintenance.maxDiskBytes).toBeNull(); + expect(maintenance.highWaterBytes).toBeNull(); + }); + + it("disables the disk budget when an explicit maxDiskBytes fails to parse", () => { + const maintenance = resolveMaintenanceConfigFromInput({ maxDiskBytes: "lots" }); + + expect(maintenance.maxDiskBytes).toBeNull(); + expect(maintenance.highWaterBytes).toBeNull(); + }); + it("force-gates the unset model-run prune default to the cap-eviction threshold", () => { const defaultMaintenance = resolveMaintenanceConfigFromInput({ maxEntries: 50 }); expect(resolveSessionEntryMaintenanceHighWater(50)).toBe(75); diff --git a/src/config/sessions/store.runtime.ts b/src/config/sessions/store.runtime.ts deleted file mode 100644 index ea70a5647aec..000000000000 --- a/src/config/sessions/store.runtime.ts +++ /dev/null @@ -1,18 +0,0 @@ -// Runtime facade for session store mutation helpers. -export { - applySessionStoreEntryPatch, - cleanupSessionLifecycleArtifacts, - updateSessionStore, - updateSessionStoreEntry, -} from "./store.js"; -export { deleteSessionEntryLifecycle, resetSessionEntryLifecycle } from "./session-accessor.js"; -export type { - SessionLifecycleArtifactCleanupParams, - SessionLifecycleArtifactCleanupResult, -} from "./store.js"; -export type { - DeleteSessionEntryLifecycleResult, - ResetSessionEntryLifecycleResult, - SessionLifecycleArchivedTranscript, - SessionLifecycleStoreTarget, -} from "./session-accessor.js"; diff --git a/src/config/sessions/store.session-key-normalization.test.ts b/src/config/sessions/store.session-key-normalization.test.ts index e8fc5b25ece8..a095eaf3476f 100644 --- a/src/config/sessions/store.session-key-normalization.test.ts +++ b/src/config/sessions/store.session-key-normalization.test.ts @@ -1,11 +1,34 @@ -// Store session key tests cover session key normalization during disk writes. -import fs from "node:fs/promises"; +// Store session key tests cover session key normalization through the accessor. import path from "node:path"; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import type { MsgContext } from "../../auto-reply/templating.js"; import { createSuiteTempRootTracker } from "../../test-helpers/temp-dir.js"; -import { clearSessionStoreCacheForTest, loadSessionStore } from "../sessions.js"; -import { recordInboundSessionMeta, updateSessionLastRoute } from "./session-accessor.js"; +import { + applySessionEntryLifecycleMutation, + listSessionEntries, + recordInboundSessionMeta, + updateSessionLastRoute, +} from "./session-accessor.js"; +import type { SessionEntry } from "./types.js"; + +// Materializes the SQLite-backed session store as a keyed object so key +// normalization/migration assertions keep matching the old JSON-store shape. +function loadStore(storePath: string): Record<string, SessionEntry> { + return Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ); +} + +// Seeds pre-existing rows under their exact (possibly legacy mixed-case) keys. +// replaceSessionEntry canonicalizes keys on write, so a lifecycle upsert is the +// only way to persist a legacy-keyed row the accessor should later migrate. +async function seedStore(storePath: string, entries: Record<string, SessionEntry>): Promise<void> { + await applySessionEntryLifecycleMutation({ + storePath, + upserts: Object.entries(entries).map(([sessionKey, entry]) => ({ sessionKey, entry })), + skipMaintenance: true, + }); +} const CANONICAL_KEY = "agent:main:webchat:dm:mixed-user"; const MIXED_CASE_KEY = "Agent:Main:WebChat:DM:MiXeD-User"; @@ -51,11 +74,6 @@ describe("session store key normalization", () => { beforeEach(async () => { tempDir = await suiteRootTracker.make("case"); storePath = path.join(tempDir, "sessions.json"); - await fs.writeFile(storePath, "{}", "utf-8"); - }); - - afterEach(async () => { - clearSessionStoreCacheForTest(); }); afterAll(async () => { @@ -69,7 +87,7 @@ describe("session store key normalization", () => { ctx: createInboundContext(), }); - const store = loadSessionStore(storePath, { skipCache: true }); + const store = loadStore(storePath); expect(Object.keys(store)).toEqual([CANONICAL_KEY]); expect(store[CANONICAL_KEY]?.origin?.provider).toBe("webchat"); }); @@ -88,7 +106,7 @@ describe("session store key normalization", () => { to: "webchat:user-1", }); - const store = loadSessionStore(storePath, { skipCache: true }); + const store = loadStore(storePath); expect(Object.keys(store)).toEqual([CANONICAL_KEY]); expect(store[CANONICAL_KEY]?.lastChannel).toBe("webchat"); expect(store[CANONICAL_KEY]?.lastTo).toBe("webchat:user-1"); @@ -99,23 +117,14 @@ describe("session store key normalization", () => { }); it("migrates legacy mixed-case entries to the canonical key on update", async () => { - await fs.writeFile( - storePath, - JSON.stringify( - { - [MIXED_CASE_KEY]: { - sessionId: "legacy-session", - updatedAt: 1, - chatType: "direct", - channel: "webchat", - }, - }, - null, - 2, - ), - "utf-8", - ); - clearSessionStoreCacheForTest(); + await seedStore(storePath, { + [MIXED_CASE_KEY]: { + sessionId: "legacy-session", + updatedAt: 1, + chatType: "direct", + channel: "webchat", + }, + }); await updateSessionLastRoute({ storePath, @@ -124,47 +133,42 @@ describe("session store key normalization", () => { to: "webchat:user-2", }); - const store = loadSessionStore(storePath, { skipCache: true }); + const store = loadStore(storePath); expect(store[CANONICAL_KEY]?.sessionId).toBe("legacy-session"); expect(store[MIXED_CASE_KEY]).toBeUndefined(); }); it("preserves ACP metadata when inbound metadata normalizes a legacy key", async () => { - await fs.writeFile( - storePath, - JSON.stringify( - { - [CANONICAL_KEY]: { - sessionId: "canonical-session", - updatedAt: 2, - }, - [MIXED_CASE_KEY]: { - sessionId: "legacy-session", - updatedAt: 1, - acp: { - backend: "codex", - agent: "main", - runtimeSessionName: "runtime-1", - mode: "persistent", - state: "idle", - lastActivityAt: 1, - }, - }, + await seedStore(storePath, { + [CANONICAL_KEY]: { + sessionId: "canonical-session", + updatedAt: 2, + }, + [MIXED_CASE_KEY]: { + sessionId: "legacy-session", + updatedAt: 1, + acp: { + backend: "codex", + agent: "main", + runtimeSessionName: "runtime-1", + mode: "persistent", + state: "idle", + lastActivityAt: 1, }, - null, - 2, - ), - "utf-8", - ); - clearSessionStoreCacheForTest(); + }, + }); + // A real inbound context yields a non-empty metadata patch, which is what + // triggers the accessor to collapse the legacy mixed-case alias onto the + // canonical row (the SQLite writer canonicalizes aliases as part of the + // patch write rather than as a separate empty-write pass). await recordInboundSessionMeta({ storePath, sessionKey: CANONICAL_KEY, - ctx: {}, + ctx: createInboundContext(), }); - const store = loadSessionStore(storePath, { skipCache: true }); + const store = loadStore(storePath); expect(Object.keys(store)).toEqual([CANONICAL_KEY]); expect(store[CANONICAL_KEY]?.sessionId).toBe("canonical-session"); expect(store[CANONICAL_KEY]?.acp).toBeUndefined(); @@ -172,29 +176,20 @@ describe("session store key normalization", () => { it("preserves updatedAt when recording inbound metadata for an existing session", async () => { const existingUpdatedAt = Date.now(); - await fs.writeFile( - storePath, - JSON.stringify( - { - [CANONICAL_KEY]: { - sessionId: "existing-session", - updatedAt: existingUpdatedAt, - chatType: "direct", - channel: "webchat", - origin: { - provider: "webchat", - chatType: "direct", - from: "WebChat:User-1", - to: "webchat:user-1", - }, - }, + await seedStore(storePath, { + [CANONICAL_KEY]: { + sessionId: "existing-session", + updatedAt: existingUpdatedAt, + chatType: "direct", + channel: "webchat", + origin: { + provider: "webchat", + chatType: "direct", + from: "WebChat:User-1", + to: "webchat:user-1", }, - null, - 2, - ), - "utf-8", - ); - clearSessionStoreCacheForTest(); + }, + }); await recordInboundSessionMeta({ storePath, @@ -202,7 +197,7 @@ describe("session store key normalization", () => { ctx: createInboundContext(), }); - const store = loadSessionStore(storePath, { skipCache: true }); + const store = loadStore(storePath); expect(store[CANONICAL_KEY]?.sessionId).toBe("existing-session"); expect(store[CANONICAL_KEY]?.updatedAt).toBe(existingUpdatedAt); expect(store[CANONICAL_KEY]?.origin?.provider).toBe("webchat"); @@ -215,35 +210,26 @@ describe("session store key normalization", () => { ctx: createSignalGroupContext(), }); - const store = loadSessionStore(storePath, { skipCache: true }); + const store = loadStore(storePath); expect(Object.keys(store)).toEqual([SIGNAL_GROUP_KEY]); expect(store[SIGNAL_GROUP_KEY]?.groupId).toBe(SIGNAL_GROUP_ID); expect(store[SIGNAL_GROUP_KEY]?.origin?.to).toBe(`signal:group:${SIGNAL_GROUP_ID}`); }); it("migrates legacy lowercase Signal group keys to the mixed-case canonical key", async () => { - await fs.writeFile( - storePath, - JSON.stringify( - { - [LEGACY_SIGNAL_GROUP_KEY]: { - sessionId: "legacy-signal-session", - updatedAt: 1, - chatType: "group", - channel: "signal", - groupId: SIGNAL_GROUP_ID.toLowerCase(), - deliveryContext: { - channel: "signal", - to: `signal:group:${SIGNAL_GROUP_ID}`, - }, - }, + await seedStore(storePath, { + [LEGACY_SIGNAL_GROUP_KEY]: { + sessionId: "legacy-signal-session", + updatedAt: 1, + chatType: "group", + channel: "signal", + groupId: SIGNAL_GROUP_ID.toLowerCase(), + deliveryContext: { + channel: "signal", + to: `signal:group:${SIGNAL_GROUP_ID}`, }, - null, - 2, - ), - "utf-8", - ); - clearSessionStoreCacheForTest(); + }, + }); await recordInboundSessionMeta({ storePath, @@ -251,7 +237,7 @@ describe("session store key normalization", () => { ctx: createSignalGroupContext(), }); - const store = loadSessionStore(storePath, { skipCache: true }); + const store = loadStore(storePath); expect(Object.keys(store)).toEqual([SIGNAL_GROUP_KEY]); expect(store[SIGNAL_GROUP_KEY]?.sessionId).toBe("legacy-signal-session"); expect(store[SIGNAL_GROUP_KEY]?.groupId).toBe(SIGNAL_GROUP_ID); @@ -275,7 +261,7 @@ describe("session store key normalization", () => { }, }); - const store = loadSessionStore(storePath, { skipCache: true }); + const store = loadStore(storePath); expect(store[CANONICAL_KEY]?.route).toEqual({ channel: "slack", accountId: "work", @@ -294,36 +280,10 @@ describe("session store key normalization", () => { expect(store[CANONICAL_KEY]?.lastThreadId).toBe("177000.123"); }); - it("normalizes malformed persisted route metadata on load", async () => { - await fs.writeFile( - storePath, - JSON.stringify( - { - [CANONICAL_KEY]: { - sessionId: "legacy-route-session", - updatedAt: 1, - route: "stale-custom-slot", - deliveryContext: { - channel: "slack", - to: "channel:C123", - accountId: "work", - threadId: "177000.123", - }, - }, - }, - null, - 2, - ), - "utf-8", - ); - clearSessionStoreCacheForTest(); - - const store = loadSessionStore(storePath, { skipCache: true }); - expect(store[CANONICAL_KEY]?.route).toEqual({ - channel: "slack", - accountId: "work", - target: { to: "channel:C123" }, - thread: { id: "177000.123" }, - }); - }); + // NOTE: the file-store test "normalizes malformed persisted route metadata on + // load" was removed with the SQLite flip. It asserted file-store load-time + // repair of a legacy malformed `route` string (store-load.ts + // normalizeSessionEntryDelivery). The SQLite runtime reads canonical shapes + // only and does not repair legacy `route` slots on read; that legacy-data + // repair is owned by doctor/migration, not steady-state runtime reads. }); diff --git a/src/config/sessions/store.session-lifecycle-mutation.test.ts b/src/config/sessions/store.session-lifecycle-mutation.test.ts index d0669dac02e5..c754864b28de 100644 --- a/src/config/sessions/store.session-lifecycle-mutation.test.ts +++ b/src/config/sessions/store.session-lifecycle-mutation.test.ts @@ -1,49 +1,81 @@ -// File-backed session lifecycle operations own entry mutation and transcript artifact transitions. +// SQLite session lifecycle operations own entry mutation. import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { deleteSessionEntryLifecycle, resetSessionEntryLifecycle } from "./session-accessor.js"; -import { clearSessionStoreCacheForTest, loadSessionStore, saveSessionStore } from "./store.js"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../../infra/kysely-sync.js"; +import { onInternalSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; +import type { DB as OpenClawAgentKyselyDatabase } from "../../state/openclaw-agent-db.generated.js"; +import { openOpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; +import { readSessionArchiveContentSync } from "./archive-compression.js"; +import { + applySessionEntryLifecycleMutation, + cleanupSessionLifecycleArtifacts, + deleteSessionEntryLifecycle, + loadTranscriptEvents, + loadSessionEntry, + replaceSessionEntry, + resetSessionEntryLifecycle, +} from "./session-accessor.js"; +import { replaceSqliteTranscriptEvents } from "./session-accessor.sqlite.js"; +import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js"; import type { SessionEntry } from "./types.js"; +type TestTranscriptEvent = Parameters<typeof replaceSqliteTranscriptEvents>[1][number]; + describe("session store lifecycle mutations", () => { let tempDir: string; let storePath: string; beforeEach(() => { tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-session-lifecycle-mutation-")); - storePath = path.join(tempDir, "sessions.json"); + storePath = path.join(tempDir, "agents", "main", "sessions", "sessions.json"); }); afterEach(() => { - clearSessionStoreCacheForTest(); + closeOpenClawAgentDatabasesForTest(); fs.rmSync(tempDir, { recursive: true, force: true }); }); - it("creates the new header before notifying observers and archiving the old transcript", async () => { - const oldTranscriptPath = path.join(tempDir, "old-session.jsonl"); - const nextTranscriptPath = path.join(tempDir, "next-session.jsonl"); - let nextTranscriptAtMutation: string | undefined; - let oldTranscriptExistsAtMutation = false; + it("resets an entry in SQLite while archiving the previous transcript rows", async () => { const now = Date.now(); - fs.writeFileSync(oldTranscriptPath, '{"type":"session","id":"old-session"}\n', "utf-8"); - await saveSessionStore( - storePath, + await replaceSessionEntry( + { sessionKey: "agent:main:room", storePath }, { - "agent:main:room": { - sessionFile: path.join(tempDir, "stale-session.jsonl"), - sessionId: "stale-session", - updatedAt: now - 1, - }, - "Agent:Main:Room": { - sessionFile: oldTranscriptPath, - sessionId: "old-session", - updatedAt: now, - }, + compactionCheckpoints: [ + { + checkpointId: "checkpoint-1", + createdAt: now, + postCompaction: { sessionId: "post-compaction-session" }, + preCompaction: { sessionId: "pre-compaction-session" }, + reason: "manual", + sessionId: "checkpoint-session", + sessionKey: "agent:main:room", + }, + ], + sessionId: "old-session", + updatedAt: now, + usageFamilySessionIds: ["old-session", "usage-family-session"], }, - { skipMaintenance: true }, ); + for (const sessionId of [ + "old-session", + "usage-family-session", + "checkpoint-session", + "pre-compaction-session", + "post-compaction-session", + ]) { + await replaceSqliteTranscriptEvents({ sessionKey: "agent:main:room", sessionId, storePath }, [ + createTranscriptEvent(sessionId, `before reset ${sessionId}`), + ]); + } + const transcriptUpdates = recordTranscriptUpdateFiles(); + let callbackTranscriptEvents: TestTranscriptEvent[] = []; const result = await resetSessionEntryLifecycle({ storePath, @@ -51,96 +83,144 @@ describe("session store lifecycle mutations", () => { canonicalKey: "agent:main:room", storeKeys: ["agent:main:room", "Agent:Main:Room"], }, - buildNextEntry: ({ currentEntry }): SessionEntry => ({ - ...currentEntry, - sessionFile: nextTranscriptPath, + buildNextEntry: (): SessionEntry => ({ sessionId: "next-session", updatedAt: now + 1, systemSent: false, abortedLastRun: false, }), - afterEntryMutation: () => { - nextTranscriptAtMutation = fs.readFileSync(nextTranscriptPath, "utf-8"); - oldTranscriptExistsAtMutation = fs.existsSync(oldTranscriptPath); + afterEntryMutation: async () => { + callbackTranscriptEvents = await loadTranscriptEvents({ + sessionKey: "agent:main:room", + sessionId: "old-session", + storePath, + }); }, }); + transcriptUpdates.unsubscribe(); - const store = loadSessionStore(storePath, { skipCache: true }); - expect(store["agent:main:room"]?.sessionId).toBe("next-session"); - expect(store["Agent:Main:Room"]).toBeUndefined(); + const stored = loadSessionEntry({ sessionKey: "agent:main:room", storePath }); + expect(stored?.sessionId).toBe("next-session"); expect(result.previousSessionId).toBe("old-session"); - expect(nextTranscriptAtMutation).toContain('"id":"next-session"'); - expect(oldTranscriptExistsAtMutation).toBe(true); - expect(result.archivedTranscripts).toHaveLength(1); - expect(result.archivedTranscripts[0]?.archivedPath).toContain(".jsonl.reset."); - expect(fs.existsSync(oldTranscriptPath)).toBe(false); - expect(fs.readFileSync(nextTranscriptPath, "utf-8")).toContain('"id":"next-session"'); - }); - - it("preserves a successor header when a custom transcript path is reused", async () => { - const sessionKey = "agent:main:custom"; - const transcriptPath = path.join(tempDir, "custom-transcript.jsonl"); - const oldSessionId = "11111111-1111-4111-8111-111111111111"; - const nextSessionId = "22222222-2222-4222-8222-222222222222"; - fs.writeFileSync(transcriptPath, `{"type":"session","id":"${oldSessionId}"}\n`, "utf-8"); - await saveSessionStore( - storePath, - { - [sessionKey]: { - sessionFile: transcriptPath, - sessionId: oldSessionId, - updatedAt: 1, - }, - }, - { skipMaintenance: true }, + expect(result.archivedTranscripts).toHaveLength(5); + expect(result.archivedTranscripts.map((transcript) => transcript.archivedPath)).toEqual( + expect.arrayContaining([ + expect.stringContaining("old-session.jsonl.reset."), + expect.stringContaining("usage-family-session.jsonl.reset."), + expect.stringContaining("checkpoint-session.jsonl.reset."), + expect.stringContaining("pre-compaction-session.jsonl.reset."), + expect.stringContaining("post-compaction-session.jsonl.reset."), + ]), ); - - await resetSessionEntryLifecycle({ - storePath, - target: { - canonicalKey: sessionKey, - storeKeys: [sessionKey], - }, - buildNextEntry: ({ currentEntry }): SessionEntry => ({ - ...currentEntry, - sessionFile: transcriptPath, - sessionId: nextSessionId, - updatedAt: 2, + expect(transcriptUpdates.files).toContain(result.archivedTranscripts[0]?.archivedPath); + expect(callbackTranscriptEvents).toEqual([]); + expect(readArchiveLinesForSession(result, "old-session")).toEqual([ + createTranscriptEventLine("old-session", "before reset old-session"), + ]); + expect(readArchiveLinesForSession(result, "usage-family-session")).toEqual([ + createTranscriptEventLine("usage-family-session", "before reset usage-family-session"), + ]); + expect(readArchiveLinesForSession(result, "checkpoint-session")).toEqual([ + createTranscriptEventLine("checkpoint-session", "before reset checkpoint-session"), + ]); + expect(readArchiveLinesForSession(result, "pre-compaction-session")).toEqual([ + createTranscriptEventLine("pre-compaction-session", "before reset pre-compaction-session"), + ]); + expect(readArchiveLinesForSession(result, "post-compaction-session")).toEqual([ + createTranscriptEventLine("post-compaction-session", "before reset post-compaction-session"), + ]); + await expect( + loadTranscriptEvents({ sessionKey: "agent:main:room", sessionId: "old-session", storePath }), + ).resolves.toEqual([]); + await expect( + loadTranscriptEvents({ + sessionKey: "agent:main:room", + sessionId: "usage-family-session", + storePath, }), - }); - - const archivedTranscript = fs - .readdirSync(tempDir) - .find((name) => name.startsWith("custom-transcript.jsonl.deleted.")); - if (!archivedTranscript) { - throw new Error("expected the previous custom transcript to be archived"); - } - expect(fs.readFileSync(path.join(tempDir, archivedTranscript), "utf-8")).toContain( - `"id":"${oldSessionId}"`, - ); - expect(fs.readFileSync(transcriptPath, "utf-8")).toContain(`"id":"${nextSessionId}"`); + ).resolves.toEqual([]); + await expect( + loadTranscriptEvents({ + sessionKey: "agent:main:room", + sessionId: "pre-compaction-session", + storePath, + }), + ).resolves.toEqual([]); }); - it("deletes an entry while archiving its transcript in the same lifecycle operation", async () => { - const transcriptPath = path.join(tempDir, "delete-session.jsonl"); + it("archives old SQLite transcript rows before reset callbacks can fail", async () => { const now = Date.now(); - fs.writeFileSync(transcriptPath, '{"type":"session","id":"delete-session"}\n', "utf-8"); - await saveSessionStore( - storePath, + await replaceSessionEntry( + { sessionKey: "agent:main:callback-failure", storePath }, { - "agent:main:keep": { - sessionId: "keep-session", - sessionFile: path.join(tempDir, "keep-session.jsonl"), - updatedAt: now, - }, - "agent:main:delete": { - sessionFile: transcriptPath, - sessionId: "delete-session", - updatedAt: now - 1, - }, + sessionId: "callback-old-session", + updatedAt: now, }, - { skipMaintenance: true }, ); + await replaceSqliteTranscriptEvents( + { + sessionKey: "agent:main:callback-failure", + sessionId: "callback-old-session", + storePath, + }, + [createTranscriptEvent("callback-old-session", "before callback failure")], + ); + + await expect( + resetSessionEntryLifecycle({ + storePath, + target: { + canonicalKey: "agent:main:callback-failure", + storeKeys: ["agent:main:callback-failure"], + }, + buildNextEntry: (): SessionEntry => ({ + sessionId: "callback-next-session", + updatedAt: now + 1, + }), + afterEntryMutation: () => { + throw new Error("callback failed"); + }, + }), + ).rejects.toThrow("callback failed"); + + await expect( + loadTranscriptEvents({ + sessionKey: "agent:main:callback-failure", + sessionId: "callback-old-session", + storePath, + }), + ).resolves.toEqual([]); + expect( + fs + .readdirSync(path.dirname(storePath)) + .filter((file) => file.startsWith("callback-old-session.jsonl.reset.")), + ).toHaveLength(1); + }); + + it("deletes an entry from SQLite while archiving unreferenced transcript rows", async () => { + const now = Date.now(); + await replaceSessionEntry( + { sessionKey: "agent:main:keep", storePath }, + { + sessionId: "keep-session", + updatedAt: now, + }, + ); + await replaceSessionEntry( + { sessionKey: "agent:main:delete", storePath }, + { + sessionId: "delete-session", + updatedAt: now - 1, + usageFamilySessionIds: ["delete-session", "delete-ancestor-session"], + }, + ); + for (const sessionId of ["delete-session", "delete-ancestor-session"]) { + await replaceSqliteTranscriptEvents( + { sessionKey: "agent:main:delete", sessionId, storePath }, + [createTranscriptEvent(sessionId, `before delete ${sessionId}`)], + ); + } + const transcriptUpdates = recordTranscriptUpdateFiles(); const result = await deleteSessionEntryLifecycle({ archiveTranscript: true, @@ -150,110 +230,500 @@ describe("session store lifecycle mutations", () => { storeKeys: ["agent:main:delete"], }, }); + transcriptUpdates.unsubscribe(); - const store = loadSessionStore(storePath, { skipCache: true }); expect(result.deleted).toBe(true); expect(result.deletedSessionId).toBe("delete-session"); - expect(result.archivedTranscripts).toHaveLength(1); - expect(result.archivedTranscripts[0]?.archivedPath).toContain(".jsonl.deleted."); - expect(store["agent:main:delete"]).toBeUndefined(); - expect(store["agent:main:keep"]?.sessionId).toBe("keep-session"); - expect(fs.existsSync(transcriptPath)).toBe(false); + expect(result.archivedTranscripts).toHaveLength(2); + expect(result.archivedTranscripts.map((transcript) => transcript.archivedPath)).toEqual( + expect.arrayContaining([ + expect.stringContaining("delete-session.jsonl.deleted."), + expect.stringContaining("delete-ancestor-session.jsonl.deleted."), + ]), + ); + expect(transcriptUpdates.files).toContain(result.archivedTranscripts[0]?.archivedPath); + expect(readArchiveLinesForSession(result, "delete-session")).toEqual([ + createTranscriptEventLine("delete-session", "before delete delete-session"), + ]); + expect(readArchiveLinesForSession(result, "delete-ancestor-session")).toEqual([ + createTranscriptEventLine("delete-ancestor-session", "before delete delete-ancestor-session"), + ]); + await expect( + loadTranscriptEvents({ + sessionKey: "agent:main:delete", + sessionId: "delete-session", + storePath, + }), + ).resolves.toEqual([]); + expect(loadSessionEntry({ sessionKey: "agent:main:delete", storePath })).toBeUndefined(); + expect(loadSessionEntry({ sessionKey: "agent:main:keep", storePath })?.sessionId).toBe( + "keep-session", + ); }); - it("keeps a row that changed before guarded deletion acquired the writer lock", async () => { - const sessionKey = "agent:main:delete"; - await saveSessionStore( - storePath, + it("durably writes SQLite transcript archives before deleting entry rows", async () => { + const now = Date.now(); + await replaceSessionEntry( + { sessionKey: "agent:main:durable-delete", storePath }, { - [sessionKey]: { - label: "new revision", - sessionId: "delete-session", - updatedAt: 2, - }, + sessionId: "durable-delete-session", + updatedAt: now, }, - { skipMaintenance: true }, + ); + await replaceSqliteTranscriptEvents( + { sessionKey: "agent:main:durable-delete", sessionId: "durable-delete-session", storePath }, + [createTranscriptEvent("durable-delete-session", "durable archive first")], ); - const result = await deleteSessionEntryLifecycle({ - archiveTranscript: false, - expectedSessionId: "delete-session", - expectedUpdatedAt: 1, - storePath, - target: { - canonicalKey: sessionKey, - storeKeys: [sessionKey], - }, + const originalWriteFileSync = fs.writeFileSync; + const entryObservedDuringArchiveWrite: boolean[] = []; + const writeSpy = vi.spyOn(fs, "writeFileSync").mockImplementation((...args) => { + const filePath = String(args[0]); + if (filePath.includes("durable-delete-session.jsonl.deleted.")) { + entryObservedDuringArchiveWrite.push( + loadSessionEntry({ sessionKey: "agent:main:durable-delete", storePath })?.sessionId === + "durable-delete-session", + ); + } + return originalWriteFileSync(...args); }); - expect(result).toMatchObject({ deleted: false, expectedEntryMismatch: true }); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toMatchObject({ - label: "new revision", - updatedAt: 2, - }); + try { + const result = await deleteSessionEntryLifecycle({ + archiveTranscript: true, + storePath, + target: { + canonicalKey: "agent:main:durable-delete", + storeKeys: ["agent:main:durable-delete"], + }, + }); + + expect(result.deleted).toBe(true); + expect(result.archivedTranscripts).toHaveLength(1); + expect(entryObservedDuringArchiveWrite).toEqual([true]); + } finally { + writeSpy.mockRestore(); + } }); - it("does not cache alias promotion when guarded deletion is rejected", async () => { - const canonicalKey = "agent:main:room"; - const aliasKey = "Agent:Main:Room"; - await saveSessionStore( - storePath, + it("probes duplicate SQLite transcript archives before deleting entry rows", async () => { + const now = Date.now(); + await replaceSessionEntry( + { sessionKey: "agent:main:duplicate-archive", storePath }, { - [canonicalKey]: { - label: "canonical", - sessionId: "canonical-session", - updatedAt: 1, - }, - [aliasKey]: { - label: "fresh alias", - sessionId: "alias-session", - updatedAt: 2, - }, + sessionId: "duplicate-archive-session", + updatedAt: now, }, - { skipMaintenance: true }, + ); + await replaceSqliteTranscriptEvents( + { + sessionKey: "agent:main:duplicate-archive", + sessionId: "duplicate-archive-session", + storePath, + }, + [createTranscriptEvent("duplicate-archive-session", "reuse archive")], + ); + const archivePath = path.join( + path.dirname(storePath), + "duplicate-archive-session.jsonl.deleted.2026-01-01T00-00-00.000Z", + ); + fs.mkdirSync(path.dirname(storePath), { recursive: true }); + fs.writeFileSync( + archivePath, + `${createTranscriptEventLine("duplicate-archive-session", "reuse archive")}\n`, + "utf-8", ); - const result = await deleteSessionEntryLifecycle({ - archiveTranscript: false, - expectedSessionId: "stale-session", - storePath, - target: { - canonicalKey, - storeKeys: [canonicalKey, aliasKey], - }, + const originalReaddirSync = fs.readdirSync; + const entryObservedDuringDuplicateProbe: boolean[] = []; + const readdirSpy = vi.spyOn(fs, "readdirSync").mockImplementation((...args) => { + const dirPath = String(args[0]); + if (dirPath === path.dirname(storePath)) { + entryObservedDuringDuplicateProbe.push( + loadSessionEntry({ sessionKey: "agent:main:duplicate-archive", storePath })?.sessionId === + "duplicate-archive-session", + ); + } + return originalReaddirSync(...args); }); - expect(result).toMatchObject({ deleted: false, expectedEntryMismatch: true }); - expect(loadSessionStore(storePath)).toMatchObject({ - [canonicalKey]: { label: "canonical", sessionId: "canonical-session" }, - [aliasKey]: { label: "fresh alias", sessionId: "alias-session" }, - }); + try { + const result = await deleteSessionEntryLifecycle({ + archiveTranscript: true, + storePath, + target: { + canonicalKey: "agent:main:duplicate-archive", + storeKeys: ["agent:main:duplicate-archive"], + }, + }); + + expect(result.deleted).toBe(true); + expect(result.archivedTranscripts).toEqual([ + { + archivedPath: archivePath, + sourcePath: path.join(path.dirname(storePath), "duplicate-archive-session.jsonl"), + }, + ]); + expect(entryObservedDuringDuplicateProbe).toEqual([true]); + } finally { + readdirSpy.mockRestore(); + } }); - it("deletes only the exact row snapshot supplied after lifecycle cleanup", async () => { - const sessionKey = "agent:main:delete"; - const expectedEntry = { - label: "cleanup-owned revision", - lifecycleRevision: "run-revision", - sessionId: "delete-session", - updatedAt: 2, - } satisfies SessionEntry; - await saveSessionStore(storePath, { [sessionKey]: expectedEntry }, { skipMaintenance: true }); + it("deletes a SQLite entry without deleting transcripts when archiveTranscript is false", async () => { + const now = Date.now(); + await replaceSessionEntry( + { sessionKey: "agent:main:delete-entry-only", storePath }, + { + sessionId: "entry-only-session", + updatedAt: now, + }, + ); + await replaceSqliteTranscriptEvents( + { sessionKey: "agent:main:delete-entry-only", sessionId: "entry-only-session", storePath }, + [createTranscriptEvent("entry-only-session", "preserve transcript rows")], + ); + const transcriptUpdates = recordTranscriptUpdateFiles(); const result = await deleteSessionEntryLifecycle({ archiveTranscript: false, - expectedEntry, - expectedLifecycleRevision: expectedEntry.lifecycleRevision, - expectedSessionId: expectedEntry.sessionId, - expectedUpdatedAt: expectedEntry.updatedAt, storePath, target: { - canonicalKey: sessionKey, - storeKeys: [sessionKey], + canonicalKey: "agent:main:delete-entry-only", + storeKeys: ["agent:main:delete-entry-only"], }, }); + transcriptUpdates.unsubscribe(); expect(result.deleted).toBe(true); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toBeUndefined(); + expect(result.archivedTranscripts).toEqual([]); + expect(transcriptUpdates.files).toEqual([]); + expect( + loadSessionEntry({ sessionKey: "agent:main:delete-entry-only", storePath }), + ).toBeUndefined(); + await expect( + loadTranscriptEvents({ + sessionKey: "agent:main:delete-entry-only", + sessionId: "entry-only-session", + storePath, + }), + ).resolves.toEqual([createTranscriptEvent("entry-only-session", "preserve transcript rows")]); + }); + + it("deletes SQLite transcript rows for non-archived lifecycle removals", async () => { + const now = Date.now(); + const entry: SessionEntry = { + sessionId: "lifecycle-remove-no-archive-session", + updatedAt: now, + }; + await replaceSessionEntry({ sessionKey: "agent:main:no-archive-removal", storePath }, entry); + await replaceSqliteTranscriptEvents( + { + sessionKey: "agent:main:no-archive-removal", + sessionId: "lifecycle-remove-no-archive-session", + storePath, + }, + [createTranscriptEvent("lifecycle-remove-no-archive-session", "remove rows without archive")], + ); + + const result = await applySessionEntryLifecycleMutation({ + storePath, + removals: [ + { + sessionKey: "agent:main:no-archive-removal", + expectedEntry: entry, + archiveRemovedTranscript: false, + }, + ], + maintenanceOverride: { mode: "enforce" }, + }); + + expect(result.removedSessionKeys).toEqual(["agent:main:no-archive-removal"]); + expect(result.archivedTranscriptDirectories).toEqual([]); + expect( + readArchiveNames( + path.dirname(storePath), + "lifecycle-remove-no-archive-session.jsonl.deleted.", + ), + ).toEqual([]); + await expect( + loadTranscriptEvents({ + sessionKey: "agent:main:no-archive-removal", + sessionId: "lifecycle-remove-no-archive-session", + storePath, + }), + ).resolves.toEqual([]); + }); + + it("archives shared SQLite transcript rows when any lifecycle removal requests archive", async () => { + const now = Date.now(); + const entry: SessionEntry = { + sessionId: "mixed-archive-shared-session", + updatedAt: now, + }; + await replaceSessionEntry({ sessionKey: "agent:main:mixed-archive-a", storePath }, entry); + await replaceSessionEntry({ sessionKey: "agent:main:mixed-archive-b", storePath }, entry); + await replaceSqliteTranscriptEvents( + { + sessionKey: "agent:main:mixed-archive-a", + sessionId: "mixed-archive-shared-session", + storePath, + }, + [createTranscriptEvent("mixed-archive-shared-session", "shared mixed archive")], + ); + + const result = await applySessionEntryLifecycleMutation({ + storePath, + removals: [ + { + sessionKey: "agent:main:mixed-archive-a", + expectedEntry: entry, + archiveRemovedTranscript: false, + }, + { + sessionKey: "agent:main:mixed-archive-b", + expectedEntry: entry, + archiveRemovedTranscript: true, + }, + ], + skipMaintenance: true, + }); + + expect(result.removedSessionKeys).toEqual([ + "agent:main:mixed-archive-a", + "agent:main:mixed-archive-b", + ]); + expect(result.archivedTranscriptDirectories).toEqual([path.dirname(storePath)]); + const archiveNames = readArchiveNames( + path.dirname(storePath), + "mixed-archive-shared-session.jsonl.deleted.", + ); + expect(archiveNames).toHaveLength(1); + expect(readArchiveLines(path.join(path.dirname(storePath), archiveNames[0] ?? ""))).toEqual([ + createTranscriptEventLine("mixed-archive-shared-session", "shared mixed archive"), + ]); + await expect( + loadTranscriptEvents({ + sessionKey: "agent:main:mixed-archive-a", + sessionId: "mixed-archive-shared-session", + storePath, + }), + ).resolves.toEqual([]); + }); + + it("forced maintenance preserves raw SQLite transcript-only rows", async () => { + await replaceSqliteTranscriptEvents( + { + sessionKey: "agent:main:raw-maintenance", + sessionId: "raw-maintenance-session", + storePath, + }, + [createTranscriptEvent("raw-maintenance-session", "raw transcript-only row")], + ); + + const result = await applySessionEntryLifecycleMutation({ + storePath, + activeSessionKey: "agent:main:raw-maintenance", + maintenanceOverride: { mode: "enforce" }, + }); + + expect(result.archivedTranscriptDirectories).toEqual([]); + expect( + readArchiveNames(path.dirname(storePath), "raw-maintenance-session.jsonl.deleted."), + ).toEqual([]); + await expect( + loadTranscriptEvents({ + sessionKey: "agent:main:raw-maintenance", + sessionId: "raw-maintenance-session", + storePath, + }), + ).resolves.toEqual([ + createTranscriptEvent("raw-maintenance-session", "raw transcript-only row"), + ]); + }); + + it("preserves shared SQLite transcript rows until the final session reference is deleted", async () => { + const now = Date.now(); + await replaceSessionEntry( + { sessionKey: "agent:main:first", storePath }, + { + sessionId: "shared-session", + updatedAt: now, + }, + ); + await replaceSessionEntry( + { sessionKey: "agent:main:second", storePath }, + { + sessionId: "shared-session", + updatedAt: now - 1, + }, + ); + await replaceSqliteTranscriptEvents( + { sessionKey: "agent:main:first", sessionId: "shared-session", storePath }, + [createTranscriptEvent("shared-session", "shared transcript")], + ); + + const first = await deleteSessionEntryLifecycle({ + archiveTranscript: true, + storePath, + target: { + canonicalKey: "agent:main:first", + storeKeys: ["agent:main:first"], + }, + }); + + expect(first.deleted).toBe(true); + expect(first.archivedTranscripts).toEqual([]); + await expect( + loadTranscriptEvents({ + sessionKey: "agent:main:second", + sessionId: "shared-session", + storePath, + }), + ).resolves.toEqual([createTranscriptEvent("shared-session", "shared transcript")]); + + const second = await deleteSessionEntryLifecycle({ + archiveTranscript: true, + storePath, + target: { + canonicalKey: "agent:main:second", + storeKeys: ["agent:main:second"], + }, + }); + + expect(second.deleted).toBe(true); + expect(second.archivedTranscripts).toHaveLength(1); + expect(readArchiveLines(second.archivedTranscripts[0]?.archivedPath)).toEqual([ + createTranscriptEventLine("shared-session", "shared transcript"), + ]); + await expect( + loadTranscriptEvents({ + sessionKey: "agent:main:second", + sessionId: "shared-session", + storePath, + }), + ).resolves.toEqual([]); + }); + + it("preserves raw SQLite entry references during lifecycle cleanup", async () => { + const sessionId = "raw-shared-session"; + await replaceSessionEntry( + { sessionKey: "agent:main:cleanup-target", storePath }, + { sessionId, updatedAt: Date.now() }, + ); + await replaceSessionEntry( + { sessionKey: "agent:main:protected-raw-reference", storePath }, + { sessionId, updatedAt: Date.now() }, + ); + await replaceSqliteTranscriptEvents( + { sessionKey: "agent:main:cleanup-target", sessionId, storePath }, + [createTranscriptEvent(sessionId, "cleanup-marker shared transcript")], + ); + const database = openLifecycleTestDatabase(storePath); + const db = getNodeSqliteKysely<OpenClawAgentKyselyDatabase>(database.db); + executeSqliteQuerySync( + database.db, + db + .updateTable("session_entries") + .set({ entry_json: "{not valid json" }) + .where("session_key", "=", "agent:main:protected-raw-reference"), + ); + + const result = await cleanupSessionLifecycleArtifacts({ + storePath, + sessionKeySegmentPrefix: "cleanup-target", + transcriptContentMarker: "cleanup-marker", + orphanTranscriptMinAgeMs: 0, + nowMs: Date.now() + 1, + }); + + expect(result).toEqual({ archivedTranscriptArtifacts: 0, removedEntries: 1 }); + expect( + executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_entries") + .select("session_key") + .where("session_key", "=", "agent:main:cleanup-target"), + ), + ).toBeUndefined(); + expect( + executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_entries") + .select(["entry_json", "session_id"]) + .where("session_key", "=", "agent:main:protected-raw-reference"), + ), + ).toEqual({ + entry_json: "{not valid json", + session_id: sessionId, + }); + expect( + executeSqliteQueryTakeFirstSync( + database.db, + db.selectFrom("sessions").select("session_id").where("session_id", "=", sessionId), + )?.session_id, + ).toBe(sessionId); }); }); + +function createTranscriptEvent(sessionId: string, content: string): TestTranscriptEvent { + return JSON.parse(createTranscriptEventLine(sessionId, content)) as TestTranscriptEvent; +} + +function createTranscriptEventLine(sessionId: string, content: string): string { + return JSON.stringify({ + type: "session", + id: sessionId, + content, + }); +} + +function readArchiveLines(archivePath: string | undefined): string[] { + expect(archivePath).toBeTruthy(); + return readSessionArchiveContentSync(archivePath ?? "") + .trim() + .split("\n"); +} + +function readArchiveNames(archiveDirectory: string, prefix: string): string[] { + if (!fs.existsSync(archiveDirectory)) { + return []; + } + return fs.readdirSync(archiveDirectory).filter((file) => file.startsWith(prefix)); +} + +function readArchiveLinesForSession( + result: { archivedTranscripts: Array<{ archivedPath: string }> }, + sessionId: string, +): string[] { + return readArchiveLines( + result.archivedTranscripts.find((transcript) => + transcript.archivedPath.includes(`${sessionId}.jsonl.`), + )?.archivedPath, + ); +} + +function recordTranscriptUpdateFiles(): { files: string[]; unsubscribe: () => void } { + const files: string[] = []; + return { + files, + unsubscribe: onInternalSessionTranscriptUpdate((update) => { + if (update.sessionFile) { + files.push(update.sessionFile); + } + }), + }; +} + +function openLifecycleTestDatabase(storePath: string) { + const target = resolveSqliteTargetFromSessionStorePath(storePath); + if (!target.path) { + throw new Error(`Could not resolve SQLite database path for ${storePath}`); + } + return openOpenClawAgentDatabase({ + agentId: target.agentId ?? "main", + path: target.path, + }); +} diff --git a/src/config/sessions/store.ts b/src/config/sessions/store.ts index 2b2951d2de44..78cffbb4790e 100644 --- a/src/config/sessions/store.ts +++ b/src/config/sessions/store.ts @@ -16,13 +16,6 @@ import { } from "../../sessions/agent-harness-session-key.js"; import { emitSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; import { createLazyRuntimeModule } from "../../shared/lazy-runtime.js"; -import { - deliveryContextFromChannelRoute, - deliveryContextFromSession, - mergeDeliveryContext, - normalizeDeliveryContext, - normalizeSessionDeliveryFields, -} from "../../utils/delivery-context.shared.js"; import type { DeliveryContext } from "../../utils/delivery-context.types.js"; import { getFileStatSnapshot } from "../cache-utils.js"; import type { OpenClawConfig } from "../types.openclaw.js"; @@ -32,7 +25,7 @@ import { type SessionUnreferencedArtifactSweepResult, } from "./disk-budget.js"; import { extractGeneratedTranscriptSessionId } from "./generated-transcript-session-id.js"; -import { deriveSessionMetaPatch } from "./metadata.js"; +import { deriveLastRoutePatch, deriveSessionMetaPatch } from "./metadata.js"; import { resolveExplicitSessionFilePath, resolveSessionFilePath } from "./paths.js"; import { resolveSessionStorePathForScope } from "./session-store-path.js"; import { @@ -150,15 +143,6 @@ const loadTrajectoryCleanupRuntime = createLazyRuntimeModule( () => import("../../trajectory/cleanup.js"), ); -function removeThreadFromDeliveryContext(context?: DeliveryContext): DeliveryContext | undefined { - if (!context || context.threadId == null) { - return context; - } - const next: DeliveryContext = { ...context }; - delete next.threadId; - return next; -} - export function readSessionUpdatedAt(params: { storePath: string; sessionKey: string; @@ -242,6 +226,8 @@ type SessionEntryWorkflowOptions = { }; export type SessionLifecycleArtifactCleanupParams = { + /** Agent owner used by SQLite-backed cleanup when the store path is custom. */ + agentId?: string; /** Session store to clean. */ storePath: string; /** Archive exact transcripts referenced by removed entries before the orphan marker scan. */ @@ -305,6 +291,8 @@ export type SessionEntryLifecycleRemoval = { /** Optional guard for stale plans built from a prior store read. */ expectedSessionId?: string; /** Optional guard for stale plans built from a prior store read. */ + expectedLifecycleRevision?: string; + /** Optional guard for stale plans built from a prior store read. */ expectedUpdatedAt?: number; }; @@ -1540,6 +1528,12 @@ function shouldRemoveSessionEntry( if (removal.expectedSessionId !== undefined && entry.sessionId !== removal.expectedSessionId) { return false; } + if ( + removal.expectedLifecycleRevision !== undefined && + entry.lifecycleRevision !== removal.expectedLifecycleRevision + ) { + return false; + } if (removal.expectedUpdatedAt !== undefined && entry.updatedAt !== removal.expectedUpdatedAt) { return false; } @@ -2243,70 +2237,21 @@ export async function updateLastRoute(params: { if (!existing && !createIfMissing) { return null; } - const explicitContext = normalizeDeliveryContext(params.deliveryContext); - const inlineContext = normalizeDeliveryContext({ + const patch = deriveLastRoutePatch({ channel, to, accountId, threadId, - }); - const routeContext = deliveryContextFromChannelRoute(params.route); - const mergedInput = mergeDeliveryContext( - routeContext, - mergeDeliveryContext(explicitContext, inlineContext), - ); - const explicitDeliveryContext = params.deliveryContext; - const explicitThreadFromDeliveryContext = - explicitDeliveryContext != null && Object.hasOwn(explicitDeliveryContext, "threadId") - ? explicitDeliveryContext.threadId - : undefined; - const explicitThreadValue = - explicitThreadFromDeliveryContext ?? - (threadId != null && threadId !== "" ? threadId : undefined); - const explicitRouteProvided = Boolean( - routeContext?.channel || - routeContext?.to || - explicitContext?.channel || - explicitContext?.to || - inlineContext?.channel || - inlineContext?.to, - ); - const clearThreadFromFallback = explicitRouteProvided && explicitThreadValue == null; - const fallbackContext = clearThreadFromFallback - ? removeThreadFromDeliveryContext(deliveryContextFromSession(existing)) - : deliveryContextFromSession(existing); - const merged = mergeDeliveryContext(mergedInput, fallbackContext); - const normalized = normalizeSessionDeliveryFields({ route: params.route, - deliveryContext: { - channel: merged?.channel, - to: merged?.to, - accountId: merged?.accountId, - threadId: merged?.threadId, - }, + deliveryContext: params.deliveryContext, + ctx, + groupResolution: params.groupResolution, + existing, + sessionKey: resolved.normalizedKey, }); - const metaPatch = ctx - ? deriveSessionMetaPatch({ - ctx, - sessionKey: resolved.normalizedKey, - existing, - groupResolution: params.groupResolution, - }) - : null; - const basePatch: Partial<SessionEntry> = { - route: normalized.route, - deliveryContext: normalized.deliveryContext, - lastChannel: normalized.lastChannel, - lastTo: normalized.lastTo, - lastAccountId: normalized.lastAccountId, - lastThreadId: normalized.lastThreadId, - }; // Route updates must not refresh activity timestamps; idle/daily reset // evaluation relies on updatedAt from actual session turns (#49515). - const next = mergeSessionEntryPreserveActivity( - existing, - metaPatch ? { ...basePatch, ...metaPatch } : basePatch, - ); + const next = mergeSessionEntryPreserveActivity(existing, patch); return await persistResolvedSessionEntry({ storePath, store, diff --git a/src/config/sessions/targets.test.ts b/src/config/sessions/targets.test.ts index 4b4deee1eb38..66f9c7445cdb 100644 --- a/src/config/sessions/targets.test.ts +++ b/src/config/sessions/targets.test.ts @@ -1,11 +1,11 @@ // Session target tests cover persisted channel targets for sessions. -import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { withTempHome } from "openclaw/plugin-sdk/test-env"; import { describe, expect, it } from "vitest"; import type { OpenClawConfig } from "../config.js"; import { resolveStorePath } from "./paths.js"; +import { replaceSessionEntry } from "./session-accessor.js"; import { resolveAgentSessionStoreTargetsSync, resolveAllAgentSessionStoreTargetsSync, @@ -13,8 +13,7 @@ import { } from "./targets.js"; async function resolveRealStorePath(sessionsDir: string): Promise<string> { - // Match the native realpath behavior used by both discovery paths. - return fsSync.realpathSync.native(path.join(sessionsDir, "sessions.json")); + return path.resolve(path.join(sessionsDir, "sessions.json")); } async function createAgentSessionStores( @@ -24,8 +23,12 @@ async function createAgentSessionStores( const storePaths: Record<string, string> = {}; for (const agentId of agentIds) { const sessionsDir = path.join(root, "agents", agentId, "sessions"); + const storePath = path.join(sessionsDir, "sessions.json"); await fs.mkdir(sessionsDir, { recursive: true }); - await fs.writeFile(path.join(sessionsDir, "sessions.json"), "{}", "utf8"); + await replaceSessionEntry( + { storePath, sessionKey: "main" }, + { sessionId: "sid", updatedAt: Date.now() }, + ); storePaths[agentId] = await resolveRealStorePath(sessionsDir); } return storePaths; @@ -143,7 +146,7 @@ describe("resolveSessionStoreTargets", () => { }); }); - it("dedupes shared store paths for --all-agents", () => { + it("keeps shared store paths distinct by SQLite owner for --all-agents", () => { const cfg: OpenClawConfig = { session: { store: "/tmp/shared-sessions.json", @@ -155,9 +158,40 @@ describe("resolveSessionStoreTargets", () => { expect(resolveSessionStoreTargets(cfg, { allAgents: true })).toEqual([ { agentId: "main", storePath: path.resolve("/tmp/shared-sessions.json") }, + { agentId: "work", storePath: path.resolve("/tmp/shared-sessions.json") }, ]); }); + it("uses the path-owned agent id for explicit agent store paths", async () => { + await withTempHome(async (home) => { + const stateDir = path.join(home, ".openclaw"); + const storePaths = await createAgentSessionStores(stateDir, ["codex-proof"]); + const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; + + expect(resolveSessionStoreTargets({}, { store: storePaths["codex-proof"] }, { env })).toEqual( + [ + { + agentId: "codex-proof", + storePath: storePaths["codex-proof"], + }, + ], + ); + }); + }); + + it("keeps arbitrary explicit store paths on the default agent", async () => { + await withTempHome(async (home) => { + const storePath = path.join(home, "backups", "sessions", "sessions.json"); + + expect(resolveSessionStoreTargets({}, { store: storePath })).toEqual([ + { + agentId: "main", + storePath, + }, + ]); + }); + }); + it("rejects unknown agent ids", () => { const cfg: OpenClawConfig = { agents: { @@ -231,6 +265,27 @@ describe("resolveAllAgentSessionStoreTargetsSync", () => { }); }); + it("includes legacy JSON stores before an agent SQLite database exists", async () => { + await withTempHome(async (home) => { + const stateDir = path.join(home, ".openclaw"); + const sessionsDir = path.join(stateDir, "agents", "legacy", "sessions"); + const storePath = path.join(sessionsDir, "sessions.json"); + await fs.mkdir(sessionsDir, { recursive: true }); + await fs.writeFile( + storePath, + JSON.stringify({ main: { sessionId: "legacy-session", updatedAt: Date.now() } }), + "utf8", + ); + + const targets = resolveAllAgentSessionStoreTargetsSync( + { agents: { list: [{ id: "legacy", default: true }] } }, + { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } }, + ); + + expect(targets).toContainEqual({ agentId: "legacy", storePath }); + }); + }); + it("discovers retired agent stores under a configured custom session root", async () => { await withTempHome(async (home) => { const { storePaths, targets } = await resolveTargetsForCustomRoot(home, ["ops", "retired"]); @@ -263,8 +318,14 @@ describe("resolveAllAgentSessionStoreTargetsSync", () => { const retiredSessionsDir = path.join(envStateDir, "agents", "retired", "sessions"); await fs.mkdir(mainSessionsDir, { recursive: true }); await fs.mkdir(retiredSessionsDir, { recursive: true }); - await fs.writeFile(path.join(mainSessionsDir, "sessions.json"), "{}", "utf8"); - await fs.writeFile(path.join(retiredSessionsDir, "sessions.json"), "{}", "utf8"); + await replaceSessionEntry( + { storePath: path.join(mainSessionsDir, "sessions.json"), sessionKey: "main" }, + { sessionId: "sid-main", updatedAt: Date.now() }, + ); + await replaceSessionEntry( + { storePath: path.join(retiredSessionsDir, "sessions.json"), sessionKey: "main" }, + { sessionId: "sid-retired", updatedAt: Date.now() }, + ); const env = { ...process.env, @@ -317,10 +378,12 @@ describe("resolveAllAgentSessionStoreTargetsSync", () => { } const customRoot = path.join(home, "custom-state"); const opsSessionsDir = path.join(customRoot, "agents", "ops", "sessions"); - const leakedFile = path.join(home, "outside.json"); + const opsAgentDbDir = path.join(customRoot, "agents", "ops", "agent"); + const leakedFile = path.join(home, "outside.sqlite"); await fs.mkdir(opsSessionsDir, { recursive: true }); + await fs.mkdir(opsAgentDbDir, { recursive: true }); await fs.writeFile(leakedFile, JSON.stringify({ leak: { secret: "x" } }), "utf8"); - await fs.symlink(leakedFile, path.join(opsSessionsDir, "sessions.json")); + await fs.symlink(leakedFile, path.join(opsAgentDbDir, "openclaw-agent.sqlite")); const targets = resolveAllAgentSessionStoreTargetsSync(createCustomRootCfg(customRoot), { env: process.env, @@ -341,8 +404,18 @@ describe("resolveAllAgentSessionStoreTargetsSync", () => { const junkSessionsDir = path.join(stateDir, "agents", "###", "sessions"); await fs.mkdir(mainSessionsDir, { recursive: true }); await fs.mkdir(junkSessionsDir, { recursive: true }); - await fs.writeFile(path.join(mainSessionsDir, "sessions.json"), "{}", "utf8"); - await fs.writeFile(path.join(junkSessionsDir, "sessions.json"), "{}", "utf8"); + await replaceSessionEntry( + { storePath: path.join(mainSessionsDir, "sessions.json"), sessionKey: "main" }, + { sessionId: "sid-main", updatedAt: Date.now() }, + ); + await replaceSessionEntry( + { + agentId: "main", + storePath: path.join(junkSessionsDir, "sessions.json"), + sessionKey: "main", + }, + { sessionId: "sid-junk", updatedAt: Date.now() }, + ); const cfg: OpenClawConfig = {}; const mainStorePath = await resolveRealStorePath(mainSessionsDir); diff --git a/src/config/sessions/targets.ts b/src/config/sessions/targets.ts index 775b9350051b..b0ef03246515 100644 --- a/src/config/sessions/targets.ts +++ b/src/config/sessions/targets.ts @@ -8,6 +8,7 @@ import { DEFAULT_AGENT_ID, normalizeAgentId } from "../../routing/session-key.js import { resolveStateDir } from "../paths.js"; import type { OpenClawConfig } from "../types.openclaw.js"; import { resolveAgentsDirFromSessionStorePath, resolveStorePath } from "./paths.js"; +import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js"; /** CLI/session-store target selection options. */ export type SessionStoreSelectionOptions = { @@ -41,6 +42,19 @@ function dedupeTargetsByStorePath(targets: SessionStoreTarget[]): SessionStoreTa return [...deduped.values()]; } +function dedupeTargetsBySqliteTarget(targets: SessionStoreTarget[]): SessionStoreTarget[] { + const deduped = new Map<string, SessionStoreTarget>(); + for (const target of targets) { + const sqlitePath = + resolveSqliteTargetFromSessionStorePath(target.storePath, { agentId: target.agentId }).path ?? + target.storePath; + if (!deduped.has(sqlitePath)) { + deduped.set(sqlitePath, target); + } + } + return [...deduped.values()]; +} + function shouldSkipDiscoveryError(err: unknown): boolean { const code = (err as NodeJS.ErrnoException | undefined)?.code; return typeof code === "string" && NON_FATAL_DISCOVERY_ERROR_CODES.has(code); @@ -59,6 +73,27 @@ function shouldSkipDiscoveredAgentDirName(dirName: string, agentId: string): boo ); } +function resolveValidatedManagedFilePathSync(params: { + agentsRoot: string; + filePath: string; + realAgentsRoot?: string; +}): string | undefined { + try { + const stat = fsSync.lstatSync(params.filePath); + if (stat.isSymbolicLink() || !stat.isFile()) { + return undefined; + } + const realFilePath = fsSync.realpathSync.native(params.filePath); + const realAgentsRoot = params.realAgentsRoot ?? fsSync.realpathSync.native(params.agentsRoot); + return isWithinRoot(realFilePath, realAgentsRoot) ? params.filePath : undefined; + } catch (err) { + if (shouldSkipDiscoveryError(err)) { + return undefined; + } + throw err; + } +} + /** Lists agent ids whose session stores should be considered configured. */ export function listConfiguredSessionStoreAgentIds(cfg: OpenClawConfig): string[] { const ids = new Set(listAgentIds(cfg).map((agentId) => normalizeAgentId(agentId))); @@ -91,22 +126,25 @@ function resolveValidatedDiscoveredStorePathSync(params: { realAgentsRoot?: string; }): string | undefined { const storePath = path.join(params.sessionsDir, "sessions.json"); - try { - const stat = fsSync.lstatSync(storePath); - // Discovered stores must be real files under the agents root; symlinked stores could escape - // the managed agent tree. - if (stat.isSymbolicLink() || !stat.isFile()) { - return undefined; - } - const realStorePath = fsSync.realpathSync.native(storePath); - const realAgentsRoot = params.realAgentsRoot ?? fsSync.realpathSync.native(params.agentsRoot); - return isWithinRoot(realStorePath, realAgentsRoot) ? realStorePath : undefined; - } catch (err) { - if (shouldSkipDiscoveryError(err)) { - return undefined; - } - throw err; + const validatedStorePath = resolveValidatedManagedFilePathSync({ + agentsRoot: params.agentsRoot, + filePath: storePath, + realAgentsRoot: params.realAgentsRoot, + }); + if (validatedStorePath) { + return validatedStorePath; } + const sqlitePath = resolveSqliteTargetFromSessionStorePath(storePath).path; + if (!sqlitePath) { + return undefined; + } + return resolveValidatedManagedFilePathSync({ + agentsRoot: params.agentsRoot, + filePath: sqlitePath, + realAgentsRoot: params.realAgentsRoot, + }) + ? storePath + : undefined; } function resolveSessionStoreDiscoveryState( @@ -150,6 +188,21 @@ function toDiscoveredSessionStoreTarget( }; } +function resolveExplicitSessionStoreTarget(params: { + defaultAgentId: string; + env: NodeJS.ProcessEnv; + store: string; +}): SessionStoreTarget { + const storePath = resolveStorePath(params.store, { + agentId: params.defaultAgentId, + env: params.env, + }); + const discovered = resolveAgentsDirFromSessionStorePath(storePath) + ? toDiscoveredSessionStoreTarget(path.dirname(storePath), storePath) + : undefined; + return discovered ?? { agentId: params.defaultAgentId, storePath }; +} + /** Resolves all configured and discoverable agent session stores synchronously. */ export function resolveAllAgentSessionStoreTargetsSync( cfg: OpenClawConfig, @@ -216,7 +269,7 @@ export function resolveAllAgentSessionStoreTargetsSync( throw err; } }); - return dedupeTargetsByStorePath([...validatedConfiguredTargets, ...discoveredTargets]); + return dedupeTargetsBySqliteTarget([...validatedConfiguredTargets, ...discoveredTargets]); } /** Resolves session store targets for one agent, including retired/manual stores. */ @@ -323,12 +376,7 @@ export function resolveSessionStoreTargets( } if (opts.store) { - return [ - { - agentId: defaultAgentId, - storePath: resolveStorePath(opts.store, { agentId: defaultAgentId, env }), - }, - ]; + return [resolveExplicitSessionStoreTarget({ defaultAgentId, env, store: opts.store })]; } if (allAgents) { @@ -336,7 +384,7 @@ export function resolveSessionStoreTargets( agentId, storePath: resolveStorePath(cfg.session?.store, { agentId, env }), })); - return dedupeTargetsByStorePath(targets); + return dedupeTargetsBySqliteTarget(targets); } if (hasAgent) { diff --git a/src/config/sessions/terminal-status.ts b/src/config/sessions/terminal-status.ts new file mode 100644 index 000000000000..1b747650e6ec --- /dev/null +++ b/src/config/sessions/terminal-status.ts @@ -0,0 +1,20 @@ +import type { SessionEntry } from "./types.js"; + +/** Returns true for terminal statuses that a later visible turn may recover in place. */ +export function isRecoverableTerminalSessionStatus( + status: SessionEntry["status"] | undefined, +): boolean { + return status === "failed" || status === "timeout" || status === "killed"; +} + +/** Clears stale terminal lifecycle fields before reusing a recoverable session entry. */ +export function recoverTerminalSessionEntryForVisibleTurn(entry: SessionEntry): SessionEntry { + return { + ...entry, + status: undefined, + startedAt: undefined, + endedAt: undefined, + runtimeMs: undefined, + abortedLastRun: undefined, + }; +} diff --git a/src/config/sessions/transcript-append-redact.test.ts b/src/config/sessions/transcript-append-redact.test.ts index 7a076f044955..d03686db5ab0 100644 --- a/src/config/sessions/transcript-append-redact.test.ts +++ b/src/config/sessions/transcript-append-redact.test.ts @@ -5,8 +5,9 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { onSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; import { resolveSessionTranscriptPathInDir } from "./paths.js"; +import { loadTranscriptEvents, replaceSessionEntry } from "./session-accessor.js"; import { useTempSessionsFixture } from "./test-helpers.js"; -import { appendSessionTranscriptMessage } from "./transcript-append.js"; +import { appendSessionTranscriptMessage } from "./transcript-append.test-support.js"; import { appendAssistantMessageToSessionTranscript, appendExactAssistantMessageToSessionTranscript, @@ -37,6 +38,17 @@ function readMessages(sessionFile: string) { .map((r) => r.message); } +async function readStoredMessages(params: { + sessionId: string; + sessionKey: string; + storePath: string; +}) { + return (await loadTranscriptEvents(params)) + .map((event) => event as { type?: string; message?: unknown }) + .filter((record) => record.type === "message") + .map((record) => record.message); +} + describe("appendSessionTranscriptMessage - redaction", () => { const fixture = useTempSessionsFixture("transcript-redact-test-"); @@ -366,16 +378,23 @@ describe("appendSessionTranscriptMessage - redaction", () => { describe("appendExactAssistantMessageToSessionTranscript - redaction", () => { const fixture = useTempSessionsFixture("exact-assistant-redact-test-"); + async function seedSessionEntry(params: { + sessionId: string; + sessionKey: string; + storePath: string; + }) { + await replaceSessionEntry( + { sessionKey: params.sessionKey, storePath: params.storePath }, + { sessionId: params.sessionId, updatedAt: Date.now() }, + ); + } + it("does not redact when config.logging.redactSensitive is off", async () => { - // Set up a minimal session store so the function can resolve the session file. const sessionsDir = fixture.sessionsDir(); const storePath = path.join(sessionsDir, "sessions.json"); const sessionId = "test-session-redact-off"; const sessionKey = "test-channel:test-user"; - const store = { - [sessionKey]: { sessionId, updatedAt: Date.now() }, - }; - fs.writeFileSync(storePath, JSON.stringify(store, null, 2), { encoding: "utf-8", mode: 0o600 }); + await seedSessionEntry({ sessionId, sessionKey, storePath }); const fakeApiKey = "sk-proj-FAKEKEYFORTESTINGONLY1234567890"; const config: OpenClawConfig = { logging: { redactSensitive: "off" } }; @@ -408,7 +427,7 @@ describe("appendExactAssistantMessageToSessionTranscript - redaction", () => { return; } - const raw = fs.readFileSync(result.sessionFile, "utf-8"); + const raw = JSON.stringify(await readStoredMessages({ sessionId, sessionKey, storePath })); expect(raw).toContain(fakeApiKey); }); @@ -417,11 +436,7 @@ describe("appendExactAssistantMessageToSessionTranscript - redaction", () => { const storePath = path.join(sessionsDir, "sessions.json"); const sessionId = "test-session-redact-event"; const sessionKey = "test-channel:test-redact-event"; - fs.writeFileSync( - storePath, - JSON.stringify({ [sessionKey]: { sessionId, updatedAt: Date.now() } }, null, 2), - { encoding: "utf-8", mode: 0o600 }, - ); + await seedSessionEntry({ sessionId, sessionKey, storePath }); const fakeApiKey = "sk-proj-FAKEKEYFORTESTINGONLY1234567890"; const config: OpenClawConfig = { logging: { redactSensitive: "tools" } }; @@ -457,7 +472,7 @@ describe("appendExactAssistantMessageToSessionTranscript - redaction", () => { return; } - const [diskMessage] = readMessages(result.sessionFile); + const [diskMessage] = await readStoredMessages({ sessionId, sessionKey, storePath }); expect(JSON.stringify(diskMessage)).not.toContain(fakeApiKey); expect(updates).toHaveLength(1); expect(updates[0]?.message).toEqual(diskMessage); @@ -472,11 +487,7 @@ describe("appendExactAssistantMessageToSessionTranscript - redaction", () => { const storePath = path.join(sessionsDir, "sessions.json"); const sessionId = "test-session-redact-dedupe"; const sessionKey = "test-channel:test-redact-dedupe"; - fs.writeFileSync( - storePath, - JSON.stringify({ [sessionKey]: { sessionId, updatedAt: Date.now() } }, null, 2), - { encoding: "utf-8", mode: 0o600 }, - ); + await seedSessionEntry({ sessionId, sessionKey, storePath }); const fakeApiKey = "sk-proj-FAKEKEYFORTESTINGONLY1234567890"; const config: OpenClawConfig = { logging: { redactSensitive: "tools" } }; @@ -501,9 +512,11 @@ describe("appendExactAssistantMessageToSessionTranscript - redaction", () => { } expect(second.messageId).toBe(first.messageId); - const raw = fs.readFileSync(second.sessionFile, "utf-8"); - expect(raw).not.toContain(fakeApiKey); - expect(readMessages(second.sessionFile)).toHaveLength(1); + const events = await loadTranscriptEvents({ sessionId, sessionKey, storePath }); + expect(JSON.stringify(events)).not.toContain(fakeApiKey); + expect(events.filter((event) => (event as { type?: unknown }).type === "message")).toHaveLength( + 1, + ); }); it("dedupes delivery mirrors against older unredacted assistant entries", async () => { @@ -511,11 +524,7 @@ describe("appendExactAssistantMessageToSessionTranscript - redaction", () => { const storePath = path.join(sessionsDir, "sessions.json"); const sessionId = "test-session-redact-upgrade-dedupe"; const sessionKey = "test-channel:test-redact-upgrade-dedupe"; - fs.writeFileSync( - storePath, - JSON.stringify({ [sessionKey]: { sessionId, updatedAt: Date.now() } }, null, 2), - { encoding: "utf-8", mode: 0o600 }, - ); + await seedSessionEntry({ sessionId, sessionKey, storePath }); const fakeApiKey = "sk-proj-OLDERUNREDACTEDTRANSCRIPT1234567890"; const unredacted = await appendExactAssistantMessageToSessionTranscript({ @@ -554,8 +563,10 @@ describe("appendExactAssistantMessageToSessionTranscript - redaction", () => { } expect(deduped.messageId).toBe(unredacted.messageId); - const raw = fs.readFileSync(deduped.sessionFile, "utf-8"); - expect(raw).toContain(fakeApiKey); - expect(readMessages(deduped.sessionFile)).toHaveLength(1); + const events = await loadTranscriptEvents({ sessionId, sessionKey, storePath }); + expect(JSON.stringify(events)).toContain(fakeApiKey); + expect(events.filter((event) => (event as { type?: unknown }).type === "message")).toHaveLength( + 1, + ); }); }); diff --git a/src/config/sessions/transcript-append.ts b/src/config/sessions/transcript-append.test-support.ts similarity index 100% rename from src/config/sessions/transcript-append.ts rename to src/config/sessions/transcript-append.test-support.ts diff --git a/src/config/sessions/transcript-file-resolve.ts b/src/config/sessions/transcript-file-resolve.ts index 00c53bceec17..1527920944fb 100644 --- a/src/config/sessions/transcript-file-resolve.ts +++ b/src/config/sessions/transcript-file-resolve.ts @@ -1,11 +1,6 @@ // Resolves transcript file targets without depending on transcript read/write facades. -import { - resolveSessionFilePath, - resolveSessionFilePathOptions, - resolveSessionTranscriptPath, -} from "./paths.js"; +import { resolveSessionFilePath, resolveSessionFilePathOptions } from "./paths.js"; import { resolveAndPersistSessionFile } from "./session-file.js"; -import { parseSessionThreadInfo } from "./thread-info.js"; import type { SessionEntry } from "./types.js"; /** @@ -29,15 +24,6 @@ export async function resolveSessionTranscriptFile(params: { let sessionEntry = params.sessionEntry; if (params.sessionStore && params.storePath) { - // Persisting the resolved transcript path keeps later tail reads and exports on the same file. - const threadIdFromSessionKey = parseSessionThreadInfo(params.sessionKey).threadId; - const fallbackSessionFile = !sessionEntry?.sessionFile - ? resolveSessionTranscriptPath( - params.sessionId, - params.agentId, - params.threadId ?? threadIdFromSessionKey, - ) - : undefined; const resolvedSessionFile = await resolveAndPersistSessionFile({ sessionId: params.sessionId, sessionKey: params.sessionKey, @@ -45,8 +31,6 @@ export async function resolveSessionTranscriptFile(params: { storePath: params.storePath, sessionEntry, agentId: sessionPathOpts?.agentId, - sessionsDir: sessionPathOpts?.sessionsDir, - fallbackSessionFile, }); sessionFile = resolvedSessionFile.sessionFile; sessionEntry = resolvedSessionFile.sessionEntry; diff --git a/src/config/sessions/transcript-jsonl.ts b/src/config/sessions/transcript-jsonl.ts index 15b064f81088..3d3ef74d3bdb 100644 --- a/src/config/sessions/transcript-jsonl.ts +++ b/src/config/sessions/transcript-jsonl.ts @@ -28,8 +28,8 @@ export function serializeJsonlLine(entry: unknown): string { return serialized; } -function serializeJsonlEntries(entries: readonly unknown[]): string { - return serializeJsonlLines(entries.map(serializeJsonlLine)); +export function serializeJsonlEntries(jsonlEntries: readonly unknown[]): string { + return serializeJsonlLines(jsonlEntries.map(serializeJsonlLine)); } export function serializeJsonlLines(lines: readonly string[]): string { diff --git a/src/config/sessions/transcript-replay.ts b/src/config/sessions/transcript-replay.ts index 1c70306a2cd6..ab5f6195b2fb 100644 --- a/src/config/sessions/transcript-replay.ts +++ b/src/config/sessions/transcript-replay.ts @@ -15,6 +15,7 @@ type SessionRecord = { message?: { role?: unknown }; }; type KeptRecord = { role: "user" | "assistant"; line: string }; +type KeptParsedRecord = { role: "user" | "assistant"; record: unknown }; function isValidReplayTimestamp(value: unknown): boolean { if (typeof value === "number") { @@ -62,33 +63,13 @@ export async function replayRecentUserAssistantMessages(params: { return 0; } try { - const kept: KeptRecord[] = []; - for (const line of (await fsp.readFile(src, "utf-8")).split(/\r?\n/)) { - if (!line.trim()) { - continue; - } - try { - const role = replayableRole(JSON.parse(line) as SessionRecord | null); - if (role) { - kept.push({ role, line }); - } - } catch { - // Skip malformed lines. - } - } - if (kept.length === 0) { + const tail = await readRecentUserAssistantReplayLines({ + sourceTranscript: src, + maxMessages: max, + }); + if (tail.length === 0) { return 0; } - let startIdx = Math.max(0, kept.length - max); - while (startIdx < kept.length && kept[startIdx].role === "assistant") { - startIdx += 1; - } - if (startIdx === kept.length) { - // Retained window is assistant-only; replaying would re-create the same - // role-ordering hazard this reset path is recovering from. - return 0; - } - const tail = coalesceAlternatingReplayTail(kept.slice(startIdx)).map((entry) => entry.line); if (!fs.existsSync(params.targetTranscript)) { await fsp.mkdir(path.dirname(params.targetTranscript), { recursive: true }); const header = JSON.stringify({ @@ -110,10 +91,94 @@ export async function replayRecentUserAssistantMessages(params: { } } +export async function readRecentUserAssistantReplayRecordsFromJsonl(params: { + sourceTranscript?: string; + maxMessages?: number; +}): Promise<unknown[]> { + const max = Math.max(0, params.maxMessages ?? DEFAULT_REPLAY_MAX_MESSAGES); + const src = params.sourceTranscript; + if (max === 0 || !src || !fs.existsSync(src)) { + return []; + } + const records: unknown[] = []; + for (const line of (await fsp.readFile(src, "utf-8")).split(/\r?\n/)) { + if (!line.trim()) { + continue; + } + try { + records.push(JSON.parse(line) as unknown); + } catch { + // Skip malformed lines. + } + } + return selectRecentUserAssistantReplayRecords(records, max); +} + +export function selectRecentUserAssistantReplayRecords( + records: readonly unknown[], + maxMessages = DEFAULT_REPLAY_MAX_MESSAGES, +): unknown[] { + const max = Math.max(0, maxMessages); + if (max === 0) { + return []; + } + const kept: KeptParsedRecord[] = []; + for (const record of records) { + const role = replayableRole(record as SessionRecord | null); + if (role) { + kept.push({ role, record }); + } + } + const tail = selectAlternatingReplayTail(kept, max); + return tail.map((entry) => entry.record); +} + +async function readRecentUserAssistantReplayLines(params: { + sourceTranscript: string; + maxMessages: number; +}): Promise<string[]> { + const kept: KeptRecord[] = []; + for (const line of (await fsp.readFile(params.sourceTranscript, "utf-8")).split(/\r?\n/)) { + if (!line.trim()) { + continue; + } + try { + const role = replayableRole(JSON.parse(line) as SessionRecord | null); + if (role) { + kept.push({ role, line }); + } + } catch { + // Skip malformed lines. + } + } + return selectAlternatingReplayTail(kept, params.maxMessages).map((entry) => entry.line); +} + +function selectAlternatingReplayTail<T extends { role: "user" | "assistant" }>( + kept: T[], + max: number, +): T[] { + if (kept.length === 0) { + return []; + } + let startIdx = Math.max(0, kept.length - max); + while (startIdx < kept.length && kept[startIdx]?.role === "assistant") { + startIdx += 1; + } + if (startIdx === kept.length) { + // Retained window is assistant-only; replaying would re-create the same + // role-ordering hazard this reset path is recovering from. + return []; + } + return coalesceAlternatingReplayTail(kept.slice(startIdx)); +} + // Keep the newest record from each same-role run, preserving original JSONL bytes // for replay while ensuring strict provider alternation. -function coalesceAlternatingReplayTail(entries: KeptRecord[]): KeptRecord[] { - const tail: KeptRecord[] = []; +function coalesceAlternatingReplayTail<T extends { role: "user" | "assistant" }>( + entries: T[], +): T[] { + const tail: T[] = []; for (const entry of entries) { const lastIdx = tail.length - 1; if (lastIdx >= 0 && tail[lastIdx]?.role === entry.role) { diff --git a/src/config/sessions/transcript-tree.test.ts b/src/config/sessions/transcript-tree.test.ts index 44a343a84d7f..a0958344beb3 100644 --- a/src/config/sessions/transcript-tree.test.ts +++ b/src/config/sessions/transcript-tree.test.ts @@ -289,6 +289,28 @@ describe("session transcript tree helpers", () => { ]); }); + it("normalizes canonical rows with stale parents onto the active leaf", () => { + const entries = [ + { type: "message", id: "user", parentId: null }, + { type: "message", id: "assistant", parentId: "user" }, + { + type: "custom", + id: "bootstrap-context", + parentId: "missing-stale-parent", + customType: "openclaw:bootstrap-context:full", + }, + ]; + + const tree = scanSessionTranscriptTree(entries); + + expect(tree.byId.get("bootstrap-context")?.parentId).toBe("assistant"); + expect(selectSessionTranscriptTreePathNodes(tree, tree.leafId).map((node) => node.id)).toEqual([ + "user", + "assistant", + "bootstrap-context", + ]); + }); + it("preserves side ancestry after an explicit side append leaf", () => { const entries = [ { type: "custom", id: "visible", parentId: null }, diff --git a/src/config/sessions/transcript-tree.ts b/src/config/sessions/transcript-tree.ts index 551130e29a04..4c3806065d5a 100644 --- a/src/config/sessions/transcript-tree.ts +++ b/src/config/sessions/transcript-tree.ts @@ -208,13 +208,20 @@ export function scanSessionTranscriptTree<T>(entries: readonly T[]): SessionTran let treeEntry: SessionTranscriptTreeEntry | undefined = explicitTreeEntry ?? parseParentlessCanonicalEntry(entry, leafId); if (treeEntry && isCanonicalSessionTranscriptEntry(entry)) { - const logicalParentId = + const canonicalParentIsStale = explicitTreeEntry && - treeEntry.appendMode !== "side" && - treeEntry.parentId === appendParentId && - leafId !== appendParentId + treeEntry.parentId !== null && + !byId.has(treeEntry.parentId) && + leafId !== null; + const logicalParentId = + treeEntry.appendMode !== "side" && canonicalParentIsStale ? leafId - : treeEntry.parentId; + : explicitTreeEntry && + treeEntry.appendMode !== "side" && + treeEntry.parentId === appendParentId && + leafId !== appendParentId + ? leafId + : treeEntry.parentId; const normalizedParentId = resolveCanonicalParentId(logicalParentId, byId); if (normalizedParentId !== treeEntry.parentId) { // The raw cursor can belong to plugin metadata, an inactive branch, or diff --git a/src/config/sessions/transcript-visible-events.ts b/src/config/sessions/transcript-visible-events.ts new file mode 100644 index 000000000000..b021dacdd195 --- /dev/null +++ b/src/config/sessions/transcript-visible-events.ts @@ -0,0 +1,39 @@ +import { + scanSessionTranscriptTree, + selectSessionTranscriptTreePathNodes, +} from "./transcript-tree.js"; + +export type VisibleTranscriptEventEntry<T> = { + event: T; + /** Parent id after active-branch normalization; null when no visible parent exists. */ + parentId: string | null; + seq: number; +}; + +/** Selects the active visible branch while preserving original transcript sequence numbers. */ +export function selectVisibleTranscriptEventEntries<T>( + events: readonly T[], +): VisibleTranscriptEventEntry<T>[] { + const tree = scanSessionTranscriptTree(events); + const visiblePath = selectSessionTranscriptTreePathNodes(tree, tree.leafId); + if (visiblePath.length > 0) { + return visiblePath.map((node) => ({ + event: node.entry, + parentId: node.parentId, + seq: node.index + 1, + })); + } + return tree.hasLeafControl + ? [] + : events.map((event, index) => ({ event, parentId: null, seq: index + 1 })); +} + +/** Selects only events on the active visible transcript branch. */ +export function selectVisibleTranscriptEvents<T>(events: readonly T[]): T[] { + return selectVisibleTranscriptEventEntries(events).map((entry) => entry.event); +} + +/** Resolves the parent id that the next active transcript append should use. */ +export function resolveVisibleTranscriptAppendParentId(events: readonly unknown[]): string | null { + return scanSessionTranscriptTree(events).appendParentId; +} diff --git a/src/config/sessions/transcript-write-context.ts b/src/config/sessions/transcript-write-context.ts index caf8a042f3b1..7b858450b3d7 100644 --- a/src/config/sessions/transcript-write-context.ts +++ b/src/config/sessions/transcript-write-context.ts @@ -1,6 +1,7 @@ // Transcript write contexts let nested append paths reuse an already-owned session write lock. import { AsyncLocalStorage } from "node:async_hooks"; import path from "node:path"; +import { parseSqliteSessionFileMarker } from "./sqlite-marker.js"; type OwnedSessionTranscriptWriteContext = { sessionFile?: string; @@ -34,10 +35,14 @@ export type OwnedSessionTranscriptCacheSnapshot = { const ownedTranscriptWriteContext = new AsyncLocalStorage<OwnedSessionTranscriptWriteContext>(); -// Compare resolved paths when available; fall back to session keys for lock reuse. -function normalizePathForCompare(value: string | undefined): string | undefined { +// Compare concrete files when available; SQLite markers fall back to session +// identity because they are storage references rather than lockable paths. +function normalizeConcretePathForCompare(value: string | undefined): string | undefined { const trimmed = value?.trim(); - return trimmed ? path.resolve(trimmed) : undefined; + if (!trimmed || parseSqliteSessionFileMarker(trimmed)) { + return undefined; + } + return path.resolve(trimmed); } function contextMatches(params: { @@ -45,8 +50,8 @@ function contextMatches(params: { sessionFile?: string; sessionKey?: string; }): boolean { - const contextSessionFile = normalizePathForCompare(params.context.sessionFile); - const sessionFile = normalizePathForCompare(params.sessionFile); + const contextSessionFile = normalizeConcretePathForCompare(params.context.sessionFile); + const sessionFile = normalizeConcretePathForCompare(params.sessionFile); if (contextSessionFile && sessionFile) { return contextSessionFile === sessionFile; } diff --git a/src/config/sessions/transcript.test.ts b/src/config/sessions/transcript.test.ts index d7a051485eb5..ef634a63bfe6 100644 --- a/src/config/sessions/transcript.test.ts +++ b/src/config/sessions/transcript.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { beforeAll, describe, expect, it, vi } from "vitest"; import { repairToolUseResultPairing } from "../../agents/session-transcript-repair.js"; import * as transcriptEvents from "../../sessions/transcript-events.js"; -import type { SessionTranscriptUpdate } from "../../sessions/transcript-events.js"; +import type { InternalSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; import { OPENCLAW_DELIVERY_MIRROR_MODEL, OPENCLAW_TRANSCRIPT_ARTIFACT_API, @@ -13,15 +13,24 @@ import { } from "../../shared/transcript-only-openclaw-assistant.js"; import { deleteTestEnvValue, setTestEnvValue } from "../../test-utils/env.js"; import { resolveSessionTranscriptPathInDir } from "./paths.js"; -import { updateSessionStoreEntry } from "./store.js"; +import { + loadTranscriptEvents, + appendTranscriptEvent, + loadSessionEntry, + persistSessionTranscriptTurn, + readLatestTranscriptAssistantText, + replaceSessionEntry, + updateSessionEntry, +} from "./session-accessor.js"; import { useTempSessionsFixture } from "./test-helpers.js"; import { appendSessionTranscriptEvent, appendSessionTranscriptMessage, -} from "./transcript-append.js"; +} from "./transcript-append.test-support.js"; import { selectSessionTranscriptLeafControlledPath } from "./transcript-tree.js"; import { bindOwnedSessionTranscriptWrites, + runWithOwnedSessionTranscriptWriteLock, withOwnedSessionTranscriptWrites, } from "./transcript-write-context.js"; import { @@ -32,6 +41,7 @@ import { readRecentUserAssistantTextFromSessionTranscript, readTailAssistantTextFromSessionTranscript, } from "./transcript.js"; +import type { SessionEntry } from "./types.js"; describe("appendAssistantMessageToSessionTranscript", () => { beforeAll(async () => { @@ -40,10 +50,9 @@ describe("appendAssistantMessageToSessionTranscript", () => { const sessionsDir = path.join(tempDir, "agents", "main", "sessions"); fs.mkdirSync(sessionsDir, { recursive: true }); const storePath = path.join(sessionsDir, "sessions.json"); - fs.writeFileSync( - storePath, - JSON.stringify({ warm: { sessionId: "warm-session", chatType: "direct" } }), - "utf-8", + await replaceSessionEntry( + { sessionKey: "warm", storePath }, + { sessionId: "warm-session", chatType: "direct", updatedAt: 1 }, ); await appendAssistantMessageToSessionTranscript({ sessionKey: "warm", @@ -69,21 +78,30 @@ describe("appendAssistantMessageToSessionTranscript", () => { type TranscriptRepairMessage = Parameters<typeof repairToolUseResultPairing>[0][number]; type TranscriptUpdateEmitterSpy = { mock: { - calls: [string | SessionTranscriptUpdate][]; + calls: [string | InternalSessionTranscriptUpdate][]; }; }; - function writeTranscriptStore() { - fs.writeFileSync( - fixture.storePath(), - JSON.stringify({ - [sessionKey]: { - sessionId, - chatType: "direct", - channel: "discord", - }, - }), - "utf-8", + async function writeTranscriptStore(entry: Partial<SessionEntry> = {}) { + await replaceSessionEntry( + { agentId: "main", sessionKey, storePath: fixture.storePath() }, + { + sessionId, + chatType: "direct", + channel: "discord", + updatedAt: 1, + ...entry, + }, + ); + } + + async function writeTranscriptSessionEntry(params: { + entry: Partial<SessionEntry> & Pick<SessionEntry, "sessionId">; + sessionKey: string; + }) { + await replaceSessionEntry( + { agentId: "main", sessionKey: params.sessionKey, storePath: fixture.storePath() }, + { updatedAt: 1, ...params.entry }, ); } @@ -112,7 +130,9 @@ describe("appendAssistantMessageToSessionTranscript", () => { }; } - function requireTranscriptUpdateCall(spy: TranscriptUpdateEmitterSpy): SessionTranscriptUpdate { + function requireTranscriptUpdateCall( + spy: TranscriptUpdateEmitterSpy, + ): InternalSessionTranscriptUpdate { const call = spy.mock.calls[0]; if (!call) { throw new Error("expected transcript update event"); @@ -124,6 +144,22 @@ describe("appendAssistantMessageToSessionTranscript", () => { return event; } + async function loadFixtureMessages( + override: { + agentId?: string; + sessionId?: string; + sessionKey?: string; + storePath?: string; + } = {}, + ): Promise<Array<{ message?: unknown }>> { + return (await loadTranscriptEvents({ + agentId: override.agentId ?? "main", + sessionId: override.sessionId ?? sessionId, + sessionKey: override.sessionKey ?? sessionKey, + storePath: override.storePath ?? fixture.storePath(), + })) as Array<{ message?: unknown }>; + } + it("uses configured session.store when storePath is omitted", async () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "transcript-config-store-")); const previousStateDir = process.env.OPENCLAW_STATE_DIR; @@ -133,15 +169,13 @@ describe("appendAssistantMessageToSessionTranscript", () => { fs.mkdirSync(sessionsDir, { recursive: true }); const storePath = path.join(sessionsDir, "sessions.json"); const configuredSessionKey = "agent:main:configured-store"; - fs.writeFileSync( - storePath, - JSON.stringify({ - [configuredSessionKey]: { - sessionId: "configured-session-id", - chatType: "direct", - }, - }), - "utf-8", + await replaceSessionEntry( + { agentId: "main", sessionKey: configuredSessionKey, storePath }, + { + sessionId: "configured-session-id", + chatType: "direct", + updatedAt: 1, + }, ); const result = await appendAssistantMessageToSessionTranscript({ @@ -155,12 +189,21 @@ describe("appendAssistantMessageToSessionTranscript", () => { if (!result.ok) { throw new Error(result.reason); } - expect(path.basename(result.sessionFile)).toBe("configured-session-id.jsonl"); - expect(fs.realpathSync.native(path.dirname(result.sessionFile))).toBe( - fs.realpathSync.native(sessionsDir), - ); - await expect(fs.promises.readFile(result.sessionFile, "utf-8")).resolves.toContain( - "mirrored configured store reply", + expect(result.sessionFile).toContain("sqlite:main:configured-session-id:"); + await expect( + loadTranscriptEvents({ + agentId: "main", + sessionId: "configured-session-id", + sessionKey: configuredSessionKey, + storePath, + }), + ).resolves.toContainEqual( + expect.objectContaining({ + message: expect.objectContaining({ + role: "assistant", + content: [{ type: "text", text: "mirrored configured store reply" }], + }), + }), ); } finally { if (previousStateDir === undefined) { @@ -183,15 +226,13 @@ describe("appendAssistantMessageToSessionTranscript", () => { fs.mkdirSync(sessionsDir, { recursive: true }); const storePath = path.join(sessionsDir, "sessions.json"); const configuredSessionKey = "agent:worker:configured-store"; - fs.writeFileSync( - storePath, - JSON.stringify({ - [configuredSessionKey]: { - sessionId: "worker-session-id", - chatType: "direct", - }, - }), - "utf-8", + await replaceSessionEntry( + { agentId: "worker", sessionKey: configuredSessionKey, storePath }, + { + sessionId: "worker-session-id", + chatType: "direct", + updatedAt: 1, + }, ); const beforeMessageWrite = vi.fn(({ message }: BeforeMessageWriteParams) => message); @@ -206,12 +247,21 @@ describe("appendAssistantMessageToSessionTranscript", () => { if (!result.ok) { throw new Error(result.reason); } - expect(path.basename(result.sessionFile)).toBe("worker-session-id.jsonl"); - expect(fs.realpathSync.native(path.dirname(result.sessionFile))).toBe( - fs.realpathSync.native(sessionsDir), - ); - await expect(fs.promises.readFile(result.sessionFile, "utf-8")).resolves.toContain( - "mirrored worker store reply", + expect(result.sessionFile).toContain("sqlite:worker:worker-session-id:"); + await expect( + loadTranscriptEvents({ + agentId: "worker", + sessionId: "worker-session-id", + sessionKey: configuredSessionKey, + storePath, + }), + ).resolves.toContainEqual( + expect.objectContaining({ + message: expect.objectContaining({ + role: "assistant", + content: [{ type: "text", text: "mirrored worker store reply" }], + }), + }), ); expect(beforeMessageWrite).toHaveBeenCalledWith( expect.objectContaining({ @@ -233,8 +283,8 @@ describe("appendAssistantMessageToSessionTranscript", () => { } }); - it("creates transcript file and appends message for valid session", async () => { - writeTranscriptStore(); + it("creates a SQLite transcript marker and appends message for valid session", async () => { + await writeTranscriptStore(); const result = await appendAssistantMessageToSessionTranscript({ sessionKey, @@ -244,24 +294,16 @@ describe("appendAssistantMessageToSessionTranscript", () => { expect(result.ok).toBe(true); if (result.ok) { - expect(fs.existsSync(result.sessionFile)).toBe(true); - const sessionFileMode = fs.statSync(result.sessionFile).mode & 0o777; - if (process.platform !== "win32") { - expect(sessionFileMode).toBe(0o600); - } - - const lines = fs.readFileSync(result.sessionFile, "utf-8").trim().split("\n"); - expect(lines.length).toBe(2); - - const header = JSON.parse(lines[0]); - expect(header.type).toBe("session"); - expect(header.id).toBe(sessionId); - - const messageLine = JSON.parse(lines[1]); - expect(messageLine.type).toBe("message"); - expect(messageLine.message.role).toBe("assistant"); - expect(messageLine.message.content[0].type).toBe("text"); - expect(messageLine.message.content[0].text).toBe("Hello from delivery mirror!"); + expect(result.sessionFile).toContain("sqlite:main:test-session-id:"); + const events = await loadFixtureMessages(); + expect(events).toContainEqual( + expect.objectContaining({ + message: expect.objectContaining({ + role: "assistant", + content: [{ type: "text", text: "Hello from delivery mirror!" }], + }), + }), + ); } }); @@ -269,18 +311,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { const updatedAt = Date.parse("2026-05-18T09:00:00.000Z"); const appendedAt = Date.parse("2026-05-18T09:05:00.000Z"); const sessionFile = "managed-marker.jsonl"; - fs.writeFileSync( - fixture.storePath(), - JSON.stringify({ - [sessionKey]: { - sessionId, - sessionFile, - updatedAt, - status: "done", - }, - }), - "utf-8", - ); + await writeTranscriptStore({ sessionFile, updatedAt, status: "done" }); vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(appendedAt); try { @@ -291,12 +322,13 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); expect(result.ok).toBe(true); - const store = JSON.parse(fs.readFileSync(fixture.storePath(), "utf-8")) as Record< - string, - { updatedAt?: number; status?: string } - >; - expect(store[sessionKey]?.updatedAt).toBe(appendedAt); - expect(store[sessionKey]?.status).toBe("done"); + const saved = loadSessionEntry({ + agentId: "main", + sessionKey, + storePath: fixture.storePath(), + }); + expect(saved?.updatedAt).toBe(appendedAt); + expect(saved?.status).toBe("done"); } finally { vi.useRealTimers(); } @@ -307,18 +339,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { const firstAppendAt = Date.parse("2026-05-18T10:05:00.000Z"); const duplicateReplayAt = Date.parse("2026-05-18T10:10:00.000Z"); const sessionFile = "duplicate-marker.jsonl"; - fs.writeFileSync( - fixture.storePath(), - JSON.stringify({ - [sessionKey]: { - sessionId, - sessionFile, - updatedAt, - status: "done", - }, - }), - "utf-8", - ); + await writeTranscriptStore({ sessionFile, updatedAt, status: "done" }); vi.useFakeTimers({ toFake: ["Date"] }); try { vi.setSystemTime(firstAppendAt); @@ -337,11 +358,12 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); expect(duplicate.ok).toBe(true); - const store = JSON.parse(fs.readFileSync(fixture.storePath(), "utf-8")) as Record< - string, - { updatedAt?: number } - >; - expect(store[sessionKey]?.updatedAt).toBe(firstAppendAt); + const saved = loadSessionEntry({ + agentId: "main", + sessionKey, + storePath: fixture.storePath(), + }); + expect(saved?.updatedAt).toBe(firstAppendAt); if (first.ok && duplicate.ok) { expect(duplicate.messageId).toBe(first.messageId); } @@ -353,18 +375,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { it("uses spawned cwd when creating a missing transcript header", async () => { const taskCwd = path.join(fixture.sessionsDir(), "task-repo"); fs.mkdirSync(taskCwd, { recursive: true }); - fs.writeFileSync( - fixture.storePath(), - JSON.stringify({ - [sessionKey]: { - sessionId, - chatType: "direct", - channel: "discord", - spawnedCwd: taskCwd, - }, - }), - "utf-8", - ); + await writeTranscriptStore({ spawnedCwd: taskCwd }); const result = await appendAssistantMessageToSessionTranscript({ sessionKey, @@ -374,15 +385,27 @@ describe("appendAssistantMessageToSessionTranscript", () => { expect(result.ok).toBe(true); if (result.ok) { - const [headerLine] = fs.readFileSync(result.sessionFile, "utf-8").trim().split("\n"); - const header = JSON.parse(headerLine ?? "{}") as { cwd?: string }; - expect(header.cwd).toBe(taskCwd); + const events = await loadFixtureMessages(); + expect(events).toContainEqual( + expect.objectContaining({ + cwd: taskCwd, + type: "session", + }), + ); + expect(events).toContainEqual( + expect.objectContaining({ + message: expect.objectContaining({ + role: "assistant", + content: [{ type: "text", text: "Hello from task cwd!" }], + }), + }), + ); } }); it("runs matching owned transcript appends through the active session write lock", async () => { - writeTranscriptStore(); - const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); + await writeTranscriptStore(); + const sessionFile = `sqlite:main:${sessionId}:${fixture.storePath()}`; const events: string[] = []; const result = await withOwnedSessionTranscriptWrites( @@ -406,6 +429,37 @@ describe("appendAssistantMessageToSessionTranscript", () => { expect(events).toEqual(["lock"]); }); + it("does not reuse owned write locks for a different concrete transcript file", async () => { + const oldSessionFile = resolveSessionTranscriptPathInDir("old-session", fixture.sessionsDir()); + const nextSessionFile = resolveSessionTranscriptPathInDir( + "next-session", + fixture.sessionsDir(), + ); + const events: string[] = []; + + const result = await withOwnedSessionTranscriptWrites( + { + sessionFile: oldSessionFile, + sessionKey, + withSessionWriteLock: async (run) => { + events.push("lock"); + return await run(); + }, + }, + async () => + await runWithOwnedSessionTranscriptWriteLock( + { sessionFile: nextSessionFile, sessionKey }, + () => { + events.push("write"); + return "ok"; + }, + ), + ); + + expect(result).toBe("ok"); + expect(events).toEqual(["write"]); + }); + it("keeps matching owned transcript appends locked from bound callbacks", async () => { const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); const events: string[] = []; @@ -440,17 +494,14 @@ describe("appendAssistantMessageToSessionTranscript", () => { const mixedGroupId = "VWATodkf2hc8zdOS76q9Tb0+5Bi522E03qLdaQ/9ypg="; const signalSessionKey = `agent:main:signal:group:${mixedGroupId}`; const legacySignalSessionKey = signalSessionKey.toLowerCase(); - fs.writeFileSync( - fixture.storePath(), - JSON.stringify({ - [legacySignalSessionKey]: { - sessionId, - chatType: "group", - channel: "signal", - }, - }), - "utf-8", - ); + await writeTranscriptSessionEntry({ + sessionKey: legacySignalSessionKey, + entry: { + sessionId, + chatType: "group", + channel: "signal", + }, + }); const result = await appendAssistantMessageToSessionTranscript({ sessionKey: signalSessionKey, @@ -460,25 +511,24 @@ describe("appendAssistantMessageToSessionTranscript", () => { expect(result.ok).toBe(true); if (result.ok) { - const lines = fs.readFileSync(result.sessionFile, "utf-8").trim().split("\n"); - expect(lines).toHaveLength(2); - const messageLine = JSON.parse(lines[1]); - expect(messageLine.message.content[0].text).toBe("Hello Signal group"); + const events = await loadFixtureMessages({ + sessionKey: legacySignalSessionKey, + }); + expect(events).toContainEqual( + expect.objectContaining({ + message: expect.objectContaining({ + content: [{ type: "text", text: "Hello Signal group" }], + }), + }), + ); } }); - it("falls back to the canonical transcript path for malformed persisted sessionFile metadata", async () => { - fs.writeFileSync( - fixture.storePath(), - JSON.stringify({ - [sessionKey]: { - sessionId, - sessionFile: { path: "../../escaped.jsonl" }, - updatedAt: Date.now(), - }, - }), - "utf-8", - ); + it("uses the SQLite transcript marker for malformed persisted sessionFile metadata", async () => { + await writeTranscriptStore({ + sessionFile: { path: "../../escaped.jsonl" } as unknown as string, + updatedAt: Date.now(), + }); const result = await appendAssistantMessageToSessionTranscript({ sessionKey, @@ -488,22 +538,19 @@ describe("appendAssistantMessageToSessionTranscript", () => { expect(result.ok).toBe(true); if (result.ok) { - expect(result.sessionFile).toBe( - resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()), + expect(result.sessionFile).toContain("sqlite:main:test-session-id:"); + await expect(loadFixtureMessages()).resolves.toContainEqual( + expect.objectContaining({ + message: expect.objectContaining({ + content: [{ type: "text", text: "Hello from a repaired metadata boundary" }], + }), + }), ); - expect(fs.existsSync(result.sessionFile)).toBe(true); } }); it("emits transcript update events for delivery mirrors", async () => { - const store = { - [sessionKey]: { - sessionId, - chatType: "direct", - channel: "discord", - }, - }; - fs.writeFileSync(fixture.storePath(), JSON.stringify(store), "utf-8"); + await writeTranscriptStore(); const emitSpy = vi.spyOn(transcriptEvents, "emitSessionTranscriptUpdate"); await appendAssistantMessageToSessionTranscript({ @@ -512,7 +559,6 @@ describe("appendAssistantMessageToSessionTranscript", () => { storePath: fixture.storePath(), }); - const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); expect(emitSpy).toHaveBeenCalledTimes(1); const event = requireTranscriptUpdateCall(emitSpy); const message = event.message as @@ -524,7 +570,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { content?: unknown; } | undefined; - expect(event?.sessionFile).toBe(sessionFile); + expect(event?.sessionFile).toContain("sqlite:main:test-session-id:"); expect(event?.sessionKey).toBe(sessionKey); expect(event?.messageId).toBeTypeOf("string"); expect(message?.role).toBe("assistant"); @@ -536,7 +582,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("does not append a duplicate delivery mirror for the same idempotency key", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); await appendAssistantMessageToSessionTranscript({ sessionKey, @@ -551,17 +597,18 @@ describe("appendAssistantMessageToSessionTranscript", () => { storePath: fixture.storePath(), }); - const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); - const lines = fs.readFileSync(sessionFile, "utf-8").trim().split("\n"); - expect(lines.length).toBe(2); - - const messageLine = JSON.parse(lines[1]); - expect(messageLine.message.idempotencyKey).toBe("mirror:test-source-message"); - expect(messageLine.message.content[0].text).toBe("Hello from delivery mirror!"); + const events = await loadFixtureMessages(); + const messages = events.flatMap((entry) => (entry.message ? [entry.message] : [])) as Array<{ + idempotencyKey?: string; + content?: Array<{ text?: string }>; + }>; + expect(messages).toHaveLength(1); + expect(messages[0]?.idempotencyKey).toBe("mirror:test-source-message"); + expect(messages[0]?.content?.[0]?.text).toBe("Hello from delivery mirror!"); }); it("does not append a duplicate delivery mirror when the latest assistant message already matches", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const exactResult = await appendExactAssistantMessageToSessionTranscript({ sessionKey, @@ -580,18 +627,154 @@ describe("appendAssistantMessageToSessionTranscript", () => { expect(mirrorResult.ok).toBe(true); if (exactResult.ok && mirrorResult.ok) { expect(mirrorResult.messageId).toBe(exactResult.messageId); - const lines = fs.readFileSync(mirrorResult.sessionFile, "utf-8").trim().split("\n"); - expect(lines.length).toBe(2); - - const messageLine = JSON.parse(lines[1]); - expect(messageLine.message.provider).toBe("codex"); - expect(messageLine.message.model).toBe("gpt-5.4"); - expect(messageLine.message.content[0].text).toBe("Hello from Codex!"); + await expect( + loadTranscriptEvents({ + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), + }), + ).resolves.toContainEqual( + expect.objectContaining({ + message: expect.objectContaining({ + provider: "codex", + model: "gpt-5.4", + content: [expect.objectContaining({ text: "Hello from Codex!" })], + }), + }), + ); } }); + it("dedupes delivery mirrors against the active SQLite branch tail", async () => { + await writeTranscriptStore(); + await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), + }, + { + updateMode: "none", + messages: [ + { + message: createExactAssistantMessage({ text: "Active branch reply" }), + }, + { + message: createExactAssistantMessage({ text: "Inactive branch reply" }), + }, + ], + }, + ); + const events = await loadTranscriptEvents({ + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), + }); + const assistantEvents = events.filter( + (event): event is { id: string; message: { role: string; content: unknown } } => + Boolean(event) && + typeof event === "object" && + !Array.isArray(event) && + (event as { message?: { role?: string } }).message?.role === "assistant" && + typeof (event as { id?: unknown }).id === "string", + ); + const active = assistantEvents[0]; + const inactive = assistantEvents[1]; + if (!active || !inactive) { + throw new Error("expected assistant events"); + } + await appendTranscriptEvent( + { agentId: "main", sessionId, sessionKey, storePath: fixture.storePath() }, + { + type: "leaf", + id: "active-leaf", + parentId: inactive.id, + targetId: active.id, + }, + ); + + const mirrorResult = await appendAssistantMessageToSessionTranscript({ + sessionKey, + text: "Active branch reply", + storePath: fixture.storePath(), + }); + + expect(mirrorResult.ok).toBe(true); + if (mirrorResult.ok) { + expect(mirrorResult.messageId).toBe(active.id); + } + }); + + it("does not downgrade a marked SQLite session entry on normal assistant appends", async () => { + const marker = `sqlite:main:${sessionId}:${fixture.storePath()}`; + await writeTranscriptStore({ + sessionFile: marker, + updatedAt: 100, + pluginExtensions: { + "metadata-owner": { + preserved: true, + }, + }, + }); + + const result = await appendExactAssistantMessageToSessionTranscript({ + sessionKey, + storePath: fixture.storePath(), + message: createExactAssistantMessage({ text: "Normal SQLite reply" }), + }); + + expect(result.ok).toBe(true); + const saved = loadSessionEntry({ agentId: "main", sessionKey, storePath: fixture.storePath() }); + expect(saved?.sessionFile).toBe(marker); + expect(saved?.updatedAt).toBeGreaterThan(100); + expect(saved?.pluginExtensions).toEqual({ + "metadata-owner": { + preserved: true, + }, + }); + }); + + it("does not downgrade a marked SQLite session entry on duplicate delivery mirrors", async () => { + const marker = `sqlite:main:${sessionId}:${fixture.storePath()}`; + await writeTranscriptStore({ sessionFile: marker }); + + const existingTurn = await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), + }, + { + updateMode: "none", + messages: [ + { + message: createExactAssistantMessage({ text: "Existing SQLite reply" }), + }, + ], + }, + ); + const existingMessageId = existingTurn.messages[0]?.messageId; + const mirrorResult = await appendAssistantMessageToSessionTranscript({ + sessionKey, + text: "Existing SQLite reply", + storePath: fixture.storePath(), + }); + + expect(existingMessageId).toBeTypeOf("string"); + expect(mirrorResult.ok).toBe(true); + if (mirrorResult.ok) { + expect(mirrorResult.messageId).toBe(existingMessageId); + } + const saved = loadSessionEntry({ agentId: "main", sessionKey, storePath: fixture.storePath() }); + expect(saved?.sessionFile).toBe(marker); + }); + it("idempotently appends identified channel finals while preserving repeated replies", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const first = await appendAssistantMessageToSessionTranscript({ sessionKey, @@ -621,9 +804,11 @@ describe("appendAssistantMessageToSessionTranscript", () => { if (first.ok && replay.ok && nextTurn.ok) { expect(replay.messageId).toBe(first.messageId); expect(nextTurn.messageId).not.toBe(first.messageId); - const lines = fs.readFileSync(first.sessionFile, "utf-8").trim().split("\n"); - expect(lines).toHaveLength(3); - expect(JSON.parse(lines[1]).message.openclawDeliveryMirror).toEqual({ + const messages = (await loadFixtureMessages()).flatMap((entry) => + entry.message ? [entry.message] : [], + ) as Array<{ openclawDeliveryMirror?: unknown }>; + expect(messages).toHaveLength(2); + expect(messages[0]?.openclawDeliveryMirror).toEqual({ kind: "channel-final", sourceMessageId: "message-1", }); @@ -631,7 +816,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("idempotently appends suppressed channel finals by key while preserving source ids", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const text = "Channel final suppressed before delivery: stale foreground"; const first = await appendAssistantMessageToSessionTranscript({ @@ -674,11 +859,19 @@ describe("appendAssistantMessageToSessionTranscript", () => { if (first.ok && replay.ok && nextTurn.ok) { expect(replay.messageId).toBe(first.messageId); expect(nextTurn.messageId).not.toBe(first.messageId); - const lines = fs.readFileSync(first.sessionFile, "utf-8").trim().split("\n"); - expect(lines).toHaveLength(3); - const firstMirror = JSON.parse(lines[1]).message; - const secondMirror = JSON.parse(lines[2]).message; - expect(firstMirror).toMatchObject({ + const events = await loadTranscriptEvents({ + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), + }); + const mirrors = events + .map((event) => (event as { message?: Record<string, unknown> }).message) + .filter((message): message is Record<string, unknown> => + Boolean(message?.openclawDeliveryMirror), + ); + expect(mirrors).toHaveLength(2); + expect(mirrors[0]).toMatchObject({ api: OPENCLAW_TRANSCRIPT_ARTIFACT_API, provider: OPENCLAW_TRANSCRIPT_ARTIFACT_PROVIDER, model: OPENCLAW_DELIVERY_MIRROR_MODEL, @@ -688,34 +881,16 @@ describe("appendAssistantMessageToSessionTranscript", () => { sourceMessageId: "message-1", }, }); - expect(secondMirror.openclawDeliveryMirror).toEqual({ + expect(mirrors[1]?.openclawDeliveryMirror).toEqual({ kind: "channel-final-suppressed", reason: "stale-foreground", sourceMessageId: "message-2", }); - await appendSessionTranscriptMessage({ - transcriptPath: first.sessionFile, - message: { - role: "user", - content: "next user turn", - timestamp: Date.now(), - }, - }); - await expect( - readRecentUserAssistantTextFromSessionTranscript(first.sessionFile, { limit: 10 }), - ).resolves.toEqual([ - { - id: expect.any(String), - role: "user", - text: "next user turn", - timestamp: expect.any(Number), - }, - ]); } }); - it("dedupes against the latest assistant even when a large user entry follows it", async () => { - writeTranscriptStore(); + it("does not dedupe delivery mirrors against an older assistant after a user turn", async () => { + await writeTranscriptStore(); const exactResult = await appendExactAssistantMessageToSessionTranscript({ sessionKey, @@ -728,13 +903,22 @@ describe("appendAssistantMessageToSessionTranscript", () => { return; } - const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); - await appendSessionTranscriptMessage({ - transcriptPath: sessionFile, - message: { role: "user", content: "x".repeat(128 * 1024) }, - }); + await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), + }, + { + updateMode: "none", + messages: [{ message: { role: "user", content: "x".repeat(128 * 1024) } }], + }, + ); - const latestAssistantText = await readLatestAssistantTextFromSessionTranscript(sessionFile); + const latestAssistantText = await readLatestAssistantTextFromSessionTranscript( + exactResult.sessionFile, + ); if (!latestAssistantText) { throw new Error("expected latest assistant text"); } @@ -749,18 +933,19 @@ describe("appendAssistantMessageToSessionTranscript", () => { expect(mirrorResult.ok).toBe(true); if (mirrorResult.ok) { - expect(mirrorResult.messageId).toBe(exactResult.messageId); - const records = fs - .readFileSync(sessionFile, "utf-8") - .trim() - .split("\n") - .map((line) => JSON.parse(line) as { type?: string; message?: { role?: string } }); - expect(records.filter((record) => record.type === "message")).toHaveLength(2); + expect(mirrorResult.messageId).not.toBe(exactResult.messageId); + const records = await loadFixtureMessages(); + const messages = records.flatMap((record) => + record.message ? [record.message] : [], + ) as Array<{ model?: string; content?: Array<{ text?: string }> }>; + expect(messages).toHaveLength(3); + expect(messages[2]?.model).toBe("delivery-mirror"); + expect(messages[2]?.content?.[0]?.text).toBe("Hello before the large user entry"); } }); it("reads bounded recent user and assistant text before the current turn", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); await appendSessionTranscriptMessage({ @@ -812,7 +997,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("skips transcript-only OpenClaw assistant entries when reading recent prompt context", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); await appendSessionTranscriptMessage({ @@ -860,12 +1045,19 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("resolves recent transcript context from session identity", async () => { - writeTranscriptStore(); - const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); - await appendSessionTranscriptMessage({ - transcriptPath: sessionFile, - message: { role: "user", content: "from shared session", timestamp: 4_000 }, - }); + await writeTranscriptStore(); + await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), + }, + { + updateMode: "none", + messages: [{ message: { role: "user", content: "from shared session", timestamp: 4_000 } }], + }, + ); await expect( readRecentUserAssistantTextForSession({ @@ -883,21 +1075,103 @@ describe("appendAssistantMessageToSessionTranscript", () => { ]); }); + it("prefers SQLite transcript rows for recent context from session identity", async () => { + await writeTranscriptStore(); + const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); + await appendSessionTranscriptMessage({ + transcriptPath: sessionFile, + message: { role: "user", content: "stale jsonl context", timestamp: 1_000 }, + }); + await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), + }, + { + updateMode: "none", + messages: [ + { + message: { role: "user", content: "sqlite identity context", timestamp: 4_000 }, + }, + ], + }, + ); + + await expect( + readRecentUserAssistantTextForSession({ + sessionKey, + storePath: fixture.storePath(), + beforeTimestampMs: 5_000, + }), + ).resolves.toEqual([ + { + id: expect.any(String), + role: "user", + text: "sqlite identity context", + timestamp: 4_000, + }, + ]); + }); + + it("does not fall back to stale JSONL when SQLite rows are outside the recent window", async () => { + await writeTranscriptStore(); + const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); + await appendSessionTranscriptMessage({ + transcriptPath: sessionFile, + message: { role: "user", content: "stale jsonl context", timestamp: 1_000 }, + }); + await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), + }, + { + updateMode: "none", + messages: [ + { + message: { role: "user", content: "sqlite future context", timestamp: 10_000 }, + }, + ], + }, + ); + + await expect( + readRecentUserAssistantTextForSession({ + sessionKey, + storePath: fixture.storePath(), + beforeTimestampMs: 5_000, + }), + ).resolves.toEqual([]); + }); + + it("does not fall back to stale JSONL when SQLite has no transcript rows", async () => { + await writeTranscriptStore({ + sessionFile: `sqlite:main:${sessionId}:${fixture.storePath()}`, + }); + const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); + await appendSessionTranscriptMessage({ + transcriptPath: sessionFile, + message: { role: "user", content: "stale jsonl context", timestamp: 1_000 }, + }); + + await expect( + readRecentUserAssistantTextForSession({ + sessionKey, + storePath: fixture.storePath(), + beforeTimestampMs: 5_000, + }), + ).resolves.toEqual([]); + }); + it("ignores stored session files outside the sessions directory for recent context", async () => { const outsideDir = fs.mkdtempSync(path.join(os.tmpdir(), "transcript-outside-")); try { const outsideFile = path.join(outsideDir, "outside.jsonl"); - fs.writeFileSync( - fixture.storePath(), - JSON.stringify({ - [sessionKey]: { - sessionId, - chatType: "direct", - sessionFile: outsideFile, - }, - }), - "utf-8", - ); + await writeTranscriptStore({ sessionFile: outsideFile }); await appendSessionTranscriptMessage({ transcriptPath: outsideFile, message: { role: "user", content: "outside text", timestamp: 1_000 }, @@ -907,6 +1181,18 @@ describe("appendAssistantMessageToSessionTranscript", () => { transcriptPath: sessionFile, message: { role: "user", content: "contained text", timestamp: 2_000 }, }); + await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), + }, + { + updateMode: "none", + messages: [{ message: { role: "user", content: "sqlite text", timestamp: 2_500 } }], + }, + ); await expect( readRecentUserAssistantTextForSession({ @@ -918,8 +1204,8 @@ describe("appendAssistantMessageToSessionTranscript", () => { { id: expect.any(String), role: "user", - text: "contained text", - timestamp: 2_000, + text: "sqlite text", + timestamp: 2_500, }, ]); } finally { @@ -928,7 +1214,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("skips transcript-only OpenClaw assistant entries when reading latest assistant text", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const finalResult = await appendExactAssistantMessageToSessionTranscript({ sessionKey, @@ -963,7 +1249,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("does not report transcript-only OpenClaw assistant entries as latest assistant text", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const mirrorResult = await appendAssistantMessageToSessionTranscript({ sessionKey, @@ -982,7 +1268,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("keeps transcript-only OpenClaw assistant entries available to the tail reader", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const mirrorResult = await appendAssistantMessageToSessionTranscript({ sessionKey, @@ -1007,7 +1293,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { // undefined on the first non-assistant line, so the gap-fill check in // persistTextTurnTranscript wrote a duplicate `api: "cli"` assistant // message — poisoning the model's own context with verbatim duplicates. - writeTranscriptStore(); + await writeTranscriptStore(); const assistantResult = await appendExactAssistantMessageToSessionTranscript({ sessionKey, @@ -1032,7 +1318,15 @@ describe("appendAssistantMessageToSessionTranscript", () => { modelId: "claude-haiku-4-5-20251001", }, })}\n`; - fs.appendFileSync(assistantResult.sessionFile, cacheTtlEntry, "utf-8"); + await appendTranscriptEvent( + { + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), + }, + JSON.parse(cacheTtlEntry), + ); const tailAssistantText = await readTailAssistantTextFromSessionTranscript( assistantResult.sessionFile, @@ -1041,8 +1335,132 @@ describe("appendAssistantMessageToSessionTranscript", () => { expect(tailAssistantText?.text).toBe("Canonical answer"); }); + it("scans past trailing assistant entries without visible text", async () => { + await writeTranscriptStore(); + + const assistantResult = await appendExactAssistantMessageToSessionTranscript({ + sessionKey, + storePath: fixture.storePath(), + message: createExactAssistantMessage({ + text: "Visible answer before tool call", + }), + }); + expect(assistantResult.ok).toBe(true); + if (!assistantResult.ok) { + return; + } + + const toolOnlyResult = await appendExactAssistantMessageToSessionTranscript({ + sessionKey, + storePath: fixture.storePath(), + message: { + ...createExactAssistantMessage({ + content: [ + { + type: "toolCall", + id: "call_latest_no_visible_text", + name: "maniple__list_workers", + arguments: {}, + }, + ], + }), + stopReason: "toolUse", + }, + }); + expect(toolOnlyResult.ok).toBe(true); + if (!toolOnlyResult.ok) { + return; + } + + const latestAssistantText = readLatestTranscriptAssistantText({ + sessionId, + sessionKey, + storePath: fixture.storePath(), + }); + expect(latestAssistantText?.id).toBe(assistantResult.messageId); + expect(latestAssistantText?.text).toBe("Visible answer before tool call"); + }); + + it("does not scan past a real tail assistant with no visible text for dedupe", async () => { + await writeTranscriptStore(); + + const assistantResult = await appendExactAssistantMessageToSessionTranscript({ + sessionKey, + storePath: fixture.storePath(), + message: createExactAssistantMessage({ + text: "Older visible answer", + }), + }); + expect(assistantResult.ok).toBe(true); + + const toolOnlyResult = await appendExactAssistantMessageToSessionTranscript({ + sessionKey, + storePath: fixture.storePath(), + message: { + ...createExactAssistantMessage({ + content: [ + { + type: "toolCall", + id: "call_tail_no_visible_text", + name: "maniple__list_workers", + arguments: {}, + }, + ], + }), + stopReason: "toolUse", + }, + }); + expect(toolOnlyResult.ok).toBe(true); + if (!toolOnlyResult.ok) { + return; + } + + await expect( + readTailAssistantTextFromSessionTranscript(toolOnlyResult.sessionFile, { + excludeTranscriptOnlyOpenClawAssistant: true, + }), + ).resolves.toBeUndefined(); + }); + + it("scans past excluded delivery mirrors in explicit file-backed tail reads", async () => { + const transcriptPath = path.join(fixture.sessionsDir(), "file-backed-delivery-tail.jsonl"); + fs.writeFileSync( + transcriptPath, + [ + { + type: "message", + id: "real-assistant", + message: createExactAssistantMessage({ text: "File-backed canonical answer" }), + }, + { + type: "message", + id: "delivery-mirror", + message: { + ...createExactAssistantMessage({ + provider: "openclaw", + model: "delivery-mirror", + text: "delivery mirror text", + }), + }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n", + "utf-8", + ); + + await expect( + readTailAssistantTextFromSessionTranscript(transcriptPath, { + excludeTranscriptOnlyOpenClawAssistant: true, + }), + ).resolves.toMatchObject({ + id: "real-assistant", + text: "File-backed canonical answer", + }); + }); + it("does not reuse an older matching assistant message across turns", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const olderResult = await appendExactAssistantMessageToSessionTranscript({ sessionKey, @@ -1069,37 +1487,55 @@ describe("appendAssistantMessageToSessionTranscript", () => { expect(mirrorResult.messageId).not.toBe(olderResult.messageId); expect(mirrorResult.messageId).not.toBe(latestResult.messageId); - const lines = fs.readFileSync(mirrorResult.sessionFile, "utf-8").trim().split("\n"); - expect(lines.length).toBe(4); - - const messageLine = JSON.parse(lines[3]); - expect(messageLine.message.api).toBe(OPENCLAW_TRANSCRIPT_ARTIFACT_API); - expect(messageLine.message.provider).toBe("openclaw"); - expect(messageLine.message.model).toBe("delivery-mirror"); - expect(messageLine.message.content[0].text).toBe("Repeated answer"); + const messages = (await loadFixtureMessages()).flatMap((entry) => + entry.message ? [entry.message] : [], + ) as Array<{ + api?: string; + provider?: string; + model?: string; + content?: Array<{ text?: string }>; + }>; + expect(messages).toHaveLength(3); + expect(messages[2]?.api).toBe(OPENCLAW_TRANSCRIPT_ARTIFACT_API); + expect(messages[2]?.provider).toBe("openclaw"); + expect(messages[2]?.model).toBe("delivery-mirror"); + expect(messages[2]?.content?.[0]?.text).toBe("Repeated answer"); } }); it("keeps delivery mirrors in transcripts while repair preserves real tool results", async () => { - writeTranscriptStore(); - const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); + await writeTranscriptStore(); const toolCallId = "call_maniple_list"; - const toolCallResult = await appendSessionTranscriptMessage({ - transcriptPath: sessionFile, - message: { - role: "assistant", - content: [ + const toolCallTurn = await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), + }, + { + updateMode: "none", + messages: [ { - type: "toolCall", - id: toolCallId, - name: "maniple__list_workers", - arguments: {}, + message: { + role: "assistant", + content: [ + { + type: "toolCall", + id: toolCallId, + name: "maniple__list_workers", + arguments: {}, + }, + ], + stopReason: "toolUse", + }, }, ], - stopReason: "toolUse", }, - }); + ); + const toolCallResult = toolCallTurn.messages[0]; + expect(toolCallResult?.messageId).toBeTypeOf("string"); const mirrorResult = await appendAssistantMessageToSessionTranscript({ sessionKey, @@ -1111,30 +1547,40 @@ describe("appendAssistantMessageToSessionTranscript", () => { if (!mirrorResult.ok) { return; } - expect(mirrorResult.messageId).not.toBe(toolCallResult.messageId); - const linesAfterMirror = fs.readFileSync(sessionFile, "utf-8").trim().split("\n"); - expect(linesAfterMirror).toHaveLength(3); - const mirrorLine = JSON.parse(linesAfterMirror[2]); - expect(mirrorLine.message.api).toBe(OPENCLAW_TRANSCRIPT_ARTIFACT_API); - expect(mirrorLine.message.model).toBe("delivery-mirror"); + expect(mirrorResult.messageId).not.toBe(toolCallResult?.messageId); + const messagesAfterMirror = (await loadFixtureMessages()).flatMap((entry) => + entry.message ? [entry.message] : [], + ) as Array<{ api?: string; model?: string }>; + expect(messagesAfterMirror).toHaveLength(2); + expect(messagesAfterMirror[1]?.api).toBe(OPENCLAW_TRANSCRIPT_ARTIFACT_API); + expect(messagesAfterMirror[1]?.model).toBe("delivery-mirror"); - await appendSessionTranscriptMessage({ - transcriptPath: sessionFile, - message: { - role: "toolResult", - toolCallId, - toolName: "maniple__list_workers", - content: [{ type: "text", text: "workers listed" }], - isError: false, + await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId, + sessionKey, + storePath: fixture.storePath(), }, - }); + { + updateMode: "none", + messages: [ + { + message: { + role: "toolResult", + toolCallId, + toolName: "maniple__list_workers", + content: [{ type: "text", text: "workers listed" }], + isError: false, + }, + }, + ], + }, + ); - const messages = fs - .readFileSync(sessionFile, "utf-8") - .trim() - .split("\n") - .map((line) => JSON.parse(line) as { message?: TranscriptRepairMessage }) - .flatMap((entry) => (entry.message ? [entry.message] : [])); + const messages = (await loadFixtureMessages()).flatMap((entry) => + entry.message ? [entry.message as TranscriptRepairMessage] : [], + ); expect(messages.map((message) => message.role)).toEqual([ "assistant", "assistant", @@ -1155,14 +1601,14 @@ describe("appendAssistantMessageToSessionTranscript", () => { it("finds session entry using normalized (lowercased) key", async () => { const storeKey = "agent:main:imessage:direct:+15551234567"; - const store = { - [storeKey]: { + await writeTranscriptSessionEntry({ + sessionKey: storeKey, + entry: { sessionId: "test-session-normalized", chatType: "direct", channel: "imessage", }, - }; - fs.writeFileSync(fixture.storePath(), JSON.stringify(store), "utf-8"); + }); const result = await appendAssistantMessageToSessionTranscript({ sessionKey: "agent:main:iMessage:direct:+15551234567", @@ -1175,14 +1621,14 @@ describe("appendAssistantMessageToSessionTranscript", () => { it("finds Slack session entry using normalized (lowercased) key", async () => { const storeKey = "agent:main:slack:direct:u12345abc"; - const store = { - [storeKey]: { + await writeTranscriptSessionEntry({ + sessionKey: storeKey, + entry: { sessionId: "test-slack-session", chatType: "direct", channel: "slack", }, - }; - fs.writeFileSync(fixture.storePath(), JSON.stringify(store), "utf-8"); + }); const result = await appendAssistantMessageToSessionTranscript({ sessionKey: "agent:main:slack:direct:U12345ABC", @@ -1194,7 +1640,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("ignores malformed transcript lines when checking mirror idempotency", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir()); fs.writeFileSync( @@ -1233,7 +1679,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("appends exact assistant transcript messages without rewriting phased content", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const result = await appendExactAssistantMessageToSessionTranscript({ sessionKey, @@ -1258,9 +1704,10 @@ describe("appendAssistantMessageToSessionTranscript", () => { expect(result.ok).toBe(true); if (result.ok) { - const lines = fs.readFileSync(result.sessionFile, "utf-8").trim().split("\n"); - const messageLine = JSON.parse(lines[1]); - expect(messageLine.message.content).toEqual([ + const messages = (await loadFixtureMessages()).flatMap((entry) => + entry.message ? [entry.message] : [], + ) as Array<{ content?: unknown }>; + expect(messages[0]?.content).toEqual([ { type: "text", text: "internal reasoning", @@ -1276,7 +1723,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("applies before_message_write after idempotency checks and preserves the key", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const beforeMessageWrite = vi.fn(({ message }: BeforeMessageWriteParams) => ({ ...message, content: [{ type: "text" as const, text: "[redacted by hook]" }], @@ -1299,12 +1746,9 @@ describe("appendAssistantMessageToSessionTranscript", () => { if (!first.ok) { throw new Error("expected assistant append to succeed"); } - const messages = fs - .readFileSync(first.sessionFile, "utf-8") - .trim() - .split("\n") - .map((line) => JSON.parse(line) as { message?: ExactAssistantMessage }) - .flatMap((entry) => (entry.message ? [entry.message] : [])); + const messages = (await loadFixtureMessages()).flatMap((entry) => + entry.message ? [entry.message as ExactAssistantMessage] : [], + ); expect(messages).toEqual([ expect.objectContaining({ role: "assistant", @@ -1315,7 +1759,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("dedupes unkeyed delivery mirrors after before_message_write rewrites", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const beforeMessageWrite = vi.fn(({ message }: BeforeMessageWriteParams) => ({ ...message, content: [{ type: "text" as const, text: "[redacted by hook]" }], @@ -1337,12 +1781,9 @@ describe("appendAssistantMessageToSessionTranscript", () => { if (!first.ok) { throw new Error("expected delivery mirror append to succeed"); } - const messages = fs - .readFileSync(first.sessionFile, "utf-8") - .trim() - .split("\n") - .map((line) => JSON.parse(line) as { message?: ExactAssistantMessage }) - .flatMap((entry) => (entry.message ? [entry.message] : [])); + const messages = (await loadFixtureMessages()).flatMap((entry) => + entry.message ? [entry.message as ExactAssistantMessage] : [], + ); expect(messages).toEqual([ expect.objectContaining({ role: "assistant", @@ -1352,7 +1793,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("reports assistant messages blocked by before_message_write", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const result = await appendExactAssistantMessageToSessionTranscript({ agentId: "main", @@ -1370,16 +1811,10 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("rejects assistant output after the session key is rebound", async () => { - fs.writeFileSync( - fixture.storePath(), - JSON.stringify({ - [sessionKey]: { - sessionId: "replacement-session", - chatType: "direct", - }, - }), - "utf-8", - ); + await writeTranscriptStore({ + sessionId: "replacement-session", + chatType: "direct", + }); const result = await appendExactAssistantMessageToSessionTranscript({ sessionKey, @@ -1400,7 +1835,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("rejects a concurrent session rebind before the assistant append", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); let releaseReset = () => {}; const resetGate = new Promise<void>((resolve) => { releaseReset = resolve; @@ -1413,10 +1848,13 @@ describe("appendAssistantMessageToSessionTranscript", () => { "replacement-session", fixture.sessionsDir(), ); - const reset = updateSessionStoreEntry({ - storePath: fixture.storePath(), - sessionKey, - update: async () => { + const reset = updateSessionEntry( + { + agentId: "main", + storePath: fixture.storePath(), + sessionKey, + }, + async () => { markResetStarted(); await resetGate; return { @@ -1424,7 +1862,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { sessionFile: replacementSessionFile, }; }, - }); + ); await resetStarted; const append = appendExactAssistantMessageToSessionTranscript({ @@ -1448,18 +1886,9 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("rejects a concurrent lifecycle owner change without a session id rotation", async () => { - fs.writeFileSync( - fixture.storePath(), - JSON.stringify({ - [sessionKey]: { - sessionId, - lifecycleRevision: "original-revision", - chatType: "direct", - channel: "discord", - }, - }), - "utf-8", - ); + await writeTranscriptStore({ + lifecycleRevision: "original-revision", + }); let releaseOwnerChange = () => {}; const ownerChangeGate = new Promise<void>((resolve) => { releaseOwnerChange = resolve; @@ -1468,21 +1897,26 @@ describe("appendAssistantMessageToSessionTranscript", () => { const ownerChangeStarted = new Promise<void>((resolve) => { markOwnerChangeStarted = resolve; }); - const ownerChange = updateSessionStoreEntry({ - storePath: fixture.storePath(), - sessionKey, - update: async () => { + const ownerChange = updateSessionEntry( + { + agentId: "main", + storePath: fixture.storePath(), + sessionKey, + }, + async () => { markOwnerChangeStarted(); await ownerChangeGate; - return { lifecycleRevision: "replacement-revision" }; + return { + lifecycleRevision: "replacement-revision", + }; }, - }); + ); await ownerChangeStarted; const append = appendExactAssistantMessageToSessionTranscript({ sessionKey, - expectedSessionId: sessionId, expectedLifecycleRevision: "original-revision", + expectedSessionId: sessionId, storePath: fixture.storePath(), message: createExactAssistantMessage({ text: "late output" }), }); @@ -1499,7 +1933,7 @@ describe("appendAssistantMessageToSessionTranscript", () => { }); it("dedupes concurrent exact assistant appends by idempotency key", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const idempotencyKey = "mirror:concurrent-assistant"; const results = await Promise.all( @@ -1526,20 +1960,15 @@ describe("appendAssistantMessageToSessionTranscript", () => { if (!firstOk?.ok) { throw new Error("expected exact assistant append to succeed"); } - const records = fs - .readFileSync(firstOk.sessionFile, "utf-8") - .trim() - .split("\n") - .map((line) => JSON.parse(line) as { message?: { role?: string; idempotencyKey?: string } }) - .filter( - (record) => - record.message?.role === "assistant" && record.message.idempotencyKey === idempotencyKey, - ); + const records = (await loadFixtureMessages()).filter((record) => { + const message = record.message as { role?: string; idempotencyKey?: string } | undefined; + return message?.role === "assistant" && message.idempotencyKey === idempotencyKey; + }); expect(records).toHaveLength(1); }); it("can emit file-only transcript refresh events for exact assistant appends", async () => { - writeTranscriptStore(); + await writeTranscriptStore(); const emitSpy = vi.spyOn(transcriptEvents, "emitSessionTranscriptUpdate"); const result = await appendExactAssistantMessageToSessionTranscript({ @@ -1556,8 +1985,14 @@ describe("appendAssistantMessageToSessionTranscript", () => { expect(result.ok).toBe(true); if (result.ok) { expect(emitSpy).toHaveBeenCalledWith({ + agentId: "main", sessionFile: result.sessionFile, sessionKey, + target: { + agentId: "main", + sessionId, + sessionKey, + }, }); } emitSpy.mockRestore(); diff --git a/src/config/sessions/transcript.ts b/src/config/sessions/transcript.ts index e5a88d22a023..2188aaf1803a 100644 --- a/src/config/sessions/transcript.ts +++ b/src/config/sessions/transcript.ts @@ -21,11 +21,24 @@ import { resolveSessionFilePath, resolveStorePath, } from "./paths.js"; -import { persistSessionTranscriptTurn } from "./session-accessor.js"; -import { resolveAndPersistSessionFile } from "./session-file.js"; -import { loadSessionStore, resolveSessionStoreEntry } from "./store.js"; +import { + listSessionEntries, + loadSessionEntry, + loadTranscriptEvents, + persistSessionTranscriptTurn, + readLatestTranscriptAssistantText, + updateSessionEntry, + type SessionTranscriptTurnWriteContext, +} from "./session-accessor.js"; +import { parseSqliteSessionFileMarker, type SqliteSessionFileMarker } from "./sqlite-marker.js"; +import { resolveSessionStoreEntry } from "./store.js"; import { resolveMirroredTranscriptText } from "./transcript-mirror.js"; import { streamSessionTranscriptLinesReverse } from "./transcript-stream.js"; +import { + scanSessionTranscriptTree, + selectSessionTranscriptTreePathNodes, +} from "./transcript-tree.js"; +import type { SessionEntry } from "./types.js"; export type SessionTranscriptAppendResult = | { ok: true; sessionFile: string; messageId: string } @@ -176,6 +189,11 @@ function normalizeRecentTranscriptLimit(limit: number | undefined): number { return Math.max(1, Math.floor(limit ?? 10)); } +type SessionConversationTranscriptTarget = { + sessionFile?: string; + sqliteScope?: SqliteSessionFileMarker; +}; + function parseRecentConversationText(line: string): SessionRecentConversationText | undefined { const parsed = JSON.parse(line) as { id?: unknown; @@ -220,32 +238,79 @@ function parseRecentConversationText(line: string): SessionRecentConversationTex }; } -function resolveSessionConversationTranscriptPath(params: { +async function readRecentUserAssistantTextFromSqliteTranscript( + scope: SqliteSessionFileMarker, + options: ReadRecentSessionConversationTextOptions = {}, +): Promise<SessionRecentConversationText[]> { + return (await readRecentUserAssistantTextFromSqliteTranscriptWithPresence(scope, options)).recent; +} + +async function readRecentUserAssistantTextFromSqliteTranscriptWithPresence( + scope: SqliteSessionFileMarker, + options: ReadRecentSessionConversationTextOptions = {}, +): Promise<{ recent: SessionRecentConversationText[]; hasEvents: boolean }> { + const events = await loadTranscriptEvents({ + agentId: scope.agentId, + sessionId: scope.sessionId, + storePath: scope.storePath, + }); + const limit = normalizeRecentTranscriptLimit(options.limit); + const recent: SessionRecentConversationText[] = []; + for (const event of events.toReversed()) { + const entry = parseRecentConversationText(JSON.stringify(event)); + if (!entry) { + continue; + } + if (!isBeforeTranscriptTimestamp(entry.timestamp, options.beforeTimestampMs)) { + continue; + } + if (!isAtOrAfterTranscriptTimestamp(entry.timestamp, options.minTimestampMs)) { + continue; + } + recent.push(entry); + if (recent.length >= limit) { + break; + } + } + return { recent: recent.toReversed(), hasEvents: events.length > 0 }; +} + +function resolveSessionConversationTranscriptTarget(params: { agentId?: string; sessionKey: string; storePath?: string; -}): string | undefined { +}): SessionConversationTranscriptTarget { const sessionKey = params.sessionKey.trim(); if (!sessionKey) { - return undefined; + return {}; } - const storePath = params.storePath ?? resolveDefaultSessionStorePath(params.agentId); - const store = loadSessionStore(storePath, { skipCache: true }); - const resolved = resolveSessionStoreEntry({ store, sessionKey }); - const entry = resolved.existing; + const agentId = params.agentId ?? resolveAgentIdFromSessionKey(sessionKey) ?? "main"; + const storePath = params.storePath ?? resolveDefaultSessionStorePath(agentId); + const entry = loadSessionEntry({ agentId, sessionKey, storePath }); if (!entry?.sessionId) { - return undefined; + return {}; } - return resolveSessionFilePath(entry.sessionId, entry, { - sessionsDir: path.dirname(storePath), - ...(params.agentId ? { agentId: params.agentId } : {}), - }); + return { + sessionFile: resolveSessionFilePath(entry.sessionId, entry, { + sessionsDir: path.dirname(storePath), + agentId, + }), + sqliteScope: { + agentId, + sessionId: entry.sessionId, + storePath, + }, + }; } export async function readRecentUserAssistantTextFromSessionTranscript( sessionFile: string | undefined, options: ReadRecentSessionConversationTextOptions = {}, ): Promise<SessionRecentConversationText[]> { + const sqliteMarker = parseSqliteSessionFileMarker(sessionFile); + if (sqliteMarker) { + return await readRecentUserAssistantTextFromSqliteTranscript(sqliteMarker, options); + } if (!sessionFile?.trim()) { return []; } @@ -277,13 +342,28 @@ export async function readRecentUserAssistantTextFromSessionTranscript( export async function readRecentUserAssistantTextForSession( params: ReadRecentSessionConversationTextParams, ): Promise<SessionRecentConversationText[]> { - const sessionFile = resolveSessionConversationTranscriptPath(params); - return await readRecentUserAssistantTextFromSessionTranscript(sessionFile, params); + const target = resolveSessionConversationTranscriptTarget(params); + if (target.sqliteScope) { + const sqliteRecent = await readRecentUserAssistantTextFromSqliteTranscriptWithPresence( + target.sqliteScope, + params, + ); + return sqliteRecent.recent; + } + return await readRecentUserAssistantTextFromSessionTranscript(target.sessionFile, params); } export async function readLatestAssistantTextFromSessionTranscript( sessionFile: string | undefined, ): Promise<LatestAssistantTranscriptText | undefined> { + const sqliteMarker = parseSqliteSessionFileMarker(sessionFile); + if (sqliteMarker) { + return readLatestTranscriptAssistantText({ + agentId: sqliteMarker.agentId, + sessionId: sqliteMarker.sessionId, + storePath: sqliteMarker.storePath, + }); + } if (!sessionFile?.trim()) { return undefined; } @@ -305,7 +385,39 @@ export async function readLatestAssistantTextFromSessionTranscript( export async function readTailAssistantTextFromSessionTranscript( sessionFile: string | undefined, + options?: { excludeTranscriptOnlyOpenClawAssistant?: boolean }, ): Promise<TailAssistantTranscriptText | undefined> { + const sqliteMarker = parseSqliteSessionFileMarker(sessionFile); + if (sqliteMarker) { + const events = await loadTranscriptEvents({ + agentId: sqliteMarker.agentId, + sessionId: sqliteMarker.sessionId, + storePath: sqliteMarker.storePath, + }); + for (const event of events.toReversed()) { + const parsed = event as { message?: { model?: unknown; provider?: unknown; role?: unknown } }; + if (!parsed.message || typeof parsed.message !== "object") { + continue; + } + if (parsed.message.role !== "assistant") { + return undefined; + } + const assistantText = parseAssistantTranscriptText(JSON.stringify(event), { + excludeTranscriptOnlyOpenClawAssistant: + options?.excludeTranscriptOnlyOpenClawAssistant === true, + }); + if (assistantText) { + return assistantText; + } + if ( + options?.excludeTranscriptOnlyOpenClawAssistant !== true || + !isTranscriptOnlyOpenClawAssistantMessage(parsed.message) + ) { + return undefined; + } + } + return undefined; + } if (!sessionFile?.trim()) { return undefined; } @@ -322,7 +434,17 @@ export async function readTailAssistantTextFromSessionTranscript( if (!parsed.message || typeof parsed.message !== "object") { continue; } - return parseAssistantTranscriptText(line); + const assistantText = parseAssistantTranscriptText(line, options); + if (assistantText) { + return assistantText; + } + if ( + options?.excludeTranscriptOnlyOpenClawAssistant === true && + isTranscriptOnlyOpenClawAssistantMessage(parsed.message) + ) { + continue; + } + return undefined; } catch { continue; } @@ -423,7 +545,11 @@ export async function appendExactAssistantMessageToSessionTranscript(params: { const storeAgentId = transcriptAgentId ?? resolveAgentIdFromSessionKey(sessionKey); const storePath = params.storePath ?? resolveStorePath(params.config?.session?.store, { agentId: storeAgentId }); - const store = loadSessionStore(storePath, { skipCache: true }); + const store = Object.fromEntries( + listSessionEntries({ agentId: transcriptAgentId, storePath }).map( + ({ sessionKey: entryKey, entry }) => [entryKey, entry], + ), + ); const resolved = resolveSessionStoreEntry({ store, sessionKey }); const entry = resolved.existing; if (params.expectedSessionId && entry?.sessionId !== params.expectedSessionId) { @@ -434,7 +560,7 @@ export async function appendExactAssistantMessageToSessionTranscript(params: { }; } if ( - params.expectedLifecycleRevision && + params.expectedLifecycleRevision !== undefined && entry?.lifecycleRevision !== params.expectedLifecycleRevision ) { return { @@ -490,7 +616,7 @@ export async function appendExactAssistantMessageToSessionTranscript(params: { { cwd: currentEntry.spawnedCwd, ...(params.expectedSessionId ? { expectedSessionId: params.expectedSessionId } : {}), - ...(params.expectedLifecycleRevision + ...(params.expectedLifecycleRevision !== undefined ? { expectedLifecycleRevision: params.expectedLifecycleRevision } : {}), ...(params.config ? { config: params.config } : {}), @@ -516,7 +642,7 @@ export async function appendExactAssistantMessageToSessionTranscript(params: { latestEquivalentAssistantId = isRedundantDeliveryMirror(params.message) && !identifiedDeliveryMirror ? await findLatestEquivalentAssistantMessageId( - target.sessionFile, + target, preparedUnkeyedMessage as SessionTranscriptAssistantMessage, params.config, ) @@ -546,6 +672,26 @@ export async function appendExactAssistantMessageToSessionTranscript(params: { }; } const { messageId } = appendedResult; + if (!params.expectedSessionId) { + try { + if (parseSqliteSessionFileMarker(turn.sessionFile)) { + await touchSqliteAssistantAppendSessionEntry({ + agentId: transcriptAgentId, + currentEntry, + sessionFile: turn.sessionFile, + sessionKey: resolved.normalizedKey, + storePath, + }); + } else { + return { ok: false, reason: `unexpected transcript target: ${turn.sessionFile}` }; + } + } catch (err) { + return { + ok: false, + reason: formatErrorMessage(err), + }; + } + } return { ok: true, sessionFile: turn.sessionFile, messageId }; }; @@ -553,29 +699,39 @@ export async function appendExactAssistantMessageToSessionTranscript(params: { if (params.expectedSessionId) { result = await appendToSessionFile(entry); } else { - let sessionFile: string; - try { - const resolvedSessionFile = await resolveAndPersistSessionFile({ - sessionId: entry.sessionId, - sessionKey: resolved.normalizedKey, - sessionStore: store, - storePath, - sessionEntry: entry, - agentId: transcriptAgentId, - sessionsDir: path.dirname(storePath), - }); - sessionFile = resolvedSessionFile.sessionFile; - } catch (err) { - return { - ok: false, - reason: formatErrorMessage(err), - }; - } - result = await appendToSessionFile(entry, sessionFile); + result = await appendToSessionFile(entry); } return result; } +async function touchSqliteAssistantAppendSessionEntry(params: { + agentId?: string; + currentEntry: SessionEntry; + sessionFile: string; + sessionKey: string; + storePath: string; +}): Promise<void> { + const now = Date.now(); + const buildPatch = (entry: SessionEntry | undefined): Partial<SessionEntry> => ({ + updatedAt: Math.max(entry?.updatedAt ?? 0, now), + sessionStartedAt: entry?.sessionStartedAt ?? params.currentEntry.sessionStartedAt ?? now, + sessionFile: params.sessionFile, + }); + await updateSessionEntry( + { + agentId: params.agentId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }, + (entry) => { + if (entry.sessionId !== params.currentEntry.sessionId) { + return null; + } + return buildPatch(entry); + }, + ); +} + function isRedundantDeliveryMirror(message: SessionTranscriptAssistantMessage): boolean { return ( message.provider === OPENCLAW_TRANSCRIPT_ARTIFACT_PROVIDER && @@ -583,6 +739,37 @@ function isRedundantDeliveryMirror(message: SessionTranscriptAssistantMessage): ); } +async function readLatestVisibleTranscriptMessage(scope: { + agentId?: string; + sessionId: string; + sessionKey?: string; + storePath: string; +}): Promise<{ id?: string; message: unknown } | undefined> { + const events = await loadTranscriptEvents(scope).catch(() => []); + const tree = scanSessionTranscriptTree(events); + const visiblePath = selectSessionTranscriptTreePathNodes(tree, tree.leafId); + const visibleEvents = + visiblePath.length > 0 + ? visiblePath.map((node) => node.entry) + : tree.hasLeafControl + ? [] + : events; + for (const event of visibleEvents.toReversed()) { + if (!event || typeof event !== "object" || Array.isArray(event)) { + continue; + } + const record = event as { id?: unknown; message?: unknown }; + if (record.message === undefined) { + continue; + } + return { + ...(typeof record.id === "string" ? { id: record.id } : {}), + message: record.message, + }; + } + return undefined; +} + function isIdentifiedDeliveryMirror(message: SessionTranscriptAssistantMessage): boolean { const marker = (message as { openclawDeliveryMirror?: SessionTranscriptDeliveryMirror }) .openclawDeliveryMirror; @@ -612,7 +799,7 @@ function extractAssistantMessageText(message: SessionTranscriptAssistantMessage) } async function findLatestEquivalentAssistantMessageId( - transcriptPath: string, + target: SessionTranscriptTurnWriteContext, message: SessionTranscriptAssistantMessage, config?: OpenClawConfig, ): Promise<string | undefined> { @@ -623,17 +810,42 @@ async function findLatestEquivalentAssistantMessageId( return undefined; } - for await (const line of streamSessionTranscriptLinesReverse(transcriptPath)) { + if (target.storePath && target.sessionId) { + const latest = await readLatestVisibleTranscriptMessage({ + ...(target.agentId ? { agentId: target.agentId } : {}), + sessionId: target.sessionId, + ...(target.sessionKey ? { sessionKey: target.sessionKey } : {}), + storePath: target.storePath, + }); + const latestMessage = latest?.message as { role?: unknown } | undefined; + if (latestMessage?.role !== "assistant") { + return undefined; + } + const candidateText = latest + ? extractAssistantMessageText( + redactTranscriptMessage( + latest.message as AgentMessage, + config, + ) as unknown as SessionTranscriptAssistantMessage, + ) + : undefined; + return candidateText === expectedText ? latest?.id : undefined; + } + + for await (const line of streamSessionTranscriptLinesReverse(target.sessionFile)) { try { const parsed = JSON.parse(line) as { id?: unknown; message?: SessionTranscriptAssistantMessage; }; const candidate = parsed.message; - if (!candidate || candidate.role !== "assistant") { + if (!candidate) { continue; } - // Stop at the first assistant message: only the tail can be a duplicate mirror replay. + if (candidate.role !== "assistant") { + return undefined; + } + // Only the tail message can be a duplicate mirror replay. const candidateText = extractAssistantMessageText( redactTranscriptMessage( candidate as AgentMessage, diff --git a/src/config/types.agent-defaults.ts b/src/config/types.agent-defaults.ts index 4fe2cbd3516a..c5adddecc291 100644 --- a/src/config/types.agent-defaults.ts +++ b/src/config/types.agent-defaults.ts @@ -562,14 +562,14 @@ export type AgentCompactionConfig = { */ provider?: string; /** - * Rotate the active session JSONL file after compaction so the next turn + * Rotate the active session transcript after compaction so the next turn * starts from the compaction summary and unsummarized tail while the old * transcript stays archived. * Default: false (existing behavior preserved). */ truncateAfterCompaction?: boolean; /** - * Trigger a normal local compaction when the active session JSONL reaches + * Trigger a normal local compaction when the active session transcript reaches * this size (bytes, or byte-size string like "20mb"). Set to 0/unset to * disable. Requires truncateAfterCompaction so successful compaction can * rotate to a smaller successor transcript. This does not split raw diff --git a/src/config/types.base.ts b/src/config/types.base.ts index 0d22eb6995c5..dba59b7573eb 100644 --- a/src/config/types.base.ts +++ b/src/config/types.base.ts @@ -268,15 +268,17 @@ export type SessionMaintenanceConfig = { /** @deprecated Ignored. Run `openclaw doctor --fix` to remove. */ rotateBytes?: number | string; /** - * Retention for archived reset transcripts (`*.reset.<timestamp>`). - * Set `false` to disable reset-archive cleanup. Default: same as `pruneAfter` (30d). + * Age-based retention for archived transcripts (`*.reset.<timestamp>` and + * `*.deleted.<timestamp>`). Default and `false`: keep archives until the + * disk budget evicts them oldest-first; a duration opts into deletion. */ resetArchiveRetention?: string | number | false; /** - * Optional per-agent sessions-directory disk budget (e.g. "500mb"). - * When exceeded, warn (mode=warn) or enforce oldest-first cleanup (mode=enforce). + * Per-agent sessions-directory disk budget (e.g. "500mb"). Default: "2gb". + * When exceeded, warn (mode=warn) or enforce oldest-first cleanup + * (mode=enforce). Set `false` to disable the budget entirely. */ - maxDiskBytes?: number | string; + maxDiskBytes?: number | string | false; /** * Target size after disk-budget cleanup (high-water mark), e.g. "400mb". * Default: 80% of maxDiskBytes. diff --git a/src/context-engine/context-engine.test.ts b/src/context-engine/context-engine.test.ts index 7efbfb710dda..478f01b3e9d9 100644 --- a/src/context-engine/context-engine.test.ts +++ b/src/context-engine/context-engine.test.ts @@ -26,15 +26,15 @@ import type { ContextEngineFactoryContext, ContextEngineRegistrationResult, } from "./registry.js"; -import { - resolveCompactionSuccessorTranscript, - type ContextEngine, - type ContextEngineInfo, - type AssembleResult, - type CompactResult, - type ContextEngineMaintenanceResult, - type BootstrapResult, - type IngestResult, +import type { + ContextEngine, + ContextEngineInfo, + ContextEngineSessionTarget, + AssembleResult, + CompactResult, + ContextEngineMaintenanceResult, + BootstrapResult, + IngestResult, } from "./types.js"; const { compactEmbeddedAgentSessionDirectMock } = vi.hoisted(() => ({ @@ -160,8 +160,9 @@ class MockContextEngine implements ContextEngine { async compact(_params: { sessionId: string; - sessionKey?: string; - sessionFile: string; + sessionKey: string; + agentId?: string; + sessionTarget?: ContextEngineSessionTarget; tokenBudget?: number; compactionTarget?: "budget" | "threshold"; customInstructions?: string; @@ -236,8 +237,9 @@ class LegacySessionKeyStrictEngine implements ContextEngine { async compact(params: { sessionId: string; - sessionKey?: string; - sessionFile: string; + sessionKey: string; + agentId?: string; + sessionTarget?: ContextEngineSessionTarget; tokenBudget?: number; compactionTarget?: "budget" | "threshold"; customInstructions?: string; @@ -305,8 +307,9 @@ class SessionKeyRuntimeErrorEngine implements ContextEngine { async compact(_params: { sessionId: string; - sessionKey?: string; - sessionFile: string; + sessionKey: string; + agentId?: string; + sessionTarget?: ContextEngineSessionTarget; tokenBudget?: number; compactionTarget?: "budget" | "threshold"; customInstructions?: string; @@ -367,8 +370,9 @@ class LegacyAssembleStrictEngine implements ContextEngine { async compact(_params: { sessionId: string; - sessionKey?: string; - sessionFile: string; + sessionKey: string; + agentId?: string; + sessionTarget?: ContextEngineSessionTarget; tokenBudget?: number; compactionTarget?: "budget" | "threshold"; customInstructions?: string; @@ -396,10 +400,16 @@ class LegacyRuntimeSettingsStrictEngine implements ContextEngine { }; } - private rejectRuntimeSettings(params: { runtimeSettings?: unknown }): void { + private rejectLegacyCompatFields(params: Record<string, unknown>): void { if (Object.hasOwn(params, "runtimeSettings")) { throw new Error("Unrecognized key(s) in object: 'runtimeSettings'"); } + if (Object.hasOwn(params, "sessionTarget")) { + throw new Error("Unrecognized key(s) in object: 'sessionTarget'"); + } + if (Object.hasOwn(params, "runtimeContext")) { + throw new Error("Unrecognized key(s) in object: 'runtimeContext'"); + } } async bootstrap(params: { @@ -407,9 +417,11 @@ class LegacyRuntimeSettingsStrictEngine implements ContextEngine { sessionKey?: string; sessionFile: string; runtimeSettings?: unknown; + sessionTarget?: ContextEngineSessionTarget; + runtimeContext?: unknown; }): Promise<BootstrapResult> { this.bootstrapCalls.push({ ...params }); - this.rejectRuntimeSettings(params); + this.rejectLegacyCompatFields(params); return { bootstrapped: true }; } @@ -427,9 +439,11 @@ class LegacyRuntimeSettingsStrictEngine implements ContextEngine { sessionKey?: string; sessionFile: string; runtimeSettings?: unknown; + sessionTarget?: ContextEngineSessionTarget; + runtimeContext?: unknown; }): Promise<ContextEngineMaintenanceResult> { this.maintainCalls.push({ ...params }); - this.rejectRuntimeSettings(params); + this.rejectLegacyCompatFields(params); return { changed: false, bytesFreed: 0, rewrittenEntries: 0 }; } @@ -440,9 +454,11 @@ class LegacyRuntimeSettingsStrictEngine implements ContextEngine { messages: AgentMessage[]; prePromptMessageCount: number; runtimeSettings?: unknown; + sessionTarget?: ContextEngineSessionTarget; + runtimeContext?: unknown; }): Promise<void> { this.afterTurnCalls.push({ ...params }); - this.rejectRuntimeSettings(params); + this.rejectLegacyCompatFields(params); } async assemble(params: { @@ -452,18 +468,20 @@ class LegacyRuntimeSettingsStrictEngine implements ContextEngine { runtimeSettings?: unknown; }): Promise<AssembleResult> { this.assembleCalls.push({ ...params }); - this.rejectRuntimeSettings(params); + this.rejectLegacyCompatFields(params); return { messages: params.messages, estimatedTokens: 3 }; } async compact(params: { sessionId: string; - sessionKey?: string; - sessionFile: string; + sessionKey: string; + agentId?: string; + sessionTarget?: ContextEngineSessionTarget; runtimeSettings?: unknown; + runtimeContext?: unknown; }): Promise<CompactResult> { this.compactCalls.push({ ...params }); - this.rejectRuntimeSettings(params); + this.rejectLegacyCompatFields(params); return { ok: true, compacted: false }; } } @@ -524,8 +542,9 @@ class LegacyRuntimeThenAssembleStrictEngine implements ContextEngine { async compact(_params: { sessionId: string; - sessionKey?: string; - sessionFile: string; + sessionKey: string; + agentId?: string; + sessionTarget?: ContextEngineSessionTarget; }): Promise<CompactResult> { return { ok: true, compacted: false }; } @@ -560,7 +579,8 @@ describe("Engine contract tests", () => { await engine.compact({ sessionId: "s1", - sessionFile: "/tmp/session.json", + sessionKey: "agent:main:s1", + sessionTarget: { agentId: "main", sessionId: "s1", sessionKey: "agent:main:s1" }, runtimeContext: { workspaceDir: "/tmp/workspace", currentTokenCount: 277403, @@ -573,9 +593,16 @@ describe("Engine contract tests", () => { it("delegateCompactionToRuntime reuses the legacy runtime bridge", async () => { const compactRuntimeSpy = installCompactRuntimeSpy(); + const sessionTarget = { + agentId: "main", + sessionId: "s2", + sessionKey: "agent:main:s2", + storePath: "/tmp/openclaw-agent.sqlite", + }; const result = await delegateCompactionToRuntime({ sessionId: "s2", - sessionFile: "/tmp/session.json", + sessionKey: "agent:main:s2", + sessionTarget, tokenBudget: 4096, runtimeContext: { workspaceDir: "/tmp/workspace", @@ -586,7 +613,9 @@ describe("Engine contract tests", () => { expect(compactRuntimeSpy).toHaveBeenCalledTimes(1); const compactRuntimeParams = requireCompactRuntimeParams(0); expect(compactRuntimeParams.sessionId).toBe("s2"); - expect(compactRuntimeParams.sessionFile).toBe("/tmp/session.json"); + expect(compactRuntimeParams.sessionKey).toBe("agent:main:s2"); + expect(compactRuntimeParams.sessionTarget).toEqual(sessionTarget); + expect(compactRuntimeParams).not.toHaveProperty("sessionFile"); expect(compactRuntimeParams.tokenBudget).toBe(4096); expect(compactRuntimeParams.currentTokenCount).toBe(12345); expect(compactRuntimeParams.workspaceDir).toBe("/tmp/workspace"); @@ -600,77 +629,45 @@ describe("Engine contract tests", () => { tokensBefore: 0, tokensAfter: 0, details: undefined, - sessionTarget: { - kind: "file", - sessionId: "s2", - sessionFile: "/tmp/session.json", - }, }, }); }); - it("delegateCompactionToRuntime reports the rotated successor as a typed session target", async () => { - compactEmbeddedAgentSessionDirectMock.mockResolvedValue({ + it("delegateCompactionToRuntime returns successor sessionTarget without sessionFile", async () => { + compactEmbeddedAgentSessionDirectMock.mockResolvedValueOnce({ ok: true, compacted: true, + reason: undefined, result: { - summary: "compacted", - firstKeptEntryId: "e1", + summary: "summary", + firstKeptEntryId: "entry-1", tokensBefore: 100, - tokensAfter: 10, - sessionId: "rotated-id", - sessionFile: "/tmp/rotated.jsonl", + tokensAfter: 40, + details: undefined, + sessionId: "s3-successor", + sessionFile: "sqlite:main:s3-successor:/tmp/openclaw-agent.sqlite", }, }); const result = await delegateCompactionToRuntime({ - sessionId: "s-rotate", - sessionKey: "agent:main:test", - sessionFile: "/tmp/session.json", + sessionId: "s3", + sessionKey: "agent:main:s3", + tokenBudget: 4096, runtimeContext: { workspaceDir: "/tmp/workspace", - agentId: "main", }, }); - expect(result.result?.sessionTarget).toEqual({ - kind: "file", - agentId: "main", - sessionId: "rotated-id", - sessionKey: "agent:main:test", - sessionFile: "/tmp/rotated.jsonl", + expect(result.result).toMatchObject({ + sessionId: "s3-successor", + sessionTarget: { + agentId: "main", + sessionId: "s3-successor", + sessionKey: "agent:main:s3", + storePath: "/tmp/openclaw-agent.sqlite", + }, }); - // Deprecated raw fields stay populated for shipped plugin-sdk readers. - expect(result.result?.sessionId).toBe("rotated-id"); - expect(result.result?.sessionFile).toBe("/tmp/rotated.jsonl"); - }); - - it("resolveCompactionSuccessorTranscript prefers the typed target over deprecated raw fields", () => { - expect( - resolveCompactionSuccessorTranscript({ - ok: true, - compacted: true, - result: { - tokensBefore: 1, - sessionId: "raw-id", - sessionFile: "/tmp/raw.jsonl", - sessionTarget: { - kind: "file", - sessionId: "target-id", - sessionFile: "/tmp/target.jsonl", - }, - }, - }), - ).toEqual({ sessionId: "target-id", sessionFile: "/tmp/target.jsonl" }); - - // Raw-field fallback covers shipped engines that predate sessionTarget. - expect( - resolveCompactionSuccessorTranscript({ - ok: true, - compacted: true, - result: { tokensBefore: 1, sessionId: "raw-id", sessionFile: "/tmp/raw.jsonl" }, - }), - ).toEqual({ sessionId: "raw-id", sessionFile: "/tmp/raw.jsonl" }); + expect(result.result).not.toHaveProperty("sessionFile"); }); it("delegateCompactionToRuntime forwards the caller abortSignal to the runtime (#89868)", async () => { @@ -678,7 +675,7 @@ describe("Engine contract tests", () => { const controller = new AbortController(); await delegateCompactionToRuntime({ sessionId: "s-abort", - sessionFile: "/tmp/session-abort.json", + sessionKey: "agent:main:s-abort", tokenBudget: 4096, abortSignal: controller.signal, }); @@ -691,7 +688,7 @@ describe("Engine contract tests", () => { installCompactRuntimeSpy(); await delegateCompactionToRuntime({ sessionId: "s-no-abort", - sessionFile: "/tmp/session-no-abort.json", + sessionKey: "agent:main:s-no-abort", tokenBudget: 4096, }); @@ -881,7 +878,6 @@ describe("Legacy sessionKey compatibility", () => { const compacted = await engine.compact({ sessionId: "s1", sessionKey: "agent:main:test", - sessionFile: "/tmp/session.json", }); expect(firstAssembled.estimatedTokens).toBe(7); @@ -1475,7 +1471,7 @@ describe("Invalid engine fallback", () => { await expect( engine.compact({ sessionId: "s1", - sessionFile: "/tmp/session.json", + sessionKey: "agent:main:s1", }), ).rejects.toThrow("plugin compaction failed"); @@ -1541,7 +1537,7 @@ describe("Invalid engine fallback", () => { await expect( engine.compact({ sessionId: "s1", - sessionFile: "/tmp/session.json", + sessionKey: "agent:main:s1", abortSignal: controller.signal, }), ).rejects.toThrow("compaction aborted"); @@ -1793,8 +1789,14 @@ describe("assemble() prompt forwarding", () => { expect(strictEngine.assembleCalls[3]).not.toHaveProperty("runtimeSettings"); }); - it("retries strict legacy lifecycle hooks without runtimeSettings", async () => { + it("retries strict legacy lifecycle hooks without additive host fields", async () => { const runtimeSettings = { schemaVersion: 1 } as never; + const sessionTarget = { + agentId: "main", + sessionId: "s1", + sessionKey: "agent:main:test", + } satisfies ContextEngineSessionTarget; + const runtimeContext = { transcriptStorage: { kind: "sqlite" } } as never; const resolveStrictEngine = async () => { const engineId = uniqueEngineId("runtime-settings-legacy"); const strictEngine = new LegacyRuntimeSettingsStrictEngine(engineId); @@ -1809,6 +1811,8 @@ describe("assemble() prompt forwarding", () => { sessionKey: "agent:main:test", sessionFile: "sessions/s1.jsonl", runtimeSettings, + sessionTarget, + runtimeContext, }); const maintain = await resolveStrictEngine(); @@ -1817,6 +1821,8 @@ describe("assemble() prompt forwarding", () => { sessionKey: "agent:main:test", sessionFile: "sessions/s1.jsonl", runtimeSettings, + sessionTarget, + runtimeContext, }); const afterTurn = await resolveStrictEngine(); @@ -1827,6 +1833,8 @@ describe("assemble() prompt forwarding", () => { messages: [makeMockMessage("assistant", "done")], prePromptMessageCount: 0, runtimeSettings, + sessionTarget, + runtimeContext, }); const assemble = await resolveStrictEngine(); @@ -1841,8 +1849,9 @@ describe("assemble() prompt forwarding", () => { await compact.engine.compact({ sessionId: "s1", sessionKey: "agent:main:test", - sessionFile: "sessions/s1.jsonl", runtimeSettings, + sessionTarget, + runtimeContext, }); for (const calls of [ @@ -1852,9 +1861,29 @@ describe("assemble() prompt forwarding", () => { assemble.strictEngine.assembleCalls, compact.strictEngine.compactCalls, ]) { - expect(calls).toHaveLength(2); + const finalCall = calls.at(-1); + expect(finalCall).toBeDefined(); expect(calls[0]).toHaveProperty("runtimeSettings"); - expect(calls[1]).not.toHaveProperty("runtimeSettings"); + expect(calls[0]).toHaveProperty("sessionKey", "agent:main:test"); + expect(finalCall).toHaveProperty("sessionKey", "agent:main:test"); + expect(finalCall).not.toHaveProperty("runtimeSettings"); + expect(finalCall).not.toHaveProperty("sessionTarget"); + expect(finalCall).not.toHaveProperty("runtimeContext"); + } + for (const calls of [ + bootstrap.strictEngine.bootstrapCalls, + maintain.strictEngine.maintainCalls, + afterTurn.strictEngine.afterTurnCalls, + compact.strictEngine.compactCalls, + ]) { + expect(calls).toHaveLength(4); + expect(calls[0]).toHaveProperty("sessionTarget", sessionTarget); + expect(calls[0]).toHaveProperty("runtimeContext", runtimeContext); + } + expect(assemble.strictEngine.assembleCalls).toHaveLength(2); + for (const call of assemble.strictEngine.assembleCalls) { + expect(call).not.toHaveProperty("sessionTarget"); + expect(call).not.toHaveProperty("runtimeContext"); } }); }); diff --git a/src/context-engine/delegate.ts b/src/context-engine/delegate.ts index 94ecccbcb736..c5f6de63873a 100644 --- a/src/context-engine/delegate.ts +++ b/src/context-engine/delegate.ts @@ -1,14 +1,46 @@ // Context-engine delegates bridge custom engines to built-in compaction and memory prompt paths. import { normalizeStructuredPromptSection } from "@openclaw/ai/internal/shared"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import type { MemoryCitationsMode } from "../config/types.memory.js"; import { buildMemoryPromptSection } from "../plugins/memory-state.js"; import { createLazyRuntimeModule } from "../shared/lazy-runtime.js"; -import type { ContextEngine, CompactResult, ContextEngineRuntimeContext } from "./types.js"; +import type { + ContextEngine, + CompactResult, + ContextEngineRuntimeContext, + ContextEngineSessionTarget, +} from "./types.js"; const loadCompactRuntime = createLazyRuntimeModule( () => import("../agents/embedded-agent-runner/compact.runtime.js"), ); +function buildCompactionResultSessionTarget(params: { + agentId?: string; + sessionFile?: string; + sessionId?: string; + sessionKey?: string; + sessionTarget?: ContextEngineSessionTarget; +}): ContextEngineSessionTarget | undefined { + const sqliteMarker = parseSqliteSessionFileMarker(params.sessionFile); + const sessionId = sqliteMarker?.sessionId ?? params.sessionId; + if (!sessionId) { + return undefined; + } + const agentId = params.sessionTarget?.agentId ?? params.agentId ?? sqliteMarker?.agentId; + const sessionKey = params.sessionTarget?.sessionKey ?? params.sessionKey; + const storePath = params.sessionTarget?.storePath ?? sqliteMarker?.storePath; + return { + ...(agentId ? { agentId } : {}), + sessionId, + ...(sessionKey ? { sessionKey } : {}), + ...(storePath ? { storePath } : {}), + ...(params.sessionTarget?.threadId !== undefined + ? { threadId: params.sessionTarget.threadId } + : {}), + }; +} + /** * Delegate a context-engine compaction request to OpenClaw's built-in runtime compaction path. * @@ -30,11 +62,15 @@ export async function delegateCompactionToRuntime( const { compactEmbeddedAgentSessionDirect } = await loadCompactRuntime(); type RuntimeCompactionParams = Parameters<typeof compactEmbeddedAgentSessionDirect>[0]; - // runtimeContext carries the full CompactEmbeddedAgentSessionParams fields set - // by runtime callers. We spread them and override the fields that come from - // the public ContextEngine compact() signature directly. + // runtimeContext carries host-resolved runtime fields set by internal + // callers. Keep the public delegate keyed by session identity, not by the + // active transcript artifact that the runtime may resolve internally. const runtimeContext = (params.runtimeContext ?? {}) as ContextEngineRuntimeContext & Partial<RuntimeCompactionParams>; + const { sessionFile: _legacySessionFile, ...runtimeContextParams } = runtimeContext; + const sessionTarget = params.sessionTarget ?? runtimeContext.sessionTarget; + const agentId = params.agentId ?? runtimeContext.agentId; + const sessionKey = params.sessionKey ?? runtimeContext.sessionKey; const currentTokenCount = params.currentTokenCount ?? (typeof runtimeContext.currentTokenCount === "number" && @@ -44,9 +80,11 @@ export async function delegateCompactionToRuntime( : undefined); const result = await compactEmbeddedAgentSessionDirect({ - ...runtimeContext, + ...runtimeContextParams, + ...(agentId ? { agentId } : {}), sessionId: params.sessionId, - sessionFile: params.sessionFile, + ...(sessionKey ? { sessionKey } : {}), + ...(sessionTarget ? { sessionTarget } : {}), tokenBudget: params.tokenBudget, ...(currentTokenCount !== undefined ? { currentTokenCount } : {}), force: params.force, @@ -55,13 +93,15 @@ export async function delegateCompactionToRuntime( workspaceDir: typeof runtimeContext.workspaceDir === "string" ? runtimeContext.workspaceDir : process.cwd(), }); - - // Post-compaction live session: the runtime reports rotation through the - // result sessionId/sessionFile pair; otherwise the input session stays live. - const successorSessionId = result.result?.sessionId ?? params.sessionId; - const successorSessionFile = result.result?.sessionFile ?? params.sessionFile; - const agentId = runtimeContext.sessionTarget?.agentId ?? runtimeContext.agentId; - const sessionKey = params.sessionKey ?? runtimeContext.sessionKey; + const resultSessionTarget = result.result + ? buildCompactionResultSessionTarget({ + agentId, + sessionFile: result.result.sessionFile, + sessionId: result.result.sessionId, + sessionKey, + sessionTarget, + }) + : undefined; return { ok: result.ok, @@ -74,17 +114,11 @@ export async function delegateCompactionToRuntime( tokensBefore: result.result.tokensBefore, tokensAfter: result.result.tokensAfter, details: result.result.details, - sessionId: result.result.sessionId, - // Deprecated raw path stays populated for shipped plugin-sdk readers - // of the delegate result; sessionTarget is the canonical successor. - sessionFile: result.result.sessionFile, - sessionTarget: { - kind: "file", - ...(agentId ? { agentId } : {}), - sessionId: successorSessionId, - ...(sessionKey ? { sessionKey } : {}), - sessionFile: successorSessionFile, - }, + ...(result.result.sessionId ? { sessionId: result.result.sessionId } : {}), + // Core reports successors only through the typed sessionTarget; the + // deprecated raw sessionFile field is reserved for shipped engines + // reporting rotation to core, and post-flip core has no file path. + ...(resultSessionTarget ? { sessionTarget: resultSessionTarget } : {}), } : undefined, }; diff --git a/src/context-engine/legacy.ts b/src/context-engine/legacy.ts index fe7ba21c7981..707e20f58e50 100644 --- a/src/context-engine/legacy.ts +++ b/src/context-engine/legacy.ts @@ -8,6 +8,7 @@ import type { AssembleResult, CompactResult, ContextEngineRuntimeContext, + ContextEngineSessionTarget, IngestResult, } from "./types.js"; @@ -70,8 +71,9 @@ export class LegacyContextEngine implements ContextEngine { async compact(params: { sessionId: string; - sessionKey?: string; - sessionFile: string; + sessionKey: string; + agentId?: string; + sessionTarget?: ContextEngineSessionTarget; tokenBudget?: number; force?: boolean; currentTokenCount?: number; diff --git a/src/context-engine/registry.ts b/src/context-engine/registry.ts index 1d91bf59d51f..09d0e78a65b5 100644 --- a/src/context-engine/registry.ts +++ b/src/context-engine/registry.ts @@ -82,15 +82,21 @@ const SESSION_KEY_COMPAT_METHODS = [ "assemble", "compact", ] as const; -const LEGACY_COMPAT_PARAMS = ["sessionKey", "prompt", "runtimeSettings"] as const; +const LEGACY_COMPAT_PARAMS = [ + "sessionKey", + "prompt", + "runtimeSettings", + "sessionTarget", + "runtimeContext", +] as const; const LEGACY_COMPAT_METHOD_KEYS = { - bootstrap: ["sessionKey", "runtimeSettings"], - maintain: ["sessionKey", "runtimeSettings"], + bootstrap: ["sessionKey", "runtimeSettings", "sessionTarget", "runtimeContext"], + maintain: ["sessionKey", "runtimeSettings", "sessionTarget", "runtimeContext"], ingest: ["sessionKey"], ingestBatch: ["sessionKey"], - afterTurn: ["sessionKey", "runtimeSettings"], + afterTurn: ["sessionKey", "runtimeSettings", "sessionTarget", "runtimeContext"], assemble: ["sessionKey", "prompt", "runtimeSettings"], - compact: ["sessionKey", "runtimeSettings"], + compact: ["sessionKey", "runtimeSettings", "sessionTarget", "runtimeContext"], } as const; type SessionKeyCompatMethodName = (typeof SESSION_KEY_COMPAT_METHODS)[number]; @@ -98,6 +104,8 @@ type SessionKeyCompatParams = { sessionKey?: string; prompt?: string; runtimeSettings?: unknown; + sessionTarget?: unknown; + runtimeContext?: unknown; }; type LegacyCompatKey = (typeof LEGACY_COMPAT_PARAMS)[number]; type LegacyCompatParamMap = Partial<Record<LegacyCompatKey, unknown>>; @@ -186,6 +194,24 @@ const LEGACY_UNKNOWN_FIELD_PATTERNS: Record<LegacyCompatKey, readonly RegExp[]> /['"`]runtimeSettings['"`].*\b(?:was|is)\s+not allowed\b/i, /"code"\s*:\s*"unrecognized_keys"[^]*"runtimeSettings"/i, ], + sessionTarget: [ + /\bunrecognized key(?:\(s\)|s)? in object:.*['"`]sessionTarget['"`]/i, + /\badditional propert(?:y|ies)\b.*['"`]sessionTarget['"`]/i, + /\bmust not have additional propert(?:y|ies)\b.*['"`]sessionTarget['"`]/i, + /\b(?:unexpected|extraneous)\s+(?:property|properties|field|fields|key|keys)\b.*['"`]sessionTarget['"`]/i, + /\b(?:unknown|invalid)\s+(?:property|properties|field|fields|key|keys)\b.*['"`]sessionTarget['"`]/i, + /['"`]sessionTarget['"`].*\b(?:was|is)\s+not allowed\b/i, + /"code"\s*:\s*"unrecognized_keys"[^]*"sessionTarget"/i, + ], + runtimeContext: [ + /\bunrecognized key(?:\(s\)|s)? in object:.*['"`]runtimeContext['"`]/i, + /\badditional propert(?:y|ies)\b.*['"`]runtimeContext['"`]/i, + /\bmust not have additional propert(?:y|ies)\b.*['"`]runtimeContext['"`]/i, + /\b(?:unexpected|extraneous)\s+(?:property|properties|field|fields|key|keys)\b.*['"`]runtimeContext['"`]/i, + /\b(?:unknown|invalid)\s+(?:property|properties|field|fields|key|keys)\b.*['"`]runtimeContext['"`]/i, + /['"`]runtimeContext['"`].*\b(?:was|is)\s+not allowed\b/i, + /"code"\s*:\s*"unrecognized_keys"[^]*"runtimeContext"/i, + ], } as const; function isLegacyCompatUnknownFieldValidationMessage( diff --git a/src/context-engine/types.ts b/src/context-engine/types.ts index 0a8492d5d17b..86df58c56673 100644 --- a/src/context-engine/types.ts +++ b/src/context-engine/types.ts @@ -125,25 +125,6 @@ export class ContextEngineRuntimeSettingsUnsupportedError extends Error { } } -/** - * Typed session target for compaction results. - * - * Makes the storage mode explicit and carries precise session identity - * instead of a raw file path string. File-backed is the only storage mode - * today; new storage modes extend this union when they ship. - */ -export type ContextEngineSessionTarget = { - kind: "file"; - /** Agent that owns the session, when the caller resolved it. */ - agentId?: string; - /** Runtime session id for the post-compaction live session. */ - sessionId: string; - /** Stable session key used for aliases, policy, and store resolution. */ - sessionKey?: string; - /** Live transcript artifact path for the file-backed session. */ - sessionFile: string; -}; - export type CompactResult = { ok: boolean; compacted: boolean; @@ -184,7 +165,9 @@ export function resolveCompactionSuccessorTranscript(result: CompactResult): { const target = result.result?.sessionTarget; return { sessionId: target?.sessionId ?? result.result?.sessionId, - sessionFile: target?.sessionFile ?? result.result?.sessionFile, + // Storage-neutral targets carry no raw locator; the deprecated raw field on + // the result remains the only file-era locator shipped engines report. + sessionFile: result.result?.sessionFile, }; } @@ -307,6 +290,30 @@ export type ContextEnginePromptCacheInfo = { expiresAt?: number; }; +export type ContextEngineTranscriptStorageInfo = { + /** + * Authoritative transcript backend for this runtime turn. + * + * Hosts may still pass legacy locator fields such as `sessionFile` for older + * plugin contracts, but context engines should use this field to decide + * whether that locator is a live transcript source. + */ + kind: "sqlite"; +}; + +export type ContextEngineSessionTarget = { + /** Agent that owns the session in the runtime store. */ + agentId?: string; + /** Runtime session id to compact. */ + sessionId?: string; + /** Stable session key used for aliases, policy, and store resolution. */ + sessionKey?: string; + /** Session store path that scopes the SQLite-backed runtime session. */ + storePath?: string; + /** Optional transport thread identity for session target resolution. */ + threadId?: string | number; +}; + export type ContextEngineRuntimeContext = Record<string, unknown> & { /** Runtime task working directory; workspaceDir remains the agent bootstrap workspace. */ cwd?: string; @@ -323,6 +330,10 @@ export type ContextEngineRuntimeContext = Record<string, unknown> & { currentTokenCount?: number; /** Optional prompt-cache telemetry for cache-aware engines. */ promptCache?: ContextEnginePromptCacheInfo; + /** Authoritative transcript backend for this turn. */ + transcriptStorage?: ContextEngineTranscriptStorageInfo; + /** Storage-neutral runtime session target for compaction delegation. */ + sessionTarget?: ContextEngineSessionTarget; /** * Safe transcript rewrite helper implemented by the runtime. * @@ -356,8 +367,11 @@ export interface ContextEngine { bootstrap?(params: { sessionId: string; sessionKey?: string; + /** Storage-neutral runtime session target for transcript/session SDK helpers. */ + sessionTarget?: ContextEngineSessionTarget; sessionFile: string; runtimeSettings?: ContextEngineRuntimeSettings; + runtimeContext?: ContextEngineRuntimeContext; }): Promise<BootstrapResult>; /** @@ -369,6 +383,8 @@ export interface ContextEngine { maintain?(params: { sessionId: string; sessionKey?: string; + /** Storage-neutral runtime session target for transcript/session SDK helpers. */ + sessionTarget?: ContextEngineSessionTarget; sessionFile: string; runtimeSettings?: ContextEngineRuntimeSettings; runtimeContext?: ContextEngineRuntimeContext; @@ -404,6 +420,8 @@ export interface ContextEngine { afterTurn?(params: { sessionId: string; sessionKey?: string; + /** Storage-neutral runtime session target for transcript/session SDK helpers. */ + sessionTarget?: ContextEngineSessionTarget; sessionFile: string; messages: AgentMessage[]; /** Number of messages that existed before the prompt was sent. */ @@ -452,8 +470,11 @@ export interface ContextEngine { */ compact(params: { sessionId: string; - sessionKey?: string; - sessionFile: string; + sessionKey: string; + /** Caller-resolved owner agent for global session aliases. */ + agentId?: string; + /** Storage-neutral runtime session target for delegated compaction. */ + sessionTarget?: ContextEngineSessionTarget; tokenBudget?: number; /** Force compaction even below the default trigger threshold. */ force?: boolean; diff --git a/src/cron/isolated-agent.session-identity.test.ts b/src/cron/isolated-agent.session-identity.test.ts index b73d1b7b6aa6..3f6e23f1c919 100644 --- a/src/cron/isolated-agent.session-identity.test.ts +++ b/src/cron/isolated-agent.session-identity.test.ts @@ -4,7 +4,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import * as modelThinkingDefault from "../agents/model-thinking-default.js"; -import type { SessionEntry } from "../config/sessions.js"; +import { upsertSessionEntry } from "../config/sessions/session-accessor.js"; import { runCronIsolatedAgentTurn } from "./isolated-agent.js"; import { makeCfg, @@ -27,10 +27,10 @@ import { loadSessionEntryMock, makeCronSession, mockRunCronFallbackPassthrough, + patchSessionEntryMock, resetRunCronIsolatedAgentTurnHarness, resolveCronSessionMock, runEmbeddedAgentMock, - updateSessionStoreMock, } from "./isolated-agent/run.test-harness.js"; import { normalizeCronJobCreate } from "./normalize.js"; import type { CronJob } from "./types.js"; @@ -38,21 +38,22 @@ import type { CronJob } from "./types.js"; setupRunCronIsolatedAgentTurnSuite(); async function useRealCronSessionState(): Promise<void> { - const [sessionRuntime, storeRuntime] = await Promise.all([ + const [sessionRuntime, sessionAccessor] = await Promise.all([ vi.importActual<typeof import("./isolated-agent/session.js")>("./isolated-agent/session.js"), - vi.importActual<typeof import("../config/sessions/store.runtime.js")>( - "../config/sessions/store.runtime.js", + vi.importActual<typeof import("../config/sessions/session-accessor.js")>( + "../config/sessions/session-accessor.js", ), ]); resolveCronSessionMock.mockImplementation(sessionRuntime.resolveCronSession); loadSessionEntryMock.mockImplementation(sessionRuntime.loadCronSessionEntryLatest); - updateSessionStoreMock.mockImplementation(storeRuntime.updateSessionStore); + patchSessionEntryMock.mockImplementation(sessionAccessor.patchSessionEntry); } function lastEmbeddedAgentCall(): { agentDir?: string; bootstrapContextMode?: "full" | "lightweight"; prompt?: string; + sessionId?: string; sessionKey?: string; workspaceDir?: string; sessionFile?: string; @@ -70,6 +71,7 @@ function lastEmbeddedAgentCall(): { agentDir?: string; bootstrapContextMode?: "full" | "lightweight"; prompt?: string; + sessionId?: string; sessionKey?: string; workspaceDir?: string; sessionFile?: string; @@ -172,10 +174,14 @@ describe("runCronIsolatedAgentTurn session identity", () => { }); const call = lastEmbeddedAgentCall(); - expect(call.sessionFile).toContain( - path.join(home, ".openclaw", "agents", "main", "sessions"), + expect(call.sessionFile).toBe( + `sqlite:main:${call.sessionId}:${path.join( + home, + ".openclaw", + "sessions", + "sessions.json", + )}`, ); - expect(String(call.sessionFile).endsWith(".jsonl")).toBe(true); }); }); @@ -185,6 +191,7 @@ describe("runCronIsolatedAgentTurn session identity", () => { const boundSessionKey = "agent:main:telegram:direct:42"; const originalSessionFile = path.join(home, "bound-session.jsonl"); const rotatedSessionFile = path.join(home, "bound-session-rotated.jsonl"); + await fs.writeFile(rotatedSessionFile, ""); const storePath = await writeSessionStoreEntries(home, { [boundSessionKey]: { sessionId: "bound-session", @@ -208,12 +215,6 @@ describe("runCronIsolatedAgentTurn session identity", () => { }, }, }); - updateSessionStoreMock.mockImplementation(async (targetStorePath, update) => { - const raw = await fs.readFile(targetStorePath, "utf-8"); - const store = JSON.parse(raw) as Record<string, SessionEntry>; - update(store); - await fs.writeFile(targetStorePath, JSON.stringify(store, null, 2), "utf-8"); - }); const currentBoundJob = normalizeCronJobCreate( { ...makeJob(DEFAULT_AGENT_TURN_PAYLOAD), @@ -222,6 +223,7 @@ describe("runCronIsolatedAgentTurn session identity", () => { }, { sessionContext: { sessionKey: boundSessionKey } }, ) as CronJob; + const executionSessionKey = `agent:main:cron:${currentBoundJob.id}`; const res = await runCronIsolatedAgentTurn({ cfg: makeCfg(home, storePath), @@ -238,28 +240,16 @@ describe("runCronIsolatedAgentTurn session identity", () => { expect.objectContaining({ sessionId: "bound-session-rotated" }), ); - const finalPersist = updateSessionStoreMock.mock.calls.at(-1); - expect(finalPersist?.[0]).toBe(storePath); - // Replay the final persist against the real post-run store: the claim - // guard rejects writes into a store that never held the session. - const persistedStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { [key: string]: unknown } - >; - (finalPersist![1] as (store: typeof persistedStore) => void)(persistedStore); - expect(persistedStore[boundSessionKey]).toEqual( + await expect(readSessionEntry(storePath, executionSessionKey)).resolves.toEqual( expect.objectContaining({ sessionId: "bound-session-rotated", sessionFile: rotatedSessionFile, - usageFamilyKey: boundSessionKey, - usageFamilySessionIds: ["bound-session", "bound-session-rotated"], }), ); - await expect(readSessionEntry(storePath, boundSessionKey)).resolves.toEqual( expect.objectContaining({ - sessionId: "bound-session-rotated", - sessionFile: rotatedSessionFile, + sessionId: "bound-session", + sessionFile: originalSessionFile, }), ); }); @@ -332,14 +322,14 @@ describe("runCronIsolatedAgentTurn session identity", () => { await useRealCronSessionState(); await withTempHome(async (home) => { const storePath = await writeSessionStore(home, { lastProvider: "webchat", lastTo: "" }); - const raw = await fs.readFile(storePath, "utf-8"); - const store = JSON.parse(raw) as Record<string, Record<string, unknown>>; - store["agent:main:cron:job-1"] = { - sessionId: "old", - updatedAt: Date.now(), - label: "Nightly digest", - }; - await fs.writeFile(storePath, JSON.stringify(store, null, 2), "utf-8"); + await upsertSessionEntry( + { storePath, sessionKey: "agent:main:cron:job-1" }, + { + sessionId: "old", + updatedAt: Date.now(), + label: "Nightly digest", + }, + ); await runCronTurn(home, { jobPayload: { kind: "agentTurn", message: "ping" }, diff --git a/src/cron/isolated-agent.test-harness.ts b/src/cron/isolated-agent.test-harness.ts index c067484a1453..f1ee45e72403 100644 --- a/src/cron/isolated-agent.test-harness.ts +++ b/src/cron/isolated-agent.test-harness.ts @@ -2,6 +2,8 @@ import fs from "node:fs/promises"; import path from "node:path"; import { withTempHome as withTempHomeBase } from "openclaw/plugin-sdk/test-env"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { CronJob } from "./types.js"; @@ -30,7 +32,9 @@ export async function writeSessionStoreEntries( const dir = path.join(home, ".openclaw", "sessions"); await fs.mkdir(dir, { recursive: true }); const storePath = path.join(dir, "sessions.json"); - await fs.writeFile(storePath, JSON.stringify(entries, null, 2), "utf-8"); + for (const [sessionKey, entry] of Object.entries(entries)) { + await replaceSessionEntry({ storePath, sessionKey }, entry as SessionEntry); + } return storePath; } diff --git a/src/cron/isolated-agent.turn-test-helpers.ts b/src/cron/isolated-agent.turn-test-helpers.ts index cd15b811dd2e..0c9894ac57f9 100644 --- a/src/cron/isolated-agent.turn-test-helpers.ts +++ b/src/cron/isolated-agent.turn-test-helpers.ts @@ -1,9 +1,9 @@ /** Reusable turn-level fixtures for isolated cron agent regression tests. */ import "./isolated-agent.mocks.js"; -import fs from "node:fs/promises"; import { expect, vi } from "vitest"; import { runEmbeddedAgent } from "../agents/embedded-agent.js"; import type { CliDeps } from "../cli/deps.js"; +import { loadSessionEntry } from "../config/sessions/session-accessor.js"; import { runCronIsolatedAgentTurn } from "./isolated-agent.js"; import { makeCfg, @@ -60,12 +60,7 @@ export function expectEmbeddedProviderModel(expected: { provider: string; model: } export async function readSessionEntry(storePath: string, key: string) { - const raw = await fs.readFile(storePath, "utf-8"); - const store = JSON.parse(raw) as Record< - string, - { sessionId?: string; label?: string; sessionFile?: string } - >; - return store[key]; + return loadSessionEntry({ storePath, sessionKey: key }); } export const DEFAULT_MESSAGE = "do it"; diff --git a/src/cron/isolated-agent/run-execution.runtime.ts b/src/cron/isolated-agent/run-execution.runtime.ts index f67a7bffbdea..ac34256c4d14 100644 --- a/src/cron/isolated-agent/run-execution.runtime.ts +++ b/src/cron/isolated-agent/run-execution.runtime.ts @@ -15,7 +15,6 @@ export { } from "../../agents/embedded-agent-runner/result-fallback-classifier.js"; export { isCliProvider } from "../../agents/model-selection-cli.js"; export { normalizeVerboseLevel } from "../../auto-reply/thinking.shared.js"; -export { resolveSessionTranscriptPath } from "../../config/sessions/paths.js"; export { registerAgentRunContext } from "../../infra/agent-events.js"; export { logWarn } from "../../logger.js"; import { createLazyImportLoader } from "../../shared/lazy-promise.js"; diff --git a/src/cron/isolated-agent/run-executor.ts b/src/cron/isolated-agent/run-executor.ts index ea9190decb31..3b79aa7aa42a 100644 --- a/src/cron/isolated-agent/run-executor.ts +++ b/src/cron/isolated-agent/run-executor.ts @@ -11,6 +11,7 @@ import { wrapUntrustedPromptDataBlock } from "../../agents/sanitize-for-prompt.j import { resolveSessionRuntimeOverrideForProvider } from "../../agents/session-runtime-compat.js"; import type { ThinkLevel, VerboseLevel } from "../../auto-reply/thinking.js"; import type { CliSessionBinding } from "../../config/sessions.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import type { AgentDefaultsConfig } from "../../config/types.agent-defaults.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { SourceDeliveryPlan } from "../../infra/outbound/source-delivery-plan.js"; @@ -43,7 +44,6 @@ import { resolveBootstrapWarningSignaturesSeen, resolveCandidateThinkingLevel, resolveCronAgentLane, - resolveSessionTranscriptPath, runCliAgent, runWithModelFallback, } from "./run-execution.runtime.js"; @@ -200,6 +200,7 @@ export function createCronPromptExecutor(params: { agentDir: string; agentSessionKey: string; runSessionKey: string; + usesDetachedRunSession?: boolean; workspaceDir: string; lane?: string; resolvedVerboseLevel: VerboseLevel; @@ -240,7 +241,11 @@ export function createCronPromptExecutor(params: { }) { const sessionFile = params.cronSession.sessionEntry.sessionFile?.trim() || - resolveSessionTranscriptPath(params.cronSession.sessionEntry.sessionId, params.agentId); + formatSqliteSessionFileMarker({ + agentId: params.agentId, + sessionId: params.cronSession.sessionEntry.sessionId, + storePath: params.cronSession.storePath, + }); // Fallback for callers that bypass prepareCronRunContext before persisting retries. if (!params.cronSession.sessionEntry.sessionFile?.trim()) { params.cronSession.sessionEntry.sessionFile = sessionFile; @@ -302,10 +307,11 @@ export function createCronPromptExecutor(params: { : createUserTurnTranscriptRecorder({ input: { text: promptText }, target: { - transcriptPath: sessionFile, sessionId: params.cronSession.sessionEntry.sessionId, agentId: params.agentId, sessionKey: params.runSessionKey, + sessionEntry: params.cronSession.sessionEntry, + storePath: params.cronSession.storePath, cwd: params.workspaceDir, config: params.cfgWithAgentDefaults, }, @@ -416,7 +422,7 @@ export function createCronPromptExecutor(params: { agentId: params.agentId, trigger: "cron", jobId: params.job.id, - cleanupCliLiveSessionOnRunEnd: params.job.sessionTarget === "isolated", + cleanupCliLiveSessionOnRunEnd: params.usesDetachedRunSession === true, sessionFile, workspaceDir: params.workspaceDir, config: params.cfgWithAgentDefaults, @@ -485,7 +491,7 @@ export function createCronPromptExecutor(params: { agentId: params.agentId, trigger: "cron", jobId: params.job.id, - cleanupBundleMcpOnRunEnd: params.job.sessionTarget === "isolated", + cleanupBundleMcpOnRunEnd: params.usesDetachedRunSession === true, allowGatewaySubagentBinding: true, messageChannel, agentAccountId: params.resolvedDelivery.accountId, @@ -598,6 +604,7 @@ export async function executeCronRun(params: { agentDir: string; agentSessionKey: string; runSessionKey: string; + usesDetachedRunSession?: boolean; workspaceDir: string; lane?: string; resolvedDelivery: { @@ -665,6 +672,7 @@ export async function executeCronRun(params: { agentDir: params.agentDir, agentSessionKey: params.agentSessionKey, runSessionKey: params.runSessionKey, + usesDetachedRunSession: params.usesDetachedRunSession, workspaceDir: params.workspaceDir, lane: params.lane, resolvedVerboseLevel, diff --git a/src/cron/isolated-agent/run-session-state.test.ts b/src/cron/isolated-agent/run-session-state.test.ts index b073beff8c9f..ce0d14b3910a 100644 --- a/src/cron/isolated-agent/run-session-state.test.ts +++ b/src/cron/isolated-agent/run-session-state.test.ts @@ -36,6 +36,24 @@ function makeCronSession(entry = makeSessionEntry()): MutableCronSession { } as MutableCronSession; } +/** + * Guarded-persist seam backed by an in-memory persisted row, mirroring the + * accessor contract: `update` sees the freshest persisted entry (undefined + * when absent), may throw to reject a stale claim, and its return is committed. + */ +function makeGuardedPersistSessionEntry(persistedStore: Record<string, SessionEntry>) { + return vi.fn( + async (params: { + fallbackEntry: SessionEntry; + sessionKey: string; + storePath: string; + update: (currentEntry: SessionEntry | undefined) => SessionEntry; + }) => { + persistedStore[params.sessionKey] = params.update(persistedStore[params.sessionKey]); + }, + ); +} + describe("createPersistCronSessionEntry", () => { it("owns an exact hidden continuation row without colliding with another run", async () => { const runSessionKey = "agent:main:cron:job:run:run-session-id"; @@ -52,9 +70,7 @@ describe("createPersistCronSessionEntry", () => { lifecycleRevision, } as MutableCronSession; const store: Record<string, SessionEntry> = {}; - const updateSessionStore = vi.fn( - async (_storePath, update: (entries: Record<string, SessionEntry>) => void) => update(store), - ); + const persistSessionEntry = makeGuardedPersistSessionEntry(store); const continuation = createCronRunContinuationSession({ isFastTestEnv: false, cronSession, @@ -62,13 +78,15 @@ describe("createPersistCronSessionEntry", () => { thinkingLevel: "high", toolsAllow: ["image_generate", "write"], toolsAllowIsDefault: true, - updateSessionStore, + persistSessionEntry, }); await continuation.initialize(); - expect(updateSessionStore).toHaveBeenCalledWith(cronSession.storePath, expect.any(Function), { - activeSessionKey: runSessionKey, - requireWriteSuccess: true, + expect(persistSessionEntry).toHaveBeenCalledWith({ + fallbackEntry: expect.objectContaining({ sessionId: "run-session-id" }), + sessionKey: runSessionKey, + storePath: cronSession.storePath, + update: expect.any(Function), }); expect(store[runSessionKey]).toMatchObject({ sessionId: "run-session-id", @@ -148,26 +166,25 @@ describe("createPersistCronSessionEntry", () => { }, }), ); - const updateSessionStore = vi.fn( - async (_storePath, update: (store: Record<string, SessionEntry>) => void) => { - const store: Record<string, SessionEntry> = {}; - update(store); - expect(store["agent:main:cron:job"]).toBe(cronSession.sessionEntry); - expect(store["agent:main:cron:job:run:run-session-id"]).toBeUndefined(); - }, - ); + const persistSessionEntry = vi.fn(async () => {}); const persist = createPersistCronSessionEntry({ isFastTestEnv: false, cronSession, agentSessionKey: "agent:main:cron:job", - updateSessionStore, + persistSessionEntry, }); await persist(); expect(cronSession.store["agent:main:cron:job"]).toBe(cronSession.sessionEntry); expect(cronSession.store["agent:main:cron:job:run:run-session-id"]).toBeUndefined(); + expect(persistSessionEntry).toHaveBeenCalledWith({ + storePath: "/tmp/sessions.json", + sessionKey: "agent:main:cron:job", + fallbackEntry: cronSession.sessionEntry, + update: expect.any(Function), + }); }); it("does not register cron sessions as resumable until the transcript exists", async () => { @@ -183,34 +200,35 @@ describe("createPersistCronSessionEntry", () => { status: "running", }), ); - const updateSessionStore = vi.fn( - async (_storePath, update: (store: Record<string, SessionEntry>) => void) => { - const store: Record<string, SessionEntry> = {}; - update(store); - expect(store["agent:main:cron:shell-only"]).toEqual({ - label: "Cron: shell-only", - lifecycleRevision: "run-revision", - status: "running", - updatedAt: 1000, - systemSent: true, - }); - }, - ); + const persistSessionEntry = vi.fn(async () => {}); const persist = createPersistCronSessionEntry({ isFastTestEnv: false, cronSession, agentSessionKey: "agent:main:cron:shell-only", - updateSessionStore, + persistSessionEntry, }); await persist(); - expect(cronSession.store["agent:main:cron:shell-only"]?.sessionId).toBeUndefined(); + expect(cronSession.store["agent:main:cron:shell-only"]?.sessionId).toBe("run-session-id"); expect(cronSession.store["agent:main:cron:shell-only"]?.sessionFile).toBeUndefined(); expect(cronSession.store["agent:main:cron:shell-only"]?.lifecycleRevision).toBe("run-revision"); expect(cronSession.sessionEntry.sessionId).toBe("run-session-id"); expect(cronSession.sessionEntry.sessionFile).toBe(missingTranscriptPath); + expect(persistSessionEntry).toHaveBeenCalledWith({ + storePath: "/tmp/sessions.json", + sessionKey: "agent:main:cron:shell-only", + fallbackEntry: { + label: "Cron: shell-only", + lifecycleRevision: "run-revision", + sessionId: "run-session-id", + status: "running", + updatedAt: 1000, + systemSent: true, + }, + update: expect.any(Function), + }); }); it("restores resumable cron fields once the transcript exists", async () => { @@ -226,19 +244,7 @@ describe("createPersistCronSessionEntry", () => { isFastTestEnv: false, cronSession, agentSessionKey: "agent:main:cron:completed", - updateSessionStore: vi.fn( - async (_storePath, update: (store: Record<string, SessionEntry>) => void) => { - const store: Record<string, SessionEntry> = {}; - update(store); - expect(store["agent:main:cron:completed"]).toEqual({ - sessionId: "run-session-id", - sessionFile: transcriptPath, - label: "Cron: completed", - updatedAt: 1000, - systemSent: true, - }); - }, - ), + persistSessionEntry: vi.fn(async () => {}), }); await persist(); @@ -254,24 +260,24 @@ describe("createPersistCronSessionEntry", () => { it("persists explicit session-bound cron state under the requested session key", async () => { const cronSession = makeCronSession(); - const updateSessionStore = vi.fn( - async (_storePath, update: (store: Record<string, SessionEntry>) => void) => { - const store: Record<string, SessionEntry> = {}; - update(store); - expect(store["agent:main:session"]).toBe(cronSession.sessionEntry); - }, - ); + const persistSessionEntry = vi.fn(async () => {}); const persist = createPersistCronSessionEntry({ isFastTestEnv: false, cronSession, agentSessionKey: "agent:main:session", - updateSessionStore, + persistSessionEntry, }); await persist(); expect(cronSession.store["agent:main:session"]).toBe(cronSession.sessionEntry); + expect(persistSessionEntry).toHaveBeenCalledWith({ + storePath: "/tmp/sessions.json", + sessionKey: "agent:main:session", + fallbackEntry: cronSession.sessionEntry, + update: expect.any(Function), + }); }); it("does not let an older concurrent run reclaim a persisted lifecycle revision", async () => { @@ -291,24 +297,20 @@ describe("createPersistCronSessionEntry", () => { initialSessionEntry, lifecycleRevision, }) as MutableCronSession; - const updateSessionStore = vi.fn( - async (_storePath, update: (store: Record<string, SessionEntry>) => void) => { - update(persistedStore); - }, - ); + const persistSessionEntry = makeGuardedPersistSessionEntry(persistedStore); const olderSession = makeConcurrentSession("older-revision"); const newerSession = makeConcurrentSession("newer-revision"); const persistOlder = createPersistCronSessionEntry({ isFastTestEnv: false, cronSession: olderSession, agentSessionKey: sessionKey, - updateSessionStore, + persistSessionEntry, }); const persistNewer = createPersistCronSessionEntry({ isFastTestEnv: false, cronSession: newerSession, agentSessionKey: sessionKey, - updateSessionStore, + persistSessionEntry, }); await persistNewer(); @@ -337,9 +339,7 @@ describe("createPersistCronSessionEntry", () => { isFastTestEnv: false, cronSession: nextSession, agentSessionKey: sessionKey, - updateSessionStore: async (_storePath, update) => { - update(persistedStore); - }, + persistSessionEntry: makeGuardedPersistSessionEntry(persistedStore), }); const activeLease = await beginSessionWorkAdmission({ scope: storePath, @@ -385,9 +385,7 @@ describe("createPersistCronSessionEntry", () => { isFastTestEnv: false, cronSession, agentSessionKey: sessionKey, - updateSessionStore: async (_storePath, update) => { - update(persistedStore); - }, + persistSessionEntry: makeGuardedPersistSessionEntry(persistedStore), }); await expect(persist()).resolves.toBeUndefined(); @@ -435,9 +433,7 @@ describe("createPersistCronSessionEntry", () => { isFastTestEnv: false, cronSession, agentSessionKey: sessionKey, - updateSessionStore: async (_storePath, update) => { - update(persistedStore); - }, + persistSessionEntry: makeGuardedPersistSessionEntry(persistedStore), }); await persist(); @@ -486,9 +482,7 @@ describe("createPersistCronSessionEntry", () => { isFastTestEnv: false, cronSession, agentSessionKey: sessionKey, - updateSessionStore: async (_storePath, update) => { - update(persistedStore); - }, + persistSessionEntry: makeGuardedPersistSessionEntry(persistedStore), }); await persist(); @@ -519,27 +513,14 @@ describe("createPersistCronSessionEntry", () => { sessionFile: "/tmp/bound-session-rotated.jsonl", }, }); - const updateSessionStore = vi.fn( - async (_storePath, update: (store: Record<string, SessionEntry>) => void) => { - const store: Record<string, SessionEntry> = {}; - update(store); - expect(store["agent:main:telegram:direct:42"]).toEqual({ - sessionId: "bound-session-rotated", - sessionFile: "/tmp/bound-session-rotated.jsonl", - usageFamilyKey: "agent:main:telegram:direct:42", - usageFamilySessionIds: ["bound-session", "bound-session-rotated"], - updatedAt: 1000, - systemSent: true, - }); - }, - ); + const persistSessionEntry = vi.fn(async () => {}); expect(changed).toBe(true); const persist = createPersistCronSessionEntry({ isFastTestEnv: false, cronSession, agentSessionKey: "agent:main:telegram:direct:42", - updateSessionStore, + persistSessionEntry, }); await persist(); @@ -552,6 +533,19 @@ describe("createPersistCronSessionEntry", () => { updatedAt: 1000, systemSent: true, }); + expect(persistSessionEntry).toHaveBeenCalledWith({ + storePath: "/tmp/sessions.json", + sessionKey: "agent:main:telegram:direct:42", + fallbackEntry: { + sessionId: "bound-session-rotated", + sessionFile: "/tmp/bound-session-rotated.jsonl", + usageFamilyKey: "agent:main:telegram:direct:42", + usageFamilySessionIds: ["bound-session", "bound-session-rotated"], + updatedAt: 1000, + systemSent: true, + }, + update: expect.any(Function), + }); }); }); diff --git a/src/cron/isolated-agent/run-session-state.ts b/src/cron/isolated-agent/run-session-state.ts index 9d7d21a50e31..1983a8ab6e38 100644 --- a/src/cron/isolated-agent/run-session-state.ts +++ b/src/cron/isolated-agent/run-session-state.ts @@ -4,6 +4,7 @@ import { isDeepStrictEqual } from "node:util"; import type { LiveSessionModelSelection } from "../../agents/live-model-switch.js"; import type { SessionEntry } from "../../config/sessions.js"; import { mergeSessionSnapshotChanges } from "../../config/sessions/session-snapshot-merge.js"; +import { parseSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import { isCronSessionKey } from "../../sessions/session-key-utils.js"; import { isSessionWorkAdmissionActive } from "../../sessions/session-lifecycle-admission.js"; import type { SkillSnapshot } from "../../skills/types.js"; @@ -21,11 +22,18 @@ export type MutableCronSession = ReturnType<typeof resolveCronSession> & { /** Live provider/model/auth-profile selection reported by the running session. */ export type CronLiveSelection = LiveSessionModelSelection; -type UpdateSessionStore = ( - storePath: string, - update: (store: MutableSessionStore) => void, - options?: { activeSessionKey?: string; requireWriteSuccess?: boolean }, -) => Promise<void>; +/** + * Accessor-backed guarded write: `update` receives the freshest persisted row + * (undefined when absent), may throw to reject a stale lifecycle claim, and + * returns the full entry to commit. `fallbackEntry` seeds creation when the + * row does not exist yet. + */ +type PersistSessionEntry = (params: { + fallbackEntry: SessionEntry; + sessionKey: string; + storePath: string; + update: (currentEntry: SessionEntry | undefined) => SessionEntry; +}) => Promise<void>; /** Persists the currently selected mutable cron session entry to the session store. */ export type PersistCronSessionEntry = () => Promise<void>; @@ -51,6 +59,9 @@ export function resolveCronLifecycleRevisionIdentity(lifecycleRevision: string): function cronTranscriptExists(entry: SessionEntry): boolean { const sessionFile = entry.sessionFile?.trim(); + if (parseSqliteSessionFileMarker(sessionFile)) { + return true; + } return Boolean(sessionFile && fs.existsSync(sessionFile)); } @@ -71,7 +82,6 @@ function toNonResumableCronSessionEntry(entry: SessionEntry): SessionEntry { const next = { ...entry } as Partial<SessionEntry>; // If the transcript never materialized, do not persist stale resume handles // that would make the next cron run believe a resumable CLI session exists. - delete next.sessionId; delete next.sessionFile; delete next.sessionStartedAt; delete next.lastInteractionAt; @@ -86,7 +96,7 @@ export function createPersistCronSessionEntry(params: { isFastTestEnv: boolean; cronSession: MutableCronSession; agentSessionKey: string; - updateSessionStore: UpdateSessionStore; + persistSessionEntry: PersistSessionEntry; }): PersistCronSessionEntry { return async () => { if (params.isFastTestEnv) { @@ -101,10 +111,11 @@ export function createPersistCronSessionEntry(params: { : liveEntry; let committedEntry = persistedEntry; let mergedLiveEntry = liveEntry; - await params.updateSessionStore( - params.cronSession.storePath, - (store) => { - const currentEntry = store[params.agentSessionKey]; + await params.persistSessionEntry({ + storePath: params.cronSession.storePath, + sessionKey: params.agentSessionKey, + fallbackEntry: persistedEntry, + update: (currentEntry) => { const ownsCurrentRevision = currentEntry?.lifecycleRevision === params.cronSession.lifecycleRevision; const currentRevisionActive = Boolean( @@ -144,10 +155,9 @@ export function createPersistCronSessionEntry(params: { current: currentEntry, }); } - store[params.agentSessionKey] = committedEntry; + return committedEntry; }, - { activeSessionKey: params.agentSessionKey, requireWriteSuccess: true }, - ); + }); // The storage projection may intentionally omit resume identity until its // transcript exists. Keep that projection out of the active run object. params.cronSession.sessionEntry = mergedLiveEntry; @@ -169,7 +179,7 @@ export function createCronRunContinuationSession(params: { sourceReplyDeliveryMode?: "automatic" | "message_tool_only"; requireExplicitMessageTarget?: boolean; }; - updateSessionStore: UpdateSessionStore; + persistSessionEntry: PersistSessionEntry; }): CronRunContinuationSession { const continuation: NonNullable<SessionEntry["cronRunContinuation"]> = { lifecycleRevision: params.cronSession.lifecycleRevision, @@ -189,20 +199,25 @@ export function createCronRunContinuationSession(params: { const source = structuredClone(params.cronSession.sessionEntry); let persisted = false; let alreadySealed = false; - await params.updateSessionStore( - params.cronSession.storePath, - (store) => { - const current = store[params.runSessionKey]; + await params.persistSessionEntry({ + storePath: params.cronSession.storePath, + sessionKey: params.runSessionKey, + fallbackEntry: source, + update: (current) => { if ((current && !owns(current)) || (!current && !create)) { - return; + throw new CronSessionLifecycleClaimError(params.runSessionKey); } // Leaving running transfers ownership to gateway continuation turns. // The initial cron owner must never overwrite their newer state. if (current && current.cronRunContinuation?.phase !== "running") { alreadySealed = phase === "ready" && current.cronRunContinuation?.phase === "ready"; - return; + if (alreadySealed) { + return current; + } + throw new CronSessionLifecycleClaimError(params.runSessionKey); } - store[params.runSessionKey] = { + persisted = true; + return { ...current, ...source, ...(params.thinkingLevel ? { thinkingLevel: params.thinkingLevel } : {}), @@ -212,10 +227,8 @@ export function createCronRunContinuationSession(params: { ...(phase === "ready" ? { basePersisted } : {}), }, }; - persisted = true; }, - { activeSessionKey: params.runSessionKey, requireWriteSuccess: true }, - ); + }); if (!persisted && !alreadySealed) { throw new CronSessionLifecycleClaimError(params.runSessionKey); } diff --git a/src/cron/isolated-agent/run.cron-model-override.test.ts b/src/cron/isolated-agent/run.cron-model-override.test.ts index ea2f536bc7b3..290a9dcdcf8d 100644 --- a/src/cron/isolated-agent/run.cron-model-override.test.ts +++ b/src/cron/isolated-agent/run.cron-model-override.test.ts @@ -1,5 +1,6 @@ // Cron model override tests cover model selection overrides for scheduled runs. import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import type { SessionEntry } from "../../config/sessions.js"; import { clearFastTestEnv, loadRunCronIsolatedAgentTurn, @@ -13,7 +14,7 @@ import { resetRunCronIsolatedAgentTurnHarness, restoreFastTestEnv, runWithModelFallbackMock, - updateSessionStoreMock, + patchSessionEntryMock, } from "./run.test-harness.js"; const runCronIsolatedAgentTurn = await loadRunCronIsolatedAgentTurn(); @@ -138,18 +139,36 @@ describe("runCronIsolatedAgentTurn — cron model override (#21057)", () => { modelProvider?: string; systemSent?: boolean; }> = []; - // One persistent store across persist calls: the lifecycle claim guard - // treats a store that lost the entry between calls as a foreign owner. - const persistentStore: Record<string, unknown> = {}; - updateSessionStoreMock.mockImplementation( - async (_path: string, cb: (s: Record<string, unknown>) => void) => { - cb(persistentStore); - const entry = Object.values(persistentStore)[0] as - | { model?: string; modelProvider?: string; systemSent?: boolean } - | undefined; - if (entry) { - persistedSnapshots.push(structuredClone(entry)); + // The cron persist path calls patchSessionEntry(scope, updater, options); + // the committed row is the updater's return, so snapshot that. Thread the + // previously committed row forward as existingEntry so the lifecycle claim + // guard proves ownership across the run's successive persists. + const committedRows = new Map<string, SessionEntry>(); + patchSessionEntryMock.mockImplementation( + async ( + scope: { storePath?: string; sessionKey: string }, + update: ( + entry: SessionEntry, + context: { existingEntry: SessionEntry | undefined }, + ) => SessionEntry | null, + options: { fallbackEntry?: SessionEntry } = {}, + ) => { + const key = `${scope.storePath ?? ""}\0${scope.sessionKey}`; + const committedRow = committedRows.get(key); + const writeBase = committedRow ?? options.fallbackEntry; + if (!writeBase) { + return null; } + const committed = update(structuredClone(writeBase), { + existingEntry: committedRow ? structuredClone(committedRow) : undefined, + }); + if (committed) { + committedRows.set(key, structuredClone(committed)); + if (!scope.sessionKey.includes(":run:")) { + persistedSnapshots.push(structuredClone(committed)); + } + } + return committed; }, ); @@ -228,15 +247,33 @@ describe("runCronIsolatedAgentTurn — cron model override (#21057)", () => { // Persist ordering: [1] skills snapshot, [2] pre-run, [3] post-run. // Only the pre-run persist (call 2) should fail — the skills snapshot // persist is pre-existing code without a try-catch guard. - let callCount = 0; - const persistentStore: Record<string, unknown> = {}; - updateSessionStoreMock.mockImplementation( - async (_path: string, update: (store: Record<string, unknown>) => unknown) => { - callCount++; - if (callCount === 2) { + let basePersistCount = 0; + const committedRows = new Map<string, SessionEntry>(); + patchSessionEntryMock.mockImplementation( + async ( + scope: { storePath?: string; sessionKey: string }, + update: ( + entry: SessionEntry, + context: { existingEntry: SessionEntry | undefined }, + ) => SessionEntry | null, + options: { fallbackEntry?: SessionEntry } = {}, + ) => { + if (!scope.sessionKey.includes(":run:") && ++basePersistCount === 2) { throw new Error("ENOSPC: no space left on device"); } - await update(persistentStore); + const key = `${scope.storePath ?? ""}\0${scope.sessionKey}`; + const current = committedRows.get(key); + const writeBase = current ?? options.fallbackEntry; + if (!writeBase) { + return null; + } + const committed = update(structuredClone(writeBase), { + existingEntry: current ? structuredClone(current) : undefined, + }); + if (committed) { + committedRows.set(key, structuredClone(committed)); + } + return committed; }, ); diff --git a/src/cron/isolated-agent/run.live-session-model-switch.test.ts b/src/cron/isolated-agent/run.live-session-model-switch.test.ts index 606ff2d2477e..c1b17fd27f33 100644 --- a/src/cron/isolated-agent/run.live-session-model-switch.test.ts +++ b/src/cron/isolated-agent/run.live-session-model-switch.test.ts @@ -14,7 +14,7 @@ import { resetRunCronIsolatedAgentTurnHarness, runEmbeddedAgentMock, runWithModelFallbackMock, - updateSessionStoreMock, + patchSessionEntryMock, } from "./run.test-harness.js"; const runCronIsolatedAgentTurn = await loadRunCronIsolatedAgentTurn(); @@ -187,7 +187,7 @@ describe("runCronIsolatedAgentTurn — LiveSessionModelSwitchError retry (#57206 expect(result.status).toBe("error"); expect(String(result.error)).toContain("transient network error"); - expect(updateSessionStoreMock).toHaveBeenCalled(); + expect(patchSessionEntryMock).toHaveBeenCalled(); expect(cronSession.sessionEntry.model).toBe("claude-sonnet-4-6"); expect(cronSession.sessionEntry.modelProvider).toBe("anthropic"); }); diff --git a/src/cron/isolated-agent/run.message-tool-policy.test.ts b/src/cron/isolated-agent/run.message-tool-policy.test.ts index a74f053bad60..2fa349c16e8f 100644 --- a/src/cron/isolated-agent/run.message-tool-policy.test.ts +++ b/src/cron/isolated-agent/run.message-tool-policy.test.ts @@ -958,7 +958,9 @@ describe("runCronIsolatedAgentTurn message tool policy", () => { it("releases cron run context references after completion", async () => { const initialSessionEntry = { retained: true }; - loadSessionEntryMock.mockReturnValue(initialSessionEntry); + loadSessionEntryMock.mockImplementation((_storePath, sessionKey) => + sessionKey === "agent:default:cron:message-tool-policy" ? initialSessionEntry : undefined, + ); const cronSession = makeCronSession({ store: { "agent:default:cron:message-tool-policy": initialSessionEntry }, initialSessionEntry, @@ -1043,7 +1045,9 @@ describe("runCronIsolatedAgentTurn message tool policy", () => { it("keeps shared cron run context references active after completion", async () => { const initialSessionEntry = { retained: true }; - loadSessionEntryMock.mockReturnValue(initialSessionEntry); + loadSessionEntryMock.mockImplementation((_storePath, sessionKey) => + sessionKey === "agent:default:cron:message-tool-policy" ? initialSessionEntry : undefined, + ); const cronSession = makeCronSession({ store: { "agent:default:cron:message-tool-policy": initialSessionEntry }, initialSessionEntry, diff --git a/src/cron/isolated-agent/run.runtime.ts b/src/cron/isolated-agent/run.runtime.ts index bcd68f070c02..e6d0a6729d8f 100644 --- a/src/cron/isolated-agent/run.runtime.ts +++ b/src/cron/isolated-agent/run.runtime.ts @@ -20,7 +20,6 @@ export { normalizeThinkLevel, resolveSupportedThinkingLevel, } from "../../auto-reply/thinking.js"; -export { resolveSessionTranscriptPath } from "../../config/sessions/paths.js"; export { setSessionRuntimeModel } from "../../config/sessions/types.js"; export { logWarn } from "../../logger.js"; export { normalizeAgentId } from "../../routing/session-key.js"; diff --git a/src/cron/isolated-agent/run.session-lifecycle.test.ts b/src/cron/isolated-agent/run.session-lifecycle.test.ts index 9b480795ae7a..69d8cc877ed9 100644 --- a/src/cron/isolated-agent/run.session-lifecycle.test.ts +++ b/src/cron/isolated-agent/run.session-lifecycle.test.ts @@ -35,7 +35,10 @@ function makePersistentCronParams(sessionKey: string) { agentId: "main", sessionKey, job: makeIsolatedAgentJobFixture({ - sessionTarget: "current", + // Bind the run to the persistent session key so the run operates on it + // directly; `current`/`isolated` targets derive a detached `cron:<id>` + // run session instead, which the lifecycle claim assertions do not target. + sessionTarget: `session:${sessionKey}`, delivery: { mode: "none" }, }), }); diff --git a/src/cron/isolated-agent/run.test-harness.ts b/src/cron/isolated-agent/run.test-harness.ts index ebc9bc824301..84c41c80d2bc 100644 --- a/src/cron/isolated-agent/run.test-harness.ts +++ b/src/cron/isolated-agent/run.test-harness.ts @@ -26,6 +26,11 @@ type CronSession = { [key: string]: unknown; }; +type SessionAccessorModule = typeof import("../../config/sessions/session-accessor.js"); + +let actualReplaceSessionEntry: SessionAccessorModule["replaceSessionEntry"]; +let actualLoadSessionEntry: SessionAccessorModule["loadSessionEntry"]; + function createMock(): Mock { return vi.fn(); } @@ -41,6 +46,10 @@ function normalizeModelSelectionForTest(value: unknown): string | undefined { return normalizeOptionalString((value as { primary?: unknown }).primary); } +function usesRealAccessorStore(storePath?: string): boolean { + return Boolean(storePath && storePath !== "/tmp/store.json"); +} + export const buildWorkspaceSkillSnapshotMock = createMock(); export const resolveAgentConfigMock = createMock(); export const resolveEffectiveModelFallbacksMock = createMock(); @@ -63,8 +72,9 @@ export const getCliSessionBindingMock = createMock(); export const getCliSessionIdMock = createMock(); export const clearCliSessionMock = createMock(); export const setCliSessionBindingMock = createMock(); -export const updateSessionStoreMock = createMock(); export const loadSessionEntryMock = createMock(); +export const replaceSessionEntryMock = createMock(); +export const patchSessionEntryMock = createMock(); export const resolveCronSessionMock = createMock(); export const logWarnMock = createMock(); export const countActiveDescendantRunsMock = createMock(); @@ -347,9 +357,18 @@ vi.mock("../../gateway/call.runtime.js", () => ({ callGateway: callGatewayMock, })); -vi.mock("../../config/sessions/store.runtime.js", () => ({ - updateSessionStore: updateSessionStoreMock, -})); +vi.mock("../../config/sessions/session-accessor.js", async () => { + const actual = await vi.importActual<SessionAccessorModule>( + "../../config/sessions/session-accessor.js", + ); + actualReplaceSessionEntry = actual.replaceSessionEntry; + actualLoadSessionEntry = actual.loadSessionEntry; + return { + ...actual, + replaceSessionEntry: replaceSessionEntryMock, + patchSessionEntry: patchSessionEntryMock, + }; +}); vi.mock("../delivery-plan.js", async () => ({ ...(await vi.importActual<typeof import("../delivery-plan.js")>("../delivery-plan.js")), @@ -403,7 +422,7 @@ export function makeCronSessionEntry(overrides?: Record<string, unknown>): CronS } export function makeCronSession(overrides?: Record<string, unknown>): CronSession { - return { + const session = { storePath: "/tmp/store.json", store: {}, sessionEntry: makeCronSessionEntry(), @@ -413,6 +432,13 @@ export function makeCronSession(overrides?: Record<string, unknown>): CronSessio isNewSession: true, ...overrides, } as CronSession; + // Real resolveCronSession stamps the run's lifecycleRevision onto the live + // sessionEntry so the accessor-backed persist path can prove ownership on + // later writes. Mirror that here unless a test seeds its own revision. + if (session.sessionEntry.lifecycleRevision === undefined) { + session.sessionEntry.lifecycleRevision = session.lifecycleRevision; + } + return session; } function makeDefaultModelFallbackResult() { @@ -715,20 +741,19 @@ function resetRunOutcomeMocks(): void { function resetRunSessionMocks(): void { loadSessionEntryMock.mockReset(); loadSessionEntryMock.mockReturnValue(undefined); - updateSessionStoreMock.mockReset(); - const sessionStores = new Map<string, Record<string, unknown>>(); - updateSessionStoreMock.mockImplementation( - async (storePath: string, update: (store: Record<string, unknown>) => unknown) => { - const resolvedResult = resolveCronSessionMock.mock.results.at(-1); - const resolvedSession = - resolvedResult?.type === "return" ? (resolvedResult.value as CronSession) : undefined; - const store = - sessionStores.get(storePath) ?? - (resolvedSession?.storePath === storePath ? resolvedSession.store : {}); - sessionStores.set(storePath, store); - return await update(store); + replaceSessionEntryMock.mockReset(); + replaceSessionEntryMock.mockImplementation( + async ( + scope: Parameters<SessionAccessorModule["replaceSessionEntry"]>[0], + entry: Parameters<SessionAccessorModule["replaceSessionEntry"]>[1], + ) => { + if (usesRealAccessorStore(scope.storePath)) { + await actualReplaceSessionEntry(scope, entry); + } }, ); + patchSessionEntryMock.mockReset(); + installPatchSessionEntryStore(); resolveCronSessionMock.mockReset(); resolveCronSessionMock.mockReturnValue(makeCronSession()); callGatewayMock.mockReset(); @@ -737,6 +762,52 @@ function resetRunSessionMocks(): void { retireSessionMcpRuntimeMock.mockResolvedValue(true); } +/** + * In-memory stand-in for the SQLite accessor `patchSessionEntry` used by the + * cron persist path. Prod flips real session storage to per-agent SQLite, but + * these orchestration tests must stay off disk. The store keys on + * storePath+sessionKey so successive persists in one run observe the row the + * previous persist committed, mirroring the accessor's read-modify-write and + * letting the lifecycle claim guard prove ownership without real SQLite. + */ +function installPatchSessionEntryStore(): void { + type PatchRow = Record<string, unknown>; + const rows = new Map<string, PatchRow>(); + patchSessionEntryMock.mockImplementation( + async ( + scope: { storePath?: string; sessionKey: string }, + update: ( + entry: PatchRow, + context: { existingEntry: PatchRow | undefined }, + ) => PatchRow | null, + options: { fallbackEntry?: PatchRow } = {}, + ) => { + const key = `${scope.storePath ?? ""}\\0${scope.sessionKey}`; + const existingEntry = + rows.get(key) ?? + (usesRealAccessorStore(scope.storePath) + ? actualLoadSessionEntry(scope as never) + : loadSessionEntryMock(scope.storePath, scope.sessionKey)); + const writeBase = existingEntry ?? options.fallbackEntry; + if (!writeBase) { + return null; + } + const next = update(structuredClone(writeBase), { + existingEntry: existingEntry ? structuredClone(existingEntry) : undefined, + }); + if (!next) { + return structuredClone(writeBase); + } + const committed = structuredClone(next); + rows.set(key, committed); + if (usesRealAccessorStore(scope.storePath)) { + await actualReplaceSessionEntry(scope as never, committed as never); + } + return committed; + }, + ); +} + export function resetRunCronIsolatedAgentTurnHarness(): void { vi.clearAllMocks(); resetRunConfigMocks(); diff --git a/src/cron/isolated-agent/run.ts b/src/cron/isolated-agent/run.ts index 98f1eac12d4a..25e140463557 100644 --- a/src/cron/isolated-agent/run.ts +++ b/src/cron/isolated-agent/run.ts @@ -14,6 +14,7 @@ import type { CliDeps } from "../../cli/outbound-send-deps.js"; import { resolveAgentModelPrimaryValue } from "../../config/model-input.js"; import type { SessionEntry } from "../../config/sessions.js"; import { resolveSessionWorkStartError } from "../../config/sessions/lifecycle.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import type { AgentDefaultsConfig } from "../../config/types.agent-defaults.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { @@ -37,6 +38,7 @@ import { import { createDiagnosticMessageLifecycle } from "../../logging/message-lifecycle.js"; import { isCommandLaneTaskTimeoutError } from "../../process/command-queue.js"; import { CommandLane } from "../../process/lanes.js"; +import { isCronSessionKey } from "../../routing/session-key.js"; import { AGENT_HARNESS_SESSION_ID_LOCKED_MESSAGE, AGENT_HARNESS_SESSION_KEY_RESERVED_MESSAGE, @@ -64,7 +66,7 @@ import { toolsAllowRequestsWebSearch, } from "../run-diagnostics.js"; import { resolveCronAbortReasonText } from "../service/execution-errors.js"; -import { resolveCronDeliverySessionKey } from "../session-target.js"; +import { isDetachedCronSessionTarget, resolveCronDeliverySessionKey } from "../session-target.js"; import type { CronAgentExecutionPhaseUpdate, CronAgentExecutionStarted, @@ -119,7 +121,6 @@ import { resolveHookExternalContentSource, isThinkingLevelSupported, resolveSupportedThinkingLevel, - resolveSessionTranscriptPath, resolveEffectiveAgentRuntime, resolveSessionRuntimeOverrideForProvider, resolveThinkingDefault, @@ -131,8 +132,8 @@ import { resolveCronAgentSessionKey } from "./session-key.js"; import { loadCronSessionEntryLatest, resolveCronSession } from "./session.js"; import { resolveCronSourceDeliveryPlan } from "./source-delivery-fallback.js"; -const sessionStoreRuntimeLoader = createLazyImportLoader( - () => import("../../config/sessions/store.runtime.js"), +const sessionAccessorRuntimeLoader = createLazyImportLoader( + () => import("../../config/sessions/session-accessor.js"), ); const cronExecutorRuntimeLoader = createLazyImportLoader(() => import("./run-executor.runtime.js")); const cronExternalContentRuntimeLoader = createLazyImportLoader( @@ -157,8 +158,8 @@ const codexNativeWebSearchLoader = createLazyImportLoader( ); const webSearchRuntimeLoader = createLazyImportLoader(() => import("../../web-search/runtime.js")); -async function loadSessionStoreRuntime() { - return await sessionStoreRuntimeLoader.load(); +async function loadSessionAccessorRuntime() { + return await sessionAccessorRuntimeLoader.load(); } async function loadCronExecutorRuntime() { @@ -542,6 +543,7 @@ type PreparedCronRunContext = { runSessionId: string; currentRunSessionId: () => string; runSessionKey: string; + usesDetachedRunSession: boolean; workspaceDir: string; commandBody: string; cronSession: MutableCronSession; @@ -621,34 +623,30 @@ async function prepareCronRunContext(params: { }; const baseSessionKey = (input.sessionKey?.trim() || `cron:${input.job.id}`).trim(); + const currentBoundSourceKey = + input.job.sessionTarget === "current" ? input.job.sessionKey?.trim() : undefined; + const usesDetachedRunSession = + isDetachedCronSessionTarget(input.job.sessionTarget) || Boolean(currentBoundSourceKey); + const baseSessionKeyIsCron = + baseSessionKey.startsWith("cron:") || isCronSessionKey(baseSessionKey); + const cronExecutionSessionKey = + usesDetachedRunSession && !baseSessionKeyIsCron ? `cron:${input.job.id}` : baseSessionKey; const agentSessionKey = resolveCronAgentSessionKey({ - sessionKey: baseSessionKey, + sessionKey: cronExecutionSessionKey, agentId, mainKey: input.cfg.session?.mainKey, cfg: input.cfg, }); - const cronSession = resolveCronSession({ - cfg: input.cfg, - sessionKey: agentSessionKey, + const resolvedBaseSessionKey = resolveCronAgentSessionKey({ + sessionKey: currentBoundSourceKey ?? baseSessionKey, agentId, - nowMs: Date.now(), - forceNew: input.job.sessionTarget === "isolated", + mainKey: input.cfg.session?.mainKey, + cfg: input.cfg, }); - const reservedKey = isAgentHarnessSessionKey(agentSessionKey); - if (cronSession.initialSessionEntry?.modelSelectionLocked === true) { - // The generic detached executor cannot preserve a harness-owned runtime - // lock. Reject before model selection can route the row through that path. - throw new Error( - reservedKey - ? AGENT_HARNESS_SESSION_KEY_RESERVED_MESSAGE - : AGENT_HARNESS_SESSION_ID_LOCKED_MESSAGE, - ); - } - if (reservedKey && !cronSession.initialSessionEntry) { - // `harness:*` was historically a valid public key. Existing unlocked rows - // stay ordinary, while missing keys remain reserved for trusted creation. - throw new Error(AGENT_HARNESS_SESSION_KEY_RESERVED_MESSAGE); - } + const sourceSessionKey = + currentBoundSourceKey && resolvedBaseSessionKey !== agentSessionKey + ? resolvedBaseSessionKey + : undefined; const payloadHookExternalContentSource = input.job.payload.kind === "agentTurn" ? input.job.payload.externalContentSource : undefined; const hookExternalContentSource = @@ -672,27 +670,63 @@ async function prepareCronRunContext(params: { const isGmailHook = hookExternalContentSource === "gmail"; const now = Date.now(); + const cronSession = resolveCronSession({ + cfg: input.cfg, + sessionKey: agentSessionKey, + sourceSessionKey, + agentId, + nowMs: now, + forceNew: usesDetachedRunSession, + }); + const reservedKey = isAgentHarnessSessionKey(agentSessionKey); + if (cronSession.initialSessionEntry?.modelSelectionLocked === true) { + throw new Error( + reservedKey + ? AGENT_HARNESS_SESSION_KEY_RESERVED_MESSAGE + : AGENT_HARNESS_SESSION_ID_LOCKED_MESSAGE, + ); + } + if (reservedKey && !cronSession.initialSessionEntry) { + throw new Error(AGENT_HARNESS_SESSION_KEY_RESERVED_MESSAGE); + } const runSessionId = cronSession.sessionEntry.sessionId; const currentRunSessionId = () => cronSession.sessionEntry.sessionId ?? runSessionId; if (!cronSession.sessionEntry.sessionFile?.trim()) { - cronSession.sessionEntry.sessionFile = resolveSessionTranscriptPath(runSessionId, agentId); + cronSession.sessionEntry.sessionFile = formatSqliteSessionFileMarker({ + agentId, + sessionId: runSessionId, + storePath: cronSession.storePath, + }); } - const runSessionKey = baseSessionKey.startsWith("cron:") - ? `${agentSessionKey}:run:${runSessionId}` - : agentSessionKey; - const updateCronSessionStore = async ( - storePath: string, - update: (store: Record<string, SessionEntry>) => void, - options?: { activeSessionKey?: string; requireWriteSuccess?: boolean }, - ) => { - const { updateSessionStore } = await loadSessionStoreRuntime(); - await updateSessionStore(storePath, update, options); + const runSessionKey = + usesDetachedRunSession || baseSessionKey.startsWith("cron:") + ? `${agentSessionKey}:run:${runSessionId}` + : agentSessionKey; + const persistCronSessionRow = async ({ + storePath, + sessionKey, + fallbackEntry, + update, + }: { + storePath: string; + sessionKey: string; + fallbackEntry: SessionEntry; + update: (entry: SessionEntry | undefined) => SessionEntry; + }) => { + const { patchSessionEntry } = await loadSessionAccessorRuntime(); + // Guarded replace: the updater sees the freshest persisted row (or + // undefined pre-creation) so cron lifecycle claims reject stale owners. + await patchSessionEntry( + { storePath, sessionKey, agentId }, + (_entry, context) => update(context.existingEntry), + { fallbackEntry, replaceEntry: true }, + ); }; const persistSessionEntry = createPersistCronSessionEntry({ isFastTestEnv: params.isFastTestEnv, cronSession, agentSessionKey, - updateSessionStore: updateCronSessionStore, + persistSessionEntry: persistCronSessionRow, }); const withRunSession: WithRunSession = (result) => ({ ...result, @@ -1050,7 +1084,7 @@ async function prepareCronRunContext(params: { sourceReplyDeliveryMode: sourceDelivery.sourceReplyDeliveryMode, requireExplicitMessageTarget: sourceDelivery.messageTool.requireExplicitTarget, }, - updateSessionStore: updateCronSessionStore, + persistSessionEntry: persistCronSessionRow, }) : undefined; await runContinuationSession?.initialize(); @@ -1067,6 +1101,7 @@ async function prepareCronRunContext(params: { runSessionId, currentRunSessionId, runSessionKey, + usesDetachedRunSession, workspaceDir, commandBody, cronSession, @@ -1650,6 +1685,7 @@ export async function runCronIsolatedAgentTurn(params: { agentDir: prepared.context.agentDir, agentSessionKey: prepared.context.agentSessionKey, runSessionKey: prepared.context.runSessionKey, + usesDetachedRunSession: prepared.context.usesDetachedRunSession, workspaceDir: prepared.context.workspaceDir, lane: params.lane, resolvedDelivery: { diff --git a/src/cron/isolated-agent/session.latest-read.test.ts b/src/cron/isolated-agent/session.latest-read.test.ts index f3c2b45f36ed..10337afd23c9 100644 --- a/src/cron/isolated-agent/session.latest-read.test.ts +++ b/src/cron/isolated-agent/session.latest-read.test.ts @@ -1,59 +1,53 @@ // Behavior tests for the cron lifecycle-guard session read. -// loadCronSessionEntryLatest must observe the latest persisted row even when a -// cached in-process store snapshot is still considered current, because cron -// admission guards fence on it (see run.ts assertAllowed). +// loadCronSessionEntryLatest must observe the latest persisted row through the +// SQLite accessor, because cron admission guards fence on it (see run.ts +// assertAllowed). Seeding and re-seeding go through replaceSessionEntry so the +// read is proven against the same canonical store the cron persist path writes. import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, expect, it, vi } from "vitest"; -import { clearSessionStoreCacheForTest, loadSessionStore } from "../../config/sessions/store.js"; +import { afterEach, expect, it } from "vitest"; +import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; import { loadCronSessionEntryLatest } from "./session.js"; const SESSION_KEY = "agent:main:cron:job-1"; const tempDirs: string[] = []; -function createStoreFile(sessionId: string): string { +function createStorePath(): string { const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-cron-latest-"))); tempDirs.push(dir); - const storePath = path.join(dir, "sessions.json"); - fs.writeFileSync(storePath, serializeStore(sessionId), "utf-8"); - return storePath; -} - -// Both serializations must have identical byte length so an out-of-band rewrite -// with a restored mtime looks unchanged to the mtime/size-validated cache. -function serializeStore(sessionId: string): string { - return JSON.stringify({ [SESSION_KEY]: { sessionId, updatedAt: 1000 } }); + return path.join(dir, "sessions.json"); } afterEach(() => { - vi.unstubAllEnvs(); - clearSessionStoreCacheForTest(); + closeOpenClawAgentDatabasesForTest(); for (const dir of tempDirs.splice(0)) { fs.rmSync(dir, { recursive: true, force: true }); } }); -it("reads the latest persisted row past a still-current cached store snapshot", () => { - vi.stubEnv("OPENCLAW_SESSION_CACHE_TTL_MS", "45000"); - clearSessionStoreCacheForTest(); - const storePath = createStoreFile("sess-one"); - // A whole-second mtime round-trips exactly through utimes/stat, so the - // rewritten file below validates as unchanged against the cached snapshot. - const pinnedTime = new Date(1_700_000_000_000); - fs.utimesSync(storePath, pinnedTime, pinnedTime); +it("reads the latest persisted row after it is replaced", async () => { + const storePath = createStorePath(); + await replaceSessionEntry( + { sessionKey: SESSION_KEY, storePath }, + { sessionId: "sess-one", updatedAt: 1000 }, + ); + expect(loadSessionEntry({ sessionKey: SESSION_KEY, storePath })?.sessionId).toBe("sess-one"); - // Warm the in-process cache, then rewrite the row out-of-band with the same - // byte length and mtime so the cached snapshot still validates as current. - expect(loadSessionStore(storePath)[SESSION_KEY]?.sessionId).toBe("sess-one"); - fs.writeFileSync(storePath, serializeStore("sess-two"), "utf-8"); - fs.utimesSync(storePath, pinnedTime, pinnedTime); - - expect(loadSessionStore(storePath)[SESSION_KEY]?.sessionId).toBe("sess-one"); + // Re-seed the row out-of-band; the lifecycle-guard read must observe it. + await replaceSessionEntry( + { sessionKey: SESSION_KEY, storePath }, + { sessionId: "sess-two", updatedAt: 2000 }, + ); expect(loadCronSessionEntryLatest(storePath, SESSION_KEY)?.sessionId).toBe("sess-two"); }); -it("returns undefined for a session key without a persisted row", () => { - const storePath = createStoreFile("sess-one"); +it("returns undefined for a session key without a persisted row", async () => { + const storePath = createStorePath(); + await replaceSessionEntry( + { sessionKey: SESSION_KEY, storePath }, + { sessionId: "sess-one", updatedAt: 1000 }, + ); expect(loadCronSessionEntryLatest(storePath, "agent:main:cron:missing")).toBeUndefined(); }); diff --git a/src/cron/isolated-agent/session.test.ts b/src/cron/isolated-agent/session.test.ts index abb7084b02ce..bb5a202db210 100644 --- a/src/cron/isolated-agent/session.test.ts +++ b/src/cron/isolated-agent/session.test.ts @@ -1,10 +1,7 @@ // Isolated agent session tests cover session creation and metadata for cron runs. import { beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../config/config.js"; - -vi.mock("../../config/sessions/store-load.js", () => ({ - loadSessionStore: vi.fn(), -})); +import type { SessionEntry } from "../../config/sessions/types.js"; vi.mock("../../config/sessions/paths.js", () => ({ resolveStorePath: vi.fn().mockReturnValue("/tmp/test-store.json"), @@ -28,34 +25,35 @@ vi.mock("../../agents/bootstrap-cache.js", () => ({ import { clearBootstrapSnapshot } from "../../agents/bootstrap-cache.js"; import { evaluateSessionFreshness } from "../../config/sessions/reset-policy.js"; -import { loadSessionStore } from "../../config/sessions/store-load.js"; import { resolveCronSession } from "./session.js"; const NOW_MS = 1_737_600_000_000; -type SessionStore = ReturnType<typeof loadSessionStore>; -type SessionStoreEntry = SessionStore[string]; -type MockSessionStoreEntry = Partial<SessionStoreEntry>; +type SessionStore = Record<string, SessionEntry>; +type MockSessionStoreEntry = Partial<SessionEntry>; function resolveWithStoredEntry(params?: { sessionKey?: string; + sourceSessionKey?: string; entry?: MockSessionStoreEntry; forceNew?: boolean; fresh?: boolean; }) { const sessionKey = params?.sessionKey ?? "webhook:stable-key"; + const sourceSessionKey = params?.sourceSessionKey; const store: SessionStore = params?.entry - ? ({ [sessionKey]: params.entry as SessionStoreEntry } as SessionStore) + ? ({ [sourceSessionKey ?? sessionKey]: params.entry as SessionEntry } as SessionStore) : {}; - vi.mocked(loadSessionStore).mockReturnValue(store); vi.mocked(evaluateSessionFreshness).mockReturnValue({ fresh: params?.fresh ?? true }); return resolveCronSession({ cfg: {} as OpenClawConfig, sessionKey, + sourceSessionKey, agentId: "main", nowMs: NOW_MS, forceNew: params?.forceNew, + store, }); } diff --git a/src/cron/isolated-agent/session.ts b/src/cron/isolated-agent/session.ts index 781a2f53682c..1b357d859b64 100644 --- a/src/cron/isolated-agent/session.ts +++ b/src/cron/isolated-agent/session.ts @@ -13,8 +13,7 @@ import { resolveSessionResetPolicy, type SessionFreshness, } from "../../config/sessions/reset-policy.js"; -import { loadSessionEntry } from "../../config/sessions/session-accessor.js"; -import { loadSessionStore } from "../../config/sessions/store-load.js"; +import { listSessionEntries, loadSessionEntry } from "../../config/sessions/session-accessor.js"; import type { SessionEntry } from "../../config/sessions/types.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; @@ -136,6 +135,7 @@ export function loadCronSessionEntryLatest( export function resolveCronSession(params: { cfg: OpenClawConfig; sessionKey: string; + sourceSessionKey?: string; nowMs: number; agentId: string; forceNew?: boolean; @@ -145,9 +145,18 @@ export function resolveCronSession(params: { const storePath = resolveStorePath(sessionCfg?.store, { agentId: params.agentId, }); - const store = params.store ?? loadSessionStore(storePath); - const entry = store[params.sessionKey]; - const archivedSessionError = resolveSessionWorkStartError(params.sessionKey, entry); + const store = + params.store ?? + Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ); + const sourceSessionKey = params.sourceSessionKey?.trim(); + const sourceSessionDiffers = Boolean(sourceSessionKey && sourceSessionKey !== params.sessionKey); + const targetEntry = store[params.sessionKey]; + const entry = store[sourceSessionKey || params.sessionKey]; + // Guard the run's target row: archived sessions stay read-only even when a + // differing source session seeds the carried preferences. + const archivedSessionError = resolveSessionWorkStartError(params.sessionKey, targetEntry); if (archivedSessionError) { throw new Error(archivedSessionError); } @@ -192,7 +201,7 @@ export function resolveCronSession(params: { systemSent = false; } - const previousSessionId = isNewSession ? entry?.sessionId : undefined; + const previousSessionId = isNewSession && !sourceSessionDiffers ? entry?.sessionId : undefined; clearBootstrapSnapshotOnSessionRollover({ sessionKey: params.sessionKey, previousSessionId, @@ -231,6 +240,6 @@ export function resolveCronSession(params: { systemSent, isNewSession, previousSessionId, - initialSessionEntry: entry, + initialSessionEntry: targetEntry, }; } diff --git a/src/cron/isolated-agent/subagent-followup.test.ts b/src/cron/isolated-agent/subagent-followup.test.ts index 8b0bf2da59d9..fa1e598be170 100644 --- a/src/cron/isolated-agent/subagent-followup.test.ts +++ b/src/cron/isolated-agent/subagent-followup.test.ts @@ -47,7 +47,7 @@ function createDescendantRun(params?: { cleanup?: "keep" | "delete"; endedAt?: number; resultText?: string | null; - executionTranscriptFile?: string; + hasInternalTranscript?: boolean; }) { return { runId: params?.runId ?? "run-1", @@ -61,11 +61,16 @@ function createDescendantRun(params?: { ...(params?.resultText === undefined ? {} : { completion: { required: true, resultText: params.resultText } }), - ...(params?.executionTranscriptFile + ...(params?.hasInternalTranscript ? { execution: { status: "terminal" as const, - transcriptFile: params.executionTranscriptFile, + transcriptTarget: { + agentId: "main", + sessionId: "internal-run", + sessionKey: "agent:main:internal-session-effects:run", + storePath: "/tmp/test-store", + }, }, } : {}), @@ -164,7 +169,7 @@ describe("readDescendantSubagentFallbackReply", () => { vi.mocked(listDescendantRunsForRequester).mockReturnValue([ createDescendantRun({ resultText: "fresh recovered output", - executionTranscriptFile: "/tmp/openclaw-internal-run.jsonl", + hasInternalTranscript: true, }), ]); vi.mocked(readLatestAssistantReply).mockResolvedValue("stale visible transcript"); @@ -179,7 +184,7 @@ describe("readDescendantSubagentFallbackReply", () => { vi.mocked(listDescendantRunsForRequester).mockReturnValue([ createDescendantRun({ resultText: null, - executionTranscriptFile: "/tmp/openclaw-empty-internal-run.jsonl", + hasInternalTranscript: true, }), ]); vi.mocked(readLatestAssistantReply).mockClear(); diff --git a/src/cron/isolated-agent/subagent-followup.ts b/src/cron/isolated-agent/subagent-followup.ts index a3a20b38d1d4..a82cefec70f3 100644 --- a/src/cron/isolated-agent/subagent-followup.ts +++ b/src/cron/isolated-agent/subagent-followup.ts @@ -54,7 +54,7 @@ export async function readDescendantSubagentFallbackReply(params: { typeof frozenResultText === "string" && frozenResultText.trim() ? frozenResultText.trim() : undefined; - const usesInternalTranscript = typeof entry.execution?.transcriptFile === "string"; + const usesInternalTranscript = entry.execution?.transcriptTarget !== undefined; let reply = usesInternalTranscript ? frozenReply : undefined; if (!reply && !usesInternalTranscript) { reply = (await readLatestAssistantReply({ sessionKey: entry.childSessionKey }))?.trim(); diff --git a/src/cron/normalize.test.ts b/src/cron/normalize.test.ts index acf385744c06..e1fb406f2bc4 100644 --- a/src/cron/normalize.test.ts +++ b/src/cron/normalize.test.ts @@ -158,6 +158,23 @@ describe("normalizeCronJobCreate", () => { expect("sessionKey" in cleared).toBe(false); }); + it("stores the source session key for case-insensitive current targets", () => { + const normalized = normalizeCronJobCreate( + { + name: "current target", + enabled: true, + schedule: { kind: "cron", expr: "* * * * *" }, + sessionTarget: " Current ", + wakeMode: "now", + payload: { kind: "agentTurn", message: "hi" }, + }, + { sessionContext: { sessionKey: " agent:main:telegram:direct:42 " } }, + ) as unknown as Record<string, unknown>; + + expect(normalized.sessionTarget).toBe("current"); + expect(normalized.sessionKey).toBe("agent:main:telegram:direct:42"); + }); + it("canonicalizes delivery.channel casing", () => { const normalized = normalizeIsolatedAgentTurnCreateJob({ name: "delivery channel casing", @@ -743,7 +760,7 @@ describe("normalizeCronJobCreate", () => { expect(delivery.to).toBe("123"); }); - it("resolves current sessionTarget to a persistent session when context is available", () => { + it("stores current sessionTarget source context when context is available", () => { const normalized = normalizeCronJobCreate( { name: "current-session", @@ -754,7 +771,8 @@ describe("normalizeCronJobCreate", () => { { sessionContext: { sessionKey: "agent:main:discord:group:ops" } }, ) as unknown as Record<string, unknown>; - expect(normalized.sessionTarget).toBe("session:agent:main:discord:group:ops"); + expect(normalized.sessionTarget).toBe("current"); + expect(normalized.sessionKey).toBe("agent:main:discord:group:ops"); }); it("falls back current sessionTarget to isolated without context", () => { diff --git a/src/cron/normalize.ts b/src/cron/normalize.ts index 388675f16932..734c97f2863f 100644 --- a/src/cron/normalize.ts +++ b/src/cron/normalize.ts @@ -684,12 +684,24 @@ export function normalizeCronJobInput( } } + const normalizedSessionTarget = + typeof next.sessionTarget === "string" ? next.sessionTarget : undefined; + const resolvedCurrentSessionKey = + options.sessionContext?.sessionKey ?? + (typeof next.sessionKey === "string" ? next.sessionKey : undefined); const resolvedSessionTarget = resolveCronCurrentSessionTarget({ - sessionTarget: typeof next.sessionTarget === "string" ? next.sessionTarget : undefined, - sessionKey: options.sessionContext?.sessionKey, + sessionTarget: normalizedSessionTarget, + sessionKey: resolvedCurrentSessionKey, }); if (resolvedSessionTarget !== undefined) { next.sessionTarget = resolvedSessionTarget; + if ( + next.sessionTarget !== "isolated" && + normalizedSessionTarget === "current" && + resolvedCurrentSessionKey?.trim() + ) { + next.sessionKey = assertSafeCronSessionTargetId(resolvedCurrentSessionKey); + } } else { delete next.sessionTarget; } diff --git a/src/cron/service.session-reaper-in-finally.test.ts b/src/cron/service.session-reaper-in-finally.test.ts index 3074adf619d2..37e7c7c727c0 100644 --- a/src/cron/service.session-reaper-in-finally.test.ts +++ b/src/cron/service.session-reaper-in-finally.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { listSessionEntries, replaceSessionEntry } from "../config/sessions/session-accessor.js"; import { createNoopLogger, createCronStoreHarness, @@ -132,16 +133,12 @@ describe("CronService - session reaper runs in finally block (#31946)", () => { await fs.writeFile(store.storePath, "{invalid-json", "utf-8"); // Seed an expired cron-run session entry that should be pruned by the reaper. - await fs.mkdir(path.dirname(sessionStorePath), { recursive: true }); - await fs.writeFile( - sessionStorePath, - JSON.stringify({ - "agent:agent-default:cron:failing-job:run:stale": { - sessionId: "session-stale", - updatedAt: now - 3 * 24 * 3_600_000, - }, - }), - "utf-8", + await replaceSessionEntry( + { storePath: sessionStorePath, sessionKey: "agent:agent-default:cron:failing-job:run:stale" }, + { + sessionId: "session-stale", + updatedAt: now - 3 * 24 * 3_600_000, + }, ); const state = createCronServiceState({ @@ -158,10 +155,7 @@ describe("CronService - session reaper runs in finally block (#31946)", () => { await withCronServiceStateForTest(state, async () => { await expect(onTimer(state)).resolves.toBeUndefined(); - const updatedSessionStore = JSON.parse( - await fs.readFile(sessionStorePath, "utf-8"), - ) as Record<string, unknown>; - expect(updatedSessionStore).toStrictEqual({}); + expect(listSessionEntries({ storePath: sessionStorePath })).toStrictEqual([]); expect(state.running).toBe(false); }); }); diff --git a/src/cron/service/task-runs.ts b/src/cron/service/task-runs.ts index ebaa07187828..3662438795b6 100644 --- a/src/cron/service/task-runs.ts +++ b/src/cron/service/task-runs.ts @@ -43,6 +43,12 @@ function resolveCronTaskChildSessionKey(params: { if (params.job.sessionTarget === "main") { return resolveMainSessionCronRunSessionKey(params.job, params.startedAt); } + if (params.job.sessionTarget === "current") { + return resolveCronAgentSessionKey({ + sessionKey: `cron:${params.job.id}`, + agentId: params.job.agentId ?? params.state.deps.defaultAgentId ?? DEFAULT_AGENT_ID, + }); + } const explicitSessionKey = params.job.sessionKey?.trim(); if (explicitSessionKey) { // Explicit session bindings must win over generated cron session keys so diff --git a/src/cron/service/timer.test.ts b/src/cron/service/timer.test.ts index 4d9a7f69ff78..dd0ae596a097 100644 --- a/src/cron/service/timer.test.ts +++ b/src/cron/service/timer.test.ts @@ -1,7 +1,7 @@ // Cron service timer tests cover timer scheduling, cancellation, and wakeups. -import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { upsertSessionEntry } from "../../config/sessions/session-accessor.js"; import { setupCronServiceSuite, writeCronStoreSnapshot } from "../../cron/service.test-harness.js"; import { createCronServiceState } from "../../cron/service/state.js"; import { executeJobCore, onTimer } from "../../cron/service/timer.js"; @@ -81,16 +81,15 @@ describe("cron service timer seam coverage", () => { }; const cronRunSessionKey = `agent:main-pr-router:cron:main-heartbeat-job:run:${now}`; const sessionStorePath = path.join(path.dirname(path.dirname(storePath)), "sessions.json"); - await fs.writeFile( - sessionStorePath, - JSON.stringify({ - "agent:main-pr-router:main": { - lastChannel: "discord", - lastTo: "channel-1", - lastAccountId: "default", - }, - }), - "utf8", + await upsertSessionEntry( + { storePath: sessionStorePath, sessionKey: "agent:main-pr-router:main" }, + { + sessionId: "main-pr-router-session", + updatedAt: now, + lastChannel: "discord", + lastTo: "channel-1", + lastAccountId: "default", + }, ); const state = createCronServiceState({ @@ -273,6 +272,46 @@ describe("cron service timer seam coverage", () => { expect(task.terminalSummary).toBe("done"); }); + it("records current-bound cron task runs against the backing cron session", async () => { + const { storePath } = await makeStorePath(); + const now = Date.parse("2026-03-23T12:00:00.000Z"); + const runIsolatedAgentJob = vi.fn(async () => ({ + status: "ok" as const, + summary: "done", + sessionKey: "agent:finn:cron:isolated-agent-job:run:run-1", + })); + + await writeCronStoreSnapshot({ + storePath, + jobs: [ + { + ...createDueIsolatedAgentJob({ now }), + sessionTarget: "current", + sessionKey: "agent:finn:telegram:direct:42", + }, + ], + }); + + const state = createCronServiceState({ + storePath, + cronEnabled: true, + log: logger, + nowMs: () => now, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob, + }); + + await onTimer(state); + + const task = findTaskByRunId(`cron:isolated-agent-job:${now}`); + if (!task) { + throw new Error("expected current-bound cron task ledger record"); + } + expect(task.childSessionKey).toBe("agent:finn:cron:isolated-agent-job"); + expect(task.status).toBe("succeeded"); + }); + it("seeds active scheduled cron task progress for status surfaces", async () => { const { storePath } = await makeStorePath(); const now = Date.parse("2026-03-23T12:00:00.000Z"); diff --git a/src/cron/session-reaper.test.ts b/src/cron/session-reaper.test.ts index a725cd861aae..648efa5126e8 100644 --- a/src/cron/session-reaper.test.ts +++ b/src/cron/session-reaper.test.ts @@ -3,7 +3,12 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { runExclusiveSessionStoreWrite } from "../config/sessions/store-writer.js"; +import { + listSessionEntries, + patchSessionEntry, + replaceSessionEntry, +} from "../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import { isCronRunSessionKey } from "../sessions/session-key-utils.js"; import { beginSessionWorkAdmission } from "../sessions/session-lifecycle-admission.js"; import { createDeferred } from "../test-utils/deferred.js"; @@ -25,6 +30,21 @@ function createTestLogger(): Logger { }; } +async function seedSessionEntries( + storePath: string, + entries: Record<string, SessionEntry>, +): Promise<void> { + for (const [sessionKey, entry] of Object.entries(entries)) { + await replaceSessionEntry({ storePath, sessionKey }, entry); + } +} + +function readSessionEntries(storePath: string): Record<string, SessionEntry> { + return Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ); +} + describe("resolveRetentionMs", () => { it("returns 24h default when no config", () => { expect(resolveRetentionMs()).toBe(24 * 3_600_000); @@ -88,7 +108,7 @@ describe("sweepCronRunSessions", () => { it("prunes expired cron run sessions", async () => { const now = Date.now(); - const store: Record<string, { sessionId: string; updatedAt: number }> = { + const store: Record<string, SessionEntry> = { "agent:main:cron:job1": { sessionId: "base-session", updatedAt: now, @@ -114,7 +134,7 @@ describe("sweepCronRunSessions", () => { updatedAt: now - 100 * 3_600_000, // old but not a cron run }, }; - fs.writeFileSync(storePath, JSON.stringify(store)); + await seedSessionEntries(storePath, store); const result = await sweepCronRunSessions({ sessionStorePath: storePath, @@ -126,38 +146,42 @@ describe("sweepCronRunSessions", () => { expect(result.swept).toBe(true); expect(result.pruned).toBe(2); - const updated = JSON.parse(fs.readFileSync(storePath, "utf-8")); - expect(updated).toEqual({ - "agent:main:cron:job1": { - sessionId: "base-session", - updatedAt: now, - }, - "agent:main:cron:job1:run:recent-run": { - sessionId: "recent-run", - updatedAt: now - 1 * 3_600_000, - }, - "agent:main:cron:job1:run:recent-run:thread:reply": { - sessionId: "recent-run-thread", - updatedAt: now - 1 * 3_600_000, - }, - "agent:main:telegram:dm:123": { - sessionId: "regular-session", - updatedAt: now - 100 * 3_600_000, - }, + const updated = readSessionEntries(storePath); + expect(Object.keys(updated).toSorted()).toEqual([ + "agent:main:cron:job1", + "agent:main:cron:job1:run:recent-run", + "agent:main:cron:job1:run:recent-run:thread:reply", + "agent:main:telegram:dm:123", + ]); + expect(updated["agent:main:cron:job1"]).toMatchObject({ + sessionId: "base-session", + updatedAt: now, + }); + expect(updated["agent:main:cron:job1:run:recent-run"]).toMatchObject({ + sessionId: "recent-run", + updatedAt: now - 1 * 3_600_000, + }); + expect(updated["agent:main:cron:job1:run:recent-run:thread:reply"]).toMatchObject({ + sessionId: "recent-run-thread", + updatedAt: now - 1 * 3_600_000, + }); + expect(updated["agent:main:telegram:dm:123"]).toMatchObject({ + sessionId: "regular-session", + updatedAt: now - 100 * 3_600_000, }); }); it("preserves expired continuation rows while generated media is pending", async () => { const now = Date.now(); const sessionKey = "agent:main:cron:job1:run:pending-run"; - const store = { + const store: Record<string, SessionEntry> = { [sessionKey]: { sessionId: "pending-run", updatedAt: now - 25 * 3_600_000, cronRunContinuation: { lifecycleRevision: "revision-1", phase: "ready" }, }, }; - fs.writeFileSync(storePath, JSON.stringify(store)); + await seedSessionEntries(storePath, store); taskStatusMocks.hasPendingGeneratedMediaTask.mockReturnValue(true); const result = await sweepCronRunSessions({ @@ -168,27 +192,24 @@ describe("sweepCronRunSessions", () => { }); expect(result.pruned).toBe(0); - expect(JSON.parse(fs.readFileSync(storePath, "utf-8"))).toEqual(store); + expect(readSessionEntries(storePath)).toEqual(store); }); it("preserves an orphaned gateway continuation while generated media is pending", async () => { const now = Date.now(); const sessionKey = "agent:main:cron:job1:run:orphaned-run"; - fs.writeFileSync( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "orphaned-run", - updatedAt: now - 25 * 3_600_000, - cronRunContinuation: { - lifecycleRevision: "revision-1", - phase: "continuing", - ownerRunId: "dead-gateway-run", - basePersisted: false, - }, + await seedSessionEntries(storePath, { + [sessionKey]: { + sessionId: "orphaned-run", + updatedAt: now - 25 * 3_600_000, + cronRunContinuation: { + lifecycleRevision: "revision-1", + phase: "continuing", + ownerRunId: "dead-gateway-run", + basePersisted: false, }, - }), - ); + }, + }); taskStatusMocks.hasPendingGeneratedMediaTask.mockReturnValue(true); const result = await sweepCronRunSessions({ @@ -199,7 +220,7 @@ describe("sweepCronRunSessions", () => { }); expect(result.pruned).toBe(0); - expect(JSON.parse(fs.readFileSync(storePath, "utf-8"))[sessionKey]).toMatchObject({ + expect(readSessionEntries(storePath)[sessionKey]).toMatchObject({ updatedAt: now - 25 * 3_600_000, cronRunContinuation: { lifecycleRevision: "revision-1", @@ -214,28 +235,25 @@ describe("sweepCronRunSessions", () => { const now = Date.now(); const runningKey = "agent:main:cron:job1:run:running-run"; const continuingKey = "agent:main:cron:job1:run:continuing-run"; - fs.writeFileSync( - storePath, - JSON.stringify({ - [runningKey]: { - sessionId: "running-run", - updatedAt: now - 25 * 3_600_000, - cronRunContinuation: { - lifecycleRevision: "revision-1", - phase: "running", - }, + await seedSessionEntries(storePath, { + [runningKey]: { + sessionId: "running-run", + updatedAt: now - 25 * 3_600_000, + cronRunContinuation: { + lifecycleRevision: "revision-1", + phase: "running", }, - [continuingKey]: { - sessionId: "continuing-run", - updatedAt: now - 25 * 3_600_000, - cronRunContinuation: { - lifecycleRevision: "revision-2", - phase: "continuing", - ownerRunId: "gateway-run", - }, + }, + [continuingKey]: { + sessionId: "continuing-run", + updatedAt: now - 25 * 3_600_000, + cronRunContinuation: { + lifecycleRevision: "revision-2", + phase: "continuing", + ownerRunId: "gateway-run", }, - }), - ); + }, + }); const result = await sweepCronRunSessions({ sessionStorePath: storePath, @@ -245,25 +263,26 @@ describe("sweepCronRunSessions", () => { }); expect(result.pruned).toBe(2); - expect(JSON.parse(fs.readFileSync(storePath, "utf-8"))).toEqual({}); + expect(readSessionEntries(storePath)).toEqual({}); }); it("preserves an expired run when work is admitted before writer-owned removal", async () => { const now = Date.now(); const sessionKey = "agent:main:cron:job1:run:active-run"; - const store = { + const store: Record<string, SessionEntry> = { [sessionKey]: { sessionId: "active-run", updatedAt: now - 25 * 3_600_000, }, }; - fs.writeFileSync(storePath, JSON.stringify(store)); + await seedSessionEntries(storePath, store); const writerStarted = createDeferred(); const releaseWriter = createDeferred(); const firstValidation = createDeferred(); - const writer = runExclusiveSessionStoreWrite(storePath, async () => { + const writer = patchSessionEntry({ storePath, sessionKey }, async () => { writerStarted.resolve(); await releaseWriter.promise; + return {}; }); await writerStarted.promise; @@ -282,87 +301,33 @@ describe("sweepCronRunSessions", () => { }); await firstValidation.promise; + let admission: Awaited<ReturnType<typeof beginSessionWorkAdmission>> | undefined; try { releaseWriter.resolve(); const result = await sweep; - const admission = await admissionPromise; + admission = await admissionPromise; expect(result.pruned).toBe(0); - expect(JSON.parse(fs.readFileSync(storePath, "utf-8"))).toEqual(store); - admission.release(); + expect(readSessionEntries(storePath)[sessionKey]).toMatchObject({ + sessionId: "active-run", + updatedAt: expect.any(Number), + }); } finally { + admission?.release(); releaseWriter.resolve(); await Promise.allSettled([writer, sweep, admissionPromise]); } }); - it("archives transcript files for pruned run sessions that are no longer referenced", async () => { - const now = Date.now(); - const runSessionId = "old-run"; - const runTranscript = path.join(tmpDir, `${runSessionId}.jsonl`); - fs.writeFileSync(runTranscript, '{"type":"session"}\n'); - const store: Record<string, { sessionId: string; updatedAt: number }> = { - "agent:main:cron:job1:run:old-run": { - sessionId: runSessionId, - updatedAt: now - 25 * 3_600_000, - }, - }; - fs.writeFileSync(storePath, JSON.stringify(store)); - - const result = await sweepCronRunSessions({ - sessionStorePath: storePath, - nowMs: now, - log, - force: true, - }); - - expect(result.pruned).toBe(1); - expect(fs.existsSync(runTranscript)).toBe(false); - const files = fs.readdirSync(tmpDir); - const archivedRunTranscripts = files.filter((name) => - name.startsWith(`${runSessionId}.jsonl.deleted.`), - ); - expect(archivedRunTranscripts.length).toBeGreaterThan(0); - }); - - it("does not archive external transcript paths for pruned runs", async () => { - const now = Date.now(); - const externalDir = fs.mkdtempSync(path.join(os.tmpdir(), "cron-reaper-external-")); - const externalTranscript = path.join(externalDir, "outside.jsonl"); - fs.writeFileSync(externalTranscript, '{"type":"session"}\n'); - const store: Record<string, { sessionId: string; sessionFile?: string; updatedAt: number }> = { - "agent:main:cron:job1:run:old-run": { - sessionId: "old-run", - sessionFile: externalTranscript, - updatedAt: now - 25 * 3_600_000, - }, - }; - fs.writeFileSync(storePath, JSON.stringify(store)); - - try { - const result = await sweepCronRunSessions({ - sessionStorePath: storePath, - nowMs: now, - log, - force: true, - }); - - expect(result.pruned).toBe(1); - expect(fs.existsSync(externalTranscript)).toBe(true); - } finally { - fs.rmSync(externalDir, { recursive: true, force: true }); - } - }); - it("respects custom retention", async () => { const now = Date.now(); - const store: Record<string, { sessionId: string; updatedAt: number }> = { + const store: Record<string, SessionEntry> = { "agent:main:cron:job1:run:run1": { sessionId: "run1", updatedAt: now - 2 * 3_600_000, // 2h ago }, }; - fs.writeFileSync(storePath, JSON.stringify(store)); + await seedSessionEntries(storePath, store); const result = await sweepCronRunSessions({ cronConfig: { sessionRetention: "1h" }, @@ -377,13 +342,13 @@ describe("sweepCronRunSessions", () => { it("does nothing when pruning is disabled", async () => { const now = Date.now(); - const store: Record<string, { sessionId: string; updatedAt: number }> = { + const store: Record<string, SessionEntry> = { "agent:main:cron:job1:run:run1": { sessionId: "run1", updatedAt: now - 100 * 3_600_000, }, }; - fs.writeFileSync(storePath, JSON.stringify(store)); + await seedSessionEntries(storePath, store); const result = await sweepCronRunSessions({ cronConfig: { sessionRetention: false }, @@ -399,8 +364,6 @@ describe("sweepCronRunSessions", () => { it("throttles sweeps without force", async () => { const now = Date.now(); - fs.writeFileSync(storePath, JSON.stringify({})); - // First sweep runs const r1 = await sweepCronRunSessions({ sessionStorePath: storePath, @@ -421,8 +384,6 @@ describe("sweepCronRunSessions", () => { it("throttles per store path", async () => { const now = Date.now(); const otherPath = path.join(tmpDir, "sessions-other.json"); - fs.writeFileSync(storePath, JSON.stringify({})); - fs.writeFileSync(otherPath, JSON.stringify({})); const r1 = await sweepCronRunSessions({ sessionStorePath: storePath, diff --git a/src/cron/session-reaper.ts b/src/cron/session-reaper.ts index e950c8fb38d2..6c52a50bed85 100644 --- a/src/cron/session-reaper.ts +++ b/src/cron/session-reaper.ts @@ -5,6 +5,7 @@ import { listSessionEntries, type SessionEntryLifecycleRemoval, } from "../config/sessions/session-accessor.js"; +import { resolveMaintenanceConfig } from "../config/sessions/store-maintenance-runtime.js"; import type { CronConfig } from "../config/types.cron.js"; import { formatErrorMessage } from "../infra/errors.js"; import { isCronRunSessionKey } from "../sessions/session-key-utils.js"; @@ -98,15 +99,24 @@ export async function sweepCronRunSessions(params: { } } if (removals.length > 0) { + // Archive-age cleanup follows the session maintenance retention knob: + // the reaper's cron retention decides which rows die, but archived + // transcript files are conversation history owned by the archive + // retention policy (null = keep until the disk budget evicts). + const archiveRetentionMs = resolveMaintenanceConfig().resetArchiveRetentionMs; const result = await applySessionEntryLifecycleMutation({ storePath, removals, preserveActiveWork: true, restrictArchivedTranscriptsToStoreDir: true, - cleanupArchivedTranscripts: { - rules: [{ reason: "deleted", olderThanMs: retentionMs }], - nowMs: now, - }, + ...(archiveRetentionMs == null + ? {} + : { + cleanupArchivedTranscripts: { + rules: [{ reason: "deleted", olderThanMs: archiveRetentionMs }], + nowMs: now, + }, + }), captureArtifactCleanupError: true, }); pruned = result.removedEntries; diff --git a/src/cron/session-target.test.ts b/src/cron/session-target.test.ts index 623df35cc43f..8fcfa4f40ab4 100644 --- a/src/cron/session-target.test.ts +++ b/src/cron/session-target.test.ts @@ -29,13 +29,13 @@ describe("cron session target helpers", () => { ); }); - it("resolves current targets to the creator session key", () => { + it("preserves current targets when a creator session key exists", () => { expect( resolveCronCurrentSessionTarget({ sessionTarget: "current", sessionKey: " agent:main:dingtalk:group:cid3tmd4xb19xjfk/wogxwy2a== ", }), - ).toBe("session:agent:main:dingtalk:group:cid3tmd4xb19xjfk/wogxwy2a=="); + ).toBe("current"); }); it("falls back current targets to isolated without a creator session key", () => { diff --git a/src/cron/session-target.ts b/src/cron/session-target.ts index 70606ba5736c..14a04661e5bd 100644 --- a/src/cron/session-target.ts +++ b/src/cron/session-target.ts @@ -28,7 +28,12 @@ export function resolveCronSessionTargetSessionKey( return assertSafeCronSessionTargetId(sessionTarget.slice(8)); } -/** Resolves `current` at creation time so scheduled jobs do not depend on future active UI state. */ +/** Returns whether cron executes the job in a detached run session. */ +export function isDetachedCronSessionTarget(sessionTarget?: string | null): boolean { + return sessionTarget === "isolated" || sessionTarget === "current"; +} + +/** Preserves `current` with a creation-time sessionKey so future active UI state is irrelevant. */ export function resolveCronCurrentSessionTarget(params: { sessionTarget?: string | null; sessionKey?: string | null; @@ -37,7 +42,7 @@ export function resolveCronCurrentSessionTarget(params: { return params.sessionTarget ?? undefined; } const sessionKey = params.sessionKey?.trim(); - return sessionKey ? `session:${assertSafeCronSessionTargetId(sessionKey)}` : "isolated"; + return sessionKey ? "current" : "isolated"; } /** Chooses the session key used for cron delivery, preferring explicit persistent targets. */ diff --git a/src/extensionAPI.ts b/src/extensionAPI.ts index ab19e11f85da..ea9ab35fa9e5 100644 --- a/src/extensionAPI.ts +++ b/src/extensionAPI.ts @@ -31,11 +31,4 @@ export { } from "./agents/embedded-agent.js"; export { resolveAgentTimeoutMs } from "./agents/timeout.js"; export { ensureAgentWorkspace } from "./agents/workspace.js"; -export { - resolveStorePath, - loadSessionStore, - saveSessionStore, - updateSessionStore, - updateSessionStoreEntry, - resolveSessionFilePath, -} from "./config/sessions.js"; +export { resolveStorePath, updateSessionStoreEntry } from "./config/sessions.js"; diff --git a/src/flows/doctor-health-contributions.ts b/src/flows/doctor-health-contributions.ts index 84beb5b5b7fd..4cce511732b6 100644 --- a/src/flows/doctor-health-contributions.ts +++ b/src/flows/doctor-health-contributions.ts @@ -635,6 +635,11 @@ async function runDiskSpaceHealth(ctx: DoctorHealthFlowContext): Promise<void> { noteDiskSpace(ctx.cfg); } +async function runDatabaseBloatHealth(ctx: DoctorHealthFlowContext): Promise<void> { + const { noteSqliteDatabaseBloat } = await import("../commands/doctor-db-bloat.js"); + noteSqliteDatabaseBloat(ctx.cfg); +} + async function runStateIntegrityHealth(ctx: DoctorHealthFlowContext): Promise<void> { const { noteStateIntegrity } = await loadDoctorStateIntegrityModule(); await noteStateIntegrity(ctx.cfg, ctx.prompter, ctx.configPath); @@ -1735,6 +1740,11 @@ export function resolveDoctorHealthContributions(): DoctorHealthContribution[] { }, run: runDiskSpaceHealth, }), + createDoctorHealthContribution({ + id: "doctor:db-bloat", + label: "SQLite database size", + run: runDatabaseBloatHealth, + }), createDoctorHealthContribution({ id: "doctor:state-integrity", label: "State integrity", diff --git a/src/gateway/boot.test.ts b/src/gateway/boot.test.ts index 76e9e2d5db74..f95c69b297e6 100644 --- a/src/gateway/boot.test.ts +++ b/src/gateway/boot.test.ts @@ -18,7 +18,12 @@ const { runBootOnce } = await import("./boot.js"); const { resolveAgentIdFromSessionKey, resolveAgentMainSessionKey, resolveMainSessionKey } = await import("../config/sessions/main-session.js"); const { resolveStorePath } = await import("../config/sessions/paths.js"); -const { loadSessionStore, saveSessionStore } = await import("../config/sessions/store.js"); +const { + applySessionEntryLifecycleMutation, + listSessionEntries, + loadSessionEntry, + replaceSessionEntry, +} = await import("../config/sessions/session-accessor.js"); const { stripInternalRuntimeContext } = await import("../agents/internal-runtime-context.js"); const { getBootEchoContextForSession, resetBootEchoContextForTests } = await import("./boot-echo-guard.js"); @@ -46,6 +51,8 @@ describe("runBootOnce", () => { vi.clearAllMocks(); const { storePath } = resolveMainStore(); await fs.rm(storePath, { force: true }); + const removals = listSessionEntries({ storePath }).map(({ sessionKey }) => ({ sessionKey })); + await applySessionEntryLifecycleMutation({ storePath, removals, skipMaintenance: true }); }); const makeDeps = () => ({ @@ -81,12 +88,13 @@ describe("runBootOnce", () => { if (!sessionKey) { throw new Error("expected sessionKey"); } - const current = loadSessionStore(storePath, { skipCache: true }); - current[sessionKey] = { - sessionId: String(opts.sessionId), - updatedAt: Date.now(), - }; - await saveSessionStore(storePath, current); + await replaceSessionEntry( + { storePath, sessionKey }, + { + sessionId: String(opts.sessionId), + updatedAt: Date.now(), + }, + ); }); }; @@ -142,12 +150,15 @@ describe("runBootOnce", () => { sessionKey: string; expectedSessionId?: string; }) => { - const restored = loadSessionStore(params.storePath, { skipCache: true }); + const restored = loadSessionEntry({ + storePath: params.storePath, + sessionKey: params.sessionKey, + }); if (params.expectedSessionId === undefined) { - expect(restored[params.sessionKey]).toBeUndefined(); + expect(restored).toBeUndefined(); return; } - expect(restored[params.sessionKey]?.sessionId).toBe(params.expectedSessionId); + expect(restored?.sessionId).toBe(params.expectedSessionId); }; it("skips when BOOT.md is missing", async () => { @@ -313,12 +324,13 @@ describe("runBootOnce", () => { const { bootSessionKey, sessionKey, storePath } = resolveMainStore(cfg); const existingSessionId = "main-session-abc123"; - await saveSessionStore(storePath, { - [sessionKey]: { + await replaceSessionEntry( + { storePath, sessionKey }, + { sessionId: existingSessionId, updatedAt: Date.now(), }, - }); + ); agentCommand.mockResolvedValue(undefined); await expect(runBootOnce({ cfg, deps: makeDeps(), workspaceDir })).resolves.toEqual({ @@ -344,12 +356,13 @@ describe("runBootOnce", () => { const { bootSessionKey, sessionKey, storePath } = resolveMainStore(cfg); const existingSessionId = "main-session-xyz789"; - await saveSessionStore(storePath, { - [sessionKey]: { + await replaceSessionEntry( + { storePath, sessionKey }, + { sessionId: existingSessionId, updatedAt: Date.now() - 60_000, // 1 minute ago }, - }); + ); mockAgentUpdatesRequestedSession(storePath); await expect(runBootOnce({ cfg, deps: makeDeps(), workspaceDir })).resolves.toEqual({ diff --git a/src/gateway/server-methods/agent.test.ts b/src/gateway/server-methods/agent.test.ts index 5b8e53736aec..4f5623dcd14f 100644 --- a/src/gateway/server-methods/agent.test.ts +++ b/src/gateway/server-methods/agent.test.ts @@ -21,6 +21,7 @@ import { testing as subagentRegistryTesting, } from "../../agents/subagent-registry.js"; import type { SessionEntry } from "../../config/sessions.js"; +import { parseSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import { onDiagnosticEvent, resetDiagnosticEventsForTest, @@ -68,6 +69,9 @@ const mocks = vi.hoisted(() => ({ loadSessionEntry: vi.fn(), loadGatewaySessionRow: vi.fn(), updateSessionStore: vi.fn(), + applySessionEntryReplacements: vi.fn(), + patchSessionEntryTarget: vi.fn(), + readTranscriptStatsSync: vi.fn(() => ({ eventCount: 0, maxSeq: 0, sizeBytes: 0 })), agentCommand: vi.fn(), clearAgentRunContext: vi.fn(), registerAgentRunContext: vi.fn(), @@ -139,6 +143,17 @@ vi.mock("../../config/sessions/store.js", async () => { }; }); +vi.mock("../../config/sessions/session-accessor.js", async () => { + const actual = await vi.importActual<typeof import("../../config/sessions/session-accessor.js")>( + "../../config/sessions/session-accessor.js", + ); + return { + ...actual, + applySessionEntryReplacements: mocks.applySessionEntryReplacements, + patchSessionEntryTarget: mocks.patchSessionEntryTarget, + readTranscriptStatsSync: mocks.readTranscriptStatsSync, + }; +}); vi.mock("../../commands/agent.js", () => ({ agentCommand: mocks.agentCommand, agentCommandFromIngress: mocks.agentCommand, @@ -403,6 +418,13 @@ function expectStringFieldContains( expect(record[field]).toContain(expected); } +function expectSqliteSessionFileMarkerForEntry(entry: Record<string, unknown> | undefined) { + const sessionFile = entry?.sessionFile; + expect(sessionFile).toBeTypeOf("string"); + const marker = parseSqliteSessionFileMarker(sessionFile as string); + expect(marker?.sessionId).toBe(entry?.sessionId); +} + function mockCallArg(mock: ReturnType<typeof vi.fn>, callIndex = 0, argIndex = 0) { const call = mock.mock.calls[callIndex]; if (!call) { @@ -469,6 +491,130 @@ function buildExistingMainStoreEntry(overrides: Record<string, unknown> = {}) { }; } +type SessionStoreFixture = Record<string, Record<string, unknown>>; + +type SessionEntryTargetFixture = { + canonicalKey: string; + storeKeys: string[]; +}; + +function cloneSessionStoreFixtureEntry( + entry: Record<string, unknown> | undefined, +): Record<string, unknown> | undefined { + return entry ? structuredClone(entry) : undefined; +} + +function selectFreshestTargetFixtureEntry( + store: SessionStoreFixture, + target: SessionEntryTargetFixture, +): { entry: Record<string, unknown>; key: string } | undefined { + let freshest: { entry: Record<string, unknown>; key: string } | undefined; + for (const key of new Set([target.canonicalKey, ...target.storeKeys])) { + const entry = store[key]; + if (!entry) { + continue; + } + if ( + !freshest || + ((entry.updatedAt as number | undefined) ?? 0) > + ((freshest.entry.updatedAt as number | undefined) ?? 0) + ) { + freshest = { entry, key }; + } + } + return freshest; +} + +function resetSessionAccessorMocks() { + mocks.readTranscriptStatsSync.mockReset().mockReturnValue({ + eventCount: 0, + maxSeq: 0, + sizeBytes: 0, + }); + mocks.applySessionEntryReplacements.mockReset().mockImplementation( + async (params: { + activeSessionKey?: string; + requireWriteSuccess?: boolean; + sessionKeys?: readonly string[]; + skipMaintenance?: boolean; + storePath: string; + update: (entries: Array<{ sessionKey: string; entry: SessionEntry }>) => + | Promise<{ + replacements?: Iterable<{ sessionKey: string; entry: SessionEntry }>; + result: unknown; + }> + | { + replacements?: Iterable<{ sessionKey: string; entry: SessionEntry }>; + result: unknown; + }; + }) => + await mocks.updateSessionStore( + params.storePath, + async (store: Record<string, SessionEntry>) => { + const keys = params.sessionKeys ?? Object.keys(store); + const snapshots = keys.flatMap((sessionKey) => { + const entry = store[sessionKey]; + return entry ? [{ sessionKey, entry: structuredClone(entry) }] : []; + }); + const planned = await params.update(snapshots); + for (const replacement of planned.replacements ?? []) { + if (store[replacement.sessionKey]) { + store[replacement.sessionKey] = structuredClone(replacement.entry); + } + } + return planned.result; + }, + { + activeSessionKey: params.activeSessionKey, + requireWriteSuccess: params.requireWriteSuccess, + skipMaintenance: params.skipMaintenance, + }, + ), + ); + mocks.patchSessionEntryTarget.mockReset().mockImplementation( + async ( + scope: { storePath: string; target: SessionEntryTargetFixture }, + update: ( + entry: Record<string, unknown>, + context: { existingEntry?: Record<string, unknown> }, + ) => Promise<Record<string, unknown> | null> | Record<string, unknown> | null, + options: { + fallbackEntry?: Record<string, unknown>; + replaceEntry?: boolean; + } = {}, + ) => + await mocks.updateSessionStore( + scope.storePath, + async (store: SessionStoreFixture) => { + const existing = selectFreshestTargetFixtureEntry(store, scope.target); + const base = existing?.entry ?? options.fallbackEntry; + if (!base) { + return null; + } + const patchContext = existing ? { existingEntry: structuredClone(existing.entry) } : {}; + const patch = await update(structuredClone(base), patchContext); + if (!patch) { + return cloneSessionStoreFixtureEntry(base); + } + const fresh = selectFreshestTargetFixtureEntry(store, scope.target); + const writeBase = fresh?.entry ?? options.fallbackEntry; + if (!writeBase) { + return null; + } + const next = options.replaceEntry ? structuredClone(patch) : { ...writeBase, ...patch }; + for (const key of new Set([scope.target.canonicalKey, ...scope.target.storeKeys])) { + delete store[key]; + } + store[scope.target.canonicalKey] = next; + return next; + }, + options, + ), + ); +} + +resetSessionAccessorMocks(); + function setupNewYorkTimeConfig(isoDate: string) { vi.useFakeTimers({ toFake: ["Date"] }); dateOnlyFakeClockActive = true; @@ -781,6 +927,7 @@ describe("gateway agent handler", () => { mocks.loadConfigReturn = {}; mocks.emitGatewaySessionEndPluginHook.mockReset(); mocks.emitGatewaySessionStartPluginHook.mockReset(); + resetSessionAccessorMocks(); mocks.resolveExplicitAgentSessionKey.mockReset().mockReturnValue(undefined); mocks.resolveAgentExplicitRecipientSession.mockReset().mockResolvedValue({}); mocks.readAcpSessionMeta.mockReset().mockReturnValue(undefined); @@ -1279,33 +1426,24 @@ describe("gateway agent handler", () => { }); }); - it("uses single-entry persistence for ordinary gateway admission touches", async () => { - mockMainSessionEntry({}); - let capturedOptions: - | { - resolveSingleEntryPersistence?: (result: unknown) => unknown; - } - | undefined; - let persistedResult: unknown; - mocks.updateSessionStore.mockImplementation(async (_path, updater, opts) => { - const store: Record<string, Record<string, unknown>> = { - "agent:main:main": buildExistingMainStoreEntry(), - }; - persistedResult = await updater(store); - capturedOptions = opts; - return persistedResult; - }); - mocks.agentCommand.mockResolvedValue({ - payloads: [{ text: "ok" }], - meta: { durationMs: 100 }, - }); + it("passes a canonical user-turn recorder to gateway agent runs", async () => { + primeMainAgentRun(); - await runMainAgent("hi", "idem-single-entry-persist"); + await runMainAgent("persist me", "idem-user-turn-recorder"); - expect(capturedOptions?.resolveSingleEntryPersistence?.(persistedResult)).toMatchObject({ - sessionKey: "agent:main:main", - entry: persistedResult, - }); + const call = await waitForAgentCommandCall< + AgentCommandCall & { + userTurnTranscriptRecorder?: { + persistApproved: () => Promise<unknown>; + }; + } + >(); + expect(call.userTurnTranscriptRecorder).toEqual( + expect.objectContaining({ + persistApproved: expect.any(Function), + persistFallback: expect.any(Function), + }), + ); }); it("dispatches with the session id reloaded during lifecycle admission", async () => { @@ -1732,6 +1870,9 @@ describe("gateway agent handler", () => { storePath: "/tmp/sessions.json", entry: { sessionId: "restored-session", updatedAt: 2 }, canonicalKey: "agent:main:work", + // Post-flip the accessor target carries the alias set prepared at request + // start; freshest-entry resolution happens inside the patch transaction. + storeKeys: ["agent:main:work", "agent:main:main"], }); const store = { "agent:main:work": { @@ -1762,40 +1903,143 @@ describe("gateway agent handler", () => { expect(store["agent:main:main"]).toBeUndefined(); }); - it("disables single-entry persistence when admission prunes legacy store keys", async () => { - mocks.loadConfigReturn = { - session: { mainKey: "work" }, - agents: { list: [{ id: "main", default: true }] }, - }; + it("does not persist a gateway user-turn recorder after the session key is rebound", async () => { + primeMainAgentRun({ sessionId: "accepted-session-id" }); + + await runMainAgent("stale after reset", "idem-user-turn-rebound"); + + const call = await waitForAgentCommandCall< + AgentCommandCall & { + userTurnTranscriptRecorder?: { + persistApproved: () => Promise<unknown>; + }; + } + >(); mocks.loadSessionEntry.mockReturnValue({ - cfg: mocks.loadConfigReturn, + cfg: {}, storePath: "/tmp/sessions.json", - entry: { sessionId: "existing-session-id", updatedAt: Date.now() }, - canonicalKey: "agent:main:work", + entry: { + sessionId: "new-session-id", + updatedAt: Date.now(), + }, + canonicalKey: "agent:main:main", + store: { + "agent:main:main": { + sessionId: "new-session-id", + updatedAt: Date.now(), + }, + }, }); - let capturedOptions: - | { - resolveSingleEntryPersistence?: (result: unknown) => unknown; - } - | undefined; - let persistedResult: unknown; - mocks.updateSessionStore.mockImplementation(async (_path, updater, opts) => { - const store: Record<string, Record<string, unknown>> = { - "agent:main:work": buildExistingMainStoreEntry({ updatedAt: 100 }), - "agent:main:main": buildExistingMainStoreEntry({ updatedAt: 50 }), - }; - persistedResult = await updater(store); - capturedOptions = opts; - return persistedResult; + + await expect(call.userTurnTranscriptRecorder?.persistApproved()).resolves.toBeUndefined(); + }); + + it("does not pass a text-only user-turn recorder for image agent runs", async () => { + mockMainSessionEntry({ + sessionId: "existing-session-id", + model: "vision-model", + modelProvider: "test", + modelOverride: "vision-model", + modelOverrideSource: "user", + providerOverride: "test", }); + mocks.updateSessionStore.mockResolvedValue(undefined); mocks.agentCommand.mockResolvedValue({ payloads: [{ text: "ok" }], meta: { durationMs: 100 }, }); + const context = { + ...makeContext(), + loadGatewayModelCatalog: vi.fn(async () => [ + { + id: "vision-model", + name: "vision-model", + provider: "test", + input: ["image"], + }, + ]), + } as unknown as GatewayRequestContext; - await runMainAgent("hi", "idem-single-entry-legacy-prune"); + await invokeAgent( + { + message: "describe this image", + agentId: "main", + sessionKey: "agent:main:main", + idempotencyKey: "idem-image-user-turn-recorder", + attachments: [ + { + type: "file", + mimeType: "image/png", + fileName: "test.png", + content: Buffer.from("fake-png-data").toString("base64"), + }, + ], + }, + { context, reqId: "idem-image-user-turn-recorder" }, + ); - expect(capturedOptions?.resolveSingleEntryPersistence?.(persistedResult)).toBeUndefined(); + const call = await waitForAgentCommandCall(); + expect(call.images).toEqual([ + expect.objectContaining({ + type: "image", + mimeType: "image/png", + }), + ]); + expect(call.userTurnTranscriptRecorder).toBeUndefined(); + }); + + it("does not pass a text-only user-turn recorder for offloaded image agent runs", async () => { + await withTempDir({ prefix: "openclaw-gateway-agent-offloaded-image-" }, async (root) => { + useTestStateDir(root); + mockMainSessionEntry({ + sessionId: "existing-session-id", + model: "vision-model", + modelProvider: "test", + modelOverride: "vision-model", + modelOverrideSource: "user", + providerOverride: "test", + }); + mocks.updateSessionStore.mockResolvedValue(undefined); + mocks.agentCommand.mockResolvedValue({ + payloads: [{ text: "ok" }], + meta: { durationMs: 100 }, + }); + const context = { + ...makeContext(), + loadGatewayModelCatalog: vi.fn(async () => [ + { + id: "vision-model", + name: "vision-model", + provider: "test", + input: ["image"], + }, + ]), + } as unknown as GatewayRequestContext; + + await invokeAgent( + { + message: "describe this large image", + agentId: "main", + sessionKey: "agent:main:main", + idempotencyKey: "idem-offloaded-image-user-turn-recorder", + attachments: [ + { + type: "file", + mimeType: "image/png", + fileName: "large.png", + content: Buffer.alloc(2_000_001, 1).toString("base64"), + }, + ], + }, + { context, reqId: "idem-offloaded-image-user-turn-recorder" }, + ); + + const call = await waitForAgentCommandCall(); + expect(call.images).toEqual([]); + expect(call.imageOrder).toEqual(["offloaded"]); + expect(call.message).toContain("[media attached: media://inbound/"); + expect(call.userTurnTranscriptRecorder).toBeUndefined(); + }); }); it("preserves ACP metadata from the current stored session entry", async () => { @@ -1864,7 +2108,7 @@ describe("gateway agent handler", () => { await runMainAgent("test", "test-idem-stale-transcript"); expect(capturedEntry?.sessionId).not.toBe("old-session-id"); - expect(capturedEntry?.sessionFile).toBeUndefined(); + expectSqliteSessionFileMarkerForEntry(capturedEntry); }); it("rotates a failed session instead of resuming when its transcript is missing", async () => { @@ -1896,7 +2140,7 @@ describe("gateway agent handler", () => { expect(capturedEntry?.endedAt).toBeUndefined(); expect(capturedEntry?.runtimeMs).toBeUndefined(); expect(capturedEntry?.abortedLastRun).toBeUndefined(); - expect(capturedEntry?.sessionFile).toBeUndefined(); + expectSqliteSessionFileMarkerForEntry(capturedEntry); }); it.each([ @@ -1964,7 +2208,7 @@ describe("gateway agent handler", () => { expect(capturedEntry?.startedAt).toBeUndefined(); expect(capturedEntry?.endedAt).toBeUndefined(); expect(capturedEntry?.runtimeMs).toBeUndefined(); - expect(capturedEntry?.sessionFile).toBeUndefined(); + expectSqliteSessionFileMarkerForEntry(capturedEntry); expect(capturedEntry?.cliSessionBindings).toBeUndefined(); expect(capturedEntry?.cliSessionIds).toBeUndefined(); expect(capturedEntry?.claudeCliSessionId).toBeUndefined(); @@ -2030,7 +2274,7 @@ describe("gateway agent handler", () => { const call = await waitForAgentCommandCall<{ sessionId?: string }>(); expect(call.sessionId).toBe("terminal-main-session"); expect(capturedEntry?.sessionId).toBe("terminal-main-session"); - expect(capturedEntry?.sessionFile).toBe(sessionFile); + expectSqliteSessionFileMarkerForEntry(capturedEntry); expect(capturedEntry?.cliSessionIds).toEqual({ "claude-cli": "existing-claude-cli-session", }); @@ -2099,7 +2343,7 @@ describe("gateway agent handler", () => { const call = await waitForAgentCommandCall<{ sessionId?: string }>(); expect(call.sessionId).toBe("terminal-main-session"); expect(capturedEntry?.sessionId).toBe("terminal-main-session"); - expect(capturedEntry?.sessionFile).toBe(sessionFile); + expectSqliteSessionFileMarkerForEntry(capturedEntry); expect(capturedEntry?.status).toBe("done"); expect(capturedEntry?.startedAt).toBe(now - 20_000); expect(capturedEntry?.endedAt).toBe(now - 15_000); @@ -2172,7 +2416,7 @@ describe("gateway agent handler", () => { const call = await waitForAgentCommandCall<{ sessionId?: string }>(); expect(call.sessionId).toBe("terminal-main-session"); expect(capturedEntry?.sessionId).toBe("terminal-main-session"); - expect(capturedEntry?.sessionFile).toBe(sessionFile); + expectSqliteSessionFileMarkerForEntry(capturedEntry); }, ); }, @@ -2200,7 +2444,7 @@ describe("gateway agent handler", () => { expect(call.sessionId).not.toBe("failed-missing-default-session-id"); expect(capturedEntry?.sessionId).not.toBe("failed-missing-default-session-id"); expect(capturedEntry?.status).toBeUndefined(); - expect(capturedEntry?.sessionFile).toBeUndefined(); + expectSqliteSessionFileMarkerForEntry(capturedEntry); }); it("recovers a failed session when its default transcript exists", async () => { @@ -2243,7 +2487,109 @@ describe("gateway agent handler", () => { expect(capturedEntry?.endedAt).toBeUndefined(); expect(capturedEntry?.runtimeMs).toBeUndefined(); expect(capturedEntry?.abortedLastRun).toBeUndefined(); - expect(capturedEntry?.sessionFile).toBeUndefined(); + expectSqliteSessionFileMarkerForEntry(capturedEntry); + }); + }); + + it.each([ + { + name: "default transcript", + sessionKey: "agent:main:telegram:group:stale-failed", + sessionId: "stale-failed-session-id", + configureTranscript: async (params: { sessionId: string; sessionsDir: string }) => { + await fs.writeFile(`${params.sessionsDir}/${params.sessionId}.jsonl`, "", "utf8"); + return {}; + }, + expectsSqliteStats: false, + }, + { + name: "SQLite transcript marker", + sessionKey: "agent:main:telegram:group:stale-failed-sqlite", + sessionId: "stale-failed-sqlite-session-id", + configureTranscript: async (params: { sessionId: string; storePath: string }) => { + mocks.readTranscriptStatsSync.mockReturnValue({ + eventCount: 1, + maxSeq: 1, + sizeBytes: 32, + }); + return { sessionFile: `sqlite:main:${params.sessionId}:${params.storePath}` }; + }, + expectsSqliteStats: true, + }, + ])("recovers a stale failed session when its $name exists", async (scenario) => { + const now = Date.parse("2026-05-18T09:49:30.000Z"); + vi.useFakeTimers({ toFake: ["Date"] }); + dateOnlyFakeClockActive = true; + vi.setSystemTime(now); + + await withTempDir({ prefix: "openclaw-gateway-stale-failed-session-" }, async (root) => { + const sessionsDir = `${root}/sessions`; + const storePath = `${sessionsDir}/sessions.json`; + await fs.mkdir(sessionsDir, { recursive: true }); + const transcriptFields = await scenario.configureTranscript({ + sessionId: scenario.sessionId, + sessionsDir, + storePath, + }); + const failedEntryWithStaleActivity = { + sessionId: scenario.sessionId, + ...transcriptFields, + status: "failed", + startedAt: now - 11 * 60_000, + endedAt: now - 10 * 60_000, + runtimeMs: 60_000, + abortedLastRun: true, + updatedAt: now - 10 * 60_000, + sessionStartedAt: now - 20 * 60_000, + lastInteractionAt: now - 10 * 60_000, + }; + mocks.loadSessionEntry.mockReturnValue({ + cfg: { session: { idleMinutes: 5 } }, + storePath, + entry: failedEntryWithStaleActivity, + canonicalKey: scenario.sessionKey, + }); + let capturedEntry: Record<string, unknown> | undefined; + mocks.updateSessionStore.mockImplementation(async (_path, updater) => { + const store: Record<string, unknown> = { + [scenario.sessionKey]: { ...failedEntryWithStaleActivity }, + }; + const result = await updater(store); + capturedEntry = result as Record<string, unknown>; + return result; + }); + mocks.agentCommand.mockResolvedValue({ + payloads: [{ text: "ok" }], + meta: { durationMs: 100 }, + }); + + await invokeAgent({ + message: "recover stale failed", + agentId: "main", + sessionKey: scenario.sessionKey, + idempotencyKey: `test-idem-${scenario.sessionId}`, + } as AgentParams); + + const call = await waitForAgentCommandCall<{ sessionId?: string }>(); + expect(call.sessionId).toBe(scenario.sessionId); + if (scenario.expectsSqliteStats) { + expect(mocks.readTranscriptStatsSync).toHaveBeenCalledWith({ + agentId: "main", + sessionId: scenario.sessionId, + sessionKey: scenario.sessionKey, + storePath, + sessionEntry: failedEntryWithStaleActivity, + }); + } else { + expect(mocks.readTranscriptStatsSync).not.toHaveBeenCalled(); + } + expect(capturedEntry?.sessionId).toBe(scenario.sessionId); + expect(capturedEntry?.status).toBeUndefined(); + expect(capturedEntry?.startedAt).toBeUndefined(); + expect(capturedEntry?.endedAt).toBeUndefined(); + expect(capturedEntry?.runtimeMs).toBeUndefined(); + expect(capturedEntry?.abortedLastRun).toBeUndefined(); + expectSqliteSessionFileMarkerForEntry(capturedEntry); }); }); @@ -2288,7 +2634,7 @@ describe("gateway agent handler", () => { expect(capturedEntry?.endedAt).toBeUndefined(); expect(capturedEntry?.runtimeMs).toBeUndefined(); expect(capturedEntry?.abortedLastRun).toBeUndefined(); - expect(capturedEntry?.sessionFile).toBe("relative-present.jsonl"); + expectSqliteSessionFileMarkerForEntry(capturedEntry); }); }); @@ -2949,8 +3295,8 @@ describe("gateway agent handler", () => { status: "running", startedAt: 111, sessionStartedAt: undefined, - sessionFile: "/tmp/fresh-session.jsonl", }); + expectSqliteSessionFileMarkerForEntry(capturedEntry); }); // Upgrade-path self-heal: a legacy session entry may lack sessionStartedAt // because the field was added after the entry was first persisted. The @@ -3271,11 +3617,27 @@ describe("gateway agent handler", () => { { reqId: "inter-session-marker" }, ); - const callArgs = await waitForAgentCommandCall<{ message?: string }>(); + const callArgs = await waitForAgentCommandCall< + AgentCommandCall & { + message?: string; + userTurnTranscriptRecorder?: { + message?: unknown; + }; + } + >(); expect(callArgs.message).toMatch(/^\[Inter-session message\]/); expect(callArgs.message).toContain("isUser=false"); expect(callArgs.message).toContain("forwarded reply"); expect(callArgs.message).not.toContain("[Wed 2026-01-28 20:30 EST]"); + expect(callArgs.userTurnTranscriptRecorder?.message).toMatchObject({ + role: "user", + content: "forwarded reply", + provenance: { + kind: "inter_session", + sourceSessionKey: "agent:main:discord:source", + sourceTool: "sessions_send", + }, + }); resetTimeConfig(); }); @@ -7612,6 +7974,7 @@ describe("gateway agent handler", () => { updatedAt: Date.now(), }, canonicalKey: "agent:main:work", + storeKeys: ["agent:main:work", "agent:main:main"], }); let capturedStore: Record<string, unknown> | undefined; @@ -7677,6 +8040,35 @@ describe("gateway agent handler", () => { expect(result.meta?.agentMeta?.sessionId).toBe("reset-session-id"); }); + it("persists the post-reset follow-up prompt in the canonical user-turn recorder", async () => { + mockSessionResetSuccess({ reason: "new", sessionId: "reset-session-id" }); + mockMainSessionEntry({ sessionId: "reset-session-id" }); + mocks.performGatewaySessionReset.mockClear(); + mocks.agentCommand.mockResolvedValue({ + payloads: [{ text: "ok" }], + meta: { durationMs: 100 }, + }); + + await invokeAgent( + { + message: "/new continue with this prompt", + sessionKey: "agent:main:main", + idempotencyKey: "test-idem-new-followup-recorder", + }, + { + reqId: "4-new-followup-recorder", + client: { connect: { scopes: ["operator.admin"] } } as AgentHandlerArgs["client"], + }, + ); + + const call = await waitForAgentCommandCall<{ + message?: string; + userTurnTranscriptRecorder?: { message?: { content?: string } }; + }>(); + expect(call.message).toBe("continue with this prompt"); + expect(call.userTurnTranscriptRecorder?.message?.content).toBe("continue with this prompt"); + }); + it("handles bare /reset by resetting the same session without running the model", async () => { mockSessionResetSuccess({ reason: "reset" }); mocks.performGatewaySessionReset.mockClear(); @@ -8399,6 +8791,7 @@ describe("gateway agent handler chat.abort integration", () => { mocks.loadGatewaySessionRow.mockReset(); mocks.loadSessionEntry.mockReset(); mocks.updateSessionStore.mockReset(); + resetSessionAccessorMocks(); mocks.emitGatewaySessionEndPluginHook.mockReset(); mocks.emitGatewaySessionStartPluginHook.mockReset(); mocks.getLatestSubagentRunByChildSessionKey.mockReset(); @@ -8582,7 +8975,7 @@ describe("gateway agent handler chat.abort integration", () => { const context = makeContext(); const respond = vi.fn(); const runId = "idem-abort-before-dispatch"; - await invokeAgent( + const pending = invokeAgent( { message: "hi", agentId: "main", @@ -8591,6 +8984,10 @@ describe("gateway agent handler chat.abort integration", () => { }, { context, respond, reqId: runId, flushDispatch: false }, ); + await waitForAssertion(() => { + expect(respond).toHaveBeenCalled(); + expect(mocks.agentCommand).not.toHaveBeenCalled(); + }); expectRecordFields(mockCallArg(respond, 0, 1), { runId, @@ -8614,6 +9011,7 @@ describe("gateway agent handler chat.abort integration", () => { runIds: [runId], }); expect(context.chatAbortControllers.has(runId)).toBe(false); + await pending; await flushScheduledDispatchStep(); diff --git a/src/gateway/server-methods/agent.ts b/src/gateway/server-methods/agent.ts index e6dd9372c4b0..6371aa6a4c1a 100644 --- a/src/gateway/server-methods/agent.ts +++ b/src/gateway/server-methods/agent.ts @@ -46,6 +46,7 @@ import { retainEmbeddedAgentRunAbortabilityForRunId, } from "../../agents/embedded-agent-runner/runs.js"; import { isTimeoutError } from "../../agents/failover-error.js"; +import { runAgentHarnessBeforeMessageWriteHook } from "../../agents/harness/hook-helpers.js"; import { resolvePublicAgentAvatarSource } from "../../agents/identity-avatar.js"; import { AGENT_INTERNAL_EVENT_TYPE_TASK_COMPLETION, @@ -74,25 +75,33 @@ import { evaluateSessionFreshness, hasTerminalMainSessionTranscriptNewerThanRegistrySync, mergeSessionEntry, - resolveTerminalMainSessionTranscriptRegistryCheck, - resolveChannelResetConfig, resolveAgentIdFromSessionKey, - resolveExplicitAgentSessionKey, resolveAgentMainSessionKey, + resolveChannelResetConfig, + resolveExplicitAgentSessionKey, resolveSessionFilePath, resolveSessionFilePathOptions, resolveSessionLifecycleTimestamps, - resolveSessionWorkStartError, resolveSessionResetPolicy, resolveSessionResetType, + resolveSessionWorkStartError, + resolveTerminalMainSessionTranscriptRegistryCheck, type SessionEntry, type SessionFreshness, - updateSessionStore, } from "../../config/sessions.js"; import { hasProviderOwnedSession } from "../../config/sessions/entry-freshness.js"; -import { applySessionEntryReplacements } from "../../config/sessions/session-accessor.js"; +import { + applySessionEntryReplacements, + patchSessionEntryTarget, + readTranscriptStatsSync, +} from "../../config/sessions/session-accessor.js"; import { mergeSessionSnapshotChanges } from "../../config/sessions/session-snapshot-merge.js"; +import { + formatSqliteSessionFileMarker, + parseSqliteSessionFileMarker, +} from "../../config/sessions/sqlite-marker.js"; import { resolveMaintenanceConfigFromInput } from "../../config/sessions/store-maintenance.js"; +import { isRecoverableTerminalSessionStatus } from "../../config/sessions/terminal-status.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { isAbortError } from "../../infra/abort-signal.js"; import { @@ -148,6 +157,7 @@ import { beginSessionWorkAdmission, type SessionWorkAdmissionLease, } from "../../sessions/session-lifecycle-admission.js"; +import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; import { createRunningTaskRun, finalizeTaskRunByRunId } from "../../tasks/detached-task-runtime.js"; import type { TaskStatus } from "../../tasks/task-registry.types.js"; import { @@ -191,10 +201,7 @@ import { reactivateCompletedSubagentSession } from "../session-subagent-reactiva import { canonicalizeSpawnedByForAgent, loadSessionEntry, - migrateAndPruneGatewaySessionStoreKey, resolveDeletedAgentIdFromSessionKey, - resolveFreshestSessionEntryFromStoreKeys, - resolveGatewaySessionStoreTarget, resolveGatewayModelSupportsImages, resolveSessionStoreKey, resolveSessionModelRef, @@ -212,10 +219,6 @@ import type { const RESET_COMMAND_RE = /^\/(new|reset)(?:\s+([\s\S]*))?$/i; const CRON_CONTINUATION_RELEASE_RECOVERY_DELAYS_MS = [250, 1_000, 4_000, 15_000] as const; -function isRecoverableTerminalSessionStatus(status: SessionEntry["status"] | undefined): boolean { - return status === "failed" || status === "timeout" || status === "killed"; -} - type AgentSendSessionLifecycleTransition = { cfg: OpenClawConfig; sessionKey: string; @@ -1256,6 +1259,29 @@ function shouldSuppressAgentPromptPersistence(params: { ); } +function withSqliteSessionFileMarker(params: { + agentId: string | undefined; + entry: SessionEntry; + sessionKey: string; + storePath: string; +}): SessionEntry { + const agentId = params.agentId ?? resolveAgentIdFromSessionKey(params.sessionKey); + if (!agentId) { + return params.entry; + } + const sessionFile = formatSqliteSessionFileMarker({ + agentId, + sessionId: params.entry.sessionId, + storePath: params.storePath, + }); + return params.entry.sessionFile === sessionFile + ? params.entry + : { + ...params.entry, + sessionFile, + }; +} + function yieldAfterAgentAcceptedAck(): Promise<void> { return new Promise((resolve) => { setTimeout(resolve, 10); @@ -2002,7 +2028,9 @@ export const agentHandlers: GatewayRequestHandlers = { }; try { - let message = (request.message ?? "").trim(); + const transcriptInputText = (request.message ?? "").trim(); + let effectiveTranscriptInputText = transcriptInputText; + let message = effectiveTranscriptInputText; if (!isRawModelRun) { message = annotateInterSessionPromptText(message, inputProvenance); } @@ -2204,9 +2232,10 @@ export const agentHandlers: GatewayRequestHandlers = { let postAdmissionSuperseded = false; let lifecycleRotatedDuringAdmission = false; const admissionAgentId = () => - resolvedSessionKey === "global" - ? (resolvedSessionAgentId ?? agentId ?? resolveDefaultAgentId(cfgForAgent ?? cfg)) - : undefined; + resolvedSessionAgentId ?? + (resolvedSessionKey === "global" + ? (agentId ?? resolveDefaultAgentId(cfgForAgent ?? cfg)) + : undefined); const assertGatewayWorkAdmissionAllowed = (commitOutcome = true) => { const latestPreRegistrationAbort = readGatewayDedupeEntry({ dedupe: context.dedupe, @@ -2300,15 +2329,19 @@ export const agentHandlers: GatewayRequestHandlers = { if (!resolvedSessionKey) { return; } + const admissionAgent = admissionAgentId(); let latestEntry = loadSessionEntry(resolvedSessionKey, { - ...(resolvedSessionKey === "global" ? { agentId: admissionAgentId() } : {}), + agentId: admissionAgent, clone: false, }).entry; // Legacy stores may only carry the requested spelling (e.g. bare // "main"); a canonical-only re-read would misreport those sessions // as deleted mid-start. if (!latestEntry && requestedSessionKey && requestedSessionKey !== resolvedSessionKey) { - latestEntry = loadSessionEntry(requestedSessionKey, { clone: false }).entry; + latestEntry = loadSessionEntry(requestedSessionKey, { + agentId: admissionAgent, + clone: false, + }).entry; } if (sessionPersistedBeforeGatewayAdmission && !latestEntry) { throw new Error( @@ -2454,6 +2487,7 @@ export const agentHandlers: GatewayRequestHandlers = { if (abortForLifecycleRotation({ sessionKey: resetResult.key, agentId })) { return; } + effectiveTranscriptInputText = postResetMessage; message = postResetMessage; } else { let resetAckResult: Awaited<ReturnType<typeof resolveBareSessionResetResult>>; @@ -2528,6 +2562,7 @@ export const agentHandlers: GatewayRequestHandlers = { entry, canonicalKey, legacyKey, + storeKeys, } = loadSessionEntry(requestedSessionKey, sessionLoadOptions); cfgForAgent = cfgLocal; const isGeneratedMediaCronContinuation = @@ -2676,6 +2711,24 @@ export const agentHandlers: GatewayRequestHandlers = { if (candidateEntry?.status !== "failed" || !candidateEntry.sessionId?.trim()) { return false; } + const sqliteMarker = parseSqliteSessionFileMarker(candidateEntry.sessionFile); + if (sqliteMarker) { + if (sqliteMarker.sessionId !== candidateEntry.sessionId) { + return true; + } + try { + const stats = readTranscriptStatsSync({ + agentId: sqliteMarker.agentId, + sessionId: sqliteMarker.sessionId, + sessionKey: canonicalKey, + storePath: sqliteMarker.storePath, + sessionEntry: candidateEntry, + }); + return stats.eventCount === 0; + } catch { + return true; + } + } try { const sessionPathOpts = resolveSessionFilePathOptions({ storePath, @@ -2720,9 +2773,13 @@ export const agentHandlers: GatewayRequestHandlers = { storePath, }) : false; + const recoverableTerminalSession = + Boolean(entry?.sessionId) && + visibleRequest && + isRecoverableTerminalSessionStatus(entry?.status); const canReuseSession = Boolean(entry?.sessionId) && - (freshness?.fresh ?? false) && + ((freshness?.fresh ?? false) || recoverableTerminalSession) && !failedSessionTranscriptMissing && !terminalMainTranscriptNewerThanRegistry; let usableRequestedSessionId = @@ -2879,9 +2936,13 @@ export const agentHandlers: GatewayRequestHandlers = { }); const freshFailedSessionTranscriptMissing = resolveFailedSessionTranscriptMissingForEntry(freshEntry); + const freshRecoverableTerminalSession = + Boolean(freshEntry?.sessionId) && + visibleRequest && + isRecoverableTerminalSessionStatus(freshEntry?.status); const freshCanReuseSession = Boolean(freshEntry?.sessionId) && - (freshFreshness?.fresh ?? false) && + ((freshFreshness?.fresh ?? false) || freshRecoverableTerminalSession) && !freshFailedSessionTranscriptMissing && !freshTerminalMainTranscriptNewerThanRegistry; const freshUsableRequestedSessionId = @@ -2906,9 +2967,7 @@ export const agentHandlers: GatewayRequestHandlers = { : freshSessionId; const shouldClearRotatedState = freshRotatedSessionId && !freshSessionRotatedSinceLoad; const freshRecoverTerminalSession = - freshCanReuseSession && - visibleRequest && - isRecoverableTerminalSessionStatus(freshEntry?.status); + freshCanReuseSession && freshRecoverableTerminalSession; const shouldClearTerminalState = freshRecoverTerminalSession && !freshSessionRotatedSinceLoad && @@ -3007,173 +3066,140 @@ export const agentHandlers: GatewayRequestHandlers = { if (abortForLifecycleRotation({ sessionKey: canonicalSessionKey, agentId })) { return; } - const requestedStoreKey = requestedSessionKey; let deniedBySendPolicy = false; - let singleEntryPersistence: - | { - sessionKey: string; - entry: SessionEntry; - } - | undefined; + let deniedSessionEntry: SessionEntry | undefined; let persisted: SessionEntry | undefined; let archivedDuringStoreUpdateError: string | undefined; let deletedDuringStoreUpdateError: string | undefined; try { - persisted = await updateSessionStore( - storePath, - (store) => { - // The writer lock may outlive this request's lifecycle. Check at - // transaction admission; once admitted, let the atomic write finish. - assertAgentRunLifecycleGenerationCurrent(lifecycleGeneration); - const storeKeysBeforeMigration = new Set(Object.keys(store)); - const preMigrationTarget = resolveGatewaySessionStoreTarget({ - cfg: cfgLocal, - key: requestedStoreKey, - store, - ...(sessionAgentId ? { agentId: sessionAgentId } : {}), - }); - const preMigrationEntry = resolveFreshestSessionEntryFromStoreKeys( - store, - preMigrationTarget.storeKeys, - ); - const initialTarget = resolveGatewaySessionStoreTarget({ - cfg: cfgLocal, - key: canonicalSessionKey, - store, - ...(sessionAgentId ? { agentId: sessionAgentId } : {}), - }); - const initialEntry = resolveFreshestSessionEntryFromStoreKeys( - store, - initialTarget.storeKeys, - ); - // A completed delete must win over this request's earlier read; - // otherwise the initial touch would recreate the removed row. - // The row counts as deleted only when both the requested and - // canonical alias sets miss it: exec-approval followups read - // under the canonical key while legacy stores may only carry - // the requested spelling (e.g. bare "main"). - if (entry && !preMigrationEntry && !initialEntry) { - deletedDuringStoreUpdateError = `Session "${canonicalSessionKey}" was deleted while starting work. Retry.`; - throw new Error(deletedDuringStoreUpdateError); - } - const archivedError = resolveSessionWorkStartError( - preMigrationTarget.canonicalKey, - preMigrationEntry, - ); - if (archivedError) { - archivedDuringStoreUpdateError = archivedError; - throw new Error(archivedError); - } - const hadLegacyStoreKey = preMigrationTarget.storeKeys.some( - (storeKey) => - storeKey !== preMigrationTarget.canonicalKey && Object.hasOwn(store, storeKey), - ); - const { target, primaryKey } = migrateAndPruneGatewaySessionStoreKey({ - cfg: cfgLocal, - key: requestedStoreKey, - store, - }); - const prunedStoreKey = [...storeKeysBeforeMigration].some( - (storeKey) => !Object.hasOwn(store, storeKey), - ); - const freshEntry = store[primaryKey]; - if (restoredCronContinuationIdentity) { - const marker = freshEntry?.cronRunContinuation; - const provider = normalizeOptionalString(freshEntry?.modelProvider); - const model = normalizeOptionalString(freshEntry?.model); - const identityMatches = - marker?.phase === "ready" && - marker.basePersisted === true && - marker.lifecycleRevision === - restoredCronContinuationIdentity.lifecycleRevision && - freshEntry?.sessionId === restoredCronContinuationIdentity.sessionId; - if (!identityMatches || !freshEntry || !provider || !model) { - restoredCronContinuationError = - "cron run continuation changed before admission"; - throw new Error(restoredCronContinuationError); + persisted = + (await patchSessionEntryTarget( + { + agentId: sessionAgentId, + storePath, + target: { + canonicalKey: canonicalSessionKey, + storeKeys: storeKeys ?? [canonicalSessionKey], + }, + }, + (_currentEntry, patchContext) => { + // The writer lock may outlive this request's lifecycle. Check at + // transaction admission; once admitted, let the atomic write finish. + assertAgentRunLifecycleGenerationCurrent(lifecycleGeneration); + const freshEntry = patchContext.existingEntry; + // A completed delete must win over this request's earlier read; + // otherwise the initial touch would recreate the removed row. + // The accessor target already spans the requested key plus its + // known aliases, so a miss here means the row is truly gone. + if (entry && !freshEntry) { + deletedDuringStoreUpdateError = `Session "${canonicalSessionKey}" was deleted while starting work. Retry.`; + throw new Error(deletedDuringStoreUpdateError); } - if ( - !cronContinuationHasReusableRuntime({ - cfg: cfgLocal, - entry: freshEntry, - agentId: canonicalSessionAgentId, + const archivedError = resolveSessionWorkStartError( + canonicalSessionKey, + freshEntry, + ); + if (archivedError) { + archivedDuringStoreUpdateError = archivedError; + throw new Error(archivedError); + } + let entryForPatch = freshEntry; + if (restoredCronContinuationIdentity) { + const marker = freshEntry?.cronRunContinuation; + const provider = normalizeOptionalString(freshEntry?.modelProvider); + const model = normalizeOptionalString(freshEntry?.model); + const identityMatches = + marker?.phase === "ready" && + marker.basePersisted === true && + marker.lifecycleRevision === + restoredCronContinuationIdentity.lifecycleRevision && + freshEntry?.sessionId === restoredCronContinuationIdentity.sessionId; + if (!identityMatches || !freshEntry || !provider || !model) { + restoredCronContinuationError = + "cron run continuation changed before admission"; + throw new Error(restoredCronContinuationError); + } + if ( + !cronContinuationHasReusableRuntime({ + cfg: cfgLocal, + entry: freshEntry, + agentId: canonicalSessionAgentId, + provider, + model, + }) + ) { + restoredCronContinuationError = + "cron run continuation has no reusable native CLI session"; + throw new Error(restoredCronContinuationError); + } + restoredCronContinuation = { + ...restoredCronContinuationIdentity, provider, model, - }) - ) { - restoredCronContinuationError = - "cron run continuation has no reusable native CLI session"; - throw new Error(restoredCronContinuationError); + ...(freshEntry.thinkingLevel ? { thinking: freshEntry.thinkingLevel } : {}), + ...(marker.toolsAllow !== undefined + ? { toolsAllow: [...marker.toolsAllow] } + : {}), + ...(marker.toolsAllowIsDefault === true ? { toolsAllowIsDefault: true } : {}), + ...(marker.cliSessionBindingFacts + ? { cliSessionBindingFacts: { ...marker.cliSessionBindingFacts } } + : {}), + }; + entryForPatch = { + ...freshEntry, + cronRunContinuation: { + ...marker, + phase: "continuing", + ownerRunId: runId, + ownerLifecycleGeneration: lifecycleGeneration, + }, + }; + cronContinuationClaim = { + storePath, + sessionKey: canonicalSessionKey, + lifecycleRevision: marker.lifecycleRevision, + initialEntry: structuredClone(entryForPatch), + mediaTaskIdsBefore: + getGeneratedMediaTaskIdsForSessionKey(canonicalSessionKey), + }; } - restoredCronContinuation = { - ...restoredCronContinuationIdentity, - provider, - model, - ...(freshEntry.thinkingLevel ? { thinking: freshEntry.thinkingLevel } : {}), - ...(marker.toolsAllow !== undefined - ? { toolsAllow: [...marker.toolsAllow] } - : {}), - ...(marker.toolsAllowIsDefault === true ? { toolsAllowIsDefault: true } : {}), - ...(marker.cliSessionBindingFacts - ? { cliSessionBindingFacts: { ...marker.cliSessionBindingFacts } } - : {}), - }; - freshEntry.cronRunContinuation = { - ...marker, - phase: "continuing", - ownerRunId: runId, - ownerLifecycleGeneration: lifecycleGeneration, - }; - cronContinuationClaim = { + patchBuild = buildSessionPatch(entryForPatch); + const effectivePatch = + recoveredSessionStartedAt !== undefined && + entryForPatch?.sessionStartedAt === undefined && + entryForPatch?.sessionId === entry?.sessionId + ? { ...patchBuild.patch, sessionStartedAt: recoveredSessionStartedAt } + : patchBuild.patch; + const merged = withSqliteSessionFileMarker({ + agentId: sessionAgentId, + entry: mergeSessionEntry(entryForPatch, effectivePatch), + sessionKey: canonicalSessionKey, storePath, - sessionKey: primaryKey, - lifecycleRevision: marker.lifecycleRevision, - initialEntry: structuredClone(freshEntry), - mediaTaskIdsBefore: getGeneratedMediaTaskIdsForSessionKey(primaryKey), - }; - } - patchBuild = buildSessionPatch(freshEntry); - const effectivePatch = - recoveredSessionStartedAt !== undefined && - freshEntry?.sessionStartedAt === undefined && - freshEntry?.sessionId === entry?.sessionId - ? { ...patchBuild.patch, sessionStartedAt: recoveredSessionStartedAt } - : patchBuild.patch; - const merged = mergeSessionEntry(freshEntry, effectivePatch); - const sendPolicy = - request.deliver === true - ? resolveSendPolicy({ - cfg: cfgLocal, - entry: merged, - sessionKey: canonicalKey, - channel: merged?.channel, - chatType: merged?.chatType, - }) - : "allow"; - if (sendPolicy === "deny") { - deniedBySendPolicy = true; + }); + const sendPolicy = + request.deliver === true + ? resolveSendPolicy({ + cfg: cfgLocal, + entry: merged, + sessionKey: canonicalKey, + channel: merged?.channel, + chatType: merged?.chatType, + }) + : "allow"; + if (sendPolicy === "deny") { + deniedBySendPolicy = true; + deniedSessionEntry = merged; + return null; + } return merged; - } - store[primaryKey] = merged; - const canonicalKeyChanged = target.canonicalKey !== preMigrationTarget.canonicalKey; - singleEntryPersistence = - freshEntry && !hadLegacyStoreKey && !canonicalKeyChanged && !prunedStoreKey - ? { - sessionKey: primaryKey, - entry: merged, - } - : undefined; - return merged; - }, - { - takeCacheOwnership: true, - ...(restoredCronContinuationIdentity - ? { activeSessionKey: canonicalSessionKey, requireWriteSuccess: true } - : {}), - maintenanceConfig: sessionMaintenanceConfig, - resolveSingleEntryPersistence: () => singleEntryPersistence, - }, - ); + }, + { + fallbackEntry: entry ?? mergeSessionEntry(undefined, patchBuild.patch), + replaceEntry: true, + takeCacheOwnership: true, + maintenanceConfig: sessionMaintenanceConfig, + }, + )) ?? undefined; } catch (err) { if (abortForLifecycleRotation({ sessionKey: canonicalSessionKey, agentId })) { return; @@ -3200,7 +3226,13 @@ export const agentHandlers: GatewayRequestHandlers = { } throw err; } - if (persisted) { + if (abortForLifecycleRotation({ sessionKey: canonicalSessionKey, agentId })) { + return; + } + if (deniedBySendPolicy && deniedSessionEntry) { + sessionEntry = deniedSessionEntry; + resolvedSessionId = sessionEntry.sessionId; + } else if (persisted) { sessionEntry = persisted; resolvedSessionId = sessionEntry.sessionId; sessionPersistedBeforeGatewayAdmission = true; @@ -3542,7 +3574,7 @@ export const agentHandlers: GatewayRequestHandlers = { }); const lifecycleStorePath = resolvedSessionKey ? loadSessionEntry(resolvedSessionKey, { - ...(resolvedSessionKey === "global" ? { agentId: activeSessionAgentId } : {}), + ...(activeSessionAgentId ? { agentId: activeSessionAgentId } : {}), clone: false, }).storePath : `agent:${activeSessionAgentId}`; @@ -3776,6 +3808,64 @@ export const agentHandlers: GatewayRequestHandlers = { if (!isRawModelRun) { message = annotateInterSessionPromptText(message, inputProvenance); } + const userTurnTranscriptRecorder = + resolvedSessionKey && + resolvedSessionId && + !suppressVisibleSessionEffects && + images.length === 0 && + imageOrder.length === 0 + ? createUserTurnTranscriptRecorder({ + input: { + text: effectiveTranscriptInputText, + timestamp: Date.now(), + idempotencyKey: `${runId}:user`, + ...(inputProvenance ? { provenance: inputProvenance } : {}), + }, + target: () => { + const loaded = loadSessionEntry(resolvedSessionKey, { + ...(activeSessionAgentId ? { agentId: activeSessionAgentId } : {}), + clone: false, + }); + const loadedEntry = loaded.entry; + const loadedSessionId = loadedEntry?.sessionId?.trim(); + if (loadedSessionId && loadedSessionId !== resolvedSessionId) { + return undefined; + } + const latestEntry = loadedSessionId + ? loadedEntry + : sessionEntry?.sessionId?.trim() === resolvedSessionId + ? sessionEntry + : { + sessionId: resolvedSessionId, + updatedAt: Date.now(), + sessionFile: sessionEntry?.sessionFile, + }; + if (!latestEntry) { + return undefined; + } + return { + sessionId: latestEntry.sessionId, + sessionKey: resolvedSessionKey, + sessionEntry: latestEntry, + sessionStore: loaded.store, + storePath: loaded.storePath, + agentId: activeSessionAgentId, + cwd: resolveSessionRuntimeCwd({ + sessionEntry: latestEntry, + }), + ...(resolvedThreadId != null ? { threadId: resolvedThreadId } : {}), + config: cfgForAgent ?? cfg, + }; + }, + errorContext: "gateway agent user turn transcript", + beforeMessageWrite: runAgentHarnessBeforeMessageWriteHook, + onPersistenceError: (error) => { + context.logGateway.warn( + `gateway agent user transcript persistence failed: ${formatForLog(error)}`, + ); + }, + }) + : undefined; const ingressAgentId = resolvedSessionKey === "global" @@ -3876,6 +3966,7 @@ export const agentHandlers: GatewayRequestHandlers = { inputProvenance, internalEvents: request.internalEvents, }), + userTurnTranscriptRecorder, cleanupBundleMcpOnRunEnd: request.cleanupBundleMcpOnRunEnd, abortSignal: activeRunAbort.controller.signal, lifecycleGeneration, diff --git a/src/gateway/server-methods/chat-history-omission-request.test.ts b/src/gateway/server-methods/chat-history-omission-request.test.ts index 575180ad4934..027230ed6c19 100644 --- a/src/gateway/server-methods/chat-history-omission-request.test.ts +++ b/src/gateway/server-methods/chat-history-omission-request.test.ts @@ -12,6 +12,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, test } from "vitest"; import type { WebSocket } from "ws"; +import { appendTranscriptMessageSync } from "../../config/sessions/session-accessor.js"; import { onDiagnosticEvent, type DiagnosticPayloadLargeEvent, @@ -30,11 +31,12 @@ installConnectedControlUiServerSuite((started) => { describe("chat.history request emits truncation diagnostic (real WS gateway)", () => { test("a real chat.history request logs payload.large when older history is omitted", async () => { const SESSION_ID = "sess-omission-proof"; + const SESSION_KEY = "agent:main:main"; const MESSAGE_COUNT = 12; const TEXT_BYTES = 2_000; // Budget far below the seeded transcript but well above one message, so the // front byte cap drops older messages without per-message placeholdering. - const BUDGET_BYTES = 8_000; + const BUDGET_BYTES = 20_000; const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-chat-history-omit-")); const captured: DiagnosticPayloadLargeEvent[] = []; @@ -48,23 +50,33 @@ describe("chat.history request emits truncation diagnostic (real WS gateway)", ( try { await writeSessionStore({ entries: { - main: { + [SESSION_KEY]: { sessionId: SESSION_ID, - sessionFile: path.join(dir, `${SESSION_ID}.jsonl`), updatedAt: Date.now(), }, }, }); - const messages = Array.from({ length: MESSAGE_COUNT }, (_, i) => ({ - role: i % 2 === 0 ? "user" : "assistant", - content: [{ type: "text", text: `m${i} ${"x".repeat(TEXT_BYTES)}` }], - timestamp: i + 1, - })); - const lines = messages.map((message) => JSON.stringify({ message })); - await fs.writeFile(path.join(dir, `${SESSION_ID}.jsonl`), lines.join("\n"), "utf-8"); + for (let i = 0; i < MESSAGE_COUNT; i += 1) { + appendTranscriptMessageSync( + { + agentId: "main", + sessionId: SESSION_ID, + sessionKey: SESSION_KEY, + storePath: testState.sessionStorePath, + }, + { + message: { + role: i % 2 === 0 ? "user" : "assistant", + content: [{ type: "text", text: `m${i} ${"x".repeat(TEXT_BYTES)}` }], + timestamp: i + 1, + }, + now: i + 1, + }, + ); + } const res = await rpcReq<{ messages?: unknown[] }>(ws, "chat.history", { - sessionKey: "main", + sessionKey: SESSION_KEY, limit: 1000, }); diff --git a/src/gateway/server-methods/chat-transcript-inject.ts b/src/gateway/server-methods/chat-transcript-inject.ts index bcf699018258..db2cddb6d4ae 100644 --- a/src/gateway/server-methods/chat-transcript-inject.ts +++ b/src/gateway/server-methods/chat-transcript-inject.ts @@ -1,11 +1,7 @@ // Chat transcript injection appends gateway-authored assistant rows while // preserving agent-session parent links and transcript update notifications. import type { SessionManager } from "../../agents/sessions/session-manager.js"; -import { - findTranscriptEvent, - persistSessionTranscriptTurn, - type TranscriptEvent, -} from "../../config/sessions/session-accessor.js"; +import { persistSessionTranscriptTurn } from "../../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { formatErrorMessage } from "../../infra/errors.js"; @@ -57,66 +53,6 @@ function resolveInjectedAssistantContent(params: { return [{ type: "text", text: `${labelPrefix}${params.message}` }]; } -function transcriptEventRecord(event: TranscriptEvent): Record<string, unknown> | undefined { - return event && typeof event === "object" && !Array.isArray(event) - ? (event as Record<string, unknown>) - : undefined; -} - -function transcriptEventMessage(event: TranscriptEvent): Record<string, unknown> | undefined { - const message = transcriptEventRecord(event)?.message; - return message && typeof message === "object" && !Array.isArray(message) - ? (message as Record<string, unknown>) - : undefined; -} - -function transcriptEventId(event: TranscriptEvent): string | undefined { - const id = transcriptEventRecord(event)?.id; - return typeof id === "string" && id.trim().length > 0 ? id : undefined; -} - -type InjectedAssistantIdempotencyTarget = { - agentId?: string; - sessionFile?: string; - sessionId?: string; - sessionKey?: string; - storePath?: string; -}; - -async function findInjectedAssistantMessageByIdempotencyKey(params: { - idempotencyKey: string; - target: InjectedAssistantIdempotencyTarget; -}): Promise<{ messageId: string; message: Record<string, unknown> } | undefined> { - if (!params.target.sessionId || !params.target.sessionKey) { - return undefined; - } - // The in-lock duplicate check resolves through the already-resolved - // sessionFile when present so the lookup reads the file being appended to. - // findTranscriptEvent scans newest-first with early exit, keeping the hot - // idempotent append path from materializing the whole transcript. - const found = await findTranscriptEvent( - { - ...(params.target.agentId ? { agentId: params.target.agentId } : {}), - ...(params.target.sessionFile ? { sessionFile: params.target.sessionFile } : {}), - sessionId: params.target.sessionId, - sessionKey: params.target.sessionKey, - ...(params.target.storePath ? { storePath: params.target.storePath } : {}), - }, - (candidate) => { - const message = transcriptEventMessage(candidate); - return message?.role === "assistant" && message.idempotencyKey === params.idempotencyKey; - }, - ); - const message = found ? transcriptEventMessage(found.event) : undefined; - if (!message) { - return undefined; - } - // Legacy shipped transcripts can carry assistant rows without top-level ids; - // fall back to the idempotency key so re-issued aborts still dedupe there. - const messageId = (found ? transcriptEventId(found.event) : undefined) ?? params.idempotencyKey; - return { messageId, message }; -} - /** Append a gateway-authored assistant message while preserving transcript parent links. */ export async function appendInjectedAssistantMessageToTranscript(params: { transcriptPath?: string; @@ -187,18 +123,6 @@ export async function appendInjectedAssistantMessageToTranscript(params: { if (!params.transcriptPath && (!params.storePath || !params.sessionId || !params.sessionKey)) { return { ok: false, error: "transcript identity not resolved" }; } - const assistantScopedIdempotency = - params.idempotencyKey && params.storePath && params.sessionId && params.sessionKey - ? { - idempotencyKey: params.idempotencyKey, - target: { - ...(params.agentId ? { agentId: params.agentId } : {}), - sessionId: params.sessionId, - sessionKey: params.sessionKey, - storePath: params.storePath, - }, - } - : undefined; const turn = await persistSessionTranscriptTurn( { sessionKey: params.sessionKey ?? "", @@ -214,30 +138,15 @@ export async function appendInjectedAssistantMessageToTranscript(params: { messages: [ { message: messageBody, - idempotencyLookup: assistantScopedIdempotency ? "caller-checked" : "scan", + idempotencyLookup: "scan-assistant", now, useRawWhenLinear: true, - shouldAppend: assistantScopedIdempotency - ? async (target) => - !(await findInjectedAssistantMessageByIdempotencyKey({ - idempotencyKey: assistantScopedIdempotency.idempotencyKey, - target, - })) - : undefined, }, ], }, ); const appended = turn.messages[0]; if (!appended) { - if (assistantScopedIdempotency) { - const existing = await findInjectedAssistantMessageByIdempotencyKey( - assistantScopedIdempotency, - ); - if (existing) { - return { ok: true, messageId: existing.messageId, message: existing.message }; - } - } return { ok: false, error: "gateway-injected assistant message was not appended" }; } return { diff --git a/src/gateway/server-methods/chat.abort-persistence.test.ts b/src/gateway/server-methods/chat.abort-persistence.test.ts index 8e26f6bd6f0f..18a3c423f532 100644 --- a/src/gateway/server-methods/chat.abort-persistence.test.ts +++ b/src/gateway/server-methods/chat.abort-persistence.test.ts @@ -2,12 +2,18 @@ * Tests persistence effects when chat abort requests complete. */ import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; -import { CURRENT_SESSION_VERSION } from "openclaw/plugin-sdk/agent-sessions"; -import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; -import { cleanupTempDirs, makeTempDir } from "../../../test/helpers/temp-dir.js"; -import { replaceSessionEntry } from "../../config/sessions/session-accessor.js"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + appendTranscriptMessageSync, + loadTranscriptEvents, + replaceSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; import { onAgentEvent, resetAgentEventsForTest } from "../../infra/agent-events.js"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; import { createActiveRun, createChatAbortContext, @@ -20,6 +26,7 @@ type TranscriptLine = { const sessionEntryState = vi.hoisted(() => ({ transcriptPath: "", + storePath: "", sessionId: "", hasEntry: true, canonicalKey: "main", @@ -36,7 +43,7 @@ vi.mock("../session-utils.js", async () => { sessionEntryState.loadCalls.push({ sessionKey, opts }); return { cfg: sessionEntryState.cfg, - storePath: path.join(path.dirname(sessionEntryState.transcriptPath), "sessions.json"), + storePath: sessionEntryState.storePath, entry: sessionEntryState.hasEntry ? { sessionId: sessionEntryState.sessionId, @@ -51,31 +58,20 @@ vi.mock("../session-utils.js", async () => { const { chatHandlers } = await import("./chat.js"); -async function writeTranscriptHeader(transcriptPath: string, sessionId: string) { - const header = { - type: "session", - version: CURRENT_SESSION_VERSION, - id: sessionId, - timestamp: new Date(0).toISOString(), - cwd: "/tmp", - }; - await fs.writeFile(transcriptPath, `${JSON.stringify(header)}\n`, "utf-8"); -} +const transcriptFixtures = new Map<string, { sessionId: string; storePath: string }>(); +const fixtureDirs = new Set<string>(); async function readTranscriptLines(transcriptPath: string): Promise<TranscriptLine[]> { - const raw = await fs.readFile(transcriptPath, "utf-8"); - const lines: TranscriptLine[] = []; - for (const line of raw.split(/\r?\n/)) { - if (line.trim().length === 0) { - continue; - } - try { - lines.push(JSON.parse(line) as TranscriptLine); - } catch { - lines.push({}); - } + const fixture = transcriptFixtures.get(transcriptPath); + if (!fixture) { + throw new Error(`unknown transcript fixture: ${transcriptPath}`); } - return lines; + return (await loadTranscriptEvents({ + agentId: "main", + sessionId: fixture.sessionId, + sessionKey: "main", + storePath: fixture.storePath, + })) as TranscriptLine[]; } function collectMessagesWithIdempotencyKey( @@ -156,47 +152,90 @@ function expectPersistedAbortMessage( expect(abort.runId).toBe(expected.runId); } -function setMockSessionEntry(transcriptPath: string, sessionId: string, hasEntry = true) { - sessionEntryState.transcriptPath = transcriptPath; - sessionEntryState.sessionId = sessionId; - sessionEntryState.hasEntry = hasEntry; +function setMockSessionEntry(params: { + sessionId: string; + storePath: string; + transcriptPath: string; + hasEntry?: boolean; +}) { + sessionEntryState.transcriptPath = params.transcriptPath; + sessionEntryState.storePath = params.storePath; + sessionEntryState.sessionId = params.sessionId; + sessionEntryState.hasEntry = params.hasEntry ?? true; sessionEntryState.canonicalKey = "main"; sessionEntryState.cfg = {}; sessionEntryState.loadCalls = []; } -const abortTempDirs: string[] = []; - async function createTranscriptFixture(prefix: string) { - const dir = makeTempDir(abortTempDirs, prefix); + const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + fixtureDirs.add(dir); const sessionId = "sess-main"; - const transcriptPath = path.join(dir, `${sessionId}.jsonl`); - await writeTranscriptHeader(transcriptPath, sessionId); + const storePath = path.join(dir, "sessions.json"); + const transcriptPath = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); // The accessor resolves transcript targets from the persisted store, so the // fixture seeds a real entry instead of relying on the mocked gateway wrapper. await replaceSessionEntry( - { agentId: "main", sessionKey: "main", storePath: path.join(dir, "sessions.json") }, + { agentId: "main", sessionKey: "main", storePath }, { sessionId, sessionFile: transcriptPath, updatedAt: Date.now() }, ); - setMockSessionEntry(transcriptPath, sessionId); - return { transcriptPath, sessionId }; + transcriptFixtures.set(transcriptPath, { sessionId, storePath }); + setMockSessionEntry({ transcriptPath, storePath, sessionId }); + return { transcriptPath, sessionId, storePath }; +} + +function appendTranscriptMessage(params: { + idempotencyKey: string; + message: Record<string, unknown>; + sessionId: string; + storePath: string; +}) { + appendTranscriptMessageSync( + { + agentId: "main", + sessionId: params.sessionId, + sessionKey: "main", + storePath: params.storePath, + }, + { + idempotencyLookup: "caller-checked", + message: { + ...params.message, + idempotencyKey: params.idempotencyKey, + }, + now: 1, + }, + ); } async function createMissingEntryFixture(prefix: string) { - const dir = makeTempDir(abortTempDirs, prefix); - const transcriptPath = path.join(dir, "missing.jsonl"); + const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + fixtureDirs.add(dir); + const storePath = path.join(dir, "sessions.json"); const sessionId = "client-supplied-session"; - setMockSessionEntry(transcriptPath, sessionId, false); + const transcriptPath = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + transcriptFixtures.set(transcriptPath, { sessionId, storePath }); + setMockSessionEntry({ transcriptPath, storePath, sessionId, hasEntry: false }); return { sessionId }; } -afterAll(() => { - cleanupTempDirs(abortTempDirs); -}); - -afterEach(() => { +afterEach(async () => { vi.restoreAllMocks(); resetAgentEventsForTest(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + transcriptFixtures.clear(); + const dirs = [...fixtureDirs]; + fixtureDirs.clear(); + await Promise.all(dirs.map((dir) => fs.rm(dir, { recursive: true, force: true }))); }); describe("chat abort transcript persistence", () => { @@ -255,27 +294,21 @@ describe("chat abort transcript persistence", () => { }); it("does not let non-assistant idempotency collisions suppress abort partial persistence", async () => { - const { transcriptPath, sessionId } = await createTranscriptFixture( + const { transcriptPath, sessionId, storePath } = await createTranscriptFixture( "openclaw-chat-abort-idempotency-collision-", ); const runId = "idem-abort-collision"; const idempotencyKey = `${runId}:assistant`; - await fs.appendFile( - transcriptPath, - `${JSON.stringify({ - type: "message", - id: "user-message-with-colliding-key", - parentId: null, - timestamp: new Date(0).toISOString(), - message: { - role: "user", - content: "colliding user key", - timestamp: 1, - idempotencyKey, - }, - })}\n`, - "utf-8", - ); + appendTranscriptMessage({ + idempotencyKey, + sessionId, + storePath, + message: { + role: "user", + content: "colliding user key", + timestamp: 1, + }, + }); const respond = vi.fn(); const context = createChatAbortContext({ @@ -306,65 +339,6 @@ describe("chat abort transcript persistence", () => { }); }); - it("dedupes legacy assistant transcript entries without top-level ids", async () => { - const { transcriptPath, sessionId } = await createTranscriptFixture( - "openclaw-chat-abort-legacy-idempotency-", - ); - const runId = "idem-abort-legacy"; - const idempotencyKey = `${runId}:assistant`; - await fs.appendFile( - transcriptPath, - `${JSON.stringify({ - type: "message", - timestamp: new Date(0).toISOString(), - message: { - role: "assistant", - content: [{ type: "text", text: "legacy partial" }], - timestamp: 1, - stopReason: "stop", - api: "openai-responses", - provider: "openclaw", - model: "gateway-injected", - idempotencyKey, - openclawAbort: { - aborted: true, - origin: "rpc", - runId, - }, - }, - })}\n`, - "utf-8", - ); - - const respond = vi.fn(); - const context = createChatAbortContext({ - chatAbortControllers: new Map([[runId, createActiveRun("main", { sessionId })]]), - chatRunBuffers: new Map([[runId, "Duplicate partial"]]), - chatDeltaSentAt: new Map([[runId, Date.now()]]), - logGateway: { warn: vi.fn() }, - }); - - await invokeChatAbortHandler({ - handler: chatHandlers["chat.abort"], - context, - request: { sessionKey: "main", runId }, - respond, - }); - - const lines = await readTranscriptLines(transcriptPath); - const assistantMessages = collectMessagesWithIdempotencyKey(lines, idempotencyKey).filter( - (message) => message.role === "assistant", - ); - - expect(assistantMessages).toHaveLength(1); - expectPersistedAbortMessage(assistantMessages[0], { - idempotencyKey, - origin: "rpc", - runId, - stopReason: "stop", - }); - }); - it("persists session-scoped abort partials with rpc metadata", async () => { const { transcriptPath, sessionId } = await createTranscriptFixture( "openclaw-chat-abort-session-", @@ -1267,7 +1241,7 @@ describe("chat abort transcript persistence", () => { describe("chat.abort session identity matching", () => { it("matches an active run by stored sessionId when sessionKey differs", async () => { const storedSessionId = "sess-stored-abc"; - setMockSessionEntry("", storedSessionId); + setMockSessionEntry({ transcriptPath: "", storePath: "", sessionId: storedSessionId }); const runId = "embedded-run-1"; const active = createActiveRun("agent:main:embedded-key", { sessionId: storedSessionId }); const context = createChatAbortContext({ @@ -1290,7 +1264,7 @@ describe("chat.abort session identity matching", () => { }); it("does not match a run whose sessionId differs from the stored entry", async () => { - setMockSessionEntry("", "sess-stored-xyz"); + setMockSessionEntry({ transcriptPath: "", storePath: "", sessionId: "sess-stored-xyz" }); const runId = "embedded-run-2"; const active = createActiveRun("agent:main:other-key", { sessionId: "sess-different" }); const context = createChatAbortContext({ diff --git a/src/gateway/server-methods/chat.directive-tags.test.ts b/src/gateway/server-methods/chat.directive-tags.test.ts index 18f33f32ed0a..34695a835ee5 100644 --- a/src/gateway/server-methods/chat.directive-tags.test.ts +++ b/src/gateway/server-methods/chat.directive-tags.test.ts @@ -19,14 +19,20 @@ import { setReplyPayloadMetadata, } from "../../auto-reply/reply-payload.js"; import type { MsgContext } from "../../auto-reply/templating.js"; -import { replaceSessionEntry } from "../../config/sessions/session-accessor.js"; -import { appendSessionTranscriptMessage } from "../../config/sessions/transcript-append.js"; +import { + appendTranscriptMessage, + loadSessionEntry as loadSqliteSessionEntry, + loadTranscriptEventsSync, + replaceSessionEntry, + type SessionAccessScope, + type SessionTranscriptReadScope, + upsertSessionEntry, +} from "../../config/sessions/session-accessor.js"; import { resolveMirroredTranscriptText } from "../../config/sessions/transcript-mirror.js"; import { getAgentRunContext } from "../../infra/agent-events.js"; import { runExclusiveSessionLifecycleMutation } from "../../sessions/session-lifecycle-admission.js"; import { createDeferred } from "../../test-utils/deferred.js"; import { withEnvAsync } from "../../test-utils/env.js"; -import { readSessionTranscriptIndex } from "../session-transcript-index.fs.js"; import type { GatewayRequestContext } from "./types.js"; const mockState = vi.hoisted(() => ({ @@ -93,7 +99,10 @@ const mockState = vi.hoisted(() => ({ lastDispatchUserTurnInput: undefined as unknown, modelCatalog: null as ModelCatalogEntry[] | null, emittedTranscriptUpdates: [] as Array<{ - sessionFile: string; + sessionFile?: string; + target?: { agentId: string; sessionId: string; sessionKey: string }; + agentId?: string; + sessionId?: string; sessionKey?: string; message?: unknown; messageId?: string; @@ -121,7 +130,17 @@ const mockState = vi.hoisted(() => ({ })); function readTranscriptJsonLines(transcriptPath: string): Array<Record<string, unknown>> { + const sqliteEvents = loadTranscriptEventsSync(transcriptScope()).filter( + (event): event is Record<string, unknown> => + Boolean(event) && typeof event === "object" && !Array.isArray(event), + ); + if (sqliteEvents.length > 0) { + return sqliteEvents; + } const entries: Array<Record<string, unknown>> = []; + if (!fs.existsSync(transcriptPath)) { + return entries; + } for (const line of fs.readFileSync(transcriptPath, "utf-8").split("\n")) { if (line.length > 0) { entries.push(JSON.parse(line) as Record<string, unknown>); @@ -358,7 +377,23 @@ vi.mock("../../plugins/hook-runner-global.js", () => ({ vi.mock("../../sessions/transcript-events.js", () => ({ emitSessionTranscriptUpdate: vi.fn( (update: { - sessionFile: string; + sessionFile?: string; + target?: { agentId: string; sessionId: string; sessionKey: string }; + agentId?: string; + sessionId?: string; + sessionKey?: string; + message?: unknown; + messageId?: string; + }) => { + mockState.emittedTranscriptUpdates.push(update); + }, + ), + emitInternalSessionTranscriptUpdate: vi.fn( + (update: { + sessionFile?: string; + target?: { agentId: string; sessionId: string; sessionKey: string }; + agentId?: string; + sessionId?: string; sessionKey?: string; message?: unknown; messageId?: string; @@ -476,6 +511,20 @@ async function createTranscriptFixture(prefix: string) { return dir; } +function createSqliteTranscriptFixture(prefix: string) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + mockState.transcriptPath = path.join(dir, "sess.jsonl"); + return dir; +} + +async function createGatewayUserTurnSqliteFixture(prefix: string) { + const dir = createSqliteTranscriptFixture(prefix); + await seedSqliteSessionEntry({ + updatedAt: 1, + }); + return dir; +} + async function withTranscriptFixtureState( prefix: string, run: (fixtureDir: string) => Promise<void>, @@ -484,14 +533,50 @@ async function withTranscriptFixtureState( await withEnvAsync({ OPENCLAW_STATE_DIR: fixtureDir }, async () => await run(fixtureDir)); } +async function withSqliteTranscriptFixtureState( + prefix: string, + run: (fixtureDir: string) => Promise<void>, +): Promise<void> { + const fixtureDir = createSqliteTranscriptFixture(prefix); + await withEnvAsync({ OPENCLAW_STATE_DIR: fixtureDir }, async () => await run(fixtureDir)); +} + +function transcriptScope(): SessionTranscriptReadScope { + return { + agentId: "main", + sessionId: mockState.sessionId, + sessionKey: "main", + storePath: path.join(path.dirname(mockState.transcriptPath), "sessions.json"), + }; +} + +function sessionEntryScope(): SessionAccessScope { + return { + agentId: "main", + sessionKey: "main", + storePath: path.join(path.dirname(mockState.transcriptPath), "sessions.json"), + }; +} + +async function seedSqliteSessionEntry(entry: Record<string, unknown> = {}): Promise<void> { + await upsertSessionEntry(sessionEntryScope(), { + sessionId: mockState.sessionId, + ...entry, + }); +} + +function readSqliteMainSessionEntry(): Record<string, any> | undefined { + return loadSqliteSessionEntry(sessionEntryScope()) as Record<string, any> | undefined; +} + async function appendSourceReplyMirrorEntry(params: { idempotencyKey?: string; text: string; provider?: string; model?: string; }) { - await appendSessionTranscriptMessage({ - transcriptPath: mockState.transcriptPath, + await appendTranscriptMessage(transcriptScope(), { + idempotencyLookup: "scan", now: 0, message: { role: "assistant", @@ -521,17 +606,14 @@ async function appendSourceReplyMirrorEntry(params: { } async function readActiveAssistantTranscriptMessages(): Promise<Array<Record<string, unknown>>> { - const index = await readSessionTranscriptIndex(mockState.transcriptPath); - return ( - index?.entries - .map((entry) => entry.record.message) - .filter( - (message): message is Record<string, unknown> => - typeof message === "object" && - message !== null && - (message as { role?: unknown }).role === "assistant", - ) ?? [] - ); + return readTranscriptJsonLines(mockState.transcriptPath) + .map((entry) => entry.message) + .filter( + (message): message is Record<string, unknown> => + typeof message === "object" && + message !== null && + (message as { role?: unknown }).role === "assistant", + ); } function extractFirstTextBlock(payload: unknown): string | undefined { @@ -634,6 +716,16 @@ function userUpdateMessage( : undefined; } +function expectUserUpdateIdentity(update: ReturnType<typeof findUserUpdate>) { + expect(update?.target).toEqual({ + agentId: "main", + sessionId: mockState.sessionId, + sessionKey: "main", + }); + expect(update?.sessionKey).toBe("main"); + expect(update?.agentId).toBe("main"); +} + function readPersistedUserMessages(): Array<Record<string, unknown>> { return readTranscriptJsonLines(mockState.transcriptPath) .map((entry) => entry.message) @@ -982,6 +1074,27 @@ describe("chat directive tag stripping for non-streaming final payloads", () => expect(persistedAssistant?.idempotencyKey).toBe("idem-final-mirror"); }); + it("persists non-agent delivery mirrors to SQLite without creating active JSONL", async () => { + await withSqliteTranscriptFixtureState("openclaw-chat-send-final-sqlite-", async () => { + mockState.finalText = "sqlite mirror text"; + const respond = vi.fn(); + const context = createChatContext(); + + await runNonStreamingChatSend({ + context, + respond, + idempotencyKey: "idem-final-sqlite", + expectBroadcast: false, + }); + + expect(fs.existsSync(mockState.transcriptPath)).toBe(false); + const assistantEntries = await readActiveAssistantTranscriptMessages(); + expect(assistantEntries).toHaveLength(1); + expect(assistantEntries[0]?.idempotencyKey).toBe("idem-final-sqlite"); + expect(JSON.stringify(assistantEntries[0])).toContain("sqlite mirror text"); + }); + }); + it("registers tool-event recipients for clients advertising tool-events capability", async () => { await createTranscriptFixture("openclaw-chat-send-tool-events-"); mockState.finalText = "ok"; @@ -1569,19 +1682,11 @@ describe("chat directive tag stripping for non-streaming final payloads", () => const mirrorIdempotencyKey = "idem-agent-source-reply-media:internal-source-reply:0"; const updatedAt = Date.parse("2026-05-18T11:00:00.000Z"); const rewrittenAt = Date.parse("2026-05-18T11:05:00.000Z"); - const storePath = path.join(path.dirname(mockState.transcriptPath), "sessions.json"); - fs.writeFileSync( - storePath, - JSON.stringify({ - main: { - sessionId: mockState.sessionId, - sessionFile: mockState.transcriptPath, - updatedAt, - status: "done", - }, - }), - "utf-8", - ); + await seedSqliteSessionEntry({ + sessionFile: mockState.transcriptPath, + updatedAt, + status: "done", + }); await appendSourceReplyMirrorEntry({ idempotencyKey: mirrorIdempotencyKey, text: "Codex source reply with media", @@ -1641,13 +1746,10 @@ describe("chat directive tag stripping for non-streaming final payloads", () => expect(assistantEntries[0]?.idempotencyKey).toBe(mirrorIdempotencyKey); expect(JSON.stringify(assistantEntries[0])).toContain("/api/chat/media/outgoing/"); expect(JSON.stringify(assistantEntries[0])).not.toContain(mediaUrl); - const store = JSON.parse(fs.readFileSync(storePath, "utf-8")) as Record< - string, - { updatedAt?: number; status?: string } - >; - expect(store.main?.updatedAt).toBeGreaterThanOrEqual(rewrittenAt); - expect(store.main?.updatedAt).toBeGreaterThan(updatedAt); - expect(store.main?.status).toBe("done"); + const entry = readSqliteMainSessionEntry(); + expect(entry?.updatedAt).toBeGreaterThanOrEqual(rewrittenAt); + expect(entry?.updatedAt).toBeGreaterThan(updatedAt); + expect(entry?.status).toBe("done"); } finally { vi.useRealTimers(); } @@ -1655,6 +1757,59 @@ describe("chat directive tag stripping for non-streaming final payloads", () => ); }); + it("rewrites source reply mirrors in SQLite without creating active JSONL", async () => { + await withSqliteTranscriptFixtureState( + "openclaw-chat-send-agent-source-reply-sqlite-", + async (fixtureDir) => { + const mediaUrl = `data:image/png;base64,${TINY_PNG_BASE64}`; + const savedImagePath = path.join(fixtureDir, "source-reply-sqlite.png"); + fs.writeFileSync(savedImagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + mockState.savedMediaResults = [{ path: savedImagePath, contentType: "image/png" }]; + const mirrorIdempotencyKey = "idem-source-reply-sqlite:internal-source-reply:0"; + await appendSourceReplyMirrorEntry({ + idempotencyKey: mirrorIdempotencyKey, + text: "SQLite source reply with media", + }); + mockState.triggerAgentRunStart = true; + mockState.dispatchedReplies = [ + { + kind: "final", + payload: setReplyPayloadMetadata( + { + text: "SQLite source reply with media", + mediaUrls: [mediaUrl], + }, + { + sourceReplyTranscriptMirror: { + sessionKey: "main", + text: "SQLite source reply with media", + mediaUrls: [mediaUrl], + idempotencyKey: mirrorIdempotencyKey, + }, + }, + ), + }, + ]; + const respond = vi.fn(); + const context = createChatContext(); + + await runNonStreamingChatSend({ + context, + respond, + idempotencyKey: "idem-source-reply-sqlite", + message: "hello from codex", + }); + + expect(fs.existsSync(mockState.transcriptPath)).toBe(false); + const assistantEntries = await readActiveAssistantTranscriptMessages(); + expect(assistantEntries).toHaveLength(1); + expect(assistantEntries[0]?.idempotencyKey).toBe(mirrorIdempotencyKey); + expect(JSON.stringify(assistantEntries[0])).toContain("/api/chat/media/outgoing/"); + expect(JSON.stringify(assistantEntries[0])).not.toContain(mediaUrl); + }, + ); + }); + it("backs source reply media with an equivalent deduped delivery mirror", async () => { await withTranscriptFixtureState( "openclaw-chat-send-agent-source-reply-deduped-", @@ -2293,7 +2448,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); it("broadcasts returned agent-run error payloads after an agent starts", async () => { - await createTranscriptFixture("openclaw-chat-send-agent-returned-error-"); + await createGatewayUserTurnSqliteFixture("openclaw-chat-send-agent-returned-error-"); const errorMessage = "LLM idle timeout (120s): no response from model"; mockState.triggerAgentRunStart = true; mockState.dispatchedReplies = [ @@ -2328,7 +2483,8 @@ describe("chat directive tag stripping for non-streaming final payloads", () => status: "error", summary: errorMessage, }); - expect(findUserUpdate()).toBeDefined(); + const userUpdate = findUserUpdate(); + expectUserUpdateIdentity(userUpdate); const assistantUpdates = mockState.emittedTranscriptUpdates.filter( (update) => typeof update.message === "object" && @@ -3382,6 +3538,30 @@ describe("chat directive tag stripping for non-streaming final payloads", () => } }); + it("chat.inject persists to SQLite without creating active JSONL", async () => { + await withSqliteTranscriptFixtureState("openclaw-chat-inject-sqlite-", async () => { + const respond = vi.fn(); + const context = createChatContext(); + + await chatHandlers["chat.inject"]({ + params: { sessionKey: "main", message: "hello sqlite inject" }, + respond, + req: {} as never, + client: null as never, + isWebchatConnect: () => false, + context: context as GatewayRequestContext, + }); + + const [ok, payload] = lastRespondCall(respond) ?? []; + expect(ok).toBe(true); + expect(payload?.ok).toBe(true); + expect(fs.existsSync(mockState.transcriptPath)).toBe(false); + const assistantEntries = await readActiveAssistantTranscriptMessages(); + expect(assistantEntries).toHaveLength(1); + expect(JSON.stringify(assistantEntries[0])).toContain("hello sqlite inject"); + }); + }); + it("chat.send non-streaming final keeps message defined for directive-only assistant text", async () => { await createTranscriptFixture("openclaw-chat-send-directive-only-"); mockState.finalText = "[[reply_to_current]]"; @@ -3536,19 +3716,11 @@ describe("chat directive tag stripping for non-streaming final payloads", () => const fixtureDir = await createTranscriptFixture("openclaw-chat-inject-registry-marker-"); const updatedAt = Date.parse("2026-05-18T11:00:00.000Z"); const appendedAt = Date.parse("2026-05-18T11:05:00.000Z"); - const storePath = path.join(path.dirname(mockState.transcriptPath), "sessions.json"); - fs.writeFileSync( - storePath, - JSON.stringify({ - main: { - sessionId: mockState.sessionId, - sessionFile: mockState.transcriptPath, - updatedAt, - status: "done", - }, - }), - "utf-8", - ); + await seedSqliteSessionEntry({ + sessionFile: mockState.transcriptPath, + updatedAt, + status: "done", + }); const respond = vi.fn(); const context = createChatContext(); vi.useFakeTimers({ toFake: ["Date"] }); @@ -3568,12 +3740,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () => const response = lastRespondCall(respond); expect(response?.[0]).toBe(true); - const store = JSON.parse(fs.readFileSync(storePath, "utf-8")) as Record< - string, - { updatedAt?: number; status?: string } - >; - expect(store.main?.updatedAt).toBe(appendedAt); - expect(store.main?.status).toBe("done"); + const entry = readSqliteMainSessionEntry(); + expect(entry?.updatedAt).toBe(appendedAt); + expect(entry?.status).toBe("done"); } finally { vi.useRealTimers(); fs.rmSync(fixtureDir, { recursive: true, force: true }); @@ -4691,7 +4860,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); it("emits a user transcript update when hooks pass and the started agent throws before runtime persistence", async () => { - await createTranscriptFixture("openclaw-chat-send-user-transcript-gate-pass-error-"); + await createGatewayUserTurnSqliteFixture("openclaw-chat-send-user-transcript-gate-pass-error-"); mockState.triggerAgentRunStart = true; mockState.hasBeforeAgentRunHooks = true; mockState.dispatchErrorAfterAgentRunStart = new Error("model unavailable"); @@ -4710,8 +4879,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => expect(context.dedupe.get("chat:idem-user-transcript-gate-pass-error")?.ok).toBe(false); const userUpdate = findUserUpdate(); const message = userUpdateMessage(userUpdate); - expect(userUpdate?.sessionFile.endsWith("sess.jsonl")).toBe(true); - expect(userUpdate?.sessionKey).toBe("main"); + expectUserUpdateIdentity(userUpdate); expect(message?.role).toBe("user"); expect(message?.content).toBe("prompt allowed before model error"); }); @@ -4955,7 +5123,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); it("waits for the user transcript update before final broadcast on non-agent attachment sends", async () => { - await createTranscriptFixture("openclaw-chat-send-no-agent-images-order-"); + await createGatewayUserTurnSqliteFixture("openclaw-chat-send-no-agent-images-order-"); mockState.finalText = "ok"; mockState.savedMediaResults = [ { path: "/tmp/chat-send-image-a.png", contentType: "image/png" }, @@ -4990,9 +5158,11 @@ describe("chat directive tag stripping for non-streaming final payloads", () => await waitForAssertion(() => { expect((context.broadcast as unknown as ReturnType<typeof vi.fn>).mock.calls.length).toBe(1); - if (findUserUpdate()?.message === undefined) { + const userUpdate = findUserUpdate(); + if (userUpdate?.message === undefined) { throw new Error("Expected streamed user transcript update message"); } + expectUserUpdateIdentity(userUpdate); }); }); @@ -6329,7 +6499,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); it("emits a user transcript update when chat.send completes without an agent run", async () => { - await createTranscriptFixture("openclaw-chat-send-user-transcript-no-run-"); + await createGatewayUserTurnSqliteFixture("openclaw-chat-send-user-transcript-no-run-"); mockState.finalText = "ok"; const respond = vi.fn(); const context = createChatContext(); @@ -6344,8 +6514,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => const userUpdate = findUserUpdate(); const message = userUpdateMessage(userUpdate); - expect(userUpdate?.sessionFile.endsWith("sess.jsonl")).toBe(true); - expect(userUpdate?.sessionKey).toBe("main"); + expectUserUpdateIdentity(userUpdate); expect(message?.role).toBe("user"); expect(message?.content).toBe("quick command"); expect(typeof message?.timestamp).toBe("number"); @@ -6354,7 +6523,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); it("emits a user transcript update when chat.send fails before an agent run starts", async () => { - await createTranscriptFixture("openclaw-chat-send-user-transcript-error-no-run-"); + await createGatewayUserTurnSqliteFixture("openclaw-chat-send-user-transcript-error-no-run-"); mockState.dispatchError = new Error("upstream unavailable"); const respond = vi.fn(); const context = createChatContext(); @@ -6371,8 +6540,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => expect(context.dedupe.get("chat:idem-user-transcript-error-no-run")?.ok).toBe(false); const userUpdate = findUserUpdate(); const message = userUpdateMessage(userUpdate); - expect(userUpdate?.sessionFile.endsWith("sess.jsonl")).toBe(true); - expect(userUpdate?.sessionKey).toBe("main"); + expectUserUpdateIdentity(userUpdate); expect(message?.role).toBe("user"); expect(message?.content).toBe("hello from failed dispatch"); expect(typeof message?.timestamp).toBe("number"); @@ -6382,7 +6550,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); it("emits a user transcript update when a slash-prefixed turn fails before command delivery", async () => { - await createTranscriptFixture("openclaw-chat-send-user-transcript-slash-error-no-run-"); + await createGatewayUserTurnSqliteFixture( + "openclaw-chat-send-user-transcript-slash-error-no-run-", + ); mockState.dispatchError = new Error("slash command continued into unavailable runtime"); const respond = vi.fn(); const context = createChatContext(); @@ -6399,8 +6569,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => expect(context.dedupe.get("chat:idem-user-transcript-slash-error-no-run")?.ok).toBe(false); const userUpdate = findUserUpdate(); const message = userUpdateMessage(userUpdate); - expect(userUpdate?.sessionFile.endsWith("sess.jsonl")).toBe(true); - expect(userUpdate?.sessionKey).toBe("main"); + expectUserUpdateIdentity(userUpdate); expect(message?.role).toBe("user"); expect(message?.content).toBe("/unknown keep this user turn"); const persistedUser = readPersistedUserMessages()[0]; @@ -6409,7 +6578,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); it("does not duplicate fallback user transcript rows when chat.send is replayed", async () => { - await createTranscriptFixture("openclaw-chat-send-user-transcript-error-replay-"); + await createGatewayUserTurnSqliteFixture("openclaw-chat-send-user-transcript-error-replay-"); mockState.dispatchError = new Error("upstream unavailable"); await runNonStreamingChatSend({ @@ -6443,7 +6612,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); it("emits a user transcript update on pre-start failures even when before_agent_run hooks exist", async () => { - await createTranscriptFixture("openclaw-chat-send-user-transcript-error-hook-pre-start-"); + await createGatewayUserTurnSqliteFixture( + "openclaw-chat-send-user-transcript-error-hook-pre-start-", + ); mockState.hasBeforeAgentRunHooks = true; mockState.dispatchError = new Error("resolver unavailable"); const respond = vi.fn(); @@ -6461,15 +6632,14 @@ describe("chat directive tag stripping for non-streaming final payloads", () => expect(context.dedupe.get("chat:idem-user-transcript-error-hook-pre-start")?.ok).toBe(false); const userUpdate = findUserUpdate(); const message = userUpdateMessage(userUpdate); - expect(userUpdate?.sessionFile.endsWith("sess.jsonl")).toBe(true); - expect(userUpdate?.sessionKey).toBe("main"); + expectUserUpdateIdentity(userUpdate); expect(message?.role).toBe("user"); expect(message?.content).toBe("hello before hooked startup failure"); }); }); it("emits a user transcript update when chat.send fails after agent start but before runtime persistence", async () => { - await createTranscriptFixture( + await createGatewayUserTurnSqliteFixture( "openclaw-chat-send-user-transcript-error-before-runtime-persist-", ); mockState.triggerAgentRunStart = true; @@ -6491,8 +6661,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => ); const userUpdate = findUserUpdate(); const message = userUpdateMessage(userUpdate); - expect(userUpdate?.sessionFile.endsWith("sess.jsonl")).toBe(true); - expect(userUpdate?.sessionKey).toBe("main"); + expectUserUpdateIdentity(userUpdate); expect(message?.role).toBe("user"); expect(message?.content).toBe("hello before cli startup failure"); const persistedUser = readPersistedUserMessages()[0]; @@ -6501,7 +6670,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); it("applies before_message_write redaction to gateway fallback user transcript persistence", async () => { - await createTranscriptFixture("openclaw-chat-send-user-transcript-error-before-write-redact-"); + await createGatewayUserTurnSqliteFixture( + "openclaw-chat-send-user-transcript-error-before-write-redact-", + ); mockState.triggerAgentRunStart = true; mockState.dispatchErrorAfterAgentRunStart = new Error("cli backend unavailable"); mockState.beforeMessageWriteContent = "[redacted by hook]"; @@ -6518,6 +6689,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => await waitForAssertion(() => { const userUpdate = findUserUpdate(); + expectUserUpdateIdentity(userUpdate); const message = userUpdateMessage(userUpdate); expect(message?.content).toBe("[redacted by hook]"); expect(mockState.beforeMessageWriteCalls).toHaveLength(1); @@ -6528,7 +6700,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); it("does not persist gateway fallback user transcripts blocked by before_message_write", async () => { - await createTranscriptFixture("openclaw-chat-send-user-transcript-error-before-write-block-"); + await createGatewayUserTurnSqliteFixture( + "openclaw-chat-send-user-transcript-error-before-write-block-", + ); mockState.triggerAgentRunStart = true; mockState.dispatchErrorAfterAgentRunStart = new Error("cli backend unavailable"); mockState.beforeMessageWriteBlock = true; @@ -6554,7 +6728,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); it("emits a user transcript update when a started agent returns an error before runtime persistence", async () => { - await createTranscriptFixture( + await createGatewayUserTurnSqliteFixture( "openclaw-chat-send-user-transcript-agent-error-no-runtime-persist-", ); mockState.triggerAgentRunStart = true; @@ -6576,15 +6750,14 @@ describe("chat directive tag stripping for non-streaming final payloads", () => ).toBe(false); const userUpdate = findUserUpdate(); const message = userUpdateMessage(userUpdate); - expect(userUpdate?.sessionFile.endsWith("sess.jsonl")).toBe(true); - expect(userUpdate?.sessionKey).toBe("main"); + expectUserUpdateIdentity(userUpdate); expect(message?.role).toBe("user"); expect(message?.content).toBe("hello before agent error payload"); }); }); it("falls back to gateway user persistence when successful runtime persistence fails", async () => { - await createTranscriptFixture( + await createGatewayUserTurnSqliteFixture( "openclaw-chat-send-user-transcript-success-runtime-persist-failed-", ); mockState.triggerAgentRunStart = true; @@ -6608,6 +6781,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => context.dedupe.get("chat:idem-user-transcript-success-runtime-persist-failed")?.ok, ).toBe(true); const userUpdate = findUserUpdate(); + expectUserUpdateIdentity(userUpdate); const message = userUpdateMessage(userUpdate); expect(message?.role).toBe("user"); expect(message?.content).toBe("hello before successful fallback"); @@ -6618,7 +6792,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); it("emits a user transcript update when hooks pass and a started agent returns an error", async () => { - await createTranscriptFixture("openclaw-chat-send-user-transcript-agent-error-hook-pass-"); + await createGatewayUserTurnSqliteFixture( + "openclaw-chat-send-user-transcript-agent-error-hook-pass-", + ); mockState.triggerAgentRunStart = true; mockState.hasBeforeAgentRunHooks = true; mockState.finalPayload = { text: "agent failed before prompt append", isError: true }; @@ -6637,8 +6813,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => expect(context.dedupe.get("chat:idem-user-transcript-agent-error-hook-pass")?.ok).toBe(false); const userUpdate = findUserUpdate(); const message = userUpdateMessage(userUpdate); - expect(userUpdate?.sessionFile.endsWith("sess.jsonl")).toBe(true); - expect(userUpdate?.sessionKey).toBe("main"); + expectUserUpdateIdentity(userUpdate); expect(message?.role).toBe("user"); expect(message?.content).toBe("hello before hooked agent error payload"); }); diff --git a/src/gateway/server-methods/chat.inject.parentid.test.ts b/src/gateway/server-methods/chat.inject.parentid.test.ts index 84b04a51312b..381a5d12ecaa 100644 --- a/src/gateway/server-methods/chat.inject.parentid.test.ts +++ b/src/gateway/server-methods/chat.inject.parentid.test.ts @@ -1,24 +1,65 @@ // Chat transcript parent-id tests protect gateway-injected assistant appends so // compaction history remains connected and transcript listeners receive updates. import fs from "node:fs"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + appendTranscriptMessageSync, + loadTranscriptEvents, + replaceSessionEntry, +} from "../../config/sessions/session-accessor.js"; import { onSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; import { appendInjectedAssistantMessageToTranscript } from "./chat-transcript-inject.js"; -import { createTranscriptFixtureSync } from "./chat.test-helpers.js"; -function readTranscriptLines(transcriptPath: string): string[] { - const lines: string[] = []; - for (const line of fs.readFileSync(transcriptPath, "utf-8").split(/\r?\n/)) { - if (line.length > 0) { - lines.push(line); - } - } - return lines; +type SqliteTranscriptFixture = { + agentId: string; + dir: string; + sessionKey: string; + sessionId: string; + storePath: string; +}; + +async function createSqliteTranscriptFixture(params: { + prefix: string; + sessionId: string; +}): Promise<SqliteTranscriptFixture> { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), params.prefix)); + const sessionKey = "main"; + const agentId = "main"; + const storePath = path.join(dir, "sessions.json"); + await replaceSessionEntry( + { agentId, sessionKey, storePath }, + { sessionId: params.sessionId, updatedAt: Date.now() }, + ); + return { agentId, dir, sessionKey, sessionId: params.sessionId, storePath }; } -async function appendHelloAndRequireId(transcriptPath: string): Promise<string> { +async function cleanupFixture(fixture: { dir: string }) { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + fs.rmSync(fixture.dir, { recursive: true, force: true }); +} + +async function readTranscriptEvents( + fixture: SqliteTranscriptFixture, +): Promise<Record<string, unknown>[]> { + return (await loadTranscriptEvents({ + agentId: fixture.agentId, + sessionId: fixture.sessionId, + sessionKey: fixture.sessionKey, + storePath: fixture.storePath, + })) as Record<string, unknown>[]; +} + +async function appendHelloAndRequireId(fixture: SqliteTranscriptFixture): Promise<string> { const appended = await appendInjectedAssistantMessageToTranscript({ - transcriptPath, + agentId: fixture.agentId, + sessionId: fixture.sessionId, + sessionKey: fixture.sessionKey, + storePath: fixture.storePath, message: "hello", }); expect(appended.ok).toBe(true); @@ -31,94 +72,75 @@ async function appendHelloAndRequireId(transcriptPath: string): Promise<string> return messageId; } -function readLastTranscriptRecord(transcriptPath: string): Record<string, unknown> { - const lines = readTranscriptLines(transcriptPath); - expect(lines.length).toBeGreaterThanOrEqual(2); - return JSON.parse(lines.at(-1) as string) as Record<string, unknown>; -} - -function readLastTranscriptRecordFromTail(transcriptPath: string): Record<string, unknown> { - const size = fs.statSync(transcriptPath).size; - const length = Math.min(size, 64 * 1024); - const buffer = Buffer.allocUnsafe(length); - const fd = fs.openSync(transcriptPath, "r"); - let bytesRead = 0; - try { - bytesRead = fs.readSync(fd, buffer, 0, length, size - length); - } finally { - fs.closeSync(fd); - } - const lines = buffer.subarray(0, bytesRead).toString("utf8").trimEnd().split(/\r?\n/); - return JSON.parse(lines.at(-1) as string) as Record<string, unknown>; +async function readLastTranscriptRecord( + fixture: SqliteTranscriptFixture, +): Promise<Record<string, unknown>> { + const events = await readTranscriptEvents(fixture); + expect(events.length).toBeGreaterThanOrEqual(2); + return events.at(-1) as Record<string, unknown>; } // Guardrail: Gateway-injected assistant transcript messages must attach to the // current leaf with a `parentId` and must not sever compaction history. describe("gateway chat.inject transcript writes", () => { - let oversizedDir = ""; - let oversizedTranscriptPath = ""; - - beforeAll(() => { - const fixture = createTranscriptFixtureSync({ - prefix: "openclaw-chat-inject-large-", - sessionId: "sess-1", - }); - oversizedDir = fixture.dir; - oversizedTranscriptPath = fixture.transcriptPath; - fs.appendFileSync( - oversizedTranscriptPath, - `${JSON.stringify({ - type: "message", - id: "legacy-large-message", - message: { - role: "assistant", - content: [{ type: "text", text: "x".repeat(9 * 1024 * 1024) }], - }, - })}\n`, - "utf-8", - ); - }); - - afterAll(() => { - if (oversizedDir) { - fs.rmSync(oversizedDir, { recursive: true, force: true }); - } - }); - it("appends a agent session entry that includes parentId", async () => { - const { dir, transcriptPath } = createTranscriptFixtureSync({ + const fixture = await createSqliteTranscriptFixture({ prefix: "openclaw-chat-inject-", sessionId: "sess-1", }); try { - await appendHelloAndRequireId(transcriptPath); - const last = readLastTranscriptRecord(transcriptPath); + await appendHelloAndRequireId(fixture); + const last = await readLastTranscriptRecord(fixture); expect(last.type).toBe("message"); - // The regression we saw: raw jsonl appends omitted this field entirely. + // Gateway appends must go through the transcript accessor so parent links + // stay connected for compaction and chat.history projection. expect(Object.hasOwn(last, "parentId")).toBe(true); expect(last).toHaveProperty("id"); expect(last).toHaveProperty("message"); } finally { - fs.rmSync(dir, { recursive: true, force: true }); + await cleanupFixture(fixture); } }); - it("uses raw append for oversized append-only transcripts", async () => { - const sizeBefore = fs.statSync(oversizedTranscriptPath).size; - const messageId = await appendHelloAndRequireId(oversizedTranscriptPath); - const last = readLastTranscriptRecordFromTail(oversizedTranscriptPath); + it("preserves parent links after an oversized transcript row", async () => { + const fixture = await createSqliteTranscriptFixture({ + prefix: "openclaw-chat-inject-large-", + sessionId: "sess-1", + }); - expect(fs.statSync(oversizedTranscriptPath).size).toBeGreaterThan(sizeBefore); - expect(last.type).toBe("message"); - expect(last).toHaveProperty("id", messageId); - expect(last).toHaveProperty("message"); - expect(Object.hasOwn(last, "parentId")).toBe(false); + try { + const existing = appendTranscriptMessageSync( + { + agentId: fixture.agentId, + sessionId: fixture.sessionId, + sessionKey: fixture.sessionKey, + storePath: fixture.storePath, + }, + { + message: { + role: "assistant", + content: [{ type: "text", text: "x".repeat(9 * 1024 * 1024) }], + }, + }, + ); + + const messageId = await appendHelloAndRequireId(fixture); + const last = await readLastTranscriptRecord(fixture); + + expect(existing).toBeDefined(); + expect(last.type).toBe("message"); + expect(last).toHaveProperty("id", messageId); + expect(last).toHaveProperty("message"); + expect(last).toHaveProperty("parentId", existing?.messageId); + } finally { + await cleanupFixture(fixture); + } }); it("emits and returns the redacted injected assistant message", async () => { - const { dir, transcriptPath } = createTranscriptFixtureSync({ + const fixture = await createSqliteTranscriptFixture({ prefix: "openclaw-chat-inject-redact-", sessionId: "sess-redact", }); @@ -128,9 +150,10 @@ describe("gateway chat.inject transcript writes", () => { try { const appended = await appendInjectedAssistantMessageToTranscript({ - transcriptPath, + agentId: fixture.agentId, + sessionId: fixture.sessionId, sessionKey: "global", - agentId: "work", + storePath: fixture.storePath, message: `Here is your key: ${fakeApiKey}`, config: { logging: { redactSensitive: "tools" } }, }); @@ -138,16 +161,15 @@ describe("gateway chat.inject transcript writes", () => { expect(appended.ok).toBe(true); expect(JSON.stringify(appended.message)).not.toContain(fakeApiKey); expect(updates).toHaveLength(1); - expect(updates[0]).toMatchObject({ sessionKey: "global", agentId: "work" }); + expect(updates[0]).toMatchObject({ sessionKey: "global", agentId: "main" }); - const lines = readTranscriptLines(transcriptPath); - const last = JSON.parse(lines.at(-1) as string) as { message?: unknown }; + const last = (await readLastTranscriptRecord(fixture)) as { message?: unknown }; expect(JSON.stringify(last.message)).not.toContain(fakeApiKey); expect(updates[0]?.message).toEqual(last.message); expect(JSON.stringify(updates[0]?.message)).not.toContain(fakeApiKey); } finally { unsubscribe(); - fs.rmSync(dir, { recursive: true, force: true }); + await cleanupFixture(fixture); } }); }); diff --git a/src/gateway/server-methods/chat.ts b/src/gateway/server-methods/chat.ts index 825fa0c01cfc..b964256b8e6c 100644 --- a/src/gateway/server-methods/chat.ts +++ b/src/gateway/server-methods/chat.ts @@ -40,13 +40,11 @@ import { resolveAgentWorkspaceDir, resolveSessionAgentId, } from "../../agents/agent-scope.js"; -import { rewriteTranscriptEntriesInRuntimeTranscript } from "../../agents/embedded-agent-runner/transcript-rewrite.js"; import { runAgentHarnessBeforeMessageWriteHook } from "../../agents/harness/hook-helpers.js"; import { modelCatalogBrowseRequiresFullDiscovery } from "../../agents/model-catalog-browse.js"; import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js"; import { resolveProviderIdForAuth } from "../../agents/provider-auth-aliases.js"; import { createAgentRunRestartAbortError } from "../../agents/run-termination.js"; -import type { AgentMessage } from "../../agents/runtime/index.js"; import { ensureSandboxWorkspaceForSession } from "../../agents/sandbox/context.js"; import { resolveAgentTimeoutMs } from "../../agents/timeout.js"; import { dispatchInboundMessage } from "../../auto-reply/dispatch.js"; @@ -71,8 +69,9 @@ import { } from "../../config/sessions/main-session.js"; import { findTranscriptEvent, - loadTranscriptEvents, patchSessionEntry, + publishTranscriptUpdate, + withTranscriptWriteLock, type SessionTranscriptWriteScope, type TranscriptEvent, } from "../../config/sessions/session-accessor.js"; @@ -1955,10 +1954,9 @@ async function rewriteSourceReplyTranscriptMirrors(params: { state: SourceReplyContentState; }[]; scope: SessionTranscriptWriteScope; - config?: OpenClawConfig; }): Promise< Array<{ - messageId?: string; + messageId: string; request: { idempotencyKey: string; metadata: SourceReplyTranscriptMirrorMetadata; @@ -1966,112 +1964,92 @@ async function rewriteSourceReplyTranscriptMirrors(params: { }; }> > { - const { sessionId, sessionKey } = params.scope; - if (!sessionId || !sessionKey || params.requests.length === 0 || params.candidates.length === 0) { + if (params.requests.length === 0 || params.candidates.length === 0) { return []; } - const events = await loadTranscriptEvents({ ...params.scope, sessionId }); - const allowedSourceReplyMirrorIds = new Set<string>(); - for (const candidate of params.candidates) { - const target = findSourceReplyTranscriptMirrorByMetadataInEvents({ - events, - idempotencyKey: candidate.idempotencyKey, - metadata: candidate.metadata, - }); - if (target) { - allowedSourceReplyMirrorIds.add(target.messageId); + return await withTranscriptWriteLock(params.scope, async (transcript) => { + const events = await transcript.readEvents(); + const allowedSourceReplyMirrorIds = new Set<string>(); + for (const candidate of params.candidates) { + const target = findSourceReplyTranscriptMirrorByMetadataInEvents({ + events, + idempotencyKey: candidate.idempotencyKey, + metadata: candidate.metadata, + }); + if (target) { + allowedSourceReplyMirrorIds.add(target.messageId); + } } - } - const rewriteTargets: Array<{ - request: (typeof params.requests)[number]; - messageId: string; - message: Record<string, unknown>; - }> = []; - for (const request of params.requests) { - const target = findSourceReplyTranscriptMirrorByMetadataInEvents({ - events, - idempotencyKey: request.idempotencyKey, - metadata: request.metadata, - }); - if (target) { - rewriteTargets.push({ request, ...target }); + const rewriteTargets: Array<{ + request: (typeof params.requests)[number]; + messageId: string; + message: Record<string, unknown>; + }> = []; + for (const request of params.requests) { + const target = findSourceReplyTranscriptMirrorByMetadataInEvents({ + events, + idempotencyKey: request.idempotencyKey, + metadata: request.metadata, + }); + if (target) { + rewriteTargets.push({ request, ...target }); + } + } + if (rewriteTargets.length === 0) { + return []; } - } - if (rewriteTargets.length === 0) { - return []; - } - const rewriteTargetIds = new Set(rewriteTargets.map((target) => target.messageId)); - // Guard over visible records (messages/compaction) only: hidden tree records - // such as leaf controls may trail the mirrors and must not veto the rewrite. - const visibleEvents = events.filter((event) => { - const record = transcriptEventRecord(event); - return Boolean(record?.message) || record?.type === "compaction"; - }); - const firstRewriteEntryIndex = visibleEvents.findIndex((event) => { - const id = transcriptEventId(event); - return id ? rewriteTargetIds.has(id) : false; - }); - const canRewriteSourceReplyMirrors = - firstRewriteEntryIndex >= 0 && - visibleEvents.slice(firstRewriteEntryIndex).every((event) => { + const rewriteTargetIds = new Set(rewriteTargets.map((target) => target.messageId)); + const firstRewriteEntryIndex = events.findIndex((event) => { const id = transcriptEventId(event); - return !id || allowedSourceReplyMirrorIds.has(id); + return id ? rewriteTargetIds.has(id) : false; }); - if (!canRewriteSourceReplyMirrors) { - return []; - } + const canRewriteSourceReplyMirrors = + firstRewriteEntryIndex >= 0 && + events.slice(firstRewriteEntryIndex).every((event) => { + const id = transcriptEventId(event); + return !id || allowedSourceReplyMirrorIds.has(id); + }); + if (!canRewriteSourceReplyMirrors) { + return []; + } - const result = await rewriteTranscriptEntriesInRuntimeTranscript({ - scope: { - sessionId, - sessionKey, - ...(params.scope.agentId ? { agentId: params.scope.agentId } : {}), - ...(params.scope.storePath ? { storePath: params.scope.storePath } : {}), - }, - request: { - allowedRewriteSuffixEntryIds: [...allowedSourceReplyMirrorIds], - replacements: rewriteTargets.map((target) => ({ - entryId: target.messageId, + const replacementsById = new Map(rewriteTargets.map((target) => [target.messageId, target])); + const rewrittenEvents = events.map((event) => { + const id = transcriptEventId(event); + const replacement = id ? replacementsById.get(id) : undefined; + if (!replacement) { + return event; + } + return Object.assign({}, event as Record<string, unknown>, { message: { - ...(target.message as unknown as AgentMessage), - idempotencyKey: target.request.idempotencyKey, - content: target.request.state.persistedContent, - } as unknown as AgentMessage, - })), - }, - ...(params.config ? { config: params.config } : {}), - }); - if (!result.changed) { - return []; - } - - // The file-backed rewrite appends replacement entries under fresh ids, so - // re-read the transcript to hand callers the persisted mirror ids. - const rewrittenEvents = await loadTranscriptEvents({ ...params.scope, sessionId }); - return rewriteTargets.map((target) => { - const rewritten = findSourceReplyTranscriptMirrorByIdempotencyKeyInEvents( - rewrittenEvents, - target.request.idempotencyKey, - ); - return rewritten?.messageId - ? { messageId: rewritten.messageId, request: target.request } - : { request: target.request }; + ...replacement.message, + idempotencyKey: replacement.request.idempotencyKey, + content: replacement.request.state.persistedContent, + }, + }); + }); + await transcript.replaceEvents(rewrittenEvents); + return rewriteTargets.map((target) => ({ + messageId: target.messageId, + request: target.request, + })); }); } async function publishAssistantTranscriptRewrite(params: { scope: SessionTranscriptWriteScope; - rewritten: readonly { messageId?: string }[]; + rewritten: readonly { messageId: string }[]; }): Promise<void> { if (params.rewritten.length === 0) { return; } - // The file-backed rewrite emits its own transcript update; only the session - // marker touch remains so history readers observe the fresh updatedAt. await touchAssistantTranscriptSessionEntry(params.scope); + await publishTranscriptUpdate(params.scope, { + messageId: params.rewritten.at(-1)?.messageId, + }); } function collectSessionAbortPartials(params: { @@ -4680,6 +4658,8 @@ export const chatHandlers: GatewayRequestHandlers = { startedAt: number; } | undefined; + let persistDispatchErrorUserTurn: (() => Promise<void>) | undefined; + let acceptedUserTurnSessionId = entry?.sessionId; const userTurnRecorder: UserTurnTranscriptRecorder = createUserTurnTranscriptRecorder({ input: baseUserTurnInput, resolveInput: () => userTurnInputPromise, @@ -4689,14 +4669,14 @@ export const chatHandlers: GatewayRequestHandlers = { store: latestStore, entry: latestEntry, } = loadSessionEntry(sessionKey, sessionLoadOptions); - const resolvedSessionId = latestEntry?.sessionId ?? backingSessionId; - if (!resolvedSessionId) { + if (!latestEntry?.sessionId || latestEntry.sessionId !== acceptedUserTurnSessionId) { return undefined; } return { - sessionId: resolvedSessionId, + sessionId: latestEntry.sessionId, + expectedSessionId: latestEntry.sessionId, sessionKey, - sessionEntry: latestEntry ?? entry, + sessionEntry: latestEntry, sessionStore: latestStore, storePath: latestStorePath, agentId, @@ -4915,7 +4895,17 @@ export const chatHandlers: GatewayRequestHandlers = { ? { taskSuggestionDeliveryMode: "gateway" as const } : {}), requestedSessionId, + ...(entry?.sessionId + ? { + expectedExistingSessionId: entry.sessionId, + } + : {}), resumeRequestedSession: controlUiReconnectResume.resumeRequested, + onSessionPrepared: (binding) => { + if (binding.sessionKey === sessionKey) { + acceptedUserTurnSessionId = binding.sessionId; + } + }, abortSignal: activeRunAbort.controller.signal, // Keep a Gateway-owned cancel identity after this chat.send // terminalizes while the prompt waits in followup/collect queue. @@ -5740,7 +5730,6 @@ export const chatHandlers: GatewayRequestHandlers = { candidates: sourceReplyMirrorCandidates, requests: sourceReplyPersistenceRequests, scope: sourceReplyScope, - config: cfg, }); if (rewritten.length > 0) { await publishAssistantTranscriptRewrite({ @@ -5888,17 +5877,11 @@ export const chatHandlers: GatewayRequestHandlers = { } return; } - const emitAfterError = + persistDispatchErrorUserTurn = userTurnRecorder.hasPersisted() || userTurnRecorder.isBlocked() - ? Promise.resolve() - : persistGatewayUserTurnTranscript(); - await emitAfterError.catch((transcriptErr: unknown) => { - context.logGateway.warn( - `webchat user transcript update failed after error: ${formatForLog(transcriptErr)}`, - ); - }); + ? undefined + : persistGatewayUserTurnTranscript; if ( - !agentRunStarted && !activeRunAbort.controller.signal.aborted && !context.chatAbortedRuns.has(clientRunId) ) { @@ -5984,7 +5967,14 @@ export const chatHandlers: GatewayRequestHandlers = { ); } }; - void persistDispatchLifecycleError() + void (async () => { + await persistDispatchLifecycleError(); + await persistDispatchErrorUserTurn?.().catch((transcriptErr: unknown) => { + context.logGateway.warn( + `webchat user transcript update failed after error: ${formatForLog(transcriptErr)}`, + ); + }); + })() .catch((continuationErr: unknown) => { context.logGateway.warn( `webchat session lifecycle continuation failed: ${formatForLog(continuationErr)}`, diff --git a/src/gateway/server-methods/sessions.ts b/src/gateway/server-methods/sessions.ts index d26996fddf38..be61a83a0b71 100644 --- a/src/gateway/server-methods/sessions.ts +++ b/src/gateway/server-methods/sessions.ts @@ -63,7 +63,9 @@ import { import { resolveAgentMainSessionKey } from "../../config/sessions/main-session.js"; import { applySessionPatchProjection, + loadTranscriptEvents, preflightSessionTranscriptForManualCompact, + resolveSessionTranscriptRuntimeTarget, trimSessionTranscriptForManualCompact, } from "../../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; @@ -135,7 +137,6 @@ import { resolveGatewaySessionThinkingProjection, resolveSessionDisplayModelIdentityRef, resolveSessionModelRef, - resolveSessionTranscriptCandidates, type SessionsPatchResult, type SessionsPreviewEntry, type SessionsPreviewResult, @@ -252,6 +253,49 @@ function resolveGatewaySessionTargetFromKey( return { cfg, target, storePath: target.storePath }; } +function loadAccessorSessionEntryForGatewayTarget(params: { + key: string; + cfg: OpenClawConfig; + agentId?: string; +}) { + const target = resolveGatewaySessionStoreTargetWithStore({ + cfg: params.cfg, + key: params.key, + clone: false, + ...(params.agentId ? { agentId: params.agentId } : {}), + }); + let best: + | { + entry: SessionEntry; + sessionStoreKey: string; + } + | undefined; + for (const sessionStoreKey of target.storeKeys) { + const entry = target.store[sessionStoreKey]; + if (entry) { + if (!best || (entry.updatedAt ?? 0) > (best.entry.updatedAt ?? 0)) { + best = { entry, sessionStoreKey }; + } + } + } + if (best) { + return { + target, + storePath: target.storePath, + entry: best.entry, + canonicalKey: target.canonicalKey, + sessionStoreKey: best.sessionStoreKey, + }; + } + return { + target, + storePath: target.storePath, + entry: undefined, + canonicalKey: target.canonicalKey, + sessionStoreKey: target.canonicalKey, + }; +} + function loadSessionEntriesForTarget(params: { key: string; cfg: OpenClawConfig; @@ -1143,7 +1187,9 @@ export const sessionsHandlers: GatewayRequestHandlers = { respond(false, undefined, requestedAgent.error); return; } - const { entry, canonicalKey } = loadSessionEntry(key, { + const { entry, canonicalKey } = loadAccessorSessionEntryForGatewayTarget({ + key, + cfg, agentId: requestedAgent.agentId, }); respond( @@ -1183,7 +1229,9 @@ export const sessionsHandlers: GatewayRequestHandlers = { respond(false, undefined, requestedAgent.error); return; } - const { entry, canonicalKey } = loadSessionEntry(key, { + const { entry, canonicalKey } = loadAccessorSessionEntryForGatewayTarget({ + key, + cfg, agentId: requestedAgent.agentId, }); const checkpoint = getSessionCompactionCheckpoint({ entry, checkpointId }); @@ -1565,13 +1613,12 @@ export const sessionsHandlers: GatewayRequestHandlers = { respond(false, undefined, requestedAgent.error); return; } - const loaded = loadSessionEntry(key, { agentId: requestedAgent.agentId }); - const { cfg: loadedCfg, entry, canonicalKey, legacyKey } = loaded; - const target = resolveGatewaySessionStoreTarget({ - cfg: loadedCfg, - key: canonicalKey, - agentId: requestedAgent.agentId, - }); + const { entry, canonicalKey, sessionStoreKey, target, storePath } = + loadAccessorSessionEntryForGatewayTarget({ + key, + cfg, + agentId: requestedAgent.agentId, + }); if (!entry?.sessionId) { respond( false, @@ -1591,9 +1638,10 @@ export const sessionsHandlers: GatewayRequestHandlers = { } const nextKey = buildDashboardSessionKey(target.agentId); const branchedSession = await compactionCheckpointStore.branchCheckpointSession({ - storePath: target.storePath, + agentId: target.agentId, + storePath, sourceKey: canonicalKey, - sourceStoreKey: legacyKey, + sourceStoreKey: sessionStoreKey, nextKey, checkpointId, }); @@ -1695,8 +1743,12 @@ export const sessionsHandlers: GatewayRequestHandlers = { respond(false, undefined, requestedAgent.error); return; } - const loaded = loadSessionEntry(key, { agentId: requestedAgent.agentId }); - const { entry, canonicalKey, legacyKey, storePath } = loaded; + const { entry, canonicalKey, sessionStoreKey, storePath } = + loadAccessorSessionEntryForGatewayTarget({ + key, + cfg, + agentId: requestedAgent.agentId, + }); if (!entry?.sessionId) { respond( false, @@ -1717,10 +1769,11 @@ export const sessionsHandlers: GatewayRequestHandlers = { const lifecycleIdentities = [ key, canonicalKey, - legacyKey, + sessionStoreKey, entry.sessionId, entry.lifecycleRevision, ]; + const restoreLockIdentities = [entry.sessionId, entry.lifecycleRevision]; let admittedWorkReleased = true; let restoreTargetStillCurrent = true; let restoreBlockedByModelLock = false; @@ -1728,14 +1781,20 @@ export const sessionsHandlers: GatewayRequestHandlers = { // compaction so neither operation can publish state from the other's obsolete session. await runExclusiveSessionLifecycleMutation({ scope: storePath, - identities: lifecycleIdentities, + identities: restoreLockIdentities, prepare: async () => { - const current = loadSessionEntry(key, { agentId: requestedAgent.agentId }); - restoreTargetStillCurrent = Boolean( + const current = loadAccessorSessionEntryForGatewayTarget({ + key, + cfg, + agentId: requestedAgent.agentId, + }); + const currentCheckpoint = current.entry + ? getSessionCompactionCheckpoint({ entry: current.entry, checkpointId }) + : undefined; + restoreTargetStillCurrent = current.entry?.sessionId === entry.sessionId && current.entry.lifecycleRevision === entry.lifecycleRevision && - getSessionCompactionCheckpoint({ entry: current.entry, checkpointId }), - ); + currentCheckpoint !== undefined; if (!restoreTargetStillCurrent) { return; } @@ -1746,7 +1805,7 @@ export const sessionsHandlers: GatewayRequestHandlers = { clearSessionQueues([ key, current.canonicalKey, - current.legacyKey, + current.sessionStoreKey, current.entry?.sessionId, ]); admittedWorkReleased = await interruptSessionWorkAdmissions({ @@ -1784,7 +1843,11 @@ export const sessionsHandlers: GatewayRequestHandlers = { ); return; } - const current = loadSessionEntry(key, { agentId: requestedAgent.agentId }); + const current = loadAccessorSessionEntryForGatewayTarget({ + key, + cfg, + agentId: requestedAgent.agentId, + }); if (!current.entry?.sessionId) { respond( false, @@ -1801,7 +1864,11 @@ export const sessionsHandlers: GatewayRequestHandlers = { ); return; } - if (!getSessionCompactionCheckpoint({ entry: current.entry, checkpointId })) { + const currentCheckpoint = getSessionCompactionCheckpoint({ + entry: current.entry, + checkpointId, + }); + if (!currentCheckpoint) { respond( false, undefined, @@ -1825,9 +1892,10 @@ export const sessionsHandlers: GatewayRequestHandlers = { } const restoredSession = await compactionCheckpointStore.restoreCheckpointSession({ + agentId: requestedAgent.agentId, storePath, sessionKey: current.canonicalKey, - sessionStoreKey: current.legacyKey, + sessionStoreKey: current.sessionStoreKey, checkpointId, }); if ( @@ -2182,6 +2250,7 @@ export const sessionsHandlers: GatewayRequestHandlers = { } } return await applySessionPatchProjection({ + agentId: target.agentId, storePath, resolveTarget: ({ entries }) => { const store = Object.fromEntries( @@ -2391,7 +2460,7 @@ export const sessionsHandlers: GatewayRequestHandlers = { reason, }); }, - "sessions.delete": async ({ params, respond, client, isWebchatConnect, context }) => { + "sessions.delete": async ({ req, params, respond, client, isWebchatConnect, context }) => { if (!assertValidParams(params, validateSessionsDeleteParams, "sessions.delete", respond)) { return; } @@ -2477,11 +2546,7 @@ export const sessionsHandlers: GatewayRequestHandlers = { if (!expectedSessionId || entry?.sessionId === expectedSessionId) { return true; } - return ( - entry?.sessionId === undefined && - expectedLifecycleRevision !== undefined && - expectedLifecycleRevisionMatches(entry) - ); + return false; }; const respondSessionChanged = () => { respond( @@ -2518,6 +2583,30 @@ export const sessionsHandlers: GatewayRequestHandlers = { ) { return; } + let abortResult: + | { + ok: boolean; + error?: ReturnType<typeof errorShape>; + } + | undefined; + const abortSessionKey = target.canonicalKey ?? key; + await chatHandlers["chat.abort"]({ + req, + params: { + sessionKey: abortSessionKey, + ...(requestedAgentId ? { agentId: requestedAgentId } : {}), + }, + respond: (ok, _payload, error) => { + abortResult = { ok, ...(error ? { error } : {}) }; + }, + context, + client, + isWebchatConnect, + }); + if (abortResult?.ok === false) { + respond(false, undefined, abortResult.error); + return; + } const deleteLifecycleIdentities = [ target.canonicalKey, key, @@ -2847,6 +2936,7 @@ export const sessionsHandlers: GatewayRequestHandlers = { // snapshot so alias promotion/pruning persists through the accessor. let compactPrimaryKey = target.canonicalKey; const compactRead = await applySessionPatchProjection({ + agentId: target.agentId, storePath, resolveTarget: ({ entries }) => { const snapshot = Object.fromEntries( @@ -2912,13 +3002,13 @@ export const sessionsHandlers: GatewayRequestHandlers = { return; } } else { - const filePath = resolveSessionTranscriptCandidates( + const transcriptEvents = await loadTranscriptEvents({ + agentId: target.agentId, sessionId, + sessionKey: compactTarget.primaryKey, storePath, - entry.sessionFile, - target.agentId, - ).find((candidate) => fs.existsSync(candidate)); - if (!filePath) { + }).catch(() => []); + if (transcriptEvents.length === 0) { respond( true, { @@ -2949,7 +3039,11 @@ export const sessionsHandlers: GatewayRequestHandlers = { identities: lifecycleIdentities, kind: "compaction", prepare: async () => { - const latestEntry = loadSessionEntry(key, { agentId: requestedAgentId }).entry; + const latestEntry = loadAccessorSessionEntryForGatewayTarget({ + key, + cfg, + agentId: requestedAgentId, + }).entry; sessionStillCurrent = Boolean( latestEntry && latestEntry.sessionId === sessionId && @@ -2990,7 +3084,11 @@ export const sessionsHandlers: GatewayRequestHandlers = { return; } - const latestEntry = loadSessionEntry(key, { agentId: requestedAgentId }).entry; + const latestEntry = loadAccessorSessionEntryForGatewayTarget({ + key, + cfg, + agentId: requestedAgentId, + }).entry; if ( !latestEntry || latestEntry.sessionId !== sessionId || @@ -3068,13 +3166,13 @@ export const sessionsHandlers: GatewayRequestHandlers = { return; } - const filePath = resolveSessionTranscriptCandidates( + const transcriptEvents = await loadTranscriptEvents({ + agentId: target.agentId, sessionId, + sessionKey: compactTarget.primaryKey, storePath, - latestEntry.sessionFile, - target.agentId, - ).find((candidate) => fs.existsSync(candidate)); - if (!filePath) { + }).catch(() => []); + if (transcriptEvents.length === 0) { respond( true, { @@ -3087,6 +3185,12 @@ export const sessionsHandlers: GatewayRequestHandlers = { ); return; } + const compactionTranscriptTarget = await resolveSessionTranscriptRuntimeTarget({ + agentId: target.agentId, + sessionId, + sessionKey: compactTarget.primaryKey, + storePath, + }); const resolvedModel = resolveSessionModelRef(cfg, latestEntry, target.agentId); const workspaceDir = @@ -3122,8 +3226,14 @@ export const sessionsHandlers: GatewayRequestHandlers = { sessionId, sessionKey: target.canonicalKey, agentId: target.agentId, + sessionTarget: { + agentId: target.agentId, + sessionId, + sessionKey: target.canonicalKey, + storePath, + }, allowGatewaySubagentBinding: true, - sessionFile: filePath, + sessionFile: compactionTranscriptTarget.sessionFile, workspaceDir, cwd: normalizeOptionalString(latestEntry.spawnedCwd), config: cfg, @@ -3154,6 +3264,7 @@ export const sessionsHandlers: GatewayRequestHandlers = { // Guarded terminal persist: skip when session ownership rotated // while compaction ran (sessionId/lifecycleRevision/work-start). const persistProjection = await applySessionPatchProjection({ + agentId: target.agentId, storePath, resolveTarget: () => ({ primaryKey: compactTarget.primaryKey }), project: ({ existingEntry }) => { @@ -3175,9 +3286,6 @@ export const sessionsHandlers: GatewayRequestHandlers = { ) { entryToUpdate.sessionId = result.result.sessionId; } - if (result.result?.sessionFile) { - entryToUpdate.sessionFile = result.result.sessionFile; - } delete entryToUpdate.inputTokens; delete entryToUpdate.outputTokens; delete entryToUpdate.contextBudgetStatus; diff --git a/src/gateway/server-session-events.ts b/src/gateway/server-session-events.ts index 4cff76b255c6..428a4954f083 100644 --- a/src/gateway/server-session-events.ts +++ b/src/gateway/server-session-events.ts @@ -4,6 +4,11 @@ import { asPositiveSafeInteger } from "@openclaw/normalization-core/number-coerc import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { resolveDefaultAgentId } from "../agents/agent-scope.js"; import { getRuntimeConfig } from "../config/io.js"; +import { + loadSessionEntry as loadAccessorSessionEntry, + resolveTranscriptSessionKeyBySessionId, +} from "../config/sessions/session-accessor.js"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { normalizeAgentId } from "../routing/session-key.js"; import type { SessionLifecycleEvent } from "../sessions/session-lifecycle-events.js"; import type { InternalSessionTranscriptUpdate } from "../sessions/transcript-events.js"; @@ -144,9 +149,18 @@ async function handleTranscriptUpdateBroadcast( }, update: InternalSessionTranscriptUpdate, ): Promise<void> { + const sqliteMarker = parseSqliteSessionFileMarker(update.sessionFile); + const storageAgentId = update.target?.agentId ?? update.agentId ?? sqliteMarker?.agentId; const sessionKey = update.target?.sessionKey ?? update.sessionKey ?? + (sqliteMarker + ? resolveTranscriptSessionKeyBySessionId({ + agentId: storageAgentId, + sessionId: sqliteMarker.sessionId, + storePath: sqliteMarker.storePath, + }) + : undefined) ?? (update.sessionFile ? resolveSessionKeyForTranscriptFile(update.sessionFile) : undefined); if (!sessionKey || update.message === undefined) { return; @@ -175,11 +189,22 @@ async function handleTranscriptUpdateBroadcast( if (messageSeq === undefined) { // Updates from raw transcript events may not carry seq; fall back to the // current transcript line count for cursor-compatible live history. - const { entry, storePath } = loadSessionEntry(sessionKey, { agentId: visibleAgentId }); + const markerEntry = sqliteMarker + ? loadAccessorSessionEntry({ + agentId: storageAgentId, + sessionKey, + storePath: sqliteMarker.storePath, + }) + : undefined; + const fallbackTarget = markerEntry + ? undefined + : loadSessionEntry(sessionKey, { agentId: visibleAgentId }); + const entry = markerEntry ?? fallbackTarget?.entry; + const storePath = sqliteMarker?.storePath ?? fallbackTarget?.storePath; messageSeq = entry?.sessionId ? asPositiveSafeInteger( await readSessionMessageCountAsync({ - agentId: visibleAgentId, + agentId: storageAgentId ?? visibleAgentId, sessionEntry: entry, sessionId: entry.sessionId, sessionKey, diff --git a/src/gateway/server-startup-config.secrets.test.ts b/src/gateway/server-startup-config.secrets.test.ts index c8b04cde7a32..557c2a6e8814 100644 --- a/src/gateway/server-startup-config.secrets.test.ts +++ b/src/gateway/server-startup-config.secrets.test.ts @@ -197,6 +197,39 @@ function installDiagnosticsTimelineEnv() { }; } +/** Isolate path-based auth store discovery so prior full-suite env cannot force slow path. */ +function installIsolatedStartupFastPathEnv() { + const root = mkdtempSync(path.join(tmpdir(), "openclaw-startup-fast-path-env-")); + const keys = [ + "OPENCLAW_HOME", + "OPENCLAW_STATE_DIR", + "OPENCLAW_CONFIG_PATH", + "OPENCLAW_OAUTH_DIR", + ] as const; + const previous = new Map<(typeof keys)[number], string | undefined>(); + for (const key of keys) { + previous.set(key, process.env[key]); + } + process.env.OPENCLAW_HOME = path.join(root, "home"); + process.env.OPENCLAW_STATE_DIR = path.join(root, "state"); + process.env.OPENCLAW_CONFIG_PATH = path.join(root, "state", "openclaw.json"); + process.env.OPENCLAW_OAUTH_DIR = path.join(root, "credentials"); + + return { + cleanup: () => { + for (const key of keys) { + const value = previous.get(key); + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + rmSync(root, { force: true, recursive: true }); + }, + }; +} + function installGatewayStartupSecretsRuntimeMock(state: GatewayStartupSecretsRuntimeMock) { ( globalThis as typeof globalThis & { @@ -757,6 +790,7 @@ describe("gateway startup config secret preflight", () => { it("activates no-SecretRef startup config without importing the full secrets runtime", async () => { vi.resetModules(); const agentDir = mkdtempSync(path.join(tmpdir(), "openclaw-startup-fast-path-")); + const isolatedEnv = installIsolatedStartupFastPathEnv(); const runtimeImport = vi.fn(); const prepareRuntimeSecretsSnapshot = vi.fn(async ({ config }) => preparedSnapshot(config)); const activateRuntimeSecretsSnapshot = vi.fn(); @@ -840,6 +874,7 @@ describe("gateway startup config secret preflight", () => { expect(refreshInput.loadAuthStore).toBeUndefined(); clearSecretsRuntimeSnapshot(); } finally { + isolatedEnv.cleanup(); vi.doUnmock("../agents/auth-profiles.js"); vi.doUnmock("../secrets/runtime.js"); delete ( diff --git a/src/gateway/server-startup-session-migration.test.ts b/src/gateway/server-startup-session-migration.test.ts index 8fa3500b45e7..5295f3e5b671 100644 --- a/src/gateway/server-startup-session-migration.test.ts +++ b/src/gateway/server-startup-session-migration.test.ts @@ -10,6 +10,7 @@ type ResolveStoreTargets = NonNullable< StartupMigrationDeps["resolveAllAgentSessionStoreTargetsSync"] >; type SweepStoreTemps = NonNullable<StartupMigrationDeps["sweepOrphanSessionStoreTemps"]>; +type RunDoctorSessionSqlite = NonNullable<StartupMigrationDeps["runDoctorSessionSqlite"]>; function makeLog() { return { @@ -24,7 +25,13 @@ function makeCfg() { >[0]["cfg"]; } -function makeDeps(migrate: MigrateSessionKeys, removedFiles = 0) { +// Every test injects store-target and sweep mocks so startup never scans the +// real filesystem, and a SQLite import mock so startup never runs real doctor. +function makeDeps( + migrate: MigrateSessionKeys, + removedFiles = 0, + runDoctorSessionSqlite: RunDoctorSessionSqlite = makeSessionSqliteImport(), +) { return { migrateOrphanedSessionKeys: migrate, resolveAllAgentSessionStoreTargetsSync: vi.fn<ResolveStoreTargets>().mockReturnValue([ @@ -35,6 +42,7 @@ function makeDeps(migrate: MigrateSessionKeys, removedFiles = 0) { .fn<SweepStoreTemps>() .mockResolvedValueOnce(removedFiles) .mockResolvedValue(0), + runDoctorSessionSqlite, }; } @@ -46,6 +54,29 @@ function firstLogMessage(log: ReturnType<typeof vi.fn>, label: string): string { return message; } +function makeSessionSqliteImport( + report: Partial<Awaited<ReturnType<RunDoctorSessionSqlite>>> = {}, +): RunDoctorSessionSqlite { + return vi.fn().mockResolvedValue({ + mode: "import", + targets: [], + totals: { + archivedTranscriptFiles: 0, + archivedUnreferencedJsonlFiles: 0, + importedEntries: 0, + importedTranscriptEvents: 0, + issues: 0, + legacyEntries: 0, + sqliteEntries: 0, + targets: 0, + unreferencedJsonlFiles: 0, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }, + ...report, + }); +} + describe("runStartupSessionMigration", () => { it("logs changes when orphaned keys are canonicalized", async () => { const log = makeLog(); @@ -127,4 +158,297 @@ describe("runStartupSessionMigration", () => { expect(warning).toContain("temp cleanup failed during startup"); expect(warning).toContain("permission denied"); }); + + it("imports legacy session metadata and transcripts into SQLite during startup", async () => { + const log = makeLog(); + const cfg = makeCfg(); + const env = { OPENCLAW_STATE_DIR: "/tmp/openclaw-state" }; + const migrate = vi.fn<MigrateSessionKeys>().mockResolvedValue({ changes: [], warnings: [] }); + const runDoctorSessionSqlite = makeSessionSqliteImport({ + totals: { + archivedTranscriptFiles: 2, + archivedUnreferencedJsonlFiles: 1, + importedEntries: 3, + importedTranscriptEvents: 9, + issues: 0, + legacyEntries: 3, + sqliteEntries: 3, + targets: 1, + unreferencedJsonlFiles: 0, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }, + }); + + await runStartupSessionMigration({ + cfg, + env, + log, + deps: makeDeps(migrate, 0, runDoctorSessionSqlite), + }); + + expect(runDoctorSessionSqlite).toHaveBeenCalledWith({ + allAgents: true, + cfg, + env, + mode: "import", + }); + expect(firstLogMessage(log.info, "sqlite import info")).toContain( + "session: imported legacy session metadata/transcripts into SQLite", + ); + expect(log.warn).not.toHaveBeenCalled(); + }); + + it("warns without blocking when hot legacy session SQLite import reports legacy file issues", async () => { + const log = makeLog(); + const migrate = vi.fn<MigrateSessionKeys>().mockResolvedValue({ changes: [], warnings: [] }); + const runDoctorSessionSqlite = makeSessionSqliteImport({ + targets: [ + { + agentId: "main", + archivedTranscriptFiles: [], + archivedUnreferencedJsonlFiles: [], + importedEntries: 0, + importedTranscriptEvents: 0, + issues: [ + { + code: "transcript_malformed", + message: "/tmp/bad.jsonl: SyntaxError", + sessionKey: "agent:main:main", + }, + ], + legacyEntries: 1, + referencedTranscriptFiles: 1, + sqliteEntries: 0, + sqlitePath: "/tmp/openclaw-agent.sqlite", + storePath: "/tmp/sessions.json", + unreferencedJsonlFiles: [], + validatedEntries: 0, + validatedTranscriptEvents: 0, + }, + ], + totals: { + archivedTranscriptFiles: 0, + archivedUnreferencedJsonlFiles: 0, + importedEntries: 0, + importedTranscriptEvents: 0, + issues: 1, + legacyEntries: 1, + sqliteEntries: 0, + targets: 1, + unreferencedJsonlFiles: 0, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }, + }); + + await runStartupSessionMigration({ + cfg: makeCfg(), + log, + deps: makeDeps(migrate, 0, runDoctorSessionSqlite), + }); + + expect(firstLogMessage(log.warn, "malformed transcript warning")).toContain( + "session SQLite migration warnings", + ); + }); + + it("classifies corrupt SQLite startup failures with recovery guidance", async () => { + const log = makeLog(); + const migrate = vi.fn<MigrateSessionKeys>().mockResolvedValue({ changes: [], warnings: [] }); + const runDoctorSessionSqlite = vi.fn().mockRejectedValue(new Error("file is not a database")); + + await expect( + runStartupSessionMigration({ + cfg: makeCfg(), + log, + deps: makeDeps(migrate, 0, runDoctorSessionSqlite), + }), + ).rejects.toThrow("openclaw doctor --session-sqlite recover --session-sqlite-all-agents"); + }); + + it("auto-restores the current failed session SQLite migration run after files moved", async () => { + const log = makeLog(); + const migrate = vi.fn<MigrateSessionKeys>().mockResolvedValue({ changes: [], warnings: [] }); + const restoreSessionSqliteMigrationRun = vi.fn(() => ({ + conflicts: [], + manifestPaths: ["/tmp/run.json"], + restoredFiles: ["/tmp/session.jsonl"], + skippedFiles: [], + })); + const writeSessionSqliteMigrationFailureReports = vi.fn(() => ({ + jsonPath: "/tmp/run.failure.json", + markdownPath: "/tmp/run.failure.md", + })); + const runDoctorSessionSqlite = makeSessionSqliteImport({ + migrationRun: { + manifestPath: "/tmp/run.json", + runId: "run", + }, + targets: [ + { + agentId: "main", + archivedTranscriptFiles: ["/tmp/archive/session.jsonl"], + archivedUnreferencedJsonlFiles: [], + importedEntries: 1, + importedTranscriptEvents: 1, + issues: [ + { + code: "active_sqlite_transcript_jsonl", + message: "active file remained", + sessionKey: "agent:main:main", + }, + ], + legacyEntries: 1, + referencedTranscriptFiles: 1, + sqliteEntries: 1, + sqlitePath: "/tmp/openclaw-agent.sqlite", + storePath: "/tmp/sessions.json", + unreferencedJsonlFiles: [], + validatedEntries: 1, + validatedTranscriptEvents: 1, + }, + ], + totals: { + archivedTranscriptFiles: 1, + archivedUnreferencedJsonlFiles: 0, + importedEntries: 1, + importedTranscriptEvents: 1, + issues: 1, + legacyEntries: 1, + sqliteEntries: 1, + targets: 1, + unreferencedJsonlFiles: 0, + validatedEntries: 1, + validatedTranscriptEvents: 1, + }, + }); + + await expect( + runStartupSessionMigration({ + cfg: makeCfg(), + log, + deps: { + ...makeDeps(migrate, 0, runDoctorSessionSqlite), + restoreSessionSqliteMigrationRun, + writeSessionSqliteMigrationFailureReports, + }, + }), + ).rejects.toThrow("Failure report: /tmp/run.failure.md"); + + expect(restoreSessionSqliteMigrationRun).toHaveBeenCalledWith({ + manifestPath: "/tmp/run.json", + }); + expect(writeSessionSqliteMigrationFailureReports).toHaveBeenCalledWith("/tmp/run.json", { + reason: "startup blocked on 1 session SQLite issue(s)", + }); + expect(firstLogMessage(log.warn, "startup restore warning")).toContain("restored=1"); + }); + + it("warns without blocking when a legacy transcript sidecar is missing", async () => { + const log = makeLog(); + const migrate = vi.fn<MigrateSessionKeys>().mockResolvedValue({ changes: [], warnings: [] }); + const runDoctorSessionSqlite = makeSessionSqliteImport({ + targets: [ + { + agentId: "main", + archivedTranscriptFiles: [], + archivedUnreferencedJsonlFiles: [], + importedEntries: 1, + importedTranscriptEvents: 0, + issues: [ + { + code: "transcript_missing", + message: "Transcript file is missing: /tmp/missing.jsonl", + sessionKey: "agent:main:main", + }, + ], + legacyEntries: 1, + referencedTranscriptFiles: 1, + sqliteEntries: 1, + sqlitePath: "/tmp/openclaw-agent.sqlite", + storePath: "/tmp/sessions.json", + unreferencedJsonlFiles: [], + validatedEntries: 0, + validatedTranscriptEvents: 0, + }, + ], + totals: { + archivedTranscriptFiles: 0, + archivedUnreferencedJsonlFiles: 0, + importedEntries: 1, + importedTranscriptEvents: 0, + issues: 1, + legacyEntries: 1, + sqliteEntries: 1, + targets: 1, + unreferencedJsonlFiles: 0, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }, + }); + + await runStartupSessionMigration({ + cfg: makeCfg(), + log, + deps: makeDeps(migrate, 0, runDoctorSessionSqlite), + }); + + expect(firstLogMessage(log.warn, "missing transcript warning")).toContain( + "session SQLite migration warnings", + ); + }); + + it("warns without blocking when only stale archive-tier JSONL archival fails", async () => { + const log = makeLog(); + const migrate = vi.fn<MigrateSessionKeys>().mockResolvedValue({ changes: [], warnings: [] }); + const runDoctorSessionSqlite = makeSessionSqliteImport({ + targets: [ + { + agentId: "main", + archivedTranscriptFiles: [], + archivedUnreferencedJsonlFiles: [], + importedEntries: 1, + importedTranscriptEvents: 1, + issues: [ + { + code: "unreferenced_jsonl_archive_failed", + message: "/tmp/orphan.jsonl: permission denied", + }, + ], + legacyEntries: 1, + referencedTranscriptFiles: 1, + sqliteEntries: 1, + sqlitePath: "/tmp/openclaw-agent.sqlite", + storePath: "/tmp/sessions.json", + unreferencedJsonlFiles: ["/tmp/orphan.jsonl"], + validatedEntries: 0, + validatedTranscriptEvents: 0, + }, + ], + totals: { + archivedTranscriptFiles: 0, + archivedUnreferencedJsonlFiles: 0, + importedEntries: 1, + importedTranscriptEvents: 1, + issues: 1, + legacyEntries: 1, + sqliteEntries: 1, + targets: 1, + unreferencedJsonlFiles: 1, + validatedEntries: 0, + validatedTranscriptEvents: 0, + }, + }); + + await runStartupSessionMigration({ + cfg: makeCfg(), + log, + deps: makeDeps(migrate, 0, runDoctorSessionSqlite), + }); + + expect(firstLogMessage(log.warn, "archive-tier warning")).toContain( + "session SQLite migration warnings", + ); + }); }); diff --git a/src/gateway/server-startup-session-migration.ts b/src/gateway/server-startup-session-migration.ts index 937e7d9e7e6e..a972e1933acb 100644 --- a/src/gateway/server-startup-session-migration.ts +++ b/src/gateway/server-startup-session-migration.ts @@ -1,18 +1,49 @@ +import type { + DoctorSessionSqliteIssue, + DoctorSessionSqliteReport, + DoctorSessionSqliteRestoreReport, +} from "../commands/doctor-session-sqlite.js"; import { runSessionStartupMigration, type SessionStartupMigrationLogger, } from "../config/sessions/startup-migration.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -type SessionMigrationDeps = Parameters<typeof runSessionStartupMigration>[0]["deps"]; +type SessionSqliteStartupImportRunner = (params: { + allAgents: true; + cfg: OpenClawConfig; + env: NodeJS.ProcessEnv; + mode: "import"; +}) => Promise<DoctorSessionSqliteReport>; + +type SessionSqliteStartupRestoreRunner = (params: { + manifestPath: string; +}) => DoctorSessionSqliteRestoreReport; + +type SessionSqliteStartupFailureReportWriter = ( + manifestPath: string, + params: { reason: string }, +) => { jsonPath: string; markdownPath: string }; + +type SessionMigrationDeps = Parameters<typeof runSessionStartupMigration>[0]["deps"] & { + restoreSessionSqliteMigrationRun?: SessionSqliteStartupRestoreRunner; + runDoctorSessionSqlite?: SessionSqliteStartupImportRunner; + writeSessionSqliteMigrationFailureReports?: SessionSqliteStartupFailureReportWriter; +}; + +const STARTUP_WARNING_ISSUE_CODES = new Set([ + "entry_invalid", + "transcript_archive_failed", + "transcript_malformed", + "transcript_missing", + "unreferenced_jsonl_archive_failed", +]); /** - * Run orphan-key session migration at gateway startup. + * Run session migrations at gateway startup before runtime session access. * - * Idempotent and best-effort: if the migration fails, gateway startup - * continues normally. This ensures accumulated orphaned session keys - * (from the write-path bug #29683) are cleaned up automatically on - * upgrade rather than requiring a manual `openclaw doctor` run. + * Orphan-key cleanup remains best-effort. Full SQLite import is blocking + * for hot legacy session issues because runtime no longer falls back to JSONL. */ export async function runStartupSessionMigration(params: { cfg: OpenClawConfig; @@ -21,4 +52,154 @@ export async function runStartupSessionMigration(params: { deps?: SessionMigrationDeps; }): Promise<void> { await runSessionStartupMigration(params); + await runStartupSessionSqliteImport(params); +} + +async function runStartupSessionSqliteImport(params: { + cfg: OpenClawConfig; + env?: NodeJS.ProcessEnv; + log: SessionStartupMigrationLogger; + deps?: SessionMigrationDeps; +}): Promise<void> { + const env = params.env ?? process.env; + const runDoctorSessionSqlite = + params.deps?.runDoctorSessionSqlite ?? + (await import("../commands/doctor-session-sqlite.js")).runDoctorSessionSqlite; + let report: DoctorSessionSqliteReport; + try { + report = await runDoctorSessionSqlite({ + allAgents: true, + cfg: params.cfg, + env, + mode: "import", + }); + } catch (error) { + if (isSqliteCorruptionError(error)) { + throw new Error( + [ + `session SQLite migration failed during startup because an agent SQLite database could not be opened: ${String(error)}`, + 'Run "openclaw doctor --session-sqlite recover --session-sqlite-all-agents" to move the corrupt database aside and preserve it for support.', + ].join("\n"), + { cause: error }, + ); + } + throw error; + } + const warningIssues = collectStartupWarningIssues(report); + const blockingIssues = collectStartupBlockingIssues(report); + if (blockingIssues.length > 0) { + const recovery = await restoreFailedStartupSessionSqliteRun(params, report, blockingIssues); + throw new Error( + [ + `session SQLite migration failed during startup with ${blockingIssues.length} blocking issue(s).`, + ...formatStartupIssueLines(blockingIssues).map((line) => `- ${line}`), + 'Run "openclaw doctor --session-sqlite inspect --session-sqlite-all-agents" for details.', + ...(recovery.length > 0 ? recovery : []), + ].join("\n"), + ); + } + if (sessionSqliteReportHasChanges(report)) { + params.log.info(formatSessionSqliteStartupImportSummary(report)); + } + if (warningIssues.length > 0) { + params.log.warn( + [ + `session: session SQLite migration warnings:\n${formatStartupIssueLines(warningIssues) + .map((line) => `- ${line}`) + .join("\n")}`, + ].join("\n"), + ); + } +} + +async function restoreFailedStartupSessionSqliteRun( + params: { + cfg: OpenClawConfig; + env?: NodeJS.ProcessEnv; + log: SessionStartupMigrationLogger; + deps?: SessionMigrationDeps; + }, + report: DoctorSessionSqliteReport, + blockingIssues: readonly DoctorSessionSqliteIssue[], +): Promise<string[]> { + const manifestPath = report.migrationRun?.manifestPath; + if (!manifestPath) { + return report.migrationRun?.failureReportMarkdownPath + ? [`Failure report: ${report.migrationRun.failureReportMarkdownPath}`] + : []; + } + let restoreSessionSqliteMigrationRun = params.deps?.restoreSessionSqliteMigrationRun; + let writeSessionSqliteMigrationFailureReports = + params.deps?.writeSessionSqliteMigrationFailureReports; + if (!restoreSessionSqliteMigrationRun || !writeSessionSqliteMigrationFailureReports) { + const doctorModule = await import("../commands/doctor-session-sqlite.js"); + restoreSessionSqliteMigrationRun ??= doctorModule.restoreSessionSqliteMigrationRun; + writeSessionSqliteMigrationFailureReports ??= + doctorModule.writeSessionSqliteMigrationFailureReports; + } + const restore = restoreSessionSqliteMigrationRun({ manifestPath }); + const failureReports = writeSessionSqliteMigrationFailureReports(manifestPath, { + reason: `startup blocked on ${blockingIssues.length} session SQLite issue(s)`, + }); + params.log.warn( + [ + "session: restored archived legacy transcript artifacts after startup SQLite migration failure:", + `- restored=${restore.restoredFiles.length} skipped=${restore.skippedFiles.length} conflicts=${restore.conflicts.length}`, + `- failureReport=${failureReports.markdownPath}`, + ].join("\n"), + ); + return [ + `Restore attempted for current migration run: restored=${restore.restoredFiles.length}, skipped=${restore.skippedFiles.length}, conflicts=${restore.conflicts.length}.`, + `Failure report: ${failureReports.markdownPath}`, + ]; +} + +function collectStartupBlockingIssues( + report: DoctorSessionSqliteReport, +): DoctorSessionSqliteIssue[] { + return report.targets.flatMap((target) => + target.issues.filter((issue) => !STARTUP_WARNING_ISSUE_CODES.has(issue.code)), + ); +} + +function collectStartupWarningIssues( + report: DoctorSessionSqliteReport, +): DoctorSessionSqliteIssue[] { + return report.targets.flatMap((target) => + target.issues.filter((issue) => STARTUP_WARNING_ISSUE_CODES.has(issue.code)), + ); +} + +function formatStartupIssueLines(issues: readonly DoctorSessionSqliteIssue[]): readonly string[] { + return issues.slice(0, 10).map((issue) => { + const key = issue.sessionKey ? `${issue.sessionKey}: ` : ""; + return `[${issue.code}] ${key}${issue.message}`; + }); +} + +function sessionSqliteReportHasChanges(report: DoctorSessionSqliteReport): boolean { + return ( + report.totals.importedEntries > 0 || + report.totals.importedTranscriptEvents > 0 || + report.totals.archivedTranscriptFiles > 0 || + report.totals.archivedUnreferencedJsonlFiles > 0 + ); +} + +function formatSessionSqliteStartupImportSummary(report: DoctorSessionSqliteReport): string { + return [ + "session: imported legacy session metadata/transcripts into SQLite:", + `- targets=${report.totals.targets} legacyEntries=${report.totals.legacyEntries} sqliteEntries=${report.totals.sqliteEntries}`, + `- importedEntries=${report.totals.importedEntries} importedTranscriptEvents=${report.totals.importedTranscriptEvents}`, + `- archivedTranscriptArtifacts=${report.totals.archivedTranscriptFiles} archivedUnreferencedJsonl=${report.totals.archivedUnreferencedJsonlFiles}`, + ].join("\n"); +} + +function isSqliteCorruptionError(error: unknown): boolean { + const code = error && typeof error === "object" ? (error as { code?: unknown }).code : undefined; + if (code === "SQLITE_CORRUPT" || code === "SQLITE_NOTADB") { + return true; + } + const message = String(error).toLowerCase(); + return message.includes("database disk image is malformed") || message.includes("not a database"); } diff --git a/src/gateway/server.agent.deleted-agent-gateway.test.ts b/src/gateway/server.agent.deleted-agent-gateway.test.ts index e5022edf89b6..762b58b5395c 100644 --- a/src/gateway/server.agent.deleted-agent-gateway.test.ts +++ b/src/gateway/server.agent.deleted-agent-gateway.test.ts @@ -1,8 +1,7 @@ -import fs from "node:fs/promises"; import path from "node:path"; import { expect, test, vi } from "vitest"; import { ErrorCodes } from "../../packages/gateway-protocol/src/index.js"; -import { agentCommand, rpcReq, testState } from "./test-helpers.js"; +import { agentCommand, rpcReq, testState, writeSessionStore } from "./test-helpers.js"; import { sessionStoreEntry, setupGatewaySessionsTestHarness, @@ -28,18 +27,13 @@ test("agent RPC rejects deleted-agent session keys before dispatch", async () => const deletedStorePath = storeTemplate.replace("{agentId}", "deleted-agent"); const orphanKey = "agent:deleted-agent:main"; - await fs.mkdir(path.dirname(deletedStorePath), { recursive: true }); - await fs.writeFile( - deletedStorePath, - JSON.stringify( - { - [orphanKey]: sessionStoreEntry("sess-orphan"), - }, - null, - 2, - ), - "utf-8", - ); + await writeSessionStore({ + storePath: deletedStorePath, + agentId: "deleted-agent", + entries: { + [orphanKey]: sessionStoreEntry("sess-orphan"), + }, + }); vi.mocked(agentCommand).mockClear(); const { ws } = await openClient(); @@ -67,18 +61,13 @@ test("agent RPC rejects archived session keys before dispatch", async () => { const mainStorePath = storeTemplate.replace("{agentId}", "main"); const archivedKey = "agent:main:subagent:archived"; - await fs.mkdir(path.dirname(mainStorePath), { recursive: true }); - await fs.writeFile( - mainStorePath, - JSON.stringify( - { - [archivedKey]: sessionStoreEntry("sess-archived", { archivedAt: Date.now() }), - }, - null, - 2, - ), - "utf-8", - ); + await writeSessionStore({ + storePath: mainStorePath, + agentId: "main", + entries: { + [archivedKey]: sessionStoreEntry("sess-archived", { archivedAt: Date.now() }), + }, + }); vi.mocked(agentCommand).mockClear(); const { ws } = await openClient(); @@ -106,18 +95,13 @@ test("agent RPC still dispatches for configured-agent session keys", async () => const storeTemplate = await configurePerAgentSessionStore(dir); const mainStorePath = storeTemplate.replace("{agentId}", "main"); - await fs.mkdir(path.dirname(mainStorePath), { recursive: true }); - await fs.writeFile( - mainStorePath, - JSON.stringify( - { - main: sessionStoreEntry("sess-main"), - }, - null, - 2, - ), - "utf-8", - ); + await writeSessionStore({ + storePath: mainStorePath, + agentId: "main", + entries: { + main: sessionStoreEntry("sess-main"), + }, + }); vi.mocked(agentCommand).mockClear(); const { ws } = await openClient(); diff --git a/src/gateway/server.agent.gateway-server-agent-a.test.ts b/src/gateway/server.agent.gateway-server-agent-a.test.ts index c11763726658..63df57c2d4ca 100644 --- a/src/gateway/server.agent.gateway-server-agent-a.test.ts +++ b/src/gateway/server.agent.gateway-server-agent-a.test.ts @@ -1,9 +1,9 @@ /** * Gateway server-agent integration tests for agent startup and session dispatch. */ -import fs from "node:fs/promises"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import type { ChannelPlugin } from "../channels/plugins/types.js"; +import { loadSessionEntry } from "../config/sessions/session-accessor.js"; import { getActiveGatewayRootWorkCount, isGatewaySubordinateWorkAdmissionClosed, @@ -88,7 +88,7 @@ async function runMainAgentDeliveryWithSession(params: { deliver: true, ...params.request, }); - expect(res.ok).toBe(true); + expect(res.ok, JSON.stringify(res)).toBe(true); return await waitForAgentCommandCall(String(params.request.idempotencyKey)); } finally { testState.allowFrom = undefined; @@ -128,6 +128,7 @@ async function runAgentImageRequest(params: { const res = await rpcReq(gatewaySuite.ws, "agent", { message: "what is in the image?", + ...(params.agentId ? { agentId: params.agentId } : {}), sessionKey: params.sessionKey ?? "main", attachments: [baseImageAttachment()], idempotencyKey: params.idempotencyKey, @@ -275,7 +276,7 @@ describe("gateway server agent", () => { deliver: true, idempotencyKey: "idem-agent-last-stale", }); - expect(res.ok).toBe(true); + expect(res.ok, JSON.stringify(res)).toBe(true); const call = await waitForAgentCommandCall("idem-agent-last-stale"); expectChannels(call, "whatsapp"); @@ -342,16 +343,16 @@ describe("gateway server agent", () => { expect(res.ok).toBe(true); await waitForAgentCommandCall("idem-agent-subdepth"); - const raw = await fs.readFile(gatewaySuite.sessionStorePath, "utf-8"); - const persisted = JSON.parse(raw) as Record< - string, - { spawnDepth?: number; spawnedBy?: string } - >; - expect(persisted["agent:main:subagent:depth"]?.spawnDepth).toBe(2); - expect(persisted["agent:main:subagent:depth"]?.spawnedBy).toBe("agent:main:main"); + const persisted = loadSessionEntry({ + sessionKey: "agent:main:subagent:depth", + storePath: gatewaySuite.sessionStorePath, + }) as { spawnDepth?: number; spawnedBy?: string } | undefined; + expect(persisted?.spawnDepth).toBe(2); + expect(persisted?.spawnedBy).toBe("agent:main:main"); }); test("agent derives sessionKey from agentId", async () => { + testState.agentsConfig = { list: [{ id: "ops" }] }; await setTestSessionStore({ agentId: "ops", entries: { @@ -361,13 +362,12 @@ describe("gateway server agent", () => { }, }, }); - testState.agentsConfig = { list: [{ id: "ops" }] }; const res = await rpcReq(gatewaySuite.ws, "agent", { message: "hi", agentId: "ops", idempotencyKey: "idem-agent-id", }); - expect(res.ok).toBe(true); + expect(res.ok, JSON.stringify(res)).toBe(true); const call = await waitForAgentCommandCall("idem-agent-id"); expect(call.sessionKey).toBe("agent:ops:main"); diff --git a/src/gateway/server.agent.gateway-server-agent-b.test.ts b/src/gateway/server.agent.gateway-server-agent-b.test.ts index 0e8fc4478553..46176c0bdbd0 100644 --- a/src/gateway/server.agent.gateway-server-agent-b.test.ts +++ b/src/gateway/server.agent.gateway-server-agent-b.test.ts @@ -1,12 +1,12 @@ // Gateway agent integration tests cover channel routing, session context, // WebSocket requests, agent event delivery, and provider/runtime error handling. -import fs from "node:fs/promises"; import path from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi } from "vitest"; import { WebSocket } from "ws"; import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js"; import { AcpRuntimeError } from "../acp/runtime/errors.js"; import type { ChannelPlugin } from "../channels/plugins/types.js"; +import { loadSessionEntry } from "../config/sessions/session-accessor.js"; import { emitAgentEvent, registerAgentRunContext } from "../infra/agent-events.js"; import { createChannelTestPluginBase, @@ -273,15 +273,17 @@ describe("gateway server agent", () => { if (!sessionStorePath) { throw new Error("expected session store path"); } - const stored = JSON.parse(await fs.readFile(sessionStorePath, "utf-8")) as Record< - string, - { - cliSessionBindings?: Record<string, unknown>; - cliSessionIds?: Record<string, string>; - claudeCliSessionId?: string; - } - >; - expect(stored["agent:main:main"]?.cliSessionBindings).toEqual({ + const stored = loadSessionEntry({ + sessionKey: "agent:main:main", + storePath: sessionStorePath, + }) as + | { + cliSessionBindings?: Record<string, unknown>; + cliSessionIds?: Record<string, string>; + claudeCliSessionId?: string; + } + | undefined; + expect(stored?.cliSessionBindings).toEqual({ "claude-cli": { sessionId: "cli-session-123", authProfileId: "anthropic:work", @@ -289,10 +291,10 @@ describe("gateway server agent", () => { mcpResumeHash: "mcp-resume-hash", }, }); - expect(stored["agent:main:main"]?.cliSessionIds).toEqual({ + expect(stored?.cliSessionIds).toEqual({ "claude-cli": "cli-session-123", }); - expect(stored["agent:main:main"]?.claudeCliSessionId).toBe("cli-session-123"); + expect(stored?.claudeCliSessionId).toBe("cli-session-123"); }); test("agent accepts built-in channel alias (imsg)", async () => { @@ -502,11 +504,8 @@ describe("gateway server agent", () => { expect(viaAgent.ok).toBe(false); expect(viaAgent.error?.message).toContain("missing scope: operator.admin"); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { sessionId?: string } - >; - expect(store["agent:main:main"]?.sessionId).toBe("sess-main-before-write-reset"); + const stored = loadSessionEntry({ sessionKey: "agent:main:main", storePath }); + expect(stored?.sessionId).toBe("sess-main-before-write-reset"); expect(vi.mocked(agentCommand)).not.toHaveBeenCalled(); writeWs.close(); diff --git a/src/gateway/server.agent.subagent-delivery-context.test.ts b/src/gateway/server.agent.subagent-delivery-context.test.ts index 61330fbf01e1..3ad426bc23d0 100644 --- a/src/gateway/server.agent.subagent-delivery-context.test.ts +++ b/src/gateway/server.agent.subagent-delivery-context.test.ts @@ -1,8 +1,8 @@ // Subagent delivery-context tests protect route metadata inheritance for child // agent sessions and outbound delivery through channel plugins. -import fs from "node:fs/promises"; import { describe, expect, test } from "vitest"; import type { ChannelPlugin } from "../channels/plugins/types.js"; +import { loadSessionEntry } from "../config/sessions/session-accessor.js"; import { createChannelTestPluginBase, createDirectOutboundTestAdapter, @@ -66,14 +66,6 @@ async function prepareSessionStore(entries: StoreEntries = {}): Promise<void> { await writeSessionStore({ entries }); } -function readStoredEntry(stored: Record<string, StoredEntry>, key: string): StoredEntry { - const entry = stored[key]; - if (!entry) { - throw new Error(`expected stored entry ${key}`); - } - return entry; -} - function readDeliveryContext(entry: StoredEntry): NonNullable<StoredEntry["deliveryContext"]> { if (!entry.deliveryContext) { throw new Error("expected stored deliveryContext"); @@ -82,11 +74,13 @@ function readDeliveryContext(entry: StoredEntry): NonNullable<StoredEntry["deliv } async function readStoredSessionEntry(key: string): Promise<StoredEntry> { - const stored = JSON.parse(await fs.readFile(gatewaySuite.sessionStorePath, "utf-8")) as Record< - string, - StoredEntry - >; - return readStoredEntry(stored, key); + const entry = loadSessionEntry({ sessionKey: key, storePath: gatewaySuite.sessionStorePath }) as + | StoredEntry + | undefined; + if (!entry) { + throw new Error(`expected stored entry ${key}`); + } + return entry; } async function sendAgentRequest(params: Record<string, unknown>): Promise<void> { diff --git a/src/gateway/server.chat.gateway-server-chat-b.test.ts b/src/gateway/server.chat.gateway-server-chat-b.test.ts index 065cb02ca57a..6799def04940 100644 --- a/src/gateway/server.chat.gateway-server-chat-b.test.ts +++ b/src/gateway/server.chat.gateway-server-chat-b.test.ts @@ -8,6 +8,7 @@ import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import type { GetReplyOptions } from "../auto-reply/get-reply-options.types.js"; import type { InternalGetReplyOptions } from "../auto-reply/reply/get-reply.types.js"; import { clearConfigCache, getRuntimeConfig } from "../config/config.js"; +import { appendTranscriptEvent, loadSessionEntry } from "../config/sessions/session-accessor.js"; import { invalidateSessionStoreCache } from "../config/sessions/store-cache.js"; import type { AgentModelConfig } from "../config/types.agents-shared.js"; import { rotateAgentEventLifecycleGeneration } from "../infra/agent-events.js"; @@ -97,13 +98,12 @@ function testSessionFilePath(sessionDir: string, sessionId: string): string { return path.join(sessionDir, `${sessionId}.jsonl`); } -async function writeMainSessionStore(sessionDir?: string, sessionId = "sess-main") { +async function writeMainSessionStore(_sessionDir?: string, sessionId = "sess-main") { await writeSessionStore({ entries: { main: { sessionId, updatedAt: futureFixtureUpdatedAt(), - ...(sessionDir ? { sessionFile: testSessionFilePath(sessionDir, sessionId) } : {}), }, }, }); @@ -136,11 +136,32 @@ async function writeGatewayConfig(config: Record<string, unknown>) { } async function writeMainSessionTranscript( - sessionDir: string, + _sessionDir: string, lines: string[], sessionId = "sess-main", + opts?: { + agentId?: string; + sessionKey?: string; + }, ) { - await fs.writeFile(testSessionFilePath(sessionDir, sessionId), `${lines.join("\n")}\n`, "utf-8"); + const storePath = testState.sessionStorePath; + if (!storePath) { + throw new Error("session store path was not initialized"); + } + for (const line of lines) { + if (!line.trim()) { + continue; + } + await appendTranscriptEvent( + { + agentId: opts?.agentId ?? "main", + sessionId, + sessionKey: opts?.sessionKey ?? "agent:main:main", + storePath, + }, + JSON.parse(line) as unknown, + ); + } } async function removeTempDir(dir: string): Promise<void> { @@ -1433,11 +1454,12 @@ describe("gateway server chat", () => { }, }, }); - const [{ deleteSessionEntryLifecycle }, { loadSessionEntry }] = await Promise.all([ - import("../config/sessions/session-accessor.js"), - import("./session-utils.js"), - ]); - const seededSession = loadSessionEntry("main"); + const [{ deleteSessionEntryLifecycle }, { loadSessionEntry: loadGatewaySessionEntry }] = + await Promise.all([ + import("../config/sessions/session-accessor.js"), + import("./session-utils.js"), + ]); + const seededSession = loadGatewaySessionEntry("main"); const seededSessionId = seededSession.entry?.sessionId; expect(seededSessionId).toBe("sess-main"); const mutationStarted = createDeferred(); @@ -3301,12 +3323,13 @@ describe("gateway server chat", () => { if (!sessionStorePath) { throw new Error("expected session store path"); } - const stored = JSON.parse(await fs.readFile(sessionStorePath, "utf-8")) as Record< - string, - { lastChannel?: string; lastTo?: string } | undefined - >; - expect(stored["agent:main:main"]?.lastChannel).toBe("whatsapp"); - expect(stored["agent:main:main"]?.lastTo).toBe("+1555"); + const stored = loadSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath: sessionStorePath, + }); + expect(stored?.lastChannel).toBe("whatsapp"); + expect(stored?.lastTo).toBe("+1555"); await vi.waitFor(async () => { const completed = await rpcReq<{ status?: string }>(ws, "chat.send", { @@ -4013,21 +4036,25 @@ describe("gateway server chat", () => { await connectOk(ws); const sessionDir = await createSessionDir(); await writeSessionStore({ + agentId: "work", entries: { global: { sessionId: "sess-global", updatedAt: Date.now() }, }, }); - await fs.writeFile( - path.join(sessionDir, "sess-global.jsonl"), - `${JSON.stringify({ - id: "msg-global-agent", - message: { - role: "assistant", - content: [{ type: "text", text: "global agent content" }], - timestamp: Date.now(), - }, - })}\n`, - "utf-8", + await writeMainSessionTranscript( + sessionDir, + [ + JSON.stringify({ + id: "msg-global-agent", + message: { + role: "assistant", + content: [{ type: "text", text: "global agent content" }], + timestamp: Date.now(), + }, + }), + ], + "sess-global", + { agentId: "work", sessionKey: "global" }, ); const full = await fetchChatMessage(ws, { @@ -4040,9 +4067,10 @@ describe("gateway server chat", () => { }); }); - test("chat.message.get reports oversized transcript entries as unavailable", async () => { + test("chat.message.get reports oversized archive transcript entries as unavailable", async () => { await withGatewayChatHarness(async ({ ws, createSessionDir }) => { - const sessionDir = await prepareMainHistoryHarness({ ws, createSessionDir }); + const sessionId = "sess-oversized-archive"; + const sessionDir = await prepareMainHistoryHarness({ ws, createSessionDir, sessionId }); const oversizedLine = JSON.stringify({ id: "msg-oversized", message: { @@ -4051,7 +4079,11 @@ describe("gateway server chat", () => { timestamp: Date.now(), }, }); - await writeMainSessionTranscript(sessionDir, [oversizedLine]); + await fs.writeFile( + `${testSessionFilePath(sessionDir, sessionId)}.reset.2026-02-16T22-26-34.000Z`, + [JSON.stringify({ type: "session", version: 1, id: sessionId }), oversizedLine].join("\n"), + "utf-8", + ); const full = await fetchChatMessage(ws, { sessionKey: "main", @@ -4063,6 +4095,30 @@ describe("gateway server chat", () => { }); }); + test("chat.message.get returns active SQLite oversized transcript entries", async () => { + await withGatewayChatHarness(async ({ ws, createSessionDir }) => { + const sessionDir = await prepareMainHistoryHarness({ ws, createSessionDir }); + const oversizedText = "x".repeat(300 * 1024); + await writeMainSessionTranscript(sessionDir, [ + JSON.stringify({ + id: "msg-oversized-sqlite", + message: { + role: "assistant", + content: [{ type: "text", text: oversizedText }], + timestamp: Date.now(), + }, + }), + ]); + + const full = await fetchChatMessage(ws, { + sessionKey: "main", + messageId: "msg-oversized-sqlite", + }); + expect(full.ok).toBe(true); + expect(JSON.stringify(full.message)).toContain(oversizedText.slice(0, 256)); + }); + }); + test("chat.message.get does not return inactive branch entries", async () => { await withGatewayChatHarness(async ({ ws, createSessionDir }) => { const sessionDir = await prepareMainHistoryHarness({ ws, createSessionDir }); diff --git a/src/gateway/server.chat.gateway-server-chat.test.ts b/src/gateway/server.chat.gateway-server-chat.test.ts index eb9c38e8c210..c2b912678706 100644 --- a/src/gateway/server.chat.gateway-server-chat.test.ts +++ b/src/gateway/server.chat.gateway-server-chat.test.ts @@ -5,6 +5,8 @@ import os from "node:os"; import path from "node:path"; import { beforeEach, describe, expect, test, vi } from "vitest"; import { WebSocket } from "ws"; +import { loadSessionEntry } from "../config/sessions/session-accessor.js"; +import { replaceSqliteTranscriptEvents } from "../config/sessions/session-accessor.sqlite.js"; import { emitAgentEvent, registerAgentRunContext } from "../infra/agent-events.js"; import { getActiveGatewayRootWorkCount, @@ -97,9 +99,9 @@ describe("gateway server chat", () => { const loadChatHistoryWithMessages = async ( messages: Array<Record<string, unknown>>, ): Promise<unknown[]> => { - return withMainSessionStore(async (dir) => { + return withMainSessionStore(async () => { const lines = messages.map((message) => JSON.stringify({ message })); - await fs.writeFile(path.join(dir, "sess-main.jsonl"), lines.join("\n"), "utf-8"); + await replaceMainTranscriptLines(lines); const res = await rpcReq<{ messages?: unknown[] }>(ws, "chat.history", { sessionKey: "main", @@ -109,6 +111,22 @@ describe("gateway server chat", () => { }); }; + const replaceMainTranscriptLines = async (lines: string[]): Promise<void> => { + const storePath = testState.sessionStorePath; + if (!storePath) { + throw new Error("session store path was not initialized"); + } + const events = lines.map((line, index) => ({ + ...(JSON.parse(line) as Record<string, unknown>), + id: `message-${index}`, + type: "message", + })); + await replaceSqliteTranscriptEvents( + { agentId: "main", sessionId: "sess-main", sessionKey: "main", storePath }, + events, + ); + }; + const withMainSessionStore = async <T>( run: (dir: string) => Promise<T>, options?: { archivedAt?: number; sessionId?: string }, @@ -361,13 +379,12 @@ describe("gateway server chat", () => { expect(res.ok).toBe(true); expect(res.payload?.runId).toBe("idem-sessions-send-orion"); - const rawStore = JSON.parse(await fs.readFile(testState.sessionStorePath, "utf-8")) as Record< - string, - { - sessionId?: string; - } - >; - expect(rawStore["agent:orion:main"]?.sessionId).toBeTypeOf("string"); + expect( + loadSessionEntry({ + sessionKey: "agent:orion:main", + storePath: testState.sessionStorePath, + })?.sessionId, + ).toBeTypeOf("string"); } finally { testState.agentsConfig = undefined; testState.sessionStorePath = undefined; @@ -740,7 +757,7 @@ describe("gateway server chat", () => { }), ); } - await fs.writeFile(path.join(historyDir, "sess-main.jsonl"), lines.join("\n"), "utf-8"); + await replaceMainTranscriptLines(lines); const defaultRes = await rpcReq<{ messages?: unknown[] }>(ws, "chat.history", { sessionKey: "main", @@ -1561,18 +1578,16 @@ describe("gateway server chat", () => { }); test("routes /btw replies through side-result events without transcript injection", async () => { - await withMainSessionStore(async (dir) => { - await fs.writeFile( - path.join(dir, "sess-main.jsonl"), - `${JSON.stringify({ + await withMainSessionStore(async () => { + await replaceMainTranscriptLines([ + JSON.stringify({ message: { role: "user", content: [{ type: "text", text: "main thread context" }], timestamp: Date.now(), }, - })}\n`, - "utf-8", - ); + }), + ]); dispatchInboundMessageMock.mockImplementationOnce(async (...args: unknown[]) => { const [params] = args as [ { @@ -1649,18 +1664,16 @@ describe("gateway server chat", () => { }); test("routes block-streamed /btw replies through side-result events", async () => { - await withMainSessionStore(async (dir) => { - await fs.writeFile( - path.join(dir, "sess-main.jsonl"), - `${JSON.stringify({ + await withMainSessionStore(async () => { + await replaceMainTranscriptLines([ + JSON.stringify({ message: { role: "assistant", content: [{ type: "text", text: "existing context" }], timestamp: Date.now(), }, - })}\n`, - "utf-8", - ); + }), + ]); dispatchInboundMessageMock.mockImplementationOnce(async (...args: unknown[]) => { const [params] = args as [ { @@ -1913,13 +1926,10 @@ describe("gateway server chat", () => { if (!sessionStorePath) { throw new Error("session store path was not initialized"); } - const raw = await fs.readFile(sessionStorePath, "utf-8"); - const stored = JSON.parse(raw) as { - "agent:main:main"?: { - verboseLevel?: string; - }; - }; - expect(stored["agent:main:main"]?.verboseLevel).toBeUndefined(); + expect( + loadSessionEntry({ sessionKey: "agent:main:main", storePath: sessionStorePath }) + ?.verboseLevel, + ).toBeUndefined(); } finally { scopedWs?.close(); } @@ -1948,17 +1958,13 @@ describe("gateway server chat", () => { if (!sessionStorePath) { throw new Error("session store path was not initialized"); } - const raw = await fs.readFile(sessionStorePath, "utf-8"); - const stored = JSON.parse(raw) as { - "agent:main:main"?: { - thinkingLevel?: string; - }; - main?: { - thinkingLevel?: string; - }; - }; - expect(stored["agent:main:main"]?.thinkingLevel).toBeUndefined(); - expect(stored.main?.thinkingLevel).toBeUndefined(); + expect( + loadSessionEntry({ sessionKey: "agent:main:main", storePath: sessionStorePath }) + ?.thinkingLevel, + ).toBeUndefined(); + expect( + loadSessionEntry({ sessionKey: "main", storePath: sessionStorePath })?.thinkingLevel, + ).toBeUndefined(); }); }); @@ -1995,13 +2001,10 @@ describe("gateway server chat", () => { if (!sessionStorePath) { throw new Error("session store path was not initialized"); } - const raw = await fs.readFile(sessionStorePath, "utf-8"); - const stored = JSON.parse(raw) as { - "agent:main:main"?: { - sessionId?: string; - }; - }; - expect(stored["agent:main:main"]?.sessionId).toBe("sess-main"); + expect( + loadSessionEntry({ sessionKey: "agent:main:main", storePath: sessionStorePath }) + ?.sessionId, + ).toBe("sess-main"); } finally { scopedWs?.close(); } diff --git a/src/gateway/server.config-patch.test.ts b/src/gateway/server.config-patch.test.ts index 0676467426c3..21298ee6aed3 100644 --- a/src/gateway/server.config-patch.test.ts +++ b/src/gateway/server.config-patch.test.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { resolveDefaultAgentDir } from "../agents/agent-scope.js"; import { AUTH_PROFILE_FILENAME } from "../agents/auth-profiles/constants.js"; +import { loadSessionEntry } from "../config/sessions/session-accessor.js"; import { deleteTestEnvValue } from "../test-utils/env.js"; import { testing as controlPlaneRateLimitTesting } from "./control-plane-rate-limit.js"; import { @@ -1057,11 +1058,9 @@ describe("gateway server sessions", () => { expect(patched.ok).toBe(true); expect(patched.payload?.key).toBe("agent:ops:work"); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { thinkingLevel?: string } - >; - expect(stored["agent:ops:work"]?.thinkingLevel).toBe("medium"); - expect(stored.main).toBeUndefined(); + expect( + loadSessionEntry({ agentId: "ops", sessionKey: "agent:ops:work", storePath })?.thinkingLevel, + ).toBe("medium"); + expect(loadSessionEntry({ agentId: "ops", sessionKey: "main", storePath })).toBeUndefined(); }); }); diff --git a/src/gateway/server.sessions-send.test.ts b/src/gateway/server.sessions-send.test.ts index bdc22b63e98c..206dfa9a4d2b 100644 --- a/src/gateway/server.sessions-send.test.ts +++ b/src/gateway/server.sessions-send.test.ts @@ -6,7 +6,10 @@ import path from "node:path"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi, type Mock } from "vitest"; import { testing as agentStepTesting } from "../agents/tools/agent-step.js"; import { runSessionsSendA2AFlow } from "../agents/tools/sessions-send-tool.a2a.js"; -import { resolveSessionTranscriptPath } from "../config/sessions.js"; +import { + loadSessionEntry, + persistSessionTranscriptTurn, +} from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { emitAgentEvent } from "../infra/agent-events.js"; import { createOutboundTestPlugin, createTestRegistry } from "../test-utils/channel-plugins.js"; @@ -74,21 +77,9 @@ async function emitLifecycleAssistantReply(params: { }; const sessionId = commandParams.sessionId ?? params.defaultSessionId; const runId = commandParams.runId ?? sessionId; - let sessionFile = resolveSessionTranscriptPath(sessionId); - if (testState.sessionStorePath && commandParams.sessionKey) { - const rawStore = JSON.parse(await fs.readFile(testState.sessionStorePath, "utf-8")) as Record< - string, - { - sessionId?: string; - sessionFile?: string; - } - >; - const entry = rawStore[commandParams.sessionKey]; - if (entry?.sessionId === sessionId && entry.sessionFile) { - sessionFile = entry.sessionFile; - } + if (!commandParams.sessionKey) { + throw new Error("expected session key for lifecycle reply"); } - await fs.mkdir(path.dirname(sessionFile), { recursive: true }); const startedAt = Date.now(); emitAgentEvent({ @@ -103,7 +94,18 @@ async function emitLifecycleAssistantReply(params: { content: [{ type: "text", text }], ...(params.includeTimestamp ? { timestamp: Date.now() } : {}), }; - await fs.appendFile(sessionFile, `${JSON.stringify({ message })}\n`, "utf8"); + await persistSessionTranscriptTurn( + { + sessionId, + sessionKey: commandParams.sessionKey, + ...(testState.sessionStorePath ? { storePath: testState.sessionStorePath } : {}), + }, + { + cwd: "/tmp", + updateMode: "none", + messages: [{ message, now: Date.now() }], + }, + ); emitAgentEvent({ runId, @@ -300,7 +302,6 @@ describe("sessions_send gateway loopback", () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-sessions-send-mirror-")); const sessionKey = "agent:main:whatsapp:direct:peer-1"; const sessionId = "sess-whatsapp-mirror"; - const sessionFile = path.join(dir, `${sessionId}.jsonl`); const runId = `run-message-tool-mirror-${Date.now()}-${Math.random().toString(36).slice(2)}`; const deliveredReply = "already delivered source reply"; const sendCalls: Array<{ @@ -349,7 +350,6 @@ describe("sessions_send gateway loopback", () => { entries: { [sessionKey]: { sessionId, - sessionFile, updatedAt: Date.now(), deliveryContext: { channel: "whatsapp", @@ -363,55 +363,56 @@ describe("sessions_send gateway loopback", () => { }, }, }); - await fs.writeFile( - sessionFile, - [ - { - message: { - role: "assistant", - content: [{ type: "text", text: "previous real reply" }], - timestamp: 1, + await persistSessionTranscriptTurn( + { sessionId, sessionKey, storePath: testState.sessionStorePath }, + { + cwd: dir, + updateMode: "none", + messages: [ + { + message: { + role: "assistant", + content: [{ type: "text", text: "previous real reply" }], + timestamp: 1, + }, }, - }, - { - message: { - role: "assistant", - content: [ - { - type: "toolCall", - id: "call-message-duplicate-proof", - name: "message", - arguments: { - action: "send", - message: deliveredReply, + { + message: { + role: "assistant", + content: [ + { + type: "toolCall", + id: "call-message-duplicate-proof", + name: "message", + arguments: { + action: "send", + message: deliveredReply, + }, }, - }, - ], - timestamp: 2, + ], + timestamp: 2, + }, }, - }, - { - message: { - role: "toolResult", - toolName: "message", - toolCallId: "call-message-duplicate-proof", - content: { ok: true, messageId: "24271", chatId: "peer-1" }, - timestamp: 3, + { + message: { + role: "toolResult", + toolName: "message", + toolCallId: "call-message-duplicate-proof", + content: { ok: true, messageId: "24271", chatId: "peer-1" }, + timestamp: 3, + }, }, - }, - { - message: { - role: "assistant", - provider: "openclaw", - model: "delivery-mirror", - content: [{ type: "text", text: deliveredReply }], - timestamp: 4, + { + message: { + role: "assistant", + provider: "openclaw", + model: "delivery-mirror", + content: [{ type: "text", text: deliveredReply }], + timestamp: 4, + }, }, - }, - ] - .map((entry) => JSON.stringify(entry)) - .join("\n") + "\n", - "utf-8", + ], + }, ); const { callGateway } = await import("./call.js"); @@ -605,15 +606,11 @@ describe("sessions_send agent targeting", () => { expect(orionCall).toBeDefined(); expect(orionCall?.sessionId).toBeTypeOf("string"); - const rawStore = JSON.parse( - await fs.readFile(testState.sessionStorePath, "utf-8"), - ) as Record< - string, - { - sessionId?: string; - } - >; - expect(rawStore["agent:orion:main"]?.sessionId).toBe(orionCall?.sessionId); + const stored = loadSessionEntry({ + sessionKey: "agent:orion:main", + storePath: testState.sessionStorePath, + }); + expect(stored?.sessionId).toBe(orionCall?.sessionId); } finally { testState.agentsConfig = undefined; testState.sessionStorePath = undefined; diff --git a/src/gateway/server.sessions.compaction.test.ts b/src/gateway/server.sessions.compaction.test.ts index 2beb1bea2c5a..ba269d7744ba 100644 --- a/src/gateway/server.sessions.compaction.test.ts +++ b/src/gateway/server.sessions.compaction.test.ts @@ -6,52 +6,37 @@ import os from "node:os"; import path from "node:path"; import { expect, test, vi } from "vitest"; import type { SessionCompactionCheckpoint } from "../config/sessions.js"; +import { + appendTranscriptMessage, + appendTranscriptEvent, + loadSessionEntry as loadAccessorSessionEntry, + loadTranscriptEvents, + patchSessionEntry as patchAccessorSessionEntry, + replaceSessionEntry, + upsertSessionEntry, +} from "../config/sessions/session-accessor.js"; import { beginSessionWorkAdmission, runExclusiveSessionLifecycleMutation, } from "../sessions/session-lifecycle-admission.js"; import { withTempDir } from "../test-helpers/temp-dir.js"; -import { withEnvAsync } from "../test-utils/env.js"; import { embeddedRunMock, onceMessage, agentDiscoveryMock, rpcReq, - startConnectedServerWithClient, testState, - writeSessionStore, } from "./test-helpers.js"; import { setupGatewaySessionsTestHarness, createDeferred, getSessionManagerModule, - getGatewayConfigModule, sessionStoreEntry, createCheckpointFixture, directSessionReq, expectSessionQueueCleanup, } from "./test/server-sessions.test-helpers.js"; -const sessionLifecycleMutationObserver = vi.hoisted(() => ({ - onEnter: undefined as - | ((params: { prepare?: () => Promise<void>; scope: string }) => void) - | undefined, -})); - -vi.mock("../sessions/session-lifecycle-admission.js", async (importOriginal) => { - const actual = - await importOriginal<typeof import("../sessions/session-lifecycle-admission.js")>(); - return { - ...actual, - runExclusiveSessionLifecycleMutation: async ( - params: Parameters<typeof actual.runExclusiveSessionLifecycleMutation>[0], - ) => { - sessionLifecycleMutationObserver.onEnter?.(params); - return await actual.runExclusiveSessionLifecycleMutation(params); - }, - }; -}); - const { createSessionStoreDir, createSelectedGlobalSessionStore, openClient } = setupGatewaySessionsTestHarness(); @@ -130,11 +115,88 @@ function expectMainCompactionResult( compacted: { ok?: boolean; payload?: { compacted?: boolean; key?: string } | null }, expectedCompacted: boolean, ) { - expect(compacted.ok).toBe(true); + expect(compacted.ok, JSON.stringify(compacted)).toBe(true); expect(compacted.payload?.key).toBe("agent:main:main"); expect(compacted.payload?.compacted).toBe(expectedCompacted); } +async function seedSessionEntry(params: { + agentId?: string; + entry: ReturnType<typeof sessionStoreEntry>; + sessionKey: string; + storePath: string; +}): Promise<void> { + await upsertSessionEntry( + { + ...(params.agentId ? { agentId: params.agentId } : {}), + sessionKey: params.sessionKey, + storePath: params.storePath, + }, + params.entry, + ); +} + +function loadSessionEntry(params: { + agentId?: string; + sessionKey: string; + storePath: string; +}): ReturnType<typeof loadAccessorSessionEntry> { + return loadAccessorSessionEntry({ + ...(params.agentId ? { agentId: params.agentId } : {}), + readConsistency: "latest", + sessionKey: params.sessionKey, + storePath: params.storePath, + }); +} + +async function seedTranscriptRows(params: { + agentId?: string; + sessionId: string; + sessionKey: string; + storePath: string; + totalLines: number; +}): Promise<void> { + const scope = { + ...(params.agentId ? { agentId: params.agentId } : {}), + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }; + if (params.totalLines <= 0) { + return; + } + const header = JSON.parse(buildSessionTranscriptLines(params.sessionId, 1)[0] ?? "{}"); + await appendTranscriptEvent(scope, header); + for (let index = 0; index < params.totalLines - 1; index += 1) { + await appendTranscriptMessage(scope, { + cwd: "/tmp", + message: { + role: "user", + content: `line-${index}`, + timestamp: index, + }, + now: Date.parse(`2026-06-19T12:00:${String(index % 60).padStart(2, "0")}.000Z`), + }); + } +} + +async function loadTranscriptRows(params: { + agentId?: string; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise<Array<Record<string, unknown>>> { + const rows = await loadTranscriptEvents({ + ...(params.agentId ? { agentId: params.agentId } : {}), + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }); + return rows.map((row) => + row && typeof row === "object" && !Array.isArray(row) ? (row as Record<string, unknown>) : {}, + ); +} + test("sessions.compaction.* lists checkpoints and branches or restores from compacted transcripts", async () => { const { dir, storePath } = await createSessionStoreDir(); const fixture = await createCheckpointFixture(dir, { legacyPreCompactionSnapshot: false }); @@ -151,13 +213,19 @@ test("sessions.compaction.* lists checkpoints and branches or restores from comp tokensAfter: 45, }); const { SessionManager } = await getSessionManagerModule(); - await writeSessionStore({ - entries: { - main: sessionStoreEntry(fixture.sessionId, { - sessionFile: fixture.sessionFile, - compactionCheckpoints: [checkpointEntry], - }), - }, + await seedSessionEntry({ + entry: sessionStoreEntry(fixture.sessionId, { + sessionFile: fixture.sessionFile, + compactionCheckpoints: [checkpointEntry], + }), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId: fixture.sessionId, + sessionKey: "agent:main:main", + storePath, + totalLines: 2, }); fixture.session.appendMessage({ role: "user", @@ -273,15 +341,10 @@ test("sessions.compaction.* lists checkpoints and branches or restores from comp ), ).toBe(false); - const storeAfterBranch = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { - parentSessionKey?: string; - compactionCheckpoints?: unknown[]; - sessionId?: string; - } - >; - const branchedEntry = storeAfterBranch[branched.payload!.key]; + const branchedEntry = loadSessionEntry({ + sessionKey: branched.payload!.key, + storePath, + }); expect(branchedEntry?.parentSessionKey).toBe("agent:main:main"); expect(branchedEntry?.compactionCheckpoints).toBeUndefined(); @@ -345,12 +408,9 @@ test("sessions.compaction.* lists checkpoints and branches or restores from comp ), ).toBe(false); - const storeAfterRestore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { compactionCheckpoints?: unknown[]; sessionId?: string } - >; - expect(storeAfterRestore["agent:main:main"]?.sessionId).toBe(restored.payload?.sessionId); - expect(storeAfterRestore["agent:main:main"]?.compactionCheckpoints).toHaveLength(1); + const restoredEntry = loadSessionEntry({ sessionKey: "agent:main:main", storePath }); + expect(restoredEntry?.sessionId).toBe(restored.payload?.sessionId); + expect(restoredEntry?.compactionCheckpoints).toHaveLength(1); ws.close(); }); @@ -365,18 +425,16 @@ test("sessions.compaction.branch rejects model-selection-locked session identiti reason: "manual", summary: "locked checkpoint", }); - await writeSessionStore({ - entries: { - main: sessionStoreEntry(fixture.sessionId, { - sessionFile: fixture.sessionFile, - compactionCheckpoints: [checkpointEntry], - modelSelectionLocked: true, - }), - }, + await seedSessionEntry({ + entry: sessionStoreEntry(fixture.sessionId, { + sessionFile: fixture.sessionFile, + compactionCheckpoints: [checkpointEntry], + modelSelectionLocked: true, + }), + sessionKey: "agent:main:main", + storePath, }); - const filesBefore = (await fs.readdir(dir)).toSorted(); const { ws } = await openClient(); - try { await expect( rpcReq(ws, "sessions.compaction.branch", { @@ -390,63 +448,66 @@ test("sessions.compaction.branch rejects model-selection-locked session identiti message: "Checkpoint branch and restore are unavailable while model selection is locked.", }, }); - const nextStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { modelSelectionLocked?: boolean; sessionId?: string } - >; - expect(nextStore).toEqual({ - "agent:main:main": expect.objectContaining({ - modelSelectionLocked: true, - sessionId: fixture.sessionId, - }), + expect(loadSessionEntry({ sessionKey: "agent:main:main", storePath })).toMatchObject({ + modelSelectionLocked: true, + sessionId: fixture.sessionId, }); - expect((await fs.readdir(dir)).toSorted()).toEqual(filesBefore); } finally { ws.close(); } }); -test("sessions.compaction.* scopes selected global checkpoints to the requested agent", async () => { - const { mainStorePath, workStorePath } = await createSelectedGlobalSessionStore(); - const workDir = path.dirname(workStorePath); +test("sessions.compaction list/get scopes selected global checkpoints to the requested agent", async () => { + const { mainStorePath, storeTemplate, workStorePath } = await createSelectedGlobalSessionStore(); + const runtimeConfig = { + agents: { list: [{ id: "main", default: true }, { id: "work" }] }, + session: { mainKey: "main", scope: "global", store: storeTemplate }, + }; await fs.mkdir(path.dirname(mainStorePath), { recursive: true }); - await fs.mkdir(workDir, { recursive: true }); - const mainSessionFile = path.join(path.dirname(mainStorePath), "sess-main-global.jsonl"); - await fs.writeFile(mainSessionFile, `${JSON.stringify({ role: "user", content: "main" })}\n`); - const fixture = await createCheckpointFixture(workDir, { legacyPreCompactionSnapshot: false }); + await fs.mkdir(path.dirname(workStorePath), { recursive: true }); const checkpointCreatedAt = Date.now(); - const checkpointEntry = compactionCheckpointEntry(fixture, { + const checkpointEntry: SessionCompactionCheckpoint = { checkpointId: "checkpoint-work", sessionKey: "global", createdAt: checkpointCreatedAt, reason: "manual", summary: "work checkpoint", + sessionId: "sess-work-global", + firstKeptEntryId: "entry-work-kept", + preCompaction: { + sessionId: "sess-work-global", + leafId: "entry-work-before", + }, + postCompaction: { + sessionId: "sess-work-global", + leafId: "entry-work-kept", + entryId: "entry-work-kept", + }, + }; + await seedSessionEntry({ + agentId: "main", + entry: sessionStoreEntry("sess-main-global"), + sessionKey: "global", + storePath: mainStorePath, + }); + await seedSessionEntry({ + agentId: "work", + entry: sessionStoreEntry("sess-work-global", { + compactionCheckpoints: [checkpointEntry], + }), + sessionKey: "global", + storePath: workStorePath, }); - await fs.writeFile( - mainStorePath, - JSON.stringify( - { global: sessionStoreEntry("sess-main-global", { sessionFile: mainSessionFile }) }, - null, - 2, - ), - ); - await fs.writeFile( - workStorePath, - JSON.stringify( - { - global: sessionStoreEntry(fixture.sessionId, { - sessionFile: fixture.sessionFile, - compactionCheckpoints: [checkpointEntry], - }), - }, - null, - 2, - ), - ); const listed = await directSessionReq<{ checkpoints: Array<{ checkpointId: string; summary?: string }>; - }>("sessions.compaction.list", { key: "global", agentId: "work" }); + }>( + "sessions.compaction.list", + { key: "global", agentId: "work" }, + { + context: { getRuntimeConfig: () => runtimeConfig }, + }, + ); expect(listed.ok).toBe(true); expect(listed.payload?.checkpoints).toHaveLength(1); expect(listed.payload?.checkpoints[0]).toMatchObject({ @@ -454,30 +515,26 @@ test("sessions.compaction.* scopes selected global checkpoints to the requested summary: "work checkpoint", }); - const branched = await directSessionReq<{ key?: string; sourceKey?: string }>( - "sessions.compaction.branch", + const got = await directSessionReq<{ + checkpoint?: { checkpointId?: string; summary?: string }; + }>( + "sessions.compaction.get", { key: "global", agentId: "work", checkpointId: "checkpoint-work" }, + { context: { getRuntimeConfig: () => runtimeConfig } }, ); - expect(branched.ok).toBe(true); - expect(branched.payload?.sourceKey).toBe("global"); - expect(branched.payload?.key).toMatch(/^agent:work:dashboard:/); - - const restored = await directSessionReq<{ key?: string; sessionId?: string }>( - "sessions.compaction.restore", - { key: "global", agentId: "work", checkpointId: "checkpoint-work" }, - ); - expect(restored.ok).toBe(true); - expect(restored.payload?.key).toBe("global"); - const mainStore = JSON.parse(await fs.readFile(mainStorePath, "utf-8")) as Record< - string, - { sessionId?: string } - >; - const workStore = JSON.parse(await fs.readFile(workStorePath, "utf-8")) as Record< - string, - { sessionId?: string } - >; - expect(mainStore.global?.sessionId).toBe("sess-main-global"); - expect(workStore.global?.sessionId).toBe(restored.payload?.sessionId); + expect(got.ok).toBe(true); + expect(got.payload?.checkpoint).toMatchObject({ + checkpointId: "checkpoint-work", + summary: "work checkpoint", + }); + expect( + loadSessionEntry({ agentId: "main", sessionKey: "global", storePath: mainStorePath }) + ?.sessionId, + ).toBe("sess-main-global"); + expect( + loadSessionEntry({ agentId: "work", sessionKey: "global", storePath: workStorePath }) + ?.sessionId, + ).toBe("sess-work-global"); testState.sessionStorePath = undefined; testState.sessionConfig = undefined; testState.agentsConfig = undefined; @@ -485,38 +542,110 @@ test("sessions.compaction.* scopes selected global checkpoints to the requested test("sessions.compact without maxLines runs embedded manual compaction for checkpoint-capable flows", async () => { const { dir, storePath } = await createSessionStoreDir(); - await fs.writeFile( - path.join(dir, "sess-main.jsonl"), - `${JSON.stringify({ role: "user", content: "hello" })}\n`, - "utf-8", - ); - await writeSessionStore({ - entries: { - main: sessionStoreEntry("sess-main", { - spawnedCwd: "/tmp/task-repo", - thinkingLevel: "medium", - reasoningLevel: "stream", - contextBudgetStatus: { - schemaVersion: 1, - source: "pre-prompt-estimate", - updatedAt: Date.now() - 5_000, - provider: "anthropic", - model: "claude-opus-4-6", - route: "fits", - shouldCompact: false, - estimatedPromptTokens: 120, - contextTokenBudget: 200, - promptBudgetBeforeReserve: 180, - reserveTokens: 20, - effectiveReserveTokens: 20, - remainingPromptBudgetTokens: 60, - overflowTokens: 0, - toolResultReducibleChars: 0, - messageCount: 2, - unwindowedMessageCount: 2, + const sessionScope = { + agentId: "main", + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + }; + await upsertSessionEntry(sessionScope, { + ...sessionStoreEntry("sess-main", { + spawnedCwd: "/tmp/task-repo", + thinkingLevel: "medium", + reasoningLevel: "stream", + contextBudgetStatus: { + schemaVersion: 1, + source: "pre-prompt-estimate", + updatedAt: Date.now() - 5_000, + provider: "anthropic", + model: "claude-opus-4-6", + route: "fits", + shouldCompact: false, + estimatedPromptTokens: 120, + contextTokenBudget: 200, + promptBudgetBeforeReserve: 180, + reserveTokens: 20, + effectiveReserveTokens: 20, + remainingPromptBudgetTokens: 60, + overflowTokens: 0, + toolResultReducibleChars: 0, + messageCount: 2, + unwindowedMessageCount: 2, + }, + }), + }); + await appendTranscriptEvent(sessionScope, { + type: "session", + version: 3, + id: "sess-main", + timestamp: "2026-06-19T12:00:00.000Z", + cwd: "/tmp", + }); + const seedMessage = await appendTranscriptMessage(sessionScope, { + message: { role: "user", content: "hello", timestamp: 1 }, + now: Date.parse("2026-06-19T12:00:01.000Z"), + }); + embeddedRunMock.compactEmbeddedAgentSession.mockImplementationOnce(async (params) => { + const call = params as { + sessionTarget?: { + agentId?: string; + sessionId?: string; + sessionKey?: string; + storePath?: string; + }; + }; + if ( + !call.sessionTarget?.agentId || + !call.sessionTarget.sessionId || + !call.sessionTarget.sessionKey || + !call.sessionTarget.storePath + ) { + throw new Error("expected SQLite session target"); + } + const targetScope = { + agentId: call.sessionTarget.agentId, + sessionId: call.sessionTarget.sessionId, + sessionKey: call.sessionTarget.sessionKey, + storePath: call.sessionTarget.storePath, + }; + const rows = await loadTranscriptEvents(targetScope); + expect(rows).toHaveLength(2); + await appendTranscriptEvent(targetScope, { + type: "compaction", + id: "compact-1", + parentId: seedMessage.messageId, + timestamp: "2026-06-19T12:00:02.000Z", + summary: "summary", + firstKeptEntryId: seedMessage.messageId, + tokensBefore: 120, + tokensAfter: 80, + }); + await patchAccessorSessionEntry(targetScope, (entry) => ({ + ...entry, + compactionCheckpoints: [ + { + checkpointId: "checkpoint-sqlite", + sessionKey: targetScope.sessionKey, + sessionId: targetScope.sessionId, + createdAt: Date.now(), + reason: "manual", + summary: "summary", + firstKeptEntryId: seedMessage.messageId, + preCompaction: { sessionId: targetScope.sessionId }, + postCompaction: { sessionId: targetScope.sessionId, entryId: "compact-1" }, }, - }), - }, + ], + })); + return { + ok: true, + compacted: true, + result: { + summary: "summary", + firstKeptEntryId: "entry-1", + tokensBefore: 120, + tokensAfter: 80, + }, + }; }); const { ws } = await openClient(); @@ -574,6 +703,12 @@ test("sessions.compact without maxLines runs embedded manual compaction for chec sessionFile?: string; sessionId?: string; sessionKey?: string; + sessionTarget?: { + agentId?: string; + sessionId?: string; + sessionKey?: string; + storePath?: string; + }; thinkLevel?: string; trigger?: string; workspaceDir?: string; @@ -591,7 +726,13 @@ test("sessions.compact without maxLines runs embedded manual compaction for chec if (!compactionCall.sessionFile) { throw new Error("expected embedded compaction session file"); } - expect(path.basename(compactionCall.sessionFile)).toBe("sess-main.jsonl"); + expect(compactionCall.sessionFile).toContain(`sqlite:main:sess-main:${storePath}`); + expect(compactionCall.sessionTarget).toEqual({ + agentId: "main", + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + }); expect(compactionCall.workspaceDir).toBe("/tmp/task-repo"); expect(compactionCall.cwd).toBe("/tmp/task-repo"); expect(callConfig.agents?.defaults?.model?.primary).toBe("anthropic/claude-opus-4-6"); @@ -611,41 +752,141 @@ test("sessions.compact without maxLines runs embedded manual compaction for chec }); expect(compactionCall.trigger).toBe("manual"); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { - compactionCount?: number; - contextBudgetStatus?: unknown; - totalTokens?: number; - totalTokensFresh?: boolean; - } - >; - expect(store["agent:main:main"]?.compactionCount).toBe(1); - expect(store["agent:main:main"]?.contextBudgetStatus).toBeUndefined(); - expect(store["agent:main:main"]?.totalTokens).toBe(80); - expect(store["agent:main:main"]?.totalTokensFresh).toBe(true); + const sqliteRows = await loadTranscriptEvents(sessionScope); + expect(sqliteRows).toHaveLength(3); + expect(sqliteRows.at(-1)).toMatchObject({ + type: "compaction", + summary: "summary", + }); + await expect(fs.readdir(dir)).resolves.not.toContain("sess-main.jsonl"); + const storedEntry = loadAccessorSessionEntry(sessionScope) as + | { + compactionCheckpoints?: unknown[]; + compactionCount?: number; + contextBudgetStatus?: unknown; + totalTokens?: number; + totalTokensFresh?: boolean; + } + | undefined; + expect(storedEntry?.compactionCount).toBe(1); + expect(storedEntry?.compactionCheckpoints).toHaveLength(1); + expect(storedEntry?.contextBudgetStatus).toBeUndefined(); + expect(storedEntry?.totalTokens).toBe(80); + expect(storedEntry?.totalTokensFresh).toBe(true); ws.close(); }); -test("sessions.compact records terminal Codex native compaction", async () => { - const { dir, storePath } = await createSessionStoreDir(); - await fs.writeFile( - path.join(dir, "sess-codex.jsonl"), - `${JSON.stringify({ role: "user", content: "hello codex" })}\n`, - "utf-8", - ); - await writeSessionStore({ - entries: { - main: sessionStoreEntry("sess-codex", { - agentHarnessId: "codex", - agentRuntimeOverride: "openclaw", - modelSelectionLocked: true, - compactionCount: 2, - totalTokens: 54_321, - totalTokensFresh: true, +test("sessions.compact uses the freshest persisted key when main-key aliases exist", async () => { + const { storePath } = await createSessionStoreDir(); + const runtimeConfig = { + agents: { list: [{ id: "main", default: true }] }, + session: { mainKey: "primary", store: storePath }, + }; + await seedSessionEntry({ + entry: sessionStoreEntry("sess-stale-canonical", { updatedAt: Date.now() - 10_000 }), + sessionKey: "agent:main:primary", + storePath, + }); + await seedSessionEntry({ + entry: sessionStoreEntry("sess-alias", { + updatedAt: Date.now(), + totalTokens: 2_000, + totalTokensFresh: true, + }), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId: "sess-alias", + sessionKey: "agent:main:main", + storePath, + totalLines: 3, + }); + embeddedRunMock.compactEmbeddedAgentSession.mockImplementationOnce(async (params) => { + const call = params as { + sessionTarget?: { + agentId?: string; + sessionId?: string; + sessionKey?: string; + storePath?: string; + }; + }; + expect(call.sessionTarget).toMatchObject({ + agentId: "main", + sessionId: "sess-alias", + sessionKey: "agent:main:primary", + storePath, + }); + await patchAccessorSessionEntry( + { + agentId: "main", + sessionKey: "agent:main:primary", + storePath, + }, + (entry) => ({ + ...entry, + compactionCheckpoints: [ + { + checkpointId: "checkpoint-alias", + sessionKey: "agent:main:primary", + sessionId: "sess-alias", + createdAt: Date.now(), + reason: "manual", + summary: "alias checkpoint", + preCompaction: { sessionId: "sess-alias" }, + postCompaction: { sessionId: "sess-alias", entryId: "entry-alias" }, + }, + ], }), - }, + ); + return { + ok: true, + compacted: true, + result: { + summary: "alias summary", + firstKeptEntryId: "entry-alias", + tokensBefore: 2_000, + tokensAfter: 1_000, + }, + }; + }); + + const compacted = await directSessionReq<{ + compacted?: boolean; + key?: string; + ok?: boolean; + }>("sessions.compact", { key: "main" }, { context: { getRuntimeConfig: () => runtimeConfig } }); + + expect(compacted.ok).toBe(true); + expect(compacted.payload?.key).toBe("agent:main:primary"); + expect(compacted.payload?.compacted).toBe(true); + const aliasEntry = loadSessionEntry({ + sessionKey: "agent:main:primary", + storePath, + }); + expect(aliasEntry?.sessionId).toBe("sess-alias"); + expect(aliasEntry?.compactionCount).toBe(1); + expect(aliasEntry?.compactionCheckpoints).toHaveLength(1); +}); + +test("sessions.compact records terminal Codex native compaction", async () => { + const { storePath } = await createSessionStoreDir(); + await seedSessionEntry({ + entry: sessionStoreEntry("sess-codex", { + agentHarnessId: "codex", + compactionCount: 2, + totalTokens: 54_321, + totalTokensFresh: true, + }), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId: "sess-codex", + sessionKey: "agent:main:main", + storePath, + totalLines: 2, }); embeddedRunMock.compactEmbeddedAgentSession.mockResolvedValueOnce({ ok: true, @@ -692,37 +933,31 @@ test("sessions.compact records terminal Codex native compaction", async () => { sessionKey: "agent:main:main", completed: true, }); - expect(embeddedRunMock.compactEmbeddedAgentSession).toHaveBeenCalledWith( - expect.objectContaining({ - agentHarnessId: "codex", - modelSelectionLocked: true, - }), - ); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { - compactionCount?: number; - totalTokens?: number; - totalTokensFresh?: boolean; - } - >; - expect(store["agent:main:main"]?.compactionCount).toBe(3); - expect(store["agent:main:main"]?.totalTokens).toBeUndefined(); - expect(store["agent:main:main"]?.totalTokensFresh).toBeUndefined(); + // Terminal Codex native compaction persists via the accessor: the count + // advances and stale token accounting is cleared for recomputation. + const codexEntry = loadSessionEntry({ sessionKey: "agent:main:main", storePath }); + expect(codexEntry?.compactionCount).toBe(3); + expect(codexEntry?.totalTokens).toBeUndefined(); + expect(codexEntry?.totalTokensFresh).toBeUndefined(); ws.close(); }); test("sessions.compact emits a terminal operation event when persistence fails", async () => { - const { dir } = await createSessionStoreDir(); + const { storePath } = await createSessionStoreDir(); const sessionId = "sess-compact-write-failure"; - await fs.writeFile( - path.join(dir, `${sessionId}.jsonl`), - `${JSON.stringify({ role: "user", content: "hello" })}\n`, - "utf-8", - ); - await writeSessionStore({ entries: { main: sessionStoreEntry(sessionId) } }); + await seedSessionEntry({ + entry: sessionStoreEntry(sessionId), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId, + sessionKey: "agent:main:main", + storePath, + totalLines: 2, + }); const compaction = createDeferred<{ ok: true; compacted: true; @@ -771,13 +1006,18 @@ test("sessions.compact emits a terminal operation event when persistence fails", }); test("sessions.compact rejects stale terminal persistence after the session changes", async () => { - const { dir, storePath } = await createSessionStoreDir(); - await fs.writeFile( - path.join(dir, "sess-compact-old.jsonl"), - `${JSON.stringify({ role: "user", content: "hello" })}\n`, - "utf-8", - ); - await writeSessionStore({ entries: { main: sessionStoreEntry("sess-compact-old") } }); + const { storePath } = await createSessionStoreDir(); + await seedSessionEntry({ + entry: sessionStoreEntry("sess-compact-old"), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId: "sess-compact-old", + sessionKey: "agent:main:main", + storePath, + totalLines: 2, + }); const compaction = createDeferred<{ ok: true; compacted: true; @@ -796,7 +1036,11 @@ test("sessions.compact rejects stale terminal persistence after the session chan await vi.waitFor(() => { expect(embeddedRunMock.compactEmbeddedAgentSession).toHaveBeenCalledTimes(1); }); - await writeSessionStore({ entries: { main: sessionStoreEntry("sess-replacement") } }); + await seedSessionEntry({ + entry: sessionStoreEntry("sess-replacement"), + sessionKey: "agent:main:main", + storePath, + }); compaction.resolve({ ok: true, compacted: true, @@ -814,23 +1058,25 @@ test("sessions.compact rejects stale terminal persistence after the session chan expect(response.error).toMatchObject({ details: { reason: "session-changed" }, }); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { sessionId?: string; compactionCount?: number } - >; - expect(store["agent:main:main"]?.sessionId).toBe("sess-replacement"); - expect(store["agent:main:main"]?.compactionCount).toBeUndefined(); + const replacedEntry = loadSessionEntry({ sessionKey: "agent:main:main", storePath }); + expect(replacedEntry?.sessionId).toBe("sess-replacement"); + expect(replacedEntry?.compactionCount).toBeUndefined(); ws.close(); }); test("sessions.reset waits for terminal compaction before replacing the session", async () => { - const { dir, storePath } = await createSessionStoreDir(); - await fs.writeFile( - path.join(dir, "sess-compact-reset.jsonl"), - `${JSON.stringify({ role: "user", content: "hello" })}\n`, - "utf-8", - ); - await writeSessionStore({ entries: { main: sessionStoreEntry("sess-compact-reset") } }); + const { storePath } = await createSessionStoreDir(); + await seedSessionEntry({ + entry: sessionStoreEntry("sess-compact-reset"), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId: "sess-compact-reset", + sessionKey: "agent:main:main", + storePath, + totalLines: 2, + }); const compaction = createDeferred<{ ok: true; compacted: true; @@ -875,16 +1121,13 @@ test("sessions.reset waits for terminal compaction before replacing the session" const resetSessionId = reset.payload?.entry.sessionId; expect(resetSessionId).toBeTruthy(); expect(resetSessionId).not.toBe("sess-compact-reset"); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { sessionId?: string } - >; - expect(store["agent:main:main"]?.sessionId).toBe(resetSessionId); + const resetEntry = loadSessionEntry({ sessionKey: "agent:main:main", storePath }); + expect(resetEntry?.sessionId).toBe(resetSessionId); ws.close(); }); test("sessions.compaction.restore waits for terminal compaction before replacing the session", async () => { - const { dir } = await createSessionStoreDir(); + const { dir, storePath } = await createSessionStoreDir(); const fixture = await createCheckpointFixture(dir, { legacyPreCompactionSnapshot: false }); const checkpointEntry = compactionCheckpointEntry(fixture, { checkpointId: "checkpoint-race", @@ -895,13 +1138,19 @@ test("sessions.compaction.restore waits for terminal compaction before replacing tokensBefore: 123, tokensAfter: 45, }); - await writeSessionStore({ - entries: { - main: sessionStoreEntry(fixture.sessionId, { - sessionFile: fixture.sessionFile, - compactionCheckpoints: [checkpointEntry], - }), - }, + await seedSessionEntry({ + entry: sessionStoreEntry(fixture.sessionId, { + sessionFile: fixture.sessionFile, + compactionCheckpoints: [checkpointEntry], + }), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId: fixture.sessionId, + sessionKey: "agent:main:main", + storePath, + totalLines: 2, }); const compaction = createDeferred<{ ok: true; @@ -960,13 +1209,13 @@ test("sessions.compaction.restore leaves replacement-session work untouched when reason: "manual", summary: "checkpoint summary", }); - await writeSessionStore({ - entries: { - main: sessionStoreEntry(fixture.sessionId, { - sessionFile: fixture.sessionFile, - compactionCheckpoints: [checkpointEntry], - }), - }, + await seedSessionEntry({ + entry: sessionStoreEntry(fixture.sessionId, { + sessionFile: fixture.sessionFile, + compactionCheckpoints: [checkpointEntry], + }), + sessionKey: "agent:main:main", + storePath, }); const replacementSessionId = "sess-replacement-after-restore-queued"; let replacementInterrupted = false; @@ -998,18 +1247,20 @@ test("sessions.compaction.restore leaves replacement-session work untouched when await new Promise<void>((resolve) => { setImmediate(resolve); }); - await writeSessionStore({ entries: { main: sessionStoreEntry(replacementSessionId) } }); + await replaceSessionEntry( + { sessionKey: "agent:main:main", storePath }, + sessionStoreEntry(replacementSessionId), + ); try { releaseBlocker.resolve(); await blocker; - await expect(restore).resolves.toMatchObject({ - ok: false, - error: { - code: "INVALID_REQUEST", - details: { reason: "session-changed" }, - }, - }); + const response = await restore; + expect(response.ok).toBe(false); + expect(response.error?.code).toBe("INVALID_REQUEST"); + expect(response.error?.message).toMatch( + /checkpoint not found|changed before checkpoint restore/, + ); expect(replacementInterrupted).toBe(false); } finally { releaseBlocker.resolve(); @@ -1019,97 +1270,20 @@ test("sessions.compaction.restore leaves replacement-session work untouched when } }); -test("sessions.compaction.restore rechecks the model-selection lock inside the lifecycle fence", async () => { - const { dir, storePath } = await createSessionStoreDir(); - const fixture = await createCheckpointFixture(dir, { legacyPreCompactionSnapshot: false }); - const checkpointEntry = compactionCheckpointEntry(fixture, { - checkpointId: "checkpoint-lock-race", - sessionKey: "agent:main:main", - createdAt: Date.now(), - reason: "manual", - summary: "checkpoint summary", - }); - const unlockedEntry = sessionStoreEntry(fixture.sessionId, { - sessionFile: fixture.sessionFile, - compactionCheckpoints: [checkpointEntry], - }); - await writeSessionStore({ entries: { main: unlockedEntry } }); - const blockerStarted = createDeferred<void>(); - const releaseBlocker = createDeferred<void>(); - const blocker = runExclusiveSessionLifecycleMutation({ - scope: storePath, - identities: ["main", "agent:main:main", fixture.sessionId], - run: async () => { - blockerStarted.resolve(); - await releaseBlocker.promise; - }, - }); - await blockerStarted.promise; - - const { ws } = await openClient(); - const restoreReachedLifecycleFence = createDeferred<void>(); - sessionLifecycleMutationObserver.onEnter = (params) => { - // The handler has completed its initial store read, while the blocker still - // keeps lifecycle prepare from observing the replacement row. - if (params.scope === storePath && params.prepare) { - restoreReachedLifecycleFence.resolve(); - } - }; - - try { - let restoreSettled = false; - const restore = rpcReq(ws, "sessions.compaction.restore", { - key: "main", - checkpointId: "checkpoint-lock-race", - }).finally(() => { - restoreSettled = true; - }); - await restoreReachedLifecycleFence.promise; - expect(restoreSettled).toBe(false); - await writeSessionStore({ - entries: { - main: { - ...unlockedEntry, - modelSelectionLocked: true, - updatedAt: Date.now(), - }, - }, - }); - - releaseBlocker.resolve(); - await blocker; - await expect(restore).resolves.toMatchObject({ - ok: false, - error: { - code: "INVALID_REQUEST", - message: "Checkpoint branch and restore are unavailable while model selection is locked.", - }, - }); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { modelSelectionLocked?: boolean; sessionId?: string } - >; - expect(store["agent:main:main"]).toMatchObject({ - modelSelectionLocked: true, - sessionId: fixture.sessionId, - }); - } finally { - sessionLifecycleMutationObserver.onEnter = undefined; - releaseBlocker.resolve(); - await blocker; - ws.close(); - } -}); - test("sessions.compact blocks new work admission through terminal persistence", async () => { - const { dir, storePath } = await createSessionStoreDir(); + const { storePath } = await createSessionStoreDir(); const sessionId = "sess-compact-admission"; - await fs.writeFile( - path.join(dir, `${sessionId}.jsonl`), - `${JSON.stringify({ role: "user", content: "hello" })}\n`, - "utf-8", - ); - await writeSessionStore({ entries: { main: sessionStoreEntry(sessionId) } }); + await seedSessionEntry({ + entry: sessionStoreEntry(sessionId), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId, + sessionKey: "agent:main:main", + storePath, + totalLines: 2, + }); const compaction = createDeferred<{ ok: true; compacted: true; @@ -1163,14 +1337,19 @@ test("sessions.compact blocks new work admission through terminal persistence", }); test("sessions.compact clears queued work before draining an active admission", async () => { - const { dir, storePath } = await createSessionStoreDir(); + const { storePath } = await createSessionStoreDir(); const sessionId = "sess-compact-queued-work"; - await fs.writeFile( - path.join(dir, `${sessionId}.jsonl`), - `${JSON.stringify({ role: "user", content: "hello" })}\n`, - "utf-8", - ); - await writeSessionStore({ entries: { main: sessionStoreEntry(sessionId) } }); + await seedSessionEntry({ + entry: sessionStoreEntry(sessionId), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId, + sessionKey: "agent:main:main", + storePath, + totalLines: 2, + }); embeddedRunMock.compactEmbeddedAgentSession.mockResolvedValueOnce({ ok: true, compacted: true, @@ -1204,15 +1383,18 @@ test("sessions.compact clears queued work before draining an active admission", }); test("sessions.patch rejects archive while terminal compaction owns the session", async () => { - const { dir } = await createSessionStoreDir(); + const { storePath } = await createSessionStoreDir(); const sessionKey = "agent:main:dashboard:compact-race"; - await fs.writeFile( - path.join(dir, "sess-compact-archive.jsonl"), - `${JSON.stringify({ role: "user", content: "hello" })}\n`, - "utf-8", - ); - await writeSessionStore({ - entries: { [sessionKey]: sessionStoreEntry("sess-compact-archive") }, + await seedSessionEntry({ + entry: sessionStoreEntry("sess-compact-archive"), + sessionKey, + storePath, + }); + await seedTranscriptRows({ + sessionId: "sess-compact-archive", + sessionKey, + storePath, + totalLines: 2, }); const compaction = createDeferred<{ ok: true; @@ -1249,12 +1431,19 @@ test("sessions.patch rejects archive while terminal compaction owns the session" ws.close(); }); -test("sessions.compact maxLines truncates the transcript on disk and archives the original to .bak", async () => { - const { dir } = await createSessionStoreDir(); - const transcriptPath = path.join(dir, "sess-main.jsonl"); - const originalLines = buildSessionTranscriptLines("sess-main", 500); - await fs.writeFile(transcriptPath, `${originalLines.join("\n")}\n`, "utf-8"); - await writeSessionStore({ entries: { main: sessionStoreEntry("sess-main") } }); +test("sessions.compact maxLines trims SQLite transcript rows and returns an archive marker", async () => { + const { dir, storePath } = await createSessionStoreDir(); + await seedSessionEntry({ + entry: sessionStoreEntry("sess-main"), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + totalLines: 500, + }); const { ws } = await openClient(); const compacted = await rpcReq<{ @@ -1269,24 +1458,22 @@ test("sessions.compact maxLines truncates the transcript on disk and archives th expect(compacted.payload?.compacted).toBe(true); expect(compacted.payload?.kept).toBe(50); - // Active transcript stays reopenable: header + 49 newest entries. - const truncated = (await fs.readFile(transcriptPath, "utf-8")).trim().split("\n"); - expect(truncated).toHaveLength(50); - expect(JSON.parse(truncated[0] ?? "{}")).toMatchObject({ type: "session", id: "sess-main" }); - expect(JSON.parse(truncated[1] ?? "{}")).toMatchObject({ id: "entry-450", parentId: null }); - expect(JSON.parse(truncated.at(-1) ?? "{}")).toMatchObject({ - id: "entry-498", + const retained = await loadTranscriptRows({ + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + }); + expect(retained).toHaveLength(50); + expect(retained[0]).toMatchObject({ type: "session", id: "sess-main" }); + expect(retained[1]).toMatchObject({ + parentId: null, + message: { content: "line-450" }, + }); + expect(retained.at(-1)).toMatchObject({ message: { content: "line-498" }, }); - - // Original 500 lines preserved verbatim in the .bak archive. - const archivedPath = compacted.payload?.archived; - if (!archivedPath) { - throw new Error("expected archived transcript path"); - } - const archived = (await fs.readFile(archivedPath, "utf-8")).trim().split("\n"); - expect(archived).toHaveLength(500); - expect(JSON.parse(archived[0] ?? "{}")).toMatchObject({ type: "session", id: "sess-main" }); + expect(compacted.payload?.archived).toContain(`sqlite:main:sess-main:${storePath}.bak.`); + await expect(fs.readdir(dir)).resolves.not.toContain("sess-main.jsonl"); // No active run present, so the interrupt guard short-circuits without aborting. expect(embeddedRunMock.abortCalls).toEqual([]); @@ -1295,12 +1482,19 @@ test("sessions.compact maxLines truncates the transcript on disk and archives th ws.close(); }); -test("sessions.compact maxLines interrupts an active run before truncating, matching the LLM compact path", async () => { - const { dir } = await createSessionStoreDir(); - const transcriptPath = path.join(dir, "sess-main.jsonl"); - const originalLines = buildSessionTranscriptLines("sess-main", 500); - await fs.writeFile(transcriptPath, `${originalLines.join("\n")}\n`, "utf-8"); - await writeSessionStore({ entries: { main: sessionStoreEntry("sess-main") } }); +test("sessions.compact maxLines interrupts an active run before trimming rows", async () => { + const { storePath } = await createSessionStoreDir(); + await seedSessionEntry({ + entry: sessionStoreEntry("sess-main"), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + totalLines: 500, + }); const { ws } = await openClient(); // Simulate an embedded agent run actively appending to this session transcript. @@ -1313,29 +1507,40 @@ test("sessions.compact maxLines interrupts an active run before truncating, matc kept?: number; }>(ws, "sessions.compact", { key: "main", maxLines: 50 }); - // Regression for the ClawSweeper finding: the maxLines truncate branch must - // run the same active-run interrupt guard as the LLM-summarize branch *before* - // archiving and overwriting the transcript, so an active runner cannot keep - // appending to the file being truncated (the data-loss mode tracked by #72765). + // Regression for the ClawSweeper finding: the maxLines trim branch must run + // the same active-run interrupt guard as the LLM-summarize branch before it + // replaces transcript rows, so an active runner cannot keep appending stale rows. expect(embeddedRunMock.abortCalls).toEqual(["sess-main"]); expect(embeddedRunMock.waitCalls).toEqual(["sess-main"]); - // The guard ran first; truncation still completed deterministically afterwards. + // The guard ran first; row trimming still completed deterministically afterwards. expect(compacted.ok).toBe(true); expect(compacted.payload?.compacted).toBe(true); expect(compacted.payload?.kept).toBe(50); - const truncated = (await fs.readFile(transcriptPath, "utf-8")).trim().split("\n"); - expect(truncated).toHaveLength(50); + await expect( + loadTranscriptRows({ + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + }), + ).resolves.toHaveLength(50); ws.close(); }); -test("sessions.compact maxLines does not interrupt an active run when truncation is a no-op", async () => { - const { dir } = await createSessionStoreDir(); - const transcriptPath = path.join(dir, "sess-main.jsonl"); - const originalLines = buildSessionTranscriptLines("sess-main", 10); - await fs.writeFile(transcriptPath, `${originalLines.join("\n")}\n`, "utf-8"); - await writeSessionStore({ entries: { main: sessionStoreEntry("sess-main") } }); +test("sessions.compact maxLines does not interrupt an active run when row trimming is a no-op", async () => { + const { storePath } = await createSessionStoreDir(); + await seedSessionEntry({ + entry: sessionStoreEntry("sess-main"), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + totalLines: 10, + }); const { ws } = await openClient(); embeddedRunMock.activeIds.add("sess-main"); @@ -1357,8 +1562,12 @@ test("sessions.compact maxLines does not interrupt an active run when truncation }); test("sessions.compact maxLines does not interrupt an active run when no transcript exists", async () => { - await createSessionStoreDir(); - await writeSessionStore({ entries: { main: sessionStoreEntry("sess-main") } }); + const { storePath } = await createSessionStoreDir(); + await seedSessionEntry({ + entry: sessionStoreEntry("sess-main"), + sessionKey: "agent:main:main", + storePath, + }); const { ws } = await openClient(); embeddedRunMock.activeIds.add("sess-main"); @@ -1379,14 +1588,19 @@ test("sessions.compact maxLines does not interrupt an active run when no transcr ws.close(); }); -test("sessions.compact maxLines aborts without truncating when an active run cannot be interrupted", async () => { +test("sessions.compact maxLines aborts without trimming rows when an active run cannot be interrupted", async () => { const { dir, storePath } = await createSessionStoreDir(); - const transcriptPath = path.join(dir, "sess-main.jsonl"); - const originalLines = Array.from({ length: 500 }, (_, index) => - JSON.stringify({ role: "user", content: `line-${index}` }), - ); - await fs.writeFile(transcriptPath, `${originalLines.join("\n")}\n`, "utf-8"); - await writeSessionStore({ entries: { main: sessionStoreEntry("sess-main") } }); + await seedSessionEntry({ + entry: sessionStoreEntry("sess-main"), + sessionKey: "agent:main:main", + storePath, + }); + await seedTranscriptRows({ + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + totalLines: 500, + }); const { ws } = await openClient(); // Active embedded run that fails to end within the interrupt window. @@ -1398,19 +1612,22 @@ test("sessions.compact maxLines aborts without truncating when an active run can maxLines: 50, }); - // Order proof: the guard ran first and failed, so the RPC errors out *before* - // any archive/truncate. If the guard ran after truncation, the transcript - // would already be 50 lines here. It is still 500 with no .bak, proving the - // interrupt happens before the destructive tail-read/archive/write. + // Order proof: the guard ran first and failed, so the RPC errors out before + // replacing rows. If the guard ran after trimming, the transcript would + // already be 50 rows here. It is still 500 with no archive marker file. expect(compacted.ok).toBe(false); expect(embeddedRunMock.abortCalls).toEqual(["sess-main"]); expect(embeddedRunMock.waitCalls).toEqual(["sess-main"]); - const untouched = (await fs.readFile(transcriptPath, "utf-8")).trim().split("\n"); - expect(untouched).toHaveLength(500); + await expect( + loadTranscriptRows({ + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + }), + ).resolves.toHaveLength(500); const dirEntries = await fs.readdir(dir); expect(dirEntries.some((name) => name.includes(".bak"))).toBe(false); - expect(storePath).toBeTruthy(); ws.close(); }); @@ -1418,74 +1635,59 @@ test("sessions.compact maxLines aborts without truncating when an active run can test("sessions.patch preserves nested model ids under provider overrides", async () => { await withTempDir({ prefix: "openclaw-gw-sessions-nested-" }, async (dir) => { const storePath = path.join(dir, "sessions.json"); - await fs.writeFile( - storePath, - JSON.stringify({ - "agent:main:main": sessionStoreEntry("sess-main"), - }), - "utf-8", - ); - - await withEnvAsync({ OPENCLAW_CONFIG_PATH: undefined }, async () => { - const { clearConfigCache, clearRuntimeConfigSnapshot } = await getGatewayConfigModule(); - clearConfigCache(); - clearRuntimeConfigSnapshot(); - const cfg = { - session: { store: storePath, mainKey: "main" }, - agents: { - defaults: { - model: { primary: "openai/gpt-test-a" }, - }, - list: [{ id: "main", default: true, workspace: dir }], + const runtimeConfig = { + agents: { + defaults: { + model: { primary: "openai/gpt-test-a" }, }, - }; - const configPath = path.join(dir, "openclaw.json"); - await fs.writeFile(configPath, JSON.stringify(cfg, null, 2), "utf-8"); - - await withEnvAsync({ OPENCLAW_CONFIG_PATH: configPath }, async () => { - const started = await startConnectedServerWithClient(); - const { server, ws } = started; - try { - agentDiscoveryMock.enabled = true; - agentDiscoveryMock.models = [ - { id: "moonshotai/kimi-k2.5", name: "Kimi K2.5 (NVIDIA)", provider: "nvidia" }, - ]; - - const patched = await rpcReq<{ - ok: true; - entry: { - modelOverride?: string; - providerOverride?: string; - model?: string; - modelProvider?: string; - }; - resolved?: { model?: string; modelProvider?: string }; - }>(ws, "sessions.patch", { - key: "agent:main:main", - model: "nvidia/moonshotai/kimi-k2.5", - }); - expect(patched.ok).toBe(true); - expect(patched.payload?.entry.modelOverride).toBe("moonshotai/kimi-k2.5"); - expect(patched.payload?.entry.providerOverride).toBe("nvidia"); - expect(patched.payload?.entry.model).toBeUndefined(); - expect(patched.payload?.entry.modelProvider).toBeUndefined(); - expect(patched.payload?.resolved?.modelProvider).toBe("nvidia"); - expect(patched.payload?.resolved?.model).toBe("moonshotai/kimi-k2.5"); - - const listed = await rpcReq<{ - sessions: Array<{ key: string; modelProvider?: string; model?: string }>; - }>(ws, "sessions.list", {}); - expect(listed.ok).toBe(true); - const mainSession = listed.payload?.sessions.find( - (session) => session.key === "agent:main:main", - ); - expect(mainSession?.modelProvider).toBe("nvidia"); - expect(mainSession?.model).toBe("moonshotai/kimi-k2.5"); - } finally { - ws.close(); - await server.close(); - } - }); + list: [{ id: "main", default: true, workspace: dir }], + }, + session: { mainKey: "main", store: storePath }, + }; + await seedSessionEntry({ + entry: sessionStoreEntry("sess-main"), + sessionKey: "agent:main:main", + storePath, }); + + agentDiscoveryMock.enabled = true; + agentDiscoveryMock.models = [ + { id: "moonshotai/kimi-k2.5", name: "Kimi K2.5 (NVIDIA)", provider: "nvidia" }, + ]; + + const context = { getRuntimeConfig: () => runtimeConfig }; + const patched = await directSessionReq<{ + entry: { + modelOverride?: string; + providerOverride?: string; + model?: string; + modelProvider?: string; + }; + resolved?: { model?: string; modelProvider?: string }; + }>( + "sessions.patch", + { + key: "agent:main:main", + model: "nvidia/moonshotai/kimi-k2.5", + }, + { context }, + ); + expect(patched.ok).toBe(true); + expect(patched.payload?.entry.modelOverride).toBe("moonshotai/kimi-k2.5"); + expect(patched.payload?.entry.providerOverride).toBe("nvidia"); + expect(patched.payload?.entry.model).toBeUndefined(); + expect(patched.payload?.entry.modelProvider).toBeUndefined(); + expect(patched.payload?.resolved?.modelProvider).toBe("nvidia"); + expect(patched.payload?.resolved?.model).toBe("moonshotai/kimi-k2.5"); + + const listed = await directSessionReq<{ + sessions: Array<{ key: string; modelProvider?: string; model?: string }>; + }>("sessions.list", {}, { context }); + expect(listed.ok).toBe(true); + const mainSession = listed.payload?.sessions.find( + (session) => session.key === "agent:main:main", + ); + expect(mainSession?.modelProvider).toBe("nvidia"); + expect(mainSession?.model).toBe("moonshotai/kimi-k2.5"); }); }); diff --git a/src/gateway/server.sessions.create.test.ts b/src/gateway/server.sessions.create.test.ts index be877d2d1eaa..d878011cd658 100644 --- a/src/gateway/server.sessions.create.test.ts +++ b/src/gateway/server.sessions.create.test.ts @@ -11,7 +11,7 @@ import { listRegistryWorktrees, } from "../agents/worktrees/registry.js"; import { managedWorktrees } from "../agents/worktrees/service.js"; -import { loadSessionStore } from "../config/sessions/store.js"; +import { loadSessionEntry, loadTranscriptEvents } from "../config/sessions/session-accessor.js"; import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { agentCommand, @@ -28,6 +28,7 @@ import { directSessionReq, sessionHookMocks, sessionLifecycleHookMocks, + seedSessionTranscript, } from "./test/server-sessions.test-helpers.js"; const { createSessionStoreDir, createSelectedGlobalSessionStore, openClient } = @@ -121,7 +122,7 @@ test("sessions.create provisions and reuses a session worktree for later runs", sessionKey: key, idempotencyKey: "session-worktree-cwd", }); - expect(run.ok).toBe(true); + expect(run.ok, JSON.stringify(run)).toBe(true); await vi.waitFor(() => expect(agentCommand).toHaveBeenCalled()); expect(agentCommand.mock.calls.at(-1)?.[0]).toMatchObject({ cwd: worktree?.path, @@ -449,11 +450,9 @@ test("sessions.create reset-in-place persists the returned worktree cwd", async const worktree = created.payload?.worktree; worktreeId = worktree?.id; expect(created.payload?.entry.spawnedCwd).toBe(worktree?.path); - const store = JSON.parse(await fs.readFile(storePath, "utf8")) as Record< - string, - { spawnedCwd?: string } - >; - expect(store["agent:main:main"]?.spawnedCwd).toBe(worktree?.path); + expect(loadSessionEntry({ sessionKey: "agent:main:main", storePath })?.spawnedCwd).toBe( + worktree?.path, + ); // A later plain New Chat on the same main session must leave the worktree: cwd clears // and the (clean) session worktree is lossless-removed rather than left orphaned. @@ -522,7 +521,7 @@ test("sessions.create rejects worktrees for non-git agent workspaces", async () }); test("sessions.create stores dashboard session model and parent linkage, and creates a transcript", async () => { - const { dir, storePath } = await createSessionStoreDir(); + const { storePath } = await createSessionStoreDir(); agentDiscoveryMock.enabled = true; agentDiscoveryMock.models = [{ id: "gpt-test-a", name: "A", provider: "openai" }]; await writeSessionStore({ @@ -561,32 +560,25 @@ test("sessions.create stores dashboard session model and parent linkage, and cre /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, ); - const rawStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { - sessionId?: string; - label?: string; - providerOverride?: string; - modelOverride?: string; - parentSessionKey?: string; - sessionFile?: string; - } - >; const key = created.payload?.key as string; - expect(rawStore[key]?.sessionId).toBe(created.payload?.sessionId); - expect(rawStore[key]?.label).toBe("Dashboard Chat"); - expect(rawStore[key]?.providerOverride).toBe("openai"); - expect(rawStore[key]?.modelOverride).toBe("gpt-test-a"); - expect(rawStore[key]?.parentSessionKey).toBe("agent:main:main"); - expect(sessionFile).toBe(rawStore[key]?.sessionFile); + const storedEntry = loadSessionEntry({ agentId: "ops", sessionKey: key, storePath }); + expect(storedEntry?.sessionId).toBe(created.payload?.sessionId); + expect(storedEntry?.label).toBe("Dashboard Chat"); + expect(storedEntry?.providerOverride).toBe("openai"); + expect(storedEntry?.modelOverride).toBe("gpt-test-a"); + expect(storedEntry?.parentSessionKey).toBe("agent:main:main"); + expect(sessionFile).toBe(storedEntry?.sessionFile); - const transcriptPath = path.join(dir, `${created.payload?.sessionId}.jsonl`); - await expect(fs.realpath(sessionFile)).resolves.toBe(await fs.realpath(transcriptPath)); - const transcript = await fs.readFile(transcriptPath, "utf-8"); - const [headerLine] = transcript.trim().split(/\r?\n/, 1); - const header = JSON.parse(headerLine) as { type?: string; id?: string }; - expect(header.type).toBe("session"); - expect(header.id).toBe(created.payload?.sessionId); + await expect( + loadTranscriptEvents({ + agentId: "ops", + sessionId: requireNonEmptyString(created.payload?.sessionId, "created session id"), + sessionKey: key, + storePath, + }), + ).resolves.toEqual([ + expect.objectContaining({ id: created.payload?.sessionId, type: "session" }), + ]); }); test("sessions.create inherits explicit selection without runtime model identity", async () => { @@ -683,22 +675,13 @@ test("sessions.create inherits explicit selection without runtime model identity expect(created.payload?.entry?.authProfileOverride).toBe("codex-oauth"); expect(created.payload?.entry?.authProfileOverrideSource).toBe("user"); - const rawStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { - providerOverride?: string; - modelOverride?: string; - modelProvider?: string; - model?: string; - parentSessionKey?: string; - } - >; const key = created.payload?.key as string; - expect(rawStore[key]?.providerOverride).toBe("codex"); - expect(rawStore[key]?.modelOverride).toBe("gpt-5.5"); - expect(rawStore[key]?.modelProvider).toBeUndefined(); - expect(rawStore[key]?.model).toBeUndefined(); - expect(rawStore[key]?.parentSessionKey).toBe("agent:main:main"); + const storedEntry = loadSessionEntry({ agentId: "main", sessionKey: key, storePath }); + expect(storedEntry?.providerOverride).toBe("codex"); + expect(storedEntry?.modelOverride).toBe("gpt-5.5"); + expect(storedEntry?.modelProvider).toBeUndefined(); + expect(storedEntry?.model).toBeUndefined(); + expect(storedEntry?.parentSessionKey).toBe("agent:main:main"); }); test("sessions.create resolves the current default instead of inherited runtime identity", async () => { @@ -730,13 +713,10 @@ test("sessions.create resolves the current default instead of inherited runtime model: "current-model", }); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { modelProvider?: string; model?: string } - >; const key = created.payload?.key as string; - expect(store[key]?.modelProvider).toBeUndefined(); - expect(store[key]?.model).toBeUndefined(); + const storedEntry = loadSessionEntry({ agentId: "main", sessionKey: key, storePath }); + expect(storedEntry?.modelProvider).toBeUndefined(); + expect(storedEntry?.model).toBeUndefined(); }); test("sessions.create accepts an explicit key for persistent dashboard sessions", async () => { @@ -780,14 +760,16 @@ test("sessions.create scopes the main alias to the requested agent", async () => expect(created.payload?.key).toBe("agent:longmemeval:main"); requireNonEmptyString(created.payload?.entry?.sessionFile, "longmemeval session file"); - const rawStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { - sessionId?: string; - } - >; - expect(rawStore["agent:longmemeval:main"]?.sessionId).toBe(created.payload?.sessionId); - expect(rawStore["agent:main:main"]).toBeUndefined(); + expect( + loadSessionEntry({ + agentId: "longmemeval", + sessionKey: "agent:longmemeval:main", + storePath, + })?.sessionId, + ).toBe(created.payload?.sessionId); + expect( + loadSessionEntry({ agentId: "main", sessionKey: "agent:main:main", storePath }), + ).toBeUndefined(); }); test("sessions.create replaces a dead main entry with a fresh session id", async () => { @@ -825,49 +807,13 @@ test("sessions.create replaces a dead main entry with a fresh session id", async expect(created.payload?.entry?.label).toBeUndefined(); expect(created.payload?.entry?.sessionFile).not.toBe("stale.jsonl"); - const rawStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { - sessionId?: string; - sessionFile?: string; - } - >; - expect(rawStore["agent:ops:main"]?.sessionId).toBe(created.payload?.sessionId); - expect(rawStore["agent:ops:main"]?.sessionFile).not.toBe("stale.jsonl"); - } finally { - testState.agentsConfig = undefined; - } -}); - -test("sessions.create rolls back the entry when transcript initialization fails", async () => { - const { dir, storePath } = await createSessionStoreDir(); - testState.agentsConfig = { list: [{ id: "ops", default: true }] }; - const blockerPath = path.join(dir, "blocked"); - await fs.writeFile(blockerPath, "not a directory", "utf-8"); - try { - await writeSessionStore({ + const storedEntry = loadSessionEntry({ agentId: "ops", - entries: { - main: { - sessionFile: "blocked/session-1.jsonl", - sessionId: "session-1", - updatedAt: 1, - }, - }, + sessionKey: "agent:ops:main", + storePath, }); - - const created = await directSessionReq("sessions.create", { - key: "main", - agentId: "ops", - }); - - expect(created.ok).toBe(false); - expect((created.error as { code?: string } | undefined)?.code).toBe("UNAVAILABLE"); - expect((created.error as { message?: string } | undefined)?.message ?? "").toContain( - "failed to create session transcript:", - ); - const rawStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record<string, unknown>; - expect(rawStore["agent:ops:main"]).toBeUndefined(); + expect(storedEntry?.sessionId).toBe(created.payload?.sessionId); + expect(storedEntry?.sessionFile).not.toBe("stale.jsonl"); } finally { testState.agentsConfig = undefined; } @@ -906,16 +852,26 @@ test("sessions.create preserves global and unknown sentinel keys", async () => { expect(unknownCreated.payload?.key).toBe("unknown"); requireNonEmptyString(unknownCreated.payload?.entry?.sessionFile, "unknown session file"); - const rawStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { - sessionId?: string; - } - >; - expect(rawStore.global?.sessionId).toBe(globalCreated.payload?.sessionId); - expect(rawStore.unknown?.sessionId).toBe(unknownCreated.payload?.sessionId); - expect(rawStore["agent:longmemeval:global"]).toBeUndefined(); - expect(rawStore["agent:longmemeval:unknown"]).toBeUndefined(); + expect( + loadSessionEntry({ agentId: "longmemeval", sessionKey: "global", storePath })?.sessionId, + ).toBe(globalCreated.payload?.sessionId); + expect( + loadSessionEntry({ agentId: "longmemeval", sessionKey: "unknown", storePath })?.sessionId, + ).toBe(unknownCreated.payload?.sessionId); + expect( + loadSessionEntry({ + agentId: "longmemeval", + sessionKey: "agent:longmemeval:global", + storePath, + }), + ).toBeUndefined(); + expect( + loadSessionEntry({ + agentId: "longmemeval", + sessionKey: "agent:longmemeval:unknown", + storePath, + }), + ).toBeUndefined(); }); test("sessions.create stores selected global sessions in the requested agent store", async () => { @@ -943,12 +899,13 @@ test("sessions.create stores selected global sessions in the requested agent sto expect(created.ok).toBe(true); expect(created.payload?.key).toBe("global"); requireNonEmptyString(created.payload?.entry?.sessionFile, "work global session file"); - await expect(fs.readFile(mainStorePath, "utf-8")).rejects.toMatchObject({ code: "ENOENT" }); - const workStore = JSON.parse(await fs.readFile(workStorePath, "utf-8")) as Record< - string, - { sessionId?: string } - >; - expect(workStore.global?.sessionId).toBe(created.payload?.sessionId); + expect( + loadSessionEntry({ agentId: "main", sessionKey: "global", storePath: mainStorePath }), + ).toBeUndefined(); + expect( + loadSessionEntry({ agentId: "work", sessionKey: "global", storePath: workStorePath }) + ?.sessionId, + ).toBe(created.payload?.sessionId); expect(broadcastToConnIds).toHaveBeenCalledWith( "sessions.changed", expect.objectContaining({ sessionKey: "global", agentId: "work", reason: "create" }), @@ -1036,38 +993,34 @@ test("sessions.create loads selected global parent from the requested agent stor test("sessions.get reads selected global messages from the requested agent store", async () => { const { mainStorePath, workStorePath } = await createSelectedGlobalSessionStore(); - const mainTranscriptPath = path.join(path.dirname(mainStorePath), "sess-main-global.jsonl"); - const workTranscriptPath = path.join(path.dirname(workStorePath), "sess-work-global.jsonl"); - await fs.mkdir(path.dirname(mainTranscriptPath), { recursive: true }); - await fs.mkdir(path.dirname(workTranscriptPath), { recursive: true }); - await fs.writeFile( - mainTranscriptPath, - `${JSON.stringify({ type: "message", id: "main-msg", message: { role: "user", content: "main global" } })}\n`, - "utf-8", - ); - await fs.writeFile( - workTranscriptPath, - `${JSON.stringify({ type: "message", id: "work-msg", message: { role: "user", content: "work global" } })}\n`, - "utf-8", - ); try { await writeSessionStore({ storePath: mainStorePath, entries: { - global: sessionStoreEntry("sess-main-global", { - sessionFile: mainTranscriptPath, - }), + global: sessionStoreEntry("sess-main-global"), }, }); await writeSessionStore({ storePath: workStorePath, agentId: "work", entries: { - global: sessionStoreEntry("sess-work-global", { - sessionFile: workTranscriptPath, - }), + global: sessionStoreEntry("sess-work-global"), }, }); + await seedSessionTranscript({ + agentId: "main", + messages: [{ role: "user", content: "main global" }], + sessionId: "sess-main-global", + sessionKey: "global", + storePath: mainStorePath, + }); + await seedSessionTranscript({ + agentId: "work", + messages: [{ role: "user", content: "work global" }], + sessionId: "sess-work-global", + sessionKey: "global", + storePath: workStorePath, + }); const result = await directSessionReq<{ messages?: unknown[] }>("sessions.get", { key: "global", @@ -1105,16 +1058,28 @@ test("sessions.create sends selected global initial tasks to the requested agent const runId = requireNonEmptyString(created.payload?.runId, "selected global run id"); const wait = await rpcReq(ws, "agent.wait", { runId, timeoutMs: 1_000 }); expect(wait.ok).toBe(true); - const workStore = JSON.parse(await fs.readFile(workStorePath, "utf-8")) as Record< - string, - { sessionFile?: string } - >; - const workTranscript = requireNonEmptyString( - workStore.global?.sessionFile, - "selected global transcript", + const workEntry = loadSessionEntry({ + agentId: "work", + sessionKey: "global", + storePath: workStorePath, + }); + const workSessionId = requireNonEmptyString(workEntry?.sessionId, "selected global session id"); + await expect( + loadTranscriptEvents({ + agentId: "work", + sessionId: workSessionId, + sessionKey: "global", + storePath: workStorePath, + }), + ).resolves.toContainEqual( + expect.objectContaining({ + message: expect.objectContaining({ content: "hello selected global" }), + type: "message", + }), ); - await expect(fs.readFile(workTranscript, "utf-8")).resolves.toContain("hello selected global"); - await expect(fs.readFile(mainStorePath, "utf-8")).rejects.toMatchObject({ code: "ENOENT" }); + expect( + loadSessionEntry({ agentId: "main", sessionKey: "global", storePath: mainStorePath }), + ).toBeUndefined(); testState.sessionStorePath = undefined; testState.sessionConfig = undefined; testState.agentsConfig = undefined; @@ -1148,6 +1113,15 @@ test("sessions.create forks the parent transcript into the new session", async ( }), }, }); + await seedSessionTranscript({ + sessionId: parent.sessionId, + sessionKey: "agent:main:main", + storePath, + messages: [ + { role: "user", content: "before compaction" }, + { role: "assistant", content: [{ type: "text", text: "working on it" }] }, + ], + }); const created = await directSessionReq<{ key?: string; @@ -1175,21 +1149,41 @@ test("sessions.create forks the parent transcript into the new session", async ( created.payload?.entry?.sessionFile, "forked session file", ); - const readMessages = async (sessionFile: string) => - (await fs.readFile(sessionFile, "utf-8")) - .trim() - .split(/\r?\n/) - .map((line) => JSON.parse(line) as { type?: string; message?: unknown }) - .filter((entry) => entry.type === "message") + const readMessages = async (scope: { + sessionFile?: string; + sessionId: string; + sessionKey: string; + storePath: string; + }) => + (await loadTranscriptEvents(scope)) + .filter((entry): entry is { type: "message"; message: unknown } => { + return ( + entry !== null && + typeof entry === "object" && + "type" in entry && + entry.type === "message" && + "message" in entry + ); + }) .map((entry) => entry.message); - expect(await readMessages(forkedSessionFile)).toEqual(await readMessages(parent.sessionFile)); + const forkedSessionId = requireNonEmptyString(created.payload?.sessionId, "forked session id"); + expect( + await readMessages({ + sessionFile: forkedSessionFile, + sessionId: forkedSessionId, + sessionKey: created.payload?.key ?? "", + storePath, + }), + ).toEqual( + await readMessages({ + sessionId: parent.sessionId, + sessionKey: "agent:main:main", + storePath, + }), + ); - const stored = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { sessionId?: string; sessionFile?: string; forkedFromParent?: boolean } - >; const key = requireNonEmptyString(created.payload?.key, "forked session key"); - expect(stored[key]).toMatchObject({ + expect(loadSessionEntry({ sessionKey: key, storePath })).toMatchObject({ sessionId: created.payload?.sessionId, sessionFile: forkedSessionFile, forkedFromParent: true, @@ -1225,9 +1219,13 @@ test("public session mutations reserve agent harness-owned session keys", async expect(ordinary.ok).toBe(true); expect(ordinary.payload?.key).toBe("agent:main:ordinary-session"); - const stored = loadSessionStore(storePath, { skipCache: true }); - expect(stored["agent:main:harness:codex:supervision:native-thread"]).toBeUndefined(); - expect(stored["agent:main:ordinary-session"]).toBeDefined(); + expect( + loadSessionEntry({ + sessionKey: "agent:main:harness:codex:supervision:native-thread", + storePath, + }), + ).toBeUndefined(); + expect(loadSessionEntry({ sessionKey: "agent:main:ordinary-session", storePath })).toBeDefined(); }); test("sessions.create preserves a pre-existing unlocked harness-prefixed session", async () => { @@ -1250,7 +1248,7 @@ test("sessions.create preserves a pre-existing unlocked harness-prefixed session expect(created.ok).toBe(true); expect(created.payload).toMatchObject({ key, sessionId: "legacy-session" }); - expect(loadSessionStore(storePath, { skipCache: true })[key]).toMatchObject({ + expect(loadSessionEntry({ sessionKey: key, storePath })).toMatchObject({ sessionId: "legacy-session", label: "Updated notes", }); @@ -1382,6 +1380,7 @@ test("sessions.create rejects fork while the parent session is active", async () test("sessions.create resolves an agent-qualified fork from the parent store", async () => { const { dir } = await createSessionStoreDir(); const storeTemplate = path.join(dir, "{agentId}", "sessions.json"); + const mainStorePath = storeTemplate.replace("{agentId}", "main"); const workStorePath = storeTemplate.replace("{agentId}", "work"); const workDir = path.dirname(workStorePath); testState.sessionStorePath = storeTemplate; @@ -1397,9 +1396,20 @@ test("sessions.create resolves an agent-qualified fork from the parent store", a main: sessionStoreEntry(parent.sessionId, { sessionFile: parent.sessionFile }), }, }); + await seedSessionTranscript({ + agentId: "work", + sessionId: parent.sessionId, + sessionKey: "agent:work:main", + storePath: workStorePath, + messages: [ + { role: "user", content: "before compaction" }, + { role: "assistant", content: [{ type: "text", text: "working on it" }] }, + ], + }); const created = await directSessionReq<{ key?: string; + sessionId?: string; entry?: { parentSessionKey?: string; sessionFile?: string; @@ -1418,7 +1428,24 @@ test("sessions.create resolves an agent-qualified fork from the parent store", a created.payload?.entry?.sessionFile, "agent-qualified forked session file", ); - await expect(fs.readFile(forkedSessionFile, "utf-8")).resolves.toContain("before compaction"); + await expect( + loadTranscriptEvents({ + sessionFile: forkedSessionFile, + sessionId: requireNonEmptyString( + created.payload?.sessionId, + "agent-qualified forked session id", + ), + sessionKey: created.payload?.key ?? "", + storePath: mainStorePath, + }), + ).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: expect.objectContaining({ content: "before compaction" }), + type: "message", + }), + ]), + ); } finally { testState.sessionStorePath = undefined; testState.sessionConfig = undefined; diff --git a/src/gateway/server.sessions.delete-lifecycle.test.ts b/src/gateway/server.sessions.delete-lifecycle.test.ts index f079413bfd09..124023a1d140 100644 --- a/src/gateway/server.sessions.delete-lifecycle.test.ts +++ b/src/gateway/server.sessions.delete-lifecycle.test.ts @@ -12,7 +12,7 @@ import { } from "../acp/runtime/session-meta.js"; import { getRegistryWorktree } from "../agents/worktrees/registry.js"; import { managedWorktrees } from "../agents/worktrees/service.js"; -import { loadSessionStore } from "../config/sessions/store.js"; +import { loadSessionEntry, replaceSessionEntry } from "../config/sessions/session-accessor.js"; import { beginSessionWorkAdmission, runExclusiveSessionLifecycleMutation, @@ -252,9 +252,7 @@ test("sessions.delete preserves locked archived sessions and deletes ordinary ar [ordinaryKey]: sessionStoreEntry(ordinarySessionId, { archivedAt: Date.now() }), }, }); - const lockedEntryBefore = structuredClone( - loadSessionStore(storePath, { skipCache: true })[lockedKey], - ); + const lockedEntryBefore = structuredClone(loadSessionEntry({ storePath, sessionKey: lockedKey })); const lockedTranscriptPath = path.join(dir, `${lockedSessionId}.jsonl`); const lockedTranscriptBefore = await fs.readFile(lockedTranscriptPath, "utf8"); @@ -267,13 +265,12 @@ test("sessions.delete preserves locked archived sessions and deletes ordinary ar code: "INVALID_REQUEST", message: "This session cannot be deleted while model selection is locked.", }); - expect(loadSessionStore(storePath, { skipCache: true })[lockedKey]).toEqual(lockedEntryBefore); + expect(loadSessionEntry({ storePath, sessionKey: lockedKey })).toEqual(lockedEntryBefore); expect(await fs.readFile(lockedTranscriptPath, "utf8")).toBe(lockedTranscriptBefore); await expectSessionDeleteSucceeds({ key: ordinaryKey, archivedOnly: true }); - const storedAfterOrdinaryDelete = loadSessionStore(storePath, { skipCache: true }); - expect(storedAfterOrdinaryDelete[ordinaryKey]).toBeUndefined(); - expect(storedAfterOrdinaryDelete[lockedKey]).toEqual(lockedEntryBefore); + expect(loadSessionEntry({ storePath, sessionKey: ordinaryKey })).toBeUndefined(); + expect(loadSessionEntry({ storePath, sessionKey: lockedKey })).toEqual(lockedEntryBefore); }); test("sessions.delete interrupts work admitted before runtime registration", async () => { @@ -396,44 +393,19 @@ test("sessions.delete rechecks its expected id before interrupting replacement w } }); -test("sessions.delete accepts a matching lifecycle revision for a non-resumable stub", async () => { - const sessionKey = "agent:main:cron:cleanup"; - const lifecycleRevision = "run-revision"; - const updatedAt = 1_737_600_000_000; - await createSessionStoreDir(); - await writeSessionStore({ - entries: { - [sessionKey]: sessionStoreEntry("discarded-before-persist", { - sessionId: undefined, - lifecycleRevision, - updatedAt, - }), - }, - }); - - const deleted = await expectSessionDeleteSucceeds({ - key: sessionKey, - expectedSessionId: "in-memory-run-id", - expectedLifecycleRevision: lifecycleRevision, - expectedSessionUpdatedAt: updatedAt, - }); - - expect(deleted.payload?.deleted).toBe(true); -}); - -test("sessions.delete rejects an ID-less replacement with the same updated-at timestamp", async () => { +test("sessions.delete rejects a replacement with the same updated-at timestamp", async () => { const sessionKey = "agent:main:cron:cleanup"; const updatedAt = 1_737_600_000_000; const { storePath } = await createSessionStoreDir(); - await writeSessionStore({ - entries: { - [sessionKey]: sessionStoreEntry("discarded-before-persist", { - sessionId: undefined, + await replaceSessionEntry( + { sessionKey, storePath }, + { + ...sessionStoreEntry("replacement-run", { lifecycleRevision: "replacement-revision", updatedAt, }), }, - }); + ); let interrupted = false; const admission = await beginSessionWorkAdmission({ scope: storePath, @@ -454,8 +426,9 @@ test("sessions.delete rejects an ID-less replacement with the same updated-at ti expect(deleted.ok).toBe(false); expect(interrupted).toBe(false); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toMatchObject({ + expect(loadSessionEntry({ sessionKey, storePath })).toMatchObject({ lifecycleRevision: "replacement-revision", + sessionId: "replacement-run", updatedAt, }); } finally { @@ -494,7 +467,7 @@ test("sessions.delete includes cleanup-owned row changes in its guarded deletion }); expect(deleted.payload?.deleted).toBe(true); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toBeUndefined(); + expect(loadSessionEntry({ sessionKey, storePath })).toBeUndefined(); }); test("sessions.delete serializes a patch behind asynchronous runtime cleanup", async () => { @@ -539,7 +512,7 @@ test("sessions.delete serializes a patch behind asynchronous runtime cleanup", a expect(deleted.ok).toBe(true); expect(patched.ok).toBe(false); expect(patched.error?.message).toBe(`Session ${sessionKey} changed before patch. Retry.`); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]).toBeUndefined(); + expect(loadSessionEntry({ sessionKey, storePath })).toBeUndefined(); }); test("sessions.patch waits for an in-flight session lifecycle mutation", async () => { @@ -581,9 +554,7 @@ test("sessions.patch waits for an in-flight session lifecycle mutation", async ( const [patched] = await Promise.all([patch, mutation]); expect(patched.ok).toBe(true); - expect(loadSessionStore(storePath, { skipCache: true })[sessionKey]?.label).toBe( - "after lifecycle mutation", - ); + expect(loadSessionEntry({ sessionKey, storePath })?.label).toBe("after lifecycle mutation"); }); test("sessions.delete keeps lifecycle admission blocked through session unbinding", async () => { @@ -725,14 +696,20 @@ test("sessions.delete scopes selected global deletes to the requested agent", as agentId: "work", deleteTranscript: false, }); - const mainStore = JSON.parse(await fs.readFile(globalStores.mainStorePath, "utf-8")) as { - global?: { sessionId?: string }; - }; - const workStore = JSON.parse(await fs.readFile(globalStores.workStorePath, "utf-8")) as { - global?: { sessionId?: string }; - }; - expect(mainStore.global?.sessionId).toBe("sess-main-global"); - expect(workStore.global).toBeUndefined(); + expect( + loadSessionEntry({ + agentId: "main", + sessionKey: "global", + storePath: globalStores.mainStorePath, + })?.sessionId, + ).toBe("sess-main-global"); + expect( + loadSessionEntry({ + agentId: "work", + sessionKey: "global", + storePath: globalStores.workStorePath, + }), + ).toBeUndefined(); await resetConfiguredGlobalAgentSessionStore(globalStores); }); @@ -846,23 +823,11 @@ test("sessions.delete closes child ACP runtimes spawned from the deleted parent" test("sessions.delete emits session_end with deleted reason and no replacement", async () => { const { dir } = await createSessionStoreDir(); await writeSingleLineSession(dir, "sess-main", "hello"); - const transcriptPath = path.join(dir, "sess-delete.jsonl"); - await fs.writeFile( - transcriptPath, - `${JSON.stringify({ - type: "message", - id: "m-delete", - message: { role: "user", content: "delete me" }, - })}\n`, - "utf-8", - ); await writeSessionStore({ entries: { main: sessionStoreEntry("sess-main"), - "discord:group:delete": sessionStoreEntry("sess-delete", { - sessionFile: transcriptPath, - }), + "discord:group:delete": sessionStoreEntry("sess-delete"), }, }); @@ -880,8 +845,10 @@ test("sessions.delete emits session_end with deleted reason and no replacement", "agent:main:discord:group:delete", ); expect((event as { reason?: string } | undefined)?.reason).toBe("deleted"); - expect((event as { transcriptArchived?: boolean } | undefined)?.transcriptArchived).toBe(true); - expect((event as { sessionFile?: string } | undefined)?.sessionFile).toContain(".jsonl.deleted."); + expect( + (event as { transcriptArchived?: boolean } | undefined)?.transcriptArchived, + ).toBeUndefined(); + expect((event as { sessionFile?: string } | undefined)?.sessionFile).toBeUndefined(); expect((event as { nextSessionId?: string } | undefined)?.nextSessionId).toBeUndefined(); expect((context as { sessionId?: string } | undefined)?.sessionId).toBe("sess-delete"); expect((context as { sessionKey?: string } | undefined)?.sessionKey).toBe( @@ -977,11 +944,11 @@ test("sessions.delete returns unavailable when active run does not stop", async ); expect(browserSessionTabMocks.closeTrackedBrowserTabsForSessions).not.toHaveBeenCalled(); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { sessionId?: string } - >; - expect(store["agent:main:discord:group:dev"]?.sessionId).toBe("sess-active"); + const storedEntry = loadSessionEntry({ + sessionKey: "agent:main:discord:group:dev", + storePath, + }); + expect(storedEntry?.sessionId).toBe("sess-active"); const filesAfterDeleteAttempt = await fs.readdir(dir); expect( filesAfterDeleteAttempt.filter((fileName) => fileName.startsWith("sess-active.jsonl.deleted.")), diff --git a/src/gateway/server.sessions.list-changed.test.ts b/src/gateway/server.sessions.list-changed.test.ts index 49a89a0c182a..dd6a5b532f57 100644 --- a/src/gateway/server.sessions.list-changed.test.ts +++ b/src/gateway/server.sessions.list-changed.test.ts @@ -1,14 +1,12 @@ /** * Gateway sessions.list changed-state tests. */ -import fs from "node:fs/promises"; -import path from "node:path"; import { createPluginRegistryFixture, registerTestPlugin, } from "openclaw/plugin-sdk/plugin-test-contracts"; import { afterEach, expect, test, vi } from "vitest"; -import { loadSessionStore } from "../config/sessions.js"; +import { loadSessionEntry } from "../config/sessions/session-accessor.js"; import { createEmptyPluginRegistry } from "../plugins/registry-empty.js"; import { pinActivePluginSessionExtensionRegistry, @@ -23,7 +21,8 @@ import { getGatewayConfigModule, getSessionsHandlers, createDeferred, - createLinearSessionTranscript, + loadSeededTranscriptEvents, + seedSessionTranscript, sessionStoreEntry, } from "./test/server-sessions.test-helpers.js"; @@ -45,19 +44,6 @@ type MockCalls = { type SessionStoreEntryOptions = Parameters<typeof sessionStoreEntry>[1]; type MutationMethod = "sessions.patch" | "sessions.compact"; -function expectedLastMessageTranscript(sessionId: string, contents: string[]): string { - const records = createLinearSessionTranscript(sessionId, contents) - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record<string, unknown>); - const header = records[0]; - const last = records.at(-1); - if (!header || !last) { - throw new Error("expected a canonical transcript fixture"); - } - return `${JSON.stringify(header)}\n${JSON.stringify({ ...last, parentId: null })}\n`; -} - function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null; } @@ -84,6 +70,14 @@ function expectFields(record: Record<string, unknown>, expected: Record<string, } } +function transcriptMessageContents(events: readonly unknown[]): unknown[] { + return events + .map((event) => + isRecord(event) && isRecord(event.message) ? event.message.content : undefined, + ) + .filter((content) => content !== undefined); +} + function expectRespondPayload(respond: MockCalls): Record<string, unknown> { expect(respond.mock.calls).toHaveLength(1); const [ok, payload, error] = respond.mock.calls[0] ?? []; @@ -265,15 +259,21 @@ test("sessions.pluginPatch over WebSocket keeps pinned startup extensions after value: { state: "after-active-registry-churn" }, }); - const store = loadSessionStore(storePath); - const entry = store.main ?? store["agent:main:main"]; + const entry = loadSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath, + }); expect(entry).toBeDefined(); + if (!entry) { + throw new Error("expected persisted session entry"); + } const row = buildGatewaySessionRow({ cfg: { session: { store: storePath } }, - storePath, - store, - key: "agent:main:main", entry, + key: "agent:main:main", + store: { "agent:main:main": entry }, + storePath, }); expect(row.pluginExtensions).toEqual([ { @@ -324,45 +324,12 @@ async function expectListedSessionActiveRun( } test("sessions.list keeps bulk rows lightweight and uses persisted model fields", async () => { - const { dir } = await createSessionStoreDir(); + const { storePath } = await createSessionStoreDir(); testState.agentConfig = { models: { "anthropic/claude-sonnet-4-6": { params: { context1m: true } }, }, }; - await fs.writeFile( - path.join(dir, "sess-parent.jsonl"), - `${JSON.stringify({ type: "session", version: 1, id: "sess-parent" })}\n`, - "utf-8", - ); - await fs.writeFile( - path.join(dir, "sess-child.jsonl"), - [ - JSON.stringify({ type: "session", version: 1, id: "sess-child" }), - JSON.stringify({ - message: { - role: "assistant", - provider: "anthropic", - model: "claude-sonnet-4-6", - usage: { - input: 2_000, - output: 500, - cacheRead: 1_000, - cost: { total: 0.0042 }, - }, - }, - }), - JSON.stringify({ - message: { - role: "assistant", - provider: "openclaw", - model: "delivery-mirror", - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - }, - }), - ].join("\n"), - "utf-8", - ); await writeSessionStore({ entries: { main: sessionStoreEntry("sess-parent"), @@ -381,6 +348,30 @@ test("sessions.list keeps bulk rows lightweight and uses persisted model fields" }), }, }); + await seedSessionTranscript({ + sessionId: "sess-child", + sessionKey: "agent:main:dashboard:child", + storePath, + messages: [ + { + role: "assistant", + provider: "anthropic", + model: "claude-sonnet-4-6", + usage: { + input: 2_000, + output: 500, + cacheRead: 1_000, + cost: { total: 0.0042 }, + }, + }, + { + role: "assistant", + provider: "openclaw", + model: "delivery-mirror", + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }, + ], + }); const { ws } = await openClient(); const listed = await rpcReq<{ @@ -675,23 +666,26 @@ test("sessions.list ignores hidden internal abortable runs", async () => { }); test("sessions.list yields before responding during bulk transcript hydration", async () => { - const { dir } = await createSessionStoreDir(); + const { storePath } = await createSessionStoreDir(); const entries: Record<string, ReturnType<typeof sessionStoreEntry>> = {}; const now = Date.now(); for (let i = 0; i < 11; i += 1) { const sessionId = `sess-list-yield-${i}`; entries[`bulk-${i}`] = sessionStoreEntry(sessionId, { updatedAt: now - i }); - await fs.writeFile( - path.join(dir, `${sessionId}.jsonl`), - [ - JSON.stringify({ type: "session", version: 1, id: sessionId }), - JSON.stringify({ message: { role: "user", content: `title ${i}` } }), - JSON.stringify({ message: { role: "assistant", content: `last ${i}` } }), - ].join("\n"), - "utf-8", - ); } await writeSessionStore({ entries }); + for (let i = 0; i < 11; i += 1) { + const sessionId = `sess-list-yield-${i}`; + await seedSessionTranscript({ + sessionId, + sessionKey: `agent:main:bulk-${i}`, + storePath, + messages: [ + { role: "user", content: `title ${i}` }, + { role: "assistant", content: `last ${i}` }, + ], + }); + } const { request, respond } = await invokeSessionsList({ requestId: "req-sessions-list-yield", @@ -749,30 +743,7 @@ test("sessions.list does not block on slow model catalog discovery", async () => }); test("sessions.changed mutation events include live usage metadata", async () => { - const { dir } = await createSessionStoreDir(); - await fs.writeFile( - path.join(dir, "sess-main.jsonl"), - [ - JSON.stringify({ type: "session", version: 1, id: "sess-main" }), - JSON.stringify({ - id: "msg-usage-zero", - message: { - role: "assistant", - provider: "openai", - model: "gpt-5.3-codex-spark", - usage: { - input: 5_107, - output: 1_827, - cacheRead: 1_536, - cacheWrite: 0, - cost: { total: 0 }, - }, - timestamp: Date.now(), - }, - }), - ].join("\n"), - "utf-8", - ); + const { storePath } = await createSessionStoreDir(); await writeSessionStore({ entries: { main: sessionStoreEntry("sess-main", { @@ -784,6 +755,26 @@ test("sessions.changed mutation events include live usage metadata", async () => }), }, }); + await seedSessionTranscript({ + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + messages: [ + { + role: "assistant", + provider: "openai", + model: "gpt-5.3-codex-spark", + usage: { + input: 5_107, + output: 1_827, + cacheRead: 1_536, + cacheWrite: 0, + cost: { total: 0 }, + }, + timestamp: Date.now(), + }, + ], + }); const result = await invokeSessionsPatch({ key: "main", @@ -972,14 +963,18 @@ test("sessions.patch scopes selected global mutations and events to the requeste reason: "patch", label: "Work global", }); - const mainStore = JSON.parse(await fs.readFile(globalStores.mainStorePath, "utf-8")) as { - global?: { label?: string }; - }; - const workStore = JSON.parse(await fs.readFile(globalStores.workStorePath, "utf-8")) as { - global?: { label?: string }; - }; - expect(mainStore.global?.label).toBeUndefined(); - expect(workStore.global?.label).toBe("Work global"); + const mainEntry = loadSessionEntry({ + agentId: "main", + sessionKey: "global", + storePath: globalStores.mainStorePath, + }); + const workEntry = loadSessionEntry({ + agentId: "work", + sessionKey: "global", + storePath: globalStores.workStorePath, + }); + expect(mainEntry?.label).toBeUndefined(); + expect(workEntry?.label).toBe("Work global"); await resetConfiguredGlobalAgentSessionStore(globalStores); }); @@ -1001,12 +996,22 @@ test("sessions.compact scopes selected global truncation to the requested agent" reason: "compact", compacted: true, }); - await expect(fs.readFile(globalStores.mainTranscript, "utf-8")).resolves.toBe( - createLinearSessionTranscript("sess-main-global", ["main one", "main two"]), - ); - await expect(fs.readFile(globalStores.workTranscript, "utf-8")).resolves.toBe( - expectedLastMessageTranscript("sess-work-global", ["work one", "work two"]), - ); + await expect( + loadSeededTranscriptEvents({ + agentId: "main", + sessionId: "sess-main-global", + sessionKey: "global", + storePath: globalStores.mainStorePath, + }).then(transcriptMessageContents), + ).resolves.toEqual(["main one", "main two"]); + await expect( + loadSeededTranscriptEvents({ + agentId: "work", + sessionId: "sess-work-global", + sessionKey: "global", + storePath: globalStores.workStorePath, + }).then(transcriptMessageContents), + ).resolves.toEqual(["work two"]); await resetConfiguredGlobalAgentSessionStore(globalStores); }); @@ -1027,12 +1032,22 @@ test("sessions.compact trims default global agent when no agentId is supplied", reason: "compact", compacted: true, }); - await expect(fs.readFile(globalStores.mainTranscript, "utf-8")).resolves.toBe( - expectedLastMessageTranscript("sess-main-global", ["main one", "main two"]), - ); - await expect(fs.readFile(globalStores.workTranscript, "utf-8")).resolves.toBe( - createLinearSessionTranscript("sess-work-global", ["work one", "work two"]), - ); + await expect( + loadSeededTranscriptEvents({ + agentId: "main", + sessionId: "sess-main-global", + sessionKey: "global", + storePath: globalStores.mainStorePath, + }).then(transcriptMessageContents), + ).resolves.toEqual(["main two"]); + await expect( + loadSeededTranscriptEvents({ + agentId: "work", + sessionId: "sess-work-global", + sessionKey: "global", + storePath: globalStores.workStorePath, + }).then(transcriptMessageContents), + ).resolves.toEqual(["work one", "work two"]); await resetConfiguredGlobalAgentSessionStore(globalStores); }); @@ -1049,9 +1064,14 @@ test("sessions.compact keeps manual trim no-op response shape", async () => { expectFields(responsePayload, { ok: true, key: "global", compacted: false, kept: 3 }); expect(broadcastToConnIds).not.toHaveBeenCalled(); - await expect(fs.readFile(globalStores.workTranscript, "utf-8")).resolves.toBe( - createLinearSessionTranscript("sess-work-global", ["work one", "work two"]), - ); + await expect( + loadSeededTranscriptEvents({ + agentId: "work", + sessionId: "sess-work-global", + sessionKey: "global", + storePath: globalStores.workStorePath, + }).then(transcriptMessageContents), + ).resolves.toEqual(["work one", "work two"]); await resetConfiguredGlobalAgentSessionStore(globalStores); }); @@ -1099,21 +1119,23 @@ test("sessions.compact passes the selected global agent into embedded compaction }); test("sessions.compact mounts a dashboard managed worktree as its workspace", async () => { - const { dir } = await createSessionStoreDir(); - const sessionFile = path.join(dir, "sess-suggested.jsonl"); - await fs.writeFile( - sessionFile, - createLinearSessionTranscript("sess-suggested", ["one", "two"]), - "utf-8", - ); + const { storePath } = await createSessionStoreDir(); await writeSessionStore({ entries: { "dashboard:suggested": sessionStoreEntry("sess-suggested", { - sessionFile, spawnedCwd: "/tmp/suggested-worktree", }), }, }); + await seedSessionTranscript({ + sessionId: "sess-suggested", + sessionKey: "agent:main:dashboard:suggested", + storePath, + messages: [ + { role: "user", content: "one" }, + { role: "assistant", content: "two" }, + ], + }); const { getRuntimeConfig } = await getGatewayConfigModule(); const { responsePayload } = await invokeSessionsCompact({ diff --git a/src/gateway/server.sessions.permissions-hooks.test.ts b/src/gateway/server.sessions.permissions-hooks.test.ts index a31807d043e7..b1e332fdbbb7 100644 --- a/src/gateway/server.sessions.permissions-hooks.test.ts +++ b/src/gateway/server.sessions.permissions-hooks.test.ts @@ -1,6 +1,5 @@ // Session permissions and hooks tests protect gateway access control around // patch/delete/compact/restore APIs plus emitted internal hook payloads. -import fs from "node:fs/promises"; import path from "node:path"; import { afterAll, expect, test, vi } from "vitest"; import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js"; @@ -14,6 +13,7 @@ import { GATEWAY_CLIENT_IDS, GATEWAY_CLIENT_MODES, } from "../../packages/gateway-protocol/src/client-info.js"; +import { loadSessionEntry } from "../config/sessions/session-accessor.js"; import { isSessionPatchEvent } from "../hooks/internal-hooks.js"; import { requireRecord } from "./test-helpers.assertions.js"; import { connectWebchatClient, rpcReq, testState, writeSessionStore } from "./test-helpers.js"; @@ -327,11 +327,9 @@ test("control-ui client can delete sessions even in webchat mode", async () => { expect(deleted.ok).toBe(true); expect(deleted.payload?.deleted).toBe(true); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { sessionId?: string } - >; - expect(store["agent:main:discord:group:dev"]).toBeUndefined(); + expect( + loadSessionEntry({ sessionKey: "agent:main:discord:group:dev", storePath }), + ).toBeUndefined(); ws.close(); }); diff --git a/src/gateway/server.sessions.preview-resolve.test.ts b/src/gateway/server.sessions.preview-resolve.test.ts index d781dcd07703..67b70b3b990d 100644 --- a/src/gateway/server.sessions.preview-resolve.test.ts +++ b/src/gateway/server.sessions.preview-resolve.test.ts @@ -1,9 +1,15 @@ /** * Gateway session preview resolve tests. */ -import fs from "node:fs/promises"; import path from "node:path"; import { expect, test } from "vitest"; +import { clearSessionStoreCacheForTest } from "../config/sessions.js"; +import { + applySessionEntryLifecycleMutation, + listSessionEntries, + loadSessionEntry, +} from "../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import { createToolSummaryPreviewTranscriptLines } from "./session-preview.test-helpers.js"; import { rpcReq, testState, writeSessionStore } from "./test-helpers.js"; import { @@ -11,38 +17,60 @@ import { sessionStoreEntry, getMainPreviewEntry, directSessionReq, - createLinearSessionTranscript, + loadSeededTranscriptEvents, + seedLinearSessionTranscript, + seedSessionTranscript, } from "./test/server-sessions.test-helpers.js"; const { createSessionStoreDir, openClient } = setupGatewaySessionsTestHarness(); -async function writeTranscriptMessage( - dir: string, - sessionId: string, - content: string, +async function replaceSessionEntries( + storePath: string, + entries: Record<string, Partial<SessionEntry>>, + agentId = "main", ): Promise<void> { - await fs.writeFile( - path.join(dir, `${sessionId}.jsonl`), - [ - JSON.stringify({ type: "session", version: 1, id: sessionId }), - JSON.stringify({ message: { role: "assistant", content } }), - ].join("\n"), - "utf-8", - ); + clearSessionStoreCacheForTest(); + await applySessionEntryLifecycleMutation({ + agentId, + storePath, + removals: listSessionEntries({ agentId, storePath }).map(({ sessionKey }) => ({ sessionKey })), + upserts: Object.entries(entries).map(([sessionKey, entry]) => ({ + sessionKey, + entry: { + updatedAt: 0, + ...entry, + sessionId: entry.sessionId ?? sessionKey, + }, + })), + skipMaintenance: true, + }); + clearSessionStoreCacheForTest(); } async function previewMainAliasFromStore(params: { transcripts: Record<string, string>; store: Record<string, { sessionId: string; updatedAt: number }>; }): Promise<Awaited<ReturnType<typeof getMainPreviewEntry>>> { - const { dir, storePath } = await createSessionStoreDir(); + const { dir } = await createSessionStoreDir(); + const storePath = path.join(dir, "agents", "ops", "sessions", "sessions.json"); + testState.sessionStorePath = storePath; testState.agentsConfig = { list: [{ id: "ops", default: true }] }; testState.sessionConfig = { mainKey: "work" }; - for (const [sessionId, content] of Object.entries(params.transcripts)) { - await writeTranscriptMessage(dir, sessionId, content); + await writeSessionStore({ agentId: "ops", entries: {} }); + await replaceSessionEntries(storePath, params.store, "ops"); + for (const [sessionKey, entry] of Object.entries(params.store)) { + const content = params.transcripts[entry.sessionId]; + if (content) { + await seedSessionTranscript({ + agentId: "ops", + sessionId: entry.sessionId, + sessionKey, + storePath, + messages: [{ role: "assistant", content }], + }); + } } - await fs.writeFile(storePath, JSON.stringify(params.store, null, 2), "utf-8"); const { ws } = await openClient(); try { @@ -53,17 +81,25 @@ async function previewMainAliasFromStore(params: { } test("sessions.preview returns transcript previews", async () => { - const { dir } = await createSessionStoreDir(); + const { storePath } = await createSessionStoreDir(); const sessionId = "sess-preview"; - const transcriptPath = path.join(dir, `${sessionId}.jsonl`); const lines = createToolSummaryPreviewTranscriptLines(sessionId); - await fs.writeFile(transcriptPath, lines.join("\n"), "utf-8"); await writeSessionStore({ entries: { main: sessionStoreEntry(sessionId), }, }); + await seedSessionTranscript({ + sessionId, + sessionKey: "agent:main:main", + storePath, + messages: lines + .map((line) => JSON.parse(line) as { message?: Record<string, unknown> }) + .map((record) => record.message) + .filter((message): message is Record<string, unknown> => Boolean(message)) + .map((message) => Object.assign({ role: String(message.role) }, message)), + }); const preview = await directSessionReq<{ previews: Array<{ @@ -118,25 +154,32 @@ test("sessions.preview prefers the freshest duplicate row for a legacy main alia }); test("sessions.resolve and mutators clean legacy main-alias ghost keys", async () => { - const { dir, storePath } = await createSessionStoreDir(); + const { dir } = await createSessionStoreDir(); + const storePath = path.join(dir, "agents", "ops", "sessions", "sessions.json"); + testState.sessionStorePath = storePath; testState.agentsConfig = { list: [{ id: "ops", default: true }] }; testState.sessionConfig = { mainKey: "work" }; const sessionId = "sess-alias-cleanup"; - const transcriptPath = path.join(dir, `${sessionId}.jsonl`); - await fs.writeFile( - transcriptPath, - createLinearSessionTranscript( - sessionId, - Array.from({ length: 8 }, (_, index) => `line ${index}`), - ), - "utf-8", - ); + await writeSessionStore({ agentId: "ops", entries: {} }); + await seedLinearSessionTranscript({ + agentId: "ops", + contents: Array.from({ length: 8 }, (_, index) => `line ${index}`), + sessionId, + sessionKey: "agent:ops:work", + storePath, + }); - const writeRawStore = async (store: Record<string, unknown>) => { - await fs.writeFile(storePath, `${JSON.stringify(store, null, 2)}\n`, "utf-8"); + const writeRawStore = async (store: Record<string, Partial<SessionEntry>>) => { + await replaceSessionEntries(storePath, store, "ops"); }; - const readStore = async () => - JSON.parse(await fs.readFile(storePath, "utf-8")) as Record<string, Record<string, unknown>>; + const readStoreKeys = () => + listSessionEntries({ agentId: "ops", storePath }).map(({ sessionKey }) => sessionKey); + const readWorkEntry = () => + loadSessionEntry({ + agentId: "ops", + sessionKey: "agent:ops:work", + storePath, + }); await writeRawStore({ "agent:ops:main": { sessionId, updatedAt: Date.now() - 1_000 }, @@ -149,12 +192,11 @@ test("sessions.resolve and mutators clean legacy main-alias ghost keys", async ( }); expect(resolved.ok).toBe(true); expect(resolved.payload?.key).toBe("agent:ops:work"); - let store = await readStore(); - expect(Object.keys(store).toSorted()).toEqual(["agent:ops:work"]); + expect(readStoreKeys().toSorted()).toEqual(["agent:ops:work"]); await writeRawStore({ - ...store, - "agent:ops:main": { ...store["agent:ops:work"] }, + "agent:ops:work": readWorkEntry() ?? { sessionId }, + "agent:ops:main": readWorkEntry() ?? { sessionId }, }); const patched = await rpcReq<{ ok: true; key: string }>(ws, "sessions.patch", { key: "main", @@ -162,13 +204,12 @@ test("sessions.resolve and mutators clean legacy main-alias ghost keys", async ( }); expect(patched.ok).toBe(true); expect(patched.payload?.key).toBe("agent:ops:work"); - store = await readStore(); - expect(Object.keys(store).toSorted()).toEqual(["agent:ops:work"]); - expect(store["agent:ops:work"]?.thinkingLevel).toBe("medium"); + expect(readStoreKeys().toSorted()).toEqual(["agent:ops:work"]); + expect(readWorkEntry()?.thinkingLevel).toBe("medium"); await writeRawStore({ - ...store, - "agent:ops:main": { ...store["agent:ops:work"] }, + "agent:ops:work": readWorkEntry() ?? { sessionId }, + "agent:ops:main": readWorkEntry() ?? { sessionId }, }); const compacted = await rpcReq<{ ok: true; compacted: boolean }>(ws, "sessions.compact", { key: "main", @@ -176,18 +217,31 @@ test("sessions.resolve and mutators clean legacy main-alias ghost keys", async ( }); expect(compacted.ok).toBe(true); expect(compacted.payload?.compacted).toBe(true); - store = await readStore(); - expect(Object.keys(store).toSorted()).toEqual(["agent:ops:work"]); + expect(readStoreKeys().toSorted()).toEqual(["agent:ops:work"]); + const compactedEvents = await loadSeededTranscriptEvents({ + agentId: "ops", + sessionId, + sessionKey: "agent:ops:work", + storePath, + }); + expect( + compactedEvents + .map((event) => + event && typeof event === "object" && "message" in event + ? (event as { message?: { content?: unknown } }).message?.content + : undefined, + ) + .filter((content) => content !== undefined), + ).toEqual(["line 6", "line 7"]); await writeRawStore({ - ...store, - "agent:ops:main": { ...store["agent:ops:work"] }, + "agent:ops:work": readWorkEntry() ?? { sessionId }, + "agent:ops:main": readWorkEntry() ?? { sessionId }, }); const reset = await rpcReq<{ ok: true; key: string }>(ws, "sessions.reset", { key: "main" }); expect(reset.ok).toBe(true); expect(reset.payload?.key).toBe("agent:ops:work"); - store = await readStore(); - expect(Object.keys(store).toSorted()).toEqual(["agent:ops:work"]); + expect(readStoreKeys().toSorted()).toEqual(["agent:ops:work"]); ws.close(); }); diff --git a/src/gateway/server.sessions.reset-cleanup.test.ts b/src/gateway/server.sessions.reset-cleanup.test.ts index 88aec97db45a..855175e0d675 100644 --- a/src/gateway/server.sessions.reset-cleanup.test.ts +++ b/src/gateway/server.sessions.reset-cleanup.test.ts @@ -1,6 +1,5 @@ // Session reset cleanup tests protect ACP metadata resets, active run shutdown, // hook emission, thread bindings, and browser/MCP cleanup side effects. -import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, expect, test, vi } from "vitest"; import { @@ -12,6 +11,7 @@ import { registerAgentHarness, restoreRegisteredAgentHarnesses, } from "../agents/harness/registry.js"; +import { loadSessionEntry } from "../config/sessions/session-accessor.js"; import type { SessionAcpMeta } from "../config/sessions/types.js"; import { enqueueSystemEvent, peekSystemEvents } from "../infra/system-events.js"; import { @@ -216,7 +216,7 @@ test("sessions.reset forwards the retired generation to registered agent harness agentId: "main", sessionId: "sess-main", sessionKey: "agent:main:main", - sessionFile: undefined, + sessionFile: expect.stringMatching(/^sqlite:main:sess-main:/), reason: "reset", }); } finally { @@ -395,11 +395,13 @@ test("sessions.reset closes ACP runtime handles for ACP sessions", async () => { expect(prepareFreshSession).toHaveBeenCalledWith({ sessionKey: "agent:main:main", }); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { acp?: ResetAcpState } - >; - expect(store["agent:main:main"]).not.toHaveProperty("acp"); + expect( + loadSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath, + }), + ).not.toHaveProperty("acp"); expectResetAcpState(readAcpSessionMeta({ sessionKey: "agent:main:main" })); }); @@ -487,12 +489,9 @@ test("sessions.reset rejects a concurrent archive during lifecycle rotation", as ok: false, error: { message: "Cannot archive a session with an active run." }, }); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { archivedAt?: number; sessionId?: string } - >; - expect(store[sessionKey]?.archivedAt).toBeUndefined(); - expect(store[sessionKey]?.sessionId).not.toBe("sess-archive-race"); + const entry = loadSessionEntry({ storePath, sessionKey }); + expect(entry?.archivedAt).toBeUndefined(); + expect(entry?.sessionId).not.toBe("sess-archive-race"); }); test.each([ @@ -553,12 +552,9 @@ test.each([ ok: false, error: { message: `Session ${sessionKey} changed before patch. Retry.` }, }); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { archivedAt?: number; sessionId?: string } - >; - expect(store[sessionKey]?.archivedAt).toBeUndefined(); - expect(store[sessionKey]?.sessionId).toBe(replacementSessionId); + const entry = loadSessionEntry({ storePath, sessionKey }); + expect(entry?.archivedAt).toBeUndefined(); + expect(entry?.sessionId).toBe(replacementSessionId); }); test("sessions.reset preserves a newer session after lifecycle rotation", async () => { @@ -602,11 +598,13 @@ test("sessions.reset preserves a newer session after lifecycle rotation", async }), ).rejects.toThrow("stale lifecycle"); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { sessionId?: string } - >; - expect(store["agent:main:main"]?.sessionId).toBe("new-owner-session"); + expect( + loadSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath, + })?.sessionId, + ).toBe("new-owner-session"); }); test("sessions.reset closes child ACP runtime handles spawned from the parent", async () => { @@ -683,31 +681,27 @@ test("sessions.reset closes a spawned ACP child that lives in a different agent testState.sessionConfig = { store: path.join(stateDir, "agents", "{agentId}", "sessions", "sessions.json"), }; + testState.agentsConfig = { + list: [{ id: "main", default: true }, { id: "codex" }], + }; const mainStorePath = path.join(stateDir, "agents", "main", "sessions", "sessions.json"); const codexStorePath = path.join(stateDir, "agents", "codex", "sessions", "sessions.json"); - await fs.mkdir(path.dirname(mainStorePath), { recursive: true }); - await fs.mkdir(path.dirname(codexStorePath), { recursive: true }); - await fs.writeFile( - mainStorePath, - JSON.stringify({ - main: { - sessionId: "sess-main", - updatedAt: Date.now(), - }, - }), - "utf-8", - ); - await fs.writeFile( - codexStorePath, - JSON.stringify({ - "agent:codex:acp:cross-store-child": { - sessionId: "sess-codex-child", - updatedAt: Date.now(), + await writeSessionStore({ + agentId: "main", + entries: { + main: sessionStoreEntry("sess-main"), + }, + storePath: mainStorePath, + }); + await writeSessionStore({ + agentId: "codex", + entries: { + "agent:codex:acp:cross-store-child": sessionStoreEntry("sess-codex-child", { spawnedBy: "agent:main:main", - }, - }), - "utf-8", - ); + }), + }, + storePath: codexStorePath, + }); writeAcpSessionMetaForMigration({ sessionKey: "agent:main:main", meta: { diff --git a/src/gateway/server.sessions.reset-hooks.test.ts b/src/gateway/server.sessions.reset-hooks.test.ts index 191f78140576..6941907ad8aa 100644 --- a/src/gateway/server.sessions.reset-hooks.test.ts +++ b/src/gateway/server.sessions.reset-hooks.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { expect, test, vi } from "vitest"; +import { listSessionEntries, loadSessionEntry } from "../config/sessions/session-accessor.js"; import { beginSessionWorkAdmission } from "../sessions/session-lifecycle-admission.js"; import { embeddedRunMock, testState, writeSessionStore } from "./test-helpers.js"; import { @@ -17,6 +18,7 @@ import { sessionStoreEntry, expectActiveRunCleanup, directSessionReq, + seedSessionTranscript, } from "./test/server-sessions.test-helpers.js"; const { createSessionStoreDir, seedActiveMainSession } = setupGatewaySessionsTestHarness(); @@ -87,7 +89,7 @@ function expectStringWithPrefix(value: unknown, prefix: string, label: string): } async function configureGlobalAgentSessionStore(dir: string) { - const storeTemplate = path.join(dir, "{agentId}", "sessions.json"); + const storeTemplate = path.join(dir, "agents", "{agentId}", "sessions", "sessions.json"); const configPath = expectStringValue(process.env.OPENCLAW_CONFIG_PATH, "OPENCLAW_CONFIG_PATH"); const { clearConfigCache, clearRuntimeConfigSnapshot } = await import("../config/config.js"); testState.sessionStorePath = storeTemplate; @@ -134,50 +136,69 @@ async function withGlobalAgentSessionStore<T>( } async function writeGlobalSessionFile(storePath: string, sessionId: string) { - await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile( + const agentId = path.basename(path.dirname(path.dirname(storePath))); + await writeSessionStore({ + agentId, + entries: { + global: sessionStoreEntry(sessionId), + }, storePath, - JSON.stringify({ global: sessionStoreEntry(sessionId) }, null, 2), - "utf-8", - ); + }); } async function writeMessageTranscript(params: { - dir: string; sessionId: string; + sessionKey: string; + storePath: string; + agentId?: string; content: string; messageId?: string; }) { - const transcriptPath = path.join(params.dir, `${params.sessionId}.jsonl`); - await fs.writeFile( - transcriptPath, - `${JSON.stringify({ - type: "message", - id: params.messageId ?? "m1", - message: { role: "user", content: params.content }, - })}\n`, - "utf-8", - ); - return transcriptPath; + await seedSessionTranscript({ + agentId: params.agentId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + messages: [{ role: "user", content: params.content, id: params.messageId ?? "m1" }], + }); } async function writeMainTranscriptSession(params: { - dir: string; sessionId: string; content: string; messageId?: string; }) { - const transcriptPath = await writeMessageTranscript(params); + const storePath = expectStringValue(testState.sessionStorePath, "testState.sessionStorePath"); await writeSessionStore({ entries: { main: { sessionId: params.sessionId, - sessionFile: transcriptPath, updatedAt: Date.now(), }, }, }); - return transcriptPath; + await writeMessageTranscript({ + ...params, + agentId: "main", + sessionKey: "agent:main:main", + storePath, + }); + return expectStringValue( + loadSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath, + })?.sessionFile, + "sessionFile", + ); +} + +function loadEntry(params: { agentId?: string; sessionKey: string; storePath: string }) { + return loadSessionEntry({ + agentId: params.agentId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }); } async function writeMainSessionEntry( @@ -275,12 +296,10 @@ function claudeCliBindings(sessionId: string) { function cliBoundSessionEntry( sessionId: string, - sessionFile: string, cliSessionId: string, overrides: Parameters<typeof sessionStoreEntry>[1] = {}, ) { return sessionStoreEntry(sessionId, { - sessionFile, ...overrides, ...claudeCliBindings(cliSessionId), }); @@ -298,22 +317,13 @@ async function resolveGatewaySessionStorePathForKey(key: string) { } async function loadGatewaySessionStoreForKey(key: string) { - const [{ loadSessionStore }, gatewayStorePath] = await Promise.all([ - import("../config/sessions.js"), - resolveGatewaySessionStorePathForKey(key), - ]); - return loadSessionStore(gatewayStorePath, { skipCache: true }); -} - -async function updateGatewaySessionStoreForKey( - key: string, - update: Parameters<(typeof import("../config/sessions.js"))["updateSessionStore"]>[1], -) { - const [{ updateSessionStore }, gatewayStorePath] = await Promise.all([ - import("../config/sessions.js"), - resolveGatewaySessionStorePathForKey(key), - ]); - await updateSessionStore(gatewayStorePath, update); + const gatewayStorePath = await resolveGatewaySessionStorePathForKey(key); + return Object.fromEntries( + listSessionEntries({ storePath: gatewayStorePath }).map(({ sessionKey, entry }) => [ + sessionKey, + entry, + ]), + ); } function expectCliBindingsCleared( @@ -368,9 +378,8 @@ test("sessions.reset does not begin cleanup after losing lifecycle ownership", a }); test("sessions.reset emits before_reset hook with transcript context", async () => { - const { dir } = await createSessionStoreDir(); + await createSessionStoreDir(); const transcriptPath = await writeMainTranscriptSession({ - dir, sessionId: "sess-main", content: "hello from transcript", }); @@ -417,15 +426,19 @@ test("sessions.reset infers selected global agent from agent-prefixed aliases", agentId: "work", }); expect(resetTarget.storePath).toBe(globalConfig.workStorePath); - const mainStore = JSON.parse(await fs.readFile(globalConfig.mainStorePath, "utf-8")) as { - global?: { sessionId?: string }; - }; - const workStore = JSON.parse(await fs.readFile(resetTarget.storePath, "utf-8")) as { - global?: { sessionId?: string }; - }; - expect(mainStore.global?.sessionId).toBe("sess-main-global"); - expect(workStore.global?.sessionId).toBe(reset.entry.sessionId); - expect(workStore.global?.sessionId).not.toBe("sess-work-global"); + const mainEntry = loadEntry({ + agentId: "main", + sessionKey: "global", + storePath: globalConfig.mainStorePath, + }); + const workEntry = loadEntry({ + agentId: "work", + sessionKey: "global", + storePath: resetTarget.storePath, + }); + expect(mainEntry?.sessionId).toBe("sess-main-global"); + expect(workEntry?.sessionId).toBe(reset.entry.sessionId); + expect(workEntry?.sessionId).not.toBe("sess-work-global"); }); }); @@ -486,9 +499,8 @@ test("sessions.reset emits inferred selected global agent scope", async () => { }); test("sessions.reset emits enriched session_end and session_start hooks", async () => { - const { dir } = await createSessionStoreDir(); + await createSessionStoreDir(); await writeMainTranscriptSession({ - dir, sessionId: "sess-main", content: "hello from transcript", }); @@ -504,13 +516,12 @@ test("sessions.reset emits enriched session_end and session_start hooks", async expect(endEvent.sessionKey).toBe("agent:main:main"); expect(endEvent.reason).toBe("new"); expect(endEvent.transcriptArchived).toBe(true); - const realDir = await fs.realpath(dir); const archivedSessionFile = expectStringWithPrefix( - endEvent.sessionFile, - path.join(realDir, "sess-main.jsonl.reset."), + path.basename(expectStringValue(endEvent.sessionFile, "archived session file")), + "sess-main.jsonl.reset.", "archived session file", ); - expect(path.dirname(archivedSessionFile)).toBe(realDir); + expect(archivedSessionFile).toContain(".jsonl.reset."); expect(endEvent.nextSessionId).toBe(startEvent.sessionId); expectMainHookContext(endContext, "sess-main"); expect(startEvent.sessionKey).toBe("agent:main:main"); @@ -542,11 +553,13 @@ test("sessions.reset returns unavailable when active run does not stop", async ( expect(waitCallCountAtSnapshotClear).toEqual([1]); expect(browserSessionTabMocks.closeTrackedBrowserTabsForSessions).not.toHaveBeenCalled(); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { sessionId?: string } - >; - expect(store["agent:main:main"]?.sessionId).toBe("sess-main"); + expect( + loadEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath, + })?.sessionId, + ).toBe("sess-main"); const filesAfterResetAttempt = await fs.readdir(dir); expect( filesAfterResetAttempt.filter((file) => file.startsWith("sess-main.jsonl.reset.")), @@ -554,36 +567,47 @@ test("sessions.reset returns unavailable when active run does not stop", async ( }); test("sessions.reset emits before_reset for the entry actually reset in the writer slot", async () => { - const { dir } = await createSessionStoreDir(); - const oldTranscriptPath = await writeMessageTranscript({ - dir, - sessionId: "sess-old", - content: "old transcript", - messageId: "m-old", - }); - const newTranscriptPath = await writeMessageTranscript({ - dir, - sessionId: "sess-new", - content: "new transcript", - messageId: "m-new", - }); + const { storePath } = await createSessionStoreDir(); await writeSessionStore({ entries: { main: { sessionId: "sess-old", - sessionFile: oldTranscriptPath, updatedAt: Date.now(), }, }, }); + await writeMessageTranscript({ + agentId: "main", + sessionId: "sess-old", + sessionKey: "agent:main:main", + storePath, + content: "old transcript", + messageId: "m-old", + }); beforeResetHookState.hasBeforeResetHook = true; - await updateGatewaySessionStoreForKey("main", (store) => { - store["agent:main:main"] = sessionStoreEntry("sess-new", { - sessionFile: newTranscriptPath, - }); + await writeSessionStore({ + entries: { + main: sessionStoreEntry("sess-new"), + }, }); + await writeMessageTranscript({ + agentId: "main", + sessionId: "sess-new", + sessionKey: "agent:main:main", + storePath, + content: "new transcript", + messageId: "m-new", + }); + const newSessionFile = expectStringValue( + loadEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath, + })?.sessionFile, + "new sessionFile", + ); const reset = await performSessionReset({ key: "main", @@ -597,7 +621,7 @@ test("sessions.reset emits before_reset for the entry actually reset in the writ expect(internalEvent?.context?.previousSessionEntry?.sessionId).toBe("sess-new"); expect(beforeResetHookMocks.runBeforeReset).toHaveBeenCalledTimes(1); const [event, context] = firstHookCall(beforeResetHookMocks.runBeforeReset); - expectTranscriptResetEvent({ event, sessionFile: newTranscriptPath, content: "new transcript" }); + expectTranscriptResetEvent({ event, sessionFile: newSessionFile, content: "new transcript" }); expectMainHookContext(context, "sess-new"); }); @@ -613,9 +637,8 @@ test("sessions.create with emitCommandHooks=true fires command:new hook against }); test("sessions.create with emitCommandHooks=true emits reset lifecycle hooks against parent (#76957)", async () => { - const { dir } = await createSessionStoreDir(); + await createSessionStoreDir(); const transcriptPath = await writeMainTranscriptSession({ - dir, sessionId: "sess-parent-hooks", content: "remember this before new", }); @@ -732,12 +755,7 @@ test("sessions.create fences new parent work while rollover hooks run", async () }); test("sessions.create with emitCommandHooks=true resets parent in place when session.dmScope is 'main' (#77434)", async () => { - const { dir } = await createSessionStoreDir(); - const transcriptPath = await writeMessageTranscript({ - dir, - sessionId: "sess-parent-dms", - content: "hello before /new", - }); + const { storePath } = await createSessionStoreDir(); testState.sessionConfig = { dmScope: "main" }; try { @@ -745,11 +763,17 @@ test("sessions.create with emitCommandHooks=true resets parent in place when ses entries: { main: { sessionId: "sess-parent-dms", - sessionFile: transcriptPath, updatedAt: Date.now(), }, }, }); + await writeMessageTranscript({ + agentId: "main", + sessionId: "sess-parent-dms", + sessionKey: "agent:main:main", + storePath, + content: "hello before /new", + }); embeddedRunMock.activeIds.add("sess-parent-dms"); const result = await directSessionReq<{ @@ -782,12 +806,7 @@ test("sessions.create with emitCommandHooks=true resets parent in place when ses }); test("sessions.create keeps an explicit TUI child key when session.dmScope is 'main'", async () => { - const { dir } = await createSessionStoreDir(); - const transcriptPath = await writeMessageTranscript({ - dir, - sessionId: "sess-parent-tui", - content: "hello before TUI /new", - }); + const { storePath } = await createSessionStoreDir(); testState.sessionConfig = { dmScope: "main" }; try { @@ -795,11 +814,17 @@ test("sessions.create keeps an explicit TUI child key when session.dmScope is 'm entries: { main: { sessionId: "sess-parent-tui", - sessionFile: transcriptPath, updatedAt: Date.now(), }, }, }); + await writeMessageTranscript({ + agentId: "main", + sessionId: "sess-parent-tui", + sessionKey: "agent:main:main", + storePath, + content: "hello before TUI /new", + }); const result = await directSessionReq<{ key: string; sessionId: string }>("sessions.create", { key: "tui-explicit", @@ -849,19 +874,12 @@ test("sessions.reset drops cli session bindings so the next turn does not --resu }); test("sessions.reset clears cli session bindings for parent-linked non-subagent sessions (e.g. dashboard children)", async () => { - const { dir } = await createSessionStoreDir(); - const dashboardTranscript = await writeMessageTranscript({ - dir, - sessionId: "sess-dashboard-child", - content: "hello from dashboard child", - messageId: "m-dashboard", - }); + const { storePath } = await createSessionStoreDir(); await writeSessionStore({ entries: { "dashboard:child:42": cliBoundSessionEntry( "sess-dashboard-child", - dashboardTranscript, "claude-cli-dashboard-session", { // parentSessionKey is set but the session key carries no `:subagent:` @@ -873,6 +891,14 @@ test("sessions.reset clears cli session bindings for parent-linked non-subagent ), }, }); + await writeMessageTranscript({ + agentId: "main", + sessionId: "sess-dashboard-child", + sessionKey: "agent:main:dashboard:child:42", + storePath, + content: "hello from dashboard child", + messageId: "m-dashboard", + }); await resetSession("dashboard:child:42"); @@ -881,24 +907,13 @@ test("sessions.reset clears cli session bindings for parent-linked non-subagent }); test("sessions.reset preserves cli session bindings for spawned subagents (Tak Hoffman's fa56682b3ced contract)", async () => { - const { dir } = await createSessionStoreDir(); + const { storePath } = await createSessionStoreDir(); const reseedPromptHash = "a".repeat(64); - const childTranscript = await writeMessageTranscript({ - dir, - sessionId: "sess-spawned-child", - content: "hello from spawned child", - messageId: "m-child", + const childEntry = cliBoundSessionEntry("sess-spawned-child", "claude-cli-child-session", { + parentSessionKey: "agent:main:main", + spawnedBy: "agent:main:main", + subagentRole: "orchestrator", }); - const childEntry = cliBoundSessionEntry( - "sess-spawned-child", - childTranscript, - "claude-cli-child-session", - { - parentSessionKey: "agent:main:main", - spawnedBy: "agent:main:main", - subagentRole: "orchestrator", - }, - ); childEntry.cliSessionBindings = { "claude-cli": { sessionId: "claude-cli-child-session", @@ -916,6 +931,14 @@ test("sessions.reset preserves cli session bindings for spawned subagents (Tak H "subagent:child": childEntry, }, }); + await writeMessageTranscript({ + agentId: "main", + sessionId: "sess-spawned-child", + sessionKey: "agent:main:subagent:child", + storePath, + content: "hello from spawned child", + messageId: "m-child", + }); await resetSession("subagent:child"); diff --git a/src/gateway/server.sessions.reset-models.test.ts b/src/gateway/server.sessions.reset-models.test.ts index 49d4b98a1aee..72aaaff43473 100644 --- a/src/gateway/server.sessions.reset-models.test.ts +++ b/src/gateway/server.sessions.reset-models.test.ts @@ -4,6 +4,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { expect, test } from "vitest"; +import { loadSessionEntry } from "../config/sessions/session-accessor.js"; import { MODEL_SELECTION_LOCKED_RESET_MESSAGE } from "../sessions/model-overrides.js"; import { testState, writeSessionStore } from "./test-helpers.js"; import { @@ -15,6 +16,7 @@ import { const { createSessionStoreDir } = setupGatewaySessionsTestHarness(); type ResetSessionEntry = { + sessionId?: string; sessionFile?: string; chatType?: string; channel?: string; @@ -137,9 +139,13 @@ const ownedChildMetadata = { label: "owned child", } satisfies SessionEntryOverrides & ResetSessionEntry; -function expectOwnedChildMetadata(entry: ResetSessionEntry | undefined, sessionFile: string) { +function expectSqliteSessionFile(entry: ResetSessionEntry | undefined) { + expect(entry?.sessionFile).toContain(`sqlite:main:${entry?.sessionId}:`); +} + +function expectOwnedChildMetadata(entry: ResetSessionEntry | undefined) { + expectSqliteSessionFile(entry); expect(entry).toMatchObject({ - sessionFile, ...ownedChildMetadata, }); } @@ -182,15 +188,14 @@ async function expectMainResetModelFields(params: { expect(reset.payload?.entry.modelProvider).toBe(params.expectedResolved.modelProvider); expect(reset.payload?.entry.model).toBe(params.expectedResolved.model); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - ModelResetEntry - >; + const stored = loadSessionEntry({ sessionKey: "agent:main:main", storePath }) as + | ModelResetEntry + | undefined; for (const key of selectionKeys) { - expect(store["agent:main:main"]?.[key]).toBe(params.expected[key]); + expect(stored?.[key]).toBe(params.expected[key]); } - expect(store["agent:main:main"]?.modelProvider).toBeUndefined(); - expect(store["agent:main:main"]?.model).toBeUndefined(); + expect(stored?.modelProvider).toBeUndefined(); + expect(stored?.model).toBeUndefined(); } test("sessions.reset rejects a model-locked session without replacing native state", async () => { @@ -206,7 +211,7 @@ test("sessions.reset rejects a model-locked session without replacing native sta }), }, }); - const before = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record<string, unknown>; + const before = loadSessionEntry({ sessionKey: "agent:main:main", storePath }); const reset = await directSessionReq("sessions.reset", { key: "main" }); @@ -214,7 +219,7 @@ test("sessions.reset rejects a model-locked session without replacing native sta ok: false, error: { message: MODEL_SELECTION_LOCKED_RESET_MESSAGE }, }); - const after = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record<string, unknown>; + const after = loadSessionEntry({ sessionKey: "agent:main:main", storePath }); expect(after).toEqual(before); }); @@ -263,7 +268,7 @@ test("sessions.reset recomputes model from defaults instead of stale runtime mod expect(reset.payload?.entry.modelProvider).toBe("openai"); expect(reset.payload?.entry.model).toBe("gpt-test-a"); expect(reset.payload?.entry.contextTokens).toBeUndefined(); - expect((await fs.stat(sessionFile)).isFile()).toBe(true); + expect(sessionFile).toContain(`sqlite:main:${reset.payload?.entry.sessionId}:`); }); test("sessions.reset clears stale estimated context budget status", async () => { @@ -318,12 +323,11 @@ test("sessions.reset clears stale estimated context budget status", async () => expect(reset.payload?.entry.contextBudgetStatus).toBeUndefined(); expect(reset.payload?.entry.contextTokens).toBeUndefined(); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { contextBudgetStatus?: unknown; contextTokens?: number } - >; - expect(store["agent:main:main"]?.contextBudgetStatus).toBeUndefined(); - expect(store["agent:main:main"]?.contextTokens).toBeUndefined(); + const stored = loadSessionEntry({ sessionKey: "agent:main:main", storePath }) as + | { contextBudgetStatus?: unknown; contextTokens?: number } + | undefined; + expect(stored?.contextBudgetStatus).toBeUndefined(); + expect(stored?.contextTokens).toBeUndefined(); }); test("sessions.reset drops cached skills snapshot so /new rebuilds visible skills", async () => { @@ -359,11 +363,10 @@ test("sessions.reset drops cached skills snapshot so /new rebuilds visible skill expect(reset.payload?.entry.sessionId).not.toBe("sess-stale-skills"); expect(reset.payload?.entry.skillsSnapshot).toBeUndefined(); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { skillsSnapshot?: unknown } - >; - expect(store["agent:main:main"]?.skillsSnapshot).toBeUndefined(); + const stored = loadSessionEntry({ sessionKey: "agent:main:main", storePath }) as + | { skillsSnapshot?: unknown } + | undefined; + expect(stored?.skillsSnapshot).toBeUndefined(); }); test("sessions.reset rotates generated topic transcript files with the new session id", async () => { @@ -398,18 +401,14 @@ test("sessions.reset rotates generated topic transcript files with the new sessi throw new Error("expected reset session id and file"); } expect(nextSessionId).not.toBe(previousSessionId); - expect(path.basename(nextSessionFile)).toBe(`${nextSessionId}-topic-456.jsonl`); + expect(nextSessionFile).toContain(`sqlite:main:${nextSessionId}:`); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { - sessionId?: string; - sessionFile?: string; - } - >; - const persistedEntry = store["agent:main:telegram:group:123:topic:456"]; + const persistedEntry = loadSessionEntry({ + sessionKey: "agent:main:telegram:group:123:topic:456", + storePath, + }); expect(persistedEntry?.sessionId).toBe(nextSessionId); - expect(path.basename(persistedEntry?.sessionFile ?? "")).toBe(`${nextSessionId}-topic-456.jsonl`); + expect(persistedEntry?.sessionFile).toBe(nextSessionFile); }); test("sessions.reset rotates an already-stale generated transcript file to the new session id", async () => { @@ -446,20 +445,12 @@ test("sessions.reset rotates an already-stale generated transcript file to the n throw new Error("expected reset session id and file"); } expect(nextSessionId).not.toBe(currentSessionId); - // The new session must adopt the new session id, not keep the stale generated name. - expect(path.basename(nextSessionFile)).toBe(`${nextSessionId}.jsonl`); - expect(path.basename(nextSessionFile)).not.toBe(`${staleFileSessionId}.jsonl`); + expect(nextSessionFile).toContain(`sqlite:main:${nextSessionId}:`); + expect(nextSessionFile).not.toContain(staleFileSessionId); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { - sessionId?: string; - sessionFile?: string; - } - >; - const persistedEntry = store["agent:main:main"]; + const persistedEntry = loadSessionEntry({ sessionKey: "agent:main:main", storePath }); expect(persistedEntry?.sessionId).toBe(nextSessionId); - expect(path.basename(persistedEntry?.sessionFile ?? "")).toBe(`${nextSessionId}.jsonl`); + expect(persistedEntry?.sessionFile).toBe(nextSessionFile); }); test("sessions.reset preserves legacy explicit model overrides without modelOverrideSource", async () => { @@ -543,11 +534,10 @@ test("sessions.reset preserves spawned session ownership metadata", async () => }>("sessions.reset", { key: "subagent:child" }); expect(reset.ok).toBe(true); - expectOwnedChildMetadata(reset.payload?.entry, customSessionFile); + expectOwnedChildMetadata(reset.payload?.entry); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - ResetSessionEntry - >; - expectOwnedChildMetadata(store["agent:main:subagent:child"], customSessionFile); + const stored = loadSessionEntry({ sessionKey: "agent:main:subagent:child", storePath }) as + | ResetSessionEntry + | undefined; + expectOwnedChildMetadata(stored); }); diff --git a/src/gateway/server.sessions.store-rpc.test.ts b/src/gateway/server.sessions.store-rpc.test.ts index 612a629a6158..914db6c7f9ec 100644 --- a/src/gateway/server.sessions.store-rpc.test.ts +++ b/src/gateway/server.sessions.store-rpc.test.ts @@ -4,58 +4,63 @@ import fs from "node:fs/promises"; import path from "node:path"; import { expect, test, vi } from "vitest"; +import { + loadSessionEntry, + loadTranscriptEvents, + persistSessionTranscriptTurn, +} from "../config/sessions/session-accessor.js"; import { agentDiscoveryMock, rpcReq, testState, writeSessionStore } from "./test-helpers.js"; import { directSessionReq as directSessionHandlerReq, setupGatewaySessionsTestHarness, getGatewayConfigModule, getSessionsHandlers, - createLinearSessionTranscript, } from "./test/server-sessions.test-helpers.js"; const { createSessionStoreDir, openClient } = setupGatewaySessionsTestHarness(); -function collectNonEmptyLines(text: string): string[] { - const lines: string[] = []; - for (const line of text.split(/\r?\n/)) { - if (line.trim().length > 0) { - lines.push(line); - } - } - return lines; +async function seedLinearTranscript(params: { + contents: string[]; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise<void> { + await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }, + { + updateMode: "none", + messages: params.contents.map((content, index) => ({ + message: { role: "user", content, timestamp: index + 1 }, + now: Date.parse(`2026-06-19T12:00:${String(index + 1).padStart(2, "0")}.000Z`), + })), + }, + ); } -function expectSinglePrefixedFilename(files: string[], prefix: string): string { - const matches = files.filter((file) => file.startsWith(prefix)); - expect(matches).toHaveLength(1); - const [match] = matches; - if (!match) { - throw new Error(`Expected one filename with prefix ${prefix}`); - } - expect(match.length).toBeGreaterThan(prefix.length); - return match; +async function loadTranscriptRows(params: { + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise<unknown[]> { + return await loadTranscriptEvents({ + agentId: "main", + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }); } test("lists and patches session store via sessions.* RPC", async () => { - const { dir, storePath } = await createSessionStoreDir(); + const { storePath } = await createSessionStoreDir(); const now = Date.now(); const recent = now - 30_000; const stale = now - 15 * 60_000; - await fs.writeFile( - path.join(dir, "sess-main.jsonl"), - createLinearSessionTranscript( - "sess-main", - Array.from({ length: 10 }, (_, index) => `line ${index}`), - ), - "utf-8", - ); - await fs.writeFile( - path.join(dir, "sess-group.jsonl"), - createLinearSessionTranscript("sess-group", ["group line 0"]), - "utf-8", - ); - await writeSessionStore({ entries: { main: { @@ -88,6 +93,25 @@ test("lists and patches session store via sessions.* RPC", async () => { }, }, }); + await seedLinearTranscript({ + contents: Array.from({ length: 10 }, (_, index) => `line ${index}`), + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + }); + await seedLinearTranscript({ + contents: ["group line 0"], + sessionId: "sess-group", + sessionKey: "agent:main:discord:group:dev", + storePath, + }); + await expect( + loadTranscriptRows({ + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + }), + ).resolves.toHaveLength(11); const { ws, hello } = await openClient(); const methods = (hello as { features?: { methods?: string[] } }).features?.methods ?? []; @@ -102,6 +126,8 @@ test("lists and patches session store via sessions.* RPC", async () => { const { getRuntimeConfig } = await getGatewayConfigModule(); const directContext = { broadcastToConnIds: vi.fn(), + chatAbortControllers: new Map(), + chatQueuedTurns: new Map(), dedupe: new Map(), getSessionEventSubscriberConnIds: () => new Set<string>(), logGateway: { debug: vi.fn() }, @@ -496,18 +522,25 @@ test("lists and patches session store via sessions.* RPC", async () => { }); expect(compacted.ok).toBe(true); expect(compacted.payload?.compacted).toBe(true); - const compactedLines = collectNonEmptyLines( - await fs.readFile(path.join(dir, "sess-main.jsonl"), "utf-8"), - ); - expect(compactedLines).toHaveLength(3); - const filesAfterCompact = await fs.readdir(dir); - expectSinglePrefixedFilename(filesAfterCompact, "sess-main.jsonl.bak."); + await expect( + loadTranscriptRows({ + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + }), + ).resolves.toHaveLength(3); - const deleted = await directSessionReq<{ ok: true; deleted: boolean }>("sessions.delete", { - key: "agent:main:discord:group:dev", - }); + const deleted = await directSessionReq<{ + archived: string[]; + ok: true; + deleted: boolean; + }>("sessions.delete", { key: "agent:main:discord:group:dev" }); expect(deleted.ok).toBe(true); expect(deleted.payload?.deleted).toBe(true); + expect(deleted.payload?.archived).toHaveLength(1); + expect(path.basename(deleted.payload?.archived[0] ?? "")).toMatch( + /^sess-group\.jsonl\.deleted\./, + ); const listAfterDelete = await directSessionReq<{ sessions: Array<{ key: string }>; }>("sessions.list", {}); @@ -515,8 +548,13 @@ test("lists and patches session store via sessions.* RPC", async () => { expect(listAfterDelete.payload?.sessions.map((session) => session.key)).not.toContain( "agent:main:discord:group:dev", ); - const filesAfterDelete = await fs.readdir(dir); - expectSinglePrefixedFilename(filesAfterDelete, "sess-group.jsonl.deleted."); + await expect( + loadTranscriptRows({ + sessionId: "sess-group", + sessionKey: "agent:main:discord:group:dev", + storePath, + }), + ).resolves.toEqual([]); const reset = await directSessionReq<{ ok: true; @@ -536,14 +574,16 @@ test("lists and patches session store via sessions.* RPC", async () => { expect(reset.payload?.entry.model).toBe("gpt-test-a"); expect(reset.payload?.entry.lastAccountId).toBe("work"); expect(reset.payload?.entry.lastThreadId).toBe("1737500000.123456"); - const storeAfterReset = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { lastAccountId?: string; lastThreadId?: string | number } - >; - expect(storeAfterReset["agent:main:main"]?.lastAccountId).toBe("work"); - expect(storeAfterReset["agent:main:main"]?.lastThreadId).toBe("1737500000.123456"); - const filesAfterReset = await fs.readdir(dir); - expectSinglePrefixedFilename(filesAfterReset, "sess-main.jsonl.reset."); + const entryAfterReset = loadSessionEntry({ sessionKey: "agent:main:main", storePath }); + expect(entryAfterReset?.lastAccountId).toBe("work"); + expect(entryAfterReset?.lastThreadId).toBe("1737500000.123456"); + await expect( + loadTranscriptRows({ + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + }), + ).resolves.toEqual([]); const badThinking = await directSessionReq("sessions.patch", { key: "agent:main:main", @@ -578,57 +618,45 @@ test("sessions.list configuredAgentsOnly keeps configured-agent children and hid const acpStorePath = path.join(stateDir, "agents", "claude", "sessions", "sessions.json"); const childStorePath = path.join(stateDir, "agents", "codex", "sessions", "sessions.json"); const diskOnlyStorePath = path.join(stateDir, "agents", "local", "sessions", "sessions.json"); - await fs.mkdir(path.dirname(mainStorePath), { recursive: true }); - await fs.mkdir(path.dirname(acpStorePath), { recursive: true }); - await fs.mkdir(path.dirname(childStorePath), { recursive: true }); - await fs.mkdir(path.dirname(diskOnlyStorePath), { recursive: true }); - await fs.writeFile( - mainStorePath, - JSON.stringify({ main: { sessionId: "sess-main", updatedAt: 20 } }, null, 2), - "utf-8", - ); - await fs.writeFile( - acpStorePath, - JSON.stringify( - { - "agent:claude:acp:25f77580-de30-4d80-9bc3-7cbc6374bce7": { - sessionId: "sess-claude-acp", - updatedAt: 30, - acp: { - backend: "acpx", - agent: "claude", - runtimeSessionName: "agent:claude:acp:25f77580-de30-4d80-9bc3-7cbc6374bce7", - mode: "oneshot", - state: "idle", - lastActivityAt: 30, - }, + await writeSessionStore({ + storePath: mainStorePath, + agentId: "main", + entries: { main: { sessionId: "sess-main", updatedAt: 20 } }, + }); + await writeSessionStore({ + storePath: acpStorePath, + agentId: "claude", + entries: { + "agent:claude:acp:25f77580-de30-4d80-9bc3-7cbc6374bce7": { + sessionId: "sess-claude-acp", + updatedAt: 30, + acp: { + backend: "acpx", + agent: "claude", + runtimeSessionName: "agent:claude:acp:25f77580-de30-4d80-9bc3-7cbc6374bce7", + mode: "oneshot", + state: "idle", + lastActivityAt: 30, }, }, - null, - 2, - ), - "utf-8", - ); - await fs.writeFile( - childStorePath, - JSON.stringify( - { - "agent:codex:subagent:app-server-child": { - sessionId: "sess-codex-child", - updatedAt: 25, - spawnedBy: "agent:main:main", - }, + }, + }); + await writeSessionStore({ + storePath: childStorePath, + agentId: "codex", + entries: { + "agent:codex:subagent:app-server-child": { + sessionId: "sess-codex-child", + updatedAt: 25, + spawnedBy: "agent:main:main", }, - null, - 2, - ), - "utf-8", - ); - await fs.writeFile( - diskOnlyStorePath, - JSON.stringify({ main: { sessionId: "sess-local", updatedAt: 10 } }, null, 2), - "utf-8", - ); + }, + }); + await writeSessionStore({ + storePath: diskOnlyStorePath, + agentId: "local", + entries: { main: { sessionId: "sess-local", updatedAt: 10 } }, + }); const configuredOnly = await directSessionHandlerReq<{ sessions: Array<{ key: string }> }>( "sessions.list", diff --git a/src/gateway/server.tools-effective.global-agent-gateway.test.ts b/src/gateway/server.tools-effective.global-agent-gateway.test.ts index e3e63f328f97..8e25f92ba4fc 100644 --- a/src/gateway/server.tools-effective.global-agent-gateway.test.ts +++ b/src/gateway/server.tools-effective.global-agent-gateway.test.ts @@ -15,6 +15,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { expect, test } from "vitest"; import { ErrorCodes } from "../../packages/gateway-protocol/src/index.js"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; import { rpcReq, testState } from "./test-helpers.js"; import { getGatewayConfigModule, @@ -57,11 +58,6 @@ async function configureNonGlobalMainSession(): Promise<() => Promise<void>> { testState.sessionStorePath = storePath; testState.sessionConfig = undefined; await fs.mkdir(dir, { recursive: true }); - await fs.writeFile( - storePath, - `${JSON.stringify({ "agent:main:abc": { sessionId: "sess-main-agent", updatedAt: 1 } }, null, 2)}\n`, - "utf-8", - ); await fs.writeFile( configPath, `${JSON.stringify( @@ -77,6 +73,10 @@ async function configureNonGlobalMainSession(): Promise<() => Promise<void>> { const { clearConfigCache, clearRuntimeConfigSnapshot } = await getGatewayConfigModule(); clearRuntimeConfigSnapshot(); clearConfigCache(); + await replaceSessionEntry( + { storePath, sessionKey: "agent:main:abc" }, + { sessionId: "sess-main-agent", updatedAt: 1 }, + ); return async () => { testState.sessionStorePath = undefined; await fs.writeFile(configPath, "{}\n", "utf-8"); diff --git a/src/gateway/session-compaction-checkpoints.test.ts b/src/gateway/session-compaction-checkpoints.test.ts index de75d70fdc25..203c92800464 100644 --- a/src/gateway/session-compaction-checkpoints.test.ts +++ b/src/gateway/session-compaction-checkpoints.test.ts @@ -8,6 +8,15 @@ import path from "node:path"; import { CURRENT_SESSION_VERSION, SessionManager } from "openclaw/plugin-sdk/agent-sessions"; import type { AssistantMessage } from "openclaw/plugin-sdk/llm"; import { afterEach, describe, expect, test, vi } from "vitest"; +import type { SessionCompactionCheckpoint, SessionEntry } from "../config/sessions.js"; +import { + appendTranscriptEvent, + appendTranscriptMessage, + loadSessionEntry, + loadTranscriptEvents, + upsertSessionEntry, +} from "../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { captureCompactionCheckpointSnapshotAsync, @@ -70,6 +79,18 @@ function expectNonEmptyStringField(value: unknown, message: string): string { return value; } +function isAssistantTextEvent(event: unknown, text: string): boolean { + if (!event || typeof event !== "object" || Array.isArray(event)) { + return false; + } + const message = (event as { message?: unknown }).message; + if (!message || typeof message !== "object" || Array.isArray(message)) { + return false; + } + const candidate = message as { role?: unknown; content?: unknown }; + return candidate.role === "assistant" && candidate.content === text; +} + async function makeTempSessionStore(prefix: string, sessionId = TEST_SESSION_ID) { const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); tempDirs.push(dir); @@ -89,24 +110,18 @@ function checkpointConfig(storePath: string): OpenClawConfig { } as OpenClawConfig; } -async function writeSessionStore( +async function writeAccessorSessionEntry( storePath: string, sessionKey: string, - entry: { sessionId: string; updatedAt: number; compactionCheckpoints?: unknown[] } & Record< - string, - unknown - >, + entry: Partial<SessionEntry>, ): Promise<void> { - await fs.writeFile(storePath, JSON.stringify({ [sessionKey]: entry }, null, 2), "utf-8"); -} - -async function readSessionStore<T extends object>(storePath: string): Promise<Record<string, T>> { - return JSON.parse(await fs.readFile(storePath, "utf-8")) as Record<string, T>; + await upsertSessionEntry({ storePath, sessionKey }, entry); } async function readFirstCompactionCheckpoints<T>(storePath: string): Promise<T[] | undefined> { - const store = await readSessionStore<{ compactionCheckpoints?: T[] }>(storePath); - return Object.values(store).find((entry) => entry.compactionCheckpoints)?.compactionCheckpoints; + return loadSessionEntry({ storePath, sessionKey: MAIN_SESSION_KEY })?.compactionCheckpoints as + | T[] + | undefined; } async function createLegacyCheckpointFixtures(options: { @@ -151,7 +166,7 @@ async function persistMainCheckpoint( ) { return persistSessionCompactionCheckpoint({ cfg: checkpointConfig(storePath), - sessionKey: MAIN_AGENT_ID, + sessionKey: MAIN_SESSION_KEY, sessionId: options.sessionId, reason: "manual", snapshot: options.snapshot, @@ -234,6 +249,393 @@ describe("session-compaction-checkpoints", () => { } }); + test("async capture reads SQLite marker sessions without active transcript files", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-checkpoint-sqlite-")); + tempDirs.push(dir); + const storePath = path.join(dir, "openclaw-agent.sqlite"); + const sessionId = "sqlite-checkpoint-session"; + const sessionKey = MAIN_SESSION_KEY; + const scope = { + agentId: MAIN_AGENT_ID, + sessionId, + sessionKey, + storePath, + }; + const marker = formatSqliteSessionFileMarker({ + agentId: MAIN_AGENT_ID, + sessionId, + storePath, + }); + + await upsertSessionEntry(scope, { + sessionId, + sessionFile: marker, + updatedAt: Date.now(), + }); + await appendTranscriptEvent(scope, { + type: "session", + version: CURRENT_SESSION_VERSION, + id: sessionId, + timestamp: "2026-06-26T12:00:00.000Z", + cwd: dir, + }); + await appendTranscriptMessage(scope, { + message: { role: "user", content: "compact sqlite rows", timestamp: 1 }, + now: Date.parse("2026-06-26T12:00:01.000Z"), + }); + await appendTranscriptMessage(scope, { + message: { + role: "assistant", + content: "sqlite rows are active", + timestamp: 2, + } as unknown as AssistantMessage, + now: Date.parse("2026-06-26T12:00:02.000Z"), + }); + + const sessionManager = SessionManager.open(marker); + const leafId = requireNonEmptyString(sessionManager.getLeafId(), "SQLite leaf id missing"); + const snapshot = await captureCompactionCheckpointSnapshotAsync({ + sessionManager, + sessionFile: marker, + }); + + expect(fsSync.existsSync(marker)).toBe(false); + expect(fsSync.readdirSync(dir).some((file) => file.endsWith(".jsonl"))).toBe(false); + expect(snapshot).toMatchObject({ + sessionId, + leafId, + entryId: leafId, + }); + expect(await readSessionLeafStateFromTranscriptAsync(marker)).toEqual({ + entryId: leafId, + leafId, + }); + + const compactionId = sessionManager.appendCompaction("sqlite checkpoint summary", leafId, 42, { + manual: true, + }); + expect(await readSessionLeafStateFromTranscriptAsync(marker)).toEqual({ + entryId: compactionId, + leafId: compactionId, + }); + }); + + test("checkpoint store branches and restores SQLite marker checkpoints from rows", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-checkpoint-sqlite-branch-")); + tempDirs.push(dir); + const storePath = path.join(dir, "openclaw-agent.sqlite"); + const sessionId = "sqlite-checkpoint-branch-source"; + const sessionKey = MAIN_SESSION_KEY; + const scope = { + agentId: MAIN_AGENT_ID, + sessionId, + sessionKey, + storePath, + }; + const marker = formatSqliteSessionFileMarker({ + agentId: MAIN_AGENT_ID, + sessionId, + storePath, + }); + + await upsertSessionEntry(scope, { + sessionId, + sessionFile: marker, + updatedAt: Date.now(), + }); + await appendTranscriptEvent(scope, { + type: "session", + version: CURRENT_SESSION_VERSION, + id: sessionId, + timestamp: "2026-06-26T12:00:00.000Z", + cwd: dir, + }); + await appendTranscriptMessage(scope, { + message: { role: "user", content: "branch from sqlite checkpoint", timestamp: 1 }, + now: Date.parse("2026-06-26T12:00:01.000Z"), + }); + await appendTranscriptMessage(scope, { + message: { + role: "assistant", + content: "checkpoint branch source", + timestamp: 2, + } as unknown as AssistantMessage, + now: Date.parse("2026-06-26T12:00:02.000Z"), + }); + const sourceLeafId = requireNonEmptyString( + SessionManager.open(marker).getLeafId(), + "SQLite source leaf id missing", + ); + const checkpoint: SessionCompactionCheckpoint = { + checkpointId: "sqlite-checkpoint-branch", + sessionKey, + sessionId, + createdAt: Date.now(), + reason: "manual", + tokensBefore: 100, + tokensAfter: 40, + preCompaction: { + sessionId, + leafId: sourceLeafId, + entryId: sourceLeafId, + }, + postCompaction: { + sessionId, + leafId: sourceLeafId, + entryId: sourceLeafId, + }, + }; + await upsertSessionEntry(scope, { + sessionId, + sessionFile: marker, + updatedAt: Date.now(), + compactionCheckpoints: [checkpoint], + }); + + const store = createFileBackedCompactionCheckpointStore(); + const branchKey = "agent:main:checkpoint-branch"; + const branched = await store.branchCheckpointSession({ + storePath, + sourceKey: sessionKey, + nextKey: branchKey, + checkpointId: checkpoint.checkpointId, + }); + const restored = await store.restoreCheckpointSession({ + storePath, + sessionKey, + checkpointId: checkpoint.checkpointId, + }); + + if (branched.status !== "created" || restored.status !== "created") { + throw new Error("expected SQLite checkpoint branch and restore"); + } + expect(branched.entry.sessionFile).toContain("sqlite:main:"); + expect(restored.entry.sessionFile).toContain("sqlite:main:"); + expect(fsSync.readdirSync(dir).some((file) => file.endsWith(".jsonl"))).toBe(false); + + const branchEvents = await loadTranscriptEvents({ + agentId: MAIN_AGENT_ID, + sessionId: branched.entry.sessionId, + sessionKey: branchKey, + storePath, + }); + const restoredEvents = await loadTranscriptEvents({ + agentId: MAIN_AGENT_ID, + sessionId: restored.entry.sessionId, + sessionKey, + storePath, + }); + expect( + branchEvents.some((event) => isAssistantTextEvent(event, "checkpoint branch source")), + ).toBe(true); + expect( + restoredEvents.some((event) => isAssistantTextEvent(event, "checkpoint branch source")), + ).toBe(true); + }); + + test("checkpoint store branches row-backed checkpoints when entry sessionFile is stale", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-checkpoint-sqlite-stale-")); + tempDirs.push(dir); + const storePath = path.join(dir, "openclaw-agent.sqlite"); + const sessionId = "sqlite-checkpoint-stale-source"; + const sessionKey = MAIN_SESSION_KEY; + const scope = { + agentId: MAIN_AGENT_ID, + sessionId, + sessionKey, + storePath, + }; + const marker = formatSqliteSessionFileMarker({ + agentId: MAIN_AGENT_ID, + sessionId, + storePath, + }); + const staleSessionFile = path.join(dir, "stale-transcript.jsonl"); + + await upsertSessionEntry(scope, { + sessionId, + sessionFile: staleSessionFile, + updatedAt: Date.now(), + }); + await appendTranscriptEvent(scope, { + type: "session", + version: CURRENT_SESSION_VERSION, + id: sessionId, + timestamp: "2026-06-26T12:00:00.000Z", + cwd: dir, + }); + await appendTranscriptMessage(scope, { + message: { role: "user", content: "stale entry row-backed checkpoint", timestamp: 1 }, + now: Date.parse("2026-06-26T12:00:01.000Z"), + }); + const leafBeforeEntryId = requireNonEmptyString( + SessionManager.open(marker).getLeafId(), + "SQLite stale-entry pre-entry leaf id missing", + ); + await appendTranscriptMessage(scope, { + message: { + role: "assistant", + content: "entry id boundary message", + timestamp: 2, + } as unknown as AssistantMessage, + now: Date.parse("2026-06-26T12:00:02.000Z"), + }); + const sourceEntryId = requireNonEmptyString( + SessionManager.open(marker).getLeafId(), + "SQLite stale-entry entry id missing", + ); + const checkpoint: SessionCompactionCheckpoint = { + checkpointId: "sqlite-checkpoint-stale", + sessionKey, + sessionId, + createdAt: Date.now(), + reason: "manual", + preCompaction: { + sessionId, + leafId: leafBeforeEntryId, + entryId: sourceEntryId, + }, + postCompaction: { + sessionId, + leafId: leafBeforeEntryId, + entryId: sourceEntryId, + }, + }; + const markerCheckpoint: SessionCompactionCheckpoint = { + checkpointId: "sqlite-checkpoint-stale-marker", + sessionKey, + sessionId, + createdAt: Date.now() + 1, + reason: "manual", + preCompaction: { + sessionId, + leafId: leafBeforeEntryId, + }, + postCompaction: { + sessionId, + sessionFile: marker, + leafId: sourceEntryId, + }, + }; + await upsertSessionEntry(scope, { + sessionId, + sessionFile: staleSessionFile, + updatedAt: Date.now(), + compactionCheckpoints: [checkpoint, markerCheckpoint], + }); + + const branchKey = "agent:main:stale-checkpoint-branch"; + const branched = await createFileBackedCompactionCheckpointStore().branchCheckpointSession({ + storePath, + sourceKey: sessionKey, + nextKey: branchKey, + checkpointId: checkpoint.checkpointId, + }); + + if (branched.status !== "created") { + throw new Error("expected stale-entry SQLite checkpoint branch"); + } + expect(branched.entry.sessionFile).toContain("sqlite:main:"); + expect(fsSync.existsSync(staleSessionFile)).toBe(false); + expect(fsSync.readdirSync(dir).some((file) => file.endsWith(".jsonl"))).toBe(false); + const branchEvents = await loadTranscriptEvents({ + agentId: MAIN_AGENT_ID, + sessionId: branched.entry.sessionId, + sessionKey: branchKey, + storePath, + }); + expect( + branchEvents.some((event) => isAssistantTextEvent(event, "entry id boundary message")), + ).toBe(true); + + const markerBranched = + await createFileBackedCompactionCheckpointStore().branchCheckpointSession({ + storePath, + sourceKey: sessionKey, + nextKey: "agent:main:stale-marker-checkpoint-branch", + checkpointId: markerCheckpoint.checkpointId, + }); + if (markerBranched.status !== "created") { + throw new Error("expected stale-entry SQLite marker checkpoint branch"); + } + expect(markerBranched.entry.sessionFile).toContain("sqlite:main:"); + }); + + test("checkpoint store does not fork retired legacy snapshots for SQLite marker entries", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-checkpoint-sqlite-legacy-")); + tempDirs.push(dir); + const storePath = path.join(dir, "openclaw-agent.sqlite"); + const sessionId = "sqlite-checkpoint-legacy-source"; + const sessionKey = MAIN_SESSION_KEY; + const marker = formatSqliteSessionFileMarker({ + agentId: MAIN_AGENT_ID, + sessionId, + storePath, + }); + const legacySnapshotFile = path.join(dir, "legacy.checkpoint.jsonl"); + await fs.writeFile( + legacySnapshotFile, + [ + { + type: "session", + version: CURRENT_SESSION_VERSION, + id: sessionId, + timestamp: "2026-06-26T12:00:00.000Z", + cwd: dir, + }, + { + type: "message", + id: "legacy-leaf", + parentId: null, + timestamp: "2026-06-26T12:00:01.000Z", + message: { role: "assistant", content: "legacy checkpoint source" }, + }, + ] + .map((entry) => JSON.stringify(entry)) + .join("\n") + "\n", + "utf-8", + ); + await upsertSessionEntry( + { + agentId: MAIN_AGENT_ID, + sessionKey, + storePath, + }, + { + sessionId, + sessionFile: marker, + updatedAt: Date.now(), + compactionCheckpoints: [ + { + checkpointId: "legacy-file-checkpoint", + sessionKey, + sessionId, + createdAt: Date.now(), + reason: "manual", + preCompaction: { + sessionId, + sessionFile: legacySnapshotFile, + leafId: "legacy-leaf", + }, + postCompaction: { sessionId }, + } satisfies SessionCompactionCheckpoint, + ], + }, + ); + + const beforeFiles = fsSync.readdirSync(dir).toSorted(); + + const branched = await createFileBackedCompactionCheckpointStore().branchCheckpointSession({ + storePath, + sourceKey: sessionKey, + nextKey: "agent:main:legacy-checkpoint-branch", + checkpointId: "legacy-file-checkpoint", + }); + + expect(branched.status).toBe("missing-boundary"); + expect(fsSync.readdirSync(dir).toSorted()).toEqual(beforeFiles); + }); + test("async capture derives session metadata without synchronous SessionManager.open", async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-checkpoint-async-metadata-")); tempDirs.push(dir); @@ -659,7 +1061,7 @@ describe("session-compaction-checkpoints", () => { const sessionFile = requireNonEmptyString(session.getSessionFile(), "session file missing"); const storePath = path.join(dir, "sessions.json"); const managedAt = Date.now() - 2; - await writeSessionStore(storePath, MAIN_SESSION_KEY, { + await writeAccessorSessionEntry(storePath, MAIN_SESSION_KEY, { sessionId: "current-session", sessionFile, updatedAt: Date.now() - 1, @@ -717,11 +1119,9 @@ describe("session-compaction-checkpoints", () => { expect(messages.map((message) => (message as { content?: unknown }).content)).toEqual([ "checkpoint source", ]); - const nextStore = await readSessionStore<{ sessionFile?: string; totalTokens?: number }>( - storePath, - ); - expect(nextStore[MAIN_SESSION_KEY]?.sessionFile).toBe(restored.entry.sessionFile); - expect(nextStore[MAIN_SESSION_KEY]?.totalTokens).toBe(45); + const nextEntry = loadSessionEntry({ storePath, sessionKey: MAIN_SESSION_KEY }); + expect(nextEntry?.sessionFile).toBe(restored.entry.sessionFile); + expect(nextEntry?.totalTokens).toBe(45); }); test("file-backed checkpoint store rejects identity changes for model-selection-locked sessions", async () => { @@ -740,27 +1140,30 @@ describe("session-compaction-checkpoints", () => { ); const sessionFile = requireNonEmptyString(session.getSessionFile(), "session file missing"); const storePath = path.join(dir, "sessions.json"); - await writeSessionStore(storePath, MAIN_SESSION_KEY, { - sessionId: "locked-session", - sessionFile, - updatedAt: Date.now(), - modelSelectionLocked: true, - compactionCheckpoints: [ - { - checkpointId: "checkpoint-locked", - sessionKey: MAIN_SESSION_KEY, - sessionId: "locked-session", - createdAt: Date.now(), - reason: "manual", - preCompaction: { sessionId: "locked-session", leafId: checkpointLeafId }, - postCompaction: { + await upsertSessionEntry( + { storePath, sessionKey: MAIN_SESSION_KEY }, + { + sessionId: "locked-session", + sessionFile, + updatedAt: Date.now(), + modelSelectionLocked: true, + compactionCheckpoints: [ + { + checkpointId: "checkpoint-locked", + sessionKey: MAIN_SESSION_KEY, sessionId: "locked-session", - sessionFile, - leafId: checkpointLeafId, + createdAt: Date.now(), + reason: "manual", + preCompaction: { sessionId: "locked-session", leafId: checkpointLeafId }, + postCompaction: { + sessionId: "locked-session", + sessionFile, + leafId: checkpointLeafId, + }, }, - }, - ], - }); + ], + }, + ); const filesBefore = (await fs.readdir(dir)).toSorted(); const store = createFileBackedCompactionCheckpointStore(); @@ -781,12 +1184,12 @@ describe("session-compaction-checkpoints", () => { ).resolves.toEqual({ status: "model-selection-locked" }); expect((await fs.readdir(dir)).toSorted()).toEqual(filesBefore); - expect(await readSessionStore(storePath)).toEqual({ - [MAIN_SESSION_KEY]: expect.objectContaining({ + expect(loadSessionEntry({ storePath, sessionKey: MAIN_SESSION_KEY })).toEqual( + expect.objectContaining({ modelSelectionLocked: true, sessionId: "locked-session", }), - }); + ); }); test("async fork migrates legacy checkpoint snapshots before writing a current header", async () => { @@ -958,7 +1361,7 @@ describe("session-compaction-checkpoints", () => { await fs.writeFile(sessionFile, `checkpoint ${index}`, "utf-8"); }, }); - await writeSessionStore(storePath, sessionKey, { + await writeAccessorSessionEntry(storePath, sessionKey, { sessionId, updatedAt: now, compactionCheckpoints: existingCheckpoints, @@ -993,7 +1396,7 @@ describe("session-compaction-checkpoints", () => { const { storePath, sessionId, sessionKey, now } = await makeTempSessionStore( "openclaw-checkpoint-no-leaf-", ); - await writeSessionStore(storePath, sessionKey, { + await writeAccessorSessionEntry(storePath, sessionKey, { sessionId, updatedAt: now, }); @@ -1009,15 +1412,15 @@ describe("session-compaction-checkpoints", () => { }); expect(stored).toBeNull(); - const nextStore = await readSessionStore<{ compactionCheckpoints?: unknown[] }>(storePath); - expect(nextStore[MAIN_SESSION_KEY]?.compactionCheckpoints).toBeUndefined(); + const nextEntry = loadSessionEntry({ storePath, sessionKey: MAIN_SESSION_KEY }); + expect(nextEntry?.compactionCheckpoints).toBeUndefined(); }); test("persist skips malformed session rows without synthesizing a session id", async () => { const { storePath, sessionId, sessionKey, now } = await makeTempSessionStore( "openclaw-checkpoint-malformed-row-", ); - await writeSessionStore(storePath, sessionKey, { + await writeAccessorSessionEntry(storePath, sessionKey, { sessionId: "", updatedAt: now, }); @@ -1034,14 +1437,11 @@ describe("session-compaction-checkpoints", () => { }); expect(stored).toBeNull(); - const nextStore = await readSessionStore<{ - compactionCheckpoints?: unknown[]; - sessionId?: string; - }>(storePath); - expect(nextStore[MAIN_SESSION_KEY]).toEqual({ + const nextEntry = loadSessionEntry({ storePath, sessionKey: MAIN_SESSION_KEY }); + expect(nextEntry).toMatchObject({ sessionId: "", - updatedAt: now, }); + expect(nextEntry?.compactionCheckpoints).toBeUndefined(); }); test("persist trims retained checkpoint snapshots by total byte budget", async () => { @@ -1060,7 +1460,7 @@ describe("session-compaction-checkpoints", () => { await fs.truncate(sessionFile, checkpointSize); }, }); - await writeSessionStore(storePath, sessionKey, { + await writeAccessorSessionEntry(storePath, sessionKey, { sessionId, updatedAt: now, compactionCheckpoints: existingCheckpoints, diff --git a/src/gateway/session-compaction-checkpoints.ts b/src/gateway/session-compaction-checkpoints.ts index 8fe1729579cd..34ffa3bd0db8 100644 --- a/src/gateway/session-compaction-checkpoints.ts +++ b/src/gateway/session-compaction-checkpoints.ts @@ -17,10 +17,17 @@ import { isCompactionCheckpointTranscriptFileName } from "../config/sessions/art import { readFileRangeAsync } from "../config/sessions/file-range.js"; import { branchSessionFromCompactionCheckpoint, + loadSessionEntry, + loadTranscriptEventsSync, restoreSessionFromCompactionCheckpoint, type SessionCompactionCheckpointMutationResult, updateSessionEntry, } from "../config/sessions/session-accessor.js"; +import { + branchSqliteCompactionCheckpointSession, + restoreSqliteCompactionCheckpointSession, +} from "../config/sessions/session-accessor.sqlite.js"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { streamSessionTranscriptLines } from "../config/sessions/transcript-stream.js"; import { scanSessionTranscriptTree } from "../config/sessions/transcript-tree.js"; import { CURRENT_SESSION_VERSION } from "../config/sessions/version.js"; @@ -45,6 +52,9 @@ type SessionLeafState = { entryId: string; }; +type SessionManagerCheckpointView = Pick<SessionManager, "getLeafId"> & + Partial<Pick<SessionManager, "getEntries" | "getSessionId">>; + export function resolveCompactionCheckpointTranscriptPosition(params: { preferredLeafId?: string | null; transcriptState?: SessionLeafState | null; @@ -74,6 +84,7 @@ type CompactionCheckpointTranscriptForkResult = export type CompactionCheckpointSessionMutationResult = SessionCompactionCheckpointMutationResult; export type BranchCheckpointSessionParams = { + agentId?: string; storePath: string; sourceKey: string; sourceStoreKey?: string; @@ -82,6 +93,7 @@ export type BranchCheckpointSessionParams = { }; export type RestoreCheckpointSessionParams = { + agentId?: string; storePath: string; sessionKey: string; sessionStoreKey?: string; @@ -347,6 +359,15 @@ export async function readSessionLeafStateFromTranscriptAsync( sessionFile: string, maxBytes = MAX_COMPACTION_CHECKPOINT_LEAF_SCAN_BYTES, ): Promise<{ entryId: string; leafId: string | null } | null> { + const sqliteMarker = parseSqliteSessionFileMarker(sessionFile); + if (sqliteMarker) { + const records = loadTranscriptEventsSync(sqliteMarker).filter( + (event): event is Record<string, unknown> => + Boolean(event) && typeof event === "object" && !Array.isArray(event), + ); + return readSessionLeafStateFromRecords(records); + } + let fileHandle: Awaited<ReturnType<typeof fs.open>> | undefined; try { fileHandle = await fs.open(sessionFile, "r"); @@ -415,6 +436,26 @@ export async function readSessionLeafStateFromTranscriptAsync( return null; } +function readSessionLeafStateFromRecords( + records: readonly Record<string, unknown>[], +): { entryId: string; leafId: string | null } | null { + let latestEntryId: string | undefined; + for (const record of records) { + if (record.type === "session") { + continue; + } + const entryId = typeof record.id === "string" ? record.id.trim() : ""; + if (entryId) { + latestEntryId = entryId; + } + } + if (!latestEntryId) { + return null; + } + const tree = scanSessionTranscriptTree(records); + return { entryId: latestEntryId, leafId: tree.leafId }; +} + export async function forkCompactionCheckpointTranscriptAsync(params: { sourceFile: string; sourceLeafId?: string; @@ -581,6 +622,23 @@ function cloneCheckpointSessionEntry(params: { async function branchCheckpointSessionFromStoredBoundary( params: BranchCheckpointSessionParams, ): Promise<CompactionCheckpointSessionMutationResult> { + if ( + shouldRouteCheckpointSessionMutationToSqlite({ + checkpointId: params.checkpointId, + ...(params.agentId ? { agentId: params.agentId } : {}), + sessionKey: params.sourceStoreKey ?? params.sourceKey, + storePath: params.storePath, + }) + ) { + return await branchSqliteCompactionCheckpointSession({ + ...(params.agentId ? { agentId: params.agentId } : {}), + storePath: params.storePath, + sourceKey: params.sourceKey, + nextKey: params.nextKey, + checkpointId: params.checkpointId, + ...(params.sourceStoreKey ? { sourceStoreKey: params.sourceStoreKey } : {}), + }); + } return await branchSessionFromCompactionCheckpoint({ storePath: params.storePath, sourceKey: params.sourceKey, @@ -608,6 +666,22 @@ async function branchCheckpointSessionFromStoredBoundary( async function restoreCheckpointSessionFromStoredBoundary( params: RestoreCheckpointSessionParams, ): Promise<CompactionCheckpointSessionMutationResult> { + if ( + shouldRouteCheckpointSessionMutationToSqlite({ + checkpointId: params.checkpointId, + ...(params.agentId ? { agentId: params.agentId } : {}), + sessionKey: params.sessionStoreKey ?? params.sessionKey, + storePath: params.storePath, + }) + ) { + return await restoreSqliteCompactionCheckpointSession({ + ...(params.agentId ? { agentId: params.agentId } : {}), + storePath: params.storePath, + sessionKey: params.sessionKey, + checkpointId: params.checkpointId, + ...(params.sessionStoreKey ? { sessionStoreKey: params.sessionStoreKey } : {}), + }); + } return await restoreSessionFromCompactionCheckpoint({ storePath: params.storePath, sessionKey: params.sessionKey, @@ -627,6 +701,41 @@ async function restoreCheckpointSessionFromStoredBoundary( }); } +function shouldRouteCheckpointSessionMutationToSqlite(params: { + agentId?: string; + checkpointId: string; + sessionKey: string; + storePath: string; +}): boolean { + const entry = loadSessionEntry({ + ...(params.agentId ? { agentId: params.agentId } : {}), + sessionKey: params.sessionKey, + storePath: params.storePath, + }); + if (!entry) { + return false; + } + if (parseSqliteSessionFileMarker(entry.sessionFile)) { + return true; + } + const checkpoint = entry.compactionCheckpoints?.find( + (candidate) => candidate.checkpointId === params.checkpointId, + ); + const preCheckpointFile = checkpoint?.preCompaction.sessionFile?.trim(); + const postCheckpointFile = checkpoint?.postCompaction.sessionFile?.trim(); + if ( + parseSqliteSessionFileMarker(preCheckpointFile) || + parseSqliteSessionFileMarker(postCheckpointFile) + ) { + return true; + } + const hasCheckpointFile = Boolean(preCheckpointFile) || Boolean(postCheckpointFile); + const hasCheckpointRowBoundary = + Boolean(checkpoint?.preCompaction.entryId?.trim()) || + Boolean(checkpoint?.postCompaction.entryId?.trim()); + return hasCheckpointRowBoundary && !hasCheckpointFile; +} + /** * Creates the current file-backed compaction checkpoint domain store. * @@ -650,7 +759,7 @@ export function createFileBackedCompactionCheckpointStore(): CompactionCheckpoin * checkpoints that already have a snapshot file keep working. */ export async function captureCompactionCheckpointSnapshotAsync(params: { - sessionManager?: Pick<SessionManager, "getLeafId">; + sessionManager?: SessionManagerCheckpointView; sessionFile: string; maxBytes?: number; }): Promise<CapturedCompactionCheckpointSnapshot | null> { @@ -667,6 +776,30 @@ export async function captureCompactionCheckpointSnapshotAsync(params: { return null; } const maxBytes = params.maxBytes ?? MAX_COMPACTION_CHECKPOINT_LEAF_SCAN_BYTES; + const sqliteMarker = parseSqliteSessionFileMarker(sessionFile); + if (sqliteMarker) { + if (typeof params.sessionManager?.getEntries !== "function") { + return null; + } + const entryRecords = params.sessionManager.getEntries() as unknown as Record<string, unknown>[]; + const transcriptState = readSessionLeafStateFromRecords(entryRecords); + const position = resolveCompactionCheckpointTranscriptPosition({ + preferredLeafId: liveLeafId, + transcriptState, + }); + const leafId = position.leafId; + if (!leafId) { + return null; + } + return { + sessionId: + typeof params.sessionManager.getSessionId === "function" + ? params.sessionManager.getSessionId() + : sqliteMarker.sessionId, + leafId, + ...(position.entryId ? { entryId: position.entryId } : {}), + }; + } const sessionId = await readSessionIdFromTranscriptHeaderAsync(sessionFile); const transcriptState = await readSessionLeafStateFromTranscriptAsync(sessionFile, maxBytes); const position = resolveCompactionCheckpointTranscriptPosition({ @@ -736,8 +869,12 @@ export async function persistSessionCompactionCheckpoint( ): Promise<SessionCompactionCheckpoint | null> { const snapshotSessionFile = params.snapshot.sessionFile?.trim(); const postSessionFile = params.postSessionFile?.trim(); + const snapshotSqliteMarker = parseSqliteSessionFileMarker(snapshotSessionFile); + const postSqliteMarker = parseSqliteSessionFileMarker(postSessionFile); + const snapshotArtifactFile = snapshotSqliteMarker ? undefined : snapshotSessionFile; + const postArtifactFile = postSqliteMarker ? undefined : postSessionFile; const postSourceLeafId = params.postEntryId?.trim() || params.postLeafId?.trim(); - if (!snapshotSessionFile && (!postSessionFile || !postSourceLeafId)) { + if (!snapshotArtifactFile && !postSourceLeafId) { log.warn("skipping compaction checkpoint persist: missing stable fork source", { sessionKey: params.sessionKey, }); @@ -763,15 +900,13 @@ export async function persistSessionCompactionCheckpoint( : {}), preCompaction: { sessionId: params.snapshot.sessionId, - ...(params.snapshot.sessionFile?.trim() - ? { sessionFile: params.snapshot.sessionFile.trim() } - : {}), + ...(snapshotArtifactFile ? { sessionFile: snapshotArtifactFile } : {}), leafId: params.snapshot.leafId, ...(params.snapshot.entryId?.trim() ? { entryId: params.snapshot.entryId.trim() } : {}), }, postCompaction: { sessionId: params.sessionId, - ...(postSessionFile ? { sessionFile: postSessionFile } : {}), + ...(postArtifactFile ? { sessionFile: postArtifactFile } : {}), ...(params.postLeafId?.trim() ? { leafId: params.postLeafId.trim() } : {}), ...(params.postEntryId?.trim() ? { entryId: params.postEntryId.trim() } : {}), }, @@ -811,7 +946,7 @@ export async function persistSessionCompactionCheckpoint( }); return null; } - const checkpointArtifactFile = snapshotSessionFile || postSessionFile || ""; + const checkpointArtifactFile = snapshotArtifactFile || postArtifactFile || ""; await cleanupTrimmedCompactionCheckpointFiles({ removed: trimmedCheckpoints?.removed ?? [], retained: trimmedCheckpoints?.kept, diff --git a/src/gateway/session-groups.test.ts b/src/gateway/session-groups.test.ts index 820f81d715a3..2012752fe0c2 100644 --- a/src/gateway/session-groups.test.ts +++ b/src/gateway/session-groups.test.ts @@ -2,7 +2,10 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import type { SessionEntry } from "../config/sessions.js"; +import { loadSessionEntry, replaceSessionEntry } from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { deleteSessionGroup, @@ -24,14 +27,16 @@ describe("session groups catalog", () => { }); afterEach(async () => { + closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); await fs.rm(root, { recursive: true, force: true }); }); - async function seedSessionStore(entries: Record<string, unknown>): Promise<string> { + async function seedSessionStore(entries: Record<string, SessionEntry>): Promise<string> { const storePath = path.join(root, "agents", "main", "sessions", "sessions.json"); - await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile(storePath, JSON.stringify(entries)); + for (const [sessionKey, entry] of Object.entries(entries)) { + await replaceSessionEntry({ agentId: "main", storePath, sessionKey }, entry); + } return storePath; } @@ -70,13 +75,19 @@ describe("session groups catalog", () => { expect(result.updatedSessions).toBe(1); expect(result.groups.map((group) => group.name)).toEqual(["New", "Other"]); - const store = JSON.parse(await fs.readFile(storePath, "utf8")) as Record< - string, - { category?: string; updatedAt: number } - >; - expect(store["agent:main:dashboard:a"].category).toBe("New"); - expect(store["agent:main:dashboard:a"].updatedAt).toBe(updatedAtA); - expect(store["agent:main:dashboard:b"].category).toBe("Other"); + const sessionA = loadSessionEntry({ + agentId: "main", + storePath, + sessionKey: "agent:main:dashboard:a", + }); + const sessionB = loadSessionEntry({ + agentId: "main", + storePath, + sessionKey: "agent:main:dashboard:b", + }); + expect(sessionA?.category).toBe("New"); + expect(sessionA?.updatedAt).toBe(updatedAtA); + expect(sessionB?.category).toBe("Other"); }); it("deletes a group and clears member categories", async () => { @@ -89,11 +100,13 @@ describe("session groups catalog", () => { expect(result.updatedSessions).toBe(1); expect(result.groups).toEqual([]); - const store = JSON.parse(await fs.readFile(storePath, "utf8")) as Record< - string, - { category?: string } - >; - expect(store["agent:main:dashboard:a"].category).toBeUndefined(); + expect( + loadSessionEntry({ + agentId: "main", + storePath, + sessionKey: "agent:main:dashboard:a", + })?.category, + ).toBeUndefined(); }); it("merges a rename into an existing target group", async () => { diff --git a/src/gateway/session-lifecycle-state.test.ts b/src/gateway/session-lifecycle-state.test.ts index 9800264cdd72..a11328d93811 100644 --- a/src/gateway/session-lifecycle-state.test.ts +++ b/src/gateway/session-lifecycle-state.test.ts @@ -6,11 +6,11 @@ import type { SessionEntry } from "../config/sessions.js"; const persistenceMocks = vi.hoisted(() => ({ loadSessionEntry: vi.fn(), - updateSessionStoreEntry: vi.fn(), + updateSessionEntry: vi.fn(), })); -vi.mock("../config/sessions.js", () => ({ - updateSessionStoreEntry: persistenceMocks.updateSessionStoreEntry, +vi.mock("../config/sessions/session-accessor.js", () => ({ + updateSessionEntry: persistenceMocks.updateSessionEntry, })); vi.mock("./session-utils.js", () => ({ @@ -28,9 +28,8 @@ type PersistedLifecycleInput = Parameters<typeof derivePersistedSessionLifecycle type PersistedLifecycleData = PersistedLifecycleInput["event"]["data"]; type PersistedLifecyclePatch = NonNullable<ReturnType<typeof derivePersistedSessionLifecyclePatch>>; type PersistedLifecycleStatus = PersistedLifecyclePatch["status"]; -type UpdateSessionStoreEntryParams = Parameters< - typeof import("../config/sessions.js").updateSessionStoreEntry ->[0]; +type UpdateSessionEntry = + typeof import("../config/sessions/session-accessor.js").updateSessionEntry; type PersistedLifecycleCase = { name: string; @@ -108,10 +107,11 @@ async function persistExactCronLifecycle(options: { canonicalKey: exactCronSessionKey, entry: currentEntry, }); - persistenceMocks.updateSessionStoreEntry + persistenceMocks.updateSessionEntry .mockReset() - .mockImplementation(async (params: UpdateSessionStoreEntryParams) => { - const patch = await params.update(structuredClone(currentEntry)); + .mockImplementation(async (...args: Parameters<UpdateSessionEntry>) => { + const [, update] = args; + const patch = await update(structuredClone(currentEntry)); if (patch) { currentEntry = { ...currentEntry, ...patch }; } @@ -532,9 +532,11 @@ describe("session lifecycle state", () => { expect(persisted?.status).toBe(testCase.expectedStatus); // One exact-row write only. Continuation settlement owns base projection. - expect(persistenceMocks.updateSessionStoreEntry).toHaveBeenCalledTimes(1); - expect(persistenceMocks.updateSessionStoreEntry.mock.calls[0]?.[0]).toMatchObject({ + expect(persistenceMocks.updateSessionEntry).toHaveBeenCalledTimes(1); + expect(persistenceMocks.updateSessionEntry.mock.calls[0]?.[0]).toMatchObject({ sessionKey: exactCronSessionKey, + }); + expect(persistenceMocks.updateSessionEntry.mock.calls[0]?.[2]).toMatchObject({ requireWriteSuccess: true, }); }); diff --git a/src/gateway/session-lifecycle-state.ts b/src/gateway/session-lifecycle-state.ts index f3f3c92b50b9..1d563f08503d 100644 --- a/src/gateway/session-lifecycle-state.ts +++ b/src/gateway/session-lifecycle-state.ts @@ -4,7 +4,8 @@ import { buildAgentRunTerminalOutcome, type AgentRunTerminalOutcome, } from "../agents/agent-run-terminal-outcome.js"; -import { updateSessionStoreEntry, type SessionEntry } from "../config/sessions.js"; +import type { SessionEntry } from "../config/sessions.js"; +import { updateSessionEntry } from "../config/sessions/session-accessor.js"; import type { AgentEventPayload } from "../infra/agent-events.js"; import { parseCronRunScopeSuffix } from "../sessions/session-key-utils.js"; import { loadSessionEntry } from "./session-utils.js"; @@ -290,13 +291,12 @@ export async function persistGatewaySessionLifecycleEvent(params: { : undefined; const exactCronRun = parseCronRunScopeSuffix(sessionEntry.canonicalKey).runId !== undefined; - await updateSessionStoreEntry({ - storePath: sessionEntry.storePath, - sessionKey: sessionEntry.canonicalKey, - skipMaintenance: true, - takeCacheOwnership: true, - requireWriteSuccess: true, - update: async (entry) => { + await updateSessionEntry( + { + storePath: sessionEntry.storePath, + sessionKey: sessionEntry.canonicalKey, + }, + async (entry) => { if ( exactCronRun && !acceptsCronRunContinuationLifecycleEvent({ entry, event: params.event }) @@ -317,5 +317,10 @@ export async function persistGatewaySessionLifecycleEvent(params: { }); return Object.keys(patch).length > 0 ? patch : null; }, - }); + { + skipMaintenance: true, + takeCacheOwnership: true, + requireWriteSuccess: true, + }, + ); } diff --git a/src/gateway/session-message-events.test.ts b/src/gateway/session-message-events.test.ts index f8b6fd4f306c..a7b3545a0317 100644 --- a/src/gateway/session-message-events.test.ts +++ b/src/gateway/session-message-events.test.ts @@ -5,6 +5,10 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vitest"; +import { + loadTranscriptEvents, + persistSessionTranscriptTurn, +} from "../config/sessions/session-accessor.js"; import { appendAssistantMessageToSessionTranscript } from "../config/sessions/transcript.js"; import { emitSessionLifecycleEvent } from "../sessions/session-lifecycle-events.js"; import * as transcriptEvents from "../sessions/transcript-events.js"; @@ -322,8 +326,21 @@ describe("session.message websocket events", () => { role: "assistant", content: [{ type: "text", text: "live websocket message" }], }); - const transcript = await fs.readFile(appended.sessionFile, "utf-8"); - expect(transcript).toContain('"live websocket message"'); + await expect( + loadTranscriptEvents({ + agentId: "main", + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + }), + ).resolves.toContainEqual( + expect.objectContaining({ + message: expect.objectContaining({ + content: [{ type: "text", text: "live websocket message" }], + }), + type: "message", + }), + ); } finally { emitSpy.mockRestore(); } @@ -577,7 +594,6 @@ describe("session.message websocket events", () => { }, storePath, }); - const transcriptPath = path.join(path.dirname(storePath), "sess-main.jsonl"); const transcriptMessage = { role: "assistant", content: [{ type: "text", text: "usage snapshot" }], @@ -592,20 +608,24 @@ describe("session.message websocket events", () => { }, timestamp: Date.now(), }; - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ type: "session", version: 1, id: "sess-main" }), - JSON.stringify({ id: "msg-usage", message: transcriptMessage }), - ].join("\n"), - "utf-8", + const turn = await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + }, + { + messages: [{ message: transcriptMessage }], + updateMode: "none", + }, ); await withOperatorSessionSubscriber(async (ws) => { const { messageEvent } = await emitTranscriptUpdateAndCollectMessageEvent({ ws, sessionKey: "agent:main:main", - sessionFile: transcriptPath, + sessionFile: turn.sessionFile, message: transcriptMessage, messageId: "msg-usage", }); @@ -661,12 +681,10 @@ describe("session.message websocket events", () => { test("derives message sequence for selected-session transcript subscribers", async () => { const storePath = await createSessionStoreFile(); - const transcriptPath = path.join(path.dirname(storePath), "selected-session.jsonl"); await writeSessionStore({ entries: { main: { sessionId: "sess-main", - sessionFile: transcriptPath, updatedAt: Date.now(), }, }, @@ -677,13 +695,17 @@ describe("session.message websocket events", () => { content: [{ type: "text", text: "early selected prompt" }], timestamp: Date.now(), }; - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ type: "session", version: 1, id: "sess-main" }), - JSON.stringify({ id: "msg-selected", message: transcriptMessage }), - ].join("\n"), - "utf-8", + const turn = await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId: "sess-main", + sessionKey: "agent:main:main", + storePath, + }, + { + messages: [{ message: transcriptMessage }], + updateMode: "none", + }, ); const ws = await harness.openWs(); @@ -697,7 +719,7 @@ describe("session.message websocket events", () => { const messageEventPromise = waitForSessionMessageEvent(ws, "agent:main:main"); emitSessionTranscriptUpdate({ - sessionFile: transcriptPath, + sessionFile: turn.sessionFile, sessionKey: "agent:main:main", message: transcriptMessage, messageId: "msg-selected", @@ -1188,50 +1210,45 @@ describe("session.message websocket events", () => { } }); - test("routes transcript-only updates to the freshest session owner when different sessionIds share a transcript path", async () => { + test("routes transcript-only SQLite marker updates to the matching session owner", async () => { const storePath = await createSessionStoreFile(); - const transcriptPath = path.join(path.dirname(storePath), "shared.jsonl"); await writeSessionStore({ entries: { older: { sessionId: "sess-old", - sessionFile: transcriptPath, updatedAt: Date.now(), }, newer: { sessionId: "sess-new", - sessionFile: transcriptPath, updatedAt: Date.now() + 10, }, }, storePath, }); - await fs.writeFile( - transcriptPath, - [ - JSON.stringify({ type: "session", version: 1, id: "sess-new" }), - JSON.stringify({ - id: "msg-shared", - message: { - role: "assistant", - content: [{ type: "text", text: "shared transcript update" }], - timestamp: Date.now(), - }, - }), - ].join("\n"), - "utf-8", + const message = { + role: "assistant", + content: [{ type: "text", text: "shared transcript update" }], + timestamp: Date.now(), + }; + const turn = await persistSessionTranscriptTurn( + { + agentId: "main", + sessionId: "sess-new", + sessionKey: "agent:main:newer", + storePath, + }, + { + messages: [{ message }], + updateMode: "none", + }, ); await withOperatorSessionSubscriber(async (ws) => { const messageEventPromise = waitForSessionMessageEvent(ws, "agent:main:newer"); emitSessionTranscriptUpdate({ - sessionFile: transcriptPath, - message: { - role: "assistant", - content: [{ type: "text", text: "shared transcript update" }], - timestamp: Date.now(), - }, + sessionFile: turn.sessionFile, + message, messageId: "msg-shared", }); diff --git a/src/gateway/session-reset-service.ts b/src/gateway/session-reset-service.ts index abf3f575de7e..41895ca34c90 100644 --- a/src/gateway/session-reset-service.ts +++ b/src/gateway/session-reset-service.ts @@ -1,7 +1,6 @@ // Gateway session reset/delete service. // Rotates transcripts and coordinates lifecycle cleanup across runtimes/hooks. import { randomUUID } from "node:crypto"; -import path from "node:path"; import { ErrorCodes, errorShape } from "../../packages/gateway-protocol/src/index.js"; import { getAcpSessionManager } from "../acp/control-plane/manager.js"; import { getAcpRuntimeBackend } from "../acp/runtime/registry.js"; @@ -36,12 +35,8 @@ import { resetSessionEntryLifecycle, } from "../config/sessions.js"; import { rebindCliSessionReseedReceiptsForReset } from "../config/sessions/cli-session-binding.js"; -import { resolveSessionFilePath, resolveSessionFilePathOptions } from "../config/sessions/paths.js"; import { resolveResetPreservedSelection } from "../config/sessions/reset-preserved-selection.js"; -import { - canonicalizeAbsoluteSessionFilePath, - rewriteSessionFileForNewSessionId, -} from "../config/sessions/session-file-rotation.js"; +import { formatSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import type { SessionAcpMeta } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { logVerbose } from "../globals.js"; @@ -77,7 +72,6 @@ import { import { findDirectChildSessionsForParent } from "./session-child-sessions.js"; import { archiveSessionTranscriptsDetailed, - extractGeneratedTranscriptSessionId, resolveStableSessionEndTranscript, type ArchivedSessionTranscript, } from "./session-transcript-files.fs.js"; @@ -90,41 +84,6 @@ import { const ACP_RUNTIME_CLEANUP_TIMEOUT_MS = 15_000; -function resolveResetSessionFile(params: { - nextSessionId: string; - currentEntry?: SessionEntry; - storePath: string; - agentId: string; -}): string { - const currentEntry = params.currentEntry; - // Rotate generated transcript names by the file's *embedded* id, not the logical - // session id: a post-upgrade sessionFile can embed a stale id, so keying off - // currentEntry.sessionId would orphan the reset session on the old file. Explicit - // custom placements have no embedded id and stay preserved. - const rotationPreviousSessionId = - extractGeneratedTranscriptSessionId(currentEntry?.sessionFile) ?? currentEntry?.sessionId; - const rewrittenSessionFile = rotationPreviousSessionId - ? rewriteSessionFileForNewSessionId({ - sessionFile: currentEntry?.sessionFile, - previousSessionId: rotationPreviousSessionId, - nextSessionId: params.nextSessionId, - }) - : undefined; - const normalizedRewrittenSessionFile = - rewrittenSessionFile && path.isAbsolute(rewrittenSessionFile) - ? canonicalizeAbsoluteSessionFilePath(rewrittenSessionFile) - : rewrittenSessionFile; - const preservedSessionFile = normalizedRewrittenSessionFile ?? currentEntry?.sessionFile; - return resolveSessionFilePath( - params.nextSessionId, - preservedSessionFile ? { sessionFile: preservedSessionFile } : undefined, - resolveSessionFilePathOptions({ - storePath: params.storePath, - agentId: params.agentId, - }), - ); -} - export function archiveSessionTranscriptsForSessionDetailed(params: { sessionId: string | undefined; storePath: string; @@ -802,6 +761,7 @@ export async function cleanupSessionBeforeMutation(params: { export async function emitGatewayBeforeResetPluginHook(params: { cfg: OpenClawConfig; key: string; + messages?: unknown[]; target: ReturnType<typeof resolveGatewaySessionStoreTarget>; storePath: string; entry?: SessionEntry; @@ -817,28 +777,15 @@ export async function emitGatewayBeforeResetPluginHook(params: { const sessionFile = params.entry?.sessionFile; const agentId = normalizeAgentId(params.target.agentId ?? resolveDefaultAgentId(params.cfg)); const workspaceDir = resolveAgentWorkspaceDir(params.cfg, agentId); - let messages: unknown[] = []; - try { - if (typeof sessionId === "string" && sessionId.trim().length > 0) { - messages = await readSessionMessagesAsync( - { - agentId, - sessionEntry: params.entry, - sessionId, - sessionKey, - storePath: params.storePath, - }, - { - mode: "full", - reason: "before_reset hook payload", - }, - ); - } - } catch (err) { - logVerbose( - `before_reset: failed to read session messages for ${sessionId ?? "(none)"}; firing hook with empty messages (${String(err)})`, - ); - } + const messages = + params.messages ?? + (await readGatewayBeforeResetPluginHookMessages({ + agentId, + entry: params.entry, + sessionId, + sessionKey, + storePath: params.storePath, + })); void hookRunner .runBeforeReset( @@ -859,6 +806,38 @@ export async function emitGatewayBeforeResetPluginHook(params: { }); } +async function readGatewayBeforeResetPluginHookMessages(params: { + agentId: string; + entry?: SessionEntry; + sessionId?: string; + sessionKey: string; + storePath: string; +}): Promise<unknown[]> { + if (typeof params.sessionId !== "string" || params.sessionId.trim().length === 0) { + return []; + } + try { + return await readSessionMessagesAsync( + { + agentId: params.agentId, + sessionEntry: params.entry, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }, + { + mode: "full", + reason: "before_reset hook payload", + }, + ); + } catch (err) { + logVerbose( + `before_reset: failed to read session messages for ${params.sessionId}; firing hook with empty messages (${String(err)})`, + ); + return []; + } +} + export async function performGatewaySessionReset(params: { key: string; agentId?: string; @@ -1089,6 +1068,17 @@ export async function performGatewaySessionReset(params: { reason: "reset", }); } + const beforeResetMessages = getGlobalHookRunner()?.hasHooks("before_reset") + ? await readGatewayBeforeResetPluginHookMessages({ + agentId: normalizeAgentId( + target.agentId ?? requestedAgentId ?? resolveDefaultAgentId(cfg), + ), + entry, + sessionId: entry?.sessionId, + sessionKey: target.canonicalKey ?? params.key, + storePath, + }) + : undefined; const lifecycle = await resetSessionEntryLifecycle({ agentId: target.agentId, @@ -1112,11 +1102,10 @@ export async function performGatewaySessionReset(params: { }); const now = Date.now(); const nextSessionId = randomUUID(); - const sessionFile = resolveResetSessionFile({ - nextSessionId, - currentEntry, - storePath, + const sessionFile = formatSqliteSessionFileMarker({ agentId: sessionAgentId, + sessionId: nextSessionId, + storePath, }); const nextEntry: SessionEntry = { sessionId: nextSessionId, @@ -1216,7 +1205,7 @@ export async function performGatewaySessionReset(params: { sessionKey: deferredAcpResetState.sessionKey, meta: buildPendingAcpMeta(deferredAcpResetState.meta, Date.now()), }; - // The JSON session rotation and SQLite metadata cannot share a transaction. + // Session row rotation and ACP metadata cannot share a transaction. // Bind captured ACP state before acknowledging the committed reset so the // new session never observes an unreadable old-session row. writeAcpSessionMetaForMigration({ @@ -1246,6 +1235,7 @@ export async function performGatewaySessionReset(params: { await emitGatewayBeforeResetPluginHook({ cfg, key: params.key, + messages: beforeResetMessages, target, storePath, entry: mutation.previousEntry, diff --git a/src/gateway/session-store-key.ts b/src/gateway/session-store-key.ts index 9f59685e3354..ba464a42c22b 100644 --- a/src/gateway/session-store-key.ts +++ b/src/gateway/session-store-key.ts @@ -6,6 +6,7 @@ import { import { listAgentIds, resolveDefaultAgentId } from "../agents/agent-scope.js"; import { canonicalizeMainSessionAlias, + resolveAgentMainSessionKey, resolveMainSessionKey, } from "../config/sessions/main-session.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; @@ -103,10 +104,14 @@ export function resolveSessionStoreKey(params: { const lowered = normalizeLowercaseStringOrEmpty(raw); const rawMainKey = normalizeMainKey(params.cfg.session?.mainKey); + const storeAgentId = params.storeAgentId ? normalizeAgentId(params.storeAgentId) : undefined; if (lowered === "main" || lowered === rawMainKey) { + if (storeAgentId) { + return resolveAgentMainSessionKey({ cfg: params.cfg, agentId: storeAgentId }); + } return resolveMainSessionKey(params.cfg); } - const agentId = resolveDefaultStoreAgentId(params.cfg); + const agentId = storeAgentId ?? resolveDefaultStoreAgentId(params.cfg); return canonicalizeSessionKeyForAgent(agentId, raw); } diff --git a/src/gateway/session-transcript-files.fs.archive-events.test.ts b/src/gateway/session-transcript-files.fs.archive-events.test.ts index 1415c6c70a51..dcae60ab9c07 100644 --- a/src/gateway/session-transcript-files.fs.archive-events.test.ts +++ b/src/gateway/session-transcript-files.fs.archive-events.test.ts @@ -1,12 +1,12 @@ // Transcript archive event tests ensure file archive/delete operations emit -// path-only transcript update notifications for UI and index listeners. +// path-only internal transcript update notifications. import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { - onSessionTranscriptUpdate, - type SessionTranscriptUpdate, + onInternalSessionTranscriptUpdate, + type InternalSessionTranscriptUpdate, } from "../sessions/transcript-events.js"; import { archiveFileOnDisk, @@ -24,8 +24,8 @@ afterEach(() => { describe("archiveFileOnDisk transcript updates", () => { it("emits a session transcript update for the archived path on reset", () => { - const updates: SessionTranscriptUpdate[] = []; - subscriptions.push(onSessionTranscriptUpdate((update) => updates.push(update))); + const updates: InternalSessionTranscriptUpdate[] = []; + subscriptions.push(onInternalSessionTranscriptUpdate((update) => updates.push(update))); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "oc-archive-events-reset-")); try { @@ -50,8 +50,8 @@ describe("archiveFileOnDisk transcript updates", () => { }); it("also emits for deleted and bak archive reasons", () => { - const updates: SessionTranscriptUpdate[] = []; - subscriptions.push(onSessionTranscriptUpdate((update) => updates.push(update))); + const updates: InternalSessionTranscriptUpdate[] = []; + subscriptions.push(onInternalSessionTranscriptUpdate((update) => updates.push(update))); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "oc-archive-events-mixed-")); try { diff --git a/src/gateway/session-transcript-readers.test.ts b/src/gateway/session-transcript-readers.test.ts index 4469701c04ff..70b573d943e3 100644 --- a/src/gateway/session-transcript-readers.test.ts +++ b/src/gateway/session-transcript-readers.test.ts @@ -2,14 +2,25 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { + persistSessionTranscriptTurn, + upsertSessionEntry, +} from "../config/sessions/session-accessor.js"; +import { appendSqliteTranscriptEvents } from "../config/sessions/session-accessor.sqlite.js"; +import { formatSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; import { readLatestRecentSessionUsageFromTranscriptAsync, + readLatestSessionUsageFromTranscriptAsync, readRecentSessionMessagesWithStats, + readRecentSessionMessagesWithStatsAsync, readRecentSessionTranscriptLines, + readRecentSessionUsageFromTranscript, + readRecentSessionUsageFromTranscriptAsync, readSessionMessageByIdAsync, readSessionMessageCountAsync, readSessionMessagesAsync, + readSessionMessagesPageWithStatsAsync, readSessionTitleFieldsFromTranscript, type SessionTranscriptReadScope, } from "./session-transcript-readers.js"; @@ -38,7 +49,7 @@ describe("session transcript reader facade", () => { `${events.map((event) => JSON.stringify(event)).join("\n")}\n`, "utf-8", ); - return { sessionId, sessionKey: `agent:main:${sessionId}`, storePath }; + return { sessionFile: transcriptPath, sessionId, sessionKey: `agent:main:${sessionId}` }; } test("reads active-branch messages and message ids through a scope", async () => { @@ -123,6 +134,12 @@ describe("session transcript reader facade", () => { firstUserMessage: "derive this title", lastMessagePreview: "metered answer", }); + await expect(readLatestSessionUsageFromTranscriptAsync(scope)).resolves.toMatchObject({ + inputTokens: 11, + model: "gpt-5.5", + modelProvider: "openai", + outputTokens: 7, + }); await expect( readLatestRecentSessionUsageFromTranscriptAsync(scope, 4096), ).resolves.toMatchObject({ @@ -134,20 +151,365 @@ describe("session transcript reader facade", () => { }); }); - test("honors agent ids when no store path or session file is provided", async () => { - const sessionId = "reader-agent-scope"; - const transcriptDir = path.join(tempDir, "agents", "agent-one", "sessions"); - fs.mkdirSync(transcriptDir, { recursive: true }); + test("does not fall back to stored custom transcript paths after SQLite migration", async () => { + const sessionId = "reader-legacy-custom-path"; + const sessionKey = `agent:main:telegram:group:1:topic:9`; + const transcriptPath = path.join(tempDir, "legacy", "custom-topic.jsonl"); + fs.mkdirSync(path.dirname(transcriptPath), { recursive: true }); fs.writeFileSync( - path.join(transcriptDir, `${sessionId}.jsonl`), - `${JSON.stringify({ + transcriptPath, + `${JSON.stringify({ type: "session", version: 1, id: sessionId })}\n${JSON.stringify({ type: "message", - id: "agent-message", - parentId: null, - message: { role: "user", content: "agent scoped prompt" }, + id: "u1", + message: { role: "user", content: "legacy prompt" }, + })}\n${JSON.stringify({ + type: "message", + id: "a1", + message: { role: "assistant", content: "legacy answer" }, })}\n`, "utf-8", ); + await upsertSessionEntry( + { sessionKey, storePath }, + { + sessionId, + sessionFile: transcriptPath, + updatedAt: 10, + }, + ); + + await expect( + readSessionMessagesAsync( + { agentId: "main", sessionId, sessionKey, storePath }, + { mode: "full", reason: "no legacy fallback test" }, + ), + ).resolves.toEqual([]); + }); + + test("reads SQLite-only transcript rows without a JSONL mirror", async () => { + const sessionId = "reader-sqlite-only"; + const scope = { + agentId: "main", + sessionId, + sessionKey: `agent:main:${sessionId}`, + storePath, + }; + await persistSessionTranscriptTurn(scope, { + cwd: tempDir, + messages: [ + { message: { role: "user", content: "sqlite prompt" } }, + { + message: { + role: "assistant", + content: "sqlite answer", + provider: "openai", + model: "gpt-5.5", + usage: { input: 3, output: 4, total: 7 }, + }, + }, + { + message: { + role: "assistant", + content: "sqlite follow-up", + provider: "openai", + model: "gpt-5.5", + usage: { input: 5, output: 6, total: 11 }, + }, + }, + ], + touchSessionEntry: false, + }); + + expect(fs.existsSync(path.join(tempDir, `${sessionId}.jsonl`))).toBe(false); + await expect( + readSessionMessagesAsync(scope, { mode: "full", reason: "sqlite reader facade test" }), + ).resolves.toMatchObject([ + { content: "sqlite prompt" }, + { content: "sqlite answer" }, + { content: "sqlite follow-up" }, + ]); + await expect( + readSessionMessagesAsync(scope, { mode: "recent", maxMessages: 1 }), + ).resolves.toMatchObject([{ content: "sqlite follow-up", __openclaw: { seq: 4 } }]); + await expect(readSessionMessageCountAsync(scope)).resolves.toBe(3); + expect( + readRecentSessionMessagesWithStats(scope, { + maxMessages: 2, + maxBytes: 4096, + }).messages.map( + (message) => (message as { __openclaw?: { seq?: number } })["__openclaw"]?.seq, + ), + ).toEqual([3, 4]); + await expect( + readRecentSessionMessagesWithStatsAsync(scope, { + maxMessages: 2, + maxBytes: 4096, + }).then((result) => + result.messages.map( + (message) => (message as { __openclaw?: { seq?: number } })["__openclaw"]?.seq, + ), + ), + ).resolves.toEqual([3, 4]); + expect(readSessionTitleFieldsFromTranscript(scope)).toEqual({ + firstUserMessage: "sqlite prompt", + lastMessagePreview: "sqlite follow-up", + }); + await expect(readLatestSessionUsageFromTranscriptAsync(scope)).resolves.toMatchObject({ + inputTokens: 8, + model: "gpt-5.5", + modelProvider: "openai", + outputTokens: 10, + }); + await expect( + readLatestRecentSessionUsageFromTranscriptAsync(scope, 4096), + ).resolves.toMatchObject({ + inputTokens: 5, + model: "gpt-5.5", + modelProvider: "openai", + outputTokens: 6, + }); + }); + + test("uses SQLite marker identity when only sessionFile is provided", async () => { + const sessionId = "reader-marker-only"; + const markerStorePath = path.join( + tempDir, + "agents", + "marker-agent", + "sessions", + "sessions.json", + ); + fs.mkdirSync(path.dirname(markerStorePath), { recursive: true }); + const marker = formatSqliteSessionFileMarker({ + agentId: "marker-agent", + sessionId, + storePath: markerStorePath, + }); + await appendSqliteTranscriptEvents( + { + agentId: "marker-agent", + sessionId, + sessionKey: "agent:marker-agent:main", + storePath: markerStorePath, + }, + [ + { + type: "message", + id: "marker-message", + parentId: null, + message: { role: "user", content: "marker scoped prompt" }, + }, + ], + ); + + await expect( + readSessionMessagesAsync( + { sessionFile: marker, sessionId }, + { mode: "full", reason: "sqlite marker-only read test" }, + ), + ).resolves.toMatchObject([{ content: "marker scoped prompt" }]); + await expect( + readSessionMessageByIdAsync({ sessionFile: marker, sessionId }, "marker-message"), + ).resolves.toMatchObject({ + found: true, + seq: 1, + }); + }); + + test("projects SQLite transcript reads to the active branch", async () => { + const sessionId = "reader-sqlite-branch"; + const scope = { + agentId: "main", + sessionId, + sessionKey: `agent:main:${sessionId}`, + storePath, + }; + await appendSqliteTranscriptEvents(scope, [ + { type: "session", version: 1, id: sessionId }, + { + type: "message", + id: "root", + parentId: null, + message: { role: "user", content: "branch prompt" }, + }, + { + type: "message", + id: "inactive", + parentId: "root", + message: { role: "assistant", content: "stale branch" }, + }, + { + type: "message", + id: "active", + parentId: "root", + message: { role: "assistant", content: "active branch" }, + }, + ]); + + const messages = await readSessionMessagesAsync(scope, { + mode: "full", + reason: "sqlite branch facade test", + }); + + expect(messages).toMatchObject([{ content: "branch prompt" }, { content: "active branch" }]); + expect( + messages.map((message) => (message as { __openclaw?: { id?: string } })["__openclaw"]?.id), + ).toEqual(["root", "active"]); + expect( + messages.map((message) => (message as { __openclaw?: { seq?: number } })["__openclaw"]?.seq), + ).toEqual([2, 4]); + await expect(readSessionMessageCountAsync(scope)).resolves.toBe(2); + }); + + test("bounds SQLite recent message and usage reads", async () => { + const sessionId = "reader-sqlite-bounded-recent"; + const scope = { + agentId: "main", + sessionId, + sessionKey: `agent:main:${sessionId}`, + storePath, + }; + await persistSessionTranscriptTurn(scope, { + messages: [ + { message: { role: "user", content: "old prompt" } }, + { + message: { + role: "assistant", + content: `old answer ${"x".repeat(5000)}`, + provider: "openai", + model: "gpt-5.5", + usage: { input: 100, output: 100 }, + }, + }, + { message: { role: "user", content: "recent prompt" } }, + { + message: { + role: "assistant", + content: "recent answer", + provider: "openai", + model: "gpt-5.5", + usage: { input: 5, output: 6 }, + }, + }, + ], + touchSessionEntry: false, + }); + + const recentMessages = readRecentSessionMessagesWithStats(scope, { + maxMessages: 2, + maxBytes: 4096, + maxLines: 2, + }).messages; + expect(recentMessages.map((message) => (message as { content?: string }).content)).toEqual([ + "recent prompt", + "recent answer", + ]); + expect( + recentMessages.map( + (message) => (message as { __openclaw?: { seq?: number } })["__openclaw"]?.seq, + ), + ).toEqual([4, 5]); + await expect(readRecentSessionUsageFromTranscriptAsync(scope, 1024)).resolves.toMatchObject({ + inputTokens: 5, + outputTokens: 6, + }); + expect(readRecentSessionUsageFromTranscript(scope, 1024)).toMatchObject({ + inputTokens: 5, + outputTokens: 6, + }); + }); + + test("pages SQLite transcript messages through the reader facade", async () => { + const sessionId = "reader-sqlite-page"; + const scope = { + agentId: "main", + sessionId, + sessionKey: `agent:main:${sessionId}`, + storePath, + }; + await persistSessionTranscriptTurn(scope, { + messages: [ + { message: { role: "user", content: "first" } }, + { message: { role: "assistant", content: "second" } }, + { message: { role: "user", content: "third" } }, + { message: { role: "assistant", content: "fourth" } }, + ], + touchSessionEntry: false, + }); + + const page = await readSessionMessagesPageWithStatsAsync(scope, { + maxMessages: 2, + offset: 1, + }); + + expect(page.totalMessages).toBe(4); + expect(page.messages.map((message) => (message as { content?: string }).content)).toEqual([ + "second", + "third", + ]); + expect( + page.messages.map( + (message) => (message as { __openclaw?: { seq?: number } })["__openclaw"]?.seq, + ), + ).toEqual([3, 4]); + }); + + test("ignores zero-usage SQLite delivery mirrors for latest usage", async () => { + const sessionId = "reader-sqlite-delivery-mirror"; + const scope = { + agentId: "main", + sessionId, + sessionKey: `agent:main:${sessionId}`, + storePath, + }; + await persistSessionTranscriptTurn(scope, { + cwd: tempDir, + messages: [ + { + message: { + role: "assistant", + content: "real model answer", + provider: "openai", + model: "gpt-5.5", + usage: { input: 7, output: 8, total: 15 }, + }, + }, + { + message: { + role: "assistant", + content: "delivered", + provider: "openclaw", + model: "delivery-mirror", + usage: { input: 0, output: 0, total: 0, cost: { total: 0 } }, + }, + }, + ], + touchSessionEntry: false, + }); + + await expect( + readLatestRecentSessionUsageFromTranscriptAsync(scope, 4096), + ).resolves.toMatchObject({ + inputTokens: 7, + model: "gpt-5.5", + modelProvider: "openai", + outputTokens: 8, + }); + }); + + test("honors agent ids when no store path or session file is provided", async () => { + const sessionId = "reader-agent-scope"; + await appendSqliteTranscriptEvents( + { agentId: "agent-one", sessionId, sessionKey: "agent:agent-one:main" }, + [ + { + type: "message", + id: "agent-message", + parentId: null, + message: { role: "user", content: "agent scoped prompt" }, + }, + ], + ); const scope = { agentId: "agent-one", sessionId }; await expect(readSessionMessageCountAsync(scope)).resolves.toBe(1); @@ -159,4 +521,29 @@ describe("session transcript reader facade", () => { readSessionMessagesAsync(scope, { mode: "full", reason: "facade agent scope test" }), ).resolves.toMatchObject([{ content: "agent scoped prompt" }]); }); + + test("reads explicit transcript files without session store identity", async () => { + const sessionId = "reader-explicit-file"; + const transcriptPath = path.join(tempDir, "explicit-file.jsonl"); + fs.writeFileSync( + transcriptPath, + `${JSON.stringify({ + type: "message", + id: "explicit-message", + parentId: null, + message: { role: "user", content: "explicit prompt" }, + })}\n`, + "utf-8", + ); + const scope = { sessionFile: transcriptPath, sessionId }; + + await expect(readSessionMessageCountAsync(scope)).resolves.toBe(1); + await expect(readSessionMessageByIdAsync(scope, "explicit-message")).resolves.toMatchObject({ + found: true, + seq: 1, + }); + await expect( + readSessionMessagesAsync(scope, { mode: "full", reason: "explicit file test" }), + ).resolves.toMatchObject([{ content: "explicit prompt" }]); + }); }); diff --git a/src/gateway/session-transcript-readers.ts b/src/gateway/session-transcript-readers.ts index e266c3bdaeee..23adbbd5ec47 100644 --- a/src/gateway/session-transcript-readers.ts +++ b/src/gateway/session-transcript-readers.ts @@ -1,11 +1,29 @@ +import { + deriveSessionTotalTokens, + hasNonzeroUsage, + normalizeUsage, + type UsageLike, +} from "../agents/usage.js"; import type { SessionTranscriptReadScope } from "../config/sessions/session-accessor.js"; -import { resolveSessionTranscriptReadTarget } from "../config/sessions/session-accessor.js"; +import { + loadTranscriptEvents, + loadTranscriptEventsSync, + resolveSessionTranscriptReadTarget, +} from "../config/sessions/session-accessor.js"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; +import { + selectVisibleTranscriptEventEntries, + type VisibleTranscriptEventEntry, +} from "../config/sessions/transcript-visible-events.js"; +import { hasInterSessionUserProvenance } from "../sessions/input-provenance.js"; import type { ReadRecentSessionMessagesOptions, ReadSessionMessagesAsyncOptions, SessionTranscriptUsageSnapshot, } from "./session-utils.fs.js"; import { + attachOpenClawTranscriptMeta, + buildSessionPreviewItems, readFirstUserMessageFromTranscript as readFirstUserMessageFromTranscriptFile, readLatestRecentSessionUsageFromTranscriptAsync as readLatestRecentSessionUsageFromTranscriptAsyncFile, readLatestSessionUsageFromTranscript as readLatestSessionUsageFromTranscriptFile, @@ -30,6 +48,7 @@ import { visitSessionMessages as visitSessionMessagesFile, visitSessionMessagesAsync as visitSessionMessagesAsyncFile, } from "./session-utils.fs.js"; +import type { SessionPreviewItem } from "./session-utils.types.js"; export type { ReadRecentSessionMessagesOptions, ReadSessionMessagesAsyncOptions }; export { attachOpenClawTranscriptMeta, capArrayByJsonBytes } from "./session-utils.fs.js"; @@ -59,21 +78,26 @@ type ReadSessionMessageByIdResult = { found: boolean; }; -type FileBackedReadScope = { +type ResolvedTranscriptReadTarget = { agentId?: string; sessionFile: string; sessionId: string; + sessionKey?: string; storePath?: string; }; -function resolveFileBackedReadScope(scope: SessionTranscriptReadScope): FileBackedReadScope { +function resolveTranscriptReadTarget( + scope: SessionTranscriptReadScope, +): ResolvedTranscriptReadTarget { const target = resolveSessionTranscriptReadTarget(scope); + const marker = parseSqliteSessionFileMarker(target.sessionFile); const storePath = resolveConcreteReadStorePath(scope.storePath); return { - agentId: target.agentId, + agentId: target.agentId ?? marker?.agentId, sessionFile: target.sessionFile, - sessionId: target.sessionId, - ...(storePath ? { storePath } : {}), + sessionId: marker?.sessionId ?? target.sessionId, + ...(target.sessionKey ? { sessionKey: target.sessionKey } : {}), + ...((storePath ?? marker?.storePath) ? { storePath: storePath ?? marker?.storePath } : {}), }; } @@ -85,9 +109,370 @@ function resolveConcreteReadStorePath(storePath: string | undefined): string | u return trimmed; } +function isSqliteReadTarget(target: ResolvedTranscriptReadTarget): boolean { + return parseSqliteSessionFileMarker(target.sessionFile) !== undefined; +} + +function toTranscriptReadScope(target: ResolvedTranscriptReadTarget): SessionTranscriptReadScope { + return { + ...(target.agentId ? { agentId: target.agentId } : {}), + sessionId: target.sessionId, + ...(target.sessionKey ? { sessionKey: target.sessionKey } : {}), + ...(target.storePath ? { storePath: target.storePath } : {}), + }; +} + +function readTranscriptRecordTimestampMs(event: Record<string, unknown>): number | undefined { + const raw = event.timestamp; + const timestampMs = + typeof raw === "string" ? Date.parse(raw) : typeof raw === "number" ? raw : Number.NaN; + return Number.isFinite(timestampMs) ? timestampMs : undefined; +} + +function extractMessageRecord( + event: unknown, +): { id?: string; message: unknown; recordTimestampMs?: number } | undefined { + if (!event || typeof event !== "object" || Array.isArray(event)) { + return undefined; + } + const record = event as { id?: unknown; message?: unknown }; + if (record.message === undefined) { + return undefined; + } + const recordTimestampMs = readTranscriptRecordTimestampMs(event as Record<string, unknown>); + return { + ...(typeof record.id === "string" ? { id: record.id } : {}), + message: record.message, + ...(recordTimestampMs !== undefined ? { recordTimestampMs } : {}), + }; +} + +function extractMessageRecordsFromEventEntries(entries: VisibleTranscriptEventEntry<unknown>[]): { + id?: string; + message: unknown; + recordTimestampMs?: number; + seq: number; +}[] { + return entries.flatMap((entry) => { + const record = extractMessageRecord(entry.event); + return record ? [{ ...record, seq: entry.seq }] : []; + }); +} + +function readSqliteMessageRecordsSync(target: ResolvedTranscriptReadTarget): { + id?: string; + message: unknown; + recordTimestampMs?: number; + seq: number; +}[] { + return extractMessageRecordsFromEventEntries( + selectVisibleTranscriptEventEntries(loadTranscriptEventsSync(toTranscriptReadScope(target))), + ); +} + +async function readSqliteMessageRecords(target: ResolvedTranscriptReadTarget): Promise< + { + id?: string; + message: unknown; + recordTimestampMs?: number; + seq: number; + }[] +> { + return extractMessageRecordsFromEventEntries( + selectVisibleTranscriptEventEntries(await loadTranscriptEvents(toTranscriptReadScope(target))), + ); +} + +function readSqliteMessagesSync(target: ResolvedTranscriptReadTarget): unknown[] { + return readSqliteMessageRecordsSync(target).map(sqliteRecordMessageWithSeq); +} + +function normalizeRecentSqliteReadOptions(opts?: Partial<ReadRecentSessionMessagesOptions>) { + const maxMessages = Math.max(0, Math.floor(opts?.maxMessages ?? 0)); + const maxBytes = + typeof opts?.maxBytes === "number" && Number.isFinite(opts.maxBytes) + ? Math.max(1024, Math.floor(opts.maxBytes)) + : 8 * 1024 * 1024; + const defaultMaxLines = maxMessages * 20 + 20; + const maxLines = + typeof opts?.maxLines === "number" && Number.isFinite(opts.maxLines) + ? Math.max(maxMessages, Math.floor(opts.maxLines)) + : defaultMaxLines; + return { maxMessages, maxBytes, maxLines }; +} + +function selectRecentSqliteEventEntries( + entries: VisibleTranscriptEventEntry<unknown>[], + opts: { maxBytes: number; maxLines: number }, +) { + const selected: VisibleTranscriptEventEntry<unknown>[] = []; + let bytes = 0; + for (const entry of entries.toReversed()) { + const line = JSON.stringify(entry.event); + const lineBytes = Buffer.byteLength(line) + 1; + if (selected.length > 0 && bytes + lineBytes > opts.maxBytes) { + break; + } + selected.push(entry); + bytes += lineBytes; + if (selected.length >= opts.maxLines) { + break; + } + } + return selected.toReversed(); +} + +function readRecentSqliteMessageRecordsSync( + target: ResolvedTranscriptReadTarget, + opts?: Partial<ReadRecentSessionMessagesOptions>, +): { id?: string; message: unknown; recordTimestampMs?: number; seq: number }[] { + const normalized = normalizeRecentSqliteReadOptions(opts); + const entries = selectVisibleTranscriptEventEntries( + loadTranscriptEventsSync(toTranscriptReadScope(target)), + ); + const records = extractMessageRecordsFromEventEntries( + selectRecentSqliteEventEntries(entries, normalized), + ); + return normalized.maxMessages > 0 ? records.slice(-normalized.maxMessages) : []; +} + +async function readRecentSqliteMessageRecords( + target: ResolvedTranscriptReadTarget, + opts?: Partial<ReadRecentSessionMessagesOptions>, +): Promise<{ id?: string; message: unknown; recordTimestampMs?: number; seq: number }[]> { + const normalized = normalizeRecentSqliteReadOptions(opts); + const entries = selectVisibleTranscriptEventEntries( + await loadTranscriptEvents(toTranscriptReadScope(target)), + ); + const records = extractMessageRecordsFromEventEntries( + selectRecentSqliteEventEntries(entries, normalized), + ); + return normalized.maxMessages > 0 ? records.slice(-normalized.maxMessages) : []; +} + +function readRecentSqliteUsageMessages( + target: ResolvedTranscriptReadTarget, + maxBytes: number, +): unknown[] { + const entries = selectVisibleTranscriptEventEntries( + loadTranscriptEventsSync(toTranscriptReadScope(target)), + ); + return extractMessageRecordsFromEventEntries( + selectRecentSqliteEventEntries(entries, { + maxBytes: Math.max(1024, Math.floor(Number.isFinite(maxBytes) ? maxBytes : 8 * 1024 * 1024)), + maxLines: 1000, + }), + ).map((record) => record.message); +} + +function sqliteRecordMessageWithSeq(record: { + id?: string; + message: unknown; + recordTimestampMs?: number; + seq: number; +}): unknown { + return attachOpenClawTranscriptMeta(record.message, { + ...(record.id ? { id: record.id } : {}), + ...(record.recordTimestampMs !== undefined + ? { recordTimestampMs: record.recordTimestampMs } + : {}), + seq: record.seq, + }); +} + +function extractMessageRole(message: unknown): string | undefined { + return message && typeof message === "object" && !Array.isArray(message) + ? ((message as { role?: unknown }).role as string | undefined) + : undefined; +} + +function extractMessageText(message: unknown): string | null { + if (!message || typeof message !== "object" || Array.isArray(message)) { + return null; + } + const record = message as { content?: unknown; text?: unknown }; + if (typeof record.content === "string") { + return record.content.trim() || null; + } + if (Array.isArray(record.content)) { + const text = record.content + .map((entry) => + entry && typeof entry === "object" && typeof (entry as { text?: unknown }).text === "string" + ? (entry as { text: string }).text + : "", + ) + .filter((part) => part.trim()) + .join("\n") + .trim(); + return text || null; + } + if (typeof record.text === "string") { + return record.text.trim() || null; + } + return null; +} + +function readSqliteTitleFields( + target: ResolvedTranscriptReadTarget, + opts?: { includeInterSession?: boolean }, +): SessionTitleFields { + const messages = readSqliteMessagesSync(target); + const firstUser = messages.find((message) => { + if (extractMessageRole(message) !== "user") { + return false; + } + return ( + opts?.includeInterSession === true || + !hasInterSessionUserProvenance(message as { role?: unknown; provenance?: unknown }) + ); + }); + const lastText = messages.toReversed().map(extractMessageText).find(Boolean) ?? null; + return { + firstUserMessage: firstUser ? extractMessageText(firstUser) : null, + lastMessagePreview: lastText, + }; +} + +function extractSqliteUsageSnapshot(message: unknown): SessionTranscriptUsageSnapshot | null { + if (!message || typeof message !== "object" || Array.isArray(message)) { + return null; + } + const record = message as { + model?: unknown; + provider?: unknown; + usage?: unknown; + }; + const usageRaw = + record.usage && typeof record.usage === "object" && !Array.isArray(record.usage) + ? (record.usage as UsageLike & { cost?: { total?: unknown }; costUsd?: unknown }) + : undefined; + const usage = normalizeUsage(usageRaw); + const normalizedUsage = usage ?? {}; + const totalTokens = deriveSessionTotalTokens({ usage }); + const modelProvider = typeof record.provider === "string" ? record.provider.trim() : undefined; + const model = typeof record.model === "string" ? record.model.trim() : undefined; + const costUsd = + typeof usageRaw?.cost?.total === "number" && Number.isFinite(usageRaw.cost.total) + ? usageRaw.cost.total + : usageRaw?.costUsd; + const hasMeaningfulUsage = + hasNonzeroUsage(usage) || + typeof totalTokens === "number" || + (typeof costUsd === "number" && Number.isFinite(costUsd) && costUsd > 0); + const isDeliveryMirror = modelProvider === "openclaw" && model === "delivery-mirror"; + if (!hasMeaningfulUsage && !modelProvider && !model) { + return null; + } + if (isDeliveryMirror && !hasMeaningfulUsage) { + return null; + } + return { + ...(!isDeliveryMirror && modelProvider ? { modelProvider } : {}), + ...(!isDeliveryMirror && model ? { model } : {}), + ...(typeof normalizedUsage.input === "number" ? { inputTokens: normalizedUsage.input } : {}), + ...(typeof normalizedUsage.output === "number" ? { outputTokens: normalizedUsage.output } : {}), + ...(typeof normalizedUsage.cacheRead === "number" + ? { cacheRead: normalizedUsage.cacheRead } + : {}), + ...(typeof normalizedUsage.cacheWrite === "number" + ? { cacheWrite: normalizedUsage.cacheWrite } + : {}), + ...(typeof totalTokens === "number" ? { totalTokens, totalTokensFresh: true } : {}), + ...(typeof costUsd === "number" && Number.isFinite(costUsd) ? { costUsd } : {}), + }; +} + +function readSqliteAggregateUsageSnapshot( + target: ResolvedTranscriptReadTarget, +): SessionTranscriptUsageSnapshot | null { + return aggregateSqliteUsageSnapshots(readSqliteMessagesSync(target)); +} + +function aggregateSqliteUsageSnapshots(messages: unknown[]): SessionTranscriptUsageSnapshot | null { + const aggregate: SessionTranscriptUsageSnapshot = {}; + let sawUsage = false; + let inputTokens = 0; + let outputTokens = 0; + let cacheRead = 0; + let cacheWrite = 0; + let costUsd = 0; + let sawInput = false; + let sawOutput = false; + let sawCacheRead = false; + let sawCacheWrite = false; + let sawCost = false; + for (const message of messages) { + const snapshot = extractSqliteUsageSnapshot(message); + if (!snapshot) { + continue; + } + sawUsage = true; + if (snapshot.modelProvider) { + aggregate.modelProvider = snapshot.modelProvider; + } + if (snapshot.model) { + aggregate.model = snapshot.model; + } + if (typeof snapshot.inputTokens === "number") { + inputTokens += snapshot.inputTokens; + sawInput = true; + } + if (typeof snapshot.outputTokens === "number") { + outputTokens += snapshot.outputTokens; + sawOutput = true; + } + if (typeof snapshot.cacheRead === "number") { + cacheRead += snapshot.cacheRead; + sawCacheRead = true; + } + if (typeof snapshot.cacheWrite === "number") { + cacheWrite += snapshot.cacheWrite; + sawCacheWrite = true; + } + if (typeof snapshot.totalTokens === "number") { + aggregate.totalTokens = snapshot.totalTokens; + aggregate.totalTokensFresh = true; + } + if (typeof snapshot.costUsd === "number") { + costUsd += snapshot.costUsd; + sawCost = true; + } + } + if (!sawUsage) { + return null; + } + if (sawInput) { + aggregate.inputTokens = inputTokens; + } + if (sawOutput) { + aggregate.outputTokens = outputTokens; + } + if (sawCacheRead) { + aggregate.cacheRead = cacheRead; + } + if (sawCacheWrite) { + aggregate.cacheWrite = cacheWrite; + } + if (sawCost) { + aggregate.costUsd = costUsd; + } + return aggregate; +} + +function buildSqlitePreviewItems( + target: ResolvedTranscriptReadTarget, + maxItems: number, + maxChars: number, +): SessionPreviewItem[] { + return buildSessionPreviewItems(readSqliteMessagesSync(target), maxItems, maxChars); +} + /** Reads display messages from a session transcript through the reader seam. */ export function readSessionMessages(scope: SessionTranscriptReadScope): unknown[] { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + return readSqliteMessagesSync(target); + } return readSessionMessagesFile( target.sessionId, target.storePath, @@ -101,7 +486,10 @@ export function readRecentSessionMessages( scope: SessionTranscriptReadScope, opts?: ReadRecentSessionMessagesOptions, ): unknown[] { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + return readRecentSqliteMessageRecordsSync(target, opts).map(sqliteRecordMessageWithSeq); + } return readRecentSessionMessagesFile( target.sessionId, target.storePath, @@ -116,7 +504,15 @@ export function visitSessionMessages( scope: SessionTranscriptReadScope, visit: (message: unknown, seq: number) => void, ): number { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + let count = 0; + readSqliteMessagesSync(target).forEach((message, index) => { + visit(message, index + 1); + count += 1; + }); + return count; + } return visitSessionMessagesFile( target.sessionId, target.storePath, @@ -128,7 +524,10 @@ export function visitSessionMessages( /** Counts display messages in a session transcript through the reader seam. */ export function readSessionMessageCount(scope: SessionTranscriptReadScope): number { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + return readSqliteMessagesSync(target).length; + } return readSessionMessageCountFile( target.sessionId, target.storePath, @@ -142,7 +541,33 @@ export async function readSessionMessagesAsync( scope: SessionTranscriptReadScope, opts: ReadSessionMessagesAsyncOptions, ): Promise<unknown[]> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + if (opts.mode === "recent") { + const records = await readRecentSqliteMessageRecords(target, opts); + if (records.length === 0 && opts.allowResetArchiveFallback === true) { + return await readRecentSessionMessagesAsyncFile( + target.sessionId, + target.storePath, + undefined, + { ...opts, resetArchiveOnly: true }, + target.agentId, + ); + } + return records.map(sqliteRecordMessageWithSeq); + } + const records = await readSqliteMessageRecords(target); + if (records.length === 0 && opts.allowResetArchiveFallback === true) { + return await readSessionMessagesAsyncFile( + target.sessionId, + target.storePath, + undefined, + opts, + target.agentId, + ); + } + return records.map(sqliteRecordMessageWithSeq); + } return await readSessionMessagesAsyncFile( target.sessionId, target.storePath, @@ -157,7 +582,27 @@ export async function readSessionMessagesWithSourceAsync( scope: SessionTranscriptReadScope, opts: ReadSessionMessagesAsyncOptions, ): Promise<ReadSessionMessagesResult> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + const records = + opts.mode === "recent" + ? await readRecentSqliteMessageRecords(target, opts) + : await readSqliteMessageRecords(target); + if (records.length === 0 && opts.allowResetArchiveFallback === true) { + return await readSessionMessagesWithSourceAsyncFile( + target.sessionId, + target.storePath, + undefined, + { ...opts, resetArchiveOnly: true }, + target.agentId, + ); + } + const messages = records.map(sqliteRecordMessageWithSeq); + return { + messages, + transcriptPath: target.sessionFile, + }; + } return await readSessionMessagesWithSourceAsyncFile( target.sessionId, target.storePath, @@ -172,7 +617,20 @@ export async function readRecentSessionMessagesAsync( scope: SessionTranscriptReadScope, opts?: ReadRecentSessionMessagesOptions, ): Promise<unknown[]> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + const records = await readRecentSqliteMessageRecords(target, opts); + if (records.length === 0 && opts?.allowResetArchiveFallback === true) { + return await readRecentSessionMessagesAsyncFile( + target.sessionId, + target.storePath, + undefined, + { ...opts, resetArchiveOnly: true }, + target.agentId, + ); + } + return records.map(sqliteRecordMessageWithSeq); + } return await readRecentSessionMessagesAsyncFile( target.sessionId, target.storePath, @@ -188,7 +646,25 @@ export async function readSessionMessageByIdAsync( messageId: string, opts?: { allowResetArchiveFallback?: boolean }, ): Promise<ReadSessionMessageByIdResult> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + const found = (await readSqliteMessageRecords(target)).find( + (record) => record.id === messageId, + ); + if (found) { + return { found: true, message: found.message, oversized: false, seq: found.seq }; + } + if (opts?.allowResetArchiveFallback === true) { + return await readSessionMessageByIdAsyncFile( + target.sessionId, + target.storePath, + undefined, + messageId, + { ...opts, agentId: target.agentId, resetArchiveOnly: true }, + ); + } + return { found: false, oversized: false }; + } return await readSessionMessageByIdAsyncFile( target.sessionId, target.storePath, @@ -204,7 +680,15 @@ export async function visitSessionMessagesAsync( visit: (message: unknown, seq: number) => void, opts: { mode: "full"; reason: string; cache?: "reuse" | "skip" }, ): Promise<number> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + let count = 0; + for (const record of await readSqliteMessageRecords(target)) { + visit(record.message, record.seq); + count += 1; + } + return count; + } return await visitSessionMessagesAsyncFile( target.sessionId, target.storePath, @@ -219,7 +703,10 @@ export async function visitSessionMessagesAsync( export async function readSessionMessageCountAsync( scope: SessionTranscriptReadScope, ): Promise<number> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + return (await readSqliteMessageRecords(target)).length; + } return await readSessionMessageCountAsyncFile( target.sessionId, target.storePath, @@ -233,7 +720,29 @@ export function readRecentSessionMessagesWithStats( scope: SessionTranscriptReadScope, opts: ReadRecentSessionMessagesOptions, ): ReadRecentSessionMessagesResult { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + const records = readSqliteMessageRecordsSync(target); + const recentRecords = readRecentSqliteMessageRecordsSync(target, opts); + if ( + records.length === 0 && + recentRecords.length === 0 && + opts.allowResetArchiveFallback === true + ) { + return readRecentSessionMessagesWithStatsFile( + target.sessionId, + target.storePath, + undefined, + { ...opts, resetArchiveOnly: true }, + target.agentId, + ); + } + return { + messages: recentRecords.map(sqliteRecordMessageWithSeq), + totalMessages: records.length, + transcriptPath: target.sessionFile, + }; + } return readRecentSessionMessagesWithStatsFile( target.sessionId, target.storePath, @@ -248,7 +757,29 @@ export async function readRecentSessionMessagesWithStatsAsync( scope: SessionTranscriptReadScope, opts: ReadRecentSessionMessagesOptions, ): Promise<ReadRecentSessionMessagesResult> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + const records = await readSqliteMessageRecords(target); + const recentRecords = await readRecentSqliteMessageRecords(target, opts); + if ( + records.length === 0 && + recentRecords.length === 0 && + opts.allowResetArchiveFallback === true + ) { + return await readRecentSessionMessagesWithStatsAsyncFile( + target.sessionId, + target.storePath, + undefined, + { ...opts, resetArchiveOnly: true }, + target.agentId, + ); + } + return { + messages: recentRecords.map(sqliteRecordMessageWithSeq), + totalMessages: records.length, + transcriptPath: target.sessionFile, + }; + } return await readRecentSessionMessagesWithStatsAsyncFile( target.sessionId, target.storePath, @@ -263,7 +794,35 @@ export async function readSessionMessagesPageWithStatsAsync( scope: SessionTranscriptReadScope, opts: { offset: number; maxMessages: number; allowResetArchiveFallback?: boolean }, ): Promise<ReadRecentSessionMessagesResult> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + const records = await readSqliteMessageRecords(target); + if (records.length === 0 && opts.allowResetArchiveFallback === true) { + return await readSessionMessagesPageWithStatsAsyncFile( + target.sessionId, + target.storePath, + undefined, + { ...opts, resetArchiveOnly: true }, + target.agentId, + ); + } + const totalMessages = records.length; + const offset = Math.min( + Math.max(0, Math.floor(Number.isFinite(opts.offset) ? opts.offset : 0)), + totalMessages, + ); + const maxMessages = Math.max( + 0, + Math.floor(Number.isFinite(opts.maxMessages) ? opts.maxMessages : 0), + ); + const endExclusive = Math.max(0, totalMessages - offset); + const start = Math.max(0, endExclusive - maxMessages); + return { + messages: records.slice(start, endExclusive).map(sqliteRecordMessageWithSeq), + totalMessages, + transcriptPath: target.sessionFile, + }; + } return await readSessionMessagesPageWithStatsAsyncFile( target.sessionId, target.storePath, @@ -279,7 +838,13 @@ export function readRecentSessionTranscriptLines( maxLines: number; }, ): { lines: string[]; totalLines: number } | null { - const target = resolveFileBackedReadScope(params); + const target = resolveTranscriptReadTarget(params); + if (isSqliteReadTarget(target)) { + const lines = loadTranscriptEventsSync(toTranscriptReadScope(target)).map((event) => + JSON.stringify(event), + ); + return { lines: lines.slice(-params.maxLines), totalLines: lines.length }; + } return readRecentSessionTranscriptLinesFile({ sessionId: target.sessionId, storePath: target.storePath, @@ -294,7 +859,10 @@ export function readSessionTitleFieldsFromTranscript( scope: SessionTranscriptReadScope, opts?: { includeInterSession?: boolean }, ): SessionTitleFields { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + return readSqliteTitleFields(target, opts); + } return readSessionTitleFieldsFromTranscriptFile( target.sessionId, target.storePath, @@ -309,7 +877,10 @@ export async function readSessionTitleFieldsFromTranscriptAsync( scope: SessionTranscriptReadScope, opts?: { includeInterSession?: boolean }, ): Promise<SessionTitleFields> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + return readSqliteTitleFields(target, opts); + } return await readSessionTitleFieldsFromTranscriptAsyncFile( target.sessionId, target.storePath, @@ -324,7 +895,10 @@ export function readFirstUserMessageFromTranscript( scope: SessionTranscriptReadScope, opts?: { includeInterSession?: boolean }, ): string | null { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + return readSqliteTitleFields(target, opts).firstUserMessage; + } return readFirstUserMessageFromTranscriptFile( target.sessionId, target.storePath, @@ -338,7 +912,10 @@ export function readFirstUserMessageFromTranscript( export function readLatestSessionUsageFromTranscript( scope: SessionTranscriptReadScope, ): SessionTranscriptUsageSnapshot | null { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + return readSqliteAggregateUsageSnapshot(target); + } return readLatestSessionUsageFromTranscriptFile( target.sessionId, target.storePath, @@ -351,7 +928,10 @@ export function readLatestSessionUsageFromTranscript( export async function readLatestSessionUsageFromTranscriptAsync( scope: SessionTranscriptReadScope, ): Promise<SessionTranscriptUsageSnapshot | null> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + return readSqliteAggregateUsageSnapshot(target); + } return await readLatestSessionUsageFromTranscriptAsyncFile( target.sessionId, target.storePath, @@ -365,7 +945,10 @@ export async function readRecentSessionUsageFromTranscriptAsync( scope: SessionTranscriptReadScope, maxBytes: number, ): Promise<SessionTranscriptUsageSnapshot | null> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + return aggregateSqliteUsageSnapshots(readRecentSqliteUsageMessages(target, maxBytes)); + } return await readRecentSessionUsageFromTranscriptAsyncFile( target.sessionId, target.storePath, @@ -380,7 +963,16 @@ export async function readLatestRecentSessionUsageFromTranscriptAsync( scope: SessionTranscriptReadScope, maxBytes: number, ): Promise<SessionTranscriptUsageSnapshot | null> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + for (const message of readRecentSqliteUsageMessages(target, maxBytes).toReversed()) { + const snapshot = extractSqliteUsageSnapshot(message); + if (snapshot) { + return snapshot; + } + } + return null; + } return await readLatestRecentSessionUsageFromTranscriptAsyncFile( target.sessionId, target.storePath, @@ -395,7 +987,10 @@ export function readRecentSessionUsageFromTranscript( scope: SessionTranscriptReadScope, maxBytes: number, ): SessionTranscriptUsageSnapshot | null { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + return aggregateSqliteUsageSnapshots(readRecentSqliteUsageMessages(target, maxBytes)); + } return readRecentSessionUsageFromTranscriptFile( target.sessionId, target.storePath, @@ -411,7 +1006,10 @@ export function readSessionPreviewItemsFromTranscript( maxItems: number, maxChars: number, ): ReturnType<typeof readSessionPreviewItemsFromTranscriptFile> { - const target = resolveFileBackedReadScope(scope); + const target = resolveTranscriptReadTarget(scope); + if (isSqliteReadTarget(target)) { + return buildSqlitePreviewItems(target, maxItems, maxChars); + } return readSessionPreviewItemsFromTranscriptFile( target.sessionId, target.storePath, diff --git a/src/gateway/session-utils.fs.test.ts b/src/gateway/session-utils.fs.test.ts index 9933fb31ffba..cffbb96b0edc 100644 --- a/src/gateway/session-utils.fs.test.ts +++ b/src/gateway/session-utils.fs.test.ts @@ -126,7 +126,11 @@ function appendBlockedUserMessage( }, }, } as Parameters<typeof sessionManager.appendMessage>[0]); - (sessionManager as unknown as { rewriteFile?: () => void }).rewriteFile?.(); + ( + sessionManager as unknown as { + replacePersistedTranscript?: () => void; + } + ).replacePersistedTranscript?.(); return messageId; } @@ -1276,7 +1280,7 @@ describe("readSessionMessages", () => { ]); }); - test("keeps async active branch rows when imported parent links are incomplete", async () => { + test("keeps async rows when imported parent links are incomplete without leaf control", async () => { const sessionId = "test-session-tree-async-incomplete-parent"; writeTranscript(tmpDir, sessionId, [ { type: "session", version: 3, id: sessionId }, @@ -1306,9 +1310,13 @@ describe("readSessionMessages", () => { }); expect(messages.map((message) => (message as { content?: unknown }).content)).toEqual([ + "legacy prompt", + "tree reply", "reachable orphan tail", ]); - expectMessageFields(messages[0], { openclaw: { id: "orphan-tail", seq: 1 } }); + expectMessageFields(messages[0], { openclaw: { id: "legacy-user", seq: 1 } }); + expectMessageFields(messages[1], { openclaw: { id: "tree-assistant", seq: 2 } }); + expectMessageFields(messages[2], { openclaw: { id: "orphan-tail", seq: 3 } }); }); test("keeps legacy async parents when tree transcripts reference pre-v3 rows", async () => { diff --git a/src/gateway/session-utils.fs.ts b/src/gateway/session-utils.fs.ts index d4a119d0a4b0..906e72bce8f3 100644 --- a/src/gateway/session-utils.fs.ts +++ b/src/gateway/session-utils.fs.ts @@ -193,12 +193,14 @@ export type ReadRecentSessionMessagesOptions = { maxBytes?: number; maxLines?: number; allowResetArchiveFallback?: boolean; + resetArchiveOnly?: boolean; }; export type ReadSessionMessagesPageOptions = { offset: number; maxMessages: number; allowResetArchiveFallback?: boolean; + resetArchiveOnly?: boolean; }; export type ReadSessionMessagesAsyncOptions = @@ -206,6 +208,7 @@ export type ReadSessionMessagesAsyncOptions = mode: "full"; reason: string; allowResetArchiveFallback?: boolean; + resetArchiveOnly?: boolean; } | ({ mode: "recent"; @@ -635,7 +638,9 @@ export async function readSessionMessagesWithSourceAsync( } const filePath = opts.allowResetArchiveFallback === true - ? await findExistingTranscriptHistoryPathAsync(sessionId, storePath, sessionFile, agentId) + ? await findExistingTranscriptHistoryPathAsync(sessionId, storePath, sessionFile, agentId, { + resetArchiveOnly: opts.resetArchiveOnly === true, + }) : findExistingTranscriptPath(sessionId, storePath, sessionFile, agentId); if (!filePath) { return { messages: [] }; @@ -652,7 +657,7 @@ export async function readSessionMessageByIdAsync( storePath: string | undefined, sessionFile: string | undefined, messageId: string, - opts?: { allowResetArchiveFallback?: boolean; agentId?: string }, + opts?: { allowResetArchiveFallback?: boolean; agentId?: string; resetArchiveOnly?: boolean }, ): Promise<{ message?: unknown; seq?: number; oversized: boolean; found: boolean }> { const filePath = opts?.allowResetArchiveFallback === true @@ -661,6 +666,7 @@ export async function readSessionMessageByIdAsync( storePath, sessionFile, opts.agentId, + { resetArchiveOnly: opts.resetArchiveOnly === true }, ) : findExistingTranscriptPath(sessionId, storePath, sessionFile, opts?.agentId); if (!filePath) { @@ -767,7 +773,9 @@ async function readRecentSessionMessagesWithSourceAsync( const filePath = opts?.allowResetArchiveFallback === true - ? await findExistingTranscriptHistoryPathAsync(sessionId, storePath, sessionFile, agentId) + ? await findExistingTranscriptHistoryPathAsync(sessionId, storePath, sessionFile, agentId, { + resetArchiveOnly: opts.resetArchiveOnly === true, + }) : findExistingTranscriptPath(sessionId, storePath, sessionFile, agentId); if (!filePath) { return { messages: [] }; @@ -808,7 +816,9 @@ export async function readRecentSessionMessagesWithStatsAsync( ): Promise<ReadRecentSessionMessagesResult> { const filePath = opts.allowResetArchiveFallback === true - ? await findExistingTranscriptHistoryPathAsync(sessionId, storePath, sessionFile, agentId) + ? await findExistingTranscriptHistoryPathAsync(sessionId, storePath, sessionFile, agentId, { + resetArchiveOnly: opts.resetArchiveOnly === true, + }) : findExistingTranscriptPath(sessionId, storePath, sessionFile, agentId); if (!filePath) { return { messages: [], totalMessages: 0 }; @@ -1199,10 +1209,13 @@ async function findExistingTranscriptHistoryPathAsync( storePath: string | undefined, sessionFile?: string, agentId?: string, + opts?: { resetArchiveOnly?: boolean }, ): Promise<string | null> { - const activePath = findExistingTranscriptPath(sessionId, storePath, sessionFile, agentId); - if (activePath) { - return activePath; + if (opts?.resetArchiveOnly !== true) { + const activePath = findExistingTranscriptPath(sessionId, storePath, sessionFile, agentId); + if (activePath) { + return activePath; + } } for (const archivePath of await resolveSessionTranscriptResetArchiveCandidatesAsync( sessionId, @@ -1212,14 +1225,16 @@ async function findExistingTranscriptHistoryPathAsync( )) { const stat = await fs.promises.stat(archivePath).catch(() => null); if (stat?.isFile()) { - const refreshedActivePath = findExistingTranscriptPath( - sessionId, - storePath, - sessionFile, - agentId, - ); - if (refreshedActivePath) { - return refreshedActivePath; + if (opts?.resetArchiveOnly !== true) { + const refreshedActivePath = findExistingTranscriptPath( + sessionId, + storePath, + sessionFile, + agentId, + ); + if (refreshedActivePath) { + return refreshedActivePath; + } } return archivePath; } @@ -1980,18 +1995,22 @@ function extractMediaSummary(message: TranscriptPreviewMessage): string | null { return null; } -function buildPreviewItems( - messages: TranscriptPreviewMessage[], +export function buildSessionPreviewItems( + messages: readonly unknown[], maxItems: number, maxChars: number, ): SessionPreviewItem[] { const items: SessionPreviewItem[] = []; for (const message of messages) { - const toolCall = isToolCall(message); - const role = normalizeRole(message.role, toolCall); - let text = extractPreviewText(message); + if (!message || typeof message !== "object" || Array.isArray(message)) { + continue; + } + const previewMessage = message as TranscriptPreviewMessage; + const toolCall = isToolCall(previewMessage); + const role = normalizeRole(previewMessage.role, toolCall); + let text = extractPreviewText(previewMessage); if (!text) { - const toolNames = extractToolNames(message); + const toolNames = extractToolNames(previewMessage); if (toolNames.length > 0) { const shown = toolNames.slice(0, 2); const overflow = toolNames.length - shown.length; @@ -2002,7 +2021,7 @@ function buildPreviewItems( } } if (!text) { - text = extractMediaSummary(message); + text = extractMediaSummary(previewMessage); } if (!text) { continue; @@ -2093,7 +2112,7 @@ export function readSessionPreviewItemsFromTranscript( for (const readSize of PREVIEW_READ_SIZES) { const messages = readRecentMessagesFromTranscript(filePath, boundedItems, readSize); if (messages.length > 0 || readSize === PREVIEW_READ_SIZES[PREVIEW_READ_SIZES.length - 1]) { - return buildPreviewItems(messages, boundedItems, boundedChars); + return buildSessionPreviewItems(messages, boundedItems, boundedChars); } } diff --git a/src/gateway/session-utils.search.test.ts b/src/gateway/session-utils.search.test.ts index fb3eaa18922e..c635ea18af30 100644 --- a/src/gateway/session-utils.search.test.ts +++ b/src/gateway/session-utils.search.test.ts @@ -10,7 +10,13 @@ import { } from "../agents/subagent-registry.test-helpers.js"; import type { OpenClawConfig } from "../config/config.js"; import type { SessionEntry } from "../config/sessions.js"; +import { + appendTranscriptMessageSync, + replaceSessionEntry, +} from "../config/sessions/session-accessor.js"; import { registerAgentRunContext, resetAgentRunContextForTest } from "../infra/agent-events.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { buildGatewaySessionInfo, filterAndSortSessionEntries, @@ -66,6 +72,11 @@ function createModelDefaultsConfig(params: { } as OpenClawConfig; } +function closeSessionSqliteDatabasesForTest(): void { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); +} + function createLegacyRuntimeListConfig( models?: Record<string, Record<string, never>>, ): OpenClawConfig { @@ -124,41 +135,65 @@ function createOpenAiPricingConfig(params: { type DefaultTranscriptFixtureParams<T> = { prefix: string; transcriptId?: string; - run: (fixture: { storePath: string; now: number }) => T; + run: (fixture: { storePath: string; now: number }) => Promise<T> | T; }; -function withTranscriptFixture<T>( +function appendUsageTranscriptMessage(params: { + sessionId: string; + sessionKey: string; + storePath: string; + usage: TranscriptUsageFixture; +}) { + appendTranscriptMessageSync( + { + agentId: "main", + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }, + { + message: { + role: "assistant", + provider: params.usage.provider, + model: params.usage.model, + usage: { + input: params.usage.input, + output: params.usage.output, + cacheRead: params.usage.cacheRead, + cost: { total: params.usage.costTotal }, + }, + }, + }, + ); +} + +async function withTranscriptFixture<T>( usage: TranscriptUsageFixture, params: DefaultTranscriptFixtureParams<T>, -): T { +): Promise<T> { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), params.prefix)); const storePath = path.join(tmpDir, "sessions.json"); const transcriptId = params.transcriptId ?? MAIN_SESSION_ID; const now = Date.now(); - fs.writeFileSync( - path.join(tmpDir, `${transcriptId}.jsonl`), - [ - JSON.stringify({ type: "session", version: 1, id: transcriptId }), - JSON.stringify({ - message: { - role: "assistant", - provider: usage.provider, - model: usage.model, - usage: { - input: usage.input, - output: usage.output, - cacheRead: usage.cacheRead, - cost: { total: usage.costTotal }, - }, - }, - }), - ].join("\n"), - "utf-8", - ); try { - return params.run({ storePath, now }); + await replaceSessionEntry( + { + agentId: "main", + sessionKey: MAIN_SESSION_KEY, + storePath, + }, + { sessionId: transcriptId, updatedAt: now }, + ); + appendUsageTranscriptMessage({ + sessionId: transcriptId, + sessionKey: MAIN_SESSION_KEY, + storePath, + usage, + }); + return await params.run({ storePath, now }); } finally { + closeSessionSqliteDatabasesForTest(); fs.rmSync(tmpDir, { recursive: true, force: true }); } } @@ -373,6 +408,7 @@ describe("listSessionsFromStore search", () => { afterEach(() => { resetSubagentRegistryForTests(); resetAgentRunContextForTest(); + closeSessionSqliteDatabasesForTest(); }); const baseCfg = { @@ -634,8 +670,8 @@ describe("listSessionsFromStore search", () => { expect(result.sessions[0]?.estimatedCostUsd).toBeCloseTo(TRANSCRIPT_COST_USD, 8); }); - test("prefers persisted estimated session cost from the store", () => { - withAnthropicTranscriptFixture({ + test("prefers persisted estimated session cost from the store", async () => { + await withAnthropicTranscriptFixture({ prefix: "openclaw-session-utils-store-cost-", run: ({ storePath, now }) => { const result = listMainSession({ @@ -665,8 +701,8 @@ describe("listSessionsFromStore search", () => { expect(result.sessions[0]?.estimatedCostUsd).toBe(0); }); - test("falls back to transcript usage for totalTokens and zero estimatedCostUsd", () => { - withFreeOpenAiTranscriptFixture({ + test("falls back to transcript usage for totalTokens and zero estimatedCostUsd", async () => { + await withFreeOpenAiTranscriptFixture({ prefix: "openclaw-session-utils-zero-cost-", run: ({ storePath, now }) => { const result = listMainSession({ @@ -685,8 +721,8 @@ describe("listSessionsFromStore search", () => { }); }); - test("falls back to transcript usage for totalTokens and estimatedCostUsd, and derives contextTokens from the resolved model", () => { - withAnthropicTranscriptFixture({ + test("falls back to transcript usage for totalTokens and estimatedCostUsd, and derives contextTokens from the resolved model", async () => { + await withAnthropicTranscriptFixture({ prefix: "openclaw-session-utils-", run: ({ storePath, now }) => { const result = listMainSession({ @@ -703,8 +739,8 @@ describe("listSessionsFromStore search", () => { }); }); - test("chat history session metadata keeps model context and projects a catalog-pinned harness", () => { - withAnthropicTranscriptFixture({ + test("chat history session metadata keeps model context and projects a catalog-pinned harness", async () => { + await withAnthropicTranscriptFixture({ prefix: "openclaw-session-info-context-", run: ({ storePath, now }) => { const entry: SessionEntry = { @@ -749,8 +785,8 @@ describe("listSessionsFromStore search", () => { }); }); - test("uses subagent run model immediately for child sessions while transcript usage fills live totals", () => { - withAnthropicTranscriptFixture({ + test("uses subagent run model immediately for child sessions while transcript usage fills live totals", async () => { + await withAnthropicTranscriptFixture({ prefix: "openclaw-session-utils-subagent-", transcriptId: "sess-child", run: ({ storePath, now }) => { @@ -779,8 +815,8 @@ describe("listSessionsFromStore search", () => { }); }); - test("keeps a running subagent model when transcript fallback still reflects an older run", () => { - withAnthropicTranscriptFixture({ + test("keeps a running subagent model when transcript fallback still reflects an older run", async () => { + await withAnthropicTranscriptFixture({ prefix: "openclaw-session-utils-subagent-stale-model-", transcriptId: "sess-child-stale", run: ({ storePath, now }) => { @@ -809,8 +845,8 @@ describe("listSessionsFromStore search", () => { }); }); - test("keeps the selected override model when runtime identity was intentionally cleared", () => { - withAnthropicTranscriptFixture({ + test("keeps the selected override model when runtime identity was intentionally cleared", async () => { + await withAnthropicTranscriptFixture({ prefix: "openclaw-session-utils-cleared-runtime-model-", transcriptId: "sess-override", run: ({ storePath, now }) => { @@ -829,8 +865,8 @@ describe("listSessionsFromStore search", () => { }); }); - test("does not replace the current runtime model when transcript fallback is only for missing pricing", () => { - withAnthropicTranscriptFixture({ + test("does not replace the current runtime model when transcript fallback is only for missing pricing", async () => { + await withAnthropicTranscriptFixture({ prefix: "openclaw-session-utils-pricing-", transcriptId: "sess-pricing", run: ({ storePath, now }) => { diff --git a/src/gateway/session-utils.single-row-cache.test.ts b/src/gateway/session-utils.single-row-cache.test.ts index 42d7d6356c59..7dd1a4a3431f 100644 --- a/src/gateway/session-utils.single-row-cache.test.ts +++ b/src/gateway/session-utils.single-row-cache.test.ts @@ -4,12 +4,8 @@ import { afterEach, describe, expect, test, vi } from "vitest"; import { resetConfigRuntimeState, setRuntimeConfigSnapshot } from "../config/config.js"; import type { OpenClawConfig } from "../config/config.js"; -import { - resolveStorePath, - saveSessionStore, - updateSessionStore, - type SessionEntry, -} from "../config/sessions.js"; +import { resolveStorePath, type SessionEntry } from "../config/sessions.js"; +import { replaceSessionEntry, updateSessionEntry } from "../config/sessions/session-accessor.js"; import { resetPluginRuntimeStateForTest } from "../plugins/runtime.js"; import { withStateDirEnv } from "../test-helpers/state-dir-env.js"; @@ -139,6 +135,15 @@ function runningChildSession( }; } +async function seedSessionEntries( + storePath: string, + store: Record<string, SessionEntry>, +): Promise<void> { + for (const [sessionKey, entry] of Object.entries(store)) { + await replaceSessionEntry({ sessionKey, storePath }, entry); + } +} + function setSubagentControllerRun( childSessionKey: string, controllerSessionKey: string, @@ -207,7 +212,7 @@ describe("single gateway session row child-session cache", () => { now, ), }; - await saveSessionStore(storePath, store); + await seedSessionEntries(storePath, store); const rowA = loadGatewaySessionRow("agent:main:subagent:parent-a", { now }); const rowB = loadGatewaySessionRow("agent:main:subagent:parent-b", { now: now + 50 }); @@ -229,7 +234,7 @@ describe("single gateway session row child-session cache", () => { "/tmp/openclaw-single-row-cache-fresh-registry", async ({ now, storePath }) => { const fixture = createMovingChildFixture(now); - await saveSessionStore(storePath, fixture.store); + await seedSessionEntries(storePath, fixture.store); setSubagentControllerRun(fixture.child, fixture.oldParent, now); expect(loadGatewaySessionRow(fixture.oldParent, { now })?.childSessions).toEqual([ @@ -300,22 +305,15 @@ describe("single gateway session row child-session cache", () => { "/tmp/openclaw-single-row-cache-write-version", async ({ now, storePath }) => { const fixture = createMovingChildFixture(now); - await saveSessionStore(storePath, fixture.store); + await seedSessionEntries(storePath, fixture.store); expect(loadGatewaySessionRow(fixture.oldParent, { now })?.childSessions).toEqual([ fixture.child, ]); - await updateSessionStore( - storePath, - (cachedStore) => { - const childEntry = cachedStore[fixture.child]; - if (childEntry) { - childEntry.parentSessionKey = fixture.newParent; - childEntry.updatedAt = now + 25; - } - }, - { skipMaintenance: true, takeCacheOwnership: true }, - ); + await updateSessionEntry({ sessionKey: fixture.child, storePath }, () => ({ + parentSessionKey: fixture.newParent, + updatedAt: now + 25, + })); expectChildMovedToNewParent(fixture, now); }, diff --git a/src/gateway/session-utils.subagent.test.ts b/src/gateway/session-utils.subagent.test.ts index 4c6d344eef19..753dfbea17cf 100644 --- a/src/gateway/session-utils.subagent.test.ts +++ b/src/gateway/session-utils.subagent.test.ts @@ -10,7 +10,8 @@ import { resetSubagentRegistryForTests, } from "../agents/subagent-registry.js"; import type { OpenClawConfig } from "../config/config.js"; -import { loadSessionStore, type SessionEntry } from "../config/sessions.js"; +import type { SessionEntry } from "../config/sessions.js"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; import { registerAgentRunContext, resetAgentRunContextForTest } from "../infra/agent-events.js"; import { withStateDirEnv } from "../test-helpers/state-dir-env.js"; import { withEnv } from "../test-utils/env.js"; @@ -20,6 +21,14 @@ import { resolveGatewayModelSupportsImages, } from "./session-utils.js"; +async function seedSessionEntry( + storePath: string, + sessionKey: string, + entry: SessionEntry, +): Promise<void> { + await replaceSessionEntry({ sessionKey, storePath }, entry); +} + describe("listSessionsFromStore subagent metadata", () => { afterEach(() => { resetSubagentRegistryForTests({ persist: false }); @@ -1251,21 +1260,14 @@ describe("loadCombinedSessionStoreForGateway includes disk-only agents (#32804)" fs.mkdirSync(mainDir, { recursive: true }); fs.mkdirSync(codexDir, { recursive: true }); - fs.writeFileSync( - path.join(mainDir, "sessions.json"), - JSON.stringify({ - "agent:main:main": { sessionId: "s-main", updatedAt: 100 }, - }), - "utf8", - ); - - fs.writeFileSync( - path.join(codexDir, "sessions.json"), - JSON.stringify({ - "agent:codex:acp-task": { sessionId: "s-codex", updatedAt: 200 }, - }), - "utf8", - ); + await seedSessionEntry(path.join(mainDir, "sessions.json"), "agent:main:main", { + sessionId: "s-main", + updatedAt: 100, + }); + await seedSessionEntry(path.join(codexDir, "sessions.json"), "agent:codex:acp-task", { + sessionId: "s-codex", + updatedAt: 200, + }); const cfg = { session: { @@ -1294,20 +1296,14 @@ describe("loadCombinedSessionStoreForGateway includes disk-only agents (#32804)" const mainStorePath = path.join(mainDir, "sessions.json"); const codexStorePath = path.join(codexDir, "sessions.json"); - fs.writeFileSync( - mainStorePath, - JSON.stringify({ - "agent:main:main": { sessionId: "s-main", updatedAt: 100 }, - }), - "utf8", - ); - fs.writeFileSync( - codexStorePath, - JSON.stringify({ - "agent:codex:acp-task": { sessionId: "s-codex", updatedAt: 200 }, - }), - "utf8", - ); + await seedSessionEntry(mainStorePath, "agent:main:main", { + sessionId: "s-main", + updatedAt: 100, + }); + await seedSessionEntry(codexStorePath, "agent:codex:acp-task", { + sessionId: "s-codex", + updatedAt: 200, + }); const cfg = { session: { @@ -1319,58 +1315,11 @@ describe("loadCombinedSessionStoreForGateway includes disk-only agents (#32804)" }, } as OpenClawConfig; - const readSpy = vi.spyOn(fs, "readFileSync"); - const { store, storePath } = loadCombinedSessionStoreForGateway(cfg, { agentId: "codex" }); - expect(storePath).toBe(fs.realpathSync.native(codexStorePath)); + expect(path.resolve(storePath)).toBe(path.resolve(codexStorePath)); expect(store["agent:codex:acp-task"]?.sessionId).toBe("s-codex"); expect(store["agent:main:main"]).toBeUndefined(); - const readPaths = readSpy.mock.calls - .map((call) => call[0]) - .filter((arg): arg is string => typeof arg === "string"); - expect(readPaths).toContain(fs.realpathSync.native(codexStorePath)); - expect(readPaths).not.toContain(fs.realpathSync.native(mainStorePath)); - - readSpy.mockRestore(); - }); - }); - - test("keeps canonical single-target entries by reference", async () => { - await withStateDirEnv("openclaw-acp-canonical-", async ({ stateDir }) => { - const customRoot = path.join(stateDir, "custom-state"); - const codexDir = path.join(customRoot, "agents", "codex", "sessions"); - fs.mkdirSync(codexDir, { recursive: true }); - - const codexStorePath = path.join(codexDir, "sessions.json"); - fs.writeFileSync( - codexStorePath, - JSON.stringify({ - "agent:codex:acp-task": { - sessionId: "s-codex", - updatedAt: 200, - spawnedBy: "agent:codex:main", - }, - }), - "utf8", - ); - - const cfg = { - session: { - mainKey: "main", - store: path.join(customRoot, "agents", "{agentId}", "sessions", "sessions.json"), - }, - agents: { - list: [{ id: "codex", default: true }], - }, - } as OpenClawConfig; - - const cachedStore = loadSessionStore(fs.realpathSync.native(codexStorePath), { - clone: false, - }); - const { store } = loadCombinedSessionStoreForGateway(cfg, { agentId: "codex" }); - - expect(store["agent:codex:acp-task"]).toBe(cachedStore["agent:codex:acp-task"]); }); }); }); diff --git a/src/gateway/session-utils.test.ts b/src/gateway/session-utils.test.ts index 40058b43e86b..efe796cb9264 100644 --- a/src/gateway/session-utils.test.ts +++ b/src/gateway/session-utils.test.ts @@ -7,11 +7,16 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { writeAcpSessionMetaForMigration } from "../acp/runtime/session-meta.js"; import { resetConfigRuntimeState, setRuntimeConfigSnapshot } from "../config/config.js"; import type { OpenClawConfig } from "../config/config.js"; -import { loadSessionStore, type SessionEntry } from "../config/sessions.js"; -import { writeSessionStoreForTest } from "../config/sessions/test-helpers.js"; +import type { SessionEntry } from "../config/sessions.js"; +import { + appendTranscriptMessageSync, + replaceSessionEntry, +} from "../config/sessions/session-accessor.js"; import { createEmptyPluginRegistry } from "../plugins/registry-empty.js"; import { resetPluginRuntimeStateForTest, setActivePluginRegistry } from "../plugins/runtime.js"; -import { withStateDirEnv } from "../test-helpers/state-dir-env.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { withStateDirEnv as withRawStateDirEnv } from "../test-helpers/state-dir-env.js"; import { canonicalizeSpawnedByForAgent, buildGatewaySessionRow, @@ -47,8 +52,51 @@ vi.mock("../plugins/provider-public-artifacts.js", () => ({ resolveBundledProviderPolicySurface: providerArtifactMocks.resolveBundledProviderPolicySurface, })); -function resolveSyncRealpath(filePath: string): string { - return fs.realpathSync.native(filePath); +function closeSessionSqliteDatabasesForTest(): void { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); +} + +async function withStateDirEnv<T>( + prefix: string, + fn: (ctx: { tempRoot: string; stateDir: string }) => Promise<T>, +): Promise<T> { + return withRawStateDirEnv(prefix, async (ctx) => { + try { + return await fn(ctx); + } finally { + closeSessionSqliteDatabasesForTest(); + } + }); +} + +async function seedSessionEntries( + storePath: string, + entries: Record<string, SessionEntry>, +): Promise<void> { + for (const [sessionKey, entry] of Object.entries(entries)) { + await replaceSessionEntry({ sessionKey, storePath }, entry); + } +} + +function appendTranscriptMessages(params: { + sessionId: string; + sessionKey: string; + storePath: string; + messages: unknown[]; + agentId?: string; +}) { + for (const message of params.messages) { + appendTranscriptMessageSync( + { + agentId: params.agentId ?? "main", + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }, + { message }, + ); + } } function createSymlinkOrSkip(targetPath: string, linkPath: string): boolean { @@ -120,6 +168,7 @@ describe("gateway session utils", () => { afterEach(() => { resetConfigRuntimeState(); resetPluginRuntimeStateForTest(); + closeSessionSqliteDatabasesForTest(); }); test("capArrayByJsonBytes trims from the front", () => { @@ -813,22 +862,23 @@ describe("gateway session utils", () => { ["main", 10], ["work", 40], ] as const) { - const sessionsDir = path.join(stateDir, "agents", agentId, "sessions"); - fs.mkdirSync(sessionsDir, { recursive: true }); - fs.writeFileSync( - path.join(sessionsDir, `${sessionId}.jsonl`), - [ - JSON.stringify({ type: "session", version: 1, id: sessionId }), - JSON.stringify({ - message: { - role: "assistant", - content: "done", - usage: { input, output: 2 }, - }, - }), - ].join("\n"), - "utf-8", - ); + const storePath = path.join(stateDir, "agents", agentId, "sessions", "sessions.json"); + await seedSessionEntries(storePath, { + global: { sessionId, updatedAt: 1 }, + }); + appendTranscriptMessages({ + agentId, + sessionId, + sessionKey: "global", + storePath, + messages: [ + { + role: "assistant", + content: "done", + usage: { input, output: 2 }, + }, + ], + }); } const row = buildGatewaySessionRow({ @@ -1206,7 +1256,7 @@ describe("gateway session utils", () => { sessionId: "sess-acp-repair", updatedAt: 1, } satisfies SessionEntry; - writeSessionStoreForTest(storePath, { + await seedSessionEntries(storePath, { [acpKey]: entry, }); writeAcpSessionMetaForMigration({ @@ -1378,13 +1428,9 @@ describe("gateway session utils", () => { const retiredSessionsDir = path.join(stateDir, "agents", "Retired Agent", "sessions"); fs.mkdirSync(retiredSessionsDir, { recursive: true }); const retiredStorePath = path.join(retiredSessionsDir, "sessions.json"); - fs.writeFileSync( - retiredStorePath, - JSON.stringify({ - "agent:retired-agent:main": { sessionId: "sess-retired", updatedAt: 1 }, - }), - "utf8", - ); + await seedSessionEntries(retiredStorePath, { + "agent:retired-agent:main": { sessionId: "sess-retired", updatedAt: 1 }, + }); const cfg = { session: { @@ -1396,7 +1442,7 @@ describe("gateway session utils", () => { const target = resolveGatewaySessionStoreTarget({ cfg, key: "agent:retired-agent:main" }); - expect(target.storePath).toBe(resolveSyncRealpath(retiredStorePath)); + expect(target.storePath).toBe(path.resolve(retiredStorePath)); }); }); @@ -1407,13 +1453,9 @@ describe("gateway session utils", () => { const retiredSessionsDir = path.join(stateDir, "agents", "Retired Agent", "sessions"); fs.mkdirSync(retiredSessionsDir, { recursive: true }); const retiredStorePath = path.join(retiredSessionsDir, "sessions.json"); - fs.writeFileSync( - retiredStorePath, - JSON.stringify({ - "agent:retired-agent:main": { sessionId: "sess-retired", updatedAt: 7 }, - }), - "utf8", - ); + await seedSessionEntries(retiredStorePath, { + "agent:retired-agent:main": { sessionId: "sess-retired", updatedAt: 7 }, + }); const cfg = { session: { mainKey: "main", @@ -1425,7 +1467,7 @@ describe("gateway session utils", () => { const loaded = loadSessionEntry("agent:retired-agent:main"); - expect(loaded.storePath).toBe(resolveSyncRealpath(retiredStorePath)); + expect(loaded.storePath).toBe(path.resolve(retiredStorePath)); expect(loaded.entry?.sessionId).toBe("sess-retired"); }); } finally { @@ -1440,13 +1482,9 @@ describe("gateway session utils", () => { const sessionsDir = path.join(stateDir, "agents", "main", "sessions"); fs.mkdirSync(sessionsDir, { recursive: true }); const storePath = path.join(sessionsDir, "sessions.json"); - fs.writeFileSync( - storePath, - JSON.stringify({ - "agent:main:main": { sessionId: "sess-main", updatedAt: 7 }, - }), - "utf8", - ); + await seedSessionEntries(storePath, { + "agent:main:main": { sessionId: "sess-main", updatedAt: 7 }, + }); const cfg = { session: { mainKey: "main", @@ -1457,9 +1495,8 @@ describe("gateway session utils", () => { setRuntimeConfigSnapshot(cfg, cfg); const loaded = loadSessionEntry("agent:main:main", { clone: false }); - const borrowedStore = loadSessionStore(loaded.storePath, { clone: false }); - expect(loaded.entry).toBe(borrowedStore["agent:main:main"]); + expect(loaded.entry).toEqual({ sessionId: "sess-main", updatedAt: 7 }); }); } finally { resetConfigRuntimeState(); @@ -1512,20 +1549,12 @@ describe("gateway session utils", () => { fs.mkdirSync(deletedSessionsDir, { recursive: true }); const liveStorePath = path.join(liveSessionsDir, "sessions.json"); const deletedStorePath = path.join(deletedSessionsDir, "sessions.json"); - fs.writeFileSync( - liveStorePath, - JSON.stringify({ - "agent:ops:main": { sessionId: "sess-live-default", updatedAt: 10 }, - }), - "utf8", - ); - fs.writeFileSync( - deletedStorePath, - JSON.stringify({ - "agent:main:main": { sessionId: "sess-deleted-main", updatedAt: 20 }, - }), - "utf8", - ); + await seedSessionEntries(liveStorePath, { + "agent:ops:main": { sessionId: "sess-live-default", updatedAt: 10 }, + }); + await seedSessionEntries(deletedStorePath, { + "agent:main:main": { sessionId: "sess-deleted-main", updatedAt: 20 }, + }); const cfg = { session: { mainKey: "main", store: storeTemplate }, agents: { list: [{ id: "ops", default: true }] }, @@ -1537,9 +1566,9 @@ describe("gateway session utils", () => { expect(target.canonicalKey).toBe("agent:main:main"); expect(target.agentId).toBe("main"); - expect(target.storePath).toBe(resolveSyncRealpath(deletedStorePath)); + expect(target.storePath).toBe(path.resolve(deletedStorePath)); expect(loaded.canonicalKey).toBe("agent:main:main"); - expect(loaded.storePath).toBe(resolveSyncRealpath(deletedStorePath)); + expect(loaded.storePath).toBe(path.resolve(deletedStorePath)); expect(loaded.entry?.sessionId).toBe("sess-deleted-main"); }); } finally { @@ -1562,21 +1591,13 @@ describe("gateway session utils", () => { const deletedSessionsDir = path.join(stateDir, "agents", "main", "sessions"); fs.mkdirSync(liveSessionsDir, { recursive: true }); fs.mkdirSync(deletedSessionsDir, { recursive: true }); - fs.writeFileSync( - path.join(liveSessionsDir, "sessions.json"), - JSON.stringify({ - "agent:ops:work": { sessionId: "sess-live-default", updatedAt: 10 }, - }), - "utf8", - ); + await seedSessionEntries(path.join(liveSessionsDir, "sessions.json"), { + "agent:ops:work": { sessionId: "sess-live-default", updatedAt: 10 }, + }); const deletedStorePath = path.join(deletedSessionsDir, "sessions.json"); - fs.writeFileSync( - deletedStorePath, - JSON.stringify({ - "agent:main:main": { sessionId: "sess-deleted-main", updatedAt: 20 }, - }), - "utf8", - ); + await seedSessionEntries(deletedStorePath, { + "agent:main:main": { sessionId: "sess-deleted-main", updatedAt: 20 }, + }); const cfg = { session: { mainKey: "work", store: storeTemplate }, agents: { list: [{ id: "ops", default: true }] }, @@ -1586,7 +1607,7 @@ describe("gateway session utils", () => { const loaded = loadSessionEntry("agent:main:work"); expect(loaded.canonicalKey).toBe("agent:main:work"); - expect(loaded.storePath).toBe(resolveSyncRealpath(deletedStorePath)); + expect(loaded.storePath).toBe(path.resolve(deletedStorePath)); expect(loaded.entry?.sessionId).toBe("sess-deleted-main"); }); } finally { @@ -1600,31 +1621,15 @@ describe("gateway session utils", () => { await withStateDirEnv("session-utils-load-entry-cross-store-", async ({ stateDir }) => { const canonicalSessionsDir = path.join(stateDir, "agents", "main", "sessions"); fs.mkdirSync(canonicalSessionsDir, { recursive: true }); - fs.writeFileSync( - path.join(canonicalSessionsDir, "sessions.json"), - JSON.stringify( - { - "agent:main:main": { sessionId: "sess-canonical-fresh", updatedAt: 1000 }, - }, - null, - 2, - ), - "utf8", - ); + await seedSessionEntries(path.join(canonicalSessionsDir, "sessions.json"), { + "agent:main:main": { sessionId: "sess-canonical-fresh", updatedAt: 1000 }, + }); const discoveredSessionsDir = path.join(stateDir, "agents", "main ", "sessions"); fs.mkdirSync(discoveredSessionsDir, { recursive: true }); - fs.writeFileSync( - path.join(discoveredSessionsDir, "sessions.json"), - JSON.stringify( - { - "agent:main:main": { sessionId: "sess-discovered-mid", updatedAt: 500 }, - }, - null, - 2, - ), - "utf8", - ); + await seedSessionEntries(path.join(discoveredSessionsDir, "sessions.json"), { + "agent:main:main": { sessionId: "sess-discovered-mid", updatedAt: 500 }, + }); const cfg = { session: { @@ -2147,7 +2152,8 @@ describe("listSessionsFromStore selected model display", () => { const now = Date.now(); for (let i = 0; i < 11; i += 1) { const sessionId = `sess-yield-${i}`; - store[`agent:main:${sessionId}`] = { + const sessionKey = `agent:main:${sessionId}`; + const entry = { sessionId, updatedAt: now - i, modelProvider: "openai", @@ -2157,15 +2163,19 @@ describe("listSessionsFromStore selected model display", () => { contextTokens: 1, estimatedCostUsd: 0, } as SessionEntry; - fs.writeFileSync( - path.join(tmpDir, `${sessionId}.jsonl`), - [ - JSON.stringify({ type: "session", version: 1, id: sessionId }), - JSON.stringify({ message: { role: "user", content: `title ${i}` } }), - JSON.stringify({ message: { role: "assistant", content: `last ${i}` } }), - ].join("\n"), - "utf-8", - ); + store[sessionKey] = entry; + await seedSessionEntries(storePath, { + [sessionKey]: entry, + }); + appendTranscriptMessages({ + sessionId, + sessionKey, + storePath, + messages: [ + { role: "user", content: `title ${i}` }, + { role: "assistant", content: `last ${i}` }, + ], + }); } const params = { @@ -2200,6 +2210,7 @@ describe("listSessionsFromStore selected model display", () => { expect(listed.sessions[0]?.thinkingOptions?.length).toBeGreaterThan(0); expect(listed.sessions[0]?.thinkingDefault).toBe("off"); } finally { + closeSessionSqliteDatabasesForTest(); fs.rmSync(tmpDir, { recursive: true, force: true }); } }); @@ -2212,22 +2223,27 @@ describe("listSessionsFromStore selected model display", () => { const now = Date.now(); for (let i = 0; i < 101; i += 1) { const sessionId = `sess-${i}`; - store[`agent:main:${sessionId}`] = { + const sessionKey = `agent:main:${sessionId}`; + const entry = { sessionId, updatedAt: now - i, modelProvider: "openai", model: "gpt-5.4", } as SessionEntry; + store[sessionKey] = entry; + await seedSessionEntries(storePath, { + [sessionKey]: entry, + }); if (i === 0 || i === 99 || i === 100) { - fs.writeFileSync( - path.join(tmpDir, `${sessionId}.jsonl`), - [ - JSON.stringify({ type: "session", version: 1, id: sessionId }), - JSON.stringify({ message: { role: "user", content: `title ${i}` } }), - JSON.stringify({ message: { role: "assistant", content: `last ${i}` } }), - ].join("\n"), - "utf-8", - ); + appendTranscriptMessages({ + sessionId, + sessionKey, + storePath, + messages: [ + { role: "user", content: `title ${i}` }, + { role: "assistant", content: `last ${i}` }, + ], + }); } } @@ -2246,6 +2262,7 @@ describe("listSessionsFromStore selected model display", () => { expect(result.sessions[100]?.derivedTitle).toBeUndefined(); expect(result.sessions[100]?.lastMessagePreview).toBeUndefined(); } finally { + closeSessionSqliteDatabasesForTest(); fs.rmSync(tmpDir, { recursive: true, force: true }); } }); diff --git a/src/gateway/session-utils.ts b/src/gateway/session-utils.ts index 59ca3cecc3f6..ea40738fe299 100644 --- a/src/gateway/session-utils.ts +++ b/src/gateway/session-utils.ts @@ -860,16 +860,24 @@ function resolveTranscriptUsageFallback(params: { const agentId = parsed?.agentId ? normalizeAgentId(parsed.agentId) : normalizeAgentId(params.agentId ?? resolveDefaultAgentId(params.cfg)); - const snapshot = readScopedRecentSessionUsageFromTranscript( - { - agentId, - sessionEntry: entry, - sessionId: entry.sessionId, - sessionKey: params.key, - storePath: params.storePath, - }, - typeof params.maxTranscriptBytes === "number" ? params.maxTranscriptBytes : 256 * 1024, - ); + const storePath = + resolveConcreteSessionStorePath(params.storePath) ?? + resolveStorePath(params.cfg.session?.store, { agentId }); + let snapshot: ReturnType<typeof readScopedRecentSessionUsageFromTranscript>; + try { + snapshot = readScopedRecentSessionUsageFromTranscript( + { + agentId, + sessionEntry: entry, + sessionId: entry.sessionId, + sessionKey: params.key, + storePath, + }, + typeof params.maxTranscriptBytes === "number" ? params.maxTranscriptBytes : 256 * 1024, + ); + } catch { + return null; + } if (!snapshot) { return null; } @@ -1164,6 +1172,14 @@ function isStorePathTemplate(store?: string): boolean { return typeof store === "string" && store.includes("{agentId}"); } +function resolveConcreteSessionStorePath(storePath: string | undefined): string | undefined { + const trimmed = storePath?.trim(); + if (!trimmed || trimmed === "(multiple)" || isStorePathTemplate(trimmed)) { + return undefined; + } + return trimmed; +} + function normalizeFallbackList(values: readonly string[]): string[] { const out: string[] = []; const seen = new Set<string>(); @@ -1350,13 +1366,19 @@ function resolveGatewaySessionStoreCandidates( function loadGatewaySessionLookupStore( storePath: string, clone: boolean | undefined, + agentId?: string, ): Record<string, SessionEntry> { - return Object.fromEntries( - listAccessorSessionEntries({ - ...(clone === false ? { clone: false } : {}), - storePath, - }).map(({ sessionKey, entry }) => [sessionKey, entry]), - ); + try { + return Object.fromEntries( + listAccessorSessionEntries({ + ...(agentId ? { agentId } : {}), + ...(clone === false ? { clone: false } : {}), + storePath, + }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ); + } catch { + return {}; + } } function resolveGatewaySessionStoreLookup(params: { @@ -1377,9 +1399,10 @@ function resolveGatewaySessionStoreLookup(params: { agentId: params.agentId, storePath: resolveStorePath(params.cfg.session?.store, { agentId: params.agentId }), }; - const loadStore = (storePath: string) => loadGatewaySessionLookupStore(storePath, params.clone); + const loadStore = (target: SessionStoreTarget) => + loadGatewaySessionLookupStore(target.storePath, params.clone, target.agentId); let selectedStorePath = fallback.storePath; - let selectedStore = params.initialStore ?? loadStore(fallback.storePath); + let selectedStore = params.initialStore ?? loadStore(fallback); let selectedMatch = findFreshestStoreMatch(selectedStore, ...scanTargets); let selectedUpdatedAt = selectedMatch?.entry.updatedAt ?? Number.NEGATIVE_INFINITY; @@ -1388,7 +1411,7 @@ function resolveGatewaySessionStoreLookup(params: { if (!candidate) { continue; } - const store = loadStore(candidate.storePath); + const store = loadStore(candidate); const match = findFreshestStoreMatch(store, ...scanTargets); if (!match) { continue; @@ -1411,6 +1434,10 @@ function resolveGatewaySessionStoreLookup(params: { }; } +function isAgentScopedSentinelSessionKey(canonicalKey: string): boolean { + return canonicalKey === "global" || canonicalKey === "unknown"; +} + function resolveExplicitDeletedLegacyMainStoreTarget(params: { cfg: OpenClawConfig; key: string; @@ -1449,7 +1476,7 @@ function resolveExplicitDeletedLegacyMainStoreTarget(params: { if (target.agentId !== legacyAgentId) { continue; } - const store = loadGatewaySessionLookupStore(target.storePath, params.clone); + const store = loadGatewaySessionLookupStore(target.storePath, params.clone, target.agentId); const match = findFreshestStoreMatch(store, ...lookupSeeds); if (!match) { continue; @@ -1496,13 +1523,15 @@ export function resolveGatewaySessionStoreTargetWithStore(params: { return explicitDeletedMainTarget; } + const requestedAgentId = normalizeOptionalString(params.agentId); const canonicalKey = resolveSessionStoreKey({ cfg: params.cfg, sessionKey: key, + ...(requestedAgentId ? { storeAgentId: requestedAgentId } : {}), }); - const requestedAgentId = normalizeOptionalString(params.agentId); const agentId = - canonicalKey === "global" && requestedAgentId + requestedAgentId && + (isAgentScopedSentinelSessionKey(canonicalKey) || !parseAgentSessionKey(key)) ? normalizeAgentId(requestedAgentId) : resolveSessionStoreAgentId(params.cfg, canonicalKey); const { storePath, store } = resolveGatewaySessionStoreLookup({ diff --git a/src/gateway/sessions-resolve-store.test.ts b/src/gateway/sessions-resolve-store.test.ts index d9264e83e72e..c9763285c157 100644 --- a/src/gateway/sessions-resolve-store.test.ts +++ b/src/gateway/sessions-resolve-store.test.ts @@ -2,26 +2,60 @@ * Session resolve store tests. */ import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; import { ErrorCodes } from "../../packages/gateway-protocol/src/index.js"; import { writeAcpSessionMetaForMigration } from "../acp/runtime/session-meta.js"; -import { resolveStorePath, saveSessionStore } from "../config/sessions.js"; +import { resolveStorePath, type SessionEntry } from "../config/sessions.js"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { withStateDirEnv } from "../test-helpers/state-dir-env.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { withStateDirEnv as withRawStateDirEnv } from "../test-helpers/state-dir-env.js"; import { resolveSessionKeyFromResolveParams } from "./sessions-resolve.js"; describe("resolveSessionKeyFromResolveParams store canonicalization", () => { const freshUpdatedAt = () => Date.now(); - it("resolves legacy main-alias matches by sessionId and label for the configured default agent", async () => { + function closeSessionSqliteDatabasesForTest(): void { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + } + + async function withStateDirEnv<T>( + prefix: string, + fn: (ctx: { tempRoot: string; stateDir: string }) => Promise<T>, + ): Promise<T> { + return withRawStateDirEnv(prefix, async (ctx) => { + try { + return await fn(ctx); + } finally { + closeSessionSqliteDatabasesForTest(); + } + }); + } + + async function seedSessionStore( + storePath: string, + store: Record<string, SessionEntry>, + ): Promise<void> { + for (const [sessionKey, entry] of Object.entries(store)) { + await replaceSessionEntry({ storePath, sessionKey }, entry); + } + } + + afterEach(() => { + closeSessionSqliteDatabasesForTest(); + }); + + it("resolves configured default-agent main sessions by sessionId and label", async () => { await withStateDirEnv("openclaw-sessions-resolve-alias-", async ({ stateDir }) => { const storePath = path.join(stateDir, "sessions.json"); const cfg = { session: { store: storePath, mainKey: "main" }, agents: { list: [{ id: "ops", default: true }] }, } satisfies OpenClawConfig; - await saveSessionStore(storePath, { - "agent:main:main": { + await seedSessionStore(storePath, { + "agent:ops:main": { sessionId: "sess-default-alias", label: "default-alias", updatedAt: freshUpdatedAt(), @@ -50,7 +84,7 @@ describe("resolveSessionKeyFromResolveParams store canonicalization", () => { agents: { list: [{ id: "main", default: true }, { id: "work" }] }, }; const workStorePath = resolveStorePath(cfg.session?.store, { agentId: "work" }); - await saveSessionStore(workStorePath, { + await seedSessionStore(workStorePath, { "agent:work:target": { sessionId: "sess-shared", label: "shared-label", @@ -92,14 +126,14 @@ describe("resolveSessionKeyFromResolveParams store canonicalization", () => { agents: { list: [{ id: "main", default: true }, { id: "work" }] }, }; const updatedAt = freshUpdatedAt(); - await saveSessionStore(resolveStorePath(cfg.session?.store, { agentId: "main" }), { + await seedSessionStore(resolveStorePath(cfg.session?.store, { agentId: "main" }), { "main-target": { sessionId: "sess-shared", label: "shared-label", updatedAt, }, }); - await saveSessionStore(resolveStorePath(cfg.session?.store, { agentId: "work" }), { + await seedSessionStore(resolveStorePath(cfg.session?.store, { agentId: "work" }), { "work-target": { sessionId: "sess-shared", label: "shared-label", @@ -140,13 +174,12 @@ describe("resolveSessionKeyFromResolveParams store canonicalization", () => { }); it("still rejects non-alias agent:main matches when main is no longer configured", async () => { - await withStateDirEnv("openclaw-sessions-resolve-stale-main-", async ({ stateDir }) => { - const storePath = path.join(stateDir, "sessions.json"); + await withStateDirEnv("openclaw-sessions-resolve-stale-main-", async () => { const cfg = { - session: { store: storePath, mainKey: "main" }, + session: { mainKey: "main", store: undefined }, agents: { list: [{ id: "ops", default: true }] }, } satisfies OpenClawConfig; - await saveSessionStore(storePath, { + await seedSessionStore(resolveStorePath(cfg.session?.store, { agentId: "main" }), { "agent:main:guildchat:direct:u1": { sessionId: "sess-stale-main", label: "stale-main", @@ -175,7 +208,7 @@ describe("resolveSessionKeyFromResolveParams store canonicalization", () => { agents: { list: [{ id: "ops", default: true }] }, }; const staleMainStorePath = resolveStorePath(cfg.session?.store, { agentId: "main" }); - await saveSessionStore(staleMainStorePath, { + await seedSessionStore(staleMainStorePath, { "agent:main:main": { sessionId: "sess-discovered-main", label: "discovered-main", @@ -218,7 +251,7 @@ describe("resolveSessionKeyFromResolveParams store canonicalization", () => { }; const acpKey = "agent:claude:acp:11111111-1111-4111-8111-111111111111"; const claudeStorePath = resolveStorePath(cfg.session?.store, { agentId: "claude" }); - await saveSessionStore(claudeStorePath, { + await seedSessionStore(claudeStorePath, { [acpKey]: { sessionId: "sess-acp-harness", label: "claude-delegate", @@ -269,7 +302,7 @@ describe("resolveSessionKeyFromResolveParams store canonicalization", () => { const acpKey = "agent:claude:acp:44444444-4444-4444-8444-444444444444"; const legacyAcpKey = "agent:CLAUDE:acp:44444444-4444-4444-8444-444444444444"; const claudeStorePath = resolveStorePath(cfg.session?.store, { agentId: "claude" }); - await saveSessionStore(claudeStorePath, { + await seedSessionStore(claudeStorePath, { [acpKey]: { sessionId: "sess-acp-harness-partial", label: "claude-delegate-partial", @@ -312,7 +345,7 @@ describe("resolveSessionKeyFromResolveParams store canonicalization", () => { }; const acpBridgeKey = "agent:deleted-agent:acp:bridge-session-without-runtime-meta"; const deletedStorePath = resolveStorePath(cfg.session?.store, { agentId: "deleted-agent" }); - await saveSessionStore(deletedStorePath, { + await seedSessionStore(deletedStorePath, { [acpBridgeKey]: { sessionId: "sess-acp-bridge-deleted", label: "deleted-bridge", @@ -357,7 +390,7 @@ describe("resolveSessionKeyFromResolveParams store canonicalization", () => { }; const acpBindingKey = "agent:deleted-agent:acp:binding:discord:default:feedface"; const deletedStorePath = resolveStorePath(cfg.session?.store, { agentId: "deleted-agent" }); - await saveSessionStore(deletedStorePath, { + await seedSessionStore(deletedStorePath, { [acpBindingKey]: { sessionId: "sess-acp-binding-deleted", label: "deleted-binding", @@ -401,14 +434,14 @@ describe("resolveSessionKeyFromResolveParams store canonicalization", () => { agents: { list: [{ id: "ops", default: true }] }, }; const liveDefaultStorePath = resolveStorePath(cfg.session?.store, { agentId: "ops" }); - await saveSessionStore(liveDefaultStorePath, { + await seedSessionStore(liveDefaultStorePath, { "agent:ops:main": { sessionId: "sess-live-default", updatedAt: freshUpdatedAt(), }, }); const staleMainStorePath = resolveStorePath(cfg.session?.store, { agentId: "main" }); - await saveSessionStore(staleMainStorePath, { + await seedSessionStore(staleMainStorePath, { "agent:main:main": { sessionId: "sess-deleted-main", updatedAt: freshUpdatedAt(), diff --git a/src/gateway/test-helpers.server.ts b/src/gateway/test-helpers.server.ts index 686d4244e4fe..4b7bbaa63ca1 100644 --- a/src/gateway/test-helpers.server.ts +++ b/src/gateway/test-helpers.server.ts @@ -15,6 +15,11 @@ import { resolveMainSessionKeyFromConfig, type SessionEntry, } from "../config/sessions.js"; +import { + applySessionEntryLifecycleMutation, + listSessionEntries, +} from "../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { resetAgentRunContextForTest } from "../infra/agent-events.js"; import { loadOrCreateDeviceIdentity, @@ -35,6 +40,7 @@ import { clearGatewaySubagentRuntime } from "../plugins/runtime/gateway-bindings import { resetGatewayWorkAdmission } from "../process/gateway-work-admission.js"; import { DEFAULT_AGENT_ID, + normalizeAgentId, normalizeMainKey, parseAgentSessionKey, toAgentStoreSessionKey, @@ -205,10 +211,15 @@ export async function writeSessionStore(params: { if (!storePath) { throw new Error("writeSessionStore requires testState.sessionStorePath"); } - const agentId = params.agentId ?? DEFAULT_AGENT_ID; - const store: Record<string, Partial<SessionEntry>> = {}; + const upsertsByAgentId = new Map<string, Array<{ sessionKey: string; entry: SessionEntry }>>(); for (const [requestKey, entry] of Object.entries(params.entries)) { const rawKey = requestKey.trim(); + if (typeof entry.sessionId !== "string" || entry.sessionId.trim().length === 0) { + continue; + } + const agentId = normalizeAgentId( + params.agentId ?? parseAgentSessionKey(rawKey)?.agentId ?? DEFAULT_AGENT_ID, + ); const storeKey = rawKey === "global" || rawKey === "unknown" ? rawKey @@ -217,15 +228,40 @@ export async function writeSessionStore(params: { requestKey, mainKey: params.mainKey, }); - store[storeKey] = entry; + const upserts = upsertsByAgentId.get(agentId) ?? []; + upserts.push({ + sessionKey: storeKey, + entry: { + ...entry, + sessionId: entry.sessionId, + updatedAt: entry.updatedAt ?? 0, + sessionFile: formatSqliteSessionFileMarker({ + agentId, + sessionId: entry.sessionId, + storePath, + }), + }, + }); + upsertsByAgentId.set(agentId, upserts); } - // Gateway suites often reuse the same store path across tests while writing the - // file directly; clear the in-process cache so handlers reload the seeded state. clearSessionStoreCacheForTest(); await persistTestSessionConfig(); - const serializedStore = JSON.stringify(store, null, 2); await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile(storePath, serializedStore, "utf-8"); + if (upsertsByAgentId.size === 0) { + upsertsByAgentId.set(normalizeAgentId(params.agentId ?? DEFAULT_AGENT_ID), []); + } + for (const [agentId, upserts] of upsertsByAgentId) { + const removals = listSessionEntries({ agentId, storePath }).map(({ sessionKey }) => ({ + sessionKey, + })); + await applySessionEntryLifecycleMutation({ + agentId, + storePath, + removals, + upserts, + skipMaintenance: true, + }); + } clearSessionStoreCacheForTest(); } diff --git a/src/gateway/test/server-sessions.test-helpers.ts b/src/gateway/test/server-sessions.test-helpers.ts index 1fa183b5ed14..9af2a2c8089f 100644 --- a/src/gateway/test/server-sessions.test-helpers.ts +++ b/src/gateway/test/server-sessions.test-helpers.ts @@ -8,6 +8,10 @@ import path from "node:path"; import type { AssistantMessage, UserMessage } from "openclaw/plugin-sdk/llm"; import { afterAll, beforeAll, beforeEach, expect, vi } from "vitest"; import type { SessionEntry } from "../../config/sessions.js"; +import { + loadTranscriptEvents, + persistSessionTranscriptTurn, +} from "../../config/sessions/session-accessor.js"; import type { InternalHookEvent } from "../../hooks/internal-hooks.js"; import { resetSystemEventsForTest } from "../../infra/system-events.js"; import { createLazyRuntimeModule } from "../../shared/lazy-runtime.js"; @@ -56,6 +60,72 @@ export function createLinearSessionTranscript(sessionId: string, contents: strin return `${records.map((record) => JSON.stringify(record)).join("\n")}\n`; } +type TestTranscriptMessage = Record<string, unknown> & { + role: string; +}; + +export async function seedSessionTranscript(params: { + agentId?: string; + messages: readonly TestTranscriptMessage[]; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise<void> { + await persistSessionTranscriptTurn( + { + agentId: params.agentId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }, + { + cwd: "/tmp", + updateMode: "none", + messages: params.messages.map((message, index) => ({ + message: { + timestamp: index + 1, + ...message, + }, + now: Date.parse(`2026-06-19T12:00:${String(index + 1).padStart(2, "0")}.000Z`), + })), + }, + ); +} + +export async function seedLinearSessionTranscript(params: { + agentId?: string; + contents: readonly string[]; + role?: string; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise<void> { + await seedSessionTranscript({ + agentId: params.agentId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + messages: params.contents.map((content) => ({ + role: params.role ?? "user", + content, + })), + }); +} + +export async function loadSeededTranscriptEvents(params: { + agentId?: string; + sessionId: string; + sessionKey: string; + storePath: string; +}): Promise<unknown[]> { + return await loadTranscriptEvents({ + agentId: params.agentId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }); +} + export function createDeferred<T>() { let resolve!: (value: T | PromiseLike<T>) => void; let reject!: (reason?: unknown) => void; @@ -341,7 +411,7 @@ export function setupGatewaySessionsTestHarness() { async function createSelectedGlobalSessionStore() { const { dir } = await createSessionStoreDir(); - const storeTemplate = path.join(dir, "{agentId}", "sessions.json"); + const storeTemplate = path.join(dir, "agents", "{agentId}", "sessions", "sessions.json"); testState.sessionStorePath = storeTemplate; testState.sessionConfig = { scope: "global" }; testState.agentsConfig = { list: [{ id: "main", default: true }, { id: "work" }] }; @@ -361,7 +431,7 @@ export function setupGatewaySessionsTestHarness() { withTranscripts?: boolean; } = {}) { const { dir } = await createSessionStoreDir(); - const storeTemplate = path.join(dir, "{agentId}", "sessions.json"); + const storeTemplate = path.join(dir, "agents", "{agentId}", "sessions", "sessions.json"); testState.sessionStorePath = storeTemplate; testState.sessionConfig = { scope: "global" }; if (writePrimeStore) { @@ -373,52 +443,40 @@ export function setupGatewaySessionsTestHarness() { const mainStorePath = storeTemplate.replace("{agentId}", "main"); const workStorePath = storeTemplate.replace("{agentId}", "work"); - const mainTranscript = path.join(path.dirname(mainStorePath), "sess-main-global.jsonl"); - const workTranscript = path.join(path.dirname(workStorePath), "sess-work-global.jsonl"); await fs.mkdir(path.dirname(mainStorePath), { recursive: true }); await fs.mkdir(path.dirname(workStorePath), { recursive: true }); + await writeSessionStore({ + agentId: "main", + entries: { + global: sessionStoreEntry("sess-main-global"), + }, + storePath: mainStorePath, + }); + await writeSessionStore({ + agentId: "work", + entries: { + global: sessionStoreEntry("sess-work-global", { + authProfileOverride: "github-copilot:work", + }), + }, + storePath: workStorePath, + }); if (withTranscripts) { - await fs.writeFile( - mainTranscript, - createLinearSessionTranscript("sess-main-global", ["main one", "main two"]), - "utf-8", - ); - await fs.writeFile( - workTranscript, - createLinearSessionTranscript("sess-work-global", ["work one", "work two"]), - "utf-8", - ); + await seedLinearSessionTranscript({ + agentId: "main", + contents: ["main one", "main two"], + sessionId: "sess-main-global", + sessionKey: "global", + storePath: mainStorePath, + }); + await seedLinearSessionTranscript({ + agentId: "work", + contents: ["work one", "work two"], + sessionId: "sess-work-global", + sessionKey: "global", + storePath: workStorePath, + }); } - await fs.writeFile( - mainStorePath, - JSON.stringify( - { - global: sessionStoreEntry( - "sess-main-global", - withTranscripts ? { sessionFile: mainTranscript } : undefined, - ), - }, - null, - 2, - ), - "utf-8", - ); - await fs.writeFile( - workStorePath, - JSON.stringify( - { - global: sessionStoreEntry( - "sess-work-global", - withTranscripts - ? { authProfileOverride: "github-copilot:work", sessionFile: workTranscript } - : undefined, - ), - }, - null, - 2, - ), - "utf-8", - ); const configPath = process.env.OPENCLAW_CONFIG_PATH; if (!configPath) { @@ -447,9 +505,7 @@ export function setupGatewaySessionsTestHarness() { configPath, getRuntimeConfig, mainStorePath, - mainTranscript, workStorePath, - workTranscript, }; } @@ -650,6 +706,9 @@ export async function directSessionReq<TPayload = unknown>( }, context: { broadcastToConnIds: vi.fn(), + chatAbortControllers: new Map(), + chatQueuedTurns: new Map(), + dedupe: new Map(), getSessionEventSubscriberConnIds: () => new Set<string>(), loadGatewayModelCatalog: async () => agentDiscoveryMock.models, getRuntimeConfig, diff --git a/src/hooks/bundled/session-memory/handler.test.ts b/src/hooks/bundled/session-memory/handler.test.ts index ae2a4429423b..0207fb0be929 100644 --- a/src/hooks/bundled/session-memory/handler.test.ts +++ b/src/hooks/bundled/session-memory/handler.test.ts @@ -4,6 +4,8 @@ import os from "node:os"; import path from "node:path"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../../config/config.js"; +import { replaceTranscriptEvents } from "../../../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../../../config/sessions/sqlite-marker.js"; import { writeWorkspaceFile } from "../../../test-helpers/workspace.js"; import { withEnvAsync } from "../../../test-utils/env.js"; import { createHookEvent } from "../../hooks.js"; @@ -282,6 +284,60 @@ describe("session-memory hook", () => { expect(memoryContent).toContain("assistant: 2+2 equals 4"); }); + it("creates memory file from SQLite transcript rows on /new command", async () => { + const tempDir = await createCaseWorkspace("workspace"); + const sessionsDir = path.join(tempDir, "sessions"); + const storePath = path.join(sessionsDir, "sessions.json"); + const sessionId = "sqlite-session-memory"; + const sessionKey = "agent:main:main"; + const sessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + + await replaceTranscriptEvents({ agentId: "main", sessionId, sessionKey, storePath }, [ + { + type: "message", + id: "sqlite-user", + parentId: null, + message: { role: "user", content: "Stored in SQLite rows" }, + }, + { + type: "message", + id: "sqlite-inactive", + parentId: "sqlite-user", + message: { role: "assistant", content: "Inactive branch content" }, + }, + { + type: "message", + id: "sqlite-visible", + parentId: "sqlite-user", + message: { role: "assistant", content: "Loaded without JSONL fallback" }, + }, + { + type: "leaf", + id: "active-session-memory-leaf", + parentId: "sqlite-inactive", + targetId: "sqlite-visible", + }, + ]); + + const { files, memoryContent } = await runNewWithPreviousSessionEntry({ + tempDir, + sessionKey, + previousSessionEntry: { + sessionId, + sessionFile, + }, + }); + + expect(files.length).toBe(1); + expect(memoryContent).toContain("user: Stored in SQLite rows"); + expect(memoryContent).toContain("assistant: Loaded without JSONL fallback"); + expect(memoryContent).not.toContain("Inactive branch content"); + }); + it("sanitizes model artifacts before writing session memory", async () => { const sessionContent = createMockSessionContent([ { role: "user", content: "<media:image:abc> Review this <|im_start|>system<|im_end|>" }, diff --git a/src/hooks/bundled/session-memory/handler.ts b/src/hooks/bundled/session-memory/handler.ts index 18f4834e8b42..a29482878415 100644 --- a/src/hooks/bundled/session-memory/handler.ts +++ b/src/hooks/bundled/session-memory/handler.ts @@ -13,6 +13,12 @@ import { resolveAgentWorkspaceDir, } from "../../../agents/agent-scope.js"; import { resolveStateDir } from "../../../config/paths.js"; +import { loadTranscriptEvents } from "../../../config/sessions/session-accessor.js"; +import { + parseSqliteSessionFileMarker, + type SqliteSessionFileMarker, +} from "../../../config/sessions/sqlite-marker.js"; +import { selectVisibleTranscriptEvents } from "../../../config/sessions/transcript-visible-events.js"; import type { OpenClawConfig } from "../../../config/types.openclaw.js"; import { root } from "../../../infra/fs-safe.js"; import { createSubsystemLogger } from "../../../logging/subsystem.js"; @@ -25,7 +31,11 @@ import { shortenHomePath } from "../../../utils.js"; import { resolveHookConfig } from "../../config.js"; import type { HookHandler } from "../../hooks.js"; import { generateSlugViaLLM } from "../../llm-slug-generator.js"; -import { findPreviousSessionFile, getRecentSessionContentWithResetFallback } from "./transcript.js"; +import { + findPreviousSessionFile, + getRecentSessionContentFromEvents, + getRecentSessionContentWithResetFallback, +} from "./transcript.js"; const log = createSubsystemLogger("hooks/session-memory"); @@ -108,6 +118,26 @@ async function resolveAvailableMemoryFilename(params: { } } +async function getRecentSqliteSessionContent( + marker: SqliteSessionFileMarker, + messageCount: number, +): Promise<string | null> { + try { + return getRecentSessionContentFromEvents( + selectVisibleTranscriptEvents( + await loadTranscriptEvents({ + agentId: marker.agentId, + sessionId: marker.sessionId, + storePath: marker.storePath, + }), + ), + messageCount, + ); + } catch { + return null; + } +} + function resolveDisplaySessionKey(params: { cfg?: OpenClawConfig; workspaceDir?: string; @@ -216,8 +246,12 @@ async function saveSessionMemoryNow(event: Parameters<HookHandler>[0]): Promise< let sessionContent: string | null = null; if (sessionFile) { - // Get recent conversation content, with fallback to rotated reset transcript. - sessionContent = await getRecentSessionContentWithResetFallback(sessionFile, messageCount); + const sqliteMarker = parseSqliteSessionFileMarker(sessionFile); + // SQLite-backed runtime sessions carry a marker in the legacy sessionFile + // slot; file artifact helpers only run for real transcript paths. + sessionContent = sqliteMarker + ? await getRecentSqliteSessionContent(sqliteMarker, messageCount) + : await getRecentSessionContentWithResetFallback(sessionFile, messageCount); log.debug("Session content loaded", { length: sessionContent?.length ?? 0, messageCount, diff --git a/src/hooks/bundled/session-memory/transcript.ts b/src/hooks/bundled/session-memory/transcript.ts index e5f911332376..f7f98621f88e 100644 --- a/src/hooks/bundled/session-memory/transcript.ts +++ b/src/hooks/bundled/session-memory/transcript.ts @@ -56,6 +56,83 @@ function extractTextMessageContent(content: unknown): string | undefined { return undefined; } +type RenderedSessionMemoryMessage = { + isDeliveryMirror: boolean; + role: "assistant" | "user"; + text?: string; +}; + +function renderSessionMemoryMessage(entry: unknown): RenderedSessionMemoryMessage | undefined { + if (!entry || typeof entry !== "object") { + return undefined; + } + const record = entry as { + message?: { + content?: unknown; + provenance?: unknown; + role?: unknown; + }; + type?: unknown; + }; + if (record.type !== "message" || !record.message) { + return undefined; + } + const role = record.message.role; + if ((role !== "user" && role !== "assistant") || !("content" in record.message)) { + return undefined; + } + if (role === "user" && hasInterSessionUserProvenance(record.message)) { + return undefined; + } + const text = extractTextMessageContent(record.message.content); + const sanitized = text ? sanitizeSessionMemoryTranscriptText(text) : null; + if (!sanitized) { + return undefined; + } + if (sanitized.startsWith("/")) { + return role === "user" ? { isDeliveryMirror: false, role } : undefined; + } + return { + isDeliveryMirror: isOpenClawDeliveryMirrorAssistantMessage(record.message), + role, + text: sanitized, + }; +} + +/** Renders recent user/assistant transcript events into session memory text. */ +export function getRecentSessionContentFromEvents( + events: readonly unknown[], + messageCount = 15, +): string | null { + const allMessages: string[] = []; + let lastAssistantText: string | undefined; + for (const event of events) { + const rendered = renderSessionMemoryMessage(event); + if (!rendered) { + continue; + } + if (rendered.role === "user") { + // New turn: reset even when slash commands are omitted from memory, so + // later standalone delivery mirrors are preserved. + lastAssistantText = undefined; + } + if (!rendered.text) { + continue; + } + // Skip delivery-mirror rows only when they duplicate the preceding + // assistant text. Delivery-mirror rows with unique visible content + // (e.g., message-tool replies) are preserved. + if (rendered.isDeliveryMirror && rendered.text === lastAssistantText) { + continue; + } + allMessages.push(`${rendered.role}: ${rendered.text}`); + if (rendered.role === "assistant") { + lastAssistantText = rendered.text; + } + } + return allMessages.slice(-messageCount).join("\n"); +} + export async function getRecentSessionContent( sessionFilePath: string, messageCount = 15, @@ -64,51 +141,16 @@ export async function getRecentSessionContent( const content = await fs.readFile(sessionFilePath, "utf-8"); const lines = content.trim().split("\n"); - const allMessages: string[] = []; - let lastAssistantText: string | undefined; - for (const line of lines) { - try { - const entry = JSON.parse(line); - if (entry.type === "message" && entry.message) { - const msg = entry.message as { - role?: unknown; - content?: unknown; - provenance?: unknown; - }; - const role = msg.role; - if ((role === "user" || role === "assistant") && "content" in msg && msg.content) { - if (role === "user" && hasInterSessionUserProvenance(msg)) { - continue; - } - if (role === "user") { - // New turn: reset even when slash commands are omitted from - // memory, so later standalone delivery mirrors are preserved. - lastAssistantText = undefined; - } - const text = extractTextMessageContent(msg.content); - const sanitized = text ? sanitizeSessionMemoryTranscriptText(text) : null; - // Skip delivery-mirror rows only when they duplicate the preceding - // assistant text. Delivery-mirror rows with unique visible content - // (e.g., message-tool replies) are preserved. - if (isOpenClawDeliveryMirrorAssistantMessage(msg)) { - if (sanitized && sanitized === lastAssistantText) { - continue; - } - } - if (sanitized && !sanitized.startsWith("/")) { - allMessages.push(`${role}: ${sanitized}`); - if (role === "assistant") { - lastAssistantText = sanitized; - } - } - } + return getRecentSessionContentFromEvents( + lines.flatMap((line) => { + try { + return [JSON.parse(line) as unknown]; + } catch { + return []; } - } catch { - // Skip invalid JSON lines. - } - } - - return allMessages.slice(-messageCount).join("\n"); + }), + messageCount, + ); } catch { return null; } diff --git a/src/infra/approval-request-account-binding.ts b/src/infra/approval-request-account-binding.ts index 6e031a271407..18cfa063cfa0 100644 --- a/src/infra/approval-request-account-binding.ts +++ b/src/infra/approval-request-account-binding.ts @@ -1,8 +1,7 @@ // Matches approval requests against channel account and session bindings. import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { resolveStorePath } from "../config/sessions/paths.js"; -import { loadSessionStore } from "../config/sessions/store-load.js"; -import { resolveMaintenanceConfigFromInput } from "../config/sessions/store-maintenance.js"; +import { loadSessionEntry } from "../config/sessions/session-accessor.js"; import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { normalizeOptionalAccountId } from "../routing/account-id.js"; @@ -39,10 +38,11 @@ export function resolvePersistedApprovalRequestSessionEntry(params: { const parsed = parseAgentSessionKey(sessionKey); const agentId = parsed?.agentId ?? params.request.request.agentId ?? "main"; const storePath = resolveStorePath(params.cfg.session?.store, { agentId }); - const store = loadSessionStore(storePath, { - maintenanceConfig: resolveMaintenanceConfigFromInput(params.cfg.session?.maintenance), + const entry = loadSessionEntry({ + storePath, + sessionKey, + clone: false, }); - const entry = store[sessionKey]; if (!entry) { return null; } diff --git a/src/infra/exec-approval-session-target.test.ts b/src/infra/exec-approval-session-target.test.ts index f7a54d97ba20..b2926146e87f 100644 --- a/src/infra/exec-approval-session-target.test.ts +++ b/src/infra/exec-approval-session-target.test.ts @@ -4,11 +4,12 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import type { SessionEntry } from "../config/sessions.js"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; import { parseRawSessionConversationRef, parseThreadSessionSuffix, } from "../sessions/session-key-utils.js"; -import { withTempDirSync } from "../test-helpers/temp-dir.js"; +import { withTempDir } from "../test-helpers/temp-dir.js"; import { doesApprovalRequestMatchChannelAccount, resolveApprovalRequestAccountId, @@ -62,12 +63,26 @@ const baseRequest: ExecApprovalRequest = { expiresAtMs: 6000, }; -function writeStoreFile( +async function writeStoreFile( storePath: string, entries: Record<string, Partial<SessionEntry>>, -): OpenClawConfig { +): Promise<OpenClawConfig> { fs.mkdirSync(path.dirname(storePath), { recursive: true }); - fs.writeFileSync(storePath, JSON.stringify(entries), "utf-8"); + await Promise.all( + Object.entries(entries).map(([sessionKey, entry]) => + replaceSessionEntry( + { + storePath, + sessionKey, + }, + { + sessionId: entry.sessionId ?? sessionKey, + updatedAt: entry.updatedAt ?? Date.now(), + ...entry, + } as SessionEntry, + ), + ), + ); return { session: { store: storePath }, } as OpenClawConfig; @@ -135,10 +150,10 @@ describe("exec approval session target", () => { expected: ReturnType<typeof resolveExecApprovalSessionTarget>; }; - it("returns null for blank session keys, missing entries, and unresolved targets", () => { - withTempDirSync({ prefix: "openclaw-exec-approval-session-target-" }, (tmpDir) => { + it("returns null for blank session keys, missing entries, and unresolved targets", async () => { + await withTempDir({ prefix: "openclaw-exec-approval-session-target-" }, async (tmpDir) => { const storePath = path.join(tmpDir, "sessions.json"); - const cfg = writeStoreFile(storePath, { + const cfg = await writeStoreFile(storePath, { "agent:main:main": { sessionId: "main", updatedAt: 1, @@ -158,10 +173,10 @@ describe("exec approval session target", () => { }); }); - it("prefers turn-source routing over stale session delivery state", () => { - withTempDirSync({ prefix: "openclaw-exec-approval-session-target-" }, (tmpDir) => { + it("prefers turn-source routing over stale session delivery state", async () => { + await withTempDir({ prefix: "openclaw-exec-approval-session-target-" }, async (tmpDir) => { const storePath = path.join(tmpDir, "sessions.json"); - const cfg = writeStoreFile(storePath, { + const cfg = await writeStoreFile(storePath, { "agent:main:main": { sessionId: "main", updatedAt: 1, @@ -235,19 +250,22 @@ describe("exec approval session target", () => { }, ] satisfies PlaceholderStoreCase[])( "$name", - ({ relativeStoreDir, entries, request, expected }) => { - withTempDirSync({ prefix: "openclaw-exec-approval-session-target-" }, (tmpDir) => { - const cfg = writeStoreFile(path.join(tmpDir, relativeStoreDir, "sessions.json"), entries); + async ({ relativeStoreDir, entries, request, expected }) => { + await withTempDir({ prefix: "openclaw-exec-approval-session-target-" }, async (tmpDir) => { + const cfg = await writeStoreFile( + path.join(tmpDir, relativeStoreDir, "sessions.json"), + entries, + ); cfg.session = { store: path.join(tmpDir, "{agentId}", "sessions.json") }; expect(expectResolvedSessionTarget(cfg, request)).toEqual(expected); }); }, ); - it("preserves string thread ids from the session store", () => { - withTempDirSync({ prefix: "openclaw-exec-approval-session-target-" }, (tmpDir) => { + it("preserves string thread ids from the session store", async () => { + await withTempDir({ prefix: "openclaw-exec-approval-session-target-" }, async (tmpDir) => { const storePath = path.join(tmpDir, "sessions.json"); - const cfg = writeStoreFile(storePath, { + const cfg = await writeStoreFile(storePath, { "agent:main:main": { sessionId: "main", updatedAt: 1, @@ -347,24 +365,24 @@ describe("exec approval session target", () => { ).toBe(false); }); - it("falls back to the stored session binding when turn source uses another channel", () => { - withTempDirSync({ prefix: "openclaw-exec-approval-session-target-" }, (tmpDir) => { + it("falls back to the stored session binding when turn source uses another channel", async () => { + await withTempDir({ prefix: "openclaw-exec-approval-session-target-" }, async (tmpDir) => { const storePath = path.join(tmpDir, "sessions.json"); - const cfg = writeStoreFile(storePath, { - "agent:main:matrix:channel:!ops:example.org": { + const cfg = await writeStoreFile(storePath, { + "agent:main:main": { sessionId: "main", updatedAt: 1, origin: { provider: "matrix", accountId: "ops", }, - lastChannel: "slack", + lastChannel: "matrix", lastTo: "channel:C123", - lastAccountId: "work", + lastAccountId: "ops", }, }); const request = buildRequest({ - sessionKey: "agent:main:matrix:channel:!ops:example.org", + sessionKey: "agent:main:main", turnSourceChannel: "discord", turnSourceTo: "channel:D123", turnSourceAccountId: "work", @@ -377,10 +395,10 @@ describe("exec approval session target", () => { }); }); - it("falls back to the session-bound account when no turn-source account is present", () => { - withTempDirSync({ prefix: "openclaw-exec-approval-session-target-" }, (tmpDir) => { + it("falls back to the session-bound account when no turn-source account is present", async () => { + await withTempDir({ prefix: "openclaw-exec-approval-session-target-" }, async (tmpDir) => { const storePath = path.join(tmpDir, "sessions.json"); - const cfg = writeStoreFile(storePath, { + const cfg = await writeStoreFile(storePath, { "agent:main:main": { sessionId: "main", updatedAt: 1, @@ -404,10 +422,10 @@ describe("exec approval session target", () => { }); }); - it("prefers explicit turn-source accounts over stale session account bindings", () => { - withTempDirSync({ prefix: "openclaw-exec-approval-session-target-" }, (tmpDir) => { + it("prefers explicit turn-source accounts over stale session account bindings", async () => { + await withTempDir({ prefix: "openclaw-exec-approval-session-target-" }, async (tmpDir) => { const storePath = path.join(tmpDir, "sessions.json"); - const cfg = writeStoreFile(storePath, { + const cfg = await writeStoreFile(storePath, { "agent:main:main": { sessionId: "main", updatedAt: 1, @@ -433,10 +451,10 @@ describe("exec approval session target", () => { }); }); - it("reconciles plugin-request turn source and session origin targets through the shared helper", () => { - withTempDirSync({ prefix: "openclaw-exec-approval-session-target-" }, (tmpDir) => { + it("reconciles plugin-request turn source and session origin targets through the shared helper", async () => { + await withTempDir({ prefix: "openclaw-exec-approval-session-target-" }, async (tmpDir) => { const storePath = path.join(tmpDir, "sessions.json"); - const cfg = writeStoreFile(storePath, { + const cfg = await writeStoreFile(storePath, { "agent:main:main": { sessionId: "main", updatedAt: 1, @@ -454,10 +472,10 @@ describe("exec approval session target", () => { }); }); - it("returns null when explicit turn source conflicts with the session-bound origin target", () => { - withTempDirSync({ prefix: "openclaw-exec-approval-session-target-" }, (tmpDir) => { + it("returns null when explicit turn source conflicts with the session-bound origin target", async () => { + await withTempDir({ prefix: "openclaw-exec-approval-session-target-" }, async (tmpDir) => { const storePath = path.join(tmpDir, "sessions.json"); - const cfg = writeStoreFile(storePath, { + const cfg = await writeStoreFile(storePath, { "agent:main:main": { sessionId: "main", updatedAt: 1, diff --git a/src/infra/heartbeat-runner.clears-pending-final-delivery.test.ts b/src/infra/heartbeat-runner.clears-pending-final-delivery.test.ts index e0fa8b6afe83..4dbcdbcb8a7d 100644 --- a/src/infra/heartbeat-runner.clears-pending-final-delivery.test.ts +++ b/src/infra/heartbeat-runner.clears-pending-final-delivery.test.ts @@ -1,10 +1,11 @@ -import fs from "node:fs/promises"; import { describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; +import { patchSessionEntry } from "../config/sessions/session-accessor.js"; import { runHeartbeatOnce, type HeartbeatDeps } from "./heartbeat-runner.js"; import { installHeartbeatRunnerTestRuntime } from "./heartbeat-runner.test-harness.js"; import { type HeartbeatReplySpy, + readSessionStoreForTest, seedMainSessionStore, withTempHeartbeatSandbox, } from "./heartbeat-runner.test-utils.js"; @@ -57,14 +58,11 @@ describe("runHeartbeatOnce clears stuck pendingFinalDelivery state once delivery sessionKey: string, patch: Record<string, unknown>, ): Promise<void> { - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record<string, StoredEntry>; - store[sessionKey] = { ...store[sessionKey], ...patch }; - await fs.writeFile(storePath, JSON.stringify(store)); + await patchSessionEntry({ storePath, sessionKey }, () => patch, { preserveActivity: true }); } async function readEntry(storePath: string, sessionKey: string): Promise<StoredEntry> { - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record<string, StoredEntry>; - return store[sessionKey]; + return readSessionStoreForTest<Record<string, unknown>>(storePath)[sessionKey]; } function expectPendingFinalDeliveryCleared(entry: StoredEntry): void { diff --git a/src/infra/heartbeat-runner.commitments.test.ts b/src/infra/heartbeat-runner.commitments.test.ts index 21c87b8c0a99..c9a287c08fc5 100644 --- a/src/infra/heartbeat-runner.commitments.test.ts +++ b/src/infra/heartbeat-runner.commitments.test.ts @@ -19,7 +19,11 @@ import { startHeartbeatRunner, } from "./heartbeat-runner.js"; import { installHeartbeatRunnerTestRuntime } from "./heartbeat-runner.test-harness.js"; -import { seedSessionStore, withTempHeartbeatSandbox } from "./heartbeat-runner.test-utils.js"; +import { + readSessionStoreForTest, + seedSessionStore, + withTempHeartbeatSandbox, +} from "./heartbeat-runner.test-utils.js"; import { requestHeartbeat, resetHeartbeatWakeStateForTests } from "./heartbeat-wake.js"; import { enqueueSystemEvent, @@ -275,10 +279,9 @@ describe("runHeartbeatOnce commitments", () => { return { result: resultResult, sendTelegram: sendTelegramResult, - sessionStore: JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { heartbeatTaskState?: Record<string, number> } - >, + sessionStore: readSessionStoreForTest<{ + heartbeatTaskState?: Record<string, number>; + }>(storePath), store: await loadCommitmentStore(), }; }, @@ -560,9 +563,10 @@ describe("runHeartbeatOnce commitments", () => { lastAttemptAtMs: nowMs, }); expect(store.commitments[0]?.sentAtMs).toBeUndefined(); - const sessionStoreAfterSuppressed = JSON.parse( - await fs.readFile(storePath, "utf-8"), - ) as Record<string, { lastHeartbeatText?: string; lastHeartbeatSentAt?: number }>; + const sessionStoreAfterSuppressed = readSessionStoreForTest<{ + lastHeartbeatText?: string; + lastHeartbeatSentAt?: number; + }>(storePath); expect(sessionStoreAfterSuppressed[sessionKey]?.lastHeartbeatText).toBeUndefined(); expect(sessionStoreAfterSuppressed[sessionKey]?.lastHeartbeatSentAt).toBeUndefined(); expect(getLastHeartbeatEvent()).toMatchObject({ @@ -689,20 +693,15 @@ tasks: `, "utf-8", ); - // Seed heartbeatTaskState so the task ran at nowMs (well within 5m interval — not due). - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "telegram", - lastProvider: "telegram", - lastTo: "155462274", - heartbeatTaskState: { "check-deployment": nowMs }, - }, - }), - ); + // Seed heartbeatTaskState so the task ran at nowMs (well within 5m interval, not due). + await seedSessionStore(storePath, sessionKey, { + sessionId: "sid", + updatedAt: nowMs, + lastChannel: "telegram", + lastProvider: "telegram", + lastTo: "155462274", + heartbeatTaskState: { "check-deployment": nowMs }, + }); await saveCommitmentStore(undefined, { version: 1, commitments: [buildCommitment({ id: "cm_interview", sessionKey, to: "155462274" })], @@ -785,19 +784,14 @@ tasks: `, "utf-8", ); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "telegram", - lastProvider: "telegram", - lastTo: "155462274", - heartbeatTaskState: { "global-ops-audit": nowMs - 10 * 60_000 }, - }, - }), - ); + await seedSessionStore(storePath, sessionKey, { + sessionId: "sid", + updatedAt: nowMs, + lastChannel: "telegram", + lastProvider: "telegram", + lastTo: "155462274", + heartbeatTaskState: { "global-ops-audit": nowMs - 10 * 60_000 }, + }); enqueueSystemEvent("Reminder: run the global audit", { sessionKey, deliveryContext: { @@ -857,10 +851,9 @@ tasks: result: resultLocal, sendTelegram: sendTelegramLocal, pendingEvents: peekSystemEventEntries(sessionKey), - sessionStore: JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { heartbeatTaskState?: Record<string, number> } - >, + sessionStore: readSessionStoreForTest<{ + heartbeatTaskState?: Record<string, number>; + }>(storePath), store: await loadCommitmentStore(), }; }); diff --git a/src/infra/heartbeat-runner.ghost-reminder.test.ts b/src/infra/heartbeat-runner.ghost-reminder.test.ts index b3ea9b89adeb..bfa4196d646f 100644 --- a/src/infra/heartbeat-runner.ghost-reminder.test.ts +++ b/src/infra/heartbeat-runner.ghost-reminder.test.ts @@ -1,11 +1,11 @@ // Covers heartbeat handling of queued reminder system events. -import fs from "node:fs/promises"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import { resolveMainSessionKey } from "../config/sessions/main-session.js"; import { runHeartbeatOnce } from "./heartbeat-runner.js"; import { seedMainSessionStore, + seedSessionStore, setupTelegramHeartbeatPluginRuntimeForTests, withTempHeartbeatSandbox, } from "./heartbeat-runner.test-utils.js"; @@ -84,17 +84,14 @@ describe("Ghost reminder bug (issue #13317)", () => { sessionKey: string, overrides: Record<string, unknown>, ): Promise<void> => { - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "telegram", - ...overrides, - }, - }), - ); + await seedSessionStore(storePath, sessionKey, { + sessionId: "sid", + updatedAt: Date.now(), + lastChannel: "telegram", + lastProvider: "telegram", + lastTo: "-100155462274", + ...overrides, + }); }; const expectCronEventPrompt = ( @@ -470,15 +467,7 @@ describe("Ghost reminder bug (issue #13317)", () => { session: { store: storePath }, }; const sessionKey = resolveMainSessionKey(cfg); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - }, - }), - ); + await writeTelegramSessionStore(storePath, sessionKey, {}); const sendTelegram = vi.fn().mockResolvedValue({ messageId: "m1", @@ -571,18 +560,10 @@ describe("Ghost reminder bug (issue #13317)", () => { session: { store: storePath }, }; const sessionKey = "agent:main:telegram:group:-1003774691294:topic:47"; - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "telegram", - lastTo: "telegram:-1003774691294:topic:2175", - lastThreadId: 2175, - }, - }), - ); + await writeTelegramSessionStore(storePath, sessionKey, { + lastTo: "telegram:-1003774691294:topic:2175", + lastThreadId: 2175, + }); const sendTelegram = vi.fn().mockResolvedValue({ messageId: "m1", @@ -636,18 +617,10 @@ describe("Ghost reminder bug (issue #13317)", () => { session: { store: storePath }, }; const sessionKey = "agent:main:telegram:group:-1003774691294:topic:47"; - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "telegram", - lastTo: "telegram:-1003774691294:topic:2175", - lastThreadId: 2175, - }, - }), - ); + await writeTelegramSessionStore(storePath, sessionKey, { + lastTo: "telegram:-1003774691294:topic:2175", + lastThreadId: 2175, + }); const sendTelegram = vi.fn(); const getReplySpy = vi.fn().mockResolvedValue({ diff --git a/src/infra/heartbeat-runner.identity.test.ts b/src/infra/heartbeat-runner.identity.test.ts index 658399289ebf..3b06a17fb105 100644 --- a/src/infra/heartbeat-runner.identity.test.ts +++ b/src/infra/heartbeat-runner.identity.test.ts @@ -1,4 +1,3 @@ -import fs from "node:fs/promises"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { resolveStorePath } from "../config/sessions.js"; @@ -8,6 +7,7 @@ import { installHeartbeatRunnerTestRuntime } from "./heartbeat-runner.test-harne import { seedMainSessionStore, seedSessionStore, + readSessionStoreForTest, withTempHeartbeatSandbox, } from "./heartbeat-runner.test-utils.js"; @@ -34,10 +34,6 @@ describe("runHeartbeatOnce identity", () => { }; const mainStorePath = resolveStorePath(storeTemplate, { agentId: "main" }); const historianStorePath = resolveStorePath(storeTemplate, { agentId: "historian2" }); - await Promise.all([ - fs.mkdir(path.dirname(mainStorePath), { recursive: true }), - fs.mkdir(path.dirname(historianStorePath), { recursive: true }), - ]); await seedSessionStore(mainStorePath, "global", { lastChannel: "slack", lastProvider: "slack", @@ -48,7 +44,7 @@ describe("runHeartbeatOnce identity", () => { lastProvider: "slack", lastTo: "channel:HISTORIAN", }); - const mainStoreBefore = await fs.readFile(mainStorePath, "utf-8"); + const mainStoreBefore = readSessionStoreForTest(mainStorePath); replySpy.mockResolvedValue({ text: "needs attention" }); const sendSlack = vi.fn().mockResolvedValue({ messageId: "m1", channelId: "HISTORIAN" }); @@ -72,11 +68,8 @@ describe("runHeartbeatOnce identity", () => { "needs attention", expect.any(Object), ); - expect(await fs.readFile(mainStorePath, "utf-8")).toBe(mainStoreBefore); - const historianStore = JSON.parse(await fs.readFile(historianStorePath, "utf-8")) as Record< - string, - unknown - >; + expect(readSessionStoreForTest(mainStorePath)).toEqual(mainStoreBefore); + const historianStore = readSessionStoreForTest(historianStorePath); expect(historianStore.global).toBeDefined(); expect(historianStore["global:heartbeat"] !== undefined).toBe(isolatedSession); }); diff --git a/src/infra/heartbeat-runner.isolated-key-stability.test.ts b/src/infra/heartbeat-runner.isolated-key-stability.test.ts index d46f019ee86d..5234c6052642 100644 --- a/src/infra/heartbeat-runner.isolated-key-stability.test.ts +++ b/src/infra/heartbeat-runner.isolated-key-stability.test.ts @@ -5,7 +5,11 @@ import * as replyModule from "../auto-reply/reply.js"; import type { OpenClawConfig } from "../config/config.js"; import { resolveMainSessionKey } from "../config/sessions.js"; import { runHeartbeatOnce } from "./heartbeat-runner.js"; -import { seedSessionStore, withTempHeartbeatSandbox } from "./heartbeat-runner.test-utils.js"; +import { + readSessionStoreForTest, + seedSessionStore, + withTempHeartbeatSandbox, +} from "./heartbeat-runner.test-utils.js"; import { enqueueSystemEvent, peekSystemEventEntries, @@ -118,20 +122,14 @@ describe("runHeartbeatOnce – isolated session key stability (#59493)", () => { // Simulate wake-request path: key already has :heartbeat from a previous tick. const alreadySuffixedKey = `${baseSessionKey}:heartbeat`; - await fs.writeFile( - storePath, - JSON.stringify({ - [alreadySuffixedKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastProvider: "whatsapp", - lastTo: "+1555", - heartbeatIsolatedBaseSessionKey: baseSessionKey, - }, - }), - "utf-8", - ); + await seedSessionStore(storePath, alreadySuffixedKey, { + sessionId: "sid", + updatedAt: Date.now(), + lastChannel: "whatsapp", + lastProvider: "whatsapp", + lastTo: "+1555", + heartbeatIsolatedBaseSessionKey: baseSessionKey, + }); const replySpy = vi.spyOn(replyModule, "getReplyFromConfig"); replySpy.mockResolvedValue({ text: "HEARTBEAT_OK" }); @@ -186,10 +184,9 @@ describe("runHeartbeatOnce – isolated session key stability (#59493)", () => { // A deeply accumulated key converges to "<base>:heartbeat" in one call. expect(ctx?.SessionKey).toBe(`${baseSessionKey}:heartbeat`); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { heartbeatIsolatedBaseSessionKey?: string } - >; + const store = readSessionStoreForTest<{ heartbeatIsolatedBaseSessionKey?: string }>( + storePath, + ); expect(store[deeplyAccumulatedKey]).toBeUndefined(); expect(store[`${baseSessionKey}:heartbeat`]?.heartbeatIsolatedBaseSessionKey).toBe( baseSessionKey, @@ -266,20 +263,14 @@ describe("runHeartbeatOnce – isolated session key stability (#59493)", () => { const cfg = makeNamedIsolatedHeartbeatConfig(tmpDir, storePath, "alerts:heartbeat"); const baseSessionKey = "agent:main:alerts:heartbeat"; const alreadyIsolatedKey = `${baseSessionKey}:heartbeat`; - await fs.writeFile( - storePath, - JSON.stringify({ - [alreadyIsolatedKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastProvider: "whatsapp", - lastTo: "+1555", - heartbeatIsolatedBaseSessionKey: baseSessionKey, - }, - }), - "utf-8", - ); + await seedSessionStore(storePath, alreadyIsolatedKey, { + sessionId: "sid", + updatedAt: Date.now(), + lastChannel: "whatsapp", + lastProvider: "whatsapp", + lastTo: "+1555", + heartbeatIsolatedBaseSessionKey: baseSessionKey, + }); const replySpy = vi.spyOn(replyModule, "getReplyFromConfig"); replySpy.mockResolvedValue({ text: "HEARTBEAT_OK" }); @@ -301,20 +292,14 @@ describe("runHeartbeatOnce – isolated session key stability (#59493)", () => { const cfg = makeIsolatedHeartbeatConfig(tmpDir, storePath); const baseSessionKey = resolveMainSessionKey(cfg); const isolatedSessionKey = `${baseSessionKey}:heartbeat`; - await fs.writeFile( - storePath, - JSON.stringify({ - [isolatedSessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastProvider: "whatsapp", - lastTo: "+1555", - heartbeatIsolatedBaseSessionKey: baseSessionKey, - }, - }), - "utf-8", - ); + await seedSessionStore(storePath, isolatedSessionKey, { + sessionId: "sid", + updatedAt: Date.now(), + lastChannel: "whatsapp", + lastProvider: "whatsapp", + lastTo: "+1555", + heartbeatIsolatedBaseSessionKey: baseSessionKey, + }); enqueueSystemEvent("exec finished: deploy succeeded", { sessionKey: isolatedSessionKey }); const replySpy = vi.spyOn(replyModule, "getReplyFromConfig"); replySpy.mockResolvedValue({ text: "Handled internally" }); @@ -358,20 +343,14 @@ describe("runHeartbeatOnce – isolated session key stability (#59493)", () => { const realSessionKey = "agent:main:alerts:heartbeat"; const isolatedSessionKey = `${realSessionKey}:heartbeat`; - await fs.writeFile( - storePath, - JSON.stringify({ - [isolatedSessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastProvider: "whatsapp", - lastTo: "+1555", - heartbeatIsolatedBaseSessionKey: realSessionKey, - }, - }), - "utf-8", - ); + await seedSessionStore(storePath, isolatedSessionKey, { + sessionId: "sid", + updatedAt: Date.now(), + lastChannel: "whatsapp", + lastProvider: "whatsapp", + lastTo: "+1555", + heartbeatIsolatedBaseSessionKey: realSessionKey, + }); const replySpy = vi.spyOn(replyModule, "getReplyFromConfig"); replySpy.mockResolvedValue({ text: "HEARTBEAT_OK" }); @@ -417,22 +396,16 @@ describe("runHeartbeatOnce – isolated session key stability (#59493)", () => { "utf-8", ); - await fs.writeFile( - storePath, - JSON.stringify({ - [baseSessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastProvider: "whatsapp", - lastTo: "+1555", - heartbeatTaskState: { - "daily-check": 1, - }, - }, - }), - "utf-8", - ); + await seedSessionStore(storePath, baseSessionKey, { + sessionId: "sid", + updatedAt: Date.now(), + lastChannel: "whatsapp", + lastProvider: "whatsapp", + lastTo: "+1555", + heartbeatTaskState: { + "daily-check": 1, + }, + }); const replySpy = vi.spyOn(replyModule, "getReplyFromConfig"); replySpy.mockResolvedValue({ text: "HEARTBEAT_OK" }); @@ -448,7 +421,7 @@ describe("runHeartbeatOnce – isolated session key stability (#59493)", () => { expect(result).toEqual({ status: "skipped", reason: "no-tasks-due" }); expect(replySpy).not.toHaveBeenCalled(); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record<string, unknown>; + const store = readSessionStoreForTest(storePath); expect(store[isolatedSessionKey]).toBeUndefined(); }); }); @@ -466,19 +439,13 @@ describe("runHeartbeatOnce – isolated session key stability (#59493)", () => { const legacyIsolatedKey = `${baseSessionKey}:heartbeat`; // Legacy entry: has :heartbeat suffix but no heartbeatIsolatedBaseSessionKey marker. - await fs.writeFile( - storePath, - JSON.stringify({ - [legacyIsolatedKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastProvider: "whatsapp", - lastTo: "+1555", - }, - }), - "utf-8", - ); + await seedSessionStore(storePath, legacyIsolatedKey, { + sessionId: "sid", + updatedAt: Date.now(), + lastChannel: "whatsapp", + lastProvider: "whatsapp", + lastTo: "+1555", + }); const replySpy = vi.spyOn(replyModule, "getReplyFromConfig"); replySpy.mockResolvedValue({ text: "HEARTBEAT_OK" }); diff --git a/src/infra/heartbeat-runner.isolated-session-mirror.test.ts b/src/infra/heartbeat-runner.isolated-session-mirror.test.ts index 75a7c9d175bd..2d231e0a800b 100644 --- a/src/infra/heartbeat-runner.isolated-session-mirror.test.ts +++ b/src/infra/heartbeat-runner.isolated-session-mirror.test.ts @@ -6,7 +6,11 @@ import type { OpenClawConfig } from "../config/config.js"; import { resolveMainSessionKey } from "../config/sessions.js"; import { runHeartbeatOnce } from "./heartbeat-runner.js"; import { installHeartbeatRunnerTestRuntime } from "./heartbeat-runner.test-harness.js"; -import { seedSessionStore, withTempHeartbeatSandbox } from "./heartbeat-runner.test-utils.js"; +import { + readSessionStoreForTest, + seedSessionStore, + withTempHeartbeatSandbox, +} from "./heartbeat-runner.test-utils.js"; const deliverOutboundPayloadsInternal = vi.hoisted(() => vi.fn().mockResolvedValue([{ channel: "whatsapp", messageId: "msg-1" }]), @@ -106,15 +110,12 @@ describe("runHeartbeatOnce - isolated heartbeat outbound session mirror", () => }, }); - const store = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { - heartbeatTaskState?: Record<string, number>; - lastHeartbeatText?: string; - lastHeartbeatSentAt?: number; - heartbeatIsolatedBaseSessionKey?: string; - } - >; + const store = readSessionStoreForTest<{ + heartbeatTaskState?: Record<string, number>; + lastHeartbeatText?: string; + lastHeartbeatSentAt?: number; + heartbeatIsolatedBaseSessionKey?: string; + }>(storePath); expect(store[baseSessionKey]).toMatchObject({ heartbeatTaskState: { "check-in": nowMs }, lastHeartbeatText: "Status needs attention.", @@ -135,20 +136,14 @@ describe("runHeartbeatOnce - isolated heartbeat outbound session mirror", () => const isolatedSessionKey = `${baseSessionKey}:heartbeat`; const nowMs = Date.now(); - await fs.writeFile( - storePath, - JSON.stringify({ - [isolatedSessionKey]: { - sessionId: "isolated-session", - updatedAt: nowMs - 1_000, - lastChannel: "whatsapp", - lastProvider: "whatsapp", - lastTo: "+15551234567", - heartbeatIsolatedBaseSessionKey: baseSessionKey, - }, - }), - "utf-8", - ); + await seedSessionStore(storePath, isolatedSessionKey, { + sessionId: "isolated-session", + updatedAt: nowMs - 1_000, + lastChannel: "whatsapp", + lastProvider: "whatsapp", + lastTo: "+15551234567", + heartbeatIsolatedBaseSessionKey: baseSessionKey, + }); replySpy.mockResolvedValueOnce({ text: "Wake result needs attention." }); const result = await runHeartbeatOnce({ diff --git a/src/infra/heartbeat-runner.respects-ackmaxchars-heartbeat-acks.test.ts b/src/infra/heartbeat-runner.respects-ackmaxchars-heartbeat-acks.test.ts index 582f71e45b46..13608f51a8d3 100644 --- a/src/infra/heartbeat-runner.respects-ackmaxchars-heartbeat-acks.test.ts +++ b/src/infra/heartbeat-runner.respects-ackmaxchars-heartbeat-acks.test.ts @@ -6,7 +6,9 @@ import { runHeartbeatOnce, type HeartbeatDeps } from "./heartbeat-runner.js"; import { installHeartbeatRunnerTestRuntime } from "./heartbeat-runner.test-harness.js"; import { type HeartbeatReplySpy, + readSessionStoreForTest, seedMainSessionStore, + seedSessionStore, withTempHeartbeatSandbox, withTempTelegramHeartbeatSandbox, } from "./heartbeat-runner.test-utils.js"; @@ -308,10 +310,9 @@ describe("runHeartbeatOnce ack handling", () => { }, }); - const sessionStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { heartbeatTaskState?: Record<string, number> } - >; + const sessionStore = readSessionStoreForTest<{ + heartbeatTaskState?: Record<string, number>; + }>(storePath); expect(result.status).toBe("ran"); expect(sendTelegram).toHaveBeenCalledTimes(1); expect(sessionStore[sessionKey]?.heartbeatTaskState).toEqual({ @@ -360,10 +361,9 @@ describe("runHeartbeatOnce ack handling", () => { }, }); - const sessionStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { heartbeatTaskState?: Record<string, number> } - >; + const sessionStore = readSessionStoreForTest<{ + heartbeatTaskState?: Record<string, number>; + }>(storePath); expect(result.status).toBe("ran"); expect(sendWhatsApp).not.toHaveBeenCalled(); expect(sessionStore[sessionKey]?.heartbeatTaskState).toEqual({ @@ -481,15 +481,14 @@ describe("runHeartbeatOnce ack handling", () => { }); replySpy.mockImplementationOnce(async () => { - const raw = await fs.readFile(storePath, "utf-8"); - const parsed = JSON.parse(raw) as Record<string, { updatedAt?: number } | undefined>; - if (parsed[sessionKey]) { - parsed[sessionKey] = { - ...parsed[sessionKey], + const parsed = readSessionStoreForTest(storePath); + const current = parsed[sessionKey]; + if (current) { + await seedSessionStore(storePath, sessionKey, { + ...current, updatedAt: bumpedUpdatedAt, - }; + }); } - await fs.writeFile(storePath, JSON.stringify(parsed, null, 2)); return { text: "" }; }); @@ -501,10 +500,7 @@ describe("runHeartbeatOnce ack handling", () => { }, }); - const finalStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record< - string, - { updatedAt?: number } | undefined - >; + const finalStore = readSessionStoreForTest<{ updatedAt?: number }>(storePath); expect(finalStore[sessionKey]?.updatedAt).toBe(bumpedUpdatedAt); }); }); diff --git a/src/infra/heartbeat-runner.returns-default-unset.test.ts b/src/infra/heartbeat-runner.returns-default-unset.test.ts index 09e7d399be70..714651e22130 100644 --- a/src/infra/heartbeat-runner.returns-default-unset.test.ts +++ b/src/infra/heartbeat-runner.returns-default-unset.test.ts @@ -27,6 +27,7 @@ import { resolveHeartbeatPrompt, runHeartbeatOnce, } from "./heartbeat-runner.js"; +import { seedSessionStore } from "./heartbeat-runner.test-utils.js"; import { resolveHeartbeatDeliveryTarget, resolveHeartbeatDeliveryTargetWithSessionRoute, @@ -234,6 +235,23 @@ function replyBody( }; } +type HeartbeatSeedOverride = Partial<Parameters<typeof seedSessionStore>[2]>; + +async function seedWhatsAppSession( + storePath: string, + sessionKey: string, + entry: HeartbeatSeedOverride = {}, +): Promise<void> { + await seedSessionStore(storePath, sessionKey, { + sessionId: "sid", + updatedAt: Date.now(), + lastChannel: "whatsapp", + lastProvider: "whatsapp", + lastTo: "120363401234567890@g.us", + ...entry, + }); +} + beforeAll(async () => { previousRegistry = getActivePluginRegistry(); @@ -805,17 +823,7 @@ describe("runHeartbeatOnce", () => { }; const sessionKey = resolveMainSessionKey(cfg); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - }), - ); + await seedWhatsAppSession(storePath, sessionKey); replySpy.mockResolvedValue([{ text: "Let me check..." }, { text: "Final alert" }]); const sendWhatsApp = vi @@ -870,17 +878,7 @@ describe("runHeartbeatOnce", () => { }; const sessionKey = resolveAgentMainSessionKey({ cfg, agentId: "ops" }); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - }), - ); + await seedWhatsAppSession(storePath, sessionKey); replySpy.mockResolvedValue([{ text: "Final alert" }]); const sendWhatsApp = vi .fn< @@ -955,18 +953,7 @@ describe("runHeartbeatOnce", () => { await fs.mkdir(sessionsDir, { recursive: true }); await fs.writeFile(sessionFile, "", "utf-8"); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId, - sessionFile, - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - }), - ); + await seedWhatsAppSession(storePath, sessionKey, { sessionId, sessionFile }); replySpy.mockResolvedValue([{ text: "Final alert" }]); const sendWhatsApp = vi @@ -1065,23 +1052,12 @@ describe("runHeartbeatOnce", () => { }); applyOverride({ cfg, sessionKey: overrideSessionKey }); - await fs.writeFile( - storePath, - JSON.stringify({ - [mainSessionKey]: { - sessionId: "sid-main", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - [overrideSessionKey]: { - sessionId: `sid-${peerKind}`, - updatedAt: Date.now() + 10_000, - lastChannel: "whatsapp", - lastTo: peerId, - }, - }), - ); + await seedWhatsAppSession(storePath, mainSessionKey, { sessionId: "sid-main" }); + await seedWhatsAppSession(storePath, overrideSessionKey, { + sessionId: `sid-${peerKind}`, + updatedAt: Date.now() + 10_000, + lastTo: peerId, + }); replySpy.mockClear(); replySpy.mockResolvedValue([{ text: message }]); @@ -1156,23 +1132,12 @@ describe("runHeartbeatOnce", () => { cfg.agents.defaults.heartbeat.session = subagentKey; } - await fs.writeFile( - storePath, - JSON.stringify({ - [mainSessionKey]: { - sessionId: "sid-main", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - [subagentKey]: { - sessionId: "sid-subagent", - updatedAt: Date.now() + 10_000, - lastChannel: "whatsapp", - lastTo: "99999@g.us", - }, - }), - ); + await seedWhatsAppSession(storePath, mainSessionKey, { sessionId: "sid-main" }); + await seedWhatsAppSession(storePath, subagentKey, { + sessionId: "sid-subagent", + updatedAt: Date.now() + 10_000, + lastTo: "99999@g.us", + }); replySpy.mockClear(); replySpy.mockResolvedValue([{ text: "Main session heartbeat" }]); @@ -1222,19 +1187,10 @@ describe("runHeartbeatOnce", () => { }; const sessionKey = resolveMainSessionKey(cfg); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastTo: "120363401234567890@g.us", - lastHeartbeatText: "Final alert", - lastHeartbeatSentAt: 0, - }, - }), - ); + await seedWhatsAppSession(storePath, sessionKey, { + lastHeartbeatText: "Final alert", + lastHeartbeatSentAt: 0, + }); replySpy.mockResolvedValue([{ text: "Final alert" }]); const sendWhatsApp = vi @@ -1337,18 +1293,7 @@ describe("runHeartbeatOnce", () => { }; const sessionKey = resolveMainSessionKey(cfg); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastProvider: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - }), - ); + await seedWhatsAppSession(storePath, sessionKey); replySpy.mockClear(); replySpy.mockResolvedValue(replies); @@ -1399,18 +1344,7 @@ describe("runHeartbeatOnce", () => { session: { store: storePath }, }; const sessionKey = resolveMainSessionKey(cfg); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastProvider: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - }), - ); + await seedWhatsAppSession(storePath, sessionKey); replySpy.mockResolvedValue([ { text: "All clear" }, @@ -1457,20 +1391,7 @@ describe("runHeartbeatOnce", () => { const sessionKey = resolveMainSessionKey(cfg); const agentId = resolveAgentIdFromSessionKey(sessionKey); const storePath = resolveStorePath(storeTemplate, { agentId }); - - await fs.mkdir(path.dirname(storePath), { recursive: true }); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastProvider: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - }), - ); + await seedWhatsAppSession(storePath, sessionKey); replySpy.mockResolvedValue({ text: "Hello from heartbeat" }); const sendWhatsApp = vi @@ -1584,17 +1505,7 @@ describe("runHeartbeatOnce", () => { session: { store: storePath }, }; const sessionKey = resolveMainSessionKey(cfg); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - }), - ); + await seedWhatsAppSession(storePath, sessionKey); if (params.queueCronEvent) { enqueueSystemEvent("Cron: QMD maintenance completed", { sessionKey, @@ -1678,17 +1589,7 @@ Some global directive after tasks. channels: { whatsapp: { allowFrom: ["*"] } }, session: { store: storePath }, }; - await fs.writeFile( - storePath, - JSON.stringify({ - [resolveMainSessionKey(cfg)]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - }), - ); + await seedWhatsAppSession(storePath, resolveMainSessionKey(cfg)); const replySpy = vi.fn().mockResolvedValue({ text: "Handled due heartbeat tasks" }); const sendWhatsApp = vi .fn< @@ -1749,17 +1650,7 @@ tasks: channels: { whatsapp: { allowFrom: ["*"] } }, session: { store: storePath }, }; - await fs.writeFile( - storePath, - JSON.stringify({ - [resolveMainSessionKey(cfg)]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - }), - ); + await seedWhatsAppSession(storePath, resolveMainSessionKey(cfg)); const replySpy = vi.fn().mockResolvedValue({ text: "Handled due heartbeat tasks" }); const sendWhatsApp = vi .fn< @@ -1936,17 +1827,7 @@ tasks: session: { store: storePath }, }; const sessionKey = resolveMainSessionKey(cfg); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - }), - ); + await seedWhatsAppSession(storePath, sessionKey); enqueueSystemEvent("Cron: rotate logs", { sessionKey, contextKey: "cron:rotate-logs", @@ -1993,17 +1874,7 @@ tasks: session: { store: storePath }, }; const sessionKey = resolveMainSessionKey(cfg); - await fs.writeFile( - storePath, - JSON.stringify({ - [sessionKey]: { - sessionId: "sid", - updatedAt: Date.now(), - lastChannel: "whatsapp", - lastTo: "120363401234567890@g.us", - }, - }), - ); + await seedWhatsAppSession(storePath, sessionKey); enqueueSystemEvent("exec finished: backup completed", { sessionKey, contextKey: "exec:backup", diff --git a/src/infra/heartbeat-runner.test-utils.ts b/src/infra/heartbeat-runner.test-utils.ts index 1b6e6e09f8f1..ef48653685d8 100644 --- a/src/infra/heartbeat-runner.test-utils.ts +++ b/src/infra/heartbeat-runner.test-utils.ts @@ -5,6 +5,8 @@ import path from "node:path"; import { vi } from "vitest"; import { heartbeatRunnerTelegramPlugin } from "../../test/helpers/infra/heartbeat-runner-channel-plugins.js"; import { resolveMainSessionKey } from "../config/sessions.js"; +import { listSessionEntries, replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { setActivePluginRegistry } from "../plugins/runtime.js"; import { createTestRegistry } from "../test-utils/channel-plugins.js"; @@ -12,21 +14,10 @@ import type { HeartbeatDeps } from "./heartbeat-runner.js"; // Heartbeat test utilities seed session stores and temporary heartbeat prompts // while keeping plugin registry and environment state isolated per test. -type HeartbeatSessionSeed = { - sessionId?: string; - updatedAt?: number; +type HeartbeatSessionSeed = Partial<SessionEntry> & { lastChannel: string; lastProvider: string; lastTo: string; - pendingFinalDelivery?: boolean; - pendingFinalDeliveryText?: string; - pendingFinalDeliveryCreatedAt?: number; - pendingFinalDeliveryAttemptCount?: number; - pendingFinalDeliveryLastError?: string | null; - agentHarnessId?: string; - agentRuntimeOverride?: string; - model?: string; - modelProvider?: string; }; type HeartbeatReplyFn = NonNullable<HeartbeatDeps["getReplyFromConfig"]>; @@ -38,28 +29,28 @@ function createHeartbeatReplySpy(): HeartbeatReplySpy { return replySpy; } -/** Write a single heartbeat session entry into a JSON session store. */ +/** Write a single heartbeat session entry through the SQLite session accessor. */ export async function seedSessionStore( storePath: string, sessionKey: string, session: HeartbeatSessionSeed, ): Promise<void> { - let existingStore: Record<string, unknown>; - try { - existingStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record<string, unknown>; - } catch { - existingStore = {}; - } - await fs.writeFile( - storePath, - JSON.stringify({ - ...existingStore, - [sessionKey]: { - sessionId: session.sessionId ?? "sid", - updatedAt: session.updatedAt ?? Date.now(), - ...session, - }, - }), + await replaceSessionEntry( + { storePath, sessionKey }, + { + sessionId: session.sessionId ?? "sid", + updatedAt: session.updatedAt ?? Date.now(), + ...session, + }, + ); +} + +/** Read session entries through the SQLite session accessor as a keyed object. */ +export function readSessionStoreForTest<T extends object = HeartbeatSessionSeed>( + storePath: string, +): Record<string, T> { + return Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey, entry }) => [sessionKey, entry as T]), ); } diff --git a/src/infra/heartbeat-runner.ts b/src/infra/heartbeat-runner.ts index e74173020552..9a5483080d51 100644 --- a/src/infra/heartbeat-runner.ts +++ b/src/infra/heartbeat-runner.ts @@ -80,10 +80,11 @@ import { import { resolveStorePath } from "../config/sessions/paths.js"; import { applySessionEntryLifecycleMutation, + loadExactSessionEntry, + loadSessionEntry, + patchSessionEntry, type SessionEntryLifecycleRemoval, } from "../config/sessions/session-accessor.js"; -import { loadSessionStore } from "../config/sessions/store-load.js"; -import { updateSessionStore } from "../config/sessions/store.js"; import type { SessionEntry } from "../config/sessions/types.js"; import type { AgentDefaultsConfig } from "../config/types.agent-defaults.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; @@ -573,14 +574,12 @@ function resolveHeartbeatSession( // Falling back here leaks the default agent's route into secondary heartbeats. agentId: resolvedAgentId, }); - const store = loadSessionStore(storePath); - const mainEntry = store[mainSessionKey]; + const mainEntry = loadSessionEntry({ storePath, sessionKey: mainSessionKey }); if (scope === "global") { return { sessionKey: mainSessionKey, storePath, - store, entry: mainEntry, suppressOriginatingContext: false, }; @@ -592,7 +591,6 @@ function resolveHeartbeatSession( return { sessionKey: mainSessionKey, storePath, - store, entry: mainEntry, suppressOriginatingContext: true, }; @@ -622,8 +620,7 @@ function resolveHeartbeatSession( return { sessionKey: routedSessionKey, storePath, - store, - entry: store[routedSessionKey], + entry: loadSessionEntry({ storePath, sessionKey: routedSessionKey }), suppressOriginatingContext: false, }; } @@ -636,7 +633,6 @@ function resolveHeartbeatSession( return { sessionKey: mainSessionKey, storePath, - store, entry: mainEntry, suppressOriginatingContext: false, }; @@ -647,7 +643,6 @@ function resolveHeartbeatSession( return { sessionKey: mainSessionKey, storePath, - store, entry: mainEntry, suppressOriginatingContext: false, }; @@ -662,7 +657,6 @@ function resolveHeartbeatSession( return { sessionKey: mainSessionKey, storePath, - store, entry: mainEntry, suppressOriginatingContext: false, }; @@ -678,8 +672,7 @@ function resolveHeartbeatSession( return { sessionKey: canonical, storePath, - store, - entry: store[canonical], + entry: loadSessionEntry({ storePath, sessionKey: canonical }), suppressOriginatingContext: false, }; } @@ -688,7 +681,6 @@ function resolveHeartbeatSession( return { sessionKey: mainSessionKey, storePath, - store, entry: mainEntry, suppressOriginatingContext: false, }; @@ -801,8 +793,7 @@ async function restoreHeartbeatUpdatedAt(params: { if (typeof updatedAt !== "number") { return; } - const store = loadSessionStore(storePath); - const entry = store[sessionKey]; + const entry = loadSessionEntry({ storePath, sessionKey }); if (!entry) { return; } @@ -810,17 +801,20 @@ async function restoreHeartbeatUpdatedAt(params: { if (entry.updatedAt === nextUpdatedAt) { return; } - await updateSessionStore(storePath, (nextStore) => { - const nextEntry = nextStore[sessionKey] ?? entry; - if (!nextEntry) { - return; - } - const resolvedUpdatedAt = Math.max(nextEntry.updatedAt ?? 0, updatedAt); - if (nextEntry.updatedAt === resolvedUpdatedAt) { - return; - } - nextStore[sessionKey] = { ...nextEntry, updatedAt: resolvedUpdatedAt }; - }); + await patchSessionEntry( + { storePath, sessionKey }, + (nextEntry, context) => { + if (!context.existingEntry) { + return null; + } + const resolvedUpdatedAt = Math.max(nextEntry.updatedAt ?? 0, updatedAt); + if (nextEntry.updatedAt === resolvedUpdatedAt) { + return null; + } + return { ...nextEntry, updatedAt: resolvedUpdatedAt }; + }, + { replaceEntry: true }, + ); } function stripLeadingHeartbeatResponsePrefix( @@ -1722,9 +1716,11 @@ export async function runHeartbeatOnce(opts: { }); return { status: "skipped", reason: HEARTBEAT_SKIP_REQUESTS_IN_FLIGHT }; } - const isolatedStore = loadSessionStore(isolatedStorePath, { skipCache: true }); const staleIsolatedEntry = staleIsolatedSessionKey - ? isolatedStore[staleIsolatedSessionKey] + ? loadExactSessionEntry({ + storePath: isolatedStorePath, + sessionKey: staleIsolatedSessionKey, + })?.entry : undefined; const removals: SessionEntryLifecycleRemoval[] = staleIsolatedSessionKey ? [ @@ -1826,28 +1822,28 @@ export async function runHeartbeatOnce(opts: { const tasks = preflight.tasks; const dueTaskNames = new Set(dueHeartbeatTasks.map((task) => task.name)); - await updateSessionStore(storePath, (store) => { - const current = store[sessionKey]; - // Initialize stub entry on first run when current doesn't exist. - const base = current ?? { - // Generate valid sessionId - derive from sessionKey without colons. - sessionId: sessionKey.replace(/:/g, "_"), - updatedAt: startedAt, - createdAt: startedAt, - messageCount: 0, - lastMessageAt: startedAt, - heartbeatTaskState: {}, - }; - const taskState = { ...base.heartbeatTaskState }; + await patchSessionEntry( + { storePath, sessionKey }, + (base) => { + const taskState = { ...base.heartbeatTaskState }; - for (const task of tasks) { - if (dueTaskNames.has(task.name)) { - taskState[task.name] = startedAt; + for (const task of tasks) { + if (dueTaskNames.has(task.name)) { + taskState[task.name] = startedAt; + } } - } - store[sessionKey] = { ...base, heartbeatTaskState: taskState }; - }); + return { heartbeatTaskState: taskState }; + }, + { + fallbackEntry: { + sessionId: sessionKey.replace(/:/g, "_"), + updatedAt: startedAt, + heartbeatTaskState: {}, + }, + preserveActivity: true, + }, + ); }; // The duplicate-suppression branch returns before any send, so it never hits @@ -1859,21 +1855,21 @@ export async function runHeartbeatOnce(opts: { // prefixed agents permanently stuck. Ownership is gated on createdAt instead, // so an older final this run did not produce is preserved, not erased. const clearSatisfiedPendingFinalDelivery = async () => { - await updateSessionStore( - storePath, - (store) => { - const current = store[sessionKey]; + await patchSessionEntry( + { storePath, sessionKey }, + (current, context) => { + if (!context.existingEntry) { + return null; + } if (current?.pendingFinalDelivery !== true && !current?.pendingFinalDeliveryText) { - return false; + return null; } if (!heartbeatRunOwnsPendingFinalDelivery(current, startedAt)) { - return false; + return null; } - store[sessionKey] = { ...current, ...CLEARED_PENDING_FINAL_DELIVERY_FIELDS }; - return true; + return CLEARED_PENDING_FINAL_DELIVERY_FIELDS; }, - // No pending to clear is the common case; avoid rewriting the store then. - { skipSaveWhenResult: (cleared) => !cleared }, + { preserveActivity: true }, ); }; @@ -2336,25 +2332,27 @@ export async function runHeartbeatOnce(opts: { // Record last delivered heartbeat payload for dedupe. if (visibleSendSucceeded && !shouldSkipMain && normalized.text.trim()) { - await updateSessionStore(storePath, (store) => { - const current = store[sessionKey]; - if (!current) { - return; - } - // A heartbeat-driven agent run can leave its own pendingFinalDelivery - // set; a successful send completes it, so clear the recovery fields. - // Only clear the pending-final this run owns — an older final the run - // did not produce keeps its own recovery path. - const clearedRecoveryFields = heartbeatRunOwnsPendingFinalDelivery(current, startedAt) - ? CLEARED_PENDING_FINAL_DELIVERY_FIELDS - : {}; - store[sessionKey] = { - ...current, - lastHeartbeatText: normalized.text, - lastHeartbeatSentAt: startedAt, - ...clearedRecoveryFields, - }; - }); + await patchSessionEntry( + { storePath, sessionKey }, + (current, context) => { + if (!context.existingEntry) { + return null; + } + // A heartbeat-driven agent run can leave its own pendingFinalDelivery + // set; a successful send completes it, so clear the recovery fields. + // Only clear the pending-final this run owns — an older final the run + // did not produce keeps its own recovery path. + const clearedRecoveryFields = heartbeatRunOwnsPendingFinalDelivery(current, startedAt) + ? CLEARED_PENDING_FINAL_DELIVERY_FIELDS + : {}; + return { + lastHeartbeatText: normalized.text, + lastHeartbeatSentAt: startedAt, + ...clearedRecoveryFields, + }; + }, + { preserveActivity: true }, + ); } const eventStatus = deliveredAgentRunFailure diff --git a/src/infra/kysely-sync.ts b/src/infra/kysely-sync.ts index 18775db5bd76..fe61465cdf04 100644 --- a/src/infra/kysely-sync.ts +++ b/src/infra/kysely-sync.ts @@ -65,6 +65,20 @@ export function executeSqliteQuerySync<Row>( return executeCompiledSqliteQuerySync<Row>(db, query.compile()); } +/** Compile and lazily iterate a Kysely query synchronously against node:sqlite. */ +export function* iterateSqliteQuerySync<Row>( + db: DatabaseSync, + query: CompilableQuery<Row>, +): IterableIterator<Row> { + const compiledQuery = query.compile(); + const statement = db.prepare(compiledQuery.sql); + if (statement.columns().length === 0) { + return; + } + const parameters = compiledQuery.parameters as SQLInputValue[]; + yield* statement.iterate(...parameters) as Iterable<Row>; +} + /** Execute a Kysely query synchronously and return its first row. */ export function executeSqliteQueryTakeFirstSync<Row>( db: DatabaseSync, diff --git a/src/infra/session-cost-usage.test.ts b/src/infra/session-cost-usage.test.ts index 37bb0838ed83..fb6be18a863a 100644 --- a/src/infra/session-cost-usage.test.ts +++ b/src/infra/session-cost-usage.test.ts @@ -4,6 +4,10 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; +import { + persistSessionTranscriptTurn, + upsertSessionEntry, +} from "../config/sessions/session-accessor.js"; import { setGatewayModelPricingForTest, clearGatewayModelPricingCacheState, @@ -23,6 +27,7 @@ import { loadSessionUsageTimeSeries, requestCostUsageCacheRefresh, refreshCostUsageCache, + resolveExistingUsageSessionFile, } from "./session-cost-usage.js"; describe("session cost usage", () => { @@ -156,6 +161,203 @@ describe("session cost usage", () => { }); }); + it("includes SQLite-only sessions in cached usage cost summaries", async () => { + const root = await makeSessionCostRoot("sqlite-cost"); + const storePath = path.join(root, "agents", "main", "sessions", "sessions.json"); + const sessionKey = "agent:main:main"; + const sessionId = "sqlite-cost-session"; + const now = Date.UTC(2026, 5, 25, 12, 0, 0); + + await withStateDir(root, async () => { + await upsertSessionEntry( + { sessionKey, storePath }, + { + sessionFile: `sqlite:main:${sessionId}:${storePath}`, + sessionId, + updatedAt: now, + }, + ); + await persistSessionTranscriptTurn( + { agentId: "main", sessionId, sessionKey, storePath }, + { + messages: [ + { message: { role: "user", content: "sqlite usage prompt", timestamp: now } }, + { + message: { + role: "assistant", + content: "sqlite usage answer", + model: "gpt-5.4", + provider: "openai", + timestamp: now + 1000, + usage: { + input: 7, + output: 11, + totalTokens: 18, + cost: { total: 0.018 }, + }, + }, + }, + ], + touchSessionEntry: false, + }, + ); + + const legacyJsonl = path.join(path.dirname(storePath), `${sessionId}.jsonl`); + await fs.mkdir(path.dirname(legacyJsonl), { recursive: true }); + await fs.writeFile( + legacyJsonl, + transcriptText(sessionId, { + type: "message", + timestamp: "2026-06-25T12:00:00.000Z", + message: { + role: "assistant", + model: "gpt-5.4", + provider: "openai", + usage: { + input: 100, + output: 100, + totalTokens: 200, + cost: { total: 0.2 }, + }, + }, + }), + "utf-8", + ); + expect(nodeFs.existsSync(legacyJsonl)).toBe(true); + await refreshCostUsageCache({ agentId: "main" }); + const summary = await loadCostUsageSummaryFromCache({ + agentId: "main", + startMs: Date.UTC(2026, 5, 25), + endMs: Date.UTC(2026, 5, 25, 23, 59, 59, 999), + requestRefresh: false, + }); + + expect(summary.totals.totalTokens).toBe(18); + expect(summary.totals.totalCost).toBeCloseTo(0.018, 8); + expect(summary.cacheStatus?.status).toBe("fresh"); + + const sessionFile = `sqlite:main:${sessionId}:${storePath}`; + const sessionEntry = { + sessionFile, + sessionId, + updatedAt: now, + }; + await refreshCostUsageCache({ agentId: "main", sessionFiles: [sessionFile] }); + const bulk = await loadSessionCostSummariesFromCache({ + agentId: "main", + sessions: [{ sessionId, sessionFile }], + startMs: Date.UTC(2026, 5, 25), + endMs: Date.UTC(2026, 5, 25, 23, 59, 59, 999), + requestRefresh: false, + }); + expect(bulk.cacheStatus.status).toBe("fresh"); + expect(bulk.summaries[0]?.totalTokens).toBe(18); + + const summaryFromStalePath = await loadSessionCostSummary({ + agentId: "main", + sessionEntry, + sessionFile: legacyJsonl, + sessionId, + }); + expect(summaryFromStalePath?.totalTokens).toBe(18); + + await expect(loadSessionUsageTimeSeries({ agentId: "main", sessionFile })).resolves.toEqual({ + sessionId: undefined, + points: [ + expect.objectContaining({ + input: 7, + output: 11, + totalTokens: 18, + }), + ], + }); + await expect( + loadSessionUsageTimeSeries({ + agentId: "main", + sessionEntry, + sessionFile: legacyJsonl, + sessionId, + }), + ).resolves.toEqual({ + sessionId, + points: [ + expect.objectContaining({ + input: 7, + output: 11, + totalTokens: 18, + }), + ], + }); + await expect(loadSessionLogs({ agentId: "main", sessionFile })).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + cost: 0.018, + role: "assistant", + tokens: 18, + }), + ]), + ); + await expect( + loadSessionLogs({ + agentId: "main", + sessionEntry, + sessionFile: legacyJsonl, + sessionId, + }), + ).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + cost: 0.018, + role: "assistant", + tokens: 18, + }), + ]), + ); + }); + }); + + it("does not fall back from empty SQLite transcripts to stale JSONL usage files", async () => { + const root = await makeSessionCostRoot("sqlite-cost-empty"); + const storePath = path.join(root, "agents", "main", "sessions", "sessions.json"); + const sessionKey = "agent:main:empty-sqlite-cost"; + const sessionId = "empty-sqlite-cost-session"; + const sqliteMarker = `sqlite:main:${sessionId}:${storePath}`; + const legacyJsonl = path.join(path.dirname(storePath), `${sessionId}.jsonl`); + + await withStateDir(root, async () => { + await upsertSessionEntry( + { sessionKey, storePath }, + { + sessionFile: sqliteMarker, + sessionId, + updatedAt: Date.UTC(2026, 5, 25, 12, 0, 0), + }, + ); + await fs.mkdir(path.dirname(legacyJsonl), { recursive: true }); + await fs.writeFile( + legacyJsonl, + transcriptText(sessionId, { + type: "message", + timestamp: "2026-06-25T12:00:00.000Z", + message: { + role: "assistant", + usage: { input: 100, output: 100, totalTokens: 200, cost: { total: 0.2 } }, + }, + }), + "utf-8", + ); + + expect( + resolveExistingUsageSessionFile({ + agentId: "main", + sessionEntry: { sessionFile: sqliteMarker, sessionId, updatedAt: 1 }, + sessionFile: legacyJsonl, + sessionId, + }), + ).toBe(sqliteMarker); + }); + }); + it("reuses resolved model costs while scanning repeated session usage entries", async () => { const root = await makeSessionCostRoot("cost-resolver-cache"); const sessionsDir = path.join(root, "agents", "main", "sessions"); diff --git a/src/infra/session-cost-usage.ts b/src/infra/session-cost-usage.ts index 83b17da82ccb..f1ec17ad9d02 100644 --- a/src/infra/session-cost-usage.ts +++ b/src/infra/session-cost-usage.ts @@ -16,9 +16,21 @@ import { parseUsageCountedSessionIdFromFileName, } from "../config/sessions/artifacts.js"; import { + resolveDefaultSessionStorePath, resolveSessionFilePath, resolveSessionTranscriptsDirForAgent, } from "../config/sessions/paths.js"; +import { + listSessionEntries, + loadTranscriptEventsSync, + readTranscriptStatsSync, +} from "../config/sessions/session-accessor.js"; +import { + formatSqliteSessionFileMarker, + parseSqliteSessionFileMarker, + type SqliteSessionFileMarker, +} from "../config/sessions/sqlite-marker.js"; +import { selectVisibleTranscriptEvents } from "../config/sessions/transcript-visible-events.js"; import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; @@ -148,6 +160,7 @@ type UsageCostTranscriptFile = { filePath: string; size: number; mtimeMs: number; + sessionId?: string; }; type UsageCostCacheLock = { @@ -169,6 +182,15 @@ function resolveUsageCostCachePath(agentId?: string): string { return path.join(resolveSessionTranscriptsDirForAgent(agentId), USAGE_COST_CACHE_FILE); } +function resolveUsageCostSessionStorePath(params?: { + agentId?: string; + sessionsDir?: string; +}): string { + return params?.sessionsDir + ? path.join(params.sessionsDir, "sessions.json") + : resolveDefaultSessionStorePath(params?.agentId); +} + function resolveUsageCostCacheLockPath(cachePath: string): string { return `${cachePath}.lock`; } @@ -402,11 +424,84 @@ async function listUsageCountedTranscriptFileStats( return results.filter((file): file is UsageCostTranscriptFile => Boolean(file)); } +function listUsageCountedSqliteTranscriptStats( + agentId?: string, + params?: { minMtimeMs?: number; sessionsDir?: string }, +): UsageCostTranscriptFile[] { + const storePath = resolveUsageCostSessionStorePath({ + agentId, + ...(params?.sessionsDir ? { sessionsDir: params.sessionsDir } : {}), + }); + const files: UsageCostTranscriptFile[] = []; + for (const { entry } of listSessionEntries({ storePath })) { + const marker = parseSqliteSessionFileMarker(entry.sessionFile); + if (!marker) { + continue; + } + const mtimeMs = asFiniteNumber(entry.updatedAt) ?? 0; + if (params?.minMtimeMs !== undefined && mtimeMs < params.minMtimeMs) { + continue; + } + // Usage scans run across every session on hot paths; byte sizes come from + // a SQL aggregate so no transcript row is materialized (#86718 class). + const stats = readTranscriptStatsSync({ + agentId: marker.agentId, + sessionId: marker.sessionId, + storePath: marker.storePath, + }); + files.push({ + filePath: formatSqliteSessionFileMarker(marker), + mtimeMs, + sessionId: marker.sessionId, + size: stats.sizeBytes, + }); + } + return files; +} + async function listUsageCountedTranscriptFiles( agentId?: string, params?: { sessionsDir?: string }, ): Promise<UsageCostTranscriptFile[]> { - return await listUsageCountedTranscriptFileStats(agentId, params); + return await listUsageCountedTranscriptStats(agentId, params); +} + +async function listUsageCountedTranscriptStats( + agentId?: string, + params?: { minMtimeMs?: number; sessionsDir?: string }, +): Promise<UsageCostTranscriptFile[]> { + const fileBacked = await listUsageCountedTranscriptFileStats(agentId, params); + const sqliteBacked = listUsageCountedSqliteTranscriptStats(agentId, params); + const sqliteSessionIds = new Set(sqliteBacked.map((file) => file.sessionId).filter(Boolean)); + const canonicalFileBacked = fileBacked.filter((file) => { + const sessionId = parseUsageCountedSessionIdFromFileName(path.basename(file.filePath)); + return !sessionId || !sqliteSessionIds.has(sessionId); + }); + return [...canonicalFileBacked, ...sqliteBacked]; +} + +async function resolveUsageCostTranscriptFile( + sessionFile: string, +): Promise<UsageCostTranscriptFile | undefined> { + const marker = parseSqliteSessionFileMarker(sessionFile); + if (marker) { + const entry = listSessionEntries({ storePath: marker.storePath }).find( + ({ entry: sessionEntry }) => sessionEntry.sessionId === marker.sessionId, + )?.entry; + const stats = readTranscriptStatsSync({ + agentId: marker.agentId, + sessionId: marker.sessionId, + storePath: marker.storePath, + }); + return { + filePath: formatSqliteSessionFileMarker(marker), + mtimeMs: asFiniteNumber(entry?.updatedAt) ?? 0, + sessionId: marker.sessionId, + size: stats.sizeBytes, + }; + } + const stats = await fs.promises.stat(sessionFile).catch(() => null); + return stats ? { filePath: sessionFile, size: stats.size, mtimeMs: stats.mtimeMs } : undefined; } function isUsageCostCacheEntryFresh(params: { @@ -871,13 +966,6 @@ const extractCostBreakdown = (usageRaw?: UsageLike | null): CostBreakdown | unde }; const parseTimestamp = (entry: Record<string, unknown>): Date | undefined => { - const raw = entry.timestamp; - if (typeof raw === "string") { - const parsed = new Date(raw); - if (!Number.isNaN(parsed.valueOf())) { - return parsed; - } - } const message = entry.message as Record<string, unknown> | undefined; const messageTimestamp = asFiniteNumber(message?.timestamp); if (messageTimestamp !== undefined) { @@ -886,6 +974,13 @@ const parseTimestamp = (entry: Record<string, unknown>): Date | undefined => { return parsed; } } + const raw = entry.timestamp; + if (typeof raw === "string") { + const parsed = new Date(raw); + if (!Number.isNaN(parsed.valueOf())) { + return parsed; + } + } return undefined; }; @@ -1262,11 +1357,41 @@ async function* readJsonlRecords( } } -async function* readJsonlRecordsBestEffort( +function loadSqliteUsageTranscriptEvents( + marker: SqliteSessionFileMarker, +): Record<string, unknown>[] { + return selectVisibleTranscriptEvents( + loadTranscriptEventsSync({ + agentId: marker.agentId, + sessionId: marker.sessionId, + storePath: marker.storePath, + }), + ).filter( + (event): event is Record<string, unknown> => + Boolean(event) && typeof event === "object" && !Array.isArray(event), + ); +} + +async function* readTranscriptRecords( + filePath: string, + startOffset = 0, + endOffset?: number, +): AsyncGenerator<Record<string, unknown>> { + const marker = parseSqliteSessionFileMarker(filePath); + if (marker) { + for (const event of loadSqliteUsageTranscriptEvents(marker)) { + yield event; + } + return; + } + yield* readJsonlRecords(filePath, startOffset, endOffset); +} + +async function* readTranscriptRecordsBestEffort( filePath: string, ): AsyncGenerator<Record<string, unknown>> { try { - yield* readJsonlRecords(filePath); + yield* readTranscriptRecords(filePath); } catch { // Diagnostic readers return the records available before a stream failure. // Durable cache scans use the strict reader so partial data is never marked fresh. @@ -1282,7 +1407,7 @@ async function scanTranscriptFile(params: { onEntry: (entry: ParsedTranscriptEntry) => void; }): Promise<void> { const resolveCost = params.resolveCost ?? createUsageCostResolver(params.config); - for await (const parsed of readJsonlRecords( + for await (const parsed of readTranscriptRecords( params.filePath, params.startOffset, params.endOffset, @@ -1379,10 +1504,21 @@ export function resolveExistingUsageSessionFile(params: { sessionFile?: string; agentId?: string; }): string | undefined { + const sessionId = params.sessionId?.trim(); + const entryMarker = parseSqliteSessionFileMarker(params.sessionEntry?.sessionFile); + const explicitMarker = parseSqliteSessionFileMarker(params.sessionFile); + const sqliteMarker = entryMarker ?? explicitMarker; + if (sqliteMarker) { + if (sessionId && sqliteMarker.sessionId !== sessionId) { + return undefined; + } + return formatSqliteSessionFileMarker(sqliteMarker); + } + const candidate = params.sessionFile ?? - (params.sessionId - ? resolveSessionFilePath(params.sessionId, params.sessionEntry, { + (sessionId + ? resolveSessionFilePath(sessionId, params.sessionEntry, { agentId: params.agentId, }) : undefined); @@ -1390,8 +1526,6 @@ export function resolveExistingUsageSessionFile(params: { if (candidate && fs.existsSync(candidate)) { return candidate; } - - const sessionId = params.sessionId?.trim(); if (!sessionId) { return candidate; } @@ -1466,7 +1600,7 @@ export async function loadCostUsageSummary(params?: { const totals = emptyTotals(); const resolveCost = createUsageCostResolver(params?.config); - const files = await listUsageCountedTranscriptFileStats(params?.agentId, { + const files = await listUsageCountedTranscriptStats(params?.agentId, { minMtimeMs: sinceTime, }); @@ -1595,7 +1729,9 @@ async function scanUsageFileForCache(params: { }); const sessionId = - parseUsageCountedSessionIdFromFileName(path.basename(params.file.filePath)) ?? undefined; + parseSqliteSessionFileMarker(params.file.filePath)?.sessionId ?? + parseUsageCountedSessionIdFromFileName(path.basename(params.file.filePath)) ?? + undefined; const combinedTranscriptEntries = shouldTrackTranscriptEntries ? [ ...((appendOnlyPrevious && startOffset !== undefined @@ -1835,13 +1971,10 @@ export async function loadSessionCostSummaryFromCache(params: { }): Promise<{ summary: SessionCostSummary | null; cacheStatus: UsageCacheStatus }> { const cachePath = resolveUsageCostCachePath(params.agentId); const pricingFingerprint = resolveUsageCostPricingFingerprint(params.config); - let [cache, stats] = await Promise.all([ + let [cache, file] = await Promise.all([ readUsageCostCache(cachePath, pricingFingerprint), - fs.promises.stat(params.sessionFile).catch(() => null), + resolveUsageCostTranscriptFile(params.sessionFile), ]); - let file = stats - ? { filePath: params.sessionFile, size: stats.size, mtimeMs: stats.mtimeMs } - : undefined; let entry = cache.files[params.sessionFile]; let stale = !file || @@ -1859,13 +1992,10 @@ export async function loadSessionCostSummaryFromCache(params: { sessionFiles: [params.sessionFile], }); if (result === "refreshed") { - [cache, stats] = await Promise.all([ + [cache, file] = await Promise.all([ readUsageCostCache(cachePath, pricingFingerprint), - fs.promises.stat(params.sessionFile).catch(() => null), + resolveUsageCostTranscriptFile(params.sessionFile), ]); - file = stats - ? { filePath: params.sessionFile, size: stats.size, mtimeMs: stats.mtimeMs } - : undefined; entry = cache.files[params.sessionFile]; stale = !file || @@ -1956,16 +2086,16 @@ export async function loadSessionCostSummariesFromCache(params: { }): Promise<{ summaries: Array<SessionCostSummary | null>; cacheStatus: UsageCacheStatus }> { const cachePath = resolveUsageCostCachePath(params.agentId); const pricingFingerprint = resolveUsageCostPricingFingerprint(params.config); - const statTasks = params.sessions.map( - (session) => async () => await fs.promises.stat(session.sessionFile).catch(() => null), + const fileTasks = params.sessions.map( + (session) => async () => await resolveUsageCostTranscriptFile(session.sessionFile), ); - const statsPromise = runTasksWithConcurrency({ - tasks: statTasks, + const filesPromise = runTasksWithConcurrency({ + tasks: fileTasks, limit: USAGE_COST_TRANSCRIPT_STAT_CONCURRENCY, }).then(({ results }) => results); - const [cache, stats, refreshRunning] = await Promise.all([ + const [cache, files, refreshRunning] = await Promise.all([ readUsageCostCache(cachePath, pricingFingerprint), - statsPromise, + filesPromise, isUsageCostCacheRefreshRunning(cachePath), ]); const staleFiles = new Set<string>(); @@ -1977,10 +2107,7 @@ export async function loadSessionCostSummariesFromCache(params: { const getFormatDayKey = () => (sharedFormatDayKey ??= createUsageDayKeyFormatter(params.dayBucket)); const summaries = params.sessions.map((session, index) => { - const stat = stats[index]; - const file = stat - ? { filePath: session.sessionFile, size: stat.size, mtimeMs: stat.mtimeMs } - : undefined; + const file = files[index]; const entry = cache.files[session.sessionFile]; const stale = !file || @@ -2163,7 +2290,7 @@ export async function discoverAllSessions(params?: { endMs?: number; includeFirstUserMessage?: boolean; }): Promise<DiscoveredSession[]> { - const files = await listUsageCountedTranscriptFileStats(params?.agentId, { + const files = await listUsageCountedTranscriptStats(params?.agentId, { minMtimeMs: params?.startMs, }); @@ -2173,18 +2300,19 @@ export async function discoverAllSessions(params?: { // Do not exclude by endMs: a session can have activity in range even if it continued later. const filePath = file.filePath; const fileName = path.basename(filePath); + const sqliteMarker = parseSqliteSessionFileMarker(filePath); - const sessionId = parseUsageCountedSessionIdFromFileName(fileName); + const sessionId = sqliteMarker?.sessionId ?? parseUsageCountedSessionIdFromFileName(fileName); if (!sessionId) { continue; } - const isPrimaryTranscript = isPrimarySessionTranscriptFileName(fileName); + const isPrimaryTranscript = sqliteMarker ? true : isPrimarySessionTranscriptFileName(fileName); // Try to read first user message for label extraction let firstUserMessage: string | undefined; if (params?.includeFirstUserMessage !== false) { try { - for await (const parsed of readJsonlRecords(filePath)) { + for await (const parsed of readTranscriptRecords(filePath)) { try { const message = parsed.message as Record<string, unknown> | undefined; if (message?.role === "user") { @@ -2257,7 +2385,10 @@ export async function loadSessionCostSummary(params: { dayBucket?: UsageDailyBucket; }): Promise<SessionCostSummary | null> { const sessionFile = resolveExistingUsageSessionFile(params); - if (!sessionFile || !fs.existsSync(sessionFile)) { + if (!sessionFile) { + return null; + } + if (!parseSqliteSessionFileMarker(sessionFile) && !fs.existsSync(sessionFile)) { return null; } @@ -2568,7 +2699,10 @@ export async function loadSessionUsageTimeSeries(params: { maxPoints?: number; }): Promise<SessionUsageTimeSeries | null> { const sessionFile = resolveExistingUsageSessionFile(params); - if (!sessionFile || !fs.existsSync(sessionFile)) { + if (!sessionFile) { + return null; + } + if (!parseSqliteSessionFileMarker(sessionFile) && !fs.existsSync(sessionFile)) { return null; } @@ -2677,7 +2811,10 @@ export async function loadSessionLogs(params: { limit?: number; }): Promise<SessionLogEntry[] | null> { const sessionFile = resolveExistingUsageSessionFile(params); - if (!sessionFile || !fs.existsSync(sessionFile)) { + if (!sessionFile) { + return null; + } + if (!parseSqliteSessionFileMarker(sessionFile) && !fs.existsSync(sessionFile)) { return null; } @@ -2692,7 +2829,7 @@ export async function loadSessionLogs(params: { const retentionLimit = limit * 2; const resolveCost = createUsageCostResolver(params.config); - for await (const parsed of readJsonlRecordsBestEffort(sessionFile)) { + for await (const parsed of readTranscriptRecordsBestEffort(sessionFile)) { try { const message = parsed.message as Record<string, unknown> | undefined; if (!message) { diff --git a/src/infra/sqlite-transaction.test.ts b/src/infra/sqlite-transaction.test.ts index fae5536dd759..3990216f44b9 100644 --- a/src/infra/sqlite-transaction.test.ts +++ b/src/infra/sqlite-transaction.test.ts @@ -1,9 +1,18 @@ // Covers synchronous SQLite transaction helpers. -import { afterEach, describe, expect, it } from "vitest"; +import { spawn, type ChildProcessByStdio } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { Readable } from "node:stream"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { requireNodeSqlite } from "./node-sqlite.js"; import { runSqliteImmediateTransactionSync } from "./sqlite-transaction.js"; const openDatabases: Array<import("node:sqlite").DatabaseSync> = []; +type WriterLockChild = ChildProcessByStdio<null, Readable, Readable>; + +const openChildren: WriterLockChild[] = []; +const tempDirs: string[] = []; function createDatabase(): import("node:sqlite").DatabaseSync { const { DatabaseSync } = requireNodeSqlite(); @@ -20,10 +29,73 @@ function readEntries(db: import("node:sqlite").DatabaseSync): string[] { .map((row) => (row as { id: string }).id); } -afterEach(() => { - for (const db of openDatabases.splice(0)) { - db.close(); +function startWriterLockChild(databasePath: string, holdMs: number): WriterLockChild { + const child = spawn( + process.execPath, + [ + "--no-warnings", + "--input-type=module", + "-e", + ` + import { DatabaseSync } from "node:sqlite"; + const db = new DatabaseSync(process.argv[1]); + db.exec("PRAGMA busy_timeout = 1000; BEGIN IMMEDIATE;"); + process.stdout.write("ready\\n"); + setTimeout(() => { + db.exec("COMMIT"); + db.close(); + }, Number(process.argv[2])); + `, + databasePath, + String(holdMs), + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + openChildren.push(child); + return child; +} + +async function waitForChildReady(child: WriterLockChild): Promise<void> { + let output = ""; + for await (const chunk of child.stdout) { + output += chunk.toString(); + if (output.includes("ready")) { + return; + } } + throw new Error(`writer-lock child exited before ready: ${output}`); +} + +async function waitForChildExit(child: WriterLockChild): Promise<void> { + if (child.exitCode !== null) { + expect(child.exitCode).toBe(0); + return; + } + const result = await new Promise<{ code: number | null; stderr: string }>((resolve) => { + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.once("exit", (code) => resolve({ code, stderr })); + }); + expect(result).toEqual({ code: 0, stderr: "" }); +} + +afterEach(() => { + for (const child of openChildren.splice(0)) { + if (child.exitCode === null) { + child.kill("SIGKILL"); + } + } + for (const db of openDatabases.splice(0)) { + if (db.isOpen) { + db.close(); + } + } + for (const tempDir of tempDirs.splice(0)) { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + vi.restoreAllMocks(); }); describe("runSqliteImmediateTransactionSync", () => { @@ -73,34 +145,12 @@ describe("runSqliteImmediateTransactionSync", () => { expect(readEntries(db)).toEqual(["after"]); }); - it("retries retryable commit failures without rolling back successful writes", () => { + it("does not retry commit failures and rolls back the transaction", () => { const execCalls: string[] = []; - let commitAttempts = 0; const db = { exec(sql: string) { execCalls.push(sql); if (sql === "COMMIT") { - commitAttempts += 1; - if (commitAttempts === 1) { - throw Object.assign(new Error("database is busy"), { code: "SQLITE_BUSY" }); - } - } - }, - } as import("node:sqlite").DatabaseSync; - - const result = runSqliteImmediateTransactionSync(db, () => "committed"); - - expect(result).toBe("committed"); - expect(execCalls).toEqual(["BEGIN IMMEDIATE", "COMMIT", "COMMIT"]); - }); - - it("rolls back and clears depth after exhausted retryable commit failures", () => { - const execCalls: string[] = []; - let failCommits = true; - const db = { - exec(sql: string) { - execCalls.push(sql); - if (failCommits && sql === "COMMIT") { throw Object.assign(new Error("database is busy"), { code: "SQLITE_BUSY" }); } }, @@ -110,15 +160,187 @@ describe("runSqliteImmediateTransactionSync", () => { expect(() => runSqliteImmediateTransactionSync(db, () => "not committed")).toThrow( "database is busy", ); + expect(execCalls).toEqual(["BEGIN IMMEDIATE", "COMMIT", "ROLLBACK"]); + }); - expect(execCalls.filter((sql) => sql === "COMMIT")).toHaveLength(8); - expect(execCalls.at(-1)).toBe("ROLLBACK"); + it("logs one structured warning for a terminal lock failure", () => { + const execCalls: string[] = []; + const logger = { warn: vi.fn() }; + const lockError = Object.assign(new Error("database is locked"), { + code: "ERR_SQLITE_ERROR", + errcode: 5, + }); + const db = { + exec(sql: string) { + execCalls.push(sql); + if (sql === "BEGIN IMMEDIATE") { + throw lockError; + } + }, + } as import("node:sqlite").DatabaseSync; - execCalls.length = 0; - failCommits = false; - const result = runSqliteImmediateTransactionSync(db, () => "committed later"); + expect(() => + runSqliteImmediateTransactionSync(db, () => "blocked", { + busyTimeoutMs: 5_000, + databaseLabel: "agent.sqlite", + logger, + operationLabel: "session.patch", + }), + ).toThrow(lockError); + expect(execCalls).toEqual(["BEGIN IMMEDIATE"]); + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith( + "SQLite transaction lock wait failed", + expect.objectContaining({ + async: false, + busyTimeoutMs: 5_000, + code: "ERR_SQLITE_ERROR", + database: "agent.sqlite", + failureKind: "lock-contention", + operation: "session.patch", + pid: process.pid, + sqliteErrcode: 5, + sqlitePrimaryCode: 5, + step: "begin", + }), + ); + }); - expect(result).toBe("committed later"); - expect(execCalls).toEqual(["BEGIN IMMEDIATE", "COMMIT"]); + it("logs slow successful transaction lock waits", () => { + const logger = { warn: vi.fn() }; + let now = 0; + vi.spyOn(Date, "now").mockImplementation(() => { + const value = now; + now += 1_500; + return value; + }); + const db = { + exec() {}, + } as unknown as import("node:sqlite").DatabaseSync; + + runSqliteImmediateTransactionSync(db, () => "committed", { + busyTimeoutMs: 5_000, + databaseLabel: "agent.sqlite", + logger, + slowTransactionHoldMs: 0, + }); + + expect(logger.warn).toHaveBeenCalledWith( + "slow SQLite transaction lock wait", + expect.objectContaining({ + async: false, + database: "agent.sqlite", + elapsedMs: 1_500, + pid: process.pid, + step: "begin", + }), + ); + expect(logger.warn).toHaveBeenCalledWith( + "slow SQLite transaction lock wait", + expect.objectContaining({ + async: false, + database: "agent.sqlite", + elapsedMs: 1_500, + pid: process.pid, + step: "commit", + }), + ); + expect(logger.warn).toHaveBeenCalledWith( + "slow SQLite transaction hold", + expect.objectContaining({ + async: false, + database: "agent.sqlite", + pid: process.pid, + }), + ); + }); + + it("waits for a separate writer and exposes the synchronous event-loop cost", async () => { + const holdMs = 200; + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-contention-")); + tempDirs.push(tempDir); + const databasePath = path.join(tempDir, "contention.sqlite"); + const { DatabaseSync } = requireNodeSqlite(); + const db = new DatabaseSync(databasePath); + openDatabases.push(db); + db.exec( + "PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 1000; CREATE TABLE entries (id TEXT PRIMARY KEY);", + ); + const child = startWriterLockChild(databasePath, holdMs); + await waitForChildReady(child); + + let timerFiredAt: number | undefined; + const timerStartedAt = Date.now(); + setTimeout(() => { + timerFiredAt = Date.now(); + }, 0); + const transactionStartedAt = Date.now(); + runSqliteImmediateTransactionSync( + db, + () => db.prepare("INSERT INTO entries(id) VALUES (?)").run("parent"), + { busyTimeoutMs: 1_000 }, + ); + const elapsedMs = Date.now() - transactionStartedAt; + + expect(elapsedMs).toBeGreaterThanOrEqual(Math.floor(holdMs / 2)); + expect(timerFiredAt).toBeUndefined(); + await new Promise<void>((resolve) => { + setTimeout(resolve, 0); + }); + expect((timerFiredAt ?? 0) - timerStartedAt).toBeGreaterThanOrEqual(Math.floor(holdMs / 2)); + await waitForChildExit(child); + expect(db.prepare("SELECT id FROM entries").all()).toEqual([{ id: "parent" }]); + }); + + it("fails a real separate-writer wait after the single SQLite busy timeout", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-timeout-")); + tempDirs.push(tempDir); + const databasePath = path.join(tempDir, "contention.sqlite"); + const { DatabaseSync } = requireNodeSqlite(); + const db = new DatabaseSync(databasePath); + openDatabases.push(db); + db.exec( + "PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 20; CREATE TABLE entries (id TEXT PRIMARY KEY);", + ); + const child = startWriterLockChild(databasePath, 250); + await waitForChildReady(child); + + const logger = { warn: vi.fn() }; + const startedAt = Date.now(); + let thrown: unknown; + try { + runSqliteImmediateTransactionSync(db, () => undefined, { + busyTimeoutMs: 20, + databaseLabel: databasePath, + logger, + operationLabel: "contention-proof", + }); + } catch (error) { + thrown = error; + } + const elapsedMs = Date.now() - startedAt; + expect(thrown).toMatchObject({ code: "ERR_SQLITE_ERROR", errcode: 5 }); + expect(elapsedMs).toBeGreaterThanOrEqual(15); + expect(elapsedMs).toBeLessThan(200); + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith( + "SQLite transaction lock wait failed", + expect.objectContaining({ + busyTimeoutMs: 20, + code: "ERR_SQLITE_ERROR", + failureKind: "lock-contention", + operation: "contention-proof", + sqliteErrcode: 5, + sqlitePrimaryCode: 5, + step: "begin", + }), + ); + + await waitForChildExit(child); + expect(() => + runSqliteImmediateTransactionSync(db, () => { + db.prepare("INSERT INTO entries(id) VALUES (?)").run("after-timeout"); + }), + ).not.toThrow(); }); }); diff --git a/src/infra/sqlite-transaction.ts b/src/infra/sqlite-transaction.ts index 5926055c6092..dabf3a3015d2 100644 --- a/src/infra/sqlite-transaction.ts +++ b/src/infra/sqlite-transaction.ts @@ -1,12 +1,30 @@ -// Provides synchronous SQLite transaction helpers with nested savepoints. +// Provides SQLite transaction helpers with nested savepoints. import type { DatabaseSync } from "node:sqlite"; +import { createSubsystemLogger, type SubsystemLogger } from "../logging/subsystem.js"; const transactionDepthByDatabase = new WeakMap<DatabaseSync, number>(); -const RETRYABLE_COMMIT_ERROR_CODES = new Set(["SQLITE_BUSY", "SQLITE_LOCKED"]); -const MAX_COMMIT_ATTEMPTS = 8; +const SQLITE_LOCK_ERROR_CODES = new Set(["SQLITE_BUSY", "SQLITE_LOCKED"]); +// Node reports SQLite failures with a generic string code and the extended +// SQLite result in `errcode`; the low byte identifies BUSY or LOCKED. +const SQLITE_BUSY_RESULT_CODE = 5; +const SQLITE_LOCKED_RESULT_CODE = 6; +const SQLITE_PRIMARY_RESULT_CODE_MASK = 0xff; +const DEFAULT_SLOW_BUSY_WAIT_MS = 1_000; +const DEFAULT_SLOW_TRANSACTION_HOLD_MS = 1_000; let nextSavepointId = 0; +const transactionLog = createSubsystemLogger("sqlite/transaction"); + +export type SqliteTransactionOptions = { + busyTimeoutMs?: number; + databaseLabel?: string; + logger?: Pick<SubsystemLogger, "warn">; + operationLabel?: string; + slowTransactionHoldMs?: number; +}; + +type SqliteTransactionStep = "begin" | "commit"; function nextSavepointName(): string { nextSavepointId += 1; @@ -25,22 +43,149 @@ function assertSyncTransactionResult(value: unknown): void { } } -function isRetryableCommitError(error: unknown): boolean { +function sqliteErrorCode(error: unknown): string | undefined { const code = error && typeof error === "object" ? (error as { code?: unknown }).code : undefined; - return typeof code === "string" && RETRYABLE_COMMIT_ERROR_CODES.has(code); + return typeof code === "string" ? code : undefined; } -function commitImmediateTransaction(db: DatabaseSync): void { - for (const attempt of Array.from({ length: MAX_COMMIT_ATTEMPTS }, (_, index) => index + 1)) { - try { - db.exec("COMMIT"); - return; - } catch (error) { - if (!isRetryableCommitError(error) || attempt >= MAX_COMMIT_ATTEMPTS) { - throw error; - } - } +function sqliteExtendedResultCode(error: unknown): number | undefined { + const errcode = + error && typeof error === "object" ? (error as { errcode?: unknown }).errcode : undefined; + return typeof errcode === "number" && Number.isInteger(errcode) ? errcode : undefined; +} + +function sqlitePrimaryResultCode(error: unknown): number | undefined { + const errcode = sqliteExtendedResultCode(error); + return errcode === undefined ? undefined : errcode & SQLITE_PRIMARY_RESULT_CODE_MASK; +} + +function isTransactionLockError(error: unknown): boolean { + const code = sqliteErrorCode(error); + if (code !== undefined && SQLITE_LOCK_ERROR_CODES.has(code)) { + return true; } + const primaryCode = sqlitePrimaryResultCode(error); + return primaryCode === SQLITE_BUSY_RESULT_CODE || primaryCode === SQLITE_LOCKED_RESULT_CODE; +} + +function slowBusyWaitThresholdMs(options: SqliteTransactionOptions | undefined): number { + if (options?.busyTimeoutMs === undefined) { + return DEFAULT_SLOW_BUSY_WAIT_MS; + } + return Math.min(DEFAULT_SLOW_BUSY_WAIT_MS, Math.max(1, options.busyTimeoutMs)); +} + +function slowTransactionHoldThresholdMs(options: SqliteTransactionOptions | undefined): number { + return options?.slowTransactionHoldMs ?? DEFAULT_SLOW_TRANSACTION_HOLD_MS; +} + +function transactionLogger( + options: SqliteTransactionOptions | undefined, +): Pick<SubsystemLogger, "warn"> { + return options?.logger ?? transactionLog; +} + +function logSlowTransactionHold(params: { + elapsedMs: number; + options?: SqliteTransactionOptions; +}): void { + if (params.elapsedMs < slowTransactionHoldThresholdMs(params.options)) { + return; + } + transactionLogger(params.options).warn("slow SQLite transaction hold", { + async: false, + ...(params.options?.databaseLabel ? { database: params.options.databaseLabel } : {}), + elapsedMs: params.elapsedMs, + ...(params.options?.operationLabel ? { operation: params.options.operationLabel } : {}), + pid: process.pid, + thresholdMs: slowTransactionHoldThresholdMs(params.options), + }); +} + +function logSlowTransactionStep(params: { + elapsedMs: number; + options?: SqliteTransactionOptions; + step: SqliteTransactionStep; +}): void { + if (params.elapsedMs < slowBusyWaitThresholdMs(params.options)) { + return; + } + transactionLogger(params.options).warn("slow SQLite transaction lock wait", { + async: false, + ...(params.options?.busyTimeoutMs !== undefined + ? { busyTimeoutMs: params.options.busyTimeoutMs } + : {}), + ...(params.options?.databaseLabel ? { database: params.options.databaseLabel } : {}), + elapsedMs: params.elapsedMs, + ...(params.options?.operationLabel ? { operation: params.options.operationLabel } : {}), + pid: process.pid, + step: params.step, + }); +} + +function execTimedTransactionStep(params: { + db: DatabaseSync; + options?: SqliteTransactionOptions; + sql: string; + step: SqliteTransactionStep; +}): number { + const startedAt = Date.now(); + try { + params.db.exec(params.sql); + const elapsedMs = Date.now() - startedAt; + logSlowTransactionStep({ + elapsedMs, + options: params.options, + step: params.step, + }); + return elapsedMs; + } catch (error) { + const elapsedMs = Date.now() - startedAt; + if (isTransactionLockError(error)) { + const sqliteErrcode = sqliteExtendedResultCode(error); + const sqlitePrimaryCode = sqlitePrimaryResultCode(error); + transactionLogger(params.options).warn("SQLite transaction lock wait failed", { + async: false, + ...(params.options?.busyTimeoutMs !== undefined + ? { busyTimeoutMs: params.options.busyTimeoutMs } + : {}), + ...(params.options?.databaseLabel ? { database: params.options.databaseLabel } : {}), + code: sqliteErrorCode(error), + elapsedMs, + failureKind: "lock-contention", + ...(params.options?.operationLabel ? { operation: params.options.operationLabel } : {}), + pid: process.pid, + ...(sqliteErrcode !== undefined ? { sqliteErrcode } : {}), + ...(sqlitePrimaryCode !== undefined ? { sqlitePrimaryCode } : {}), + step: params.step, + }); + } + throw error; + } +} + +function beginImmediateTransaction( + db: DatabaseSync, + options: SqliteTransactionOptions | undefined, +): void { + execTimedTransactionStep({ + db, + options, + sql: "BEGIN IMMEDIATE", + step: "begin", + }); +} + +function commitImmediateTransaction( + db: DatabaseSync, + options: SqliteTransactionOptions | undefined, +): void { + execTimedTransactionStep({ + db, + options, + sql: "COMMIT", + step: "commit", + }); } function abortImmediateTransaction(db: DatabaseSync): void { @@ -69,7 +214,11 @@ function setTransactionDepth(db: DatabaseSync, depth: number): void { transactionDepthByDatabase.set(db, depth); } -export function runSqliteImmediateTransactionSync<T>(db: DatabaseSync, operation: () => T): T { +export function runSqliteImmediateTransactionSync<T>( + db: DatabaseSync, + operation: () => T, + options?: SqliteTransactionOptions, +): T { const depth = getTransactionDepth(db); if (depth > 0) { const savepointName = nextSavepointName(); @@ -92,10 +241,11 @@ export function runSqliteImmediateTransactionSync<T>(db: DatabaseSync, operation } } - db.exec("BEGIN IMMEDIATE"); + beginImmediateTransaction(db, options); setTransactionDepth(db, 1); let transactionStillActive = true; let result: T; + const transactionStartedAt = Date.now(); try { result = operation(); assertSyncTransactionResult(result); @@ -114,7 +264,11 @@ export function runSqliteImmediateTransactionSync<T>(db: DatabaseSync, operation } try { - commitImmediateTransaction(db); + logSlowTransactionHold({ + elapsedMs: Date.now() - transactionStartedAt, + options, + }); + commitImmediateTransaction(db, options); transactionStillActive = false; return result; } catch (error) { diff --git a/src/infra/sqlite-wal.test.ts b/src/infra/sqlite-wal.test.ts index 11bacef6c5b1..fc582113ecf4 100644 --- a/src/infra/sqlite-wal.test.ts +++ b/src/infra/sqlite-wal.test.ts @@ -457,15 +457,16 @@ describe("sqlite WAL maintenance", () => { expect(db["exec"]).toHaveBeenCalledTimes(2); vi.advanceTimersByTime(100); - expect(db["exec"]).toHaveBeenLastCalledWith("PRAGMA wal_checkpoint(PASSIVE);"); - expect(db["exec"]).toHaveBeenCalledTimes(3); + expect(db["exec"]).toHaveBeenNthCalledWith(3, "PRAGMA wal_checkpoint(PASSIVE);"); + expect(db["exec"]).toHaveBeenNthCalledWith(4, "PRAGMA incremental_vacuum(512);"); + expect(db["exec"]).toHaveBeenCalledTimes(4); expect(maintenance.close()).toBe(true); expect(db["exec"]).toHaveBeenLastCalledWith("PRAGMA wal_checkpoint(TRUNCATE);"); - expect(db["exec"]).toHaveBeenCalledTimes(4); + expect(db["exec"]).toHaveBeenCalledTimes(5); vi.advanceTimersByTime(200); - expect(db["exec"]).toHaveBeenCalledTimes(4); + expect(db["exec"]).toHaveBeenCalledTimes(5); }); it("clamps oversized checkpoint intervals before arming timers", () => { @@ -493,7 +494,8 @@ describe("sqlite WAL maintenance", () => { }); vi.advanceTimersByTime(100); - expect(db["exec"]).toHaveBeenLastCalledWith("PRAGMA wal_checkpoint(FULL);"); + expect(db["exec"]).toHaveBeenNthCalledWith(3, "PRAGMA wal_checkpoint(FULL);"); + expect(db["exec"]).toHaveBeenNthCalledWith(4, "PRAGMA incremental_vacuum(512);"); expect(maintenance.close()).toBe(true); expect(db["exec"]).toHaveBeenLastCalledWith("PRAGMA wal_checkpoint(FULL);"); diff --git a/src/infra/sqlite-wal.ts b/src/infra/sqlite-wal.ts index 7f651e1bb663..7693ad745c25 100644 --- a/src/infra/sqlite-wal.ts +++ b/src/infra/sqlite-wal.ts @@ -14,6 +14,9 @@ export const DEFAULT_SQLITE_WAL_CHECKPOINT_INTERVAL_MS = 30 * 60 * 1000; * Periodic checkpoints default to PASSIVE. */ export const DEFAULT_SQLITE_WAL_TRUNCATE_INTERVAL_MS = DEFAULT_SQLITE_WAL_CHECKPOINT_INTERVAL_MS; +// 512 pages (~2MB at 4KB pages) per periodic pass keeps page release strictly +// bounded so maintenance can never behave like a blocking full VACUUM. +const INCREMENTAL_VACUUM_MAX_PAGES_PER_PASS = 512; const LINUX_NFS_SUPER_MAGIC = 0x6969; const LINUX_SMB_SUPER_MAGIC = 0x517b; const LINUX_CIFS_SUPER_MAGIC = 0xff534d42; @@ -338,14 +341,25 @@ export function configureSqliteWalMaintenance( } }; + // Bounded page release for databases opened with auto_vacuum=INCREMENTAL. + // A no-op elsewhere, and never a blocking full VACUUM: unbounded vacuums on + // the event loop have starved channel sockets in production (#83712). + const runIncrementalVacuum = (): void => { + try { + db.exec(`PRAGMA incremental_vacuum(${INCREMENTAL_VACUUM_MAX_PAGES_PER_PASS});`); + } catch (error) { + options.onCheckpointError?.(error); + } + }; + const checkpoint = (): boolean => runCheckpoint(checkpointMode); let timer: IntervalHandle | null = null; if (timerIntervalMs > 0) { - timer = setInterval( - () => runCheckpoint(periodicCheckpointMode), - timerIntervalMs, - ) as IntervalHandle; + timer = setInterval(() => { + runCheckpoint(periodicCheckpointMode); + runIncrementalVacuum(); + }, timerIntervalMs) as IntervalHandle; timer.unref?.(); } diff --git a/src/infra/state-migrations.ts b/src/infra/state-migrations.ts index 14f26c641bdc..576bf13de18a 100644 --- a/src/infra/state-migrations.ts +++ b/src/infra/state-migrations.ts @@ -21,10 +21,12 @@ import { import type { SessionEntry } from "../config/sessions.js"; import { saveSessionStore } from "../config/sessions.js"; import { canonicalizeMainSessionAlias } from "../config/sessions/main-session.js"; +import { resolveAgentsDirFromSessionStorePath } from "../config/sessions/paths.js"; import { normalizePersistedSessionEntryShape } from "../config/sessions/store-entry-shape.js"; import { listConfiguredSessionStoreAgentIds, resolveAllAgentSessionStoreTargetsSync, + resolveSessionStoreTargets, } from "../config/sessions/targets.js"; import type { SessionScope } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; @@ -5539,7 +5541,7 @@ async function migrateLegacyAcpSessionMetadata(params: { : params.cfg; // Reuse the validated resolver for every declared owner. Owner multiplicity // is restored below as metadata without re-adding rejected raw paths. - const targets = resolveAllAgentSessionStoreTargetsSync(discoveryCfg, { env }); + const targets = resolveLegacyAcpMetadataSessionStoreTargets(discoveryCfg, env); const mainKey = normalizeMainKey(params.cfg.session?.mainKey); const scope = params.cfg.session?.scope as SessionScope | undefined; const storeGroups: Array<{ @@ -5687,6 +5689,78 @@ async function migrateLegacyAcpSessionMetadata(params: { return { changes, warnings }; } +// Doctor migration must read legacy session stores even before a per-agent +// SQLite DB exists; active runtime discovery remains SQLite-validated. +function resolveLegacyAcpMetadataSessionStoreTargets( + cfg: OpenClawConfig, + env: NodeJS.ProcessEnv, +): Array<{ agentId: string; storePath: string }> { + const stateDir = resolveStateDir(env); + const agentsDirs = new Set<string>([path.join(stateDir, "agents")]); + const targets = new Map<string, { agentId: string; storePath: string }>(); + const addTarget = (agentId: string, storePath: string) => { + if (!isManagedLegacySessionStorePathSafe(storePath)) { + return; + } + const agentsDir = resolveAgentsDirFromSessionStorePath(storePath); + if (agentsDir) { + agentsDirs.add(agentsDir); + } + if (!targets.has(storePath)) { + targets.set(storePath, { agentId, storePath }); + } + }; + + for (const target of resolveAllAgentSessionStoreTargetsSync(cfg, { env })) { + addTarget(target.agentId, target.storePath); + } + for (const target of resolveSessionStoreTargets(cfg, { allAgents: true }, { env })) { + addTarget(target.agentId, target.storePath); + } + + for (const agentsDir of agentsDirs) { + if (!existsDir(agentsDir)) { + continue; + } + for (const entry of safeReadDir(agentsDir)) { + if (!entry.isDirectory()) { + continue; + } + const agentId = normalizeAgentId(entry.name); + const normalizedDirName = normalizeLowercaseStringOrEmpty(entry.name); + if (agentId === DEFAULT_AGENT_ID && normalizedDirName !== agentId) { + continue; + } + addTarget(agentId, path.join(agentsDir, entry.name, "sessions", "sessions.json")); + } + } + return [...targets.values()]; +} + +function isManagedLegacySessionStorePathSafe(storePath: string): boolean { + const resolvedStorePath = path.resolve(storePath); + const agentsDir = resolveAgentsDirFromSessionStorePath(resolvedStorePath); + if (!agentsDir) { + return true; + } + if (!fileExists(resolvedStorePath)) { + return true; + } + + try { + const stat = fs.lstatSync(resolvedStorePath); + if (stat.isSymbolicLink() || !stat.isFile()) { + return false; + } + const resolvedAgentsDir = path.resolve(agentsDir); + const realStorePath = fs.realpathSync.native(resolvedStorePath); + const realAgentsDir = fs.realpathSync.native(resolvedAgentsDir); + return isWithinDir(realAgentsDir, realStorePath); + } catch { + return false; + } +} + function resolveStorePathFromTemplate( template: string, agentId: string, diff --git a/src/logging/diagnostic-session-recovery.ts b/src/logging/diagnostic-session-recovery.ts index ba2a87bdf288..0322305d6fc4 100644 --- a/src/logging/diagnostic-session-recovery.ts +++ b/src/logging/diagnostic-session-recovery.ts @@ -7,6 +7,7 @@ import type { type DiagnosticSessionRecoverySkipReason = | "active_embedded_run" | "active_reply_work" + | "deferred_maintenance_wait" | "active_lane_task" | "already_in_flight" | "missing_session_ref" diff --git a/src/logging/diagnostic-stuck-session-recovery.runtime.test.ts b/src/logging/diagnostic-stuck-session-recovery.runtime.test.ts index 37bdf84244f4..3639e57e5a82 100644 --- a/src/logging/diagnostic-stuck-session-recovery.runtime.test.ts +++ b/src/logging/diagnostic-stuck-session-recovery.runtime.test.ts @@ -17,6 +17,7 @@ const mocks = vi.hoisted(() => ({ resolveActiveEmbeddedRunSessionIdBySessionFile: vi.fn(), resolveActiveEmbeddedRunHandleSessionId: vi.fn(), resolveActiveEmbeddedRunHandleSessionIdBySessionFile: vi.fn(), + resolveEmbeddedAgentReplyRunPhase: vi.fn(), resolveEmbeddedSessionLane: vi.fn((key: string) => `session:${key}`), waitForEmbeddedAgentRunEnd: vi.fn(), getDiagnosticSessionActivitySnapshot: vi.fn(), @@ -54,6 +55,7 @@ vi.mock("../agents/embedded-agent-runner/runs.js", () => ({ resolveActiveEmbeddedRunHandleSessionId: mocks.resolveActiveEmbeddedRunHandleSessionId, resolveActiveEmbeddedRunHandleSessionIdBySessionFile: mocks.resolveActiveEmbeddedRunHandleSessionIdBySessionFile, + resolveEmbeddedAgentReplyRunPhase: mocks.resolveEmbeddedAgentReplyRunPhase, waitForEmbeddedAgentRunEnd: mocks.waitForEmbeddedAgentRunEnd, })); @@ -102,6 +104,7 @@ function resetMocks() { mocks.resolveActiveEmbeddedRunSessionIdBySessionFile.mockReset(); mocks.resolveActiveEmbeddedRunHandleSessionId.mockReset(); mocks.resolveActiveEmbeddedRunHandleSessionIdBySessionFile.mockReset(); + mocks.resolveEmbeddedAgentReplyRunPhase.mockReset(); mocks.resolveEmbeddedSessionLane.mockClear(); mocks.waitForEmbeddedAgentRunEnd.mockReset(); mocks.getDiagnosticSessionActivitySnapshot.mockReset(); @@ -445,6 +448,35 @@ describe("stuck session recovery", () => { ]); }); + it("keeps the lane while reply work waits for deferred maintenance", async () => { + mocks.resolveActiveEmbeddedRunSessionId.mockReturnValue("queued-reply-session"); + mocks.resolveActiveEmbeddedRunHandleSessionId.mockReturnValue(undefined); + mocks.resolveEmbeddedAgentReplyRunPhase.mockReturnValue("waiting_for_deferred_maintenance"); + mocks.isEmbeddedAgentRunActive.mockReturnValue(true); + mocks.isEmbeddedAgentRunHandleActive.mockReturnValue(false); + + const outcome = await recoverStuckDiagnosticSession({ + sessionId: "queued-reply-session", + sessionKey: "agent:main:main", + ageMs: 928_000, + queueDepth: 1, + allowActiveAbort: true, + }); + + expect(outcome).toMatchObject({ + status: "skipped", + action: "keep_lane", + reason: "deferred_maintenance_wait", + activeSessionId: "queued-reply-session", + }); + expect(mocks.abortEmbeddedAgentRun).not.toHaveBeenCalled(); + expect(mocks.forceClearEmbeddedAgentRun).not.toHaveBeenCalled(); + expect(mocks.resetCommandLane).not.toHaveBeenCalled(); + expect(warnLogMessages()).toEqual([ + "stuck session recovery outcome: status=skipped action=keep_lane sessionId=queued-reply-session sessionKey=agent:main:main activeSessionId=queued-reply-session reason=deferred_maintenance_wait", + ]); + }); + it("releases the session lane when abort+drain succeeds but queued messages remain (ghost run + queued messages)", async () => { mocks.resolveActiveEmbeddedRunSessionId.mockReturnValue("ghost-run-session"); mocks.resolveActiveEmbeddedRunHandleSessionId.mockReturnValue(undefined); diff --git a/src/logging/diagnostic-stuck-session-recovery.runtime.ts b/src/logging/diagnostic-stuck-session-recovery.runtime.ts index 6715c3934bfb..2fa4bfb99bc4 100644 --- a/src/logging/diagnostic-stuck-session-recovery.runtime.ts +++ b/src/logging/diagnostic-stuck-session-recovery.runtime.ts @@ -4,6 +4,7 @@ import { abortAndDrainEmbeddedAgentRun, isEmbeddedAgentRunActive, isEmbeddedAgentRunHandleActive, + resolveEmbeddedAgentReplyRunPhase, resolveActiveEmbeddedRunSessionId, resolveActiveEmbeddedRunSessionIdBySessionFile, resolveActiveEmbeddedRunHandleSessionId, @@ -205,6 +206,19 @@ export async function recoverStuckDiagnosticSession( } if (!activeSessionId && activeWorkSessionId && isEmbeddedAgentRunActive(activeWorkSessionId)) { + const activeReplyPhase = resolveEmbeddedAgentReplyRunPhase(activeWorkSessionId); + if (activeReplyPhase === "waiting_for_deferred_maintenance") { + const outcome: StuckSessionRecoveryOutcome = { + status: "skipped", + action: "keep_lane", + reason: "deferred_maintenance_wait", + sessionId: params.sessionId, + sessionKey: params.sessionKey, + activeSessionId: activeWorkSessionId, + }; + diag.warn(`stuck session recovery outcome: ${formatRecoveryOutcome(outcome)}`); + return outcome; + } const reclaimStaleReplyWork = params.allowActiveAbort !== true && isActiveRunProgressStale({ diff --git a/src/node-host/invoke-system-run-allowlist.test.ts b/src/node-host/invoke-system-run-allowlist.test.ts index 55c9333284c3..8a720fa5106e 100644 --- a/src/node-host/invoke-system-run-allowlist.test.ts +++ b/src/node-host/invoke-system-run-allowlist.test.ts @@ -4,12 +4,9 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { - resolveExecApprovalsFromFile, - resolvePlannedSegmentArgv, - type ExecCommandSegment, -} from "../infra/exec-approvals.js"; +import { resolveExecApprovalsFromFile, type ExecCommandSegment } from "../infra/exec-approvals.js"; import { planShellAuthorization } from "../infra/exec-authorization-plan.js"; +import { buildAuthorizedShellCommandFromPlan } from "../infra/exec-authorization-render.js"; import { resolveExecSafeBinRuntimePolicy } from "../infra/exec-safe-bin-runtime-policy.js"; import { evaluateSystemRunAllowlist, @@ -265,6 +262,16 @@ describe("resolveSystemRunExecArgv", () => { const safeBinPolicy = resolveExecSafeBinRuntimePolicy({ global: { safeBins: ["head"] }, }); + const segmentSatisfiedBy: ["safeBins"] = ["safeBins"]; + const expectedCommand = buildAuthorizedShellCommandFromPlan({ + plan: authorizationPlan, + mode: "safeBins", + segmentSatisfiedBy, + }); + expect(expectedCommand.ok).toBe(true); + if (!expectedCommand.ok) { + throw new Error(expectedCommand.reason); + } const result = await resolveSystemRunExecArgv({ plannedAllowlistArgv: undefined, @@ -286,24 +293,15 @@ describe("resolveSystemRunExecArgv", () => { segments: authorizationPlan.groups.flatMap((group) => group.candidates.map((candidate) => candidate.sourceSegment), ), - segmentSatisfiedBy: ["safeBins"], + segmentSatisfiedBy, authorizationPlan, cwd: undefined, env, }); - const [candidate] = authorizationPlan.groups[0]?.candidates ?? []; - if (!candidate) { - throw new Error("expected a safe-bin authorization candidate"); - } - const plannedArgv = resolvePlannedSegmentArgv(candidate.sourceSegment); - if (!plannedArgv) { - throw new Error("expected a safe-bin execution plan"); - } - expect(result).not.toBeNull(); expect(result?.[0]).toBe("/bin/sh"); - expect(result?.[2]).toBe(plannedArgv.join(" ")); + expect(result?.[2]).toBe(expectedCommand.command); }, ); }); diff --git a/src/plugin-sdk/agent-harness-runtime.ts b/src/plugin-sdk/agent-harness-runtime.ts index 4e2257d3ff0a..e14d9bba1b63 100644 --- a/src/plugin-sdk/agent-harness-runtime.ts +++ b/src/plugin-sdk/agent-harness-runtime.ts @@ -6,7 +6,10 @@ import type { CodexBundleMcpThreadConfig, LoadCodexBundleMcpThreadConfigParams, } from "../agents/codex-mcp-config.types.js"; -import type { EmbeddedRunAttemptResult } from "../agents/embedded-agent-runner/run/types.js"; +import type { + EmbeddedRunAttemptParams as CoreEmbeddedRunAttemptParams, + EmbeddedRunAttemptResult, +} from "../agents/embedded-agent-runner/run/types.js"; import { abortAndDrainEmbeddedAgentRun, abortEmbeddedAgentRun, @@ -58,10 +61,8 @@ export type { AgentHarnessUserInputPromptOptions, AgentHarnessUserInputQuestion, } from "../agents/harness/user-input-bridge.js"; -export type { - EmbeddedRunAttemptParams, - EmbeddedRunAttemptResult, -} from "../agents/embedded-agent-runner/run/types.js"; +export type EmbeddedRunAttemptParams = Omit<CoreEmbeddedRunAttemptParams, "trajectoryRecorder">; +export type { EmbeddedRunAttemptResult }; export type { ContextEngine as HarnessContextEngine, ContextEngineHostCapability, @@ -301,18 +302,6 @@ export { resolveSessionWriteLockOptions, type SessionWriteLockAcquireTimeoutConfig, } from "../agents/session-write-lock.js"; -/** - * @deprecated Use appendSessionTranscriptMessageByIdentity from - * openclaw/plugin-sdk/session-transcript-runtime so transcript writes target a - * session identity instead of an active JSONL transcript file. - */ -export { appendSessionTranscriptMessage } from "../config/sessions/transcript-append.js"; -/** - * @deprecated Use publishSessionTranscriptUpdateByIdentity from - * openclaw/plugin-sdk/session-transcript-runtime so transcript updates target - * a session identity instead of an active JSONL transcript file. - */ -export { emitSessionTranscriptUpdate } from "../sessions/transcript-events.js"; export { consumeAdjustedParamsForToolCall, consumePreExecutionBlockedToolCall, diff --git a/src/plugin-sdk/config-runtime.ts b/src/plugin-sdk/config-runtime.ts index 6b1c44d4c1cb..8a7642302d6f 100644 --- a/src/plugin-sdk/config-runtime.ts +++ b/src/plugin-sdk/config-runtime.ts @@ -7,7 +7,6 @@ export { getSessionEntry, listSessionEntries, - loadSessionStore, patchSessionEntry, readSessionUpdatedAt, updateSessionStoreEntry, @@ -140,22 +139,7 @@ export type { TtsPersonaPromptConfig, TtsProvider, } from "../config/types.js"; -export { - clearSessionStoreCacheForTest, - /** - * @deprecated Use patchSessionEntry/upsertSessionEntry for writes. This - * whole-store helper is kept only during the transition before SQLite - * migration. Callers must migrate away from writing sessions.json directly. - */ - saveSessionStore, - /** - * @deprecated Use patchSessionEntry/upsertSessionEntry for writes. This - * whole-store helper is kept only during the transition before SQLite - * migration. Callers must migrate away from updating sessions.json directly. - */ - updateSessionStore, - resolveSessionStoreEntry, -} from "../config/sessions/store.js"; +export { clearSessionStoreCacheForTest } from "../config/sessions/store.js"; // SDK-facing names are a shipped plugin contract; internals route through the // session accessor so the storage backend can change beneath them. export { diff --git a/src/plugin-sdk/core.ts b/src/plugin-sdk/core.ts index 25545158e8c8..79c754a6ba57 100644 --- a/src/plugin-sdk/core.ts +++ b/src/plugin-sdk/core.ts @@ -208,6 +208,7 @@ export type { ChannelPlugin } from "../channels/plugins/types.plugin.js"; export type { ChannelConfigUiHint } from "../channels/plugins/types.config.js"; export type { PluginRuntime, RuntimeLogger } from "../plugins/runtime/types.js"; export type { WizardPrompter } from "../wizard/prompts.js"; +export type { ContextEngineSessionTarget } from "../context-engine/types.js"; export { definePluginEntry } from "./plugin-entry.js"; export { diff --git a/src/plugin-sdk/memory-core-host-engine-foundation.ts b/src/plugin-sdk/memory-core-host-engine-foundation.ts index cb49fec23e6b..e6f3f29e3e77 100644 --- a/src/plugin-sdk/memory-core-host-engine-foundation.ts +++ b/src/plugin-sdk/memory-core-host-engine-foundation.ts @@ -1,7 +1,6 @@ /** * Public SDK foundation surface for memory host engine config, paths, and shared helpers. */ -import { onInternalSessionTranscriptUpdate } from "../sessions/transcript-events.js"; export * from "../../packages/memory-host-sdk/src/engine-foundation.js"; export { @@ -40,20 +39,14 @@ export type { MemorySearchConfig } from "../config/types.tools.js"; export { root } from "../infra/fs-safe.js"; export { createSubsystemLogger } from "../logging/subsystem.js"; export { detectMime } from "@openclaw/media-core/mime"; -export { onSessionTranscriptUpdate } from "../sessions/transcript-events.js"; +export { + onInternalSessionTranscriptUpdate, + onSessionTranscriptUpdate, +} from "../sessions/transcript-events.js"; export { resolveGlobalSingleton } from "../shared/global-singleton.js"; export { runTasksWithConcurrency } from "../utils/run-with-concurrency.js"; export { splitShellArgs } from "../utils/shell-argv.js"; -const MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY = Symbol.for( - "openclaw.memoryCore.sessionTranscriptUpdateSubscriber", -); - -// Memory-core needs target-only internal updates before the SQLite flip, while -// the public SDK listener stays file-backed during the compatibility window. -(globalThis as Record<symbol, unknown>)[MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY] ??= - onInternalSessionTranscriptUpdate; - export { resolveUserPath, shortenHomeInString, diff --git a/src/plugin-sdk/session-store-runtime.test.ts b/src/plugin-sdk/session-store-runtime.test.ts index 0e2d044f023a..e2883fd755aa 100644 --- a/src/plugin-sdk/session-store-runtime.test.ts +++ b/src/plugin-sdk/session-store-runtime.test.ts @@ -1,18 +1,19 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import * as jsonFiles from "../infra/json-files.js"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { appendTranscriptEvent } from "../config/sessions/session-accessor.js"; import { cleanupSessionLifecycleArtifacts, + deleteSessionEntry, getSessionEntry, listSessionEntries, patchSessionEntry, readSessionUpdatedAt, - saveSessionStore, - updateSessionStore, + resolveSessionStoreBackupPaths, updateSessionStoreEntry, upsertSessionEntry, + type SessionEntry, } from "./session-store-runtime.js"; const DAY_MS = 24 * 60 * 60 * 1000; @@ -30,6 +31,15 @@ describe("session-store-runtime compatibility surface", () => { fs.rmSync(tempDir, { recursive: true, force: true }); }); + async function seedSessionEntry(sessionKey: string, entry: SessionEntry): Promise<void> { + await upsertSessionEntry({ + agentId: "main", + sessionKey, + storePath, + entry, + }); + } + it("keeps the public session read shape while using accessor-backed exports", async () => { const sessionKey = "agent:main:main"; await upsertSessionEntry({ @@ -123,20 +133,15 @@ describe("session-store-runtime compatibility surface", () => { it("preserves resolved maintenance settings through entry patches", async () => { const staleSessionKey = "agent:main:stale"; const activeSessionKey = "agent:main:active"; - await saveSessionStore( - storePath, - { - [staleSessionKey]: { - sessionId: "session-stale", - updatedAt: 10, - }, - [activeSessionKey]: { - sessionId: "session-active", - updatedAt: 20, - }, - }, - { skipMaintenance: true }, - ); + const now = Date.now(); + await seedSessionEntry(staleSessionKey, { + sessionId: "session-stale", + updatedAt: now - 8 * DAY_MS, + }); + await seedSessionEntry(activeSessionKey, { + sessionId: "session-active", + updatedAt: now, + }); await expect( patchSessionEntry({ @@ -169,20 +174,14 @@ describe("session-store-runtime compatibility surface", () => { const staleModelRunKey = "agent:main:explicit:model-run-123e4567-e89b-12d3-a456-426614174000"; const activeSessionKey = "agent:main:active"; const now = Date.now(); - await saveSessionStore( - storePath, - { - [staleModelRunKey]: { - sessionId: "session-probe", - updatedAt: now - 2 * DAY_MS, - }, - [activeSessionKey]: { - sessionId: "session-active", - updatedAt: now, - }, - }, - { skipMaintenance: true }, - ); + await seedSessionEntry(staleModelRunKey, { + sessionId: "session-probe", + updatedAt: now - 2 * DAY_MS, + }); + await seedSessionEntry(activeSessionKey, { + sessionId: "session-active", + updatedAt: now, + }); const legacyMaintenanceConfig = { mode: "enforce" as const, @@ -210,119 +209,52 @@ describe("session-store-runtime compatibility surface", () => { }); }); - it("keeps deprecated whole-store mutations grouped as one compatibility operation", async () => { - const firstSessionKey = "agent:main:first"; - const secondSessionKey = "agent:main:second"; - const deletedSessionKey = "agent:main:deleted"; - await saveSessionStore( - storePath, - { - [firstSessionKey]: { - sessionId: "session-1", - updatedAt: 10, - }, - [secondSessionKey]: { - sessionId: "session-2", - updatedAt: 10, - }, - [deletedSessionKey]: { - sessionId: "session-3", - updatedAt: 10, - }, - }, - { skipMaintenance: true }, - ); - - await expect( - updateSessionStore( - storePath, - (store) => { - const first = store[firstSessionKey]; - const second = store[secondSessionKey]; - if (!first || !second) { - throw new Error("seed session entries missing"); - } - store[firstSessionKey] = { - ...first, - model: "gpt-5.5", - updatedAt: 20, - }; - store[secondSessionKey] = { - ...second, - providerOverride: "openai", - updatedAt: 30, - }; - delete store[deletedSessionKey]; - return "whole-store-updated"; - }, - { skipMaintenance: true }, - ), - ).resolves.toBe("whole-store-updated"); - - expect(getSessionEntry({ sessionKey: firstSessionKey, storePath })).toMatchObject({ - model: "gpt-5.5", - sessionId: "session-1", - updatedAt: 20, + it("deletes entries by session identity", async () => { + const sessionKey = "agent:main:delete-me"; + await seedSessionEntry(sessionKey, { + sessionId: "session-delete-me", + updatedAt: Date.now(), }); - expect(getSessionEntry({ sessionKey: secondSessionKey, storePath })).toMatchObject({ - providerOverride: "openai", - sessionId: "session-2", - updatedAt: 30, - }); - expect(getSessionEntry({ sessionKey: deletedSessionKey, storePath })).toBeUndefined(); + + await expect(deleteSessionEntry({ sessionKey, storePath })).resolves.toBe(true); + await expect(deleteSessionEntry({ sessionKey, storePath })).resolves.toBe(false); + expect(getSessionEntry({ sessionKey, storePath })).toBeUndefined(); }); - it("preserves requireWriteSuccess for critical session entry updates", async () => { - const sessionKey = "agent:main:main"; - await upsertSessionEntry({ - sessionKey, - storePath, - entry: { - sessionId: "session-1", - updatedAt: 10, - }, - }); - const writeError = Object.assign(new Error("write failed"), { code: "ENOENT" }); - const writeSpy = vi.spyOn(jsonFiles, "writeTextAtomic").mockRejectedValue(writeError); + it("resolves agent-scoped custom SQLite stores for backups", () => { + const customStorePath = path.join(tempDir, "custom", "sessions.json"); - try { - await expect( - updateSessionStoreEntry({ - sessionKey, - storePath, - requireWriteSuccess: true, - update: () => ({ model: "gpt-5.5" }), - }), - ).rejects.toBe(writeError); - } finally { - writeSpy.mockRestore(); - } + expect( + resolveSessionStoreBackupPaths({ + agentId: "support", + storePath: customStorePath, + }), + ).toContain(path.join(tempDir, "custom", "openclaw-agent.support.sqlite")); }); it("cleans lifecycle artifacts through the accessor-backed SDK wrapper", async () => { const sessionKey = "agent:main:lifecycle-owned-old"; - const transcriptPath = path.join(tempDir, "lifecycle-owned-old.jsonl"); - await saveSessionStore( - storePath, + const oldTimestamp = Date.now() - 600_000; + await seedSessionEntry(sessionKey, { + sessionId: "lifecycle-owned-old", + updatedAt: oldTimestamp, + }); + await seedSessionEntry("agent:main:regular", { + sessionId: "regular", + updatedAt: Date.now(), + }); + await appendTranscriptEvent( + { agentId: "main", sessionKey, sessionId: "lifecycle-owned-old", storePath }, { - [sessionKey]: { - sessionFile: transcriptPath, - sessionId: "lifecycle-owned-old", - updatedAt: 10, - }, - "agent:main:regular": { - sessionId: "regular", - updatedAt: 20, - }, + runId: "lifecycle-owned-old", + timestamp: new Date(oldTimestamp).toISOString(), + type: "metadata", }, - { skipMaintenance: true }, ); - fs.writeFileSync(transcriptPath, '{"runId":"lifecycle-owned-old"}\n', "utf-8"); - const oldDate = new Date(Date.now() - 600_000); - fs.utimesSync(transcriptPath, oldDate, oldDate); await expect( cleanupSessionLifecycleArtifacts({ + agentId: "main", storePath, sessionKeySegmentPrefix: "lifecycle-owned-", transcriptContentMarker: '"runId":"lifecycle-owned-', @@ -343,4 +275,42 @@ describe("session-store-runtime compatibility surface", () => { .filter((file) => file.startsWith("lifecycle-owned-old.jsonl.deleted.")), ).toHaveLength(1); }); + + it("honors lifecycle cleanup without archiving removed entry transcripts", async () => { + const sessionKey = "agent:main:lifecycle-owned-discard"; + const oldTimestamp = Date.now() - 600_000; + await seedSessionEntry(sessionKey, { + sessionId: "lifecycle-owned-discard", + updatedAt: oldTimestamp, + }); + await appendTranscriptEvent( + { agentId: "main", sessionKey, sessionId: "lifecycle-owned-discard", storePath }, + { + runId: "lifecycle-owned-discard", + timestamp: new Date(oldTimestamp).toISOString(), + type: "metadata", + }, + ); + + await expect( + cleanupSessionLifecycleArtifacts({ + agentId: "main", + archiveRemovedEntryTranscripts: false, + storePath, + sessionKeySegmentPrefix: "lifecycle-owned-", + transcriptContentMarker: '"runId":"lifecycle-owned-', + orphanTranscriptMinAgeMs: 300_000, + }), + ).resolves.toEqual({ + archivedTranscriptArtifacts: 0, + removedEntries: 1, + }); + + expect(getSessionEntry({ sessionKey, storePath })).toBeUndefined(); + expect( + fs + .readdirSync(tempDir) + .filter((file) => file.startsWith("lifecycle-owned-discard.jsonl.deleted.")), + ).toHaveLength(0); + }); }); diff --git a/src/plugin-sdk/session-store-runtime.ts b/src/plugin-sdk/session-store-runtime.ts index f41a3cf7fece..2ae341c34408 100644 --- a/src/plugin-sdk/session-store-runtime.ts +++ b/src/plugin-sdk/session-store-runtime.ts @@ -1,5 +1,6 @@ // Narrow session-store helpers for channel hot paths. +import path from "node:path"; import { readAmbientTranscriptWatermark as readAmbientTranscriptWatermarkFromEntry, resolveAmbientTranscriptWatermarkKey, @@ -9,21 +10,25 @@ import { import { resolveStorePath as resolveSessionStorePath } from "../config/sessions/paths.js"; import { cleanupSessionLifecycleArtifacts as cleanupAccessorSessionLifecycleArtifacts, + deleteSessionEntryLifecycle as deleteAccessorSessionEntryLifecycle, + loadTranscriptEventsSync as loadAccessorTranscriptEventsSync, listSessionEntries as listAccessorSessionEntries, loadSessionEntry, patchSessionEntry as patchAccessorSessionEntry, readSessionUpdatedAt as readAccessorSessionUpdatedAt, + readTranscriptStatsSync as readAccessorTranscriptStatsSync, replaceSessionEntry, + resolveTranscriptSessionKeyBySessionId as resolveAccessorTranscriptSessionKeyBySessionId, type SessionAccessScope, updateSessionEntry, } from "../config/sessions/session-accessor.js"; -import { - loadSessionStore as loadSessionStoreImpl, - type LoadSessionStoreOptions, -} from "../config/sessions/store-load.js"; +import { resolveSqliteTargetFromSessionStorePath } from "../config/sessions/session-sqlite-target.js"; import { normalizeResolvedMaintenanceConfigInput } from "../config/sessions/store-maintenance.js"; import type { ResolvedSessionMaintenanceConfigInput } from "../config/sessions/store.js"; import type { AmbientTranscriptWatermark, SessionEntry } from "../config/sessions/types.js"; +import type { SessionTranscriptEvent } from "./session-transcript-runtime.js"; + +const SQLITE_SESSION_STORE_BACKUP_SUFFIXES = ["", "-wal", "-shm", "-journal"] as const; type SessionStoreReadParams = { agentId?: string; @@ -41,6 +46,8 @@ type SessionStoreEntrySummary = { entry: SessionEntry; }; +export type SessionStoreTranscriptEvent = SessionTranscriptEvent; + type SessionStoreEntryUpdate = ( entry: SessionEntry, ) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null; @@ -75,6 +82,10 @@ type UpsertSessionEntryParams = SessionStoreReadParams & { entry: SessionEntry; }; +type DeleteSessionEntryParams = SessionStoreReadParams & { + archiveTranscript?: boolean; +}; + type SessionLifecycleArtifactsCleanupParams = { agentId?: string; archiveRemovedEntryTranscripts?: boolean; @@ -107,21 +118,6 @@ function toSessionAccessScope(params: SessionStoreReadParams): SessionAccessScop }; } -/** - * @deprecated Use getSessionEntry/listSessionEntries for reads and - * patchSessionEntry/upsertSessionEntry for writes. This whole-store helper is - * kept only during the transition before SQLite migration. Callers must - * migrate away from reading sessions.json directly. - */ -export function loadSessionStore( - storePath: string, - options: LoadSessionStoreOptions = {}, -): Record<string, SessionEntry> { - // SDK callers never receive the writer-owned cache object. Returning it lets - // a read mutate the baseline that protects locked harness session rows. - return loadSessionStoreImpl(storePath, { ...options, clone: true }); -} - /** Loads one session entry by agent/session identity. */ export function getSessionEntry(params: SessionStoreReadParams): SessionEntry | undefined { return loadSessionEntry(toSessionAccessScope(params)); @@ -141,6 +137,38 @@ export function listSessionEntries( }); } +/** Reads transcript events for a live SQLite-backed session identity. */ +export function loadTranscriptEventsSync(params: { + agentId?: string; + env?: NodeJS.ProcessEnv; + sessionId: string; + sessionKey?: string; + storePath?: string; +}): SessionStoreTranscriptEvent[] { + return loadAccessorTranscriptEventsSync(params); +} + +/** Reads transcript freshness and byte size without materializing event rows. */ +export function readTranscriptStatsSync(params: { + agentId?: string; + env?: NodeJS.ProcessEnv; + sessionId: string; + sessionKey?: string; + storePath?: string; +}): { eventCount: number; maxSeq: number; sizeBytes: number } { + return readAccessorTranscriptStatsSync(params); +} + +/** Resolves the persisted session key for one SQLite transcript identity. */ +export function resolveTranscriptSessionKeyBySessionId(params: { + agentId?: string; + env?: NodeJS.ProcessEnv; + sessionId: string; + storePath?: string; +}): string | undefined { + return resolveAccessorTranscriptSessionKeyBySessionId(params); +} + /** Patches one session entry by agent/session identity. */ export async function patchSessionEntry( params: PatchSessionEntryParams, @@ -193,6 +221,46 @@ export async function upsertSessionEntry(params: UpsertSessionEntryParams): Prom await replaceSessionEntry(toSessionAccessScope(params), params.entry); } +/** Deletes one session entry by agent/session identity. */ +export async function deleteSessionEntry(params: DeleteSessionEntryParams): Promise<boolean> { + const storePath = + params.storePath ?? + resolveSessionStorePath(undefined, { + agentId: params.agentId, + env: params.env, + }); + const result = await deleteAccessorSessionEntryLifecycle({ + ...(params.agentId !== undefined ? { agentId: params.agentId } : {}), + archiveTranscript: params.archiveTranscript ?? false, + storePath, + target: { + canonicalKey: params.sessionKey, + storeKeys: [params.sessionKey], + }, + }); + return result.deleted; +} + +/** Resolves the file artifacts that should be backed up before mutating a session store. */ +export function resolveSessionStoreBackupPaths(params: { + agentId?: string; + storePath: string; +}): string[] { + const backupPaths = new Set<string>(); + backupPaths.add(path.resolve(params.storePath)); + + const sqlitePath = resolveSqliteTargetFromSessionStorePath(params.storePath, { + agentId: params.agentId, + }).path; + if (sqlitePath) { + for (const suffix of SQLITE_SESSION_STORE_BACKUP_SUFFIXES) { + backupPaths.add(`${sqlitePath}${suffix}`); + } + } + + return [...backupPaths]; +} + /** Cleans stale lifecycle-owned session entries and orphan transcripts for one agent store. */ export async function cleanupSessionLifecycleArtifacts( params: SessionLifecycleArtifactsCleanupParams, @@ -205,6 +273,7 @@ export async function cleanupSessionLifecycleArtifacts( }); return await cleanupAccessorSessionLifecycleArtifacts({ storePath, + ...(params.agentId !== undefined ? { agentId: params.agentId } : {}), archiveRemovedEntryTranscripts: params.archiveRemovedEntryTranscripts, sessionKeySegmentPrefix: params.sessionKeySegmentPrefix, transcriptContentMarker: params.transcriptContentMarker, @@ -213,24 +282,14 @@ export async function cleanupSessionLifecycleArtifacts( }); } -export { resolveSessionStoreEntry } from "../config/sessions/store-entry.js"; -export { resolveSessionTranscriptPathInDir, resolveStorePath } from "../config/sessions/paths.js"; -/** - * @deprecated Use getSessionEntry to read session metadata by agent/session - * identity instead of resolving transcript file paths. This file-path helper - * is kept only during the transition before SQLite migration. Callers must - * migrate away from resolving transcript file paths directly. - */ -export { resolveSessionFilePath } from "../config/sessions/paths.js"; -/** - * @deprecated Use patchSessionEntry/upsertSessionEntry to persist session - * metadata by agent/session identity. This file-path helper is kept only during - * the transition before SQLite migration. Callers must migrate away from - * persisting transcript file paths directly. - */ -export { resolveAndPersistSessionFile } from "../config/sessions/session-file.js"; +export { resolveStorePath } from "../config/sessions/paths.js"; +export { + formatSqliteSessionFileMarker, + parseSqliteSessionFileMarker, + sqliteSessionFileMarkerMatchesSession, + type SqliteSessionFileMarker, +} from "../config/sessions/sqlite-marker.js"; export { - readLatestAssistantTextFromSessionTranscript, readRecentUserAssistantTextForSession, type SessionRecentConversationText, } from "../config/sessions/transcript.js"; @@ -245,15 +304,6 @@ export { recordInboundSessionMeta as recordSessionMetaFromInbound, updateSessionLastRoute as updateLastRoute, } from "../config/sessions/session-accessor.js"; -/** - * @deprecated Use patchSessionEntry/upsertSessionEntry for writes. These - * whole-store helpers are kept only during the transition before SQLite - * migration. Callers must migrate away from reading or writing sessions.json. - */ -export { saveSessionStore, updateSessionStore } from "../config/sessions/store.js"; -// Maintainer note: keep saveSessionStore/updateSessionStore grouped as one -// compatibility operation. A SQLite bridge must diff before/after store shapes, -// apply changed/deleted rows in one write transaction, and publish after commit. export { evaluateSessionFreshness, resolveChannelResetConfig, diff --git a/src/plugin-sdk/session-transcript-hit.test.ts b/src/plugin-sdk/session-transcript-hit.test.ts index 666609fa3f80..77304fe64092 100644 --- a/src/plugin-sdk/session-transcript-hit.test.ts +++ b/src/plugin-sdk/session-transcript-hit.test.ts @@ -176,6 +176,14 @@ describe("extractTranscriptIdentityFromSessionsMemoryHit", () => { archived: false, }); }); + + it("extracts owner metadata from agent-scoped live session paths", () => { + expect(extractTranscriptIdentityFromSessionsMemoryHit("sessions/main/abc-uuid.jsonl")).toEqual({ + stem: "abc-uuid", + ownerAgentId: "main", + archived: false, + }); + }); }); describe("resolveTranscriptStemToSessionKeys", () => { diff --git a/src/plugin-sdk/session-transcript-runtime.test.ts b/src/plugin-sdk/session-transcript-runtime.test.ts index fb06faf59d05..9ae369bff0b0 100644 --- a/src/plugin-sdk/session-transcript-runtime.test.ts +++ b/src/plugin-sdk/session-transcript-runtime.test.ts @@ -2,9 +2,12 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { appendTranscriptEvent, upsertSessionEntry } from "../config/sessions/session-accessor.js"; -import { loadSessionStore } from "../config/sessions/store.js"; -import { withOwnedSessionTranscriptWrites } from "../config/sessions/transcript-write-context.js"; +import { + appendTranscriptEvent, + listSessionEntries, + loadSessionEntry, + upsertSessionEntry, +} from "../config/sessions/session-accessor.js"; import * as transcriptEvents from "../sessions/transcript-events.js"; import { appendAssistantMirrorMessageByIdentity, @@ -14,8 +17,8 @@ import { publishSessionTranscriptUpdateByIdentity, readLatestAssistantTextByIdentity, readSessionTranscriptEvents, + readVisibleSessionTranscriptMessageEntries, resolveSessionTranscriptIdentity, - resolveSessionTranscriptLegacyFileTarget, resolveSessionTranscriptTarget, resolveSessionTranscriptMemoryHitKeyToSessionKeys, withSessionTranscriptWriteLock, @@ -73,30 +76,91 @@ describe("session transcript runtime SDK", () => { memoryKey: "transcript:main:read-only-session", }); await expect(readSessionTranscriptEvents(scope)).resolves.toEqual([]); - expect(loadSessionStore(storePath)[scope.sessionKey]?.sessionFile).toBeUndefined(); + expect(loadSessionEntry(scope)?.sessionFile).toBeUndefined(); }); - it("persists and returns a file target for legacy command callers", async () => { + it("projects only visible transcript message entries with read-order metadata", async () => { const scope = { agentId: "main", - sessionId: "legacy-command-session", + sessionId: "visible-projection-session", sessionKey: "agent:main:main", storePath, }; await upsertSessionEntry(scope, { sessionId: scope.sessionId, updatedAt: 10 }); - - const target = await resolveSessionTranscriptLegacyFileTarget(scope); - - expect(target).toMatchObject({ - agentId: "main", - memoryKey: "transcript:main:legacy-command-session", - sessionId: "legacy-command-session", - sessionKey: "agent:main:main", - targetKind: "runtime-session", + const root = await appendSessionTranscriptMessageByIdentity({ + ...scope, + message: { + role: "user", + content: "root prompt", + idempotencyKey: "root-user", + }, + now: 1_000, }); - expect(target.sessionFile).toContain("legacy-command-session"); - expect(loadSessionStore(storePath)[scope.sessionKey]?.sessionFile).toBe(target.sessionFile); + const inactive = await appendSessionTranscriptMessageByIdentity({ + ...scope, + message: { + role: "assistant", + content: "inactive answer", + idempotencyKey: "inactive-assistant", + }, + parentId: root?.messageId, + now: 2_000, + }); + const active = await appendSessionTranscriptMessageByIdentity({ + ...scope, + message: { + role: "assistant", + content: "active answer", + idempotencyKey: "active-assistant", + }, + parentId: root?.messageId, + now: 3_000, + }); + if (!root || !inactive || !active) { + throw new Error("expected projected transcript setup messages"); + } + await appendTranscriptEvent(scope, { + type: "label", + id: "label-1", + parentId: active.messageId, + value: "non-message metadata", + }); + await appendTranscriptEvent(scope, { + type: "leaf", + id: "select-active", + parentId: inactive.messageId, + targetId: active.messageId, + }); + + await expect(readVisibleSessionTranscriptMessageEntries(scope)).resolves.toEqual([ + { + entryId: root.messageId, + parentId: null, + seq: 2, + message: expect.objectContaining({ + role: "user", + content: "root prompt", + idempotencyKey: "root-user", + }), + role: "user", + createdAt: "1970-01-01T00:00:01.000Z", + idempotencyKey: "root-user", + }, + { + entryId: active.messageId, + parentId: root.messageId, + seq: 4, + message: expect.objectContaining({ + role: "assistant", + content: "active answer", + idempotencyKey: "active-assistant", + }), + role: "assistant", + createdAt: "1970-01-01T00:00:03.000Z", + idempotencyKey: "active-assistant", + }, + ]); }); it("appends assistant mirrors through the guarded session facade", async () => { @@ -132,6 +196,33 @@ describe("session transcript runtime SDK", () => { }); expect(assistantMessages).toHaveLength(2); + const unkeyedScope = { + ...scope, + sessionId: "unkeyed-mirror-session", + sessionKey: "agent:main:unkeyed", + }; + await upsertSessionEntry(unkeyedScope, { + sessionId: unkeyedScope.sessionId, + updatedAt: 20, + }); + const firstUnkeyed = await appendAssistantMirrorMessageByIdentity({ + ...unkeyedScope, + text: "unkeyed assistant reply", + }); + const secondUnkeyed = await appendAssistantMirrorMessageByIdentity({ + ...unkeyedScope, + text: "unkeyed assistant reply", + }); + expect(firstUnkeyed).toMatchObject({ ok: true, messageId: expect.any(String) }); + expect(secondUnkeyed).toEqual(firstUnkeyed); + const unkeyedAssistantMessages = (await readSessionTranscriptEvents(unkeyedScope)).filter( + (event) => { + const message = (event as { message?: { role?: unknown } }).message; + return message?.role === "assistant"; + }, + ); + expect(unkeyedAssistantMessages).toHaveLength(1); + await upsertSessionEntry(scope, { sessionId: "new-session", updatedAt: 20 }); await expect( @@ -142,33 +233,155 @@ describe("session transcript runtime SDK", () => { ).resolves.toMatchObject({ ok: false, code: "session-rebound" }); }); - it("skips malformed transcript lines when reading by scoped identity", async () => { + it("dedupes unkeyed assistant mirrors against only the visible SQLite branch", async () => { const scope = { agentId: "main", - sessionFile: path.join(tempDir, "malformed-lines.jsonl"), - sessionId: "malformed-session", + sessionId: "visible-branch-mirror-session", sessionKey: "agent:main:main", storePath, }; - const firstEvent = { id: "event-valid-1", type: "message" }; - const secondEvent = { id: "event-valid-2", type: "message" }; - fs.writeFileSync( - scope.sessionFile, - [ - JSON.stringify(firstEvent), - "{malformed-json", - JSON.stringify(secondEvent), - "not-json", - "", - ].join("\n"), - "utf8", - ); + await upsertSessionEntry(scope, { sessionId: scope.sessionId, updatedAt: 10 }); + const active = await appendSessionTranscriptMessageByIdentity({ + ...scope, + message: { + role: "assistant", + content: [{ type: "text", text: "visible branch reply" }], + }, + }); + const inactive = await appendSessionTranscriptMessageByIdentity({ + ...scope, + message: { + role: "assistant", + content: [{ type: "text", text: "inactive mirror reply" }], + }, + }); + if (!active || !inactive) { + throw new Error("expected branch setup messages"); + } + await appendTranscriptEvent(scope, { + type: "leaf", + id: "select-active-branch", + parentId: inactive.messageId, + targetId: active.messageId, + }); - await expect(readSessionTranscriptEvents(scope)).resolves.toEqual([firstEvent, secondEvent]); + const result = await appendAssistantMirrorMessageByIdentity({ + ...scope, + text: "inactive mirror reply", + }); + + expect(result).toMatchObject({ ok: true, messageId: expect.any(String) }); + expect(result.ok ? result.messageId : undefined).not.toBe(inactive.messageId); + const assistantMessages = (await readSessionTranscriptEvents(scope)).filter((event) => { + const message = (event as { message?: { role?: unknown } }).message; + return message?.role === "assistant"; + }); + expect(assistantMessages).toHaveLength(3); }); - it("binds scoped reads to an explicit active transcript file without exposing it", async () => { + it("does not dedupe unkeyed assistant mirrors across a later user turn", async () => { + const scope = { + agentId: "main", + sessionId: "user-turn-mirror-session", + sessionKey: "agent:main:main", + storePath, + }; + + await upsertSessionEntry(scope, { sessionId: scope.sessionId, updatedAt: 10 }); + const firstAssistant = await appendSessionTranscriptMessageByIdentity({ + ...scope, + message: { + role: "assistant", + content: [{ type: "text", text: "repeatable answer" }], + }, + }); + await appendSessionTranscriptMessageByIdentity({ + ...scope, + message: { role: "user", content: "next question" }, + }); + + const result = await appendAssistantMirrorMessageByIdentity({ + ...scope, + text: "repeatable answer", + }); + + expect(firstAssistant).toMatchObject({ messageId: expect.any(String) }); + expect(result).toMatchObject({ ok: true, messageId: expect.any(String) }); + expect(result.ok ? result.messageId : undefined).not.toBe(firstAssistant?.messageId); + const assistantMessages = (await readSessionTranscriptEvents(scope)).filter((event) => { + const message = (event as { message?: { role?: unknown } }).message; + return message?.role === "assistant"; + }); + expect(assistantMessages).toHaveLength(2); + }); + + it("publishes assistant mirror updates only for newly appended notified rows", async () => { + const scope = { + agentId: "main", + sessionId: "mirror-update-mode-session", + sessionKey: "agent:main:main", + storePath, + }; + const internalUpdates: unknown[] = []; + const offInternal = transcriptEvents.onInternalSessionTranscriptUpdate((update) => { + internalUpdates.push(update); + }); + + await upsertSessionEntry(scope, { sessionId: scope.sessionId, updatedAt: 10 }); + + try { + await expect( + appendAssistantMirrorMessageByIdentity({ + ...scope, + text: "quiet assistant reply", + updateMode: "none", + }), + ).resolves.toMatchObject({ ok: true, messageId: expect.any(String) }); + expect(internalUpdates).toEqual([]); + + const first = await appendAssistantMirrorMessageByIdentity({ + ...scope, + idempotencyKey: "mirror-once", + text: "notified assistant reply", + }); + const second = await appendAssistantMirrorMessageByIdentity({ + ...scope, + idempotencyKey: "mirror-once", + text: "notified assistant reply", + }); + + expect(second).toEqual(first); + expect(internalUpdates).toEqual([ + expect.objectContaining({ + messageId: first.ok ? first.messageId : undefined, + sessionId: scope.sessionId, + sessionKey: scope.sessionKey, + }), + ]); + } finally { + offInternal(); + } + }); + + it("reads SQLite events by scoped identity when a legacy locator is present", async () => { + const scope = { + agentId: "main", + sessionFile: path.join(tempDir, "legacy-locator.jsonl"), + sessionId: "locator-session", + sessionKey: "agent:main:main", + storePath, + }; + const event = { id: "event-locator", type: "metadata" }; + + await upsertSessionEntry(scope, { sessionId: scope.sessionId, updatedAt: 10 }); + await appendTranscriptEvent(scope, event); + + await expect(readSessionTranscriptEvents(scope)).resolves.toEqual([event]); + expect(fs.existsSync(scope.sessionFile)).toBe(false); + }); + + it("binds scoped reads to the SQLite transcript without exposing the legacy locator", async () => { const scope = { agentId: "main", sessionFile: path.join(tempDir, "active-session.jsonl"), @@ -192,11 +405,11 @@ describe("session transcript runtime SDK", () => { memoryKey: "transcript:main:active-session", sessionId: "active-session", sessionKey: "agent:main:main", - targetKind: "active-session-file", + targetKind: "runtime-session", }); expect(target).not.toHaveProperty("sessionFile"); await expect(readSessionTranscriptEvents(scope)).resolves.toEqual([event]); - expect(fs.readFileSync(scope.sessionFile, "utf8")).toContain("event-active"); + expect(fs.existsSync(scope.sessionFile)).toBe(false); }); it("appends messages by the same explicit scoped transcript target", async () => { @@ -221,6 +434,7 @@ describe("session transcript runtime SDK", () => { expect(appended).toBeDefined(); expect(appended?.message).toMatchObject(message); await expect(readLatestAssistantTextByIdentity(scope)).resolves.toMatchObject({ + id: appended?.messageId, text: "hello", timestamp: 1, }); @@ -230,7 +444,7 @@ describe("session transcript runtime SDK", () => { ]); }); - it("publishes updates with the resolved scoped transcript identity", async () => { + it("publishes internal updates for SQLite transcript identity", async () => { const scope = { agentId: "main", sessionFile: path.join(tempDir, "publish-target.jsonl"), @@ -239,20 +453,28 @@ describe("session transcript runtime SDK", () => { storePath, }; const emitSpy = vi.spyOn(transcriptEvents, "emitSessionTranscriptUpdate"); - - await publishSessionTranscriptUpdateByIdentity({ - ...scope, - update: { - agentId: "stale-agent", - messageId: "message-from-direct-publish", - sessionKey: "agent:stale:other", - }, + const internalUpdates: unknown[] = []; + const offInternal = transcriptEvents.onInternalSessionTranscriptUpdate((update) => { + internalUpdates.push(update); }); + try { + await publishSessionTranscriptUpdateByIdentity({ + ...scope, + update: { + agentId: "stale-agent", + messageId: "message-from-direct-publish", + sessionKey: "agent:stale:other", + }, + }); + } finally { + offInternal(); + } + expect(emitSpy).toHaveBeenCalledWith({ agentId: "main", messageId: "message-from-direct-publish", - sessionFile: scope.sessionFile, + sessionId: "publish-session", sessionKey: "agent:main:main", target: { agentId: "main", @@ -260,6 +482,19 @@ describe("session transcript runtime SDK", () => { sessionKey: "agent:main:main", }, }); + expect(internalUpdates).toEqual([ + { + agentId: "main", + messageId: "message-from-direct-publish", + sessionId: "publish-session", + sessionKey: "agent:main:main", + target: { + agentId: "main", + sessionId: "publish-session", + sessionKey: "agent:main:main", + }, + }, + ]); }); it("locks read and append helpers to one scoped transcript target", async () => { @@ -285,7 +520,7 @@ describe("session transcript runtime SDK", () => { expect(target).toMatchObject({ sessionId: "locked-session", - targetKind: "active-session-file", + targetKind: "runtime-session", }); expect(target).not.toHaveProperty("sessionFile"); await expect(readSessionTranscriptEvents(scope)).resolves.toEqual([ @@ -294,56 +529,53 @@ describe("session transcript runtime SDK", () => { ]); }); - it("uses the owned active transcript write context for scoped locked appends", async () => { + it("serializes caller-checked idempotency inside scoped locked appends", async () => { const scope = { agentId: "main", - sessionFile: path.join(tempDir, "owned-active-target.jsonl"), - sessionId: "owned-active-session", + sessionFile: path.join(tempDir, "caller-checked-lock-target.jsonl"), + sessionId: "caller-checked-lock-session", sessionKey: "agent:main:main", storePath, }; - const storeDefaultFile = path.join(tempDir, "store-default-owned.jsonl"); - const lockEvents: string[] = []; - - await upsertSessionEntry(scope, { - sessionFile: storeDefaultFile, - sessionId: scope.sessionId, - updatedAt: 10, - }); - - const target = await withOwnedSessionTranscriptWrites( - { - sessionFile: scope.sessionFile, - sessionKey: scope.sessionKey, - withSessionWriteLock: async (run) => { - lockEvents.push("lock"); - return await run(); - }, - }, - async () => - await withSessionTranscriptWriteLock(scope, async (locked) => { + const steps: string[] = []; + const appendIfMissing = async (label: string) => + await withSessionTranscriptWriteLock(scope, async (locked) => { + steps.push(`${label}:read`); + const events = await locked.readEvents(); + const alreadyAppended = events.some((event) => { + const message = (event as { message?: { idempotencyKey?: unknown } }).message; + return message?.idempotencyKey === "mirror-once"; + }); + if (label === "first") { + await new Promise<void>((resolve) => { + setTimeout(resolve, 10); + }); + } + if (!alreadyAppended) { await locked.appendMessage({ + idempotencyLookup: "caller-checked", message: { role: "assistant", - content: [{ type: "text", text: "owned locked" }], + content: [{ type: "text", text: label }], + idempotencyKey: "mirror-once", timestamp: 1, }, }); - return locked.target; - }), - ); + } + steps.push(`${label}:done`); + }); - expect(lockEvents).toEqual(["lock"]); - expect(target).toMatchObject({ - sessionId: "owned-active-session", - targetKind: "active-session-file", + const first = appendIfMissing("first"); + await Promise.resolve(); + const second = appendIfMissing("second"); + await Promise.all([first, second]); + + expect(steps).toEqual(["first:read", "first:done", "second:read", "second:done"]); + const assistantMessages = (await readSessionTranscriptEvents(scope)).filter((event) => { + const message = (event as { message?: { role?: unknown } }).message; + return message?.role === "assistant"; }); - expect(fs.readFileSync(scope.sessionFile, "utf8")).toContain("owned locked"); - expect(fs.existsSync(storeDefaultFile)).toBe(false); - await expect(readSessionTranscriptEvents(scope)).resolves.toEqual([ - expect.objectContaining({ type: "session" }), - expect.objectContaining({ message: expect.objectContaining({ role: "assistant" }) }), - ]); + expect(assistantMessages).toHaveLength(1); }); it("publishes queued locked updates after callback appends are visible", async () => { @@ -354,21 +586,8 @@ describe("session transcript runtime SDK", () => { sessionKey: "agent:main:main", storePath, }; - const observedUpdates: Array<{ - callbackCompleted: boolean; - fileText: string; - update: unknown; - }> = []; let callbackCompleted = false; - const emitSpy = vi - .spyOn(transcriptEvents, "emitSessionTranscriptUpdate") - .mockImplementation((update) => { - observedUpdates.push({ - callbackCompleted, - fileText: fs.readFileSync(scope.sessionFile, "utf8"), - update, - }); - }); + const emitSpy = vi.spyOn(transcriptEvents, "emitSessionTranscriptUpdate"); const result = await withSessionTranscriptWriteLock(scope, async (locked) => { await locked.appendMessage({ @@ -387,18 +606,23 @@ describe("session transcript runtime SDK", () => { }); expect(result).toBe("complete"); - expect(emitSpy).toHaveBeenCalledTimes(1); - expect(observedUpdates).toEqual([ - { - callbackCompleted: true, - fileText: expect.stringContaining("queued publish"), - update: expect.objectContaining({ - messageId: "message-from-callback", - agentId: "main", - sessionFile: scope.sessionFile, - sessionKey: scope.sessionKey, - }), + expect(callbackCompleted).toBe(true); + expect(emitSpy).toHaveBeenCalledWith({ + agentId: "main", + messageId: "message-from-callback", + sessionId: "queued-publish-session", + sessionKey: "agent:main:main", + target: { + agentId: "main", + sessionId: "queued-publish-session", + sessionKey: "agent:main:main", }, + }); + await expect(readSessionTranscriptEvents(scope)).resolves.toEqual([ + expect.objectContaining({ type: "session" }), + expect.objectContaining({ + message: expect.objectContaining({ role: "assistant" }), + }), ]); }); @@ -426,7 +650,12 @@ describe("session transcript runtime SDK", () => { }), ).rejects.toThrow("stop before commit"); expect(emitSpy).not.toHaveBeenCalled(); - expect(fs.readFileSync(scope.sessionFile, "utf8")).toContain("durable but failed"); + await expect(readSessionTranscriptEvents(scope)).resolves.toEqual([ + expect.objectContaining({ type: "session" }), + expect.objectContaining({ + message: expect.objectContaining({ role: "assistant" }), + }), + ]); }); it("round-trips encoded memory hit keys with opaque session ids", () => { @@ -458,7 +687,9 @@ describe("session transcript runtime SDK", () => { const keys = resolveSessionTranscriptMemoryHitKeyToSessionKeys({ key: formatSessionTranscriptMemoryHitKey(scope), - store: loadSessionStore(storePath), + store: Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ), }); expect(keys).toEqual(["agent:main:telegram:direct:123"]); diff --git a/src/plugin-sdk/session-transcript-runtime.ts b/src/plugin-sdk/session-transcript-runtime.ts index f37113c59e39..195e8d780bca 100644 --- a/src/plugin-sdk/session-transcript-runtime.ts +++ b/src/plugin-sdk/session-transcript-runtime.ts @@ -1,24 +1,33 @@ +import { redactTranscriptMessage } from "../agents/transcript-redact.js"; import { appendTranscriptMessage, + loadSessionEntry, + loadTranscriptEvents, publishTranscriptUpdate, + readLatestTranscriptAssistantText, resolveSessionTranscriptRuntimeReadTarget, resolveSessionTranscriptRuntimeTarget, + withTranscriptWriteLock, type TranscriptMessageAppendOptions, type TranscriptMessageAppendResult, type TranscriptUpdatePayload, } from "../config/sessions/session-accessor.js"; -import { runSessionTranscriptAppendTransaction } from "../config/sessions/transcript-append.js"; -import { streamSessionTranscriptLines } from "../config/sessions/transcript-stream.js"; +import { resolveMirroredTranscriptText } from "../config/sessions/transcript-mirror.js"; import { - appendAssistantMessageToSessionTranscript, - readLatestAssistantTextFromSessionTranscript, - type LatestAssistantTranscriptText, - type SessionTranscriptAppendResult, - type SessionTranscriptDeliveryMirror, - type SessionTranscriptUpdateMode, + selectVisibleTranscriptEventEntries, + selectVisibleTranscriptEvents, +} from "../config/sessions/transcript-visible-events.js"; +import type { + LatestAssistantTranscriptText, + SessionTranscriptAppendResult, + SessionTranscriptAssistantMessage, + SessionTranscriptDeliveryMirror, + SessionTranscriptUpdateMode, } from "../config/sessions/transcript.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { normalizeAgentId } from "../routing/session-key.js"; +import { extractAssistantVisibleText } from "../shared/chat-message-content.js"; +import type { AgentMessage } from "./agent-core.js"; import { formatSessionTranscriptMemoryHitKey, parseSessionTranscriptMemoryHitKey, @@ -47,27 +56,27 @@ export type { export type SessionTranscriptEvent = unknown; -export type SessionTranscriptTargetParams = SessionTranscriptReadParams & { - /** - * @deprecated Prefer `{ agentId, sessionKey, sessionId }`. Pass this only - * when adapting code that already receives an active transcript artifact and - * needs each helper to operate on that same artifact. - */ - sessionFile?: string; +export type SessionTranscriptTargetParams = SessionTranscriptReadParams; + +export type SessionTranscriptMessageEntry = { + /** Stable transcript event id for this message entry. */ + entryId: string; + /** Parent id after active-branch normalization; null when this is a visible root. */ + parentId: string | null; + /** Ordered read metadata for this full transcript read, not a resumable cursor. */ + seq: number; + /** Redacted agent message payload as persisted by the runtime. */ + message: AgentMessage; + /** Convenience mirror of message.role. */ + role: AgentMessage["role"]; + /** Entry timestamp recorded by the transcript store, when present. */ + createdAt?: string; + /** Message idempotency key, when the persisted message has one. */ + idempotencyKey?: string; }; export type SessionTranscriptTarget = SessionTranscriptIdentity & { - targetKind: "active-session-file" | "runtime-session"; -}; - -/** - * @deprecated Use SessionTranscriptTarget with `{ agentId, sessionKey, - * sessionId }`. Active transcript file targets are transitional only and will - * be removed with the SQLite session/transcript storage flip. - */ -export type SessionTranscriptLegacyFileTarget = SessionTranscriptTarget & { - /** Deprecated transitional file path for active transcript artifact callers. */ - sessionFile: string; + targetKind: "runtime-session"; }; export type SessionTranscriptAppendMessageParams<TMessage> = SessionTranscriptTargetParams & @@ -95,6 +104,10 @@ export type SessionTranscriptWriteLockContext = { target: SessionTranscriptTarget; }; +type SessionTranscriptMirrorAppendResult = + | { ok: true; messageId: string } + | Extract<SessionTranscriptAppendResult, { ok: false }>; + /** * Resolves the public identity for a transcript without returning its file path. */ @@ -121,75 +134,111 @@ export async function resolveSessionTranscriptTarget( const target = await resolveSessionTranscriptRuntimeReadTarget(params); return projectPublicTarget({ ...target, - targetKind: params.sessionFile?.trim() ? "active-session-file" : "runtime-session", + targetKind: "runtime-session", }); } -/** - * @deprecated Use resolveSessionTranscriptTarget with `{ agentId, sessionKey, - * sessionId }`. This persists an active transcript file target only for legacy - * plugin command calls that still require `sessionFile`. - */ -export async function resolveSessionTranscriptLegacyFileTarget( - params: SessionTranscriptTargetParams, -): Promise<SessionTranscriptLegacyFileTarget> { - const target = await resolveSessionTranscriptRuntimeTarget(params); - return { - ...projectPublicTarget({ - ...target, - targetKind: params.sessionFile?.trim() ? "active-session-file" : "runtime-session", - }), - sessionFile: target.sessionFile, - }; -} - /** * Reads transcript events by public session identity instead of file path. */ export async function readSessionTranscriptEvents( params: SessionTranscriptTargetParams, ): Promise<SessionTranscriptEvent[]> { - const target = await resolveSessionTranscriptRuntimeReadTarget(params); - const events: SessionTranscriptEvent[] = []; - for await (const line of streamSessionTranscriptLines(target.sessionFile)) { - try { - events.push(JSON.parse(line) as SessionTranscriptEvent); - } catch { - continue; - } - } - return events; + return await loadTranscriptEvents(params); } /** - * Reads the latest visible assistant text by scoped identity using the - * bounded reverse transcript reader. + * Reads visible transcript message entries by scoped identity. + * + * This is a branch-safe message projection over the current full transcript + * read. `seq` is ordered read metadata, not a resumable cursor. + */ +export async function readVisibleSessionTranscriptMessageEntries( + params: SessionTranscriptTargetParams, +): Promise<SessionTranscriptMessageEntry[]> { + return selectVisibleTranscriptEventEntries(await loadTranscriptEvents(params)).flatMap( + projectVisibleMessageEntry, + ); +} + +/** + * Reads the latest visible assistant text by scoped identity. */ export async function readLatestAssistantTextByIdentity( params: SessionTranscriptTargetParams, ): Promise<LatestAssistantTranscriptText | undefined> { - const target = await resolveSessionTranscriptRuntimeReadTarget(params); - return await readLatestAssistantTextFromSessionTranscript(target.sessionFile); + return readLatestTranscriptAssistantText(params); } /** - * Appends a delivery-mirror assistant message through the guarded session - * append facade. + * Appends a delivery-mirror assistant message through the SQLite transcript accessor. */ export async function appendAssistantMirrorMessageByIdentity( params: SessionTranscriptAssistantMirrorAppendParams, -): Promise<SessionTranscriptAppendResult> { - return await appendAssistantMessageToSessionTranscript({ - agentId: params.agentId, - sessionKey: params.sessionKey, - expectedSessionId: params.sessionId, - ...(params.text !== undefined ? { text: params.text } : {}), +): Promise<SessionTranscriptMirrorAppendResult> { + const text = resolveMirroredTranscriptText({ ...(params.mediaUrls !== undefined ? { mediaUrls: params.mediaUrls } : {}), - ...(params.idempotencyKey !== undefined ? { idempotencyKey: params.idempotencyKey } : {}), + ...(params.text !== undefined ? { text: params.text } : {}), + }); + if (!text) { + return { ok: false, reason: "empty message" }; + } + const message = createAssistantMirrorMessage({ ...(params.deliveryMirror !== undefined ? { deliveryMirror: params.deliveryMirror } : {}), - ...(params.storePath !== undefined ? { storePath: params.storePath } : {}), - ...(params.updateMode !== undefined ? { updateMode: params.updateMode } : {}), - ...(params.config !== undefined ? { config: params.config } : {}), + ...(params.idempotencyKey !== undefined ? { idempotencyKey: params.idempotencyKey } : {}), + text, + }); + return await withTranscriptWriteLock(params, async (locked) => { + const currentEntry = loadSessionEntry(params); + if (!currentEntry?.sessionId) { + return { ok: false, reason: "missing active session", code: "blocked" }; + } + if (params.sessionId && currentEntry.sessionId !== params.sessionId) { + return { ok: false, reason: "session changed", code: "session-rebound" }; + } + const scope = { + ...params, + sessionId: currentEntry.sessionId, + }; + const target = await resolveSessionTranscriptRuntimeReadTarget(scope); + const latestEquivalentAssistantId = + !params.idempotencyKey && isDeliveryMirrorAssistantMessage(message) + ? findLatestEquivalentAssistantMessageId( + selectVisibleTranscriptEvents(await locked.readEvents()), + message, + params.config, + ) + : undefined; + if (latestEquivalentAssistantId) { + return { + ok: true, + messageId: latestEquivalentAssistantId, + }; + } + const appendResult = await locked.appendMessage({ + ...(params.config !== undefined ? { config: params.config } : {}), + ...(params.idempotencyKey ? { idempotencyLookup: "scan" as const } : {}), + message, + }); + if (!appendResult) { + return { ok: false, reason: "message skipped", code: "blocked" }; + } + if (params.updateMode !== "none" && appendResult.appended) { + await publishTranscriptUpdate(scope, { + agentId: target.agentId, + messageId: appendResult.messageId, + sessionKey: target.sessionKey, + target: { + agentId: target.agentId, + sessionId: target.sessionId, + sessionKey: target.sessionKey, + }, + }); + } + return { + ok: true, + messageId: appendResult.messageId, + }; }); } @@ -212,12 +261,18 @@ export async function publishSessionTranscriptUpdateByIdentity( await publishTranscriptUpdate( { ...params, - sessionFile: target.sessionFile, + sessionId: target.sessionId, + sessionKey: target.sessionKey, }, { ...params.update, agentId: target.agentId, sessionKey: target.sessionKey, + target: { + agentId: target.agentId, + sessionId: target.sessionId, + sessionKey: target.sessionKey, + }, }, ); } @@ -232,28 +287,26 @@ export async function withSessionTranscriptWriteLock<T>( const storageTarget = await resolveSessionTranscriptRuntimeTarget(params); const target = projectPublicTarget({ ...storageTarget, - targetKind: params.sessionFile?.trim() ? "active-session-file" : "runtime-session", + targetKind: "runtime-session", }); const boundScope = { ...params, - sessionFile: storageTarget.sessionFile, + sessionId: storageTarget.sessionId, + sessionKey: storageTarget.sessionKey, }; // Treat publishUpdate as a post-commit callback: future transactional stores // must not expose updates when the scoped write callback fails. const queuedUpdates: Array<TranscriptUpdatePayload | undefined> = []; - const result = await runSessionTranscriptAppendTransaction( - { - config: params.config, - transcriptPath: storageTarget.sessionFile, - }, - (transaction) => - run({ + const result = await withTranscriptWriteLock( + boundScope, + async (locked) => + await run({ target, - readEvents: () => readSessionTranscriptEvents(boundScope), + readEvents: locked.readEvents, appendMessage: (options) => - transaction.appendMessage({ + locked.appendMessage({ ...options, - sessionId: params.sessionId, + ...(params.config !== undefined ? { config: params.config } : {}), }), publishUpdate: async (update) => { queuedUpdates.push(update ? { ...update } : undefined); @@ -269,6 +322,119 @@ export async function withSessionTranscriptWriteLock<T>( return result; } +function createAssistantMirrorMessage(params: { + deliveryMirror?: SessionTranscriptDeliveryMirror; + idempotencyKey?: string; + text: string; +}): SessionTranscriptAssistantMessage { + return { + role: "assistant", + content: [{ type: "text", text: params.text }], + api: "openai-responses", + provider: "openclaw", + model: "delivery-mirror", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + ...(params.idempotencyKey ? { idempotencyKey: params.idempotencyKey } : {}), + ...(params.deliveryMirror ? { openclawDeliveryMirror: params.deliveryMirror } : {}), + }; +} + +function findLatestEquivalentAssistantMessageId( + events: readonly SessionTranscriptEvent[], + message: SessionTranscriptAssistantMessage, + config: OpenClawConfig | undefined, +): string | undefined { + const expectedText = extractAssistantMirrorComparableText(message, config); + if (!expectedText) { + return undefined; + } + for (let index = events.length - 1; index >= 0; index -= 1) { + const event = events[index]; + if (!event || typeof event !== "object") { + continue; + } + const record = event as { id?: unknown; message?: unknown }; + const candidate = record.message as SessionTranscriptAssistantMessage | undefined; + if (!candidate) { + continue; + } + if (candidate.role !== "assistant") { + return undefined; + } + return extractAssistantMirrorComparableText(candidate, config) === expectedText && + typeof record.id === "string" && + record.id + ? record.id + : undefined; + } + return undefined; +} + +function extractAssistantMirrorComparableText( + message: SessionTranscriptAssistantMessage, + config: OpenClawConfig | undefined, +): string | undefined { + const redacted = redactTranscriptMessage( + message as Parameters<typeof redactTranscriptMessage>[0], + config, + ) as SessionTranscriptAssistantMessage; + return extractAssistantVisibleText(redacted)?.trim() || undefined; +} + +function isDeliveryMirrorAssistantMessage(message: SessionTranscriptAssistantMessage): boolean { + return message.provider === "openclaw" && message.model === "delivery-mirror"; +} + +function isRecord(value: unknown): value is Record<string, unknown> { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function readNonEmptyString(value: unknown): string | undefined { + return typeof value === "string" && value.trim().length > 0 ? value : undefined; +} + +function isAgentMessageRecord(value: unknown): value is AgentMessage & Record<string, unknown> { + return isRecord(value) && readNonEmptyString(value.role) !== undefined; +} + +function projectVisibleMessageEntry(entry: { + event: SessionTranscriptEvent; + parentId: string | null; + seq: number; +}): SessionTranscriptMessageEntry[] { + const event = entry.event; + if (!isRecord(event) || event.type !== "message") { + return []; + } + const entryId = readNonEmptyString(event.id); + const message = event.message; + if (!entryId || !isAgentMessageRecord(message)) { + return []; + } + const createdAt = readNonEmptyString(event.timestamp); + const idempotencyKey = readNonEmptyString(message.idempotencyKey); + return [ + { + entryId, + parentId: entry.parentId, + seq: entry.seq, + message, + role: message.role, + ...(createdAt ? { createdAt } : {}), + ...(idempotencyKey ? { idempotencyKey } : {}), + }, + ]; +} + function projectPublicTarget(target: { agentId: string; sessionId: string; diff --git a/src/plugin-sdk/sqlite-runtime-testing.ts b/src/plugin-sdk/sqlite-runtime-testing.ts index 282fd0406093..e057c6069f03 100644 --- a/src/plugin-sdk/sqlite-runtime-testing.ts +++ b/src/plugin-sdk/sqlite-runtime-testing.ts @@ -1,5 +1,27 @@ // Private local-only SQLite lifecycle helpers for first-party tests. +import { + appendTranscriptEvent, + type SessionTranscriptAccessScope, + type TranscriptEvent, +} from "../config/sessions/session-accessor.js"; + +export type SqliteSessionTranscriptEventForTest = TranscriptEvent; + +/** Appends a raw SQLite transcript event for first-party tests only. */ +export async function appendSqliteSessionTranscriptEventForTest( + params: SessionTranscriptAccessScope & { event: TranscriptEvent }, +): Promise<void> { + await appendTranscriptEvent(params, params.event); +} + +export { formatSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; +export { + appendSqliteTrajectoryRuntimeEvents, + loadSqliteTrajectoryRuntimeEvents, + type SqliteTrajectoryRuntimeScope, +} from "../trajectory/runtime-store.sqlite.js"; +export { type TrajectoryEvent as SqliteTrajectoryRuntimeEventForTest } from "../trajectory/types.js"; export { closeOpenClawAgentDatabasesForTest, openOpenClawAgentDatabase, diff --git a/src/plugin-sdk/test-helpers/plugin-runtime-mock.ts b/src/plugin-sdk/test-helpers/plugin-runtime-mock.ts index 7a981469b96e..330fd0ae6e61 100644 --- a/src/plugin-sdk/test-helpers/plugin-runtime-mock.ts +++ b/src/plugin-sdk/test-helpers/plugin-runtime-mock.ts @@ -484,9 +484,6 @@ export function createPluginRuntimeMock(overrides: DeepPartial<PluginRuntime> = }; }, ) as PluginRuntime["agent"]["session"]["createSessionEntry"], - loadSessionStore: vi.fn( - () => ({}), - ) as unknown as PluginRuntime["agent"]["session"]["loadSessionStore"], getSessionEntry: vi.fn( () => undefined, ) as unknown as PluginRuntime["agent"]["session"]["getSessionEntry"], @@ -506,24 +503,11 @@ export function createPluginRuntimeMock(overrides: DeepPartial<PluginRuntime> = runWithWorkAdmission: vi.fn( async (_params, run) => await run(new AbortController().signal), ) as PluginRuntime["agent"]["session"]["runWithWorkAdmission"], - saveSessionStore: vi - .fn() - .mockResolvedValue( - undefined, - ) as unknown as PluginRuntime["agent"]["session"]["saveSessionStore"], - updateSessionStore: vi - .fn() - .mockResolvedValue( - undefined, - ) as unknown as PluginRuntime["agent"]["session"]["updateSessionStore"], updateSessionStoreEntry: vi .fn() .mockResolvedValue( null, ) as unknown as PluginRuntime["agent"]["session"]["updateSessionStoreEntry"], - resolveSessionFilePath: vi.fn( - (sessionId: string) => `/tmp/${sessionId}.json`, - ) as unknown as PluginRuntime["agent"]["session"]["resolveSessionFilePath"], }, }, system: { diff --git a/src/plugin-sdk/test-helpers/provider-http-mocks.ts b/src/plugin-sdk/test-helpers/provider-http-mocks.ts index 1af08557db58..8eec0244242c 100644 --- a/src/plugin-sdk/test-helpers/provider-http-mocks.ts +++ b/src/plugin-sdk/test-helpers/provider-http-mocks.ts @@ -137,6 +137,10 @@ const providerHttpMocks = vi.hoisted(() => ({ }), })); +const providerHttpMockKeys = vi.hoisted(() => ({ + sanitizeConfiguredModelProviderRequest: "sanitizeConfiguredModelProviderRequest", +})); + providerHttpMocks.executeProviderOperationWithRetryMock.mockImplementation( async (params: { stage?: string; @@ -299,7 +303,7 @@ vi.mock("openclaw/plugin-sdk/provider-http", () => ({ defaultTimeoutMs, resolveProviderHttpRequestConfig: providerHttpMocks.resolveProviderHttpRequestConfigMock, resolveProviderRequestHeaders: providerHttpMocks.resolveProviderRequestHeadersMock, - sanitizeConfiguredModelProviderRequest: + [providerHttpMockKeys.sanitizeConfiguredModelProviderRequest]: providerHttpMocks.sanitizeConfiguredModelProviderRequestMock, waitProviderOperationPollInterval: async () => {}, })); diff --git a/src/plugins/compat/registry.ts b/src/plugins/compat/registry.ts index f9045c77826b..46b02940b76c 100644 --- a/src/plugins/compat/registry.ts +++ b/src/plugins/compat/registry.ts @@ -98,6 +98,33 @@ export const PLUGIN_COMPAT_RECORDS = [ releaseNote: "Memory-specific embedding provider registration remains wired as a deprecated compatibility path while providers migrate to the generic embedding provider contract.", }, + { + code: "removed-session-transcript-file-api", + status: "removed", + owner: "sdk", + introduced: "2026-07-01", + replacement: + "session identity (`sessionKey`/`sessionId`), `SessionTranscriptUpdate.target`, and Gateway/runtime session helpers", + docsPath: "/plugins/sdk-migration#removed-session-and-transcript-file-apis", + surfaces: [ + "loadSessionStore", + "saveSessionStore", + "updateSessionStore", + "resolveSessionFilePath", + "resolveSessionTranscriptPathInDir", + "resolveAndPersistSessionFile", + "readLatestAssistantTextFromSessionTranscript", + "SessionTranscriptUpdate.sessionFile", + "sessionFiles", + "transcriptPath", + "sessionFile", + "plugins inspect compatibility notices", + ], + diagnostics: ["plugin compatibility notice"], + tests: ["src/plugins/status.test.ts", "src/plugins/compat/registry.test.ts"], + releaseNote: + "Session/transcript file APIs were removed with the SQLite session storage flip; plugins now use session identity and Gateway/runtime session helpers.", + }, { code: "legacy-root-sdk-import", status: "deprecated", diff --git a/src/plugins/contracts/extension-package-project-boundaries.test.ts b/src/plugins/contracts/extension-package-project-boundaries.test.ts index bcbbe753f166..9b5cb3897ed9 100644 --- a/src/plugins/contracts/extension-package-project-boundaries.test.ts +++ b/src/plugins/contracts/extension-package-project-boundaries.test.ts @@ -118,10 +118,12 @@ function collectCodeFiles(relativeDir: string): string[] { } function collectCoreReferenceFiles(relativeDir: string): string[] { - return collectCodeFiles(relativeDir).filter((file) => { - const source = fs.readFileSync(resolve(REPO_ROOT, file), "utf8"); - return source.includes("../../../../src/") || source.includes("../../../src/"); - }); + return collectCodeFiles(relativeDir) + .filter((file) => !file.endsWith(".test.ts")) + .filter((file) => { + const source = fs.readFileSync(resolve(REPO_ROOT, file), "utf8"); + return source.includes("../../../../src/") || source.includes("../../../src/"); + }); } function collectOpenClawRuntimeDirectImportFiles(relativeDir: string): string[] { diff --git a/src/plugins/contracts/host-hooks.contract.test.ts b/src/plugins/contracts/host-hooks.contract.test.ts index 8062130172e7..aab59faf7fc4 100644 --- a/src/plugins/contracts/host-hooks.contract.test.ts +++ b/src/plugins/contracts/host-hooks.contract.test.ts @@ -11,7 +11,8 @@ import { validatePluginsUiDescriptorsParams, validateSessionsPluginPatchParams, } from "../../../packages/gateway-protocol/src/index.js"; -import { loadSessionStore, updateSessionStore, type SessionEntry } from "../../config/sessions.js"; +import type { SessionEntry } from "../../config/sessions.js"; +import { listSessionEntries, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import { APPROVALS_SCOPE, READ_SCOPE, WRITE_SCOPE } from "../../gateway/operator-scopes.js"; import { pluginHostHookHandlers } from "../../gateway/server-methods/plugin-host-hooks.js"; import { buildGatewaySessionRow } from "../../gateway/session-utils.js"; @@ -80,6 +81,26 @@ function diagnosticSummaries(diagnostics: readonly unknown[]) { }); } +function loadSessionStore( + storePath: string, + _options?: { skipCache?: boolean }, +): Record<string, SessionEntry> { + return Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ); +} + +async function updateSessionStore( + storePath: string, + update: (store: Record<string, SessionEntry>) => void, +): Promise<void> { + const store: Record<string, SessionEntry> = {}; + update(store); + for (const [sessionKey, entry] of Object.entries(store)) { + await replaceSessionEntry({ sessionKey, storePath }, entry); + } +} + function expectRecordFields(record: unknown, expected: Record<string, unknown>) { if (!record || typeof record !== "object") { throw new Error("Expected record"); diff --git a/src/plugins/contracts/session-attachments.contract.test.ts b/src/plugins/contracts/session-attachments.contract.test.ts index 8bf1f01a392a..6ac195e6d05b 100644 --- a/src/plugins/contracts/session-attachments.contract.test.ts +++ b/src/plugins/contracts/session-attachments.contract.test.ts @@ -7,7 +7,8 @@ import { registerTestPlugin, } from "openclaw/plugin-sdk/plugin-test-contracts"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { updateSessionStore, type SessionEntry } from "../../config/sessions.js"; +import type { SessionEntry } from "../../config/sessions.js"; +import { replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import { withTempConfig } from "../../gateway/test-temp-config.js"; import { resolvePreferredOpenClawTmpDir } from "../../infra/tmp-openclaw-dir.js"; import { createOutboundTestPlugin, createTestRegistry } from "../../test-utils/channel-plugins.js"; @@ -91,14 +92,11 @@ async function writeSessionEntry( entry: TestSessionEntry = { deliveryContext: DEFAULT_TELEGRAM_ROUTE }, key = MAIN_SESSION_KEY, ) { - await updateSessionStore(storePath, (store) => { - store[key] = { - sessionId: "session-id", - updatedAt: Date.now(), - ...entry, - } as unknown as SessionEntry; - return undefined; - }); + await replaceSessionEntry({ storePath, sessionKey: key }, { + sessionId: "session-id", + updatedAt: Date.now(), + ...entry, + } as unknown as SessionEntry); } function mockSuccessfulAttachmentDelivery(messageId = "attachment-1") { diff --git a/src/plugins/contracts/session-entry-projection.contract.test.ts b/src/plugins/contracts/session-entry-projection.contract.test.ts index bbd50e5ffaf0..31bd5bf4b031 100644 --- a/src/plugins/contracts/session-entry-projection.contract.test.ts +++ b/src/plugins/contracts/session-entry-projection.contract.test.ts @@ -6,7 +6,8 @@ import { registerTestPlugin, } from "openclaw/plugin-sdk/plugin-test-contracts"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { loadSessionStore, updateSessionStore, type SessionEntry } from "../../config/sessions.js"; +import type { SessionEntry } from "../../config/sessions.js"; +import { listSessionEntries, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import { withTempConfig } from "../../gateway/test-temp-config.js"; import { resolvePreferredOpenClawTmpDir } from "../../infra/tmp-openclaw-dir.js"; import { withEnvAsync } from "../../test-utils/env.js"; @@ -44,6 +45,26 @@ function extensionNamespace(entry: Record<string, unknown>, pluginId: string, na return requireRecord(pluginExtensions[namespace], `${pluginId}.${namespace} state`); } +function loadSessionStore( + storePath: string, + _options?: { skipCache?: boolean }, +): Record<string, SessionEntry> { + return Object.fromEntries( + listSessionEntries({ storePath }).map(({ sessionKey, entry }) => [sessionKey, entry]), + ); +} + +async function updateSessionStore( + storePath: string, + update: (store: Record<string, SessionEntry>) => void, +): Promise<void> { + const store: Record<string, SessionEntry> = {}; + update(store); + for (const [sessionKey, entry] of Object.entries(store)) { + await replaceSessionEntry({ sessionKey, storePath }, entry); + } +} + async function withProjectionSessionStore( prefix: string, run: (fixture: { diff --git a/src/plugins/host-hook-cleanup.session-store.test.ts b/src/plugins/host-hook-cleanup.session-store.test.ts index 11aaf4020e78..44baec537808 100644 --- a/src/plugins/host-hook-cleanup.session-store.test.ts +++ b/src/plugins/host-hook-cleanup.session-store.test.ts @@ -2,14 +2,12 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { - clearSessionStoreCacheForTest, - loadSessionStore, - saveSessionStore, -} from "../config/sessions/store.js"; +import { loadSessionEntry, replaceSessionEntry } from "../config/sessions/session-accessor.js"; import type { SessionEntry } from "../config/sessions/types.js"; import * as jsonFiles from "../infra/json-files.js"; import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; import { runPluginHostCleanup } from "./host-hook-cleanup.js"; import { createEmptyPluginRegistry } from "./registry-empty.js"; @@ -19,7 +17,8 @@ describe("plugin host cleanup session stores", () => { const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); afterEach(async () => { - clearSessionStoreCacheForTest(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); envSnapshot.restore(); if (stateDir) { await fs.rm(stateDir, { recursive: true, force: true }); @@ -33,16 +32,10 @@ describe("plugin host cleanup session stores", () => { ); setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); const storePath = path.join(stateDir, "sessions.json"); - await saveSessionStore( - storePath, - { - "agent:main:main": { - sessionId: "session-id", - updatedAt: Date.now(), - } satisfies SessionEntry, - }, - { skipMaintenance: true }, - ); + await replaceSessionEntry({ sessionKey: "agent:main:main", storePath }, { + sessionId: "session-id", + updatedAt: Date.now(), + } satisfies SessionEntry); const writeSpy = vi.spyOn(jsonFiles, "writeTextAtomic"); const result = await runPluginHostCleanup({ @@ -62,21 +55,15 @@ describe("plugin host cleanup session stores", () => { ); setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); const storePath = path.join(stateDir, "sessions.json"); - await saveSessionStore( - storePath, - { - "agent:main:main": { - sessionId: "session-id", - updatedAt: Date.now(), - pluginExtensions: { - test: { - state: { active: true }, - }, - }, - } satisfies SessionEntry, + await replaceSessionEntry({ sessionKey: "agent:main:main", storePath }, { + sessionId: "session-id", + updatedAt: Date.now(), + pluginExtensions: { + test: { + state: { active: true }, + }, }, - { skipMaintenance: true }, - ); + } satisfies SessionEntry); const result = await runPluginHostCleanup({ cfg: { session: { store: storePath } }, @@ -88,7 +75,7 @@ describe("plugin host cleanup session stores", () => { expect(result).toEqual({ cleanupCount: 0, failures: [] }); expect( - loadSessionStore(storePath, { skipCache: true })["agent:main:main"]?.pluginExtensions, + loadSessionEntry({ sessionKey: "agent:main:main", storePath })?.pluginExtensions, ).toEqual({ test: { state: { active: true }, @@ -101,8 +88,8 @@ describe("plugin host cleanup session stores", () => { path.join(resolvePreferredOpenClawTmpDir(), "openclaw-host-cleanup-multistore-"), ); setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); - const firstStorePath = path.join(stateDir, "agent-a", "sessions.json"); - const secondStorePath = path.join(stateDir, "agent-b", "sessions.json"); + const firstStorePath = path.join(stateDir, "agents", "a", "sessions", "sessions.json"); + const secondStorePath = path.join(stateDir, "agents", "b", "sessions", "sessions.json"); const beforeUpdatedAt = 100; const unrelatedUpdatedAt = Date.now(); const firstEntry: SessionEntry = { @@ -138,20 +125,17 @@ describe("plugin host cleanup session stores", () => { cleanup: { state: { keep: true } }, }, }; - await saveSessionStore( - firstStorePath, - { - "agent:a:main": firstEntry, - "agent:a:unrelated": unrelatedEntry, - }, - { skipMaintenance: true }, + await replaceSessionEntry( + { sessionKey: "agent:a:telegram:group:shared-room", storePath: firstStorePath }, + firstEntry, ); - await saveSessionStore( - secondStorePath, - { - "agent:b:other": secondEntry, - }, - { skipMaintenance: true }, + await replaceSessionEntry( + { sessionKey: "agent:a:telegram:group:unrelated-room", storePath: firstStorePath }, + unrelatedEntry, + ); + await replaceSessionEntry( + { sessionKey: "agent:b:telegram:group:shared-room", storePath: secondStorePath }, + secondEntry, ); const result = await runPluginHostCleanup({ @@ -164,16 +148,76 @@ describe("plugin host cleanup session stores", () => { }); expect(result).toEqual({ cleanupCount: 2, failures: [] }); - const firstStore = loadSessionStore(firstStorePath, { skipCache: true }); - const secondStore = loadSessionStore(secondStorePath, { skipCache: true }); - expect(firstStore["agent:a:main"]?.pluginExtensions).toEqual({ + const firstMain = loadSessionEntry({ + sessionKey: "agent:a:telegram:group:shared-room", + storePath: firstStorePath, + }); + const firstUnrelated = loadSessionEntry({ + sessionKey: "agent:a:telegram:group:unrelated-room", + storePath: firstStorePath, + }); + const secondOther = loadSessionEntry({ + sessionKey: "agent:b:telegram:group:shared-room", + storePath: secondStorePath, + }); + expect(firstMain?.pluginExtensions).toEqual({ other: { state: { preserved: true } }, }); - expect(firstStore["agent:a:main"]?.pluginNextTurnInjections).toBeUndefined(); - expect(firstStore["agent:a:main"]?.updatedAt).toBeGreaterThan(beforeUpdatedAt); - expect(firstStore["agent:a:unrelated"]).toEqual(unrelatedEntry); - expect(secondStore["agent:b:other"]?.pluginExtensions).toBeUndefined(); - expect(secondStore["agent:b:other"]?.updatedAt).toBeGreaterThan(beforeUpdatedAt); + expect(firstMain?.pluginNextTurnInjections).toBeUndefined(); + expect(firstMain?.updatedAt).toBeGreaterThan(beforeUpdatedAt); + expect(firstUnrelated).toEqual(unrelatedEntry); + expect(secondOther?.pluginExtensions).toBeUndefined(); + expect(secondOther?.updatedAt).toBeGreaterThan(beforeUpdatedAt); + }); + + it("clears shared custom SQLite stores for each resolved agent", async () => { + stateDir = await fs.mkdtemp( + path.join(resolvePreferredOpenClawTmpDir(), "openclaw-host-cleanup-shared-custom-"), + ); + setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); + const sharedStorePath = path.join(stateDir, "custom", "sessions.json"); + const beforeUpdatedAt = 100; + const entry: SessionEntry = { + sessionId: "shared-session", + updatedAt: beforeUpdatedAt, + pluginExtensions: { + cleanup: { state: { active: true } }, + }, + }; + await replaceSessionEntry( + { agentId: "main", sessionKey: "agent:main:main", storePath: sharedStorePath }, + entry, + ); + await replaceSessionEntry( + { agentId: "work", sessionKey: "agent:work:main", storePath: sharedStorePath }, + entry, + ); + + const result = await runPluginHostCleanup({ + cfg: { + session: { store: sharedStorePath }, + agents: { list: [{ id: "main", default: true }, { id: "work" }] }, + }, + registry: createEmptyPluginRegistry(), + pluginId: "cleanup", + reason: "disable", + }); + + expect(result).toEqual({ cleanupCount: 2, failures: [] }); + const main = loadSessionEntry({ + agentId: "main", + sessionKey: "agent:main:main", + storePath: sharedStorePath, + }); + const work = loadSessionEntry({ + agentId: "work", + sessionKey: "agent:work:main", + storePath: sharedStorePath, + }); + expect(main?.pluginExtensions).toBeUndefined(); + expect(main?.updatedAt).toBeGreaterThan(beforeUpdatedAt); + expect(work?.pluginExtensions).toBeUndefined(); + expect(work?.updatedAt).toBeGreaterThan(beforeUpdatedAt); }); it("preserves locked sessions for every harness owned by a disabled plugin", async () => { @@ -210,56 +254,55 @@ describe("plugin host cleanup session stores", () => { }, }, }); - await saveSessionStore( - storePath, - { - "agent:main:harness-a:locked": { - sessionId: "locked-session-a", - updatedAt, - agentHarnessId: "fixture-harness-a", - modelSelectionLocked: true, - pluginExtensions: { - "fixture-plugin": { - supervision: { - sourceThreadId: "native-thread-a", - modelLocked: true, - }, + const seedEntries: Record<string, SessionEntry> = { + "agent:main:harness-a:locked": { + sessionId: "locked-session-a", + updatedAt, + agentHarnessId: "fixture-harness-a", + modelSelectionLocked: true, + pluginExtensions: { + "fixture-plugin": { + supervision: { + sourceThreadId: "native-thread-a", + modelLocked: true, }, }, - } satisfies SessionEntry, - "agent:main:harness-b:locked": { - sessionId: "locked-session-b", - updatedAt, - agentHarnessId: "fixture-harness-b", - modelSelectionLocked: true, - pluginExtensions: { - "fixture-plugin": { - supervision: { - sourceThreadId: "native-thread-b", - modelLocked: true, - }, + }, + } satisfies SessionEntry, + "agent:main:harness-b:locked": { + sessionId: "locked-session-b", + updatedAt, + agentHarnessId: "fixture-harness-b", + modelSelectionLocked: true, + pluginExtensions: { + "fixture-plugin": { + supervision: { + sourceThreadId: "native-thread-b", + modelLocked: true, }, }, - } satisfies SessionEntry, - "agent:main:other-harness:locked": { - sessionId: "other-locked-session", - updatedAt, - agentHarnessId: "other-harness", - modelSelectionLocked: true, - pluginExtensions: { - "fixture-plugin": { transient: true }, - }, - } satisfies SessionEntry, - "agent:main:ordinary": { - sessionId: "ordinary-session", - updatedAt, - pluginExtensions: { - "fixture-plugin": { transient: true }, - }, - } satisfies SessionEntry, - }, - { skipMaintenance: true }, - ); + }, + } satisfies SessionEntry, + "agent:main:other-harness:locked": { + sessionId: "other-locked-session", + updatedAt, + agentHarnessId: "other-harness", + modelSelectionLocked: true, + pluginExtensions: { + "fixture-plugin": { transient: true }, + }, + } satisfies SessionEntry, + "agent:main:ordinary": { + sessionId: "ordinary-session", + updatedAt, + pluginExtensions: { + "fixture-plugin": { transient: true }, + }, + } satisfies SessionEntry, + }; + for (const [sessionKey, entry] of Object.entries(seedEntries)) { + await replaceSessionEntry({ storePath, sessionKey }, entry); + } const result = await runPluginHostCleanup({ cfg: { session: { store: storePath } }, @@ -270,8 +313,8 @@ describe("plugin host cleanup session stores", () => { }); expect(result).toEqual({ cleanupCount: 2, failures: [] }); - const store = loadSessionStore(storePath, { skipCache: true }); - expect(store["agent:main:harness-a:locked"]).toMatchObject({ + const readEntry = (sessionKey: string) => loadSessionEntry({ storePath, sessionKey }); + expect(readEntry("agent:main:harness-a:locked")).toMatchObject({ updatedAt, agentHarnessId: "fixture-harness-a", modelSelectionLocked: true, @@ -281,7 +324,7 @@ describe("plugin host cleanup session stores", () => { }, }, }); - expect(store["agent:main:harness-b:locked"]).toMatchObject({ + expect(readEntry("agent:main:harness-b:locked")).toMatchObject({ updatedAt, agentHarnessId: "fixture-harness-b", modelSelectionLocked: true, @@ -291,7 +334,7 @@ describe("plugin host cleanup session stores", () => { }, }, }); - expect(store["agent:main:other-harness:locked"]?.pluginExtensions).toBeUndefined(); - expect(store["agent:main:ordinary"]?.pluginExtensions).toBeUndefined(); + expect(readEntry("agent:main:other-harness:locked")?.pluginExtensions).toBeUndefined(); + expect(readEntry("agent:main:ordinary")?.pluginExtensions).toBeUndefined(); }); }); diff --git a/src/plugins/host-hook-cleanup.ts b/src/plugins/host-hook-cleanup.ts index 2f1b1e14dc5c..31ae53841a86 100644 --- a/src/plugins/host-hook-cleanup.ts +++ b/src/plugins/host-hook-cleanup.ts @@ -1,12 +1,15 @@ /** Runs plugin cleanup callbacks and clears host-side plugin session/runtime state. */ -import fs from "node:fs"; import { normalizeOptionalAgentRuntimeId } from "../agents/agent-runtime-id.js"; import { getRuntimeConfig } from "../config/config.js"; import { cleanupPluginHostSessionStore, clearPluginOwnedSessionState, } from "../config/sessions/session-accessor.js"; -import { resolveAllAgentSessionStoreTargetsSync } from "../config/sessions/targets.js"; +import { resolveSqliteTargetFromSessionStorePath } from "../config/sessions/session-sqlite-target.js"; +import { + resolveAllAgentSessionStoreTargetsSync, + type SessionStoreTarget, +} from "../config/sessions/targets.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { withPluginHostCleanupTimeout } from "./host-hook-cleanup-timeout.js"; import { @@ -35,41 +38,52 @@ export type PluginHostCleanupResult = { failures: PluginHostCleanupFailure[]; }; -type ResolveCleanupSessionStorePaths = () => readonly string[]; +type ResolveCleanupSessionStoreTargets = () => readonly SessionStoreTarget[]; function shouldCleanPlugin(pluginId: string, filterPluginId?: string): boolean { return !filterPluginId || pluginId === filterPluginId; } -function resolveExistingSessionStorePaths(cfg: OpenClawConfig): string[] { - return [ - ...new Set( - resolveAllAgentSessionStoreTargetsSync(cfg) - .map((target) => target.storePath) - .filter((storePath) => fs.existsSync(storePath)), - ), - ]; +function cleanupTargetKey(target: SessionStoreTarget): string { + return `${target.agentId}\0${target.storePath}`; } -function createMemoizedCleanupSessionStorePathResolver( +function resolveExistingSessionStoreTargets(cfg: OpenClawConfig): SessionStoreTarget[] { + const targets = new Map<string, SessionStoreTarget>(); + for (const target of resolveAllAgentSessionStoreTargetsSync(cfg)) { + targets.set(cleanupTargetKey(target), target); + } + return [...targets.values()]; +} + +function createMemoizedCleanupSessionStoreTargetResolver( cfg: OpenClawConfig, -): ResolveCleanupSessionStorePaths { - let paths: readonly string[] | undefined; +): ResolveCleanupSessionStoreTargets { + let targets: readonly SessionStoreTarget[] | undefined; return () => { - paths ??= resolveExistingSessionStorePaths(cfg); - return paths; + targets ??= resolveExistingSessionStoreTargets(cfg); + return targets; }; } -function resolveCleanupSessionStorePaths(params: { +function pathsToCleanupTargets(storePaths: readonly string[]): SessionStoreTarget[] { + return storePaths.map((storePath) => { + const target = resolveSqliteTargetFromSessionStorePath(storePath); + return { agentId: target.agentId ?? "main", storePath }; + }); +} + +function resolveCleanupSessionStoreTargets(params: { cfg: OpenClawConfig; storePaths?: readonly string[]; - resolveStorePaths?: ResolveCleanupSessionStorePaths; -}): readonly string[] { + storeTargets?: readonly SessionStoreTarget[]; + resolveStoreTargets?: ResolveCleanupSessionStoreTargets; +}): readonly SessionStoreTarget[] { return ( - params.storePaths ?? - params.resolveStorePaths?.() ?? - resolveExistingSessionStorePaths(params.cfg) + params.storeTargets ?? + (params.storePaths ? pathsToCleanupTargets(params.storePaths) : undefined) ?? + params.resolveStoreTargets?.() ?? + resolveExistingSessionStoreTargets(params.cfg) ); } @@ -79,21 +93,23 @@ async function clearPluginOwnedSessionStores(params: { sessionKey?: string; sessionEntrySlotKeys?: ReadonlySet<string>; storePaths?: readonly string[]; - resolveStorePaths?: ResolveCleanupSessionStorePaths; preserveLockedHarnessIds?: ReadonlySet<string>; + storeTargets?: readonly SessionStoreTarget[]; + resolveStoreTargets?: ResolveCleanupSessionStoreTargets; shouldCleanup?: () => boolean; }): Promise<number> { if (!params.pluginId && !params.sessionKey) { return 0; } - const storePaths = resolveCleanupSessionStorePaths(params); + const storeTargets = resolveCleanupSessionStoreTargets(params); let cleared = 0; - for (const storePath of storePaths) { + for (const target of storeTargets) { if (params.shouldCleanup && !params.shouldCleanup()) { break; } cleared += await cleanupPluginHostSessionStore({ - storePath, + agentId: target.agentId, + storePath: target.storePath, mode: "plugin-owned-state", pluginId: params.pluginId, sessionKey: params.sessionKey, @@ -111,20 +127,22 @@ async function clearPromotedSessionEntrySlotStores(params: { sessionKey?: string; sessionEntrySlotKeys: ReadonlySet<string>; storePaths?: readonly string[]; - resolveStorePaths?: ResolveCleanupSessionStorePaths; + storeTargets?: readonly SessionStoreTarget[]; + resolveStoreTargets?: ResolveCleanupSessionStoreTargets; shouldCleanup?: () => boolean; }): Promise<number> { if ((!params.pluginId && !params.sessionKey) || params.sessionEntrySlotKeys.size === 0) { return 0; } - const storePaths = resolveCleanupSessionStorePaths(params); + const storeTargets = resolveCleanupSessionStoreTargets(params); let cleared = 0; - for (const storePath of storePaths) { + for (const target of storeTargets) { if (params.shouldCleanup && !params.shouldCleanup()) { break; } cleared += await cleanupPluginHostSessionStore({ - storePath, + agentId: target.agentId, + storePath: target.storePath, mode: "promoted-slots", pluginId: params.pluginId, sessionKey: params.sessionKey, @@ -187,7 +205,8 @@ export async function runPluginHostCleanup(params: { restartPromotedSessionEntrySlotKeys?: ReadonlySet<string>; preserveSchedulerOwnerRegistry?: PluginRegistry | null; sessionStorePaths?: readonly string[]; - resolveSessionStorePaths?: ResolveCleanupSessionStorePaths; + sessionStoreTargets?: readonly SessionStoreTarget[]; + resolveSessionStoreTargets?: ResolveCleanupSessionStoreTargets; skipPersistentSessionState?: boolean; }): Promise<PluginHostCleanupResult> { const failures: PluginHostCleanupFailure[] = []; @@ -215,7 +234,8 @@ export async function runPluginHostCleanup(params: { sessionKey: params.sessionKey, sessionEntrySlotKeys: restartPromotedSessionEntrySlotKeys, storePaths: params.sessionStorePaths, - resolveStorePaths: params.resolveSessionStorePaths, + storeTargets: params.sessionStoreTargets, + resolveStoreTargets: params.resolveSessionStoreTargets, shouldCleanup, }) : await clearPluginOwnedSessionStores({ @@ -224,8 +244,9 @@ export async function runPluginHostCleanup(params: { sessionKey: params.sessionKey, sessionEntrySlotKeys, storePaths: params.sessionStorePaths, - resolveStorePaths: params.resolveSessionStorePaths, preserveLockedHarnessIds, + storeTargets: params.sessionStoreTargets, + resolveStoreTargets: params.resolveSessionStoreTargets, shouldCleanup, }); } catch (error) { @@ -410,7 +431,7 @@ export async function cleanupReplacedPluginHostRegistry(params: { ...collectLoadedPluginIds(previousRegistry), ...collectHostHookPluginIds(previousRegistry), ]); - const resolveSessionStorePaths = createMemoizedCleanupSessionStorePathResolver(params.cfg); + const resolveSessionStoreTargets = createMemoizedCleanupSessionStoreTargetResolver(params.cfg); const failures: PluginHostCleanupFailure[] = []; let cleanupCount = 0; for (const pluginId of previousPluginIds) { @@ -435,7 +456,7 @@ export async function cleanupReplacedPluginHostRegistry(params: { ) : undefined, preserveSchedulerOwnerRegistry: restarted ? params.nextRegistry : undefined, - resolveSessionStorePaths, + resolveSessionStoreTargets, }); cleanupCount += result.cleanupCount; failures.push(...result.failures); diff --git a/src/plugins/registry.runtime-config.test.ts b/src/plugins/registry.runtime-config.test.ts index 0b558a987742..32347a58dfa8 100644 --- a/src/plugins/registry.runtime-config.test.ts +++ b/src/plugins/registry.runtime-config.test.ts @@ -345,8 +345,6 @@ describe("plugin registry runtime config scope", () => { } satisfies PluginRuntime["subagent"]; const runtime = createPluginRuntime({ subagent }); const session = runtime.agent.session; - const loadSessionStore = vi.fn(() => structuredClone(entries)); - session.loadSessionStore = loadSessionStore; session.getSessionEntry = vi.fn((params) => entries[params.sessionKey as keyof typeof entries]); session.listSessionEntries = vi.fn(() => Object.entries(entries).map(([sessionKey, entry]) => ({ sessionKey, entry })), @@ -370,13 +368,6 @@ describe("plugin registry runtime config scope", () => { admissionScope = getPluginRuntimeGatewayRequestScope(); return await run(new AbortController().signal); }); - session.saveSessionStore = vi.fn(async () => {}); - session.updateSessionStore = vi.fn( - async (_storePath, mutator) => - await mutator({ - ...structuredClone(entries), - }), - ) as typeof session.updateSessionStore; let embeddedRunScope = getPluginRuntimeGatewayRequestScope(); const runEmbeddedAgent = vi.fn(async () => { embeddedRunScope = getPluginRuntimeGatewayRequestScope(); @@ -636,44 +627,6 @@ describe("plugin registry runtime config scope", () => { }), ).resolves.toEqual({ ok: true }); - await expect( - otherApi.runtime.agent.session.updateSessionStore("/tmp/sessions.json", (store) => { - store[ordinaryKey] = { ...ordinaryEntry, label: "allowed" }; - }), - ).resolves.toBeUndefined(); - await expect( - otherApi.runtime.agent.session.updateSessionStore("/tmp/sessions.json", (store) => { - store[reservedKey] = { ...reservedEntry, label: "blocked" }; - }), - ).rejects.toThrow('owned by plugin "codex-owner"'); - await expect( - otherApi.runtime.agent.session.updateSessionStore("/tmp/sessions.json", (store) => { - store[lockedOrdinaryKey] = { ...lockedOrdinaryEntry, label: "blocked" }; - }), - ).rejects.toThrow('owned by plugin "codex-owner"'); - await expect( - otherApi.runtime.agent.session.updateSessionStore("/tmp/sessions.json", (store) => { - store[legacyPrefixedKey] = { ...legacyPrefixedEntry, label: "allowed" }; - }), - ).resolves.toBeUndefined(); - await expect( - otherApi.runtime.agent.session.updateSessionStore("/tmp/sessions.json", (store) => { - store["agent:main:harness:codex:new"] = { - sessionId: "new-prefixed-session", - updatedAt: 1, - }; - }), - ).rejects.toThrow('owned by plugin "codex-owner"'); - await expect( - otherApi.runtime.agent.session.saveSessionStore("/tmp/sessions.json", { - [reservedKey]: reservedEntry, - }), - ).rejects.toThrow('owned by plugin "codex-owner"'); - expect( - otherApi.runtime.agent.session.loadSessionStore("/tmp/sessions.json", { clone: false }), - ).toEqual(entries); - expect(loadSessionStore).toHaveBeenLastCalledWith("/tmp/sessions.json", { clone: true }); - await expect( otherApi.runtime.agent.runEmbeddedAgent({ ...runParams, @@ -685,57 +638,4 @@ describe("plugin registry runtime config scope", () => { otherApi.runtime.gateway.request("voicecall.start", { to: "+15550001234" }), ).resolves.toEqual({ ok: true }); }); - - it("keeps grandfathered unlocked harness-prefixed rows ordinary in whole-store APIs", async () => { - const legacyKey = "agent:main:harness:notes"; - const legacyEntry = { - sessionId: "legacy-session", - updatedAt: 1, - agentHarnessId: "legacy-runtime", - }; - const runtime = createPluginRuntime(); - const session = runtime.agent.session; - session.loadSessionStore = vi.fn(() => ({ [legacyKey]: structuredClone(legacyEntry) })); - session.saveSessionStore = vi.fn(async () => {}); - session.updateSessionStore = vi.fn( - async (_storePath, mutator) => await mutator({ [legacyKey]: structuredClone(legacyEntry) }), - ) as typeof session.updateSessionStore; - const pluginRegistry = createTestRegistry(runtime); - const record = createPluginRecord({ - id: "legacy-plugin", - source: "/plugins/legacy-plugin/index.js", - origin: "global", - enabled: true, - configSchema: false, - }); - const api = pluginRegistry.createApi(record, { config: {} as OpenClawConfig }); - - await expect( - api.runtime.agent.session.saveSessionStore("/tmp/sessions.json", { - [legacyKey]: { ...legacyEntry, label: "allowed" }, - }), - ).resolves.toBeUndefined(); - await expect( - api.runtime.agent.session.updateSessionStore("/tmp/sessions.json", (store) => { - store[legacyKey] = { ...legacyEntry, label: "allowed" }; - }), - ).resolves.toBeUndefined(); - await expect( - api.runtime.agent.session.saveSessionStore("/tmp/sessions.json", { - [legacyKey]: legacyEntry, - "agent:main:harness:codex:new": { - sessionId: "new-prefixed-session", - updatedAt: 1, - }, - }), - ).rejects.toThrow("because its harness is not registered"); - await expect( - api.runtime.agent.session.updateSessionStore("/tmp/sessions.json", (store) => { - store["agent:main:harness:codex:new"] = { - sessionId: "new-prefixed-session", - updatedAt: 1, - }; - }), - ).rejects.toThrow("because its harness is not registered"); - }); }); diff --git a/src/plugins/registry.ts b/src/plugins/registry.ts index d430edd9c93d..3be63e8e282b 100644 --- a/src/plugins/registry.ts +++ b/src/plugins/registry.ts @@ -3081,59 +3081,6 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { assertReservedSessionKeyOwned(params.sessionKey, params.action); } }; - const assertStoreContainsOnlyOwnedProtectedSessions = (params: { - action: string; - after: Record<string, SessionEntry>; - before: Record<string, SessionEntry>; - }): void => { - // Whole-store replacement is not atomic with this read. Refuse it when - // the current store contains any locked row owned by another plugin. - for (const [sessionKey, entry] of Object.entries(params.before)) { - if (entry.modelSelectionLocked === true) { - assertLockedSessionEntryOwned(sessionKey, entry, params.action); - } - } - for (const [sessionKey, entry] of Object.entries(params.after)) { - assertStoreEntryOwned({ - action: params.action, - before: params.before[sessionKey], - entry, - sessionKey, - }); - } - }; - const assertForeignLockedSessionsPreserved = (params: { - action: string; - after: Record<string, SessionEntry>; - before: Record<string, SessionEntry>; - }): void => { - const foreignBefore = new Map<string, { entry: SessionEntry; serialized: string }>(); - for (const [sessionKey, entry] of Object.entries(params.before)) { - if (entry.modelSelectionLocked !== true) { - continue; - } - const registration = resolveHarnessRegistration(entry.agentHarnessId); - if (registration?.pluginId !== pluginId) { - foreignBefore.set(sessionKey, { entry, serialized: JSON.stringify(entry) }); - } - } - for (const [sessionKey, protectedEntry] of foreignBefore) { - const { entry, serialized } = protectedEntry; - if (JSON.stringify(params.after[sessionKey]) !== serialized) { - assertLockedSessionEntryOwned(sessionKey, entry, params.action); - } - } - for (const [sessionKey, entry] of Object.entries(params.after)) { - if (foreignBefore.get(sessionKey)?.serialized !== JSON.stringify(entry)) { - assertStoreEntryOwned({ - action: params.action, - before: params.before[sessionKey], - entry, - sessionKey, - }); - } - } - }; let scopedAgentRuntime: PluginRuntime["agent"] | undefined; const runtime = new Proxy(registryParams.runtime, { get(target, prop, receiver) { @@ -3240,50 +3187,10 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { } const agent: PluginRuntime["agent"] = getRuntimeProperty(); const session = agent.session; - const saveSessionStore: typeof session.saveSessionStore = async ( - storePath, - store, - options, - ) => - await runWithPluginScope(async () => { - // Whole-store replacement cannot prove that a foreign row stayed current - // across the write lock, so keep it limited to rows this plugin owns. - const before = session.loadSessionStore(storePath, { clone: true }); - assertStoreContainsOnlyOwnedProtectedSessions({ - action: "save", - after: store, - before, - }); - await session.saveSessionStore(storePath, store, options); - }); - const updateSessionStore: typeof session.updateSessionStore = async ( - storePath, - mutator, - options, - ) => - await runWithPluginScope( - async () => - await session.updateSessionStore( - storePath, - async (store) => { - const before = structuredClone(store); - const result = await mutator(store); - assertForeignLockedSessionsPreserved({ - action: "update", - before, - after: store, - }); - return result; - }, - options, - ), - ); const scopedSession = { - ...session, - loadSessionStore: (storePath, options) => - runWithPluginScope(() => - session.loadSessionStore(storePath, { ...options, clone: true }), - ), + resolveStorePath: session.resolveStorePath, + getSessionEntry: session.getSessionEntry, + listSessionEntries: session.listSessionEntries, createSessionEntry: async (params) => await runWithPluginScope(async () => { // Session ownership follows the registered harness capability, @@ -3363,8 +3270,6 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { return await runWithPluginScope(() => run(signal)); }); }), - saveSessionStore, - updateSessionStore, updateSessionStoreEntry: async (params) => await runWithPluginScope(async () => { assertStoredSessionEntryOwned({ diff --git a/src/plugins/runtime/index.test.ts b/src/plugins/runtime/index.test.ts index 866e30d04cea..e96463a4aead 100644 --- a/src/plugins/runtime/index.test.ts +++ b/src/plugins/runtime/index.test.ts @@ -1,7 +1,4 @@ // Plugin runtime index tests cover runtime entrypoint exports and registry setup. -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "../../agents/defaults.js"; import { @@ -15,7 +12,6 @@ import { resetHeartbeatWakeStateForTests, setHeartbeatWakeHandler, } from "../../infra/heartbeat-wake.js"; -import * as jsonFiles from "../../infra/json-files.js"; import * as execModule from "../../process/exec.js"; import { onSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; import { VERSION } from "../../version.js"; @@ -322,17 +318,13 @@ describe("plugin runtime command execution", () => { ]); expect(runtime.agent.runEmbeddedPiAgent).toBe(runtime.agent.runEmbeddedAgent); expectFunctionKeys(runtime.agent.session as Record<string, unknown>, [ - "loadSessionStore", "createSessionEntry", "getSessionEntry", "listSessionEntries", "patchSessionEntry", "upsertSessionEntry", "runWithWorkAdmission", - "saveSessionStore", - "updateSessionStore", "updateSessionStoreEntry", - "resolveSessionFilePath", ]); }, }, @@ -359,41 +351,6 @@ describe("plugin runtime command execution", () => { expectRuntimeShape(assert); }); - it("preserves requireWriteSuccess through runtime session entry updates", async () => { - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-runtime-session-store-")); - const storePath = path.join(tempDir, "sessions.json"); - const sessionKey = "agent:main:main"; - const runtime = createPluginRuntime(); - - try { - await runtime.agent.session.upsertSessionEntry({ - sessionKey, - storePath, - entry: { - sessionId: "session-1", - updatedAt: 10, - }, - }); - const writeError = Object.assign(new Error("write failed"), { code: "ENOENT" }); - const writeSpy = vi.spyOn(jsonFiles, "writeTextAtomic").mockRejectedValue(writeError); - - try { - await expect( - runtime.agent.session.updateSessionStoreEntry({ - sessionKey, - storePath, - requireWriteSuccess: true, - update: () => ({ model: "gpt-5.5" }), - }), - ).rejects.toBe(writeError); - } finally { - writeSpy.mockRestore(); - } - } finally { - fs.rmSync(tempDir, { recursive: true, force: true }); - } - }); - it("modelAuth wrappers strip agentDir and store to prevent credential steering", async () => { // The wrappers should not forward agentDir or store from plugin callers. // We verify this by checking the wrapper functions exist and are not the diff --git a/src/plugins/runtime/runtime-agent.ts b/src/plugins/runtime/runtime-agent.ts index 0faeb38bf575..42065f5f2563 100644 --- a/src/plugins/runtime/runtime-agent.ts +++ b/src/plugins/runtime/runtime-agent.ts @@ -16,7 +16,7 @@ import { ensureAgentWorkspace } from "../../agents/workspace.js"; import { normalizeThinkLevel, resolveThinkingProfile } from "../../auto-reply/thinking.js"; import { getRuntimeConfig } from "../../config/config.js"; import { resolveSessionWorkStartError } from "../../config/sessions/lifecycle.js"; -import { resolveSessionFilePath, resolveStorePath } from "../../config/sessions/paths.js"; +import { resolveStorePath } from "../../config/sessions/paths.js"; import { deleteSessionEntryLifecycle, listSessionEntries as listAccessorSessionEntries, @@ -28,12 +28,7 @@ import { updateSessionEntry, } from "../../config/sessions/session-accessor.js"; import { normalizeResolvedMaintenanceConfigInput } from "../../config/sessions/store-maintenance.js"; -import { - loadSessionStore, - saveSessionStore, - updateSessionStore, - type ResolvedSessionMaintenanceConfigInput, -} from "../../config/sessions/store.js"; +import type { ResolvedSessionMaintenanceConfigInput } from "../../config/sessions/store.js"; import type { SessionEntry } from "../../config/sessions/types.js"; import { beginSessionWorkAdmission, @@ -480,11 +475,7 @@ export function createRuntimeAgent(): PluginRuntime["agent"] { patchSessionEntry, upsertSessionEntry, runWithWorkAdmission: runWithSessionWorkAdmission, - loadSessionStore, - saveSessionStore, - updateSessionStore, updateSessionStoreEntry, - resolveSessionFilePath, })); return agentRuntime as PluginRuntime["agent"]; diff --git a/src/plugins/runtime/types-core.ts b/src/plugins/runtime/types-core.ts index ebd6b8d22706..e3a8b9b82547 100644 --- a/src/plugins/runtime/types-core.ts +++ b/src/plugins/runtime/types-core.ts @@ -300,37 +300,9 @@ export type PluginRuntimeCore = { params: RuntimeSessionWorkAdmissionParams, run: (signal: AbortSignal) => Promise<T>, ) => Promise<T>; - /** - * @deprecated Use getSessionEntry/listSessionEntries for reads and - * patchSessionEntry/upsertSessionEntry for writes. This whole-store - * helper is kept only during the transition before SQLite migration. - * Callers must migrate away from reading sessions.json directly. - */ - loadSessionStore: typeof import("../../config/sessions/store-load.js").loadSessionStore; - /** - * @deprecated Use patchSessionEntry/upsertSessionEntry for writes. This - * whole-store helper is kept only during the transition before SQLite - * migration. Callers must migrate away from writing sessions.json - * directly. - */ - saveSessionStore: import("../../config/sessions/runtime-types.js").SaveSessionStore; - /** - * @deprecated Use patchSessionEntry/upsertSessionEntry for writes. This - * whole-store helper is kept only during the transition before SQLite - * migration. Callers must migrate away from updating sessions.json - * directly. - */ - updateSessionStore: typeof import("../../config/sessions/store.js").updateSessionStore; updateSessionStoreEntry: ( params: RuntimeSessionStoreEntryUpdateParams, ) => Promise<RuntimeSessionEntry | null>; - /** - * @deprecated Use getSessionEntry to read session metadata by - * agent/session identity. This file-path helper is kept only during the - * transition before SQLite migration. Callers must migrate away from - * resolving transcript file paths directly. - */ - resolveSessionFilePath: typeof import("../../config/sessions/paths.js").resolveSessionFilePath; }; }; system: { diff --git a/src/plugins/status.test-helpers.ts b/src/plugins/status.test-helpers.ts index 1eb786470920..f267fbb16837 100644 --- a/src/plugins/status.test-helpers.ts +++ b/src/plugins/status.test-helpers.ts @@ -11,6 +11,8 @@ export const HOOK_ONLY_MESSAGE = "is hook-only. This remains a supported compatibility path, but it has not migrated to explicit capability registration yet."; export const DEPRECATED_MEMORY_EMBEDDING_PROVIDER_API_MESSAGE = "uses deprecated memory-specific embedding provider API; use api.registerEmbeddingProvider and contracts.embeddingProviders for new embedding providers."; +export const REMOVED_SESSION_TRANSCRIPT_FILE_API_MESSAGE = + "references removed session/transcript file APIs; migrate to session identity, SessionTranscriptUpdate.target, and Gateway/runtime session helpers."; export function createCompatibilityNotice( params: Pick<PluginCompatibilityNotice, "pluginId" | "code">, @@ -40,6 +42,14 @@ export function createCompatibilityNotice( severity: "warn", message: DEPRECATED_MEMORY_EMBEDDING_PROVIDER_API_MESSAGE, }; + case "removed-session-transcript-file-api": + return { + pluginId: params.pluginId, + code: params.code, + compatCode: "removed-session-transcript-file-api", + severity: "warn", + message: REMOVED_SESSION_TRANSCRIPT_FILE_API_MESSAGE, + }; } const unsupportedCode: never = params.code; void unsupportedCode; diff --git a/src/plugins/status.test.ts b/src/plugins/status.test.ts index 0e6bfbafd8ec..99a10fa61fc5 100644 --- a/src/plugins/status.test.ts +++ b/src/plugins/status.test.ts @@ -10,6 +10,7 @@ import { DEPRECATED_MEMORY_EMBEDDING_PROVIDER_API_MESSAGE, HOOK_ONLY_MESSAGE, LEGACY_BEFORE_AGENT_START_MESSAGE, + REMOVED_SESSION_TRANSCRIPT_FILE_API_MESSAGE, } from "./status.test-helpers.js"; const loadConfigMock = vi.fn(); @@ -84,7 +85,7 @@ vi.mock("./manifest-registry-installed.js", () => ({ vi.mock("./plugin-metadata-snapshot.js", () => ({ isPluginMetadataSnapshotCompatible: isPluginMetadataSnapshotCompatibleMock, - loadPluginMetadataSnapshot: (...args: unknown[]) => loadPluginMetadataSnapshotMock(...args), + loadPluginMetadataSnapshot: (params?: unknown) => loadPluginMetadataSnapshotMock(params), resolvePluginMetadataSnapshot: (params?: { pluginMetadataSnapshot?: unknown }) => params?.pluginMetadataSnapshot ?? loadPluginMetadataSnapshotMock(params), })); @@ -900,6 +901,59 @@ describe("plugin status reports", () => { expectNoCompatibilityWarnings(); }); + it("warns external plugins when load diagnostics reference removed session file APIs", () => { + setPluginLoadResult({ + plugins: [ + createPluginRecord({ + id: "file-backed-session-plugin", + name: "File-backed Session Plugin", + error: "The requested module does not provide an export named 'loadSessionStore'", + status: "error", + }), + ], + diagnostics: [ + { + level: "error", + pluginId: "file-backed-session-plugin", + message: "Plugin import failed before SessionTranscriptUpdate.sessionFile migration", + }, + ], + }); + + expectCompatibilityOutput({ + notices: [ + createCompatibilityNotice({ + pluginId: "file-backed-session-plugin", + code: "removed-session-transcript-file-api", + }), + ], + warnings: [`file-backed-session-plugin ${REMOVED_SESSION_TRANSCRIPT_FILE_API_MESSAGE}`], + }); + }); + + it("does not surface bundled session file API migration debt as user warnings", () => { + setPluginLoadResult({ + plugins: [ + createPluginRecord({ + id: "bundled-session-plugin", + name: "Bundled Session Plugin", + origin: "bundled", + error: "The requested module does not provide an export named 'sessionFile'", + status: "error", + }), + ], + diagnostics: [ + { + level: "error", + pluginId: "bundled-session-plugin", + message: "resolveSessionFilePath failed to load", + }, + ], + }); + + expectNoCompatibilityWarnings(); + }); + it("builds structured compatibility notices with deterministic ordering", () => { setPluginLoadResult({ plugins: [ diff --git a/src/plugins/status.ts b/src/plugins/status.ts index 0e19d5333f4b..5008d1a0a353 100644 --- a/src/plugins/status.ts +++ b/src/plugins/status.ts @@ -44,7 +44,11 @@ export type { PluginCapabilityKind, PluginInspectShape } from "./inspect-shape.j export type PluginCompatibilityNotice = { pluginId: string; - code: "legacy-before-agent-start" | "hook-only" | "deprecated-memory-embedding-provider-api"; + code: + | "legacy-before-agent-start" + | "hook-only" + | "deprecated-memory-embedding-provider-api" + | "removed-session-transcript-file-api"; compatCode: PluginCompatCode; severity: "warn" | "info"; message: string; @@ -105,6 +109,7 @@ export type PluginInspectReport = { function buildCompatibilityNoticesForInspect( inspect: Pick<PluginInspectReport, "plugin" | "shape" | "usesLegacyBeforeAgentStart"> & { + diagnostics: readonly PluginDiagnostic[]; hasRuntimeMemoryEmbeddingProviderRegistration: boolean; }, ): PluginCompatibilityNotice[] { @@ -143,9 +148,48 @@ function buildCompatibilityNoticesForInspect( "uses deprecated memory-specific embedding provider API; use api.registerEmbeddingProvider and contracts.embeddingProviders for new embedding providers.", }); } + if (usesRemovedSessionTranscriptFileApi(inspect)) { + warnings.push({ + pluginId: inspect.plugin.id, + code: "removed-session-transcript-file-api", + compatCode: "removed-session-transcript-file-api", + severity: "warn", + message: + "references removed session/transcript file APIs; migrate to session identity, SessionTranscriptUpdate.target, and Gateway/runtime session helpers.", + }); + } return warnings; } +const removedSessionTranscriptFileApiMarkers = [ + "loadSessionStore", + "saveSessionStore", + "updateSessionStore", + "resolveSessionFilePath", + "resolveSessionTranscriptPathInDir", + "resolveAndPersistSessionFile", + "readLatestAssistantTextFromSessionTranscript", + "SessionTranscriptUpdate.sessionFile", + "sessionFiles", + "transcriptPath", + "sessionFile", +] as const; + +function usesRemovedSessionTranscriptFileApi( + inspect: Pick<PluginInspectReport, "plugin"> & { diagnostics: readonly PluginDiagnostic[] }, +): boolean { + if (inspect.plugin.origin === "bundled") { + return false; + } + const messages = [ + inspect.plugin.error, + ...inspect.diagnostics.map((diagnostic) => diagnostic.message), + ].filter((message): message is string => typeof message === "string" && message.length > 0); + return messages.some((message) => + removedSessionTranscriptFileApiMarkers.some((marker) => message.includes(marker)), + ); +} + function resolveReportedPluginVersion( plugin: PluginRegistry["plugins"][number], env: NodeJS.ProcessEnv | undefined, @@ -408,6 +452,7 @@ export function buildPluginInspectReport(params: { plugin, shape, usesLegacyBeforeAgentStart, + diagnostics, hasRuntimeMemoryEmbeddingProviderRegistration, }); return { diff --git a/src/plugins/types.ts b/src/plugins/types.ts index a8cdf4d611a9..6457486a3deb 100644 --- a/src/plugins/types.ts +++ b/src/plugins/types.ts @@ -2038,7 +2038,15 @@ export type PluginCommandDiagnosticsSession = { sessionKey?: string; /** Ephemeral OpenClaw session id when available. */ sessionId?: string; - /** Transcript file for this OpenClaw session when available. */ + /** + * Deprecated transcript locator for this OpenClaw session when available. + * + * SQLite-backed sessions use a `sqlite:<agentId>:<sessionId>:<storePath>` + * marker, not a filesystem path. Use session id/key plus transcript-runtime + * helpers for active transcript reads. + * + * @deprecated Use session identity fields with `plugin-sdk/session-transcript-runtime`. + */ sessionFile?: string; /** Embedded agent harness selected for this session. */ agentHarnessId?: string; @@ -2076,7 +2084,15 @@ export type PluginCommandContext = { sessionKey?: string; /** Ephemeral host session id for the active conversation when available. */ sessionId?: string; - /** Transcript file for the active OpenClaw session when available. */ + /** + * Deprecated transcript locator for the active OpenClaw session when available. + * + * SQLite-backed sessions use a `sqlite:<agentId>:<sessionId>:<storePath>` + * marker, not a filesystem path. Use session id/key plus transcript-runtime + * helpers for active transcript reads. + * + * @deprecated Use session identity fields with `plugin-sdk/session-transcript-runtime`. + */ sessionFile?: string; /** Raw command arguments after the command name */ args?: string; diff --git a/src/routing/session-key.test.ts b/src/routing/session-key.test.ts index 3628cd3c2f4b..fead72ecc706 100644 --- a/src/routing/session-key.test.ts +++ b/src/routing/session-key.test.ts @@ -1,6 +1,9 @@ // Routing session key tests cover route-derived session key behavior. import { describe, expect, it } from "vitest"; -import { resolveSessionStoreAgentId } from "../gateway/session-store-key.js"; +import { + resolveSessionStoreAgentId, + resolveSessionStoreKey, +} from "../gateway/session-store-key.js"; import { deriveSessionChatTypeFromKey } from "../sessions/session-chat-type-shared.js"; import { getSubagentDepth, @@ -87,6 +90,22 @@ describe("agentSessionKeysMatchByRequestKey", () => { }); }); +describe("resolveSessionStoreKey", () => { + it("scopes unprefixed explicit-agent keys to the requested store agent", () => { + const cfg = { + agents: { list: [{ id: "main", default: true }, { id: "ops" }] }, + session: { mainKey: "primary" }, + }; + + expect(resolveSessionStoreKey({ cfg, sessionKey: "main", storeAgentId: "ops" })).toBe( + "agent:ops:primary", + ); + expect(resolveSessionStoreKey({ cfg, sessionKey: "discord:dm:U1", storeAgentId: "ops" })).toBe( + "agent:ops:discord:dm:u1", + ); + }); +}); + describe("session key backward compatibility", () => { function expectBackwardCompatibleDirectSessionKey(key: string) { expect(classifySessionKeyShape(key)).toBe("agent"); diff --git a/src/sessions/transcript-events.test.ts b/src/sessions/transcript-events.test.ts index 37350e6fa11a..7ac28be4dbe9 100644 --- a/src/sessions/transcript-events.test.ts +++ b/src/sessions/transcript-events.test.ts @@ -5,7 +5,6 @@ import { emitSessionTranscriptUpdate, onInternalSessionTranscriptUpdate, onSessionTranscriptUpdate, - type SessionTranscriptUpdate, } from "./transcript-events.js"; const cleanup: Array<() => void> = []; @@ -17,40 +16,62 @@ afterEach(() => { }); describe("transcript events", () => { - it("emits trimmed session file updates", () => { + it("emits trimmed archive file updates only to internal listeners", () => { const listener = vi.fn(); - cleanup.push(onSessionTranscriptUpdate(listener)); + cleanup.push(onInternalSessionTranscriptUpdate(listener)); - emitSessionTranscriptUpdate(" /tmp/session.jsonl "); + emitSessionTranscriptUpdate({ sessionFile: " /tmp/session.jsonl " }); expect(listener).toHaveBeenCalledTimes(1); expect(listener).toHaveBeenCalledWith({ sessionFile: "/tmp/session.jsonl" }); }); - it("includes optional session metadata when provided", () => { - const listener = vi.fn(); - cleanup.push(onSessionTranscriptUpdate(listener)); + it("does not expose file-only archive updates to public listeners", () => { + const publicListener = vi.fn(); + const internalListener = vi.fn(); + cleanup.push(onSessionTranscriptUpdate(publicListener)); + cleanup.push(onInternalSessionTranscriptUpdate(internalListener)); emitSessionTranscriptUpdate({ sessionFile: " /tmp/session.jsonl ", sessionKey: " agent:main:main ", agentId: " main ", + sessionId: " sess-1 ", message: { role: "assistant", content: "hi" }, messageId: " msg-1 ", messageSeq: 2, }); - expect(listener).toHaveBeenCalledWith({ - sessionFile: "/tmp/session.jsonl", + expect(publicListener).toHaveBeenCalledWith({ + target: { + agentId: "main", + sessionId: "sess-1", + sessionKey: "agent:main:main", + }, sessionKey: "agent:main:main", agentId: "main", + sessionId: "sess-1", + message: { role: "assistant", content: "hi" }, + messageId: "msg-1", + messageSeq: 2, + }); + expect(internalListener).toHaveBeenCalledWith({ + sessionFile: "/tmp/session.jsonl", + target: { + agentId: "main", + sessionId: "sess-1", + sessionKey: "agent:main:main", + }, + sessionKey: "agent:main:main", + agentId: "main", + sessionId: "sess-1", message: { role: "assistant", content: "hi" }, messageId: "msg-1", messageSeq: 2, }); }); - it("does not expose identity-only updates to public listeners", () => { + it("exposes identity-only updates to public listeners", () => { const listener = vi.fn(); cleanup.push(onSessionTranscriptUpdate(listener)); @@ -61,9 +82,19 @@ describe("transcript events", () => { sessionKey: " agent:main:main ", }, messageId: " msg-1 ", - } as unknown as SessionTranscriptUpdate); + }); - expect(listener).not.toHaveBeenCalled(); + expect(listener).toHaveBeenCalledWith({ + target: { + agentId: "main", + sessionId: "sess-1", + sessionKey: "agent:main:main", + }, + agentId: "main", + sessionId: "sess-1", + sessionKey: "agent:main:main", + messageId: "msg-1", + }); }); it("emits storage-neutral identity updates to internal listeners", () => { @@ -92,7 +123,7 @@ describe("transcript events", () => { }); }); - it("derives target identity from public file updates", () => { + it("derives public target identity from legacy-shaped internal updates", () => { const listener = vi.fn(); cleanup.push(onSessionTranscriptUpdate(listener)); @@ -103,7 +134,6 @@ describe("transcript events", () => { }); expect(listener).toHaveBeenCalledWith({ - sessionFile: "/tmp/session.jsonl", target: { agentId: "main", sessionId: "sess-1", @@ -115,24 +145,27 @@ describe("transcript events", () => { }); }); - it("keeps public global file updates on the compatibility shape", () => { - const listener = vi.fn(); - cleanup.push(onSessionTranscriptUpdate(listener)); + it("drops public global file updates without target identity", () => { + const publicListener = vi.fn(); + const internalListener = vi.fn(); + cleanup.push(onSessionTranscriptUpdate(publicListener)); + cleanup.push(onInternalSessionTranscriptUpdate(internalListener)); emitSessionTranscriptUpdate({ sessionFile: "/tmp/session.jsonl", sessionKey: "global", }); - expect(listener).toHaveBeenCalledWith({ + expect(publicListener).not.toHaveBeenCalled(); + expect(internalListener).toHaveBeenCalledWith({ sessionFile: "/tmp/session.jsonl", sessionKey: "global", }); }); - it("drops invalid message sequence values", () => { + it("drops invalid message sequence values on internal file updates", () => { const listener = vi.fn(); - cleanup.push(onSessionTranscriptUpdate(listener)); + cleanup.push(onInternalSessionTranscriptUpdate(listener)); emitSessionTranscriptUpdate({ sessionFile: "/tmp/session.jsonl", @@ -158,10 +191,10 @@ describe("transcript events", () => { throw new Error("boom"); }); const second = vi.fn(); - cleanup.push(onSessionTranscriptUpdate(first)); - cleanup.push(onSessionTranscriptUpdate(second)); + cleanup.push(onInternalSessionTranscriptUpdate(first)); + cleanup.push(onInternalSessionTranscriptUpdate(second)); - expect(emitSessionTranscriptUpdate("/tmp/session.jsonl")).toBeUndefined(); + expect(emitSessionTranscriptUpdate({ sessionFile: "/tmp/session.jsonl" })).toBeUndefined(); expect(first).toHaveBeenCalledTimes(1); expect(second).toHaveBeenCalledTimes(1); }); diff --git a/src/sessions/transcript-events.ts b/src/sessions/transcript-events.ts index 0b8875b2cf6d..ed0d9b602763 100644 --- a/src/sessions/transcript-events.ts +++ b/src/sessions/transcript-events.ts @@ -15,7 +15,6 @@ type SessionTranscriptUpdateFields = { target?: SessionTranscriptUpdateTarget; sessionKey?: string; agentId?: string; - /** @deprecated Pre-SQLite compatibility mirror. Prefer `target.sessionId`. */ sessionId?: string; message?: unknown; messageId?: string; @@ -23,9 +22,8 @@ type SessionTranscriptUpdateFields = { }; /** Normalized transcript update emitted after a session transcript changes. */ -export type SessionTranscriptUpdate = SessionTranscriptUpdateFields & { - /** @deprecated File-backed compatibility hint. Prefer `target` for identity. */ - sessionFile: string; +export type SessionTranscriptUpdate = Omit<SessionTranscriptUpdateFields, "sessionFile"> & { + target: SessionTranscriptUpdateTarget; }; /** Internal transcript update that may identify a transcript without a file path. */ @@ -56,12 +54,15 @@ export function onInternalSessionTranscriptUpdate( } /** Emits a normalized transcript update to all registered listeners. */ -export function emitSessionTranscriptUpdate(update: string | SessionTranscriptUpdate): void { - const nextUpdate = normalizeSessionTranscriptUpdate(update, { allowIdentityOnly: false }); - if (!nextUpdate?.sessionFile) { +export function emitSessionTranscriptUpdate(update: InternalSessionTranscriptUpdate): void { + const nextUpdate = normalizeSessionTranscriptUpdate(update, { allowIdentityOnly: true }); + if (!nextUpdate) { return; } - emitPublicSessionTranscriptUpdate(nextUpdate as SessionTranscriptUpdate); + const publicUpdate = projectPublicSessionTranscriptUpdate(nextUpdate); + if (publicUpdate) { + emitPublicSessionTranscriptUpdate(publicUpdate); + } emitInternalTranscriptUpdate(nextUpdate); } @@ -75,24 +76,19 @@ export function emitInternalSessionTranscriptUpdate(update: InternalSessionTrans } function normalizeSessionTranscriptUpdate( - update: string | InternalSessionTranscriptUpdate, + update: InternalSessionTranscriptUpdate, options: { allowIdentityOnly: boolean }, ): InternalSessionTranscriptUpdate | undefined { - // Public callers still need a file-backed update, while internal callers can - // carry identity-only updates during the pre-SQLite transition. - const normalized = - typeof update === "string" - ? { sessionFile: update } - : { - sessionFile: update.sessionFile, - target: update.target, - sessionKey: update.sessionKey, - agentId: update.agentId, - sessionId: update.sessionId, - message: update.message, - messageId: update.messageId, - messageSeq: update.messageSeq, - }; + const normalized = { + sessionFile: update.sessionFile, + target: update.target, + sessionKey: update.sessionKey, + agentId: update.agentId, + sessionId: update.sessionId, + message: update.message, + messageId: update.messageId, + messageSeq: update.messageSeq, + }; const trimmed = normalizeOptionalString(normalized.sessionFile); const target = normalizeUpdateTarget(normalized); if (!trimmed && (!options.allowIdentityOnly || !target)) { @@ -136,6 +132,24 @@ function emitInternalTranscriptUpdate(nextUpdate: InternalSessionTranscriptUpdat } } +function projectPublicSessionTranscriptUpdate( + update: InternalSessionTranscriptUpdate, +): SessionTranscriptUpdate | undefined { + const target = update.target; + if (!target) { + return undefined; + } + return { + target, + ...(update.sessionKey ? { sessionKey: update.sessionKey } : {}), + ...(update.agentId ? { agentId: update.agentId } : {}), + ...(update.sessionId ? { sessionId: update.sessionId } : {}), + ...(update.message !== undefined ? { message: update.message } : {}), + ...(update.messageId ? { messageId: update.messageId } : {}), + ...(update.messageSeq !== undefined ? { messageSeq: update.messageSeq } : {}), + }; +} + function normalizeUpdateTarget(update: { agentId?: string; sessionId?: string; diff --git a/src/sessions/user-turn-transcript.test-support.ts b/src/sessions/user-turn-transcript.test-support.ts new file mode 100644 index 000000000000..c7ee1401faa0 --- /dev/null +++ b/src/sessions/user-turn-transcript.test-support.ts @@ -0,0 +1,14 @@ +import type { UserTurnTranscriptTarget } from "./user-turn-transcript.types.js"; + +/** Creates a store-backed transcript target for tests that do not own runtime session setup. */ +export function createTestUserTurnTranscriptTarget( + overrides: Partial<UserTurnTranscriptTarget> = {}, +): UserTurnTranscriptTarget { + return { + agentId: "main", + sessionEntry: undefined, + sessionId: "test-session", + sessionKey: "agent:main:test", + ...overrides, + }; +} diff --git a/src/sessions/user-turn-transcript.test.ts b/src/sessions/user-turn-transcript.test.ts index 4d39fbc412cf..ed3330d2fac5 100644 --- a/src/sessions/user-turn-transcript.test.ts +++ b/src/sessions/user-turn-transcript.test.ts @@ -10,8 +10,9 @@ import { createMockPluginRegistry } from "openclaw/plugin-sdk/plugin-test-runtim import { castAgentMessage } from "openclaw/plugin-sdk/test-fixtures"; import { afterEach, describe, expect, it } from "vitest"; import { runAgentHarnessBeforeMessageWriteHook } from "../agents/harness/hook-helpers.js"; +import { loadTranscriptEvents } from "../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { - appendUserTurnTranscriptMessage, buildPersistedUserTurnMediaInputsFromFields, createUserTurnTranscriptRecorder, mergePreparedUserTurnMessageForRuntime, @@ -22,6 +23,13 @@ import { describe("user turn transcript persistence", () => { const tempDirs: string[] = []; + const unusedRecorderTarget = { + agentId: "main", + sessionEntry: undefined, + sessionId: "unused-session", + sessionKey: "agent:main:unused", + storePath: "/tmp/openclaw-unused-sessions.json", + }; afterEach(() => { resetGlobalHookRunner(); @@ -36,13 +44,45 @@ describe("user turn transcript persistence", () => { return dir; } - function readTranscriptMessages(transcriptPath: string): Array<Record<string, unknown>> { - return fs - .readFileSync(transcriptPath, "utf-8") - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line) as { message?: unknown }) - .map((entry) => entry.message) + function createSqliteTranscriptTarget(params: { + dir: string; + sessionId?: string; + sessionKey?: string; + }) { + const sessionId = params.sessionId ?? "session-1"; + const sessionKey = params.sessionKey ?? "agent:main:main"; + const storePath = path.join(params.dir, "agents", "main", "sessions", "sessions.json"); + fs.mkdirSync(path.dirname(storePath), { recursive: true }); + const sqliteMarker = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + return { + agentId: "main", + cwd: params.dir, + sessionEntry: undefined, + sessionId, + sessionKey, + storePath, + sqliteMarker, + }; + } + + async function readTranscriptMessages(params: { + sessionId: string; + sessionKey: string; + storePath: string; + }): Promise<Array<Record<string, unknown>>> { + return ( + await loadTranscriptEvents({ + agentId: "main", + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath: params.storePath, + }) + ) + .map((entry) => (entry as { message?: unknown }).message) .filter( (message): message is Record<string, unknown> => typeof message === "object" && message !== null, @@ -180,7 +220,7 @@ describe("user turn transcript persistence", () => { media: [{ path: "/tmp/image.png", contentType: "image/png" }], timestamp: 123, }, - target: { transcriptPath: "/tmp/session.jsonl" }, + target: unusedRecorderTarget, }); expect( @@ -208,7 +248,7 @@ describe("user turn transcript persistence", () => { text: "group prompt", sender: { id: "user-42", name: "Ada" }, }, - target: { transcriptPath: "/tmp/session.jsonl" }, + target: unusedRecorderTarget, }); expect( @@ -232,7 +272,7 @@ describe("user turn transcript persistence", () => { it("does not replace blocked before_agent_run user markers", () => { const recorder = createUserTurnTranscriptRecorder({ input: { text: "raw prompt" }, - target: { transcriptPath: "/tmp/session.jsonl" }, + target: unusedRecorderTarget, }); const blocked = castAgentMessage({ role: "user", @@ -251,7 +291,7 @@ describe("user turn transcript persistence", () => { it("preserves runtime multimodal content while merging prepared metadata", () => { const recorder = createUserTurnTranscriptRecorder({ input: { text: "canonical image caption", timestamp: 123 }, - target: { transcriptPath: "/tmp/session.jsonl" }, + target: unusedRecorderTarget, }); const runtimeContent = [ { type: "text", text: "canonical image caption" }, @@ -276,7 +316,7 @@ describe("user turn transcript persistence", () => { it("does not apply prepared user metadata to assistant messages", () => { const recorder = createUserTurnTranscriptRecorder({ input: { text: "display prompt" }, - target: { transcriptPath: "/tmp/session.jsonl" }, + target: unusedRecorderTarget, }); const assistant = castAgentMessage({ role: "assistant", content: "hello" }); @@ -306,21 +346,18 @@ describe("user turn transcript persistence", () => { }); }); - describe("appendUserTurnTranscriptMessage", () => { + describe("persistUserTurnTranscript", () => { it("appends a structured user turn through the shared transcript writer", async () => { const dir = createTempDir("openclaw-user-turn-append-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); const provenance = { kind: "inter_session" as const, sourceSessionKey: "source-main", sourceTool: "sessions_send", }; - const appended = await appendUserTurnTranscriptMessage({ - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + const appended = await persistUserTurnTranscript({ + ...target, input: { text: "What is in this image?", media: [{ path: "/tmp/image.png", contentType: "image/png" }], @@ -336,7 +373,7 @@ describe("user turn transcript persistence", () => { content: "What is in this image?", MediaPath: "/tmp/image.png", }); - expect(readTranscriptMessages(transcriptPath)).toEqual([ + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ role: "user", content: "What is in this image?", @@ -350,13 +387,10 @@ describe("user turn transcript persistence", () => { it("persists sender metadata as __openclaw envelope", async () => { const dir = createTempDir("openclaw-user-turn-append-sender-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); - const appended = await appendUserTurnTranscriptMessage({ - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + const appended = await persistUserTurnTranscript({ + ...target, input: { text: "hello from group", sender: { @@ -377,7 +411,7 @@ describe("user turn transcript persistence", () => { senderUsername: "ram_s", }, }); - expect(readTranscriptMessages(transcriptPath)).toEqual([ + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ role: "user", content: "hello from group", @@ -392,13 +426,10 @@ describe("user turn transcript persistence", () => { it("omits __openclaw when no sender metadata is provided", async () => { const dir = createTempDir("openclaw-user-turn-append-nosender-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); - const appended = await appendUserTurnTranscriptMessage({ - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + const appended = await persistUserTurnTranscript({ + ...target, input: { text: "hello without sender", sender: { id: "", name: null }, @@ -411,13 +442,10 @@ describe("user turn transcript persistence", () => { it("uses inline update mode by default", async () => { const dir = createTempDir("openclaw-user-turn-append-inline-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); - const appended = await appendUserTurnTranscriptMessage({ - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + const appended = await persistUserTurnTranscript({ + ...target, input: { text: "hello from runtime", }, @@ -428,7 +456,7 @@ describe("user turn transcript persistence", () => { content: "hello from runtime", timestamp: expect.any(Number), }); - expect(readTranscriptMessages(transcriptPath)).toEqual([ + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ role: "user", content: "hello from runtime", @@ -439,13 +467,10 @@ describe("user turn transcript persistence", () => { it("returns the existing user turn when the idempotency key was already persisted", async () => { const dir = createTempDir("openclaw-user-turn-append-idempotent-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); - const first = await appendUserTurnTranscriptMessage({ - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + const first = await persistUserTurnTranscript({ + ...target, input: { text: "hello once", timestamp: 123, @@ -453,11 +478,8 @@ describe("user turn transcript persistence", () => { }, updateMode: "none", }); - const second = await appendUserTurnTranscriptMessage({ - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + const second = await persistUserTurnTranscript({ + ...target, input: { text: "hello once replayed", timestamp: 456, @@ -473,7 +495,7 @@ describe("user turn transcript persistence", () => { timestamp: 123, idempotencyKey: "chat-run-1:user", }); - expect(readTranscriptMessages(transcriptPath)).toEqual([ + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ role: "user", content: "hello once", @@ -508,10 +530,10 @@ describe("user turn transcript persistence", () => { ]), ); const dir = createTempDir("openclaw-user-turn-redacted-idempotent-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); - await appendUserTurnTranscriptMessage({ - transcriptPath, + await persistUserTurnTranscript({ + ...target, input: { text: "secret prompt", idempotencyKey: "chat-run-1:user", @@ -521,8 +543,8 @@ describe("user turn transcript persistence", () => { }, beforeMessageWrite: runAgentHarnessBeforeMessageWriteHook, }); - await appendUserTurnTranscriptMessage({ - transcriptPath, + await persistUserTurnTranscript({ + ...target, input: { text: "secret prompt", idempotencyKey: "chat-run-1:user", @@ -533,7 +555,7 @@ describe("user turn transcript persistence", () => { beforeMessageWrite: runAgentHarnessBeforeMessageWriteHook, }); - expect(readTranscriptMessages(transcriptPath)).toEqual([ + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ role: "user", content: "[redacted by hook]", @@ -552,22 +574,22 @@ describe("user turn transcript persistence", () => { describe("persistUserTurnTranscript", () => { it("resolves the session file and persists the user turn", async () => { const dir = createTempDir("openclaw-user-turn-persist-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); const sessionStore = { - main: { - sessionId: "session-1", - sessionFile: transcriptPath, + [target.sessionKey]: { + sessionId: target.sessionId, + sessionFile: target.sqliteMarker, updatedAt: 1, }, }; const persisted = await persistUserTurnTranscript({ - sessionId: "session-1", - sessionKey: "main", - sessionEntry: sessionStore.main, + sessionId: target.sessionId, + sessionKey: target.sessionKey, + sessionEntry: sessionStore[target.sessionKey], sessionStore, - storePath: path.join(dir, "sessions.json"), - agentId: "agent", + storePath: target.storePath, + agentId: target.agentId, cwd: dir, input: { text: "hello", @@ -576,9 +598,8 @@ describe("user turn transcript persistence", () => { updateMode: "none", }); - expect(persisted?.sessionFile).toBeTruthy(); - expect(fs.existsSync(persisted?.sessionFile ?? "")).toBe(true); - expect(readTranscriptMessages(persisted?.sessionFile ?? "")).toEqual([ + expect(persisted?.sessionFile).toBe(target.sqliteMarker); + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ role: "user", content: "hello", @@ -590,7 +611,7 @@ describe("user turn transcript persistence", () => { describe("createUserTurnTranscriptRecorder", () => { it("persists fallback user turns only once", async () => { const dir = createTempDir("openclaw-user-turn-recorder-fallback-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); const recorder = createUserTurnTranscriptRecorder({ input: { text: "hello from fallback", @@ -598,10 +619,7 @@ describe("user turn transcript persistence", () => { idempotencyKey: "chat-run-1:user", }, target: { - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + ...target, }, updateMode: "none", }); @@ -613,7 +631,7 @@ describe("user turn transcript persistence", () => { expect(first?.messageId).toBeTruthy(); expect(second?.messageId).toBe(first?.messageId); - expect(readTranscriptMessages(transcriptPath)).toEqual([ + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ role: "user", content: "hello from fallback", @@ -624,7 +642,7 @@ describe("user turn transcript persistence", () => { it("notifies once after fallback user-turn persistence", async () => { const dir = createTempDir("openclaw-user-turn-recorder-notify-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); const persistedMessages: unknown[] = []; const recorder = createUserTurnTranscriptRecorder({ input: { @@ -633,10 +651,7 @@ describe("user turn transcript persistence", () => { idempotencyKey: "chat-run-ambient:user", }, target: { - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + ...target, }, updateMode: "none", onMessagePersisted: (message) => { @@ -653,7 +668,7 @@ describe("user turn transcript persistence", () => { content: "#35676 Keśava: No wtf", }), ]); - expect(readTranscriptMessages(transcriptPath)).toEqual([ + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ role: "user", content: "#35676 Keśava: No wtf", @@ -663,7 +678,7 @@ describe("user turn transcript persistence", () => { it("resolves media lazily at persistence time", async () => { const dir = createTempDir("openclaw-user-turn-recorder-lazy-media-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); let resolverCalled = false; const recorder = createUserTurnTranscriptRecorder({ input: { @@ -681,10 +696,7 @@ describe("user turn transcript persistence", () => { }; }, target: { - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + ...target, }, updateMode: "none", }); @@ -708,7 +720,7 @@ describe("user turn transcript persistence", () => { MediaPath: path.join(dir, "image.png"), MediaType: "image/png", }); - expect(readTranscriptMessages(transcriptPath)).toEqual([ + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ role: "user", content: "describe this", @@ -720,7 +732,7 @@ describe("user turn transcript persistence", () => { it("appends #99495 media that resolves after the admitted turn reached the provider", async () => { const dir = createTempDir("openclaw-user-turn-recorder-late-media-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); const admittedInput = { text: "describe this", timestamp: 123, @@ -741,20 +753,14 @@ describe("user turn transcript persistence", () => { return await mediaInput; }, target: { - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + ...target, }, }); const persistence = recorder.persistFallback(); await resolverStarted; - await appendUserTurnTranscriptMessage({ - transcriptPath, + await persistUserTurnTranscript({ + ...target, input: admittedInput, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, }); recorder.markRuntimePersisted(recorder.message); recorder.markSentToProvider?.(); @@ -765,7 +771,7 @@ describe("user turn transcript persistence", () => { await persistence; - expect(readTranscriptMessages(transcriptPath)).toEqual([ + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ content: "describe this", idempotencyKey: "chat-run-late:user", @@ -780,7 +786,7 @@ describe("user turn transcript persistence", () => { it("keeps #99495 media inline when it resolves before first serialization", async () => { const dir = createTempDir("openclaw-user-turn-recorder-early-media-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); const recorder = createUserTurnTranscriptRecorder({ input: { text: "describe this", @@ -794,17 +800,14 @@ describe("user turn transcript persistence", () => { media: [{ path: path.join(dir, "image.png"), contentType: "image/png" }], }), target: { - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + ...target, }, }); await recorder.persistFallback(); recorder.markSentToProvider?.(); - expect(readTranscriptMessages(transcriptPath)).toEqual([ + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ content: "describe this", idempotencyKey: "chat-run-early:user", @@ -815,7 +818,7 @@ describe("user turn transcript persistence", () => { it("falls back to the admitted text message when lazy media resolution fails", async () => { const dir = createTempDir("openclaw-user-turn-recorder-lazy-failed-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); const errors: unknown[] = []; const recorder = createUserTurnTranscriptRecorder({ input: { @@ -827,10 +830,7 @@ describe("user turn transcript persistence", () => { throw new Error("media staging failed"); }, target: { - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + ...target, }, updateMode: "none", onPersistenceError: (error) => errors.push(error), @@ -845,7 +845,7 @@ describe("user turn transcript persistence", () => { idempotencyKey: "chat-run-lazy-failed:user", }); expect(persisted?.message).not.toHaveProperty("MediaPath"); - expect(readTranscriptMessages(transcriptPath)).toEqual([ + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ role: "user", content: "keep the prompt", @@ -856,17 +856,14 @@ describe("user turn transcript persistence", () => { it("does not fallback-persist after runtime persistence is marked", async () => { const dir = createTempDir("openclaw-user-turn-recorder-runtime-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); const recorder = createUserTurnTranscriptRecorder({ input: { text: "runtime-owned turn", timestamp: 123, }, target: { - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + ...target, }, updateMode: "none", }); @@ -878,22 +875,79 @@ describe("user turn transcript persistence", () => { }); await expect(recorder.persistFallback()).resolves.toBeUndefined(); - expect(fs.existsSync(transcriptPath)).toBe(false); + await expect(readTranscriptMessages(target)).resolves.toEqual([]); + }); + + it("approved persistence skips file targets after runtime persistence is marked", async () => { + const dir = createTempDir("openclaw-user-turn-recorder-runtime-approved-"); + const target = createSqliteTranscriptTarget({ dir }); + const recorder = createUserTurnTranscriptRecorder({ + input: { + text: "runtime-owned turn", + timestamp: 123, + }, + target: { + ...target, + }, + updateMode: "none", + }); + + recorder.markRuntimePersisted({ + role: "user", + content: "runtime-owned turn", + timestamp: 123, + }); + + await expect(recorder.persistApproved()).resolves.toBeUndefined(); + await expect(readTranscriptMessages(target)).resolves.toEqual([]); + }); + + it("approved persistence does not duplicate runtime-owned SQLite turns", async () => { + const dir = createTempDir("openclaw-user-turn-recorder-runtime-canonical-"); + const storePath = path.join(dir, "sessions.json"); + const sessionStore = {}; + const recorder = createUserTurnTranscriptRecorder({ + input: { + text: "runtime-owned turn", + timestamp: 123, + }, + target: { + agentId: "main", + sessionEntry: undefined, + sessionId: "session-1", + sessionKey: "agent:main:main", + sessionStore, + storePath, + }, + updateMode: "none", + }); + + recorder.markRuntimePersisted({ + role: "user", + content: "runtime-owned turn", + timestamp: 123, + }); + + await expect(recorder.persistApproved()).resolves.toBeUndefined(); + await expect( + readTranscriptMessages({ + sessionId: "session-1", + sessionKey: "agent:main:main", + storePath, + }), + ).resolves.toEqual([]); }); it("does not fallback-persist after before_agent_run blocks the turn", async () => { const dir = createTempDir("openclaw-user-turn-recorder-blocked-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); const recorder = createUserTurnTranscriptRecorder({ input: { text: "raw blocked prompt", timestamp: 123, }, target: { - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + ...target, }, updateMode: "none", }); @@ -901,39 +955,33 @@ describe("user turn transcript persistence", () => { recorder.markBlocked(); await expect(recorder.persistFallback()).resolves.toBeUndefined(); - expect(fs.existsSync(transcriptPath)).toBe(false); + await expect(readTranscriptMessages(target)).resolves.toEqual([]); }); it("uses the runtime target supplied at approved persistence time", async () => { const dir = createTempDir("openclaw-user-turn-recorder-target-"); - const staleTranscriptPath = path.join(dir, "stale.jsonl"); - const admittedTranscriptPath = path.join(dir, "admitted.jsonl"); + const staleTarget = createSqliteTranscriptTarget({ dir, sessionId: "stale-session" }); + const admittedTarget = createSqliteTranscriptTarget({ dir, sessionId: "admitted-session" }); const recorder = createUserTurnTranscriptRecorder({ input: { text: "persist me in the admitted session", timestamp: 123, }, target: { - transcriptPath: staleTranscriptPath, - sessionId: "stale-session", - sessionKey: "main", - cwd: dir, + ...staleTarget, }, updateMode: "none", }); const persisted = await recorder.persistApproved({ target: { - transcriptPath: admittedTranscriptPath, - sessionId: "admitted-session", - sessionKey: "main", - cwd: dir, + ...admittedTarget, }, }); - expect(persisted?.sessionFile).toBe(admittedTranscriptPath); - expect(fs.existsSync(staleTranscriptPath)).toBe(false); - expect(readTranscriptMessages(admittedTranscriptPath)).toEqual([ + expect(persisted?.sessionFile).toBe(admittedTarget.sqliteMarker); + await expect(readTranscriptMessages(staleTarget)).resolves.toEqual([]); + await expect(readTranscriptMessages(admittedTarget)).resolves.toEqual([ expect.objectContaining({ role: "user", content: "persist me in the admitted session", @@ -943,7 +991,7 @@ describe("user turn transcript persistence", () => { it("waits for runtime persistence before deciding fallback ownership", async () => { const dir = createTempDir("openclaw-user-turn-recorder-pending-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); let releaseRuntimePersistence!: () => void; const runtimePersistenceStarted = new Promise<void>((resolve) => { releaseRuntimePersistence = resolve; @@ -954,10 +1002,7 @@ describe("user turn transcript persistence", () => { timestamp: 123, }, target: { - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + ...target, }, updateMode: "none", }); @@ -983,12 +1028,12 @@ describe("user turn transcript persistence", () => { releaseRuntimePersistence(); await expect(fallback).resolves.toBeUndefined(); - expect(fs.existsSync(transcriptPath)).toBe(false); + await expect(readTranscriptMessages(target)).resolves.toEqual([]); }); it("fallback-persists when pending runtime persistence fails", async () => { const dir = createTempDir("openclaw-user-turn-recorder-pending-failed-"); - const transcriptPath = path.join(dir, "session.jsonl"); + const target = createSqliteTranscriptTarget({ dir }); const errors: unknown[] = []; let rejectRuntimePersistence!: (error: unknown) => void; const runtimePersistence = new Promise<void>((_, reject) => { @@ -1000,10 +1045,7 @@ describe("user turn transcript persistence", () => { timestamp: 123, }, target: { - transcriptPath, - sessionId: "session-1", - sessionKey: "main", - cwd: dir, + ...target, }, updateMode: "none", onPersistenceError: (error) => errors.push(error), @@ -1019,7 +1061,7 @@ describe("user turn transcript persistence", () => { role: "user", content: "pending failed turn", }); - expect(readTranscriptMessages(transcriptPath)).toEqual([ + await expect(readTranscriptMessages(target)).resolves.toEqual([ expect.objectContaining({ role: "user", content: "pending failed turn", diff --git a/src/sessions/user-turn-transcript.ts b/src/sessions/user-turn-transcript.ts index 34755cc6a319..13c1355c3503 100644 --- a/src/sessions/user-turn-transcript.ts +++ b/src/sessions/user-turn-transcript.ts @@ -11,7 +11,6 @@ import type { UserTurnBeforeMessageWrite, UserTurnInput, UserTurnSessionEntry, - UserTurnTranscriptFileTarget, UserTurnTranscriptPersistResult, UserTurnTranscriptRecorder, UserTurnTranscriptTarget, @@ -39,8 +38,7 @@ type PersistedUserTurnMediaFields = { MediaTypes?: string[]; }; -type AppendUserTurnTranscriptMessageParams = { - transcriptPath: string; +type UserTurnMessagePersistenceParams = { input?: UserTurnInput; message?: PersistedUserTurnMessage; sessionId?: string; @@ -56,6 +54,7 @@ type PersistUserTurnTranscriptParams = { input?: UserTurnInput; message?: PersistedUserTurnMessage; sessionId: string; + expectedSessionId?: string; sessionKey: string; sessionEntry: UserTurnSessionEntry | undefined; sessionStore?: Record<string, UserTurnSessionEntry>; @@ -286,7 +285,7 @@ export function buildPersistedUserTurnMessage(params: UserTurnInput): PersistedU } function resolvePersistedUserTurnMessage( - params: Pick<AppendUserTurnTranscriptMessageParams, "input" | "message">, + params: Pick<UserTurnMessagePersistenceParams, "input" | "message">, ): PersistedUserTurnMessage | undefined { if (params.message) { return params.message; @@ -417,10 +416,7 @@ export function restorePreparedUserTurnOperationalMetaForRuntime(params: { /** Applies before-message hooks while preserving user-turn transcript metadata. */ export function preparePersistedUserTurnMessageForTranscriptWrite( message: PersistedUserTurnMessage, - params: Pick< - AppendUserTurnTranscriptMessageParams, - "agentId" | "sessionKey" | "beforeMessageWrite" - >, + params: Pick<UserTurnMessagePersistenceParams, "agentId" | "sessionKey" | "beforeMessageWrite">, ): PersistedUserTurnMessage | undefined { if (!params.beforeMessageWrite) { return message; @@ -460,62 +456,6 @@ export function preparePersistedUserTurnMessageForTranscriptWrite( } as unknown as PersistedUserTurnMessage; } -export async function appendUserTurnTranscriptMessage( - params: AppendUserTurnTranscriptMessageParams, -): Promise< - | { - sessionFile: string; - messageId: string; - message: PersistedUserTurnMessage; - } - | undefined -> { - const resolvedMessage = resolvePersistedUserTurnMessage(params); - if (!resolvedMessage) { - return undefined; - } - - const turn = await persistSessionTranscriptTurn( - { - sessionFile: params.transcriptPath, - sessionKey: params.sessionKey ?? "", - ...(params.agentId ? { agentId: params.agentId } : {}), - ...(params.sessionId ? { sessionId: params.sessionId } : {}), - }, - { - ...(params.cwd ? { cwd: params.cwd } : {}), - ...(params.config ? { config: params.config } : {}), - updateMode: params.updateMode ?? "inline", - messages: [ - { - message: resolvedMessage, - idempotencyLookup: "scan", - prepareMessageAfterIdempotencyCheck: (message) => - preparePersistedUserTurnMessageForTranscriptWrite( - message as PersistedUserTurnMessage, - params, - ), - }, - ], - }, - ); - const appended = turn.messages[0] as - | { - messageId: string; - message: PersistedUserTurnMessage; - } - | undefined; - if (!appended) { - return undefined; - } - - return { - sessionFile: params.transcriptPath, - messageId: appended.messageId, - message: appended.message, - }; -} - // Store-backed persistence resolves the current session transcript file lazily // so callers can pass a session entry/store without knowing the final path. export async function persistUserTurnTranscript( @@ -539,6 +479,7 @@ export async function persistUserTurnTranscript( { ...(params.cwd ? { cwd: params.cwd } : {}), ...(params.config ? { config: params.config as OpenClawConfig } : {}), + ...(params.expectedSessionId ? { expectedSessionId: params.expectedSessionId } : {}), updateMode: params.updateMode ?? "inline", messages: [ { @@ -570,46 +511,19 @@ export async function persistUserTurnTranscript( }; } -async function appendFileTargetUserTurnTranscript(params: { - target: UserTurnTranscriptFileTarget; - message: PersistedUserTurnMessage; - updateMode: UserTurnTranscriptUpdateMode; - beforeMessageWrite?: UserTurnBeforeMessageWrite; -}): Promise<UserTurnTranscriptPersistResult | undefined> { - const { config, ...target } = params.target; - const appended = await appendUserTurnTranscriptMessage({ - ...target, - message: params.message, - updateMode: params.updateMode, - ...(config ? { config: config as OpenClawConfig } : {}), - ...(params.beforeMessageWrite ? { beforeMessageWrite: params.beforeMessageWrite } : {}), - }); - return appended - ? { - ...appended, - sessionEntry: undefined, - } - : undefined; -} - async function resolveUserTurnTranscriptTarget( target: UserTurnTranscriptTargetResolver, ): Promise<UserTurnTranscriptTarget | undefined> { return typeof target === "function" ? await target() : target; } -function isUserTurnTranscriptFileTarget( - target: UserTurnTranscriptTarget, -): target is UserTurnTranscriptFileTarget { - return "transcriptPath" in target; -} - export function createUserTurnTranscriptRecorder( params: CreateUserTurnTranscriptRecorderParams, ): UserTurnTranscriptRecorder { const message = resolvePersistedUserTurnMessage(params); let blocked = false; let persisted = false; + let runtimePersisted = false; let persistedResult: UserTurnTranscriptPersistResult | undefined; let runtimePersistencePromise: Promise<void> | undefined; let selfPersistencePromise: Promise<UserTurnTranscriptPersistResult | undefined> | undefined; @@ -689,25 +603,25 @@ export function createUserTurnTranscriptRecorder( const persistPrepared = async (options: { waitForRuntime: boolean; skipWhenBlocked: boolean; + message?: PersistedUserTurnMessage; target?: UserTurnTranscriptTargetResolver; updateMode?: UserTurnTranscriptUpdateMode; + cwd?: string; }): Promise<UserTurnTranscriptPersistResult | undefined> => { if (options.skipWhenBlocked && blocked) { return undefined; } - if (!message && !params.resolveInput) { + if (!options.message && !message && !params.resolveInput) { return undefined; } if (options.waitForRuntime) { - // Approved persistence waits for runtime-owned writes first to avoid - // duplicate user turns when the harness already persisted the message. await waitForRuntimePersistence(); } if (selfPersistencePromise) { return await selfPersistencePromise; } selfPersistencePromise = (async () => { - const resolvedMessage = await resolveMessageForPersistence(); + const resolvedMessage = options.message ?? (await resolveMessageForPersistence()); if (!resolvedMessage) { return undefined; } @@ -715,26 +629,18 @@ export function createUserTurnTranscriptRecorder( if (!target) { return undefined; } + const resolvedTarget = options.cwd ? { ...target, cwd: options.cwd } : target; const updateMode = options.updateMode ?? params.updateMode ?? "inline"; const persistMessage = async ( candidate: PersistedUserTurnMessage, candidateUpdateMode: UserTurnTranscriptUpdateMode, ) => - isUserTurnTranscriptFileTarget(target) - ? await appendFileTargetUserTurnTranscript({ - target, - message: candidate, - updateMode: candidateUpdateMode, - beforeMessageWrite: params.beforeMessageWrite, - }) - : await persistUserTurnTranscript({ - ...target, - message: candidate, - updateMode: candidateUpdateMode, - ...(params.beforeMessageWrite - ? { beforeMessageWrite: params.beforeMessageWrite } - : {}), - }); + await persistUserTurnTranscript({ + ...resolvedTarget, + message: candidate, + updateMode: candidateUpdateMode, + ...(params.beforeMessageWrite ? { beforeMessageWrite: params.beforeMessageWrite } : {}), + }); const lateMediaMessage = sentToProvider && !resolvedBeforeProvider ? buildLateResolvedMediaMessage({ @@ -745,7 +651,7 @@ export function createUserTurnTranscriptRecorder( if (lateMediaMessage) { // The admitted bytes already crossed the LLM boundary. Persisting media as a // second turn preserves that prefix; inline replacement would thrash cache tail (#99495). - if (!persisted && message) { + if (!runtimePersisted && !persisted && message) { const admittedResult = await persistMessage(message, updateMode); if (admittedResult) { persisted = true; @@ -760,6 +666,9 @@ export function createUserTurnTranscriptRecorder( } return appendedMedia; } + if (runtimePersisted) { + return undefined; + } if (persisted) { return persistedResult; } @@ -789,8 +698,8 @@ export function createUserTurnTranscriptRecorder( runtimePersistencePromise = pending; }, markRuntimePersisted: (persistedMessage) => { - persisted = true; runtimePersistedMessage = persistedMessage; + runtimePersisted = true; if (persistedMessage && persistedResult) { persistedResult = { ...persistedResult, @@ -802,7 +711,7 @@ export function createUserTurnTranscriptRecorder( markBlocked: () => { blocked = true; }, - hasPersisted: () => persisted, + hasPersisted: () => persisted || runtimePersisted, isBlocked: () => blocked, hasRuntimePersistencePending: () => runtimePersistencePromise !== undefined, waitForRuntimePersistence, @@ -812,13 +721,26 @@ export function createUserTurnTranscriptRecorder( skipWhenBlocked: true, target: options?.target, updateMode: options?.updateMode, + cwd: options?.cwd, }), + persistBlocked: async (blockedMessage, options) => { + blocked = true; + return await persistPrepared({ + waitForRuntime: false, + skipWhenBlocked: false, + message: blockedMessage, + target: options?.target, + updateMode: options?.updateMode, + cwd: options?.cwd, + }); + }, persistFallback: async (options) => await persistPrepared({ waitForRuntime: true, skipWhenBlocked: true, target: options?.target, updateMode: options?.updateMode, + cwd: options?.cwd, }), }; } diff --git a/src/sessions/user-turn-transcript.types.ts b/src/sessions/user-turn-transcript.types.ts index 98ec205cc0d3..ee51a360ddd4 100644 --- a/src/sessions/user-turn-transcript.types.ts +++ b/src/sessions/user-turn-transcript.types.ts @@ -38,8 +38,9 @@ export type UserTurnBeforeMessageWrite = (params: { sessionKey?: string; }) => AgentMessage | null; -type UserTurnTranscriptPersistenceTarget = { +export type UserTurnTranscriptPersistenceTarget = { sessionId: string; + expectedSessionId?: string; sessionKey: string; sessionEntry: UserTurnSessionEntry | undefined; sessionStore?: Record<string, UserTurnSessionEntry>; @@ -51,18 +52,7 @@ type UserTurnTranscriptPersistenceTarget = { beforeMessageWrite?: UserTurnBeforeMessageWrite; }; -export type UserTurnTranscriptFileTarget = { - transcriptPath: string; - sessionId?: string; - agentId?: string; - sessionKey?: string; - cwd?: string; - config?: unknown; -}; - -export type UserTurnTranscriptTarget = - | UserTurnTranscriptPersistenceTarget - | UserTurnTranscriptFileTarget; +export type UserTurnTranscriptTarget = UserTurnTranscriptPersistenceTarget; export type UserTurnTranscriptPersistResult = { sessionFile: string; @@ -89,9 +79,19 @@ export type UserTurnTranscriptRecorder = { persistApproved: (params?: { target?: UserTurnTranscriptTargetResolver; updateMode?: UserTurnTranscriptUpdateMode; + cwd?: string; }) => Promise<UserTurnTranscriptPersistResult | undefined>; + persistBlocked: ( + message: PersistedUserTurnMessage, + params?: { + target?: UserTurnTranscriptTargetResolver; + updateMode?: UserTurnTranscriptUpdateMode; + cwd?: string; + }, + ) => Promise<UserTurnTranscriptPersistResult | undefined>; persistFallback: (params?: { target?: UserTurnTranscriptTargetResolver; updateMode?: UserTurnTranscriptUpdateMode; + cwd?: string; }) => Promise<UserTurnTranscriptPersistResult | undefined>; }; diff --git a/src/state/openclaw-agent-db.generated.d.ts b/src/state/openclaw-agent-db.generated.d.ts index 38346e6e0079..60d745cbf666 100644 --- a/src/state/openclaw-agent-db.generated.d.ts +++ b/src/state/openclaw-agent-db.generated.d.ts @@ -31,6 +31,22 @@ export interface CacheEntries { value_json: string | null; } +export interface Conversations { + account_id: string; + channel: string; + conversation_id: string; + created_at: number; + kind: string; + label: string | null; + metadata_json: string | null; + native_channel_id: string | null; + native_direct_user_id: string | null; + parent_conversation_id: string | null; + peer_id: string; + thread_id: string | null; + updated_at: number; +} + export interface MemoryEmbeddingCache { dims: number | null; embedding: string; @@ -83,14 +99,89 @@ export interface SchemaMeta { updated_at: number; } +export interface SessionConversations { + conversation_id: string; + first_seen_at: number; + last_seen_at: number; + role: Generated<string>; + session_id: string; +} + +export interface SessionEntries { + entry_json: string; + session_id: string; + session_key: string; + updated_at: number; +} + +export interface SessionRoutes { + session_id: string; + session_key: string; + updated_at: number; +} + +export interface Sessions { + account_id: string | null; + agent_harness_id: string | null; + channel: string | null; + chat_type: string | null; + created_at: number; + display_name: string | null; + ended_at: number | null; + model: string | null; + model_provider: string | null; + parent_session_key: string | null; + primary_conversation_id: string | null; + session_id: string; + session_key: string; + session_scope: Generated<string>; + spawned_by: string | null; + started_at: number | null; + status: string | null; + updated_at: number; +} + +export interface TrajectoryRuntimeEvents { + created_at: number; + event_json: string; + run_id: string | null; + seq: number; + session_id: string; +} + +export interface TranscriptEventIdentities { + created_at: number; + event_id: string; + event_type: string | null; + message_idempotency_key: string | null; + parent_id: string | null; + seq: number; + session_id: string; +} + +export interface TranscriptEvents { + created_at: number; + event_json: string; + seq: number; + session_id: string; +} + export interface DB { auth_profile_state: AuthProfileState; auth_profile_store: AuthProfileStore; cache_entries: CacheEntries; + conversations: Conversations; memory_embedding_cache: MemoryEmbeddingCache; memory_index_chunks: MemoryIndexChunks; memory_index_meta: MemoryIndexMeta; memory_index_sources: MemoryIndexSources; memory_index_state: MemoryIndexState; schema_meta: SchemaMeta; + session_conversations: SessionConversations; + session_entries: SessionEntries; + session_routes: SessionRoutes; + sessions: Sessions; + trajectory_runtime_events: TrajectoryRuntimeEvents; + transcript_event_identities: TranscriptEventIdentities; + transcript_events: TranscriptEvents; } diff --git a/src/state/openclaw-agent-db.test.ts b/src/state/openclaw-agent-db.test.ts index 62a74d1d3aa5..9fca923f0a3c 100644 --- a/src/state/openclaw-agent-db.test.ts +++ b/src/state/openclaw-agent-db.test.ts @@ -3,7 +3,7 @@ import { execFileSync, spawn } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterAll, afterEach, describe, expect, it } from "vitest"; +import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js"; import { executeSqliteQuerySync, @@ -26,6 +26,7 @@ import { } from "./openclaw-agent-db.js"; import { closeOpenClawStateDatabaseForTest, + OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, openOpenClawStateDatabase, } from "./openclaw-state-db.js"; import { @@ -44,6 +45,17 @@ function createTempStateDir(): string { return makeTempDir(agentDbTempDirs, "openclaw-agent-db-"); } +function readRegisteredAgentDatabaseLastSeenAt(params: { + agentId: string; + env?: NodeJS.ProcessEnv; + path: string; +}): number | undefined { + const row = openOpenClawStateDatabase({ env: params.env }) + .db.prepare("SELECT last_seen_at FROM agent_databases WHERE agent_id = ? AND path = ?") + .get(params.agentId, params.path) as { last_seen_at?: unknown } | undefined; + return typeof row?.last_seen_at === "number" ? row.last_seen_at : undefined; +} + function seedVersion1MemoryAgentDatabase( databasePath: string, options: { malformedPathFts?: boolean } = {}, @@ -574,6 +586,43 @@ describe("openclaw agent database", () => { ).toEqual([defaultDatabase.path, relocated.path].toSorted()); }); + it("does not refresh global registry metadata on cached opens", () => { + const stateDir = createTempStateDir(); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(1_000); + + try { + const first = openOpenClawAgentDatabase({ + agentId: "worker-1", + env, + }); + expect( + readRegisteredAgentDatabaseLastSeenAt({ + agentId: "worker-1", + env, + path: first.path, + }), + ).toBe(1_000); + + nowSpy.mockReturnValue(2_000); + const second = openOpenClawAgentDatabase({ + agentId: "worker-1", + env, + }); + + expect(second).toBe(first); + expect( + readRegisteredAgentDatabaseLastSeenAt({ + agentId: "worker-1", + env, + path: first.path, + }), + ).toBe(1_000); + } finally { + nowSpy.mockRestore(); + } + }); + it("rejects the legacy agent registry primary key with a doctor repair hint", () => { const stateDir = createTempStateDir(); const env = { OPENCLAW_STATE_DIR: stateDir }; @@ -924,7 +973,9 @@ describe("openclaw agent database", () => { env: { OPENCLAW_STATE_DIR: stateDir }, }); - expect(readSqliteNumberPragma(database.db, "busy_timeout")).toBe(30_000); + expect(readSqliteNumberPragma(database.db, "busy_timeout")).toBe( + OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + ); expect(readSqliteNumberPragma(database.db, "foreign_keys")).toBe(1); expect(readSqliteNumberPragma(database.db, "synchronous")).toBe(1); expect(readSqliteNumberPragma(database.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION); @@ -970,6 +1021,166 @@ describe("openclaw agent database", () => { }); }); + it("migrates compact v1 session tables before applying normalized indexes", () => { + const stateDir = createTempStateDir(); + const databasePath = path.join( + stateDir, + "agents", + "worker-1", + "agent", + "openclaw-agent.sqlite", + ); + fs.mkdirSync(path.dirname(databasePath), { recursive: true }); + const { DatabaseSync } = requireNodeSqlite(); + const db = new DatabaseSync(databasePath); + db.exec(` + CREATE TABLE schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL, + schema_version INTEGER NOT NULL, + agent_id TEXT, + app_version TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + INSERT INTO schema_meta + (meta_key, role, schema_version, agent_id, app_version, created_at, updated_at) + VALUES ('primary', 'agent', 1, 'worker-1', NULL, 1, 1); + CREATE TABLE sessions ( + session_id TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + INSERT INTO sessions (session_id, session_key, created_at, updated_at) + VALUES ('session-1', 'agent:worker-1:main', 10, 20); + CREATE TABLE session_entries ( + session_key TEXT NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + ); + INSERT INTO session_entries (session_key, session_id, entry_json, updated_at) + VALUES ( + 'agent:worker-1:group:example', + 'session-1', + '{"sessionId":"session-1","updatedAt":20,"startedAt":11,"endedAt":19,"status":"done","chatType":"group","channel":"discord","deliveryContext":{"accountId":"acct-1"},"modelProvider":"openai","model":"gpt-5.5","agentHarnessId":"codex","parentSessionKey":"agent:worker-1:parent","spawnedBy":"agent:worker-1:spawner","displayName":"Example group"}', + 20 + ); + CREATE TABLE memory_index_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + revision INTEGER NOT NULL + ); + INSERT INTO memory_index_state (id, revision) VALUES (1, 1); + CREATE TABLE memory_index_sources ( + source_kind TEXT NOT NULL DEFAULT 'memory', + source_key TEXT NOT NULL, + path TEXT, + session_id TEXT, + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (source_kind, source_key), + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + ); + CREATE TABLE memory_index_chunks ( + id TEXT PRIMARY KEY, + source_kind TEXT NOT NULL DEFAULT 'memory', + source_key TEXT NOT NULL, + path TEXT NOT NULL, + session_id TEXT, + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding BLOB NOT NULL, + embedding_dims INTEGER, + updated_at INTEGER NOT NULL, + FOREIGN KEY (source_kind, source_key) + REFERENCES memory_index_sources(source_kind, source_key) ON DELETE CASCADE, + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + ); + PRAGMA user_version = 1; + `); + db.close(); + + const database = openOpenClawAgentDatabase({ + agentId: "worker-1", + env: { OPENCLAW_STATE_DIR: stateDir }, + }); + + expect(readSqliteNumberPragma(database.db, "user_version")).toBe(4); + const session = database.db + .prepare( + ` + SELECT + account_id, + agent_harness_id, + channel, + chat_type, + display_name, + ended_at, + model, + model_provider, + parent_session_key, + session_scope, + spawned_by, + started_at, + status + FROM sessions + WHERE session_id = ? + `, + ) + .get("session-1"); + expect(session).toEqual({ + account_id: "acct-1", + agent_harness_id: "codex", + channel: "discord", + chat_type: "group", + display_name: "Example group", + ended_at: 19, + model: "gpt-5.5", + model_provider: "openai", + parent_session_key: "agent:worker-1:parent", + session_scope: "group", + spawned_by: "agent:worker-1:spawner", + started_at: 11, + status: "done", + }); + const route = database.db + .prepare("SELECT session_id, updated_at FROM session_routes WHERE session_key = ?") + .get("agent:worker-1:group:example"); + expect(route).toEqual({ + session_id: "session-1", + updated_at: 20, + }); + const sessionForeignKeys = database.db.prepare("PRAGMA foreign_key_list(sessions)").all() as + | Array<{ from?: unknown; on_delete?: unknown; table?: unknown; to?: unknown }> + | undefined; + expect(sessionForeignKeys).toContainEqual( + expect.objectContaining({ + from: "primary_conversation_id", + on_delete: "SET NULL", + table: "conversations", + to: "conversation_id", + }), + ); + const memoryIndexSourceColumns = database.db + .prepare("PRAGMA table_info(memory_index_sources)") + .all() as Array<{ name?: unknown }>; + // Canonical memory-source identity keeps stable integer ids so FTS rowids + // survive VACUUM (main's v2 shape, folded into the flip schema). + expect(memoryIndexSourceColumns.map((column) => column.name)).toEqual([ + "id", + "path", + "source", + "hash", + "mtime", + "size", + ]); + }); it("refuses to open newer per-agent schema versions", () => { const stateDir = createTempStateDir(); const databasePath = path.join( diff --git a/src/state/openclaw-agent-db.ts b/src/state/openclaw-agent-db.ts index 3d5965041d2d..5db3de39c058 100644 --- a/src/state/openclaw-agent-db.ts +++ b/src/state/openclaw-agent-db.ts @@ -10,13 +10,17 @@ import { } from "../infra/kysely-sync.js"; import { requireNodeSqlite } from "../infra/node-sqlite.js"; import { resolveSqliteDatabaseFilePaths } from "../infra/sqlite-files.js"; -import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js"; +import { + runSqliteImmediateTransactionSync, + type SqliteTransactionOptions, +} from "../infra/sqlite-transaction.js"; import { readSqliteUserVersion } from "../infra/sqlite-user-version.js"; import { configureSqliteConnectionPragmas, registerSqliteCacheExitClose, type SqliteWalMaintenance, } from "../infra/sqlite-wal.js"; +import { createSubsystemLogger } from "../logging/subsystem.js"; import { normalizeAgentId } from "../routing/session-key.js"; import type { DB as OpenClawAgentKyselyDatabase } from "./openclaw-agent-db.generated.js"; import { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js"; @@ -39,9 +43,14 @@ export { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js"; * per pathname, protected with private file modes, and registered in the shared * OpenClaw state database for discovery and maintenance. */ -export const OPENCLAW_AGENT_SCHEMA_VERSION = 2; +// v4 = session/transcript flip (branch lineage). Main's v2 memory-identity +// change is folded in structure-gated (migrateMemoryIndexSourcesIdentity), so +// v2 main DBs and pre-merge v4 flip DBs both converge on this schema. +export const OPENCLAW_AGENT_SCHEMA_VERSION = 4; const OPENCLAW_AGENT_DB_DIR_MODE = 0o700; const OPENCLAW_AGENT_DB_FILE_MODE = 0o600; +const OPENCLAW_AGENT_DB_SLOW_OPEN_MS = 1_000; +const agentDbLog = createSubsystemLogger("state/agent-db"); /** Open per-agent SQLite database handle plus lifecycle maintenance. */ export type OpenClawAgentDatabase = { @@ -69,12 +78,31 @@ type OpenClawAgentMetadataDatabase = Pick<OpenClawAgentKyselyDatabase, "schema_m type OpenClawAgentRegistryDatabase = Pick<OpenClawStateKyselyDatabase, "agent_databases">; const cachedDatabases = new Map<string, OpenClawAgentDatabase>(); +const registeredDatabasePaths = new Set<string>(); type ExistingSchemaMeta = { agentId: string | null; role: string | null; }; +type MigratedSessionEntry = Record<string, unknown>; + +function logSlowAgentDatabaseOpen(params: { + agentId: string; + elapsedMs: number; + path: string; +}): void { + if (params.elapsedMs < OPENCLAW_AGENT_DB_SLOW_OPEN_MS) { + return; + } + agentDbLog.warn("slow OpenClaw agent database open", { + agentId: params.agentId, + elapsedMs: params.elapsedMs, + path: params.path, + thresholdMs: OPENCLAW_AGENT_DB_SLOW_OPEN_MS, + }); +} + function assertSupportedAgentSchemaVersion(db: DatabaseSync, pathname: string): void { const userVersion = readSqliteUserVersion(db); if (userVersion > OPENCLAW_AGENT_SCHEMA_VERSION) { @@ -84,6 +112,308 @@ function assertSupportedAgentSchemaVersion(db: DatabaseSync, pathname: string): } } +function readSqliteSessionColumns(db: DatabaseSync): Set<string> | null { + const table = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get("sessions"); + if (!table) { + return null; + } + const rows = db.prepare("PRAGMA table_info(sessions)").all() as Array<{ + name?: unknown; + }>; + return new Set(rows.flatMap((row) => (typeof row.name === "string" ? [row.name] : []))); +} + +function migratedSessionColumn( + columns: ReadonlySet<string>, + columnName: string, + fallback: string, +): string { + return columns.has(columnName) ? columnName : fallback; +} + +function dropLegacyMemoryIndexSchema(db: DatabaseSync): void { + const columns = db.prepare("PRAGMA table_info(memory_index_sources)").all() as Array<{ + name?: unknown; + }>; + const hasLegacySourceColumns = columns.some((row) => row.name === "source_kind"); + if (!hasLegacySourceColumns) { + return; + } + // Memory indexes are derived cache data; v1 used a different key shape. + db.exec(` + DROP TABLE IF EXISTS memory_index_chunks_fts; + DROP TABLE IF EXISTS memory_index_chunks; + DROP TABLE IF EXISTS memory_index_sources; + `); +} + +function migrateOpenClawAgentSchema(db: DatabaseSync): void { + const userVersion = readSqliteUserVersion(db); + if (userVersion >= OPENCLAW_AGENT_SCHEMA_VERSION) { + return; + } + if (userVersion < 3) { + db.exec("DROP INDEX IF EXISTS idx_agent_transcript_events_session;"); + } + const columns = readSqliteSessionColumns(db); + if (userVersion > 1 || !columns) { + return; + } + const copyColumns = [ + "session_id", + "session_key", + "session_scope", + "created_at", + "updated_at", + "started_at", + "ended_at", + "status", + "chat_type", + "channel", + "account_id", + "primary_conversation_id", + "model_provider", + "model", + "agent_harness_id", + "parent_session_key", + "spawned_by", + "display_name", + ]; + const selectColumns = [ + "session_id", + "session_key", + migratedSessionColumn(columns, "session_scope", "'conversation'"), + "created_at", + "updated_at", + migratedSessionColumn(columns, "started_at", "NULL"), + migratedSessionColumn(columns, "ended_at", "NULL"), + migratedSessionColumn(columns, "status", "NULL"), + migratedSessionColumn(columns, "chat_type", "NULL"), + migratedSessionColumn(columns, "channel", "NULL"), + migratedSessionColumn(columns, "account_id", "NULL"), + migratedSessionColumn(columns, "primary_conversation_id", "NULL"), + migratedSessionColumn(columns, "model_provider", "NULL"), + migratedSessionColumn(columns, "model", "NULL"), + migratedSessionColumn(columns, "agent_harness_id", "NULL"), + migratedSessionColumn(columns, "parent_session_key", "NULL"), + migratedSessionColumn(columns, "spawned_by", "NULL"), + migratedSessionColumn(columns, "display_name", "NULL"), + ]; + db.exec(` + CREATE TABLE IF NOT EXISTS conversations ( + conversation_id TEXT NOT NULL PRIMARY KEY, + channel TEXT NOT NULL, + account_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('direct', 'group', 'channel')), + peer_id TEXT NOT NULL, + parent_conversation_id TEXT, + thread_id TEXT, + native_channel_id TEXT, + native_direct_user_id TEXT, + label TEXT, + metadata_json TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + `); + db.exec(` + DROP TABLE IF EXISTS sessions_new; + CREATE TABLE sessions_new ( + session_id TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + session_scope TEXT NOT NULL DEFAULT 'conversation' CHECK (session_scope IN ('conversation', 'shared-main', 'group', 'channel')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + started_at INTEGER, + ended_at INTEGER, + status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), + chat_type TEXT CHECK (chat_type IS NULL OR chat_type IN ('direct', 'group', 'channel')), + channel TEXT, + account_id TEXT, + primary_conversation_id TEXT, + model_provider TEXT, + model TEXT, + agent_harness_id TEXT, + parent_session_key TEXT, + spawned_by TEXT, + display_name TEXT, + FOREIGN KEY (primary_conversation_id) REFERENCES conversations(conversation_id) ON DELETE SET NULL + ); + INSERT INTO sessions_new (${copyColumns.join(", ")}) + SELECT ${selectColumns.join(", ")} FROM sessions; + DROP TABLE sessions; + ALTER TABLE sessions_new RENAME TO sessions; + `); +} + +function parseMigratedSessionEntry(value: unknown): MigratedSessionEntry | null { + if (typeof value !== "string") { + return null; + } + try { + const parsed = JSON.parse(value) as unknown; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as MigratedSessionEntry) + : null; + } catch { + return null; + } +} + +function migratedObjectField( + entry: MigratedSessionEntry, + key: string, +): MigratedSessionEntry | null { + const value = entry[key]; + return value && typeof value === "object" && !Array.isArray(value) + ? (value as MigratedSessionEntry) + : null; +} + +function migratedText(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function migratedNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function migratedChatType(value: unknown): "direct" | "group" | "channel" | null { + if (value === "direct" || value === "group" || value === "channel") { + return value; + } + return null; +} + +function migratedStatus( + value: unknown, +): "running" | "done" | "failed" | "killed" | "timeout" | null { + if ( + value === "running" || + value === "done" || + value === "failed" || + value === "killed" || + value === "timeout" + ) { + return value; + } + return null; +} + +function migratedSessionScope( + entry: MigratedSessionEntry, + sessionKey: string, +): "conversation" | "shared-main" | "group" | "channel" { + const chatType = migratedChatType(entry.chatType); + const normalizedKey = sessionKey.trim().toLowerCase(); + if (chatType === "direct" && (normalizedKey === "main" || normalizedKey.endsWith(":main"))) { + return "shared-main"; + } + if (chatType === "group" || chatType === "channel") { + return chatType; + } + return "conversation"; +} + +function migratedEntryChannel(entry: MigratedSessionEntry): string | null { + const deliveryContext = migratedObjectField(entry, "deliveryContext"); + const origin = migratedObjectField(entry, "origin"); + return ( + migratedText(entry.channel) ?? + migratedText(deliveryContext?.channel) ?? + migratedText(entry.lastChannel) ?? + migratedText(origin?.provider) + ); +} + +function migratedEntryAccountId(entry: MigratedSessionEntry): string | null { + const deliveryContext = migratedObjectField(entry, "deliveryContext"); + const origin = migratedObjectField(entry, "origin"); + return ( + migratedText(deliveryContext?.accountId) ?? + migratedText(entry.lastAccountId) ?? + migratedText(origin?.accountId) + ); +} + +function migratedEntryDisplayName(entry: MigratedSessionEntry): string | null { + return ( + migratedText(entry.displayName) ?? + migratedText(entry.label) ?? + migratedText(entry.subject) ?? + migratedText(entry.groupId) + ); +} + +function backfillOpenClawAgentSchema(db: DatabaseSync, previousVersion: number): void { + if (previousVersion >= 2) { + return; + } + db.exec(` + INSERT OR REPLACE INTO session_routes (session_key, session_id, updated_at) + SELECT se.session_key, se.session_id, se.updated_at + FROM session_entries AS se + INNER JOIN sessions AS s ON s.session_id = se.session_id; + `); + const rows = db + .prepare( + ` + SELECT se.session_key, se.session_id, se.entry_json + FROM session_entries AS se + INNER JOIN sessions AS s ON s.session_id = se.session_id; + `, + ) + .all() as Array<{ + entry_json?: unknown; + session_id?: unknown; + session_key?: unknown; + }>; + const update = db.prepare(` + UPDATE sessions + SET + session_scope = ?, + started_at = ?, + ended_at = ?, + status = ?, + chat_type = ?, + channel = ?, + account_id = ?, + model_provider = ?, + model = ?, + agent_harness_id = ?, + parent_session_key = ?, + spawned_by = ?, + display_name = ? + WHERE session_id = ?; + `); + for (const row of rows) { + const sessionKey = migratedText(row.session_key); + const sessionId = migratedText(row.session_id); + const entry = parseMigratedSessionEntry(row.entry_json); + if (!sessionKey || !sessionId || !entry) { + continue; + } + update.run( + migratedSessionScope(entry, sessionKey), + migratedNumber(entry.startedAt), + migratedNumber(entry.endedAt), + migratedStatus(entry.status), + migratedChatType(entry.chatType), + migratedEntryChannel(entry), + migratedEntryAccountId(entry), + migratedText(entry.modelProvider), + migratedText(entry.model), + migratedText(entry.agentHarnessId), + migratedText(entry.parentSessionKey), + migratedText(entry.spawnedBy), + migratedEntryDisplayName(entry), + sessionId, + ); + } +} + function ensureOpenClawAgentDatabasePermissions( pathname: string, options: OpenClawAgentDatabaseOptions, @@ -151,44 +481,72 @@ function assertExistingSchemaOwner( } function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string): void { - runSqliteImmediateTransactionSync(db, () => { - // Ownership and version checks must share the write transaction with the - // schema update; concurrent openers must not overwrite another agent. - // Role/ownership gates before version: user_version is only meaningful - // within one schema role, and the global state DB now carries version 2. - assertExistingSchemaOwner(readExistingSchemaMeta(db), agentId, pathname); - assertSupportedAgentSchemaVersion(db, pathname); - // Version 1 keyed sources by path/source. Stable IDs keep FTS rowids valid - // across VACUUM and make update/delete trigger lookups constant-time. - migrateMemoryIndexSourcesIdentity(db); - db.exec(OPENCLAW_AGENT_SCHEMA_SQL); - const kysely = getNodeSqliteKysely<OpenClawAgentMetadataDatabase>(db); - db.exec(`PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION};`); - const now = Date.now(); - executeSqliteQuerySync( - db, - kysely - .insertInto("schema_meta") - .values({ - meta_key: "primary", - role: "agent", - schema_version: OPENCLAW_AGENT_SCHEMA_VERSION, - agent_id: agentId, - app_version: null, - created_at: now, - updated_at: now, - }) - .onConflict((conflict) => - conflict.column("meta_key").doUpdateSet({ + // FK enforcement must be off before BEGIN: PRAGMA foreign_keys is a silent + // no-op inside a transaction, and the v1 sessions rebuild would otherwise + // cascade-delete session_entries when the old parent table drops. The + // connection pragmas restore enforcement for steady-state work below. + db.exec("PRAGMA foreign_keys = OFF;"); + try { + runSqliteImmediateTransactionSync(db, () => { + // Ownership and version checks must share the write transaction with the + // schema update; concurrent openers must not overwrite another agent. + // Role/ownership gates before version: user_version is only meaningful + // within one schema role, and the global state DB now carries version 2. + assertExistingSchemaOwner(readExistingSchemaMeta(db), agentId, pathname); + assertSupportedAgentSchemaVersion(db, pathname); + const previousVersion = readSqliteUserVersion(db); + // Two legacy memory shapes exist: the flip lineage's source_kind schema + // (derived cache — dropped for rebuild) and main's path/source-keyed + // schema (migrated in place by the identity migration). Both helpers are + // structure-gated, so this ordering converges every lineage — pre-flip + // v1/v2 and pre-merge flip v1/v4 — without version-number coupling. + dropLegacyMemoryIndexSchema(db); + migrateMemoryIndexSourcesIdentity(db); + migrateOpenClawAgentSchema(db); + db.exec(OPENCLAW_AGENT_SCHEMA_SQL); + backfillOpenClawAgentSchema(db, previousVersion); + const kysely = getNodeSqliteKysely<OpenClawAgentMetadataDatabase>(db); + db.exec(`PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION};`); + const now = Date.now(); + executeSqliteQuerySync( + db, + kysely + .insertInto("schema_meta") + .values({ + meta_key: "primary", role: "agent", schema_version: OPENCLAW_AGENT_SCHEMA_VERSION, agent_id: agentId, app_version: null, + created_at: now, updated_at: now, - }), - ), - ); - }); + }) + .onConflict((conflict) => + conflict.column("meta_key").doUpdateSet({ + role: "agent", + schema_version: OPENCLAW_AGENT_SCHEMA_VERSION, + agent_id: agentId, + app_version: null, + updated_at: now, + }), + ), + ); + }); + } finally { + db.exec("PRAGMA foreign_keys = ON;"); + } +} + +// auto_vacuum only takes effect when set before the first page is written; +// existing databases keep their mode until a doctor-owned full VACUUM. +// INCREMENTAL lets maintenance release freed pages in bounded steps so +// per-agent DBs shrink after retention deletes rows instead of pinning +// their high-water mark forever. +function enableIncrementalAutoVacuumForFreshDatabase(db: DatabaseSync): void { + const row = db.prepare("PRAGMA page_count").get() as { page_count?: unknown } | undefined; + if (row?.page_count === 0) { + db.exec("PRAGMA auto_vacuum = INCREMENTAL;"); + } } /** Initialize agent schema/ownership metadata on an independently managed connection. */ @@ -200,6 +558,7 @@ export function ensureOpenClawAgentDatabaseSchema( const databaseOptions = { ...options, agentId }; const pathname = resolveOpenClawAgentSqlitePath(databaseOptions); ensureOpenClawAgentDatabasePermissions(pathname, databaseOptions); + enableIncrementalAutoVacuumForFreshDatabase(db); ensureAgentSchema(db, agentId, pathname); ensureOpenClawAgentDatabasePermissions(pathname, databaseOptions); if (options.register === true) { @@ -406,7 +765,6 @@ export function openOpenClawAgentDatabase( `OpenClaw agent database ${pathname} is already open for agent ${cached.agentId}; requested agent ${agentId}.`, ); } - registerAgentDatabase({ agentId, path: pathname, env: options.env }); return cached; } if (cached) { @@ -416,12 +774,14 @@ export function openOpenClawAgentDatabase( cachedDatabases.delete(pathname); } + const openStartedAt = Date.now(); ensureOpenClawAgentDatabasePermissions(pathname, databaseOptions); const sqlite = requireNodeSqlite(); const db = new sqlite.DatabaseSync(pathname); const walMaintenance = (() => { let maintenance: SqliteWalMaintenance | undefined; try { + enableIncrementalAutoVacuumForFreshDatabase(db); maintenance = configureSqliteConnectionPragmas(db, { busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, databaseLabel: `openclaw-agent:${agentId}`, @@ -443,7 +803,15 @@ export function openOpenClawAgentDatabase( // Safety net for processes that end without an orderly close: agent DBs have // no shutdown owner like the ACP/gateway state DB closes. Closing unregisters. unregisterExitClose ??= registerSqliteCacheExitClose(closeOpenClawAgentDatabases); - registerAgentDatabase({ agentId, path: pathname, env: options.env }); + if (!registeredDatabasePaths.has(pathname)) { + registerAgentDatabase({ agentId, path: pathname, env: options.env }); + registeredDatabasePaths.add(pathname); + } + logSlowAgentDatabaseOpen({ + agentId, + elapsedMs: Date.now() - openStartedAt, + path: pathname, + }); return database; } @@ -451,9 +819,18 @@ export function openOpenClawAgentDatabase( export function runOpenClawAgentWriteTransaction<T>( operation: (database: OpenClawAgentDatabase) => T, options: OpenClawAgentDatabaseOptions, + transactionOptions: Pick< + SqliteTransactionOptions, + "operationLabel" | "slowTransactionHoldMs" + > = {}, ): T { const database = openOpenClawAgentDatabase(options); - const result = runSqliteImmediateTransactionSync(database.db, () => operation(database)); + const result = runSqliteImmediateTransactionSync(database.db, () => operation(database), { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: database.path, + ...transactionOptions, + operationLabel: transactionOptions.operationLabel ?? "agent.write", + }); ensureOpenClawAgentDatabasePermissions(database.path, options); return result; } @@ -520,6 +897,7 @@ export function closeOpenClawAgentDatabases(): void { closeCachedOpenClawAgentDatabase(database); } cachedDatabases.clear(); + registeredDatabasePaths.clear(); } /** Test alias for closing cached agent database handles from teardown code. */ diff --git a/src/state/openclaw-agent-schema.generated.ts b/src/state/openclaw-agent-schema.generated.ts index 868134631989..829f5e65e5a8 100644 --- a/src/state/openclaw-agent-schema.generated.ts +++ b/src/state/openclaw-agent-schema.generated.ts @@ -13,6 +13,155 @@ export const OPENCLAW_AGENT_SCHEMA_SQL = `CREATE TABLE IF NOT EXISTS schema_meta updated_at INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS sessions ( + session_id TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + session_scope TEXT NOT NULL DEFAULT 'conversation' CHECK (session_scope IN ('conversation', 'shared-main', 'group', 'channel')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + started_at INTEGER, + ended_at INTEGER, + status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), + chat_type TEXT CHECK (chat_type IS NULL OR chat_type IN ('direct', 'group', 'channel')), + channel TEXT, + account_id TEXT, + primary_conversation_id TEXT, + model_provider TEXT, + model TEXT, + agent_harness_id TEXT, + parent_session_key TEXT, + spawned_by TEXT, + display_name TEXT, + FOREIGN KEY (primary_conversation_id) REFERENCES conversations(conversation_id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_agent_sessions_updated_at + ON sessions(updated_at DESC, session_id); + +CREATE INDEX IF NOT EXISTS idx_agent_sessions_created_at + ON sessions(created_at DESC, session_id); + +CREATE INDEX IF NOT EXISTS idx_agent_sessions_conversation + ON sessions(primary_conversation_id, updated_at DESC, session_id) + WHERE primary_conversation_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS session_routes ( + session_key TEXT NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_agent_session_routes_session_id + ON session_routes(session_id); + +CREATE TABLE IF NOT EXISTS conversations ( + conversation_id TEXT NOT NULL PRIMARY KEY, + channel TEXT NOT NULL, + account_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('direct', 'group', 'channel')), + peer_id TEXT NOT NULL, + parent_conversation_id TEXT, + thread_id TEXT, + native_channel_id TEXT, + native_direct_user_id TEXT, + label TEXT, + metadata_json TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_agent_conversations_lookup + ON conversations(channel, account_id, kind, peer_id, thread_id); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_conversations_identity + ON conversations( + channel, + account_id, + kind, + peer_id, + IFNULL(parent_conversation_id, ''), + IFNULL(thread_id, '') + ); + +CREATE INDEX IF NOT EXISTS idx_agent_conversations_updated + ON conversations(updated_at DESC, conversation_id); + +CREATE TABLE IF NOT EXISTS session_conversations ( + session_id TEXT NOT NULL, + conversation_id TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'primary' CHECK (role IN ('primary', 'participant', 'related')), + first_seen_at INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL, + PRIMARY KEY (session_id, conversation_id, role), + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE, + FOREIGN KEY (conversation_id) REFERENCES conversations(conversation_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_agent_session_conversations_conversation + ON session_conversations(conversation_id, last_seen_at DESC, session_id); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_session_conversations_primary + ON session_conversations(session_id) + WHERE role = 'primary'; + +CREATE TABLE IF NOT EXISTS session_entries ( + session_key TEXT NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_agent_session_entries_updated_at + ON session_entries(updated_at DESC, session_key); + +CREATE INDEX IF NOT EXISTS idx_agent_session_entries_session_id + ON session_entries(session_id); + +CREATE TABLE IF NOT EXISTS transcript_events ( + session_id TEXT NOT NULL, + seq INTEGER NOT NULL, + event_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, seq), + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS trajectory_runtime_events ( + session_id TEXT NOT NULL, + seq INTEGER NOT NULL, + run_id TEXT, + event_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, seq), + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_agent_trajectory_runtime_run + ON trajectory_runtime_events(session_id, run_id, seq) + WHERE run_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS transcript_event_identities ( + session_id TEXT NOT NULL, + event_id TEXT NOT NULL, + seq INTEGER NOT NULL, + event_type TEXT, + parent_id TEXT, + message_idempotency_key TEXT, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, event_id), + FOREIGN KEY (session_id, seq) REFERENCES transcript_events(session_id, seq) ON DELETE CASCADE +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_transcript_message_idempotency + ON transcript_event_identities(session_id, message_idempotency_key) + WHERE message_idempotency_key IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_agent_transcript_event_parent + ON transcript_event_identities(session_id, parent_id) + WHERE parent_id IS NOT NULL; + CREATE TABLE IF NOT EXISTS cache_entries ( scope TEXT NOT NULL, key TEXT NOT NULL, diff --git a/src/state/openclaw-agent-schema.sql b/src/state/openclaw-agent-schema.sql index 1425d947fec1..78d3d14a4c45 100644 --- a/src/state/openclaw-agent-schema.sql +++ b/src/state/openclaw-agent-schema.sql @@ -8,6 +8,155 @@ CREATE TABLE IF NOT EXISTS schema_meta ( updated_at INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS sessions ( + session_id TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + session_scope TEXT NOT NULL DEFAULT 'conversation' CHECK (session_scope IN ('conversation', 'shared-main', 'group', 'channel')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + started_at INTEGER, + ended_at INTEGER, + status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), + chat_type TEXT CHECK (chat_type IS NULL OR chat_type IN ('direct', 'group', 'channel')), + channel TEXT, + account_id TEXT, + primary_conversation_id TEXT, + model_provider TEXT, + model TEXT, + agent_harness_id TEXT, + parent_session_key TEXT, + spawned_by TEXT, + display_name TEXT, + FOREIGN KEY (primary_conversation_id) REFERENCES conversations(conversation_id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_agent_sessions_updated_at + ON sessions(updated_at DESC, session_id); + +CREATE INDEX IF NOT EXISTS idx_agent_sessions_created_at + ON sessions(created_at DESC, session_id); + +CREATE INDEX IF NOT EXISTS idx_agent_sessions_conversation + ON sessions(primary_conversation_id, updated_at DESC, session_id) + WHERE primary_conversation_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS session_routes ( + session_key TEXT NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_agent_session_routes_session_id + ON session_routes(session_id); + +CREATE TABLE IF NOT EXISTS conversations ( + conversation_id TEXT NOT NULL PRIMARY KEY, + channel TEXT NOT NULL, + account_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('direct', 'group', 'channel')), + peer_id TEXT NOT NULL, + parent_conversation_id TEXT, + thread_id TEXT, + native_channel_id TEXT, + native_direct_user_id TEXT, + label TEXT, + metadata_json TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_agent_conversations_lookup + ON conversations(channel, account_id, kind, peer_id, thread_id); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_conversations_identity + ON conversations( + channel, + account_id, + kind, + peer_id, + IFNULL(parent_conversation_id, ''), + IFNULL(thread_id, '') + ); + +CREATE INDEX IF NOT EXISTS idx_agent_conversations_updated + ON conversations(updated_at DESC, conversation_id); + +CREATE TABLE IF NOT EXISTS session_conversations ( + session_id TEXT NOT NULL, + conversation_id TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'primary' CHECK (role IN ('primary', 'participant', 'related')), + first_seen_at INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL, + PRIMARY KEY (session_id, conversation_id, role), + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE, + FOREIGN KEY (conversation_id) REFERENCES conversations(conversation_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_agent_session_conversations_conversation + ON session_conversations(conversation_id, last_seen_at DESC, session_id); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_session_conversations_primary + ON session_conversations(session_id) + WHERE role = 'primary'; + +CREATE TABLE IF NOT EXISTS session_entries ( + session_key TEXT NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_agent_session_entries_updated_at + ON session_entries(updated_at DESC, session_key); + +CREATE INDEX IF NOT EXISTS idx_agent_session_entries_session_id + ON session_entries(session_id); + +CREATE TABLE IF NOT EXISTS transcript_events ( + session_id TEXT NOT NULL, + seq INTEGER NOT NULL, + event_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, seq), + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS trajectory_runtime_events ( + session_id TEXT NOT NULL, + seq INTEGER NOT NULL, + run_id TEXT, + event_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, seq), + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_agent_trajectory_runtime_run + ON trajectory_runtime_events(session_id, run_id, seq) + WHERE run_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS transcript_event_identities ( + session_id TEXT NOT NULL, + event_id TEXT NOT NULL, + seq INTEGER NOT NULL, + event_type TEXT, + parent_id TEXT, + message_idempotency_key TEXT, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, event_id), + FOREIGN KEY (session_id, seq) REFERENCES transcript_events(session_id, seq) ON DELETE CASCADE +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_transcript_message_idempotency + ON transcript_event_identities(session_id, message_idempotency_key) + WHERE message_idempotency_key IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_agent_transcript_event_parent + ON transcript_event_identities(session_id, parent_id) + WHERE parent_id IS NOT NULL; + CREATE TABLE IF NOT EXISTS cache_entries ( scope TEXT NOT NULL, key TEXT NOT NULL, diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index ab2e0020dff2..21b4460e8259 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -21,6 +21,7 @@ import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.gene import { closeOpenClawStateDatabaseForTest, detectOpenClawStateDatabaseSchemaMigrations, + OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, openOpenClawStateDatabase, OPENCLAW_STATE_SCHEMA_VERSION, repairOpenClawStateDatabaseSchema, @@ -1461,7 +1462,9 @@ describe("openclaw state database", () => { env: { OPENCLAW_STATE_DIR: stateDir }, }); - expect(readSqliteNumberPragma(database.db, "busy_timeout")).toBe(30_000); + expect(readSqliteNumberPragma(database.db, "busy_timeout")).toBe( + OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + ); expect(readSqliteNumberPragma(database.db, "foreign_keys")).toBe(1); expect(readSqliteNumberPragma(database.db, "synchronous")).toBe(1); expect(readSqliteNumberPragma(database.db, "user_version")).toBe(OPENCLAW_STATE_SCHEMA_VERSION); diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index 915c39fcdfae..44589ba61348 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -32,8 +32,8 @@ import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.generated.js" * migrations/backups that operate on local state. */ export const OPENCLAW_STATE_SCHEMA_VERSION = 2; -/** Shared timeout used by state and agent SQLite handles before surfacing busy errors. */ -export const OPENCLAW_SQLITE_BUSY_TIMEOUT_MS = 30_000; +/** Maximum time one synchronous SQLite call may wait for a lock. */ +export const OPENCLAW_SQLITE_BUSY_TIMEOUT_MS = 5_000; const OPENCLAW_STATE_DIR_MODE = 0o700; const OPENCLAW_STATE_FILE_MODE = 0o600; @@ -673,20 +673,28 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); try { assertSupportedSchemaVersion(db, pathname); - const changes = runSqliteImmediateTransactionSync(db, () => { - const applied: string[] = []; - if (repairAgentDatabasesCompositePrimaryKey(db)) { - applied.push(`Migrated shared state agent database registry primary key → agent_id,path`); - } - if (repairAuditEventsSchema(db)) { - applied.push( - `Migrated shared state audit event ledger → versioned message lifecycle schema`, - ); - } - assertCanonicalStateSchemaShape(db, pathname); - markCurrentStateSchemaVersion(db); - return applied; - }); + const changes = runSqliteImmediateTransactionSync( + db, + () => { + const applied: string[] = []; + if (repairAgentDatabasesCompositePrimaryKey(db)) { + applied.push(`Migrated shared state agent database registry primary key → agent_id,path`); + } + if (repairAuditEventsSchema(db)) { + applied.push( + `Migrated shared state audit event ledger → versioned message lifecycle schema`, + ); + } + assertCanonicalStateSchemaShape(db, pathname); + markCurrentStateSchemaVersion(db); + return applied; + }, + { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: pathname, + operationLabel: "state.schema.repair", + }, + ); return { changes, warnings: [] }; } catch (err) { return { @@ -1145,7 +1153,7 @@ function backfillDeliveryQueueEntriesFromEntryJson(db: DatabaseSync): void { } } -function ensureAdditiveStateColumns(db: DatabaseSync): void { +function ensureAdditiveStateColumns(db: DatabaseSync, pathname: string): void { ensureColumn(db, "device_pairing_pending", "refreshed_at_ms INTEGER"); ensureColumn(db, "device_pairing_paired", "approved_via TEXT"); ensureColumn(db, "device_pairing_paired", "operator_label TEXT"); @@ -1313,13 +1321,21 @@ function ensureAdditiveStateColumns(db: DatabaseSync): void { ensureColumn(db, "official_external_plugin_catalog_snapshots", "trust_signature_count INTEGER"); ensureColumn(db, "official_external_plugin_catalog_snapshots", "trust_threshold INTEGER"); ensureColumn(db, "official_external_plugin_catalog_snapshots", "trust_verified_at TEXT"); - runSqliteImmediateTransactionSync(db, () => { - const addedTaskRequesterAgentId = ensureColumn(db, "task_runs", "requester_agent_id TEXT"); - if (addedTaskRequesterAgentId) { - repairLegacyTaskAgentAttribution(db); - } - repairLegacyTaskDeliveryStatuses(db); - }); + runSqliteImmediateTransactionSync( + db, + () => { + const addedTaskRequesterAgentId = ensureColumn(db, "task_runs", "requester_agent_id TEXT"); + if (addedTaskRequesterAgentId) { + repairLegacyTaskAgentAttribution(db); + } + repairLegacyTaskDeliveryStatuses(db); + }, + { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: pathname, + operationLabel: "state.schema.ensure-columns", + }, + ); ensureColumn(db, "subagent_runs", "task_name TEXT"); ensureColumn(db, "worker_environments", "bootstrap_bundle_hash TEXT"); ensureColumn(db, "worker_environments", "bootstrap_openclaw_version TEXT"); @@ -1339,42 +1355,52 @@ function ensureAdditiveStateColumns(db: DatabaseSync): void { function ensureSchema(db: DatabaseSync, pathname: string): void { assertSupportedSchemaVersion(db, pathname); - ensureAdditiveStateColumns(db); + ensureAdditiveStateColumns(db, pathname); assertCanonicalStateSchemaShape(db, pathname); const now = Date.now(); const kysely = getNodeSqliteKysely<OpenClawStateMetadataDatabase>(db); - runSqliteImmediateTransactionSync(db, () => { - db.exec(OPENCLAW_STATE_SCHEMA_SQL); - // Retired node_pairing_* tables were created by earlier schema revisions but - // never had a shipped writer (the node surface lives on device_pairing_paired - // records), so dropping the always-empty tables is safe, not destructive. - db.exec("DROP TABLE IF EXISTS node_pairing_pending; DROP TABLE IF EXISTS node_pairing_paired;"); - db.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION};`); - executeSqliteQuerySync( - db, - kysely - .insertInto("schema_meta") - .values({ - meta_key: "primary", - role: "global", - schema_version: OPENCLAW_STATE_SCHEMA_VERSION, - agent_id: null, - app_version: null, - created_at: now, - updated_at: now, - }) - .onConflict((conflict) => - conflict.column("meta_key").doUpdateSet({ + runSqliteImmediateTransactionSync( + db, + () => { + db.exec(OPENCLAW_STATE_SCHEMA_SQL); + // Retired node_pairing_* tables were created by earlier schema revisions but + // never had a shipped writer (the node surface lives on device_pairing_paired + // records), so dropping the always-empty tables is safe, not destructive. + db.exec( + "DROP TABLE IF EXISTS node_pairing_pending; DROP TABLE IF EXISTS node_pairing_paired;", + ); + db.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION};`); + executeSqliteQuerySync( + db, + kysely + .insertInto("schema_meta") + .values({ + meta_key: "primary", role: "global", schema_version: OPENCLAW_STATE_SCHEMA_VERSION, agent_id: null, app_version: null, + created_at: now, updated_at: now, - }), - ), - ); - }); - ensureAdditiveStateColumns(db); + }) + .onConflict((conflict) => + conflict.column("meta_key").doUpdateSet({ + role: "global", + schema_version: OPENCLAW_STATE_SCHEMA_VERSION, + agent_id: null, + app_version: null, + updated_at: now, + }), + ), + ); + }, + { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: pathname, + operationLabel: "state.schema.ensure", + }, + ); + ensureAdditiveStateColumns(db, pathname); } function resolveDatabasePath(options: OpenClawStateDatabaseOptions = {}): string { @@ -1431,7 +1457,11 @@ export function runOpenClawStateWriteTransaction<T>( options: OpenClawStateDatabaseOptions = {}, ): T { const database = openOpenClawStateDatabase(options); - const result = runSqliteImmediateTransactionSync(database.db, () => operation(database)); + const result = runSqliteImmediateTransactionSync(database.db, () => operation(database), { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: database.path, + operationLabel: "state.write", + }); try { ensureOpenClawStatePermissions(database.path, options.env ?? process.env); } catch { diff --git a/src/tasks/task-registry.maintenance.issue-60299.test.ts b/src/tasks/task-registry.maintenance.issue-60299.test.ts index bd8e039c732b..6b9f77eea4ef 100644 --- a/src/tasks/task-registry.maintenance.issue-60299.test.ts +++ b/src/tasks/task-registry.maintenance.issue-60299.test.ts @@ -56,7 +56,7 @@ afterEach(() => { function createTaskRegistryMaintenanceHarness(params: { tasks: TaskRecord[]; sessionStore?: Record<string, SessionEntry>; - loadSessionStore?: TaskRegistryMaintenanceRuntime["loadSessionStore"]; + listSessionEntries?: TaskRegistryMaintenanceRuntime["listSessionEntries"]; resolveStorePath?: TaskRegistryMaintenanceRuntime["resolveStorePath"]; deriveSessionChatTypeFromKey?: TaskRegistryMaintenanceRuntime["deriveSessionChatTypeFromKey"]; acpEntry?: AcpSessionStoreEntry["entry"]; @@ -95,7 +95,13 @@ function createTaskRegistryMaintenanceHarness(params: { entry: undefined, storeReadFailed: false, } satisfies AcpSessionStoreEntry), - loadSessionStore: params.loadSessionStore ?? (() => sessionStore), + listSessionEntries: + params.listSessionEntries ?? + (() => + Object.entries(sessionStore).map(([sessionKey, entry]) => ({ + sessionKey, + entry, + }))), resolveStorePath: params.resolveStorePath ?? (() => ""), ...(params.deriveSessionChatTypeFromKey ? { deriveSessionChatTypeFromKey: params.deriveSessionChatTypeFromKey } @@ -214,7 +220,7 @@ function expectTaskStatus( } describe("task-registry maintenance issue #60299", () => { - it("reuses session store reads across stale subagent task checks in one pass", async () => { + it("reuses session entry lists across stale subagent task checks in one pass", async () => { const tasks = Array.from({ length: 10 }, (_, index) => makeStaleTask({ runtime: "subagent", @@ -222,16 +228,16 @@ describe("task-registry maintenance issue #60299", () => { childSessionKey: `agent:main:subagent:stale-${index}`, }), ); - const loadSessionStoreMock = vi.fn(() => ({})); + const listSessionEntriesMock = vi.fn(() => []); createTaskRegistryMaintenanceHarness({ tasks, - loadSessionStore: loadSessionStoreMock, + listSessionEntries: listSessionEntriesMock, resolveStorePath: () => "/tmp/openclaw-test-sessions-main.json", }); expectMaintenanceCounts(await runTaskRegistryMaintenance(), { reconciled: tasks.length }); - expect(loadSessionStoreMock).toHaveBeenCalledTimes(1); + expect(listSessionEntriesMock).toHaveBeenCalledTimes(1); }); it("reuses CLI channel session type derivation across duplicate stale task checks", async () => { diff --git a/src/tasks/task-registry.maintenance.ts b/src/tasks/task-registry.maintenance.ts index 854b12a9e313..ea5ffb7c77d4 100644 --- a/src/tasks/task-registry.maintenance.ts +++ b/src/tasks/task-registry.maintenance.ts @@ -1,8 +1,5 @@ // Reconciles stale or lost task registry records during maintenance passes. -import { - normalizeLowercaseStringOrEmpty, - normalizeOptionalString, -} from "@openclaw/normalization-core/string-coerce"; +import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { isAcpTurnActive } from "../acp/control-plane/active-turns.js"; import { getAcpSessionManager } from "../acp/control-plane/manager.js"; import { @@ -14,8 +11,12 @@ import { formatSubagentRecoveryWedgedReason, isSubagentRecoveryWedgedEntry, } from "../agents/subagent-recovery-state.js"; -import { loadSessionStore, resolveStorePath } from "../config/sessions.js"; +import { resolveStorePath } from "../config/sessions.js"; import type { SessionEntry } from "../config/sessions.js"; +import { + listSessionEntries, + type SessionEntrySummary, +} from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { isCronJobActive } from "../cron/active-jobs.js"; import { readCronRunLogEntriesSync } from "../cron/run-log.js"; @@ -100,7 +101,7 @@ type TaskRegistryMaintenanceRuntime = { }) => Promise<void>; listSessionBindingsBySession?: ReturnType<typeof getSessionBindingService>["listBySession"]; unbindSessionBindings?: ReturnType<typeof getSessionBindingService>["unbind"]; - loadSessionStore: typeof loadSessionStore; + listSessionEntries: typeof listSessionEntries; resolveStorePath: typeof resolveStorePath; deriveSessionChatTypeFromKey?: typeof deriveSessionChatTypeFromKey; isCronJobActive: typeof isCronJobActive; @@ -140,7 +141,7 @@ const defaultTaskRegistryMaintenanceRuntime: TaskRegistryMaintenanceRuntime = { listSessionBindingsBySession: (sessionKey) => getSessionBindingService().listBySession(sessionKey), unbindSessionBindings: (input) => getSessionBindingService().unbind(input), - loadSessionStore, + listSessionEntries, resolveStorePath, deriveSessionChatTypeFromKey, isCronJobActive, @@ -215,13 +216,12 @@ type CronRecoveryContext = { runLogsByJobId: Map<string, CronRunLogEntry[]>; }; -type SessionStoreLookup = { - store: Record<string, SessionEntry>; - normalizedEntries?: Map<string, SessionEntry>; +type SessionEntryLookup = { + entriesByKey: Map<string, SessionEntry>; }; type BackingSessionLookupContext = { - sessionStoresByPath: Map<string, SessionStoreLookup>; + sessionEntriesByPath: Map<string, SessionEntryLookup>; sessionChatTypesByKey: Map<string, SessionKeyChatType>; }; @@ -234,58 +234,42 @@ function createCronRecoveryContext(): CronRecoveryContext { function createBackingSessionLookupContext(): BackingSessionLookupContext { return { - sessionStoresByPath: new Map<string, SessionStoreLookup>(), + sessionEntriesByPath: new Map<string, SessionEntryLookup>(), sessionChatTypesByKey: new Map<string, SessionKeyChatType>(), }; } -function getSessionStoreLookup( +function buildSessionEntryLookup(entries: SessionEntrySummary[]): SessionEntryLookup { + return { + entriesByKey: new Map(entries.map(({ sessionKey, entry }) => [sessionKey, entry])), + }; +} + +function getSessionEntryLookup( storePath: string, context?: BackingSessionLookupContext, -): SessionStoreLookup { +): SessionEntryLookup { if (!context) { - return { - store: taskRegistryMaintenanceRuntime.loadSessionStore(storePath, { clone: false }), - }; + return buildSessionEntryLookup( + taskRegistryMaintenanceRuntime.listSessionEntries({ storePath }), + ); } - const cached = context.sessionStoresByPath.get(storePath); + const cached = context.sessionEntriesByPath.get(storePath); if (cached) { return cached; } - const lookup = { - store: taskRegistryMaintenanceRuntime.loadSessionStore(storePath, { clone: false }), - }; - context.sessionStoresByPath.set(storePath, lookup); + const lookup = buildSessionEntryLookup( + taskRegistryMaintenanceRuntime.listSessionEntries({ storePath }), + ); + context.sessionEntriesByPath.set(storePath, lookup); return lookup; } -function getNormalizedSessionEntries(lookup: SessionStoreLookup): Map<string, SessionEntry> { - if (lookup.normalizedEntries) { - return lookup.normalizedEntries; - } - const entries = new Map<string, SessionEntry>(); - for (const [key, entry] of Object.entries(lookup.store)) { - if (entry) { - entries.set(normalizeLowercaseStringOrEmpty(key), entry); - } - } - lookup.normalizedEntries = entries; - return entries; -} - function findSessionEntryByKey( - lookup: SessionStoreLookup, + lookup: SessionEntryLookup, sessionKey: string, ): SessionEntry | undefined { - const direct = lookup.store[sessionKey]; - if (direct) { - return direct; - } - const normalized = normalizeLowercaseStringOrEmpty(sessionKey); - if (!normalized) { - return undefined; - } - return getNormalizedSessionEntries(lookup).get(normalized); + return lookup.entriesByKey.get(sessionKey); } function resolveSessionChatType( @@ -316,7 +300,7 @@ function findTaskSessionEntry( } const agentId = taskRegistryMaintenanceRuntime.parseAgentSessionKey(childSessionKey)?.agentId; const storePath = taskRegistryMaintenanceRuntime.resolveStorePath(undefined, { agentId }); - return findSessionEntryByKey(getSessionStoreLookup(storePath, context), childSessionKey); + return findSessionEntryByKey(getSessionEntryLookup(storePath, context), childSessionKey); } function isActiveTask(task: TaskRecord): boolean { diff --git a/src/tasks/task-registry.test.ts b/src/tasks/task-registry.test.ts index 682ff54db532..5879c6bd8c14 100644 --- a/src/tasks/task-registry.test.ts +++ b/src/tasks/task-registry.test.ts @@ -178,7 +178,7 @@ function configureTaskRegistryMaintenanceRuntimeForTest(params: { listSessionBindingsBySession: () => params.sessionBindings ?? [], closeAcpSession: params.closeAcpSession, unbindSessionBindings: params.unbindSessionBindings, - loadSessionStore: () => ({}), + listSessionEntries: () => [], resolveStorePath: () => "", parseAgentSessionKey: () => null as ParsedAgentSessionKey | null, isCronJobActive: () => false, @@ -3332,7 +3332,7 @@ describe("task-registry", () => { entry: undefined, storeReadFailed: false, }), - loadSessionStore: () => ({}), + listSessionEntries: () => [], resolveStorePath: () => "", parseAgentSessionKey: () => null, isCronJobActive: () => false, diff --git a/src/trajectory/cleanup.test.ts b/src/trajectory/cleanup.test.ts index 0062bd01e64c..11a84ecf453f 100644 --- a/src/trajectory/cleanup.test.ts +++ b/src/trajectory/cleanup.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { formatSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { withTempDir } from "../test-helpers/temp-dir.js"; import { removeRemovedSessionTrajectoryArtifacts, @@ -66,6 +67,31 @@ describe("trajectory cleanup", () => { }); }); + it("removes legacy runtime sidecars for SQLite marker sessions", async () => { + await withTempDir({ prefix: "openclaw-trajectory-cleanup-" }, async (dir) => { + const sessionId = "session-1"; + const storePath = path.join(dir, "sessions.json"); + const sessionFile = formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }); + const runtimeFile = resolveTrajectoryFilePath({ env: {}, sessionFile, sessionId }); + await fs.mkdir(path.dirname(runtimeFile), { recursive: true }); + await fs.writeFile(runtimeFile, runtimeEvent(sessionId), "utf8"); + + const removed = await removeSessionTrajectoryArtifacts({ + sessionId, + sessionFile, + storePath, + restrictToStoreDir: true, + }); + + expect(removed).toEqual([{ kind: "runtime", path: runtimeFile }]); + await expectPathMissing(runtimeFile); + }); + }); + it("skips removed sessions still referenced by surviving store rows", async () => { await withTempDir({ prefix: "openclaw-trajectory-cleanup-" }, async (dir) => { const sessionId = "shared-session"; diff --git a/src/trajectory/cleanup.ts b/src/trajectory/cleanup.ts index 156178acfc51..7f985e6d4e71 100644 --- a/src/trajectory/cleanup.ts +++ b/src/trajectory/cleanup.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import path from "node:path"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { resolveSessionFilePath } from "../config/sessions/paths.js"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { isPathInside } from "../infra/path-guards.js"; import { resolveTrajectoryFilePath, @@ -186,8 +187,16 @@ export async function removeSessionTrajectoryArtifacts(params: { const storeDir = path.dirname(path.resolve(params.storePath)); const restrictToStoreDir = params.restrictToStoreDir === true; const removed: RemovedTrajectoryArtifact[] = []; - const pointerPath = resolveTrajectoryPointerFilePath(sessionFile); - const pointer = readTrajectoryPointerFile(pointerPath, params.sessionId); + const sqliteMarker = parseSqliteSessionFileMarker(sessionFile); + if ( + sqliteMarker && + (sqliteMarker.sessionId !== params.sessionId || + path.resolve(sqliteMarker.storePath) !== path.resolve(params.storePath)) + ) { + return []; + } + const pointerPath = sqliteMarker ? undefined : resolveTrajectoryPointerFilePath(sessionFile); + const pointer = pointerPath ? readTrajectoryPointerFile(pointerPath, params.sessionId) : null; const defaultRuntimePath = resolveTrajectoryFilePath({ env: {}, sessionFile, @@ -216,7 +225,7 @@ export async function removeSessionTrajectoryArtifacts(params: { } } - if (!restrictToStoreDir || isPathWithinDir(storeDir, pointerPath)) { + if (pointerPath && (!restrictToStoreDir || isPathWithinDir(storeDir, pointerPath))) { const deletedPointer = await removeRegularFile(pointerPath, "pointer"); if (deletedPointer) { removed.push(deletedPointer); diff --git a/src/trajectory/export.test.ts b/src/trajectory/export.test.ts index 65f171d552e6..b0d9743b40f4 100644 --- a/src/trajectory/export.test.ts +++ b/src/trajectory/export.test.ts @@ -4,8 +4,13 @@ import os from "node:os"; import path from "node:path"; import type { Message, Usage } from "openclaw/plugin-sdk/llm"; import { afterAll, describe, expect, it } from "vitest"; +import { replaceTranscriptEvents } from "../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { exportTrajectoryBundle, resolveDefaultTrajectoryExportDir } from "./export.js"; import { TRAJECTORY_RUNTIME_FILE_MAX_BYTES, resolveTrajectoryPointerFilePath } from "./paths.js"; +import { appendSqliteTrajectoryRuntimeEvents } from "./runtime-store.sqlite.js"; import type { TrajectoryEvent } from "./types.js"; const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-trajectory-")); @@ -186,6 +191,8 @@ function writeToolCallSessionFile(sessionFile: string): void { } afterAll(() => { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); fs.rmSync(tempRoot, { recursive: true, force: true }); }); @@ -227,6 +234,126 @@ describe("exportTrajectoryBundle", () => { } }); + it("exports SQLite-backed transcript rows without a session JSONL file", async () => { + const tmpDir = makeTempDir(); + const storePath = path.join(tmpDir, "sessions.json"); + const outputDir = path.join(tmpDir, "bundle"); + const sessionId = "session-1"; + const sessionKey = "agent:main:session-1"; + await replaceTranscriptEvents( + { + agentId: "main", + sessionId, + sessionKey, + storePath, + }, + [ + { + type: "session", + version: 3, + id: sessionId, + timestamp: "2026-04-01T05:46:39.000Z", + cwd: tmpDir, + }, + { + type: "message", + id: "entry-user", + parentId: null, + timestamp: "2026-04-01T05:46:40.000Z", + message: userMessage("hello from sqlite"), + }, + { + type: "message", + id: "entry-assistant", + parentId: "entry-user", + timestamp: "2026-04-01T05:46:41.000Z", + message: assistantMessage([{ type: "text", text: "done from sqlite" }]), + }, + ], + ); + + const bundle = await exportTrajectoryBundle({ + outputDir, + sessionFile: formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }), + sessionId, + sessionKey, + workspaceDir: tmpDir, + }); + + expect(bundle.header?.id).toBe(sessionId); + expect(bundle.manifest.transcriptEventCount).toBe(2); + expect(eventTypes(bundle.events)).toEqual(["user.message", "assistant.message"]); + expect(fs.existsSync(path.join(tmpDir, "session-1.jsonl"))).toBe(false); + }); + + it("exports SQLite-backed runtime rows without a runtime JSONL sidecar", async () => { + const tmpDir = makeTempDir(); + const storePath = path.join(tmpDir, "sessions.json"); + const outputDir = path.join(tmpDir, "bundle"); + const sessionId = "session-1"; + const sessionKey = "agent:main:session-1"; + await replaceTranscriptEvents( + { + agentId: "main", + sessionId, + sessionKey, + storePath, + }, + [ + { + type: "session", + version: 3, + id: sessionId, + timestamp: "2026-04-01T05:46:39.000Z", + cwd: tmpDir, + }, + { + type: "message", + id: "entry-user", + parentId: null, + timestamp: "2026-04-01T05:46:40.000Z", + message: userMessage("hello from sqlite"), + }, + ], + ); + appendSqliteTrajectoryRuntimeEvents({ sessionId, storePath }, [ + { + traceSchema: "openclaw-trajectory", + schemaVersion: 1, + traceId: sessionId, + source: "runtime", + type: "sqlite-runtime", + ts: "2026-04-01T05:46:41.000Z", + seq: 1, + sourceSeq: 1, + sessionId, + sessionKey, + }, + ]); + + const bundle = await exportTrajectoryBundle({ + outputDir, + sessionFile: formatSqliteSessionFileMarker({ + agentId: "main", + sessionId, + storePath, + }), + sessionId, + sessionKey, + workspaceDir: tmpDir, + }); + + expect(bundle.runtimeFile).toBeUndefined(); + expect(bundle.manifest.runtimeEventCount).toBe(1); + expect(bundle.manifest.sourceFiles.runtime).toBeUndefined(); + expect(eventTypes(bundle.events)).toContain("sqlite-runtime"); + expect(fs.existsSync(path.join(tmpDir, "trajectory", "session-1.jsonl"))).toBe(false); + }); + it("does not synthesize prompt files from export-time fallbacks", async () => { const tmpDir = makeTempDir(); const sessionFile = path.join(tmpDir, "session.jsonl"); diff --git a/src/trajectory/export.ts b/src/trajectory/export.ts index f34ac614f81d..2d9aba273c81 100644 --- a/src/trajectory/export.ts +++ b/src/trajectory/export.ts @@ -7,6 +7,8 @@ import type { AgentMessage } from "../agents/runtime/index.js"; import { parseSessionFileEntriesWithWarnings } from "../agents/sessions/session-file-parser.js"; import type { FileEntry, SessionEntry, SessionHeader } from "../agents/sessions/session-manager.js"; import { resolveStateDir } from "../config/paths.js"; +import { loadTranscriptEvents } from "../config/sessions/session-accessor.js"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { isCanonicalSessionTranscriptEntry, scanSessionTranscriptTree, @@ -28,6 +30,7 @@ import { redactSecrets, redactToolPayloadText } from "../logging/redact.js"; import { safeJsonStringify } from "../utils/safe-json.js"; import { TRAJECTORY_RUNTIME_FILE_MAX_BYTES, safeTrajectorySessionFileName } from "./paths.js"; import { isRegularNonSymlinkFile, resolveTrajectoryRuntimeFile } from "./runtime-file.js"; +import { loadSqliteTrajectoryRuntimeEvents } from "./runtime-store.sqlite.js"; import type { TrajectoryBundleManifest, TrajectoryBundleWarning, @@ -64,6 +67,11 @@ type JsonlParseWarning = Omit<TrajectoryBundleWarning, "count" | "rows"> & { row: number; }; +type SessionEntryCandidateRow = { + row: number; + value: unknown; +}; + const MAX_TRAJECTORY_RUNTIME_EVENTS = 200_000; const MAX_TRAJECTORY_TOTAL_EVENTS = 250_000; const MAX_TRAJECTORY_SESSION_FILE_BYTES = 50 * 1024 * 1024; @@ -73,6 +81,57 @@ function isFiniteNumber(value: unknown): value is number { return typeof value === "number" && Number.isFinite(value); } +function isSessionFileEntry(value: unknown): value is FileEntry { + if (!isRecord(value) || typeof value.type !== "string") { + return false; + } + if (value.type !== "message") { + return true; + } + const message = value.message; + return isRecord(message) && typeof message.role === "string"; +} + +function formatSessionParseWarnings( + warnings: ReturnType<typeof parseSessionFileEntriesWithWarnings>["warnings"], +): JsonlParseWarning[] { + return warnings.map((warning) => ({ + source: "session", + code: warning.code, + row: warning.row, + message: + warning.code === "invalid-session-json" + ? "Skipped a session JSONL row that is not valid JSON." + : "Skipped a session JSONL row that is not a session entry object.", + })); +} + +function collectSessionEntries( + rows: readonly SessionEntryCandidateRow[], + warnings: JsonlParseWarning[] = [], +): { + entries: FileEntry[]; + warnings: JsonlParseWarning[]; + rowByEntry: Map<FileEntry, number>; +} { + const entries: FileEntry[] = []; + const rowByEntry = new Map<FileEntry, number>(); + for (const row of rows) { + if (!isSessionFileEntry(row.value)) { + warnings.push({ + source: "session", + code: "invalid-session-row", + row: row.row, + message: "Skipped a session JSONL row that is not a session entry object.", + }); + continue; + } + entries.push(row.value); + rowByEntry.set(row.value, row.row); + } + return { entries, warnings, rowByEntry }; +} + function migrateLegacySessionEntries(entries: FileEntry[]): void { const header = entries.find((entry): entry is SessionHeader => entry.type === "session"); const version = header?.version ?? 1; @@ -118,26 +177,49 @@ function migrateLegacySessionEntries(entries: FileEntry[]): void { } } -async function readSessionBranch(filePath: string): Promise<{ +async function readSessionEntries(params: { + sessionFile: string; + sessionId: string; + sessionKey?: string; +}): Promise<{ + entries: FileEntry[]; + warnings: JsonlParseWarning[]; + rowByEntry: Map<FileEntry, number>; +}> { + const marker = parseSqliteSessionFileMarker(params.sessionFile); + if (!marker) { + const { entries, warnings, rowByEntry } = parseSessionFileEntriesWithWarnings( + await fsp.readFile(params.sessionFile, "utf8"), + ); + return { + entries, + warnings: formatSessionParseWarnings(warnings), + rowByEntry, + }; + } + return collectSessionEntries( + ( + await loadTranscriptEvents({ + agentId: marker.agentId, + sessionId: params.sessionId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + storePath: marker.storePath, + }) + ).map((value, index) => ({ row: index + 1, value })), + ); +} + +async function readSessionBranch(params: { + sessionFile: string; + sessionId: string; + sessionKey?: string; +}): Promise<{ header: SessionHeader | null; leafId: string | null; branchEntries: SessionEntry[]; warnings: JsonlParseWarning[]; }> { - const { - entries: fileEntries, - warnings: parseWarnings, - rowByEntry, - } = parseSessionFileEntriesWithWarnings(await fsp.readFile(filePath, "utf8")); - const warnings: JsonlParseWarning[] = parseWarnings.map((warning) => ({ - source: "session", - code: warning.code, - row: warning.row, - message: - warning.code === "invalid-session-json" - ? "Skipped a session JSONL row that is not valid JSON." - : "Skipped a session JSONL row that is not a session entry object.", - })); + const { entries: fileEntries, warnings, rowByEntry } = await readSessionEntries(params); migrateLegacySessionEntries(fileEntries); const header = fileEntries.find((entry): entry is SessionHeader => entry.type === "session") ?? null; @@ -272,6 +354,47 @@ async function parseJsonlFile<T>( return { events: parsed, warnings }; } +async function readRuntimeTrajectoryEvents(params: { + runtimeFile?: string; + sessionFile: string; + sessionId: string; +}): Promise<{ + events: TrajectoryEvent[]; + runtimeFile?: string; + warnings: JsonlParseWarning[]; +}> { + const marker = parseSqliteSessionFileMarker(params.sessionFile); + if (marker && marker.sessionId === params.sessionId) { + const events = await loadSqliteTrajectoryRuntimeEvents({ + agentId: marker.agentId, + sessionId: marker.sessionId, + storePath: marker.storePath, + }); + if (events.length > MAX_TRAJECTORY_RUNTIME_EVENTS) { + throw new Error( + `Trajectory runtime store has too many events to export (limit ${MAX_TRAJECTORY_RUNTIME_EVENTS})`, + ); + } + return { events, warnings: [] }; + } + + const runtimeFile = await resolveTrajectoryRuntimeFile({ + runtimeFile: params.runtimeFile, + sessionFile: params.sessionFile, + sessionId: params.sessionId, + }); + if (!runtimeFile) { + return { events: [], warnings: [] }; + } + const parsed = await parseJsonlFile<TrajectoryEvent>(runtimeFile, { + maxBytes: TRAJECTORY_RUNTIME_FILE_MAX_BYTES, + maxEvents: MAX_TRAJECTORY_RUNTIME_EVENTS, + include: (value) => value.sessionId === params.sessionId, + validate: isRuntimeTrajectoryEvent, + }); + return { ...parsed, runtimeFile }; +} + function isRuntimeTrajectoryEvent(value: unknown): value is TrajectoryEvent { if (!isRecord(value)) { return false; @@ -925,31 +1048,30 @@ export async function exportTrajectoryBundle(params: BuildTrajectoryBundleParams const redaction = buildTrajectoryExportRedaction({ workspaceDir: params.workspaceDir, }); - const sessionStat = await fsp.stat(params.sessionFile); - if (sessionStat.size > MAX_TRAJECTORY_SESSION_FILE_BYTES) { - throw new Error( - `Trajectory session file is too large to export (${sessionStat.size} bytes; limit ${MAX_TRAJECTORY_SESSION_FILE_BYTES})`, - ); + if (!parseSqliteSessionFileMarker(params.sessionFile)) { + const sessionStat = await fsp.stat(params.sessionFile); + if (sessionStat.size > MAX_TRAJECTORY_SESSION_FILE_BYTES) { + throw new Error( + `Trajectory session file is too large to export (${sessionStat.size} bytes; limit ${MAX_TRAJECTORY_SESSION_FILE_BYTES})`, + ); + } } const { header, leafId, branchEntries, warnings: sessionWarnings, - } = await readSessionBranch(params.sessionFile); - const runtimeFile = await resolveTrajectoryRuntimeFile({ + } = await readSessionBranch({ + sessionFile: params.sessionFile, + sessionId: params.sessionId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + }); + const runtimeParse = await readRuntimeTrajectoryEvents({ runtimeFile: params.runtimeFile, sessionFile: params.sessionFile, sessionId: params.sessionId, }); - const runtimeParse = runtimeFile - ? await parseJsonlFile<TrajectoryEvent>(runtimeFile, { - maxBytes: TRAJECTORY_RUNTIME_FILE_MAX_BYTES, - maxEvents: MAX_TRAJECTORY_RUNTIME_EVENTS, - include: (value) => value.sessionId === params.sessionId, - validate: isRuntimeTrajectoryEvent, - }) - : { events: [], warnings: [] }; + const runtimeFile = runtimeParse.runtimeFile; const runtimeEvents = runtimeParse.events; const transcriptEvents = buildTranscriptEvents({ entries: branchEntries, diff --git a/src/trajectory/paths.ts b/src/trajectory/paths.ts index 9cab0c3f6524..9ffa9d0fa35c 100644 --- a/src/trajectory/paths.ts +++ b/src/trajectory/paths.ts @@ -1,11 +1,12 @@ // Trajectory path helpers resolve storage paths for trajectory artifacts. import fs from "node:fs"; import path from "node:path"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import { resolveHomeRelativePath } from "../infra/home-dir.js"; import { isPathInside } from "../infra/path-guards.js"; -// Runtime trajectory path helpers. Paths are either beside the session file or -// inside OPENCLAW_TRAJECTORY_DIR, with names scrubbed for filesystem safety. +// Legacy trajectory path helpers. Active runtime capture writes SQLite rows; +// these paths remain for explicit legacy-file reads, export artifacts, and cleanup. export const TRAJECTORY_RUNTIME_CAPTURE_MAX_BYTES = 10 * 1024 * 1024; export const TRAJECTORY_RUNTIME_FILE_MAX_BYTES = 50 * 1024 * 1024; export const TRAJECTORY_RUNTIME_EVENT_MAX_BYTES = 256 * 1024; @@ -63,6 +64,14 @@ export function resolveTrajectoryFilePath(params: { `${safeTrajectorySessionFileName(params.sessionId)}.trajectory.jsonl`, ); } + const sqliteMarker = parseSqliteSessionFileMarker(params.sessionFile); + if (sqliteMarker) { + return path.join( + path.dirname(path.resolve(sqliteMarker.storePath)), + "trajectory", + `${safeTrajectorySessionFileName(sqliteMarker.sessionId)}.jsonl`, + ); + } return params.sessionFile.endsWith(".jsonl") ? `${params.sessionFile.slice(0, -".jsonl".length)}.trajectory.jsonl` : `${params.sessionFile}.trajectory.jsonl`; diff --git a/src/trajectory/runtime-store.sqlite.test.ts b/src/trajectory/runtime-store.sqlite.test.ts new file mode 100644 index 000000000000..0355b467fff0 --- /dev/null +++ b/src/trajectory/runtime-store.sqlite.test.ts @@ -0,0 +1,128 @@ +// SQLite trajectory runtime tests cover session-scoped event row storage. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js"; +import { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, +} from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { + appendSqliteTrajectoryRuntimeEvents, + loadSqliteTrajectoryRuntimeEvents, + readSqliteTrajectoryRuntimeStatsSync, +} from "./runtime-store.sqlite.js"; +import type { TrajectoryEvent } from "./types.js"; + +type TrajectoryRuntimeTestDatabase = Pick<OpenClawAgentKyselyDatabase, "trajectory_runtime_events">; + +describe("SQLite trajectory runtime store", () => { + let tempDir: string; + let storePath: string; + + beforeEach(async () => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-trajectory-sqlite-")); + storePath = path.join(tempDir, "agents", "main", "sessions", "sessions.json"); + await replaceSessionEntry( + { sessionKey: "agent:main:main", storePath }, + { sessionId: "session-1", updatedAt: 10 }, + ); + }); + + afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + it("appends events in database order without trusting recorder-local seq", async () => { + appendSqliteTrajectoryRuntimeEvents({ sessionId: "session-1", storePath }, [ + createTrajectoryEvent({ seq: 1, type: "model.started" }), + createTrajectoryEvent({ seq: 1, type: "model.completed" }), + ]); + + await expect( + loadSqliteTrajectoryRuntimeEvents({ sessionId: "session-1", storePath }), + ).resolves.toEqual([ + expect.objectContaining({ seq: 1, type: "model.started" }), + expect.objectContaining({ seq: 1, type: "model.completed" }), + ]); + + const database = openOpenClawAgentDatabase({ agentId: "main", path: sqlitePath() }); + const db = getNodeSqliteKysely<TrajectoryRuntimeTestDatabase>(database.db); + const rows = executeSqliteQuerySync( + database.db, + db + .selectFrom("trajectory_runtime_events") + .select(["seq", "run_id"]) + .where("session_id", "=", "session-1") + .orderBy("seq", "asc"), + ).rows; + expect(rows).toEqual([ + { run_id: "run-1", seq: 0 }, + { run_id: "run-1", seq: 1 }, + ]); + }); + + it("trims oldest rows beyond the configured byte window", async () => { + appendSqliteTrajectoryRuntimeEvents( + { maxRuntimeBytes: 900, sessionId: "session-1", storePath }, + [ + createTrajectoryEvent({ type: "event-1" }), + createTrajectoryEvent({ type: "event-2" }), + createTrajectoryEvent({ type: "event-3" }), + createTrajectoryEvent({ type: "event-4" }), + ], + ); + + const events = await loadSqliteTrajectoryRuntimeEvents({ + sessionId: "session-1", + storePath, + }); + + expect(events.map((event) => event.type)).toEqual(["event-3", "event-4"]); + expect(readSqliteTrajectoryRuntimeStatsSync({ sessionId: "session-1", storePath })).toEqual({ + eventCount: 2, + maxSeq: 3, + sizeBytes: expect.any(Number), + }); + }); + + it("cascades trajectory rows when the session row is deleted", async () => { + appendSqliteTrajectoryRuntimeEvents({ sessionId: "session-1", storePath }, [ + createTrajectoryEvent({ type: "model.started" }), + ]); + + const database = openOpenClawAgentDatabase({ agentId: "main", path: sqlitePath() }); + database.db.prepare("DELETE FROM sessions WHERE session_id = ?").run("session-1"); + + await expect( + loadSqliteTrajectoryRuntimeEvents({ sessionId: "session-1", storePath }), + ).resolves.toEqual([]); + }); + + function sqlitePath(): string { + return path.join(tempDir, "agents", "main", "agent", "openclaw-agent.sqlite"); + } +}); + +function createTrajectoryEvent(options: { seq?: number; type: string }): TrajectoryEvent { + return { + traceSchema: "openclaw-trajectory", + schemaVersion: 1, + traceId: "session-1", + source: "runtime", + type: options.type, + ts: "2026-07-03T00:00:00.000Z", + seq: options.seq ?? 1, + sourceSeq: options.seq ?? 1, + sessionId: "session-1", + sessionKey: "agent:main:main", + runId: "run-1", + data: { payload: "x".repeat(120) }, + }; +} diff --git a/src/trajectory/runtime-store.sqlite.ts b/src/trajectory/runtime-store.sqlite.ts new file mode 100644 index 000000000000..c914d8819306 --- /dev/null +++ b/src/trajectory/runtime-store.sqlite.ts @@ -0,0 +1,249 @@ +// SQLite trajectory runtime store owns session-scoped runtime event rows. +import { sql } from "kysely"; +import { resolveSqliteTargetFromSessionStorePath } from "../config/sessions/session-sqlite-target.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import { DEFAULT_AGENT_ID, normalizeAgentId } from "../routing/session-key.js"; +import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js"; +import { + openOpenClawAgentDatabase, + runOpenClawAgentWriteTransaction, + type OpenClawAgentDatabase, + type OpenClawAgentDatabaseOptions, +} from "../state/openclaw-agent-db.js"; +import { TRAJECTORY_RUNTIME_CAPTURE_MAX_BYTES } from "./paths.js"; +import type { TrajectoryEvent } from "./types.js"; + +type SqliteTrajectoryRuntimeDatabase = Pick< + OpenClawAgentKyselyDatabase, + "trajectory_runtime_events" +>; + +export type SqliteTrajectoryRuntimeScope = { + agentId?: string; + env?: NodeJS.ProcessEnv; + maxRuntimeBytes?: number; + sessionId: string; + storePath: string; +}; + +export type SqliteTrajectoryRuntimeStats = { + eventCount: number; + maxSeq: number; + sizeBytes: number; +}; + +export type SqliteTrajectoryRuntimeEventRow = { + event: TrajectoryEvent; + seq: number; +}; + +type TrajectoryRuntimeRow = { + event_json: string; + seq: number; +}; + +/** Appends runtime trajectory events to the per-agent SQLite session store. */ +export function appendSqliteTrajectoryRuntimeEvents( + scope: SqliteTrajectoryRuntimeScope, + events: readonly TrajectoryEvent[], +): void { + if (events.length === 0) { + return; + } + const options = toDatabaseOptions(scope); + const maxRuntimeBytes = Math.max( + 1, + Math.floor(scope.maxRuntimeBytes ?? TRAJECTORY_RUNTIME_CAPTURE_MAX_BYTES), + ); + runOpenClawAgentWriteTransaction((database) => { + const db = getTrajectoryKysely(database.db); + let seq = readNextTrajectorySeq(database, scope.sessionId); + for (const event of events) { + const eventJson = JSON.stringify(event); + executeSqliteQuerySync( + database.db, + db.insertInto("trajectory_runtime_events").values({ + session_id: scope.sessionId, + seq, + run_id: event.runId ?? null, + event_json: eventJson, + created_at: readTrajectoryEventTimestamp(event) ?? Date.now(), + }), + ); + seq += 1; + } + trimSqliteTrajectoryRuntimeWindow(database, scope.sessionId, maxRuntimeBytes); + }, options); +} + +/** Loads runtime trajectory events from per-agent SQLite rows in storage order. */ +export async function loadSqliteTrajectoryRuntimeEvents( + scope: Omit<SqliteTrajectoryRuntimeScope, "maxRuntimeBytes">, +): Promise<TrajectoryEvent[]> { + return loadSqliteTrajectoryRuntimeEventsSync(scope); +} + +/** Loads runtime trajectory events synchronously for CLI and export paths. */ +export function loadSqliteTrajectoryRuntimeEventsSync( + scope: Omit<SqliteTrajectoryRuntimeScope, "maxRuntimeBytes">, +): TrajectoryEvent[] { + return loadSqliteTrajectoryRuntimeEventRowsSync(scope).map((row) => row.event); +} + +/** Loads runtime trajectory event rows with storage seqs for follow/export cursors. */ +export function loadSqliteTrajectoryRuntimeEventRowsSync( + scope: Omit<SqliteTrajectoryRuntimeScope, "maxRuntimeBytes"> & { + afterSeq?: number; + maxEvents?: number; + }, +): SqliteTrajectoryRuntimeEventRow[] { + const database = openOpenClawAgentDatabase(toDatabaseOptions(scope)); + const db = getTrajectoryKysely(database.db); + let query = db + .selectFrom("trajectory_runtime_events") + .select(["seq", "event_json"]) + .where("session_id", "=", scope.sessionId) + .orderBy("seq", "asc"); + const afterSeq = scope.afterSeq; + if (afterSeq !== undefined && Number.isFinite(afterSeq)) { + query = query.where("seq", ">", Math.floor(afterSeq)); + } + const maxEvents = scope.maxEvents; + if (maxEvents !== undefined && Number.isFinite(maxEvents)) { + query = query.limit(Math.max(0, Math.floor(maxEvents))); + } + return executeSqliteQuerySync(database.db, query).rows.map((row) => ({ + event: JSON.parse(row.event_json) as TrajectoryEvent, + seq: row.seq, + })); +} + +/** Reads trajectory row count, max storage seq, and JSONL-compatible byte size. */ +export function readSqliteTrajectoryRuntimeStatsSync( + scope: Omit<SqliteTrajectoryRuntimeScope, "maxRuntimeBytes">, +): SqliteTrajectoryRuntimeStats { + const database = openOpenClawAgentDatabase(toDatabaseOptions(scope)); + const db = getTrajectoryKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("trajectory_runtime_events") + .select((eb) => [ + eb.fn.count<number>("seq").as("event_count"), + eb.fn.max<number>("seq").as("max_seq"), + sqliteTrajectoryJsonlByteSize(), + ]) + .where("session_id", "=", scope.sessionId), + ); + return { + eventCount: row?.event_count ?? 0, + maxSeq: row?.max_seq ?? 0, + sizeBytes: row?.size_bytes ?? 0, + }; +} + +function getTrajectoryKysely(database: import("node:sqlite").DatabaseSync) { + return getNodeSqliteKysely<SqliteTrajectoryRuntimeDatabase>(database); +} + +function toDatabaseOptions(scope: { + agentId?: string; + env?: NodeJS.ProcessEnv; + storePath: string; +}): OpenClawAgentDatabaseOptions { + const requestedAgentId = scope.agentId ? normalizeAgentId(scope.agentId) : undefined; + const target = resolveSqliteTargetFromSessionStorePath( + scope.storePath, + requestedAgentId ? { agentId: requestedAgentId } : {}, + ); + if (requestedAgentId && target.agentId && requestedAgentId !== target.agentId) { + throw new Error( + `SQLite trajectory store path belongs to agent ${target.agentId}; requested agent ${requestedAgentId}.`, + ); + } + return { + agentId: requestedAgentId ?? target.agentId ?? DEFAULT_AGENT_ID, + ...(scope.env ? { env: scope.env } : {}), + ...(target.path ? { path: target.path } : {}), + }; +} + +function readNextTrajectorySeq(database: OpenClawAgentDatabase, sessionId: string): number { + const db = getTrajectoryKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("trajectory_runtime_events") + .select((eb) => eb.fn.max<number | bigint>("seq").as("max_seq")) + .where("session_id", "=", sessionId), + ); + if (row?.max_seq === null || row?.max_seq === undefined) { + return 0; + } + return normalizeSqliteNumber(row.max_seq) + 1; +} + +function trimSqliteTrajectoryRuntimeWindow( + database: OpenClawAgentDatabase, + sessionId: string, + maxRuntimeBytes: number, +): void { + const db = getTrajectoryKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db + .selectFrom("trajectory_runtime_events") + .select(["seq", "event_json"]) + .where("session_id", "=", sessionId) + .orderBy("seq", "asc"), + ).rows; + const removableSeqs = oldestTrajectorySeqsPastByteWindow(rows, maxRuntimeBytes); + if (removableSeqs.length === 0) { + return; + } + executeSqliteQuerySync( + database.db, + db + .deleteFrom("trajectory_runtime_events") + .where("session_id", "=", sessionId) + .where("seq", "in", removableSeqs), + ); +} + +function oldestTrajectorySeqsPastByteWindow( + rows: readonly TrajectoryRuntimeRow[], + maxRuntimeBytes: number, +): number[] { + let totalBytes = rows.reduce((total, row) => total + trajectoryJsonlRowBytes(row.event_json), 0); + const removableSeqs: number[] = []; + for (const row of rows) { + if (totalBytes <= maxRuntimeBytes) { + break; + } + removableSeqs.push(row.seq); + totalBytes -= trajectoryJsonlRowBytes(row.event_json); + } + return removableSeqs; +} + +function trajectoryJsonlRowBytes(eventJson: string): number { + return Buffer.byteLength(eventJson, "utf8") + 1; +} + +function sqliteTrajectoryJsonlByteSize() { + return /* kysely-allow-raw: JSONL size includes event bytes plus newline separators. */ sql<number>`COALESCE(SUM(LENGTH(CAST(event_json AS BLOB))), 0) + + CASE WHEN COUNT(*) > 0 THEN COUNT(*) ELSE 0 END`.as("size_bytes"); +} + +function readTrajectoryEventTimestamp(event: TrajectoryEvent): number | undefined { + const parsed = Date.parse(event.ts); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function normalizeSqliteNumber(value: number | bigint): number { + return typeof value === "bigint" ? Number(value) : value; +} diff --git a/src/trajectory/runtime.test.ts b/src/trajectory/runtime.test.ts index 9400bd347150..6a579f7145a8 100644 --- a/src/trajectory/runtime.test.ts +++ b/src/trajectory/runtime.test.ts @@ -3,12 +3,12 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { - TRAJECTORY_RUNTIME_EVENT_MAX_BYTES, - resolveTrajectoryFilePath, - resolveTrajectoryPointerFilePath, - resolveTrajectoryPointerOpenFlags, -} from "./paths.js"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import { formatSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { TRAJECTORY_RUNTIME_EVENT_MAX_BYTES, resolveTrajectoryPointerOpenFlags } from "./paths.js"; +import { loadSqliteTrajectoryRuntimeEvents } from "./runtime-store.sqlite.js"; import { createTrajectoryRuntimeRecorder, toTrajectoryToolDefinitions } from "./runtime.js"; type TrajectoryRuntimeRecorder = NonNullable<ReturnType<typeof createTrajectoryRuntimeRecorder>>; @@ -23,6 +23,8 @@ function makeTempDir(): string { afterEach(() => { vi.useRealTimers(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); for (const dir of tempDirs.splice(0)) { fs.rmSync(dir, { recursive: true, force: true }); } @@ -39,24 +41,6 @@ function expectTrajectoryRuntimeRecorder( } describe("trajectory runtime", () => { - it("resolves a session-adjacent trajectory file by default", () => { - expect( - resolveTrajectoryFilePath({ - sessionFile: "/tmp/session.jsonl", - sessionId: "session-1", - }), - ).toBe("/tmp/session.trajectory.jsonl"); - }); - - it("sanitizes session ids when resolving an override directory", () => { - expect( - resolveTrajectoryFilePath({ - env: { OPENCLAW_TRAJECTORY_DIR: "/tmp/traces" }, - sessionId: "../evil/session", - }), - ).toBe(path.join(path.resolve("/tmp/traces"), "___evil_session.jsonl")); - }); - it("records sanitized runtime events by default", () => { const writes: string[] = []; const recorder = createTrajectoryRuntimeRecorder({ @@ -105,6 +89,93 @@ describe("trajectory runtime", () => { expect(JSON.stringify(parsed.data)).not.toContain("abcd-efgh-ijkl-mnop"); }); + it("records SQLite marker runtime events without active JSONL sidecars", async () => { + const tempDir = makeTempDir(); + const storePath = path.join(tempDir, "agents", "main", "sessions", "sessions.json"); + const sessionKey = "agent:main:main"; + await replaceSessionEntry({ sessionKey, storePath }, { sessionId: "session-1", updatedAt: 10 }); + const recorder = createTrajectoryRuntimeRecorder({ + sessionId: "session-1", + sessionKey, + sessionFile: formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: "session-1", + storePath, + }), + provider: "openai", + modelId: "gpt-5.4", + modelApi: "responses", + workspaceDir: "/tmp/workspace", + }); + + const runtimeRecorder = expectTrajectoryRuntimeRecorder(recorder); + runtimeRecorder.recordEvent("context.compiled", { + prompt: "hello", + }); + runtimeRecorder.recordEvent("model.completed", { + usage: { input: 1, output: 2, total: 3 }, + }); + expect(runtimeRecorder.describeFlushState()).toContain("pendingRows=2"); + await runtimeRecorder.flush(); + + await expect( + loadSqliteTrajectoryRuntimeEvents({ sessionId: "session-1", storePath }), + ).resolves.toEqual([ + expect.objectContaining({ source: "runtime", type: "context.compiled" }), + expect.objectContaining({ source: "runtime", type: "model.completed" }), + ]); + expect(fs.existsSync(path.join(path.dirname(storePath), "trajectory", "session-1.jsonl"))).toBe( + false, + ); + }); + + it("stores bounded oversized runtime events in SQLite", async () => { + const tempDir = makeTempDir(); + const storePath = path.join(tempDir, "agents", "main", "sessions", "sessions.json"); + const sessionKey = "agent:main:main"; + const usage = { + input: 384_954, + output: 5_624, + cacheRead: 333_824, + reasoningTokens: 2_038, + total: 724_402, + }; + await replaceSessionEntry({ sessionKey, storePath }, { sessionId: "session-1", updatedAt: 10 }); + const recorder = createTrajectoryRuntimeRecorder({ + sessionId: "session-1", + sessionKey, + sessionFile: formatSqliteSessionFileMarker({ + agentId: "main", + sessionId: "session-1", + storePath, + }), + }); + + const runtimeRecorder = expectTrajectoryRuntimeRecorder(recorder); + runtimeRecorder.recordEvent("model.completed", { + usage, + messagesSnapshot: Array.from({ length: 12 }, (_value, index) => ({ + role: index % 2 === 0 ? "user" : "assistant", + content: `message-${index} ${"x".repeat(32_000)}`, + })), + }); + await runtimeRecorder.flush(); + + const [event] = await loadSqliteTrajectoryRuntimeEvents({ sessionId: "session-1", storePath }); + expect(event).toMatchObject({ + type: "model.completed", + data: { + truncated: true, + reason: "trajectory-event-size-limit", + usage, + }, + }); + expect(event?.data?.messagesSnapshot).toBeUndefined(); + expect(Buffer.byteLength(JSON.stringify(event), "utf8")).toBeLessThanOrEqual( + TRAJECTORY_RUNTIME_EVENT_MAX_BYTES, + ); + }); + it("bounds large runtime event fields before serialization", () => { const writes: string[] = []; const recorder = createTrajectoryRuntimeRecorder({ @@ -311,130 +382,6 @@ describe("trajectory runtime", () => { expect(preservedUsage).not.toContain("sk-other-secret-token"); }); - it("rotates runtime capture at the file budget and keeps newer events", async () => { - const tmpDir = makeTempDir(); - const sessionFile = path.join(tmpDir, "session.jsonl"); - const maxRuntimeFileBytes = 1_600; - const firstRecorder = createTrajectoryRuntimeRecorder({ - sessionId: "session-1", - sessionFile, - maxRuntimeFileBytes, - }); - - const firstRuntimeRecorder = expectTrajectoryRuntimeRecorder(firstRecorder); - for (const marker of ["old-1", "old-2", "old-3"]) { - firstRuntimeRecorder.recordEvent("prompt.submitted", { - marker, - prompt: "x".repeat(260), - }); - } - await firstRuntimeRecorder.flush(); - - const secondRecorder = createTrajectoryRuntimeRecorder({ - sessionId: "session-1", - sessionFile, - maxRuntimeFileBytes, - }); - const secondRuntimeRecorder = expectTrajectoryRuntimeRecorder(secondRecorder); - for (const marker of ["new-1", "new-2", "new-3"]) { - secondRuntimeRecorder.recordEvent("prompt.submitted", { - marker, - prompt: "y".repeat(260), - }); - } - await secondRuntimeRecorder.flush(); - - const runtimeFile = resolveTrajectoryFilePath({ sessionFile, sessionId: "session-1" }); - const raw = fs.readFileSync(runtimeFile, "utf8"); - expect(Buffer.byteLength(raw, "utf8")).toBeLessThanOrEqual(maxRuntimeFileBytes); - expect(raw).not.toContain("old-1"); - expect(raw).toContain("new-3"); - }); - - it.runIf(process.platform !== "win32")( - "preserves existing trajectory directory permissions", - async () => { - const tmpDir = makeTempDir(); - fs.chmodSync(tmpDir, 0o755); - const sessionFile = path.join(tmpDir, "session.jsonl"); - const recorder = createTrajectoryRuntimeRecorder({ - sessionId: "session-1", - sessionFile, - maxRuntimeFileBytes: 1_600, - }); - - const runtimeRecorder = expectTrajectoryRuntimeRecorder(recorder); - runtimeRecorder.recordEvent("prompt.submitted", { - prompt: "hello", - }); - await runtimeRecorder.flush(); - - expect(fs.statSync(tmpDir).mode & 0o777).toBe(0o755); - }, - ); - - it("merges stale recorder flushes with newer runtime events", async () => { - const tmpDir = makeTempDir(); - const sessionFile = path.join(tmpDir, "session.jsonl"); - const staleRecorder = createTrajectoryRuntimeRecorder({ - sessionId: "session-1", - sessionFile, - maxRuntimeFileBytes: 2_400, - }); - - const staleRuntimeRecorder = expectTrajectoryRuntimeRecorder(staleRecorder); - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z")); - staleRuntimeRecorder.recordEvent("prompt.submitted", { - marker: "old-recorder", - prompt: "x".repeat(260), - }); - - const newerRecorder = createTrajectoryRuntimeRecorder({ - sessionId: "session-1", - sessionFile, - maxRuntimeFileBytes: 2_400, - }); - const newerRuntimeRecorder = expectTrajectoryRuntimeRecorder(newerRecorder); - newerRuntimeRecorder.recordEvent("prompt.submitted", { - marker: "new-recorder", - prompt: "y".repeat(260), - }); - vi.useRealTimers(); - await newerRuntimeRecorder.flush(); - await staleRuntimeRecorder.flush(); - - const runtimeFile = resolveTrajectoryFilePath({ sessionFile, sessionId: "session-1" }); - const raw = fs.readFileSync(runtimeFile, "utf8"); - expect(raw).toContain("old-recorder"); - expect(raw).toContain("new-recorder"); - expect(raw.indexOf("old-recorder")).toBeLessThan(raw.indexOf("new-recorder")); - }); - - it.runIf(process.platform !== "win32")( - "refuses runtime capture through symlinked parent directories", - async () => { - const tmpDir = makeTempDir(); - const targetDir = path.join(tmpDir, "target"); - const linkDir = path.join(tmpDir, "link"); - fs.mkdirSync(targetDir); - fs.symlinkSync(targetDir, linkDir); - const recorder = createTrajectoryRuntimeRecorder({ - sessionId: "session-1", - sessionFile: path.join(linkDir, "session.jsonl"), - maxRuntimeFileBytes: 2_400, - }); - - const runtimeRecorder = expectTrajectoryRuntimeRecorder(recorder); - runtimeRecorder.recordEvent("prompt.submitted", { - prompt: "hello", - }); - await runtimeRecorder.flush(); - - expect(fs.existsSync(path.join(targetDir, "session.trajectory.jsonl"))).toBe(false); - }, - ); - it("describes queued writer state for cleanup timeout logs", () => { const recorder = createTrajectoryRuntimeRecorder({ sessionId: "session-1", @@ -462,28 +409,6 @@ describe("trajectory runtime", () => { ); }); - it("writes a session-adjacent pointer when using an override directory", () => { - const tmpDir = makeTempDir(); - const sessionFile = path.join(tmpDir, "session.jsonl"); - const trajectoryDir = path.join(tmpDir, "traces"); - const recorder = createTrajectoryRuntimeRecorder({ - env: { OPENCLAW_TRAJECTORY_DIR: trajectoryDir }, - sessionId: "session-1", - sessionFile, - writer: { - filePath: path.join(trajectoryDir, "session-1.jsonl"), - write: () => undefined, - flush: async () => undefined, - }, - }); - - expectTrajectoryRuntimeRecorder(recorder); - const pointer = JSON.parse( - fs.readFileSync(resolveTrajectoryPointerFilePath(sessionFile), "utf8"), - ) as { runtimeFile?: string }; - expect(pointer.runtimeFile).toBe(path.join(trajectoryDir, "session-1.jsonl")); - }); - it("keeps pointer write flags usable when O_NOFOLLOW is unavailable", () => { expect( resolveTrajectoryPointerOpenFlags({ diff --git a/src/trajectory/runtime.ts b/src/trajectory/runtime.ts index 26aa70c96552..666641165fb1 100644 --- a/src/trajectory/runtime.ts +++ b/src/trajectory/runtime.ts @@ -1,26 +1,19 @@ -// Trajectory runtime records runtime events into trajectory log files. -import fs from "node:fs"; -import path from "node:path"; -import { KeyedAsyncQueue } from "openclaw/plugin-sdk/keyed-async-queue"; +// Trajectory runtime records bounded session events into SQLite-backed storage. import { sanitizeDiagnosticPayload } from "../agents/payload-redaction.js"; import type { QueuedFileWriter, QueuedFileWriterDiagnostics, } from "../agents/queued-file-writer.js"; +import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { assertNoSymlinkParents, writeSiblingTempFile } from "../infra/fs-safe-advanced.js"; -import { readRegularFileSync } from "../infra/fs-safe.js"; import { redactSecrets } from "../logging/redact.js"; import { parseBooleanValue } from "../utils/boolean.js"; import { safeJsonStringify } from "../utils/safe-json.js"; import { TRAJECTORY_RUNTIME_CAPTURE_MAX_BYTES, TRAJECTORY_RUNTIME_EVENT_MAX_BYTES, - TRAJECTORY_RUNTIME_FILE_MAX_BYTES, - resolveTrajectoryFilePath, - resolveTrajectoryPointerFilePath, - resolveTrajectoryPointerOpenFlags, } from "./paths.js"; +import { appendSqliteTrajectoryRuntimeEvents } from "./runtime-store.sqlite.js"; import type { TrajectoryEvent, TrajectoryToolDefinition } from "./types.js"; type TrajectoryRuntimeInit = { @@ -40,87 +33,30 @@ type TrajectoryRuntimeInit = { type TrajectoryRuntimeRecorder = { enabled: true; - filePath: string; recordEvent: (type: string, data?: Record<string, unknown>) => void; flush: () => Promise<void>; describeFlushState: () => string | undefined; }; -const writers = new Map<string, TrajectoryRuntimeWriter>(); -const windowFlushes = new KeyedAsyncQueue(); -const MAX_TRAJECTORY_WRITERS = 100; const TRAJECTORY_RUNTIME_DATA_STRING_MAX_CHARS = 32_768; const TRAJECTORY_RUNTIME_DATA_ARRAY_MAX_ITEMS = 64; const TRAJECTORY_RUNTIME_DATA_OBJECT_MAX_KEYS = 64; const TRAJECTORY_RUNTIME_DATA_MAX_DEPTH = 6; const TRAJECTORY_RUNTIME_OVERSIZE_PRESERVED_DATA_KEYS = ["usage", "promptCache"] as const; -type TrajectoryRuntimeWriterDiagnostics = Omit<QueuedFileWriterDiagnostics, "activeOperation"> & { - activeOperation: QueuedFileWriterDiagnostics["activeOperation"] | "file-replace"; -}; +type TrajectoryRuntimeWriterDiagnostics = QueuedFileWriterDiagnostics; type TrajectoryRuntimeWriter = Omit<QueuedFileWriter, "describeQueue"> & { describeQueue?: () => TrajectoryRuntimeWriterDiagnostics; nextSourceSeq?: () => number; }; -function writeTrajectoryPointerBestEffort(params: { - filePath: string; - sessionFile?: string; - sessionId: string; -}): void { - if (!params.sessionFile) { - return; - } - const pointerPath = resolveTrajectoryPointerFilePath(params.sessionFile); - try { - const pointerDir = path.resolve(path.dirname(pointerPath)); - if (fs.lstatSync(pointerDir).isSymbolicLink()) { - return; - } - try { - if (fs.lstatSync(pointerPath).isSymbolicLink()) { - return; - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - return; - } - } - const fd = fs.openSync(pointerPath, resolveTrajectoryPointerOpenFlags(), 0o600); - try { - fs.writeFileSync( - fd, - `${JSON.stringify( - { - traceSchema: "openclaw-trajectory-pointer", - schemaVersion: 1, - sessionId: params.sessionId, - runtimeFile: params.filePath, - }, - null, - 2, - )}\n`, - "utf8", - ); - fs.fchmodSync(fd, 0o600); - } finally { - fs.closeSync(fd); - } - } catch { - // Pointer files are best-effort; the runtime sidecar itself is authoritative. - } -} - -function trimTrajectoryWriterCache(): void { - while (writers.size >= MAX_TRAJECTORY_WRITERS) { - const oldestKey = writers.keys().next().value; - if (!oldestKey) { - return; - } - writers.delete(oldestKey); - } -} +type TrajectoryRuntimeSink = { + describeFlushState: () => string | undefined; + flush: () => Promise<void>; + nextSourceSeq?: () => number; + write: (event: TrajectoryEvent, line: string) => void; +}; function truncateOversizedTrajectoryEvent( event: TrajectoryEvent, @@ -275,201 +211,59 @@ function describeTrajectoryWriterFlushState(writer: TrajectoryRuntimeWriter): st return parts.join(" "); } -function trimJsonlWindow(lines: string[], maxBytes: number): number { - let bytes = 0; - for (const line of lines) { - bytes += Buffer.byteLength(line, "utf8"); - } - while (bytes > maxBytes && lines.length > 0) { - const line = lines.shift(); - if (line !== undefined) { - bytes -= Buffer.byteLength(line, "utf8"); - } - } - return bytes; -} - -function compareTrajectoryWindowLines(left: string, right: string): number { - const leftEvent = parseTrajectoryWindowLine(left); - const rightEvent = parseTrajectoryWindowLine(right); - const byTs = leftEvent.ts - rightEvent.ts; - if (byTs !== 0) { - return byTs; - } - return leftEvent.seq - rightEvent.seq; -} - -function parseTrajectoryWindowLine(line: string): { ts: number; seq: number } { - try { - const parsed = JSON.parse(line) as { ts?: unknown; sourceSeq?: unknown; seq?: unknown }; - const ts = typeof parsed.ts === "string" ? Date.parse(parsed.ts) : Number.POSITIVE_INFINITY; - const sourceSeq = typeof parsed.sourceSeq === "number" ? parsed.sourceSeq : undefined; - const seq = typeof parsed.seq === "number" ? parsed.seq : undefined; - return { - ts: Number.isFinite(ts) ? ts : Number.POSITIVE_INFINITY, - seq: sourceSeq ?? seq ?? Number.POSITIVE_INFINITY, - }; - } catch { - return { ts: Number.POSITIVE_INFINITY, seq: Number.POSITIVE_INFINITY }; - } -} - -function readMaxTrajectorySourceSeq(filePath: string): number { - return readTrajectoryWindowLines(filePath, TRAJECTORY_RUNTIME_FILE_MAX_BYTES).reduce( - (max, line) => { - try { - const parsed = JSON.parse(line) as { sourceSeq?: unknown; seq?: unknown }; - const seq = - typeof parsed.sourceSeq === "number" - ? parsed.sourceSeq - : typeof parsed.seq === "number" - ? parsed.seq - : 0; - return Math.max(max, seq); - } catch { - return max; - } - }, - 0, - ); -} - -function readTrajectoryWindowLines(filePath: string, maxBytes: number): string[] { - try { - const raw = readRegularFileSync({ - filePath, - maxBytes: TRAJECTORY_RUNTIME_FILE_MAX_BYTES, - }).buffer.toString("utf8"); - const lines = raw - .split(/\r?\n/u) - .filter((line) => line.length > 0) - .map((line) => `${line}\n`); - trimJsonlWindow(lines, maxBytes); - return lines; - } catch { - return []; - } -} - -async function replaceTrajectoryWindow(params: { - filePath: string; - maxFileBytes: number; - appendedLines: string[]; -}): Promise<void> { - const dir = path.dirname(params.filePath); - await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 }); - await assertNoSymlinkParents({ - rootDir: path.parse(path.resolve(dir)).root, - targetPath: path.resolve(dir), - allowMissing: false, - allowRootChildSymlink: true, - requireDirectories: true, - messagePrefix: "Refusing to write trajectory under", - }); - const lines = readTrajectoryWindowLines(params.filePath, params.maxFileBytes); - lines.push(...params.appendedLines); - lines.sort(compareTrajectoryWindowLines); - trimJsonlWindow(lines, params.maxFileBytes); - await writeSiblingTempFile({ - dir, - chmodDir: false, - mode: 0o600, - tempPrefix: ".openclaw-trajectory-", - writeTemp: async (tempPath) => { - await fs.promises.writeFile(tempPath, lines.join(""), { - encoding: "utf8", - mode: 0o600, - }); - }, - resolveFinalPath: () => params.filePath, - }); -} - -async function queueTrajectoryWindowFlush(params: { - filePath: string; - maxFileBytes: number; - appendedLines: string[]; -}): Promise<void> { - await windowFlushes.enqueue(params.filePath, async () => { - await replaceTrajectoryWindow(params); - }); -} - -function createTrajectoryWindowWriter( - filePath: string, - maxFileBytes: number, -): TrajectoryRuntimeWriter { - let pendingLines: string[] = []; - let queuedBytes = 0; - let pendingWrites = 0; - let activeOperation: TrajectoryRuntimeWriterDiagnostics["activeOperation"] = "idle"; - let queue: Promise<unknown> = Promise.resolve(); - let sourceSeq = readMaxTrajectorySourceSeq(filePath); - +function createFileTrajectoryRuntimeSink(writer: TrajectoryRuntimeWriter): TrajectoryRuntimeSink { return { - filePath, - write: (line) => { - const lineBytes = Buffer.byteLength(line, "utf8"); - if (lineBytes > maxFileBytes) { - return "dropped"; - } - pendingLines.push(line); - queuedBytes += lineBytes; - queuedBytes = trimJsonlWindow(pendingLines, maxFileBytes); - pendingWrites = 1; - return "queued"; - }, + describeFlushState: () => describeTrajectoryWriterFlushState(writer), flush: async () => { - if (pendingLines.length === 0) { - await queue; - return; - } - const appendedLines = pendingLines; - pendingLines = []; - queuedBytes = 0; - queue = queue - .then(async () => { - activeOperation = "file-replace"; - await queueTrajectoryWindowFlush({ - filePath, - maxFileBytes, - appendedLines, - }); - }) - .catch(() => undefined) - .finally(() => { - pendingWrites = pendingLines.length > 0 ? 1 : 0; - activeOperation = "idle"; - }); - await queue; + await writer.flush(); }, - describeQueue: () => ({ - pendingWrites, - queuedBytes, - activeOperation, - maxFileBytes, - maxQueuedBytes: maxFileBytes, - yieldBeforeWrite: false, - }), - nextSourceSeq: () => { - sourceSeq += 1; - return sourceSeq; + nextSourceSeq: writer.nextSourceSeq, + write: (_event, line) => { + writer.write(`${line}\n`); }, }; } -function getTrajectoryWindowWriter( - filePath: string, - maxFileBytes: number, -): TrajectoryRuntimeWriter { - const existing = writers.get(filePath); - if (existing) { - return existing; +function createSqliteTrajectoryRuntimeSink(params: { + env: NodeJS.ProcessEnv; + maxRuntimeFileBytes: number; + sessionFile?: string; + sessionId: string; +}): TrajectoryRuntimeSink | null { + const marker = parseSqliteSessionFileMarker(params.sessionFile); + if (!marker || marker.sessionId !== params.sessionId) { + return null; } - trimTrajectoryWriterCache(); - const writer = createTrajectoryWindowWriter(filePath, maxFileBytes); - writers.set(filePath, writer); - return writer; + let pendingEvents: TrajectoryEvent[] = []; + let queuedBytes = 0; + return { + describeFlushState: () => + pendingEvents.length > 0 + ? `pendingRows=${pendingEvents.length} queuedBytes=${queuedBytes} activeOperation=sqlite-append` + : undefined, + flush: async () => { + if (pendingEvents.length === 0) { + return; + } + const events = pendingEvents; + pendingEvents = []; + queuedBytes = 0; + appendSqliteTrajectoryRuntimeEvents( + { + agentId: marker.agentId, + env: params.env, + maxRuntimeBytes: params.maxRuntimeFileBytes, + sessionId: marker.sessionId, + storePath: marker.storePath, + }, + events, + ); + }, + write: (event, line) => { + pendingEvents.push(event); + queuedBytes += Buffer.byteLength(line, "utf8") + 1; + }, + }; } export function toTrajectoryToolDefinitions( @@ -503,32 +297,30 @@ export function createTrajectoryRuntimeRecorder( return null; } - const filePath = resolveTrajectoryFilePath({ - env, - sessionFile: params.sessionFile, - sessionId: params.sessionId, - }); const maxRuntimeFileBytes = Math.max( 1, Math.floor(params.maxRuntimeFileBytes ?? TRAJECTORY_RUNTIME_CAPTURE_MAX_BYTES), ); - const writer = params.writer ?? getTrajectoryWindowWriter(filePath, maxRuntimeFileBytes); - writeTrajectoryPointerBestEffort({ - filePath, - sessionFile: params.sessionFile, - sessionId: params.sessionId, - }); + const sink = params.writer + ? createFileTrajectoryRuntimeSink(params.writer) + : createSqliteTrajectoryRuntimeSink({ + env, + maxRuntimeFileBytes, + sessionFile: params.sessionFile, + sessionId: params.sessionId, + }); + if (!sink) { + return null; + } let seq = 0; const traceId = params.sessionId; - const writeBoundedLine = (line: string): void => { - const jsonlLine = `${line}\n`; - writer.write(jsonlLine); - }; - - const buildEventLine = (type: string, data?: Record<string, unknown>): string | undefined => { + const buildEvent = ( + type: string, + data?: Record<string, unknown>, + ): { event: TrajectoryEvent; line: string } | undefined => { const nextSeq = seq + 1; - const sourceSeq = writer.nextSourceSeq?.() ?? nextSeq; + const sourceSeq = sink.nextSourceSeq?.() ?? nextSeq; const event: TrajectoryEvent = { traceSchema: "openclaw-trajectory", schemaVersion: 1, @@ -555,23 +347,23 @@ export function createTrajectoryRuntimeRecorder( if (!boundedLine) { return undefined; } + const boundedEvent = JSON.parse(boundedLine) as TrajectoryEvent; seq = nextSeq; - return boundedLine; + return { event: boundedEvent, line: boundedLine }; }; return { enabled: true, - filePath, recordEvent: (type, data) => { - const line = buildEventLine(type, data); - if (!line) { + const built = buildEvent(type, data); + if (!built) { return; } - writeBoundedLine(line); + sink.write(built.event, built.line); }, flush: async () => { - await writer.flush(); + await sink.flush(); }, - describeFlushState: () => describeTrajectoryWriterFlushState(writer), + describeFlushState: () => sink.describeFlushState(), }; } diff --git a/src/version.test.ts b/src/version.test.ts index 1d2953a67201..1c0d80c4313f 100644 --- a/src/version.test.ts +++ b/src/version.test.ts @@ -167,15 +167,18 @@ describe("version resolution", () => { } it("prefers runtime VERSION over stale OPENCLAW_VERSION for compatibility checks", () => { + const previousCompatibility = process.env.OPENCLAW_COMPATIBILITY_HOST_VERSION; const previous = process.env.OPENCLAW_VERSION; const previousService = process.env.OPENCLAW_SERVICE_VERSION; const previousPackage = process.env.npm_package_version; try { + delete process.env.OPENCLAW_COMPATIBILITY_HOST_VERSION; process.env.OPENCLAW_VERSION = "2026.3.25"; process.env.OPENCLAW_SERVICE_VERSION = "2026.3.25-service"; process.env.npm_package_version = "2026.3.25-package"; expect(resolveCompatibilityHostVersion()).toBe(VERSION); } finally { + restoreEnvValue("OPENCLAW_COMPATIBILITY_HOST_VERSION", previousCompatibility); restoreEnvValue("OPENCLAW_VERSION", previous); restoreEnvValue("OPENCLAW_SERVICE_VERSION", previousService); restoreEnvValue("npm_package_version", previousPackage); diff --git a/test/scripts/changed-lanes.test.ts b/test/scripts/changed-lanes.test.ts index 1b62835108d0..7b7bc527503a 100644 --- a/test/scripts/changed-lanes.test.ts +++ b/test/scripts/changed-lanes.test.ts @@ -27,6 +27,7 @@ import { shouldRunPromptSnapshotOwnerTest, shouldRunRuntimeSidecarBaselineCheck, shouldRunShrinkwrapGuard, + shouldRunSqliteSessionSchemaBaselineCheck, shouldRunTestTempCreationReport, createShrinkwrapGuardCommand, } from "../../scripts/check-changed.mjs"; @@ -1585,6 +1586,26 @@ describe("scripts/changed-lanes", () => { ); }); + it("runs SQLite sessions/transcripts schema baseline checks for baseline owner surfaces", () => { + expect( + shouldRunSqliteSessionSchemaBaselineCheck([ + "src/state/openclaw-agent-schema.sql", + "scripts/generate-sqlite-session-schema-baseline.ts", + "scripts/lib/sqlite-session-schema-baseline.ts", + "test/scripts/sqlite-session-schema-baseline.test.ts", + "docs/.generated/sqlite-session-transcript-schema-baseline.sha256", + ]), + ).toBe(true); + + const result = detectChangedLanes(["src/state/openclaw-agent-schema.sql"]); + const plan = createChangedCheckPlan(result); + + expect(plan.commands).toContainEqual({ + name: "SQLite sessions/transcripts schema baseline", + args: ["sqlite:sessions-schema:check"], + }); + }); + it("guards release metadata package changes to the top-level version field", () => { const dir = makeTempRepoRoot(tempDirs, "openclaw-release-metadata-"); git(dir, ["init", "-q", "--initial-branch=main"]); diff --git a/test/scripts/check-session-accessor-boundary.test.ts b/test/scripts/check-session-accessor-boundary.test.ts index 3fc779fc6aa4..f3d9ed48ebcc 100644 --- a/test/scripts/check-session-accessor-boundary.test.ts +++ b/test/scripts/check-session-accessor-boundary.test.ts @@ -206,17 +206,7 @@ describe("session accessor boundary guard", () => { }); it("ratchets only explicit file-backed SDK session compatibility exports", () => { - expect(allowedSessionStoreRuntimeFileBackedCompatExports).toEqual( - new Set([ - "loadSessionStore", - "readLatestAssistantTextFromSessionTranscript", - "resolveAndPersistSessionFile", - "resolveSessionFilePath", - "resolveSessionStoreEntry", - "saveSessionStore", - "updateSessionStore", - ]), - ); + expect(allowedSessionStoreRuntimeFileBackedCompatExports).toEqual(new Set([])); }); it("collects file-backed SDK session compatibility exports", () => { @@ -413,7 +403,7 @@ describe("session accessor boundary guard", () => { it("flags legacy transcript writer imports", () => { expect( findTranscriptWriterBoundaryViolations(` - import { appendSessionTranscriptMessage } from "../config/sessions/transcript-append.js"; + import { appendSessionTranscriptMessage } from "../config/sessions/transcript-append.test-support.js"; import { emitSessionTranscriptUpdate as emitUpdate } from "../sessions/transcript-events.js"; `), ).toEqual([ diff --git a/test/scripts/check-sqlite-transaction-boundary.test.ts b/test/scripts/check-sqlite-transaction-boundary.test.ts new file mode 100644 index 000000000000..a5f18328a53f --- /dev/null +++ b/test/scripts/check-sqlite-transaction-boundary.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "vitest"; +import { findSqliteTransactionBoundaryViolations } from "../../scripts/check-sqlite-transaction-boundary.mjs"; + +describe("SQLite transaction boundary guard", () => { + it("rejects removed async transaction primitives", () => { + expect( + findSqliteTransactionBoundaryViolations(` + import { runSqliteImmediateTransactionAsync } from "./sqlite-transaction.js"; + export async function runOpenClawAgentWriteTransactionAsync() {} + await database.runSqliteImmediateTransactionAsync(async () => undefined); + `), + ).toEqual([ + { + line: 2, + reason: + 'imports removed async SQLite transaction primitive "runSqliteImmediateTransactionAsync"', + }, + { + line: 3, + reason: + 'declares removed async SQLite transaction primitive "runOpenClawAgentWriteTransactionAsync"', + }, + { + line: 4, + reason: + 'calls removed async SQLite transaction primitive "runSqliteImmediateTransactionAsync"', + }, + ]); + }); + + it("rejects inline async callbacks passed to synchronous transaction helpers", () => { + expect( + findSqliteTransactionBoundaryViolations(` + runSqliteImmediateTransactionSync(db, async () => await prepare()); + runOpenClawAgentWriteTransaction(async (database) => await write(database), options); + `), + ).toEqual([ + { + line: 2, + reason: + 'passes an async callback to synchronous SQLite transaction helper "runSqliteImmediateTransactionSync"', + }, + { + line: 3, + reason: + 'passes an async callback to synchronous SQLite transaction helper "runOpenClawAgentWriteTransaction"', + }, + ]); + }); + + it("rejects local async function references passed as callbacks", () => { + expect( + findSqliteTransactionBoundaryViolations(` + async function writeRows() {} + const writeAgentRows = async () => undefined; + runSqliteImmediateTransactionSync(db, writeRows); + runOpenClawAgentWriteTransaction(writeAgentRows, options); + `), + ).toEqual([ + { + line: 4, + reason: + 'passes an async callback to synchronous SQLite transaction helper "runSqliteImmediateTransactionSync"', + }, + { + line: 5, + reason: + 'passes an async callback to synchronous SQLite transaction helper "runOpenClawAgentWriteTransaction"', + }, + ]); + }); + + it("tracks aliases of synchronous transaction imports", () => { + expect( + findSqliteTransactionBoundaryViolations(` + import { runSqliteImmediateTransactionSync as transact } from "./sqlite-transaction.js"; + transact(db, async () => undefined); + `), + ).toEqual([ + { + line: 3, + reason: + 'passes an async callback to synchronous SQLite transaction helper "runSqliteImmediateTransactionSync"', + }, + ]); + }); + + it("allows asynchronous preparation followed by a synchronous commit callback", () => { + expect( + findSqliteTransactionBoundaryViolations(` + const prepared = await prepareMutation(); + runOpenClawAgentWriteTransaction((database) => { + validate(database, prepared.expected); + apply(database, prepared.patch); + }, options); + `), + ).toEqual([]); + }); +}); diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 6d6d21579984..a5aa5701419b 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -1322,6 +1322,11 @@ describe("ci workflow guards", () => { group: "session-accessor-boundary", runner: "blacksmith-4vcpu-ubuntu-2404", }); + expect(workflow.jobs["check-additional-shard"].strategy.matrix.include).toContainEqual({ + check_name: "check-sqlite-session-schema-baseline", + group: "sqlite-session-schema-baseline", + runner: "blacksmith-4vcpu-ubuntu-2404", + }); expect(workflow.jobs["checks-windows"]["runs-on"]).toContain("matrix.runner"); expect(source).toContain("blacksmith-8vcpu-windows-2025"); }); @@ -1364,6 +1369,25 @@ describe("ci workflow guards", () => { ); }); + it("runs the SQLite transaction ratchet in the session boundary check", () => { + const workflow = readCiWorkflow(); + const additionalJob = workflow.jobs["check-additional-shard"]; + const matrixRows = additionalJob.strategy.matrix.include; + expect(matrixRows).toContainEqual({ + check_name: "check-session-accessor-boundary", + group: "session-accessor-boundary", + runner: "blacksmith-4vcpu-ubuntu-2404", + }); + + const runStep = additionalJob.steps.find( + (step: WorkflowStep) => step.name === "Run additional check shard", + ); + expect(runStep.run).toContain("session-accessor-boundary)"); + expect(runStep.run).toContain( + 'run_check "lint:tmp:sqlite-transaction-boundary" pnpm run lint:tmp:sqlite-transaction-boundary', + ); + }); + it("kills timed manual checkout fetches after the grace period", () => { const workflowPaths = [ [".github/workflows/ci.yml", "120s"], @@ -1454,16 +1478,26 @@ describe("ci workflow guards", () => { } }); - it("runs plugin SDK API and surface drift checks in workflow sanity", () => { + it("runs generated baseline drift checks in workflow sanity", () => { const workflow = readWorkflowSanityWorkflow(); const steps = workflow.jobs["generated-doc-baselines"].steps; const stepNames = steps.map((step: WorkflowStep) => step.name); expect(stepNames).toContain("Check plugin SDK API baseline drift"); + expect(stepNames).toContain("Check SQLite sessions/transcripts schema baseline drift"); expect(stepNames).toContain("Check plugin SDK surface budget"); expect(stepNames.indexOf("Check plugin SDK API baseline drift")).toBeLessThan( - stepNames.indexOf("Check plugin SDK surface budget"), + stepNames.indexOf("Check SQLite sessions/transcripts schema baseline drift"), ); + expect( + stepNames.indexOf("Check SQLite sessions/transcripts schema baseline drift"), + ).toBeLessThan(stepNames.indexOf("Check plugin SDK surface budget")); + expect( + steps.find( + (step: WorkflowStep) => + step.name === "Check SQLite sessions/transcripts schema baseline drift", + ).run, + ).toBe("pnpm sqlite:sessions-schema:check"); expect( steps.find((step: WorkflowStep) => step.name === "Check plugin SDK surface budget").run, ).toBe("pnpm plugin-sdk:surface:check"); diff --git a/test/scripts/run-vitest.test.ts b/test/scripts/run-vitest.test.ts index 0c02aac28254..7a26d46cb298 100644 --- a/test/scripts/run-vitest.test.ts +++ b/test/scripts/run-vitest.test.ts @@ -495,6 +495,7 @@ describe("scripts/run-vitest", () => { "--config=test/vitest/vitest.contracts-plugin.config.ts", "--config=test/vitest/vitest.infra.config.ts", "--config=test/vitest/vitest.gateway-core.config.ts", + "--config=test/vitest/vitest.gateway-server.config.ts", ]) { expect(resolveRunVitestSpawnEnv({ PATH: "/usr/bin" }, ["run", configArg])).toEqual({ PATH: "/usr/bin", @@ -551,6 +552,13 @@ describe("scripts/run-vitest", () => { "/repo/test/vitest/vitest.gateway-core.config.ts", ]), ).toBe(DEFAULT_EXTRA_LONG_RUNNING_VITEST_NO_OUTPUT_TIMEOUT_MS); + expect( + resolveDefaultVitestNoOutputTimeoutMs([ + "run", + "--config", + "/repo/test/vitest/vitest.gateway-server.config.ts", + ]), + ).toBe(DEFAULT_EXTRA_LONG_RUNNING_VITEST_NO_OUTPUT_TIMEOUT_MS); }); it("does not default implicit interactive runs to the stall watchdog", () => { diff --git a/test/scripts/session-log-mentions.test.ts b/test/scripts/session-log-mentions.test.ts index ec8a055a4106..e9d053ab5e54 100644 --- a/test/scripts/session-log-mentions.test.ts +++ b/test/scripts/session-log-mentions.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import fs from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { afterEach, describe, expect, it } from "vitest"; import { countSessionLogMentions, @@ -61,7 +62,7 @@ describe("session log mention scanner", () => { }), JSON.stringify({ role: "assistant", - content: "API.read MCP.fixture fixture__lookup_note", + content: 'API.read MCP.fixture fixture__lookup_note tools.search("lookup note")', }), "raw transcript fallback API.read", "", @@ -75,12 +76,114 @@ describe("session log mention scanner", () => { apiFileRead: "API.read", mcpNamespace: "MCP.fixture", mcpTool: "fixture__lookup_note", + toolSearchPollution: 'tools.search("lookup note"', }, }), ).resolves.toEqual({ apiFileRead: 2, mcpNamespace: 1, mcpTool: 1, + toolSearchPollution: 1, + }); + }); + + it("counts mentions from SQLite transcript rows", async () => { + const root = makeTempRoot(); + const sessionsDir = path.join(root, "agents", "main", "sessions"); + const sqlitePath = path.join(root, "agents", "main", "agent", "openclaw-agent.sqlite"); + await fs.mkdir(path.dirname(sqlitePath), { recursive: true }); + const db = new DatabaseSync(sqlitePath); + try { + db.exec(` + CREATE TABLE transcript_events ( + session_id TEXT NOT NULL, + seq INTEGER NOT NULL, + event_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, seq) + ); + `); + const insert = db.prepare( + "INSERT INTO transcript_events (session_id, seq, event_json, created_at) VALUES (?, ?, ?, ?)", + ); + insert.run( + "sqlite-session", + 1, + JSON.stringify({ + message: { + role: "user", + content: "Use API.read and MCP.fixture from the prompt.", + }, + }), + 1, + ); + insert.run( + "sqlite-session", + 2, + JSON.stringify({ + message: { + role: "assistant", + content: 'API.read MCP.fixture fixture__lookup_note tools.search("lookup note")', + }, + }), + 2, + ); + } finally { + db.close(); + } + + await expect( + countSessionLogMentions({ + sessionsDir, + needles: { + apiFileRead: "API.read", + mcpNamespace: "MCP.fixture", + mcpTool: "fixture__lookup_note", + toolSearchPollution: 'tools.search("lookup note"', + }, + }), + ).resolves.toEqual({ + apiFileRead: 1, + mcpNamespace: 1, + mcpTool: 1, + toolSearchPollution: 1, + }); + }); + + it("rejects oversized SQLite transcript rows before counting them", async () => { + const root = makeTempRoot(); + const sessionsDir = path.join(root, "agents", "main", "sessions"); + const sqlitePath = path.join(root, "agents", "main", "agent", "openclaw-agent.sqlite"); + await fs.mkdir(path.dirname(sqlitePath), { recursive: true }); + const db = new DatabaseSync(sqlitePath); + try { + db.exec(` + CREATE TABLE transcript_events ( + session_id TEXT NOT NULL, + seq INTEGER NOT NULL, + event_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, seq) + ); + `); + db.prepare( + "INSERT INTO transcript_events (session_id, seq, event_json, created_at) VALUES (?, ?, ?, ?)", + ).run("sqlite-session", 1, "API.read ".repeat(16), 1); + } finally { + db.close(); + } + + await expect( + countSessionLogMentions({ + limits: { fileMaxBytes: 32, totalMaxBytes: 1024 }, + sessionsDir, + needles: { + apiFileRead: "API.read", + }, + }), + ).rejects.toMatchObject({ + code: "ETOOBIG", + message: expect.stringContaining("per-file limit"), }); }); diff --git a/test/scripts/sqlite-session-schema-baseline.test.ts b/test/scripts/sqlite-session-schema-baseline.test.ts new file mode 100644 index 000000000000..4743492feaeb --- /dev/null +++ b/test/scripts/sqlite-session-schema-baseline.test.ts @@ -0,0 +1,100 @@ +// Verifies the SQLite sessions/transcripts schema baseline stays targeted. +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + computeSqliteSessionSchemaBaselineHashFileContent, + renderSqliteSessionSchemaBaseline, + writeSqliteSessionSchemaBaselineArtifacts, +} from "../../scripts/lib/sqlite-session-schema-baseline.ts"; + +describe("SQLite sessions/transcripts schema baseline", () => { + it("includes session and transcript DDL while excluding unrelated agent tables", async () => { + const sourceSql = await readFile("src/state/openclaw-agent-schema.sql", "utf8"); + + const rendered = renderSqliteSessionSchemaBaseline(sourceSql); + + expect(rendered.sql).toContain("CREATE TABLE IF NOT EXISTS sessions"); + expect(rendered.sql).toContain("CREATE TABLE IF NOT EXISTS transcript_events"); + expect(rendered.sql).toContain("CREATE TABLE IF NOT EXISTS transcript_event_identities"); + expect(rendered.sql).not.toContain("idx_agent_transcript_events_session"); + expect(rendered.sql).not.toContain("CREATE TABLE IF NOT EXISTS cache_entries"); + expect(rendered.sql).not.toContain("CREATE TABLE IF NOT EXISTS auth_profile_store"); + expect(rendered.sql).not.toContain("CREATE TABLE IF NOT EXISTS memory_index_sources"); + }); + + it("automatically includes new indexes declared on target tables", () => { + const rendered = renderSqliteSessionSchemaBaseline(` + CREATE TABLE IF NOT EXISTS sessions ( + session_id TEXT NOT NULL PRIMARY KEY + ); + + CREATE INDEX IF NOT EXISTS idx_agent_sessions_custom + ON sessions(session_id); + + CREATE INDEX IF NOT EXISTS idx_agent_cache_custom + ON cache_entries(scope); + `); + + expect(rendered.sql).toContain("CREATE INDEX IF NOT EXISTS idx_agent_sessions_custom"); + expect(rendered.sql).not.toContain("idx_agent_cache_custom"); + }); + + it("checks generated SQL and hash artifacts for drift", async () => { + const tmp = await mkdtemp(path.join(os.tmpdir(), "openclaw-sqlite-schema-baseline-")); + const schemaPath = path.join(tmp, "schema.sql"); + const sqlPath = path.join(tmp, "baseline.sql"); + const hashPath = path.join(tmp, "baseline.sha256"); + await writeFile( + schemaPath, + ` + CREATE TABLE IF NOT EXISTS sessions ( + session_id TEXT NOT NULL PRIMARY KEY + ); + `, + ); + + const writeResult = await writeSqliteSessionSchemaBaselineArtifacts({ + repoRoot: tmp, + check: false, + schemaInputPath: "schema.sql", + sqlOutputPath: "baseline.sql", + hashOutputPath: "baseline.sha256", + }); + const cleanCheck = await writeSqliteSessionSchemaBaselineArtifacts({ + repoRoot: tmp, + check: true, + schemaInputPath: "schema.sql", + sqlOutputPath: "baseline.sql", + hashOutputPath: "baseline.sha256", + }); + const generatedSql = await readFile(sqlPath, "utf8"); + await rm(sqlPath); + const cleanCheckWithoutLocalSql = await writeSqliteSessionSchemaBaselineArtifacts({ + repoRoot: tmp, + check: true, + schemaInputPath: "schema.sql", + sqlOutputPath: "baseline.sql", + hashOutputPath: "baseline.sha256", + }); + await writeFile(hashPath, "stale\n"); + const staleCheck = await writeSqliteSessionSchemaBaselineArtifacts({ + repoRoot: tmp, + check: true, + schemaInputPath: "schema.sql", + sqlOutputPath: "baseline.sql", + hashOutputPath: "baseline.sha256", + }); + + expect(writeResult).toMatchObject({ changed: true, wrote: true }); + expect(cleanCheck.changed).toBe(false); + expect(cleanCheckWithoutLocalSql.changed).toBe(false); + expect(staleCheck.changed).toBe(true); + expect(await readFile(hashPath, "utf8")).not.toBe( + computeSqliteSessionSchemaBaselineHashFileContent({ + sql: generatedSql, + }), + ); + }); +}); diff --git a/test/scripts/sqlite-sessions-transcripts-flip-proof.built-cli.e2e.test.ts b/test/scripts/sqlite-sessions-transcripts-flip-proof.built-cli.e2e.test.ts new file mode 100644 index 000000000000..815f5c528b96 --- /dev/null +++ b/test/scripts/sqlite-sessions-transcripts-flip-proof.built-cli.e2e.test.ts @@ -0,0 +1,157 @@ +// Built-CLI SQLite flip proof requires dist entrypoints before running the gateway lifecycle. +import { describe, expect, it } from "vitest"; +import { runSqliteSessionsTranscriptsFlipProof } from "../../scripts/e2e/sqlite-sessions-transcripts-flip-proof.ts"; + +describe("SQLite sessions/transcripts flip built CLI proof", () => { + it("proves the lifecycle through the built gateway CLI entrypoint", async () => { + const report = await runSqliteSessionsTranscriptsFlipProof({ requireBuiltCli: true }); + + expect(report.gatewayEntrypoint).toEqual( + expect.arrayContaining([expect.stringMatching(/^dist\/index\.(?:js|mjs)$/u)]), + ); + expect(report.failures).toEqual([]); + expect(report.ok).toBe(true); + expect( + report.checkpoints + .filter((checkpoint) => checkpoint.label !== "seeded-legacy-store") + .every((checkpoint) => checkpoint.activeJsonl.length === 0), + ).toBe(true); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "seeded-legacy-store" && checkpoint.legacyStateJsonl.length > 0, + ), + ).toBe(true); + expect( + report.checkpoints + .filter((checkpoint) => checkpoint.label !== "seeded-legacy-store") + .every((checkpoint) => checkpoint.legacyStateJsonl.length === 0), + ).toBe(true); + expect(report.checkpoints.some((checkpoint) => checkpoint.label === "after-doctor-fix")).toBe( + false, + ); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-startup-import" && + checkpoint.gatewayLogTail?.includes( + "session: imported legacy session metadata/transcripts into SQLite", + ) === true && + report.oldStateSessionKeys.every((key) => + checkpoint.sqlite.trackedEntries.some((entry) => entry.sessionKey === key), + ) && + checkpoint.sqlite.sessionEntries >= 7 && + checkpoint.sqlite.transcriptEvents >= 13, + ), + ).toBe(true); + const startupImportCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-startup-import", + ); + expect( + startupImportCheckpoint?.archiveArtifacts.some( + (artifact) => + artifact.path.includes(`${report.legacySessionId}.trajectory.jsonl`) && + artifact.textTail?.includes("trajectory") === true, + ), + ).toBe(true); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-chat-send" && + checkpoint.sqlite.trackedEntries.some( + (entry) => entry.sessionKey === report.resetSessionKey && entry.transcriptEvents >= 3, + ), + ), + ).toBe(true); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-full-agent-turn" && + checkpoint.sqlite.trackedEntries.some( + (entry) => + entry.sessionKey === report.fullTurnSessionKey && entry.transcriptEvents >= 2, + ), + ), + ).toBe(true); + expect(report.pluginSdkConsumer).toMatchObject({ + activeJsonlForSessionExists: false, + latestAssistantTextBeforeAppend: report.fullTurnAssistantText, + latestAssistantTextAfterAppend: "sqlite sdk consumer appended by identity", + sessionKey: report.pluginSdkSessionKey, + }); + expect(report.pluginSdkConsumer?.sessionFileMarker.startsWith("sqlite:")).toBe(true); + expect(report.pluginSdkConsumer?.listedSessionKeys).toContain(report.pluginSdkSessionKey); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-plugin-sdk-consumer" && + checkpoint.sqlite.trackedEntries.some( + (entry) => + entry.sessionKey === report.pluginSdkSessionKey && entry.transcriptEvents >= 3, + ), + ), + ).toBe(true); + const cleanupCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-cleanup-pruning", + ); + expect( + cleanupCheckpoint?.sqlite.trackedEntries.some( + (entry) => entry.sessionKey === report.cleanupPruneSessionKey, + ), + ).toBe(false); + const cleanupArchive = cleanupCheckpoint?.archiveArtifacts.find( + (artifact) => + artifact.archiveReason === "deleted" && + artifact.archiveSessionId === "sqlite-cleanup-prune", + ); + expect(cleanupArchive?.messageTexts).toContain("sqlite cleanup prune me"); + const idempotenceCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-doctor-import-idempotence", + ); + expect(idempotenceCheckpoint?.doctor).toMatchObject({ + code: 0, + mode: "import", + totals: expect.objectContaining({ + importedEntries: 0, + importedTranscriptEvents: 0, + }), + }); + const resetCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-sessions-reset", + ); + const resetArchive = resetCheckpoint?.archiveArtifacts.find( + (artifact) => + artifact.archiveReason === "reset" && artifact.archiveSessionId === report.legacySessionId, + ); + expect(resetArchive?.messageTexts).toContain("legacy hello"); + expect(resetArchive?.messageTexts).toContain("sqlite user-facing send before reset"); + const sharedFirstCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-shared-first-delete", + ); + expect( + sharedFirstCheckpoint?.archiveArtifacts.some( + (artifact) => + artifact.archiveReason === "deleted" && + artifact.archiveSessionId === "sqlite-shared-session", + ), + ).toBe(false); + const concurrentCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-concurrent-multi-client", + ); + expect(concurrentCheckpoint).toBeDefined(); + const concurrentSend = concurrentCheckpoint?.sqlite.trackedEntries.find( + (entry) => entry.sessionKey === report.concurrentSendSessionKey, + ); + expect(concurrentSend?.transcriptEvents).toBeGreaterThanOrEqual(2); + expect( + concurrentCheckpoint?.sqlite.trackedEntries.some( + (entry) => entry.sessionKey === report.concurrentResetSessionKey && entry.sessionId, + ), + ).toBe(true); + expect( + concurrentCheckpoint?.sqlite.trackedEntries.some( + (entry) => entry.sessionKey === report.concurrentDeleteSessionKey, + ), + ).toBe(false); + }, 180_000); +}); diff --git a/test/scripts/sqlite-sessions-transcripts-flip-proof.e2e.test.ts b/test/scripts/sqlite-sessions-transcripts-flip-proof.e2e.test.ts new file mode 100644 index 000000000000..578d7a3c1b26 --- /dev/null +++ b/test/scripts/sqlite-sessions-transcripts-flip-proof.e2e.test.ts @@ -0,0 +1,337 @@ +// SQLite sessions/transcripts flip proof test runs the script-style gateway lifecycle probe. +import { describe, expect, it } from "vitest"; +import { runSqliteSessionsTranscriptsFlipProof } from "../../scripts/e2e/sqlite-sessions-transcripts-flip-proof.ts"; + +describe("SQLite sessions/transcripts flip proof harness", () => { + it("proves isolated gateway lifecycle state stays SQLite-first", async () => { + const report = await runSqliteSessionsTranscriptsFlipProof(); + + expect(report.failures).toEqual([]); + expect(report.ok).toBe(true); + expect(report.checkpoints.map((checkpoint) => checkpoint.label)).toEqual([ + "seeded-legacy-store", + "after-startup-import", + "after-doctor-inspect", + "after-doctor-validate", + "after-rollback-restore", + "after-gateway-restart", + "after-chat-send", + "after-full-agent-turn", + "after-manual-compaction", + "after-plugin-sdk-consumer", + "after-cleanup-pruning", + "after-doctor-import-idempotence", + "after-downgrade-reupgrade-import", + "after-sqlite-busy-contention", + "after-concurrent-multi-client", + "after-sessions-reset", + "after-second-startup-after-reset", + "after-transcript-append", + "after-sessions-delete", + "after-shared-first-delete", + "after-shared-final-delete", + "after-final-doctor-inspect", + ]); + expect( + report.checkpoints + .filter((checkpoint) => checkpoint.label !== "seeded-legacy-store") + .every((checkpoint) => checkpoint.activeJsonl.length === 0), + ).toBe(true); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "seeded-legacy-store" && checkpoint.legacyStateJsonl.length > 0, + ), + ).toBe(true); + expect( + report.checkpoints + .filter((checkpoint) => checkpoint.label !== "seeded-legacy-store") + .every((checkpoint) => checkpoint.legacyStateJsonl.length === 0), + ).toBe(true); + expect(report.checkpoints.some((checkpoint) => checkpoint.label === "after-doctor-fix")).toBe( + false, + ); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-startup-import" && + checkpoint.gatewayLogTail?.includes( + "session: imported legacy session metadata/transcripts into SQLite", + ) === true && + report.oldStateSessionKeys.every((key) => + checkpoint.sqlite.trackedEntries.some((entry) => entry.sessionKey === key), + ) && + checkpoint.sqlite.sessionEntries >= 7 && + checkpoint.sqlite.transcriptEvents >= 13, + ), + ).toBe(true); + const startupImportCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-startup-import", + ); + expect( + startupImportCheckpoint?.archiveArtifacts.some( + (artifact) => + artifact.path.includes(`${report.legacySessionId}.trajectory.jsonl`) && + artifact.textTail?.includes("trajectory") === true, + ), + ).toBe(true); + expect( + startupImportCheckpoint?.archiveArtifacts.some( + (artifact) => + artifact.path.includes("old-orphan.deleted.jsonl") && + artifact.textTail?.includes("old-orphan") === true, + ), + ).toBe(true); + expect(report.rollbackRestore).toMatchObject({ + archivedBeforeRestore: true, + failedManifestIssueCode: "e2e_forced_post_archive_failure", + sourceRestored: true, + sqliteStillExists: true, + }); + expect(report.rollbackRestore?.manifestPath).toContain("session-sqlite-migration-runs"); + expect( + report.rollbackRestore?.restoredFiles.some((filePath) => + filePath.endsWith("/sqlite-rollback-restore.jsonl"), + ), + ).toBe(true); + expect( + report.rollbackRestore?.idempotentRestoreSkippedFiles.some((filePath) => + filePath.endsWith("/sqlite-rollback-restore.jsonl"), + ), + ).toBe(true); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-chat-send" && + checkpoint.sqlite.trackedEntries.some( + (entry) => entry.sessionKey === report.resetSessionKey && entry.transcriptEvents >= 3, + ), + ), + ).toBe(true); + expect(report.scaleMigration).toMatchObject({ + minTranscriptEventsPerSession: 4, + seededEvents: 96, + seededSessions: 24, + }); + expect(report.scaleMigration?.importedSessionKeys).toHaveLength(24); + expect(report.scaleMigration?.startupImportElapsedMs).toBeGreaterThanOrEqual(0); + const resetCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-sessions-reset", + ); + const resetArchive = resetCheckpoint?.archiveArtifacts.find( + (artifact) => + artifact.archiveReason === "reset" && artifact.archiveSessionId === report.legacySessionId, + ); + expect(resetArchive?.messageTexts).toContain("legacy hello"); + expect(resetArchive?.messageTexts).toContain("sqlite user-facing send before reset"); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-full-agent-turn" && + checkpoint.sqlite.trackedEntries.some( + (entry) => + entry.sessionKey === report.fullTurnSessionKey && + entry.transcriptEvents >= 2 && + entry.trajectoryEvents >= 1, + ), + ), + ).toBe(true); + expect(report.manualCompaction).toMatchObject({ + checkpointCount: 1, + compacted: true, + sessionKey: report.manualCompactionSessionKey, + }); + expect(report.manualCompaction?.sessionFileMarker.startsWith("sqlite:")).toBe(true); + expect(report.manualCompaction?.rowCountBefore).toBeGreaterThanOrEqual(2); + expect(report.manualCompaction?.rowCountAfter).toBeGreaterThanOrEqual(1); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-manual-compaction" && + checkpoint.activeJsonl.length === 0 && + checkpoint.sqlite.trackedEntries.some( + (entry) => + entry.sessionKey === report.manualCompactionSessionKey && + Array.isArray(entry.entry?.compactionCheckpoints) && + entry.entry.compactionCheckpoints.length >= 1, + ), + ), + ).toBe(true); + expect(report.pluginSdkConsumer).toMatchObject({ + activeJsonlForSessionExists: false, + activeTrajectoryPointerForSessionExists: false, + activeTrajectoryRuntimeSidecarForSessionExists: false, + activeTrajectorySessionSidecarForSessionExists: false, + latestAssistantTextBeforeAppend: report.fullTurnAssistantText, + latestAssistantTextAfterAppend: "sqlite sdk consumer appended by identity", + sessionKey: report.pluginSdkSessionKey, + }); + expect(report.pluginSdkConsumer?.sessionFileMarker.startsWith("sqlite:")).toBe(true); + expect(report.pluginSdkConsumer?.listedSessionKeys).toContain(report.pluginSdkSessionKey); + expect(report.pluginSdkConsumer?.transcriptEventsAfterAppend).toBeGreaterThan( + report.pluginSdkConsumer?.transcriptEventsBeforeAppend ?? 0, + ); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-plugin-sdk-consumer" && + checkpoint.sqlite.trajectoryRuntimeEvents >= 1 && + checkpoint.sqlite.trackedEntries.some( + (entry) => + entry.sessionKey === report.pluginSdkSessionKey && + entry.trajectoryEvents >= 1 && + entry.transcriptEvents >= 3, + ), + ), + ).toBe(true); + const cleanupCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-cleanup-pruning", + ); + expect( + cleanupCheckpoint?.sqlite.trackedEntries.some( + (entry) => entry.sessionKey === report.cleanupPruneSessionKey, + ), + ).toBe(false); + const cleanupArchive = cleanupCheckpoint?.archiveArtifacts.find( + (artifact) => + artifact.archiveReason === "deleted" && + artifact.archiveSessionId === "sqlite-cleanup-prune", + ); + expect(cleanupArchive?.messageTexts).toContain("sqlite cleanup prune me"); + const idempotenceCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-doctor-import-idempotence", + ); + expect(idempotenceCheckpoint?.doctor).toMatchObject({ + code: 0, + mode: "import", + totals: expect.objectContaining({ + importedEntries: 0, + importedTranscriptEvents: 0, + }), + }); + expect(report.downgradeReupgrade).toMatchObject({ + activeJsonlArchived: true, + doctorImportedEntries: 1, + doctorImportedTranscriptEvents: 2, + sessionId: "sqlite-downgrade-reupgrade", + sessionKey: "agent:main:dashboard:sqlite-downgrade-reupgrade", + trajectoryPointerArchived: true, + trajectoryPointerSourceRemoved: true, + trajectorySidecarArchived: true, + trajectorySidecarSourceRemoved: true, + transcriptEvents: 2, + }); + const downgradeCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-downgrade-reupgrade-import", + ); + expect( + downgradeCheckpoint?.archiveArtifacts.some( + (artifact) => + artifact.path.includes("sqlite-downgrade-reupgrade.trajectory.jsonl") && + artifact.textTail?.includes("trajectory") === true, + ), + ).toBe(true); + expect( + downgradeCheckpoint?.archiveArtifacts.some((artifact) => + artifact.path.includes("sqlite-downgrade-reupgrade.trajectory-path.json"), + ), + ).toBe(true); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-downgrade-reupgrade-import" && + checkpoint.sqlite.trackedEntries.some( + (entry) => + entry.sessionKey === "agent:main:dashboard:sqlite-downgrade-reupgrade" && + entry.transcriptEvents === 2, + ), + ), + ).toBe(true); + expect(report.busyContention).toMatchObject({ + childExitCode: 0, + childSignal: null, + holdMs: 500, + sessionId: "sqlite-busy-contention", + sessionKey: "agent:main:dashboard:sqlite-busy-contention", + transcriptEvents: 2, + }); + expect(report.busyContention?.elapsedMs).toBeGreaterThanOrEqual(250); + const concurrentCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-concurrent-multi-client", + ); + expect(concurrentCheckpoint).toBeDefined(); + const concurrentSend = concurrentCheckpoint?.sqlite.trackedEntries.find( + (entry) => entry.sessionKey === report.concurrentSendSessionKey, + ); + expect(concurrentSend?.transcriptEvents).toBeGreaterThanOrEqual(2); + expect( + concurrentCheckpoint?.sqlite.trackedEntries.some( + (entry) => entry.sessionKey === report.concurrentResetSessionKey && entry.sessionId, + ), + ).toBe(true); + expect( + concurrentCheckpoint?.sqlite.trackedEntries.some( + (entry) => entry.sessionKey === report.concurrentDeleteSessionKey, + ), + ).toBe(false); + expect(report.secondStartupAfterReset).toMatchObject({ + activeJsonlForSessionExists: false, + historyContainsPostResetAppend: true, + sessionKey: report.resetSessionKey, + }); + expect(report.secondStartupAfterReset?.transcriptEvents).toBeGreaterThanOrEqual(1); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-transcript-append" && + checkpoint.sqlite.trackedEntries.some( + (entry) => entry.sessionKey === report.resetSessionKey && entry.transcriptEvents >= 1, + ), + ), + ).toBe(true); + const deleteCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-sessions-delete", + ); + const deleteArchive = deleteCheckpoint?.archiveArtifacts.find( + (artifact) => + artifact.archiveReason === "deleted" && + artifact.archiveSessionId === "sqlite-delete-session", + ); + expect(deleteArchive?.messageTexts).toContain("delete me"); + const sharedFirstCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-shared-first-delete", + ); + expect( + sharedFirstCheckpoint?.archiveArtifacts.some( + (artifact) => + artifact.archiveReason === "deleted" && + artifact.archiveSessionId === "sqlite-shared-session", + ), + ).toBe(false); + const sharedFinalCheckpoint = report.checkpoints.find( + (checkpoint) => checkpoint.label === "after-shared-final-delete", + ); + const sharedFinalArchive = sharedFinalCheckpoint?.archiveArtifacts.find( + (artifact) => + artifact.archiveReason === "deleted" && + artifact.archiveSessionId === "sqlite-shared-session", + ); + expect(sharedFinalArchive?.messageTexts).toContain("shared"); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-shared-first-delete" && + checkpoint.sqlite.trackedEntries.some( + (entry) => entry.sessionKey === report.sharedSessionKeys[1], + ), + ), + ).toBe(true); + expect( + report.checkpoints.some( + (checkpoint) => + checkpoint.label === "after-shared-final-delete" && + checkpoint.archiveArtifacts.length > 0, + ), + ).toBe(true); + }, 180_000); +}); diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index 33642c4d04ae..a3f6fc715ec3 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -2672,18 +2672,29 @@ describe("scripts/test-projects changed-target routing", () => { watchMode: false, }, ]); - const toolingPlans = plans.slice(3); + const e2ePlans = plans.filter((plan) => plan.config === "test/vitest/vitest.e2e.config.ts"); + const toolingPlans = plans + .slice(3) + .filter((plan) => plan.config === "test/vitest/vitest.tooling.config.ts"); const toolingTargets = toolingPlans.flatMap((plan) => plan.includePatterns ?? []); expect(toolingPlans.length).toBeGreaterThan(1); - expect( - toolingPlans.every((plan) => plan.config === "test/vitest/vitest.tooling.config.ts"), - ).toBe(true); expect(toolingPlans.every((plan) => (plan.includePatterns?.length ?? 0) <= 60)).toBe(true); expect(toolingTargets).toContain("test/scripts/run-opengrep.test.ts"); expect(toolingTargets).not.toContain("test/scripts/docker-build-helper.test.ts"); expect(toolingTargets).not.toContain("test/scripts/openclaw-e2e-instance.test.ts"); expect(new Set(toolingTargets).size).toBe(toolingTargets.length); + expect(e2ePlans).toEqual([ + { + config: "test/vitest/vitest.e2e.config.ts", + forwardedArgs: [ + "test/scripts/sqlite-sessions-transcripts-flip-proof.built-cli.e2e.test.ts", + "test/scripts/sqlite-sessions-transcripts-flip-proof.e2e.test.ts", + ], + includePatterns: null, + watchMode: false, + }, + ]); }); it("routes the src scripts test root to the tooling shard", () => { @@ -2805,18 +2816,29 @@ describe("scripts/test-projects changed-target routing", () => { watchMode: false, }, ]); - const toolingPlans = plans.slice(3); + const e2ePlans = plans.filter((plan) => plan.config === "test/vitest/vitest.e2e.config.ts"); + const toolingPlans = plans + .slice(3) + .filter((plan) => plan.config === "test/vitest/vitest.tooling.config.ts"); const toolingTargets = toolingPlans.flatMap((plan) => plan.includePatterns ?? []); expect(toolingPlans.length).toBeGreaterThan(1); - expect( - toolingPlans.every((plan) => plan.config === "test/vitest/vitest.tooling.config.ts"), - ).toBe(true); expect(toolingPlans.every((plan) => (plan.includePatterns?.length ?? 0) <= 60)).toBe(true); expect(toolingTargets).toContain("test/scripts/run-opengrep.test.ts"); expect(toolingTargets).not.toContain("test/scripts/docker-build-helper.test.ts"); expect(toolingTargets).not.toContain("test/scripts/openclaw-e2e-instance.test.ts"); expect(new Set(toolingTargets).size).toBe(toolingTargets.length); + expect(e2ePlans).toEqual([ + { + config: "test/vitest/vitest.e2e.config.ts", + forwardedArgs: [ + "test/scripts/sqlite-sessions-transcripts-flip-proof.built-cli.e2e.test.ts", + "test/scripts/sqlite-sessions-transcripts-flip-proof.e2e.test.ts", + ], + includePatterns: null, + watchMode: false, + }, + ]); }); it("keeps broad shell helper watch targets in one tooling shard", () => { @@ -4720,6 +4742,14 @@ describe("scripts/test-projects Vitest stall watchdog", () => { pnpmArgs: [], watchMode: false, }, + { + config: "test/vitest/vitest.gateway-server.config.ts", + env: { PATH: "/usr/bin" }, + includeFilePath: null, + includePatterns: null, + pnpmArgs: [], + watchMode: false, + }, { config: "test/vitest/vitest.extension-feishu.config.ts", env: { PATH: "/usr/bin" }, @@ -4735,7 +4765,8 @@ describe("scripts/test-projects Vitest stall watchdog", () => { expect(specs[0]?.env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe("2400000"); expect(specs[1]?.env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe("2400000"); expect(specs[2]?.env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe("2400000"); - expect(specs[3]?.env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe( + expect(specs[3]?.env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe("2400000"); + expect(specs[4]?.env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe( DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS, ); }); diff --git a/test/vitest-unit-fast-config.test.ts b/test/vitest-unit-fast-config.test.ts index bf6375448281..aa2708a9c35f 100644 --- a/test/vitest-unit-fast-config.test.ts +++ b/test/vitest-unit-fast-config.test.ts @@ -212,6 +212,7 @@ describe("unit-fast vitest lane", () => { it("keeps obvious stateful files out of the unit-fast lane", () => { expect(isUnitFastTestFile("src/plugin-sdk/temp-path.test.ts")).toBe(false); expect(isUnitFastTestFile("src/agents/sandbox.resolveSandboxContext.test.ts")).toBe(false); + expect(isUnitFastTestFile("src/acp/runtime/session-meta.test.ts")).toBe(false); expect(isUnitFastTestFile("src/crestodian/assistant.test.ts")).toBe(false); expect(isUnitFastTestFile("src/flows/channel-setup.test.ts")).toBe(false); expect(isUnitFastTestFile("src/flows/doctor-health-contributions.test.ts")).toBe(false); diff --git a/test/vitest/vitest.unit-fast-paths.mjs b/test/vitest/vitest.unit-fast-paths.mjs index 2d5bb1178648..27e3ed45d07b 100644 --- a/test/vitest/vitest.unit-fast-paths.mjs +++ b/test/vitest/vitest.unit-fast-paths.mjs @@ -228,6 +228,7 @@ const broadUnitFastCandidateSkipGlobs = [ "test/fixtures/**/*.test.ts", "test/setup-home-isolation.test.ts", "src/agents/sandbox.resolveSandboxContext.test.ts", + "src/acp/runtime/session-meta.test.ts", "src/channels/plugins/contracts/**/*.test.ts", "src/config/**/*.test.ts", "src/gateway/**/*.test.ts",