mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
0a8e3604ba
* refactor(sessions): migrate runtime storage to sqlite * test(sessions): fix sqlite CI regressions * test(sessions): align remaining sqlite fixtures * fix(codex): require sqlite trajectory recorder * test(sessions): align orphan recovery sqlite fixture * test(sessions): align sqlite rebase fixtures * fix(sessions): finish current-main integration of the sqlite flip Resolve the whole-store SDK removal across its owner boundary: drop the loadSessionStore re-export and the registry whole-store wrappers, wire hasTrackedActiveSessionRun into gateway chat, complete the preserveLockedHarnessIds cleanup contract, flip the codex thread-history import to storePath targets, and port remaining main-side tests from file-store helpers to session accessor reads. * chore: drop committed pebbles log, revert plugin-inspector bump, refresh generated docs Remove the 1.8k-line .pebbles/events.jsonl work log from the branch, restore the plugin-inspector advisory lane to main's pinned 0.3.10 so the supply-chain bump gets its own review, and regenerate docs_map, the plugin SDK API baseline, and the export-surface ratchet for the merged tree. * feat(sessions): keep archived transcripts by default with zstd cold storage Codex-style retention: deleting or resetting a session archives its transcript as a zstd-compressed JSONL artifact (plain when the runtime lacks node:zlib zstd) and keeps it until the disk budget evicts oldest first. resetArchiveRetention now governs both deleted and reset archives and defaults to keep; maxDiskBytes defaults to 2gb so retention stays bounded, with archives evicted before live sessions. The cron reaper follows the same knob instead of deleting archives on its own timer. * fix(state): converge agent DB migration lineages and bound database growth Merge coherence: run both structure-gated legacy memory-schema repairs (flip-lineage drop, main-lineage identity rebuild) before the flip migration so pre-flip v1/v2 and pre-merge flip v1/v4 databases all converge, and hoist foreign_keys=OFF outside the schema transaction where the pragma was silently ignored and the v1 sessions rebuild cascade-deleted session_entries. Growth guards: fresh agent DBs enable auto_vacuum=INCREMENTAL, WAL maintenance releases freed pages in bounded passes (never a blocking full VACUUM), and doctor reports state/agent DB bloat from freelist stats. * fix(codex): resolve the store path for thread-history import via the SDK The supervision catalog passed the legacy sessionFile locator to the storePath-targeted transcript mirror; resolve the agent store path with the session-store SDK helper instead of a runtime-object seam so test fakes and headless callers need no extra surface. Drop the obsolete missing-session-id preprocessing case: sessions rows are NOT NULL on session_id and upsert repairs id-less patches at write time. * fix(sessions): fail safe on malformed disk-budget config and doctor stat errors A malformed explicit maxDiskBytes disables the budget instead of falling back to the destructive 2gb default the user never chose, and the doctor bloat check skips databases whose paths stat-fail instead of aborting doctor. * fix(sessions): complete sqlite conflict translations * test(sqlite): align hardening checks with maintenance * test(sessions): inspect compressed transcript archives * fix(tests): await session seeds and drop unused helpers flagged by CI lint The five unawaited writeSessionStoreSeed calls raced their SQLite seeds against the assertions, failing compact shards; the bloat probe drops a useless initializer and the merged tests drop now-unused helpers. * test(sessions): type legacy proof events directly * test(sessions): align hardening contracts * perf(sessions): read usage transcript sizes from SQL aggregates Usage/cost scans walked every session and materialized every transcript event just to re-stringify it for a byte estimate — the #86718 stall class reborn on the DB. readTranscriptStatsSync sums stored JSON bytes in SQLite without loading a single row. * fix(sessions): re-root foreign-root transcript paths onto the current sessions dir Restored backups, moved OPENCLAW_STATE_DIR, and rehearsal copies carry absolute sessionFile paths from the old root; the containment fallback kept those foreign paths, so migration read (and would archive) files in the original root and reported local copies missing. Re-root the canonical agents/<id>/sessions suffix onto the current dir when the file exists there; genuine cross-root layouts still fall through unchanged. * test(agents): seed harness admission through sqlite * fix(sqlite): close agent db on pragma setup failure * fix(doctor): compact and retrofit incremental auto-vacuum after session import The migration is the sanctioned offline window: post-import compact reclaims import churn and applies auto_vacuum=INCREMENTAL to databases created before the fresh-DB pragma existed, so runtime maintenance can release pages in bounded passes on every install. --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
288 lines
8.7 KiB
TypeScript
288 lines
8.7 KiB
TypeScript
// Codex tests cover mirrored session-history branch selection.
|
|
import fs from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { CURRENT_SESSION_VERSION } from "openclaw/plugin-sdk/agent-sessions";
|
|
import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime";
|
|
import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime";
|
|
import { afterEach, describe, expect, it } from "vitest";
|
|
import { readCodexMirroredSessionHistoryMessages } from "./session-history.js";
|
|
|
|
const tempDirs: string[] = [];
|
|
|
|
afterEach(async () => {
|
|
for (const dir of tempDirs.splice(0)) {
|
|
await fs.rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
async function writeSession(records: unknown[]): Promise<string> {
|
|
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-session-history-"));
|
|
tempDirs.push(dir);
|
|
const sessionFile = path.join(dir, "session.jsonl");
|
|
const header = {
|
|
type: "session",
|
|
version: CURRENT_SESSION_VERSION,
|
|
id: "codex-session",
|
|
timestamp: "2026-06-15T00:00:00.000Z",
|
|
cwd: dir,
|
|
};
|
|
await fs.writeFile(
|
|
sessionFile,
|
|
[header, ...records].map((record) => JSON.stringify(record)).join("\n") + "\n",
|
|
);
|
|
return sessionFile;
|
|
}
|
|
|
|
// Fixtures keep legacy string content on purpose: session ingest normalizes
|
|
// assistant strings into [{ type: "text" }] blocks, so expectations below
|
|
// assert the canonical block-array shape for assistant rows.
|
|
function messageEntry(params: {
|
|
id: string;
|
|
parentId: string | null;
|
|
role: "user" | "assistant";
|
|
content: string;
|
|
}) {
|
|
return {
|
|
type: "message",
|
|
id: params.id,
|
|
parentId: params.parentId,
|
|
timestamp: "2026-06-15T00:00:00.000Z",
|
|
message: {
|
|
role: params.role,
|
|
content: params.content,
|
|
timestamp: 1,
|
|
},
|
|
};
|
|
}
|
|
|
|
function mirroredTarget(sessionFile: string) {
|
|
return {
|
|
sessionFile,
|
|
sessionId: "codex-session",
|
|
sessionKey: "codex-session",
|
|
};
|
|
}
|
|
|
|
async function writeSqliteSession(params: { storedSessionFile?: string } = {}): Promise<{
|
|
marker: string;
|
|
sessionKey: string;
|
|
}> {
|
|
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-session-history-sqlite-"));
|
|
tempDirs.push(dir);
|
|
const storePath = path.join(dir, "openclaw-agent.sqlite");
|
|
const sessionId = "codex-sqlite-session";
|
|
const sessionKey = "agent:main:codex-sqlite";
|
|
const marker = `sqlite:main:${sessionId}:${storePath}`;
|
|
const scope = {
|
|
agentId: "main",
|
|
sessionId,
|
|
sessionKey,
|
|
storePath,
|
|
};
|
|
await upsertSessionEntry({
|
|
...scope,
|
|
entry: {
|
|
sessionFile: params.storedSessionFile ?? marker,
|
|
sessionId,
|
|
updatedAt: 1,
|
|
},
|
|
});
|
|
await appendSessionTranscriptMessageByIdentity({
|
|
...scope,
|
|
message: { role: "user", content: "sqlite prompt", timestamp: 1 },
|
|
});
|
|
await appendSessionTranscriptMessageByIdentity({
|
|
...scope,
|
|
message: { role: "assistant", content: "sqlite answer", timestamp: 2 },
|
|
});
|
|
return { marker, sessionKey };
|
|
}
|
|
|
|
describe("readCodexMirroredSessionHistoryMessages", () => {
|
|
it("treats a missing mirrored session file as empty history", async () => {
|
|
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-session-history-"));
|
|
tempDirs.push(dir);
|
|
const sessionFile = path.join(dir, "session.jsonl");
|
|
|
|
await expect(
|
|
readCodexMirroredSessionHistoryMessages(mirroredTarget(sessionFile)),
|
|
).resolves.toEqual([]);
|
|
});
|
|
|
|
it("returns undefined for malformed non-empty mirrored session files", async () => {
|
|
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-session-history-"));
|
|
tempDirs.push(dir);
|
|
const sessionFile = path.join(dir, "session.jsonl");
|
|
await fs.writeFile(sessionFile, JSON.stringify({ type: "message", id: "orphan" }) + "\n");
|
|
|
|
await expect(
|
|
readCodexMirroredSessionHistoryMessages(mirroredTarget(sessionFile)),
|
|
).resolves.toBeUndefined();
|
|
});
|
|
|
|
it("replays SQLite marker history by session identity", async () => {
|
|
const { marker, sessionKey } = await writeSqliteSession();
|
|
|
|
await expect(
|
|
readCodexMirroredSessionHistoryMessages({
|
|
agentId: "main",
|
|
sessionFile: marker,
|
|
sessionId: "codex-sqlite-session",
|
|
sessionKey,
|
|
}),
|
|
).resolves.toMatchObject([
|
|
{ role: "user", content: "sqlite prompt" },
|
|
{ role: "assistant", content: "sqlite answer" },
|
|
]);
|
|
});
|
|
|
|
it("resolves SQLite marker history when the caller has no session key", async () => {
|
|
const { marker } = await writeSqliteSession();
|
|
|
|
await expect(
|
|
readCodexMirroredSessionHistoryMessages({
|
|
agentId: "main",
|
|
sessionFile: marker,
|
|
sessionId: "codex-sqlite-session",
|
|
}),
|
|
).resolves.toMatchObject([
|
|
{ role: "user", content: "sqlite prompt" },
|
|
{ role: "assistant", content: "sqlite answer" },
|
|
]);
|
|
});
|
|
|
|
it("resolves synthesized SQLite markers for stale file-backed session metadata", async () => {
|
|
const { marker } = await writeSqliteSession({
|
|
storedSessionFile: "/tmp/legacy-session.jsonl",
|
|
});
|
|
|
|
await expect(
|
|
readCodexMirroredSessionHistoryMessages({
|
|
agentId: "main",
|
|
sessionFile: marker,
|
|
sessionId: "codex-sqlite-session",
|
|
}),
|
|
).resolves.toMatchObject([
|
|
{ role: "user", content: "sqlite prompt" },
|
|
{ role: "assistant", content: "sqlite answer" },
|
|
]);
|
|
});
|
|
|
|
it("replays only the branch selected by a leaf control", async () => {
|
|
const sessionFile = await writeSession([
|
|
messageEntry({ id: "root", parentId: null, role: "user", content: "root prompt" }),
|
|
messageEntry({
|
|
id: "active",
|
|
parentId: "root",
|
|
role: "assistant",
|
|
content: "active answer",
|
|
}),
|
|
messageEntry({
|
|
id: "inactive",
|
|
parentId: "root",
|
|
role: "assistant",
|
|
content: "inactive answer",
|
|
}),
|
|
{
|
|
type: "leaf",
|
|
id: "active-leaf",
|
|
parentId: "inactive",
|
|
targetId: "active",
|
|
},
|
|
]);
|
|
|
|
await expect(
|
|
readCodexMirroredSessionHistoryMessages(mirroredTarget(sessionFile)),
|
|
).resolves.toMatchObject([
|
|
{ role: "user", content: "root prompt" },
|
|
{ role: "assistant", content: [{ type: "text", text: "active answer" }] },
|
|
]);
|
|
});
|
|
|
|
it("honors explicit navigation to an empty branch", async () => {
|
|
const sessionFile = await writeSession([
|
|
messageEntry({ id: "old", parentId: null, role: "user", content: "old prompt" }),
|
|
{
|
|
type: "leaf",
|
|
id: "empty-leaf",
|
|
parentId: "old",
|
|
targetId: null,
|
|
appendParentId: "old",
|
|
},
|
|
]);
|
|
|
|
await expect(
|
|
readCodexMirroredSessionHistoryMessages(mirroredTarget(sessionFile)),
|
|
).resolves.toEqual([]);
|
|
});
|
|
|
|
it("keeps visible history when continuation rows use a disjoint append cursor", async () => {
|
|
const sessionFile = await writeSession([
|
|
messageEntry({ id: "visible", parentId: null, role: "user", content: "visible prompt" }),
|
|
messageEntry({
|
|
id: "inactive",
|
|
parentId: "visible",
|
|
role: "assistant",
|
|
content: "inactive answer",
|
|
}),
|
|
{
|
|
type: "metadata",
|
|
id: "append-metadata",
|
|
parentId: "inactive",
|
|
},
|
|
{
|
|
type: "leaf",
|
|
id: "active-leaf",
|
|
parentId: "inactive",
|
|
targetId: "visible",
|
|
appendParentId: "append-metadata",
|
|
},
|
|
messageEntry({
|
|
id: "continued",
|
|
parentId: "append-metadata",
|
|
role: "assistant",
|
|
content: "continued answer",
|
|
}),
|
|
]);
|
|
|
|
await expect(
|
|
readCodexMirroredSessionHistoryMessages(mirroredTarget(sessionFile)),
|
|
).resolves.toMatchObject([
|
|
{ role: "user", content: "visible prompt" },
|
|
{ role: "assistant", content: [{ type: "text", text: "continued answer" }] },
|
|
]);
|
|
});
|
|
|
|
it("keeps visible history when a continuation references the leaf marker", async () => {
|
|
const sessionFile = await writeSession([
|
|
messageEntry({ id: "visible", parentId: null, role: "user", content: "visible prompt" }),
|
|
messageEntry({
|
|
id: "inactive",
|
|
parentId: "visible",
|
|
role: "assistant",
|
|
content: "inactive answer",
|
|
}),
|
|
{
|
|
type: "leaf",
|
|
id: "active-leaf",
|
|
parentId: "inactive",
|
|
targetId: "visible",
|
|
},
|
|
messageEntry({
|
|
id: "continued",
|
|
parentId: "active-leaf",
|
|
role: "assistant",
|
|
content: "continued answer",
|
|
}),
|
|
]);
|
|
|
|
await expect(
|
|
readCodexMirroredSessionHistoryMessages(mirroredTarget(sessionFile)),
|
|
).resolves.toMatchObject([
|
|
{ role: "user", content: "visible prompt" },
|
|
{ role: "assistant", content: [{ type: "text", text: "continued answer" }] },
|
|
]);
|
|
});
|
|
});
|