From 7401d21bdc37c826c3d459fd183e5e4c2a7d2803 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 25 Aug 2026 05:16:50 -0700 Subject: [PATCH] refactor(scripts): remove obsolete sqlite incident reproductions (#129212) --- config/knip.config.ts | 2 - scripts/memory-index-manager.sync-repro.ts | 347 ------------------ ...lite-session-entry-cache-lifetime-proof.ts | 174 --------- 3 files changed, 523 deletions(-) delete mode 100644 scripts/memory-index-manager.sync-repro.ts delete mode 100644 scripts/sqlite-session-entry-cache-lifetime-proof.ts diff --git a/config/knip.config.ts b/config/knip.config.ts index bf5ea7f088b5..7265d6842828 100644 --- a/config/knip.config.ts +++ b/config/knip.config.ts @@ -74,7 +74,6 @@ const repositoryScriptEntries = [ // Invoked by scripts/lib/live-docker-stage.sh during container validation. "scripts/live-docker-normalize-config.ts!", "scripts/mcp-code-mode-gateway-e2e.ts!", - "scripts/memory-index-manager.sync-repro.ts!", // Mantis invokes the trusted proof collector through its workflow shell step. "scripts/mantis/telegram-visible-proof.mjs!", "scripts/openclaw-release-clawhub-plan.ts!", @@ -95,7 +94,6 @@ const repositoryScriptEntries = [ // Changed-file checks invoke this targeted UI Stylelint entrypoint by path. "scripts/run-stylelint.mts!", "scripts/secrets/openclaw-bws-resolver.mjs!", - "scripts/sqlite-session-entry-cache-lifetime-proof.ts!", "scripts/sync-labels.ts!", "scripts/test-built-bundled-channel-entry-smoke.mts!", "scripts/update-clawtributors.ts!", diff --git a/scripts/memory-index-manager.sync-repro.ts b/scripts/memory-index-manager.sync-repro.ts deleted file mode 100644 index 861ecdb0cb48..000000000000 --- a/scripts/memory-index-manager.sync-repro.ts +++ /dev/null @@ -1,347 +0,0 @@ -import { execFileSync } from "node:child_process"; -import fs from "node:fs/promises"; -import path from "node:path"; -import { DatabaseSync } from "node:sqlite"; -import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; -import { resolveSessionTranscriptsDirForAgent } from "openclaw/plugin-sdk/memory-core-host-runtime-core"; -import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; -import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; -import { resolveOpenClawAgentSqlitePath } from "openclaw/plugin-sdk/sqlite-runtime"; -import { - closeAllMemorySearchManagers, - getMemorySearchManager, -} from "../extensions/memory-core/src/memory/index.ts"; - -const proofRoot = process.argv[2]; -const exactHead = process.argv[3]; -if (!proofRoot || !exactHead?.match(/^[0-9a-f]{40}$/)) { - throw new Error("proof root and exact 40-character head are required"); -} -const observedHead = execFileSync("git", ["rev-parse", "HEAD"], { - encoding: "utf8", -}).trim(); -if (observedHead !== exactHead) { - throw new Error(`exact head mismatch: expected ${exactHead}, observed ${observedHead}`); -} -const dirtyState = execFileSync("git", ["status", "--porcelain"], { - encoding: "utf8", -}).trim(); -if (dirtyState) { - throw new Error("the production repro requires a clean exact-head worktree"); -} - -const agentId = "main"; -const stateDir = path.join(proofRoot, "state"); -const workspaceDir = path.join(proofRoot, "workspace"); -const markers = { - blocker: "BLOCKER_LOCKED_SYNC_729", - retained: "RETAINED_RETRY_TARGET_729", - trigger: "CONCURRENT_TRIGGER_TARGET_729", - archive: "RETAINED_ARCHIVE_TARGET_729", -}; - -Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir); -await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true }); -await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Proof workspace\n"); - -const cfg: OpenClawConfig = { - memory: { - search: { - provider: "none", - sources: ["sessions"], - rememberAcrossConversations: true, - store: { vector: { enabled: false } }, - query: { minScore: 0 }, - }, - }, - agents: { - defaults: { workspace: workspaceDir }, - list: [{ id: agentId, default: true }], - }, -}; - -async function seedSession(sessionId: string, marker: string): Promise { - const sessionsDir = resolveSessionTranscriptsDirForAgent(agentId); - const storePath = path.join(sessionsDir, "sessions.json"); - const sessionKey = `agent:${agentId}:proof:${sessionId}`; - await fs.mkdir(sessionsDir, { recursive: true }); - await upsertSessionEntry({ - agentId, - sessionKey, - storePath, - entry: { sessionId, updatedAt: Date.now() }, - }); - await appendSessionTranscriptMessageByIdentity({ - agentId, - sessionId, - sessionKey, - storePath, - message: { - role: "user", - timestamp: Date.now(), - content: [{ type: "text", text: marker }], - }, - }); - return sessionKey; -} - -function openExclusiveLock(dbPath: string): DatabaseSync { - const db = new DatabaseSync(dbPath); - db.exec("PRAGMA busy_timeout = 0"); - db.exec("BEGIN EXCLUSIVE"); - return db; -} - -function releaseExclusiveLock(db: DatabaseSync | null): void { - if (!db) { - return; - } - try { - db.exec("ROLLBACK"); - } finally { - db.close(); - } -} - -function describeSqliteFailure(failure: unknown): string { - const details = [String(failure)]; - if (failure && typeof failure === "object") { - const record = failure as Record; - for (const key of ["message", "code"] as const) { - if (typeof record[key] === "string") { - details.push(record[key]); - } - } - if (record.cause && typeof record.cause === "object") { - const cause = record.cause as Record; - for (const key of ["message", "code"] as const) { - if (typeof cause[key] === "string") { - details.push(cause[key]); - } - } - } - } - return details.join(" "); -} - -function isSqliteLockFailure(failure: unknown): boolean { - return /SQLITE_(?:BUSY|LOCKED)|database is (?:busy|locked)/i.test(describeSqliteFailure(failure)); -} - -async function withTimeout(promise: Promise, timeoutMs: number, label: string): Promise { - let timer: ReturnType | undefined; - const timeout = new Promise((_resolve, reject) => { - timer = setTimeout(() => { - reject(new Error(`${label} timed out after ${timeoutMs}ms`)); - }, timeoutMs); - }); - try { - return await Promise.race([promise, timeout]); - } finally { - if (timer) { - clearTimeout(timer); - } - } -} - -let lock: DatabaseSync | null = null; -try { - const result = await getMemorySearchManager({ cfg, agentId }); - if (!result.manager) { - throw new Error(`memory manager unavailable: ${result.error ?? "unknown"}`); - } - const manager = result.manager; - const sync = manager.sync?.bind(manager); - if (!sync) { - throw new Error("memory manager sync is unavailable"); - } - await sync({ reason: "proof-baseline", force: true }); - - const blockerKey = await seedSession("proof-blocker", markers.blocker); - const retainedKey = await seedSession("proof-retained", markers.retained); - const triggerKey = await seedSession("proof-trigger", markers.trigger); - const archiveFile = path.join( - resolveSessionTranscriptsDirForAgent(agentId), - "proof-archive.jsonl.deleted.2026-07-29T00-00-00.000Z", - ); - await fs.writeFile( - archiveFile, - [ - JSON.stringify({ - type: "session", - id: "proof-archive", - timestamp: new Date().toISOString(), - }), - JSON.stringify({ - type: "message", - message: { - role: "user", - timestamp: Date.now(), - content: [{ type: "text", text: markers.archive }], - }, - }), - ].join("\n") + "\n", - "utf8", - ); - const dbPath = resolveOpenClawAgentSqlitePath({ agentId }); - - lock = openExclusiveLock(dbPath); - const blockedOwner = sync({ - reason: "proof-locked-owner", - sessions: [{ agentId, sessionId: "proof-blocker", sessionKey: blockerKey }], - }); - const failedQueued = sync({ - reason: "proof-queued-retained", - sessions: [{ agentId, sessionId: "proof-retained", sessionKey: retainedKey }], - archiveFiles: [archiveFile], - }); - const failures = await Promise.allSettled([blockedOwner, failedQueued]); - const lockedSyncFailures = failures.filter((entry) => entry.status === "rejected").length; - const sqliteLockFailures = failures.filter( - (entry) => entry.status === "rejected" && isSqliteLockFailure(entry.reason), - ).length; - releaseExclusiveLock(lock); - lock = null; - if (lockedSyncFailures !== 2) { - throw new Error(`expected two locked sync failures, received ${lockedSyncFailures}`); - } - if (sqliteLockFailures !== 2) { - throw new Error(`expected two SQLite lock failures, received ${sqliteLockFailures}`); - } - - const observer = new DatabaseSync(dbPath, { readOnly: true }); - const retainedBefore = - ( - observer - .prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE text LIKE ?") - .get(`%${markers.retained}%`) as { count: number } - ).count > 0; - const triggerBefore = - ( - observer - .prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE text LIKE ?") - .get(`%${markers.trigger}%`) as { count: number } - ).count > 0; - const archiveBefore = - ( - observer - .prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE text LIKE ?") - .get(`%${markers.archive}%`) as { count: number } - ).count > 0; - observer.close(); - if (retainedBefore || triggerBefore || archiveBefore) { - throw new Error("a recovery target was unexpectedly indexed before the idle trigger"); - } - - const recoveryState = manager as unknown as { - syncing: Promise | null; - queuedArchiveFiles: Set; - queuedSessions: Map; - sessionsDirtyFiles: Set; - sessionsFullRetryDirty: boolean; - }; - const retainedQueueBeforeRecovery = recoveryState.queuedSessions.size; - const retainedArchiveQueueBeforeRecovery = recoveryState.queuedArchiveFiles.size; - const dirtySessionFilesBeforeRecovery = recoveryState.sessionsDirtyFiles.size; - const fullRetryBeforeRecovery = recoveryState.sessionsFullRetryDirty; - if ( - recoveryState.syncing !== null || - retainedQueueBeforeRecovery !== 1 || - retainedArchiveQueueBeforeRecovery !== 1 - ) { - throw new Error("manager was not idle with exactly one retained session and archive target"); - } - - const recoveryProgress: Array<{ completed: number; total: number; label?: string }> = []; - const recovery = sync({ - reason: "proof-idle-recovery-trigger", - sessions: [{ agentId, sessionId: "proof-trigger", sessionKey: triggerKey }], - progress: (update) => recoveryProgress.push(update), - }); - // Start an untargeted sync before the retained queue owner resumes. - // Its distinct public progress callback proves that this call, rather than - // an implementation token, reached competing production admission. - const competingUntargetedProgress: Array<{ completed: number; total: number; label?: string }> = - []; - const competingUntargetedSync = sync({ - reason: "proof-competing-untargeted-sync", - progress: (update) => competingUntargetedProgress.push(update), - }); - const queueSettlementTimeoutMs = 15_000; - const recoveryResults = await withTimeout( - Promise.allSettled([recovery, competingUntargetedSync]), - queueSettlementTimeoutMs, - "queue-owner self-deadlock check", - ); - const recoveryStatus = recoveryResults[0]?.status; - const competingUntargetedStatus = recoveryResults[1]?.status; - if (recoveryStatus !== "fulfilled" || competingUntargetedStatus !== "fulfilled") { - throw new Error( - `concurrent recovery did not settle: recovery=${recoveryStatus ?? "missing"} untargeted=${competingUntargetedStatus ?? "missing"}`, - ); - } - if (competingUntargetedProgress.length === 0) { - throw new Error("competing untargeted sync emitted no public progress updates"); - } - - const recoveryObserver = new DatabaseSync(dbPath, { readOnly: true }); - const indexedCount = (marker: string) => - ( - recoveryObserver - .prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE text LIKE ?") - .get(`%${marker}%`) as { count: number } - ).count; - const retainedAfter = indexedCount(markers.retained) > 0; - const triggerAfter = indexedCount(markers.trigger) > 0; - const archiveAfter = indexedCount(markers.archive) > 0; - recoveryObserver.close(); - if (!retainedAfter || !triggerAfter || !archiveAfter) { - throw new Error( - `recovery result mismatch: retained=${String(retainedAfter)} trigger=${String(triggerAfter)} archive=${String(archiveAfter)}`, - ); - } - if (recoveryProgress.length === 0) { - throw new Error("idle recovery trigger did not receive progress"); - } - const retainedQueueAfterRecovery = recoveryState.queuedSessions.size; - const retainedArchiveQueueAfterRecovery = recoveryState.queuedArchiveFiles.size; - if ( - dirtySessionFilesBeforeRecovery !== 0 || - fullRetryBeforeRecovery || - retainedQueueAfterRecovery !== 0 || - retainedArchiveQueueAfterRecovery !== 0 - ) { - throw new Error( - `unexpected recovery ownership state: dirty=${dirtySessionFilesBeforeRecovery} fullRetry=${String(fullRetryBeforeRecovery)} retainedAfter=${retainedQueueAfterRecovery} archiveAfter=${retainedArchiveQueueAfterRecovery}`, - ); - } - - console.log(`exact_head=${exactHead}`); - console.log("test_runner=none"); - console.log("entrypoint=MemoryIndexManager.sync"); - console.log("owners=memory-manager,session-store,sqlite"); - console.log(`locked_sync_failures=${lockedSyncFailures}`); - console.log("locked_sync_failure_kind=sqlite-busy"); - console.log("recovery_manager_state=idle"); - console.log("recovery_input_sessions=proof-trigger"); - console.log(`recovery_progress_updates=${recoveryProgress.length}`); - console.log(`competing_untargeted_sync_progress_updates=${competingUntargetedProgress.length}`); - console.log(`recovery_sync_status=${recoveryStatus}`); - console.log(`competing_untargeted_sync_status=${competingUntargetedStatus}`); - console.log(`queue_settlement_timeout_ms=${queueSettlementTimeoutMs}`); - console.log(`retained_queue_before_recovery=${retainedQueueBeforeRecovery}`); - console.log(`retained_archive_queue_before_recovery=${retainedArchiveQueueBeforeRecovery}`); - console.log(`sessions_dirty_files_before_recovery=${dirtySessionFilesBeforeRecovery}`); - console.log(`sessions_full_retry_dirty_before_recovery=${String(fullRetryBeforeRecovery)}`); - console.log("retained_target_before_recovery=absent"); - console.log("retained_archive_target_before_recovery=absent"); - console.log("retained_target_after_recovery=indexed"); - console.log("retained_archive_target_after_recovery=indexed"); - console.log("idle_trigger_after_recovery=indexed"); - console.log(`retained_queue_after_recovery=${retainedQueueAfterRecovery}`); - console.log(`retained_archive_queue_after_recovery=${retainedArchiveQueueAfterRecovery}`); - console.log("verdict=pass"); -} finally { - releaseExclusiveLock(lock); - await closeAllMemorySearchManagers(); -} diff --git a/scripts/sqlite-session-entry-cache-lifetime-proof.ts b/scripts/sqlite-session-entry-cache-lifetime-proof.ts deleted file mode 100644 index e06e01f3751b..000000000000 --- a/scripts/sqlite-session-entry-cache-lifetime-proof.ts +++ /dev/null @@ -1,174 +0,0 @@ -import assert from "node:assert/strict"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; - -type SessionAccessorModule = typeof import("../src/config/sessions/session-accessor.js"); -type EntryCacheModule = - typeof import("../src/config/sessions/session-accessor.sqlite-entry-cache.js"); -type AgentDatabaseModule = typeof import("../src/state/openclaw-agent-db.js"); -type ReadOnlyDatabaseModule = typeof import("../src/state/openclaw-agent-db-readonly.js"); -type StateDatabaseModule = typeof import("../src/state/openclaw-state-db.js"); - -const repoRoot = process.env.PROOF_REPO_ROOT ?? process.cwd(); -const expectation = process.env.PROOF_EXPECTATION ?? "released"; -if (expectation !== "retained" && expectation !== "released") { - throw new Error("PROOF_EXPECTATION must be retained or released"); -} -const forceGc = globalThis.gc; -if (typeof forceGc !== "function") { - throw new Error("run with node --expose-gc"); -} - -const dataRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-entry-cache-proof-")); -const importSource = async (relativePath: string) => - import(pathToFileURL(path.join(repoRoot, relativePath)).href); -const { upsertSessionEntryCore } = (await importSource( - "src/config/sessions/session-accessor.js", -)) as SessionAccessorModule; -const { readSessionEntryCache } = (await importSource( - "src/config/sessions/session-accessor.sqlite-entry-cache.js", -)) as EntryCacheModule; -const { - OPENCLAW_AGENT_DB_OPEN_HANDLE_CAP, - closeOpenClawAgentDatabases, - isOpenClawAgentDatabaseOpen, - openOpenClawAgentDatabase, -} = (await importSource("src/state/openclaw-agent-db.js")) as AgentDatabaseModule; -const { withOpenClawAgentDatabaseReadOnly } = (await importSource( - "src/state/openclaw-agent-db-readonly.js", -)) as ReadOnlyDatabaseModule; -const { closeOpenClawStateDatabase } = (await importSource( - "src/state/openclaw-state-db.js", -)) as StateDatabaseModule; - -const readOnlyEntryCount = 20; -const readOnlyPayloadBytes = 1_000_000; -const lruHandleCount = OPENCLAW_AGENT_DB_OPEN_HANDLE_CAP + 2; -const lruPayloadBytes = 100_000; - -const createScope = (group: string, index: number) => ({ - agentId: "main", - env: { - ...process.env, - OPENCLAW_STATE_DIR: path.join(dataRoot, group, `agent-${index}`), - }, - sessionKey: `agent:main:${group}-${index}`, -}); - -const settleGc = async () => { - for (let attempt = 0; attempt < 12; attempt += 1) { - forceGc(); - await new Promise((resolve) => { - setImmediate(resolve); - }); - } -}; - -try { - const readOnlyScopes = Array.from({ length: readOnlyEntryCount }, (_, index) => - createScope("read-only", index), - ); - for (const [index, scope] of readOnlyScopes.entries()) { - fs.mkdirSync(scope.env.OPENCLAW_STATE_DIR, { recursive: true }); - await upsertSessionEntryCore(scope, { - label: `${index}:${"x".repeat(readOnlyPayloadBytes)}`, - sessionId: `read-only-${index}`, - updatedAt: index + 1, - }); - } - closeOpenClawAgentDatabases(); - closeOpenClawStateDatabase(); - - await settleGc(); - const readOnlyBaselineHeap = process.memoryUsage().heapUsed; - const readOnlyConnectionRefs: Array> = []; - - for (const scope of readOnlyScopes) { - const result = withOpenClawAgentDatabaseReadOnly((database) => { - const snapshot = readSessionEntryCache(database, { cache: true }); - assert.deepEqual(snapshot.keys, [scope.sessionKey]); - readOnlyConnectionRefs.push(new WeakRef(database.db)); - return snapshot.keys.length; - }, scope); - assert.equal(result.found, true); - } - - await settleGc(); - const readOnlyHeapDeltaBytes = process.memoryUsage().heapUsed - readOnlyBaselineHeap; - const readOnlyRetainedConnections = readOnlyConnectionRefs.filter((reference) => - reference.deref(), - ).length; - - const lruConnectionRefs: Array> = []; - const lruPaths: string[] = []; - for (let index = 0; index < lruHandleCount; index += 1) { - const scope = createScope("lru", index); - fs.mkdirSync(scope.env.OPENCLAW_STATE_DIR, { recursive: true }); - await upsertSessionEntryCore(scope, { - label: `${index}:${"y".repeat(lruPayloadBytes)}`, - sessionId: `lru-${index}`, - updatedAt: index + 1, - }); - const database = openOpenClawAgentDatabase(scope); - const snapshot = readSessionEntryCache(database, { cache: true }); - assert.deepEqual(snapshot.keys, [scope.sessionKey]); - lruConnectionRefs.push(new WeakRef(database.db)); - lruPaths.push(database.path); - } - - const lruEvictedCount = lruHandleCount - OPENCLAW_AGENT_DB_OPEN_HANDLE_CAP; - assert.equal(isOpenClawAgentDatabaseOpen(lruPaths[0]!), false); - assert.equal(isOpenClawAgentDatabaseOpen(lruPaths.at(-1)!), true); - await settleGc(); - const lruEvictedRetainedConnections = lruConnectionRefs - .slice(0, lruEvictedCount) - .filter((reference) => reference.deref()).length; - const lruLiveRetainedConnections = lruConnectionRefs - .slice(lruEvictedCount) - .filter((reference) => reference.deref()).length; - - const retained = - readOnlyRetainedConnections === readOnlyEntryCount && - readOnlyHeapDeltaBytes > 6_000_000 && - lruEvictedRetainedConnections === lruEvictedCount && - lruLiveRetainedConnections === OPENCLAW_AGENT_DB_OPEN_HANDLE_CAP; - const released = - readOnlyRetainedConnections <= 1 && - readOnlyHeapDeltaBytes < 6_000_000 && - lruEvictedRetainedConnections === 0 && - lruLiveRetainedConnections === OPENCLAW_AGENT_DB_OPEN_HANDLE_CAP; - const verdict = expectation === "released" ? released : retained; - - process.stdout.write( - `${JSON.stringify({ - schema: "sqlite-session-entry-cache-lifetime-proof-v2", - test_runner: "none", - affected_owners: { - database: "real-node-sqlite", - read_only_lifecycle: "withOpenClawAgentDatabaseReadOnly", - writable_lifecycle: "openOpenClawAgentDatabase LRU", - cache: "readSessionEntryCache", - }, - read_only_entry_count: readOnlyEntryCount, - read_only_payload_bytes_per_entry: readOnlyPayloadBytes, - read_only_retained_connections: readOnlyRetainedConnections, - read_only_heap_delta_bytes: readOnlyHeapDeltaBytes, - lru_handle_cap: OPENCLAW_AGENT_DB_OPEN_HANDLE_CAP, - lru_opened_handles: lruHandleCount, - lru_evicted_connections: lruEvictedCount, - lru_evicted_retained_connections: lruEvictedRetainedConnections, - lru_live_retained_connections: lruLiveRetainedConnections, - expectation, - verdict: verdict ? "PASS" : "FAIL", - })}\n`, - ); - if (!verdict) { - process.exitCode = 1; - } -} finally { - closeOpenClawAgentDatabases(); - closeOpenClawStateDatabase(); - fs.rmSync(dataRoot, { force: true, recursive: true }); -}