refactor(scripts): remove obsolete sqlite incident reproductions (#129212)

This commit is contained in:
Peter Steinberger
2026-08-25 05:16:50 -07:00
committed by GitHub
parent 0b867e14fe
commit 7401d21bdc
3 changed files with 0 additions and 523 deletions
-2
View File
@@ -74,7 +74,6 @@ const repositoryScriptEntries = [
// Invoked by scripts/lib/live-docker-stage.sh during container validation.
"scripts/live-docker-normalize-config.ts!",
"scripts/mcp-code-mode-gateway-e2e.ts!",
"scripts/memory-index-manager.sync-repro.ts!",
// Mantis invokes the trusted proof collector through its workflow shell step.
"scripts/mantis/telegram-visible-proof.mjs!",
"scripts/openclaw-release-clawhub-plan.ts!",
@@ -95,7 +94,6 @@ const repositoryScriptEntries = [
// Changed-file checks invoke this targeted UI Stylelint entrypoint by path.
"scripts/run-stylelint.mts!",
"scripts/secrets/openclaw-bws-resolver.mjs!",
"scripts/sqlite-session-entry-cache-lifetime-proof.ts!",
"scripts/sync-labels.ts!",
"scripts/test-built-bundled-channel-entry-smoke.mts!",
"scripts/update-clawtributors.ts!",
-347
View File
@@ -1,347 +0,0 @@
import { execFileSync } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { DatabaseSync } from "node:sqlite";
import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
import { resolveSessionTranscriptsDirForAgent } from "openclaw/plugin-sdk/memory-core-host-runtime-core";
import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime";
import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime";
import { resolveOpenClawAgentSqlitePath } from "openclaw/plugin-sdk/sqlite-runtime";
import {
closeAllMemorySearchManagers,
getMemorySearchManager,
} from "../extensions/memory-core/src/memory/index.ts";
const proofRoot = process.argv[2];
const exactHead = process.argv[3];
if (!proofRoot || !exactHead?.match(/^[0-9a-f]{40}$/)) {
throw new Error("proof root and exact 40-character head are required");
}
const observedHead = execFileSync("git", ["rev-parse", "HEAD"], {
encoding: "utf8",
}).trim();
if (observedHead !== exactHead) {
throw new Error(`exact head mismatch: expected ${exactHead}, observed ${observedHead}`);
}
const dirtyState = execFileSync("git", ["status", "--porcelain"], {
encoding: "utf8",
}).trim();
if (dirtyState) {
throw new Error("the production repro requires a clean exact-head worktree");
}
const agentId = "main";
const stateDir = path.join(proofRoot, "state");
const workspaceDir = path.join(proofRoot, "workspace");
const markers = {
blocker: "BLOCKER_LOCKED_SYNC_729",
retained: "RETAINED_RETRY_TARGET_729",
trigger: "CONCURRENT_TRIGGER_TARGET_729",
archive: "RETAINED_ARCHIVE_TARGET_729",
};
Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir);
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Proof workspace\n");
const cfg: OpenClawConfig = {
memory: {
search: {
provider: "none",
sources: ["sessions"],
rememberAcrossConversations: true,
store: { vector: { enabled: false } },
query: { minScore: 0 },
},
},
agents: {
defaults: { workspace: workspaceDir },
list: [{ id: agentId, default: true }],
},
};
async function seedSession(sessionId: string, marker: string): Promise<string> {
const sessionsDir = resolveSessionTranscriptsDirForAgent(agentId);
const storePath = path.join(sessionsDir, "sessions.json");
const sessionKey = `agent:${agentId}:proof:${sessionId}`;
await fs.mkdir(sessionsDir, { recursive: true });
await upsertSessionEntry({
agentId,
sessionKey,
storePath,
entry: { sessionId, updatedAt: Date.now() },
});
await appendSessionTranscriptMessageByIdentity({
agentId,
sessionId,
sessionKey,
storePath,
message: {
role: "user",
timestamp: Date.now(),
content: [{ type: "text", text: marker }],
},
});
return sessionKey;
}
function openExclusiveLock(dbPath: string): DatabaseSync {
const db = new DatabaseSync(dbPath);
db.exec("PRAGMA busy_timeout = 0");
db.exec("BEGIN EXCLUSIVE");
return db;
}
function releaseExclusiveLock(db: DatabaseSync | null): void {
if (!db) {
return;
}
try {
db.exec("ROLLBACK");
} finally {
db.close();
}
}
function describeSqliteFailure(failure: unknown): string {
const details = [String(failure)];
if (failure && typeof failure === "object") {
const record = failure as Record<string, unknown>;
for (const key of ["message", "code"] as const) {
if (typeof record[key] === "string") {
details.push(record[key]);
}
}
if (record.cause && typeof record.cause === "object") {
const cause = record.cause as Record<string, unknown>;
for (const key of ["message", "code"] as const) {
if (typeof cause[key] === "string") {
details.push(cause[key]);
}
}
}
}
return details.join(" ");
}
function isSqliteLockFailure(failure: unknown): boolean {
return /SQLITE_(?:BUSY|LOCKED)|database is (?:busy|locked)/i.test(describeSqliteFailure(failure));
}
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number, label: string): Promise<T> {
let timer: ReturnType<typeof setTimeout> | undefined;
const timeout = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => {
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
}, timeoutMs);
});
try {
return await Promise.race([promise, timeout]);
} finally {
if (timer) {
clearTimeout(timer);
}
}
}
let lock: DatabaseSync | null = null;
try {
const result = await getMemorySearchManager({ cfg, agentId });
if (!result.manager) {
throw new Error(`memory manager unavailable: ${result.error ?? "unknown"}`);
}
const manager = result.manager;
const sync = manager.sync?.bind(manager);
if (!sync) {
throw new Error("memory manager sync is unavailable");
}
await sync({ reason: "proof-baseline", force: true });
const blockerKey = await seedSession("proof-blocker", markers.blocker);
const retainedKey = await seedSession("proof-retained", markers.retained);
const triggerKey = await seedSession("proof-trigger", markers.trigger);
const archiveFile = path.join(
resolveSessionTranscriptsDirForAgent(agentId),
"proof-archive.jsonl.deleted.2026-07-29T00-00-00.000Z",
);
await fs.writeFile(
archiveFile,
[
JSON.stringify({
type: "session",
id: "proof-archive",
timestamp: new Date().toISOString(),
}),
JSON.stringify({
type: "message",
message: {
role: "user",
timestamp: Date.now(),
content: [{ type: "text", text: markers.archive }],
},
}),
].join("\n") + "\n",
"utf8",
);
const dbPath = resolveOpenClawAgentSqlitePath({ agentId });
lock = openExclusiveLock(dbPath);
const blockedOwner = sync({
reason: "proof-locked-owner",
sessions: [{ agentId, sessionId: "proof-blocker", sessionKey: blockerKey }],
});
const failedQueued = sync({
reason: "proof-queued-retained",
sessions: [{ agentId, sessionId: "proof-retained", sessionKey: retainedKey }],
archiveFiles: [archiveFile],
});
const failures = await Promise.allSettled([blockedOwner, failedQueued]);
const lockedSyncFailures = failures.filter((entry) => entry.status === "rejected").length;
const sqliteLockFailures = failures.filter(
(entry) => entry.status === "rejected" && isSqliteLockFailure(entry.reason),
).length;
releaseExclusiveLock(lock);
lock = null;
if (lockedSyncFailures !== 2) {
throw new Error(`expected two locked sync failures, received ${lockedSyncFailures}`);
}
if (sqliteLockFailures !== 2) {
throw new Error(`expected two SQLite lock failures, received ${sqliteLockFailures}`);
}
const observer = new DatabaseSync(dbPath, { readOnly: true });
const retainedBefore =
(
observer
.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE text LIKE ?")
.get(`%${markers.retained}%`) as { count: number }
).count > 0;
const triggerBefore =
(
observer
.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE text LIKE ?")
.get(`%${markers.trigger}%`) as { count: number }
).count > 0;
const archiveBefore =
(
observer
.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE text LIKE ?")
.get(`%${markers.archive}%`) as { count: number }
).count > 0;
observer.close();
if (retainedBefore || triggerBefore || archiveBefore) {
throw new Error("a recovery target was unexpectedly indexed before the idle trigger");
}
const recoveryState = manager as unknown as {
syncing: Promise<void> | null;
queuedArchiveFiles: Set<string>;
queuedSessions: Map<string, unknown>;
sessionsDirtyFiles: Set<string>;
sessionsFullRetryDirty: boolean;
};
const retainedQueueBeforeRecovery = recoveryState.queuedSessions.size;
const retainedArchiveQueueBeforeRecovery = recoveryState.queuedArchiveFiles.size;
const dirtySessionFilesBeforeRecovery = recoveryState.sessionsDirtyFiles.size;
const fullRetryBeforeRecovery = recoveryState.sessionsFullRetryDirty;
if (
recoveryState.syncing !== null ||
retainedQueueBeforeRecovery !== 1 ||
retainedArchiveQueueBeforeRecovery !== 1
) {
throw new Error("manager was not idle with exactly one retained session and archive target");
}
const recoveryProgress: Array<{ completed: number; total: number; label?: string }> = [];
const recovery = sync({
reason: "proof-idle-recovery-trigger",
sessions: [{ agentId, sessionId: "proof-trigger", sessionKey: triggerKey }],
progress: (update) => recoveryProgress.push(update),
});
// Start an untargeted sync before the retained queue owner resumes.
// Its distinct public progress callback proves that this call, rather than
// an implementation token, reached competing production admission.
const competingUntargetedProgress: Array<{ completed: number; total: number; label?: string }> =
[];
const competingUntargetedSync = sync({
reason: "proof-competing-untargeted-sync",
progress: (update) => competingUntargetedProgress.push(update),
});
const queueSettlementTimeoutMs = 15_000;
const recoveryResults = await withTimeout(
Promise.allSettled([recovery, competingUntargetedSync]),
queueSettlementTimeoutMs,
"queue-owner self-deadlock check",
);
const recoveryStatus = recoveryResults[0]?.status;
const competingUntargetedStatus = recoveryResults[1]?.status;
if (recoveryStatus !== "fulfilled" || competingUntargetedStatus !== "fulfilled") {
throw new Error(
`concurrent recovery did not settle: recovery=${recoveryStatus ?? "missing"} untargeted=${competingUntargetedStatus ?? "missing"}`,
);
}
if (competingUntargetedProgress.length === 0) {
throw new Error("competing untargeted sync emitted no public progress updates");
}
const recoveryObserver = new DatabaseSync(dbPath, { readOnly: true });
const indexedCount = (marker: string) =>
(
recoveryObserver
.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE text LIKE ?")
.get(`%${marker}%`) as { count: number }
).count;
const retainedAfter = indexedCount(markers.retained) > 0;
const triggerAfter = indexedCount(markers.trigger) > 0;
const archiveAfter = indexedCount(markers.archive) > 0;
recoveryObserver.close();
if (!retainedAfter || !triggerAfter || !archiveAfter) {
throw new Error(
`recovery result mismatch: retained=${String(retainedAfter)} trigger=${String(triggerAfter)} archive=${String(archiveAfter)}`,
);
}
if (recoveryProgress.length === 0) {
throw new Error("idle recovery trigger did not receive progress");
}
const retainedQueueAfterRecovery = recoveryState.queuedSessions.size;
const retainedArchiveQueueAfterRecovery = recoveryState.queuedArchiveFiles.size;
if (
dirtySessionFilesBeforeRecovery !== 0 ||
fullRetryBeforeRecovery ||
retainedQueueAfterRecovery !== 0 ||
retainedArchiveQueueAfterRecovery !== 0
) {
throw new Error(
`unexpected recovery ownership state: dirty=${dirtySessionFilesBeforeRecovery} fullRetry=${String(fullRetryBeforeRecovery)} retainedAfter=${retainedQueueAfterRecovery} archiveAfter=${retainedArchiveQueueAfterRecovery}`,
);
}
console.log(`exact_head=${exactHead}`);
console.log("test_runner=none");
console.log("entrypoint=MemoryIndexManager.sync");
console.log("owners=memory-manager,session-store,sqlite");
console.log(`locked_sync_failures=${lockedSyncFailures}`);
console.log("locked_sync_failure_kind=sqlite-busy");
console.log("recovery_manager_state=idle");
console.log("recovery_input_sessions=proof-trigger");
console.log(`recovery_progress_updates=${recoveryProgress.length}`);
console.log(`competing_untargeted_sync_progress_updates=${competingUntargetedProgress.length}`);
console.log(`recovery_sync_status=${recoveryStatus}`);
console.log(`competing_untargeted_sync_status=${competingUntargetedStatus}`);
console.log(`queue_settlement_timeout_ms=${queueSettlementTimeoutMs}`);
console.log(`retained_queue_before_recovery=${retainedQueueBeforeRecovery}`);
console.log(`retained_archive_queue_before_recovery=${retainedArchiveQueueBeforeRecovery}`);
console.log(`sessions_dirty_files_before_recovery=${dirtySessionFilesBeforeRecovery}`);
console.log(`sessions_full_retry_dirty_before_recovery=${String(fullRetryBeforeRecovery)}`);
console.log("retained_target_before_recovery=absent");
console.log("retained_archive_target_before_recovery=absent");
console.log("retained_target_after_recovery=indexed");
console.log("retained_archive_target_after_recovery=indexed");
console.log("idle_trigger_after_recovery=indexed");
console.log(`retained_queue_after_recovery=${retainedQueueAfterRecovery}`);
console.log(`retained_archive_queue_after_recovery=${retainedArchiveQueueAfterRecovery}`);
console.log("verdict=pass");
} finally {
releaseExclusiveLock(lock);
await closeAllMemorySearchManagers();
}
@@ -1,174 +0,0 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
type SessionAccessorModule = typeof import("../src/config/sessions/session-accessor.js");
type EntryCacheModule =
typeof import("../src/config/sessions/session-accessor.sqlite-entry-cache.js");
type AgentDatabaseModule = typeof import("../src/state/openclaw-agent-db.js");
type ReadOnlyDatabaseModule = typeof import("../src/state/openclaw-agent-db-readonly.js");
type StateDatabaseModule = typeof import("../src/state/openclaw-state-db.js");
const repoRoot = process.env.PROOF_REPO_ROOT ?? process.cwd();
const expectation = process.env.PROOF_EXPECTATION ?? "released";
if (expectation !== "retained" && expectation !== "released") {
throw new Error("PROOF_EXPECTATION must be retained or released");
}
const forceGc = globalThis.gc;
if (typeof forceGc !== "function") {
throw new Error("run with node --expose-gc");
}
const dataRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-entry-cache-proof-"));
const importSource = async (relativePath: string) =>
import(pathToFileURL(path.join(repoRoot, relativePath)).href);
const { upsertSessionEntryCore } = (await importSource(
"src/config/sessions/session-accessor.js",
)) as SessionAccessorModule;
const { readSessionEntryCache } = (await importSource(
"src/config/sessions/session-accessor.sqlite-entry-cache.js",
)) as EntryCacheModule;
const {
OPENCLAW_AGENT_DB_OPEN_HANDLE_CAP,
closeOpenClawAgentDatabases,
isOpenClawAgentDatabaseOpen,
openOpenClawAgentDatabase,
} = (await importSource("src/state/openclaw-agent-db.js")) as AgentDatabaseModule;
const { withOpenClawAgentDatabaseReadOnly } = (await importSource(
"src/state/openclaw-agent-db-readonly.js",
)) as ReadOnlyDatabaseModule;
const { closeOpenClawStateDatabase } = (await importSource(
"src/state/openclaw-state-db.js",
)) as StateDatabaseModule;
const readOnlyEntryCount = 20;
const readOnlyPayloadBytes = 1_000_000;
const lruHandleCount = OPENCLAW_AGENT_DB_OPEN_HANDLE_CAP + 2;
const lruPayloadBytes = 100_000;
const createScope = (group: string, index: number) => ({
agentId: "main",
env: {
...process.env,
OPENCLAW_STATE_DIR: path.join(dataRoot, group, `agent-${index}`),
},
sessionKey: `agent:main:${group}-${index}`,
});
const settleGc = async () => {
for (let attempt = 0; attempt < 12; attempt += 1) {
forceGc();
await new Promise<void>((resolve) => {
setImmediate(resolve);
});
}
};
try {
const readOnlyScopes = Array.from({ length: readOnlyEntryCount }, (_, index) =>
createScope("read-only", index),
);
for (const [index, scope] of readOnlyScopes.entries()) {
fs.mkdirSync(scope.env.OPENCLAW_STATE_DIR, { recursive: true });
await upsertSessionEntryCore(scope, {
label: `${index}:${"x".repeat(readOnlyPayloadBytes)}`,
sessionId: `read-only-${index}`,
updatedAt: index + 1,
});
}
closeOpenClawAgentDatabases();
closeOpenClawStateDatabase();
await settleGc();
const readOnlyBaselineHeap = process.memoryUsage().heapUsed;
const readOnlyConnectionRefs: Array<WeakRef<object>> = [];
for (const scope of readOnlyScopes) {
const result = withOpenClawAgentDatabaseReadOnly((database) => {
const snapshot = readSessionEntryCache(database, { cache: true });
assert.deepEqual(snapshot.keys, [scope.sessionKey]);
readOnlyConnectionRefs.push(new WeakRef(database.db));
return snapshot.keys.length;
}, scope);
assert.equal(result.found, true);
}
await settleGc();
const readOnlyHeapDeltaBytes = process.memoryUsage().heapUsed - readOnlyBaselineHeap;
const readOnlyRetainedConnections = readOnlyConnectionRefs.filter((reference) =>
reference.deref(),
).length;
const lruConnectionRefs: Array<WeakRef<object>> = [];
const lruPaths: string[] = [];
for (let index = 0; index < lruHandleCount; index += 1) {
const scope = createScope("lru", index);
fs.mkdirSync(scope.env.OPENCLAW_STATE_DIR, { recursive: true });
await upsertSessionEntryCore(scope, {
label: `${index}:${"y".repeat(lruPayloadBytes)}`,
sessionId: `lru-${index}`,
updatedAt: index + 1,
});
const database = openOpenClawAgentDatabase(scope);
const snapshot = readSessionEntryCache(database, { cache: true });
assert.deepEqual(snapshot.keys, [scope.sessionKey]);
lruConnectionRefs.push(new WeakRef(database.db));
lruPaths.push(database.path);
}
const lruEvictedCount = lruHandleCount - OPENCLAW_AGENT_DB_OPEN_HANDLE_CAP;
assert.equal(isOpenClawAgentDatabaseOpen(lruPaths[0]!), false);
assert.equal(isOpenClawAgentDatabaseOpen(lruPaths.at(-1)!), true);
await settleGc();
const lruEvictedRetainedConnections = lruConnectionRefs
.slice(0, lruEvictedCount)
.filter((reference) => reference.deref()).length;
const lruLiveRetainedConnections = lruConnectionRefs
.slice(lruEvictedCount)
.filter((reference) => reference.deref()).length;
const retained =
readOnlyRetainedConnections === readOnlyEntryCount &&
readOnlyHeapDeltaBytes > 6_000_000 &&
lruEvictedRetainedConnections === lruEvictedCount &&
lruLiveRetainedConnections === OPENCLAW_AGENT_DB_OPEN_HANDLE_CAP;
const released =
readOnlyRetainedConnections <= 1 &&
readOnlyHeapDeltaBytes < 6_000_000 &&
lruEvictedRetainedConnections === 0 &&
lruLiveRetainedConnections === OPENCLAW_AGENT_DB_OPEN_HANDLE_CAP;
const verdict = expectation === "released" ? released : retained;
process.stdout.write(
`${JSON.stringify({
schema: "sqlite-session-entry-cache-lifetime-proof-v2",
test_runner: "none",
affected_owners: {
database: "real-node-sqlite",
read_only_lifecycle: "withOpenClawAgentDatabaseReadOnly",
writable_lifecycle: "openOpenClawAgentDatabase LRU",
cache: "readSessionEntryCache",
},
read_only_entry_count: readOnlyEntryCount,
read_only_payload_bytes_per_entry: readOnlyPayloadBytes,
read_only_retained_connections: readOnlyRetainedConnections,
read_only_heap_delta_bytes: readOnlyHeapDeltaBytes,
lru_handle_cap: OPENCLAW_AGENT_DB_OPEN_HANDLE_CAP,
lru_opened_handles: lruHandleCount,
lru_evicted_connections: lruEvictedCount,
lru_evicted_retained_connections: lruEvictedRetainedConnections,
lru_live_retained_connections: lruLiveRetainedConnections,
expectation,
verdict: verdict ? "PASS" : "FAIL",
})}\n`,
);
if (!verdict) {
process.exitCode = 1;
}
} finally {
closeOpenClawAgentDatabases();
closeOpenClawStateDatabase();
fs.rmSync(dataRoot, { force: true, recursive: true });
}