mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 05:16:23 -06:00
fix(sessions): warn when a fire-and-forget disk-budget sweep fails (#124681)
The throttled budget kick swallowed every sweep failure into an empty catch. Retrying on the next kick is right, but a persistently failing sweep (corrupt store, permission loss) meant unbounded disk growth with no operator signal — silent failure on the default maintenance path. Warn with the error and store path before the retry.
This commit is contained in:
committed by
GitHub
parent
2404d41de2
commit
2b8e94ea4d
@@ -1,7 +1,23 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const evictionWarnSpy = vi.hoisted(() => vi.fn());
|
||||
vi.mock("../../logging/subsystem.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../../logging/subsystem.js")>(
|
||||
"../../logging/subsystem.js",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
createSubsystemLogger: (subsystem: string) => {
|
||||
const logger = actual.createSubsystemLogger(subsystem);
|
||||
return subsystem === "sessions/history-eviction"
|
||||
? { ...logger, warn: evictionWarnSpy }
|
||||
: logger;
|
||||
},
|
||||
};
|
||||
});
|
||||
import { executeSqliteQuerySync } from "../../infra/kysely-sync.js";
|
||||
import { beginSessionWorkAdmission } from "../../sessions/session-lifecycle-admission.js";
|
||||
import {
|
||||
@@ -25,6 +41,7 @@ import { getSessionKysely } from "./session-accessor.sqlite-scope.js";
|
||||
import {
|
||||
enforceSqliteSessionHistoryDiskBudget,
|
||||
inspectSqliteSessionHistoryDiskBudget,
|
||||
kickSessionHistoryDiskBudgetMaintenance,
|
||||
} from "./session-history-eviction.js";
|
||||
import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js";
|
||||
|
||||
@@ -344,6 +361,33 @@ describe("SQLite historical session disk budget", () => {
|
||||
expect(sessionExists("stale-live")).toBe(true);
|
||||
});
|
||||
|
||||
it("warns when a fire-and-forget budget sweep fails instead of swallowing it", async () => {
|
||||
evictionWarnSpy.mockClear();
|
||||
// The kick gate reads maxDiskBytes twice synchronously; the queued sweep
|
||||
// re-reads it asynchronously. Throwing on the later read rejects the
|
||||
// fire-and-forget promise, exercising the catch path deterministically.
|
||||
let maxDiskBytesReads = 0;
|
||||
const maintenanceConfig = {
|
||||
mode: "enforce",
|
||||
highWaterBytes: 1,
|
||||
get maxDiskBytes() {
|
||||
maxDiskBytesReads += 1;
|
||||
if (maxDiskBytesReads > 1) {
|
||||
throw new Error("sweep exploded");
|
||||
}
|
||||
return 1;
|
||||
},
|
||||
} as never;
|
||||
|
||||
kickSessionHistoryDiskBudgetMaintenance({ storePath, force: true, maintenanceConfig });
|
||||
await vi.waitFor(() => {
|
||||
expect(evictionWarnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("disk-budget sweep failed"),
|
||||
expect.objectContaining({ storePath }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("warn mode reports physical overage without extracting or deleting history", async () => {
|
||||
await createHistoricalTranscript({
|
||||
content: "warn history",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { executeSqliteQuerySync } from "../../infra/kysely-sync.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import {
|
||||
collectActiveSessionWorkAdmissionIdentities,
|
||||
runExclusiveSessionLifecycleMutation,
|
||||
@@ -441,6 +442,8 @@ async function pruneAllSessionTranscriptArchivesToHighWater(params: {
|
||||
};
|
||||
}
|
||||
|
||||
const log = createSubsystemLogger("sessions/history-eviction");
|
||||
|
||||
const PHYSICAL_BUDGET_CHECK_INTERVAL_MS = 30 * 60 * 1000;
|
||||
// Single-slot per store: ordinary entry writes kick a throttled background
|
||||
// budget pass so an over-budget database self-heals without waiting for a
|
||||
@@ -503,8 +506,14 @@ export function kickSessionHistoryDiskBudgetMaintenance(params: {
|
||||
mode: maintenance.mode,
|
||||
maintenance,
|
||||
})
|
||||
.catch(() => {
|
||||
// Best-effort: budget pressure is retried on the next throttled kick.
|
||||
.catch((error: unknown) => {
|
||||
// Best-effort: budget pressure is retried on the next throttled kick,
|
||||
// but a persistently failing sweep must stay operator-visible — silent
|
||||
// failure here means unbounded disk growth with no signal.
|
||||
log.warn("session history disk-budget sweep failed; retrying on next kick", {
|
||||
error,
|
||||
storePath: params.storePath,
|
||||
});
|
||||
})
|
||||
.finally(() => {
|
||||
state.running = false;
|
||||
|
||||
Reference in New Issue
Block a user