fix(sessions): warn when a fire-and-forget disk-budget sweep fails (#124681)

The throttled budget kick swallowed every sweep failure into an empty
catch. Retrying on the next kick is right, but a persistently failing
sweep (corrupt store, permission loss) meant unbounded disk growth with
no operator signal — silent failure on the default maintenance path.
Warn with the error and store path before the retry.
This commit is contained in:
Peter Steinberger
2026-08-16 09:15:12 -07:00
committed by GitHub
parent 2404d41de2
commit 2b8e94ea4d
2 changed files with 56 additions and 3 deletions
@@ -1,7 +1,23 @@
import { randomBytes } from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const evictionWarnSpy = vi.hoisted(() => vi.fn());
vi.mock("../../logging/subsystem.js", async () => {
const actual = await vi.importActual<typeof import("../../logging/subsystem.js")>(
"../../logging/subsystem.js",
);
return {
...actual,
createSubsystemLogger: (subsystem: string) => {
const logger = actual.createSubsystemLogger(subsystem);
return subsystem === "sessions/history-eviction"
? { ...logger, warn: evictionWarnSpy }
: logger;
},
};
});
import { executeSqliteQuerySync } from "../../infra/kysely-sync.js";
import { beginSessionWorkAdmission } from "../../sessions/session-lifecycle-admission.js";
import {
@@ -25,6 +41,7 @@ import { getSessionKysely } from "./session-accessor.sqlite-scope.js";
import {
enforceSqliteSessionHistoryDiskBudget,
inspectSqliteSessionHistoryDiskBudget,
kickSessionHistoryDiskBudgetMaintenance,
} from "./session-history-eviction.js";
import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js";
@@ -344,6 +361,33 @@ describe("SQLite historical session disk budget", () => {
expect(sessionExists("stale-live")).toBe(true);
});
it("warns when a fire-and-forget budget sweep fails instead of swallowing it", async () => {
evictionWarnSpy.mockClear();
// The kick gate reads maxDiskBytes twice synchronously; the queued sweep
// re-reads it asynchronously. Throwing on the later read rejects the
// fire-and-forget promise, exercising the catch path deterministically.
let maxDiskBytesReads = 0;
const maintenanceConfig = {
mode: "enforce",
highWaterBytes: 1,
get maxDiskBytes() {
maxDiskBytesReads += 1;
if (maxDiskBytesReads > 1) {
throw new Error("sweep exploded");
}
return 1;
},
} as never;
kickSessionHistoryDiskBudgetMaintenance({ storePath, force: true, maintenanceConfig });
await vi.waitFor(() => {
expect(evictionWarnSpy).toHaveBeenCalledWith(
expect.stringContaining("disk-budget sweep failed"),
expect.objectContaining({ storePath }),
);
});
});
it("warn mode reports physical overage without extracting or deleting history", async () => {
await createHistoricalTranscript({
content: "warn history",
@@ -1,6 +1,7 @@
import fs from "node:fs";
import path from "node:path";
import { executeSqliteQuerySync } from "../../infra/kysely-sync.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
import {
collectActiveSessionWorkAdmissionIdentities,
runExclusiveSessionLifecycleMutation,
@@ -441,6 +442,8 @@ async function pruneAllSessionTranscriptArchivesToHighWater(params: {
};
}
const log = createSubsystemLogger("sessions/history-eviction");
const PHYSICAL_BUDGET_CHECK_INTERVAL_MS = 30 * 60 * 1000;
// Single-slot per store: ordinary entry writes kick a throttled background
// budget pass so an over-budget database self-heals without waiting for a
@@ -503,8 +506,14 @@ export function kickSessionHistoryDiskBudgetMaintenance(params: {
mode: maintenance.mode,
maintenance,
})
.catch(() => {
// Best-effort: budget pressure is retried on the next throttled kick.
.catch((error: unknown) => {
// Best-effort: budget pressure is retried on the next throttled kick,
// but a persistently failing sweep must stay operator-visible — silent
// failure here means unbounded disk growth with no signal.
log.warn("session history disk-budget sweep failed; retrying on next kick", {
error,
storePath: params.storePath,
});
})
.finally(() => {
state.running = false;