diff --git a/src/logging/diagnostic-memory.test.ts b/src/logging/diagnostic-memory.test.ts index dce15fead67b..ff87998e9c73 100644 --- a/src/logging/diagnostic-memory.test.ts +++ b/src/logging/diagnostic-memory.test.ts @@ -156,6 +156,75 @@ describe("diagnostic memory", () => { expect(events.map((event) => event.type)).toEqual(["diagnostic.memory.pressure"]); }); + it("scales default heap pressure thresholds with enlarged V8 limits", () => { + const events: DiagnosticEventPayload[] = []; + const stop = onDiagnosticEvent((event) => events.push(event)); + const gb = 1024 ** 3; + + emitDiagnosticMemorySample({ + now: 1000, + heapSizeLimitBytes: 8 * gb, + memoryUsage: memoryUsage({ heapUsed: 2.1 * gb }), + }); + expect(events.filter((event) => event.type === "diagnostic.memory.pressure")).toEqual([]); + + emitDiagnosticMemorySample({ + now: 2000, + heapSizeLimitBytes: 8 * gb, + memoryUsage: memoryUsage({ heapUsed: 4.1 * gb }), + }); + emitDiagnosticMemorySample({ + now: 3000, + heapSizeLimitBytes: 8 * gb, + memoryUsage: memoryUsage({ heapUsed: 6.1 * gb }), + }); + stop(); + + expect( + events + .filter((event) => event.type === "diagnostic.memory.pressure") + .map((event) => ({ + level: event.level, + reason: event.reason, + threshold: event.thresholdBytes, + })), + ).toEqual([ + { level: "warning", reason: "heap_threshold", threshold: 4 * gb }, + { level: "critical", reason: "heap_threshold", threshold: 6 * gb }, + ]); + }); + + it("scales default heap pressure thresholds down for constrained V8 limits", () => { + const events: DiagnosticEventPayload[] = []; + const stop = onDiagnosticEvent((event) => events.push(event)); + const mb = 1024 ** 2; + + emitDiagnosticMemorySample({ + now: 1000, + heapSizeLimitBytes: 1024 * mb, + memoryUsage: memoryUsage({ heapUsed: 600 * mb }), + }); + emitDiagnosticMemorySample({ + now: 2000, + heapSizeLimitBytes: 1024 * mb, + memoryUsage: memoryUsage({ heapUsed: 800 * mb }), + }); + stop(); + + expect( + events + .filter((event) => event.type === "diagnostic.memory.pressure") + .map((event) => ({ + level: event.level, + reason: event.reason, + threshold: event.thresholdBytes, + })), + ).toEqual([ + { level: "warning", reason: "heap_threshold", threshold: 512 * mb }, + { level: "critical", reason: "heap_threshold", threshold: 768 * mb }, + ]); + }); + it("emits pressure when RSS grows quickly", () => { const events: DiagnosticEventPayload[] = []; const stop = onDiagnosticEvent((event) => events.push(event)); diff --git a/src/logging/diagnostic-memory.ts b/src/logging/diagnostic-memory.ts index 26fc94eeb759..d34fad3c5388 100644 --- a/src/logging/diagnostic-memory.ts +++ b/src/logging/diagnostic-memory.ts @@ -1,4 +1,5 @@ // Diagnostic memory helpers capture process memory facts for support diagnostics. +import { getHeapStatistics } from "node:v8"; import { emitInternalDiagnosticEvent as emitDiagnosticEvent, type DiagnosticMemoryPressureEvent, @@ -9,10 +10,15 @@ import { createSubsystemLogger } from "./subsystem.js"; // Diagnostic memory sampler with threshold/growth pressure detection and repeat suppression. const MB = 1024 * 1024; +const GB = 1024 * MB; const DEFAULT_RSS_WARNING_BYTES = 1536 * MB; const DEFAULT_RSS_CRITICAL_BYTES = 3072 * MB; const DEFAULT_HEAP_WARNING_BYTES = 1024 * MB; const DEFAULT_HEAP_CRITICAL_BYTES = 2048 * MB; +const DEFAULT_HEAP_WARNING_RATIO = 0.5; +const DEFAULT_HEAP_CRITICAL_RATIO = 0.75; +const DEFAULT_HEAP_WARNING_MAX_BYTES = 4 * GB; +const DEFAULT_HEAP_CRITICAL_MAX_BYTES = 6 * GB; const DEFAULT_RSS_GROWTH_WARNING_BYTES = 512 * MB; const DEFAULT_RSS_GROWTH_CRITICAL_BYTES = 1024 * MB; const DEFAULT_GROWTH_WINDOW_MS = 10 * 60 * 1000; @@ -60,12 +66,31 @@ function normalizeMemoryUsage(memory: NodeJS.MemoryUsage): DiagnosticMemoryUsage function resolveThresholds( thresholds?: DiagnosticMemoryThresholds, + heapSizeLimitBytes?: number, ): Required { + const hasHeapLimit = + typeof heapSizeLimitBytes === "number" && + Number.isFinite(heapSizeLimitBytes) && + heapSizeLimitBytes > 0; + // Scale both directions with V8's effective limit, but keep a warning/critical + // ceiling so very large heaps still surface actionable pressure diagnostics. + const heapWarningBytes = hasHeapLimit + ? Math.min( + Math.floor(heapSizeLimitBytes * DEFAULT_HEAP_WARNING_RATIO), + DEFAULT_HEAP_WARNING_MAX_BYTES, + ) + : DEFAULT_HEAP_WARNING_BYTES; + const heapCriticalBytes = hasHeapLimit + ? Math.min( + Math.floor(heapSizeLimitBytes * DEFAULT_HEAP_CRITICAL_RATIO), + DEFAULT_HEAP_CRITICAL_MAX_BYTES, + ) + : DEFAULT_HEAP_CRITICAL_BYTES; return { rssWarningBytes: thresholds?.rssWarningBytes ?? DEFAULT_RSS_WARNING_BYTES, rssCriticalBytes: thresholds?.rssCriticalBytes ?? DEFAULT_RSS_CRITICAL_BYTES, - heapUsedWarningBytes: thresholds?.heapUsedWarningBytes ?? DEFAULT_HEAP_WARNING_BYTES, - heapUsedCriticalBytes: thresholds?.heapUsedCriticalBytes ?? DEFAULT_HEAP_CRITICAL_BYTES, + heapUsedWarningBytes: thresholds?.heapUsedWarningBytes ?? heapWarningBytes, + heapUsedCriticalBytes: thresholds?.heapUsedCriticalBytes ?? heapCriticalBytes, rssGrowthWarningBytes: thresholds?.rssGrowthWarningBytes ?? DEFAULT_RSS_GROWTH_WARNING_BYTES, rssGrowthCriticalBytes: thresholds?.rssGrowthCriticalBytes ?? DEFAULT_RSS_GROWTH_CRITICAL_BYTES, growthWindowMs: thresholds?.growthWindowMs ?? DEFAULT_GROWTH_WINDOW_MS, @@ -263,6 +288,7 @@ function logMemoryPressure(params: { export function emitDiagnosticMemorySample(options?: { now?: number; memoryUsage?: NodeJS.MemoryUsage; + heapSizeLimitBytes?: number; uptimeMs?: number; thresholds?: DiagnosticMemoryThresholds; emitSample?: boolean; @@ -274,7 +300,10 @@ export function emitDiagnosticMemorySample(options?: { const now = options?.now ?? Date.now(); const memory = normalizeMemoryUsage(options?.memoryUsage ?? process.memoryUsage()); const current = { ts: now, memory }; - const thresholds = resolveThresholds(options?.thresholds); + const thresholds = resolveThresholds( + options?.thresholds, + options?.heapSizeLimitBytes ?? getHeapStatistics().heap_size_limit, + ); const shouldEmitSample = options?.emitSample !== false; if (shouldEmitSample) {