From d946a02a13d099bac1dead2c35f09278b16a2aed Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 22 May 2026 20:28:13 +0100 Subject: [PATCH] fix(gateway): coalesce provider auth rewarms Coalesce provider auth-state rewarms after auth-profile failures and include event-loop delay in provider auth warm logs. --- CHANGELOG.md | 1 + .../server-startup-post-attach.test.ts | 17 ++- src/gateway/server-startup-post-attach.ts | 105 ++++++++++++++---- 3 files changed, 98 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fd89a1f7b42..8530830219a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ Docs: https://docs.openclaw.ai - Agents: bound embedded auto-compaction session write-lock watchdogs to the compaction timeout instead of the full run timeout, so stuck compaction cannot hold the live session lock for the whole run window. (#84949) Thanks @luoyanglang. - Gateway/agents: return phase-aware `agent.wait` timeout attribution and only cool auth profiles on provider-started timeouts. Refs #65504. Thanks @100yenadmin. - Gateway: defer provider auth-state prewarm until after startup readiness so early gateway tool/session requests are not blocked by provider auth discovery. (#85272) Thanks @dutifulbob. +- Gateway/models: coalesce provider auth-state rewarms after auth-profile failures and log event-loop delay for warm/rewarm work, so provider auth bursts no longer stack full auth sweeps behind channel replies. - Agents/Codex: show the first plan update as a transient chat status notice without counting it as final assistant content. - CLI/update: walk the macOS process ancestry and honor the inherited Gateway runtime PID before package updates stop the managed Gateway service, so nested in-band updater children can refuse instead of killing the LaunchAgent-supervised Gateway that owns them. Fixes #85120. - Gateway/LaunchAgent: wait for launchd reload bootout to finish and fall back to kickstart when bootstrap races, so reload handoff does not leave the service deregistered. Fixes #84630. (#84641) Thanks @NianJiuZst. diff --git a/src/gateway/server-startup-post-attach.test.ts b/src/gateway/server-startup-post-attach.test.ts index 0951228033a4..22eeb137e248 100644 --- a/src/gateway/server-startup-post-attach.test.ts +++ b/src/gateway/server-startup-post-attach.test.ts @@ -978,7 +978,12 @@ describe("startGatewayPostAttachRuntime", () => { const hook = hoisted.setAuthProfileFailureHook.mock.calls[0]?.[0] as (() => void) | undefined; hook?.(); expect(hoisted.clearCurrentProviderAuthState).toHaveBeenCalledTimes(1); - expect(hoisted.warmCurrentProviderAuthState).toHaveBeenCalledTimes(2); + expect(hoisted.warmCurrentProviderAuthState).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(1_000); + await vi.waitFor(() => { + expect(hoisted.warmCurrentProviderAuthState).toHaveBeenCalledTimes(2); + }); } finally { vi.useRealTimers(); } @@ -1044,11 +1049,15 @@ describe("startGatewayPostAttachRuntime", () => { hook(); currentCfg = afterFailureCfg; hook(); + expect(hoisted.warmCurrentProviderAuthState).toHaveBeenCalledTimes(1); expect(hoisted.clearCurrentProviderAuthState).toHaveBeenCalledTimes(2); - expect(hoisted.warmCurrentProviderAuthState).toHaveBeenCalledTimes(3); + + await vi.advanceTimersByTimeAsync(1_000); + await vi.waitFor(() => { + expect(hoisted.warmCurrentProviderAuthState).toHaveBeenCalledTimes(2); + }); expect(hoisted.warmCurrentProviderAuthState.mock.calls[0]?.[0]).toBe(reloadedCfg); - expect(hoisted.warmCurrentProviderAuthState.mock.calls[1]?.[0]).toBe(reloadedCfg); - expect(hoisted.warmCurrentProviderAuthState.mock.calls[2]?.[0]).toBe(afterFailureCfg); + expect(hoisted.warmCurrentProviderAuthState.mock.calls[1]?.[0]).toBe(afterFailureCfg); } finally { vi.useRealTimers(); } diff --git a/src/gateway/server-startup-post-attach.ts b/src/gateway/server-startup-post-attach.ts index 9b2acd1f7dae..e22bd1948978 100644 --- a/src/gateway/server-startup-post-attach.ts +++ b/src/gateway/server-startup-post-attach.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { monitorEventLoopDelay, performance } from "node:perf_hooks"; import { setTimeout as sleep } from "node:timers/promises"; import type { CliDeps } from "../cli/deps.types.js"; import type { GatewayTailscaleMode } from "../config/types.gateway.js"; @@ -28,6 +29,7 @@ const ACP_BACKEND_READY_POLL_MS = 50; const PRIMARY_MODEL_PREWARM_TIMEOUT_MS = 5_000; const STARTUP_PROVIDER_DISCOVERY_TIMEOUT_MS = 5_000; const PROVIDER_AUTH_PREWARM_START_DELAY_MS = 1_000; +const PROVIDER_AUTH_REWARM_DELAY_MS = 1_000; const SKIP_STARTUP_MODEL_PREWARM_ENV = "OPENCLAW_SKIP_STARTUP_MODEL_PREWARM"; const QMD_STARTUP_IDLE_DELAY_MS = 120_000; const RESTART_SENTINEL_FILENAME = "restart-sentinel.json"; @@ -69,6 +71,31 @@ async function measureStartup( return startupTrace ? startupTrace.measure(name, run) : await run(); } +async function measureProviderAuthWarm(run: () => Promise): Promise<{ + elapsedMs: number; + eventLoopMaxMs: number; +}> { + const eventLoopDelay = monitorEventLoopDelay({ resolution: 10 }); + eventLoopDelay.enable(); + const startMs = performance.now(); + try { + await run(); + } finally { + eventLoopDelay.disable(); + } + return { + elapsedMs: performance.now() - startMs, + eventLoopMaxMs: eventLoopDelay.max / 1_000_000, + }; +} + +function formatProviderAuthWarmMetrics(metrics: { + elapsedMs: number; + eventLoopMaxMs: number; +}): string { + return `in ${metrics.elapsedMs.toFixed(0)}ms eventLoopMax=${metrics.eventLoopMaxMs.toFixed(1)}ms`; +} + function shouldCheckRestartSentinel(env: NodeJS.ProcessEnv = process.env): boolean { return !env.VITEST && env.NODE_ENV !== "test"; } @@ -155,29 +182,58 @@ function scheduleProviderAuthStatePrewarm(params: { delayMs?: number; }): GatewayPostReadySidecarHandle { let stopped = false; - let timer: ReturnType | undefined; + let startupTimer: ReturnType | undefined; + let rewarmTimer: ReturnType | undefined; + let rewarmInFlight = false; + let pendingRewarmReason: string | undefined; const isStopped = () => stopped; const delayMs = params.delayMs ?? PROVIDER_AUTH_PREWARM_START_DELAY_MS; void (async () => { const { clearCurrentProviderAuthState, warmCurrentProviderAuthState } = await import("../agents/model-provider-auth.js"); const { setAuthProfileFailureHook } = await import("../agents/auth-profiles.js"); - const scheduleAuthMapRewarm = (reason: string) => { + const runRewarm = async (reason: string) => { if (isStopped()) { return; } const cfg = params.getConfig(); - const startMs = Date.now(); - void warmCurrentProviderAuthState(cfg, { isCancelled: isStopped }) - .then(() => { - if (isStopped()) { - return; - } - params.log.info(`provider auth state re-warmed (${reason}) in ${Date.now() - startMs}ms`); - }) - .catch((err) => { - params.log.warn(`provider auth state rewarm failed: ${String(err)}`); - }); + rewarmInFlight = true; + try { + const metrics = await measureProviderAuthWarm(() => + warmCurrentProviderAuthState(cfg, { isCancelled: isStopped }), + ); + if (isStopped()) { + return; + } + params.log.info( + `provider auth state re-warmed (${reason}) ${formatProviderAuthWarmMetrics(metrics)}`, + ); + } catch (err) { + params.log.warn(`provider auth state rewarm failed: ${String(err)}`); + } finally { + rewarmInFlight = false; + const nextReason = pendingRewarmReason; + pendingRewarmReason = undefined; + if (nextReason && !isStopped()) { + scheduleAuthMapRewarm(nextReason); + } + } + }; + const scheduleAuthMapRewarm = (reason: string) => { + if (isStopped()) { + return; + } + pendingRewarmReason = reason; + if (rewarmTimer || rewarmInFlight) { + return; + } + rewarmTimer = setTimeout(() => { + rewarmTimer = undefined; + const nextReason = pendingRewarmReason ?? reason; + pendingRewarmReason = undefined; + void runRewarm(nextReason); + }, PROVIDER_AUTH_REWARM_DELAY_MS); + rewarmTimer.unref?.(); }; if (isStopped()) { return; @@ -189,35 +245,42 @@ function scheduleProviderAuthStatePrewarm(params: { clearCurrentProviderAuthState(); scheduleAuthMapRewarm("auth-profile-failure"); }); - timer = setTimeout( + startupTimer = setTimeout( () => { void (async () => { if (isStopped()) { return; } const cfg = params.getConfig(); - const startMs = Date.now(); - await warmCurrentProviderAuthState(cfg, { isCancelled: isStopped }); + const metrics = await measureProviderAuthWarm(() => + warmCurrentProviderAuthState(cfg, { isCancelled: isStopped }), + ); if (isStopped()) { return; } - params.log.info(`provider auth state pre-warmed in ${Date.now() - startMs}ms`); + params.log.info( + `provider auth state pre-warmed ${formatProviderAuthWarmMetrics(metrics)}`, + ); })().catch((err) => { params.log.warn(`provider auth state pre-warm failed: ${String(err)}`); }); }, Math.max(0, delayMs), ); - timer.unref?.(); + startupTimer.unref?.(); })().catch((err) => { params.log.warn(`provider auth state pre-warm setup failed: ${String(err)}`); }); return { stop: () => { stopped = true; - if (timer) { - clearTimeout(timer); - timer = undefined; + if (startupTimer) { + clearTimeout(startupTimer); + startupTimer = undefined; + } + if (rewarmTimer) { + clearTimeout(rewarmTimer); + rewarmTimer = undefined; } }, };