diff --git a/docs/concepts/agent-loop.md b/docs/concepts/agent-loop.md index ba35ceb7b135..10a752defa4d 100644 --- a/docs/concepts/agent-loop.md +++ b/docs/concepts/agent-loop.md @@ -167,7 +167,7 @@ surfaces, while Codex native hooks remain a separate lower-level Codex mechanism - Agent runtime: `agents.defaults.timeoutSeconds` default 172800s (48 hours); enforced in `runEmbeddedAgent` abort timer. - Cron runtime: isolated agent-turn `timeoutSeconds` is owned by cron. The scheduler starts that timer when execution begins, aborts the underlying run at the configured deadline, then runs bounded cleanup before recording the timeout so a stale child session cannot keep the lane stuck. - Session liveness diagnostics: with diagnostics enabled, `diagnostics.stuckSessionWarnMs` classifies long `processing` sessions that have no observed reply, tool, status, block, or ACP progress. Active embedded runs, model calls, and tool calls report as `session.long_running`; owned silent model calls also stay `session.long_running` until `diagnostics.stuckSessionAbortMs` so slow or non-streaming providers are not reported as stalled too early. Active work with no recent progress reports as `session.stalled`; owned model calls switch to `session.stalled` at or after the abort threshold, and ownerless stale model/tool activity is not hidden as long-running. `session.stuck` is reserved for recoverable stale session bookkeeping, including idle queued sessions with stale ownerless model/tool activity. Stale session bookkeeping releases the affected session lane immediately after recovery gates pass; stalled embedded runs are abort-drained only after `diagnostics.stuckSessionAbortMs` (default: at least 5 minutes and 3x the warning threshold) so queued work can resume without cutting off merely slow runs. Recovery emits structured requested/completed outcomes, and diagnostic state is marked idle only if the same processing generation is still current. Repeated `session.stuck` diagnostics back off while the session remains unchanged. -- Model idle timeout: OpenClaw aborts a model request when no response chunks arrive before the idle window. `models.providers..timeoutSeconds` extends this idle watchdog for slow local/self-hosted providers, but it is still bounded by any lower `agents.defaults.timeoutSeconds` or run-specific timeout because those control the whole agent run. Otherwise OpenClaw uses `agents.defaults.timeoutSeconds` when configured, capped at 120s by default. Cron-triggered cloud model runs with no explicit model or agent timeout use the same default idle watchdog; cron-triggered local or self-hosted model runs disable the implicit watchdog unless an explicit timeout is configured, so slow local providers should set `models.providers..timeoutSeconds`. +- Model idle timeout: OpenClaw aborts a model request when no response chunks arrive before the idle window. `models.providers..timeoutSeconds` extends this idle watchdog for slow local/self-hosted providers, but it is still bounded by any lower `agents.defaults.timeoutSeconds` or run-specific timeout because those control the whole agent run. Otherwise OpenClaw uses `agents.defaults.timeoutSeconds` when configured, capped at 120s by default. Cron-triggered cloud model runs with no explicit model or agent timeout use the same default idle watchdog; with an explicit cron run timeout, cloud model stream stalls are capped at 60s so configured model fallbacks can run before the outer cron deadline. Cron-triggered local or self-hosted model runs disable the implicit watchdog unless an explicit timeout is configured, and explicit cron run timeouts remain the idle window for local/self-hosted providers, so slow local providers should set `models.providers..timeoutSeconds`. - Provider HTTP request timeout: `models.providers..timeoutSeconds` applies to that provider's model HTTP fetches, including connect, headers, body, SDK request timeout, total guarded-fetch abort handling, and model stream idle watchdog. Use this for slow local/self-hosted providers such as Ollama before raising the whole agent runtime timeout, and keep the agent/runtime timeout at least as high when the model request needs to run longer. ## Where things can end early diff --git a/src/agents/embedded-agent-runner/run/attempt.ts b/src/agents/embedded-agent-runner/run/attempt.ts index c818c1225cd3..9dd0ec3e2368 100644 --- a/src/agents/embedded-agent-runner/run/attempt.ts +++ b/src/agents/embedded-agent-runner/run/attempt.ts @@ -3129,7 +3129,11 @@ export async function runEmbeddedAttempt( trigger: params.trigger, runTimeoutMs: resolvedRunTimeoutMs, modelRequestTimeoutMs: (params.model as { requestTimeoutMs?: number }).requestTimeoutMs, - model: params.model as { baseUrl?: string }, + model: { + baseUrl: params.model.baseUrl, + id: params.modelId, + provider: params.provider, + }, }); if (idleTimeoutMs > 0) { activeSession.agent.streamFn = streamWithIdleTimeout( diff --git a/src/agents/embedded-agent-runner/run/llm-idle-timeout.test.ts b/src/agents/embedded-agent-runner/run/llm-idle-timeout.test.ts index 6459a6365a98..65a6dd1849ce 100644 --- a/src/agents/embedded-agent-runner/run/llm-idle-timeout.test.ts +++ b/src/agents/embedded-agent-runner/run/llm-idle-timeout.test.ts @@ -12,6 +12,7 @@ import type { StreamFn } from "../../runtime/index.js"; import { resolveLlmIdleTimeoutMs, streamWithIdleTimeout } from "./llm-idle-timeout.js"; const DEFAULT_LLM_IDLE_TIMEOUT_MS = 120_000; +const CRON_LLM_IDLE_TIMEOUT_MS = 60_000; describe("resolveLlmIdleTimeoutMs", () => { it("returns default when config is undefined", () => { @@ -41,8 +42,153 @@ describe("resolveLlmIdleTimeoutMs", () => { expect(resolveLlmIdleTimeoutMs({ runTimeoutMs: 30_000 })).toBe(30_000); }); - it("honors explicit cron run timeouts as the idle watchdog ceiling", () => { - expect(resolveLlmIdleTimeoutMs({ trigger: "cron", runTimeoutMs: 600_000 })).toBe(600_000); + it("caps explicit cron run timeouts so stream stalls can reach model fallbacks", () => { + expect(resolveLlmIdleTimeoutMs({ trigger: "cron", runTimeoutMs: 600_000 })).toBe( + CRON_LLM_IDLE_TIMEOUT_MS, + ); + }); + + it("uses shorter explicit cron run timeouts as the idle watchdog ceiling", () => { + expect(resolveLlmIdleTimeoutMs({ trigger: "cron", runTimeoutMs: 30_000 })).toBe(30_000); + }); + + it("honors explicit cron run timeouts for local provider model calls", () => { + expect( + resolveLlmIdleTimeoutMs({ + trigger: "cron", + runTimeoutMs: 600_000, + model: { baseUrl: "http://127.0.0.1:11434" }, + }), + ).toBe(600_000); + }); + + it.each([ + ["ollama", "http://ollama-host:11434"], + ["ollama-beelink", "http://ollama-host:11434"], + ["lmstudio", "http://lmstudio-box:1234/v1"], + ["lmstudio-mac", "http://lmstudio-box:1234/v1"], + ["vllm", "http://vllm-rig:8000/v1"], + ["sglang", "http://sglang-rig:30000/v1"], + ])( + "honors explicit cron run timeouts for self-hosted provider %s hostname %s", + (provider, baseUrl) => { + expect( + resolveLlmIdleTimeoutMs({ + trigger: "cron", + runTimeoutMs: 600_000, + model: { provider, baseUrl }, + }), + ).toBe(600_000); + }, + ); + + it("honors explicit cron run timeouts for explicit local host aliases", () => { + expect( + resolveLlmIdleTimeoutMs({ + trigger: "cron", + runTimeoutMs: 600_000, + model: { baseUrl: "http://host.docker.internal:11434" }, + }), + ).toBe(600_000); + }); + + it("honors explicit cron run timeouts for custom local provider markers on bare hostnames", () => { + const cfg = { + models: { + providers: { + gpu: { + baseUrl: "http://gpu-box:8000/v1", + api: "openai-completions", + apiKey: "custom-local", + models: [], + }, + "local-ollama": { + baseUrl: "http://ollama-box:11434", + api: "ollama", + apiKey: "ollama-local", + models: [], + }, + }, + }, + } as unknown as OpenClawConfig; + + expect( + resolveLlmIdleTimeoutMs({ + cfg, + trigger: "cron", + runTimeoutMs: 600_000, + model: { provider: "gpu", baseUrl: "http://gpu-box:8000/v1" }, + }), + ).toBe(600_000); + expect( + resolveLlmIdleTimeoutMs({ + cfg, + trigger: "cron", + runTimeoutMs: 600_000, + model: { provider: "local-ollama", baseUrl: "http://ollama-box:11434" }, + }), + ).toBe(600_000); + }); + + it("honors explicit cron run timeouts for provider-owned local services on bare hostnames", () => { + const cfg = { + models: { + providers: { + ds4: { + baseUrl: "http://ds4-box:8000/v1", + api: "openai-completions", + localService: { + command: "/opt/ds4/ds4-server", + healthUrl: "http://ds4-box:8000/v1/models", + }, + models: [], + }, + }, + }, + } as unknown as OpenClawConfig; + + expect( + resolveLlmIdleTimeoutMs({ + cfg, + trigger: "cron", + runTimeoutMs: 600_000, + model: { provider: "ds4", baseUrl: "http://ds4-box:8000/v1" }, + }), + ).toBe(600_000); + }); + + it.each([ + ["openai", "openai/gpt-5.5", "http://api:8080/v1"], + ["custom-proxy", "custom-proxy/gpt-5.5", "http://gateway:4000/v1"], + ["ollama-cloud", "ollama-cloud/kimi-k2.6", "http://ollama-host:11434"], + ])( + "keeps the cron stall cap for cloud provider %s routed through single-label host %s", + (provider, id, baseUrl) => { + expect( + resolveLlmIdleTimeoutMs({ + trigger: "cron", + runTimeoutMs: 600_000, + model: { provider, id, baseUrl }, + }), + ).toBe(CRON_LLM_IDLE_TIMEOUT_MS); + }, + ); + + it("keeps the cron stall cap for remote or cloud hostnames", () => { + expect( + resolveLlmIdleTimeoutMs({ + trigger: "cron", + runTimeoutMs: 600_000, + model: { provider: "openai", id: "openai/gpt-5.5", baseUrl: "https://api.openai.com/v1" }, + }), + ).toBe(CRON_LLM_IDLE_TIMEOUT_MS); + expect( + resolveLlmIdleTimeoutMs({ + trigger: "cron", + runTimeoutMs: 600_000, + model: { provider: "ollama", id: "ollama/gpt-oss:cloud", baseUrl: "http://ollama-host" }, + }), + ).toBe(CRON_LLM_IDLE_TIMEOUT_MS); }); it("disables the idle watchdog when an explicit run timeout disables timeouts", () => { diff --git a/src/agents/embedded-agent-runner/run/llm-idle-timeout.ts b/src/agents/embedded-agent-runner/run/llm-idle-timeout.ts index 212eae429d21..af5a50c1e923 100644 --- a/src/agents/embedded-agent-runner/run/llm-idle-timeout.ts +++ b/src/agents/embedded-agent-runner/run/llm-idle-timeout.ts @@ -18,6 +18,16 @@ import type { EmbeddedRunTrigger } from "./params.js"; * Default idle timeout for LLM streaming responses in milliseconds. */ const DEFAULT_LLM_IDLE_TIMEOUT_MS = 120_000; +// Cron has its own outer watchdog; stream stalls must fail early enough for +// the existing model fallback chain to try the next configured candidate. +const CRON_LLM_IDLE_TIMEOUT_MS = 60_000; +const LOCAL_PROVIDER_AUTH_MARKERS = new Set(["custom-local", "ollama-local"]); +const SELF_HOSTED_PROVIDER_ID_PREFIXES = ["ollama", "lmstudio", "vllm", "sglang", "llama-cpp"]; + +type IdleTimeoutProviderConfig = { + apiKey?: unknown; + localService?: unknown; +}; /** * Detects loopback / private-network / `.local` base URLs. Local providers @@ -37,11 +47,9 @@ const DEFAULT_LLM_IDLE_TIMEOUT_MS = 120_000; * matched, mirroring the SSRF-policy helper in * `src/cron/isolated-agent/model-preflight.runtime.ts`. * - DNS-resolved local aliases (e.g. an `/etc/hosts` entry mapping a custom - * hostname to a private IP) are not detected: classification keys on - * `URL.hostname` so resolution would have to happen here, and adding - * sync/async DNS to the watchdog hot path is disproportionate. Affected - * users can use the IP directly or set - * `models.providers..timeoutSeconds` explicitly. + * hostname to a private IP) are not detected for the implicit watchdog opt-out: + * classification keys on `URL.hostname` so resolution would have to happen + * here, and adding sync/async DNS to the watchdog hot path is disproportionate. */ function isLocalProviderBaseUrl(baseUrl: string): boolean { let host: string; @@ -95,6 +103,82 @@ function isLocalProviderBaseUrl(baseUrl: string): boolean { ); } +function isExplicitLocalHostnameBaseUrl(baseUrl: string): boolean { + let host: string; + try { + host = new URL(baseUrl).hostname.toLowerCase(); + } catch { + return false; + } + + if ( + host === "docker.orb.internal" || + host === "host.docker.internal" || + host === "host.orb.internal" + ) { + return true; + } + return false; +} + +function isBareProviderHostnameBaseUrl(baseUrl: string): boolean { + let host: string; + try { + host = new URL(baseUrl).hostname.toLowerCase(); + } catch { + return false; + } + + if (host.includes(".") || host.includes(":")) { + return false; + } + return /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(host); +} + +function isSelfHostedProviderId(provider: string | undefined): boolean { + const normalized = provider?.trim().toLowerCase(); + if (!normalized || normalized === "ollama-cloud") { + return false; + } + return SELF_HOSTED_PROVIDER_ID_PREFIXES.some( + (prefix) => normalized === prefix || normalized.startsWith(`${prefix}-`), + ); +} + +function findConfiguredProviderConfig( + cfg: OpenClawConfig | undefined, + provider: string | undefined, +): IdleTimeoutProviderConfig | undefined { + const normalizedProvider = provider?.trim().toLowerCase(); + if (!normalizedProvider) { + return undefined; + } + const providers = cfg?.models?.providers as + | Record + | undefined; + const exact = providers?.[normalizedProvider]; + if (exact) { + return exact; + } + return Object.entries(providers ?? {}).find( + ([key]) => key.trim().toLowerCase() === normalizedProvider, + )?.[1]; +} + +function hasLocalProviderAuthMarker(apiKey: unknown): boolean { + return typeof apiKey === "string" && LOCAL_PROVIDER_AUTH_MARKERS.has(apiKey.trim().toLowerCase()); +} + +function hasConfiguredLocalProviderSignal(params: { + cfg: OpenClawConfig | undefined; + provider: string | undefined; +}): boolean { + const providerConfig = findConfiguredProviderConfig(params.cfg, params.provider); + return Boolean( + providerConfig?.localService || hasLocalProviderAuthMarker(providerConfig?.apiKey), + ); +} + function isOllamaCloudModel(model: { id?: string; provider?: string } | undefined): boolean { const rawModelId = model?.id; if (typeof rawModelId !== "string") { @@ -134,6 +218,22 @@ export function resolveLlmIdleTimeoutMs(params?: { const hasExplicitRunTimeout = typeof runTimeoutMs === "number" && Number.isFinite(runTimeoutMs) && runTimeoutMs > 0; const runTimeoutIsNoTimeout = hasExplicitRunTimeout && runTimeoutMs >= MAX_TIMER_TIMEOUT_MS; + const baseUrl = params?.model?.baseUrl; + const isLocalProvider = + typeof baseUrl === "string" && baseUrl.length > 0 && isLocalProviderBaseUrl(baseUrl); + const isLocalRuntimeModel = isLocalProvider && !isOllamaCloudModel(params?.model); + const isExplicitLocalHostnameRuntimeModel = + typeof baseUrl === "string" && + baseUrl.length > 0 && + isExplicitLocalHostnameBaseUrl(baseUrl) && + !isOllamaCloudModel(params?.model); + const isSelfHostedHostnameRuntimeModel = + typeof baseUrl === "string" && + baseUrl.length > 0 && + isBareProviderHostnameBaseUrl(baseUrl) && + (isSelfHostedProviderId(params?.model?.provider) || + hasConfiguredLocalProviderSignal({ cfg: params?.cfg, provider: params?.model?.provider })) && + !isOllamaCloudModel(params?.model); const timeoutBounds = [ runTimeoutIsNoTimeout ? undefined : runTimeoutMs, hasExplicitRunTimeout ? undefined : agentTimeoutMs, @@ -174,7 +274,14 @@ export function resolveLlmIdleTimeoutMs(params?: { return 0; } if (params?.trigger === "cron") { - return clampTimeoutMs(runTimeoutMs); + if ( + isLocalRuntimeModel || + isExplicitLocalHostnameRuntimeModel || + isSelfHostedHostnameRuntimeModel + ) { + return clampTimeoutMs(runTimeoutMs); + } + return clampTimeoutMs(Math.min(runTimeoutMs, CRON_LLM_IDLE_TIMEOUT_MS)); } return clampImplicitTimeoutMs(runTimeoutMs); } @@ -190,10 +297,7 @@ export function resolveLlmIdleTimeoutMs(params?: { // baseUrl pointing at loopback / private-network / `.local`. Ollama cloud // models are still hosted remotely even when proxied through local Ollama, so // keep the cloud watchdog for `*:cloud` model ids. - const baseUrl = params?.model?.baseUrl; - const isLocalProvider = - typeof baseUrl === "string" && baseUrl.length > 0 && isLocalProviderBaseUrl(baseUrl); - if (isLocalProvider && !isOllamaCloudModel(params?.model)) { + if (isLocalRuntimeModel) { return 0; }