From 8190c326ce491324331c8b9793f3f4dddd5c7d91 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 11 Aug 2026 23:36:33 -0700 Subject: [PATCH] feat(gateway): add /startupz startup probe and fix deployment template admission (#122477) * feat(gateway): add /startupz startup probe with auth-gated version detail Startup/traffic-admission probe that excludes downstream channel health: 200 started once startup work completes and the gateway is not draining, 503 starting/draining otherwise. Version and uptime are only included for local-direct or authenticated callers, reusing the /readyz detail gate. * fix(deploy): use /startupz for traffic admission in bundled templates fly.toml gains its missing HTTP check; render.yaml stops using pure liveness as admission; k8s pins an immutable image tag, seeds config only when missing, and adds a startupProbe; stale Fly healthcheck-port doc corrected (healthcheck follows the active gateway lock port since bc4221a07e9). * docs(k8s): make persistent-file config ownership explicit with reseed path --- Dockerfile | 5 +- docs/gateway/health.md | 14 ++ docs/install/docker.md | 4 +- docs/install/fly.md | 14 +- docs/install/kubernetes.md | 14 +- docs/install/render.mdx | 16 +- fly.toml | 7 + render.yaml | 2 +- scripts/k8s/manifests/deployment.yaml | 21 ++- src/gateway/gateway-http-route-contracts.ts | 6 +- src/gateway/server-http.probe.test.ts | 175 +++++++++++++++++++- src/gateway/server-http.ts | 91 ++++++++-- src/gateway/server-runtime-state-prepare.ts | 11 +- src/gateway/server-runtime-state.ts | 4 +- src/gateway/server.plugin-http-auth.test.ts | 16 +- src/gateway/server/readiness.ts | 65 ++++++-- 16 files changed, 401 insertions(+), 64 deletions(-) diff --git a/Dockerfile b/Dockerfile index f1f5a18f861f..1963be82cab9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -377,8 +377,9 @@ USER node # - Override --bind to "lan" (0.0.0.0) and set auth credentials # # Built-in probe endpoints for container health checks: -# - GET /healthz (liveness) and GET /readyz (readiness) -# - aliases: /health and /ready +# - GET /healthz (liveness), GET /startupz (startup/traffic admission), +# and GET /readyz (channel-aware readiness) +# - aliases: /health, /startup, and /ready # For external access from host/ingress, override bind to "lan" and set auth. HEALTHCHECK --interval=3m --timeout=10s --start-period=15s --retries=3 \ CMD ["node", "dist/docker-healthcheck.js"] diff --git a/docs/gateway/health.md b/docs/gateway/health.md index 74be0fd2a188..72549401362d 100644 --- a/docs/gateway/health.md +++ b/docs/gateway/health.md @@ -52,6 +52,20 @@ Channel connectivity and inbound admission are separate failure domains. A chann - If the restarts keep repeating, the cause is not transient. Check the logged ingress failure: a plugin denied the `openChannelIngressQueue` capability, for example, needs operator action rather than another restart. - Channels that never report ingress state are unaffected: absence means "no signal", never "broken". There is no traffic-staleness heuristic, so a genuinely quiet channel is never marked unhealthy for having received nothing. +## HTTP probes + +The Gateway exposes three unauthenticated `GET`/`HEAD` probe pairs: + +| Endpoints | Meaning | Use | +| ----------------------- | ------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | +| `/health`, `/healthz` | The HTTP server is live. | Process liveness and restart decisions. | +| `/startup`, `/startupz` | Startup work is complete and the Gateway is not draining. Channel health is not consulted. | Orchestrator startup and traffic admission. | +| `/ready`, `/readyz` | Startup is complete, the Gateway is not draining, and configured channel accounts pass deep readiness checks. | Operator monitoring that should surface hard channel failures. | + +`/startupz` returns `503` with `status: "starting"` while startup sidecars are pending, `503` with `status: "draining"` during drain, and `200` with `status: "started"` otherwise. Use it for Kubernetes, Fly, Render, and similar traffic admission. A broken Telegram or other channel account can make `/readyz` return `503` without taking a healthy Control UI out of service through `/startupz`. + +Remote unauthenticated startup responses contain only `ok` and `status`. Local-direct and authenticated callers also receive `version`, `uptimeMs`, and `pendingReason` while startup is pending. Readiness details follow the same local-or-authenticated gate because they can name failing subsystems. + ## Uptime monitoring External uptime monitoring services should use the dedicated `/health` endpoint, not `/v1/chat/completions`. diff --git a/docs/install/docker.md b/docs/install/docker.md index b37a1e72d731..1331411c30b8 100644 --- a/docs/install/docker.md +++ b/docs/install/docker.md @@ -280,10 +280,12 @@ Container probe endpoints (no auth required): ```bash curl -fsS http://127.0.0.1:18789/healthz # liveness -curl -fsS http://127.0.0.1:18789/readyz # readiness +curl -fsS http://127.0.0.1:18789/startupz # startup and traffic admission +curl -fsS http://127.0.0.1:18789/readyz # deep, channel-aware readiness ``` The image's built-in `HEALTHCHECK` pings `/healthz`; repeated failures mark the container `unhealthy` so orchestrators can restart or replace it. +Use `/startupz` for an orchestrator startup or readiness probe so a failed channel account does not remove the otherwise healthy Gateway and Control UI from service. Use `/readyz` for monitoring that intentionally treats hard channel failures as not ready. See [Health checks](/gateway/health#http-probes) for response details. Authenticated deep health snapshot: diff --git a/docs/install/fly.md b/docs/install/fly.md index e729b58229e9..3bd205a34b84 100644 --- a/docs/install/fly.md +++ b/docs/install/fly.md @@ -66,6 +66,13 @@ read_when: min_machines_running = 1 processes = ["app"] + [[http_service.checks]] + grace_period = "2m" + interval = "15s" + method = "GET" + timeout = "5s" + path = "/startupz" + [[vm]] size = "shared-cpu-2x" memory = "2048mb" @@ -84,6 +91,7 @@ read_when: | `--bind lan` | Binds to `0.0.0.0` so Fly's proxy can reach the gateway | | `--allow-unconfigured` | Starts without a config file (you create one after) | | `internal_port = 3000` | Must match `--port 3000` (or `OPENCLAW_GATEWAY_PORT`) for Fly health checks | + | `path = "/startupz"` | Admits traffic after Gateway startup finishes, independent of channel health | | `memory = "2048mb"` | 512MB is too small; 2GB recommended | | `OPENCLAW_STATE_DIR = "/data"` | Persists state on the volume | @@ -123,7 +131,7 @@ read_when: fly logs ``` - Gateway startup logs `gateway ready` once the HTTP/WebSocket listener is up. Fly's own health check watches `internal_port = 3000` per `fly.toml`; the image's Docker `HEALTHCHECK` directive additionally polls `/healthz` on its default port 18789, which is unused here since this deployment overrides the gateway to `--port 3000`. + Gateway startup logs `gateway ready` once the HTTP/WebSocket listener is up. Fly checks `/startupz` on `internal_port = 3000` and admits traffic after startup work finishes. The image's Docker `HEALTHCHECK` resolves the active Gateway lock port, so its `/healthz` liveness check also follows this deployment's `--port 3000` override. @@ -247,9 +255,9 @@ The gateway is binding to `127.0.0.1` instead of `0.0.0.0`. ### Health checks failing / connection refused -Fly cannot reach the gateway on the configured port. +Fly cannot reach the gateway on the configured port, or `/startupz` is still reporting startup work. -**Fix:** ensure `internal_port` matches the gateway port (`--port 3000` or `OPENCLAW_GATEWAY_PORT=3000`). +**Fix:** ensure `internal_port` matches the gateway port (`--port 3000` or `OPENCLAW_GATEWAY_PORT=3000`), then inspect `fly logs` for the pending startup step. ### OOM / memory issues diff --git a/docs/install/kubernetes.md b/docs/install/kubernetes.md index 26b62c670e68..ab9023e33b6e 100644 --- a/docs/install/kubernetes.md +++ b/docs/install/kubernetes.md @@ -90,6 +90,8 @@ Namespace: openclaw (configurable via OPENCLAW_NAMESPACE) └── Secret/openclaw-secrets # Gateway token + API keys ``` +The Deployment uses `/startupz` for both startup and traffic-readiness probes, with a five-minute startup budget. Channel failures do not evict a healthy Gateway or Control UI from Service endpoints. `/healthz` remains the liveness probe; use `/readyz` separately when monitoring should include channel-account health. + ## Customization ### Agent instructions @@ -104,6 +106,15 @@ Edit the `AGENTS.md` in `scripts/k8s/manifests/configmap.yaml` and redeploy: Edit `openclaw.json` in `scripts/k8s/manifests/configmap.yaml`. See [Gateway configuration](/gateway/configuration) for the full reference. +The init container seeds `openclaw.json` and workspace `AGENTS.md` only when each file is missing from the PVC. The persisted copy is the source of truth after first boot: changes made through OpenClaw (`onboard`, `channels add`, `doctor --fix`, Control UI) survive pod restarts, and updating the ConfigMap does not overwrite an existing PVC copy. To intentionally reseed a file from an updated ConfigMap, delete the persisted copy and restart: + +```bash +kubectl exec -n openclaw deploy/openclaw -- rm /home/node/.openclaw/openclaw.json +kubectl rollout restart -n openclaw deploy/openclaw +``` + +Deployments created from the previous template applied ConfigMap edits on every pod start (and discarded any config changes made through OpenClaw). If you relied on that flow, use the reseed commands above after ConfigMap edits. + ### Add providers Re-run with additional keys exported: @@ -136,7 +147,8 @@ OPENCLAW_NAMESPACE=my-namespace ./scripts/k8s/deploy.sh Edit the `image` field in `scripts/k8s/manifests/deployment.yaml`: ```yaml -image: ghcr.io/openclaw/openclaw:slim # primary; official Docker Hub mirror: openclaw/openclaw +# Bump this immutable versioned tag when upgrading OpenClaw. +image: ghcr.io/openclaw/openclaw:2026.7.1-2-slim ``` ### Expose beyond port-forward diff --git a/docs/install/render.mdx b/docs/install/render.mdx index c2f03585fb88..be2d8be21695 100644 --- a/docs/install/render.mdx +++ b/docs/install/render.mdx @@ -27,7 +27,7 @@ services: name: openclaw runtime: docker plan: starter - healthCheckPath: /health + healthCheckPath: /startupz envVars: - key: OPENCLAW_GATEWAY_PORT value: "8080" @@ -43,12 +43,12 @@ services: sizeGB: 1 ``` -| Feature | Purpose | -| --------------------- | ---------------------------------------------------------- | -| `runtime: docker` | Builds from the repo's Dockerfile | -| `healthCheckPath` | Render monitors `/health` and restarts unhealthy instances | -| `generateValue: true` | Auto-generates a cryptographically secure value | -| `disk` | Persistent storage that survives redeploys | +| Feature | Purpose | +| --------------------- | ---------------------------------------------------------------- | +| `runtime: docker` | Builds from the repo's Dockerfile | +| `healthCheckPath` | Render admits traffic after `/startupz` reports startup complete | +| `generateValue: true` | Auto-generates a cryptographically secure value | +| `disk` | Persistent storage that survives redeploys | ## Choosing a plan @@ -123,7 +123,7 @@ Happens on the free tier (no persistent disk). Upgrade to a paid plan, or regula ### Health check failures -If builds succeed but deploys fail, the service may be taking too long to start or `/health` may not be reachable. Check: +If builds succeed but deploys fail, the service may be taking too long to start or `/startupz` may not be reachable. Check: - Build logs for errors - Whether the container runs locally with `docker build && docker run` diff --git a/fly.toml b/fly.toml index 9aca608c5c7b..86af98ad8e26 100644 --- a/fly.toml +++ b/fly.toml @@ -25,6 +25,13 @@ auto_start_machines = true min_machines_running = 1 processes = ["app"] +[[http_service.checks]] +grace_period = "2m" +interval = "15s" +method = "GET" +timeout = "5s" +path = "/startupz" + [[vm]] size = "shared-cpu-2x" memory = "2048mb" diff --git a/render.yaml b/render.yaml index c99d5131f2e3..8b364059c60f 100644 --- a/render.yaml +++ b/render.yaml @@ -3,7 +3,7 @@ services: name: openclaw runtime: docker plan: starter - healthCheckPath: /health + healthCheckPath: /startupz envVars: - key: OPENCLAW_GATEWAY_PORT value: "8080" diff --git a/scripts/k8s/manifests/deployment.yaml b/scripts/k8s/manifests/deployment.yaml index f87c266930b8..95376373e41a 100644 --- a/scripts/k8s/manifests/deployment.yaml +++ b/scripts/k8s/manifests/deployment.yaml @@ -29,9 +29,12 @@ spec: - sh - -c - | - cp /config/openclaw.json /home/node/.openclaw/openclaw.json + # Seed-if-missing: the PVC copy owns config after first boot so edits made + # through OpenClaw (onboard, channels add, doctor --fix, Control UI) survive + # restarts. ConfigMap edits need an explicit reseed (see docs/install/kubernetes.md). + [ -f /home/node/.openclaw/openclaw.json ] || cp /config/openclaw.json /home/node/.openclaw/openclaw.json mkdir -p /home/node/.openclaw/workspace - cp /config/AGENTS.md /home/node/.openclaw/workspace/AGENTS.md + [ -f /home/node/.openclaw/workspace/AGENTS.md ] || cp /config/AGENTS.md /home/node/.openclaw/workspace/AGENTS.md securityContext: runAsUser: 1000 runAsGroup: 1000 @@ -49,7 +52,8 @@ spec: mountPath: /config containers: - name: gateway - image: ghcr.io/openclaw/openclaw:slim + # Bump this immutable versioned tag when upgrading OpenClaw. + image: ghcr.io/openclaw/openclaw:2026.7.1-2-slim imagePullPolicy: IfNotPresent command: - node @@ -103,6 +107,15 @@ spec: limits: memory: 2Gi cpu: "1" + startupProbe: + exec: + command: + - node + - -e + - "require('http').get('http://127.0.0.1:18789/startupz', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))" + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 30 livenessProbe: exec: command: @@ -117,7 +130,7 @@ spec: command: - node - -e - - "require('http').get('http://127.0.0.1:18789/readyz', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))" + - "require('http').get('http://127.0.0.1:18789/startupz', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))" initialDelaySeconds: 15 periodSeconds: 10 timeoutSeconds: 5 diff --git a/src/gateway/gateway-http-route-contracts.ts b/src/gateway/gateway-http-route-contracts.ts index 1e7c18979d62..256b2602e1ae 100644 --- a/src/gateway/gateway-http-route-contracts.ts +++ b/src/gateway/gateway-http-route-contracts.ts @@ -1,8 +1,10 @@ -const GATEWAY_PROBE_ROUTES = new Map([ +const GATEWAY_PROBE_ROUTES = new Map([ ["/health", "live"], ["/healthz", "live"], ["/ready", "ready"], ["/readyz", "ready"], + ["/startup", "startup"], + ["/startupz", "startup"], ]); export const MCP_APP_STANDALONE_PATH = "/__openclaw__/mcp-app"; @@ -10,7 +12,7 @@ export const MCP_APP_STANDALONE_VIEW_PATH = `${MCP_APP_STANDALONE_PATH}/view`; export function classifyGatewayProbePath( pathname: string, -): "live" | "ready" | "namespace" | "outside" { +): "live" | "ready" | "startup" | "namespace" | "outside" { for (const [root, status] of GATEWAY_PROBE_ROUTES) { if (pathname === root) { return status; diff --git a/src/gateway/server-http.probe.test.ts b/src/gateway/server-http.probe.test.ts index fa561ebc9795..775553f38826 100644 --- a/src/gateway/server-http.probe.test.ts +++ b/src/gateway/server-http.probe.test.ts @@ -14,6 +14,7 @@ import { getActiveGatewayRootWorkCount, resetGatewayWorkAdmission, } from "../process/gateway-work-admission.js"; +import { resolveRuntimeServiceVersion } from "../version.js"; import type { ChannelManager } from "./server-channels.js"; import { AUTH_TOKEN, @@ -23,7 +24,12 @@ import { dispatchRequest, withGatewayServer, } from "./server-http.test-harness.js"; -import { createReadinessChecker, type ReadinessChecker } from "./server/readiness.js"; +import { + createReadinessChecker, + createStartupChecker, + type ReadinessChecker, + type StartupChecker, +} from "./server/readiness.js"; import { withTempConfig } from "./test-temp-config.js"; type GatewayServerHarness = Parameters[0]; @@ -352,7 +358,14 @@ describe("gateway probe endpoints", () => { expect(exact.res.statusCode).toBe(503); expect(JSON.parse(exact.getBody())).toMatchObject({ ready: false }); - for (const routePath of ["/health/", "/healthz/details", "/ready/", "/readyz/details"]) { + for (const routePath of [ + "/health/", + "/healthz/details", + "/ready/", + "/readyz/details", + "/startup/", + "/startupz/details", + ]) { const { res, getBody } = await sendGatewayRequest(server, { path: routePath }); expect(res.statusCode, routePath).toBe(404); expect(getBody(), routePath).toBe("Not Found"); @@ -760,6 +773,133 @@ describe("gateway probe endpoints", () => { }); }); + it("reports startup lifecycle independently of hard channel failures", async () => { + let startupPending = true; + let gatewayDraining = false; + const startedAt = Date.now() - 5_000; + const account = { + accountId: "default", + running: true, + connected: true, + enabled: true, + configured: true, + lifecycle: "blocked" as const, + lastStartAt: startedAt, + }; + const channelManager = { + getRuntimeSnapshot: () => ({ + channels: { telegram: account }, + channelAccounts: { telegram: { default: account } }, + }), + getAutostartSuppression: () => null, + isAmbientAutostartSuppressed: () => false, + } as unknown as ChannelManager; + const startupDeps = { + startedAt, + getStartupPending: () => startupPending, + getStartupPendingReason: () => "plugin-convergence", + getGatewayDraining: () => gatewayDraining, + }; + const getStartup = createStartupChecker(startupDeps); + const getReadiness = createReadinessChecker({ + channelManager, + ...startupDeps, + cacheTtlMs: 0, + }); + + await withGatewayServer({ + prefix: "probe-startup-lifecycle", + resolvedAuth: AUTH_NONE, + overrides: { getReadiness, getStartup }, + run: async (server) => { + const starting = await sendGatewayRequest(server, { path: "/startupz" }); + expect(starting.res.statusCode).toBe(503); + expect(JSON.parse(starting.getBody())).toMatchObject({ + ok: false, + status: "starting", + version: resolveRuntimeServiceVersion(process.env), + uptimeMs: expect.any(Number), + pendingReason: "plugin-convergence", + }); + + startupPending = false; + const started = await sendGatewayRequest(server, { path: "/startupz" }); + expect(started.res.statusCode).toBe(200); + expect(JSON.parse(started.getBody())).toMatchObject({ + ok: true, + status: "started", + version: resolveRuntimeServiceVersion(process.env), + uptimeMs: expect.any(Number), + }); + + const readiness = await sendGatewayRequest(server, { path: "/readyz" }); + expect(readiness.res.statusCode).toBe(503); + expect(JSON.parse(readiness.getBody())).toMatchObject({ + ready: false, + failing: ["telegram"], + }); + + const channelIndependentStartup = await sendGatewayRequest(server, { + path: "/startupz", + }); + expect(channelIndependentStartup.res.statusCode).toBe(200); + expect(JSON.parse(channelIndependentStartup.getBody())).toMatchObject({ + ok: true, + status: "started", + }); + + gatewayDraining = true; + const draining = await sendGatewayRequest(server, { path: "/startupz" }); + expect(draining.res.statusCode).toBe(503); + expect(JSON.parse(draining.getBody())).toMatchObject({ + ok: false, + status: "draining", + version: resolveRuntimeServiceVersion(process.env), + uptimeMs: expect.any(Number), + }); + }, + }); + }); + + it("gates startup details to local or authenticated callers", async () => { + const getStartup = createStartupChecker({ + startedAt: Date.now() - 8_000, + getStartupPending: () => true, + getStartupPendingReason: () => "startup-sidecars", + getGatewayDraining: () => false, + }); + + await withGatewayServer({ + prefix: "probe-startup-details", + resolvedAuth: AUTH_TOKEN, + overrides: { getStartup }, + run: async (server) => { + const remote = await sendGatewayRequest(server, { + path: "/startupz", + remoteAddress: "10.0.0.8", + host: "gateway.test", + }); + expect(remote.res.statusCode).toBe(503); + expect(JSON.parse(remote.getBody())).toEqual({ ok: false, status: "starting" }); + + const authenticated = await sendGatewayRequest(server, { + path: "/startupz", + remoteAddress: "10.0.0.8", + host: "gateway.test", + authorization: "Bearer test-token", + }); + expect(authenticated.res.statusCode).toBe(503); + expect(JSON.parse(authenticated.getBody())).toMatchObject({ + ok: false, + status: "starting", + version: resolveRuntimeServiceVersion(process.env), + uptimeMs: expect.any(Number), + pendingReason: "startup-sidecars", + }); + }, + }); + }); + it("serves /healthz before loading gateway config", async () => { const getRuntimeConfig = vi.fn(() => { throw new Error("config load blocked"); @@ -837,6 +977,37 @@ describe("gateway probe endpoints", () => { }); }); + it("keeps GET and HEAD /startupz status and Content-Length in parity", async () => { + const getStartup: StartupChecker = () => ({ + ok: false, + status: "draining", + uptimeMs: 5_000, + }); + + await withGatewayServer({ + prefix: "probe-startupz-head", + resolvedAuth: AUTH_NONE, + overrides: { getStartup }, + run: async (server) => { + const get = await sendGatewayRequest(server, { path: "/startupz" }); + const head = createResponse(); + await dispatchRequest( + server, + createRequest({ path: "/startupz", method: "HEAD" }), + head.res, + ); + + expect(get.res.statusCode).toBe(503); + expect(head.res.statusCode).toBe(503); + expect(head.getBody()).toBe(""); + expect(head.setHeader).toHaveBeenCalledWith( + "Content-Length", + String(Buffer.byteLength(get.getBody())), + ); + }, + }); + }); + it("sends Content-Length on HEAD probe responses matching the GET body", async () => { await withGatewayServer({ prefix: "probe-head-content-length", diff --git a/src/gateway/server-http.ts b/src/gateway/server-http.ts index 8d3307ecb8a2..4a40852e4ef0 100644 --- a/src/gateway/server-http.ts +++ b/src/gateway/server-http.ts @@ -24,6 +24,7 @@ import { isGatewayWorkAdmissionClosed, } from "../process/gateway-work-admission.js"; import { createLazyRuntimeModule } from "../shared/lazy-runtime.js"; +import { resolveRuntimeServiceVersion } from "../version.js"; import { resolveAssistantIdentity } from "./assistant-identity.js"; import type { AuthRateLimiter } from "./auth-rate-limit.js"; import { @@ -69,7 +70,7 @@ import { type PluginRoutePathContext, } from "./server/plugins-http/path-context.js"; import type { PreauthConnectionBudget } from "./server/preauth-connection-budget.js"; -import type { ReadinessChecker } from "./server/readiness.js"; +import type { ReadinessChecker, StartupChecker, StartupResult } from "./server/readiness.js"; import { GATEWAY_WS_CONNECTION_KIND_PROPERTY, GATEWAY_WS_PREAUTH_BUDGET_PROPERTY, @@ -185,7 +186,46 @@ function shouldEnforceDefaultPluginGatewayAuth(pathContext: PluginRoutePathConte ); } -/** Handles live/ready probe endpoints before normal gateway routing. */ +async function shouldIncludeGatewayProbeDetails(params: { + req: IncomingMessage; + resolvedAuth: ResolvedGatewayAuth; + trustedProxies: string[]; + allowRealIpFallback: boolean; +}): Promise { + if (isLocalDirectRequest(params.req, params.trustedProxies, params.allowRealIpFallback)) { + return true; + } + if (params.resolvedAuth.mode === "none") { + return false; + } + const { getBearerToken, resolveHttpBrowserOriginPolicy } = await getHttpAuthUtilsModule(); + const bearerToken = getBearerToken(params.req); + return ( + await authorizeHttpGatewayConnect({ + auth: params.resolvedAuth, + connectAuth: bearerToken ? { token: bearerToken, password: bearerToken } : null, + req: params.req, + trustedProxies: params.trustedProxies, + allowRealIpFallback: params.allowRealIpFallback, + browserOriginPolicy: resolveHttpBrowserOriginPolicy(params.req), + }) + ).ok; +} + +function startupProbeBody(result: StartupResult, includeDetails: boolean): string { + if (!includeDetails) { + return JSON.stringify({ ok: result.ok, status: result.status }); + } + return JSON.stringify({ + ok: result.ok, + status: result.status, + version: resolveRuntimeServiceVersion(process.env), + uptimeMs: result.uptimeMs, + ...(result.status === "starting" ? { pendingReason: result.pendingReason } : {}), + }); +} + +/** Handles live/ready/startup probe endpoints before normal gateway routing. */ async function handleGatewayProbeRequest( req: IncomingMessage, res: ServerResponse, @@ -194,6 +234,7 @@ async function handleGatewayProbeRequest( trustedProxies: string[], allowRealIpFallback: boolean, getReadiness?: ReadinessChecker, + getStartup?: StartupChecker, ): Promise { const status = classifyGatewayProbePath(requestPath); if (status === "namespace" || status === "outside") { @@ -217,21 +258,12 @@ async function handleGatewayProbeRequest( if (status === "ready" && getReadiness) { // Readiness details expose subsystem names, so only local direct or authenticated // callers receive them; unauthenticated remote probes get the aggregate boolean. - let includeDetails = isLocalDirectRequest(req, trustedProxies, allowRealIpFallback); - if (!includeDetails && resolvedAuth.mode !== "none") { - const { getBearerToken, resolveHttpBrowserOriginPolicy } = await getHttpAuthUtilsModule(); - const bearerToken = getBearerToken(req); - includeDetails = ( - await authorizeHttpGatewayConnect({ - auth: resolvedAuth, - connectAuth: bearerToken ? { token: bearerToken, password: bearerToken } : null, - req, - trustedProxies, - allowRealIpFallback, - browserOriginPolicy: resolveHttpBrowserOriginPolicy(req), - }) - ).ok; - } + const includeDetails = await shouldIncludeGatewayProbeDetails({ + req, + resolvedAuth, + trustedProxies, + allowRealIpFallback, + }); try { const result = getReadiness(); statusCode = result.ready ? 200 : 503; @@ -242,6 +274,27 @@ async function handleGatewayProbeRequest( includeDetails ? { ready: false, failing: ["internal"], uptimeMs: 0 } : { ready: false }, ); } + } else if (status === "startup") { + const includeDetails = await shouldIncludeGatewayProbeDetails({ + req, + resolvedAuth, + trustedProxies, + allowRealIpFallback, + }); + try { + const result = getStartup?.() ?? { ok: true, status: "started", uptimeMs: 0 }; + statusCode = result.ok ? 200 : 503; + body = startupProbeBody(result, includeDetails); + } catch { + const result: StartupResult = { + ok: false, + status: "starting", + uptimeMs: 0, + pendingReason: "internal", + }; + statusCode = 503; + body = startupProbeBody(result, includeDetails); + } } else { statusCode = 200; body = JSON.stringify({ ok: true, status }); @@ -346,6 +399,7 @@ export function createGatewayHttpServer(opts: { /** Optional rate limiter for auth brute-force protection. */ rateLimiter?: AuthRateLimiter; getReadiness?: ReadinessChecker; + getStartup?: StartupChecker; getRuntimeConfig?: () => OpenClawConfig; isStartupPluginRuntimeReady?: () => boolean; isTerminalEnabled?: () => boolean; @@ -368,6 +422,7 @@ export function createGatewayHttpServer(opts: { resolvedAuth, rateLimiter, getReadiness, + getStartup, } = opts; const getResolvedAuth = opts.getResolvedAuth ?? (() => resolvedAuth); const loadGatewayConfig = opts.getRuntimeConfig ?? getRuntimeConfig; @@ -420,6 +475,7 @@ export function createGatewayHttpServer(opts: { [], false, getReadiness, + getStartup, ); return; } @@ -471,6 +527,7 @@ export function createGatewayHttpServer(opts: { trustedProxies, allowRealIpFallback, getReadiness, + getStartup, ), }, ]; diff --git a/src/gateway/server-runtime-state-prepare.ts b/src/gateway/server-runtime-state-prepare.ts index 263c39458120..097e35ea5d2d 100644 --- a/src/gateway/server-runtime-state-prepare.ts +++ b/src/gateway/server-runtime-state-prepare.ts @@ -29,7 +29,7 @@ import { createGatewayTransportBridge } from "./server-transport-bridge.js"; import { createWizardSessionTracker } from "./server-wizard-sessions.js"; import { createGatewayEventLoopHealthMonitor } from "./server/event-loop-health.js"; import { resolveHookClientIpConfig } from "./server/hook-client-ip-config.js"; -import { createReadinessChecker } from "./server/readiness.js"; +import { createReadinessChecker, createStartupChecker } from "./server/readiness.js"; import { resolveSharedGatewaySessionGeneration } from "./server/ws-shared-generation.js"; type GatewayBootstrap = Awaited>; @@ -351,12 +351,16 @@ export async function prepareGatewayKernelState(params: { channelManager.setAutostartSuppression(opts.channelAutostartSuppression ?? null); const sidecarStartup = opts.sidecarStartup ?? "start"; const isGatewayStartupPending = () => !startupState.sidecarsReady && sidecarStartup === "start"; - const getReadiness = createReadinessChecker({ - channelManager, + const startupCheckerDeps = { startedAt: serverStartedAt, getStartupPending: isGatewayStartupPending, getStartupPendingReason: () => startupState.pendingReason, getGatewayDraining: isGatewayDraining, + }; + const getStartup = createStartupChecker(startupCheckerDeps); + const getReadiness = createReadinessChecker({ + channelManager, + ...startupCheckerDeps, getEventLoopHealth: readinessEventLoopHealth.snapshot, shouldSkipChannelReadiness: () => isTruthyEnvValue(process.env.OPENCLAW_SKIP_CHANNELS) || @@ -407,6 +411,7 @@ export async function prepareGatewayKernelState(params: { logHooks, logPlugins, getReadiness, + getStartup, handleWatchNodeRequest: async (req: IncomingMessage, res: ServerResponse) => (await watchNodeRequestHandler.current?.(req, res)) ?? false, workerIngressEnabled: Boolean(workerEnvironmentService), diff --git a/src/gateway/server-runtime-state.ts b/src/gateway/server-runtime-state.ts index d8f847a4a0a6..03f95d017929 100644 --- a/src/gateway/server-runtime-state.ts +++ b/src/gateway/server-runtime-state.ts @@ -41,7 +41,7 @@ import { createPreauthConnectionBudget, type PreauthConnectionBudget, } from "./server/preauth-connection-budget.js"; -import type { ReadinessChecker } from "./server/readiness.js"; +import type { ReadinessChecker, StartupChecker } from "./server/readiness.js"; import type { GatewayWsClient } from "./server/ws-types.js"; import type { WorkerDesktopTunnels } from "./worker-environments/desktop-tunnel.js"; @@ -114,6 +114,7 @@ export async function createGatewayHttpTransport(params: { logHooks: ReturnType; logPlugins: ReturnType; getReadiness?: ReadinessChecker; + getStartup?: StartupChecker; isTerminalEnabled: () => boolean; handleWatchNodeRequest?: (req: IncomingMessage, res: ServerResponse) => Promise; workerIngressEnabled?: boolean; @@ -282,6 +283,7 @@ export async function createGatewayHttpTransport(params: { getResolvedAuth: params.getResolvedAuth, rateLimiter: params.rateLimiter, getReadiness: params.getReadiness, + getStartup: params.getStartup, getRuntimeConfig: loadRuntimeConfig, isStartupPluginRuntimeReady: params.isStartupPluginRuntimeReady, isTerminalEnabled: params.isTerminalEnabled, diff --git a/src/gateway/server.plugin-http-auth.test.ts b/src/gateway/server.plugin-http-auth.test.ts index d184cf388e11..4758b820cdfe 100644 --- a/src/gateway/server.plugin-http-auth.test.ts +++ b/src/gateway/server.plugin-http-auth.test.ts @@ -97,15 +97,25 @@ const PROBE_CASES = [ { path: "/healthz", status: "live" }, { path: "/ready", status: "ready" }, { path: "/readyz", status: "ready" }, + { path: "/startup", status: "started" }, + { path: "/startupz", status: "started" }, ] as const; async function expectProbeRoutesHealthy(server: Parameters[0]) { for (const probeCase of PROBE_CASES) { const response = await sendRequest(server, { path: probeCase.path }); expect(response.res.statusCode, probeCase.path).toBe(200); - expect(response.getBody(), probeCase.path).toBe( - JSON.stringify({ ok: true, status: probeCase.status }), - ); + const body = JSON.parse(response.getBody()); + if (probeCase.status === "started") { + expect(body, probeCase.path).toMatchObject({ + ok: true, + status: "started", + version: expect.any(String), + uptimeMs: expect.any(Number), + }); + } else { + expect(body, probeCase.path).toEqual({ ok: true, status: probeCase.status }); + } } } diff --git a/src/gateway/server/readiness.ts b/src/gateway/server/readiness.ts index 774a7de08c9e..d0f4cc6e2f8a 100644 --- a/src/gateway/server/readiness.ts +++ b/src/gateway/server/readiness.ts @@ -22,8 +22,42 @@ type ReadinessResult = { /** Function form used by HTTP readiness endpoints and tests. */ export type ReadinessChecker = () => ReadinessResult; +export type StartupResult = + | { ok: true; status: "started"; uptimeMs: number } + | { ok: false; status: "starting"; uptimeMs: number; pendingReason: string } + | { ok: false; status: "draining"; uptimeMs: number }; + +/** Function form used by HTTP startup endpoints and tests. */ +export type StartupChecker = () => StartupResult; + +type GatewayStartupStateDeps = { + startedAt: number; + getStartupPending?: () => boolean; + getStartupPendingReason?: () => string | undefined; + getGatewayDraining?: () => boolean; +}; + const DEFAULT_READINESS_CACHE_TTL_MS = 1_000; +/** Create a startup checker that excludes downstream channel health. */ +export function createStartupChecker(deps: GatewayStartupStateDeps): StartupChecker { + return (): StartupResult => { + const uptimeMs = Date.now() - deps.startedAt; + if (deps.getStartupPending?.()) { + return { + ok: false, + status: "starting", + uptimeMs, + pendingReason: deps.getStartupPendingReason?.() ?? "startup-sidecars", + }; + } + if (deps.getGatewayDraining?.()) { + return { ok: false, status: "draining", uptimeMs }; + } + return { ok: true, status: "started", uptimeMs }; + }; +} + function shouldIgnoreReadinessFailure( accountSnapshot: ChannelAccountSnapshot, health: ChannelHealthEvaluation, @@ -49,32 +83,31 @@ function shouldIgnoreReadinessFailure( } /** Create a cached readiness checker over channel runtime health. */ -export function createReadinessChecker(deps: { - channelManager: ChannelManager; - startedAt: number; - getStartupPending?: () => boolean; - getStartupPendingReason?: () => string | undefined; - getGatewayDraining?: () => boolean; - getEventLoopHealth?: () => GatewayEventLoopHealth | undefined; - shouldSkipChannelReadiness?: () => boolean; - cacheTtlMs?: number; -}): ReadinessChecker { +export function createReadinessChecker( + deps: GatewayStartupStateDeps & { + channelManager: ChannelManager; + getEventLoopHealth?: () => GatewayEventLoopHealth | undefined; + shouldSkipChannelReadiness?: () => boolean; + cacheTtlMs?: number; + }, +): ReadinessChecker { const { channelManager, startedAt } = deps; + const getStartup = createStartupChecker(deps); const cacheTtlMs = Math.max(0, deps.cacheTtlMs ?? DEFAULT_READINESS_CACHE_TTL_MS); let cachedAt = 0; let cachedState: Omit | null = null; return (): ReadinessResult => { - const now = Date.now(); - const uptimeMs = now - startedAt; - if (deps.getStartupPending?.()) { - const reason = deps.getStartupPendingReason?.() ?? "startup-sidecars"; + const startup = getStartup(); + const uptimeMs = startup.uptimeMs; + const now = startedAt + uptimeMs; + if (startup.status === "starting") { return withEventLoopHealth( - { ready: false, failing: [reason], uptimeMs }, + { ready: false, failing: [startup.pendingReason], uptimeMs }, deps.getEventLoopHealth, ); } - if (deps.getGatewayDraining?.()) { + if (startup.status === "draining") { return withEventLoopHealth( { ready: false, failing: ["gateway-draining"], uptimeMs }, deps.getEventLoopHealth,