feat(gateway): add /startupz startup probe and fix deployment template admission (#122477)

* feat(gateway): add /startupz startup probe with auth-gated version detail Startup/traffic-admission probe that excludes downstream channel health: 200 started once startup work completes and the gateway is not draining, 503 starting/draining otherwise. Version and uptime are only included for local-direct or authenticated callers, reusing the /readyz detail gate. * fix(deploy): use /startupz for traffic admission in bundled templates fly.toml gains its missing HTTP check; render.yaml stops using pure liveness as admission; k8s pins an immutable image tag, seeds config only when missing, and adds a startupProbe; stale Fly healthcheck-port doc corrected (healthcheck follows the active gateway lock port since bc4221a07e). * docs(k8s): make persistent-file config ownership explicit with reseed path
2026-08-12 21:53:00 -06:00 · 2026-08-11 23:36:33 -07:00
parent 0b4701677b
commit 8190c326ce
16 changed files with 401 additions and 64 deletions
@@ -377,8 +377,9 @@ USER node
 #   - Override --bind to "lan" (0.0.0.0) and set auth credentials
 #
 # Built-in probe endpoints for container health checks:
-#   - GET /healthz (liveness) and GET /readyz (readiness)
-#   - aliases: /health and /ready
+#   - GET /healthz (liveness), GET /startupz (startup/traffic admission),
+#     and GET /readyz (channel-aware readiness)
+#   - aliases: /health, /startup, and /ready
 # For external access from host/ingress, override bind to "lan" and set auth.
 HEALTHCHECK --interval=3m --timeout=10s --start-period=15s --retries=3 \
  CMD ["node", "dist/docker-healthcheck.js"]
@@ -52,6 +52,20 @@ Channel connectivity and inbound admission are separate failure domains. A chann
 - If the restarts keep repeating, the cause is not transient. Check the logged ingress failure: a plugin denied the `openChannelIngressQueue` capability, for example, needs operator action rather than another restart.
 - Channels that never report ingress state are unaffected: absence means "no signal", never "broken". There is no traffic-staleness heuristic, so a genuinely quiet channel is never marked unhealthy for having received nothing.

+## HTTP probes
+
+The Gateway exposes three unauthenticated `GET`/`HEAD` probe pairs:
+
+| Endpoints               | Meaning                                                                                                       | Use                                                            |
+| ----------------------- | ------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- |
+| `/health`, `/healthz`   | The HTTP server is live.                                                                                      | Process liveness and restart decisions.                        |
+| `/startup`, `/startupz` | Startup work is complete and the Gateway is not draining. Channel health is not consulted.                    | Orchestrator startup and traffic admission.                    |
+| `/ready`, `/readyz`     | Startup is complete, the Gateway is not draining, and configured channel accounts pass deep readiness checks. | Operator monitoring that should surface hard channel failures. |
+
+`/startupz` returns `503` with `status: "starting"` while startup sidecars are pending, `503` with `status: "draining"` during drain, and `200` with `status: "started"` otherwise. Use it for Kubernetes, Fly, Render, and similar traffic admission. A broken Telegram or other channel account can make `/readyz` return `503` without taking a healthy Control UI out of service through `/startupz`.
+
+Remote unauthenticated startup responses contain only `ok` and `status`. Local-direct and authenticated callers also receive `version`, `uptimeMs`, and `pendingReason` while startup is pending. Readiness details follow the same local-or-authenticated gate because they can name failing subsystems.
+
 ## Uptime monitoring

 External uptime monitoring services should use the dedicated `/health` endpoint, not `/v1/chat/completions`.
@@ -280,10 +280,12 @@ Container probe endpoints (no auth required):

 ```bash
 curl -fsS http://127.0.0.1:18789/healthz   # liveness
-curl -fsS http://127.0.0.1:18789/readyz     # readiness
+curl -fsS http://127.0.0.1:18789/startupz  # startup and traffic admission
+curl -fsS http://127.0.0.1:18789/readyz    # deep, channel-aware readiness
 ```

 The image's built-in `HEALTHCHECK` pings `/healthz`; repeated failures mark the container `unhealthy` so orchestrators can restart or replace it.
+Use `/startupz` for an orchestrator startup or readiness probe so a failed channel account does not remove the otherwise healthy Gateway and Control UI from service. Use `/readyz` for monitoring that intentionally treats hard channel failures as not ready. See [Health checks](/gateway/health#http-probes) for response details.

 Authenticated deep health snapshot:

@@ -66,6 +66,13 @@ read_when:
      min_machines_running = 1
      processes = ["app"]

+    [[http_service.checks]]
+      grace_period = "2m"
+      interval = "15s"
+      method = "GET"
+      timeout = "5s"
+      path = "/startupz"
+
    [[vm]]
      size = "shared-cpu-2x"
      memory = "2048mb"
@@ -84,6 +91,7 @@ read_when:
    | `--bind lan`                   | Binds to `0.0.0.0` so Fly's proxy can reach the gateway                     |
    | `--allow-unconfigured`         | Starts without a config file (you create one after)                        |
    | `internal_port = 3000`         | Must match `--port 3000` (or `OPENCLAW_GATEWAY_PORT`) for Fly health checks |
+    | `path = "/startupz"`          | Admits traffic after Gateway startup finishes, independent of channel health |
    | `memory = "2048mb"`            | 512MB is too small; 2GB recommended                                         |
    | `OPENCLAW_STATE_DIR = "/data"` | Persists state on the volume                                                |

@@ -123,7 +131,7 @@ read_when:
    fly logs
    ```

-    Gateway startup logs `gateway ready` once the HTTP/WebSocket listener is up. Fly's own health check watches `internal_port = 3000` per `fly.toml`; the image's Docker `HEALTHCHECK` directive additionally polls `/healthz` on its default port 18789, which is unused here since this deployment overrides the gateway to `--port 3000`.
+    Gateway startup logs `gateway ready` once the HTTP/WebSocket listener is up. Fly checks `/startupz` on `internal_port = 3000` and admits traffic after startup work finishes. The image's Docker `HEALTHCHECK` resolves the active Gateway lock port, so its `/healthz` liveness check also follows this deployment's `--port 3000` override.

  </Step>

@@ -247,9 +255,9 @@ The gateway is binding to `127.0.0.1` instead of `0.0.0.0`.

 ### Health checks failing / connection refused

-Fly cannot reach the gateway on the configured port.
+Fly cannot reach the gateway on the configured port, or `/startupz` is still reporting startup work.

-**Fix:** ensure `internal_port` matches the gateway port (`--port 3000` or `OPENCLAW_GATEWAY_PORT=3000`).
+**Fix:** ensure `internal_port` matches the gateway port (`--port 3000` or `OPENCLAW_GATEWAY_PORT=3000`), then inspect `fly logs` for the pending startup step.

 ### OOM / memory issues

@@ -90,6 +90,8 @@ Namespace: openclaw (configurable via OPENCLAW_NAMESPACE)
 └── Secret/openclaw-secrets    # Gateway token + API keys
 ```

+The Deployment uses `/startupz` for both startup and traffic-readiness probes, with a five-minute startup budget. Channel failures do not evict a healthy Gateway or Control UI from Service endpoints. `/healthz` remains the liveness probe; use `/readyz` separately when monitoring should include channel-account health.
+
 ## Customization

 ### Agent instructions
@@ -104,6 +106,15 @@ Edit the `AGENTS.md` in `scripts/k8s/manifests/configmap.yaml` and redeploy:

 Edit `openclaw.json` in `scripts/k8s/manifests/configmap.yaml`. See [Gateway configuration](/gateway/configuration) for the full reference.

+The init container seeds `openclaw.json` and workspace `AGENTS.md` only when each file is missing from the PVC. The persisted copy is the source of truth after first boot: changes made through OpenClaw (`onboard`, `channels add`, `doctor --fix`, Control UI) survive pod restarts, and updating the ConfigMap does not overwrite an existing PVC copy. To intentionally reseed a file from an updated ConfigMap, delete the persisted copy and restart:
+
+```bash
+kubectl exec -n openclaw deploy/openclaw -- rm /home/node/.openclaw/openclaw.json
+kubectl rollout restart -n openclaw deploy/openclaw
+```
+
+Deployments created from the previous template applied ConfigMap edits on every pod start (and discarded any config changes made through OpenClaw). If you relied on that flow, use the reseed commands above after ConfigMap edits.
+
 ### Add providers

 Re-run with additional keys exported:
@@ -136,7 +147,8 @@ OPENCLAW_NAMESPACE=my-namespace ./scripts/k8s/deploy.sh
 Edit the `image` field in `scripts/k8s/manifests/deployment.yaml`:

 ```yaml
-image: ghcr.io/openclaw/openclaw:slim # primary; official Docker Hub mirror: openclaw/openclaw
+# Bump this immutable versioned tag when upgrading OpenClaw.
+image: ghcr.io/openclaw/openclaw:2026.7.1-2-slim
 ```

 ### Expose beyond port-forward
@@ -27,7 +27,7 @@ services:
    name: openclaw
    runtime: docker
    plan: starter
-    healthCheckPath: /health
+    healthCheckPath: /startupz
    envVars:
      - key: OPENCLAW_GATEWAY_PORT
        value: "8080"
@@ -43,12 +43,12 @@ services:
      sizeGB: 1
 ```

-| Feature               | Purpose                                                    |
-| --------------------- | ---------------------------------------------------------- |
-| `runtime: docker`     | Builds from the repo's Dockerfile                          |
-| `healthCheckPath`     | Render monitors `/health` and restarts unhealthy instances |
-| `generateValue: true` | Auto-generates a cryptographically secure value            |
-| `disk`                | Persistent storage that survives redeploys                 |
+| Feature               | Purpose                                                          |
+| --------------------- | ---------------------------------------------------------------- |
+| `runtime: docker`     | Builds from the repo's Dockerfile                                |
+| `healthCheckPath`     | Render admits traffic after `/startupz` reports startup complete |
+| `generateValue: true` | Auto-generates a cryptographically secure value                  |
+| `disk`                | Persistent storage that survives redeploys                       |

 ## Choosing a plan

@@ -123,7 +123,7 @@ Happens on the free tier (no persistent disk). Upgrade to a paid plan, or regula

 ### Health check failures

-If builds succeed but deploys fail, the service may be taking too long to start or `/health` may not be reachable. Check:
+If builds succeed but deploys fail, the service may be taking too long to start or `/startupz` may not be reachable. Check:

 - Build logs for errors
 - Whether the container runs locally with `docker build && docker run`
@@ -25,6 +25,13 @@ auto_start_machines = true
 min_machines_running = 1
 processes = ["app"]

+[[http_service.checks]]
+grace_period = "2m"
+interval = "15s"
+method = "GET"
+timeout = "5s"
+path = "/startupz"
+
 [[vm]]
 size = "shared-cpu-2x"
 memory = "2048mb"
@@ -3,7 +3,7 @@ services:
    name: openclaw
    runtime: docker
    plan: starter
-    healthCheckPath: /health
+    healthCheckPath: /startupz
    envVars:
      - key: OPENCLAW_GATEWAY_PORT
        value: "8080"
@@ -29,9 +29,12 @@ spec:
            - sh
            - -c
            - |
-              cp /config/openclaw.json /home/node/.openclaw/openclaw.json
+              # Seed-if-missing: the PVC copy owns config after first boot so edits made
+              # through OpenClaw (onboard, channels add, doctor --fix, Control UI) survive
+              # restarts. ConfigMap edits need an explicit reseed (see docs/install/kubernetes.md).
+              [ -f /home/node/.openclaw/openclaw.json ] || cp /config/openclaw.json /home/node/.openclaw/openclaw.json
              mkdir -p /home/node/.openclaw/workspace
-              cp /config/AGENTS.md /home/node/.openclaw/workspace/AGENTS.md
+              [ -f /home/node/.openclaw/workspace/AGENTS.md ] || cp /config/AGENTS.md /home/node/.openclaw/workspace/AGENTS.md
          securityContext:
            runAsUser: 1000
            runAsGroup: 1000
@@ -49,7 +52,8 @@ spec:
              mountPath: /config
      containers:
        - name: gateway
-          image: ghcr.io/openclaw/openclaw:slim
+          # Bump this immutable versioned tag when upgrading OpenClaw.
+          image: ghcr.io/openclaw/openclaw:2026.7.1-2-slim
          imagePullPolicy: IfNotPresent
          command:
            - node
@@ -103,6 +107,15 @@ spec:
            limits:
              memory: 2Gi
              cpu: "1"
+          startupProbe:
+            exec:
+              command:
+                - node
+                - -e
+                - "require('http').get('http://127.0.0.1:18789/startupz', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))"
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 30
          livenessProbe:
            exec:
              command:
@@ -117,7 +130,7 @@ spec:
              command:
                - node
                - -e
-                - "require('http').get('http://127.0.0.1:18789/readyz', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))"
+                - "require('http').get('http://127.0.0.1:18789/startupz', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))"
            initialDelaySeconds: 15
            periodSeconds: 10
            timeoutSeconds: 5
@@ -1,8 +1,10 @@
-const GATEWAY_PROBE_ROUTES = new Map<string, "live" | "ready">([
+const GATEWAY_PROBE_ROUTES = new Map<string, "live" | "ready" | "startup">([
  ["/health", "live"],
  ["/healthz", "live"],
  ["/ready", "ready"],
  ["/readyz", "ready"],
+  ["/startup", "startup"],
+  ["/startupz", "startup"],
 ]);

 export const MCP_APP_STANDALONE_PATH = "/__openclaw__/mcp-app";
@@ -10,7 +12,7 @@ export const MCP_APP_STANDALONE_VIEW_PATH = `${MCP_APP_STANDALONE_PATH}/view`;

 export function classifyGatewayProbePath(
  pathname: string,
-): "live" | "ready" | "namespace" | "outside" {
+): "live" | "ready" | "startup" | "namespace" | "outside" {
  for (const [root, status] of GATEWAY_PROBE_ROUTES) {
    if (pathname === root) {
      return status;
@@ -14,6 +14,7 @@ import {
  getActiveGatewayRootWorkCount,
  resetGatewayWorkAdmission,
 } from "../process/gateway-work-admission.js";
+import { resolveRuntimeServiceVersion } from "../version.js";
 import type { ChannelManager } from "./server-channels.js";
 import {
  AUTH_TOKEN,
@@ -23,7 +24,12 @@ import {
  dispatchRequest,
  withGatewayServer,
 } from "./server-http.test-harness.js";
-import { createReadinessChecker, type ReadinessChecker } from "./server/readiness.js";
+import {
+  createReadinessChecker,
+  createStartupChecker,
+  type ReadinessChecker,
+  type StartupChecker,
+} from "./server/readiness.js";
 import { withTempConfig } from "./test-temp-config.js";

 type GatewayServerHarness = Parameters<typeof dispatchRequest>[0];
@@ -352,7 +358,14 @@ describe("gateway probe endpoints", () => {
          expect(exact.res.statusCode).toBe(503);
          expect(JSON.parse(exact.getBody())).toMatchObject({ ready: false });

-          for (const routePath of ["/health/", "/healthz/details", "/ready/", "/readyz/details"]) {
+          for (const routePath of [
+            "/health/",
+            "/healthz/details",
+            "/ready/",
+            "/readyz/details",
+            "/startup/",
+            "/startupz/details",
+          ]) {
            const { res, getBody } = await sendGatewayRequest(server, { path: routePath });
            expect(res.statusCode, routePath).toBe(404);
            expect(getBody(), routePath).toBe("Not Found");
@@ -760,6 +773,133 @@ describe("gateway probe endpoints", () => {
    });
  });

+  it("reports startup lifecycle independently of hard channel failures", async () => {
+    let startupPending = true;
+    let gatewayDraining = false;
+    const startedAt = Date.now() - 5_000;
+    const account = {
+      accountId: "default",
+      running: true,
+      connected: true,
+      enabled: true,
+      configured: true,
+      lifecycle: "blocked" as const,
+      lastStartAt: startedAt,
+    };
+    const channelManager = {
+      getRuntimeSnapshot: () => ({
+        channels: { telegram: account },
+        channelAccounts: { telegram: { default: account } },
+      }),
+      getAutostartSuppression: () => null,
+      isAmbientAutostartSuppressed: () => false,
+    } as unknown as ChannelManager;
+    const startupDeps = {
+      startedAt,
+      getStartupPending: () => startupPending,
+      getStartupPendingReason: () => "plugin-convergence",
+      getGatewayDraining: () => gatewayDraining,
+    };
+    const getStartup = createStartupChecker(startupDeps);
+    const getReadiness = createReadinessChecker({
+      channelManager,
+      ...startupDeps,
+      cacheTtlMs: 0,
+    });
+
+    await withGatewayServer({
+      prefix: "probe-startup-lifecycle",
+      resolvedAuth: AUTH_NONE,
+      overrides: { getReadiness, getStartup },
+      run: async (server) => {
+        const starting = await sendGatewayRequest(server, { path: "/startupz" });
+        expect(starting.res.statusCode).toBe(503);
+        expect(JSON.parse(starting.getBody())).toMatchObject({
+          ok: false,
+          status: "starting",
+          version: resolveRuntimeServiceVersion(process.env),
+          uptimeMs: expect.any(Number),
+          pendingReason: "plugin-convergence",
+        });
+
+        startupPending = false;
+        const started = await sendGatewayRequest(server, { path: "/startupz" });
+        expect(started.res.statusCode).toBe(200);
+        expect(JSON.parse(started.getBody())).toMatchObject({
+          ok: true,
+          status: "started",
+          version: resolveRuntimeServiceVersion(process.env),
+          uptimeMs: expect.any(Number),
+        });
+
+        const readiness = await sendGatewayRequest(server, { path: "/readyz" });
+        expect(readiness.res.statusCode).toBe(503);
+        expect(JSON.parse(readiness.getBody())).toMatchObject({
+          ready: false,
+          failing: ["telegram"],
+        });
+
+        const channelIndependentStartup = await sendGatewayRequest(server, {
+          path: "/startupz",
+        });
+        expect(channelIndependentStartup.res.statusCode).toBe(200);
+        expect(JSON.parse(channelIndependentStartup.getBody())).toMatchObject({
+          ok: true,
+          status: "started",
+        });
+
+        gatewayDraining = true;
+        const draining = await sendGatewayRequest(server, { path: "/startupz" });
+        expect(draining.res.statusCode).toBe(503);
+        expect(JSON.parse(draining.getBody())).toMatchObject({
+          ok: false,
+          status: "draining",
+          version: resolveRuntimeServiceVersion(process.env),
+          uptimeMs: expect.any(Number),
+        });
+      },
+    });
+  });
+
+  it("gates startup details to local or authenticated callers", async () => {
+    const getStartup = createStartupChecker({
+      startedAt: Date.now() - 8_000,
+      getStartupPending: () => true,
+      getStartupPendingReason: () => "startup-sidecars",
+      getGatewayDraining: () => false,
+    });
+
+    await withGatewayServer({
+      prefix: "probe-startup-details",
+      resolvedAuth: AUTH_TOKEN,
+      overrides: { getStartup },
+      run: async (server) => {
+        const remote = await sendGatewayRequest(server, {
+          path: "/startupz",
+          remoteAddress: "10.0.0.8",
+          host: "gateway.test",
+        });
+        expect(remote.res.statusCode).toBe(503);
+        expect(JSON.parse(remote.getBody())).toEqual({ ok: false, status: "starting" });
+
+        const authenticated = await sendGatewayRequest(server, {
+          path: "/startupz",
+          remoteAddress: "10.0.0.8",
+          host: "gateway.test",
+          authorization: "Bearer test-token",
+        });
+        expect(authenticated.res.statusCode).toBe(503);
+        expect(JSON.parse(authenticated.getBody())).toMatchObject({
+          ok: false,
+          status: "starting",
+          version: resolveRuntimeServiceVersion(process.env),
+          uptimeMs: expect.any(Number),
+          pendingReason: "startup-sidecars",
+        });
+      },
+    });
+  });
+
  it("serves /healthz before loading gateway config", async () => {
    const getRuntimeConfig = vi.fn(() => {
      throw new Error("config load blocked");
@@ -837,6 +977,37 @@ describe("gateway probe endpoints", () => {
    });
  });

+  it("keeps GET and HEAD /startupz status and Content-Length in parity", async () => {
+    const getStartup: StartupChecker = () => ({
+      ok: false,
+      status: "draining",
+      uptimeMs: 5_000,
+    });
+
+    await withGatewayServer({
+      prefix: "probe-startupz-head",
+      resolvedAuth: AUTH_NONE,
+      overrides: { getStartup },
+      run: async (server) => {
+        const get = await sendGatewayRequest(server, { path: "/startupz" });
+        const head = createResponse();
+        await dispatchRequest(
+          server,
+          createRequest({ path: "/startupz", method: "HEAD" }),
+          head.res,
+        );
+
+        expect(get.res.statusCode).toBe(503);
+        expect(head.res.statusCode).toBe(503);
+        expect(head.getBody()).toBe("");
+        expect(head.setHeader).toHaveBeenCalledWith(
+          "Content-Length",
+          String(Buffer.byteLength(get.getBody())),
+        );
+      },
+    });
+  });
+
  it("sends Content-Length on HEAD probe responses matching the GET body", async () => {
    await withGatewayServer({
      prefix: "probe-head-content-length",
@@ -24,6 +24,7 @@ import {
  isGatewayWorkAdmissionClosed,
 } from "../process/gateway-work-admission.js";
 import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
+import { resolveRuntimeServiceVersion } from "../version.js";
 import { resolveAssistantIdentity } from "./assistant-identity.js";
 import type { AuthRateLimiter } from "./auth-rate-limit.js";
 import {
@@ -69,7 +70,7 @@ import {
  type PluginRoutePathContext,
 } from "./server/plugins-http/path-context.js";
 import type { PreauthConnectionBudget } from "./server/preauth-connection-budget.js";
-import type { ReadinessChecker } from "./server/readiness.js";
+import type { ReadinessChecker, StartupChecker, StartupResult } from "./server/readiness.js";
 import {
  GATEWAY_WS_CONNECTION_KIND_PROPERTY,
  GATEWAY_WS_PREAUTH_BUDGET_PROPERTY,
@@ -185,7 +186,46 @@ function shouldEnforceDefaultPluginGatewayAuth(pathContext: PluginRoutePathConte
  );
 }

-/** Handles live/ready probe endpoints before normal gateway routing. */
+async function shouldIncludeGatewayProbeDetails(params: {
+  req: IncomingMessage;
+  resolvedAuth: ResolvedGatewayAuth;
+  trustedProxies: string[];
+  allowRealIpFallback: boolean;
+}): Promise<boolean> {
+  if (isLocalDirectRequest(params.req, params.trustedProxies, params.allowRealIpFallback)) {
+    return true;
+  }
+  if (params.resolvedAuth.mode === "none") {
+    return false;
+  }
+  const { getBearerToken, resolveHttpBrowserOriginPolicy } = await getHttpAuthUtilsModule();
+  const bearerToken = getBearerToken(params.req);
+  return (
+    await authorizeHttpGatewayConnect({
+      auth: params.resolvedAuth,
+      connectAuth: bearerToken ? { token: bearerToken, password: bearerToken } : null,
+      req: params.req,
+      trustedProxies: params.trustedProxies,
+      allowRealIpFallback: params.allowRealIpFallback,
+      browserOriginPolicy: resolveHttpBrowserOriginPolicy(params.req),
+    })
+  ).ok;
+}
+
+function startupProbeBody(result: StartupResult, includeDetails: boolean): string {
+  if (!includeDetails) {
+    return JSON.stringify({ ok: result.ok, status: result.status });
+  }
+  return JSON.stringify({
+    ok: result.ok,
+    status: result.status,
+    version: resolveRuntimeServiceVersion(process.env),
+    uptimeMs: result.uptimeMs,
+    ...(result.status === "starting" ? { pendingReason: result.pendingReason } : {}),
+  });
+}
+
+/** Handles live/ready/startup probe endpoints before normal gateway routing. */
 async function handleGatewayProbeRequest(
  req: IncomingMessage,
  res: ServerResponse,
@@ -194,6 +234,7 @@ async function handleGatewayProbeRequest(
  trustedProxies: string[],
  allowRealIpFallback: boolean,
  getReadiness?: ReadinessChecker,
+  getStartup?: StartupChecker,
 ): Promise<boolean> {
  const status = classifyGatewayProbePath(requestPath);
  if (status === "namespace" || status === "outside") {
@@ -217,21 +258,12 @@ async function handleGatewayProbeRequest(
  if (status === "ready" && getReadiness) {
    // Readiness details expose subsystem names, so only local direct or authenticated
    // callers receive them; unauthenticated remote probes get the aggregate boolean.
-    let includeDetails = isLocalDirectRequest(req, trustedProxies, allowRealIpFallback);
-    if (!includeDetails && resolvedAuth.mode !== "none") {
-      const { getBearerToken, resolveHttpBrowserOriginPolicy } = await getHttpAuthUtilsModule();
-      const bearerToken = getBearerToken(req);
-      includeDetails = (
-        await authorizeHttpGatewayConnect({
-          auth: resolvedAuth,
-          connectAuth: bearerToken ? { token: bearerToken, password: bearerToken } : null,
-          req,
-          trustedProxies,
-          allowRealIpFallback,
-          browserOriginPolicy: resolveHttpBrowserOriginPolicy(req),
-        })
-      ).ok;
-    }
+    const includeDetails = await shouldIncludeGatewayProbeDetails({
+      req,
+      resolvedAuth,
+      trustedProxies,
+      allowRealIpFallback,
+    });
    try {
      const result = getReadiness();
      statusCode = result.ready ? 200 : 503;
@@ -242,6 +274,27 @@ async function handleGatewayProbeRequest(
        includeDetails ? { ready: false, failing: ["internal"], uptimeMs: 0 } : { ready: false },
      );
    }
+  } else if (status === "startup") {
+    const includeDetails = await shouldIncludeGatewayProbeDetails({
+      req,
+      resolvedAuth,
+      trustedProxies,
+      allowRealIpFallback,
+    });
+    try {
+      const result = getStartup?.() ?? { ok: true, status: "started", uptimeMs: 0 };
+      statusCode = result.ok ? 200 : 503;
+      body = startupProbeBody(result, includeDetails);
+    } catch {
+      const result: StartupResult = {
+        ok: false,
+        status: "starting",
+        uptimeMs: 0,
+        pendingReason: "internal",
+      };
+      statusCode = 503;
+      body = startupProbeBody(result, includeDetails);
+    }
  } else {
    statusCode = 200;
    body = JSON.stringify({ ok: true, status });
@@ -346,6 +399,7 @@ export function createGatewayHttpServer(opts: {
  /** Optional rate limiter for auth brute-force protection. */
  rateLimiter?: AuthRateLimiter;
  getReadiness?: ReadinessChecker;
+  getStartup?: StartupChecker;
  getRuntimeConfig?: () => OpenClawConfig;
  isStartupPluginRuntimeReady?: () => boolean;
  isTerminalEnabled?: () => boolean;
@@ -368,6 +422,7 @@ export function createGatewayHttpServer(opts: {
    resolvedAuth,
    rateLimiter,
    getReadiness,
+    getStartup,
  } = opts;
  const getResolvedAuth = opts.getResolvedAuth ?? (() => resolvedAuth);
  const loadGatewayConfig = opts.getRuntimeConfig ?? getRuntimeConfig;
@@ -420,6 +475,7 @@ export function createGatewayHttpServer(opts: {
          [],
          false,
          getReadiness,
+          getStartup,
        );
        return;
      }
@@ -471,6 +527,7 @@ export function createGatewayHttpServer(opts: {
              trustedProxies,
              allowRealIpFallback,
              getReadiness,
+              getStartup,
            ),
        },
      ];
@@ -29,7 +29,7 @@ import { createGatewayTransportBridge } from "./server-transport-bridge.js";
 import { createWizardSessionTracker } from "./server-wizard-sessions.js";
 import { createGatewayEventLoopHealthMonitor } from "./server/event-loop-health.js";
 import { resolveHookClientIpConfig } from "./server/hook-client-ip-config.js";
-import { createReadinessChecker } from "./server/readiness.js";
+import { createReadinessChecker, createStartupChecker } from "./server/readiness.js";
 import { resolveSharedGatewaySessionGeneration } from "./server/ws-shared-generation.js";

 type GatewayBootstrap = Awaited<ReturnType<typeof prepareGatewayServerBootstrap>>;
@@ -351,12 +351,16 @@ export async function prepareGatewayKernelState(params: {
  channelManager.setAutostartSuppression(opts.channelAutostartSuppression ?? null);
  const sidecarStartup = opts.sidecarStartup ?? "start";
  const isGatewayStartupPending = () => !startupState.sidecarsReady && sidecarStartup === "start";
-  const getReadiness = createReadinessChecker({
-    channelManager,
+  const startupCheckerDeps = {
    startedAt: serverStartedAt,
    getStartupPending: isGatewayStartupPending,
    getStartupPendingReason: () => startupState.pendingReason,
    getGatewayDraining: isGatewayDraining,
+  };
+  const getStartup = createStartupChecker(startupCheckerDeps);
+  const getReadiness = createReadinessChecker({
+    channelManager,
+    ...startupCheckerDeps,
    getEventLoopHealth: readinessEventLoopHealth.snapshot,
    shouldSkipChannelReadiness: () =>
      isTruthyEnvValue(process.env.OPENCLAW_SKIP_CHANNELS) ||
@@ -407,6 +411,7 @@ export async function prepareGatewayKernelState(params: {
    logHooks,
    logPlugins,
    getReadiness,
+    getStartup,
    handleWatchNodeRequest: async (req: IncomingMessage, res: ServerResponse) =>
      (await watchNodeRequestHandler.current?.(req, res)) ?? false,
    workerIngressEnabled: Boolean(workerEnvironmentService),
@@ -41,7 +41,7 @@ import {
  createPreauthConnectionBudget,
  type PreauthConnectionBudget,
 } from "./server/preauth-connection-budget.js";
-import type { ReadinessChecker } from "./server/readiness.js";
+import type { ReadinessChecker, StartupChecker } from "./server/readiness.js";
 import type { GatewayWsClient } from "./server/ws-types.js";
 import type { WorkerDesktopTunnels } from "./worker-environments/desktop-tunnel.js";

@@ -114,6 +114,7 @@ export async function createGatewayHttpTransport(params: {
  logHooks: ReturnType<typeof createSubsystemLogger>;
  logPlugins: ReturnType<typeof createSubsystemLogger>;
  getReadiness?: ReadinessChecker;
+  getStartup?: StartupChecker;
  isTerminalEnabled: () => boolean;
  handleWatchNodeRequest?: (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
  workerIngressEnabled?: boolean;
@@ -282,6 +283,7 @@ export async function createGatewayHttpTransport(params: {
      getResolvedAuth: params.getResolvedAuth,
      rateLimiter: params.rateLimiter,
      getReadiness: params.getReadiness,
+      getStartup: params.getStartup,
      getRuntimeConfig: loadRuntimeConfig,
      isStartupPluginRuntimeReady: params.isStartupPluginRuntimeReady,
      isTerminalEnabled: params.isTerminalEnabled,
@@ -97,15 +97,25 @@ const PROBE_CASES = [
  { path: "/healthz", status: "live" },
  { path: "/ready", status: "ready" },
  { path: "/readyz", status: "ready" },
+  { path: "/startup", status: "started" },
+  { path: "/startupz", status: "started" },
 ] as const;

 async function expectProbeRoutesHealthy(server: Parameters<typeof sendRequest>[0]) {
  for (const probeCase of PROBE_CASES) {
    const response = await sendRequest(server, { path: probeCase.path });
    expect(response.res.statusCode, probeCase.path).toBe(200);
-    expect(response.getBody(), probeCase.path).toBe(
-      JSON.stringify({ ok: true, status: probeCase.status }),
-    );
+    const body = JSON.parse(response.getBody());
+    if (probeCase.status === "started") {
+      expect(body, probeCase.path).toMatchObject({
+        ok: true,
+        status: "started",
+        version: expect.any(String),
+        uptimeMs: expect.any(Number),
+      });
+    } else {
+      expect(body, probeCase.path).toEqual({ ok: true, status: probeCase.status });
+    }
  }
 }

@@ -22,8 +22,42 @@ type ReadinessResult = {
 /** Function form used by HTTP readiness endpoints and tests. */
 export type ReadinessChecker = () => ReadinessResult;

+export type StartupResult =
+  | { ok: true; status: "started"; uptimeMs: number }
+  | { ok: false; status: "starting"; uptimeMs: number; pendingReason: string }
+  | { ok: false; status: "draining"; uptimeMs: number };
+
+/** Function form used by HTTP startup endpoints and tests. */
+export type StartupChecker = () => StartupResult;
+
+type GatewayStartupStateDeps = {
+  startedAt: number;
+  getStartupPending?: () => boolean;
+  getStartupPendingReason?: () => string | undefined;
+  getGatewayDraining?: () => boolean;
+};
+
 const DEFAULT_READINESS_CACHE_TTL_MS = 1_000;

+/** Create a startup checker that excludes downstream channel health. */
+export function createStartupChecker(deps: GatewayStartupStateDeps): StartupChecker {
+  return (): StartupResult => {
+    const uptimeMs = Date.now() - deps.startedAt;
+    if (deps.getStartupPending?.()) {
+      return {
+        ok: false,
+        status: "starting",
+        uptimeMs,
+        pendingReason: deps.getStartupPendingReason?.() ?? "startup-sidecars",
+      };
+    }
+    if (deps.getGatewayDraining?.()) {
+      return { ok: false, status: "draining", uptimeMs };
+    }
+    return { ok: true, status: "started", uptimeMs };
+  };
+}
+
 function shouldIgnoreReadinessFailure(
  accountSnapshot: ChannelAccountSnapshot,
  health: ChannelHealthEvaluation,
@@ -49,32 +83,31 @@ function shouldIgnoreReadinessFailure(
 }

 /** Create a cached readiness checker over channel runtime health. */
-export function createReadinessChecker(deps: {
-  channelManager: ChannelManager;
-  startedAt: number;
-  getStartupPending?: () => boolean;
-  getStartupPendingReason?: () => string | undefined;
-  getGatewayDraining?: () => boolean;
-  getEventLoopHealth?: () => GatewayEventLoopHealth | undefined;
-  shouldSkipChannelReadiness?: () => boolean;
-  cacheTtlMs?: number;
-}): ReadinessChecker {
+export function createReadinessChecker(
+  deps: GatewayStartupStateDeps & {
+    channelManager: ChannelManager;
+    getEventLoopHealth?: () => GatewayEventLoopHealth | undefined;
+    shouldSkipChannelReadiness?: () => boolean;
+    cacheTtlMs?: number;
+  },
+): ReadinessChecker {
  const { channelManager, startedAt } = deps;
+  const getStartup = createStartupChecker(deps);
  const cacheTtlMs = Math.max(0, deps.cacheTtlMs ?? DEFAULT_READINESS_CACHE_TTL_MS);
  let cachedAt = 0;
  let cachedState: Omit<ReadinessResult, "uptimeMs"> | null = null;

  return (): ReadinessResult => {
-    const now = Date.now();
-    const uptimeMs = now - startedAt;
-    if (deps.getStartupPending?.()) {
-      const reason = deps.getStartupPendingReason?.() ?? "startup-sidecars";
+    const startup = getStartup();
+    const uptimeMs = startup.uptimeMs;
+    const now = startedAt + uptimeMs;
+    if (startup.status === "starting") {
      return withEventLoopHealth(
-        { ready: false, failing: [reason], uptimeMs },
+        { ready: false, failing: [startup.pendingReason], uptimeMs },
        deps.getEventLoopHealth,
      );
    }
-    if (deps.getGatewayDraining?.()) {
+    if (startup.status === "draining") {
      return withEventLoopHealth(
        { ready: false, failing: ["gateway-draining"], uptimeMs },
        deps.getEventLoopHealth,