fix(deploy): repair probes that silently pass and a blueprint that cannot boot (#122963)

Three defects proven by running the shipped configs:

Kubernetes probes checked only the status code against /startupz, but the
pinned image predates that route and the Control UI answers unknown paths
with a catch-all 200. A wedged pod was therefore marked Ready forever.
Probes now assert the JSON probe contract and target routes the pinned
image actually serves; verified in a kind cluster where the old command
exits 0 on the missing route and the new one exits 1.

render.yaml set no dockerCommand, so the image CMD ran without
--allow-unconfigured and a fresh Render disk exited 78 with 'Missing
config' before binding. Reproduced locally with Render's exact env.

The Cloudflare Container readiness poll had the same route mismatch
against operator-supplied official image digests; it now polls /healthz,
which every published image serves.

Also replaces an R2 verification step that could never fail: wrangler
cannot list object keys, so the documented command 404'd into || true.
This commit is contained in:
Peter Steinberger
2026-08-12 20:49:54 -07:00
committed by GitHub
parent 89a5507602
commit 2c3e537cb8
7 changed files with 42 additions and 16 deletions
+4 -3
View File
@@ -21,7 +21,7 @@ OpenClaw + Litestream container :8080
+--> R2 S3 API (SQLite replicas)
```
Every HTTP and WebSocket request is forwarded to port `8080`. The Container helper checks `GET /startupz` before admitting traffic. `max_instances: 1` and the single Durable Object name are the installation's outer single-writer fence.
Every HTTP and WebSocket request is forwarded to port `8080`. The Container helper polls `GET /healthz`, which every published image serves, before admitting traffic. `max_instances: 1` and the single Durable Object name are the installation's outer single-writer fence.
## Prerequisites
@@ -128,7 +128,7 @@ Keep the exact bootstrap recipe in a private, reproducible runbook. Litestream d
## 5. Verify before relying on it
```bash
curl -sS https://<worker-subdomain>.workers.dev/startupz
curl -sS https://<worker-subdomain>.workers.dev/healthz
npx wrangler tail
```
@@ -170,7 +170,8 @@ Treat rollbacks like restores: stop traffic where possible, preserve the current
## Troubleshooting
- **Container never becomes ready:** the image must be `linux/amd64` and pulled from a public registry, referenced by digest rather than a moving tag.
- **Requests time out after a successful deploy:** the Container helper waits for `GET /startupz` on port `8080`; confirm the Gateway still binds that port.
- **Requests time out after a successful deploy:** the Container helper waits for `GET /healthz` on port `8080`; confirm the Gateway still binds that port.
- **A probe passes but nothing serves:** the Control UI answers unknown paths with a catch-all `200`, so probing a route the image does not serve looks healthy forever; assert the JSON body, not just the status.
- **Litestream authentication or signature errors:** Litestream needs R2 _S3 API_ credentials, not a Cloudflare API token, and `LITESTREAM_ENDPOINT` must contain the account ID.
- **First boot reports no databases to restore:** expected on an empty bucket; the entrypoint treats that as a fresh installation.
- **`/readyz` is 503 while `/startupz` is 200:** by design. Startup finished and a channel account is unhealthy; inspect channel status instead of restarting.
+8 -1
View File
@@ -53,7 +53,14 @@ function buildContainerEnv(env: OpenClawContainerEnv): Record<string, string> {
export class OpenClawContainer extends Container<OpenClawContainerEnv> {
override defaultPort = 8080;
override pingEndpoint = "localhost/startupz";
// /healthz exists in every published OpenClaw image and answers as soon as the
// Gateway's listener is up, which is exactly what this readiness poll asks.
// Do not point this at a route the pinned image may not serve: the Control UI
// answers unknown paths with a catch-all 200, so a missing route would look
// permanently healthy instead of failing. /startupz additionally waits for
// startup work to finish and is the better signal once the derived image comes
// from a release that serves it.
override pingEndpoint = "localhost/healthz";
override sleepAfter = "10m";
private readonly webhookOnly: boolean;
+10 -3
View File
@@ -107,12 +107,19 @@ spec:
limits:
memory: 2Gi
cpu: "1"
# Probes assert the JSON probe contract, not just the status code. The
# Control UI serves a catch-all 200 for unknown paths, so a status-only
# check would pass forever against an image whose probe route does not
# exist yet. Failing closed makes that mismatch loud instead of silent.
# /startupz (startup admission, ignores channel health) needs an image
# built from the release that added it; switch the startup and
# readiness probes to it after pinning such an image.
startupProbe:
exec:
command:
- node
- -e
- "require('http').get('http://127.0.0.1:18789/startupz', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))"
- "require('http').get('http://127.0.0.1:18789/readyz', r => { let b = ''; r.on('data', d => b += d); r.on('end', () => { try { process.exit(r.statusCode < 400 && JSON.parse(b).ready === true ? 0 : 1); } catch { process.exit(1); } }); }).on('error', () => process.exit(1))"
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 30
@@ -121,7 +128,7 @@ spec:
command:
- node
- -e
- "require('http').get('http://127.0.0.1:18789/healthz', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))"
- "require('http').get('http://127.0.0.1:18789/healthz', r => { let b = ''; r.on('data', d => b += d); r.on('end', () => { try { process.exit(r.statusCode < 400 && JSON.parse(b).ok === true ? 0 : 1); } catch { process.exit(1); } }); }).on('error', () => process.exit(1))"
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
@@ -130,7 +137,7 @@ spec:
command:
- node
- -e
- "require('http').get('http://127.0.0.1:18789/startupz', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))"
- "require('http').get('http://127.0.0.1:18789/readyz', r => { let b = ''; r.on('data', d => b += d); r.on('end', () => { try { process.exit(r.statusCode < 400 && JSON.parse(b).ready === true ? 0 : 1); } catch { process.exit(1); } }); }).on('error', () => process.exit(1))"
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 5