fix(gateway): avoid Tailscale crash loops after upgrades (#126069)

* fix(gateway): migrate legacy Tailscale routes on upgrade

* fix(gateway): preserve unattributable Tailscale routes
This commit is contained in:
Josh Avant
2026-08-18 19:14:49 -05:00
committed by GitHub
parent fe816d69ef
commit 8d6cbee1b5
7 changed files with 157 additions and 0 deletions
+1
View File
@@ -117,6 +117,7 @@ This compatibility path does not grant managed Tailscale semantics: `gateway.aut
- Tailscale Serve/Funnel requires the `tailscale` CLI installed and logged in.
- `tailscale.mode: "funnel"` refuses to start unless auth mode is `password`, to avoid public exposure.
- OpenClaw holds Serve/Funnel as a foreground Tailscale claim. Gateway startup succeeds only after the claim is active, and stopping or losing the Gateway releases it automatically.
- When upgrading from an older OpenClaw release, a persistent HTTPS root route may still own port 443. Because that route is indistinguishable from an operator-managed route, OpenClaw does not change it: startup explains the conflict and exits with status 78 instead of entering a systemd restart loop. Inspect `tailscale serve status`. If you confirm it is the stale route created by older managed Serve/Funnel, remove only its root handler with `tailscale serve --yes --https=443 --set-path=/ off` or `tailscale funnel --yes --https=443 --set-path=/ off`, then restart the Gateway. If the route is intentionally external, disable managed Tailscale ingress and use the `trustedProxies` path above.
- Named Tailscale Services are not supported by managed ingress because Tailscale requires them to run as persistent background routes. Existing `gateway.tailscale.serviceName` installs must run `openclaw doctor --fix`; Doctor disables managed ingress and removes the key. Inspect the retained Service route, clear it with `tailscale serve clear <service-name>`, then enable device Serve with `gateway.tailscale.mode: "serve"` if desired.
- Older releases could advertise an externally configured default HTTPS Serve route that targeted a `gateway.bind: "lan"` listener. That route does not automatically gain trusted ingress provenance. Prefer migrating to managed ingress: run `openclaw doctor` to preview an atomic change to `gateway.bind: "loopback"` plus `gateway.tailscale.mode: "serve"`; apply it with `openclaw doctor --fix`, then restart the Gateway. Doctor does not reset Tailscale state or guess how to rewrite custom Serve ports and Tailscale Services. If another service must retain ownership, use the explicit `trustedProxies` compatibility path above.
- `gateway.tailscale.preserveFunnel: true` is a deprecated migration guard. It detects an externally configured `tailscale funnel` route before reapplying Serve. If that route still targets the ordinary Gateway listener, OpenClaw leaves it unchanged and warns because the route is not managed ingress. Gateway-authenticated routes work only through the explicit `trustedProxies` compatibility path above and continue to require the configured auth; plugin-authenticated webhook routes such as Google Chat and SMS keep using their own signature/auth checks. To migrate, first configure a durable `gateway.auth.password` (prefer a SecretRef) or `OPENCLAW_GATEWAY_PASSWORD`, set `gateway.auth.mode` to `password`, run `openclaw config set gateway.tailscale.mode funnel`, then `openclaw config unset gateway.tailscale.preserveFunnel`.
@@ -2,6 +2,7 @@
import { createServer } from "node:http";
import { describe, expect, it, vi } from "vitest";
import { GatewayLockError } from "../../infra/gateway-lock.js";
import { TailscaleRouteOwnershipConflictError } from "../../infra/tailscale-route-ownership-error.js";
import { OpenClawAgentDatabaseMediaMigrationRequiredError } from "../../state/openclaw-agent-db-migration-required.js";
import { testing } from "./run.test-support.js";
@@ -21,6 +22,12 @@ function createLogger() {
}
describe("supervised gateway lock recovery", () => {
it("uses exit 78 for an ambiguous persistent Tailscale route", () => {
expect(
testing.resolveGatewayStartupFailureExitCode(new TailscaleRouteOwnershipConflictError()),
).toBe(78);
});
it("uses exit 78 for offline agent database migration requirements", () => {
expect(
testing.resolveGatewayStartupFailureExitCode(
+2
View File
@@ -55,6 +55,7 @@ import {
formatGatewayPidList,
} from "../../infra/gateway-processes.js";
import type { RespawnSupervisor } from "../../infra/supervisor-markers.js";
import { isTailscaleRouteOwnershipConflictError } from "../../infra/tailscale-route-ownership-error.js";
import { setConsoleSubsystemFilter, setConsoleTimestampPrefix } from "../../logging/console.js";
import { withDiagnosticPhase } from "../../logging/diagnostic-phase.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
@@ -478,6 +479,7 @@ function resolveGatewayLockErrorExitCode(err: unknown): number {
function resolveGatewayStartupFailureExitCode(err: unknown): number {
return isInvalidConfigError(err) ||
isTailscaleRouteOwnershipConflictError(err) ||
findOpenClawAgentDatabaseMediaMigrationRequiredError(err) ||
findOpenClawStateDatabaseSchemaMigrationRequiredError(err)
? EXIT_CONFIG_ERROR
@@ -0,0 +1,86 @@
import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { afterEach, describe, expect, it } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { captureEnv } from "../test-utils/env.js";
import { startGatewayTailscaleExposure } from "./server-tailscale.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
describe("managed Tailscale upgrade", () => {
const legacyRoute = (funnel = false, proxyPort = 18789) => {
const host = "fixture.tailnet.ts.net:443";
return {
TCP: { "443": { HTTPS: true } },
Web: { [host]: { Handlers: { "/": { Proxy: `http://127.0.0.1:${proxyPort}/` } } } },
...(funnel ? { AllowFunnel: { [host]: true } } : {}),
};
};
const installFixture = async (config: object, mode: "serve" | "funnel") => {
const fixture = fileURLToPath(
new URL("../../test/fixtures/tailscale-legacy-route-fixture.mjs", import.meta.url),
);
const marker = path.join(tempDirs.make("openclaw-tailscale-upgrade-"), "state");
await writeFile(marker, JSON.stringify(config));
process.env.OPENCLAW_TEST_TAILSCALE_BINARY = fixture;
process.env.OPENCLAW_TEST_TAILSCALE_FIXTURE_MARKER = marker;
process.env.OPENCLAW_TEST_TAILSCALE_FIXTURE_MODE = mode;
process.env.VITEST ??= "true";
return marker;
};
it.each(["serve", "funnel"] as const)(
"does not infer ownership from a matching persistent %s route",
async (mode) => {
const env = captureEnv([
"OPENCLAW_TEST_TAILSCALE_BINARY",
"OPENCLAW_TEST_TAILSCALE_FIXTURE_MARKER",
"OPENCLAW_TEST_TAILSCALE_FIXTURE_MODE",
"VITEST",
]);
const marker = await installFixture(legacyRoute(mode === "funnel"), mode);
const before = await readFile(marker, "utf8");
try {
await expect(
startGatewayTailscaleExposure({
tailscaleMode: mode,
port: 18789,
backend: { host: "127.0.0.1", port: 19000 },
logTailscale: { info: () => undefined, warn: () => undefined },
}),
).rejects.toThrow("ownership OpenClaw cannot prove; it was not modified");
expect(await readFile(marker, "utf8")).toBe(before);
} finally {
env.restore();
}
},
);
it("does not mutate an independent Tailscale Service", async () => {
const env = captureEnv([
"OPENCLAW_TEST_TAILSCALE_BINARY",
"OPENCLAW_TEST_TAILSCALE_FIXTURE_MARKER",
"OPENCLAW_TEST_TAILSCALE_FIXTURE_MODE",
"VITEST",
]);
const marker = await installFixture({ Services: { "svc:other": legacyRoute() } }, "serve");
const before = await readFile(marker, "utf8");
try {
const cleanup = await startGatewayTailscaleExposure({
tailscaleMode: "serve",
port: 18789,
backend: { host: "127.0.0.1", port: 19000 },
logTailscale: { info: () => undefined, warn: () => undefined },
});
expect(await readFile(marker, "utf8")).toBe(before);
await cleanup?.();
} finally {
env.restore();
}
});
});
@@ -0,0 +1,24 @@
import { asNullableObjectRecord as readRecord } from "@openclaw/normalization-core/record-coerce";
const TAILSCALE_ROUTE_OWNERSHIP_CONFLICT_CODE = "TAILSCALE_ROUTE_OWNERSHIP_CONFLICT";
export class TailscaleRouteOwnershipConflictError extends Error {
readonly code = TAILSCALE_ROUTE_OWNERSHIP_CONFLICT_CODE;
constructor() {
super(
"Tailscale HTTPS port 443 is already owned by a route whose ownership OpenClaw cannot prove; it was not modified. " +
"Inspect `tailscale serve status`. If it is a stale route from an older OpenClaw release, remove its root handler with " +
"`tailscale serve --yes --https=443 --set-path=/ off` or `tailscale funnel --yes --https=443 --set-path=/ off`, then restart the Gateway. " +
"Otherwise disable managed Tailscale ingress or reconfigure the route before restarting.",
);
this.name = "TailscaleRouteOwnershipConflictError";
}
}
export function isTailscaleRouteOwnershipConflictError(error: unknown): boolean {
return (
error instanceof TailscaleRouteOwnershipConflictError ||
readRecord(error)?.code === TAILSCALE_ROUTE_OWNERSHIP_CONFLICT_CODE
);
}
+8
View File
@@ -21,6 +21,7 @@ import {
TAILSCALE_ROUTE_OWNER_ARG,
type TailscaleRouteOwnerMessage,
} from "./tailscale-route-owner-protocol.js";
import { TailscaleRouteOwnershipConflictError } from "./tailscale-route-ownership-error.js";
const TAILSCALE_STATUS_ATTEMPTS = 3;
const TAILSCALE_STATUS_RETRY_DELAY_MS = 500;
@@ -237,7 +238,14 @@ type TailscaleRouteOwnerFailure = Pick<
"code" | "stdout" | "stderr"
>;
function isPort443RouteConflict(message: TailscaleRouteOwnerFailure): boolean {
return /listener already exists for port 443/i.test(`${message.stderr}\n${message.stdout}`);
}
function routeClaimError(message: TailscaleRouteOwnerFailure): Error {
if (isPort443RouteConflict(message)) {
return new TailscaleRouteOwnershipConflictError();
}
const detail = [message.stderr.trim(), message.stdout.trim()].find(Boolean);
return Object.assign(new Error(detail || "Tailscale route owner exited before claiming route"), {
code: message.code,
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env node
import { readFile } from "node:fs/promises";
const args = process.argv.slice(2);
const marker = process.env.OPENCLAW_TEST_TAILSCALE_FIXTURE_MARKER;
const mode = process.env.OPENCLAW_TEST_TAILSCALE_FIXTURE_MODE;
if (!marker || (mode !== "serve" && mode !== "funnel")) {
process.stderr.write("missing fixture state\n");
process.exit(2);
}
if (JSON.stringify(args) === JSON.stringify(["status", "--json"])) {
process.stdout.write(JSON.stringify({ Self: { DNSName: "fixture.tailnet.ts.net." } }));
process.exit(0);
}
if (JSON.stringify(args) === JSON.stringify([mode, "--yes", "--bg=false", "19000"])) {
const state = await readFile(marker, "utf8");
const status = JSON.parse(state);
if (status.TCP?.["443"]) {
process.stderr.write("listener already exists for port 443\n");
process.exit(1);
}
process.stdout.write("Press Ctrl+C to exit.\n");
setInterval(() => {}, 1000);
} else {
process.stderr.write(`unexpected arguments: ${JSON.stringify(args)}\n`);
process.exit(2);
}