mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-19 09:01:39 -06:00
fix(gateway): avoid Tailscale crash loops after upgrades (#126069)
* fix(gateway): migrate legacy Tailscale routes on upgrade * fix(gateway): preserve unattributable Tailscale routes
This commit is contained in:
@@ -117,6 +117,7 @@ This compatibility path does not grant managed Tailscale semantics: `gateway.aut
|
||||
- Tailscale Serve/Funnel requires the `tailscale` CLI installed and logged in.
|
||||
- `tailscale.mode: "funnel"` refuses to start unless auth mode is `password`, to avoid public exposure.
|
||||
- OpenClaw holds Serve/Funnel as a foreground Tailscale claim. Gateway startup succeeds only after the claim is active, and stopping or losing the Gateway releases it automatically.
|
||||
- When upgrading from an older OpenClaw release, a persistent HTTPS root route may still own port 443. Because that route is indistinguishable from an operator-managed route, OpenClaw does not change it: startup explains the conflict and exits with status 78 instead of entering a systemd restart loop. Inspect `tailscale serve status`. If you confirm it is the stale route created by older managed Serve/Funnel, remove only its root handler with `tailscale serve --yes --https=443 --set-path=/ off` or `tailscale funnel --yes --https=443 --set-path=/ off`, then restart the Gateway. If the route is intentionally external, disable managed Tailscale ingress and use the `trustedProxies` path above.
|
||||
- Named Tailscale Services are not supported by managed ingress because Tailscale requires them to run as persistent background routes. Existing `gateway.tailscale.serviceName` installs must run `openclaw doctor --fix`; Doctor disables managed ingress and removes the key. Inspect the retained Service route, clear it with `tailscale serve clear <service-name>`, then enable device Serve with `gateway.tailscale.mode: "serve"` if desired.
|
||||
- Older releases could advertise an externally configured default HTTPS Serve route that targeted a `gateway.bind: "lan"` listener. That route does not automatically gain trusted ingress provenance. Prefer migrating to managed ingress: run `openclaw doctor` to preview an atomic change to `gateway.bind: "loopback"` plus `gateway.tailscale.mode: "serve"`; apply it with `openclaw doctor --fix`, then restart the Gateway. Doctor does not reset Tailscale state or guess how to rewrite custom Serve ports and Tailscale Services. If another service must retain ownership, use the explicit `trustedProxies` compatibility path above.
|
||||
- `gateway.tailscale.preserveFunnel: true` is a deprecated migration guard. It detects an externally configured `tailscale funnel` route before reapplying Serve. If that route still targets the ordinary Gateway listener, OpenClaw leaves it unchanged and warns because the route is not managed ingress. Gateway-authenticated routes work only through the explicit `trustedProxies` compatibility path above and continue to require the configured auth; plugin-authenticated webhook routes such as Google Chat and SMS keep using their own signature/auth checks. To migrate, first configure a durable `gateway.auth.password` (prefer a SecretRef) or `OPENCLAW_GATEWAY_PASSWORD`, set `gateway.auth.mode` to `password`, run `openclaw config set gateway.tailscale.mode funnel`, then `openclaw config unset gateway.tailscale.preserveFunnel`.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import { createServer } from "node:http";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { GatewayLockError } from "../../infra/gateway-lock.js";
|
||||
import { TailscaleRouteOwnershipConflictError } from "../../infra/tailscale-route-ownership-error.js";
|
||||
import { OpenClawAgentDatabaseMediaMigrationRequiredError } from "../../state/openclaw-agent-db-migration-required.js";
|
||||
import { testing } from "./run.test-support.js";
|
||||
|
||||
@@ -21,6 +22,12 @@ function createLogger() {
|
||||
}
|
||||
|
||||
describe("supervised gateway lock recovery", () => {
|
||||
it("uses exit 78 for an ambiguous persistent Tailscale route", () => {
|
||||
expect(
|
||||
testing.resolveGatewayStartupFailureExitCode(new TailscaleRouteOwnershipConflictError()),
|
||||
).toBe(78);
|
||||
});
|
||||
|
||||
it("uses exit 78 for offline agent database migration requirements", () => {
|
||||
expect(
|
||||
testing.resolveGatewayStartupFailureExitCode(
|
||||
|
||||
@@ -55,6 +55,7 @@ import {
|
||||
formatGatewayPidList,
|
||||
} from "../../infra/gateway-processes.js";
|
||||
import type { RespawnSupervisor } from "../../infra/supervisor-markers.js";
|
||||
import { isTailscaleRouteOwnershipConflictError } from "../../infra/tailscale-route-ownership-error.js";
|
||||
import { setConsoleSubsystemFilter, setConsoleTimestampPrefix } from "../../logging/console.js";
|
||||
import { withDiagnosticPhase } from "../../logging/diagnostic-phase.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
@@ -478,6 +479,7 @@ function resolveGatewayLockErrorExitCode(err: unknown): number {
|
||||
|
||||
function resolveGatewayStartupFailureExitCode(err: unknown): number {
|
||||
return isInvalidConfigError(err) ||
|
||||
isTailscaleRouteOwnershipConflictError(err) ||
|
||||
findOpenClawAgentDatabaseMediaMigrationRequiredError(err) ||
|
||||
findOpenClawStateDatabaseSchemaMigrationRequiredError(err)
|
||||
? EXIT_CONFIG_ERROR
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { captureEnv } from "../test-utils/env.js";
|
||||
import { startGatewayTailscaleExposure } from "./server-tailscale.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
describe("managed Tailscale upgrade", () => {
|
||||
const legacyRoute = (funnel = false, proxyPort = 18789) => {
|
||||
const host = "fixture.tailnet.ts.net:443";
|
||||
return {
|
||||
TCP: { "443": { HTTPS: true } },
|
||||
Web: { [host]: { Handlers: { "/": { Proxy: `http://127.0.0.1:${proxyPort}/` } } } },
|
||||
...(funnel ? { AllowFunnel: { [host]: true } } : {}),
|
||||
};
|
||||
};
|
||||
|
||||
const installFixture = async (config: object, mode: "serve" | "funnel") => {
|
||||
const fixture = fileURLToPath(
|
||||
new URL("../../test/fixtures/tailscale-legacy-route-fixture.mjs", import.meta.url),
|
||||
);
|
||||
const marker = path.join(tempDirs.make("openclaw-tailscale-upgrade-"), "state");
|
||||
await writeFile(marker, JSON.stringify(config));
|
||||
process.env.OPENCLAW_TEST_TAILSCALE_BINARY = fixture;
|
||||
process.env.OPENCLAW_TEST_TAILSCALE_FIXTURE_MARKER = marker;
|
||||
process.env.OPENCLAW_TEST_TAILSCALE_FIXTURE_MODE = mode;
|
||||
process.env.VITEST ??= "true";
|
||||
return marker;
|
||||
};
|
||||
|
||||
it.each(["serve", "funnel"] as const)(
|
||||
"does not infer ownership from a matching persistent %s route",
|
||||
async (mode) => {
|
||||
const env = captureEnv([
|
||||
"OPENCLAW_TEST_TAILSCALE_BINARY",
|
||||
"OPENCLAW_TEST_TAILSCALE_FIXTURE_MARKER",
|
||||
"OPENCLAW_TEST_TAILSCALE_FIXTURE_MODE",
|
||||
"VITEST",
|
||||
]);
|
||||
const marker = await installFixture(legacyRoute(mode === "funnel"), mode);
|
||||
const before = await readFile(marker, "utf8");
|
||||
|
||||
try {
|
||||
await expect(
|
||||
startGatewayTailscaleExposure({
|
||||
tailscaleMode: mode,
|
||||
port: 18789,
|
||||
backend: { host: "127.0.0.1", port: 19000 },
|
||||
logTailscale: { info: () => undefined, warn: () => undefined },
|
||||
}),
|
||||
).rejects.toThrow("ownership OpenClaw cannot prove; it was not modified");
|
||||
expect(await readFile(marker, "utf8")).toBe(before);
|
||||
} finally {
|
||||
env.restore();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("does not mutate an independent Tailscale Service", async () => {
|
||||
const env = captureEnv([
|
||||
"OPENCLAW_TEST_TAILSCALE_BINARY",
|
||||
"OPENCLAW_TEST_TAILSCALE_FIXTURE_MARKER",
|
||||
"OPENCLAW_TEST_TAILSCALE_FIXTURE_MODE",
|
||||
"VITEST",
|
||||
]);
|
||||
const marker = await installFixture({ Services: { "svc:other": legacyRoute() } }, "serve");
|
||||
const before = await readFile(marker, "utf8");
|
||||
|
||||
try {
|
||||
const cleanup = await startGatewayTailscaleExposure({
|
||||
tailscaleMode: "serve",
|
||||
port: 18789,
|
||||
backend: { host: "127.0.0.1", port: 19000 },
|
||||
logTailscale: { info: () => undefined, warn: () => undefined },
|
||||
});
|
||||
|
||||
expect(await readFile(marker, "utf8")).toBe(before);
|
||||
await cleanup?.();
|
||||
} finally {
|
||||
env.restore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,24 @@
|
||||
import { asNullableObjectRecord as readRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
|
||||
const TAILSCALE_ROUTE_OWNERSHIP_CONFLICT_CODE = "TAILSCALE_ROUTE_OWNERSHIP_CONFLICT";
|
||||
|
||||
export class TailscaleRouteOwnershipConflictError extends Error {
|
||||
readonly code = TAILSCALE_ROUTE_OWNERSHIP_CONFLICT_CODE;
|
||||
|
||||
constructor() {
|
||||
super(
|
||||
"Tailscale HTTPS port 443 is already owned by a route whose ownership OpenClaw cannot prove; it was not modified. " +
|
||||
"Inspect `tailscale serve status`. If it is a stale route from an older OpenClaw release, remove its root handler with " +
|
||||
"`tailscale serve --yes --https=443 --set-path=/ off` or `tailscale funnel --yes --https=443 --set-path=/ off`, then restart the Gateway. " +
|
||||
"Otherwise disable managed Tailscale ingress or reconfigure the route before restarting.",
|
||||
);
|
||||
this.name = "TailscaleRouteOwnershipConflictError";
|
||||
}
|
||||
}
|
||||
|
||||
export function isTailscaleRouteOwnershipConflictError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof TailscaleRouteOwnershipConflictError ||
|
||||
readRecord(error)?.code === TAILSCALE_ROUTE_OWNERSHIP_CONFLICT_CODE
|
||||
);
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
TAILSCALE_ROUTE_OWNER_ARG,
|
||||
type TailscaleRouteOwnerMessage,
|
||||
} from "./tailscale-route-owner-protocol.js";
|
||||
import { TailscaleRouteOwnershipConflictError } from "./tailscale-route-ownership-error.js";
|
||||
|
||||
const TAILSCALE_STATUS_ATTEMPTS = 3;
|
||||
const TAILSCALE_STATUS_RETRY_DELAY_MS = 500;
|
||||
@@ -237,7 +238,14 @@ type TailscaleRouteOwnerFailure = Pick<
|
||||
"code" | "stdout" | "stderr"
|
||||
>;
|
||||
|
||||
function isPort443RouteConflict(message: TailscaleRouteOwnerFailure): boolean {
|
||||
return /listener already exists for port 443/i.test(`${message.stderr}\n${message.stdout}`);
|
||||
}
|
||||
|
||||
function routeClaimError(message: TailscaleRouteOwnerFailure): Error {
|
||||
if (isPort443RouteConflict(message)) {
|
||||
return new TailscaleRouteOwnershipConflictError();
|
||||
}
|
||||
const detail = [message.stderr.trim(), message.stdout.trim()].find(Boolean);
|
||||
return Object.assign(new Error(detail || "Tailscale route owner exited before claiming route"), {
|
||||
code: message.code,
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env node
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const marker = process.env.OPENCLAW_TEST_TAILSCALE_FIXTURE_MARKER;
|
||||
const mode = process.env.OPENCLAW_TEST_TAILSCALE_FIXTURE_MODE;
|
||||
if (!marker || (mode !== "serve" && mode !== "funnel")) {
|
||||
process.stderr.write("missing fixture state\n");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
if (JSON.stringify(args) === JSON.stringify(["status", "--json"])) {
|
||||
process.stdout.write(JSON.stringify({ Self: { DNSName: "fixture.tailnet.ts.net." } }));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (JSON.stringify(args) === JSON.stringify([mode, "--yes", "--bg=false", "19000"])) {
|
||||
const state = await readFile(marker, "utf8");
|
||||
const status = JSON.parse(state);
|
||||
if (status.TCP?.["443"]) {
|
||||
process.stderr.write("listener already exists for port 443\n");
|
||||
process.exit(1);
|
||||
}
|
||||
process.stdout.write("Press Ctrl+C to exit.\n");
|
||||
setInterval(() => {}, 1000);
|
||||
} else {
|
||||
process.stderr.write(`unexpected arguments: ${JSON.stringify(args)}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
Reference in New Issue
Block a user