From 3a5b2764f8a5e98e574d4e45d6d782048d1306df Mon Sep 17 00:00:00 2001 From: TUARAN <729922845@qq.com> Date: Tue, 21 Jul 2026 23:43:56 +0800 Subject: [PATCH] fix(tailscale): retry status json after serve startup (#91553) * fix(tailscale): restrict status retries * test(tailscale): cover retry eligibility * refactor(tailscale): add status retry helper * refactor(tailscale): split status retry helper * fix(tailscale): scope status retry to serve startup Co-authored-by: TUARAN <729922845@qq.com> * fix(tailscale): recognize app service startup errors Co-authored-by: TUARAN <729922845@qq.com> --------- Co-authored-by: Peter Steinberger --- src/gateway/server-tailscale.test.ts | 11 +++- src/gateway/server-tailscale.ts | 7 ++- src/infra/tailscale.test.ts | 62 ++++++++++++++++++++ src/infra/tailscale.ts | 88 ++++++++++++++++++++++------ 4 files changed, 147 insertions(+), 21 deletions(-) diff --git a/src/gateway/server-tailscale.test.ts b/src/gateway/server-tailscale.test.ts index be0fefae3cfe..4a065d366592 100644 --- a/src/gateway/server-tailscale.test.ts +++ b/src/gateway/server-tailscale.test.ts @@ -8,6 +8,7 @@ const mocks = vi.hoisted(() => ({ enableTailscaleFunnel: vi.fn(async (_port: number) => undefined), disableTailscaleFunnel: vi.fn(async () => undefined), getTailnetHostname: vi.fn<() => Promise>(async () => null), + getTailnetHostnameAfterServe: vi.fn<() => Promise>(async () => null), hasTailscaleFunnelRouteForPort: vi.fn(async (_port: number) => false), })); @@ -17,6 +18,7 @@ vi.mock("../infra/tailscale.js", () => ({ enableTailscaleFunnel: mocks.enableTailscaleFunnel, disableTailscaleFunnel: mocks.disableTailscaleFunnel, getTailnetHostname: mocks.getTailnetHostname, + getTailnetHostnameAfterServe: mocks.getTailnetHostnameAfterServe, hasTailscaleFunnelRouteForPort: mocks.hasTailscaleFunnelRouteForPort, })); @@ -41,6 +43,7 @@ afterEach(() => { mocks.enableTailscaleFunnel.mockResolvedValue(undefined); mocks.disableTailscaleFunnel.mockResolvedValue(undefined); mocks.getTailnetHostname.mockResolvedValue(null); + mocks.getTailnetHostnameAfterServe.mockResolvedValue(null); mocks.hasTailscaleFunnelRouteForPort.mockResolvedValue(false); }); @@ -55,6 +58,8 @@ describe("startGatewayTailscaleExposure preserveFunnel", () => { }); expect(mocks.enableTailscaleServe).toHaveBeenCalledWith(18789); + expect(mocks.getTailnetHostnameAfterServe).toHaveBeenCalledOnce(); + expect(mocks.getTailnetHostname).not.toHaveBeenCalled(); expect(mocks.hasTailscaleFunnelRouteForPort).not.toHaveBeenCalled(); }); @@ -113,7 +118,7 @@ describe("startGatewayTailscaleExposure preserveFunnel", () => { it("passes serviceName through to Tailscale Serve setup and cleanup", async () => { const logTailscale = createLogger(); - mocks.getTailnetHostname.mockResolvedValue("node.tailnet.ts.net"); + mocks.getTailnetHostnameAfterServe.mockResolvedValue("node.tailnet.ts.net"); const cleanup = await startGatewayTailscaleExposure({ tailscaleMode: "serve", @@ -158,7 +163,7 @@ describe("startGatewayTailscaleExposure preserveFunnel", () => { }); it("prepares one tailnet-only Serve origin for the Gateway lifecycle", async () => { - mocks.getTailnetHostname.mockResolvedValue("node.tailnet.ts.net"); + mocks.getTailnetHostnameAfterServe.mockResolvedValue("node.tailnet.ts.net"); const cleanup = await startGatewayTailscaleExposure({ tailscaleMode: "serve", @@ -202,7 +207,7 @@ describe("startGatewayTailscaleExposure preserveFunnel", () => { ["omits the DNS suffix", "node"], ])("does not derive a Service URL when Tailscale %s", async (_name, hostname) => { const logTailscale = createLogger(); - mocks.getTailnetHostname.mockResolvedValue(hostname); + mocks.getTailnetHostnameAfterServe.mockResolvedValue(hostname); await startGatewayTailscaleExposure({ tailscaleMode: "serve", diff --git a/src/gateway/server-tailscale.ts b/src/gateway/server-tailscale.ts index 0bdeff1311b2..fa901249936c 100644 --- a/src/gateway/server-tailscale.ts +++ b/src/gateway/server-tailscale.ts @@ -7,6 +7,7 @@ import { enableTailscaleFunnel, enableTailscaleServe, getTailnetHostname, + getTailnetHostnameAfterServe, hasTailscaleFunnelRouteForPort, } from "../infra/tailscale.js"; import { resolveTailscalePublishedHost } from "../shared/tailscale-status.js"; @@ -55,7 +56,11 @@ export async function startGatewayTailscaleExposure(params: { } else { await enableTailscaleFunnel(params.port); } - const host = await getTailnetHostname().catch(() => null); + const host = await ( + params.tailscaleMode === "serve" && !preservedFunnel + ? getTailnetHostnameAfterServe() + : getTailnetHostname() + ).catch(() => null); if (host) { const uiPath = params.controlUiBasePath ? `${params.controlUiBasePath}/` : "/"; const publicHost = resolveTailscalePublishedHost({ diff --git a/src/infra/tailscale.test.ts b/src/infra/tailscale.test.ts index 9e0b06a5639c..1509063d35b0 100644 --- a/src/infra/tailscale.test.ts +++ b/src/infra/tailscale.test.ts @@ -5,6 +5,7 @@ import * as tailscale from "./tailscale.js"; const { getTailnetHostname, + getTailnetHostnameAfterServe, readTailscaleWhoisIdentity, enableTailscaleServe, disableTailscaleServe, @@ -75,6 +76,67 @@ describe("tailscale helpers", () => { expect(host).toBe("noisy.tailnet.ts.net"); }); + it.each([ + [new Error("Failed to connect to local Tailscale daemon; not running?")], + [new Error("failed to connect to local Tailscale service; is Tailscale running?")], + [Object.assign(new Error("Command timed out"), { timedOut: true, signal: "SIGTERM" })], + ])("retries post-Serve status after a transient failure", async (failure) => { + vi.useFakeTimers(); + const exec = vi + .fn() + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce({ + stdout: JSON.stringify({ + Self: { DNSName: "retry.tailnet.ts.net.", TailscaleIPs: ["100.7.7.7"] }, + }), + }); + + const hostPromise = getTailnetHostnameAfterServe(exec); + await vi.runAllTimersAsync(); + const host = await hostPromise; + + expect(host).toBe("retry.tailnet.ts.net"); + expect(exec).toHaveBeenCalledTimes(2); + expectExecCall(exec, 1, tailscaleBin, ["status", "--json"], { + timeoutMs: 5000, + maxBuffer: 400_000, + logOutput: false, + }); + expectExecCall(exec, 2, tailscaleBin, ["status", "--json"], { + timeoutMs: 5000, + maxBuffer: 400_000, + logOutput: false, + }); + }); + + it.each([ + ["missing binary", new Error("spawn tailscale ENOENT")], + ["permission failure", new Error("permission denied")], + ])("does not retry post-Serve status after a permanent %s", async (_name, failure) => { + const exec = vi.fn().mockRejectedValue(failure); + + await expect(getTailnetHostnameAfterServe(exec)).rejects.toThrow(failure.message); + + expect(exec).toHaveBeenCalledTimes(1); + }); + + it("does not retry malformed post-Serve status JSON", async () => { + const exec = vi.fn().mockResolvedValue({ stdout: "{not json}" }); + + await expect(getTailnetHostnameAfterServe(exec)).rejects.toThrow(SyntaxError); + + expect(exec).toHaveBeenCalledTimes(1); + }); + + it("keeps ordinary hostname lookup single-attempt", async () => { + const failure = new Error("Failed to connect to local Tailscale daemon; not running?"); + const exec = vi.fn().mockRejectedValue(failure); + + await expect(getTailnetHostname(exec, tailscaleBin)).rejects.toThrow(failure.message); + + expect(exec).toHaveBeenCalledTimes(1); + }); + it("parses noisy JSON output from tailscale whois", async () => { const exec = vi.fn().mockResolvedValue({ stdout: diff --git a/src/infra/tailscale.ts b/src/infra/tailscale.ts index 994e0ff7952d..58bcf6cd7f4a 100644 --- a/src/infra/tailscale.ts +++ b/src/infra/tailscale.ts @@ -12,6 +12,10 @@ import { import { logVerbose } from "../globals.js"; import { runExec } from "../process/exec.js"; import { toErrorObject } from "./errors.js"; +import { retryAsync } from "./retry.js"; + +const TAILSCALE_STATUS_ATTEMPTS = 3; +const TAILSCALE_STATUS_RETRY_DELAY_MS = 500; function parsePossiblyNoisyJsonObject(stdout: string): Record { const trimmed = stdout.trim(); @@ -23,6 +27,45 @@ function parsePossiblyNoisyJsonObject(stdout: string): Record { return JSON.parse(trimmed) as Record; } +function tailnetHostnameFromStatus(parsed: Record): string { + const self = + typeof parsed.Self === "object" && parsed.Self !== null + ? (parsed.Self as Record) + : undefined; + const dns = typeof self?.DNSName === "string" ? self.DNSName : undefined; + const ips = Array.isArray(self?.TailscaleIPs) + ? ((parsed.Self as { TailscaleIPs?: string[] }).TailscaleIPs ?? []) + : []; + if (dns && dns.length > 0) { + return dns.replace(/\.$/, ""); + } + const [firstIp] = ips; + if (firstIp !== undefined) { + return firstIp; + } + throw new Error("Could not determine Tailscale DNS or IP"); +} + +function isTransientTailscaleStatusError(error: unknown): boolean { + const record = readRecord(error); + const detail = [ + error instanceof Error ? error.message : undefined, + typeof record?.stderr === "string" ? record.stderr : undefined, + typeof record?.stdout === "string" ? record.stdout : undefined, + ] + .filter((value): value is string => Boolean(value)) + .join("\n") + .toLowerCase(); + + return ( + record?.timedOut === true || + detail.includes("failed to connect to local tailscale daemon") || + detail.includes("failed to connect to local tailscale service") || + detail.includes("connection refused") || + detail.includes("503 service unavailable") + ); +} + /** * Locate Tailscale binary using multiple strategies: * 1. PATH lookup (via which command) @@ -121,23 +164,7 @@ export async function getTailnetHostname(exec: typeof runExec = runExec, detecte timeoutMs: 5000, maxBuffer: 400_000, }); - const parsed = stdout ? parsePossiblyNoisyJsonObject(stdout) : {}; - const self = - typeof parsed.Self === "object" && parsed.Self !== null - ? (parsed.Self as Record) - : undefined; - const dns = typeof self?.DNSName === "string" ? self.DNSName : undefined; - const ips = Array.isArray(self?.TailscaleIPs) - ? ((parsed.Self as { TailscaleIPs?: string[] }).TailscaleIPs ?? []) - : []; - if (dns && dns.length > 0) { - return dns.replace(/\.$/, ""); - } - const [firstIp] = ips; - if (firstIp !== undefined) { - return firstIp; - } - throw new Error("Could not determine Tailscale DNS or IP"); + return tailnetHostnameFromStatus(stdout ? parsePossiblyNoisyJsonObject(stdout) : {}); } catch (err) { lastError = err; } @@ -179,6 +206,33 @@ async function getTailscaleBinary(): Promise { return cachedTailscaleBinary ?? "tailscale"; } +/** Resolve the hostname after Serve startup, while the local daemon may still be settling. */ +export async function getTailnetHostnameAfterServe( + exec: typeof runExec = runExec, +): Promise { + const candidate = await getTailscaleBinary(); + const parsed = await retryAsync( + async () => { + const { stdout } = await exec(candidate, ["status", "--json"], { + timeoutMs: 5000, + maxBuffer: 400_000, + // Hostname discovery is best-effort. Avoid scary command-failure logs while the + // local daemon settles after Serve configuration. + logOutput: false, + }); + return stdout ? parsePossiblyNoisyJsonObject(stdout) : {}; + }, + { + attempts: TAILSCALE_STATUS_ATTEMPTS, + minDelayMs: TAILSCALE_STATUS_RETRY_DELAY_MS, + maxDelayMs: TAILSCALE_STATUS_RETRY_DELAY_MS, + jitter: 0, + shouldRetry: isTransientTailscaleStatusError, + }, + ); + return tailnetHostnameFromStatus(parsed); +} + type ExecErrorDetails = { stdout?: unknown; stderr?: unknown;