fix(doctor): report degraded gateway secret owners (#126998)

This commit is contained in:
Peter Steinberger
2026-08-20 21:43:45 -07:00
committed by GitHub
parent 9bc772ba50
commit 2e1b882845
8 changed files with 531 additions and 127 deletions
+5 -8
View File
@@ -18,9 +18,9 @@ import type {
import { collectChannelStatusIssues } from "../infra/channels-status-issues.js";
import { formatErrorMessage } from "../infra/errors.js";
import type { RuntimeEnv } from "../runtime.js";
import { redactSecretDegradationReason } from "../secrets/runtime-degraded-state.js";
import type { StatusSummary } from "../status/types.js";
import { VERSION } from "../version.js";
import { projectDoctorSecretRuntimeDegradations } from "./doctor-secret-runtime-degradation.js";
import {
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE,
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE,
@@ -94,14 +94,11 @@ export async function checkGatewayHealth(params: {
});
healthOk = true;
noteCliGatewayVersionSkew(status);
if (status.degradedSecretOwners && status.degradedSecretOwners.length > 0) {
const secretDegradations = projectDoctorSecretRuntimeDegradations(status);
if (secretDegradations.length > 0) {
note(
status.degradedSecretOwners
.map(
(owner) =>
`- ${owner.degradationState ?? "cold"} ${owner.ownerKind}:${owner.ownerId} (${owner.paths.join(", ")}): ${redactSecretDegradationReason(owner.reason)}` +
"\n Retry: openclaw secrets reload",
)
secretDegradations
.map((owner) => `- ${owner.message}\n Retry: ${owner.retryHint}`)
.join("\n"),
"Secret runtime degradation",
);
+111 -10
View File
@@ -6,6 +6,8 @@ import path from "node:path";
import { DatabaseSync } from "node:sqlite";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import * as bundledHealthChecks from "../flows/bundled-health-checks.js";
import { CORE_HEALTH_CHECKS } from "../flows/doctor-core-checks.js";
import { clearHealthChecksForTest, registerHealthCheck } from "../flows/health-check-registry.js";
import { clearLoadInstalledPluginIndexInstallRecordsCache } from "../plugins/installed-plugin-index-record-cache.js";
import { writePersistedInstalledPluginIndexInstallRecords } from "../plugins/installed-plugin-index-records.js";
@@ -17,6 +19,8 @@ const mocks = vi.hoisted(() => ({
actualOpenNodeSqliteDatabase: vi.fn(),
actualPrepareSqliteReadOnlyLocationSync: vi.fn(),
actualReadConfigFileSnapshot: vi.fn(),
buildGatewayProbeConnectionDetails: vi.fn(),
callGateway: vi.fn(),
openNodeSqliteDatabase: vi.fn(),
prepareSqliteReadOnlyLocationSync: vi.fn(),
readConfigFileSnapshot: vi.fn(),
@@ -66,7 +70,14 @@ vi.mock("../flows/doctor-health-contributions.js", async (importOriginal) => {
mocks.resolveDoctorContributionHealthChecks(...args),
};
});
vi.mock("../gateway/call.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../gateway/call.js")>();
return {
...actual,
buildGatewayProbeConnectionDetails: mocks.buildGatewayProbeConnectionDetails,
callGateway: mocks.callGateway,
};
});
const runtime = {
log: vi.fn(),
error: vi.fn(),
@@ -79,6 +90,10 @@ describe("runDoctorLintCli", () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.readConfigFileSnapshot.mockReset();
mocks.buildGatewayProbeConnectionDetails.mockReset().mockResolvedValue({
url: "ws://127.0.0.1:18789",
});
mocks.callGateway.mockReset().mockResolvedValue({ degradedSecretOwners: [] });
mocks.openNodeSqliteDatabase.mockImplementation((...args: unknown[]) =>
mocks.actualOpenNodeSqliteDatabase(...args),
);
@@ -113,29 +128,115 @@ describe("runDoctorLintCli", () => {
}
});
it("reports the visible finding count in human output", async () => {
it.each([
{ label: "--only JSON", selection: "only", json: true },
{ label: "--only human text", selection: "only", json: false },
{ label: "--all JSON", selection: "all", json: true },
{ label: "--all human text", selection: "all", json: false },
{ label: "default JSON", selection: "default", json: true },
{ label: "default human text", selection: "default", json: false },
] as const)("keeps Gateway-owned secret degradation observable through $label", async (entry) => {
const gatewayCheck = CORE_HEALTH_CHECKS.find(
(check) => check.id === "core/doctor/gateway-health",
);
expect(gatewayCheck).toBeDefined();
const previousResolveChecks =
mocks.resolveDoctorContributionHealthChecks.getMockImplementation();
mocks.resolveDoctorContributionHealthChecks.mockResolvedValue([gatewayCheck]);
const registerChecks = vi
.spyOn(bundledHealthChecks, "registerBundledHealthChecks")
.mockImplementation(() => {});
const resolveStateMode = vi
.spyOn(bundledHealthChecks, "resolveBundledHealthCheckPluginStateMode")
.mockReturnValue("direct");
mocks.readConfigFileSnapshot.mockResolvedValue({
exists: true,
valid: true,
config: {},
config: {
gateway: {
mode: "local",
auth: { mode: "token", token: "SYNTHETIC_GATEWAY_SECRET" },
},
},
path: "/tmp/openclaw.json",
});
mocks.callGateway.mockResolvedValue({
degradedSecretOwners: [
{
ownerKind: "account",
ownerId: "discord:ops",
state: "unavailable",
paths: ["channels.discord.accounts.ops.token"],
reason:
"secret reference was not found (env:default:PRIVATE_REF_ID=SYNTHETIC_OWNER_SECRET)",
},
],
});
const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
const originalIsTTY = process.stdout.isTTY;
Object.defineProperty(process.stdout, "isTTY", { configurable: true, value: true });
Object.defineProperty(process.stdout, "isTTY", { configurable: true, value: !entry.json });
try {
const exitCode = await runDoctorLintCli(runtime, {
severityMin: "error",
onlyIds: ["core/doctor/final-config-validation"],
...(entry.json ? { json: true } : {}),
...(entry.selection === "only"
? { onlyIds: ["core/doctor/gateway-health"] }
: entry.selection === "all"
? { includeAllChecks: true }
: {}),
});
const output = stdout.mock.calls.map(([line]) => String(line)).join("");
expect(exitCode).toBe(0);
expect(String(stdout.mock.calls[0]?.[0])).toContain("0 finding(s)");
expect(String(stdout.mock.calls[1]?.[0])).toBe(" no findings\n");
if (entry.selection === "default") {
expect(exitCode).toBe(0);
if (entry.json) {
expect(JSON.parse(output)).toMatchObject({
checksRun: 0,
checksSkipped: 1,
findings: [],
});
} else {
expect(output).toContain("0 finding(s)");
expect(output).toContain(" no findings\n");
}
expect(mocks.buildGatewayProbeConnectionDetails).not.toHaveBeenCalled();
expect(mocks.callGateway).not.toHaveBeenCalled();
return;
}
expect(exitCode).toBe(1);
expect(output).toContain("core/doctor/gateway-health");
expect(output).toContain("cold account:discord:ops");
expect(output).toContain("channels.discord.accounts.ops.token");
expect(output).toContain("openclaw secrets reload");
expect(output).not.toContain("SYNTHETIC_GATEWAY_SECRET");
expect(output).not.toContain("SYNTHETIC_OWNER_SECRET");
expect(output).not.toContain("PRIVATE_REF_ID");
expect(mocks.callGateway).toHaveBeenCalledOnce();
if (entry.json) {
expect(JSON.parse(output)).toMatchObject({
ok: false,
checksRun: 1,
findings: [
{
checkId: "core/doctor/gateway-health",
severity: "warning",
path: "channels.discord.accounts.ops.token",
target: "account:discord:ops",
},
],
});
} else {
expect(output).toContain("[warning] core/doctor/gateway-health");
}
} finally {
Object.defineProperty(process.stdout, "isTTY", { configurable: true, value: originalIsTTY });
stdout.mockRestore();
registerChecks.mockRestore();
resolveStateMode.mockRestore();
if (previousResolveChecks) {
mocks.resolveDoctorContributionHealthChecks.mockImplementation(previousResolveChecks);
}
}
});
@@ -0,0 +1,41 @@
import { redactSensitiveUrlLikeString } from "@openclaw/net-policy/redact-sensitive-url";
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
import { sanitizeTerminalText } from "../../packages/terminal-core/src/safe-text.js";
import {
redactSecretDegradationReason,
SECRET_DEGRADATION_RETRY_HINT,
} from "../secrets/runtime-degraded-state.js";
import type { StatusSummary } from "../status/types.js";
const DOCTOR_SECRET_OWNER_ID_MAX_CHARS = 96;
const DOCTOR_SECRET_OWNER_PATH_MAX_CHARS = 120;
const DOCTOR_SECRET_OWNER_VISIBLE_PATHS = 3;
function safeDoctorSecretOwnerText(value: string, maxChars: number): string {
const safe = sanitizeTerminalText(redactSensitiveUrlLikeString(value));
return safe.length <= maxChars ? safe : `${truncateUtf16Safe(safe, maxChars - 1)}`;
}
/** Projects Gateway-owned secret degradation into the shared bounded Doctor display shape. */
export function projectDoctorSecretRuntimeDegradations(
status: Pick<StatusSummary, "degradedSecretOwners">,
) {
return (status.degradedSecretOwners ?? []).map((owner) => {
const ownerId = safeDoctorSecretOwnerText(owner.ownerId, DOCTOR_SECRET_OWNER_ID_MAX_CHARS);
const target = `${owner.ownerKind}:${ownerId}`;
const visiblePaths = owner.paths
.slice(0, DOCTOR_SECRET_OWNER_VISIBLE_PATHS)
.map((configPath) =>
safeDoctorSecretOwnerText(configPath, DOCTOR_SECRET_OWNER_PATH_MAX_CHARS),
);
const omittedPaths = owner.paths.length - visiblePaths.length;
const paths =
visiblePaths.join(", ") + (omittedPaths > 0 ? ` (+${omittedPaths} paths omitted)` : "");
return {
message: `${owner.degradationState ?? "cold"} ${target} (${paths || "no affected paths reported"}): ${redactSecretDegradationReason(owner.reason)}`,
path: visiblePaths[0] ?? "gateway",
target,
retryHint: SECRET_DEGRADATION_RETRY_HINT,
};
});
}
+199 -50
View File
@@ -1,7 +1,9 @@
// Doctor runtime check tests cover runtime-backed doctor checks.
import { beforeEach, describe, expect, it, vi } from "vitest";
import { GatewayClientRequestError } from "../../packages/gateway-client/src/index.js";
import type { AnyAgentTool } from "../agents/tools/common.js";
import { GATEWAY_HEALTH_RATE_LIMITED_MESSAGE } from "../commands/gateway-health-auth-diagnostic.js";
import { GatewaySecretRefUnavailableError } from "../gateway/credentials.js";
import { setPluginToolMeta } from "../plugins/tools.js";
const mocks = vi.hoisted(() => ({
@@ -11,7 +13,8 @@ const mocks = vi.hoisted(() => ({
loadModelCatalog: vi.fn(async (): Promise<Array<Record<string, unknown>>> => []),
normalizeProviderToolSchemasWithPlugin: vi.fn(),
buildGatewayProbeConnectionDetails: vi.fn(),
probeGatewayStatus: vi.fn(),
callGateway: vi.fn(),
isGatewayCredentialsRequiredError: vi.fn(),
readGatewayServiceState: vi.fn(),
resolveGatewayService: vi.fn(() => ({ label: "openclaw-gateway" })),
resolvePluginProvidersCore: vi.fn((): Array<Record<string, unknown>> => []),
@@ -46,10 +49,8 @@ vi.mock("../agents/agent-tools.js", () => ({
vi.mock("../gateway/call.js", () => ({
buildGatewayProbeConnectionDetails: mocks.buildGatewayProbeConnectionDetails,
}));
vi.mock("../cli/daemon-cli/probe.js", () => ({
probeGatewayStatus: mocks.probeGatewayStatus,
callGateway: mocks.callGateway,
isGatewayCredentialsRequiredError: mocks.isGatewayCredentialsRequiredError,
}));
vi.mock("../daemon/service.js", () => ({
@@ -105,13 +106,6 @@ describe("doctor runtime tool schema checks", () => {
mocks.normalizeProviderToolSchemasWithPlugin
.mockReset()
.mockImplementation(({ context }) => context.tools);
mocks.buildGatewayProbeConnectionDetails.mockReset().mockResolvedValue({
url: "http://127.0.0.1:5829",
});
mocks.probeGatewayStatus.mockReset().mockResolvedValue({
ok: true,
server: { version: "2026.6.26" },
});
mocks.readGatewayServiceState.mockReset().mockResolvedValue({
installed: true,
loadState: { status: "loaded" },
@@ -567,10 +561,8 @@ describe("doctor gateway runtime checks", () => {
mocks.buildGatewayProbeConnectionDetails.mockReset().mockResolvedValue({
url: "http://127.0.0.1:5829",
});
mocks.probeGatewayStatus.mockReset().mockResolvedValue({
ok: true,
server: { version: "2026.6.26" },
});
mocks.callGateway.mockReset().mockResolvedValue({ degradedSecretOwners: [] });
mocks.isGatewayCredentialsRequiredError.mockReset().mockReturnValue(false);
mocks.readGatewayServiceState.mockReset().mockResolvedValue({
installed: true,
loadState: { status: "loaded" },
@@ -582,52 +574,209 @@ describe("doctor gateway runtime checks", () => {
mocks.resolveGatewayService.mockReset().mockReturnValue({ label: "openclaw-gateway" });
});
it("reports unreachable gateway health probes", async () => {
mocks.probeGatewayStatus.mockResolvedValueOnce({
ok: false,
error: "connect ECONNREFUSED 127.0.0.1:5829",
it("projects every degraded SecretRef owner from exactly one authenticated read-only status RPC", async () => {
const cfg = { gateway: { mode: "local" as const } };
const privateToken = "SYNTHETIC_PRIVATE_URL_TOKEN";
mocks.buildGatewayProbeConnectionDetails.mockResolvedValueOnce({
url: "wss://127.0.0.1:5829",
tlsFingerprint: "sha256:test-doctor-fingerprint",
preauthHandshakeTimeoutMs: 1200,
});
mocks.callGateway.mockResolvedValueOnce({
degradedSecretOwners: [
{
ownerKind: "account",
ownerId: "discord:ops",
state: "unavailable",
paths: ["channels.discord.accounts.ops.token"],
reason: "secret reference was not found (env:default:PRIVATE_REF_ID)",
},
{
ownerKind: "capability",
ownerId: "tts",
state: "unavailable",
degradationState: "stale",
paths: ["tts.providers.elevenlabs.apiKey", "tts.providers.elevenlabs.voiceId"],
reason: "secret provider policy denied resolution",
},
{
ownerKind: "provider",
ownerId: `vault\u001b]52;c;attack\u0007:https://user:${privateToken}@secret.test/${"a".repeat(500)}`,
state: "unavailable",
paths: Array.from(
{ length: 12 },
(_, index) =>
`providers.example.${index}.https://secret.test/value?token=${privateToken}\n${"z".repeat(400)}`,
),
reason: `secret provider failed: ${privateToken}\nref PRIVATE_REF_ID`,
},
],
degradedPlugins: [{ pluginId: "not-this-check" }],
});
await expect(
collectGatewayHealthFindings({ cfg: { gateway: { mode: "local" } } }),
).resolves.toContainEqual({
checkId: "core/doctor/gateway-health",
severity: "warning",
message: "Gateway is not reachable: connect ECONNREFUSED 127.0.0.1:5829",
path: "gateway.mode",
target: "http://127.0.0.1:5829",
fixHint:
"Start the Gateway service or run `openclaw doctor --fix` for service repair prompts.",
const findings = await collectGatewayHealthFindings({
cfg,
configPath: "/tmp/selected-openclaw.json",
});
expect(mocks.callGateway).toHaveBeenCalledExactlyOnceWith({
method: "status",
params: { includeChannelSummary: false },
timeoutMs: 3000,
sharedStateMode: "read-only",
config: cfg,
configPath: "/tmp/selected-openclaw.json",
tlsFingerprint: "sha256:test-doctor-fingerprint",
preauthHandshakeTimeoutMs: 1200,
});
expect(findings).toEqual([
expect.objectContaining({
checkId: "core/doctor/gateway-health",
severity: "warning",
message: expect.stringContaining("cold account:discord:ops"),
path: "channels.discord.accounts.ops.token",
target: "account:discord:ops",
fixHint: expect.stringContaining("openclaw secrets reload"),
}),
expect.objectContaining({
checkId: "core/doctor/gateway-health",
severity: "warning",
message: expect.stringContaining("stale capability:tts"),
path: "tts.providers.elevenlabs.apiKey",
target: "capability:tts",
fixHint: expect.stringContaining("openclaw secrets reload"),
}),
expect.objectContaining({
checkId: "core/doctor/gateway-health",
severity: "warning",
message: expect.stringContaining("provider:vault"),
path: expect.stringContaining("providers.example.0"),
target: expect.stringContaining("provider:vault"),
}),
]);
expect(findings[1]?.message).toContain("tts.providers.elevenlabs.voiceId");
const finding = findings[2];
const rendered = JSON.stringify(findings);
expect(finding?.message).toContain("omitted");
expect(finding?.message).toContain("secret resolution failed");
expect(finding?.message.length).toBeLessThanOrEqual(700);
expect(finding?.target?.length).toBeLessThanOrEqual(150);
expect(finding?.path?.length).toBeLessThanOrEqual(180);
expect(rendered).not.toContain(privateToken);
expect(rendered).not.toContain("PRIVATE_REF_ID");
expect(rendered).not.toContain("not-this-check");
expect(rendered).not.toContain("\u001b");
expect(rendered).not.toContain("\u0007");
});
it("reports temporary Gateway authentication lockouts with wait-and-retry guidance", async () => {
mocks.probeGatewayStatus.mockResolvedValueOnce({
ok: false,
error: "connect failed",
connectFailure: { kind: "rate-limited", detailCode: "AUTH_RATE_LIMITED" },
});
it.each([
{
label: "missing Gateway authentication",
error: new Error("auth token SYNTHETIC_PRIVATE_TOKEN\nref PRIVATE_REF_ID"),
credentialsRequired: true,
message:
"Gateway status could not be inspected because this CLI has no usable token/password or paired device token for read-scope RPCs.",
},
{
label: "an unavailable Gateway authentication SecretRef",
error: new GatewaySecretRefUnavailableError("gateway.auth.token"),
credentialsRequired: false,
message:
"Gateway status could not be inspected because this CLI has no usable token/password or paired device token for read-scope RPCs.",
},
{
label: "temporary Gateway authentication rate limiting",
error: new GatewayClientRequestError({
code: "INVALID_REQUEST",
message: "unauthorized: too many failed authentication attempts (retry later)",
details: { code: "AUTH_RATE_LIMITED", authReason: "rate_limited" },
retryable: true,
}),
credentialsRequired: false,
message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
},
{
label: "an unreachable Gateway with terminal control characters",
error: new Error("connect ECONNREFUSED 127.0.0.1:5829\u001b]52;c;attack\u0007\u009b"),
credentialsRequired: false,
message: "Gateway status could not be inspected: connect ECONNREFUSED 127.0.0.1:5829",
},
])("reports $label from exactly one sanitized status attempt", async (entry) => {
mocks.callGateway.mockRejectedValueOnce(entry.error);
mocks.isGatewayCredentialsRequiredError.mockReturnValueOnce(entry.credentialsRequired);
const findings = await collectGatewayHealthFindings({ cfg: { gateway: { mode: "local" } } });
expect(findings).toEqual([
expect.objectContaining({
checkId: "core/doctor/gateway-health",
severity: "warning",
message: entry.message,
path: "gateway.mode",
target: "http://127.0.0.1:5829",
}),
]);
expect(JSON.stringify(findings)).not.toContain("SYNTHETIC_PRIVATE_TOKEN");
expect(JSON.stringify(findings)).not.toContain("PRIVATE_REF_ID");
expect(mocks.callGateway).toHaveBeenCalledOnce();
});
it("reports preparation failures without exposing URL credentials or control characters", async () => {
mocks.buildGatewayProbeConnectionDetails.mockRejectedValueOnce(
new Error(
`invalid wss://user:${"SYNTHETIC_PRIVATE_TOKEN".repeat(20)}@gateway.test/rpc\nmore`,
),
);
const findings = await collectGatewayHealthFindings({ cfg: {} });
expect(findings).toEqual([
expect.objectContaining({
severity: "warning",
message: expect.stringContaining("Gateway health inspection could not be prepared"),
path: "gateway",
}),
]);
expect(JSON.stringify(findings)).not.toContain("SYNTHETIC_PRIVATE_TOKEN");
expect(mocks.callGateway).not.toHaveBeenCalled();
});
it("prepares the target but skips the RPC for active exec credentials unless execution is allowed", async () => {
const cfg = {
gateway: {
mode: "local" as const,
auth: {
mode: "token" as const,
token: { source: "exec" as const, provider: "vault", id: "PRIVATE_REF_ID" },
},
},
};
const findings = await collectGatewayHealthFindings({ cfg, env: {} });
expect(findings).toEqual([
expect.objectContaining({
checkId: "core/doctor/gateway-health",
severity: "warning",
message: expect.stringContaining("intentionally skipped"),
fixHint: expect.stringContaining("--allow-exec"),
}),
]);
expect(JSON.stringify(findings)).not.toContain("PRIVATE_REF_ID");
expect(mocks.buildGatewayProbeConnectionDetails).toHaveBeenCalledOnce();
expect(mocks.callGateway).not.toHaveBeenCalled();
await expect(
collectGatewayHealthFindings({ cfg: { gateway: { mode: "local" } } }),
).resolves.toContainEqual({
checkId: "core/doctor/gateway-health",
severity: "warning",
message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
path: "gateway.mode",
target: "http://127.0.0.1:5829",
fixHint: "Wait for the temporary authentication lockout to expire, then rerun doctor.",
});
collectGatewayHealthFindings({ cfg, env: {}, allowExecSecretRefs: true }),
).resolves.toEqual([]);
expect(mocks.callGateway).toHaveBeenCalledOnce();
});
it("redacts sensitive remote gateway URLs from health finding targets", async () => {
mocks.buildGatewayProbeConnectionDetails.mockResolvedValueOnce({
url: "wss://user:pass@gateway.example.test/rpc?token=secret&safe=value",
});
mocks.probeGatewayStatus.mockResolvedValueOnce({
ok: false,
error: "remote gateway did not answer",
});
mocks.callGateway.mockRejectedValueOnce(new Error("remote gateway did not answer"));
const findings = await collectGatewayHealthFindings({
cfg: { gateway: { mode: "remote", remote: { url: "wss://gateway.example.test/rpc" } } },
@@ -636,7 +785,7 @@ describe("doctor gateway runtime checks", () => {
expect(findings).toContainEqual({
checkId: "core/doctor/gateway-health",
severity: "warning",
message: "Gateway is not reachable: remote gateway did not answer",
message: "Gateway status could not be inspected: remote gateway did not answer",
path: "gateway.remote.url",
target: "wss://***:***@gateway.example.test/rpc?token=***&safe=value",
fixHint: "Verify the remote Gateway URL, network path, TLS settings, and credentials.",
+89 -57
View File
@@ -1,5 +1,6 @@
// Doctor runtime checks inspect tool names, browser residue, and runtime state.
import { redactSensitiveUrlLikeString } from "@openclaw/net-policy/redact-sensitive-url";
import { sanitizeTerminalText } from "../../packages/terminal-core/src/safe-text.js";
import { TOOL_NAME_SEPARATOR } from "../agents/agent-bundle-mcp-names.js";
import {
type McpToolCatalogDiagnostic,
@@ -29,12 +30,11 @@ import {
type RuntimeToolSchemaDiagnostic,
} from "../agents/tool-schema-projection.js";
import type { AnyAgentTool } from "../agents/tools/common.js";
import { probeGatewayStatus } from "../cli/daemon-cli/probe.js";
import { projectDoctorSecretRuntimeDegradations } from "../commands/doctor-secret-runtime-degradation.js";
import { collectUnavailableAgentSkills } from "../commands/doctor-skills-core.js";
import {
GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
gatewayProbeResultSawGateway,
gatewayProbeResultWasRateLimited,
gatewayConnectErrorWasRateLimited,
} from "../commands/gateway-health-auth-diagnostic.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import {
@@ -42,7 +42,12 @@ import {
type GatewayServiceRuntime,
} from "../daemon/service-runtime.js";
import { resolveGatewayService, readGatewayServiceState } from "../daemon/service.js";
import { buildGatewayProbeConnectionDetails } from "../gateway/call.js";
import {
buildGatewayProbeConnectionDetails,
callGateway,
isGatewayCredentialsRequiredError,
} from "../gateway/call.js";
import { isGatewaySecretRefUnavailableError } from "../gateway/credentials.js";
import { formatErrorMessage } from "../infra/errors.js";
import {
formatLocalAudioSelection,
@@ -54,14 +59,18 @@ import { getPluginToolMeta, setPluginToolMeta } from "../plugins/tools.js";
import type { ProviderCatalogOrder, ProviderPlugin } from "../plugins/types.js";
import { normalizeAgentId } from "../routing/session-key.js";
import { buildWorkspaceSkillStatus } from "../skills/discovery/status.js";
import type { StatusSummary } from "../status/types.js";
import { scrubDoctorErrorMessage } from "./doctor-error-message.js";
import { hasActiveGatewayExecCredential } from "./doctor-gateway-exec-credential.js";
import type { HealthCheckContext, HealthFinding } from "./health-checks.js";
type BundleMcpToolRuntime = Awaited<ReturnType<typeof createBundleMcpToolRuntime>>;
const PROVIDER_CATALOG_ORDERS = ["simple", "profile", "paired", "late"] as const;
const PROVIDER_CATALOG_ORDER_SET = new Set<ProviderCatalogOrder>(PROVIDER_CATALOG_ORDERS);
function formatGatewayHealthTarget(url: string): string {
return redactSensitiveUrlLikeString(url);
function formatGatewayHealthDiagnostic(value: unknown): string {
const raw = value instanceof Error ? value.message : String(value);
return scrubDoctorErrorMessage(sanitizeTerminalText(redactSensitiveUrlLikeString(raw)));
}
export function detectUnavailableSkills(cfg: OpenClawConfig, workspaceDir: string) {
@@ -105,64 +114,87 @@ export async function collectLocalAudioAccelerationFindings(): Promise<readonly
}
export async function collectGatewayHealthFindings(
ctx: Pick<HealthCheckContext, "cfg" | "configPath">,
ctx: Pick<HealthCheckContext, "cfg" | "configPath" | "env" | "allowExecSecretRefs">,
): Promise<readonly HealthFinding[]> {
let probeDetails: Awaited<ReturnType<typeof buildGatewayProbeConnectionDetails>>;
const mode = ctx.cfg.gateway?.mode === "remote" ? "remote" : "local";
const gatewayPath = mode === "remote" ? "gateway.remote.url" : "gateway.mode";
let probeDetails: Awaited<ReturnType<typeof buildGatewayProbeConnectionDetails>> | undefined;
const warning = (message: string, fixHint: string): HealthFinding => ({
checkId: "core/doctor/gateway-health",
severity: "warning",
message,
path: probeDetails || mode === "remote" ? gatewayPath : "gateway",
...(probeDetails ? { target: formatGatewayHealthDiagnostic(probeDetails.url) } : {}),
fixHint,
});
try {
probeDetails = await buildGatewayProbeConnectionDetails({
config: ctx.cfg,
...(ctx.configPath ? { configPath: ctx.configPath } : {}),
configPath: ctx.configPath,
});
} catch (error) {
return [
{
checkId: "core/doctor/gateway-health",
severity: "warning",
message: `Gateway health probe could not be prepared: ${formatErrorMessage(error)}`,
path: ctx.cfg.gateway?.mode === "remote" ? "gateway.remote.url" : "gateway",
fixHint:
"Fix Gateway connection configuration, then rerun `openclaw doctor --lint --only core/doctor/gateway-health`.",
},
];
}
const probe = await probeGatewayStatus({
url: probeDetails.url,
timeoutMs: 3000,
tlsFingerprint: probeDetails.tlsFingerprint,
preauthHandshakeTimeoutMs: probeDetails.preauthHandshakeTimeoutMs,
config: ctx.cfg,
json: true,
});
const mode = ctx.cfg.gateway?.mode === "remote" ? "remote" : "local";
if (gatewayProbeResultWasRateLimited(probe)) {
return [
{
checkId: "core/doctor/gateway-health",
severity: "warning",
message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
path: mode === "remote" ? "gateway.remote.url" : "gateway.mode",
target: formatGatewayHealthTarget(probeDetails.url),
fixHint: "Wait for the temporary authentication lockout to expire, then rerun doctor.",
},
];
}
if (gatewayProbeResultSawGateway(probe)) {
return [];
}
return [
{
if (
ctx.allowExecSecretRefs !== true &&
(await hasActiveGatewayExecCredential({
cfg: ctx.cfg,
env: ctx.env,
targetUrl: probeDetails.url,
}))
) {
return [
warning(
"Authenticated Gateway health inspection was intentionally skipped because an active credential uses an exec SecretRef.",
"Rerun `openclaw doctor --lint --only core/doctor/gateway-health --allow-exec` to permit configured secret execution.",
),
];
}
const status = await callGateway<StatusSummary>({
method: "status",
params: { includeChannelSummary: false },
timeoutMs: 3000,
sharedStateMode: "read-only",
config: ctx.cfg,
configPath: ctx.configPath,
tlsFingerprint: probeDetails.tlsFingerprint,
preauthHandshakeTimeoutMs: probeDetails.preauthHandshakeTimeoutMs,
});
return projectDoctorSecretRuntimeDegradations(status).map((owner) => ({
checkId: "core/doctor/gateway-health",
severity: "warning",
message: `Gateway is not reachable: ${probe.error ?? "status probe failed"}`,
path: mode === "remote" ? "gateway.remote.url" : "gateway.mode",
target: formatGatewayHealthTarget(probeDetails.url),
fixHint:
mode === "remote"
? "Verify the remote Gateway URL, network path, TLS settings, and credentials."
: "Start the Gateway service or run `openclaw doctor --fix` for service repair prompts.",
},
];
message: `Secret runtime degradation: ${owner.message}`,
path: owner.path,
target: owner.target,
fixHint: `Retry: ${owner.retryHint}`,
}));
} catch (error) {
if (!probeDetails) {
return [
warning(
`Gateway health inspection could not be prepared: ${formatGatewayHealthDiagnostic(error)}`,
"Fix Gateway connection configuration, then rerun `openclaw doctor --lint --only core/doctor/gateway-health`.",
),
];
}
const diagnostic = gatewayConnectErrorWasRateLimited(error)
? {
message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
fixHint: "Wait for the temporary authentication lockout to expire, then rerun doctor.",
}
: isGatewayCredentialsRequiredError(error) || isGatewaySecretRefUnavailableError(error)
? {
message:
"Gateway status could not be inspected because this CLI has no usable token/password or paired device token for read-scope RPCs.",
fixHint:
"Configure the Gateway token/password or pair this device, then rerun the selected health check.",
}
: {
message: `Gateway status could not be inspected: ${formatGatewayHealthDiagnostic(error)}`,
fixHint:
mode === "remote"
? "Verify the remote Gateway URL, network path, TLS settings, and credentials."
: "Start the Gateway service or run `openclaw doctor --fix` for service repair prompts.",
};
return [warning(diagnostic.message, diagnostic.fixHint)];
}
}
function gatewayRuntimeStatus(runtime: GatewayServiceRuntime | undefined): string | undefined {
+1 -1
View File
@@ -1063,7 +1063,7 @@ function createGatewayHealthCheck(deps: CoreHealthCheckDeps): SplitHealthCheckDe
return {
id: GATEWAY_HEALTH_CHECK_ID,
kind: "core",
description: "Gateway reachability is represented as structured findings.",
description: "Authenticated Gateway health and degraded secret owners are structured findings.",
source: "doctor",
defaultEnabled: false,
async detect(ctx) {
@@ -0,0 +1,64 @@
import { describe, expect, it } from "vitest";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import type { SecretInput } from "../config/types.secrets.js";
import { hasActiveGatewayExecCredential } from "./doctor-gateway-exec-credential.js";
const execRef = { source: "exec", provider: "vault", id: "PRIVATE_EDGE_REF_ID" } as const;
type GatewayExecEdgeCredentialCase = {
label: string;
mode: "local" | "remote";
targetUrl?: string;
edgeAuth: Record<string, SecretInput>;
expected: boolean;
};
describe("hasActiveGatewayExecCredential", () => {
it.each<GatewayExecEdgeCredentialCase>([
{
label: "the configured remote target uses an exec-backed edge header",
mode: "remote" as const,
edgeAuth: { "X-Edge-Auth": execRef },
expected: true,
},
{
label: "a matching target differs only by query parameters",
mode: "remote" as const,
targetUrl: "wss://gateway.example.test/rpc?profile=two",
edgeAuth: { "X-Edge-Auth": execRef },
expected: true,
},
{
label: "the effective target is a different gateway",
mode: "remote" as const,
targetUrl: "wss://other-gateway.example.test/rpc",
edgeAuth: { "X-Edge-Auth": execRef },
expected: false,
},
{
label: "a local gateway cannot use unrelated remote edge headers",
mode: "local" as const,
edgeAuth: { "X-Edge-Auth": execRef },
expected: false,
},
{
label: "matching literal and environment-backed headers do not execute",
mode: "remote" as const,
edgeAuth: {
"X-Literal": "literal-edge-value",
"X-Environment": { source: "env", provider: "default", id: "EDGE_TOKEN" } as const,
},
expected: false,
},
])("detects only effective exec edge credentials when $label", async (entry) => {
const cfg: OpenClawConfig = {
gateway: {
mode: entry.mode,
remote: { url: "wss://gateway.example.test/rpc", edgeAuth: entry.edgeAuth },
},
};
await expect(
hasActiveGatewayExecCredential({ cfg, env: {}, targetUrl: entry.targetUrl }),
).resolves.toBe(entry.expected);
});
});
+21 -1
View File
@@ -3,6 +3,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
export async function hasActiveGatewayExecCredential(params: {
cfg: OpenClawConfig;
env?: NodeJS.ProcessEnv;
targetUrl?: string;
}): Promise<boolean> {
const [{ resolveSecretInputRef }, { gatewaySecretInputPathCanWin }, secretPaths] =
await Promise.all([
@@ -11,7 +12,7 @@ export async function hasActiveGatewayExecCredential(params: {
import("../gateway/secret-input-paths.js"),
]);
const mode = params.cfg.gateway?.mode === "remote" ? "remote" : "local";
return secretPaths.ALL_GATEWAY_SECRET_INPUT_PATHS.some((path) => {
const hasExecCredential = secretPaths.ALL_GATEWAY_SECRET_INPUT_PATHS.some((path) => {
if (
!gatewaySecretInputPathCanWin({
config: params.cfg,
@@ -28,4 +29,23 @@ export async function hasActiveGatewayExecCredential(params: {
}).ref;
return ref?.source === "exec";
});
if (hasExecCredential || !params.cfg.gateway?.remote?.edgeAuth) {
return hasExecCredential;
}
const [{ buildGatewayProbeConnectionDetails }, edgeAuth] = await Promise.all([
import("../gateway/call.js"),
import("../gateway/edge-auth.js"),
]);
const targetUrl =
params.targetUrl ?? (await buildGatewayProbeConnectionDetails({ config: params.cfg })).url;
const { gatewayEdgeAuthValueForTarget, normalizeEdgeAuthHeadersConfig } = edgeAuth;
const headers = normalizeEdgeAuthHeadersConfig(
gatewayEdgeAuthValueForTarget({ config: params.cfg, targetUrl }),
);
return Object.values(headers ?? {}).some(
(value) =>
resolveSecretInputRef({ value, defaults: params.cfg.secrets?.defaults }).ref?.source ===
"exec",
);
}