mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-24 11:25:50 -06:00
fix(doctor): report degraded gateway secret owners (#126998)
This commit is contained in:
committed by
GitHub
parent
9bc772ba50
commit
2e1b882845
@@ -18,9 +18,9 @@ import type {
|
||||
import { collectChannelStatusIssues } from "../infra/channels-status-issues.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import { redactSecretDegradationReason } from "../secrets/runtime-degraded-state.js";
|
||||
import type { StatusSummary } from "../status/types.js";
|
||||
import { VERSION } from "../version.js";
|
||||
import { projectDoctorSecretRuntimeDegradations } from "./doctor-secret-runtime-degradation.js";
|
||||
import {
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE,
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE,
|
||||
@@ -94,14 +94,11 @@ export async function checkGatewayHealth(params: {
|
||||
});
|
||||
healthOk = true;
|
||||
noteCliGatewayVersionSkew(status);
|
||||
if (status.degradedSecretOwners && status.degradedSecretOwners.length > 0) {
|
||||
const secretDegradations = projectDoctorSecretRuntimeDegradations(status);
|
||||
if (secretDegradations.length > 0) {
|
||||
note(
|
||||
status.degradedSecretOwners
|
||||
.map(
|
||||
(owner) =>
|
||||
`- ${owner.degradationState ?? "cold"} ${owner.ownerKind}:${owner.ownerId} (${owner.paths.join(", ")}): ${redactSecretDegradationReason(owner.reason)}` +
|
||||
"\n Retry: openclaw secrets reload",
|
||||
)
|
||||
secretDegradations
|
||||
.map((owner) => `- ${owner.message}\n Retry: ${owner.retryHint}`)
|
||||
.join("\n"),
|
||||
"Secret runtime degradation",
|
||||
);
|
||||
|
||||
@@ -6,6 +6,8 @@ import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import * as bundledHealthChecks from "../flows/bundled-health-checks.js";
|
||||
import { CORE_HEALTH_CHECKS } from "../flows/doctor-core-checks.js";
|
||||
import { clearHealthChecksForTest, registerHealthCheck } from "../flows/health-check-registry.js";
|
||||
import { clearLoadInstalledPluginIndexInstallRecordsCache } from "../plugins/installed-plugin-index-record-cache.js";
|
||||
import { writePersistedInstalledPluginIndexInstallRecords } from "../plugins/installed-plugin-index-records.js";
|
||||
@@ -17,6 +19,8 @@ const mocks = vi.hoisted(() => ({
|
||||
actualOpenNodeSqliteDatabase: vi.fn(),
|
||||
actualPrepareSqliteReadOnlyLocationSync: vi.fn(),
|
||||
actualReadConfigFileSnapshot: vi.fn(),
|
||||
buildGatewayProbeConnectionDetails: vi.fn(),
|
||||
callGateway: vi.fn(),
|
||||
openNodeSqliteDatabase: vi.fn(),
|
||||
prepareSqliteReadOnlyLocationSync: vi.fn(),
|
||||
readConfigFileSnapshot: vi.fn(),
|
||||
@@ -66,7 +70,14 @@ vi.mock("../flows/doctor-health-contributions.js", async (importOriginal) => {
|
||||
mocks.resolveDoctorContributionHealthChecks(...args),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../gateway/call.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../gateway/call.js")>();
|
||||
return {
|
||||
...actual,
|
||||
buildGatewayProbeConnectionDetails: mocks.buildGatewayProbeConnectionDetails,
|
||||
callGateway: mocks.callGateway,
|
||||
};
|
||||
});
|
||||
const runtime = {
|
||||
log: vi.fn(),
|
||||
error: vi.fn(),
|
||||
@@ -79,6 +90,10 @@ describe("runDoctorLintCli", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.readConfigFileSnapshot.mockReset();
|
||||
mocks.buildGatewayProbeConnectionDetails.mockReset().mockResolvedValue({
|
||||
url: "ws://127.0.0.1:18789",
|
||||
});
|
||||
mocks.callGateway.mockReset().mockResolvedValue({ degradedSecretOwners: [] });
|
||||
mocks.openNodeSqliteDatabase.mockImplementation((...args: unknown[]) =>
|
||||
mocks.actualOpenNodeSqliteDatabase(...args),
|
||||
);
|
||||
@@ -113,29 +128,115 @@ describe("runDoctorLintCli", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("reports the visible finding count in human output", async () => {
|
||||
it.each([
|
||||
{ label: "--only JSON", selection: "only", json: true },
|
||||
{ label: "--only human text", selection: "only", json: false },
|
||||
{ label: "--all JSON", selection: "all", json: true },
|
||||
{ label: "--all human text", selection: "all", json: false },
|
||||
{ label: "default JSON", selection: "default", json: true },
|
||||
{ label: "default human text", selection: "default", json: false },
|
||||
] as const)("keeps Gateway-owned secret degradation observable through $label", async (entry) => {
|
||||
const gatewayCheck = CORE_HEALTH_CHECKS.find(
|
||||
(check) => check.id === "core/doctor/gateway-health",
|
||||
);
|
||||
expect(gatewayCheck).toBeDefined();
|
||||
const previousResolveChecks =
|
||||
mocks.resolveDoctorContributionHealthChecks.getMockImplementation();
|
||||
mocks.resolveDoctorContributionHealthChecks.mockResolvedValue([gatewayCheck]);
|
||||
const registerChecks = vi
|
||||
.spyOn(bundledHealthChecks, "registerBundledHealthChecks")
|
||||
.mockImplementation(() => {});
|
||||
const resolveStateMode = vi
|
||||
.spyOn(bundledHealthChecks, "resolveBundledHealthCheckPluginStateMode")
|
||||
.mockReturnValue("direct");
|
||||
mocks.readConfigFileSnapshot.mockResolvedValue({
|
||||
exists: true,
|
||||
valid: true,
|
||||
config: {},
|
||||
config: {
|
||||
gateway: {
|
||||
mode: "local",
|
||||
auth: { mode: "token", token: "SYNTHETIC_GATEWAY_SECRET" },
|
||||
},
|
||||
},
|
||||
path: "/tmp/openclaw.json",
|
||||
});
|
||||
|
||||
mocks.callGateway.mockResolvedValue({
|
||||
degradedSecretOwners: [
|
||||
{
|
||||
ownerKind: "account",
|
||||
ownerId: "discord:ops",
|
||||
state: "unavailable",
|
||||
paths: ["channels.discord.accounts.ops.token"],
|
||||
reason:
|
||||
"secret reference was not found (env:default:PRIVATE_REF_ID=SYNTHETIC_OWNER_SECRET)",
|
||||
},
|
||||
],
|
||||
});
|
||||
const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
|
||||
const originalIsTTY = process.stdout.isTTY;
|
||||
Object.defineProperty(process.stdout, "isTTY", { configurable: true, value: true });
|
||||
Object.defineProperty(process.stdout, "isTTY", { configurable: true, value: !entry.json });
|
||||
|
||||
try {
|
||||
const exitCode = await runDoctorLintCli(runtime, {
|
||||
severityMin: "error",
|
||||
onlyIds: ["core/doctor/final-config-validation"],
|
||||
...(entry.json ? { json: true } : {}),
|
||||
...(entry.selection === "only"
|
||||
? { onlyIds: ["core/doctor/gateway-health"] }
|
||||
: entry.selection === "all"
|
||||
? { includeAllChecks: true }
|
||||
: {}),
|
||||
});
|
||||
const output = stdout.mock.calls.map(([line]) => String(line)).join("");
|
||||
|
||||
expect(exitCode).toBe(0);
|
||||
expect(String(stdout.mock.calls[0]?.[0])).toContain("0 finding(s)");
|
||||
expect(String(stdout.mock.calls[1]?.[0])).toBe(" no findings\n");
|
||||
if (entry.selection === "default") {
|
||||
expect(exitCode).toBe(0);
|
||||
if (entry.json) {
|
||||
expect(JSON.parse(output)).toMatchObject({
|
||||
checksRun: 0,
|
||||
checksSkipped: 1,
|
||||
findings: [],
|
||||
});
|
||||
} else {
|
||||
expect(output).toContain("0 finding(s)");
|
||||
expect(output).toContain(" no findings\n");
|
||||
}
|
||||
expect(mocks.buildGatewayProbeConnectionDetails).not.toHaveBeenCalled();
|
||||
expect(mocks.callGateway).not.toHaveBeenCalled();
|
||||
return;
|
||||
}
|
||||
|
||||
expect(exitCode).toBe(1);
|
||||
expect(output).toContain("core/doctor/gateway-health");
|
||||
expect(output).toContain("cold account:discord:ops");
|
||||
expect(output).toContain("channels.discord.accounts.ops.token");
|
||||
expect(output).toContain("openclaw secrets reload");
|
||||
expect(output).not.toContain("SYNTHETIC_GATEWAY_SECRET");
|
||||
expect(output).not.toContain("SYNTHETIC_OWNER_SECRET");
|
||||
expect(output).not.toContain("PRIVATE_REF_ID");
|
||||
expect(mocks.callGateway).toHaveBeenCalledOnce();
|
||||
if (entry.json) {
|
||||
expect(JSON.parse(output)).toMatchObject({
|
||||
ok: false,
|
||||
checksRun: 1,
|
||||
findings: [
|
||||
{
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
path: "channels.discord.accounts.ops.token",
|
||||
target: "account:discord:ops",
|
||||
},
|
||||
],
|
||||
});
|
||||
} else {
|
||||
expect(output).toContain("[warning] core/doctor/gateway-health");
|
||||
}
|
||||
} finally {
|
||||
Object.defineProperty(process.stdout, "isTTY", { configurable: true, value: originalIsTTY });
|
||||
stdout.mockRestore();
|
||||
registerChecks.mockRestore();
|
||||
resolveStateMode.mockRestore();
|
||||
if (previousResolveChecks) {
|
||||
mocks.resolveDoctorContributionHealthChecks.mockImplementation(previousResolveChecks);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import { redactSensitiveUrlLikeString } from "@openclaw/net-policy/redact-sensitive-url";
|
||||
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
import { sanitizeTerminalText } from "../../packages/terminal-core/src/safe-text.js";
|
||||
import {
|
||||
redactSecretDegradationReason,
|
||||
SECRET_DEGRADATION_RETRY_HINT,
|
||||
} from "../secrets/runtime-degraded-state.js";
|
||||
import type { StatusSummary } from "../status/types.js";
|
||||
|
||||
const DOCTOR_SECRET_OWNER_ID_MAX_CHARS = 96;
|
||||
const DOCTOR_SECRET_OWNER_PATH_MAX_CHARS = 120;
|
||||
const DOCTOR_SECRET_OWNER_VISIBLE_PATHS = 3;
|
||||
|
||||
function safeDoctorSecretOwnerText(value: string, maxChars: number): string {
|
||||
const safe = sanitizeTerminalText(redactSensitiveUrlLikeString(value));
|
||||
return safe.length <= maxChars ? safe : `${truncateUtf16Safe(safe, maxChars - 1)}…`;
|
||||
}
|
||||
|
||||
/** Projects Gateway-owned secret degradation into the shared bounded Doctor display shape. */
|
||||
export function projectDoctorSecretRuntimeDegradations(
|
||||
status: Pick<StatusSummary, "degradedSecretOwners">,
|
||||
) {
|
||||
return (status.degradedSecretOwners ?? []).map((owner) => {
|
||||
const ownerId = safeDoctorSecretOwnerText(owner.ownerId, DOCTOR_SECRET_OWNER_ID_MAX_CHARS);
|
||||
const target = `${owner.ownerKind}:${ownerId}`;
|
||||
const visiblePaths = owner.paths
|
||||
.slice(0, DOCTOR_SECRET_OWNER_VISIBLE_PATHS)
|
||||
.map((configPath) =>
|
||||
safeDoctorSecretOwnerText(configPath, DOCTOR_SECRET_OWNER_PATH_MAX_CHARS),
|
||||
);
|
||||
const omittedPaths = owner.paths.length - visiblePaths.length;
|
||||
const paths =
|
||||
visiblePaths.join(", ") + (omittedPaths > 0 ? ` (+${omittedPaths} paths omitted)` : "");
|
||||
return {
|
||||
message: `${owner.degradationState ?? "cold"} ${target} (${paths || "no affected paths reported"}): ${redactSecretDegradationReason(owner.reason)}`,
|
||||
path: visiblePaths[0] ?? "gateway",
|
||||
target,
|
||||
retryHint: SECRET_DEGRADATION_RETRY_HINT,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
// Doctor runtime check tests cover runtime-backed doctor checks.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { GatewayClientRequestError } from "../../packages/gateway-client/src/index.js";
|
||||
import type { AnyAgentTool } from "../agents/tools/common.js";
|
||||
import { GATEWAY_HEALTH_RATE_LIMITED_MESSAGE } from "../commands/gateway-health-auth-diagnostic.js";
|
||||
import { GatewaySecretRefUnavailableError } from "../gateway/credentials.js";
|
||||
import { setPluginToolMeta } from "../plugins/tools.js";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
@@ -11,7 +13,8 @@ const mocks = vi.hoisted(() => ({
|
||||
loadModelCatalog: vi.fn(async (): Promise<Array<Record<string, unknown>>> => []),
|
||||
normalizeProviderToolSchemasWithPlugin: vi.fn(),
|
||||
buildGatewayProbeConnectionDetails: vi.fn(),
|
||||
probeGatewayStatus: vi.fn(),
|
||||
callGateway: vi.fn(),
|
||||
isGatewayCredentialsRequiredError: vi.fn(),
|
||||
readGatewayServiceState: vi.fn(),
|
||||
resolveGatewayService: vi.fn(() => ({ label: "openclaw-gateway" })),
|
||||
resolvePluginProvidersCore: vi.fn((): Array<Record<string, unknown>> => []),
|
||||
@@ -46,10 +49,8 @@ vi.mock("../agents/agent-tools.js", () => ({
|
||||
|
||||
vi.mock("../gateway/call.js", () => ({
|
||||
buildGatewayProbeConnectionDetails: mocks.buildGatewayProbeConnectionDetails,
|
||||
}));
|
||||
|
||||
vi.mock("../cli/daemon-cli/probe.js", () => ({
|
||||
probeGatewayStatus: mocks.probeGatewayStatus,
|
||||
callGateway: mocks.callGateway,
|
||||
isGatewayCredentialsRequiredError: mocks.isGatewayCredentialsRequiredError,
|
||||
}));
|
||||
|
||||
vi.mock("../daemon/service.js", () => ({
|
||||
@@ -105,13 +106,6 @@ describe("doctor runtime tool schema checks", () => {
|
||||
mocks.normalizeProviderToolSchemasWithPlugin
|
||||
.mockReset()
|
||||
.mockImplementation(({ context }) => context.tools);
|
||||
mocks.buildGatewayProbeConnectionDetails.mockReset().mockResolvedValue({
|
||||
url: "http://127.0.0.1:5829",
|
||||
});
|
||||
mocks.probeGatewayStatus.mockReset().mockResolvedValue({
|
||||
ok: true,
|
||||
server: { version: "2026.6.26" },
|
||||
});
|
||||
mocks.readGatewayServiceState.mockReset().mockResolvedValue({
|
||||
installed: true,
|
||||
loadState: { status: "loaded" },
|
||||
@@ -567,10 +561,8 @@ describe("doctor gateway runtime checks", () => {
|
||||
mocks.buildGatewayProbeConnectionDetails.mockReset().mockResolvedValue({
|
||||
url: "http://127.0.0.1:5829",
|
||||
});
|
||||
mocks.probeGatewayStatus.mockReset().mockResolvedValue({
|
||||
ok: true,
|
||||
server: { version: "2026.6.26" },
|
||||
});
|
||||
mocks.callGateway.mockReset().mockResolvedValue({ degradedSecretOwners: [] });
|
||||
mocks.isGatewayCredentialsRequiredError.mockReset().mockReturnValue(false);
|
||||
mocks.readGatewayServiceState.mockReset().mockResolvedValue({
|
||||
installed: true,
|
||||
loadState: { status: "loaded" },
|
||||
@@ -582,52 +574,209 @@ describe("doctor gateway runtime checks", () => {
|
||||
mocks.resolveGatewayService.mockReset().mockReturnValue({ label: "openclaw-gateway" });
|
||||
});
|
||||
|
||||
it("reports unreachable gateway health probes", async () => {
|
||||
mocks.probeGatewayStatus.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
error: "connect ECONNREFUSED 127.0.0.1:5829",
|
||||
it("projects every degraded SecretRef owner from exactly one authenticated read-only status RPC", async () => {
|
||||
const cfg = { gateway: { mode: "local" as const } };
|
||||
const privateToken = "SYNTHETIC_PRIVATE_URL_TOKEN";
|
||||
mocks.buildGatewayProbeConnectionDetails.mockResolvedValueOnce({
|
||||
url: "wss://127.0.0.1:5829",
|
||||
tlsFingerprint: "sha256:test-doctor-fingerprint",
|
||||
preauthHandshakeTimeoutMs: 1200,
|
||||
});
|
||||
mocks.callGateway.mockResolvedValueOnce({
|
||||
degradedSecretOwners: [
|
||||
{
|
||||
ownerKind: "account",
|
||||
ownerId: "discord:ops",
|
||||
state: "unavailable",
|
||||
paths: ["channels.discord.accounts.ops.token"],
|
||||
reason: "secret reference was not found (env:default:PRIVATE_REF_ID)",
|
||||
},
|
||||
{
|
||||
ownerKind: "capability",
|
||||
ownerId: "tts",
|
||||
state: "unavailable",
|
||||
degradationState: "stale",
|
||||
paths: ["tts.providers.elevenlabs.apiKey", "tts.providers.elevenlabs.voiceId"],
|
||||
reason: "secret provider policy denied resolution",
|
||||
},
|
||||
{
|
||||
ownerKind: "provider",
|
||||
ownerId: `vault\u001b]52;c;attack\u0007:https://user:${privateToken}@secret.test/${"a".repeat(500)}`,
|
||||
state: "unavailable",
|
||||
paths: Array.from(
|
||||
{ length: 12 },
|
||||
(_, index) =>
|
||||
`providers.example.${index}.https://secret.test/value?token=${privateToken}\n${"z".repeat(400)}`,
|
||||
),
|
||||
reason: `secret provider failed: ${privateToken}\nref PRIVATE_REF_ID`,
|
||||
},
|
||||
],
|
||||
degradedPlugins: [{ pluginId: "not-this-check" }],
|
||||
});
|
||||
|
||||
await expect(
|
||||
collectGatewayHealthFindings({ cfg: { gateway: { mode: "local" } } }),
|
||||
).resolves.toContainEqual({
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: "Gateway is not reachable: connect ECONNREFUSED 127.0.0.1:5829",
|
||||
path: "gateway.mode",
|
||||
target: "http://127.0.0.1:5829",
|
||||
fixHint:
|
||||
"Start the Gateway service or run `openclaw doctor --fix` for service repair prompts.",
|
||||
const findings = await collectGatewayHealthFindings({
|
||||
cfg,
|
||||
configPath: "/tmp/selected-openclaw.json",
|
||||
});
|
||||
|
||||
expect(mocks.callGateway).toHaveBeenCalledExactlyOnceWith({
|
||||
method: "status",
|
||||
params: { includeChannelSummary: false },
|
||||
timeoutMs: 3000,
|
||||
sharedStateMode: "read-only",
|
||||
config: cfg,
|
||||
configPath: "/tmp/selected-openclaw.json",
|
||||
tlsFingerprint: "sha256:test-doctor-fingerprint",
|
||||
preauthHandshakeTimeoutMs: 1200,
|
||||
});
|
||||
expect(findings).toEqual([
|
||||
expect.objectContaining({
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: expect.stringContaining("cold account:discord:ops"),
|
||||
path: "channels.discord.accounts.ops.token",
|
||||
target: "account:discord:ops",
|
||||
fixHint: expect.stringContaining("openclaw secrets reload"),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: expect.stringContaining("stale capability:tts"),
|
||||
path: "tts.providers.elevenlabs.apiKey",
|
||||
target: "capability:tts",
|
||||
fixHint: expect.stringContaining("openclaw secrets reload"),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: expect.stringContaining("provider:vault"),
|
||||
path: expect.stringContaining("providers.example.0"),
|
||||
target: expect.stringContaining("provider:vault"),
|
||||
}),
|
||||
]);
|
||||
expect(findings[1]?.message).toContain("tts.providers.elevenlabs.voiceId");
|
||||
const finding = findings[2];
|
||||
const rendered = JSON.stringify(findings);
|
||||
expect(finding?.message).toContain("omitted");
|
||||
expect(finding?.message).toContain("secret resolution failed");
|
||||
expect(finding?.message.length).toBeLessThanOrEqual(700);
|
||||
expect(finding?.target?.length).toBeLessThanOrEqual(150);
|
||||
expect(finding?.path?.length).toBeLessThanOrEqual(180);
|
||||
expect(rendered).not.toContain(privateToken);
|
||||
expect(rendered).not.toContain("PRIVATE_REF_ID");
|
||||
expect(rendered).not.toContain("not-this-check");
|
||||
expect(rendered).not.toContain("\u001b");
|
||||
expect(rendered).not.toContain("\u0007");
|
||||
});
|
||||
|
||||
it("reports temporary Gateway authentication lockouts with wait-and-retry guidance", async () => {
|
||||
mocks.probeGatewayStatus.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
error: "connect failed",
|
||||
connectFailure: { kind: "rate-limited", detailCode: "AUTH_RATE_LIMITED" },
|
||||
});
|
||||
it.each([
|
||||
{
|
||||
label: "missing Gateway authentication",
|
||||
error: new Error("auth token SYNTHETIC_PRIVATE_TOKEN\nref PRIVATE_REF_ID"),
|
||||
credentialsRequired: true,
|
||||
message:
|
||||
"Gateway status could not be inspected because this CLI has no usable token/password or paired device token for read-scope RPCs.",
|
||||
},
|
||||
{
|
||||
label: "an unavailable Gateway authentication SecretRef",
|
||||
error: new GatewaySecretRefUnavailableError("gateway.auth.token"),
|
||||
credentialsRequired: false,
|
||||
message:
|
||||
"Gateway status could not be inspected because this CLI has no usable token/password or paired device token for read-scope RPCs.",
|
||||
},
|
||||
{
|
||||
label: "temporary Gateway authentication rate limiting",
|
||||
error: new GatewayClientRequestError({
|
||||
code: "INVALID_REQUEST",
|
||||
message: "unauthorized: too many failed authentication attempts (retry later)",
|
||||
details: { code: "AUTH_RATE_LIMITED", authReason: "rate_limited" },
|
||||
retryable: true,
|
||||
}),
|
||||
credentialsRequired: false,
|
||||
message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
},
|
||||
{
|
||||
label: "an unreachable Gateway with terminal control characters",
|
||||
error: new Error("connect ECONNREFUSED 127.0.0.1:5829\u001b]52;c;attack\u0007\u009b"),
|
||||
credentialsRequired: false,
|
||||
message: "Gateway status could not be inspected: connect ECONNREFUSED 127.0.0.1:5829",
|
||||
},
|
||||
])("reports $label from exactly one sanitized status attempt", async (entry) => {
|
||||
mocks.callGateway.mockRejectedValueOnce(entry.error);
|
||||
mocks.isGatewayCredentialsRequiredError.mockReturnValueOnce(entry.credentialsRequired);
|
||||
|
||||
const findings = await collectGatewayHealthFindings({ cfg: { gateway: { mode: "local" } } });
|
||||
|
||||
expect(findings).toEqual([
|
||||
expect.objectContaining({
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: entry.message,
|
||||
path: "gateway.mode",
|
||||
target: "http://127.0.0.1:5829",
|
||||
}),
|
||||
]);
|
||||
expect(JSON.stringify(findings)).not.toContain("SYNTHETIC_PRIVATE_TOKEN");
|
||||
expect(JSON.stringify(findings)).not.toContain("PRIVATE_REF_ID");
|
||||
expect(mocks.callGateway).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("reports preparation failures without exposing URL credentials or control characters", async () => {
|
||||
mocks.buildGatewayProbeConnectionDetails.mockRejectedValueOnce(
|
||||
new Error(
|
||||
`invalid wss://user:${"SYNTHETIC_PRIVATE_TOKEN".repeat(20)}@gateway.test/rpc\nmore`,
|
||||
),
|
||||
);
|
||||
|
||||
const findings = await collectGatewayHealthFindings({ cfg: {} });
|
||||
|
||||
expect(findings).toEqual([
|
||||
expect.objectContaining({
|
||||
severity: "warning",
|
||||
message: expect.stringContaining("Gateway health inspection could not be prepared"),
|
||||
path: "gateway",
|
||||
}),
|
||||
]);
|
||||
expect(JSON.stringify(findings)).not.toContain("SYNTHETIC_PRIVATE_TOKEN");
|
||||
expect(mocks.callGateway).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("prepares the target but skips the RPC for active exec credentials unless execution is allowed", async () => {
|
||||
const cfg = {
|
||||
gateway: {
|
||||
mode: "local" as const,
|
||||
auth: {
|
||||
mode: "token" as const,
|
||||
token: { source: "exec" as const, provider: "vault", id: "PRIVATE_REF_ID" },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const findings = await collectGatewayHealthFindings({ cfg, env: {} });
|
||||
|
||||
expect(findings).toEqual([
|
||||
expect.objectContaining({
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: expect.stringContaining("intentionally skipped"),
|
||||
fixHint: expect.stringContaining("--allow-exec"),
|
||||
}),
|
||||
]);
|
||||
expect(JSON.stringify(findings)).not.toContain("PRIVATE_REF_ID");
|
||||
expect(mocks.buildGatewayProbeConnectionDetails).toHaveBeenCalledOnce();
|
||||
expect(mocks.callGateway).not.toHaveBeenCalled();
|
||||
|
||||
await expect(
|
||||
collectGatewayHealthFindings({ cfg: { gateway: { mode: "local" } } }),
|
||||
).resolves.toContainEqual({
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
path: "gateway.mode",
|
||||
target: "http://127.0.0.1:5829",
|
||||
fixHint: "Wait for the temporary authentication lockout to expire, then rerun doctor.",
|
||||
});
|
||||
collectGatewayHealthFindings({ cfg, env: {}, allowExecSecretRefs: true }),
|
||||
).resolves.toEqual([]);
|
||||
expect(mocks.callGateway).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("redacts sensitive remote gateway URLs from health finding targets", async () => {
|
||||
mocks.buildGatewayProbeConnectionDetails.mockResolvedValueOnce({
|
||||
url: "wss://user:pass@gateway.example.test/rpc?token=secret&safe=value",
|
||||
});
|
||||
mocks.probeGatewayStatus.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
error: "remote gateway did not answer",
|
||||
});
|
||||
mocks.callGateway.mockRejectedValueOnce(new Error("remote gateway did not answer"));
|
||||
|
||||
const findings = await collectGatewayHealthFindings({
|
||||
cfg: { gateway: { mode: "remote", remote: { url: "wss://gateway.example.test/rpc" } } },
|
||||
@@ -636,7 +785,7 @@ describe("doctor gateway runtime checks", () => {
|
||||
expect(findings).toContainEqual({
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: "Gateway is not reachable: remote gateway did not answer",
|
||||
message: "Gateway status could not be inspected: remote gateway did not answer",
|
||||
path: "gateway.remote.url",
|
||||
target: "wss://***:***@gateway.example.test/rpc?token=***&safe=value",
|
||||
fixHint: "Verify the remote Gateway URL, network path, TLS settings, and credentials.",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Doctor runtime checks inspect tool names, browser residue, and runtime state.
|
||||
import { redactSensitiveUrlLikeString } from "@openclaw/net-policy/redact-sensitive-url";
|
||||
import { sanitizeTerminalText } from "../../packages/terminal-core/src/safe-text.js";
|
||||
import { TOOL_NAME_SEPARATOR } from "../agents/agent-bundle-mcp-names.js";
|
||||
import {
|
||||
type McpToolCatalogDiagnostic,
|
||||
@@ -29,12 +30,11 @@ import {
|
||||
type RuntimeToolSchemaDiagnostic,
|
||||
} from "../agents/tool-schema-projection.js";
|
||||
import type { AnyAgentTool } from "../agents/tools/common.js";
|
||||
import { probeGatewayStatus } from "../cli/daemon-cli/probe.js";
|
||||
import { projectDoctorSecretRuntimeDegradations } from "../commands/doctor-secret-runtime-degradation.js";
|
||||
import { collectUnavailableAgentSkills } from "../commands/doctor-skills-core.js";
|
||||
import {
|
||||
GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
gatewayProbeResultSawGateway,
|
||||
gatewayProbeResultWasRateLimited,
|
||||
gatewayConnectErrorWasRateLimited,
|
||||
} from "../commands/gateway-health-auth-diagnostic.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import {
|
||||
@@ -42,7 +42,12 @@ import {
|
||||
type GatewayServiceRuntime,
|
||||
} from "../daemon/service-runtime.js";
|
||||
import { resolveGatewayService, readGatewayServiceState } from "../daemon/service.js";
|
||||
import { buildGatewayProbeConnectionDetails } from "../gateway/call.js";
|
||||
import {
|
||||
buildGatewayProbeConnectionDetails,
|
||||
callGateway,
|
||||
isGatewayCredentialsRequiredError,
|
||||
} from "../gateway/call.js";
|
||||
import { isGatewaySecretRefUnavailableError } from "../gateway/credentials.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import {
|
||||
formatLocalAudioSelection,
|
||||
@@ -54,14 +59,18 @@ import { getPluginToolMeta, setPluginToolMeta } from "../plugins/tools.js";
|
||||
import type { ProviderCatalogOrder, ProviderPlugin } from "../plugins/types.js";
|
||||
import { normalizeAgentId } from "../routing/session-key.js";
|
||||
import { buildWorkspaceSkillStatus } from "../skills/discovery/status.js";
|
||||
import type { StatusSummary } from "../status/types.js";
|
||||
import { scrubDoctorErrorMessage } from "./doctor-error-message.js";
|
||||
import { hasActiveGatewayExecCredential } from "./doctor-gateway-exec-credential.js";
|
||||
import type { HealthCheckContext, HealthFinding } from "./health-checks.js";
|
||||
|
||||
type BundleMcpToolRuntime = Awaited<ReturnType<typeof createBundleMcpToolRuntime>>;
|
||||
const PROVIDER_CATALOG_ORDERS = ["simple", "profile", "paired", "late"] as const;
|
||||
const PROVIDER_CATALOG_ORDER_SET = new Set<ProviderCatalogOrder>(PROVIDER_CATALOG_ORDERS);
|
||||
|
||||
function formatGatewayHealthTarget(url: string): string {
|
||||
return redactSensitiveUrlLikeString(url);
|
||||
function formatGatewayHealthDiagnostic(value: unknown): string {
|
||||
const raw = value instanceof Error ? value.message : String(value);
|
||||
return scrubDoctorErrorMessage(sanitizeTerminalText(redactSensitiveUrlLikeString(raw)));
|
||||
}
|
||||
|
||||
export function detectUnavailableSkills(cfg: OpenClawConfig, workspaceDir: string) {
|
||||
@@ -105,64 +114,87 @@ export async function collectLocalAudioAccelerationFindings(): Promise<readonly
|
||||
}
|
||||
|
||||
export async function collectGatewayHealthFindings(
|
||||
ctx: Pick<HealthCheckContext, "cfg" | "configPath">,
|
||||
ctx: Pick<HealthCheckContext, "cfg" | "configPath" | "env" | "allowExecSecretRefs">,
|
||||
): Promise<readonly HealthFinding[]> {
|
||||
let probeDetails: Awaited<ReturnType<typeof buildGatewayProbeConnectionDetails>>;
|
||||
const mode = ctx.cfg.gateway?.mode === "remote" ? "remote" : "local";
|
||||
const gatewayPath = mode === "remote" ? "gateway.remote.url" : "gateway.mode";
|
||||
let probeDetails: Awaited<ReturnType<typeof buildGatewayProbeConnectionDetails>> | undefined;
|
||||
const warning = (message: string, fixHint: string): HealthFinding => ({
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message,
|
||||
path: probeDetails || mode === "remote" ? gatewayPath : "gateway",
|
||||
...(probeDetails ? { target: formatGatewayHealthDiagnostic(probeDetails.url) } : {}),
|
||||
fixHint,
|
||||
});
|
||||
try {
|
||||
probeDetails = await buildGatewayProbeConnectionDetails({
|
||||
config: ctx.cfg,
|
||||
...(ctx.configPath ? { configPath: ctx.configPath } : {}),
|
||||
configPath: ctx.configPath,
|
||||
});
|
||||
} catch (error) {
|
||||
return [
|
||||
{
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: `Gateway health probe could not be prepared: ${formatErrorMessage(error)}`,
|
||||
path: ctx.cfg.gateway?.mode === "remote" ? "gateway.remote.url" : "gateway",
|
||||
fixHint:
|
||||
"Fix Gateway connection configuration, then rerun `openclaw doctor --lint --only core/doctor/gateway-health`.",
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
const probe = await probeGatewayStatus({
|
||||
url: probeDetails.url,
|
||||
timeoutMs: 3000,
|
||||
tlsFingerprint: probeDetails.tlsFingerprint,
|
||||
preauthHandshakeTimeoutMs: probeDetails.preauthHandshakeTimeoutMs,
|
||||
config: ctx.cfg,
|
||||
json: true,
|
||||
});
|
||||
const mode = ctx.cfg.gateway?.mode === "remote" ? "remote" : "local";
|
||||
if (gatewayProbeResultWasRateLimited(probe)) {
|
||||
return [
|
||||
{
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
path: mode === "remote" ? "gateway.remote.url" : "gateway.mode",
|
||||
target: formatGatewayHealthTarget(probeDetails.url),
|
||||
fixHint: "Wait for the temporary authentication lockout to expire, then rerun doctor.",
|
||||
},
|
||||
];
|
||||
}
|
||||
if (gatewayProbeResultSawGateway(probe)) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
if (
|
||||
ctx.allowExecSecretRefs !== true &&
|
||||
(await hasActiveGatewayExecCredential({
|
||||
cfg: ctx.cfg,
|
||||
env: ctx.env,
|
||||
targetUrl: probeDetails.url,
|
||||
}))
|
||||
) {
|
||||
return [
|
||||
warning(
|
||||
"Authenticated Gateway health inspection was intentionally skipped because an active credential uses an exec SecretRef.",
|
||||
"Rerun `openclaw doctor --lint --only core/doctor/gateway-health --allow-exec` to permit configured secret execution.",
|
||||
),
|
||||
];
|
||||
}
|
||||
const status = await callGateway<StatusSummary>({
|
||||
method: "status",
|
||||
params: { includeChannelSummary: false },
|
||||
timeoutMs: 3000,
|
||||
sharedStateMode: "read-only",
|
||||
config: ctx.cfg,
|
||||
configPath: ctx.configPath,
|
||||
tlsFingerprint: probeDetails.tlsFingerprint,
|
||||
preauthHandshakeTimeoutMs: probeDetails.preauthHandshakeTimeoutMs,
|
||||
});
|
||||
return projectDoctorSecretRuntimeDegradations(status).map((owner) => ({
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: `Gateway is not reachable: ${probe.error ?? "status probe failed"}`,
|
||||
path: mode === "remote" ? "gateway.remote.url" : "gateway.mode",
|
||||
target: formatGatewayHealthTarget(probeDetails.url),
|
||||
fixHint:
|
||||
mode === "remote"
|
||||
? "Verify the remote Gateway URL, network path, TLS settings, and credentials."
|
||||
: "Start the Gateway service or run `openclaw doctor --fix` for service repair prompts.",
|
||||
},
|
||||
];
|
||||
message: `Secret runtime degradation: ${owner.message}`,
|
||||
path: owner.path,
|
||||
target: owner.target,
|
||||
fixHint: `Retry: ${owner.retryHint}`,
|
||||
}));
|
||||
} catch (error) {
|
||||
if (!probeDetails) {
|
||||
return [
|
||||
warning(
|
||||
`Gateway health inspection could not be prepared: ${formatGatewayHealthDiagnostic(error)}`,
|
||||
"Fix Gateway connection configuration, then rerun `openclaw doctor --lint --only core/doctor/gateway-health`.",
|
||||
),
|
||||
];
|
||||
}
|
||||
const diagnostic = gatewayConnectErrorWasRateLimited(error)
|
||||
? {
|
||||
message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
fixHint: "Wait for the temporary authentication lockout to expire, then rerun doctor.",
|
||||
}
|
||||
: isGatewayCredentialsRequiredError(error) || isGatewaySecretRefUnavailableError(error)
|
||||
? {
|
||||
message:
|
||||
"Gateway status could not be inspected because this CLI has no usable token/password or paired device token for read-scope RPCs.",
|
||||
fixHint:
|
||||
"Configure the Gateway token/password or pair this device, then rerun the selected health check.",
|
||||
}
|
||||
: {
|
||||
message: `Gateway status could not be inspected: ${formatGatewayHealthDiagnostic(error)}`,
|
||||
fixHint:
|
||||
mode === "remote"
|
||||
? "Verify the remote Gateway URL, network path, TLS settings, and credentials."
|
||||
: "Start the Gateway service or run `openclaw doctor --fix` for service repair prompts.",
|
||||
};
|
||||
return [warning(diagnostic.message, diagnostic.fixHint)];
|
||||
}
|
||||
}
|
||||
|
||||
function gatewayRuntimeStatus(runtime: GatewayServiceRuntime | undefined): string | undefined {
|
||||
|
||||
@@ -1063,7 +1063,7 @@ function createGatewayHealthCheck(deps: CoreHealthCheckDeps): SplitHealthCheckDe
|
||||
return {
|
||||
id: GATEWAY_HEALTH_CHECK_ID,
|
||||
kind: "core",
|
||||
description: "Gateway reachability is represented as structured findings.",
|
||||
description: "Authenticated Gateway health and degraded secret owners are structured findings.",
|
||||
source: "doctor",
|
||||
defaultEnabled: false,
|
||||
async detect(ctx) {
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { SecretInput } from "../config/types.secrets.js";
|
||||
import { hasActiveGatewayExecCredential } from "./doctor-gateway-exec-credential.js";
|
||||
|
||||
const execRef = { source: "exec", provider: "vault", id: "PRIVATE_EDGE_REF_ID" } as const;
|
||||
type GatewayExecEdgeCredentialCase = {
|
||||
label: string;
|
||||
mode: "local" | "remote";
|
||||
targetUrl?: string;
|
||||
edgeAuth: Record<string, SecretInput>;
|
||||
expected: boolean;
|
||||
};
|
||||
|
||||
describe("hasActiveGatewayExecCredential", () => {
|
||||
it.each<GatewayExecEdgeCredentialCase>([
|
||||
{
|
||||
label: "the configured remote target uses an exec-backed edge header",
|
||||
mode: "remote" as const,
|
||||
edgeAuth: { "X-Edge-Auth": execRef },
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
label: "a matching target differs only by query parameters",
|
||||
mode: "remote" as const,
|
||||
targetUrl: "wss://gateway.example.test/rpc?profile=two",
|
||||
edgeAuth: { "X-Edge-Auth": execRef },
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
label: "the effective target is a different gateway",
|
||||
mode: "remote" as const,
|
||||
targetUrl: "wss://other-gateway.example.test/rpc",
|
||||
edgeAuth: { "X-Edge-Auth": execRef },
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
label: "a local gateway cannot use unrelated remote edge headers",
|
||||
mode: "local" as const,
|
||||
edgeAuth: { "X-Edge-Auth": execRef },
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
label: "matching literal and environment-backed headers do not execute",
|
||||
mode: "remote" as const,
|
||||
edgeAuth: {
|
||||
"X-Literal": "literal-edge-value",
|
||||
"X-Environment": { source: "env", provider: "default", id: "EDGE_TOKEN" } as const,
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
])("detects only effective exec edge credentials when $label", async (entry) => {
|
||||
const cfg: OpenClawConfig = {
|
||||
gateway: {
|
||||
mode: entry.mode,
|
||||
remote: { url: "wss://gateway.example.test/rpc", edgeAuth: entry.edgeAuth },
|
||||
},
|
||||
};
|
||||
|
||||
await expect(
|
||||
hasActiveGatewayExecCredential({ cfg, env: {}, targetUrl: entry.targetUrl }),
|
||||
).resolves.toBe(entry.expected);
|
||||
});
|
||||
});
|
||||
@@ -3,6 +3,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
export async function hasActiveGatewayExecCredential(params: {
|
||||
cfg: OpenClawConfig;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
targetUrl?: string;
|
||||
}): Promise<boolean> {
|
||||
const [{ resolveSecretInputRef }, { gatewaySecretInputPathCanWin }, secretPaths] =
|
||||
await Promise.all([
|
||||
@@ -11,7 +12,7 @@ export async function hasActiveGatewayExecCredential(params: {
|
||||
import("../gateway/secret-input-paths.js"),
|
||||
]);
|
||||
const mode = params.cfg.gateway?.mode === "remote" ? "remote" : "local";
|
||||
return secretPaths.ALL_GATEWAY_SECRET_INPUT_PATHS.some((path) => {
|
||||
const hasExecCredential = secretPaths.ALL_GATEWAY_SECRET_INPUT_PATHS.some((path) => {
|
||||
if (
|
||||
!gatewaySecretInputPathCanWin({
|
||||
config: params.cfg,
|
||||
@@ -28,4 +29,23 @@ export async function hasActiveGatewayExecCredential(params: {
|
||||
}).ref;
|
||||
return ref?.source === "exec";
|
||||
});
|
||||
if (hasExecCredential || !params.cfg.gateway?.remote?.edgeAuth) {
|
||||
return hasExecCredential;
|
||||
}
|
||||
|
||||
const [{ buildGatewayProbeConnectionDetails }, edgeAuth] = await Promise.all([
|
||||
import("../gateway/call.js"),
|
||||
import("../gateway/edge-auth.js"),
|
||||
]);
|
||||
const targetUrl =
|
||||
params.targetUrl ?? (await buildGatewayProbeConnectionDetails({ config: params.cfg })).url;
|
||||
const { gatewayEdgeAuthValueForTarget, normalizeEdgeAuthHeadersConfig } = edgeAuth;
|
||||
const headers = normalizeEdgeAuthHeadersConfig(
|
||||
gatewayEdgeAuthValueForTarget({ config: params.cfg, targetUrl }),
|
||||
);
|
||||
return Object.values(headers ?? {}).some(
|
||||
(value) =>
|
||||
resolveSecretInputRef({ value, defaults: params.cfg.secrets?.defaults }).ref?.source ===
|
||||
"exec",
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user