mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 21:07:01 -06:00
feat(secrets): opt-in traffic allowlist for the secret egress proxy (#129880)
* feat(secrets): add opt-in traffic allowlist to the secret egress proxy Add secrets.egressProxy.allowedHosts: when present, the egress proxy refuses non-sentinel requests and CONNECT tunnels to hosts outside the effective allowed set (configured list, hosts bound to the run's registered secrets, and bypassHosts) with a typed host-not-allowed refusal naming the remediation. Empty array is lockdown; omitting the key keeps the previous unrestricted behavior. Sentinel substitution, per-secret destination binding, proxy auth, and bypass tunnels are unchanged; the allowlist is defense in depth for cooperating traffic since bypass-surviving sentinels remain the primary defense. * fix(secrets): validate egress proxy allowlist hostnames at the config boundary Extract the exact-host contract from the secret store into a canonical normalizeExactAllowedHost helper and validate secrets.egressProxy allowedHosts and bypassHosts entries through it. Schemes, ports, wildcards, and malformed hostnames are now rejected when config is accepted instead of throwing during egress-proxy startup, which the runtime normalizer did for both keys before this change. * fix(doctor): repair unusable secret egress proxy host entries Tightening the egress-proxy host schema can invalidate an existing config: a disabled proxy with a malformed bypassHosts entry loaded fine before and now fails validation, which exits the Gateway with code 78. Add the matching doctor --fix migration. Invalid entries never functioned (the proxy's own hostname normalizer threw on them at startup), so they are dropped rather than canonicalized; enabled and valid entries are preserved untouched.
This commit is contained in:
committed by
GitHub
parent
9f9ad63246
commit
4708c0b607
@@ -1303,6 +1303,7 @@ Default-off Gateway-owned substitution for shared-store `secret` entries used by
|
||||
secrets: {
|
||||
egressProxy: {
|
||||
enabled: false,
|
||||
allowedHosts: ["api.example.com"],
|
||||
bypassHosts: ["pinned-api.example.com"],
|
||||
},
|
||||
},
|
||||
@@ -1310,6 +1311,7 @@ Default-off Gateway-owned substitution for shared-store `secret` entries used by
|
||||
```
|
||||
|
||||
- `enabled`: starts the loopback proxy and ephemeral CA at Gateway startup. Default: `false`. Changing it requires a Gateway restart.
|
||||
- `allowedHosts`: optional exact-hostname traffic allowlist for proxy requests and CONNECT tunnels. When present, only listed hosts, hosts bound to a registered secret, and `bypassHosts` are reachable. An empty array permits only bound or bypassed hosts. Changing it requires a Gateway restart.
|
||||
- `bypassHosts`: optional exact-hostname list for authenticated blind CONNECT tunnels used by certificate-pinned clients. Sentinels are not substituted on bypassed hosts and fail vendor authentication without exposing plaintext.
|
||||
|
||||
See [Secret egress proxy](/gateway/secrets#secret-egress-proxy) for subprocess environment wiring, authentication, fail-closed behavior, and limitations.
|
||||
|
||||
@@ -348,6 +348,7 @@ Equivalent config:
|
||||
secrets: {
|
||||
egressProxy: {
|
||||
enabled: true,
|
||||
allowedHosts: ["api.openai.com"],
|
||||
bypassHosts: ["pinned-api.example.com"],
|
||||
},
|
||||
},
|
||||
@@ -373,8 +374,21 @@ The CA is generated once per Gateway start under the state directory. Its direct
|
||||
|
||||
`bypassHosts` contains exact hostnames that must remain end-to-end TLS for certificate-pinned clients. Those hosts use an authenticated blind CONNECT tunnel. No substitution is possible inside the tunnel; a sentinel sent there is safe by construction because it is authenticated ciphertext rather than a credential, so the vendor sees an invalid credential and rejects it.
|
||||
|
||||
### Traffic allowlist
|
||||
|
||||
Destination binding protects secrets, not traffic: a request that carries no sentinel can reach any host once a run holds proxy credentials. Set `secrets.egressProxy.allowedHosts` to also restrict where non-sentinel traffic may go:
|
||||
|
||||
```bash
|
||||
openclaw config set secrets.egressProxy.allowedHosts '["api.openai.com"]' --strict-json
|
||||
```
|
||||
|
||||
When the list is present, the proxy forwards only to hostnames in the list, hosts bound to a secret registered for the current agent run, and `bypassHosts`, so an existing `--allow-host` binding keeps working without listing its host twice. A request or CONNECT tunnel to any other host is refused with `Host "<host>" is not in the secret egress proxy traffic allowlist. Add it to secrets.egressProxy.allowedHosts or bind a store secret to it with: openclaw secrets store set <NAME> --allow-host <host>, then restart the Gateway.`
|
||||
|
||||
An empty array is lockdown mode: only per-secret bound hosts and `bypassHosts` remain reachable. Omitting `allowedHosts` leaves traffic unrestricted. Hostnames follow the same rules as secret bindings: exact lowercase ASCII/punycode match, no wildcards or ports. Restart the Gateway after changing the allowlist.
|
||||
|
||||
Current limits:
|
||||
|
||||
- The traffic allowlist constrains only cooperating clients that honor the proxy environment (`HTTPS_PROXY` and the CA variables). A subprocess can ignore those variables and open raw sockets, so the allowlist is defense in depth; destination-bound sentinels remain the primary defense because they survive proxy bypass.
|
||||
- HTTP/2 upstream connections are not supported; the proxy uses HTTP/1.1 upstream.
|
||||
- WebSocket rewriting is not supported.
|
||||
- Non-443 HTTPS substitution is not a supported compatibility target.
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { findLegacyConfigIssues } from "../../../config/legacy.js";
|
||||
import { validateConfigObjectRaw } from "../../../config/validation.js";
|
||||
import { LEGACY_CONFIG_MIGRATION_RUNTIME_SECRETS_EGRESS } from "./legacy-config-migrations.runtime.secrets-egress.js";
|
||||
|
||||
function applyAll(raw: Record<string, unknown>) {
|
||||
const changes: string[] = [];
|
||||
LEGACY_CONFIG_MIGRATION_RUNTIME_SECRETS_EGRESS.apply(raw, changes);
|
||||
return { raw, changes };
|
||||
}
|
||||
|
||||
describe("secret egress proxy hostname config migration", () => {
|
||||
it("repairs a disabled proxy with an unusable bypass host into a valid config", () => {
|
||||
const raw = {
|
||||
secrets: { egressProxy: { enabled: false, bypassHosts: ["api.example.com:443"] } },
|
||||
};
|
||||
|
||||
expect(validateConfigObjectRaw(raw).ok).toBe(false);
|
||||
expect(findLegacyConfigIssues(raw)).toContainEqual({
|
||||
path: "secrets.egressProxy.bypassHosts",
|
||||
message: expect.stringContaining("not usable hostnames"),
|
||||
});
|
||||
|
||||
const result = applyAll(raw);
|
||||
|
||||
expect(result.raw).toEqual({ secrets: { egressProxy: { enabled: false } } });
|
||||
expect(result.changes).toEqual([
|
||||
'Removed unusable secrets.egressProxy.bypassHosts entries: "api.example.com:443".',
|
||||
]);
|
||||
expect(validateConfigObjectRaw(result.raw).ok).toBe(true);
|
||||
});
|
||||
|
||||
it.each(["bypassHosts", "allowedHosts"] as const)(
|
||||
"preserves valid %s entries while dropping unusable entries",
|
||||
(key) => {
|
||||
const raw = {
|
||||
secrets: {
|
||||
egressProxy: {
|
||||
[key]: ["good.example.com", "https://bad.example.com"],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
expect(findLegacyConfigIssues(raw)).toContainEqual({
|
||||
path: `secrets.egressProxy.${key}`,
|
||||
message: expect.stringContaining("not usable hostnames"),
|
||||
});
|
||||
|
||||
const result = applyAll(raw);
|
||||
|
||||
expect(result.raw).toEqual({
|
||||
secrets: { egressProxy: { [key]: ["good.example.com"] } },
|
||||
});
|
||||
expect(result.changes).toEqual([
|
||||
`Removed unusable secrets.egressProxy.${key} entries: "https://bad.example.com".`,
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it("leaves valid host arrays unchanged without reporting legacy issues", () => {
|
||||
const raw = {
|
||||
secrets: {
|
||||
egressProxy: {
|
||||
enabled: false,
|
||||
allowedHosts: ["API.example.com.", "127.0.0.1", "API.example.com."],
|
||||
bypassHosts: ["good.example.com"],
|
||||
},
|
||||
},
|
||||
};
|
||||
const original = structuredClone(raw);
|
||||
|
||||
expect(findLegacyConfigIssues(raw)).toEqual([]);
|
||||
|
||||
const result = applyAll(raw);
|
||||
|
||||
expect(result.raw).toEqual(original);
|
||||
expect(result.changes).toEqual([]);
|
||||
});
|
||||
|
||||
it("detects and drops non-string host entries without throwing", () => {
|
||||
const raw = { secrets: { egressProxy: { bypassHosts: [123] } } };
|
||||
|
||||
expect(findLegacyConfigIssues(raw)).toContainEqual({
|
||||
path: "secrets.egressProxy.bypassHosts",
|
||||
message: expect.stringContaining("not usable hostnames"),
|
||||
});
|
||||
|
||||
const result = applyAll(raw);
|
||||
|
||||
expect(result.raw).toEqual({ secrets: { egressProxy: {} } });
|
||||
expect(result.changes).toEqual([
|
||||
"Removed unusable secrets.egressProxy.bypassHosts entries: 123.",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
import {
|
||||
defineLegacyConfigMigration,
|
||||
getRecord,
|
||||
type LegacyConfigMigrationSpec,
|
||||
type LegacyConfigRule,
|
||||
} from "../../../config/legacy.shared.js";
|
||||
import { normalizeExactAllowedHost } from "../../../secrets/exact-hostname.js";
|
||||
|
||||
const HOST_KEYS = ["bypassHosts", "allowedHosts"] as const;
|
||||
|
||||
const rule = (
|
||||
path: string[],
|
||||
message: string,
|
||||
match?: LegacyConfigRule["match"],
|
||||
): LegacyConfigRule => ({
|
||||
path,
|
||||
message: `${message} Run "openclaw doctor --fix".`,
|
||||
...(match ? { match } : {}),
|
||||
});
|
||||
|
||||
function isValidExactHostname(value: string): boolean {
|
||||
try {
|
||||
normalizeExactAllowedHost(value);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export const LEGACY_CONFIG_MIGRATION_RUNTIME_SECRETS_EGRESS: LegacyConfigMigrationSpec =
|
||||
defineLegacyConfigMigration({
|
||||
id: "runtime.secrets-egress-proxy-hosts",
|
||||
describe: "Drop unusable secret egress proxy host entries",
|
||||
legacyRules: HOST_KEYS.map((key) =>
|
||||
rule(
|
||||
["secrets", "egressProxy", key],
|
||||
`secrets.egressProxy.${key} contains entries that are not usable hostnames.`,
|
||||
(value) =>
|
||||
Array.isArray(value) &&
|
||||
value.some((entry) => typeof entry !== "string" || !isValidExactHostname(entry)),
|
||||
),
|
||||
),
|
||||
apply(raw, changes) {
|
||||
const egressProxy = getRecord(getRecord(raw.secrets)?.egressProxy);
|
||||
if (!egressProxy) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const key of HOST_KEYS) {
|
||||
const hosts = egressProxy[key];
|
||||
if (!Array.isArray(hosts)) {
|
||||
continue;
|
||||
}
|
||||
const validHosts = hosts.filter(
|
||||
(entry): entry is string => typeof entry === "string" && isValidExactHostname(entry),
|
||||
);
|
||||
if (validHosts.length === hosts.length) {
|
||||
continue;
|
||||
}
|
||||
const invalidHosts = hosts.filter(
|
||||
(entry) => typeof entry !== "string" || !isValidExactHostname(entry),
|
||||
);
|
||||
|
||||
// Invalid entries already prevented proxy startup, so dropping them loses no working policy.
|
||||
if (validHosts.length > 0) {
|
||||
egressProxy[key] = validHosts;
|
||||
} else {
|
||||
delete egressProxy[key];
|
||||
}
|
||||
changes.push(
|
||||
`Removed unusable secrets.egressProxy.${key} entries: ${invalidHosts
|
||||
.map((entry) => JSON.stringify(entry))
|
||||
.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -10,6 +10,7 @@ import { LEGACY_CONFIG_MIGRATIONS_RUNTIME_MCP } from "./legacy-config-migrations
|
||||
import { LEGACY_CONFIG_MIGRATIONS_RUNTIME_MODELS } from "./legacy-config-migrations.runtime.models.js";
|
||||
import { LEGACY_CONFIG_MIGRATIONS_RUNTIME_PROVIDERS } from "./legacy-config-migrations.runtime.providers.js";
|
||||
import { LEGACY_CONFIG_MIGRATIONS_RUNTIME_RETIRED } from "./legacy-config-migrations.runtime.retired.js";
|
||||
import { LEGACY_CONFIG_MIGRATION_RUNTIME_SECRETS_EGRESS } from "./legacy-config-migrations.runtime.secrets-egress.js";
|
||||
import { LEGACY_CONFIG_MIGRATIONS_RUNTIME_SESSION } from "./legacy-config-migrations.runtime.session.js";
|
||||
import { LEGACY_CONFIG_MIGRATIONS_RUNTIME_SKILLS } from "./legacy-config-migrations.runtime.skills.js";
|
||||
import { LEGACY_CONFIG_MIGRATIONS_RUNTIME_SYSTEM_AGENT } from "./legacy-config-migrations.runtime.system-agent.js";
|
||||
@@ -27,6 +28,7 @@ export const LEGACY_CONFIG_MIGRATIONS_RUNTIME: LegacyConfigMigrationSpec[] = [
|
||||
...LEGACY_CONFIG_MIGRATIONS_RUNTIME_MODELS,
|
||||
...LEGACY_CONFIG_MIGRATIONS_RUNTIME_PROVIDERS,
|
||||
...LEGACY_CONFIG_MIGRATIONS_RUNTIME_RETIRED,
|
||||
LEGACY_CONFIG_MIGRATION_RUNTIME_SECRETS_EGRESS,
|
||||
...LEGACY_CONFIG_MIGRATIONS_RUNTIME_SESSION,
|
||||
...LEGACY_CONFIG_MIGRATIONS_RUNTIME_SKILLS,
|
||||
...LEGACY_CONFIG_MIGRATIONS_RUNTIME_SYSTEM_AGENT,
|
||||
|
||||
@@ -26,6 +26,7 @@ describe("config secret refs schema", () => {
|
||||
secrets: {
|
||||
egressProxy: {
|
||||
enabled: true,
|
||||
allowedHosts: ["api.example.com"],
|
||||
bypassHosts: ["pinned.example.com"],
|
||||
},
|
||||
providers: {
|
||||
@@ -65,19 +66,37 @@ describe("config secret refs schema", () => {
|
||||
if (result.ok) {
|
||||
expect(result.config.secrets?.egressProxy).toEqual({
|
||||
enabled: true,
|
||||
allowedHosts: ["api.example.com"],
|
||||
bypassHosts: ["pinned.example.com"],
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects empty secret egress bypass hosts", () => {
|
||||
it.each(
|
||||
(["allowedHosts", "bypassHosts"] as const).flatMap((field) =>
|
||||
["", "https://api.example.com", "api.example.com:443", "*.example.com", "bad host"].map(
|
||||
(host) => ({ field, host }),
|
||||
),
|
||||
),
|
||||
)("rejects invalid secret egress $field entry $host", ({ field, host }) => {
|
||||
const result = validateConfigObjectRaw({
|
||||
secrets: { egressProxy: { enabled: false, bypassHosts: [""] } },
|
||||
secrets: { egressProxy: { enabled: false, [field]: [host] } },
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["allowedHosts", "bypassHosts"] as const)(
|
||||
"accepts exact hostname and IP secret egress %s entries",
|
||||
(field) => {
|
||||
const result = validateConfigObjectRaw({
|
||||
secrets: { egressProxy: { enabled: false, [field]: ["API.example.com.", "127.0.0.1"] } },
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects store refs outside the env-name grammar", () => {
|
||||
expect(
|
||||
validateOpenAiApiKeyRef({ source: "store", provider: "default", id: "lowercase" }).ok,
|
||||
|
||||
@@ -31,6 +31,8 @@ export const CORE_FIELD_HELP: Record<string, string> = {
|
||||
"Gateway-owned loopback proxy that replaces shared-store secret sentinels only at outbound request time. Restart the Gateway after changing this startup-scoped section.",
|
||||
"secrets.egressProxy.enabled":
|
||||
"Enables secret egress substitution for Gateway-hosted agent subprocesses. Default: false.",
|
||||
"secrets.egressProxy.allowedHosts":
|
||||
"Opt-in traffic allowlist for requests and CONNECT tunnels. When present, destinations must appear in this list, a per-secret host binding, or bypassHosts. An empty list permits only bound or bypassed hosts. Restart the Gateway after changing it.",
|
||||
"secrets.egressProxy.bypassHosts":
|
||||
"Exact hostnames that use authenticated blind CONNECT tunnels for certificate-pinned clients. Sentinels remain ciphertext and will fail vendor authentication instead of exposing plaintext.",
|
||||
wizard:
|
||||
|
||||
@@ -21,6 +21,7 @@ export const FIELD_LABELS: Record<string, string> = {
|
||||
secrets: "Secrets",
|
||||
"secrets.egressProxy": "Secret Egress Proxy",
|
||||
"secrets.egressProxy.enabled": "Secret Egress Proxy Enabled",
|
||||
"secrets.egressProxy.allowedHosts": "Secret Egress Proxy Allowed Hosts",
|
||||
"secrets.egressProxy.bypassHosts": "Secret Egress Proxy Bypass Hosts",
|
||||
wizard: "Setup Preferences",
|
||||
"wizard.accessMode": "Setup Discovery Access",
|
||||
|
||||
@@ -383,6 +383,7 @@ export type SecretProviderConfig =
|
||||
export type SecretsConfig = {
|
||||
egressProxy?: {
|
||||
enabled?: boolean;
|
||||
allowedHosts?: string[];
|
||||
bypassHosts?: string[];
|
||||
};
|
||||
providers?: Record<string, SecretProviderConfig>;
|
||||
|
||||
@@ -3,6 +3,7 @@ import path from "node:path";
|
||||
import { normalizeStringEntries } from "@openclaw/normalization-core/string-normalization";
|
||||
import { z } from "zod";
|
||||
import { isSafeExecutableValue } from "../infra/exec-safety.js";
|
||||
import { normalizeExactAllowedHost } from "../secrets/exact-hostname.js";
|
||||
import {
|
||||
formatExecSecretRefIdValidationMessage,
|
||||
isValidExecSecretRefId,
|
||||
@@ -195,6 +196,24 @@ const SecretsExecProviderSchema = z.union([
|
||||
|
||||
const SecretsStoreProviderSchema = z.object({ source: z.literal("store") }).strict();
|
||||
|
||||
// Same exact-host contract as per-secret destination bindings: rejecting schemes,
|
||||
// ports, wildcards, and malformed hostnames here keeps invalid entries out of the
|
||||
// egress-proxy startup path, which would otherwise throw while starting the Gateway.
|
||||
const EgressProxyExactHostSchema = z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.superRefine((host, ctx) => {
|
||||
try {
|
||||
normalizeExactAllowedHost(host);
|
||||
} catch (error) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
message: error instanceof Error ? error.message : "Invalid allowed host",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/** Schema for one configured env/file/exec/store secret provider entry. */
|
||||
export const SecretProviderSchema = z.union([
|
||||
SecretsEnvProviderSchema,
|
||||
@@ -209,7 +228,8 @@ export const SecretsConfigSchema = z
|
||||
egressProxy: z
|
||||
.object({
|
||||
enabled: z.boolean().optional(),
|
||||
bypassHosts: z.array(z.string().trim().min(1)).max(256).optional(),
|
||||
allowedHosts: z.array(EgressProxyExactHostSchema).max(256).optional(),
|
||||
bypassHosts: z.array(EgressProxyExactHostSchema).max(256).optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
|
||||
@@ -380,6 +380,7 @@ describe("buildGatewayReloadPlan", () => {
|
||||
"plugins.load.paths.0",
|
||||
"gateway.auth.mode",
|
||||
"secrets.egressProxy.enabled",
|
||||
"secrets.egressProxy.allowedHosts",
|
||||
"secrets.egressProxy.bypassHosts",
|
||||
])("keeps restart-owned path restart-backed: %s", (path) => {
|
||||
const plan = buildGatewayReloadPlan([path]);
|
||||
|
||||
@@ -159,11 +159,14 @@ export async function startGatewayCoreRuntime(input: {
|
||||
const secretEgressProxy =
|
||||
cfgAtStart.secrets?.egressProxy?.enabled === true
|
||||
? await import("../secrets/egress-proxy/runtime.js").then((egressRuntime) =>
|
||||
egressRuntime.startGatewaySecretEgressProxy(
|
||||
cfgAtStart.secrets?.egressProxy?.bypassHosts
|
||||
egressRuntime.startGatewaySecretEgressProxy({
|
||||
...(cfgAtStart.secrets?.egressProxy?.allowedHosts !== undefined
|
||||
? { allowedHosts: cfgAtStart.secrets.egressProxy.allowedHosts }
|
||||
: {}),
|
||||
...(cfgAtStart.secrets?.egressProxy?.bypassHosts
|
||||
? { bypassHosts: cfgAtStart.secrets.egressProxy.bypassHosts }
|
||||
: {},
|
||||
),
|
||||
: {}),
|
||||
}),
|
||||
)
|
||||
: undefined;
|
||||
if (secretEgressProxy) {
|
||||
|
||||
@@ -178,8 +178,12 @@ async function requestThroughTunnel(params: {
|
||||
return { body, status };
|
||||
}
|
||||
|
||||
async function forwardedRequest(auth?: string, protocol = "https"): Promise<number> {
|
||||
const proxyUrl = new URL(proxy.proxyOrigin);
|
||||
async function forwardedRequest(
|
||||
auth?: string,
|
||||
protocol = "https",
|
||||
proxyOrigin = proxy.proxyOrigin,
|
||||
): Promise<number> {
|
||||
const proxyUrl = new URL(proxyOrigin);
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
const request = httpRequest(
|
||||
{
|
||||
@@ -310,6 +314,115 @@ describe("secret egress proxy", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("forwards requests without sentinels to traffic-allowlisted hosts", async () => {
|
||||
const allowedEvents: SecretEgressProxyAuditEvent[] = [];
|
||||
const allowedProxy = await startSecretEgressProxyServer({
|
||||
caDir,
|
||||
allowedHosts: ["localhost"],
|
||||
onAudit: (event) => allowedEvents.push(event),
|
||||
});
|
||||
proxies.push(allowedProxy);
|
||||
|
||||
await expect(
|
||||
requestThroughTunnel({
|
||||
caPath: allowedProxy.caCertPath,
|
||||
proxyEnv: allowedProxy.registerRun(run),
|
||||
}),
|
||||
).resolves.toMatchObject({ body: "ok", status: 200 });
|
||||
|
||||
expect(originRequests).toHaveLength(1);
|
||||
expect(allowedEvents).toEqual([
|
||||
expect.objectContaining({ kind: "forwarded", host: "localhost", substituted: false }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("refuses unlisted tunnels and direct requests with traffic-allowlist remediation", async () => {
|
||||
const refusedEvents: SecretEgressProxyAuditEvent[] = [];
|
||||
const restrictedProxy = await startSecretEgressProxyServer({
|
||||
caDir,
|
||||
allowedHosts: ["api.example.com"],
|
||||
onAudit: (event) => refusedEvents.push(event),
|
||||
});
|
||||
proxies.push(restrictedProxy);
|
||||
const restrictedEnv = restrictedProxy.registerRun(run);
|
||||
const auth = basicProxyAuth(registeredPassword(restrictedEnv));
|
||||
|
||||
const refused = await rawConnect({ auth, proxyOrigin: restrictedProxy.proxyOrigin });
|
||||
expect(refused.response).toContain("403 Forbidden");
|
||||
expect(refused.response).toContain("secrets.egressProxy.allowedHosts");
|
||||
expect(refused.response).toContain("--allow-host localhost");
|
||||
refused.socket.destroy();
|
||||
|
||||
await expect(forwardedRequest(auth, "https", restrictedProxy.proxyOrigin)).resolves.toBe(403);
|
||||
expect(originRequests).toEqual([]);
|
||||
expect(refusedEvents).toEqual([
|
||||
expect.objectContaining({ kind: "refused", host: "localhost", reason: "host-not-allowed" }),
|
||||
expect.objectContaining({ kind: "refused", host: "localhost", reason: "host-not-allowed" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("allows sentinel-bound hosts during traffic-allowlist lockdown", async () => {
|
||||
const lockdownEvents: SecretEgressProxyAuditEvent[] = [];
|
||||
const lockdownProxy = await startSecretEgressProxyServer({
|
||||
caDir,
|
||||
allowedHosts: [],
|
||||
onAudit: (event) => lockdownEvents.push(event),
|
||||
});
|
||||
proxies.push(lockdownProxy);
|
||||
const secret = "lockdown-secret-value";
|
||||
const sentinel = mintSecretSentinel(secret, { label: "egress-lockdown" });
|
||||
|
||||
await expect(
|
||||
requestThroughTunnel({
|
||||
caPath: lockdownProxy.caCertPath,
|
||||
headers: { Authorization: `Bearer ${sentinel}` },
|
||||
proxyEnv: registerSentinel({
|
||||
sentinel,
|
||||
allowedHosts: ["localhost"],
|
||||
targetProxy: lockdownProxy,
|
||||
}),
|
||||
}),
|
||||
).resolves.toMatchObject({ body: "ok", status: 200 });
|
||||
|
||||
expect(originRequests.at(-1)?.headers.authorization).toBe(`Bearer ${secret}`);
|
||||
expect(lockdownEvents.at(-1)).toMatchObject({
|
||||
kind: "forwarded",
|
||||
host: "localhost",
|
||||
substituted: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps per-secret destination bindings narrower than the traffic allowlist", async () => {
|
||||
const restrictedEvents: SecretEgressProxyAuditEvent[] = [];
|
||||
const restrictedProxy = await startSecretEgressProxyServer({
|
||||
caDir,
|
||||
allowedHosts: ["localhost"],
|
||||
onAudit: (event) => restrictedEvents.push(event),
|
||||
});
|
||||
proxies.push(restrictedProxy);
|
||||
const secret = "wrong-destination-secret";
|
||||
const sentinel = mintSecretSentinel(secret, { label: "egress-wrong-destination" });
|
||||
|
||||
const result = await requestThroughTunnel({
|
||||
caPath: restrictedProxy.caCertPath,
|
||||
headers: { Authorization: `Bearer ${sentinel}` },
|
||||
proxyEnv: registerSentinel({
|
||||
sentinel,
|
||||
allowedHosts: ["api.example.com"],
|
||||
targetProxy: restrictedProxy,
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.status).toBe(502);
|
||||
expect(result.body).toContain("--allow-host localhost");
|
||||
expect(originRequests).toEqual([]);
|
||||
expect(restrictedEvents.at(-1)).toMatchObject({
|
||||
kind: "refused",
|
||||
host: "localhost",
|
||||
reason: "destination-not-allowed",
|
||||
});
|
||||
});
|
||||
|
||||
it("substitutes an authenticated header and strips proxy authorization upstream", async () => {
|
||||
const secret = "header-secret-value";
|
||||
const sentinel = mintSecretSentinel(secret, { label: "egress-header" });
|
||||
|
||||
@@ -284,6 +284,7 @@ function createUpstreamRequestOptions(params: {
|
||||
/** Starts one authenticated, loopback-only substitution proxy. */
|
||||
export async function startSecretEgressProxyServer(params: {
|
||||
caDir: string;
|
||||
allowedHosts?: readonly string[];
|
||||
bypassHosts?: readonly string[];
|
||||
onAudit: (event: SecretEgressProxyAuditEvent) => void;
|
||||
}): Promise<SecretEgressProxyHandle> {
|
||||
@@ -295,11 +296,28 @@ export async function startSecretEgressProxyServer(params: {
|
||||
ca: [...rootCertificates, caPem],
|
||||
});
|
||||
const bypassHosts = new Set((params.bypassHosts ?? []).map(normalizeHostname));
|
||||
const allowedHosts =
|
||||
params.allowedHosts === undefined
|
||||
? undefined
|
||||
: new Set(params.allowedHosts.map(normalizeHostname));
|
||||
const tokens = new Map<string, RegisteredRun>();
|
||||
const sockets = new Set<Socket>();
|
||||
const tlsServers = new Map<string, Promise<HttpsServer>>();
|
||||
|
||||
const audit = (event: SecretEgressProxyAuditEvent) => params.onAudit(event);
|
||||
const hostAllowed = (host: string, registered: RegisteredRun): boolean => {
|
||||
if (allowedHosts === undefined || allowedHosts.has(host) || bypassHosts.has(host)) {
|
||||
return true;
|
||||
}
|
||||
for (const binding of registered.sentinelBindings.values()) {
|
||||
if (binding.allowedHosts.has(host)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
const hostNotAllowedBody = (host: string): string =>
|
||||
`Host "${host}" is not in the secret egress proxy traffic allowlist. Add it to secrets.egressProxy.allowedHosts or bind a store secret to it with: openclaw secrets store set <NAME> --allow-host ${host}, then restart the Gateway.\n`;
|
||||
const authorize = (
|
||||
headers: IncomingHttpHeaders,
|
||||
): RegisteredRun | Exclude<SecretEgressRefusalReason, "destination-not-allowed"> => {
|
||||
@@ -338,6 +356,12 @@ export async function startSecretEgressProxyServer(params: {
|
||||
forward.request.resume();
|
||||
return;
|
||||
}
|
||||
if (!hostAllowed(host, forward.registered)) {
|
||||
audit({ kind: "refused", host, substituted: false, reason: "host-not-allowed" });
|
||||
sendHttpRefusal(forward.response, 403, hostNotAllowedBody(host));
|
||||
forward.request.resume();
|
||||
return;
|
||||
}
|
||||
let substituted = false;
|
||||
let target: URL;
|
||||
let headers: IncomingHttpHeaders;
|
||||
@@ -535,6 +559,19 @@ export async function startSecretEgressProxyServer(params: {
|
||||
upstream.once("error", () => clientSocket.destroy());
|
||||
return;
|
||||
}
|
||||
if (!hostAllowed(target.hostname, authorization)) {
|
||||
const body = hostNotAllowedBody(target.hostname);
|
||||
audit({
|
||||
kind: "refused",
|
||||
host: target.hostname,
|
||||
substituted: false,
|
||||
reason: "host-not-allowed",
|
||||
});
|
||||
clientSocket.end(
|
||||
`HTTP/1.1 403 Forbidden\r\nConnection: close\r\nContent-Length: ${Buffer.byteLength(body)}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n${body}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const tlsServer = await tlsServerFor(target, authorization);
|
||||
clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n");
|
||||
|
||||
@@ -27,6 +27,7 @@ function removeStaleProxyDirs(parentDir: string): void {
|
||||
|
||||
/** Starts the process-local proxy and registers it as the current Gateway owner. */
|
||||
export async function startGatewaySecretEgressProxy(params: {
|
||||
allowedHosts?: readonly string[];
|
||||
bypassHosts?: readonly string[];
|
||||
}): Promise<SecretEgressProxyHandle> {
|
||||
const parentDir = path.join(resolveStateDir(), "secret-egress-proxy");
|
||||
@@ -39,6 +40,7 @@ export async function startGatewaySecretEgressProxy(params: {
|
||||
try {
|
||||
proxy = await startSecretEgressProxyServer({
|
||||
caDir: proxyDir,
|
||||
...(params.allowedHosts !== undefined ? { allowedHosts: params.allowedHosts } : {}),
|
||||
...(params.bypassHosts ? { bypassHosts: params.bypassHosts } : {}),
|
||||
onAudit: (event) => log.info("secret egress request", event),
|
||||
});
|
||||
|
||||
@@ -10,6 +10,7 @@ const SENTINEL_PREFIX_BYTES = Buffer.from(SECRET_SENTINEL_PREFIX);
|
||||
const SENTINEL_SUFFIX_BYTES = Buffer.from(SECRET_SENTINEL_SUFFIX);
|
||||
|
||||
export type SecretEgressRefusalReason =
|
||||
| "host-not-allowed"
|
||||
| "invalid-proxy-auth"
|
||||
| "missing-proxy-auth"
|
||||
| "non-https-request"
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import net from "node:net";
|
||||
import { domainToASCII } from "node:url";
|
||||
|
||||
/**
|
||||
* Canonical exact-host contract shared by per-secret destination bindings and the
|
||||
* egress-proxy config allowlists: lowercase ASCII/punycode, unbracketed IP literals,
|
||||
* no wildcard, scheme, path, or port. Throws with an operator-actionable message so
|
||||
* config validation and the secret store surface the same policy.
|
||||
*/
|
||||
export function normalizeExactAllowedHost(raw: string): string {
|
||||
const trimmed = raw.trim().toLowerCase().replace(/\.+$/u, "");
|
||||
if (trimmed.includes("*")) {
|
||||
throw new Error(`Allowed host "${raw}" cannot contain a wildcard; use one exact hostname.`);
|
||||
}
|
||||
const unbracketed =
|
||||
trimmed.startsWith("[") && trimmed.endsWith("]") ? trimmed.slice(1, -1) : trimmed;
|
||||
if (net.isIP(unbracketed)) {
|
||||
return unbracketed;
|
||||
}
|
||||
if (!unbracketed || unbracketed.includes(":") || /[\s/?#@]/u.test(unbracketed)) {
|
||||
throw new Error(
|
||||
`Allowed host "${raw}" must be a hostname without a scheme, path, wildcard, or port.`,
|
||||
);
|
||||
}
|
||||
const ascii = domainToASCII(unbracketed);
|
||||
if (
|
||||
!ascii ||
|
||||
ascii.length > 253 ||
|
||||
ascii
|
||||
.split(".")
|
||||
.some(
|
||||
(label) =>
|
||||
!label ||
|
||||
label.length > 63 ||
|
||||
label.startsWith("-") ||
|
||||
label.endsWith("-") ||
|
||||
!/^[a-z0-9-]+$/u.test(label),
|
||||
)
|
||||
) {
|
||||
throw new Error(`Allowed host "${raw}" is not a valid hostname.`);
|
||||
}
|
||||
return ascii;
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
import net from "node:net";
|
||||
import { domainToASCII } from "node:url";
|
||||
import { err, ok, type Result } from "@openclaw/normalization-core/result";
|
||||
import type { Selectable } from "kysely";
|
||||
import { ENV_SECRET_REF_ID_RE } from "../../config/types.secrets.js";
|
||||
@@ -18,6 +16,7 @@ import {
|
||||
runOpenClawStateWriteTransaction,
|
||||
type OpenClawStateDatabaseOptions,
|
||||
} from "../../state/openclaw-state-db.js";
|
||||
import { normalizeExactAllowedHost } from "../exact-hostname.js";
|
||||
import { mintSecretSentinel } from "../sentinel.js";
|
||||
import {
|
||||
classifyHiddenGitHubStoreName,
|
||||
@@ -121,45 +120,14 @@ function assertSecretStoreValue(value: string, kind: SecretStoreKind): void {
|
||||
}
|
||||
|
||||
function normalizeSecretAllowedHost(raw: string): string {
|
||||
const trimmed = raw.trim().toLowerCase().replace(/\.+$/u, "");
|
||||
if (trimmed.includes("*")) {
|
||||
try {
|
||||
return normalizeExactAllowedHost(raw);
|
||||
} catch (error) {
|
||||
throw new SecretStoreValidationError(
|
||||
"SECRET_STORE_INVALID_ALLOWED_HOST",
|
||||
`Allowed host "${raw}" cannot contain a wildcard; use one exact hostname.`,
|
||||
error instanceof Error ? error.message : `Allowed host "${raw}" is not a valid hostname.`,
|
||||
);
|
||||
}
|
||||
const unbracketed =
|
||||
trimmed.startsWith("[") && trimmed.endsWith("]") ? trimmed.slice(1, -1) : trimmed;
|
||||
if (net.isIP(unbracketed)) {
|
||||
return unbracketed;
|
||||
}
|
||||
if (!unbracketed || unbracketed.includes(":") || /[\s/?#@]/u.test(unbracketed)) {
|
||||
throw new SecretStoreValidationError(
|
||||
"SECRET_STORE_INVALID_ALLOWED_HOST",
|
||||
`Allowed host "${raw}" must be a hostname without a scheme, path, wildcard, or port.`,
|
||||
);
|
||||
}
|
||||
const ascii = domainToASCII(unbracketed);
|
||||
if (
|
||||
!ascii ||
|
||||
ascii.length > 253 ||
|
||||
ascii
|
||||
.split(".")
|
||||
.some(
|
||||
(label) =>
|
||||
!label ||
|
||||
label.length > 63 ||
|
||||
label.startsWith("-") ||
|
||||
label.endsWith("-") ||
|
||||
!/^[a-z0-9-]+$/u.test(label),
|
||||
)
|
||||
) {
|
||||
throw new SecretStoreValidationError(
|
||||
"SECRET_STORE_INVALID_ALLOWED_HOST",
|
||||
`Allowed host "${raw}" is not a valid hostname.`,
|
||||
);
|
||||
}
|
||||
return ascii;
|
||||
}
|
||||
|
||||
export function normalizeSecretAllowedHosts(hosts: readonly string[]): string[] {
|
||||
|
||||
Reference in New Issue
Block a user