fix(net-policy): redact sig and x-* auth params in URLs and bodies (#116957)

* fix(redact): redact sig and x-* auth params (redact-sensitive-url.ts)

* test(redact): redact sig and x-* auth params (redact-sensitive-url.test.ts)

* fix(redact): redact sig and x-* auth params (redact.ts)

* test(redact): redact sig and x-* auth params (redact.test.ts)

* fix(logging): unify URL credential redaction

---------

Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
This commit is contained in:
Paco Cartones
2026-08-01 04:04:13 +02:00
committed by GitHub
parent fdcb321bfa
commit cb3cd94756
4 changed files with 143 additions and 15 deletions
@@ -22,6 +22,16 @@ describe("redactSensitiveUrl", () => {
);
});
it("redacts signed and x-* auth aliases without matching adjacent metadata", () => {
expect(
redactSensitiveUrl(
"https://example.com/mcp?sig=one&X-Api-Key=two&x_access_token=three&x-auth-token=four&signal=keep&x-api-version=1",
),
).toBe(
"https://example.com/mcp?sig=***&X-Api-Key=***&x_access_token=***&x-auth-token=***&signal=keep&x-api-version=1",
);
});
it("redacts encoded and invisible-spliced sensitive query param names", () => {
expect(
redactSensitiveUrl("https://example.com/mcp?client%5Fse%E2%80%8Bcret=secret&safe=value"),
@@ -199,6 +209,16 @@ describe("redactSensitiveUrlLikeString", () => {
);
});
it("redacts signed and x-* auth aliases in invalid URL-like strings", () => {
expect(
redactSensitiveUrlLikeString(
"//example.com/mcp?sig=one&x-api-key=two&x-access-token=three&x-auth-token=four&safe=value",
),
).toBe(
"//example.com/mcp?sig=***&x-api-key=***&x-access-token=***&x-auth-token=***&safe=value",
);
});
it("redacts encoded and invisible-spliced query names in invalid URL-like strings", () => {
expect(
redactSensitiveUrlLikeString("//example.com/mcp?client%5Fse%E2%80%8Bcret=secret&safe=value"),
@@ -263,6 +283,14 @@ describe("isSensitiveUrlQueryParamName", () => {
expect(isSensitiveUrlQueryParamName("client_se+cret")).toBe(true);
expect(isSensitiveUrlQueryParamName("client_se\u3164cret")).toBe(true);
expect(isSensitiveUrlQueryParamName("credential")).toBe(true);
expect(isSensitiveUrlQueryParamName("sig")).toBe(true);
expect(isSensitiveUrlQueryParamName("X-Api-Key")).toBe(true);
expect(isSensitiveUrlQueryParamName("x-access-token")).toBe(true);
expect(isSensitiveUrlQueryParamName("x-auth-token")).toBe(true);
expect(isSensitiveUrlQueryParamName("signal")).toBe(false);
expect(isSensitiveUrlQueryParamName("sigmoid")).toBe(false);
expect(isSensitiveUrlQueryParamName("x-api-version")).toBe(false);
expect(isSensitiveUrlQueryParamName("x-request-id")).toBe(false);
expect(isSensitiveUrlQueryParamName("safe")).toBe(false);
});
});
@@ -36,6 +36,11 @@ const SENSITIVE_URL_QUERY_PARAM_NAMES = new Set([
"private_key",
"credential",
"authorization",
// Common signed/API gateway aliases that do not contain an existing secret-name marker.
"sig",
"x_api_key",
"x_access_token",
"x_auth_token",
]);
// Align with FORM_BODY_KEY_SEPARATOR_RE: category-Lo Hangul fillers can splice sensitive names.
const URL_QUERY_NAME_SEPARATOR_RE = /[\p{C}\p{Z}\u115F\u1160\u3164\uFFA0+]/gu;
+65 -2
View File
@@ -5,6 +5,7 @@ import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { withEnv } from "../test-utils/env.js";
import {
computeSensitiveRedactionBitmap,
getDefaultRedactPatterns,
redactSecrets,
redactSensitiveFieldValue,
@@ -755,11 +756,11 @@ describe("redactSensitiveText", () => {
it("masks punctuation inside unquoted credential-style header values", () => {
const keyHeader = ["api", "-", "key"].join("");
const output = redactSensitiveText(`${keyHeader}: prefix)sensitive-suffix`, {
const output = redactSensitiveText(`${keyHeader}: prefix)sensitive&suffix#tail`, {
mode: "tools",
});
expect(output).not.toContain("sensitive-suffix");
expect(output).not.toContain("sensitive&suffix#tail");
});
it("does not redact ordinary authorization prose", () => {
@@ -787,6 +788,31 @@ describe("redactSensitiveText", () => {
expect(output).not.toContain(apiKey);
});
it("masks URL punctuation inside named Gateway header values", () => {
expect(redactSensitiveText("X-Api-Key: prefix&secret#suffix", { mode: "tools" })).toBe(
"X-Api-Key: prefix…ffix",
);
expect(
redactSensitiveText("X-OpenClaw-Token=prefix&actual-secret#tail", { mode: "tools" }),
).toBe("X-OpenClaw-Token=prefix…tail");
expect(redactSensitiveText("x-access-token=prefix&actual-secret#tail", { mode: "tools" })).toBe(
"x-access-token=prefix…tail",
);
});
it("keeps equals-assignment bitmap masking aligned with form parsing", () => {
const resolved = resolveRedactOptions({ mode: "tools" });
const form = "x-access-token=short-at-123&safe=value";
const formBitmap = computeSensitiveRedactionBitmap(form, resolved);
const safePairStart = form.indexOf("&safe=");
expect(formBitmap.slice(form.indexOf("=") + 1, safePairStart).every(Boolean)).toBe(true);
expect(formBitmap.slice(safePairStart).some(Boolean)).toBe(false);
const header = "X-OpenClaw-Token=prefix&actual-secret#tail";
const headerBitmap = computeSensitiveRedactionBitmap(header, resolved);
expect(headerBitmap.slice(header.indexOf("=") + 1).every(Boolean)).toBe(true);
});
it("masks token prefixes embedded after adjacent text", () => {
const token = `ghp_${"a".repeat(5_000)}`;
const output = redactSensitiveText(`prefix-${token} suffix`, { mode: "tools" });
@@ -967,6 +993,38 @@ describe("redactSensitiveText", () => {
);
});
it("masks canonical URL auth aliases in form bodies without consuming safe fields", () => {
const input =
"sig=short-sig-123&x-api-key=short-key-123&x-access-token=short-at-123&x-auth-token=short-authtok&safe=value";
const output = redactSensitiveText(input, { mode: "tools" });
expect(output).toBe("sig=***&x-api-key=***&x-access-token=***&x-auth-token=***&safe=value");
expect(output).not.toContain("short-sig-123");
expect(output).not.toContain("short-key-123");
});
it("masks canonical URL auth aliases in generic URL text", () => {
const input =
"GET https://example.test/cb?sig=short-sig-123&X-Api-Key=long-api-key-1234567890&x-access-token=long-access-token-1234567890&x-auth-token=short-authtok&safe=value";
expect(redactSensitiveText(input, { mode: "tools" })).toBe(
"GET https://example.test/cb?sig=***&X-Api-Key=long-a…7890&x-access-token=long-a…7890&x-auth-token=***&safe=value",
);
});
it("reaches sig-only URLs and form bodies through the default prefilter", () => {
expect(redactSensitiveText("https://example.test/cb?sig=opaque-signed-value")).toBe(
"https://example.test/cb?sig=opaque…alue",
);
expect(redactSensitiveText("sig=opaque-signed-value&safe=visible")).toBe(
"sig=***&safe=visible",
);
});
it("preserves non-secret query names adjacent to signed and x-* aliases", () => {
const input =
"GET https://example.test/cb?signal=visible&sigmoid=visible&signature_algorithm=v4&x-api-version=1&x-request-id=req-123";
expect(redactSensitiveText(input, { mode: "tools" })).toBe(input);
});
it("masks URL userinfo and database connection-string passwords", () => {
const input = [
"https://browser-user:browser-password-1234567890@api.example.test/v1",
@@ -1754,6 +1812,11 @@ describe("redactSensitiveText", () => {
expect(redactSensitiveText("https://example.test/callback?security_code=123456")).toBe(
"https://example.test/callback?security_code=***",
);
expect(
redactSensitiveText(
"https://example.test/callback?id_token=id-value&private_key=private-value&x-amz-security-token=aws-value",
),
).toBe("https://example.test/callback?id_token=***&private_key=***&x-amz-security-token=***");
});
it("redacts standalone bearer tokens after the default prefilter", () => {
+45 -13
View File
@@ -1,5 +1,4 @@
import {
CREDENTIAL_STYLE_HEADER_REDACT_PATTERN,
findStructuredAuthParamRanges,
HTTP_AUTH_HEADER_BOUNDARY_PATTERN,
HTTP_AUTH_LEGACY_VALUE_WHITESPACE_PATTERN,
@@ -10,6 +9,7 @@ import {
HTTP_AUTH_SERIALIZED_QUOTE_PATTERN,
redactStructuredAuthHeaders,
} from "@openclaw/acp-core";
import { isSensitiveUrlQueryParamName } from "@openclaw/net-policy/redact-sensitive-url";
import { expectDefined } from "@openclaw/normalization-core";
// Redaction helpers scrub secrets and sensitive identifiers from log output.
import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
@@ -159,12 +159,27 @@ const AWS_SECRET_ACCESS_KEY_VALUE_PATTERN = String.raw`(?=[A-Za-z0-9/+=]{40}(?![
const AWS_SECRET_ACCESS_KEY_VALUE_REDACT_PATTERN = String.raw`/${AWS_SECRET_ACCESS_KEY_VALUE_BOUNDARY}(${AWS_SECRET_ACCESS_KEY_VALUE_PATTERN})(?!_)/g`;
const TELEGRAM_BOT_TOKEN_REDACT_PATTERN = String.raw`\bbot(\d{6,}:[A-Za-z0-9_-]{20,})\b`;
const TELEGRAM_TOKEN_REDACT_PATTERN = String.raw`\b(\d{6,}:[A-Za-z0-9_-]{20,})\b`;
const CREDENTIAL_STYLE_HEADER_KEYS = "x-goog-api-key|api-key|apikey|x-api-token|x-access-token";
const GATEWAY_SECURITY_HEADER_KEYS =
"X-OpenClaw-Token|x-pomerium-jwt-assertion|X-Api-Key|X-Auth-Token";
// Colons identify HTTP headers. Equals assignments may be form bodies, so stop only before an
// actual following `&key=` pair; otherwise opaque credential punctuation stays fully masked.
const LOG_HEADER_BOUNDARY_PATTERN = String.raw`(^|[^A-Za-z0-9_?&-]|\\{1,64}[rn])`;
const CREDENTIAL_STYLE_COLON_HEADER_REDACT_PATTERN = String.raw`${LOG_HEADER_BOUNDARY_PATTERN}(?:${CREDENTIAL_STYLE_HEADER_KEYS})${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*:${HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}([^\s\\"',;]+)`;
const CREDENTIAL_STYLE_EQUALS_ASSIGNMENT_REDACT_PATTERN = String.raw`${LOG_HEADER_BOUNDARY_PATTERN}(?:${CREDENTIAL_STYLE_HEADER_KEYS})${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*=${HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}([^\s\\"',;]+)`;
const GATEWAY_SECURITY_COLON_HEADER_REDACT_PATTERN = String.raw`${LOG_HEADER_BOUNDARY_PATTERN}(?:${GATEWAY_SECURITY_HEADER_KEYS})\s*:\s*([^\s"',;]+)`;
const GATEWAY_SECURITY_EQUALS_ASSIGNMENT_REDACT_PATTERN = String.raw`${LOG_HEADER_BOUNDARY_PATTERN}(?:${GATEWAY_SECURITY_HEADER_KEYS})\s*=\s*([^\s"',;]+)`;
const FORM_AWARE_EQUALS_ASSIGNMENT_PATTERN_SOURCES = new Set([
CREDENTIAL_STYLE_EQUALS_ASSIGNMENT_REDACT_PATTERN,
GATEWAY_SECURITY_EQUALS_ASSIGNMENT_REDACT_PATTERN,
]);
const HTTP_AUTH_HEADER_REDACT_PATTERNS = [
String.raw`${HTTP_AUTH_HEADER_BOUNDARY_PATTERN}Proxy-Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}${HTTP_AUTH_SCHEME_PATTERN}${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}(${HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN})`,
String.raw`${HTTP_AUTH_HEADER_BOUNDARY_PATTERN}Proxy-Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}(${HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN})[ \t]*(?=${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}(?:$|[,;)}\]]|\r?\n(?![ \t])))`,
String.raw`${HTTP_AUTH_HEADER_BOUNDARY_PATTERN}Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}(?!(?:Bearer|Basic|Bot)(?=${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}))${HTTP_AUTH_SCHEME_PATTERN}${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}(${HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN})`,
String.raw`${HTTP_AUTH_HEADER_BOUNDARY_PATTERN}Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}(?!(?:Bearer|Basic|Bot)(?=${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}))(${HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN})[ \t]*(?=${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}(?:$|[,;)}\]]|\r?\n(?![ \t])))`,
CREDENTIAL_STYLE_HEADER_REDACT_PATTERN,
CREDENTIAL_STYLE_COLON_HEADER_REDACT_PATTERN,
CREDENTIAL_STYLE_EQUALS_ASSIGNMENT_REDACT_PATTERN,
] as const;
const AUTHORIZATION_BEARER_REDACT_PATTERN = String.raw`Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_LEGACY_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}Bearer${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}(${HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN})`;
const AUTHORIZATION_BASIC_REDACT_PATTERN = String.raw`Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_LEGACY_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}Basic${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}(${HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN})`;
@@ -190,15 +205,15 @@ const shellReferencePreservingPatterns = new WeakSet<RegExp>();
// Patterns whose left-context assertions or complete token can cross a chunk boundary must run
// against the full string; chunking can invent a `^` boundary or split the secret itself.
const chunkUnsafePatterns = new WeakSet<RegExp>();
const formAwareEqualsAssignmentPatterns = new WeakSet<RegExp>();
const DEFAULT_REDACT_PATTERNS: string[] = [
// ENV-style assignments. Keep this case-sensitive so diagnostics like
// `Unrecognized key: "llm"` do not lose the actual config key.
ENV_ASSIGNMENT_REDACT_PATTERN,
ESCAPED_ENV_ASSIGNMENT_REDACT_PATTERN,
// URL query parameters. Keep this separate from ENV-style assignments so
// lower-case URL secrets stay redacted without hiding config-key diagnostics.
String.raw`/[?&](?:${AUTH_QUERY_KEYS}|${PAYMENT_CREDENTIAL_QUERY_KEYS})=([^&#\s<>]+)/gi`,
// URL pairs run through redactUrlQueryPairs first so net-policy owns both ordinary and
// obfuscated query-name classification without a second regex masking the value twice.
// JSON fields.
String.raw`"(?:apiKey|api_key|apiToken|api_token|bearerToken|bearer_token|token|secret|password|passwd|${AWS_SECRET_ACCESS_KEY_FIELD_KEYS}|credential|authorization|accessToken|access_token|refreshToken|refresh_token|idToken|id_token|authToken|auth_token|clientSecret|client_secret|privateKey|private_key|secret_value|raw_secret|secret_input|key_material|${PAYMENT_CREDENTIAL_JSON_KEYS})"\s*:\s*"([^"]+)"`,
// HTTP client diagnostics often stringify request config objects using
@@ -213,7 +228,8 @@ const DEFAULT_REDACT_PATTERNS: string[] = [
AUTHORIZATION_BASIC_REDACT_PATTERN,
AUTHORIZATION_BOT_REDACT_PATTERN,
...HTTP_AUTH_HEADER_REDACT_PATTERNS,
String.raw`(?:X-OpenClaw-Token|x-pomerium-jwt-assertion|X-Api-Key|X-Auth-Token)\s*[:=]\s*([^\s"',;]+)`,
GATEWAY_SECURITY_COLON_HEADER_REDACT_PATTERN,
GATEWAY_SECURITY_EQUALS_ASSIGNMENT_REDACT_PATTERN,
STANDALONE_BEARER_REDACT_PATTERN,
// URL userinfo and common connection-string password slots.
String.raw`\b(?:https?|wss?|ftp):\/\/[^\/\s:@]*:([^\/\s@]+)@`,
@@ -342,7 +358,7 @@ let defaultResolvedPatterns: RegExp[] | undefined;
const DEFAULT_REDACT_PREFILTER_SOURCES: string[] = [
// Sensitive key names shared by the env/JSON/query/form/header/assignment families.
String.raw`KEY|TOKEN|SECRET|PASSWORD|PASSWD|AUTH|COOKIE|SIGNATURE|CREDENTIAL|CARD|CVC|CVV|PAYMENT|PRIVATE KEY`,
String.raw`security[-_]?code|\bpass\s*[=:]|\bpassphrase\s*[=:]|_(?:password|pass|passphrase|passwd)\s*[=:]|jwt\s*[=:]|session=|code=`,
String.raw`security[-_]?code|\bpass\s*[=:]|\bpassphrase\s*[=:]|_(?:password|pass|passphrase|passwd)\s*[=:]|jwt\s*[=:]|session=|code=|\bsig\s*=`,
String.raw`\bBearer\s+`,
// URL userinfo and connection-string password slots (`scheme://user:pass@host`).
String.raw`:\/\/[^\/\s:@]*:[^\/\s@]+@`,
@@ -404,6 +420,9 @@ function parsePattern(raw: RedactPattern): RegExp | null {
if (pattern && typeof raw === "string" && SHELL_REFERENCE_PRESERVING_PATTERN_SOURCES.has(raw)) {
shellReferencePreservingPatterns.add(pattern);
}
if (pattern && typeof raw === "string" && FORM_AWARE_EQUALS_ASSIGNMENT_PATTERN_SOURCES.has(raw)) {
formAwareEqualsAssignmentPatterns.add(pattern);
}
if (
pattern &&
typeof raw === "string" &&
@@ -498,6 +517,13 @@ function splitSecretValueForMask(token: string): {
};
}
function splitFormAwareCredentialValue(token: string): { secret: string; suffix: string } {
const pairBoundary = token.search(/&[A-Za-z_][A-Za-z0-9_.-]*=/u);
return pairBoundary < 0
? { secret: token, suffix: "" }
: { secret: token.slice(0, pairBoundary), suffix: token.slice(pairBoundary) };
}
function maskSecretValue(token: string, options?: { hinted?: boolean }): string {
const { maskable, suffix } = splitSecretValueForMask(token);
return `${options?.hinted ? maskToken(maskable) : "***"}${suffix}`;
@@ -516,7 +542,7 @@ function normalizeSensitiveKeyName(value: string): string {
}
function isSensitiveBodyKey(key: string): boolean {
return BODY_SECRET_KEYS.has(normalizeSensitiveKeyName(key));
return isSensitiveUrlQueryParamName(key) || BODY_SECRET_KEYS.has(normalizeSensitiveKeyName(key));
}
function hasEncodedOrInvisibleFormKey(key: string): boolean {
@@ -595,7 +621,7 @@ function redactUrlQueryPairs(text: string): string {
return text;
}
return text.replace(URL_QUERY_PAIR_RE, (match, prefix: string, key: string, token: string) => {
if (!hasEncodedOrInvisibleFormKey(key) || !isSensitiveBodyKey(key)) {
if (!isSensitiveBodyKey(key)) {
return match;
}
return `${prefix}${key}=${maskSecretValue(token, { hinted: true })}`;
@@ -613,7 +639,7 @@ function markUrlQueryPairRedactions(text: string, bitmap: boolean[]): void {
const prefix = match[1] ?? "";
const key = match[2] ?? "";
const token = match[3] ?? "";
if (!hasEncodedOrInvisibleFormKey(key) || !isSensitiveBodyKey(key)) {
if (!isSensitiveBodyKey(key)) {
continue;
}
const secretValue = splitSecretValueForMask(token);
@@ -899,9 +925,12 @@ function redactMatch(
}
const selected = selectSecretCapture(match, groups);
const token = selected.value;
const formAwareValue = formAwareEqualsAssignmentPatterns.has(pattern)
? splitFormAwareCredentialValue(token)
: { secret: token, suffix: "" };
// An earlier pass (form-body or quoted-assignment masking) may already have replaced this
// value with ***; re-masking would strip its quote wrapper around the placeholder.
if (splitSecretValueForMask(token).maskable === "***") {
if (splitSecretValueForMask(formAwareValue.secret).maskable === "***") {
return match;
}
const isShellReferencePattern = shellReferencePreservingPatterns.has(pattern);
@@ -919,7 +948,7 @@ function redactMatch(
// retained hint instead of being exposed by delimiter-aware masking.
const masked = isShellReferencePattern
? maskToken(token)
: maskSecretValue(token, { hinted: true });
: `${maskSecretValue(formAwareValue.secret, { hinted: true })}${formAwareValue.suffix}`;
if (token === match) {
return masked;
}
@@ -1012,7 +1041,10 @@ function markPatternMatchRedaction(
if (tokenStart < 0) {
return;
}
const secretValue = splitSecretValueForMask(selected.value);
const selectedSecret = formAwareEqualsAssignmentPatterns.has(pattern)
? splitFormAwareCredentialValue(selected.value).secret
: selected.value;
const secretValue = splitSecretValueForMask(selectedSecret);
markBitmapRange(
bitmap,
match.index + tokenStart + secretValue.maskStart,