mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
fix(redaction): match SQLAlchemy driver schemes; add FE prefilter bailout
Connection-string redaction (the output_guard pattern and its frontend mirror) only enumerated bare dialects plus +psycopg, so SQLAlchemy dialect+driver URLs — postgresql+psycopg2://, postgresql+asyncpg://, mysql+pymysql:// — leaked the password through every redaction surface. The scheme now takes an optional +suffix instead of enumerating drivers. redactCredentials() also gains a single early-exit prefilter scan ahead of its sixteen replace passes, for plain-log tool output on card render. The prefilter is documented and pinned as a superset of the pattern set's required substrings, so a miss is provably a no-op: new smoke cases assert bare sk-/AKIA/Bearer credentials with no '=', quote or '@' anywhere in the text still redact, alongside the fast-path no-op and the driver-scheme URLs on both sides of the mirror.
This commit is contained in:
@@ -1328,6 +1328,28 @@ def test_redact_credentials_runtime_smoke() -> None:
|
||||
+ "const ak = redactCredentials('api_key=abcdefghijklmnopqrstuvwxyz');\n"
|
||||
+ "if (ak !== '[REDACTED:api_key]') "
|
||||
+ "throw new Error('api_key= clean redact failed: ' + ak);\n"
|
||||
+ "// Prefilter fast path: plain text with no anchor substring is unchanged\n"
|
||||
+ "const fp = redactCredentials('build ok in 42s - 3 tests passed');\n"
|
||||
+ "if (fp !== 'build ok in 42s - 3 tests passed') "
|
||||
+ "throw new Error('prefilter fast-path no-op failed: ' + fp);\n"
|
||||
+ "// Bare credentials with no =, quote or @ anywhere must still redact\n"
|
||||
+ "// (these pin the prefilter as a superset of the pattern set)\n"
|
||||
+ "const bk = redactCredentials('loaded sk-abcdefghijklmnopqrstuvwx');\n"
|
||||
+ "if (bk !== 'loaded [REDACTED:api_key]') "
|
||||
+ "throw new Error('bare sk- redact failed: ' + bk);\n"
|
||||
+ "const aw = redactCredentials('using AKIAABCDEFGHIJKLMNOP now');\n"
|
||||
+ "if (aw !== 'using [REDACTED:api_key] now') "
|
||||
+ "throw new Error('bare AKIA redact failed: ' + aw);\n"
|
||||
+ "const bt = redactCredentials('Bearer abcdefghijklmnopqrstuvwxyz');\n"
|
||||
+ "if (bt !== '[REDACTED:api_key]') "
|
||||
+ "throw new Error('bare bearer redact failed: ' + bt);\n"
|
||||
+ "// SQLAlchemy dialect+driver connection URLs (psycopg2/asyncpg)\n"
|
||||
+ "const pg2 = redactCredentials('postgresql+psycopg2://user:s3cret@db:5432/app');\n"
|
||||
+ "if (pg2 !== 'postgresql+psycopg2://user:[REDACTED:password]@db:5432/app') "
|
||||
+ "throw new Error('psycopg2 conn redact failed: ' + pg2);\n"
|
||||
+ "const apg = redactCredentials('postgresql+asyncpg://user:s3cret@db/app');\n"
|
||||
+ "if (apg !== 'postgresql+asyncpg://user:[REDACTED:password]@db/app') "
|
||||
+ "throw new Error('asyncpg conn redact failed: ' + apg);\n"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".mjs", delete=False) as f:
|
||||
f.write(harness)
|
||||
|
||||
@@ -236,6 +236,20 @@ class TestCredentialLeakage:
|
||||
assert r.sanitized is not None
|
||||
assert "s3cretpw" not in r.sanitized
|
||||
|
||||
def test_sqlalchemy_driver_connection_string(self) -> None:
|
||||
# SQLAlchemy dialect+driver URLs must match — the bare-dialect
|
||||
# list alone leaked these (only +psycopg was enumerated).
|
||||
for url in (
|
||||
"postgresql+psycopg2://admin:s3cret_pass@db.internal:5432/prod",
|
||||
"postgresql+asyncpg://admin:s3cret_pass@db.internal/prod",
|
||||
"mysql+pymysql://admin:s3cret_pass@db.internal/prod",
|
||||
):
|
||||
r = evaluate_output(url)
|
||||
assert "connection_string_leak" in r.flags, url
|
||||
assert r.sanitized is not None, url
|
||||
assert "s3cret_pass" not in r.sanitized, url
|
||||
assert ":[REDACTED:password]@" in r.sanitized, url
|
||||
|
||||
def test_bearer_scheme_case_insensitive(self) -> None:
|
||||
# RFC 7235 scheme name is case-insensitive.
|
||||
r = evaluate_output("authorization: bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.sig12345")
|
||||
|
||||
@@ -94,9 +94,12 @@ _RE_PRIVATE_KEY_BLOCK = re.compile(
|
||||
# ``redact_credentials`` — error persistence, audit details,
|
||||
# coordinator inspect/wait surfaces. The structural form ``[^:@\s]+:
|
||||
# [^@\s]+@`` is specific enough that ``https://example.com:8080/path``
|
||||
# (host:port without ``@``) doesn't match.
|
||||
# (host:port without ``@``) doesn't match. The optional ``+suffix``
|
||||
# covers SQLAlchemy dialect+driver URLs (``postgresql+psycopg2``,
|
||||
# ``postgresql+asyncpg``, ``mysql+pymysql``) and ``mongodb+srv`` —
|
||||
# enumerating drivers is a losing game, the suffix shape isn't.
|
||||
_RE_CONNECTION_STRING = re.compile(
|
||||
r"(?:postgresql\+?(?:psycopg)?|mysql|mongodb(?:\+srv)?|rediss?|amqps?|sqlite|https?)"
|
||||
r"(?:postgresql|mysql|mongodb|rediss?|amqps?|sqlite|https?)(?:\+[a-z0-9]*)?"
|
||||
r"://[^:@\s]+:[^@\s]+@",
|
||||
)
|
||||
_RE_ENV_SECRET_LINE = re.compile(r"[A-Z][A-Z_0-9]+=\S+")
|
||||
|
||||
@@ -18,6 +18,10 @@
|
||||
// 6. JSON secret keys → "secret": "[REDACTED:secret]"
|
||||
// 7. ENV secret lines → SECRET_KEY=[REDACTED:secret]
|
||||
//
|
||||
// A single prefilter scan (_RE_PREFILTER) bails out before all of the
|
||||
// above when the text cannot contain any credential — the common case
|
||||
// for plain-log tool output.
|
||||
//
|
||||
// House style: no innerHTML, no DOM access, no side-effects.
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -30,9 +34,13 @@ const _RE_PRIVATE_KEY_BLOCK =
|
||||
// Connection strings — preserves protocol + user, redacts only the password
|
||||
// postgresql://user:pass@host → postgresql://user:[REDACTED:password]@host
|
||||
// https://user:token@api.example.com → https://user:[REDACTED:password]@api.example.com
|
||||
// The optional +suffix covers SQLAlchemy dialect+driver URLs
|
||||
// (postgresql+psycopg2, postgresql+asyncpg, mysql+pymysql) and
|
||||
// mongodb+srv — enumerating drivers is a losing game, the suffix
|
||||
// shape isn't.
|
||||
// ---------------------------------------------------------------------------
|
||||
const _RE_CONNECTION_STRING =
|
||||
/(?:postgresql\+?(?:psycopg)?|mysql|mongodb(?:\+srv)?|rediss?|amqps?|sqlite|https?):\/\/[^:@\s]+:[^@\s]+@/g;
|
||||
/(?:postgresql|mysql|mongodb|rediss?|amqps?|sqlite|https?)(?:\+[a-z0-9]*)?:\/\/[^:@\s]+:[^@\s]+@/g;
|
||||
|
||||
const _RE_CONN_USERINFO = /:\/\/([^:@\s]+):([^@\s]+)@/;
|
||||
|
||||
@@ -64,11 +72,17 @@ const _CREDENTIAL_REPLACEMENTS = [
|
||||
// like "monkey=" or "turkey=". Bare "token=" included via negative
|
||||
// lookbehind so standalone assignments still match (token=abcdef...)
|
||||
// without matching word suffixes like "over_tokenized=".
|
||||
[/(?:(?:access|refresh|auth|api|session|bearer|secret)_?token|(?<![a-zA-Z0-9_])token)=[a-zA-Z0-9]{20,}/g, "[REDACTED:api_key]"],
|
||||
[
|
||||
/(?:(?:access|refresh|auth|api|session|bearer|secret)_?token|(?<![a-zA-Z0-9_])token)=[a-zA-Z0-9]{20,}/g,
|
||||
"[REDACTED:api_key]",
|
||||
],
|
||||
// key=<value> (20+). Same bounded prefix approach: api_key=/secret_key= etc.
|
||||
// but not monkey= or turkey=. Bare "key=" included with negative lookbehind.
|
||||
// Multi-segment keys secret_access_key / aws_secret_access_key included explicitly.
|
||||
[/(?:(?:api|secret|session|auth|encryption|signing|private|public|access|secret_access|aws_secret_access)_?key|(?<![a-zA-Z0-9_])key)=[a-zA-Z0-9]{20,}/g, "[REDACTED:api_key]"],
|
||||
[
|
||||
/(?:(?:api|secret|session|auth|encryption|signing|private|public|access|secret_access|aws_secret_access)_?key|(?<![a-zA-Z0-9_])key)=[a-zA-Z0-9]{20,}/g,
|
||||
"[REDACTED:api_key]",
|
||||
],
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -76,7 +90,8 @@ const _CREDENTIAL_REPLACEMENTS = [
|
||||
// ?api_key=abc123 → ?api_key=*** (legacy _redactApiKeys compat)
|
||||
// &secret=value → &secret=***
|
||||
// ---------------------------------------------------------------------------
|
||||
const _RE_QUERY_CRED = /(?:api_key|apiKey|api-key|(?<![a-zA-Z0-9_])token|secret|password|auth)=[^&\s"]+/g;
|
||||
const _RE_QUERY_CRED =
|
||||
/(?:api_key|apiKey|api-key|(?<![a-zA-Z0-9_])token|secret|password|auth)=[^&\s"]+/g;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// JSON-style simple redaction (legacy _redactApiKeys compat)
|
||||
@@ -119,6 +134,24 @@ function _redactEnvLine(match) {
|
||||
return match;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Prefilter — one early-exit scan deciding whether the pipeline can match.
|
||||
// MUST remain a superset of every pattern above: each pattern requires at
|
||||
// least one of these substrings, so skipping on a prefilter miss is sound.
|
||||
// Anchor → patterns:
|
||||
// = env lines, key=/token= assignments, query-string creds
|
||||
// " ' JSON-style and JSON-secret forms
|
||||
// @ connection-string userinfo
|
||||
// -----BEGIN PEM private key blocks
|
||||
// sk- ghp_ gho_ AKIA AIza bearer well-known key prefixes ("bearer" is
|
||||
// case-insensitive per RFC 7235; /i over-approximates the
|
||||
// case-sensitive prefixes, which only costs a full scan)
|
||||
// Adding a pattern above without an anchor here is a SILENT REDACTION
|
||||
// BYPASS — extend this regex and the runtime smoke test together
|
||||
// (tests/test_app_js.py::test_redact_credentials_runtime_smoke).
|
||||
// ---------------------------------------------------------------------------
|
||||
const _RE_PREFILTER = /[='"@]|-----BEGIN|sk-|ghp_|gho_|AKIA|AIza|bearer/i;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -138,6 +171,11 @@ export function redactCredentials(text) {
|
||||
|
||||
let result = String(text);
|
||||
|
||||
// Fast bailout — most tool output (plain logs, timestamps, table data)
|
||||
// carries no anchor substring; one early-exit scan skips the sixteen
|
||||
// replace passes below. Soundness argument lives on _RE_PREFILTER.
|
||||
if (!_RE_PREFILTER.test(result)) return result;
|
||||
|
||||
// 1. PEM private key blocks (whole-block removal)
|
||||
result = result.replace(_RE_PRIVATE_KEY_BLOCK, "[REDACTED:private_key]");
|
||||
|
||||
|
||||
Reference in New Issue
Block a user