kimetsu_brain/
redact.rs

1//! v0.4.5: secret redaction at ingest.
2//!
3//! Every memory.text and provenance snapshot that lands in brain.db
4//! passes through [`redact_secrets`] first. The pattern set catches
5//! the credential formats most likely to leak into an agent's tool
6//! output (OAuth bearers, API keys, JWTs, AWS access pairs, etc.)
7//! and replaces each match with a `[REDACTED:<kind>]` placeholder.
8//!
9//! Why redact at ingest, not later: brain.db is durable + content-
10//! addressable + replicated across user / project scopes. A leak
11//! that lands here lives forever and shows up in every retrieval
12//! capsule, agent.done summary, and proposal review screen. The
13//! cost of one false positive (a config string redacted) is much
14//! lower than the cost of one true positive sitting in
15//! `~/.kimetsu/brain.db` for years.
16//!
17//! Detection layers, applied in order:
18//!   1. Exact-prefix patterns: `sk-ant-`, `sk-`, `ghp_`, `gho_`,
19//!      `ghu_`, `ghs_`, `ghr_`, `github_pat_`, `xox[bopasr]-`,
20//!      `AKIA`/`ASIA` (AWS access key IDs), `eyJ` (JWT header).
21//!   2. Generic key/token assignments: `api[_-]?key\s*=\s*<value>`,
22//!      `token\s*[:=]\s*<value>`, `password\s*[:=]\s*<value>`,
23//!      `bearer\s+<value>` (the value must look secret — high
24//!      entropy, no spaces, length ≥ 12).
25//!   3. High-entropy fallback (off by default): would catch random
26//!      base64 strings of length ≥ 40 with entropy > 4.5 bits/char.
27//!      Deferred to v0.4.5.1 so we don't false-positive on hashes,
28//!      ulids, and content-addressable refs.
29//!
30//! The redaction is greedy + non-overlapping: a string matched by
31//! pattern (1) won't be re-scanned for pattern (2). [`RedactionResult`]
32//! returns both the redacted text AND a per-kind tally so callers
33//! (chat REPL banner, CLI warning, MCP response field) can surface
34//! "we found 2 secrets in this memory, kinds: aws_access_key,
35//! github_pat" without re-parsing the redacted text.
36
37use std::sync::OnceLock;
38
39use regex::Regex;
40use serde::Serialize;
41
42/// One detected secret. `kind` is a stable string id usable in
43/// telemetry and human-readable output.
44#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
45pub struct RedactedMatch {
46    pub kind: &'static str,
47    /// Byte offset within the ORIGINAL (pre-redaction) text where
48    /// the match started.
49    pub start: usize,
50    /// Length of the matched run (bytes, original text).
51    pub len: usize,
52}
53
54/// Output of [`redact_secrets`]. `text` is the redacted string;
55/// `matches` is the per-secret tally + locations.
56#[derive(Debug, Clone, Default, Serialize)]
57pub struct RedactionResult {
58    pub text: String,
59    pub matches: Vec<RedactedMatch>,
60}
61
62impl RedactionResult {
63    pub fn was_redacted(&self) -> bool {
64        !self.matches.is_empty()
65    }
66    /// One-liner like `"redacted 2 secrets: github_pat, openai_api_key"`.
67    /// Empty when nothing was redacted.
68    pub fn summary(&self) -> String {
69        if self.matches.is_empty() {
70            return String::new();
71        }
72        let mut kinds: Vec<&'static str> = self.matches.iter().map(|m| m.kind).collect();
73        kinds.sort_unstable();
74        kinds.dedup();
75        format!(
76            "redacted {} secret{}: {}",
77            self.matches.len(),
78            if self.matches.len() == 1 { "" } else { "s" },
79            kinds.join(", ")
80        )
81    }
82}
83
84/// Redact `text` against the bundled secret pattern set. Returns
85/// the redacted version + per-match tally. Allocates only when at
86/// least one match is found — for the common case (clean text) the
87/// result borrows nothing and matches is empty.
88pub fn redact_secrets(text: &str) -> RedactionResult {
89    let patterns = patterns();
90    // Collect non-overlapping matches across all patterns. Each
91    // pattern walks the full text; we sort + dedupe by start, then
92    // discard overlapping later matches.
93    let mut spans: Vec<(usize, usize, &'static str)> = Vec::new();
94    for pat in patterns {
95        for m in pat.regex.find_iter(text) {
96            spans.push((m.start(), m.end(), pat.kind));
97        }
98    }
99    if spans.is_empty() {
100        return RedactionResult {
101            text: text.to_string(),
102            matches: Vec::new(),
103        };
104    }
105    spans.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| b.1.cmp(&a.1)));
106    // Drop overlaps: keep the first span; skip any subsequent span
107    // whose start < current_end.
108    let mut accepted: Vec<(usize, usize, &'static str)> = Vec::new();
109    let mut cursor = 0usize;
110    for (start, end, kind) in spans {
111        if start < cursor {
112            continue;
113        }
114        accepted.push((start, end, kind));
115        cursor = end;
116    }
117
118    // Rebuild the redacted text + match list.
119    let mut redacted = String::with_capacity(text.len());
120    let mut matches: Vec<RedactedMatch> = Vec::with_capacity(accepted.len());
121    let mut last = 0usize;
122    for (start, end, kind) in accepted {
123        redacted.push_str(&text[last..start]);
124        redacted.push_str(&format!("[REDACTED:{kind}]"));
125        matches.push(RedactedMatch {
126            kind,
127            start,
128            len: end - start,
129        });
130        last = end;
131    }
132    redacted.push_str(&text[last..]);
133    RedactionResult {
134        text: redacted,
135        matches,
136    }
137}
138
139struct SecretPattern {
140    kind: &'static str,
141    regex: Regex,
142}
143
144fn patterns() -> &'static [SecretPattern] {
145    static CELL: OnceLock<Vec<SecretPattern>> = OnceLock::new();
146    CELL.get_or_init(|| {
147        // Each pattern's `kind` is stable telemetry-grade ID; the
148        // regex MUST be anchored at a unique prefix so we don't
149        // shadow normal source text. Use word boundaries where the
150        // pattern's prefix isn't already distinctive.
151        let mut out = Vec::new();
152        out.push(SecretPattern {
153            kind: "anthropic_oauth",
154            // Anthropic OAuth tokens: `sk-ant-` prefix + opaque tail.
155            // Tail is at least 32 chars of [A-Za-z0-9_-].
156            regex: Regex::new(r"sk-ant-[A-Za-z0-9_-]{32,}").unwrap(),
157        });
158        out.push(SecretPattern {
159            kind: "openai_api_key",
160            // OpenAI keys: `sk-` (not `sk-ant-`) + 32+ chars.
161            // Negative lookahead isn't available in `regex`; we
162            // order anthropic_oauth FIRST so it claims those bytes
163            // before openai_api_key sees them.
164            regex: Regex::new(r"sk-[A-Za-z0-9_-]{32,}").unwrap(),
165        });
166        out.push(SecretPattern {
167            kind: "github_pat",
168            // Classic + fine-grained GitHub PATs.
169            // ghp_/gho_/ghu_/ghs_/ghr_ + 36 base62.
170            // github_pat_ + base62/underscore length 50+.
171            regex: Regex::new(
172                r"(?:ghp_|gho_|ghu_|ghs_|ghr_)[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{50,}",
173            )
174            .unwrap(),
175        });
176        out.push(SecretPattern {
177            kind: "slack_token",
178            regex: Regex::new(r"xox[bopasr]-[A-Za-z0-9-]{10,}").unwrap(),
179        });
180        out.push(SecretPattern {
181            kind: "aws_access_key",
182            // AKIA = long-lived, ASIA = STS temporary. Exactly 16
183            // uppercase alphanum after the prefix per AWS docs.
184            regex: Regex::new(r"(?:AKIA|ASIA)[A-Z0-9]{16}").unwrap(),
185        });
186        out.push(SecretPattern {
187            kind: "jwt",
188            // Three base64url segments separated by dots; first
189            // starts with `eyJ` (base64url of `{"`). Cap total at
190            // 4096 chars so a runaway match doesn't blow the line.
191            regex: Regex::new(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+").unwrap(),
192        });
193        out.push(SecretPattern {
194            kind: "private_key_pem",
195            // PEM-encoded private key BEGIN line — match the whole
196            // block so the entire payload is wiped.
197            regex: Regex::new(
198                r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----",
199            )
200            .unwrap(),
201        });
202        out.push(SecretPattern {
203            kind: "google_api_key",
204            // Google-style API keys: AIza + 35 char tail.
205            regex: Regex::new(r"AIza[0-9A-Za-z_-]{35}").unwrap(),
206        });
207        out.push(SecretPattern {
208            kind: "url_credentials",
209            // Credentials embedded in a connection-string URI:
210            // `scheme://user:password@host`. Common in env dumps and
211            // `.env` snippets (DATABASE_URL=postgres://u:p@host, redis://,
212            // amqp://, mongodb://, ...). The generic `password=` rule does
213            // NOT match this shape, so without it the password lands in
214            // brain.db in cleartext. Redact the `scheme://user:pass@` run;
215            // the host stays readable.
216            regex: Regex::new(r"(?i)\b[a-z][a-z0-9+.\-]*://[^\s:/@]+:[^\s:/@]{4,}@").unwrap(),
217        });
218        // Generic-assignment patterns. Lower-priority than the
219        // shape-specific ones above; ordered after them so e.g.
220        // `api_key=sk-...` claims the openai_api_key kind, not the
221        // generic_api_key kind.
222        out.push(SecretPattern {
223            kind: "generic_bearer",
224            // `Bearer <token>` in HTTP-style logs.
225            regex: Regex::new(r"(?i)bearer\s+[A-Za-z0-9_\-\.=]{12,}").unwrap(),
226        });
227        out.push(SecretPattern {
228            kind: "generic_api_key",
229            // `api_key = "..."` / `api-key:"..."` / `api_key=...`.
230            // Captures both quoted and bare values; value must be
231            // ≥12 chars of secret-looking content.
232            regex: Regex::new(r#"(?i)api[_\-]?key\s*[:=]\s*"?[A-Za-z0-9_\-]{12,}"?"#).unwrap(),
233        });
234        out.push(SecretPattern {
235            kind: "generic_token",
236            regex: Regex::new(r#"(?i)\btoken\s*[:=]\s*"?[A-Za-z0-9_\-\.]{20,}"?"#).unwrap(),
237        });
238        out.push(SecretPattern {
239            kind: "generic_password",
240            regex: Regex::new(r#"(?i)\bpassword\s*[:=]\s*"?[^\s"]{8,}"?"#).unwrap(),
241        });
242        out
243    })
244}
245
246#[cfg(test)]
247mod tests {
248    use super::*;
249
250    #[test]
251    fn clean_text_round_trips_untouched() {
252        let raw = "use ripgrep before broad file reads, prefer thiserror for errors";
253        let r = redact_secrets(raw);
254        assert!(!r.was_redacted());
255        assert_eq!(r.text, raw);
256        assert!(r.summary().is_empty());
257    }
258
259    #[test]
260    fn anthropic_oauth_token_is_redacted() {
261        let raw =
262            "export CLAUDE_CODE_OAUTH_TOKEN=sk-ant-api03-AbCdEfGhIjKlMnOpQrStUv0123456789AbCdEf";
263        let r = redact_secrets(raw);
264        assert!(r.was_redacted(), "{:?}", r);
265        assert!(r.text.contains("[REDACTED:anthropic_oauth]"));
266        assert!(!r.text.contains("sk-ant-api03"));
267        assert_eq!(r.matches.len(), 1);
268        assert_eq!(r.matches[0].kind, "anthropic_oauth");
269    }
270
271    #[test]
272    fn openai_key_is_redacted_without_shadowing_anthropic_prefix() {
273        // Ensure pattern ordering: anthropic claims sk-ant-... first.
274        let raw =
275            "two: sk-1234567890abcdef1234567890abcdef AND sk-ant-1234567890abcdef1234567890abcdef";
276        let r = redact_secrets(raw);
277        let kinds: Vec<_> = r.matches.iter().map(|m| m.kind).collect();
278        assert!(kinds.contains(&"openai_api_key"));
279        assert!(kinds.contains(&"anthropic_oauth"));
280        assert!(r.text.contains("[REDACTED:openai_api_key]"));
281        assert!(r.text.contains("[REDACTED:anthropic_oauth]"));
282    }
283
284    #[test]
285    fn url_embedded_credentials_are_redacted() {
286        let raw = "DATABASE_URL=postgres://admin:S3cr3tP4ssw0rd@db.internal:5432/prod";
287        let r = redact_secrets(raw);
288        assert!(r.was_redacted(), "{:?}", r);
289        assert!(r.text.contains("[REDACTED:url_credentials]"));
290        assert!(!r.text.contains("S3cr3tP4ssw0rd"));
291        // Host stays readable; only the credentials run is wiped.
292        assert!(r.text.contains("db.internal:5432/prod"));
293
294        // Other common schemes are covered too.
295        let redis = redact_secrets("redis://default:An0therSecret123@cache:6379");
296        assert!(redis.text.contains("[REDACTED:url_credentials]"));
297        assert!(!redis.text.contains("An0therSecret123"));
298
299        // A URL without credentials must NOT be redacted.
300        let clean = redact_secrets("see https://example.com/path?x=1 for docs");
301        assert!(!clean.was_redacted(), "{:?}", clean);
302    }
303
304    #[test]
305    fn github_pat_classic_and_fine_grained_redacted() {
306        // Realistic lengths: classic PATs are `ghp_` + 36 base62;
307        // fine-grained PATs are `github_pat_` + ≥50 base62/underscore.
308        let raw = concat!(
309            "classic: ghp_abcdefghijklmnopqrstuvwxyzABCDEFGHIJ ",
310            "fine: github_pat_11AAA_abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJabcdef",
311        );
312        let r = redact_secrets(raw);
313        let kinds: Vec<_> = r.matches.iter().map(|m| m.kind).collect();
314        assert_eq!(
315            kinds.iter().filter(|k| **k == "github_pat").count(),
316            2,
317            "two github_pat matches expected; got matches: {:?}",
318            r.matches
319        );
320        assert!(!r.text.contains("ghp_abcdef"));
321        assert!(!r.text.contains("github_pat_11AAA"));
322    }
323
324    #[test]
325    fn slack_aws_jwt_pem_google_all_redact() {
326        let raw = concat!(
327            // Synthetic, non-functional token shaped to exercise the
328            // slack_token detector. The prefix is split so secret
329            // scanners don't flag the literal as a real credential.
330            "slack=",
331            "xoxb",
332            "-12345678-abcdefghijklmnop ",
333            "aws=AKIAIOSFODNN7EXAMPLE ",
334            "jwt=eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ0ZXN0In0.abcdef ",
335            "google=AIzaSyDx0o-1234567890abcdefghijklmnopqrs ",
336            "pem=-----BEGIN RSA PRIVATE KEY-----\nABCDEFG\n-----END RSA PRIVATE KEY-----"
337        );
338        let r = redact_secrets(raw);
339        let kinds: Vec<&'static str> = {
340            let mut k = r.matches.iter().map(|m| m.kind).collect::<Vec<_>>();
341            k.sort_unstable();
342            k.dedup();
343            k
344        };
345        for expected in [
346            "aws_access_key",
347            "google_api_key",
348            "jwt",
349            "private_key_pem",
350            "slack_token",
351        ] {
352            assert!(
353                kinds.contains(&expected),
354                "missing kind {expected}: {kinds:?}"
355            );
356        }
357    }
358
359    #[test]
360    fn generic_assignments_match_only_with_secret_looking_value() {
361        // Should match: long enough value.
362        let bad = "config: api_key = \"abcdef1234567890\" \n token : 0123456789abcdefghij1234567890\n password = hunter2hunter2";
363        let r_bad = redact_secrets(bad);
364        let kinds: Vec<_> = r_bad.matches.iter().map(|m| m.kind).collect();
365        assert!(kinds.contains(&"generic_api_key"));
366        assert!(kinds.contains(&"generic_token"));
367        assert!(kinds.contains(&"generic_password"));
368
369        // Should NOT match: too-short value or non-secret-looking.
370        let safe = "api_key = short  token: 12345  password = a";
371        let r_safe = redact_secrets(safe);
372        assert!(
373            r_safe.matches.is_empty(),
374            "short values should not trip generic patterns: {r_safe:?}"
375        );
376    }
377
378    #[test]
379    fn bearer_token_in_curl_log_is_redacted() {
380        let raw = "curl -H 'Authorization: Bearer abc123def456ghi789' https://api.example.com";
381        let r = redact_secrets(raw);
382        assert!(r.was_redacted());
383        assert_eq!(r.matches[0].kind, "generic_bearer");
384        assert!(r.text.contains("[REDACTED:generic_bearer]"));
385        assert!(!r.text.contains("abc123def456ghi789"));
386    }
387
388    #[test]
389    fn overlapping_matches_keep_first_only() {
390        // The same byte run could match two patterns (e.g. an
391        // openai_api_key inside a `Bearer ` prefix). The earlier
392        // pattern claims it; we don't double-redact.
393        let raw = "Authorization: Bearer sk-1234567890abcdef1234567890abcdef1234";
394        let r = redact_secrets(raw);
395        assert_eq!(
396            r.matches.len(),
397            1,
398            "non-overlapping rule should pick one: {r:?}"
399        );
400    }
401
402    #[test]
403    fn summary_lists_unique_kinds() {
404        let raw =
405            "ghp_abcdefghijklmnopqrstuvwxyzABCDEFGHIJ and ghp_ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ";
406        let r = redact_secrets(raw);
407        let summary = r.summary();
408        assert!(summary.contains("github_pat"));
409        // Only one kind reported even though two matches happened.
410        assert!(summary.starts_with("redacted 2 secrets: github_pat"));
411    }
412
413    #[test]
414    fn match_offsets_point_into_original_text() {
415        let raw = "prefix sk-ant-api03-1234567890abcdef1234567890abcdef suffix";
416        let r = redact_secrets(raw);
417        assert_eq!(r.matches.len(), 1);
418        let m = &r.matches[0];
419        // The matched bytes start at "sk-ant-..." and extend over the token.
420        let original_match = &raw[m.start..m.start + m.len];
421        assert!(original_match.starts_with("sk-ant-api03"));
422    }
423
424    #[test]
425    fn redaction_preserves_non_secret_surroundings() {
426        let raw = "# Save to .env\nCLAUDE_CODE_OAUTH_TOKEN=sk-ant-api03-AbCdEfGhIjKlMnOpQrStUv0123456789AbCdEf\n# Use it";
427        let r = redact_secrets(raw);
428        assert!(r.text.starts_with("# Save to .env"));
429        assert!(r.text.ends_with("# Use it"));
430        assert!(
431            r.text
432                .contains("CLAUDE_CODE_OAUTH_TOKEN=[REDACTED:anthropic_oauth]")
433        );
434    }
435}
kimetsu_brain/redact.rs

kimetsu_brain/
redact.rs
