Skip to main content

second_brain_core/redact/
mod.rs

1use std::sync::LazyLock;
2
3use regex::{Captures, Regex};
4
5use crate::schema::Memory;
6
7mod assign;
8mod keys;
9
10use assign::redact_assignments;
11use keys::redact_private_keys;
12
13#[non_exhaustive]
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
15pub enum SecretKind {
16    PrivateKey,
17    AwsAccessKeyId,
18    AwsSecretKey,
19    GithubToken,
20    ApiKey,
21    SlackToken,
22    SlackWebhook,
23    GoogleApiKey,
24    Jwt,
25    BearerToken,
26    BasicAuth,
27    UrlCredentials,
28    Secret,
29}
30
31impl SecretKind {
32    /// The `<kind>` written into `[REDACTED:<kind>]`.
33    pub fn as_str(self) -> &'static str {
34        match self {
35            SecretKind::PrivateKey => "private_key",
36            SecretKind::AwsAccessKeyId => "aws_access_key_id",
37            SecretKind::AwsSecretKey => "aws_secret_key",
38            SecretKind::GithubToken => "github_token",
39            SecretKind::ApiKey => "api_key",
40            SecretKind::SlackToken => "slack_token",
41            SecretKind::SlackWebhook => "slack_webhook",
42            SecretKind::GoogleApiKey => "google_api_key",
43            SecretKind::Jwt => "jwt",
44            SecretKind::BearerToken => "bearer_token",
45            SecretKind::BasicAuth => "basic_auth",
46            SecretKind::UrlCredentials => "url_credentials",
47            SecretKind::Secret => "secret",
48        }
49    }
50}
51
52#[non_exhaustive]
53#[derive(Debug, Clone, PartialEq, Eq)]
54pub struct Finding {
55    pub kind: SecretKind,
56    pub count: usize,
57}
58
59#[non_exhaustive]
60#[derive(Debug, Clone, PartialEq, Eq)]
61pub struct Redacted {
62    pub text: String,
63    /// One entry per kind found, in the order the kinds were first replaced.
64    pub findings: Vec<Finding>,
65}
66
67impl Redacted {
68    pub fn changed(&self) -> bool {
69        !self.findings.is_empty()
70    }
71}
72
73/// Replaces every secret in `text` with `[REDACTED:<kind>]`. Pure and
74/// idempotent: redacting already-redacted text returns it unchanged.
75pub fn redact(text: &str) -> Redacted {
76    let mut findings = Vec::new();
77    let mut current = redact_private_keys(text, &mut findings);
78    for rule in RULES.iter() {
79        if let Some(next) = rule.apply(current.as_deref().unwrap_or(text), &mut findings) {
80            current = Some(next);
81        }
82    }
83    if let Some(next) = redact_assignments(current.as_deref().unwrap_or(text), &mut findings) {
84        current = Some(next);
85    }
86    Redacted {
87        text: current.unwrap_or_else(|| text.to_string()),
88        findings,
89    }
90}
91
92/// The text every entry path embeds and stores: `redact`, logging the kinds
93/// found (never the values).
94pub fn redact_for_store(text: &str) -> String {
95    let redacted = redact(text);
96    if redacted.changed() {
97        let kinds = redacted
98            .findings
99            .iter()
100            .map(|f| format!("{}={}", f.kind.as_str(), f.count))
101            .collect::<Vec<_>>()
102            .join(",");
103        tracing::info!(%kinds, "redacted secrets before storing");
104    }
105    redacted.text
106}
107
108fn record(findings: &mut Vec<Finding>, kind: SecretKind, count: usize) {
109    match findings.iter_mut().find(|f| f.kind == kind) {
110        Some(f) => f.count += count,
111        None => findings.push(Finding { kind, count }),
112    }
113}
114
115fn marker(kind: SecretKind) -> String {
116    format!("[REDACTED:{}]", kind.as_str())
117}
118
119/// Decides from the captures and the final value whether a match is a secret.
120type Accept = fn(&Captures, &str) -> bool;
121
122struct Rule {
123    kind: SecretKind,
124    re: Regex,
125    accept: Accept,
126}
127
128impl Rule {
129    fn new(kind: SecretKind, pattern: &str, accept: Accept) -> Self {
130        Self {
131            kind,
132            re: Regex::new(pattern).expect("redaction pattern compiles"),
133            accept,
134        }
135    }
136
137    /// Replaces the `secret` group when the pattern has one, else the whole
138    /// match. A value glued to a marker is what an earlier redaction left, so
139    /// it is kept and a second pass changes nothing. `None` when nothing was
140    /// replaced.
141    ///
142    /// A rejected match resumes at its `secret` group, which may hold the
143    /// next match, but only when that group is at most `MAX_VALUE` bytes, so
144    /// each byte is rescanned a bounded number of times.
145    fn apply(&self, text: &str, findings: &mut Vec<Finding>) -> Option<String> {
146        let mut out = String::new();
147        let mut last = 0;
148        let mut count = 0;
149        let mut pos = 0;
150        while let Some(caps) = self.re.captures_at(text, pos) {
151            let whole = caps.get(0).expect("group 0 always matches");
152            let span = caps.name("secret").unwrap_or(whole);
153            let (start, end) = (span.start(), span.end());
154            let accepted = start >= last
155                && !text[end..].starts_with("[REDACTED:")
156                && (self.accept)(&caps, span.as_str());
157            if !accepted {
158                pos = if start > whole.start() && span.len() <= MAX_VALUE {
159                    start
160                } else {
161                    whole.end()
162                };
163                continue;
164            }
165            out.push_str(&text[last..start]);
166            out.push_str(&marker(self.kind));
167            last = end;
168            pos = end.max(whole.end());
169            count += 1;
170        }
171        if count == 0 {
172            return None;
173        }
174        out.push_str(&text[last..]);
175        record(findings, self.kind, count);
176        Some(out)
177    }
178}
179
180/// The longest value rescanned after a rejected match, extended over `&` or
181/// scanned after a secret-like name, which keeps every pass linear.
182const MAX_VALUE: usize = 512;
183
184fn always(_: &Captures, _: &str) -> bool {
185    true
186}
187
188fn group<'t>(caps: &Captures<'t>, name: &str) -> &'t str {
189    caps.name(name).map_or("", |m| m.as_str())
190}
191
192/// A run of at least `min` `member` bytes mixing upper and lower case and
193/// digits.
194fn has_random_run(s: &[u8], min: usize, member: fn(u8) -> bool) -> bool {
195    let (mut run, mut upper, mut lower, mut digit) = (0, false, false, false);
196    for &b in s {
197        if member(b) {
198            run += 1;
199            upper |= b.is_ascii_uppercase();
200            lower |= b.is_ascii_lowercase();
201            digit |= b.is_ascii_digit();
202        } else {
203            (run, upper, lower, digit) = (0, false, false, false);
204        }
205        if run >= min && upper && lower && digit {
206            return true;
207        }
208    }
209    false
210}
211
212/// Capitalised words, each ending in its digits (`Uint8Array`,
213/// `Ed25519PrivateKey`, `GPT2Tokenizer`). Random strings almost never have
214/// every capital followed by a lowercase letter.
215static TYPE_NAME: LazyLock<Regex> = LazyLock::new(|| {
216    Regex::new(r"^(?:[A-Z]{2,}[0-9]*)?(?:[A-Z][a-z]+[0-9]*)+$").expect("pattern compiles")
217});
218
219fn has_digit(s: &str) -> bool {
220    s.bytes().any(|b| b.is_ascii_digit())
221}
222
223fn mixed_case_with_digit(s: &str) -> bool {
224    s.bytes().any(|b| b.is_ascii_uppercase())
225        && s.bytes().any(|b| b.is_ascii_lowercase())
226        && has_digit(s)
227}
228
229fn is_placeholder(s: &str) -> bool {
230    s.starts_with(['$', '{', '<', '*', '%'])
231}
232
233fn sk_key_body(caps: &Captures, _: &str) -> bool {
234    let body = group(caps, "body");
235    let lowercase_run = body
236        .split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit()))
237        .any(|run| {
238            run.len() >= 32 && has_digit(run) && run.bytes().any(|b| b.is_ascii_lowercase())
239        });
240    mixed_case_with_digit(body) || lowercase_run || has_uuid(body)
241}
242
243/// A password up to the last `@` before the path, or else one holding `/`
244/// up to the first `@`. A `host:443/@scope` port, a password equal to the
245/// user and a stand-in such as `password` or `****` are not secrets.
246fn url_credentials(caps: &Captures, _: &str) -> bool {
247    let user = group(caps, "user");
248    let pass = caps
249        .name("pass")
250        .or_else(|| caps.name("slashed"))
251        .map_or("", |m| m.as_str());
252    let port = caps.name("slashed").is_some_and(|m| {
253        let digits = m.as_str().bytes().take_while(u8::is_ascii_digit).count();
254        digits > 0 && m.as_str()[digits..].starts_with('/')
255    });
256    let stand_in = pass == user
257        || ["password", "pass", "secret", "changeme"]
258            .iter()
259            .any(|word| pass.eq_ignore_ascii_case(word))
260        || pass.bytes().all(|b| matches!(b, b'*' | b'x' | b'X'));
261    let scp_path = caps.name("slashed").is_some() && user.contains('@');
262    caps.name("digest").is_none()
263        && !scp_path
264        && !is_placeholder(user)
265        && !is_placeholder(pass)
266        && !port
267        && !stand_in
268}
269
270/// Langfuse keys are `sk-lf-` and a UUID.
271fn has_uuid(s: &str) -> bool {
272    UUID.is_match(s)
273}
274
275static UUID: LazyLock<Regex> = LazyLock::new(|| {
276    Regex::new(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
277        .expect("pattern compiles")
278});
279
280/// `mysql` as the command: after a line start, a separator, a prompt, a JSON
281/// string or an escaped newline, or after a word that is not a flag (`sudo`,
282/// `VAR=value`, `exec db`, `ssh host`, `--`), so `--name mysql` and
283/// `/var/lib/mysql` are not.
284const MYSQL_COMMAND: &str = r#"(?m)(?:(?:^|[ \t])(?:[^\s\-]\S*|--)[ \t]+|(?:^|[;&|(`"']|\$|\\n)[ \t]*)(?:\S*bin/)?(?:mysql(?:dump|admin|import|check|show)?|mariadb)[ \t]"#;
285
286/// The start of the text, a non-word byte, or an escape in JSON-escaped text
287/// (`\nsk-...`). Never `]`: a token glued to the one before it gains a word
288/// boundary once that one becomes a marker, and redacting again must not find
289/// it.
290const TOKEN_START: &str = r"(?:^|[^A-Za-z0-9_\]]|\\[nrt])";
291
292static RULES: LazyLock<Vec<Rule>> = LazyLock::new(|| {
293    let token = |pattern: &str| format!("{TOKEN_START}(?P<secret>{pattern})");
294    vec![
295        Rule::new(
296            SecretKind::Jwt,
297            &token(r"eyJ[A-Za-z0-9_\-]{10,}\.eyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]*"),
298            always,
299        ),
300        Rule::new(
301            SecretKind::GithubToken,
302            &token(r"gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{22,}"),
303            always,
304        ),
305        Rule::new(
306            SecretKind::ApiKey,
307            &token(r"sk-(?P<body>[A-Za-z0-9_\-]{20,})"),
308            sk_key_body,
309        ),
310        Rule::new(
311            SecretKind::ApiKey,
312            &token(r"[rs]k_(?:live|test)_[A-Za-z0-9]{20,}"),
313            always,
314        ),
315        Rule::new(
316            SecretKind::ApiKey,
317            &token(r"glpat-[A-Za-z0-9_\-]{20,}|npm_[A-Za-z0-9]{36,}|hf_[A-Za-z0-9]{34,}"),
318            always,
319        ),
320        Rule::new(
321            SecretKind::SlackToken,
322            &token(
323                r"xoxd-[A-Za-z0-9%+/=\-]{10,}|(?:xoxe\.)?(?:xox[abposrce]|xapp)-[A-Za-z0-9\-]{10,}",
324            ),
325            |_, value| {
326                let digit_run = value
327                    .split(|c: char| !c.is_ascii_digit())
328                    .any(|run| run.len() >= 8);
329                digit_run || has_random_run(value.as_bytes(), 16, |b| b.is_ascii_alphanumeric())
330            },
331        ),
332        Rule::new(
333            SecretKind::SlackWebhook,
334            r"https://hooks\.slack\.com/services/[A-Za-z0-9/_\-]{20,}",
335            always,
336        ),
337        Rule::new(
338            SecretKind::GoogleApiKey,
339            &token(r"AIza[0-9A-Za-z_\-]{35}"),
340            always,
341        ),
342        Rule::new(
343            SecretKind::AwsAccessKeyId,
344            &token(
345                r"(?:AKIA|ASIA|ABIA|ACCA|AGPA|AIDA|AIPA|ANPA|ANVA|APKA|AROA|ASCA)[A-Z0-9]{16}\b",
346            ),
347            always,
348        ),
349        Rule::new(
350            SecretKind::AwsSecretKey,
351            r"\[REDACTED:aws_access_key_id\][ \t:,]+(?P<secret>[A-Za-z0-9/+=]{40})(?:[^A-Za-z0-9/+=]|$)",
352            always,
353        ),
354        Rule::new(
355            SecretKind::AwsSecretKey,
356            r#"(?i)(?:aws[_ ]?)?secret[_ ]?(?:access[_ ]?)?key(?:\\?["'])?(?:[ \t]*\[[^\]\n]*\])?(?:[ \t]*(?:=|:=|:|=>)[ \t]*|[ \t]+)(?:\\?["'])?(?P<secret>[A-Za-z0-9/+=]{40})(?:[^A-Za-z0-9/+=]|$)"#,
357            always,
358        ),
359        Rule::new(
360            SecretKind::BearerToken,
361            r#"(?i)\b(?:bearer|authorization(?:\\?["'])?\s*:\s*(?:\\?["'])?token)\s+(?P<secret>[A-Za-z0-9\-._~+/]{16,}=*)"#,
362            |_, value| {
363                has_digit(value)
364                    || (value.len() >= 20
365                        && value.bytes().any(|b| b.is_ascii_uppercase())
366                        && value.bytes().any(|b| b.is_ascii_lowercase())
367                        && !TYPE_NAME.is_match(value))
368            },
369        ),
370        Rule::new(
371            SecretKind::BasicAuth,
372            r#"(?i)\bauthorization(?:\\?["'])?\s*:\s*(?:\\?["'])?basic\s+(?P<secret>[A-Za-z0-9+/]{8,}={0,2})"#,
373            always,
374        ),
375        Rule::new(
376            SecretKind::UrlCredentials,
377            r#"\b[A-Za-z][A-Za-z0-9+.\-]*://(?P<secret>(?P<user>[^\s:/\[\]"'<>]*):(?:(?P<pass>[^\s/?#\[\]"'<>]+)|(?P<slashed>[^\s?#\[\]"'<>@]+)))@(?P<digest>sha(?:256|512):)?"#,
378            url_credentials,
379        ),
380        Rule::new(
381            SecretKind::BasicAuth,
382            r#"\bcurl\b[^\n|;&]*?\s(?:-u|--user)[ \t]*["']?(?P<user>[^\s:'"]+):(?P<secret>[^\s'"\[]+)"#,
383            |caps, value| !is_placeholder(group(caps, "user")) && !is_placeholder(value),
384        ),
385        Rule::new(
386            SecretKind::Secret,
387            &format!(r#"{MYSQL_COMMAND}(?:[^\n|;&]*?[ \t])?-p["']?(?P<secret>[^\s'"\[]+)"#),
388            |_, value| !is_placeholder(value),
389        ),
390        Rule::new(
391            SecretKind::Secret,
392            r#"\b(?:docker|podman|nerdctl|helm(?:[ \t]+registry)?)[ \t]+login\b[^\n|;&]*?\s-p[ \t]+["']?(?P<secret>[^\s'"\[]+)"#,
393            |_, value| !is_placeholder(value),
394        ),
395        Rule::new(
396            SecretKind::Secret,
397            r#"\bsshpass[ \t]+-p[ \t]*["']?(?P<secret>[^\s'"\[]+)"#,
398            |_, value| !is_placeholder(value),
399        ),
400        Rule::new(
401            SecretKind::Secret,
402            r"\b(?:machine\s+\S+|default)\s+login\s+\S+\s+password\s+(?P<secret>[^\s\[]+)",
403            |_, value| !is_placeholder(value),
404        ),
405    ]
406});
407
408/// Redacts content and context in place; `true` when either changed.
409pub fn redact_memory(memory: &mut Memory) -> bool {
410    let content = redact_for_store(&memory.content);
411    let mut changed = content != memory.content;
412    memory.content = content;
413    if let Some(context) = memory.context.as_deref() {
414        let redacted = redact_for_store(context);
415        changed |= redacted != context;
416        memory.context = Some(redacted);
417    }
418    changed
419}
420
421#[cfg(test)]
422mod tests;