Skip to main content

mnemo_engine/
redact.rs

1//! 12.2 step 4: redacting known secret shapes from text before it's stored
2//! (`docs/design/12.2-onboard.md`, section 5). The hooks' transcript ingest
3//! always runs it; `mnemo thread ingest --redact` turns it on for a direct
4//! call.
5//!
6//! It's a best effort: it removes shapes it knows, each starting and ending
7//! at a word boundary and with a minimum length taken from the vendor's real
8//! format, so prose isn't redacted. Each match becomes `[redacted: <kind>]`.
9//! It works on any text: every position it cuts at is next to an ASCII
10//! byte, so it never splits a character. The shapes:
11//! - PEM private-key blocks, from the `BEGIN` line to the `END` line. A
12//!   marker with no partner (a block split across texts) takes the text up
13//!   to the end, or from the start, only when key material is next to it (a
14//!   `Proc-Type` header, or 40 or more base64 characters on the lines
15//!   beside it); a marker only mentioned in prose loses the marker alone;
16//! - AWS access key ids: `AKIA` or `ASIA` and exactly 16 of `[A-Z0-9]`;
17//! - GitHub tokens: `ghp_`, `gho_`, `ghu_`, `ghs_` or `ghr_` and at least
18//!   36 of `[A-Za-z0-9]`, or `github_pat_` and at least 22 of
19//!   `[A-Za-z0-9_]`;
20//! - Slack tokens: `xoxa-`, `xoxb-`, `xoxp-`, `xoxr-` or `xoxs-` and at
21//!   least 10 of `[A-Za-z0-9-]`;
22//! - API keys: `sk-` (including `sk-ant-` and `sk-proj-`) and at least 20
23//!   of `[A-Za-z0-9_-]`; `sk_live_` or `rk_live_` and at least 20 of
24//!   `[A-Za-z0-9]`; `AIza` and at least 35 of `[A-Za-z0-9_-]`; `glpat-`
25//!   and at least 20 of `[A-Za-z0-9_-]`; `npm_` and at least 36 of
26//!   `[A-Za-z0-9]`; `hf_` and at least 30 of `[A-Za-z0-9]`;
27//! - JWTs: three base64url parts joined by dots, the first starting `eyJ`,
28//!   each at least 10 characters;
29//! - MNemo recovery codes (`MN1-…`), checked with the recovery-code parser,
30//!   with hyphens, spaces or line breaks between the groups;
31//! - the value in `NAME=value`, `NAME: value` or `"name": "value"`:
32//!   - when the name ends in `PASSWORD` or `PASSPHRASE` (any case): any
33//!     value of 8 or more characters, and an unquoted one runs to the end
34//!     of the line;
35//!   - when it ends in `TOKEN`, `SECRET`, `_KEY`, `-KEY` or `APIKEY`: a
36//!     value of 8 or more characters that isn't a plain word (it has a
37//!     digit or a character outside `[A-Za-z_.-]`, or it mixes upper and
38//!     lower case and is 20 or more letters). Names whose last part before
39//!     `_KEY` or `-KEY` is `cache`, `sort`, `partition`, `primary`,
40//!     `foreign`, `unique`, `idempotency`, `dedup` or `row` are database
41//!     words, and stay;
42//!   - a value starting with `$`, `<`, `{{` or `%{` is a reference or a
43//!     placeholder, and stays;
44//! - the credential in `Authorization: <scheme> <credential>` (8 or more
45//!   characters), and after `Bearer ` (20 or more);
46//! - the word after `--passphrase` or `--password` (8 or more characters),
47//!   and after `--token` or `--key` when it's 16 or more of
48//!   `[A-Za-z0-9+/=_-]` and isn't a plain word (so a memory key like
49//!   `design.v2` or a file path stays);
50//! - the password in a URL's `scheme://user:password@`.
51
52use std::borrow::Cow;
53
54/// The kinds, in the order counts are reported.
55pub const KINDS: [&str; 8] = [
56    "private-key",
57    "aws-key-id",
58    "github-token",
59    "slack-token",
60    "api-key",
61    "jwt",
62    "recovery-code",
63    "secret-value",
64];
65
66/// What [`secrets`] found and removed.
67#[derive(Debug, Clone, PartialEq, Eq)]
68pub struct Redacted<'a> {
69    /// The text with every match replaced (borrowed when nothing matched).
70    pub text: Cow<'a, str>,
71    /// How many matches of each kind, in [`KINDS`] order, only those found.
72    pub counts: Vec<(&'static str, usize)>,
73}
74
75impl Redacted<'_> {
76    /// How many matches in all.
77    pub fn total(&self) -> usize {
78        self.counts.iter().map(|(_, n)| n).sum()
79    }
80}
81
82/// Sum the counts of several redactions, in [`KINDS`] order.
83pub fn add_counts(into: &mut Vec<(&'static str, usize)>, more: &[(&'static str, usize)]) {
84    for (k, n) in more {
85        match into.iter_mut().find(|(have, _)| have == k) {
86            Some((_, m)) => *m += n,
87            None => into.push((k, *n)),
88        }
89    }
90    into.sort_by_key(|(k, _)| KINDS.iter().position(|x| x == k).unwrap_or(usize::MAX));
91}
92
93type Spans = Vec<(usize, usize, &'static str)>;
94
95/// Replace every known secret shape in `text` with `[redacted: <kind>]`.
96pub fn secrets(text: &str) -> Redacted<'_> {
97    let mut spans: Spans = Vec::new();
98    pem_blocks(text, &mut spans);
99    tokens(text, &mut spans);
100    named_values(text, &mut spans);
101    flag_values(text, &mut spans);
102    url_passwords(text, &mut spans);
103    if spans.is_empty() {
104        return Redacted {
105            text: Cow::Borrowed(text),
106            counts: Vec::new(),
107        };
108    }
109    // Earliest first, the longest at a start; overlaps go to the first.
110    spans.sort_by(|a, b| a.0.cmp(&b.0).then(b.1.cmp(&a.1)));
111    let mut out = String::with_capacity(text.len());
112    let mut counts = Vec::new();
113    let mut at = 0;
114    for (s, e, kind) in spans {
115        if s < at {
116            continue;
117        }
118        out.push_str(&text[at..s]);
119        out.push_str("[redacted: ");
120        out.push_str(kind);
121        out.push(']');
122        add_counts(&mut counts, &[(kind, 1)]);
123        at = e;
124    }
125    out.push_str(&text[at..]);
126    Redacted {
127        text: Cow::Owned(out),
128        counts,
129    }
130}
131
132/// Whether `text` has `p` at byte `i`, ignoring ASCII case. Compares bytes,
133/// so `i` may be anywhere.
134fn has_at(text: &str, i: usize, p: &str) -> bool {
135    text.as_bytes()
136        .get(i..i + p.len())
137        .is_some_and(|b| b.eq_ignore_ascii_case(p.as_bytes()))
138}
139
140/// Whether `text` has `p` at byte `i`, exactly.
141fn has_exact(text: &str, i: usize, p: &str) -> bool {
142    text.as_bytes().get(i..i + p.len()) == Some(p.as_bytes())
143}
144
145const PEM_BEGIN: &str = "-----BEGIN ";
146const PEM_END: &str = "-----END ";
147const PEM_TAIL: &str = "PRIVATE KEY-----";
148
149/// The end of a PEM marker line starting at `at` (`-----BEGIN …` or
150/// `-----END …`) when it names a private key: the index after its closing
151/// `-----`.
152fn pem_marker_end(text: &str, at: usize, lead: &str) -> Option<usize> {
153    let rest = &text[at + lead.len()..];
154    let line_end = rest.find('\n').unwrap_or(rest.len());
155    let line = &rest[..line_end];
156    let i = line.find(PEM_TAIL)?;
157    // Only letters and spaces between the lead and the tail ("RSA ", "EC ",
158    // "OPENSSH ", "ENCRYPTED ", or nothing).
159    if !line[..i]
160        .chars()
161        .all(|c| c.is_ascii_uppercase() || c == ' ')
162    {
163        return None;
164    }
165    Some(at + lead.len() + i + PEM_TAIL.len())
166}
167
168fn base64_line(line: &str) -> bool {
169    !line.is_empty()
170        && line
171            .bytes()
172            .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'='))
173}
174
175/// How much key material the lines give, in base64 characters, counting
176/// whole base64 lines from the first until one isn't.
177fn key_material<'a>(lines: impl Iterator<Item = &'a str>) -> usize {
178    lines
179        .map(|l| l.trim_end_matches('\r'))
180        .take_while(|l| base64_line(l))
181        .map(str::len)
182        .sum()
183}
184
185/// Whether key material follows a BEGIN marker ending at `head_end`.
186fn material_after(text: &str, head_end: usize) -> bool {
187    let rest = &text[head_end..];
188    let Some(nl) = rest.find('\n') else {
189        return false;
190    };
191    let body = &rest[nl + 1..];
192    body.starts_with("Proc-Type:") || key_material(body.split('\n')) >= 40
193}
194
195/// Whether key material comes just before an END marker at `at`.
196fn material_before(text: &str, at: usize) -> bool {
197    let Some(head) = text[..at].strip_suffix('\n') else {
198        return false;
199    };
200    key_material(head.rsplit('\n')) >= 40
201}
202
203fn pem_blocks(text: &str, spans: &mut Spans) {
204    let mut from = 0;
205    let mut first_begin: Option<usize> = None;
206    while let Some(i) = text[from..].find(PEM_BEGIN).map(|i| from + i) {
207        if let Some(head_end) = pem_marker_end(text, i, PEM_BEGIN) {
208            first_begin.get_or_insert(i);
209            match find_end_marker(text, head_end) {
210                Some((_, end)) => {
211                    spans.push((i, end, "private-key"));
212                    from = end;
213                }
214                // No END: the block's other half is in a later text when key
215                // material follows; otherwise the marker is only mentioned.
216                None if material_after(text, head_end) => {
217                    spans.push((i, text.len(), "private-key"));
218                    from = text.len();
219                }
220                None => {
221                    spans.push((i, head_end, "private-key"));
222                    from = head_end;
223                }
224            }
225        } else {
226            from = i + PEM_BEGIN.len();
227        }
228    }
229    // END lines with no BEGIN before them: the first one's block began in an
230    // earlier text when key material comes before it.
231    let before = first_begin.unwrap_or(text.len());
232    let mut at = 0;
233    let mut first = true;
234    while let Some((s, e)) = find_end_marker(&text[..before], at) {
235        let start = if first && material_before(text, s) {
236            0
237        } else {
238            s
239        };
240        spans.push((start, e, "private-key"));
241        first = false;
242        at = e;
243    }
244}
245
246/// The first private-key END marker at or after `from`: its start, and the
247/// index after it.
248fn find_end_marker(text: &str, from: usize) -> Option<(usize, usize)> {
249    let mut at = from;
250    while let Some(i) = text[at..].find(PEM_END).map(|i| at + i) {
251        if let Some(e) = pem_marker_end(text, i, PEM_END) {
252            return Some((i, e));
253        }
254        at = i + PEM_END.len();
255    }
256    None
257}
258
259/// Whether the byte before `i` leaves a word boundary (any byte that isn't
260/// an ASCII letter, digit or `_`, including part of a wider character).
261fn boundary_before(text: &str, i: usize) -> bool {
262    i == 0 || {
263        let c = text.as_bytes()[i - 1];
264        !(c.is_ascii_alphanumeric() || c == b'_')
265    }
266}
267
268/// How many bytes from `at` are in `alphabet`.
269fn run(text: &str, at: usize, alphabet: impl Fn(u8) -> bool) -> usize {
270    text.as_bytes()
271        .get(at..)
272        .unwrap_or_default()
273        .iter()
274        .take_while(|b| alphabet(**b))
275        .count()
276}
277
278fn alnum(b: u8) -> bool {
279    b.is_ascii_alphanumeric()
280}
281
282fn upper_digit(b: u8) -> bool {
283    b.is_ascii_uppercase() || b.is_ascii_digit()
284}
285
286fn word(b: u8) -> bool {
287    b.is_ascii_alphanumeric() || b == b'_' || b == b'-'
288}
289
290/// Prefix tokens: `(start, end, kind)` for each match at a word boundary.
291fn tokens(text: &str, spans: &mut Spans) {
292    let bytes = text.as_bytes();
293    let mut i = 0;
294    while i < bytes.len() {
295        if !bytes[i].is_ascii_alphabetic() || !boundary_before(text, i) {
296            i += 1;
297            continue;
298        }
299        if let Some((end, kind)) = token_at(text, i) {
300            spans.push((i, end, kind));
301            i = end;
302        } else {
303            i += 1;
304        }
305    }
306}
307
308/// Prefixes followed by at least `min` bytes of an alphabet, and the kind.
309type Prefix = (&'static str, usize, fn(u8) -> bool, &'static str);
310
311const PREFIXES: [Prefix; 12] = [
312    (
313        "github_pat_",
314        22,
315        |b| b.is_ascii_alphanumeric() || b == b'_',
316        "github-token",
317    ),
318    (
319        "xoxa-",
320        10,
321        |b| b.is_ascii_alphanumeric() || b == b'-',
322        "slack-token",
323    ),
324    (
325        "xoxb-",
326        10,
327        |b| b.is_ascii_alphanumeric() || b == b'-',
328        "slack-token",
329    ),
330    (
331        "xoxp-",
332        10,
333        |b| b.is_ascii_alphanumeric() || b == b'-',
334        "slack-token",
335    ),
336    (
337        "xoxr-",
338        10,
339        |b| b.is_ascii_alphanumeric() || b == b'-',
340        "slack-token",
341    ),
342    (
343        "xoxs-",
344        10,
345        |b| b.is_ascii_alphanumeric() || b == b'-',
346        "slack-token",
347    ),
348    ("sk_live_", 20, alnum, "api-key"),
349    ("rk_live_", 20, alnum, "api-key"),
350    ("AIza", 35, word, "api-key"),
351    ("glpat-", 20, word, "api-key"),
352    ("npm_", 36, alnum, "api-key"),
353    ("hf_", 30, alnum, "api-key"),
354];
355
356/// The token starting at `i`, if one does, and where it ends.
357fn token_at(text: &str, i: usize) -> Option<(usize, &'static str)> {
358    let starts = |p: &str| has_exact(text, i, p);
359    // AWS: exactly 16 after the prefix, then not another of the alphabet.
360    for p in ["AKIA", "ASIA"] {
361        if starts(p) {
362            let n = run(text, i + 4, upper_digit);
363            if n == 16 && run(text, i + 4 + n, alnum) == 0 {
364                return Some((i + 4 + n, "aws-key-id"));
365            }
366        }
367    }
368    for p in ["ghp_", "gho_", "ghu_", "ghs_", "ghr_"] {
369        if starts(p) {
370            let n = run(text, i + 4, alnum);
371            if n >= 36 && run(text, i + 4 + n, |b| b == b'_') == 0 {
372                return Some((i + 4 + n, "github-token"));
373            }
374        }
375    }
376    for (p, min, alphabet, kind) in PREFIXES {
377        if starts(p) {
378            let n = run(text, i + p.len(), alphabet);
379            if n >= min {
380                return Some((i + p.len() + n, kind));
381            }
382        }
383    }
384    if starts("sk-") {
385        let n = run(text, i + 3, word);
386        if n >= 20 {
387            return Some((i + 3 + n, "api-key"));
388        }
389    }
390    if starts("eyJ") {
391        if let Some(end) = jwt_end(text, i) {
392            return Some((end, "jwt"));
393        }
394    }
395    if let Some(end) = recovery_code_end(text, i) {
396        return Some((end, "recovery-code"));
397    }
398    None
399}
400
401/// Bytes of an HTTP credential (RFC 9110's token68, and `=` padding).
402fn token68(b: u8) -> bool {
403    b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_' | b'~' | b'+' | b'/' | b'=')
404}
405
406/// The end of a recovery code at `i`: `MN1` and 65 symbols, with up to 8
407/// separators (hyphens, spaces, line breaks) between symbols, when the
408/// parser accepts it.
409fn recovery_code_end(text: &str, i: usize) -> Option<usize> {
410    let b = text.as_bytes();
411    if !has_at(text, i, "MN1")
412        || !b
413            .get(i + 3)
414            .is_some_and(|c| *c == b'-' || c.is_ascii_whitespace())
415    {
416        return None;
417    }
418    let sep = |c: u8| c == b'-' || c.is_ascii_whitespace();
419    let mut at = i + 3;
420    let mut symbols = 0;
421    while symbols < 65 {
422        let gap = run(text, at, sep);
423        if gap > 8 || !b.get(at + gap).is_some_and(|c| c.is_ascii_alphanumeric()) {
424            return None;
425        }
426        at += gap + 1;
427        symbols += 1;
428    }
429    crate::keys::recovery::parse(&text[i..at]).ok().map(|_| at)
430}
431
432/// The end of a JWT starting at `i` (three base64url parts of 10 or more,
433/// joined by dots).
434fn jwt_end(text: &str, i: usize) -> Option<usize> {
435    let b64 = |b: u8| b.is_ascii_alphanumeric() || b == b'_' || b == b'-';
436    let mut at = i;
437    for part in 0..3 {
438        let n = run(text, at, b64);
439        if n < 10 {
440            return None;
441        }
442        at += n;
443        if part < 2 {
444            if text.as_bytes().get(at) != Some(&b'.') {
445                return None;
446            }
447            at += 1;
448        }
449    }
450    Some(at)
451}
452
453/// Names whose values are secret at any length of 8 or more.
454const PASSWORD_SUFFIXES: [&str; 2] = ["password", "passphrase"];
455/// Names whose values are secret unless they're plain words.
456const NAME_SUFFIXES: [&str; 5] = ["token", "secret", "_key", "-key", "apikey"];
457/// The last part of a `…_key` name that makes it a database or cache word.
458const KEY_STEMS: [&str; 9] = [
459    "cache",
460    "sort",
461    "partition",
462    "primary",
463    "foreign",
464    "unique",
465    "idempotency",
466    "dedup",
467    "row",
468];
469
470/// How a name's value is judged.
471#[derive(Clone, Copy, PartialEq)]
472enum Name {
473    Password,
474    Secret,
475}
476
477fn name_kind(name: &str) -> Option<Name> {
478    let lower = name.to_ascii_lowercase();
479    if PASSWORD_SUFFIXES.iter().any(|s| lower.ends_with(s)) {
480        return Some(Name::Password);
481    }
482    let s = NAME_SUFFIXES.iter().find(|s| lower.ends_with(**s))?;
483    if s.ends_with("key") && s.len() == 4 {
484        let stem = &lower[..lower.len() - 4];
485        let last = stem.rsplit(['_', '-']).next().unwrap_or("");
486        if KEY_STEMS.contains(&last) {
487            return None;
488        }
489    }
490    Some(Name::Secret)
491}
492
493/// Values of names that look secret: `NAME=value`, `NAME: value`,
494/// `"name": "value"`; and `Authorization: <scheme> <credential>`.
495fn named_values(text: &str, spans: &mut Spans) {
496    let bytes = text.as_bytes();
497    let mut i = 0;
498    while i < bytes.len() {
499        let is_name = |b: u8| b.is_ascii_alphanumeric() || b == b'_' || b == b'-';
500        if !is_name(bytes[i]) || !boundary_before(text, i) {
501            i += 1;
502            continue;
503        }
504        let n = run(text, i, is_name);
505        let name = &text[i..i + n];
506        let after = i + n;
507        i = after.max(i + 1);
508        if name.eq_ignore_ascii_case("authorization") {
509            if let Some((s, e)) = authorization_value(text, after) {
510                spans.push((s, e, "secret-value"));
511                i = e;
512            }
513            continue;
514        }
515        // `Bearer <credential>`: the credential goes, the word stays.
516        if name.eq_ignore_ascii_case("bearer") && bytes.get(after) == Some(&b' ') {
517            let s = after + run(text, after, |c| c == b' ');
518            let n = run(text, s, token68);
519            if n >= 20 && !has_exact(text, s, "eyJ") {
520                spans.push((s, s + n, "secret-value"));
521                i = s + n;
522            }
523            continue;
524        }
525        // A flag's value is `flag_values`'s (a memory key after `--key=`
526        // isn't a secret).
527        if name.starts_with('-') {
528            continue;
529        }
530        let Some(kind) = name_kind(name) else {
531            continue;
532        };
533        if let Some((s, e, quoted)) = value_after(text, after) {
534            let e = if kind == Name::Password && !quoted {
535                line_end(text, s)
536            } else {
537                e
538            };
539            let v = &text[s..e];
540            let secret = match kind {
541                Name::Password => v.len() >= 8 && !reference(v),
542                Name::Secret => !plain(v),
543            };
544            if secret {
545                spans.push((s, e, "secret-value"));
546                i = e;
547            }
548        }
549    }
550}
551
552/// The end of the line at `s`, without trailing spaces or `\r`.
553fn line_end(text: &str, s: usize) -> usize {
554    let line = &text[s..];
555    let n = line.find('\n').unwrap_or(line.len());
556    s + line[..n].trim_end().len()
557}
558
559/// The credential after `Authorization`, ending at `at`: `: <scheme>
560/// <credential>` (or `=`, or quoted), when the credential is 8 or more.
561fn authorization_value(text: &str, at: usize) -> Option<(usize, usize)> {
562    let b = text.as_bytes();
563    let (s, _, _) = value_after(text, at)?;
564    let scheme = run(text, s, |c| c.is_ascii_alphabetic());
565    if scheme == 0 || b.get(s + scheme) != Some(&b' ') {
566        return None;
567    }
568    let c = s + scheme + run(text, s + scheme, |c| c == b' ');
569    let n = run(text, c, token68);
570    (n >= 8 && !has_exact(text, c, "eyJ")).then_some((c, c + n))
571}
572
573/// The value after a name ending at `at`: an optional closing quote, spaces,
574/// `=` or `:`, spaces, then a quoted or bare value. The third field says
575/// whether it was quoted.
576fn value_after(text: &str, at: usize) -> Option<(usize, usize, bool)> {
577    let b = text.as_bytes();
578    let mut j = at;
579    if b.get(j) == Some(&b'"') || b.get(j) == Some(&b'\'') {
580        j += 1;
581    }
582    while b.get(j) == Some(&b' ') {
583        j += 1;
584    }
585    if b.get(j) != Some(&b'=') && b.get(j) != Some(&b':') {
586        return None;
587    }
588    j += 1;
589    while b.get(j) == Some(&b' ') {
590        j += 1;
591    }
592    match b.get(j) {
593        Some(&q) if q == b'"' || q == b'\'' => {
594            let s = j + 1;
595            let len = b[s..].iter().take_while(|&&c| c != q && c != b'\n').count();
596            Some((s, s + len, true))
597        }
598        Some(_) => {
599            let len = b[j..]
600                .iter()
601                .take_while(|&&c| {
602                    !c.is_ascii_whitespace() && !matches!(c, b',' | b';' | b'}' | b')' | b'&')
603                })
604                .count();
605            Some((j, j + len, false))
606        }
607        None => None,
608    }
609}
610
611/// Whether a value is a reference or a placeholder (`$X`, `${x}`, `<x>`,
612/// `{{ x }}`, `%{x}`).
613fn reference(v: &str) -> bool {
614    v.starts_with('$') || v.starts_with('<') || v.starts_with("{{") || v.starts_with("%{")
615}
616
617/// Whether a value is too short or too plain to be a secret.
618fn plain(v: &str) -> bool {
619    if v.len() < 8 || reference(v) {
620        return true;
621    }
622    let has_digit = v.bytes().any(|c| c.is_ascii_digit());
623    let other = v
624        .bytes()
625        .any(|c| !(c.is_ascii_alphabetic() || matches!(c, b'_' | b'.' | b'-')));
626    let upper = v.bytes().any(|c| c.is_ascii_uppercase());
627    let lower = v.bytes().any(|c| c.is_ascii_lowercase());
628    // Mixed case alone counts only at a length an identifier rarely has
629    // (`useEnvironment` stays).
630    !(has_digit || other || (upper && lower && v.len() >= 20))
631}
632
633/// The word after `--passphrase`, `--password`, `--token` or `--key`.
634fn flag_values(text: &str, spans: &mut Spans) {
635    let b = text.as_bytes();
636    let mut from = 0;
637    while let Some(i) = text[from..].find("--").map(|i| from + i) {
638        from = i + 2;
639        if !boundary_before(text, i) || (i > 0 && b[i - 1] == b'-') {
640            continue;
641        }
642        let n = run(text, i + 2, |c| c.is_ascii_alphabetic());
643        let flag = &text[i + 2..i + 2 + n];
644        let password = ["passphrase", "password"].contains(&flag);
645        if !password && flag != "token" && flag != "key" {
646            continue;
647        }
648        let mut s = i + 2 + n;
649        match b.get(s) {
650            Some(b'=') => s += 1,
651            Some(b' ') => s += run(text, s, |c| c == b' '),
652            _ => continue,
653        }
654        let (s, e) = match b.get(s) {
655            Some(&q) if q == b'"' || q == b'\'' => {
656                let len = run(text, s + 1, |c| c != q && c != b'\n');
657                (s + 1, s + 1 + len)
658            }
659            _ => (s, s + run(text, s, |c| !c.is_ascii_whitespace())),
660        };
661        let v = &text[s..e];
662        let secret = if password {
663            v.len() >= 8 && !reference(v) && !v.starts_with('-')
664        } else {
665            v.len() >= 16
666                && v.bytes().all(|c| {
667                    c.is_ascii_alphanumeric() || matches!(c, b'+' | b'/' | b'=' | b'_' | b'-')
668                })
669                && !plain(v)
670        };
671        if secret {
672            spans.push((s, e, "secret-value"));
673            from = e;
674        }
675    }
676}
677
678/// The password in `scheme://user:password@host`.
679fn url_passwords(text: &str, spans: &mut Spans) {
680    let b = text.as_bytes();
681    let mut from = 0;
682    while let Some(i) = text[from..].find("://").map(|i| from + i) {
683        from = i + 3;
684        if i == 0 || !b[i - 1].is_ascii_alphanumeric() {
685            continue;
686        }
687        let s = i + 3;
688        let n = run(text, s, |c| {
689            !c.is_ascii_whitespace()
690                && !matches!(c, b'/' | b'?' | b'#' | b'@' | b'"' | b'\'' | b'<' | b'>')
691        });
692        if b.get(s + n) != Some(&b'@') {
693            continue;
694        }
695        let Some(colon) = text[s..s + n].find(':') else {
696            continue;
697        };
698        let (ps, pe) = (s + colon + 1, s + n);
699        if pe > ps && !reference(&text[ps..pe]) {
700            spans.push((ps, pe, "secret-value"));
701            from = pe;
702        }
703    }
704}
705
706#[cfg(test)]
707mod tests {
708    use super::*;
709
710    // Every secret-shaped test value is built at run time from parts, so
711    // no file holds one a secret scanner would flag (GitHub's flagged an
712    // `ASIA…` literal here once). AWS's documented example id stays.
713    const PK: &str = "PRIVATE KEY";
714
715    fn aws(prefix: &str) -> String {
716        format!("{prefix}{}", "ABCDEFGHIJKLMNOP")
717    }
718
719    fn kinds(text: &str) -> Vec<&'static str> {
720        secrets(text).counts.iter().map(|(k, _)| *k).collect()
721    }
722
723    /// Each true case is redacted, with its kind, and its secret part gone.
724    #[test]
725    fn the_true_cases() {
726        let key = crate::keys::recovery::encode(&[7u8; 32]);
727        let cases: Vec<(String, &str, &str)> = vec![
728            (
729                "id AKIAIOSFODNN7EXAMPLE here".into(),
730                "aws-key-id",
731                "AKIAIOSFODNN7EXAMPLE",
732            ),
733            (
734                format!("{}.", aws(concat!("AS", "IA"))),
735                "aws-key-id",
736                "ABCDEFGHIJKLMNOP",
737            ),
738            (
739                format!("token ghp_{}", "a1B2".repeat(9)),
740                "github-token",
741                "ghp_a1B2",
742            ),
743            (
744                format!("gho_{} end", "Z9".repeat(18)),
745                "github-token",
746                "gho_Z9",
747            ),
748            (
749                format!("github_pat_{}", "11ABC_def".repeat(3)),
750                "github-token",
751                "github_pat_11",
752            ),
753            (
754                format!("{}-1234567890-abcdefghij", concat!("xo", "xb")),
755                "slack-token",
756                "1234567890-abcd",
757            ),
758            (
759                format!("key {}-{}", concat!("s", "k"), "abcdefghijklmnopqrstuvwx"),
760                "api-key",
761                "abcdefghijklm",
762            ),
763            (
764                format!("sk-ant-api03-{}", "Ab9_".repeat(10)),
765                "api-key",
766                "sk-ant-api03",
767            ),
768            (
769                format!("sk-proj-{}", "x".repeat(30)),
770                "api-key",
771                "sk-proj-xxx",
772            ),
773            (
774                format!(
775                    "Bearer {}.{}.{}",
776                    concat!("ey", "JhbGciOiJIUzI1NiJ9"),
777                    concat!("ey", "JzdWIiOiIxMjM0NTY3ODkwIn0"),
778                    "dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"
779                ),
780                "jwt",
781                "eyJhbGciOiJIUzI1NiJ9",
782            ),
783            (
784                format!("the code is {}", key.as_str()),
785                "recovery-code",
786                &key.as_str()[4..14],
787            ),
788            (key.to_lowercase(), "recovery-code", "mn1-"),
789            ("API_KEY=a8f3k2m9q7".into(), "secret-value", "a8f3k2m9q7"),
790            (
791                "export GITHUB_TOKEN=Hunter2Hunter".into(),
792                "secret-value",
793                "Hunter2Hunter",
794            ),
795            (
796                "{\"password\": \"correct horse 99\"}".into(),
797                "secret-value",
798                "correct horse 99",
799            ),
800            (
801                "db_password: CorrectHorse".into(),
802                "secret-value",
803                "CorrectHorse",
804            ),
805            (
806                "MNEMO_KEY=4f2a9c0e1b3d5f7a".into(),
807                "secret-value",
808                "4f2a9c0e1b3d5f7a",
809            ),
810            (
811                format!(
812                    "-----BEGIN RSA {PK}-----\nMIIEpAIBAAKCAQEA\n-----END RSA {PK}-----\nafter"
813                ),
814                "private-key",
815                "MIIEpAIBAAKCAQEA",
816            ),
817            (
818                format!(
819                    "-----BEGIN OPENSSH {PK}-----\nb3BlbnNzaC1rZXk\n-----END OPENSSH {PK}-----"
820                ),
821                "private-key",
822                "b3BlbnNzaC1rZXk",
823            ),
824        ];
825        for (text, kind, gone) in &cases {
826            let r = secrets(text);
827            assert_eq!(kinds(text), [*kind], "{text}");
828            assert!(!r.text.contains(gone), "{text} -> {}", r.text);
829            assert!(
830                r.text.contains(&format!("[redacted: {kind}]")),
831                "{}",
832                r.text
833            );
834        }
835        // The text around a match stays.
836        assert_eq!(
837            secrets("id AKIAIOSFODNN7EXAMPLE here").text,
838            "id [redacted: aws-key-id] here"
839        );
840        assert_eq!(
841            secrets("API_KEY=a8f3k2m9q7 ok").text,
842            "API_KEY=[redacted: secret-value] ok"
843        );
844        let pem = format!("-----BEGIN RSA {PK}-----\nx\n-----END RSA {PK}-----\nafter");
845        assert!(secrets(&pem).text.ends_with("\nafter"));
846    }
847
848    const FALSE_CASES: [&str; 23] = [
849        "task-list",
850        "sk-1",
851        "the risk-assessment is done",
852        concat!("as", "k-for-help-with-this-long-name-here"),
853        "PRIMARY_KEY=id",
854        "SORT_KEY=created_at",
855        "password: hunter",
856        "the password field is required",
857        "commit 9fceb02d0ae598e95dc970b74767f19372d61af8",
858        "ULID 01M7F1G7YYRRRDBS43Y36C1QHR",
859        "uuid 123e4567-e89b-12d3-a456-426614174000",
860        "a ghp_ token goes here",
861        "eyJ alone",
862        "AKIA plus some words",
863        "AKIAIOSFODNN7EXAMPLEX is too long",
864        "MN1-0000-not-a-code",
865        "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}",
866        "API_KEY=<your key>",
867        "MNEMO_KEY_FILE=/home/me/brain.key",
868        "-----BEGIN PUBLIC KEY-----\nMFkwEwYH\n-----END PUBLIC KEY-----",
869        "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----",
870        "xoxb-short",
871        "sk-learn is a library",
872    ];
873
874    /// None of the false cases is redacted.
875    #[test]
876    fn the_false_cases() {
877        for c in FALSE_CASES {
878            let r = secrets(c);
879            assert_eq!(r.text, c, "redacted a false case: {c}");
880            assert!(r.counts.is_empty(), "{c}: {:?}", r.counts);
881        }
882    }
883
884    /// A line of key material, as a PEM body has (64 base64 characters).
885    fn b64_line(seed: &str) -> String {
886        seed.repeat(64).chars().take(64).collect()
887    }
888
889    /// A PEM block split across two texts (two turns): each half goes.
890    #[test]
891    fn a_pem_block_split_across_texts() {
892        let a = format!(
893            "here it is\n-----BEGIN EC {PK}-----\n{}\nMHcCAQEEIOab",
894            b64_line("MHcCAQEEIOab")
895        );
896        let b = format!(
897            "{}\ncdEFgh12\n-----END EC {PK}-----\nthat was it",
898            b64_line("cdEFgh12")
899        );
900        let ra = secrets(&a);
901        assert_eq!(ra.text, "here it is\n[redacted: private-key]");
902        let rb = secrets(&b);
903        assert_eq!(rb.text, "[redacted: private-key]\nthat was it");
904        // An encrypted key's header counts as key material.
905        let c = format!("x\n-----BEGIN RSA {PK}-----\nProc-Type: 4,ENCRYPTED\nDEK");
906        assert_eq!(secrets(&c).text, "x\n[redacted: private-key]");
907    }
908
909    /// A marker only mentioned in prose loses the marker, not the turn
910    /// (RB-011.4).
911    #[test]
912    fn a_pem_marker_in_prose_takes_only_the_marker() {
913        let begin = format!(
914            "Look for the line `-----BEGIN {PK}-----` in the file. Then tell me what \
915             follows, because the rest of this turn matters a lot."
916        );
917        assert_eq!(
918            secrets(&begin).text,
919            "Look for the line `[redacted: private-key]` in the file. Then tell me what \
920             follows, because the rest of this turn matters a lot."
921        );
922        let end = format!(
923            "A long explanation of the format comes first and it matters. The block \
924             closes with `-----END RSA {PK}-----` and that is all."
925        );
926        assert_eq!(
927            secrets(&end).text,
928            "A long explanation of the format comes first and it matters. The block \
929             closes with `[redacted: private-key]` and that is all."
930        );
931        // A short line after BEGIN isn't key material either.
932        let short = format!("-----BEGIN {PK}-----\nwhere does it go?");
933        assert_eq!(
934            secrets(&short).text,
935            "[redacted: private-key]\nwhere does it go?"
936        );
937    }
938
939    /// The shapes added for RB-011.2 (mnemo's own secrets first, then
940    /// common ones), each with its secret part gone.
941    #[test]
942    fn the_added_true_cases() {
943        let key = crate::keys::recovery::encode(&[9u8; 32]);
944        let spaced = key.replace('-', " ");
945        let groups: Vec<&str> = key.split('-').collect();
946        let wrapped = format!("{}\n{}", groups[..7].join("-"), groups[7..].join("-"));
947        let cases: Vec<(String, &str, &str)> = vec![
948            (
949                "export MNEMO_PASSPHRASE=correcthorsebattery".into(),
950                "secret-value",
951                "correcthorse",
952            ),
953            (
954                "MNEMO_PASSPHRASE=correct horse battery staple\nnext".into(),
955                "secret-value",
956                "horse battery",
957            ),
958            (
959                "mnemo init --passphrase Tr0ub4dor&3xyz".into(),
960                "secret-value",
961                "Tr0ub4dor",
962            ),
963            (
964                "tool --password=\"long enough words\"".into(),
965                "secret-value",
966                "enough",
967            ),
968            (
969                format!("gh --token {}", "a1B2c3D4".repeat(3)),
970                "secret-value",
971                "a1B2c3D4",
972            ),
973            (format!("code: {spaced}"), "recovery-code", groups[3]),
974            (format!("code:\n{wrapped}"), "recovery-code", groups[9]),
975            (
976                "Authorization: Bearer abcDEF123456ghiJKL789".into(),
977                "secret-value",
978                "abcDEF123456",
979            ),
980            (
981                "\"Authorization\": \"Basic dXNlcjpwYXNzd29yZA==\"".into(),
982                "secret-value",
983                "dXNlcjpw",
984            ),
985            (
986                "curl -H 'bearer abcDEF123456ghiJKL789xyz'".into(),
987                "secret-value",
988                "abcDEF123456",
989            ),
990            (
991                "X-Api-Key: abcDEF123456ghiJKL789".into(),
992                "secret-value",
993                "abcDEF123456",
994            ),
995            (
996                "{\"apiKey\": \"abcDEF123456ghiJKL789\"}".into(),
997                "secret-value",
998                "abcDEF123456",
999            ),
1000            (
1001                format!("{}_live_{}", concat!("s", "k"), "a1B2".repeat(6)),
1002                "api-key",
1003                "a1B2a1B2",
1004            ),
1005            (
1006                format!("{}_live_{}", concat!("r", "k"), "Z9y8".repeat(6)),
1007                "api-key",
1008                "Z9y8Z9y8",
1009            ),
1010            (
1011                format!("key={}{}", concat!("AI", "za"), "Sy_1-".repeat(7)),
1012                "api-key",
1013                "Sy_1-Sy_1",
1014            ),
1015            (
1016                format!("{}-{}", concat!("gl", "pat"), "x1Y2z3".repeat(4)),
1017                "api-key",
1018                "x1Y2z3x1",
1019            ),
1020            (
1021                format!("{}_{}", concat!("n", "pm"), "q7".repeat(18)),
1022                "api-key",
1023                "q7q7q7",
1024            ),
1025            (
1026                format!("{}_{}", concat!("h", "f"), "AbCd".repeat(8)),
1027                "api-key",
1028                "AbCdAbCd",
1029            ),
1030            (
1031                "postgres://admin:S3cr3tPassw0rd@db.example.com/app".into(),
1032                "secret-value",
1033                "S3cr3t",
1034            ),
1035        ];
1036        for (text, kind, gone) in &cases {
1037            let r = secrets(text);
1038            assert_eq!(kinds(text), [*kind], "{text} -> {}", r.text);
1039            assert!(!r.text.contains(gone), "{text} -> {}", r.text);
1040        }
1041        // Only the secret part goes.
1042        assert_eq!(
1043            secrets("Authorization: Bearer abcDEF123456ghiJKL789").text,
1044            "Authorization: Bearer [redacted: secret-value]"
1045        );
1046        assert_eq!(
1047            secrets("postgres://admin:S3cr3tPassw0rd@db.example.com/app").text,
1048            "postgres://admin:[redacted: secret-value]@db.example.com/app"
1049        );
1050        assert_eq!(
1051            secrets("mnemo init --passphrase Tr0ub4dor&3xyz now").text,
1052            "mnemo init --passphrase [redacted: secret-value] now"
1053        );
1054    }
1055
1056    /// The false cases found in RB-011.4, and a near miss beside each shape
1057    /// added for RB-011.2.
1058    #[test]
1059    fn the_added_false_cases() {
1060        let cases = [
1061            "cache_key: 3f9a2b7c",
1062            "SORT_KEY=created_at/2024",
1063            "client_secret: useEnvironment",
1064            "password: \"{{ vault_db_password }}\"",
1065            "PASSWORD=%{db_password}",
1066            "partition_key: tenant_42",
1067            "row-key: r1234567",
1068            "redis_cache_key=user:42:profile",
1069            "MNEMO_PASSPHRASE: set",
1070            "the passphrase is set",
1071            "--passphrase $PP",
1072            "--password-stdin < file",
1073            "--keychain on",
1074            "use the --token flag",
1075            "remember --key design.122-post-release-order",
1076            "curl --key ~/.ssh/id_ed25519 --cert c.pem",
1077            "Authorization: required",
1078            "the authorization: see below",
1079            "Bearer token",
1080            "AIza short",
1081            concat!("gl", "pat-short"),
1082            "npm_install",
1083            "hf_hub_download",
1084            concat!("s", "k_live_short"),
1085            "https://example.com:8080/path",
1086            "ssh://git@github.com/owner/repo",
1087            "postgres://user:${DB_PASS}@host/db",
1088            "MN1 is the code's prefix",
1089        ];
1090        for c in cases {
1091            let r = secrets(c);
1092            assert_eq!(r.text, c, "redacted a false case: {c}");
1093        }
1094    }
1095
1096    /// Text that isn't ASCII, around the cases and on its own (RB-011.1:
1097    /// these panicked).
1098    #[test]
1099    fn text_that_isnt_ascii() {
1100        for t in [
1101            "it’s done",
1102            "don’t do that",
1103            "the café is open",
1104            "so—yes",
1105            "a😀",
1106            "ok日本語",
1107            "run it ok→ done",
1108            "I’m fine",
1109            "MN😀",
1110            "mn1-é",
1111            "sk-é",
1112            "eyJé",
1113            "postgres://ü:é@x",
1114            "--key é",
1115            "Bearer 日本語",
1116            "-----BEGIN PRIVATE KEY-----\né",
1117        ] {
1118            let _ = secrets(t);
1119        }
1120        // The suffix starts a line: a password's unquoted value runs to the end
1121        // of its line.
1122        let around = |t: &str| format!("it’s — café 日本 😀 {t}\n— ok’s 🎉");
1123        for t in ["it’s done", "the café is open", "so—yes", "a😀", "ok日本語"] {
1124            assert_eq!(secrets(t).text, t);
1125        }
1126        let hit = "id AKIAIOSFODNN7EXAMPLE here";
1127        assert_eq!(
1128            secrets(&around(hit)).text,
1129            around("id [redacted: aws-key-id] here")
1130        );
1131        // The false tables, wrapped, stay as they are; the true ones are
1132        // still found.
1133        for c in FALSE_CASES {
1134            let w = around(c);
1135            assert_eq!(secrets(&w).text, w, "{c}");
1136        }
1137        for c in ["API_KEY=a8f3k2m9q7", "db_password: CorrectHorse"] {
1138            let w = around(c);
1139            assert_eq!(secrets(&w).counts, [("secret-value", 1)], "{w}");
1140        }
1141    }
1142
1143    proptest::proptest! {
1144        /// No input panics, and the output differs from the input exactly
1145        /// when something was counted.
1146        #[test]
1147        fn never_panics(parts in proptest::collection::vec(proptest::prop_oneof![
1148            proptest::prelude::any::<String>(),
1149            "[A-Za-z0-9]{0,40}",
1150            proptest::strategy::Just("MN1-".to_string()),
1151            proptest::strategy::Just("-----BEGIN PRIVATE KEY-----\n".to_string()),
1152            proptest::strategy::Just("-----END RSA PRIVATE KEY-----".to_string()),
1153            proptest::strategy::Just("password: ".to_string()),
1154            proptest::strategy::Just("Authorization: Basic ".to_string()),
1155            proptest::strategy::Just("--key ".to_string()),
1156            proptest::strategy::Just("://u:".to_string()),
1157            proptest::strategy::Just("@".to_string()),
1158            proptest::strategy::Just("Bearer ".to_string()),
1159            proptest::strategy::Just("’é😀→日".to_string()),
1160            proptest::strategy::Just("sk-".to_string()),
1161            proptest::strategy::Just("eyJ".to_string()),
1162            proptest::strategy::Just(" \n=:\"".to_string()),
1163        ], 0..24)) {
1164            let text = parts.concat();
1165            let r = secrets(&text);
1166            proptest::prop_assert_eq!(r.text == text, r.counts.is_empty());
1167        }
1168
1169        /// Words, spaces and punctuation outside ASCII are never redacted.
1170        #[test]
1171        fn prose_stays(text in "([a-z]{1,9}|[ ’—–é日本😀→“”,.!?]){0,60}") {
1172            proptest::prop_assert_eq!(secrets(&text).text, text.as_str());
1173        }
1174    }
1175
1176    #[test]
1177    fn counts_add_up_in_kind_order() {
1178        let t = format!(
1179            "AKIAIOSFODNN7EXAMPLE and ghp_{} and {}",
1180            "a".repeat(36),
1181            aws(concat!("AK", "IA"))
1182        );
1183        let r = secrets(&t);
1184        assert_eq!(r.counts, [("aws-key-id", 2), ("github-token", 1)]);
1185        assert_eq!(r.total(), 3);
1186        let mut all = vec![("github-token", 1)];
1187        add_counts(&mut all, &[("aws-key-id", 2), ("github-token", 1)]);
1188        assert_eq!(all, [("aws-key-id", 2), ("github-token", 2)]);
1189        // Nothing found borrows.
1190        assert!(matches!(secrets("plain text").text, Cow::Borrowed(_)));
1191    }
1192}