tokenfold_core/transforms/
redaction.rs

1//! `secret_redaction` (canonical transform id: `"secret_redaction"`): a mandatory safety
2//! preprocessor that scans text for common enterprise secret shapes (JWTs, API keys, AWS
3//! access key IDs, basic-auth-in-URL credentials, and bearer tokens) and replaces each match
4//! with a fixed marker naming which kind of secret was found, without ever echoing the
5//! secret's actual value back into the output.
6//!
7//! This module implements only the pure scan-and-replace behavior. The policy wiring that
8//! makes `secret_redaction` non-disableable lives elsewhere (see `CompressionPolicyBuilder`
9//! in `budget.rs`).
10//!
11//! All patterns run on Rust's `regex` crate, whose matching engine is provably linear-time in
12//! the length of the input (no backtracking), which is why it is the only regex engine
13//! permitted in this file's problem space — see `deny.toml` at the workspace root.
14
15use regex::Regex;
16use std::sync::OnceLock;
17
18/// Result of running [`redact`] over a byte buffer.
19pub struct RedactionOutcome {
20    /// The redacted text, re-encoded as bytes.
21    pub bytes: Vec<u8>,
22    /// Total number of secret matches replaced across all patterns.
23    pub redacted_count: usize,
24}
25
26fn jwt_pattern() -> &'static Regex {
27    static RE: OnceLock<Regex> = OnceLock::new();
28    RE.get_or_init(|| {
29        Regex::new(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+")
30            .expect("jwt_pattern regex is a fixed valid literal")
31    })
32}
33
34fn api_key_pattern() -> &'static Regex {
35    static RE: OnceLock<Regex> = OnceLock::new();
36    RE.get_or_init(|| {
37        Regex::new(r"sk-[A-Za-z0-9]{20,}").expect("api_key_pattern regex is a fixed valid literal")
38    })
39}
40
41fn aws_key_pattern() -> &'static Regex {
42    static RE: OnceLock<Regex> = OnceLock::new();
43    RE.get_or_init(|| {
44        Regex::new(r"AKIA[0-9A-Z]{16}").expect("aws_key_pattern regex is a fixed valid literal")
45    })
46}
47
48fn basic_auth_pattern() -> &'static Regex {
49    static RE: OnceLock<Regex> = OnceLock::new();
50    RE.get_or_init(|| {
51        Regex::new(r"(https?://)[^:/\s@]+:[^@/\s]+@")
52            .expect("basic_auth_pattern regex is a fixed valid literal")
53    })
54}
55
56fn bearer_pattern() -> &'static Regex {
57    static RE: OnceLock<Regex> = OnceLock::new();
58    RE.get_or_init(|| {
59        Regex::new(r"(?i)Bearer\s+[A-Za-z0-9\-._~+/]+=*")
60            .expect("bearer_pattern regex is a fixed valid literal")
61    })
62}
63
64/// Cheaply tests whether any known secret pattern matches `input`, without allocating a
65/// redacted copy. Used by `retrieval_store` to gate what may ever be persisted: nothing that
66/// would be redacted by [`redact`] is eligible for storage.
67pub fn contains_secret(input: &[u8]) -> bool {
68    let text = String::from_utf8_lossy(input);
69    jwt_pattern().is_match(&text)
70        || api_key_pattern().is_match(&text)
71        || aws_key_pattern().is_match(&text)
72        || basic_auth_pattern().is_match(&text)
73        || bearer_pattern().is_match(&text)
74}
75
76/// Scans `input` (treated as best-effort UTF-8 via lossy conversion) for known secret shapes
77/// and replaces each match with a marker identifying the kind of secret found. Returns the
78/// redacted bytes plus a count of total replacements made.
79///
80/// This is a best-effort safety net, not a strict parser: invalid UTF-8 in `input` is handled
81/// via `String::from_utf8_lossy`, which substitutes the standard replacement character for any
82/// invalid byte sequences before the patterns below are applied.
83pub fn redact(input: &[u8]) -> RedactionOutcome {
84    let mut text = String::from_utf8_lossy(input).into_owned();
85    let mut redacted_count = 0usize;
86
87    let re = jwt_pattern();
88    redacted_count += re.find_iter(&text).count();
89    text = re.replace_all(&text, "[REDACTED:jwt]").into_owned();
90
91    let re = api_key_pattern();
92    redacted_count += re.find_iter(&text).count();
93    text = re.replace_all(&text, "[REDACTED:api_key]").into_owned();
94
95    let re = aws_key_pattern();
96    redacted_count += re.find_iter(&text).count();
97    text = re.replace_all(&text, "[REDACTED:aws_key]").into_owned();
98
99    let re = basic_auth_pattern();
100    redacted_count += re.find_iter(&text).count();
101    // No colon inside this marker: "scheme://[REDACTED:basic_auth]@" would itself re-match
102    // `basic_auth_pattern` on a second pass (the colon makes it look like another
103    // "user:pass" segment), which breaks redaction idempotency and the pipeline's
104    // no-bypass safety check (`safety::no_redaction_bypass`, which re-runs `redact` over its
105    // own output and expects zero further matches).
106    text = re
107        .replace_all(&text, "${1}[REDACTED-BASIC-AUTH]@")
108        .into_owned();
109
110    let re = bearer_pattern();
111    redacted_count += re.find_iter(&text).count();
112    text = re
113        .replace_all(&text, "Bearer [REDACTED:bearer_token]")
114        .into_owned();
115
116    RedactionOutcome {
117        bytes: text.into_bytes(),
118        redacted_count,
119    }
120}
121
122#[cfg(test)]
123mod tests {
124    use super::*;
125
126    fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
127        !needle.is_empty()
128            && haystack.len() >= needle.len()
129            && haystack.windows(needle.len()).any(|w| w == needle)
130    }
131
132    #[test]
133    fn jwt_token_is_redacted() {
134        let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dQw4w9WgXcQ";
135        let input = format!("Authorization header carried: {jwt} in the clear");
136        let outcome = redact(input.as_bytes());
137        let output = String::from_utf8(outcome.bytes).unwrap();
138
139        assert!(!output.contains(jwt));
140        assert!(output.contains("[REDACTED:jwt]"));
141        assert!(outcome.redacted_count > 0);
142    }
143
144    #[test]
145    fn openai_api_key_is_redacted() {
146        let key = "sk-ABCDEFGHIJ1234567890abcdefgh";
147        let input = format!("OPENAI_API_KEY={key}");
148        let outcome = redact(input.as_bytes());
149        let output = String::from_utf8(outcome.bytes).unwrap();
150
151        assert!(!output.contains(key));
152        assert!(output.contains("[REDACTED:api_key]"));
153        assert!(outcome.redacted_count > 0);
154    }
155
156    #[test]
157    fn aws_access_key_is_redacted() {
158        // AKIAIOSFODNN7EXAMPLE is AWS's own published documentation placeholder key, not a
159        // real credential: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html
160        let key = "AKIAIOSFODNN7EXAMPLE";
161        let input = format!("AWS_ACCESS_KEY_ID={key}");
162        let outcome = redact(input.as_bytes());
163        let output = String::from_utf8(outcome.bytes).unwrap();
164
165        assert!(!output.contains(key));
166        assert!(output.contains("[REDACTED:aws_key]"));
167        assert!(outcome.redacted_count > 0);
168    }
169
170    #[test]
171    fn basic_auth_in_url_is_redacted() {
172        let input = "see https://alice:hunter2@example.com/path for details";
173        let outcome = redact(input.as_bytes());
174        let output = String::from_utf8(outcome.bytes).unwrap();
175
176        assert!(!output.contains("alice:hunter2"));
177        assert!(output.contains("example.com/path"));
178        assert!(output.contains("[REDACTED-BASIC-AUTH]"));
179        assert!(outcome.redacted_count > 0);
180    }
181
182    #[test]
183    fn redacting_already_redacted_basic_auth_output_finds_no_further_matches() {
184        // Regression test for the marker self-match bug: the basic-auth marker must not
185        // contain a colon, or "scheme://MARKER@" would look like another credential to the
186        // same pattern on a second pass.
187        let input = "see https://alice:hunter2@example.com/path for details";
188        let once = redact(input.as_bytes());
189        let twice = redact(&once.bytes);
190        assert_eq!(twice.redacted_count, 0);
191        assert_eq!(once.bytes, twice.bytes);
192    }
193
194    #[test]
195    fn bearer_token_is_redacted() {
196        let input = "Authorization: Bearer abcDEF123.token-value";
197        let outcome = redact(input.as_bytes());
198        let output = String::from_utf8(outcome.bytes).unwrap();
199
200        assert!(!output.contains("abcDEF123.token-value"));
201        assert!(output.contains("Bearer [REDACTED:bearer_token]"));
202        assert!(outcome.redacted_count > 0);
203    }
204
205    /// Regression canary: this input is the kind of string that pathologically stalls a
206    /// backtracking regex engine (e.g. `(a+)+` / naive nested-quantifier patterns) because it
207    /// almost-but-never-quite matches, forcing exponential blowup. It is not a real
208    /// vulnerability test — Rust's `regex` crate guarantees linear-time matching regardless of
209    /// pattern shape — but it proves the redaction path stays on that guaranteed-linear-time
210    /// engine and never regresses onto a backtracking one.
211    #[test]
212    fn redos_canary_completes_within_time_budget() {
213        let long_run_of_a = "a".repeat(40_000);
214        let start = std::time::Instant::now();
215        let outcome = redact(long_run_of_a.as_bytes());
216        assert!(start.elapsed() < std::time::Duration::from_secs(2));
217        assert_eq!(outcome.redacted_count, 0);
218
219        let nested_quantifier_bait = "a".repeat(20_000) + "!";
220        let start = std::time::Instant::now();
221        let outcome = redact(nested_quantifier_bait.as_bytes());
222        assert!(start.elapsed() < std::time::Duration::from_secs(2));
223        assert_eq!(outcome.redacted_count, 0);
224    }
225
226    #[test]
227    fn contains_secret_matches_the_same_inputs_redact_would_change() {
228        assert!(contains_secret(b"AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE"));
229        assert!(contains_secret(
230            b"Authorization: Bearer abcDEF123.token-value"
231        ));
232        assert!(!contains_secret(
233            b"just a normal log line, nothing sensitive"
234        ));
235    }
236
237    #[test]
238    fn text_with_no_secrets_is_returned_unchanged() {
239        let input = b"just a normal log line with nothing sensitive in it at all.";
240        let outcome = redact(input);
241
242        assert_eq!(outcome.bytes, input.to_vec());
243        assert_eq!(outcome.redacted_count, 0);
244    }
245
246    #[test]
247    fn redacted_output_never_contains_literal_secret_bytes() {
248        let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dQw4w9WgXcQ";
249        let api_key = "sk-ABCDEFGHIJ1234567890abcdefgh";
250        let input = format!("jwt={jwt} key={api_key}");
251        let outcome = redact(input.as_bytes());
252
253        assert!(!contains_subslice(&outcome.bytes, jwt.as_bytes()));
254        assert!(!contains_subslice(&outcome.bytes, api_key.as_bytes()));
255    }
256}
tokenfold_core/transforms/redaction.rs

tokenfold_core/transforms/
redaction.rs
