victauri_plugin/
redaction.rs

1use regex::RegexSet;
2
3const BUILTIN_PATTERNS: &[&str] = &[
4    // API keys: sk-..., pk-..., key-... (20+ chars)
5    r"(?i)\b(sk|pk|key)[-_][a-zA-Z0-9]{20,}\b",
6    // Bearer tokens in output
7    r"(?i)bearer\s+[a-zA-Z0-9\-_.~+/]{20,}",
8    // AWS keys
9    r"\bAKIA[0-9A-Z]{16}\b",
10    // JWT tokens (3 base64 sections separated by dots)
11    r"\beyJ[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_-]{10,}\b",
12    // Generic long hex secrets (40+ hex chars — SHA1 hashes, API keys)
13    r"\b[0-9a-fA-F]{40,}\b",
14    // Email addresses
15    r"\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b",
16    // Credit card numbers (basic patterns)
17    r"\b\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}\b",
18    // OpenAI-style keys
19    r"\bsk-[a-zA-Z0-9]{32,}\b",
20    // Anthropic keys
21    r"\bsk-ant-[a-zA-Z0-9\-]{20,}\b",
22    // GitHub tokens
23    r"\b(ghp|gho|ghu|ghs|ghr)_[a-zA-Z0-9]{36,}\b",
24    // Stripe keys
25    r"\b(sk|pk|rk)_(test|live)_[a-zA-Z0-9]{20,}\b",
26];
27
28const SENSITIVE_JSON_KEYS: &[&str] = &[
29    "api_key",
30    "apiKey",
31    "api-key",
32    "secret",
33    "password",
34    "passwd",
35    "token",
36    "access_token",
37    "refresh_token",
38    "private_key",
39    "privateKey",
40    "secret_key",
41    "secretKey",
42    "authorization",
43    "auth_token",
44    "session_token",
45    "cookie",
46    "credentials",
47    "ssn",
48    "credit_card",
49    "card_number",
50];
51
52/// URL query/fragment parameters whose value is a credential. Matched only as a whole parameter
53/// name right after `?`, `&` or `#`, so `?keyboard=` or `&tokens_used=` are left alone.
54const SECRET_URL_PARAMS: &[&str] = &[
55    "access_token",
56    "access-token",
57    "refresh_token",
58    "id_token",
59    "auth_token",
60    "token",
61    "api_key",
62    "api-key",
63    "apikey",
64    "key",
65    "client_secret",
66    "secret",
67    "password",
68    "passwd",
69    "pwd",
70    "auth",
71    "sig",
72    "signature",
73    "x-amz-signature",
74    "x-amz-credential",
75    "x-amz-security-token",
76    "x-goog-signature",
77    "x-goog-credential",
78];
79
80/// `?name=value` / `&name=value` / `#name=value` for a [`SECRET_URL_PARAMS`] name; the value runs
81/// to the next separator, whitespace, quote or backslash (the end of a JSON string).
82static URL_SECRET_PARAM_RE: std::sync::LazyLock<Option<regex::Regex>> =
83    std::sync::LazyLock::new(|| {
84        let names: Vec<String> = SECRET_URL_PARAMS.iter().map(|n| regex::escape(n)).collect();
85        compile_builtin(&format!(
86            r#"(?i)([?&#](?:{})=)[^&#\s"'<>\\]+"#,
87            names.join("|")
88        ))
89    });
90
91/// `"<key containing a sensitive word>": "<string>"` or `: <number>` in free text — JSON
92/// embedded in a non-JSON output, a truncated JSON fragment, or the content of a JSON string. A
93/// string value cut off by the end of the text (a truncated body) is redacted to the end.
94static TEXT_KEY_VALUE_RE: std::sync::LazyLock<Option<regex::Regex>> = std::sync::LazyLock::new(
95    || {
96        let mut words: Vec<String> = SENSITIVE_JSON_KEYS
97            .iter()
98            .map(|k| regex::escape(&k.to_lowercase()))
99            .collect();
100        words.sort();
101        words.dedup();
102        compile_builtin(&format!(
103            r#"(?i)("[^"\\\n]{{0,64}}?(?:{})[^"\\\n]{{0,64}}?"\s*:\s*)("(?:[^"\\]|\\.)*(?:"|\z)|-?\d[\d.eE+-]*)"#,
104            words.join("|")
105        ))
106    },
107);
108
109/// `<name ending in a sensitive word>=<value>` in free text — the shape of `tracing` span fields
110/// (`login{password="…" api_key=…}`, which `#[instrument]` records for every argument), log lines
111/// like `token=abc123`, and `key=value` config dumps. The name is the sensitive word, optionally
112/// after `prefix_` / `prefix-` / `prefix.` components (`db_password`, `x-api_key`) — a word that
113/// merely starts the name (`tokens_used`, `token_count`) is not a credential. The value is a
114/// quoted string (to its closing quote, or the end of a truncated text) or a bare word.
115static TEXT_KEY_EQ_RE: std::sync::LazyLock<Option<regex::Regex>> = std::sync::LazyLock::new(|| {
116    let mut words: Vec<String> = SENSITIVE_JSON_KEYS
117        .iter()
118        .map(|k| regex::escape(&k.to_lowercase()))
119        .collect();
120    words.sort();
121    words.dedup();
122    compile_builtin(&format!(
123        r#"(?i)(\b(?:[a-z0-9]{{1,32}}[_.-]){{0,4}}(?:{})\s*=\s*)("(?:[^"\\]|\\.)*(?:"|\z)|'[^'\n]*(?:'|\z)|[^\s,;&}})\]"'\[][^\s,;&}})\]"']*)"#,
124        words.join("|")
125    ))
126});
127
128/// Compile one of the built-in patterns; a failure is a bug, logged instead of panicking.
129fn compile_builtin(pattern: &str) -> Option<regex::Regex> {
130    regex::Regex::new(pattern)
131        .inspect_err(|e| tracing::error!("BUG: built-in redaction pattern failed to compile: {e}"))
132        .ok()
133}
134
135/// Replace the value of every sensitive `"key": value` and `key=value` pair found in free text.
136fn redact_text_key_values(text: &str) -> std::borrow::Cow<'_, str> {
137    let colon = match TEXT_KEY_VALUE_RE.as_ref() {
138        Some(re) => re.replace_all(text, r#"${1}"[REDACTED]""#),
139        None => std::borrow::Cow::Borrowed(text),
140    };
141    match TEXT_KEY_EQ_RE.as_ref() {
142        Some(re) if re.is_match(&colon) => {
143            std::borrow::Cow::Owned(re.replace_all(&colon, "${1}[REDACTED]").into_owned())
144        }
145        _ => colon,
146    }
147}
148
149/// Output redactor that scrubs API keys, tokens, emails, and sensitive JSON keys
150/// from MCP tool output. Applies built-in patterns plus optional custom regexes.
151pub struct Redactor {
152    builtin_set: RegexSet,
153    builtin_compiled: Vec<regex::Regex>,
154    custom_set: Option<RegexSet>,
155    custom_compiled: Vec<regex::Regex>,
156}
157
158impl Redactor {
159    /// Build a redactor with custom patterns.
160    ///
161    /// # Errors
162    ///
163    /// Returns [`regex::Error`] if any custom pattern (or a built-in pattern) fails to compile.
164    pub fn try_new(custom_patterns: &[String]) -> Result<Self, regex::Error> {
165        let builtin_set = RegexSet::new(BUILTIN_PATTERNS)?;
166        let builtin_compiled: Vec<regex::Regex> = BUILTIN_PATTERNS
167            .iter()
168            .filter_map(|p| regex::Regex::new(p).ok())
169            .collect();
170
171        let (custom_set, custom_compiled) = if custom_patterns.is_empty() {
172            (None, Vec::new())
173        } else {
174            let set = RegexSet::new(custom_patterns)?;
175            let compiled: Vec<regex::Regex> = custom_patterns
176                .iter()
177                .map(|p| regex::Regex::new(p))
178                .collect::<Result<Vec<_>, _>>()?;
179            (Some(set), compiled)
180        };
181
182        Ok(Self {
183            builtin_set,
184            builtin_compiled,
185            custom_set,
186            custom_compiled,
187        })
188    }
189
190    /// Build a redactor with custom patterns, logging a warning and skipping any invalid patterns.
191    ///
192    /// If the built-in redaction patterns fail to compile (a bug), falls back to
193    /// an empty set and logs an error rather than panicking.
194    pub fn new(custom_patterns: &[String]) -> Self {
195        let (builtin_set, builtin_compiled) = match RegexSet::new(BUILTIN_PATTERNS) {
196            Ok(set) => {
197                let compiled: Vec<regex::Regex> = BUILTIN_PATTERNS
198                    .iter()
199                    .filter_map(|p| regex::Regex::new(p).ok())
200                    .collect();
201                (set, compiled)
202            }
203            Err(e) => {
204                tracing::error!(
205                    "BUG: built-in redaction patterns failed to compile: {e}. \
206                     Redaction will be disabled."
207                );
208                // Fall back to an empty set so the process survives.
209                // An empty RegexSet always compiles successfully.
210                let empty: Vec<String> = Vec::new();
211                let empty_set = RegexSet::new(&empty).unwrap_or_else(|_| unreachable!());
212                (empty_set, Vec::new())
213            }
214        };
215
216        let (custom_set, custom_compiled) = if custom_patterns.is_empty() {
217            (None, Vec::new())
218        } else {
219            match RegexSet::new(custom_patterns) {
220                Ok(set) => {
221                    let compiled: Vec<regex::Regex> = custom_patterns
222                        .iter()
223                        .filter_map(|p| regex::Regex::new(p).ok())
224                        .collect();
225                    (Some(set), compiled)
226                }
227                Err(e) => {
228                    tracing::warn!("Failed to compile custom redaction patterns: {e}");
229                    (None, Vec::new())
230                }
231            }
232        };
233
234        Self {
235            builtin_set,
236            builtin_compiled,
237            custom_set,
238            custom_compiled,
239        }
240    }
241
242    /// Scrub sensitive data from `input` using regex patterns and JSON-key matching.
243    ///
244    /// Passes, in order: the value patterns (API keys, bearer tokens, JWTs, emails, … plus any
245    /// custom patterns); credential parameters in URLs (`?access_token=…`, `&sig=…`); then keys.
246    /// Key-based redaction applies to the whole output when it is JSON — including JSON carried
247    /// inside its string values (IPC/network bodies are JSON-encoded strings) — and otherwise to
248    /// `"key": value` pairs found in the text. Best effort: see the limits in `docs/src/security.md`.
249    #[must_use]
250    pub fn redact(&self, input: &str) -> String {
251        let mut output = self.redact_regex(input);
252        if let Some(re) = URL_SECRET_PARAM_RE.as_ref()
253            && re.is_match(&output)
254        {
255            output = re.replace_all(&output, "${1}[REDACTED]").into_owned();
256        }
257        output = self.redact_json_keys(&output);
258        output
259    }
260
261    fn redact_regex(&self, input: &str) -> String {
262        let has_builtin = self.builtin_set.is_match(input);
263        let has_custom = self.custom_set.as_ref().is_some_and(|c| c.is_match(input));
264
265        if !has_builtin && !has_custom {
266            return input.to_string();
267        }
268
269        let mut output = input.to_string();
270
271        if has_builtin {
272            for re in &self.builtin_compiled {
273                output = re.replace_all(&output, "[REDACTED]").to_string();
274            }
275        }
276
277        if has_custom {
278            for re in &self.custom_compiled {
279                output = re.replace_all(&output, "[REDACTED]").to_string();
280            }
281        }
282
283        output
284    }
285
286    fn redact_json_keys(&self, input: &str) -> String {
287        match serde_json::from_str::<serde_json::Value>(input) {
288            Ok(value) => match redact_json_value(&value, 0) {
289                Some(redacted) => {
290                    serde_json::to_string(&redacted).unwrap_or_else(|_| input.to_string())
291                }
292                None => input.to_string(),
293            },
294            // Not JSON as a whole: redact the `"key": value` pairs of any JSON inside the text.
295            Err(_) => redact_text_key_values(input).into_owned(),
296        }
297    }
298}
299
300/// How deep JSON nested inside JSON strings is followed.
301const MAX_NESTED_JSON_DEPTH: usize = 4;
302
303fn is_sensitive_key(key: &str) -> bool {
304    let lower_key = key.to_lowercase();
305    SENSITIVE_JSON_KEYS
306        .iter()
307        .any(|k| lower_key.contains(&k.to_lowercase()))
308}
309
310/// `value` with every sensitive key's value redacted, or `None` if nothing needed redacting.
311/// A string value that holds JSON is redacted inside (and re-encoded); any other string has the
312/// `"key": value` pairs in its text redacted (a truncated JSON body, say).
313fn redact_json_value(value: &serde_json::Value, depth: usize) -> Option<serde_json::Value> {
314    match value {
315        serde_json::Value::Object(map) => {
316            let mut changed = false;
317            let mut new_map = serde_json::Map::new();
318            for (key, val) in map {
319                let new_val = if is_sensitive_key(key) && !val.is_boolean() {
320                    // Booleans like `has_api_key: true` only indicate presence — kept.
321                    Some(serde_json::Value::String("[REDACTED]".into()))
322                } else {
323                    redact_json_value(val, depth)
324                };
325                changed |= new_val.is_some();
326                new_map.insert(key.clone(), new_val.unwrap_or_else(|| val.clone()));
327            }
328            changed.then_some(serde_json::Value::Object(new_map))
329        }
330        serde_json::Value::Array(arr) => {
331            let redacted: Vec<Option<serde_json::Value>> =
332                arr.iter().map(|v| redact_json_value(v, depth)).collect();
333            redacted.iter().any(Option::is_some).then(|| {
334                serde_json::Value::Array(
335                    redacted
336                        .into_iter()
337                        .zip(arr)
338                        .map(|(new, old)| new.unwrap_or_else(|| old.clone()))
339                        .collect(),
340                )
341            })
342        }
343        serde_json::Value::String(text) => redact_json_string(text, depth),
344        _ => None,
345    }
346}
347
348fn redact_json_string(text: &str, depth: usize) -> Option<serde_json::Value> {
349    let trimmed = text.trim_start();
350    if depth < MAX_NESTED_JSON_DEPTH
351        && (trimmed.starts_with('{') || trimmed.starts_with('['))
352        && let Ok(inner) = serde_json::from_str::<serde_json::Value>(text)
353    {
354        let redacted = redact_json_value(&inner, depth + 1)?;
355        return serde_json::to_string(&redacted)
356            .ok()
357            .map(serde_json::Value::String);
358    }
359    match redact_text_key_values(text) {
360        std::borrow::Cow::Owned(redacted) => Some(serde_json::Value::String(redacted)),
361        std::borrow::Cow::Borrowed(_) => None,
362    }
363}
364
365impl Default for Redactor {
366    fn default() -> Self {
367        Self::new(&[])
368    }
369}
370
371#[cfg(test)]
372mod tests {
373    use super::*;
374
375    #[test]
376    fn redacts_api_keys() {
377        let r = Redactor::default();
378        assert!(
379            r.redact("key is sk-abc123def456ghi789jkl012mno")
380                .contains("[REDACTED]")
381        );
382    }
383
384    #[test]
385    fn redacts_bearer_tokens() {
386        let r = Redactor::default();
387        let input = "Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U";
388        let output = r.redact(input);
389        assert!(output.contains("[REDACTED]"));
390        assert!(!output.contains("eyJhbGci"));
391    }
392
393    #[test]
394    fn redacts_emails() {
395        let r = Redactor::default();
396        assert!(
397            r.redact("contact user@example.com for help")
398                .contains("[REDACTED]")
399        );
400    }
401
402    #[test]
403    fn passes_through_clean_text() {
404        let r = Redactor::default();
405        let input = r#"{"ok": true, "title": "My App"}"#;
406        assert_eq!(r.redact(input), input);
407    }
408
409    #[test]
410    fn custom_patterns_work() {
411        let r = Redactor::new(&["secret_\\w+".to_string()]);
412        assert!(
413            r.redact("found secret_project_alpha here")
414                .contains("[REDACTED]")
415        );
416    }
417
418    #[test]
419    fn redacts_json_sensitive_keys() {
420        let r = Redactor::default();
421        let input = r#"{"api_key":"sk-test-12345","name":"John","token":"abc123"}"#;
422        let output = r.redact(input);
423        assert!(output.contains("[REDACTED]"));
424        assert!(output.contains("John"));
425        assert!(!output.contains("sk-test-12345"));
426    }
427
428    #[test]
429    fn preserves_boolean_sensitive_keys() {
430        let r = Redactor::default();
431        let input = r#"{"has_api_key":true,"api_key":"secret-value-here"}"#;
432        let output = r.redact(input);
433        let parsed: serde_json::Value = serde_json::from_str(&output).unwrap();
434        assert_eq!(parsed["has_api_key"], serde_json::Value::Bool(true));
435        assert_eq!(
436            parsed["api_key"],
437            serde_json::Value::String("[REDACTED]".into())
438        );
439    }
440
441    #[test]
442    fn redacts_nested_json_keys() {
443        let r = Redactor::default();
444        let input = r#"{"config":{"llm":{"api_key":"sk-live-xxx","model":"gpt-4"}}}"#;
445        let output = r.redact(input);
446        assert!(output.contains("[REDACTED]"));
447        assert!(output.contains("gpt-4"));
448        assert!(!output.contains("sk-live-xxx"));
449    }
450
451    /// R5B-REDACT1: secrets in URL query strings were not matched (tokens ride in IPC/network
452    /// URLs), and key-based redaction only applied when the WHOLE output parsed as JSON.
453    #[test]
454    fn redacts_secret_query_parameters() {
455        let r = Redactor::default();
456        let out = r.redact(
457            "GET https://api.example.com/v1/items?user=7&access_token=abc123def&page=2              https://x.test/cb#id_token=eyJraWQ&state=s https://h.test/?API_KEY=k9k9k9              https://blob.test/f?sv=2024&sig=Zx%2Fq&se=1",
458        );
459        for secret in ["abc123def", "eyJraWQ", "k9k9k9", "Zx%2Fq"] {
460            assert!(!out.contains(secret), "{secret} survived: {out}");
461        }
462        for kept in ["user=7", "page=2", "state=s", "sv=2024", "se=1"] {
463            assert!(out.contains(kept), "{kept} was redacted: {out}");
464        }
465        // Not a secret parameter: only whole parameter names match.
466        let clean = "https://x.test/?keyboard=us&monkey=1&tokens_used=5";
467        assert_eq!(r.redact(clean), clean);
468        // Inside a JSON string, the URL is redacted and the JSON stays valid.
469        let json = r#"{"url":"https://x.test/a?token=zzz111&b=1"}"#;
470        let v: serde_json::Value = serde_json::from_str(&r.redact(json)).unwrap();
471        assert_eq!(v["url"], "https://x.test/a?token=[REDACTED]&b=1");
472    }
473
474    #[test]
475    fn redacts_sensitive_keys_in_json_embedded_in_text() {
476        let r = Redactor::default();
477        let out =
478            r.redact(r#"invoke login -> {"ok":true,"session_token":"s3cr3t-value","n":1} done"#);
479        assert!(!out.contains("s3cr3t-value"), "{out}");
480        assert!(out.contains(r#""ok":true"#), "{out}");
481        // A truncated JSON fragment (log fields are cut at 4 KB) still has its keys redacted.
482        let out = r.redact(r#"body: {"password": "hunter2", "data": "xxxxxxxx…(truncated"#);
483        assert!(!out.contains("hunter2"), "{out}");
484        // …including when the cut falls inside the secret itself.
485        let out = r.redact(r#"body: {"id": 1, "secret": "abcdefgh"#);
486        assert!(!out.contains("abcdefgh"), "{out}");
487        assert!(out.contains(r#""id": 1"#), "{out}");
488    }
489
490    #[test]
491    fn redacts_sensitive_keys_in_json_nested_inside_json_strings() {
492        let r = Redactor::default();
493        // IPC/network logs carry bodies as JSON-encoded strings inside the JSON result.
494        let input = serde_json::json!({
495            "command": "get_settings",
496            "response": "{\"llm\":{\"api_key\":\"sk-live-abc\",\"model\":\"m1\"}}",
497            "truncated_body": "{\"refresh_token\":\"rt-999\",\"x\":\"…"
498        })
499        .to_string();
500        let out = r.redact(&input);
501        let v: serde_json::Value = serde_json::from_str(&out).expect("still valid JSON");
502        let response = v["response"].as_str().unwrap();
503        assert!(!response.contains("sk-live-abc"), "{out}");
504        assert!(response.contains("m1"), "{out}");
505        assert!(
506            !v["truncated_body"].as_str().unwrap().contains("rt-999"),
507            "{out}"
508        );
509        assert_eq!(v["command"], "get_settings");
510    }
511
512    /// Backend log span context is `name{key=value …}` text (`#[instrument]` records every
513    /// argument) — the same secret that is redacted as an event field must be redacted there.
514    #[test]
515    fn redacts_key_equals_value_pairs_in_text() {
516        let r = Redactor::default();
517        let input = serde_json::json!({
518            "spans": ["login{user=\"ann\" password=\"hunter2xyz\" api_key=plainsecret}"],
519            "message": "retrying with token=abc123secret after 3s",
520            "fields": {"tokens_used": 5, "elapsed_ms": 40}
521        })
522        .to_string();
523        let out = r.redact(&input);
524        let v: serde_json::Value = serde_json::from_str(&out).expect("still valid JSON");
525        let span = v["spans"][0].as_str().unwrap();
526        assert!(
527            !span.contains("hunter2xyz") && !span.contains("plainsecret"),
528            "{out}"
529        );
530        assert!(
531            span.contains("user=\"ann\""),
532            "non-sensitive fields survive: {out}"
533        );
534        assert!(!out.contains("abc123secret"), "{out}");
535        assert!(v["message"].as_str().unwrap().contains("after 3s"), "{out}");
536        assert_eq!(v["fields"]["elapsed_ms"], 40);
537        // Plain (non-JSON) text too, including a value cut off by truncation.
538        let plain = r.redact("connect secret='s3cr3t' then password=\"unterminated");
539        assert!(
540            !plain.contains("s3cr3t") && !plain.contains("unterminated"),
541            "{plain}"
542        );
543        let prefixed = r.redact("db_password=pw1 x-api_key=k2 tokens_used=5 token_count=9");
544        assert!(
545            !prefixed.contains("pw1") && !prefixed.contains("k2"),
546            "{prefixed}"
547        );
548        assert!(
549            prefixed.contains("tokens_used=5") && prefixed.contains("token_count=9"),
550            "{prefixed}"
551        );
552        // Already-redacted values are left alone (no doubled markers).
553        assert_eq!(r.redact("token=[REDACTED] x"), "token=[REDACTED] x");
554    }
555
556    #[test]
557    fn redacts_github_tokens() {
558        let r = Redactor::default();
559        assert!(
560            r.redact("ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmno")
561                .contains("[REDACTED]")
562        );
563    }
564
565    #[test]
566    fn redacts_stripe_keys() {
567        let r = Redactor::default();
568        assert!(
569            r.redact("sk_test_ABCDEFGHIJKLMNOPQRSTUVWXYZab")
570                .contains("[REDACTED]")
571        );
572    }
573
574    #[test]
575    fn try_new_valid_patterns() {
576        let r = Redactor::try_new(&["secret_\\w+".to_string()]);
577        assert!(r.is_ok());
578        let r = r.unwrap();
579        assert!(r.redact("found secret_alpha here").contains("[REDACTED]"));
580    }
581
582    #[test]
583    fn try_new_invalid_pattern_returns_error() {
584        let r = Redactor::try_new(&["[invalid".to_string()]);
585        assert!(r.is_err());
586    }
587
588    #[test]
589    fn try_new_empty_patterns() {
590        let r = Redactor::try_new(&[]);
591        assert!(r.is_ok());
592    }
593}
victauri_plugin/redaction.rs

victauri_plugin/
redaction.rs
