Skip to main content

host/
secrets.rs

1//! Secret masking of cell output, ported from ultron/secrets.ts (SecretDetector over rlm/secret-patterns.json),
2//! ultron/rlm/output-secrets.ts (known secret values, `ULTRON_MASK_SECRETS`) and ultron/sensitive.ts.
3//!
4//! The rules stay data: the shipped `secret-patterns.json` is compiled in, so masker, runtime and release scan
5//! still share one file. JS regex semantics are kept where they differ from Rust's: `\b`, `\d` and `\w` are
6//! ASCII and `\s` is JavaScript's whitespace set (see [`js_regex`]).
7
8use fancy_regex::Regex;
9use serde_json::Value;
10use std::collections::HashMap;
11use std::sync::{Mutex, OnceLock};
12
13const PATTERNS: &str = ultron_assets::SECRET_PATTERNS;
14const KNOWN_SECRET_MIN_LENGTH: usize = 8;
15/// JavaScript's (ASCII) word character, for `\b` and `\B`: fancy-regex cannot switch Unicode mode off inline.
16const W: &str = "[A-Za-z0-9_]";
17/// JavaScript's `\s` (WhiteSpace and LineTerminator), without the class brackets.
18const JS_SPACE: &str =
19    r"\t\n\x0B\x0C\r \u{A0}\u{1680}\u{2000}-\u{200A}\u{2028}\u{2029}\u{202F}\u{205F}\u{3000}\u{FEFF}";
20
21/// Compile a JavaScript pattern (flags `i`, `m` only) with JavaScript's ASCII `\b`, `\d`, `\w` and its `\s`.
22pub fn js_regex(pattern: &str, flags: &str) -> Result<Regex, String> {
23    let mut out = String::new();
24    for flag in flags.chars() {
25        match flag {
26            'i' | 'm' => out.push_str(&format!("(?{flag})")),
27            _ => return Err(format!("secret pattern flag {flag} is not portable")),
28        }
29    }
30    let mut chars = pattern.chars();
31    let mut in_class = false;
32    while let Some(c) = chars.next() {
33        match c {
34            '\\' => {
35                let Some(next) = chars.next() else {
36                    out.push('\\');
37                    break;
38                };
39                match (next, in_class) {
40                    ('b', false) => out.push_str(&format!("(?:(?<={W})(?!{W})|(?<!{W})(?={W}))")),
41                    ('B', false) => out.push_str(&format!("(?:(?<={W})(?={W})|(?<!{W})(?!{W}))")),
42                    ('d', false) => out.push_str("[0-9]"),
43                    ('d', true) => out.push_str("0-9"),
44                    ('D', false) => out.push_str("[^0-9]"),
45                    ('w', false) => out.push_str("[A-Za-z0-9_]"),
46                    ('w', true) => out.push_str("A-Za-z0-9_"),
47                    ('W', false) => out.push_str("[^A-Za-z0-9_]"),
48                    ('s', false) => out.push_str(&format!("[{JS_SPACE}]")),
49                    ('S', false) => out.push_str(&format!("[^{JS_SPACE}]")),
50                    _ => {
51                        out.push('\\');
52                        out.push(next);
53                    }
54                }
55            }
56            '[' if !in_class => {
57                in_class = true;
58                out.push(c);
59            }
60            ']' if in_class => {
61                in_class = false;
62                out.push(c);
63            }
64            _ => out.push(c),
65        }
66    }
67    Regex::new(&out).map_err(|error| format!("{pattern}: {error}"))
68}
69
70/// A finding: the rule id and the byte offsets of the value in the scanned text.
71#[derive(Clone, Debug, PartialEq, Eq)]
72pub struct SecretFinding {
73    pub kind: String,
74    pub start: usize,
75    pub end: usize,
76    pub preview: String,
77}
78
79struct Rule {
80    id: String,
81    regex: Regex,
82    group: usize,
83    mask: bool,
84    scan: bool,
85    keywords: Vec<String>,
86    caseless: bool,
87    min_entropy: f64,
88    letter_and_digit: bool,
89    reject: Option<Regex>,
90    allow: Option<Regex>,
91}
92
93pub struct SecretDetector {
94    rules: Vec<Rule>,
95    safe: Vec<Regex>,
96    test_value: Regex,
97}
98
99/// Bits of Shannon entropy per character (code points counted, divided by the UTF-16 length as JS does).
100pub fn shannon_entropy(value: &str) -> f64 {
101    if value.is_empty() {
102        return 0.0;
103    }
104    let mut counts: Vec<(char, usize)> = Vec::new();
105    for c in value.chars() {
106        match counts.iter_mut().find(|(seen, _)| *seen == c) {
107            Some((_, count)) => *count += 1,
108            None => counts.push((c, 1)),
109        }
110    }
111    let length = jsrt::utf16::len(value) as f64;
112    counts.iter().fold(0.0, |bits, (_, count)| {
113        let p = *count as f64 / length;
114        bits - p * p.log2()
115    })
116}
117
118/// A redacted preview: up to 4 leading UTF-16 units (a quarter of the value at most) and the length.
119pub fn preview_secret(value: &str) -> String {
120    let units: Vec<u16> = value.encode_utf16().collect();
121    let keep = 4.min(units.len() / 4);
122    format!("{}…({} chars)", String::from_utf16_lossy(&units[..keep]), units.len())
123}
124
125fn string_field<'a>(value: &'a Value, key: &str) -> Option<&'a str> {
126    value.get(key).and_then(Value::as_str)
127}
128
129impl SecretDetector {
130    /// The detector for a `secret-patterns.json` document.
131    pub fn new(data: &Value) -> Result<Self, String> {
132        let empty = Vec::new();
133        let list = |key: &str| data.get(key).and_then(Value::as_array).unwrap_or(&empty).clone();
134        let mut rules = Vec::new();
135        for rule in list("rules") {
136            let flags = string_field(&rule, "flags").unwrap_or_default();
137            let caseless = flags.contains('i');
138            let scopes: Vec<String> = rule
139                .get("scopes")
140                .and_then(Value::as_array)
141                .map(|scopes| scopes.iter().filter_map(|s| s.as_str().map(str::to_owned)).collect())
142                .unwrap_or_else(|| vec!["mask".into(), "scan".into()]);
143            let optional = |key: &str, flags_key: &str| -> Result<Option<Regex>, String> {
144                string_field(&rule, key)
145                    .map(|pattern| js_regex(pattern, string_field(&rule, flags_key).unwrap_or_default()))
146                    .transpose()
147            };
148            rules.push(Rule {
149                id: string_field(&rule, "id").unwrap_or_default().to_owned(),
150                regex: js_regex(string_field(&rule, "regex").unwrap_or_default(), flags)?,
151                group: rule.get("group").and_then(Value::as_u64).unwrap_or(0) as usize,
152                mask: scopes.iter().any(|s| s == "mask"),
153                scan: scopes.iter().any(|s| s == "scan"),
154                keywords: rule
155                    .get("keywords")
156                    .and_then(Value::as_array)
157                    .map(|words| {
158                        words
159                            .iter()
160                            .filter_map(Value::as_str)
161                            .map(|word| if caseless { word.to_lowercase() } else { word.to_owned() })
162                            .collect()
163                    })
164                    .unwrap_or_default(),
165                caseless,
166                min_entropy: rule.get("minEntropy").and_then(Value::as_f64).unwrap_or(0.0),
167                letter_and_digit: rule.get("requireLetterAndDigit") == Some(&Value::Bool(true)),
168                reject: optional("rejectValue", "rejectValueFlags")?,
169                allow: optional("allowValue", "allowValueFlags")?,
170            });
171        }
172        let safe = list("safeSpans")
173            .iter()
174            .map(|span| js_regex(string_field(span, "regex").unwrap_or_default(), ""))
175            .collect::<Result<_, _>>()?;
176        let test = data.get("testValue").cloned().unwrap_or(Value::Null);
177        let test_value = js_regex(
178            string_field(&test, "regex").unwrap_or_default(),
179            string_field(&test, "flags").unwrap_or_default(),
180        )?;
181        Ok(SecretDetector {
182            rules,
183            safe,
184            test_value,
185        })
186    }
187
188    /// The detector for the shipped rules (rlm/secret-patterns.json), built once.
189    pub fn shipped() -> &'static SecretDetector {
190        static DETECTOR: OnceLock<SecretDetector> = OnceLock::new();
191        DETECTOR.get_or_init(|| {
192            let data: Value = serde_json::from_str(PATTERNS).expect("secret-patterns.json is JSON");
193            SecretDetector::new(&data).expect("secret-patterns.json compiles")
194        })
195    }
196
197    /// Rule ids, in priority order.
198    pub fn kinds(&self) -> Vec<&str> {
199        self.rules.iter().map(|rule| rule.id.as_str()).collect()
200    }
201
202    /// Non-overlapping findings in `text`, in order; `mask` selects the masking scope, else the scan scope.
203    pub fn scan(&self, text: &str, mask: bool) -> Vec<SecretFinding> {
204        if text.is_empty() {
205            return Vec::new();
206        }
207        let lower = OnceLock::new();
208        let safe = OnceLock::new();
209        let mut found: Vec<(SecretFinding, usize)> = Vec::new();
210        for (order, rule) in self.rules.iter().enumerate() {
211            if !(if mask { rule.mask } else { rule.scan }) {
212                continue;
213            }
214            if !rule.keywords.is_empty() {
215                let haystack = if rule.caseless {
216                    lower.get_or_init(|| text.to_lowercase()).as_str()
217                } else {
218                    text
219                };
220                if !rule.keywords.iter().any(|word| haystack.contains(word.as_str())) {
221                    continue;
222                }
223            }
224            // ponytail: a regex runtime error (backtrack limit) ends this rule's matches, like a non-match.
225            for captures in rule.regex.captures_iter(text).map_while(Result::ok) {
226                let Some(span) = captures.get(rule.group) else { continue };
227                let (start, end) = (span.start(), span.end());
228                let value = span.as_str();
229                if !self.plausible(rule, value) {
230                    continue;
231                }
232                let safe: &Vec<(usize, usize)> = safe.get_or_init(|| self.safe_spans(text));
233                if safe.iter().any(|&(from, to)| from <= start && end <= to) {
234                    continue;
235                }
236                let finding = SecretFinding {
237                    kind: rule.id.clone(),
238                    start,
239                    end,
240                    preview: preview_secret(value),
241                };
242                found.push((finding, order));
243            }
244        }
245        // Earlier first; at the same start the longer match, then the earlier rule.
246        found.sort_by(|(a, ao), (b, bo)| {
247            a.start
248                .cmp(&b.start)
249                .then((b.end - b.start).cmp(&(a.end - a.start)))
250                .then(ao.cmp(bo))
251        });
252        let mut kept: Vec<SecretFinding> = Vec::new();
253        for (finding, _) in found {
254            if kept.last().is_some_and(|last| finding.start < last.end) {
255                continue;
256            }
257            kept.push(finding);
258        }
259        kept
260    }
261
262    /// `text` with every masking finding replaced by `[REDACTED:<kind>]`, and how many were replaced.
263    pub fn redact_counted(&self, text: &str) -> (String, usize) {
264        let findings = self.scan(text, true);
265        let mut out = String::new();
266        let mut cursor = 0;
267        for finding in &findings {
268            out.push_str(&text[cursor..finding.start]);
269            out.push_str(&format!("[REDACTED:{}]", finding.kind));
270            cursor = finding.end;
271        }
272        out.push_str(&text[cursor..]);
273        (out, findings.len())
274    }
275
276    fn plausible(&self, rule: &Rule, value: &str) -> bool {
277        let test = |regex: &Regex| regex.is_match(value).unwrap_or(false);
278        if value.is_empty() {
279            return false;
280        }
281        if rule.letter_and_digit
282            && !(value.bytes().any(|b| b.is_ascii_alphabetic()) && value.bytes().any(|b| b.is_ascii_digit()))
283        {
284            return false;
285        }
286        if rule.min_entropy > 0.0 && shannon_entropy(value) < rule.min_entropy {
287            return false;
288        }
289        if rule.reject.as_ref().is_some_and(test) || rule.allow.as_ref().is_some_and(test) {
290            return false;
291        }
292        !test(&self.test_value)
293    }
294
295    fn safe_spans(&self, text: &str) -> Vec<(usize, usize)> {
296        let mut spans = Vec::new();
297        for regex in &self.safe {
298            for found in regex.find_iter(text).map_while(Result::ok) {
299                spans.push((found.start(), found.end()));
300            }
301        }
302        spans
303    }
304}
305
306/// Whether cell output is masked: `ULTRON_MASK_SECRETS` (passed as its value) set to off, 0, false or no turns it off.
307pub fn mask_secrets_enabled(value: Option<&str>) -> bool {
308    let value = value.map(|v| v.trim().to_lowercase()).unwrap_or_default();
309    value.is_empty() || !["off", "0", "false", "no"].contains(&value.as_str())
310}
311
312/// Cell output masking: values known to be secret (MCP env values, headers, OAuth tokens) and the shipped rules.
313#[derive(Default)]
314pub struct OutputMasker {
315    known: Mutex<Vec<String>>,
316}
317
318impl OutputMasker {
319    /// Mask these exact values from now on; values shorter than 8 characters after trimming are skipped.
320    pub fn register_secret_values(&self, values: &[Value]) {
321        let mut known = self.known.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
322        for value in values.iter().filter_map(Value::as_str) {
323            let value = value.trim();
324            if jsrt::utf16::len(value) >= KNOWN_SECRET_MIN_LENGTH && !known.iter().any(|k| k == value) {
325                known.push(value.to_owned());
326            }
327        }
328    }
329
330    /// `text` with secrets masked and how many were, unless `mask_secrets` (ULTRON_MASK_SECRETS) turns masking off.
331    pub fn mask_counted(&self, text: &str, mask_secrets: Option<&str>) -> (String, usize) {
332        if text.is_empty() || !mask_secrets_enabled(mask_secrets) {
333            return (text.to_owned(), 0);
334        }
335        let mut out = text.to_owned();
336        let mut masked = 0;
337        for secret in self
338            .known
339            .lock()
340            .unwrap_or_else(std::sync::PoisonError::into_inner)
341            .iter()
342        {
343            masked += out.matches(secret.as_str()).count();
344            out = out.replace(secret.as_str(), "[REDACTED:known_secret]");
345        }
346        let (out, detected) = SecretDetector::shipped().redact_counted(&out);
347        (out, masked + detected)
348    }
349}
350
351fn any_match(patterns: &[(&str, &str)], text: &str) -> bool {
352    static CACHE: OnceLock<Mutex<HashMap<String, Regex>>> = OnceLock::new();
353    let mut cache = CACHE
354        .get_or_init(Default::default)
355        .lock()
356        .unwrap_or_else(std::sync::PoisonError::into_inner);
357    patterns.iter().any(|(pattern, flags)| {
358        cache
359            .entry(format!("{flags}/{pattern}"))
360            .or_insert_with(|| js_regex(pattern, flags).expect("sensitive pattern compiles"))
361            .is_match(text)
362            .unwrap_or(false)
363    }) || !SecretDetector::shipped().scan(text, true).is_empty()
364}
365
366/// Whether memory text holds a secret or personal identifier (sensitive.ts containsSensitiveMemory).
367pub fn contains_sensitive_memory(text: &str) -> bool {
368    any_match(
369        &[
370            (r"\b(?:sk|ghp|github_pat|xox[baprs]|AIza|AKIA)[A-Za-z0-9_-]{8,}\b", "i"),
371            (r"-----BEGIN [A-Z ]*PRIVATE KEY-----", "i"),
372            (
373                r"\b(?:api[_ -]?key|access[_ -]?token|auth[_ -]?token|password|passwd|secret|private[_ -]?key)\s*[:=]\s*\S+",
374                "i",
375            ),
376            (r"\bbearer\s+[A-Za-z0-9._~+/=-]{16,}\b", "i"),
377            (r"\b\d{3}-\d{2}-\d{4}\b", ""),
378        ],
379        text,
380    )
381}
382
383/// Whether a code skill proposal holds a secret (sensitive.ts containsCodeSecret).
384pub fn contains_code_secret(text: &str) -> bool {
385    any_match(
386        &[
387            (
388                r"\b(?:sk-(?:proj-|live-|test-)?|sk_live_|sk_test_|ghp_|gho_|github_pat_|xox[baprs]-|AIza|AKIA|ASIA)[A-Za-z0-9_-]{12,}",
389                "",
390            ),
391            (r"-----BEGIN [A-Z ]*PRIVATE KEY-----", ""),
392            (
393                r#"\b(?:api[_-]?key|access[_-]?token|auth[_-]?token|password|passwd|secret|private[_-]?key)\s*[:=]\s*["'][^"'\s]{8,}["']"#,
394                "i",
395            ),
396            (r"\bbearer\s+[A-Za-z0-9._~+/=-]{20,}", "i"),
397            (r"\b\d{3}-\d{2}-\d{4}\b", ""),
398        ],
399        text,
400    )
401}