1use fancy_regex::Regex;
9use serde_json::Value;
10use std::collections::HashMap;
11use std::sync::{Mutex, OnceLock};
12
13const PATTERNS: &str = ultron_assets::SECRET_PATTERNS;
14const KNOWN_SECRET_MIN_LENGTH: usize = 8;
15const W: &str = "[A-Za-z0-9_]";
17const JS_SPACE: &str =
19 r"\t\n\x0B\x0C\r \u{A0}\u{1680}\u{2000}-\u{200A}\u{2028}\u{2029}\u{202F}\u{205F}\u{3000}\u{FEFF}";
20
21pub fn js_regex(pattern: &str, flags: &str) -> Result<Regex, String> {
23 let mut out = String::new();
24 for flag in flags.chars() {
25 match flag {
26 'i' | 'm' => out.push_str(&format!("(?{flag})")),
27 _ => return Err(format!("secret pattern flag {flag} is not portable")),
28 }
29 }
30 let mut chars = pattern.chars();
31 let mut in_class = false;
32 while let Some(c) = chars.next() {
33 match c {
34 '\\' => {
35 let Some(next) = chars.next() else {
36 out.push('\\');
37 break;
38 };
39 match (next, in_class) {
40 ('b', false) => out.push_str(&format!("(?:(?<={W})(?!{W})|(?<!{W})(?={W}))")),
41 ('B', false) => out.push_str(&format!("(?:(?<={W})(?={W})|(?<!{W})(?!{W}))")),
42 ('d', false) => out.push_str("[0-9]"),
43 ('d', true) => out.push_str("0-9"),
44 ('D', false) => out.push_str("[^0-9]"),
45 ('w', false) => out.push_str("[A-Za-z0-9_]"),
46 ('w', true) => out.push_str("A-Za-z0-9_"),
47 ('W', false) => out.push_str("[^A-Za-z0-9_]"),
48 ('s', false) => out.push_str(&format!("[{JS_SPACE}]")),
49 ('S', false) => out.push_str(&format!("[^{JS_SPACE}]")),
50 _ => {
51 out.push('\\');
52 out.push(next);
53 }
54 }
55 }
56 '[' if !in_class => {
57 in_class = true;
58 out.push(c);
59 }
60 ']' if in_class => {
61 in_class = false;
62 out.push(c);
63 }
64 _ => out.push(c),
65 }
66 }
67 Regex::new(&out).map_err(|error| format!("{pattern}: {error}"))
68}
69
70#[derive(Clone, Debug, PartialEq, Eq)]
72pub struct SecretFinding {
73 pub kind: String,
74 pub start: usize,
75 pub end: usize,
76 pub preview: String,
77}
78
79struct Rule {
80 id: String,
81 regex: Regex,
82 group: usize,
83 mask: bool,
84 scan: bool,
85 keywords: Vec<String>,
86 caseless: bool,
87 min_entropy: f64,
88 letter_and_digit: bool,
89 reject: Option<Regex>,
90 allow: Option<Regex>,
91}
92
93pub struct SecretDetector {
94 rules: Vec<Rule>,
95 safe: Vec<Regex>,
96 test_value: Regex,
97}
98
99pub fn shannon_entropy(value: &str) -> f64 {
101 if value.is_empty() {
102 return 0.0;
103 }
104 let mut counts: Vec<(char, usize)> = Vec::new();
105 for c in value.chars() {
106 match counts.iter_mut().find(|(seen, _)| *seen == c) {
107 Some((_, count)) => *count += 1,
108 None => counts.push((c, 1)),
109 }
110 }
111 let length = jsrt::utf16::len(value) as f64;
112 counts.iter().fold(0.0, |bits, (_, count)| {
113 let p = *count as f64 / length;
114 bits - p * p.log2()
115 })
116}
117
118pub fn preview_secret(value: &str) -> String {
120 let units: Vec<u16> = value.encode_utf16().collect();
121 let keep = 4.min(units.len() / 4);
122 format!("{}…({} chars)", String::from_utf16_lossy(&units[..keep]), units.len())
123}
124
125fn string_field<'a>(value: &'a Value, key: &str) -> Option<&'a str> {
126 value.get(key).and_then(Value::as_str)
127}
128
129impl SecretDetector {
130 pub fn new(data: &Value) -> Result<Self, String> {
132 let empty = Vec::new();
133 let list = |key: &str| data.get(key).and_then(Value::as_array).unwrap_or(&empty).clone();
134 let mut rules = Vec::new();
135 for rule in list("rules") {
136 let flags = string_field(&rule, "flags").unwrap_or_default();
137 let caseless = flags.contains('i');
138 let scopes: Vec<String> = rule
139 .get("scopes")
140 .and_then(Value::as_array)
141 .map(|scopes| scopes.iter().filter_map(|s| s.as_str().map(str::to_owned)).collect())
142 .unwrap_or_else(|| vec!["mask".into(), "scan".into()]);
143 let optional = |key: &str, flags_key: &str| -> Result<Option<Regex>, String> {
144 string_field(&rule, key)
145 .map(|pattern| js_regex(pattern, string_field(&rule, flags_key).unwrap_or_default()))
146 .transpose()
147 };
148 rules.push(Rule {
149 id: string_field(&rule, "id").unwrap_or_default().to_owned(),
150 regex: js_regex(string_field(&rule, "regex").unwrap_or_default(), flags)?,
151 group: rule.get("group").and_then(Value::as_u64).unwrap_or(0) as usize,
152 mask: scopes.iter().any(|s| s == "mask"),
153 scan: scopes.iter().any(|s| s == "scan"),
154 keywords: rule
155 .get("keywords")
156 .and_then(Value::as_array)
157 .map(|words| {
158 words
159 .iter()
160 .filter_map(Value::as_str)
161 .map(|word| if caseless { word.to_lowercase() } else { word.to_owned() })
162 .collect()
163 })
164 .unwrap_or_default(),
165 caseless,
166 min_entropy: rule.get("minEntropy").and_then(Value::as_f64).unwrap_or(0.0),
167 letter_and_digit: rule.get("requireLetterAndDigit") == Some(&Value::Bool(true)),
168 reject: optional("rejectValue", "rejectValueFlags")?,
169 allow: optional("allowValue", "allowValueFlags")?,
170 });
171 }
172 let safe = list("safeSpans")
173 .iter()
174 .map(|span| js_regex(string_field(span, "regex").unwrap_or_default(), ""))
175 .collect::<Result<_, _>>()?;
176 let test = data.get("testValue").cloned().unwrap_or(Value::Null);
177 let test_value = js_regex(
178 string_field(&test, "regex").unwrap_or_default(),
179 string_field(&test, "flags").unwrap_or_default(),
180 )?;
181 Ok(SecretDetector {
182 rules,
183 safe,
184 test_value,
185 })
186 }
187
188 pub fn shipped() -> &'static SecretDetector {
190 static DETECTOR: OnceLock<SecretDetector> = OnceLock::new();
191 DETECTOR.get_or_init(|| {
192 let data: Value = serde_json::from_str(PATTERNS).expect("secret-patterns.json is JSON");
193 SecretDetector::new(&data).expect("secret-patterns.json compiles")
194 })
195 }
196
197 pub fn kinds(&self) -> Vec<&str> {
199 self.rules.iter().map(|rule| rule.id.as_str()).collect()
200 }
201
202 pub fn scan(&self, text: &str, mask: bool) -> Vec<SecretFinding> {
204 if text.is_empty() {
205 return Vec::new();
206 }
207 let lower = OnceLock::new();
208 let safe = OnceLock::new();
209 let mut found: Vec<(SecretFinding, usize)> = Vec::new();
210 for (order, rule) in self.rules.iter().enumerate() {
211 if !(if mask { rule.mask } else { rule.scan }) {
212 continue;
213 }
214 if !rule.keywords.is_empty() {
215 let haystack = if rule.caseless {
216 lower.get_or_init(|| text.to_lowercase()).as_str()
217 } else {
218 text
219 };
220 if !rule.keywords.iter().any(|word| haystack.contains(word.as_str())) {
221 continue;
222 }
223 }
224 for captures in rule.regex.captures_iter(text).map_while(Result::ok) {
226 let Some(span) = captures.get(rule.group) else { continue };
227 let (start, end) = (span.start(), span.end());
228 let value = span.as_str();
229 if !self.plausible(rule, value) {
230 continue;
231 }
232 let safe: &Vec<(usize, usize)> = safe.get_or_init(|| self.safe_spans(text));
233 if safe.iter().any(|&(from, to)| from <= start && end <= to) {
234 continue;
235 }
236 let finding = SecretFinding {
237 kind: rule.id.clone(),
238 start,
239 end,
240 preview: preview_secret(value),
241 };
242 found.push((finding, order));
243 }
244 }
245 found.sort_by(|(a, ao), (b, bo)| {
247 a.start
248 .cmp(&b.start)
249 .then((b.end - b.start).cmp(&(a.end - a.start)))
250 .then(ao.cmp(bo))
251 });
252 let mut kept: Vec<SecretFinding> = Vec::new();
253 for (finding, _) in found {
254 if kept.last().is_some_and(|last| finding.start < last.end) {
255 continue;
256 }
257 kept.push(finding);
258 }
259 kept
260 }
261
262 pub fn redact_counted(&self, text: &str) -> (String, usize) {
264 let findings = self.scan(text, true);
265 let mut out = String::new();
266 let mut cursor = 0;
267 for finding in &findings {
268 out.push_str(&text[cursor..finding.start]);
269 out.push_str(&format!("[REDACTED:{}]", finding.kind));
270 cursor = finding.end;
271 }
272 out.push_str(&text[cursor..]);
273 (out, findings.len())
274 }
275
276 fn plausible(&self, rule: &Rule, value: &str) -> bool {
277 let test = |regex: &Regex| regex.is_match(value).unwrap_or(false);
278 if value.is_empty() {
279 return false;
280 }
281 if rule.letter_and_digit
282 && !(value.bytes().any(|b| b.is_ascii_alphabetic()) && value.bytes().any(|b| b.is_ascii_digit()))
283 {
284 return false;
285 }
286 if rule.min_entropy > 0.0 && shannon_entropy(value) < rule.min_entropy {
287 return false;
288 }
289 if rule.reject.as_ref().is_some_and(test) || rule.allow.as_ref().is_some_and(test) {
290 return false;
291 }
292 !test(&self.test_value)
293 }
294
295 fn safe_spans(&self, text: &str) -> Vec<(usize, usize)> {
296 let mut spans = Vec::new();
297 for regex in &self.safe {
298 for found in regex.find_iter(text).map_while(Result::ok) {
299 spans.push((found.start(), found.end()));
300 }
301 }
302 spans
303 }
304}
305
306pub fn mask_secrets_enabled(value: Option<&str>) -> bool {
308 let value = value.map(|v| v.trim().to_lowercase()).unwrap_or_default();
309 value.is_empty() || !["off", "0", "false", "no"].contains(&value.as_str())
310}
311
312#[derive(Default)]
314pub struct OutputMasker {
315 known: Mutex<Vec<String>>,
316}
317
318impl OutputMasker {
319 pub fn register_secret_values(&self, values: &[Value]) {
321 let mut known = self.known.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
322 for value in values.iter().filter_map(Value::as_str) {
323 let value = value.trim();
324 if jsrt::utf16::len(value) >= KNOWN_SECRET_MIN_LENGTH && !known.iter().any(|k| k == value) {
325 known.push(value.to_owned());
326 }
327 }
328 }
329
330 pub fn mask_counted(&self, text: &str, mask_secrets: Option<&str>) -> (String, usize) {
332 if text.is_empty() || !mask_secrets_enabled(mask_secrets) {
333 return (text.to_owned(), 0);
334 }
335 let mut out = text.to_owned();
336 let mut masked = 0;
337 for secret in self
338 .known
339 .lock()
340 .unwrap_or_else(std::sync::PoisonError::into_inner)
341 .iter()
342 {
343 masked += out.matches(secret.as_str()).count();
344 out = out.replace(secret.as_str(), "[REDACTED:known_secret]");
345 }
346 let (out, detected) = SecretDetector::shipped().redact_counted(&out);
347 (out, masked + detected)
348 }
349}
350
351fn any_match(patterns: &[(&str, &str)], text: &str) -> bool {
352 static CACHE: OnceLock<Mutex<HashMap<String, Regex>>> = OnceLock::new();
353 let mut cache = CACHE
354 .get_or_init(Default::default)
355 .lock()
356 .unwrap_or_else(std::sync::PoisonError::into_inner);
357 patterns.iter().any(|(pattern, flags)| {
358 cache
359 .entry(format!("{flags}/{pattern}"))
360 .or_insert_with(|| js_regex(pattern, flags).expect("sensitive pattern compiles"))
361 .is_match(text)
362 .unwrap_or(false)
363 }) || !SecretDetector::shipped().scan(text, true).is_empty()
364}
365
366pub fn contains_sensitive_memory(text: &str) -> bool {
368 any_match(
369 &[
370 (r"\b(?:sk|ghp|github_pat|xox[baprs]|AIza|AKIA)[A-Za-z0-9_-]{8,}\b", "i"),
371 (r"-----BEGIN [A-Z ]*PRIVATE KEY-----", "i"),
372 (
373 r"\b(?:api[_ -]?key|access[_ -]?token|auth[_ -]?token|password|passwd|secret|private[_ -]?key)\s*[:=]\s*\S+",
374 "i",
375 ),
376 (r"\bbearer\s+[A-Za-z0-9._~+/=-]{16,}\b", "i"),
377 (r"\b\d{3}-\d{2}-\d{4}\b", ""),
378 ],
379 text,
380 )
381}
382
383pub fn contains_code_secret(text: &str) -> bool {
385 any_match(
386 &[
387 (
388 r"\b(?:sk-(?:proj-|live-|test-)?|sk_live_|sk_test_|ghp_|gho_|github_pat_|xox[baprs]-|AIza|AKIA|ASIA)[A-Za-z0-9_-]{12,}",
389 "",
390 ),
391 (r"-----BEGIN [A-Z ]*PRIVATE KEY-----", ""),
392 (
393 r#"\b(?:api[_-]?key|access[_-]?token|auth[_-]?token|password|passwd|secret|private[_-]?key)\s*[:=]\s*["'][^"'\s]{8,}["']"#,
394 "i",
395 ),
396 (r"\bbearer\s+[A-Za-z0-9._~+/=-]{20,}", "i"),
397 (r"\b\d{3}-\d{2}-\d{4}\b", ""),
398 ],
399 text,
400 )
401}