cargo_context_core/scrub/
entropy.rs

1//! Shannon-entropy-based secret detection.
2//!
3//! Scans for `KEY=VALUE` / `KEY: VALUE` assignments where KEY matches a
4//! configured context pattern (e.g. `secret`, `api_key`, `token`) and the
5//! VALUE has high entropy. Redacts matching values while preserving the key
6//! and surrounding structure.
7//!
8//! This is the noisy-by-design layer: it catches rotated keys that the
9//! regex-patterns layer misses, at the cost of occasional false positives on
10//! genuinely random-looking config values (e.g. hashes in Cargo.lock).
11
12use regex::Regex;
13use serde::Deserialize;
14
15use crate::error::Result;
16use crate::scrub::{AllowList, Redaction, Severity, hash4};
17
18/// Raw (deserialized) entropy configuration, before regex compilation.
19#[derive(Debug, Clone, Deserialize)]
20pub struct EntropyConfigRaw {
21    #[serde(default = "default_enabled")]
22    pub enabled: bool,
23    #[serde(default = "default_min_length")]
24    pub min_length: usize,
25    #[serde(default = "default_threshold")]
26    pub threshold: f64,
27    #[serde(default = "default_context_keys")]
28    pub context_keys: Vec<String>,
29}
30
31impl Default for EntropyConfigRaw {
32    fn default() -> Self {
33        Self {
34            enabled: default_enabled(),
35            min_length: default_min_length(),
36            threshold: default_threshold(),
37            context_keys: default_context_keys(),
38        }
39    }
40}
41
42fn default_enabled() -> bool {
43    true
44}
45fn default_min_length() -> usize {
46    20
47}
48fn default_threshold() -> f64 {
49    4.5
50}
51fn default_context_keys() -> Vec<String> {
52    vec![
53        "key".into(),
54        "secret".into(),
55        "token".into(),
56        "password".into(),
57        "credential".into(),
58        "api[_-]?key".into(),
59        "auth".into(),
60    ]
61}
62
63/// Compiled entropy detector.
64#[derive(Debug)]
65pub struct EntropyConfig {
66    pub enabled: bool,
67    pub min_length: usize,
68    pub threshold: f64,
69    /// Regex matching `<key><sep><quoted_or_unquoted_value>`.
70    scanner: Option<Regex>,
71}
72
73impl Default for EntropyConfig {
74    fn default() -> Self {
75        Self::from_raw(&EntropyConfigRaw::default()).expect("defaults must compile")
76    }
77}
78
79impl EntropyConfig {
80    pub fn from_raw(raw: &EntropyConfigRaw) -> Result<Self> {
81        let scanner = if raw.enabled && !raw.context_keys.is_empty() {
82            Some(build_scanner(&raw.context_keys)?)
83        } else {
84            None
85        };
86        Ok(Self {
87            enabled: raw.enabled,
88            min_length: raw.min_length,
89            threshold: raw.threshold,
90            scanner,
91        })
92    }
93
94    /// Walk `input` looking for `<context_key><sep><value>` shapes. For each
95    /// match whose value clears the length + entropy bar, replace the value
96    /// with a fingerprint and record a [`Redaction`].
97    pub(crate) fn scan_and_redact(
98        &self,
99        input: &str,
100        allowlist: &AllowList,
101    ) -> (String, Vec<Redaction>) {
102        let scanner = match &self.scanner {
103            Some(s) => s,
104            None => return (input.to_string(), Vec::new()),
105        };
106
107        let mut redactions: Vec<Redaction> = Vec::new();
108        let out = scanner
109            .replace_all(input, |caps: &regex::Captures<'_>| {
110                let full = &caps[0];
111                let value = caps.name("value").map_or("", |m| m.as_str());
112
113                if value.len() < self.min_length {
114                    return full.to_string();
115                }
116                if allowlist.is_allowed(value) {
117                    return full.to_string();
118                }
119                if shannon_entropy(value) < self.threshold {
120                    return full.to_string();
121                }
122
123                let key_part = caps.name("key").map_or("", |m| m.as_str());
124                let sep_part = caps.name("sep").map_or("", |m| m.as_str());
125                let open_quote = caps.name("open").map_or("", |m| m.as_str());
126                let close_quote = caps.name("close").map_or("", |m| m.as_str());
127
128                let hash = hash4(value);
129                redactions.push(Redaction {
130                    rule_id: "entropy".into(),
131                    category: "entropy".into(),
132                    severity: Severity::Medium,
133                    hash4: hash.clone(),
134                });
135
136                format!("{key_part}{sep_part}{open_quote}<REDACTED:entropy:{hash}>{close_quote}",)
137            })
138            .into_owned();
139
140        (out, redactions)
141    }
142}
143
144/// Build the scanner regex: `(?i)(?P<key>KEY_PATTERN)(?P<sep>\s*[:=]\s*)(?P<open>["']?)(?P<value>[^\s"'\n]+)(?P<close>["']?)`.
145fn build_scanner(context_keys: &[String]) -> Result<Regex> {
146    let alt = context_keys
147        .iter()
148        .map(|s| format!("(?:{s})"))
149        .collect::<Vec<_>>()
150        .join("|");
151    // Require the key to be anchored at a word boundary so we don't match
152    // substrings inside identifiers (e.g. `ahtoken` wouldn't match `token`).
153    let pattern = format!(
154        r#"(?i)(?P<key>\b(?:{alt})\w*)(?P<sep>\s*[:=]\s*)(?P<open>["']?)(?P<value>[^\s"'\n]+)(?P<close>["']?)"#
155    );
156    Ok(Regex::new(&pattern)?)
157}
158
159/// Shannon entropy (in bits per character) of `s`.
160pub fn shannon_entropy(s: &str) -> f64 {
161    if s.is_empty() {
162        return 0.0;
163    }
164    let mut counts = [0u32; 256];
165    let mut total = 0u32;
166    for b in s.bytes() {
167        counts[b as usize] += 1;
168        total += 1;
169    }
170    let total_f = total as f64;
171    let mut h = 0.0_f64;
172    for c in counts.iter() {
173        if *c == 0 {
174            continue;
175        }
176        let p = (*c as f64) / total_f;
177        h -= p * p.log2();
178    }
179    h
180}
181
182#[cfg(test)]
183mod tests {
184    use super::*;
185
186    #[test]
187    fn entropy_of_random_text_is_high() {
188        let random_like = "aB3xK9mP7qR4vN2wZ8sT5uY1";
189        let h = shannon_entropy(random_like);
190        assert!(
191            h > 4.0,
192            "random-like string should have high entropy, got {h}"
193        );
194    }
195
196    #[test]
197    fn entropy_of_repeated_char_is_zero() {
198        assert_eq!(shannon_entropy("aaaaaaaaaaaaaaaa"), 0.0);
199    }
200
201    #[test]
202    fn entropy_of_simple_word_is_low() {
203        assert!(shannon_entropy("password") < 3.5);
204    }
205
206    #[test]
207    fn scans_env_style_assignment() {
208        let cfg = EntropyConfig::default();
209        let allow = AllowList::default();
210        let input = "API_KEY=aB3xK9mP7qR4vN2wZ8sT5uY1\nother=line";
211        let (out, redactions) = cfg.scan_and_redact(input, &allow);
212        assert!(!out.contains("aB3xK9mP7qR4vN2wZ8sT5uY1"));
213        assert!(out.contains("<REDACTED:entropy:"));
214        assert_eq!(redactions.len(), 1);
215    }
216
217    #[test]
218    fn scans_yaml_style_assignment() {
219        let cfg = EntropyConfig::default();
220        let allow = AllowList::default();
221        let input = r#"secret: "aB3xK9mP7qR4vN2wZ8sT5uY1""#;
222        let (out, redactions) = cfg.scan_and_redact(input, &allow);
223        assert!(!out.contains("aB3xK9mP7qR4vN2wZ8sT5uY1"));
224        assert_eq!(redactions.len(), 1);
225    }
226
227    #[test]
228    fn skips_values_below_min_length() {
229        let cfg = EntropyConfig::default();
230        let allow = AllowList::default();
231        let input = "password=short";
232        let (out, redactions) = cfg.scan_and_redact(input, &allow);
233        assert_eq!(out, input);
234        assert_eq!(redactions.len(), 0);
235    }
236
237    #[test]
238    fn skips_low_entropy_values() {
239        // `version = "1.0.0.0.0.0.0.0.0.0"` is long enough but low-entropy.
240        let cfg = EntropyConfig::default();
241        let allow = AllowList::default();
242        let input = "auth=aaaaaaaaaaaaaaaaaaaaaaa";
243        let (out, _) = cfg.scan_and_redact(input, &allow);
244        assert_eq!(out, input);
245    }
246
247    #[test]
248    fn allowlist_bypasses_entropy_redaction() {
249        let cfg = EntropyConfig::default();
250        let mut allow = AllowList::default();
251        allow.exact.push("aB3xK9mP7qR4vN2wZ8sT5uY1".to_string());
252        let input = "API_KEY=aB3xK9mP7qR4vN2wZ8sT5uY1";
253        let (out, redactions) = cfg.scan_and_redact(input, &allow);
254        assert_eq!(out, input);
255        assert!(redactions.is_empty());
256    }
257
258    #[test]
259    fn leaves_non_suspicious_keys_alone() {
260        let cfg = EntropyConfig::default();
261        let allow = AllowList::default();
262        let input = "version=aB3xK9mP7qR4vN2wZ8sT5uY1";
263        let (out, _) = cfg.scan_and_redact(input, &allow);
264        assert_eq!(
265            out, input,
266            "'version' is not a context key, should not be scrubbed"
267        );
268    }
269
270    #[test]
271    fn disabled_config_is_noop() {
272        let raw = EntropyConfigRaw {
273            enabled: false,
274            ..Default::default()
275        };
276        let cfg = EntropyConfig::from_raw(&raw).unwrap();
277        let allow = AllowList::default();
278        let input = "API_KEY=aB3xK9mP7qR4vN2wZ8sT5uY1";
279        let (out, _) = cfg.scan_and_redact(input, &allow);
280        assert_eq!(out, input);
281    }
282}
cargo_context_core/scrub/entropy.rs

cargo_context_core/scrub/
entropy.rs
