scour_secrets/
entropy.rs

1//! Shannon-entropy token detection.
2//!
3//! Splits input on token delimiters and replaces tokens whose per-character
4//! Shannon entropy meets a configured threshold. Runs AFTER pattern scanning
5//! so already-replaced values (now placeholders) won't double-fire —
6//! placeholders have low entropy by design.
7//!
8//! Configs come from `kind: entropy` secrets-file entries or the CLI's
9//! `--entropy-threshold` flag, and are applied both by the CLI's dispatch
10//! layer for directly-processed files and by
11//! [`ArchiveProcessor`](crate::ArchiveProcessor) for archive entries.
12
13use crate::category::Category;
14use crate::scanner::ScanStats;
15use crate::store::MappingStore;
16use std::collections::HashMap;
17
18/// Character set an entropy candidate token must consist of exclusively.
19#[derive(Debug, Clone, PartialEq, Eq)]
20pub enum EntropyCharset {
21    Alphanumeric,
22    Base64,
23    Hex,
24    Any,
25}
26
27impl EntropyCharset {
28    /// Parse a charset name (`"base64"`, `"hex"`, `"any"`); anything else
29    /// falls back to `Alphanumeric`, the default.
30    #[must_use]
31    pub fn parse(s: &str) -> Self {
32        match s {
33            "base64" => Self::Base64,
34            "hex" => Self::Hex,
35            "any" => Self::Any,
36            _ => Self::Alphanumeric,
37        }
38    }
39
40    /// Human-readable charset name for reports and calibration output.
41    #[must_use]
42    pub fn describe(&self) -> &'static str {
43        match self {
44            Self::Alphanumeric => "alphanumeric",
45            Self::Base64 => "base64",
46            Self::Hex => "hex",
47            Self::Any => "any printable",
48        }
49    }
50
51    /// Whether every byte of `token` belongs to this charset.
52    #[must_use]
53    pub fn matches_all(&self, token: &[u8]) -> bool {
54        token.iter().all(|&b| match self {
55            Self::Alphanumeric => b.is_ascii_alphanumeric(),
56            Self::Base64 => b.is_ascii_alphanumeric() || b == b'+' || b == b'/' || b == b'=',
57            Self::Hex => b.is_ascii_hexdigit(),
58            Self::Any => b.is_ascii_graphic(),
59        })
60    }
61}
62
63/// Configuration for one entropy-detection pass.
64#[derive(Debug, Clone)]
65pub struct EntropyConfig {
66    /// Minimum token length to consider (default: 20).
67    pub min_length: usize,
68    /// Maximum token length to consider (default: 200).
69    pub max_length: usize,
70    /// Shannon entropy threshold in bits per character (default: 4.5).
71    pub threshold: f64,
72    /// Character set the token must consist of exclusively.
73    pub charset: EntropyCharset,
74    /// Label used in reports and summaries.
75    pub label: String,
76    /// Replacement category for matched tokens.
77    pub category: Category,
78}
79
80impl Default for EntropyConfig {
81    fn default() -> Self {
82        Self {
83            min_length: 20,
84            max_length: 200,
85            threshold: 4.5,
86            charset: EntropyCharset::Alphanumeric,
87            label: "high_entropy_token".into(),
88            category: Category::AuthToken,
89        }
90    }
91}
92
93/// Byte values that delimit tokens for entropy analysis.
94pub const ENTROPY_DELIMITERS: &[u8] = b" \t\n\r\"'`=:,;()[]{}|<>@#\\/^~!?&%$*";
95
96/// Shannon entropy of `data` in bits per byte.
97#[must_use]
98#[allow(clippy::cast_precision_loss)] // token lengths are far below 2^52
99pub fn shannon_entropy(data: &[u8]) -> f64 {
100    if data.is_empty() {
101        return 0.0;
102    }
103    let mut counts = [0u32; 256];
104    for &b in data {
105        counts[b as usize] += 1;
106    }
107    let len = data.len() as f64;
108    counts
109        .iter()
110        .filter(|&&c| c > 0)
111        .map(|&c| {
112            let p = f64::from(c) / len;
113            -p * p.log2()
114        })
115        .sum()
116}
117
118/// Scan `input` for high-entropy tokens and replace them using `store`.
119/// Returns `(output_bytes, per_label_counts)`.
120#[must_use]
121pub fn entropy_scan_bytes(
122    input: &[u8],
123    configs: &[EntropyConfig],
124    store: &MappingStore,
125) -> (Vec<u8>, HashMap<String, u64>) {
126    if configs.is_empty() || input.is_empty() {
127        return (input.to_vec(), HashMap::new());
128    }
129
130    let mut output = Vec::with_capacity(input.len());
131    let mut label_counts: HashMap<String, u64> = HashMap::new();
132    let mut pos = 0;
133
134    while pos < input.len() {
135        let token_start = pos;
136        let token_end = input[pos..]
137            .iter()
138            .position(|b| ENTROPY_DELIMITERS.contains(b))
139            .map_or(input.len(), |p| pos + p);
140
141        let token = &input[token_start..token_end];
142
143        let replaced = if token.is_empty() {
144            false
145        } else {
146            let hit = configs.iter().find(|cfg| {
147                token.len() >= cfg.min_length
148                    && token.len() <= cfg.max_length
149                    && cfg.charset.matches_all(token)
150                    && shannon_entropy(token) >= cfg.threshold
151            });
152
153            if let (Some(cfg), Ok(token_str)) = (hit, std::str::from_utf8(token)) {
154                if let Ok(replacement) = store.get_or_insert(&cfg.category, token_str) {
155                    output.extend_from_slice(replacement.as_bytes());
156                } else {
157                    output.extend_from_slice(token);
158                }
159                *label_counts.entry(cfg.label.clone()).or_insert(0) += 1;
160                true
161            } else {
162                false
163            }
164        };
165
166        if !replaced {
167            output.extend_from_slice(token);
168        }
169
170        if token_end < input.len() {
171            output.push(input[token_end]);
172            pos = token_end + 1;
173        } else {
174            pos = token_end;
175        }
176    }
177
178    (output, label_counts)
179}
180
181/// Merge per-label entropy replacement counts into `stats`.
182pub fn merge_entropy_counts(
183    stats: &mut ScanStats,
184    label_counts: impl IntoIterator<Item = (String, u64)>,
185) {
186    let mut total = 0u64;
187    for (label, count) in label_counts {
188        total += count;
189        *stats.pattern_counts.entry(label).or_insert(0) += count;
190    }
191    stats.matches_found += total;
192    stats.replacements_applied += total;
193}
194
195#[cfg(test)]
196mod tests {
197    use super::*;
198    use crate::generator::HmacGenerator;
199    use std::sync::Arc;
200
201    fn store() -> MappingStore {
202        MappingStore::new(Arc::new(HmacGenerator::new([9u8; 32])), None)
203    }
204
205    #[test]
206    fn replaces_high_entropy_token_and_keeps_prose() {
207        let cfg = EntropyConfig {
208            min_length: 40,
209            charset: EntropyCharset::Base64,
210            ..Default::default()
211        };
212        let input = b"secret FLwnxzdPdfIwcrav7PpjWEoYEc55HAcl2Aad0w8rfSsUvYoJHOlqlngm5UzH1zKJ here";
213        let (out, counts) = entropy_scan_bytes(input, &[cfg], &store());
214        let out = String::from_utf8(out).unwrap();
215        assert!(!out.contains("FLwnxzdPdfIw"), "token replaced: {out}");
216        assert!(out.starts_with("secret ") && out.ends_with(" here"));
217        assert_eq!(counts["high_entropy_token"], 1);
218    }
219
220    #[test]
221    fn low_entropy_and_short_tokens_pass_through() {
222        let cfg = EntropyConfig::default();
223        let input = b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa short words";
224        let (out, counts) = entropy_scan_bytes(input, &[cfg], &store());
225        assert_eq!(out, input.to_vec());
226        assert!(counts.is_empty());
227    }
228
229    #[test]
230    fn empty_configs_are_a_noop() {
231        let input = b"anything at all";
232        let (out, counts) = entropy_scan_bytes(input, &[], &store());
233        assert_eq!(out, input.to_vec());
234        assert!(counts.is_empty());
235    }
236}
scour_secrets/entropy.rs

scour_secrets/
entropy.rs
