harn_kernel/pure/
secret_scan.rs

1use std::collections::BTreeMap;
2use std::sync::OnceLock;
3
4use harn_secret_catalog::{SecretPatternSpec, DEFAULT_SECRET_PATTERN_SPECS, PRECISION_HEURISTIC};
5use regex::Regex;
6use serde::{Deserialize, Serialize};
7use sha2::Digest;
8
9const HIGH_ENTROPY_THRESHOLD: f64 = 3.5;
10
11#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
12pub struct SecretFinding {
13    pub detector: String,
14    pub source: String,
15    pub title: String,
16    pub precision: String,
17    pub line: usize,
18    pub column_start: usize,
19    pub column_end: usize,
20    pub start_offset: usize,
21    pub end_offset: usize,
22    pub redacted: String,
23    pub fingerprint: String,
24}
25
26/// One catalog entry paired with its process-wide compiled matcher.
27///
28/// Scanning and redaction are different questions asked of the same catalog, so
29/// they share one compiled copy rather than each paying to build their own.
30pub struct CompiledSecretPattern {
31    pub spec: &'static SecretPatternSpec,
32    pub regex: Regex,
33}
34
35static DEFAULT_PATTERNS: OnceLock<Vec<CompiledSecretPattern>> = OnceLock::new();
36static HIGH_ENTROPY_ASSIGNMENT: OnceLock<Regex> = OnceLock::new();
37
38/// The shared catalog, compiled on first use.
39pub fn compiled_secret_patterns() -> &'static [CompiledSecretPattern] {
40    DEFAULT_PATTERNS.get_or_init(|| {
41        DEFAULT_SECRET_PATTERN_SPECS
42            .iter()
43            .map(|spec| CompiledSecretPattern {
44                spec,
45                regex: Regex::new(spec.regex).unwrap_or_else(|error| {
46                    panic!("invalid {} secret regex: {error}", spec.detector)
47                }),
48            })
49            .collect()
50    })
51}
52
53/// Whether the catalog has already been compiled. Hosts that warm it eagerly
54/// during startup assert against this so the cost cannot silently move back
55/// onto a deep call stack.
56pub fn secret_patterns_compiled() -> bool {
57    DEFAULT_PATTERNS.get().is_some()
58}
59
60fn high_entropy_assignment() -> &'static Regex {
61    HIGH_ENTROPY_ASSIGNMENT.get_or_init(|| {
62        Regex::new(
63            r#"(?im)(?:secret|token|api[_-]?key|access[_-]?key|password|passwd|pwd|client[_-]?secret|private[_-]?key)[^\n:=]{0,32}(?::|=)\s*["']([A-Za-z0-9+/=_\.-]{20,})["']"#,
64        )
65        .expect("high-entropy secret pattern is valid")
66    })
67}
68
69pub fn scan_secrets(content: &str) -> Vec<SecretFinding> {
70    let line_starts = line_starts(content);
71    let mut findings = Vec::new();
72
73    for rule in compiled_secret_patterns() {
74        for matched in rule.regex.find_iter(content) {
75            findings.push(build_finding(
76                content,
77                &line_starts,
78                rule.spec.detector,
79                rule.spec.source,
80                rule.spec.title,
81                rule.spec.precision,
82                matched.start(),
83                matched.end(),
84                matched.as_str(),
85            ));
86        }
87    }
88
89    for captures in high_entropy_assignment().captures_iter(content) {
90        let Some(secret) = captures.get(1) else {
91            continue;
92        };
93        if shannon_entropy(secret.as_str()) < HIGH_ENTROPY_THRESHOLD {
94            continue;
95        }
96        findings.push(build_finding(
97            content,
98            &line_starts,
99            "high-entropy-credential-assignment",
100            "trufflehog",
101            "High-entropy secret assignment",
102            PRECISION_HEURISTIC,
103            secret.start(),
104            secret.end(),
105            secret.as_str(),
106        ));
107    }
108
109    findings.sort_by(|left, right| {
110        left.start_offset
111            .cmp(&right.start_offset)
112            .then(left.end_offset.cmp(&right.end_offset))
113            .then(left.detector.cmp(&right.detector))
114    });
115    let spans = findings
116        .iter()
117        .map(|finding| {
118            (
119                finding.start_offset,
120                finding.end_offset,
121                detector_specificity(&finding.detector),
122            )
123        })
124        .collect::<Vec<_>>();
125    findings.retain(|finding| {
126        let specificity = detector_specificity(&finding.detector);
127        !spans.iter().any(|(start, end, other_specificity)| {
128            *other_specificity > specificity
129                && finding.start_offset < *end
130                && *start < finding.end_offset
131        })
132    });
133    findings.dedup_by(|left, right| {
134        left.detector == right.detector
135            && left.start_offset == right.start_offset
136            && left.end_offset == right.end_offset
137    });
138    findings
139}
140
141fn detector_specificity(detector: &str) -> u8 {
142    match detector {
143        "sensitive-assignment" => 0,
144        "high-entropy-credential-assignment" => 1,
145        _ => 2,
146    }
147}
148
149#[allow(clippy::too_many_arguments)]
150fn build_finding(
151    content: &str,
152    line_starts: &[usize],
153    detector: &str,
154    source: &str,
155    title: &str,
156    precision: &str,
157    start_offset: usize,
158    end_offset: usize,
159    matched: &str,
160) -> SecretFinding {
161    let (line, column_start) = offset_to_line_col(content, line_starts, start_offset);
162    let (_, column_end) = offset_to_line_col(content, line_starts, end_offset);
163    SecretFinding {
164        detector: detector.to_string(),
165        source: source.to_string(),
166        title: title.to_string(),
167        precision: precision.to_string(),
168        line,
169        column_start,
170        column_end,
171        start_offset,
172        end_offset,
173        redacted: redact_match(matched),
174        fingerprint: fingerprint(matched),
175    }
176}
177
178fn line_starts(content: &str) -> Vec<usize> {
179    std::iter::once(0)
180        .chain(
181            content
182                .bytes()
183                .enumerate()
184                .filter_map(|(index, byte)| (byte == b'\n').then_some(index + 1)),
185        )
186        .collect()
187}
188
189fn offset_to_line_col(content: &str, starts: &[usize], offset: usize) -> (usize, usize) {
190    let line_index = starts
191        .partition_point(|start| *start <= offset)
192        .saturating_sub(1);
193    let line_start = starts[line_index];
194    (
195        line_index + 1,
196        content[line_start..offset].chars().count() + 1,
197    )
198}
199
200fn redact_match(matched: &str) -> String {
201    if matched.starts_with("-----BEGIN ") {
202        return format!(
203            "{} …",
204            matched
205                .lines()
206                .next()
207                .unwrap_or("-----BEGIN PRIVATE KEY-----")
208        );
209    }
210    let chars = matched.chars().collect::<Vec<_>>();
211    if chars.len() <= 8 {
212        return "*".repeat(chars.len());
213    }
214    let prefix = chars.iter().take(4).collect::<String>();
215    let suffix = chars[chars.len() - 4..].iter().collect::<String>();
216    format!("{prefix}…{suffix}")
217}
218
219fn fingerprint(matched: &str) -> String {
220    let digest = sha2::Sha256::digest(matched.as_bytes());
221    hex::encode(&digest[..8])
222}
223
224fn shannon_entropy(value: &str) -> f64 {
225    let mut counts = BTreeMap::new();
226    for character in value.chars() {
227        *counts.entry(character).or_insert(0_usize) += 1;
228    }
229    let length = value.chars().count() as f64;
230    counts
231        .values()
232        .map(|count| {
233            let probability = *count as f64 / length;
234            -(probability * probability.log2())
235        })
236        .sum()
237}
238
239#[cfg(test)]
240mod tests {
241    use super::*;
242
243    #[test]
244    fn scans_and_deduplicates_the_canonical_catalog() {
245        let findings = scan_secrets(r#"token = "ghp_1234567890abcdefghijklmnopqrstuvwxyzAB""#);
246        assert_eq!(findings.len(), 1);
247        assert_eq!(findings[0].detector, "github-token");
248        assert_eq!(findings[0].precision, "high");
249    }
250
251    #[test]
252    fn source_with_secretish_identifiers_remains_clean() {
253        assert!(scan_secrets("pub const Token = struct { kind: u8 };\n").is_empty());
254    }
255}
harn_kernel/pure/secret_scan.rs

harn_kernel/pure/
secret_scan.rs
