harn_kernel/pure/
secret_scan.rs

1use std::collections::BTreeMap;
2use std::sync::OnceLock;
3
4use harn_secret_catalog::{SecretPatternSpec, DEFAULT_SECRET_PATTERN_SPECS, PRECISION_HEURISTIC};
5use regex::Regex;
6use serde::{Deserialize, Serialize};
7use sha2::Digest;
8
9const HIGH_ENTROPY_THRESHOLD: f64 = 3.5;
10
11#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
12pub struct SecretFinding {
13    pub detector: String,
14    pub source: String,
15    pub title: String,
16    pub precision: String,
17    pub line: usize,
18    pub column_start: usize,
19    pub column_end: usize,
20    pub start_offset: usize,
21    pub end_offset: usize,
22    pub redacted: String,
23    pub fingerprint: String,
24}
25
26/// One catalog entry paired with its process-wide compiled matcher.
27///
28/// Scanning and redaction are different questions asked of the same catalog, so
29/// they share one compiled copy rather than each paying to build their own.
30pub struct CompiledSecretPattern {
31    pub spec: &'static SecretPatternSpec,
32    pub regex: Regex,
33}
34
35impl CompiledSecretPattern {
36    pub fn accepts_match(&self, input: &str, start: usize, end: usize) -> bool {
37        self.spec.accepts_match(input, start, end)
38    }
39}
40
41static DEFAULT_PATTERNS: OnceLock<Vec<CompiledSecretPattern>> = OnceLock::new();
42static HIGH_ENTROPY_ASSIGNMENT: OnceLock<Regex> = OnceLock::new();
43
44/// The shared catalog, compiled on first use.
45pub fn compiled_secret_patterns() -> &'static [CompiledSecretPattern] {
46    DEFAULT_PATTERNS.get_or_init(|| {
47        DEFAULT_SECRET_PATTERN_SPECS
48            .iter()
49            .map(|spec| CompiledSecretPattern {
50                spec,
51                regex: Regex::new(spec.regex).unwrap_or_else(|error| {
52                    panic!("invalid {} secret regex: {error}", spec.detector)
53                }),
54            })
55            .collect()
56    })
57}
58
59/// Whether the catalog has already been compiled. Hosts that warm it eagerly
60/// during startup assert against this so the cost cannot silently move back
61/// onto a deep call stack.
62pub fn secret_patterns_compiled() -> bool {
63    DEFAULT_PATTERNS.get().is_some()
64}
65
66fn high_entropy_assignment() -> &'static Regex {
67    HIGH_ENTROPY_ASSIGNMENT.get_or_init(|| {
68        Regex::new(
69            r#"(?im)(?:secret|token|api[_-]?key|access[_-]?key|password|passwd|pwd|client[_-]?secret|private[_-]?key)[^\n:=]{0,32}(?::|=)\s*["']([A-Za-z0-9+/=_\.-]{20,})["']"#,
70        )
71        .expect("high-entropy secret pattern is valid")
72    })
73}
74
75pub fn scan_secrets(content: &str) -> Vec<SecretFinding> {
76    let line_starts = line_starts(content);
77    let mut findings = Vec::new();
78
79    for rule in compiled_secret_patterns() {
80        for matched in rule.regex.find_iter(content) {
81            if !rule.accepts_match(content, matched.start(), matched.end()) {
82                continue;
83            }
84            findings.push(build_finding(
85                content,
86                &line_starts,
87                rule.spec.detector,
88                rule.spec.source,
89                rule.spec.title,
90                rule.spec.precision,
91                matched.start(),
92                matched.end(),
93                matched.as_str(),
94            ));
95        }
96    }
97
98    for captures in high_entropy_assignment().captures_iter(content) {
99        let Some(secret) = captures.get(1) else {
100            continue;
101        };
102        if shannon_entropy(secret.as_str()) < HIGH_ENTROPY_THRESHOLD {
103            continue;
104        }
105        findings.push(build_finding(
106            content,
107            &line_starts,
108            "high-entropy-credential-assignment",
109            "trufflehog",
110            "High-entropy secret assignment",
111            PRECISION_HEURISTIC,
112            secret.start(),
113            secret.end(),
114            secret.as_str(),
115        ));
116    }
117
118    findings.sort_by(|left, right| {
119        left.start_offset
120            .cmp(&right.start_offset)
121            .then(left.end_offset.cmp(&right.end_offset))
122            .then(left.detector.cmp(&right.detector))
123    });
124    let spans = findings
125        .iter()
126        .map(|finding| {
127            (
128                finding.start_offset,
129                finding.end_offset,
130                detector_specificity(&finding.detector),
131            )
132        })
133        .collect::<Vec<_>>();
134    findings.retain(|finding| {
135        let specificity = detector_specificity(&finding.detector);
136        !spans.iter().any(|(start, end, other_specificity)| {
137            *other_specificity > specificity
138                && finding.start_offset < *end
139                && *start < finding.end_offset
140        })
141    });
142    findings.dedup_by(|left, right| {
143        left.detector == right.detector
144            && left.start_offset == right.start_offset
145            && left.end_offset == right.end_offset
146    });
147    findings
148}
149
150fn detector_specificity(detector: &str) -> u8 {
151    match detector {
152        "sensitive-assignment" => 0,
153        "high-entropy-credential-assignment" => 1,
154        _ => 2,
155    }
156}
157
158#[allow(clippy::too_many_arguments)]
159fn build_finding(
160    content: &str,
161    line_starts: &[usize],
162    detector: &str,
163    source: &str,
164    title: &str,
165    precision: &str,
166    start_offset: usize,
167    end_offset: usize,
168    matched: &str,
169) -> SecretFinding {
170    let (line, column_start) = offset_to_line_col(content, line_starts, start_offset);
171    let (_, column_end) = offset_to_line_col(content, line_starts, end_offset);
172    SecretFinding {
173        detector: detector.to_string(),
174        source: source.to_string(),
175        title: title.to_string(),
176        precision: precision.to_string(),
177        line,
178        column_start,
179        column_end,
180        start_offset,
181        end_offset,
182        redacted: redact_match(matched),
183        fingerprint: fingerprint(matched),
184    }
185}
186
187fn line_starts(content: &str) -> Vec<usize> {
188    std::iter::once(0)
189        .chain(
190            content
191                .bytes()
192                .enumerate()
193                .filter_map(|(index, byte)| (byte == b'\n').then_some(index + 1)),
194        )
195        .collect()
196}
197
198#[expect(
199    clippy::string_slice,
200    reason = "line starts and regex match offsets are char boundaries"
201)]
202fn offset_to_line_col(content: &str, starts: &[usize], offset: usize) -> (usize, usize) {
203    let line_index = starts
204        .partition_point(|start| *start <= offset)
205        .saturating_sub(1);
206    let line_start = starts[line_index];
207    (
208        line_index + 1,
209        content[line_start..offset].chars().count() + 1,
210    )
211}
212
213fn redact_match(matched: &str) -> String {
214    if matched.starts_with("-----BEGIN ") {
215        return format!(
216            "{} …",
217            matched
218                .lines()
219                .next()
220                .unwrap_or("-----BEGIN PRIVATE KEY-----")
221        );
222    }
223    let chars = matched.chars().collect::<Vec<_>>();
224    if chars.len() <= 8 {
225        return "*".repeat(chars.len());
226    }
227    let prefix = chars.iter().take(4).collect::<String>();
228    let suffix = chars[chars.len() - 4..].iter().collect::<String>();
229    format!("{prefix}…{suffix}")
230}
231
232fn fingerprint(matched: &str) -> String {
233    let digest = sha2::Sha256::digest(matched.as_bytes());
234    hex::encode(&digest[..8])
235}
236
237fn shannon_entropy(value: &str) -> f64 {
238    let mut counts = BTreeMap::new();
239    for character in value.chars() {
240        *counts.entry(character).or_insert(0_usize) += 1;
241    }
242    let length = value.chars().count() as f64;
243    counts
244        .values()
245        .map(|count| {
246            let probability = *count as f64 / length;
247            -(probability * probability.log2())
248        })
249        .sum()
250}
251
252#[cfg(test)]
253mod tests {
254    use super::*;
255
256    #[test]
257    fn scans_and_deduplicates_the_canonical_catalog() {
258        let findings = scan_secrets(r#"token = "ghp_1234567890abcdefghijklmnopqrstuvwxyzAB""#);
259        assert_eq!(findings.len(), 1);
260        assert_eq!(findings[0].detector, "github-token");
261        assert_eq!(findings[0].precision, "high");
262    }
263
264    #[test]
265    fn source_with_secretish_identifiers_remains_clean() {
266        assert!(scan_secrets("pub const Token = struct { kind: u8 };\n").is_empty());
267    }
268
269    #[test]
270    fn source_expressions_do_not_become_secret_findings() {
271        let source = "const token = process.env.GITHUB_TOKEN ?? \"\";\nlet apiKey = readKey(path);\nlet accessToken = readKey2(path);\nconst password = credentials.current;";
272        assert!(scan_secrets(source).is_empty());
273    }
274}
harn_kernel/pure/secret_scan.rs

harn_kernel/pure/
secret_scan.rs
