harn_kernel/pure/
secret_scan.rs

1use std::collections::BTreeMap;
2use std::sync::OnceLock;
3
4use harn_secret_catalog::{SecretPatternSpec, DEFAULT_SECRET_PATTERN_SPECS, PRECISION_HEURISTIC};
5use regex::Regex;
6use serde::{Deserialize, Serialize};
7use sha2::Digest;
8
9const HIGH_ENTROPY_THRESHOLD: f64 = 3.5;
10
11/// Public catalog data in detector-attribution order, without compiled matchers.
12#[derive(Debug, Clone, Copy, Serialize)]
13pub struct SecretPattern {
14    pub redaction_name: &'static str,
15    pub detector: &'static str,
16    pub source: &'static str,
17    pub title: &'static str,
18    pub regex: &'static str,
19    pub precision: &'static str,
20}
21
22pub fn secret_pattern_catalog() -> Vec<SecretPattern> {
23    DEFAULT_SECRET_PATTERN_SPECS
24        .iter()
25        .map(|spec| SecretPattern {
26            redaction_name: spec.redaction_name,
27            detector: spec.detector,
28            source: spec.source,
29            title: spec.title,
30            regex: spec.regex,
31            precision: spec.precision,
32        })
33        .collect()
34}
35
36#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
37pub struct SecretFinding {
38    pub detector: String,
39    pub source: String,
40    pub title: String,
41    pub precision: String,
42    pub line: usize,
43    pub column_start: usize,
44    pub column_end: usize,
45    pub start_offset: usize,
46    pub end_offset: usize,
47    pub redacted: String,
48    pub fingerprint: String,
49}
50
51/// One catalog entry paired with its process-wide compiled matcher.
52///
53/// Scanning and redaction are different questions asked of the same catalog, so
54/// they share one compiled copy rather than each paying to build their own.
55pub struct CompiledSecretPattern {
56    pub spec: &'static SecretPatternSpec,
57    pub regex: Regex,
58}
59
60impl CompiledSecretPattern {
61    pub fn accepts_match(&self, input: &str, start: usize, end: usize) -> bool {
62        self.spec.accepts_match(input, start, end)
63    }
64}
65
66static DEFAULT_PATTERNS: OnceLock<Vec<CompiledSecretPattern>> = OnceLock::new();
67static HIGH_ENTROPY_ASSIGNMENT: OnceLock<Regex> = OnceLock::new();
68
69/// The shared catalog, compiled on first use.
70pub fn compiled_secret_patterns() -> &'static [CompiledSecretPattern] {
71    DEFAULT_PATTERNS.get_or_init(|| {
72        DEFAULT_SECRET_PATTERN_SPECS
73            .iter()
74            .map(|spec| CompiledSecretPattern {
75                spec,
76                regex: Regex::new(spec.regex).unwrap_or_else(|error| {
77                    panic!("invalid {} secret regex: {error}", spec.detector)
78                }),
79            })
80            .collect()
81    })
82}
83
84/// Whether the catalog has already been compiled. Hosts that warm it eagerly
85/// during startup assert against this so the cost cannot silently move back
86/// onto a deep call stack.
87pub fn secret_patterns_compiled() -> bool {
88    DEFAULT_PATTERNS.get().is_some()
89}
90
91fn high_entropy_assignment() -> &'static Regex {
92    HIGH_ENTROPY_ASSIGNMENT.get_or_init(|| {
93        Regex::new(
94            r#"(?im)(?:secret|token|api[_-]?key|access[_-]?key|password|passwd|pwd|client[_-]?secret|private[_-]?key)[^\n:=]{0,32}(?::|=)\s*["']([A-Za-z0-9+/=_\.-]{20,})["']"#,
95        )
96        .expect("high-entropy secret pattern is valid")
97    })
98}
99
100pub fn scan_secrets(content: &str) -> Vec<SecretFinding> {
101    let line_starts = line_starts(content);
102    let mut findings = Vec::new();
103
104    for rule in compiled_secret_patterns() {
105        for matched in rule.regex.find_iter(content) {
106            if !rule.accepts_match(content, matched.start(), matched.end()) {
107                continue;
108            }
109            findings.push(build_finding(
110                content,
111                &line_starts,
112                rule.spec.detector,
113                rule.spec.source,
114                rule.spec.title,
115                rule.spec.precision,
116                matched.start(),
117                matched.end(),
118                matched.as_str(),
119            ));
120        }
121    }
122
123    for captures in high_entropy_assignment().captures_iter(content) {
124        let Some(secret) = captures.get(1) else {
125            continue;
126        };
127        if shannon_entropy(secret.as_str()) < HIGH_ENTROPY_THRESHOLD {
128            continue;
129        }
130        findings.push(build_finding(
131            content,
132            &line_starts,
133            "high-entropy-credential-assignment",
134            "trufflehog",
135            "High-entropy secret assignment",
136            PRECISION_HEURISTIC,
137            secret.start(),
138            secret.end(),
139            secret.as_str(),
140        ));
141    }
142
143    findings.sort_by(|left, right| {
144        left.start_offset
145            .cmp(&right.start_offset)
146            .then(left.end_offset.cmp(&right.end_offset))
147            .then(left.detector.cmp(&right.detector))
148    });
149    let spans = findings
150        .iter()
151        .map(|finding| {
152            (
153                finding.start_offset,
154                finding.end_offset,
155                detector_specificity(&finding.detector),
156            )
157        })
158        .collect::<Vec<_>>();
159    findings.retain(|finding| {
160        let specificity = detector_specificity(&finding.detector);
161        !spans.iter().any(|(start, end, other_specificity)| {
162            *other_specificity > specificity
163                && finding.start_offset < *end
164                && *start < finding.end_offset
165        })
166    });
167    findings.dedup_by(|left, right| {
168        left.detector == right.detector
169            && left.start_offset == right.start_offset
170            && left.end_offset == right.end_offset
171    });
172    findings
173}
174
175fn detector_specificity(detector: &str) -> u8 {
176    match detector {
177        "sensitive-assignment" => 0,
178        "high-entropy-credential-assignment" => 1,
179        _ => 2,
180    }
181}
182
183#[allow(clippy::too_many_arguments)]
184fn build_finding(
185    content: &str,
186    line_starts: &[usize],
187    detector: &str,
188    source: &str,
189    title: &str,
190    precision: &str,
191    start_offset: usize,
192    end_offset: usize,
193    matched: &str,
194) -> SecretFinding {
195    let (line, column_start) = offset_to_line_col(content, line_starts, start_offset);
196    let (_, column_end) = offset_to_line_col(content, line_starts, end_offset);
197    SecretFinding {
198        detector: detector.to_string(),
199        source: source.to_string(),
200        title: title.to_string(),
201        precision: precision.to_string(),
202        line,
203        column_start,
204        column_end,
205        start_offset,
206        end_offset,
207        redacted: redact_match(matched),
208        fingerprint: fingerprint(matched),
209    }
210}
211
212fn line_starts(content: &str) -> Vec<usize> {
213    std::iter::once(0)
214        .chain(
215            content
216                .bytes()
217                .enumerate()
218                .filter_map(|(index, byte)| (byte == b'\n').then_some(index + 1)),
219        )
220        .collect()
221}
222
223#[expect(
224    clippy::string_slice,
225    reason = "line starts and regex match offsets are char boundaries"
226)]
227fn offset_to_line_col(content: &str, starts: &[usize], offset: usize) -> (usize, usize) {
228    let line_index = starts
229        .partition_point(|start| *start <= offset)
230        .saturating_sub(1);
231    let line_start = starts[line_index];
232    (
233        line_index + 1,
234        content[line_start..offset].chars().count() + 1,
235    )
236}
237
238fn redact_match(matched: &str) -> String {
239    if matched.starts_with("-----BEGIN ") {
240        return format!(
241            "{} …",
242            matched
243                .lines()
244                .next()
245                .unwrap_or("-----BEGIN PRIVATE KEY-----")
246        );
247    }
248    let chars = matched.chars().collect::<Vec<_>>();
249    if chars.len() <= 8 {
250        return "*".repeat(chars.len());
251    }
252    let prefix = chars.iter().take(4).collect::<String>();
253    let suffix = chars[chars.len() - 4..].iter().collect::<String>();
254    format!("{prefix}…{suffix}")
255}
256
257fn fingerprint(matched: &str) -> String {
258    let digest = sha2::Sha256::digest(matched.as_bytes());
259    hex::encode(&digest[..8])
260}
261
262fn shannon_entropy(value: &str) -> f64 {
263    let mut counts = BTreeMap::new();
264    for character in value.chars() {
265        *counts.entry(character).or_insert(0_usize) += 1;
266    }
267    let length = value.chars().count() as f64;
268    counts
269        .values()
270        .map(|count| {
271            let probability = *count as f64 / length;
272            -(probability * probability.log2())
273        })
274        .sum()
275}
276
277#[cfg(test)]
278mod tests {
279    use super::*;
280
281    #[test]
282    fn scans_and_deduplicates_the_canonical_catalog() {
283        let findings = scan_secrets(r#"token = "ghp_1234567890abcdefghijklmnopqrstuvwxyzAB""#);
284        assert_eq!(findings.len(), 1);
285        assert_eq!(findings[0].detector, "github-token");
286        assert_eq!(findings[0].precision, "high");
287    }
288
289    #[test]
290    fn source_with_secretish_identifiers_remains_clean() {
291        assert!(scan_secrets("pub const Token = struct { kind: u8 };\n").is_empty());
292    }
293
294    #[test]
295    fn source_expressions_do_not_become_secret_findings() {
296        let source = "const token = process.env.GITHUB_TOKEN ?? \"\";\nlet apiKey = readKey(path);\nlet accessToken = readKey2(path);\nconst password = credentials.current;";
297        assert!(scan_secrets(source).is_empty());
298    }
299}
harn_kernel/pure/secret_scan.rs

harn_kernel/pure/
secret_scan.rs
