harn_kernel/pure/
secret_scan.rs

1use std::collections::{BTreeMap, BTreeSet, HashMap};
2use std::sync::OnceLock;
3
4use harn_secret_catalog::{
5    SecretPatternSpec, DEFAULT_SECRET_PATTERN_SPECS, PRECISION_HEURISTIC, PRECISION_HIGH,
6};
7use regex::Regex;
8use serde::{Deserialize, Serialize};
9use sha2::Digest;
10
11const HIGH_ENTROPY_THRESHOLD: f64 = 3.5;
12
13/// Public catalog data in detector-attribution order, without compiled matchers.
14#[derive(Debug, Clone, Copy, Serialize)]
15pub struct SecretPattern {
16    pub redaction_name: &'static str,
17    pub detector: &'static str,
18    pub source: &'static str,
19    pub title: &'static str,
20    pub regex: &'static str,
21    pub precision: &'static str,
22}
23
24pub fn secret_pattern_catalog() -> Vec<SecretPattern> {
25    DEFAULT_SECRET_PATTERN_SPECS
26        .iter()
27        .map(|spec| SecretPattern {
28            redaction_name: spec.redaction_name,
29            detector: spec.detector,
30            source: spec.source,
31            title: spec.title,
32            regex: spec.regex,
33            precision: spec.precision,
34        })
35        .collect()
36}
37
38#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
39pub struct SecretFinding {
40    pub detector: String,
41    pub source: String,
42    pub title: String,
43    pub precision: String,
44    pub line: usize,
45    pub column_start: usize,
46    pub column_end: usize,
47    pub start_offset: usize,
48    pub end_offset: usize,
49    pub redacted: String,
50    pub fingerprint: String,
51}
52
53/// One catalog entry paired with its process-wide compiled matcher.
54///
55/// Scanning and redaction are different questions asked of the same catalog, so
56/// they share one compiled copy rather than each paying to build their own.
57pub struct CompiledSecretPattern {
58    pub spec: &'static SecretPatternSpec,
59    pub regex: Regex,
60}
61
62impl CompiledSecretPattern {
63    pub fn accepts_match(&self, input: &str, start: usize, end: usize) -> bool {
64        self.spec.accepts_match(input, start, end)
65    }
66}
67
68static DEFAULT_PATTERNS: OnceLock<Vec<CompiledSecretPattern>> = OnceLock::new();
69static HIGH_ENTROPY_ASSIGNMENT: OnceLock<Regex> = OnceLock::new();
70static HASHED_TOKEN: OnceLock<Regex> = OnceLock::new();
71
72pub fn scan_secret_hashes(content: &str, hashes: &[String]) -> Result<Vec<SecretFinding>, String> {
73    if hashes.is_empty()
74        || hashes.iter().any(|hash| {
75            hash.len() != 64
76                || !hash
77                    .bytes()
78                    .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
79        })
80    {
81        return Err(
82            "secret_scan: hashes must be a nonempty list of lowercase SHA-256 digests".into(),
83        );
84    }
85    let denied: BTreeSet<&str> = hashes.iter().map(String::as_str).collect();
86    let pattern = HASHED_TOKEN
87        .get_or_init(|| Regex::new(r"[A-Za-z0-9._-]+").expect("hashed token pattern is valid"));
88    let mut verdicts = HashMap::<String, Option<String>>::new();
89    let mut findings = Vec::new();
90    let mut scanned = 0;
91    let mut line = 1;
92    let mut column = 1;
93    for token in pattern.find_iter(content) {
94        let digest = verdicts
95            .entry(token.as_str().to_ascii_lowercase())
96            .or_insert_with_key(|normalized| {
97                let digest = super::sha256_hex(normalized.as_bytes());
98                denied.contains(digest.as_str()).then_some(digest)
99            });
100        let Some(digest) = digest else {
101            continue;
102        };
103        for character in content
104            .get(scanned..token.start())
105            .expect("regex boundaries are UTF-8 boundaries")
106            .chars()
107        {
108            if character == '\n' {
109                line += 1;
110                column = 1;
111            } else {
112                column += 1;
113            }
114        }
115        let column_start = column;
116        column += token.len();
117        scanned = token.end();
118        findings.push(SecretFinding {
119            detector: "sha256-denylist".into(),
120            source: "custom".into(),
121            title: "Denied token".into(),
122            precision: PRECISION_HIGH.into(),
123            line,
124            column_start,
125            column_end: column,
126            start_offset: token.start(),
127            end_offset: token.end(),
128            redacted: "[hash match]".into(),
129            fingerprint: digest.clone(),
130        });
131    }
132    Ok(findings)
133}
134
135/// The shared catalog, compiled on first use.
136pub fn compiled_secret_patterns() -> &'static [CompiledSecretPattern] {
137    DEFAULT_PATTERNS.get_or_init(|| {
138        DEFAULT_SECRET_PATTERN_SPECS
139            .iter()
140            .map(|spec| CompiledSecretPattern {
141                spec,
142                regex: Regex::new(spec.regex).unwrap_or_else(|error| {
143                    panic!("invalid {} secret regex: {error}", spec.detector)
144                }),
145            })
146            .collect()
147    })
148}
149
150/// Whether the catalog has already been compiled. Hosts that warm it eagerly
151/// during startup assert against this so the cost cannot silently move back
152/// onto a deep call stack.
153pub fn secret_patterns_compiled() -> bool {
154    DEFAULT_PATTERNS.get().is_some()
155}
156
157fn high_entropy_assignment() -> &'static Regex {
158    HIGH_ENTROPY_ASSIGNMENT.get_or_init(|| {
159        Regex::new(
160            r#"(?im)(?:secret|token|api[_-]?key|access[_-]?key|password|passwd|pwd|client[_-]?secret|private[_-]?key)[^\n:=]{0,32}(?::|=)\s*["']([A-Za-z0-9+/=_\.-]{20,})["']"#,
161        )
162        .expect("high-entropy secret pattern is valid")
163    })
164}
165
166pub fn scan_secrets(content: &str) -> Vec<SecretFinding> {
167    let line_starts = line_starts(content);
168    let mut findings = Vec::new();
169
170    for rule in compiled_secret_patterns() {
171        for matched in rule.regex.find_iter(content) {
172            if !rule.accepts_match(content, matched.start(), matched.end()) {
173                continue;
174            }
175            findings.push(build_finding(
176                content,
177                &line_starts,
178                rule.spec.detector,
179                rule.spec.source,
180                rule.spec.title,
181                rule.spec.precision,
182                matched.start(),
183                matched.end(),
184                matched.as_str(),
185            ));
186        }
187    }
188
189    for captures in high_entropy_assignment().captures_iter(content) {
190        let Some(secret) = captures.get(1) else {
191            continue;
192        };
193        if shannon_entropy(secret.as_str()) < HIGH_ENTROPY_THRESHOLD {
194            continue;
195        }
196        findings.push(build_finding(
197            content,
198            &line_starts,
199            "high-entropy-credential-assignment",
200            "trufflehog",
201            "High-entropy secret assignment",
202            PRECISION_HEURISTIC,
203            secret.start(),
204            secret.end(),
205            secret.as_str(),
206        ));
207    }
208
209    findings.sort_by(|left, right| {
210        left.start_offset
211            .cmp(&right.start_offset)
212            .then(left.end_offset.cmp(&right.end_offset))
213            .then(left.detector.cmp(&right.detector))
214    });
215    let spans = findings
216        .iter()
217        .map(|finding| {
218            (
219                finding.start_offset,
220                finding.end_offset,
221                detector_specificity(&finding.detector),
222            )
223        })
224        .collect::<Vec<_>>();
225    findings.retain(|finding| {
226        let specificity = detector_specificity(&finding.detector);
227        !spans.iter().any(|(start, end, other_specificity)| {
228            *other_specificity > specificity
229                && finding.start_offset < *end
230                && *start < finding.end_offset
231        })
232    });
233    findings.dedup_by(|left, right| {
234        left.detector == right.detector
235            && left.start_offset == right.start_offset
236            && left.end_offset == right.end_offset
237    });
238    findings
239}
240
241fn detector_specificity(detector: &str) -> u8 {
242    match detector {
243        "sensitive-assignment" => 0,
244        "high-entropy-credential-assignment" => 1,
245        _ => 2,
246    }
247}
248
249#[allow(clippy::too_many_arguments)]
250fn build_finding(
251    content: &str,
252    line_starts: &[usize],
253    detector: &str,
254    source: &str,
255    title: &str,
256    precision: &str,
257    start_offset: usize,
258    end_offset: usize,
259    matched: &str,
260) -> SecretFinding {
261    let (line, column_start) = offset_to_line_col(content, line_starts, start_offset);
262    let (_, column_end) = offset_to_line_col(content, line_starts, end_offset);
263    SecretFinding {
264        detector: detector.to_string(),
265        source: source.to_string(),
266        title: title.to_string(),
267        precision: precision.to_string(),
268        line,
269        column_start,
270        column_end,
271        start_offset,
272        end_offset,
273        redacted: redact_match(matched),
274        fingerprint: fingerprint(matched),
275    }
276}
277
278fn line_starts(content: &str) -> Vec<usize> {
279    std::iter::once(0)
280        .chain(
281            content
282                .bytes()
283                .enumerate()
284                .filter_map(|(index, byte)| (byte == b'\n').then_some(index + 1)),
285        )
286        .collect()
287}
288
289#[expect(
290    clippy::string_slice,
291    reason = "line starts and regex match offsets are char boundaries"
292)]
293fn offset_to_line_col(content: &str, starts: &[usize], offset: usize) -> (usize, usize) {
294    let line_index = starts
295        .partition_point(|start| *start <= offset)
296        .saturating_sub(1);
297    let line_start = starts[line_index];
298    (
299        line_index + 1,
300        content[line_start..offset].chars().count() + 1,
301    )
302}
303
304fn redact_match(matched: &str) -> String {
305    if matched.starts_with("-----BEGIN ") {
306        return format!(
307            "{} …",
308            matched
309                .lines()
310                .next()
311                .unwrap_or("-----BEGIN PRIVATE KEY-----")
312        );
313    }
314    let chars = matched.chars().collect::<Vec<_>>();
315    if chars.len() <= 8 {
316        return "*".repeat(chars.len());
317    }
318    let prefix = chars.iter().take(4).collect::<String>();
319    let suffix = chars[chars.len() - 4..].iter().collect::<String>();
320    format!("{prefix}…{suffix}")
321}
322
323fn fingerprint(matched: &str) -> String {
324    let digest = sha2::Sha256::digest(matched.as_bytes());
325    hex::encode(&digest[..8])
326}
327
328fn shannon_entropy(value: &str) -> f64 {
329    let mut counts = BTreeMap::new();
330    for character in value.chars() {
331        *counts.entry(character).or_insert(0_usize) += 1;
332    }
333    let length = value.chars().count() as f64;
334    counts
335        .values()
336        .map(|count| {
337            let probability = *count as f64 / length;
338            -(probability * probability.log2())
339        })
340        .sum()
341}
342
343#[cfg(test)]
344mod tests {
345    use super::*;
346
347    #[test]
348    fn scans_and_deduplicates_the_canonical_catalog() {
349        let findings = scan_secrets(r#"token = "ghp_1234567890abcdefghijklmnopqrstuvwxyzAB""#);
350        assert_eq!(findings.len(), 1);
351        assert_eq!(findings[0].detector, "github-token");
352        assert_eq!(findings[0].precision, "high");
353    }
354
355    #[test]
356    fn source_with_secretish_identifiers_remains_clean() {
357        assert!(scan_secrets("pub const Token = struct { kind: u8 };\n").is_empty());
358    }
359
360    #[test]
361    fn source_expressions_do_not_become_secret_findings() {
362        let source = "const token = process.env.GITHUB_TOKEN ?? \"\";\nlet apiKey = readKey(path);\nlet accessToken = readKey2(path);\nconst password = credentials.current;";
363        assert!(scan_secrets(source).is_empty());
364    }
365}
harn_kernel/pure/secret_scan.rs

harn_kernel/pure/
secret_scan.rs
