harn_vm/stdlib/
secret_scan.rs

1use std::collections::BTreeMap;
2use std::sync::LazyLock;
3
4use regex::Regex;
5use serde::{Deserialize, Serialize};
6use sha2::Digest;
7
8use crate::event_log::{active_event_log, EventLog, LogEvent, Topic};
9use crate::secret_patterns::{compiled_default_secret_patterns, PRECISION_HEURISTIC};
10use crate::stdlib::macros::harn_builtin;
11use crate::value::{VmError, VmValue};
12use crate::vm::Vm;
13
14pub const SECRET_SCAN_AUDIT_TOPIC: &str = "audit.secret_scan";
15const HIGH_ENTROPY_THRESHOLD: f64 = 3.5;
16
17#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
18pub struct SecretFinding {
19    pub detector: String,
20    pub source: String,
21    pub title: String,
22    /// Detector precision class (`"high"` for self-identifying token shapes,
23    /// `"heuristic"` for keyword/context matches). Lets consumers pick a policy
24    /// per class — e.g. hard-block only `"high"` findings — without hard-coding
25    /// detector names. See [`crate::secret_patterns::SecretPatternSpec`].
26    pub precision: String,
27    pub line: usize,
28    pub column_start: usize,
29    pub column_end: usize,
30    pub start_offset: usize,
31    pub end_offset: usize,
32    pub redacted: String,
33    pub fingerprint: String,
34}
35
36static HIGH_ENTROPY_ASSIGNMENT_RULE: LazyLock<Regex> = LazyLock::new(|| {
37    Regex::new(
38        r#"(?im)(?:secret|token|api[_-]?key|access[_-]?key|password|passwd|pwd|client[_-]?secret|private[_-]?key)[^\n:=]{0,32}(?::|=)\s*["']([A-Za-z0-9+/=_\.-]{20,})["']"#,
39    )
40    .unwrap()
41});
42
43pub fn scan_content(content: &str) -> Vec<SecretFinding> {
44    let line_starts = line_starts(content);
45    let mut findings = Vec::new();
46
47    for rule in compiled_default_secret_patterns() {
48        for mat in rule.regex.find_iter(content) {
49            findings.push(build_finding(
50                content,
51                &line_starts,
52                rule.spec.detector,
53                rule.spec.source,
54                rule.spec.title,
55                rule.spec.precision,
56                mat.start(),
57                mat.end(),
58                mat.as_str(),
59            ));
60        }
61    }
62
63    for captures in HIGH_ENTROPY_ASSIGNMENT_RULE.captures_iter(content) {
64        let Some(secret) = captures.get(1) else {
65            continue;
66        };
67        if shannon_entropy(secret.as_str()) < HIGH_ENTROPY_THRESHOLD {
68            continue;
69        }
70        findings.push(build_finding(
71            content,
72            &line_starts,
73            "high-entropy-credential-assignment",
74            "trufflehog",
75            "High-entropy secret assignment",
76            PRECISION_HEURISTIC,
77            secret.start(),
78            secret.end(),
79            secret.as_str(),
80        ));
81    }
82
83    findings.sort_by(|left, right| {
84        left.start_offset
85            .cmp(&right.start_offset)
86            .then(left.end_offset.cmp(&right.end_offset))
87            .then(left.detector.cmp(&right.detector))
88    });
89    let higher_specificity_spans = findings
90        .iter()
91        .map(|finding| {
92            (
93                finding.start_offset,
94                finding.end_offset,
95                detector_specificity(&finding.detector),
96            )
97        })
98        .collect::<Vec<_>>();
99    findings.retain(|finding| {
100        let specificity = detector_specificity(&finding.detector);
101        !higher_specificity_spans
102            .iter()
103            .any(|(start, end, other_specificity)| {
104                *other_specificity > specificity
105                    && spans_overlap((finding.start_offset, finding.end_offset), (*start, *end))
106            })
107    });
108    findings.dedup_by(|left, right| {
109        left.detector == right.detector
110            && left.start_offset == right.start_offset
111            && left.end_offset == right.end_offset
112    });
113    findings
114}
115
116fn detector_specificity(detector: &str) -> u8 {
117    match detector {
118        "sensitive-assignment" => 0,
119        "high-entropy-credential-assignment" => 1,
120        _ => 2,
121    }
122}
123
124fn spans_overlap(left: (usize, usize), right: (usize, usize)) -> bool {
125    left.0 < right.1 && right.0 < left.1
126}
127
128pub async fn append_secret_scan_audit<L: EventLog + ?Sized>(
129    event_log: &L,
130    caller: &str,
131    content_len: usize,
132    findings: &[SecretFinding],
133) -> Result<(), crate::event_log::LogError> {
134    let payload = serde_json::json!({
135        "caller": caller,
136        "content_len": content_len,
137        "finding_count": findings.len(),
138        "clean": findings.is_empty(),
139        "findings": findings
140            .iter()
141            .map(|finding| {
142                serde_json::json!({
143                    "detector": finding.detector,
144                    "source": finding.source,
145                    "title": finding.title,
146                    "precision": finding.precision,
147                    "line": finding.line,
148                    "column_start": finding.column_start,
149                    "column_end": finding.column_end,
150                    "start_offset": finding.start_offset,
151                    "end_offset": finding.end_offset,
152                    "fingerprint": finding.fingerprint,
153                    "redacted": finding.redacted,
154                })
155            })
156            .collect::<Vec<_>>(),
157        "observed_at": crate::orchestration::now_unix_seconds_text(),
158    });
159    let topic = Topic::new(SECRET_SCAN_AUDIT_TOPIC).expect("secret scan audit topic is valid");
160    let kind = if findings.is_empty() {
161        "scan_clean"
162    } else {
163        "scan_detected"
164    };
165    event_log
166        .append(&topic, LogEvent::new(kind, payload))
167        .await?;
168    Ok(())
169}
170
171pub async fn audit_secret_scan_active(
172    caller: &str,
173    content_len: usize,
174    findings: &[SecretFinding],
175) {
176    emit_secret_scan_log(caller, content_len, findings);
177
178    let Some(event_log) = active_event_log() else {
179        return;
180    };
181
182    if let Err(error) =
183        append_secret_scan_audit(event_log.as_ref(), caller, content_len, findings).await
184    {
185        crate::events::log_warn(
186            "secret_scan.audit",
187            &format!("failed to append secret scan audit event: {error}"),
188        );
189    }
190}
191
192pub(crate) fn register_secret_scan_builtins(vm: &mut Vm) {
193    vm.register_builtin_def(&SECRET_SCAN_IMPL_DEF);
194}
195
196#[harn_builtin(
197    exposure = "pure",
198    effects = [],
199    sig = "secret_scan(content: any) -> list",
200    category = "security"
201)]
202fn secret_scan_impl(args: &[VmValue], _out: &mut String) -> Result<VmValue, VmError> {
203    let content = match args.first() {
204        Some(VmValue::Nil) | None => {
205            return Err(VmError::Runtime("secret_scan: content is required".into()));
206        }
207        Some(value) => value.display(),
208    };
209    let findings = scan_content(&content);
210    let value = serde_json::to_value(findings)
211        .map_err(|error| VmError::Runtime(format!("secret_scan: {error}")))?;
212    Ok(crate::schema::json_to_vm_value(&value))
213}
214
215#[allow(clippy::too_many_arguments)]
216fn build_finding(
217    content: &str,
218    line_starts: &[usize],
219    detector: &str,
220    source: &str,
221    title: &str,
222    precision: &str,
223    start_offset: usize,
224    end_offset: usize,
225    matched: &str,
226) -> SecretFinding {
227    let (line, column_start) = offset_to_line_col(content, line_starts, start_offset);
228    let (_, column_end) = offset_to_line_col(content, line_starts, end_offset);
229    SecretFinding {
230        detector: detector.to_string(),
231        source: source.to_string(),
232        title: title.to_string(),
233        precision: precision.to_string(),
234        line,
235        column_start,
236        column_end,
237        start_offset,
238        end_offset,
239        redacted: redact_match(matched),
240        fingerprint: fingerprint(matched),
241    }
242}
243
244fn line_starts(content: &str) -> Vec<usize> {
245    let mut starts = vec![0];
246    for (index, byte) in content.bytes().enumerate() {
247        if byte == b'\n' {
248            starts.push(index + 1);
249        }
250    }
251    starts
252}
253
254fn offset_to_line_col(content: &str, line_starts: &[usize], offset: usize) -> (usize, usize) {
255    let line_index = line_starts
256        .partition_point(|start| *start <= offset)
257        .saturating_sub(1);
258    let line_start = line_starts[line_index];
259    let column = content[line_start..offset].chars().count() + 1;
260    (line_index + 1, column)
261}
262
263fn redact_match(matched: &str) -> String {
264    if matched.starts_with("-----BEGIN ") {
265        return format!(
266            "{} …",
267            matched
268                .lines()
269                .next()
270                .unwrap_or("-----BEGIN PRIVATE KEY-----")
271        );
272    }
273
274    let chars: Vec<char> = matched.chars().collect();
275    if chars.len() <= 8 {
276        return "*".repeat(chars.len());
277    }
278    let prefix: String = chars.iter().take(4).collect();
279    let suffix: String = chars[chars.len().saturating_sub(4)..].iter().collect();
280    format!("{prefix}…{suffix}")
281}
282
283fn fingerprint(matched: &str) -> String {
284    let hash = sha2::Sha256::digest(matched.as_bytes());
285    let hex: String = hash.iter().map(|byte| format!("{byte:02x}")).collect();
286    hex[..16].to_string()
287}
288
289fn shannon_entropy(value: &str) -> f64 {
290    let mut counts = BTreeMap::new();
291    for ch in value.chars() {
292        *counts.entry(ch).or_insert(0usize) += 1;
293    }
294    let len = value.chars().count() as f64;
295    counts
296        .values()
297        .map(|count| {
298            let probability = *count as f64 / len;
299            -(probability * probability.log2())
300        })
301        .sum()
302}
303
304fn emit_secret_scan_log(caller: &str, content_len: usize, findings: &[SecretFinding]) {
305    let metadata = serde_json::json!({
306        "topic": SECRET_SCAN_AUDIT_TOPIC,
307        "caller": caller,
308        "content_len": content_len,
309        "finding_count": findings.len(),
310        "clean": findings.is_empty(),
311        "findings": findings
312            .iter()
313            .map(|finding| serde_json::json!({
314                "detector": finding.detector,
315                "source": finding.source,
316                "line": finding.line,
317                "fingerprint": finding.fingerprint,
318                "redacted": finding.redacted,
319            }))
320            .collect::<Vec<_>>(),
321    });
322    let metadata = metadata
323        .as_object()
324        .cloned()
325        .map(|object| object.into_iter().collect::<BTreeMap<_, _>>())
326        .unwrap_or_default();
327    crate::events::log_info_meta("secret_scan.audit", "secret scan completed", metadata);
328}
329
330#[cfg(test)]
331mod tests {
332    use super::*;
333
334    use crate::event_log::{EventLog, MemoryEventLog};
335    use std::collections::BTreeSet;
336
337    #[test]
338    fn scan_content_detects_specific_rules_and_entropy_rule() {
339        let findings = scan_content(
340            r#"
341github_token = "ghp_1234567890abcdefghijklmnopqrstuvwxyzAB"
342config = { client_secret: "QWxhZGRpbjpPcGVuU2VzYW1lQWNjZXNzVG9rZW4=" }
343"#,
344        );
345
346        assert!(findings
347            .iter()
348            .any(|finding| finding.detector == "github-token"));
349        assert!(findings
350            .iter()
351            .any(|finding| finding.detector == "high-entropy-credential-assignment"));
352        assert!(!findings
353            .iter()
354            .any(|finding| finding.detector == "sensitive-assignment"));
355    }
356
357    #[test]
358    fn scan_content_deduplicates_generic_assignment_overlaps() {
359        let findings = scan_content(r#"token = "ghp_1234567890abcdefghijklmnopqrstuvwxyzAB""#);
360        assert_eq!(findings.len(), 1);
361        assert_eq!(findings[0].detector, "github-token");
362    }
363
364    #[test]
365    fn precision_class_splits_token_shapes_from_keyword_heuristics() {
366        let findings = scan_content(
367            "ghp_1234567890abcdefghijklmnopqrstuvwxyzAB\npassword = \"s3cr3t-value-here\"",
368        );
369        let precision = |detector: &str| {
370            findings
371                .iter()
372                .find(|finding| finding.detector == detector)
373                .map(|finding| finding.precision.as_str())
374        };
375        // A self-identifying token shape is high precision (safe to hard-block).
376        assert_eq!(precision("github-token"), Some("high"));
377        // A keyword/context match is heuristic (redaction-only, over-blocks).
378        assert_eq!(precision("sensitive-assignment"), Some("heuristic"));
379        // Every finding is classified.
380        assert!(findings
381            .iter()
382            .all(|finding| finding.precision == "high" || finding.precision == "heuristic"));
383    }
384
385    #[test]
386    fn scan_content_keeps_generic_assignment_without_specific_detector() {
387        let findings = scan_content(r#"token = "secret123""#);
388        assert_eq!(findings.len(), 1);
389        assert_eq!(findings[0].detector, "sensitive-assignment");
390    }
391
392    #[test]
393    fn scan_content_preserves_source_declarations_with_secretish_identifiers() {
394        let findings = scan_content("pub const Token = struct { kind: u8 };\n");
395        assert!(findings.is_empty());
396    }
397
398    #[test]
399    fn scan_content_redacts_private_key_blocks() {
400        let findings = scan_content(
401            "-----BEGIN OPENSSH PRIVATE KEY-----\nZXhhbXBsZQ==\n-----END OPENSSH PRIVATE KEY-----\n",
402        );
403        assert_eq!(findings.len(), 1);
404        assert_eq!(findings[0].detector, "private-key-block");
405        assert_eq!(
406            findings[0].end_offset - findings[0].start_offset,
407            "-----BEGIN OPENSSH PRIVATE KEY-----\nZXhhbXBsZQ==\n-----END OPENSSH PRIVATE KEY-----"
408                .len()
409        );
410        assert_eq!(
411            findings[0].redacted,
412            "-----BEGIN OPENSSH PRIVATE KEY----- …"
413        );
414    }
415
416    #[test]
417    fn scan_content_covers_redaction_only_token_shapes() {
418        let findings = scan_content(
419            "Authorization: Bearer abcDEFghi123_-+/=xyz\njwt=eyJabcd.eyJefgh.signature_pad\n",
420        );
421        let detectors = findings
422            .iter()
423            .map(|finding| finding.detector.as_str())
424            .collect::<BTreeSet<_>>();
425        assert!(detectors.contains("bearer-token"));
426        assert!(detectors.contains("jwt-token"));
427    }
428
429    #[test]
430    fn scan_content_covers_ai_provider_token_shapes() {
431        let huggingface = format!("hf_{}", "a".repeat(24));
432        let cerebras = format!("csk-{}", "b".repeat(48));
433        let together = format!("tgp_v1_{}", "c".repeat(32));
434        let google = format!("AIza{}", "D".repeat(35));
435        let content = format!("{huggingface}\n{cerebras}\n{together}\n{google}\n");
436
437        let findings = scan_content(&content);
438        let detectors = findings
439            .iter()
440            .map(|finding| (finding.detector.as_str(), finding.source.as_str()))
441            .collect::<BTreeSet<_>>();
442
443        assert!(detectors.contains(&("huggingface-token", "huggingface-docs")));
444        assert!(detectors.contains(&("cerebras-api-key", "cerebras-docs")));
445        assert!(detectors.contains(&("together-api-key", "together-bug-report")));
446        assert!(detectors.contains(&("google-api-key", "microsoft-purview")));
447        for secret in [&huggingface, &cerebras, &together, &google] {
448            assert!(!findings
449                .iter()
450                .any(|finding| finding.redacted.contains(secret)));
451        }
452    }
453
454    #[tokio::test(flavor = "current_thread")]
455    async fn append_secret_scan_audit_writes_redacted_event() {
456        let log = MemoryEventLog::new(32);
457        let findings = scan_content(r#"token = "sk-abcdefghijklmnopqrstuvwx123456""#);
458        append_secret_scan_audit(&log, "test.secret_scan", 44, &findings)
459            .await
460            .unwrap();
461
462        let topic = Topic::new(SECRET_SCAN_AUDIT_TOPIC).unwrap();
463        let events = log.read_range(&topic, None, 10).await.unwrap();
464        assert_eq!(events.len(), 1);
465        assert_eq!(events[0].1.kind, "scan_detected");
466        assert_eq!(events[0].1.payload["caller"], "test.secret_scan");
467        let redacted = events[0].1.payload["findings"][0]["redacted"]
468            .as_str()
469            .unwrap();
470        assert!(redacted.contains('…'));
471        assert!(!redacted.contains("abcdefghijklmnopqrstuvwx123456"));
472    }
473}
harn_vm/stdlib/secret_scan.rs

harn_vm/stdlib/
secret_scan.rs
