harn_vm/stdlib/
secret_scan.rs

1use std::collections::BTreeMap;
2use std::sync::LazyLock;
3
4use regex::Regex;
5use serde::{Deserialize, Serialize};
6use sha2::Digest;
7
8use crate::event_log::{active_event_log, EventLog, LogEvent, Topic};
9use crate::secret_patterns::{compiled_default_secret_patterns, PRECISION_HEURISTIC};
10use crate::value::{VmError, VmValue};
11use crate::vm::Vm;
12
13pub const SECRET_SCAN_AUDIT_TOPIC: &str = "audit.secret_scan";
14const HIGH_ENTROPY_THRESHOLD: f64 = 3.5;
15
16#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
17pub struct SecretFinding {
18    pub detector: String,
19    pub source: String,
20    pub title: String,
21    /// Detector precision class (`"high"` for self-identifying token shapes,
22    /// `"heuristic"` for keyword/context matches). Lets consumers pick a policy
23    /// per class — e.g. hard-block only `"high"` findings — without hard-coding
24    /// detector names. See [`crate::secret_patterns::SecretPatternSpec`].
25    pub precision: String,
26    pub line: usize,
27    pub column_start: usize,
28    pub column_end: usize,
29    pub start_offset: usize,
30    pub end_offset: usize,
31    pub redacted: String,
32    pub fingerprint: String,
33}
34
35static HIGH_ENTROPY_ASSIGNMENT_RULE: LazyLock<Regex> = LazyLock::new(|| {
36    Regex::new(
37        r#"(?im)(?:secret|token|api[_-]?key|access[_-]?key|password|passwd|pwd|client[_-]?secret|private[_-]?key)[^\n:=]{0,32}(?::|=)\s*["']([A-Za-z0-9+/=_\.-]{20,})["']"#,
38    )
39    .unwrap()
40});
41
42pub fn scan_content(content: &str) -> Vec<SecretFinding> {
43    let line_starts = line_starts(content);
44    let mut findings = Vec::new();
45
46    for rule in compiled_default_secret_patterns() {
47        for mat in rule.regex.find_iter(content) {
48            findings.push(build_finding(
49                content,
50                &line_starts,
51                rule.spec.detector,
52                rule.spec.source,
53                rule.spec.title,
54                rule.spec.precision,
55                mat.start(),
56                mat.end(),
57                mat.as_str(),
58            ));
59        }
60    }
61
62    for captures in HIGH_ENTROPY_ASSIGNMENT_RULE.captures_iter(content) {
63        let Some(secret) = captures.get(1) else {
64            continue;
65        };
66        if shannon_entropy(secret.as_str()) < HIGH_ENTROPY_THRESHOLD {
67            continue;
68        }
69        findings.push(build_finding(
70            content,
71            &line_starts,
72            "high-entropy-credential-assignment",
73            "trufflehog",
74            "High-entropy secret assignment",
75            PRECISION_HEURISTIC,
76            secret.start(),
77            secret.end(),
78            secret.as_str(),
79        ));
80    }
81
82    findings.sort_by(|left, right| {
83        left.start_offset
84            .cmp(&right.start_offset)
85            .then(left.end_offset.cmp(&right.end_offset))
86            .then(left.detector.cmp(&right.detector))
87    });
88    let higher_specificity_spans = findings
89        .iter()
90        .map(|finding| {
91            (
92                finding.start_offset,
93                finding.end_offset,
94                detector_specificity(&finding.detector),
95            )
96        })
97        .collect::<Vec<_>>();
98    findings.retain(|finding| {
99        let specificity = detector_specificity(&finding.detector);
100        !higher_specificity_spans
101            .iter()
102            .any(|(start, end, other_specificity)| {
103                *other_specificity > specificity
104                    && spans_overlap((finding.start_offset, finding.end_offset), (*start, *end))
105            })
106    });
107    findings.dedup_by(|left, right| {
108        left.detector == right.detector
109            && left.start_offset == right.start_offset
110            && left.end_offset == right.end_offset
111    });
112    findings
113}
114
115fn detector_specificity(detector: &str) -> u8 {
116    match detector {
117        "sensitive-assignment" => 0,
118        "high-entropy-credential-assignment" => 1,
119        _ => 2,
120    }
121}
122
123fn spans_overlap(left: (usize, usize), right: (usize, usize)) -> bool {
124    left.0 < right.1 && right.0 < left.1
125}
126
127pub async fn append_secret_scan_audit<L: EventLog + ?Sized>(
128    event_log: &L,
129    caller: &str,
130    content_len: usize,
131    findings: &[SecretFinding],
132) -> Result<(), crate::event_log::LogError> {
133    let payload = serde_json::json!({
134        "caller": caller,
135        "content_len": content_len,
136        "finding_count": findings.len(),
137        "clean": findings.is_empty(),
138        "findings": findings
139            .iter()
140            .map(|finding| {
141                serde_json::json!({
142                    "detector": finding.detector,
143                    "source": finding.source,
144                    "title": finding.title,
145                    "precision": finding.precision,
146                    "line": finding.line,
147                    "column_start": finding.column_start,
148                    "column_end": finding.column_end,
149                    "start_offset": finding.start_offset,
150                    "end_offset": finding.end_offset,
151                    "fingerprint": finding.fingerprint,
152                    "redacted": finding.redacted,
153                })
154            })
155            .collect::<Vec<_>>(),
156        "observed_at": crate::orchestration::now_unix_seconds_text(),
157    });
158    let topic = Topic::new(SECRET_SCAN_AUDIT_TOPIC).expect("secret scan audit topic is valid");
159    let kind = if findings.is_empty() {
160        "scan_clean"
161    } else {
162        "scan_detected"
163    };
164    event_log
165        .append(&topic, LogEvent::new(kind, payload))
166        .await?;
167    Ok(())
168}
169
170pub async fn audit_secret_scan_active(
171    caller: &str,
172    content_len: usize,
173    findings: &[SecretFinding],
174) {
175    emit_secret_scan_log(caller, content_len, findings);
176
177    let Some(event_log) = active_event_log() else {
178        return;
179    };
180
181    if let Err(error) =
182        append_secret_scan_audit(event_log.as_ref(), caller, content_len, findings).await
183    {
184        crate::events::log_warn(
185            "secret_scan.audit",
186            &format!("failed to append secret scan audit event: {error}"),
187        );
188    }
189}
190
191pub(crate) fn register_secret_scan_builtins(vm: &mut Vm) {
192    vm.register_async_builtin("secret_scan", |_ctx, args| async move {
193        let content = match args.first() {
194            Some(VmValue::Nil) | None => {
195                return Err(VmError::Runtime("secret_scan: content is required".into()));
196            }
197            Some(value) => value.display(),
198        };
199        let audit = secret_scan_audit_option(args.get(1))?;
200
201        let findings = scan_content(&content);
202        if audit {
203            audit_secret_scan_active("stdlib.secret_scan", content.len(), &findings).await;
204        }
205
206        let value = serde_json::to_value(findings)
207            .map_err(|error| VmError::Runtime(format!("secret_scan: {error}")))?;
208        Ok(crate::schema::json_to_vm_value(&value))
209    });
210}
211
212/// Parse the optional `{ audit: bool }` second argument. Audit defaults to
213/// `true` (back-compatible with the one-arg form). Callers on a hot path — e.g.
214/// a per-edit or per-command guard that only needs the catalog-backed findings
215/// — pass `{audit: false}` to skip appending an `audit.secret_scan` event on
216/// every call.
217fn secret_scan_audit_option(value: Option<&VmValue>) -> Result<bool, VmError> {
218    let map = match value {
219        None | Some(VmValue::Nil) => return Ok(true),
220        Some(VmValue::Dict(map)) => map,
221        Some(other) => {
222            return Err(VmError::Runtime(format!(
223                "secret_scan: options must be a dict or nil; got {}",
224                other.type_name()
225            )));
226        }
227    };
228    match map.get("audit") {
229        None | Some(VmValue::Nil) => Ok(true),
230        Some(VmValue::Bool(flag)) => Ok(*flag),
231        Some(other) => Err(VmError::Runtime(format!(
232            "secret_scan: options.audit must be a bool; got {}",
233            other.type_name()
234        ))),
235    }
236}
237
238#[allow(clippy::too_many_arguments)]
239fn build_finding(
240    content: &str,
241    line_starts: &[usize],
242    detector: &str,
243    source: &str,
244    title: &str,
245    precision: &str,
246    start_offset: usize,
247    end_offset: usize,
248    matched: &str,
249) -> SecretFinding {
250    let (line, column_start) = offset_to_line_col(content, line_starts, start_offset);
251    let (_, column_end) = offset_to_line_col(content, line_starts, end_offset);
252    SecretFinding {
253        detector: detector.to_string(),
254        source: source.to_string(),
255        title: title.to_string(),
256        precision: precision.to_string(),
257        line,
258        column_start,
259        column_end,
260        start_offset,
261        end_offset,
262        redacted: redact_match(matched),
263        fingerprint: fingerprint(matched),
264    }
265}
266
267fn line_starts(content: &str) -> Vec<usize> {
268    let mut starts = vec![0];
269    for (index, byte) in content.bytes().enumerate() {
270        if byte == b'\n' {
271            starts.push(index + 1);
272        }
273    }
274    starts
275}
276
277fn offset_to_line_col(content: &str, line_starts: &[usize], offset: usize) -> (usize, usize) {
278    let line_index = line_starts
279        .partition_point(|start| *start <= offset)
280        .saturating_sub(1);
281    let line_start = line_starts[line_index];
282    let column = content[line_start..offset].chars().count() + 1;
283    (line_index + 1, column)
284}
285
286fn redact_match(matched: &str) -> String {
287    if matched.starts_with("-----BEGIN ") {
288        return format!(
289            "{} …",
290            matched
291                .lines()
292                .next()
293                .unwrap_or("-----BEGIN PRIVATE KEY-----")
294        );
295    }
296
297    let chars: Vec<char> = matched.chars().collect();
298    if chars.len() <= 8 {
299        return "*".repeat(chars.len());
300    }
301    let prefix: String = chars.iter().take(4).collect();
302    let suffix: String = chars[chars.len().saturating_sub(4)..].iter().collect();
303    format!("{prefix}…{suffix}")
304}
305
306fn fingerprint(matched: &str) -> String {
307    let hash = sha2::Sha256::digest(matched.as_bytes());
308    let hex: String = hash.iter().map(|byte| format!("{byte:02x}")).collect();
309    hex[..16].to_string()
310}
311
312fn shannon_entropy(value: &str) -> f64 {
313    let mut counts = BTreeMap::new();
314    for ch in value.chars() {
315        *counts.entry(ch).or_insert(0usize) += 1;
316    }
317    let len = value.chars().count() as f64;
318    counts
319        .values()
320        .map(|count| {
321            let probability = *count as f64 / len;
322            -(probability * probability.log2())
323        })
324        .sum()
325}
326
327fn emit_secret_scan_log(caller: &str, content_len: usize, findings: &[SecretFinding]) {
328    let metadata = serde_json::json!({
329        "topic": SECRET_SCAN_AUDIT_TOPIC,
330        "caller": caller,
331        "content_len": content_len,
332        "finding_count": findings.len(),
333        "clean": findings.is_empty(),
334        "findings": findings
335            .iter()
336            .map(|finding| serde_json::json!({
337                "detector": finding.detector,
338                "source": finding.source,
339                "line": finding.line,
340                "fingerprint": finding.fingerprint,
341                "redacted": finding.redacted,
342            }))
343            .collect::<Vec<_>>(),
344    });
345    let metadata = metadata
346        .as_object()
347        .cloned()
348        .map(|object| object.into_iter().collect::<BTreeMap<_, _>>())
349        .unwrap_or_default();
350    crate::events::log_info_meta("secret_scan.audit", "secret scan completed", metadata);
351}
352
353#[cfg(test)]
354mod tests {
355    use super::*;
356
357    use crate::event_log::{EventLog, MemoryEventLog};
358    use std::collections::BTreeSet;
359
360    #[test]
361    fn scan_content_detects_specific_rules_and_entropy_rule() {
362        let findings = scan_content(
363            r#"
364github_token = "ghp_1234567890abcdefghijklmnopqrstuvwxyzAB"
365config = { client_secret: "QWxhZGRpbjpPcGVuU2VzYW1lQWNjZXNzVG9rZW4=" }
366"#,
367        );
368
369        assert!(findings
370            .iter()
371            .any(|finding| finding.detector == "github-token"));
372        assert!(findings
373            .iter()
374            .any(|finding| finding.detector == "high-entropy-credential-assignment"));
375        assert!(!findings
376            .iter()
377            .any(|finding| finding.detector == "sensitive-assignment"));
378    }
379
380    #[test]
381    fn scan_content_deduplicates_generic_assignment_overlaps() {
382        let findings = scan_content(r#"token = "ghp_1234567890abcdefghijklmnopqrstuvwxyzAB""#);
383        assert_eq!(findings.len(), 1);
384        assert_eq!(findings[0].detector, "github-token");
385    }
386
387    #[test]
388    fn precision_class_splits_token_shapes_from_keyword_heuristics() {
389        let findings = scan_content(
390            "ghp_1234567890abcdefghijklmnopqrstuvwxyzAB\npassword = \"s3cr3t-value-here\"",
391        );
392        let precision = |detector: &str| {
393            findings
394                .iter()
395                .find(|finding| finding.detector == detector)
396                .map(|finding| finding.precision.as_str())
397        };
398        // A self-identifying token shape is high precision (safe to hard-block).
399        assert_eq!(precision("github-token"), Some("high"));
400        // A keyword/context match is heuristic (redaction-only, over-blocks).
401        assert_eq!(precision("sensitive-assignment"), Some("heuristic"));
402        // Every finding is classified.
403        assert!(findings
404            .iter()
405            .all(|finding| finding.precision == "high" || finding.precision == "heuristic"));
406    }
407
408    #[test]
409    fn scan_content_keeps_generic_assignment_without_specific_detector() {
410        let findings = scan_content(r#"token = "secret123""#);
411        assert_eq!(findings.len(), 1);
412        assert_eq!(findings[0].detector, "sensitive-assignment");
413    }
414
415    #[test]
416    fn scan_content_preserves_source_declarations_with_secretish_identifiers() {
417        let findings = scan_content("pub const Token = struct { kind: u8 };\n");
418        assert!(findings.is_empty());
419    }
420
421    #[test]
422    fn scan_content_redacts_private_key_blocks() {
423        let findings = scan_content(
424            "-----BEGIN OPENSSH PRIVATE KEY-----\nZXhhbXBsZQ==\n-----END OPENSSH PRIVATE KEY-----\n",
425        );
426        assert_eq!(findings.len(), 1);
427        assert_eq!(findings[0].detector, "private-key-block");
428        assert_eq!(
429            findings[0].end_offset - findings[0].start_offset,
430            "-----BEGIN OPENSSH PRIVATE KEY-----\nZXhhbXBsZQ==\n-----END OPENSSH PRIVATE KEY-----"
431                .len()
432        );
433        assert_eq!(
434            findings[0].redacted,
435            "-----BEGIN OPENSSH PRIVATE KEY----- …"
436        );
437    }
438
439    #[test]
440    fn scan_content_covers_redaction_only_token_shapes() {
441        let findings = scan_content(
442            "Authorization: Bearer abcDEFghi123_-+/=xyz\njwt=eyJabcd.eyJefgh.signature_pad\n",
443        );
444        let detectors = findings
445            .iter()
446            .map(|finding| finding.detector.as_str())
447            .collect::<BTreeSet<_>>();
448        assert!(detectors.contains("bearer-token"));
449        assert!(detectors.contains("jwt-token"));
450    }
451
452    #[test]
453    fn scan_content_covers_ai_provider_token_shapes() {
454        let huggingface = format!("hf_{}", "a".repeat(24));
455        let cerebras = format!("csk-{}", "b".repeat(48));
456        let together = format!("tgp_v1_{}", "c".repeat(32));
457        let google = format!("AIza{}", "D".repeat(35));
458        let content = format!("{huggingface}\n{cerebras}\n{together}\n{google}\n");
459
460        let findings = scan_content(&content);
461        let detectors = findings
462            .iter()
463            .map(|finding| (finding.detector.as_str(), finding.source.as_str()))
464            .collect::<BTreeSet<_>>();
465
466        assert!(detectors.contains(&("huggingface-token", "huggingface-docs")));
467        assert!(detectors.contains(&("cerebras-api-key", "cerebras-docs")));
468        assert!(detectors.contains(&("together-api-key", "together-bug-report")));
469        assert!(detectors.contains(&("google-api-key", "microsoft-purview")));
470        for secret in [&huggingface, &cerebras, &together, &google] {
471            assert!(!findings
472                .iter()
473                .any(|finding| finding.redacted.contains(secret)));
474        }
475    }
476
477    #[tokio::test(flavor = "current_thread")]
478    async fn append_secret_scan_audit_writes_redacted_event() {
479        let log = MemoryEventLog::new(32);
480        let findings = scan_content(r#"token = "sk-abcdefghijklmnopqrstuvwx123456""#);
481        append_secret_scan_audit(&log, "test.secret_scan", 44, &findings)
482            .await
483            .unwrap();
484
485        let topic = Topic::new(SECRET_SCAN_AUDIT_TOPIC).unwrap();
486        let events = log.read_range(&topic, None, 10).await.unwrap();
487        assert_eq!(events.len(), 1);
488        assert_eq!(events[0].1.kind, "scan_detected");
489        assert_eq!(events[0].1.payload["caller"], "test.secret_scan");
490        let redacted = events[0].1.payload["findings"][0]["redacted"]
491            .as_str()
492            .unwrap();
493        assert!(redacted.contains('…'));
494        assert!(!redacted.contains("abcdefghijklmnopqrstuvwx123456"));
495    }
496}
harn_vm/stdlib/secret_scan.rs

harn_vm/stdlib/
secret_scan.rs
