cargo_context_core/scrub/
mod.rs

1//! Secret scrubber.
2//!
3//! Three detection layers, applied in this order:
4//!
5//! 1. **Pattern** — compiled regex rules (built-ins + user-defined).
6//! 2. **Entropy** — Shannon entropy on values adjacent to suspicious keys.
7//! 3. **Path** — whole-file redaction when a file path matches a glob.
8//!
9//! An allowlist of exact strings / regexes can override any layer.
10//!
11//! The scrubber is a value type; build one with [`Scrubber::with_builtins`]
12//! for the default set, [`Scrubber::from_config`] for a fully-specified
13//! config, or [`Scrubber::with_workspace`] which auto-loads
14//! `.cargo-context/scrub.yaml` if present.
15
16pub mod config;
17pub mod entropy;
18pub mod paths;
19
20use std::path::Path;
21
22use regex::Regex;
23use serde::{Deserialize, Serialize};
24use sha2::{Digest, Sha256};
25
26use crate::error::Result;
27
28pub use config::{BuiltinsMode, ReportConfig, ScrubConfig};
29pub use entropy::EntropyConfig;
30pub use paths::PathRules;
31
32#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
33#[serde(rename_all = "lowercase")]
34pub enum Severity {
35    Low,
36    Medium,
37    #[default]
38    High,
39    Critical,
40}
41
42#[derive(Debug, Clone, Serialize, Deserialize)]
43pub struct Pattern {
44    pub id: String,
45    pub regex: String,
46    #[serde(default = "default_category")]
47    pub category: String,
48    #[serde(default)]
49    pub replacement: Option<String>,
50    #[serde(default)]
51    pub severity: Severity,
52}
53
54fn default_category() -> String {
55    "generic".into()
56}
57
58#[derive(Debug)]
59struct CompiledPattern {
60    id: String,
61    category: String,
62    regex: Regex,
63    replacement: String,
64    severity: Severity,
65}
66
67/// Allowlist entries are applied *before* rules fire: an input segment that
68/// matches any allowlist entry is passed through unchanged.
69#[derive(Debug, Default)]
70pub(crate) struct AllowList {
71    pub(crate) exact: Vec<String>,
72    pub(crate) regex: Vec<Regex>,
73}
74
75impl AllowList {
76    pub(crate) fn is_allowed(&self, s: &str) -> bool {
77        self.exact.iter().any(|e| e == s) || self.regex.iter().any(|r| r.is_match(s))
78    }
79}
80
81/// A single redaction action taken by the scrubber.
82#[derive(Debug, Clone, Serialize, Deserialize)]
83pub struct Redaction {
84    pub rule_id: String,
85    pub category: String,
86    pub severity: Severity,
87    pub hash4: String,
88}
89
90/// Aggregate view of everything the scrubber touched on one input.
91#[derive(Debug, Default, Clone, Serialize, Deserialize)]
92pub struct ScrubReport {
93    pub redactions: Vec<Redaction>,
94}
95
96impl ScrubReport {
97    pub fn is_empty(&self) -> bool {
98        self.redactions.is_empty()
99    }
100
101    pub fn count_by_category(&self) -> std::collections::BTreeMap<String, usize> {
102        let mut out: std::collections::BTreeMap<String, usize> = Default::default();
103        for r in &self.redactions {
104            *out.entry(r.category.clone()).or_default() += 1;
105        }
106        out
107    }
108
109    pub fn has_severity_at_least(&self, min: Severity) -> bool {
110        let rank = |s: Severity| match s {
111            Severity::Low => 0,
112            Severity::Medium => 1,
113            Severity::High => 2,
114            Severity::Critical => 3,
115        };
116        self.redactions
117            .iter()
118            .any(|r| rank(r.severity) >= rank(min))
119    }
120
121    /// One-line human summary, e.g. "4 redacted (aws_key:1, jwt:2, entropy:1)".
122    pub fn summary(&self) -> String {
123        if self.redactions.is_empty() {
124            return "0 redacted".into();
125        }
126        let groups = self.count_by_category();
127        let parts: Vec<String> = groups.iter().map(|(k, v)| format!("{k}:{v}")).collect();
128        format!("{} redacted ({})", self.redactions.len(), parts.join(", "))
129    }
130}
131
132#[derive(Debug, Default)]
133pub struct Scrubber {
134    patterns: Vec<CompiledPattern>,
135    allowlist: AllowList,
136    entropy: EntropyConfig,
137    paths: PathRules,
138    report: ReportConfig,
139    /// Count of effectively disabled built-in rules (for `scrub --check`
140    /// reporting). Includes both `disable_builtins` removals and the whole
141    /// set dropped under `builtins: replace`.
142    effective_builtin_count: usize,
143    /// Count of user-defined patterns loaded from `patterns[]`.
144    effective_custom_count: usize,
145}
146
147impl Scrubber {
148    pub fn empty() -> Self {
149        Self::default()
150    }
151
152    /// Built-in regex patterns only; no entropy, no path rules.
153    pub fn with_builtins() -> Result<Self> {
154        let patterns = builtin_patterns()?;
155        let effective_builtin_count = patterns.len();
156        Ok(Self {
157            patterns,
158            allowlist: AllowList::default(),
159            entropy: EntropyConfig::default(),
160            paths: PathRules::default(),
161            report: ReportConfig::default(),
162            effective_builtin_count,
163            effective_custom_count: 0,
164        })
165    }
166
167    /// Build a scrubber from a parsed [`ScrubConfig`].
168    pub fn from_config(config: &ScrubConfig) -> Result<Self> {
169        let mut patterns: Vec<CompiledPattern> = match config.builtins {
170            BuiltinsMode::Replace => Vec::new(),
171            BuiltinsMode::Extend | BuiltinsMode::Disable => builtin_patterns()?,
172        };
173
174        // Filter out explicitly disabled built-in ids.
175        if !config.disable_builtins.is_empty() {
176            let drop: std::collections::HashSet<&str> =
177                config.disable_builtins.iter().map(String::as_str).collect();
178            patterns.retain(|p| !drop.contains(p.id.as_str()));
179        }
180        let effective_builtin_count = patterns.len();
181
182        // User-defined patterns.
183        let effective_custom_count = config.patterns.len();
184        for p in &config.patterns {
185            patterns.push(compile_pattern(p)?);
186        }
187
188        let mut allowlist = AllowList::default();
189        for entry in &config.allowlist {
190            if let Some(ex) = &entry.exact {
191                allowlist.exact.push(ex.clone());
192            }
193            if let Some(rg) = &entry.regex {
194                allowlist.regex.push(Regex::new(rg)?);
195            }
196        }
197
198        Ok(Self {
199            patterns,
200            allowlist,
201            entropy: EntropyConfig::from_raw(&config.entropy)?,
202            paths: PathRules::from_raw(&config.paths)?,
203            report: config.report.clone(),
204            effective_builtin_count,
205            effective_custom_count,
206        })
207    }
208
209    /// Load `.cargo-context/scrub.yaml` from `workspace_root` if present and
210    /// return a fully-configured scrubber. Falls back to built-ins on absence
211    /// or parse failure.
212    pub fn with_workspace(workspace_root: &Path) -> Result<Self> {
213        let path = workspace_root.join(".cargo-context/scrub.yaml");
214        if !path.exists() {
215            return Self::with_builtins();
216        }
217        let raw = std::fs::read_to_string(&path)?;
218        match serde_yaml::from_str::<ScrubConfig>(&raw) {
219            Ok(cfg) => Self::from_config(&cfg),
220            Err(_) => Self::with_builtins(), // malformed config shouldn't break pack builds
221        }
222    }
223
224    /// Extend the scrubber with extra user-defined patterns (e.g. loaded
225    /// from a non-default config path).
226    pub fn extend(&mut self, extra: Vec<Pattern>) -> Result<()> {
227        for p in extra {
228            self.patterns.push(compile_pattern(&p)?);
229        }
230        Ok(())
231    }
232
233    /// Redact secrets in `input`. Returns only the scrubbed string.
234    pub fn scrub(&self, input: &str) -> String {
235        self.scrub_with_report(input).0
236    }
237
238    /// Redact and also produce a [`ScrubReport`] listing every redaction.
239    pub fn scrub_with_report(&self, input: &str) -> (String, ScrubReport) {
240        let mut out = input.to_string();
241        let mut report = ScrubReport::default();
242
243        // Layer 1: regex patterns.
244        for p in &self.patterns {
245            out = p
246                .regex
247                .replace_all(&out, |caps: &regex::Captures<'_>| {
248                    let matched = &caps[0];
249                    if self.allowlist.is_allowed(matched) {
250                        return matched.to_string();
251                    }
252                    let hash = hash4(matched);
253                    report.redactions.push(Redaction {
254                        rule_id: p.id.clone(),
255                        category: p.category.clone(),
256                        severity: p.severity,
257                        hash4: hash.clone(),
258                    });
259                    render_replacement(&p.replacement, &p.category, &hash)
260                })
261                .into_owned();
262        }
263
264        // Layer 2: entropy detection on values adjacent to context keys.
265        if self.entropy.enabled {
266            let (new_out, entropy_redactions) = self.entropy.scan_and_redact(&out, &self.allowlist);
267            out = new_out;
268            report.redactions.extend(entropy_redactions);
269        }
270
271        (out, report)
272    }
273
274    /// Scrub a file's content, honoring path-based rules (whole-file
275    /// redaction for `redact_whole` matches; bypass for `exclude` matches).
276    pub fn scrub_file(&self, path: &Path, content: &str) -> (String, ScrubReport) {
277        if self.paths.is_excluded(path) {
278            return (content.to_string(), ScrubReport::default());
279        }
280        if self.paths.is_redact_whole(path) {
281            let mut report = ScrubReport::default();
282            report.redactions.push(Redaction {
283                rule_id: "path_redact_whole".into(),
284                category: "file".into(),
285                severity: Severity::High,
286                hash4: hash4(content),
287            });
288            return (format!("[REDACTED FILE: {}]", path.display()), report);
289        }
290        self.scrub_with_report(content)
291    }
292
293    /// Return `true` if this scrubber's path rules would redact `path` whole.
294    pub fn is_path_redacted(&self, path: &Path) -> bool {
295        !self.paths.is_excluded(path) && self.paths.is_redact_whole(path)
296    }
297
298    /// Return `true` if `path` is on the exclude list (bypasses all scrubbing).
299    pub fn is_path_excluded(&self, path: &Path) -> bool {
300        self.paths.is_excluded(path)
301    }
302
303    /// Report config (typically loaded from `.cargo-context/scrub.yaml`).
304    pub fn report_config(&self) -> &ReportConfig {
305        &self.report
306    }
307
308    /// Number of built-in patterns that survived `disable_builtins` and
309    /// `BuiltinsMode::Replace` filters. Used by `cargo context scrub --check`.
310    pub fn effective_builtin_count(&self) -> usize {
311        self.effective_builtin_count
312    }
313
314    /// Number of user-defined `patterns[]` entries from the config.
315    pub fn effective_custom_count(&self) -> usize {
316        self.effective_custom_count
317    }
318
319    /// Append each redaction to `report.log_file` as JSON Lines, if configured.
320    /// Values are never written — only `rule_id`, `category`, `severity`, and
321    /// the SHA-256 fingerprint (`hash4`). Timestamps are Unix epoch seconds.
322    ///
323    /// When `report.max_entries` is set, the log is truncated after writes to
324    /// retain only the most recent N entries.
325    pub fn log_redactions(&self, report: &ScrubReport) -> Result<()> {
326        let Some(path) = &self.report.log_file else {
327            return Ok(());
328        };
329        if report.redactions.is_empty() {
330            return Ok(());
331        }
332        append_log_lines(path, &report.redactions, self.report.max_entries)
333    }
334}
335
336fn append_log_lines(
337    path: &Path,
338    redactions: &[Redaction],
339    max_entries: Option<usize>,
340) -> Result<()> {
341    use std::io::{BufRead, BufReader, Seek, SeekFrom, Write};
342
343    if let Some(parent) = path.parent()
344        && !parent.as_os_str().is_empty()
345    {
346        std::fs::create_dir_all(parent)?;
347    }
348
349    let mut file = std::fs::OpenOptions::new()
350        .create(true)
351        .append(true)
352        .read(true)
353        .open(path)?;
354
355    let ts = std::time::SystemTime::now()
356        .duration_since(std::time::UNIX_EPOCH)
357        .map(|d| d.as_secs())
358        .unwrap_or(0);
359
360    for r in redactions {
361        let line = serde_json::to_string(&LogEntry {
362            ts,
363            rule_id: &r.rule_id,
364            category: &r.category,
365            severity: r.severity,
366            hash4: &r.hash4,
367        })?;
368        writeln!(file, "{line}")?;
369    }
370    file.flush()?;
371
372    if let Some(cap) = max_entries {
373        file.seek(SeekFrom::Start(0))?;
374        let reader = BufReader::new(&mut file);
375        let lines: Vec<String> = reader.lines().collect::<std::io::Result<_>>()?;
376        if lines.len() > cap {
377            let kept = &lines[lines.len() - cap..];
378            drop(file);
379            let mut new_file = std::fs::OpenOptions::new()
380                .write(true)
381                .truncate(true)
382                .open(path)?;
383            for line in kept {
384                writeln!(new_file, "{line}")?;
385            }
386            new_file.flush()?;
387        }
388    }
389
390    Ok(())
391}
392
393#[derive(Serialize)]
394struct LogEntry<'a> {
395    ts: u64,
396    rule_id: &'a str,
397    category: &'a str,
398    severity: Severity,
399    hash4: &'a str,
400}
401
402fn compile_pattern(p: &Pattern) -> Result<CompiledPattern> {
403    let replacement = p
404        .replacement
405        .clone()
406        .unwrap_or_else(|| format!("<REDACTED:{}:{{hash4}}>", p.category));
407    Ok(CompiledPattern {
408        id: p.id.clone(),
409        category: p.category.clone(),
410        regex: Regex::new(&p.regex)?,
411        replacement,
412        severity: p.severity,
413    })
414}
415
416fn builtin_patterns() -> Result<Vec<CompiledPattern>> {
417    let builtins: &[(&str, &str, &str, Severity)] = &[
418        (
419            "aws_access_key",
420            r"AKIA[0-9A-Z]{16}",
421            "aws_key",
422            Severity::Critical,
423        ),
424        (
425            "github_pat",
426            r"ghp_[A-Za-z0-9]{36,}",
427            "github",
428            Severity::Critical,
429        ),
430        (
431            "github_oauth",
432            r"gho_[A-Za-z0-9]{36,}",
433            "github",
434            Severity::Critical,
435        ),
436        (
437            "openai_key",
438            r"sk-[A-Za-z0-9]{20,}",
439            "api_key",
440            Severity::Critical,
441        ),
442        (
443            "anthropic_key",
444            r"sk-ant-[A-Za-z0-9\-_]{20,}",
445            "api_key",
446            Severity::Critical,
447        ),
448        (
449            "huggingface_token",
450            r"hf_[A-Za-z0-9]{30,}",
451            "api_key",
452            Severity::High,
453        ),
454        (
455            "google_api_key",
456            r"AIza[0-9A-Za-z\-_]{35}",
457            "api_key",
458            Severity::High,
459        ),
460        (
461            "slack_token",
462            r"xox[baprs]-[0-9A-Za-z\-]{10,}",
463            "slack",
464            Severity::High,
465        ),
466        (
467            "jwt",
468            r"eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+",
469            "jwt",
470            Severity::High,
471        ),
472        (
473            "private_key_pem",
474            r"-----BEGIN (RSA|EC|OPENSSH|PGP) PRIVATE KEY-----",
475            "pem",
476            Severity::Critical,
477        ),
478    ];
479    let mut patterns = Vec::with_capacity(builtins.len());
480    for (id, re, cat, sev) in builtins {
481        patterns.push(CompiledPattern {
482            id: (*id).into(),
483            category: (*cat).into(),
484            regex: Regex::new(re)?,
485            replacement: format!("<REDACTED:{cat}:{{hash4}}>"),
486            severity: *sev,
487        });
488    }
489    Ok(patterns)
490}
491
492fn render_replacement(template: &str, category: &str, hash4_val: &str) -> String {
493    template
494        .replace("{hash4}", hash4_val)
495        .replace("{category}", category)
496}
497
498pub(crate) fn hash4(s: &str) -> String {
499    let mut h = Sha256::new();
500    h.update(s.as_bytes());
501    let digest = h.finalize();
502    format!("{:02x}{:02x}", digest[0], digest[1])
503}
504
505#[cfg(test)]
506mod tests {
507    use super::*;
508
509    #[test]
510    fn redacts_aws_key() {
511        let s = Scrubber::with_builtins().unwrap();
512        let out = s.scrub("AWS_KEY=AKIAIOSFODNN7EXAMPLE trailing");
513        assert!(!out.contains("AKIAIOSFODNN7EXAMPLE"));
514        assert!(out.contains("<REDACTED:aws_key:"));
515        assert!(out.contains("trailing"));
516    }
517
518    #[test]
519    fn redacts_jwt() {
520        let s = Scrubber::with_builtins().unwrap();
521        let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NSJ9.abcDEF-_xyz";
522        let out = s.scrub(jwt);
523        assert!(!out.contains(jwt));
524        assert!(out.contains("<REDACTED:jwt:"));
525    }
526
527    #[test]
528    fn leaves_benign_text_alone() {
529        let s = Scrubber::with_builtins().unwrap();
530        let out = s.scrub("just some code // no secrets here");
531        assert_eq!(out, "just some code // no secrets here");
532    }
533
534    #[test]
535    fn hash_is_stable() {
536        assert_eq!(hash4("hello"), hash4("hello"));
537        assert_ne!(hash4("hello"), hash4("world"));
538    }
539
540    #[test]
541    fn report_counts_by_category() {
542        let s = Scrubber::with_builtins().unwrap();
543        let (_, report) = s.scrub_with_report(
544            "AKIAIOSFODNN7EXAMPLE and another AKIAIOSFODNN7EXAMPLW and ghp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
545        );
546        let by_cat = report.count_by_category();
547        assert_eq!(by_cat.get("aws_key").copied(), Some(2));
548        assert_eq!(by_cat.get("github").copied(), Some(1));
549    }
550
551    #[test]
552    fn report_summary_is_human_readable() {
553        let s = Scrubber::with_builtins().unwrap();
554        let (_, report) = s.scrub_with_report("AKIAIOSFODNN7EXAMPLE");
555        assert!(report.summary().contains("redacted"));
556        assert!(report.summary().contains("aws_key"));
557    }
558
559    #[test]
560    fn allowlist_bypasses_regex() {
561        let config = ScrubConfig {
562            allowlist: vec![config::AllowlistEntry {
563                exact: Some("AKIAEXAMPLEALLOWED".into()),
564                regex: None,
565            }],
566            ..Default::default()
567        };
568        let s = Scrubber::from_config(&config).unwrap();
569        let out = s.scrub("AKIAEXAMPLEALLOWED");
570        // The exact string matches AWS regex but is allowlisted.
571        assert_eq!(out, "AKIAEXAMPLEALLOWED");
572    }
573
574    #[test]
575    fn disable_builtins_drops_named_rule() {
576        let config = ScrubConfig {
577            disable_builtins: vec!["jwt".into()],
578            ..Default::default()
579        };
580        let s = Scrubber::from_config(&config).unwrap();
581        let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NSJ9.abcDEF-_xyz";
582        let out = s.scrub(jwt);
583        assert_eq!(out, jwt, "jwt rule should have been disabled");
584    }
585
586    #[test]
587    fn builtins_replace_drops_all() {
588        let config = ScrubConfig {
589            builtins: BuiltinsMode::Replace,
590            ..Default::default()
591        };
592        let s = Scrubber::from_config(&config).unwrap();
593        assert_eq!(s.scrub("AKIAIOSFODNN7EXAMPLE"), "AKIAIOSFODNN7EXAMPLE");
594    }
595
596    #[test]
597    fn log_redactions_writes_values_free_jsonl() {
598        let tmp = tempfile::tempdir().unwrap();
599        let log_path = tmp.path().join("scrub.log");
600        let config = ScrubConfig {
601            report: ReportConfig {
602                log_file: Some(log_path.clone()),
603                ..Default::default()
604            },
605            ..Default::default()
606        };
607        let scrubber = Scrubber::from_config(&config).unwrap();
608        let (_, report) = scrubber.scrub_with_report(
609            "AWS_KEY=AKIAIOSFODNN7EXAMPLE and jwt: eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NSJ9.abcDEF-_xyz",
610        );
611        assert!(!report.is_empty());
612
613        scrubber.log_redactions(&report).unwrap();
614        let contents = std::fs::read_to_string(&log_path).unwrap();
615        let lines: Vec<&str> = contents.lines().collect();
616        assert_eq!(lines.len(), report.redactions.len());
617        // Each line is JSON with no actual secret values.
618        for (line, r) in lines.iter().zip(&report.redactions) {
619            let parsed: serde_json::Value = serde_json::from_str(line).unwrap();
620            assert_eq!(parsed["rule_id"], r.rule_id);
621            assert_eq!(parsed["category"], r.category);
622            assert_eq!(parsed["hash4"], r.hash4);
623            assert!(parsed["ts"].as_u64().unwrap() > 0);
624            // Values are never written.
625            assert!(!line.contains("AKIAIOSFODNN7EXAMPLE"));
626            assert!(!line.contains("eyJhbGciOi"));
627        }
628    }
629
630    #[test]
631    fn log_redactions_noop_when_unconfigured() {
632        let scrubber = Scrubber::with_builtins().unwrap();
633        let (_, report) = scrubber.scrub_with_report("AWS_KEY=AKIAIOSFODNN7EXAMPLE");
634        // No log_file configured → no-op, no error.
635        scrubber.log_redactions(&report).unwrap();
636    }
637
638    #[test]
639    fn log_redactions_creates_parent_dirs() {
640        let tmp = tempfile::tempdir().unwrap();
641        let log_path = tmp.path().join("nested/dir/scrub.log");
642        let config = ScrubConfig {
643            report: ReportConfig {
644                log_file: Some(log_path.clone()),
645                ..Default::default()
646            },
647            ..Default::default()
648        };
649        let scrubber = Scrubber::from_config(&config).unwrap();
650        let (_, report) = scrubber.scrub_with_report("AWS_KEY=AKIAIOSFODNN7EXAMPLE");
651        scrubber.log_redactions(&report).unwrap();
652        assert!(log_path.exists());
653    }
654
655    #[test]
656    fn effective_counts_reflect_config() {
657        let config = ScrubConfig {
658            disable_builtins: vec!["jwt".into()],
659            patterns: vec![Pattern {
660                id: "x".into(),
661                regex: "xx".into(),
662                category: "test".into(),
663                replacement: None,
664                severity: Severity::Low,
665            }],
666            ..Default::default()
667        };
668        let s = Scrubber::from_config(&config).unwrap();
669        // Built-ins minus jwt.
670        assert!(s.effective_builtin_count() > 0);
671        assert_eq!(s.effective_custom_count(), 1);
672    }
673
674    #[test]
675    fn has_severity_at_least_ranks_correctly() {
676        let mut r = ScrubReport::default();
677        r.redactions.push(Redaction {
678            rule_id: "x".into(),
679            category: "x".into(),
680            severity: Severity::Medium,
681            hash4: "0000".into(),
682        });
683        assert!(r.has_severity_at_least(Severity::Low));
684        assert!(r.has_severity_at_least(Severity::Medium));
685        assert!(!r.has_severity_at_least(Severity::High));
686    }
687}
cargo_context_core/scrub/mod.rs

cargo_context_core/scrub/
mod.rs
