cargo_context_core/scrub/
mod.rs

1//! Secret scrubber.
2//!
3//! Three detection layers, applied in this order:
4//!
5//! 1. **Pattern** — compiled regex rules (built-ins + user-defined).
6//! 2. **Entropy** — Shannon entropy on values adjacent to suspicious keys.
7//! 3. **Path** — whole-file redaction when a file path matches a glob.
8//!
9//! An allowlist of exact strings / regexes can override any layer.
10//!
11//! The scrubber is a value type; build one with [`Scrubber::with_builtins`]
12//! for the default set, [`Scrubber::from_config`] for a fully-specified
13//! config, or [`Scrubber::with_workspace`] which auto-loads
14//! `.cargo-context/scrub.yaml` if present.
15
16pub mod config;
17pub mod entropy;
18pub mod paths;
19
20use std::path::Path;
21
22use regex::Regex;
23use serde::{Deserialize, Serialize};
24use sha2::{Digest, Sha256};
25
26use crate::error::Result;
27
28pub use config::{BuiltinsMode, ReportConfig, ScrubConfig};
29pub use entropy::EntropyConfig;
30pub use paths::PathRules;
31
32#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
33#[serde(rename_all = "lowercase")]
34pub enum Severity {
35    Low,
36    Medium,
37    #[default]
38    High,
39    Critical,
40}
41
42#[derive(Debug, Clone, Serialize, Deserialize)]
43pub struct Pattern {
44    pub id: String,
45    pub regex: String,
46    #[serde(default = "default_category")]
47    pub category: String,
48    #[serde(default)]
49    pub replacement: Option<String>,
50    #[serde(default)]
51    pub severity: Severity,
52}
53
54fn default_category() -> String {
55    "generic".into()
56}
57
58#[derive(Debug)]
59struct CompiledPattern {
60    id: String,
61    category: String,
62    regex: Regex,
63    replacement: String,
64    severity: Severity,
65}
66
67/// Allowlist entries are applied *before* rules fire: an input segment that
68/// matches any allowlist entry is passed through unchanged.
69#[derive(Debug, Default)]
70pub(crate) struct AllowList {
71    pub(crate) exact: Vec<String>,
72    pub(crate) regex: Vec<Regex>,
73}
74
75impl AllowList {
76    pub(crate) fn is_allowed(&self, s: &str) -> bool {
77        self.exact.iter().any(|e| e == s) || self.regex.iter().any(|r| r.is_match(s))
78    }
79}
80
81/// A single redaction action taken by the scrubber.
82#[derive(Debug, Clone, Serialize, Deserialize)]
83pub struct Redaction {
84    pub rule_id: String,
85    pub category: String,
86    pub severity: Severity,
87    pub hash4: String,
88}
89
90/// Aggregate view of everything the scrubber touched on one input.
91#[derive(Debug, Default, Clone, Serialize, Deserialize)]
92pub struct ScrubReport {
93    pub redactions: Vec<Redaction>,
94}
95
96impl ScrubReport {
97    pub fn is_empty(&self) -> bool {
98        self.redactions.is_empty()
99    }
100
101    pub fn count_by_category(&self) -> std::collections::BTreeMap<String, usize> {
102        let mut out: std::collections::BTreeMap<String, usize> = Default::default();
103        for r in &self.redactions {
104            *out.entry(r.category.clone()).or_default() += 1;
105        }
106        out
107    }
108
109    pub fn has_severity_at_least(&self, min: Severity) -> bool {
110        let rank = |s: Severity| match s {
111            Severity::Low => 0,
112            Severity::Medium => 1,
113            Severity::High => 2,
114            Severity::Critical => 3,
115        };
116        self.redactions
117            .iter()
118            .any(|r| rank(r.severity) >= rank(min))
119    }
120
121    /// One-line human summary, e.g. "4 redacted (aws_key:1, jwt:2, entropy:1)".
122    pub fn summary(&self) -> String {
123        if self.redactions.is_empty() {
124            return "0 redacted".into();
125        }
126        let groups = self.count_by_category();
127        let parts: Vec<String> = groups.iter().map(|(k, v)| format!("{k}:{v}")).collect();
128        format!("{} redacted ({})", self.redactions.len(), parts.join(", "))
129    }
130}
131
132#[derive(Debug, Default)]
133pub struct Scrubber {
134    patterns: Vec<CompiledPattern>,
135    allowlist: AllowList,
136    entropy: EntropyConfig,
137    paths: PathRules,
138    report: ReportConfig,
139    /// Count of effectively disabled built-in rules (for `scrub --check`
140    /// reporting). Includes both `disable_builtins` removals and the whole
141    /// set dropped under `builtins: replace`.
142    effective_builtin_count: usize,
143    /// Count of user-defined patterns loaded from `patterns[]`.
144    effective_custom_count: usize,
145}
146
147impl Scrubber {
148    pub fn empty() -> Self {
149        Self::default()
150    }
151
152    /// Built-in regex patterns only; no entropy, no path rules.
153    pub fn with_builtins() -> Result<Self> {
154        let patterns = builtin_patterns()?;
155        let effective_builtin_count = patterns.len();
156        Ok(Self {
157            patterns,
158            allowlist: AllowList::default(),
159            entropy: EntropyConfig::default(),
160            paths: PathRules::default(),
161            report: ReportConfig::default(),
162            effective_builtin_count,
163            effective_custom_count: 0,
164        })
165    }
166
167    /// Build a scrubber from a parsed [`ScrubConfig`].
168    pub fn from_config(config: &ScrubConfig) -> Result<Self> {
169        let mut patterns: Vec<CompiledPattern> = match config.builtins {
170            BuiltinsMode::Replace => Vec::new(),
171            BuiltinsMode::Extend | BuiltinsMode::Disable => builtin_patterns()?,
172        };
173
174        // Filter out explicitly disabled built-in ids.
175        if !config.disable_builtins.is_empty() {
176            let drop: std::collections::HashSet<&str> =
177                config.disable_builtins.iter().map(String::as_str).collect();
178            patterns.retain(|p| !drop.contains(p.id.as_str()));
179        }
180        let effective_builtin_count = patterns.len();
181
182        // User-defined patterns.
183        let effective_custom_count = config.patterns.len();
184        for p in &config.patterns {
185            patterns.push(compile_pattern(p)?);
186        }
187
188        let mut allowlist = AllowList::default();
189        for entry in &config.allowlist {
190            if let Some(ex) = &entry.exact {
191                allowlist.exact.push(ex.clone());
192            }
193            if let Some(rg) = &entry.regex {
194                allowlist.regex.push(Regex::new(rg)?);
195            }
196        }
197
198        Ok(Self {
199            patterns,
200            allowlist,
201            entropy: EntropyConfig::from_raw(&config.entropy)?,
202            paths: PathRules::from_raw(&config.paths)?,
203            report: config.report.clone(),
204            effective_builtin_count,
205            effective_custom_count,
206        })
207    }
208
209    /// Load `.cargo-context/scrub.yaml` from `workspace_root` if present and
210    /// return a fully-configured scrubber. Falls back to built-ins on absence
211    /// or parse failure.
212    pub fn with_workspace(workspace_root: &Path) -> Result<Self> {
213        let path = workspace_root.join(".cargo-context/scrub.yaml");
214        if !path.exists() {
215            return Self::with_builtins();
216        }
217        let raw = std::fs::read_to_string(&path)?;
218        match serde_yaml::from_str::<ScrubConfig>(&raw) {
219            Ok(cfg) => Self::from_config(&cfg),
220            Err(_) => Self::with_builtins(), // malformed config shouldn't break pack builds
221        }
222    }
223
224    /// Extend the scrubber with extra user-defined patterns (e.g. loaded
225    /// from a non-default config path).
226    pub fn extend(&mut self, extra: Vec<Pattern>) -> Result<()> {
227        for p in extra {
228            self.patterns.push(compile_pattern(&p)?);
229        }
230        Ok(())
231    }
232
233    /// Redact secrets in `input`. Returns only the scrubbed string.
234    pub fn scrub(&self, input: &str) -> String {
235        self.scrub_with_report(input).0
236    }
237
238    /// Redact and also produce a [`ScrubReport`] listing every redaction.
239    pub fn scrub_with_report(&self, input: &str) -> (String, ScrubReport) {
240        let mut out = input.to_string();
241        let mut report = ScrubReport::default();
242
243        // Layer 1: regex patterns.
244        for p in &self.patterns {
245            out = p
246                .regex
247                .replace_all(&out, |caps: &regex::Captures<'_>| {
248                    let matched = &caps[0];
249                    if self.allowlist.is_allowed(matched) {
250                        return matched.to_string();
251                    }
252                    let hash = hash4(matched);
253                    report.redactions.push(Redaction {
254                        rule_id: p.id.clone(),
255                        category: p.category.clone(),
256                        severity: p.severity,
257                        hash4: hash.clone(),
258                    });
259                    render_replacement(&p.replacement, &p.category, &hash)
260                })
261                .into_owned();
262        }
263
264        // Layer 2: entropy detection on values adjacent to context keys.
265        if self.entropy.enabled {
266            let (new_out, entropy_redactions) = self.entropy.scan_and_redact(&out, &self.allowlist);
267            out = new_out;
268            report.redactions.extend(entropy_redactions);
269        }
270
271        (out, report)
272    }
273
274    /// Scrub a file's content, honoring path-based rules (whole-file
275    /// redaction for `redact_whole` matches; bypass for `exclude` matches).
276    pub fn scrub_file(&self, path: &Path, content: &str) -> (String, ScrubReport) {
277        if self.paths.is_excluded(path) {
278            return (content.to_string(), ScrubReport::default());
279        }
280        if self.paths.is_redact_whole(path) {
281            let mut report = ScrubReport::default();
282            report.redactions.push(Redaction {
283                rule_id: "path_redact_whole".into(),
284                category: "file".into(),
285                severity: Severity::High,
286                hash4: hash4(content),
287            });
288            return (format!("[REDACTED FILE: {}]", path.display()), report);
289        }
290        self.scrub_with_report(content)
291    }
292
293    /// Return `true` if this scrubber's path rules would redact `path` whole.
294    pub fn is_path_redacted(&self, path: &Path) -> bool {
295        !self.paths.is_excluded(path) && self.paths.is_redact_whole(path)
296    }
297
298    /// Return `true` if `path` is on the exclude list (bypasses all scrubbing).
299    pub fn is_path_excluded(&self, path: &Path) -> bool {
300        self.paths.is_excluded(path)
301    }
302
303    /// Report config (typically loaded from `.cargo-context/scrub.yaml`).
304    pub fn report_config(&self) -> &ReportConfig {
305        &self.report
306    }
307
308    /// Number of built-in patterns that survived `disable_builtins` and
309    /// `BuiltinsMode::Replace` filters. Used by `cargo context scrub --check`.
310    pub fn effective_builtin_count(&self) -> usize {
311        self.effective_builtin_count
312    }
313
314    /// Number of user-defined `patterns[]` entries from the config.
315    pub fn effective_custom_count(&self) -> usize {
316        self.effective_custom_count
317    }
318
319    /// Append each redaction to `report.log_file` as JSON Lines, if configured.
320    /// Values are never written — only `rule_id`, `category`, `severity`, and
321    /// the SHA-256 fingerprint (`hash4`). Timestamps are Unix epoch seconds.
322    pub fn log_redactions(&self, report: &ScrubReport) -> Result<()> {
323        let Some(path) = &self.report.log_file else {
324            return Ok(());
325        };
326        if report.redactions.is_empty() {
327            return Ok(());
328        }
329        append_log_lines(path, &report.redactions)
330    }
331}
332
333fn append_log_lines(path: &Path, redactions: &[Redaction]) -> Result<()> {
334    use std::io::Write;
335
336    if let Some(parent) = path.parent()
337        && !parent.as_os_str().is_empty()
338    {
339        std::fs::create_dir_all(parent)?;
340    }
341
342    let mut file = std::fs::OpenOptions::new()
343        .create(true)
344        .append(true)
345        .open(path)?;
346
347    let ts = std::time::SystemTime::now()
348        .duration_since(std::time::UNIX_EPOCH)
349        .map(|d| d.as_secs())
350        .unwrap_or(0);
351
352    for r in redactions {
353        let line = serde_json::to_string(&LogEntry {
354            ts,
355            rule_id: &r.rule_id,
356            category: &r.category,
357            severity: r.severity,
358            hash4: &r.hash4,
359        })?;
360        writeln!(file, "{line}")?;
361    }
362    file.flush()?;
363    Ok(())
364}
365
366#[derive(Serialize)]
367struct LogEntry<'a> {
368    ts: u64,
369    rule_id: &'a str,
370    category: &'a str,
371    severity: Severity,
372    hash4: &'a str,
373}
374
375fn compile_pattern(p: &Pattern) -> Result<CompiledPattern> {
376    let replacement = p
377        .replacement
378        .clone()
379        .unwrap_or_else(|| format!("<REDACTED:{}:{{hash4}}>", p.category));
380    Ok(CompiledPattern {
381        id: p.id.clone(),
382        category: p.category.clone(),
383        regex: Regex::new(&p.regex)?,
384        replacement,
385        severity: p.severity,
386    })
387}
388
389fn builtin_patterns() -> Result<Vec<CompiledPattern>> {
390    let builtins: &[(&str, &str, &str, Severity)] = &[
391        (
392            "aws_access_key",
393            r"AKIA[0-9A-Z]{16}",
394            "aws_key",
395            Severity::Critical,
396        ),
397        (
398            "github_pat",
399            r"ghp_[A-Za-z0-9]{36,}",
400            "github",
401            Severity::Critical,
402        ),
403        (
404            "github_oauth",
405            r"gho_[A-Za-z0-9]{36,}",
406            "github",
407            Severity::Critical,
408        ),
409        (
410            "openai_key",
411            r"sk-[A-Za-z0-9]{20,}",
412            "api_key",
413            Severity::Critical,
414        ),
415        (
416            "anthropic_key",
417            r"sk-ant-[A-Za-z0-9\-_]{20,}",
418            "api_key",
419            Severity::Critical,
420        ),
421        (
422            "huggingface_token",
423            r"hf_[A-Za-z0-9]{30,}",
424            "api_key",
425            Severity::High,
426        ),
427        (
428            "google_api_key",
429            r"AIza[0-9A-Za-z\-_]{35}",
430            "api_key",
431            Severity::High,
432        ),
433        (
434            "slack_token",
435            r"xox[baprs]-[0-9A-Za-z\-]{10,}",
436            "slack",
437            Severity::High,
438        ),
439        (
440            "jwt",
441            r"eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+",
442            "jwt",
443            Severity::High,
444        ),
445        (
446            "private_key_pem",
447            r"-----BEGIN (RSA|EC|OPENSSH|PGP) PRIVATE KEY-----",
448            "pem",
449            Severity::Critical,
450        ),
451    ];
452    let mut patterns = Vec::with_capacity(builtins.len());
453    for (id, re, cat, sev) in builtins {
454        patterns.push(CompiledPattern {
455            id: (*id).into(),
456            category: (*cat).into(),
457            regex: Regex::new(re)?,
458            replacement: format!("<REDACTED:{cat}:{{hash4}}>"),
459            severity: *sev,
460        });
461    }
462    Ok(patterns)
463}
464
465fn render_replacement(template: &str, category: &str, hash4_val: &str) -> String {
466    template
467        .replace("{hash4}", hash4_val)
468        .replace("{category}", category)
469}
470
471pub(crate) fn hash4(s: &str) -> String {
472    let mut h = Sha256::new();
473    h.update(s.as_bytes());
474    let digest = h.finalize();
475    format!("{:02x}{:02x}", digest[0], digest[1])
476}
477
478#[cfg(test)]
479mod tests {
480    use super::*;
481
482    #[test]
483    fn redacts_aws_key() {
484        let s = Scrubber::with_builtins().unwrap();
485        let out = s.scrub("AWS_KEY=AKIAIOSFODNN7EXAMPLE trailing");
486        assert!(!out.contains("AKIAIOSFODNN7EXAMPLE"));
487        assert!(out.contains("<REDACTED:aws_key:"));
488        assert!(out.contains("trailing"));
489    }
490
491    #[test]
492    fn redacts_jwt() {
493        let s = Scrubber::with_builtins().unwrap();
494        let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NSJ9.abcDEF-_xyz";
495        let out = s.scrub(jwt);
496        assert!(!out.contains(jwt));
497        assert!(out.contains("<REDACTED:jwt:"));
498    }
499
500    #[test]
501    fn leaves_benign_text_alone() {
502        let s = Scrubber::with_builtins().unwrap();
503        let out = s.scrub("just some code // no secrets here");
504        assert_eq!(out, "just some code // no secrets here");
505    }
506
507    #[test]
508    fn hash_is_stable() {
509        assert_eq!(hash4("hello"), hash4("hello"));
510        assert_ne!(hash4("hello"), hash4("world"));
511    }
512
513    #[test]
514    fn report_counts_by_category() {
515        let s = Scrubber::with_builtins().unwrap();
516        let (_, report) = s.scrub_with_report(
517            "AKIAIOSFODNN7EXAMPLE and another AKIAIOSFODNN7EXAMPLW and ghp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
518        );
519        let by_cat = report.count_by_category();
520        assert_eq!(by_cat.get("aws_key").copied(), Some(2));
521        assert_eq!(by_cat.get("github").copied(), Some(1));
522    }
523
524    #[test]
525    fn report_summary_is_human_readable() {
526        let s = Scrubber::with_builtins().unwrap();
527        let (_, report) = s.scrub_with_report("AKIAIOSFODNN7EXAMPLE");
528        assert!(report.summary().contains("redacted"));
529        assert!(report.summary().contains("aws_key"));
530    }
531
532    #[test]
533    fn allowlist_bypasses_regex() {
534        let config = ScrubConfig {
535            allowlist: vec![config::AllowlistEntry {
536                exact: Some("AKIAEXAMPLEALLOWED".into()),
537                regex: None,
538            }],
539            ..Default::default()
540        };
541        let s = Scrubber::from_config(&config).unwrap();
542        let out = s.scrub("AKIAEXAMPLEALLOWED");
543        // The exact string matches AWS regex but is allowlisted.
544        assert_eq!(out, "AKIAEXAMPLEALLOWED");
545    }
546
547    #[test]
548    fn disable_builtins_drops_named_rule() {
549        let config = ScrubConfig {
550            disable_builtins: vec!["jwt".into()],
551            ..Default::default()
552        };
553        let s = Scrubber::from_config(&config).unwrap();
554        let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NSJ9.abcDEF-_xyz";
555        let out = s.scrub(jwt);
556        assert_eq!(out, jwt, "jwt rule should have been disabled");
557    }
558
559    #[test]
560    fn builtins_replace_drops_all() {
561        let config = ScrubConfig {
562            builtins: BuiltinsMode::Replace,
563            ..Default::default()
564        };
565        let s = Scrubber::from_config(&config).unwrap();
566        assert_eq!(s.scrub("AKIAIOSFODNN7EXAMPLE"), "AKIAIOSFODNN7EXAMPLE");
567    }
568
569    #[test]
570    fn log_redactions_writes_values_free_jsonl() {
571        let tmp = tempfile::tempdir().unwrap();
572        let log_path = tmp.path().join("scrub.log");
573        let config = ScrubConfig {
574            report: ReportConfig {
575                log_file: Some(log_path.clone()),
576                ..Default::default()
577            },
578            ..Default::default()
579        };
580        let scrubber = Scrubber::from_config(&config).unwrap();
581        let (_, report) = scrubber.scrub_with_report(
582            "AWS_KEY=AKIAIOSFODNN7EXAMPLE and jwt: eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NSJ9.abcDEF-_xyz",
583        );
584        assert!(!report.is_empty());
585
586        scrubber.log_redactions(&report).unwrap();
587        let contents = std::fs::read_to_string(&log_path).unwrap();
588        let lines: Vec<&str> = contents.lines().collect();
589        assert_eq!(lines.len(), report.redactions.len());
590        // Each line is JSON with no actual secret values.
591        for (line, r) in lines.iter().zip(&report.redactions) {
592            let parsed: serde_json::Value = serde_json::from_str(line).unwrap();
593            assert_eq!(parsed["rule_id"], r.rule_id);
594            assert_eq!(parsed["category"], r.category);
595            assert_eq!(parsed["hash4"], r.hash4);
596            assert!(parsed["ts"].as_u64().unwrap() > 0);
597            // Values are never written.
598            assert!(!line.contains("AKIAIOSFODNN7EXAMPLE"));
599            assert!(!line.contains("eyJhbGciOi"));
600        }
601    }
602
603    #[test]
604    fn log_redactions_noop_when_unconfigured() {
605        let scrubber = Scrubber::with_builtins().unwrap();
606        let (_, report) = scrubber.scrub_with_report("AWS_KEY=AKIAIOSFODNN7EXAMPLE");
607        // No log_file configured → no-op, no error.
608        scrubber.log_redactions(&report).unwrap();
609    }
610
611    #[test]
612    fn log_redactions_creates_parent_dirs() {
613        let tmp = tempfile::tempdir().unwrap();
614        let log_path = tmp.path().join("nested/dir/scrub.log");
615        let config = ScrubConfig {
616            report: ReportConfig {
617                log_file: Some(log_path.clone()),
618                ..Default::default()
619            },
620            ..Default::default()
621        };
622        let scrubber = Scrubber::from_config(&config).unwrap();
623        let (_, report) = scrubber.scrub_with_report("AWS_KEY=AKIAIOSFODNN7EXAMPLE");
624        scrubber.log_redactions(&report).unwrap();
625        assert!(log_path.exists());
626    }
627
628    #[test]
629    fn effective_counts_reflect_config() {
630        let config = ScrubConfig {
631            disable_builtins: vec!["jwt".into()],
632            patterns: vec![Pattern {
633                id: "x".into(),
634                regex: "xx".into(),
635                category: "test".into(),
636                replacement: None,
637                severity: Severity::Low,
638            }],
639            ..Default::default()
640        };
641        let s = Scrubber::from_config(&config).unwrap();
642        // Built-ins minus jwt.
643        assert!(s.effective_builtin_count() > 0);
644        assert_eq!(s.effective_custom_count(), 1);
645    }
646
647    #[test]
648    fn has_severity_at_least_ranks_correctly() {
649        let mut r = ScrubReport::default();
650        r.redactions.push(Redaction {
651            rule_id: "x".into(),
652            category: "x".into(),
653            severity: Severity::Medium,
654            hash4: "0000".into(),
655        });
656        assert!(r.has_severity_at_least(Severity::Low));
657        assert!(r.has_severity_at_least(Severity::Medium));
658        assert!(!r.has_severity_at_least(Severity::High));
659    }
660}
cargo_context_core/scrub/mod.rs

cargo_context_core/scrub/
mod.rs
