Skip to main content

truman_core/
scanner.rs

1//! Parallel file scanner with streaming callbacks.
2//!
3//! Detection-line rule: files are read as bytes and decoded with lossy
4//! UTF-8. Content is split on `\n`; each raw line has at most one trailing
5//! `\r` stripped (i.e. we rstripped only `\r\n`, never other whitespace).
6//! Detector byte positions (`RawFinding.col_start/col_end`) index into
7//! this rstripped line, and [`crate::finding::Span`] stores the full
8//! rstripped `line_text` so positions are exact and reproducible.
9//!
10//! Staged pipeline per file:
11//! `skip_file` → read lines → per line (`1`-indexed):
12//! `skip_line` (with previous line for nextline pragmas) → per detector
13//! `detect()` → per `RawFinding`: `skip_secret_at` (explicit byte offset
14//! for `likely_id`) → `verify()` → verification-policy
15//! (`skip_due_to_verification`) → build [`Finding`](crate::finding::Finding)
16//! (SHA1 hex, byte-exact [`Span`](crate::finding::Span)) → emit
17//! [`ScanEvent::Finding`] immediately (streaming, no buffering).
18//!
19//! Parallelism: files are independent, so both [`Scanner::scan_paths`]
20//! and [`Scanner::scan_paths_streaming`] scan files in parallel across
21//! all available cores by default (see [`ScanConfig::jobs`]). Lines within
22//! a file stay sequential (nextline-pragma state), and detectors within a
23//! line stay in registration order. Per-file event order
24//! (`FileStart` → `Finding*` → `FileDone`) is always preserved; across
25//! files, completion order is not guaranteed when `jobs != 1`.
26
27use rayon::prelude::*;
28use sha1::{Digest, Sha1};
29use std::path::{Path, PathBuf};
30use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
31use std::sync::Arc;
32
33use crate::detector::{Detector, DetectorContext, DetectorRegistry};
34use crate::error::TrumanError;
35use crate::filetype::determine_file_type;
36use crate::filter::{FilterConfig, FilterSet};
37use crate::finding::{Finding, Location, Span, VerifiedResult};
38use crate::prefilter::Prefilter;
39
40/// How to scan.
41#[derive(Debug, Clone, Default)]
42pub struct ScanConfig {
43    /// Filter toggles + exclusion patterns.
44    pub filters: FilterConfig,
45    /// Detector names to run; empty means "all registered".
46    pub detectors: Vec<String>,
47    /// Forwarded to each detector as `ctx.enable_eager_search`.
48    pub enable_eager: bool,
49    /// Scan all files, including gitignored ones. When false (default),
50    /// directory walks respect `.gitignore` (via the `ignore` crate);
51    /// when true, ignored files are also scanned. Mirrors the CLI
52    /// `--all-files` flag.
53    pub all_files: bool,
54    /// Number of worker threads for file scans.
55    ///
56    /// `0` (default) means automatic: all available cores
57    /// (`std::thread::available_parallelism`, honouring `RAYON_NUM_THREADS`
58    /// when set). `1` forces single-threaded, deterministic file order.
59    /// `N > 1` uses exactly `N` threads. Mirrors the CLI `--jobs/-j` flag.
60    pub jobs: usize,
61}
62
63impl ScanConfig {
64    /// Effective worker thread count: `jobs` when non-zero, otherwise all
65    /// available cores. Always at least 1.
66    ///
67    /// When `jobs == 0`, `RAYON_NUM_THREADS` (if set to a positive integer)
68    /// takes precedence over `available_parallelism`, so the collect
69    /// (`rayon`) and streaming (scoped-thread) engines agree.
70    pub fn effective_threads(&self) -> usize {
71        if self.jobs > 0 {
72            return self.jobs.max(1);
73        }
74        if let Ok(v) = std::env::var("RAYON_NUM_THREADS") {
75            if let Ok(n) = v.trim().parse::<usize>() {
76                if n > 0 {
77                    return n;
78                }
79            }
80        }
81        std::thread::available_parallelism()
82            .map(|n| n.get())
83            .unwrap_or(4)
84            .max(1)
85    }
86}
87
88/// Per-file statistics emitted with `ScanEvent::FileDone`.
89#[derive(Debug, Clone, PartialEq, Eq)]
90pub struct FileStats {
91    /// Scanned file.
92    pub file: PathBuf,
93    pub lines_scanned: u64,
94    pub findings: u64,
95}
96
97impl FileStats {
98    /// Path of the scanned file (alias for [`FileStats::file`]).
99    pub fn path(&self) -> &Path {
100        &self.file
101    }
102}
103
104/// Streaming scan events.
105///
106/// `scan_file_with_callback` delivers these in file order. For multi-file
107/// scans, per-file subsequences (`FileStart` → `Finding*` → `FileDone`) are
108/// always ordered, but files may complete out of order when scanning with
109/// `jobs != 1`. The callback returns `true` to continue or `false` to abort.
110#[derive(Debug)]
111pub enum ScanEvent {
112    FileStart(PathBuf),
113    Finding(Finding),
114    FileDone(FileStats),
115    Error(TrumanError),
116}
117
118/// Compute lowercase SHA1 hex of the plaintext secret.
119pub fn hash_secret(secret: &str) -> String {
120    let mut h = Sha1::new();
121    h.update(secret.as_bytes());
122    hex::encode(h.finalize())
123}
124
125/// Fallback hex encoding without an extra dependency: sha1 digest -> hex.
126mod hex {
127    pub fn encode(digest: impl AsRef<[u8]>) -> String {
128        let mut s = String::with_capacity(digest.as_ref().len() * 2);
129        for b in digest.as_ref() {
130            s.push(char::from_digit((b >> 4) as u32, 16).unwrap());
131            s.push(char::from_digit((b & 0xf) as u32, 16).unwrap());
132        }
133        s
134    }
135}
136
137/// The scanner: registry + config + filter set.
138#[derive(Debug, Clone)]
139pub struct Scanner {
140    plan: Arc<DetectionPlan>,
141    config: ScanConfig,
142    filters: FilterSet,
143}
144
145#[derive(Debug)]
146struct DetectionPlan {
147    detectors: Vec<Arc<dyn Detector>>,
148    prefilter: Prefilter,
149}
150
151impl Scanner {
152    pub fn new(registry: DetectorRegistry, config: ScanConfig) -> Self {
153        Self::with_shared_registry(Arc::new(registry), config)
154    }
155
156    pub fn with_shared_registry(registry: Arc<DetectorRegistry>, config: ScanConfig) -> Self {
157        let filters = FilterSet::new(config.filters.clone());
158        let detectors = if config.detectors.is_empty() {
159            registry.detectors().to_vec()
160        } else {
161            registry
162                .detectors()
163                .iter()
164                .filter(|d| config.detectors.iter().any(|n| n == d.name()))
165                .cloned()
166                .collect()
167        };
168        let prefilter = Prefilter::new(&detectors, config.enable_eager);
169        Self {
170            plan: Arc::new(DetectionPlan {
171                detectors,
172                prefilter,
173            }),
174            config,
175            filters,
176        }
177    }
178
179    /// Scan one file, streaming events to `callback`.
180    ///
181    /// Staged pipeline: `skip_file` → lossy-UTF8 read → per line
182    /// (`1`-indexed, single trailing `\r` stripped): `skip_line` (with
183    /// previous line) → per detector `detect()` → per finding:
184    /// `skip_secret_at(col_start)` → `verify()` →
185    /// `skip_due_to_verification` → emit `Finding` immediately.
186    ///
187    /// `callback` is a `&mut dyn FnMut` (trait object) so callers may pass
188    /// either a inline closure or an erased `ScanCallback`-style object.
189    ///
190    /// Returns `true` if the file completed, `false` if the callback
191    /// aborted early.
192    pub fn scan_file_with_callback(
193        &self,
194        path: &Path,
195        callback: &mut dyn FnMut(ScanEvent) -> bool,
196    ) -> bool {
197        if !callback(ScanEvent::FileStart(path.to_path_buf())) {
198            return false;
199        }
200
201        // File stage: invalid / non-text / excluded / lock /
202        // swagger files are skipped without reading (mirrors Python's
203        // `_is_filtered_out(filename)` gate in `scan_file`).
204        if self.filters.skip_file(path) {
205            return callback(ScanEvent::FileDone(FileStats {
206                file: path.to_path_buf(),
207                lines_scanned: 0,
208                findings: 0,
209            }));
210        }
211
212        let bytes = match std::fs::read(path) {
213            Ok(b) => b,
214            Err(e) => {
215                callback(ScanEvent::Error(TrumanError::Io {
216                    path: path.to_string_lossy().to_string(),
217                    source: e,
218                }));
219                return true;
220            }
221        };
222        // Lossy UTF-8 (tolerant of binary / non-UTF8 files).
223        let content = String::from_utf8_lossy(&bytes);
224
225        let filetype = determine_file_type(path);
226        let mut candidates = self.plan.prefilter.scratch();
227        let mut prev_line: Option<&str> = None;
228        let mut lines_scanned: u64 = 0;
229        let mut findings: u64 = 0;
230
231        // Split on '\n', strip a single trailing '\r' per line.
232        for (idx, raw) in content.split('\n').enumerate() {
233            let line = raw.strip_suffix('\r').unwrap_or(raw);
234            let line_number = (idx + 1) as u32;
235            lines_scanned += 1;
236
237            let dominated = self.filters.skip_line(line, prev_line, path);
238            // Remember current line for nextline-pragma evaluation even
239            // when skipped.
240            let ctx = DetectorContext {
241                file: path,
242                filetype,
243                line_number,
244                enable_eager_search: self.config.enable_eager,
245            };
246
247            if !dominated {
248                self.plan.prefilter.select(line, &mut candidates);
249                for id in candidates.iter() {
250                    let detector = &self.plan.detectors[id];
251                    let raw_findings = detector.detect(line, &ctx);
252                    for rf in raw_findings {
253                        // Validate byte indices defensively.
254                        if rf.col_end < rf.col_start || rf.col_end > line.len() {
255                            continue;
256                        }
257                        if !line.is_char_boundary(rf.col_start)
258                            || !line.is_char_boundary(rf.col_end)
259                        {
260                            continue;
261                        }
262                        if self.filters.skip_secret_at(
263                            &rf.secret,
264                            line,
265                            rf.col_start,
266                            detector.name(),
267                        ) {
268                            continue;
269                        }
270                        let verified = if self.config.filters.no_verify {
271                            VerifiedResult::Unverified
272                        } else {
273                            detector.verify(&rf.secret, &ctx)
274                        };
275                        if self.filters.skip_due_to_verification(verified) {
276                            continue;
277                        }
278                        // Backward-compatible context hook (always false);
279                        // the real policy decision happened above.
280                        if self
281                            .filters
282                            .skip_context(&rf.secret, line, path, line_number)
283                        {
284                            continue;
285                        }
286                        let finding = Finding {
287                            detector: detector.name().to_string(),
288                            secret_type: detector.secret_type().to_string(),
289                            hashed_secret: hash_secret(&rf.secret),
290                            secret: rf.secret,
291                            location: Location {
292                                file: path.to_path_buf(),
293                                span: Span::new(
294                                    line_number,
295                                    rf.col_start,
296                                    rf.col_end,
297                                    line.to_string(),
298                                ),
299                            },
300                            verified,
301                        };
302                        findings += 1;
303                        if !callback(ScanEvent::Finding(finding)) {
304                            return false;
305                        }
306                    }
307                }
308            }
309            prev_line = Some(line);
310        }
311
312        callback(ScanEvent::FileDone(FileStats {
313            file: path.to_path_buf(),
314            lines_scanned,
315            findings,
316        }))
317    }
318
319    /// Effective worker thread count (see [`ScanConfig::jobs`]).
320    ///
321    /// `0` means all available cores; `1` means single-threaded.
322    pub fn num_threads(&self) -> usize {
323        self.config.effective_threads()
324    }
325
326    /// Collect candidate files for one input path.
327    ///
328    /// Directories are walked with the `ignore` crate (hidden files
329    /// included) with a `walkdir` fallback when the ignore walker yields
330    /// nothing. Unless `config.all_files` is set, `.gitignore` /
331    /// `.git/exclude` are respected; with `all_files` ignored files are
332    /// also scanned (mirrors the CLI `--all-files` flag). Results are
333    /// sorted for deterministic discovery order. Plain files are returned as-is;
334    /// existence / `skip_file` filtering happens in
335    /// `scan_file_with_callback` (so explicitly-passed files still emit
336    /// `FileStart`/`FileDone`).
337    fn collect_files(&self, path: &Path) -> Vec<PathBuf> {
338        if !path.is_dir() {
339            return vec![path.to_path_buf()];
340        }
341        let mut files = Vec::new();
342        let walker = ignore::WalkBuilder::new(path)
343            .hidden(false)
344            .git_ignore(!self.config.all_files)
345            .git_exclude(!self.config.all_files)
346            .build();
347        for entry in walker.flatten() {
348            let fp = entry.path().to_path_buf();
349            if fp.is_file() {
350                files.push(fp);
351            }
352        }
353        if files.is_empty() {
354            // Fallback: plain recursive walk (e.g. outside a git repo or
355            // when ignore rules hide everything).
356            for entry in walkdir::WalkDir::new(path).into_iter().flatten() {
357                let fp = entry.path().to_path_buf();
358                if fp.is_file() {
359                    files.push(fp);
360                }
361            }
362        }
363        files.sort();
364        files
365    }
366
367    /// Collect candidate files for all input paths.
368    ///
369    /// Each input path is walked via [`Scanner::collect_files`] (sorted
370    /// per path); the concatenated list preserves input order. Used by both
371    /// [`Scanner::scan_paths`] and [`Scanner::scan_paths_streaming`] so the
372    /// two engines discover identical file sets.
373    fn collect_all_files(&self, paths: &[impl AsRef<Path>]) -> Vec<PathBuf> {
374        let mut files: Vec<PathBuf> = Vec::new();
375        for p in paths {
376            files.extend(self.collect_files(p.as_ref()));
377        }
378        files
379    }
380
381    /// Scan one file, collecting only findings.
382    ///
383    /// Thin wrapper over [`Scanner::scan_file_with_callback`] that drops
384    /// `FileStart`/`FileDone`/`Error` events. Used by [`Scanner::scan_paths`].
385    fn scan_one_file_collect(&self, path: &Path) -> Vec<Finding> {
386        let mut local = Vec::new();
387        self.scan_file_with_callback(path, &mut |ev| {
388            if let ScanEvent::Finding(f) = ev {
389                local.push(f);
390            }
391            true
392        });
393        local
394    }
395
396    /// Non-streaming convenience: scan files/dirs, return all findings.
397    ///
398    /// Directories are walked recursively (gitignore-aware via `ignore`,
399    /// `walkdir` fallback). Files rejected by `skip_file` emit no findings
400    /// (handled inside `scan_file_with_callback`). Individual-file IO
401    /// errors are ignored (use the streaming API to observe them).
402    ///
403    /// File scans run in parallel across [`ScanConfig::jobs`] threads
404    /// (default: all available cores); per-file line order and detector
405    /// order are preserved, while file completion order is not guaranteed.
406    /// Results are sorted by (file, line, columns, hash) for
407    /// deterministic output. `jobs == 1` forces a sequential scan.
408    pub fn scan_paths(&self, paths: &[impl AsRef<Path>]) -> Vec<Finding> {
409        let files = self.collect_all_files(paths);
410        if files.is_empty() {
411            return Vec::new();
412        }
413        let threads = self.config.effective_threads().min(files.len()).max(1);
414        let mut out: Vec<Finding> = if threads <= 1 {
415            files
416                .iter()
417                .flat_map(|fp| self.scan_one_file_collect(fp))
418                .collect()
419        } else if self.config.jobs == 0 {
420            // Default: rayon global pool (sized to all cores / RAYON_NUM_THREADS).
421            files
422                .par_iter()
423                .flat_map(|fp| self.scan_one_file_collect(fp))
424                .collect()
425        } else {
426            // Explicit `--jobs N`: dedicated pool with exactly N threads.
427            match rayon::ThreadPoolBuilder::new().num_threads(threads).build() {
428                Ok(pool) => pool.install(|| {
429                    files
430                        .par_iter()
431                        .flat_map(|fp| self.scan_one_file_collect(fp))
432                        .collect()
433                }),
434                Err(_) => files
435                    .par_iter()
436                    .flat_map(|fp| self.scan_one_file_collect(fp))
437                    .collect(),
438            }
439        };
440        out.sort_by(|a, b| {
441            a.location
442                .file
443                .cmp(&b.location.file)
444                .then(a.location.span.line.cmp(&b.location.span.line))
445                .then(a.location.span.col_start.cmp(&b.location.span.col_start))
446                .then(a.location.span.col_end.cmp(&b.location.span.col_end))
447                .then(a.hashed_secret.cmp(&b.hashed_secret))
448        });
449        out
450    }
451
452    /// Streaming scan over files/dirs.
453    ///
454    /// Like [`Scanner::scan_paths`] but emits every [`ScanEvent`]
455    /// (`FileStart` / `Finding` / `FileDone` / `Error`) to `callback`
456    /// instead of buffering. Returns `true` when all files completed,
457    /// `false` if the callback aborted early.
458    ///
459    /// Files are scanned in parallel across [`ScanConfig::jobs`] threads
460    /// (default: all available cores) and events are forwarded to the
461    /// single-threaded `callback` as workers complete. Per-file event order
462    /// (`FileStart` → `Finding*` → `FileDone`) is always preserved; across
463    /// files, completion order is not guaranteed when `jobs != 1` (pass
464    /// `jobs = 1` for deterministic sorted-file order). The callback must
465    /// stay fast; abort (`false`) stops new files promptly (in-flight files
466    /// finish their current send), and the scan then returns `false`.
467    pub fn scan_paths_streaming(
468        &self,
469        paths: &[impl AsRef<Path>],
470        callback: &mut dyn FnMut(ScanEvent) -> bool,
471    ) -> bool {
472        let files = self.collect_all_files(paths);
473        if files.is_empty() {
474            return true;
475        }
476        let threads = self.config.effective_threads().min(files.len()).max(1);
477        if threads <= 1 || files.len() <= 1 {
478            // Sequential fast path: exact file order, immediate abort.
479            for fp in &files {
480                if !self.scan_file_with_callback(fp, callback) {
481                    return false;
482                }
483            }
484            return true;
485        }
486        self.scan_paths_streaming_parallel(&files, threads, callback)
487    }
488
489    /// Parallel streaming engine: scoped work-stealing workers + bounded channel.
490    ///
491    /// Workers pull file indices via an atomic counter (dynamic balancing for
492    /// mixed file sizes), scan with [`Scanner::scan_file_with_callback`], and
493    /// forward each event over a bounded `sync_channel`. The caller (scope
494    /// owner) is the sole consumer and invokes `callback` serially, so a
495    /// `&mut dyn FnMut` (which is `!Send`) never crosses threads. `Scanner`
496    /// itself is `Send + Sync` (`Arc<DetectionPlan>` + `Regex` filters), so
497    /// `&self` shares freely.
498    ///
499    /// Abort: the consumer sets `aborted` and drops the receiver to unblock
500    /// workers parked on a full channel; workers also poll `aborted` before
501    /// each file and after each send, so no new files start after abort.
502    fn scan_paths_streaming_parallel(
503        &self,
504        files: &[PathBuf],
505        threads: usize,
506        callback: &mut dyn FnMut(ScanEvent) -> bool,
507    ) -> bool {
508        // Bounded backpressure: enough for in-flight files plus findings
509        // without unbounded buffering of huge finding bursts.
510        let capacity = (threads.saturating_mul(32)).clamp(64, 2048);
511        let (tx, rx) = std::sync::mpsc::sync_channel::<ScanEvent>(capacity);
512        let next = AtomicUsize::new(0);
513        let aborted = AtomicBool::new(false);
514
515        std::thread::scope(|s| {
516            for _ in 0..threads {
517                let tx = tx.clone();
518                let next = &next;
519                let aborted = &aborted;
520                s.spawn(move || {
521                    loop {
522                        if aborted.load(Ordering::Relaxed) {
523                            break;
524                        }
525                        let idx = next.fetch_add(1, Ordering::Relaxed);
526                        let Some(fp) = files.get(idx) else {
527                            break;
528                        };
529                        let mut send = |ev: ScanEvent| -> bool {
530                            if aborted.load(Ordering::Relaxed) {
531                                return false;
532                            }
533                            match tx.send(ev) {
534                                Ok(()) => !aborted.load(Ordering::Relaxed),
535                                Err(_) => {
536                                    aborted.store(true, Ordering::Relaxed);
537                                    false
538                                }
539                            }
540                        };
541                        if !self.scan_file_with_callback(fp, &mut send) {
542                            // Aborted mid-file (consumer asked, or send failed).
543                            aborted.store(true, Ordering::Relaxed);
544                            break;
545                        }
546                    }
547                });
548            }
549            drop(tx);
550
551            let mut completed = true;
552            loop {
553                match rx.recv() {
554                    Ok(ev) => {
555                        if aborted.load(Ordering::Relaxed) {
556                            completed = false;
557                            break;
558                        }
559                        if !callback(ev) {
560                            completed = false;
561                            aborted.store(true, Ordering::Relaxed);
562                            break;
563                        }
564                    }
565                    Err(_) => break, // all workers done
566                }
567            }
568            // Drop the receiver before joining so workers blocked on a full
569            // channel observe `Disconnected` instead of deadlocking.
570            drop(rx);
571            // Ensure workers observe abort even if they are between files.
572            if !completed {
573                aborted.store(true, Ordering::Relaxed);
574            }
575            completed
576        })
577    }
578}
579
580/// Free-function streaming scan: build a [`Scanner`] from `registry` +
581/// `config` and stream `paths` to `callback`.
582///
583/// Files are scanned in parallel across `config.jobs` threads (default: all
584/// cores); per-file event order is preserved while file completion order is
585/// not guaranteed when `jobs != 1`.
586///
587/// Returns `true` when all files completed, `false` on early abort.
588pub fn scan_paths_streaming(
589    registry: DetectorRegistry,
590    config: ScanConfig,
591    paths: &[impl AsRef<Path>],
592    callback: &mut dyn FnMut(ScanEvent) -> bool,
593) -> bool {
594    Scanner::new(registry, config).scan_paths_streaming(paths, callback)
595}
596
597#[cfg(test)]
598mod tests {
599    use super::*;
600    use crate::detector::{Detector, RawFinding};
601    use std::io::Write;
602
603    #[derive(Debug)]
604    struct WordDetector;
605    impl Detector for WordDetector {
606        fn name(&self) -> &'static str {
607            "WordDetector"
608        }
609        fn secret_type(&self) -> &str {
610            "Word"
611        }
612        fn detect(&self, line: &str, _ctx: &DetectorContext) -> Vec<RawFinding> {
613            match line.find("hunter2") {
614                Some(s) => vec![RawFinding::new("hunter2".into(), s, s + 7)],
615                None => vec![],
616            }
617        }
618    }
619
620    fn scanner() -> Scanner {
621        let mut reg = DetectorRegistry::new();
622        reg.register(WordDetector);
623        // WordDetector is not in the regex-based set and "hunter2" lines
624        // used here carry no id-like prefix, so defaults are safe; disable
625        // the rest explicitly to keep this unit focused on plumbing.
626        let mut filters = FilterConfig::default();
627        filters.likely_id = false;
628        filters.sequential = false;
629        Scanner::new(
630            reg,
631            ScanConfig {
632                filters,
633                detectors: vec![],
634                enable_eager: false,
635                all_files: false,
636                jobs: 0,
637            },
638        )
639    }
640
641    #[allow(dead_code)]
642    fn scanner_with_filters(_filters: FilterConfig) -> Scanner {
643        // Kept for symmetry with FilterConfig-focused tests; currently
644        // unused by default (all callers build via Scanner::new).
645        let mut reg = DetectorRegistry::new();
646        reg.register(WordDetector);
647        Scanner::new(reg, ScanConfig::default())
648    }
649
650    #[test]
651    fn positions_are_exact_and_sha1_matches() {
652        let dir = std::env::temp_dir();
653        let path = dir.join(format!("truman-test-{}.txt", std::process::id()));
654        let mut f = std::fs::File::create(&path).unwrap();
655        writeln!(f, "pass = hunter2").unwrap();
656        drop(f);
657        let findings = scanner().scan_paths(&[&path]);
658        std::fs::remove_file(&path).ok();
659        assert_eq!(findings.len(), 1);
660        let finding = &findings[0];
661        // "pass = hunter2": hunter2 starts at byte 7, ends at 14.
662        assert_eq!(finding.location.span.col_start, 7);
663        assert_eq!(finding.location.span.col_end, 14);
664        assert_eq!(finding.location.span.snippet(), Some("hunter2"));
665        // SHA1("hunter2") — cross-checked against Python hashlib.
666        let mut h = Sha1::new();
667        h.update(b"hunter2");
668        let expected = {
669            let d = h.finalize();
670            d.iter().map(|b| format!("{b:02x}")).collect::<String>()
671        };
672        assert_eq!(finding.hashed_secret, expected);
673    }
674
675    #[test]
676    fn allowlist_skips_line() {
677        let dir = std::env::temp_dir();
678        let path = dir.join(format!("truman-allow-{}.py", std::process::id()));
679        std::fs::write(&path, "pass = 'hunter2'  # pragma: allowlist secret\n").unwrap();
680        let mut reg = DetectorRegistry::new();
681        reg.register(WordDetector);
682        let mut cfg = ScanConfig::default();
683        cfg.filters.allowlist = true;
684        let findings = Scanner::new(reg, cfg).scan_paths(&[&path]);
685        std::fs::remove_file(&path).ok();
686        assert!(findings.is_empty());
687    }
688
689    #[test]
690    fn streams_events_in_order_and_supports_abort() {
691        let dir = std::env::temp_dir();
692        let path = dir.join(format!("truman-stream-{}.txt", std::process::id()));
693        std::fs::write(&path, "a hunter2\nb hunter2\n").unwrap();
694        let s = scanner();
695        let mut kinds: Vec<&str> = Vec::new();
696        let mut finding_count = 0u64;
697        let mut stats: Option<(u64, u64)> = None;
698        s.scan_file_with_callback(&path, &mut |ev| {
699            match ev {
700                ScanEvent::FileStart(_) => kinds.push("start"),
701                ScanEvent::Finding(_) => {
702                    kinds.push("finding");
703                    finding_count += 1;
704                }
705                ScanEvent::FileDone(st) => {
706                    kinds.push("done");
707                    stats = Some((st.lines_scanned, st.findings));
708                }
709                ScanEvent::Error(_) => kinds.push("error"),
710            }
711            true
712        });
713        std::fs::remove_file(&path).ok();
714        assert_eq!(kinds, vec!["start", "finding", "finding", "done"]);
715        assert_eq!(finding_count, 2);
716        assert_eq!(stats, Some((3, 2))); // trailing newline yields a 3rd empty line
717    }
718
719    #[test]
720    fn abort_stops_scan_early() {
721        let dir = std::env::temp_dir();
722        let path = dir.join(format!("truman-abort-{}.txt", std::process::id()));
723        std::fs::write(&path, "hunter2\nhunter2\nhunter2\n").unwrap();
724        let s = scanner();
725        let mut seen = 0;
726        let completed = s.scan_file_with_callback(&path, &mut |ev| {
727            if matches!(ev, ScanEvent::Finding(_)) {
728                seen += 1;
729                return false; // abort on first finding
730            }
731            true
732        });
733        std::fs::remove_file(&path).ok();
734        assert!(!completed);
735        assert_eq!(seen, 1);
736    }
737
738    #[test]
739    fn non_text_and_lock_files_are_skipped() {
740        let dir = std::env::temp_dir();
741        let png = dir.join(format!("truman-skip-{}.png", std::process::id()));
742        std::fs::write(&png, "hunter2\n").unwrap();
743        let findings = scanner().scan_paths(&[&png]);
744        std::fs::remove_file(&png).ok();
745        assert!(findings.is_empty());
746    }
747
748    #[test]
749    fn crlf_positions_relative_to_rstripped_line() {
750        let dir = std::env::temp_dir();
751        let path = dir.join(format!("truman-crlf-{}.txt", std::process::id()));
752        std::fs::write(&path, "pass = hunter2\r\nsecond\r\n").unwrap();
753        let findings = scanner().scan_paths(&[&path]);
754        std::fs::remove_file(&path).ok();
755        assert_eq!(findings.len(), 1);
756        assert_eq!(findings[0].location.span.col_start, 7);
757        assert_eq!(findings[0].location.span.col_end, 14);
758        assert_eq!(findings[0].location.span.line_text, "pass = hunter2");
759    }
760
761    #[test]
762    fn scan_paths_streaming_method_streams_all_files() {
763        let dir = std::env::temp_dir();
764        let a = dir.join(format!("truman-ms-a-{}.txt", std::process::id()));
765        let b = dir.join(format!("truman-ms-b-{}.txt", std::process::id()));
766        std::fs::write(&a, "hunter2\n").unwrap();
767        std::fs::write(&b, "nothing\n").unwrap();
768        let s = scanner();
769        let mut starts = 0;
770        let mut findings_n = 0;
771        let mut dones = 0;
772        let ok = s.scan_paths_streaming(&[&a, &b], &mut |ev| {
773            match ev {
774                ScanEvent::FileStart(_) => starts += 1,
775                ScanEvent::Finding(_) => findings_n += 1,
776                ScanEvent::FileDone(_) => dones += 1,
777                ScanEvent::Error(_) => {}
778            }
779            true
780        });
781        std::fs::remove_file(&a).ok();
782        std::fs::remove_file(&b).ok();
783        assert!(ok);
784        assert_eq!(starts, 2);
785        assert_eq!(dones, 2);
786        assert_eq!(findings_n, 1);
787    }
788
789    #[test]
790    fn verification_policy_only_verified() {
791        #[derive(Debug)]
792        struct AlwaysFalse;
793        impl Detector for AlwaysFalse {
794            fn name(&self) -> &'static str {
795                "AlwaysFalse"
796            }
797            fn secret_type(&self) -> &str {
798                "T"
799            }
800            fn detect(&self, line: &str, _ctx: &DetectorContext) -> Vec<RawFinding> {
801                if line.contains("hunter2") {
802                    vec![RawFinding::new("hunter2".into(), 0, 7)]
803                } else {
804                    vec![]
805                }
806            }
807            fn verify(&self, _s: &str, _c: &DetectorContext) -> VerifiedResult {
808                VerifiedResult::VerifiedFalse
809            }
810        }
811        let dir = std::env::temp_dir();
812        let path = dir.join(format!("truman-verify-{}.txt", std::process::id()));
813        std::fs::write(&path, "hunter2\n").unwrap();
814
815        // Default policy drops VerifiedFalse.
816        let mut reg = DetectorRegistry::new();
817        reg.register(AlwaysFalse);
818        let s = Scanner::new(reg, ScanConfig::default());
819        assert!(s.scan_paths(&[&path]).is_empty());
820
821        // no_verify keeps it (as Unverified).
822        let mut reg2 = DetectorRegistry::new();
823        reg2.register(AlwaysFalse);
824        let mut cfg = ScanConfig::default();
825        cfg.filters.no_verify = true;
826        let s2 = Scanner::new(reg2, cfg);
827        assert_eq!(s2.scan_paths(&[&path]).len(), 1);
828        std::fs::remove_file(&path).ok();
829    }
830
831    #[test]
832    fn scan_paths_is_sorted_and_parallel_safe() {
833        let dir = std::env::temp_dir();
834        let files: Vec<PathBuf> = (0..8)
835            .map(|i| dir.join(format!("truman-par-{}-{}.txt", std::process::id(), i)))
836            .collect();
837        for (i, p) in files.iter().enumerate() {
838            std::fs::write(p, format!("line hunter2 {}\n", i)).unwrap();
839        }
840        let findings = scanner().scan_paths(&files);
841        for p in &files {
842            std::fs::remove_file(p).ok();
843        }
844        assert_eq!(findings.len(), 8);
845        let mut sorted = findings.clone();
846        sorted.sort_by(|a, b| a.location.file.cmp(&b.location.file));
847        assert_eq!(findings, sorted);
848    }
849
850    fn scanner_with_jobs(jobs: usize) -> Scanner {
851        let mut reg = DetectorRegistry::new();
852        reg.register(WordDetector);
853        let mut filters = FilterConfig::default();
854        filters.likely_id = false;
855        filters.sequential = false;
856        let mut cfg = ScanConfig {
857            filters,
858            detectors: vec![],
859            enable_eager: false,
860            all_files: false,
861            jobs,
862        };
863        // Keep the unit focused on threading, not verification policy.
864        cfg.filters.no_verify = true;
865        Scanner::new(reg, cfg)
866    }
867
868    #[test]
869    fn effective_threads_respects_jobs() {
870        let mut cfg = ScanConfig::default();
871        cfg.jobs = 1;
872        assert_eq!(cfg.effective_threads(), 1);
873        cfg.jobs = 4;
874        assert_eq!(cfg.effective_threads(), 4);
875        cfg.jobs = 0;
876        assert!(cfg.effective_threads() >= 1);
877        assert_eq!(scanner_with_jobs(2).num_threads(), 2);
878    }
879
880    #[test]
881    fn scan_paths_jobs_variants_agree() {
882        let dir = std::env::temp_dir();
883        let files: Vec<PathBuf> = (0..8)
884            .map(|i| dir.join(format!("truman-jobs-{}-{}.txt", std::process::id(), i)))
885            .collect();
886        for (i, p) in files.iter().enumerate() {
887            std::fs::write(p, format!("line hunter2 {}\n", i)).unwrap();
888        }
889        let seq = scanner_with_jobs(1).scan_paths(&files);
890        let auto = scanner_with_jobs(0).scan_paths(&files);
891        let four = scanner_with_jobs(4).scan_paths(&files);
892        for p in &files {
893            std::fs::remove_file(p).ok();
894        }
895        assert_eq!(seq.len(), 8);
896        assert_eq!(auto, seq);
897        assert_eq!(four, seq);
898    }
899
900    #[test]
901    fn scan_paths_streaming_parallel_preserves_counts_and_per_file_order() {
902        let dir = std::env::temp_dir();
903        let files: Vec<PathBuf> = (0..16)
904            .map(|i| dir.join(format!("truman-pstream-{}-{}.txt", std::process::id(), i)))
905            .collect();
906        for (i, p) in files.iter().enumerate() {
907            if i % 2 == 0 {
908                std::fs::write(p, "a hunter2\nb hunter2\n").unwrap();
909            } else {
910                std::fs::write(p, "nothing here\n").unwrap();
911            }
912        }
913        for jobs in [0usize, 1, 4] {
914            let s = scanner_with_jobs(jobs);
915            let mut starts = 0usize;
916            let mut dones = 0usize;
917            let mut findings_n = 0usize;
918            // Per-file order: FileStart must precede its Findings/FileDone.
919            let mut open: std::collections::HashSet<PathBuf> = std::collections::HashSet::new();
920            let ok = s.scan_paths_streaming(&files, &mut |ev| {
921                match ev {
922                    ScanEvent::FileStart(p) => {
923                        starts += 1;
924                        assert!(open.insert(p));
925                    }
926                    ScanEvent::Finding(f) => {
927                        findings_n += 1;
928                        assert!(open.contains(&f.location.file));
929                    }
930                    ScanEvent::FileDone(st) => {
931                        dones += 1;
932                        assert!(open.remove(&st.file));
933                    }
934                    ScanEvent::Error(_) => {}
935                }
936                true
937            });
938            assert!(ok, "jobs={jobs}");
939            assert_eq!(starts, 16, "jobs={jobs}");
940            assert_eq!(dones, 16, "jobs={jobs}");
941            assert_eq!(findings_n, 16, "jobs={jobs}"); // 8 files x 2 findings
942            assert!(open.is_empty(), "jobs={jobs}");
943        }
944        for p in &files {
945            std::fs::remove_file(p).ok();
946        }
947    }
948
949    #[test]
950    fn scan_paths_streaming_parallel_abort_returns_false() {
951        let dir = std::env::temp_dir();
952        let files: Vec<PathBuf> = (0..8)
953            .map(|i| dir.join(format!("truman-pabort-{}-{}.txt", std::process::id(), i)))
954            .collect();
955        for p in &files {
956            std::fs::write(p, "hunter2\n").unwrap();
957        }
958        let s = scanner_with_jobs(4);
959        let mut seen = 0usize;
960        let completed = s.scan_paths_streaming(&files, &mut |ev| {
961            if matches!(ev, ScanEvent::Finding(_)) {
962                seen += 1;
963                return false;
964            }
965            true
966        });
967        for p in &files {
968            std::fs::remove_file(p).ok();
969        }
970        assert!(!completed);
971        assert!(seen >= 1);
972    }
973}