use rayon::prelude::*;
use sha1::{Digest, Sha1};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use crate::detector::{Detector, DetectorContext, DetectorRegistry};
use crate::error::TrumanError;
use crate::filetype::determine_file_type;
use crate::filter::{FilterConfig, FilterSet};
use crate::finding::{Finding, Location, Span, VerifiedResult};
use crate::prefilter::Prefilter;
#[derive(Debug, Clone, Default)]
pub struct ScanConfig {
pub filters: FilterConfig,
pub detectors: Vec<String>,
pub enable_eager: bool,
pub all_files: bool,
pub jobs: usize,
}
impl ScanConfig {
pub fn effective_threads(&self) -> usize {
if self.jobs > 0 {
return self.jobs.max(1);
}
if let Ok(v) = std::env::var("RAYON_NUM_THREADS") {
if let Ok(n) = v.trim().parse::<usize>() {
if n > 0 {
return n;
}
}
}
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(4)
.max(1)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileStats {
pub file: PathBuf,
pub lines_scanned: u64,
pub findings: u64,
}
impl FileStats {
pub fn path(&self) -> &Path {
&self.file
}
}
#[derive(Debug)]
pub enum ScanEvent {
FileStart(PathBuf),
Finding(Finding),
FileDone(FileStats),
Error(TrumanError),
}
pub fn hash_secret(secret: &str) -> String {
let mut h = Sha1::new();
h.update(secret.as_bytes());
hex::encode(h.finalize())
}
mod hex {
pub fn encode(digest: impl AsRef<[u8]>) -> String {
let mut s = String::with_capacity(digest.as_ref().len() * 2);
for b in digest.as_ref() {
s.push(char::from_digit((b >> 4) as u32, 16).unwrap());
s.push(char::from_digit((b & 0xf) as u32, 16).unwrap());
}
s
}
}
#[derive(Debug, Clone)]
pub struct Scanner {
plan: Arc<DetectionPlan>,
config: ScanConfig,
filters: FilterSet,
}
#[derive(Debug)]
struct DetectionPlan {
detectors: Vec<Arc<dyn Detector>>,
prefilter: Prefilter,
}
impl Scanner {
pub fn new(registry: DetectorRegistry, config: ScanConfig) -> Self {
Self::with_shared_registry(Arc::new(registry), config)
}
pub fn with_shared_registry(registry: Arc<DetectorRegistry>, config: ScanConfig) -> Self {
let filters = FilterSet::new(config.filters.clone());
let detectors = if config.detectors.is_empty() {
registry.detectors().to_vec()
} else {
registry
.detectors()
.iter()
.filter(|d| config.detectors.iter().any(|n| n == d.name()))
.cloned()
.collect()
};
let prefilter = Prefilter::new(&detectors, config.enable_eager);
Self {
plan: Arc::new(DetectionPlan {
detectors,
prefilter,
}),
config,
filters,
}
}
pub fn scan_file_with_callback(
&self,
path: &Path,
callback: &mut dyn FnMut(ScanEvent) -> bool,
) -> bool {
if !callback(ScanEvent::FileStart(path.to_path_buf())) {
return false;
}
if self.filters.skip_file(path) {
return callback(ScanEvent::FileDone(FileStats {
file: path.to_path_buf(),
lines_scanned: 0,
findings: 0,
}));
}
let bytes = match std::fs::read(path) {
Ok(b) => b,
Err(e) => {
callback(ScanEvent::Error(TrumanError::Io {
path: path.to_string_lossy().to_string(),
source: e,
}));
return true;
}
};
let content = String::from_utf8_lossy(&bytes);
let filetype = determine_file_type(path);
let mut candidates = self.plan.prefilter.scratch();
let mut prev_line: Option<&str> = None;
let mut lines_scanned: u64 = 0;
let mut findings: u64 = 0;
for (idx, raw) in content.split('\n').enumerate() {
let line = raw.strip_suffix('\r').unwrap_or(raw);
let line_number = (idx + 1) as u32;
lines_scanned += 1;
let dominated = self.filters.skip_line(line, prev_line, path);
let ctx = DetectorContext {
file: path,
filetype,
line_number,
enable_eager_search: self.config.enable_eager,
};
if !dominated {
self.plan.prefilter.select(line, &mut candidates);
for id in candidates.iter() {
let detector = &self.plan.detectors[id];
let raw_findings = detector.detect(line, &ctx);
for rf in raw_findings {
if rf.col_end < rf.col_start || rf.col_end > line.len() {
continue;
}
if !line.is_char_boundary(rf.col_start)
|| !line.is_char_boundary(rf.col_end)
{
continue;
}
if self.filters.skip_secret_at(
&rf.secret,
line,
rf.col_start,
detector.name(),
) {
continue;
}
let verified = if self.config.filters.no_verify {
VerifiedResult::Unverified
} else {
detector.verify(&rf.secret, &ctx)
};
if self.filters.skip_due_to_verification(verified) {
continue;
}
if self
.filters
.skip_context(&rf.secret, line, path, line_number)
{
continue;
}
let finding = Finding {
detector: detector.name().to_string(),
secret_type: detector.secret_type().to_string(),
hashed_secret: hash_secret(&rf.secret),
secret: rf.secret,
location: Location {
file: path.to_path_buf(),
span: Span::new(
line_number,
rf.col_start,
rf.col_end,
line.to_string(),
),
},
verified,
};
findings += 1;
if !callback(ScanEvent::Finding(finding)) {
return false;
}
}
}
}
prev_line = Some(line);
}
callback(ScanEvent::FileDone(FileStats {
file: path.to_path_buf(),
lines_scanned,
findings,
}))
}
pub fn num_threads(&self) -> usize {
self.config.effective_threads()
}
fn collect_files(&self, path: &Path) -> Vec<PathBuf> {
if !path.is_dir() {
return vec![path.to_path_buf()];
}
let mut files = Vec::new();
let walker = ignore::WalkBuilder::new(path)
.hidden(false)
.git_ignore(!self.config.all_files)
.git_exclude(!self.config.all_files)
.build();
for entry in walker.flatten() {
let fp = entry.path().to_path_buf();
if fp.is_file() {
files.push(fp);
}
}
if files.is_empty() {
for entry in walkdir::WalkDir::new(path).into_iter().flatten() {
let fp = entry.path().to_path_buf();
if fp.is_file() {
files.push(fp);
}
}
}
files.sort();
files
}
fn collect_all_files(&self, paths: &[impl AsRef<Path>]) -> Vec<PathBuf> {
let mut files: Vec<PathBuf> = Vec::new();
for p in paths {
files.extend(self.collect_files(p.as_ref()));
}
files
}
fn scan_one_file_collect(&self, path: &Path) -> Vec<Finding> {
let mut local = Vec::new();
self.scan_file_with_callback(path, &mut |ev| {
if let ScanEvent::Finding(f) = ev {
local.push(f);
}
true
});
local
}
pub fn scan_paths(&self, paths: &[impl AsRef<Path>]) -> Vec<Finding> {
let files = self.collect_all_files(paths);
if files.is_empty() {
return Vec::new();
}
let threads = self.config.effective_threads().min(files.len()).max(1);
let mut out: Vec<Finding> = if threads <= 1 {
files
.iter()
.flat_map(|fp| self.scan_one_file_collect(fp))
.collect()
} else if self.config.jobs == 0 {
files
.par_iter()
.flat_map(|fp| self.scan_one_file_collect(fp))
.collect()
} else {
match rayon::ThreadPoolBuilder::new().num_threads(threads).build() {
Ok(pool) => pool.install(|| {
files
.par_iter()
.flat_map(|fp| self.scan_one_file_collect(fp))
.collect()
}),
Err(_) => files
.par_iter()
.flat_map(|fp| self.scan_one_file_collect(fp))
.collect(),
}
};
out.sort_by(|a, b| {
a.location
.file
.cmp(&b.location.file)
.then(a.location.span.line.cmp(&b.location.span.line))
.then(a.location.span.col_start.cmp(&b.location.span.col_start))
.then(a.location.span.col_end.cmp(&b.location.span.col_end))
.then(a.hashed_secret.cmp(&b.hashed_secret))
});
out
}
pub fn scan_paths_streaming(
&self,
paths: &[impl AsRef<Path>],
callback: &mut dyn FnMut(ScanEvent) -> bool,
) -> bool {
let files = self.collect_all_files(paths);
if files.is_empty() {
return true;
}
let threads = self.config.effective_threads().min(files.len()).max(1);
if threads <= 1 || files.len() <= 1 {
for fp in &files {
if !self.scan_file_with_callback(fp, callback) {
return false;
}
}
return true;
}
self.scan_paths_streaming_parallel(&files, threads, callback)
}
fn scan_paths_streaming_parallel(
&self,
files: &[PathBuf],
threads: usize,
callback: &mut dyn FnMut(ScanEvent) -> bool,
) -> bool {
let capacity = (threads.saturating_mul(32)).clamp(64, 2048);
let (tx, rx) = std::sync::mpsc::sync_channel::<ScanEvent>(capacity);
let next = AtomicUsize::new(0);
let aborted = AtomicBool::new(false);
std::thread::scope(|s| {
for _ in 0..threads {
let tx = tx.clone();
let next = &next;
let aborted = &aborted;
s.spawn(move || {
loop {
if aborted.load(Ordering::Relaxed) {
break;
}
let idx = next.fetch_add(1, Ordering::Relaxed);
let Some(fp) = files.get(idx) else {
break;
};
let mut send = |ev: ScanEvent| -> bool {
if aborted.load(Ordering::Relaxed) {
return false;
}
match tx.send(ev) {
Ok(()) => !aborted.load(Ordering::Relaxed),
Err(_) => {
aborted.store(true, Ordering::Relaxed);
false
}
}
};
if !self.scan_file_with_callback(fp, &mut send) {
aborted.store(true, Ordering::Relaxed);
break;
}
}
});
}
drop(tx);
let mut completed = true;
loop {
match rx.recv() {
Ok(ev) => {
if aborted.load(Ordering::Relaxed) {
completed = false;
break;
}
if !callback(ev) {
completed = false;
aborted.store(true, Ordering::Relaxed);
break;
}
}
Err(_) => break, }
}
drop(rx);
if !completed {
aborted.store(true, Ordering::Relaxed);
}
completed
})
}
}
pub fn scan_paths_streaming(
registry: DetectorRegistry,
config: ScanConfig,
paths: &[impl AsRef<Path>],
callback: &mut dyn FnMut(ScanEvent) -> bool,
) -> bool {
Scanner::new(registry, config).scan_paths_streaming(paths, callback)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::detector::{Detector, RawFinding};
use std::io::Write;
#[derive(Debug)]
struct WordDetector;
impl Detector for WordDetector {
fn name(&self) -> &'static str {
"WordDetector"
}
fn secret_type(&self) -> &str {
"Word"
}
fn detect(&self, line: &str, _ctx: &DetectorContext) -> Vec<RawFinding> {
match line.find("hunter2") {
Some(s) => vec![RawFinding::new("hunter2".into(), s, s + 7)],
None => vec![],
}
}
}
fn scanner() -> Scanner {
let mut reg = DetectorRegistry::new();
reg.register(WordDetector);
let mut filters = FilterConfig::default();
filters.likely_id = false;
filters.sequential = false;
Scanner::new(
reg,
ScanConfig {
filters,
detectors: vec![],
enable_eager: false,
all_files: false,
jobs: 0,
},
)
}
#[allow(dead_code)]
fn scanner_with_filters(_filters: FilterConfig) -> Scanner {
let mut reg = DetectorRegistry::new();
reg.register(WordDetector);
Scanner::new(reg, ScanConfig::default())
}
#[test]
fn positions_are_exact_and_sha1_matches() {
let dir = std::env::temp_dir();
let path = dir.join(format!("truman-test-{}.txt", std::process::id()));
let mut f = std::fs::File::create(&path).unwrap();
writeln!(f, "pass = hunter2").unwrap();
drop(f);
let findings = scanner().scan_paths(&[&path]);
std::fs::remove_file(&path).ok();
assert_eq!(findings.len(), 1);
let finding = &findings[0];
assert_eq!(finding.location.span.col_start, 7);
assert_eq!(finding.location.span.col_end, 14);
assert_eq!(finding.location.span.snippet(), Some("hunter2"));
let mut h = Sha1::new();
h.update(b"hunter2");
let expected = {
let d = h.finalize();
d.iter().map(|b| format!("{b:02x}")).collect::<String>()
};
assert_eq!(finding.hashed_secret, expected);
}
#[test]
fn allowlist_skips_line() {
let dir = std::env::temp_dir();
let path = dir.join(format!("truman-allow-{}.py", std::process::id()));
std::fs::write(&path, "pass = 'hunter2' # pragma: allowlist secret\n").unwrap();
let mut reg = DetectorRegistry::new();
reg.register(WordDetector);
let mut cfg = ScanConfig::default();
cfg.filters.allowlist = true;
let findings = Scanner::new(reg, cfg).scan_paths(&[&path]);
std::fs::remove_file(&path).ok();
assert!(findings.is_empty());
}
#[test]
fn streams_events_in_order_and_supports_abort() {
let dir = std::env::temp_dir();
let path = dir.join(format!("truman-stream-{}.txt", std::process::id()));
std::fs::write(&path, "a hunter2\nb hunter2\n").unwrap();
let s = scanner();
let mut kinds: Vec<&str> = Vec::new();
let mut finding_count = 0u64;
let mut stats: Option<(u64, u64)> = None;
s.scan_file_with_callback(&path, &mut |ev| {
match ev {
ScanEvent::FileStart(_) => kinds.push("start"),
ScanEvent::Finding(_) => {
kinds.push("finding");
finding_count += 1;
}
ScanEvent::FileDone(st) => {
kinds.push("done");
stats = Some((st.lines_scanned, st.findings));
}
ScanEvent::Error(_) => kinds.push("error"),
}
true
});
std::fs::remove_file(&path).ok();
assert_eq!(kinds, vec!["start", "finding", "finding", "done"]);
assert_eq!(finding_count, 2);
assert_eq!(stats, Some((3, 2))); }
#[test]
fn abort_stops_scan_early() {
let dir = std::env::temp_dir();
let path = dir.join(format!("truman-abort-{}.txt", std::process::id()));
std::fs::write(&path, "hunter2\nhunter2\nhunter2\n").unwrap();
let s = scanner();
let mut seen = 0;
let completed = s.scan_file_with_callback(&path, &mut |ev| {
if matches!(ev, ScanEvent::Finding(_)) {
seen += 1;
return false; }
true
});
std::fs::remove_file(&path).ok();
assert!(!completed);
assert_eq!(seen, 1);
}
#[test]
fn non_text_and_lock_files_are_skipped() {
let dir = std::env::temp_dir();
let png = dir.join(format!("truman-skip-{}.png", std::process::id()));
std::fs::write(&png, "hunter2\n").unwrap();
let findings = scanner().scan_paths(&[&png]);
std::fs::remove_file(&png).ok();
assert!(findings.is_empty());
}
#[test]
fn crlf_positions_relative_to_rstripped_line() {
let dir = std::env::temp_dir();
let path = dir.join(format!("truman-crlf-{}.txt", std::process::id()));
std::fs::write(&path, "pass = hunter2\r\nsecond\r\n").unwrap();
let findings = scanner().scan_paths(&[&path]);
std::fs::remove_file(&path).ok();
assert_eq!(findings.len(), 1);
assert_eq!(findings[0].location.span.col_start, 7);
assert_eq!(findings[0].location.span.col_end, 14);
assert_eq!(findings[0].location.span.line_text, "pass = hunter2");
}
#[test]
fn scan_paths_streaming_method_streams_all_files() {
let dir = std::env::temp_dir();
let a = dir.join(format!("truman-ms-a-{}.txt", std::process::id()));
let b = dir.join(format!("truman-ms-b-{}.txt", std::process::id()));
std::fs::write(&a, "hunter2\n").unwrap();
std::fs::write(&b, "nothing\n").unwrap();
let s = scanner();
let mut starts = 0;
let mut findings_n = 0;
let mut dones = 0;
let ok = s.scan_paths_streaming(&[&a, &b], &mut |ev| {
match ev {
ScanEvent::FileStart(_) => starts += 1,
ScanEvent::Finding(_) => findings_n += 1,
ScanEvent::FileDone(_) => dones += 1,
ScanEvent::Error(_) => {}
}
true
});
std::fs::remove_file(&a).ok();
std::fs::remove_file(&b).ok();
assert!(ok);
assert_eq!(starts, 2);
assert_eq!(dones, 2);
assert_eq!(findings_n, 1);
}
#[test]
fn verification_policy_only_verified() {
#[derive(Debug)]
struct AlwaysFalse;
impl Detector for AlwaysFalse {
fn name(&self) -> &'static str {
"AlwaysFalse"
}
fn secret_type(&self) -> &str {
"T"
}
fn detect(&self, line: &str, _ctx: &DetectorContext) -> Vec<RawFinding> {
if line.contains("hunter2") {
vec![RawFinding::new("hunter2".into(), 0, 7)]
} else {
vec![]
}
}
fn verify(&self, _s: &str, _c: &DetectorContext) -> VerifiedResult {
VerifiedResult::VerifiedFalse
}
}
let dir = std::env::temp_dir();
let path = dir.join(format!("truman-verify-{}.txt", std::process::id()));
std::fs::write(&path, "hunter2\n").unwrap();
let mut reg = DetectorRegistry::new();
reg.register(AlwaysFalse);
let s = Scanner::new(reg, ScanConfig::default());
assert!(s.scan_paths(&[&path]).is_empty());
let mut reg2 = DetectorRegistry::new();
reg2.register(AlwaysFalse);
let mut cfg = ScanConfig::default();
cfg.filters.no_verify = true;
let s2 = Scanner::new(reg2, cfg);
assert_eq!(s2.scan_paths(&[&path]).len(), 1);
std::fs::remove_file(&path).ok();
}
#[test]
fn scan_paths_is_sorted_and_parallel_safe() {
let dir = std::env::temp_dir();
let files: Vec<PathBuf> = (0..8)
.map(|i| dir.join(format!("truman-par-{}-{}.txt", std::process::id(), i)))
.collect();
for (i, p) in files.iter().enumerate() {
std::fs::write(p, format!("line hunter2 {}\n", i)).unwrap();
}
let findings = scanner().scan_paths(&files);
for p in &files {
std::fs::remove_file(p).ok();
}
assert_eq!(findings.len(), 8);
let mut sorted = findings.clone();
sorted.sort_by(|a, b| a.location.file.cmp(&b.location.file));
assert_eq!(findings, sorted);
}
fn scanner_with_jobs(jobs: usize) -> Scanner {
let mut reg = DetectorRegistry::new();
reg.register(WordDetector);
let mut filters = FilterConfig::default();
filters.likely_id = false;
filters.sequential = false;
let mut cfg = ScanConfig {
filters,
detectors: vec![],
enable_eager: false,
all_files: false,
jobs,
};
cfg.filters.no_verify = true;
Scanner::new(reg, cfg)
}
#[test]
fn effective_threads_respects_jobs() {
let mut cfg = ScanConfig::default();
cfg.jobs = 1;
assert_eq!(cfg.effective_threads(), 1);
cfg.jobs = 4;
assert_eq!(cfg.effective_threads(), 4);
cfg.jobs = 0;
assert!(cfg.effective_threads() >= 1);
assert_eq!(scanner_with_jobs(2).num_threads(), 2);
}
#[test]
fn scan_paths_jobs_variants_agree() {
let dir = std::env::temp_dir();
let files: Vec<PathBuf> = (0..8)
.map(|i| dir.join(format!("truman-jobs-{}-{}.txt", std::process::id(), i)))
.collect();
for (i, p) in files.iter().enumerate() {
std::fs::write(p, format!("line hunter2 {}\n", i)).unwrap();
}
let seq = scanner_with_jobs(1).scan_paths(&files);
let auto = scanner_with_jobs(0).scan_paths(&files);
let four = scanner_with_jobs(4).scan_paths(&files);
for p in &files {
std::fs::remove_file(p).ok();
}
assert_eq!(seq.len(), 8);
assert_eq!(auto, seq);
assert_eq!(four, seq);
}
#[test]
fn scan_paths_streaming_parallel_preserves_counts_and_per_file_order() {
let dir = std::env::temp_dir();
let files: Vec<PathBuf> = (0..16)
.map(|i| dir.join(format!("truman-pstream-{}-{}.txt", std::process::id(), i)))
.collect();
for (i, p) in files.iter().enumerate() {
if i % 2 == 0 {
std::fs::write(p, "a hunter2\nb hunter2\n").unwrap();
} else {
std::fs::write(p, "nothing here\n").unwrap();
}
}
for jobs in [0usize, 1, 4] {
let s = scanner_with_jobs(jobs);
let mut starts = 0usize;
let mut dones = 0usize;
let mut findings_n = 0usize;
let mut open: std::collections::HashSet<PathBuf> = std::collections::HashSet::new();
let ok = s.scan_paths_streaming(&files, &mut |ev| {
match ev {
ScanEvent::FileStart(p) => {
starts += 1;
assert!(open.insert(p));
}
ScanEvent::Finding(f) => {
findings_n += 1;
assert!(open.contains(&f.location.file));
}
ScanEvent::FileDone(st) => {
dones += 1;
assert!(open.remove(&st.file));
}
ScanEvent::Error(_) => {}
}
true
});
assert!(ok, "jobs={jobs}");
assert_eq!(starts, 16, "jobs={jobs}");
assert_eq!(dones, 16, "jobs={jobs}");
assert_eq!(findings_n, 16, "jobs={jobs}"); assert!(open.is_empty(), "jobs={jobs}");
}
for p in &files {
std::fs::remove_file(p).ok();
}
}
#[test]
fn scan_paths_streaming_parallel_abort_returns_false() {
let dir = std::env::temp_dir();
let files: Vec<PathBuf> = (0..8)
.map(|i| dir.join(format!("truman-pabort-{}-{}.txt", std::process::id(), i)))
.collect();
for p in &files {
std::fs::write(p, "hunter2\n").unwrap();
}
let s = scanner_with_jobs(4);
let mut seen = 0usize;
let completed = s.scan_paths_streaming(&files, &mut |ev| {
if matches!(ev, ScanEvent::Finding(_)) {
seen += 1;
return false;
}
true
});
for p in &files {
std::fs::remove_file(p).ok();
}
assert!(!completed);
assert!(seen >= 1);
}
}