use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::sync::mpsc;
use ignore::WalkBuilder;
use crate::caches::CacheRoots;
use crate::classify::{Classifier, Finding, SkipReason};
pub const NEVER_DESCEND: &[&str] = &[".git", ".hg", ".svn"];
#[cfg(unix)]
mod guided;
#[cfg(not(unix))]
#[path = "guided_portable.rs"]
mod guided;
mod patterns;
mod visitor;
pub use patterns::PatternSet;
use visitor::Visitor;
#[expect(
clippy::struct_excessive_bools,
reason = "these mirror command-line flags, which are bools by nature"
)]
#[derive(Debug)]
pub struct ScanOptions {
pub jobs: Option<usize>,
pub one_file_system: bool,
pub collect_skips: bool,
pub exclude: PatternSet,
pub include: PatternSet,
pub caches: CacheRoots,
pub collect_repositories: bool,
pub clean_tagged: bool,
pub guide_repositories: bool,
}
impl Default for ScanOptions {
fn default() -> Self {
Self {
jobs: None,
one_file_system: true,
collect_skips: false,
exclude: PatternSet::default(),
include: PatternSet::default(),
caches: CacheRoots::default(),
collect_repositories: false,
clean_tagged: false,
guide_repositories: true,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Skipped {
pub path: PathBuf,
pub reason: SkipReason,
}
#[derive(Debug, Clone)]
pub struct WalkFailure {
pub path: Option<PathBuf>,
pub message: String,
pub transient: bool,
pub inaccessible: bool,
}
const INTERRUPTED_RETRIES: usize = 3;
#[derive(Debug, Default)]
pub struct Scan {
pub findings: Vec<Finding>,
pub skips: Vec<Skipped>,
pub skipped_count: usize,
pub failures: Vec<WalkFailure>,
pub repositories: Vec<PathBuf>,
}
enum Message {
Found(Box<Finding>),
Skipped(Box<Skipped>),
SkippedCount,
Failure(Box<WalkFailure>),
Repository(PathBuf),
Guide(PathBuf),
}
#[must_use]
pub fn scan(roots: &[PathBuf], classifier: &Classifier, options: &ScanOptions) -> Scan {
let mut scan = Scan::default();
for root in roots {
scan_root(root, classifier, options, &mut scan);
}
scan
}
fn scan_root(root: &Path, classifier: &Classifier, options: &ScanOptions, scan: &mut Scan) {
let first_failure = scan.failures.len();
walk_into(root, root, classifier, options, scan);
let found = scan.findings.len();
retry_interrupted(root, classifier, options, scan, first_failure);
dedupe_findings(scan, found);
}
fn dedupe_findings(scan: &mut Scan, from: usize) {
if scan.findings.len() <= from {
return;
}
let mut seen: HashSet<PathBuf> = scan.findings[..from]
.iter()
.map(|finding| finding.path.clone())
.collect();
let retried = scan.findings.split_off(from);
scan.findings
.extend(retried.into_iter().filter(|finding| seen.insert(finding.path.clone())));
}
fn walk_into(start: &Path, root: &Path, classifier: &Classifier, options: &ScanOptions, scan: &mut Scan) {
let visitor = Visitor {
root,
classifier,
options,
guide: options.guide_repositories && guided::supported(root),
};
let mut work_trees = Vec::new();
let mut starts = vec![start.to_path_buf()];
if visitor.guide && start == root && guided::is_work_tree(start) {
work_trees = std::mem::take(&mut starts);
}
loop {
if !starts.is_empty() {
let (sender, receiver) = mpsc::channel();
let mut builder = WalkBuilder::new(&starts[0]);
for more in &starts[1..] {
builder.add(more);
}
configure(&mut builder, options).build_parallel().run(|| {
let sender = sender.clone();
Box::new(move |result| visitor.visit(result, &sender))
});
drop(sender);
collect(receiver, scan, &mut work_trees);
}
if work_trees.is_empty() {
return;
}
let (sender, receiver) = mpsc::channel();
starts = guided::walk(visitor, &std::mem::take(&mut work_trees), &sender);
drop(sender);
collect(receiver, scan, &mut work_trees);
}
}
fn collect(receiver: mpsc::Receiver<Message>, scan: &mut Scan, work_trees: &mut Vec<PathBuf>) {
for message in receiver {
match message {
Message::Found(finding) => scan.findings.push(*finding),
Message::Skipped(skipped) => {
scan.skipped_count += 1;
scan.skips.push(*skipped);
}
Message::SkippedCount => scan.skipped_count += 1,
Message::Failure(failure) => scan.failures.push(*failure),
Message::Repository(path) => scan.repositories.push(path),
Message::Guide(path) => work_trees.push(path),
}
}
}
fn retry_interrupted(root: &Path, classifier: &Classifier, options: &ScanOptions, scan: &mut Scan, from: usize) {
for _ in 0..INTERRUPTED_RETRIES {
let mut pending = Vec::new();
let mut kept = Vec::new();
for failure in scan.failures.split_off(from) {
match failure.path.clone().filter(|_| failure.transient) {
Some(path) => pending.push(path),
None => kept.push(failure),
}
}
scan.failures.extend(kept);
if pending.is_empty() {
return;
}
for path in pending {
walk_into(&path, root, classifier, options, scan);
}
}
}
fn configure<'b>(builder: &'b mut WalkBuilder, options: &ScanOptions) -> &'b mut WalkBuilder {
builder
.git_ignore(false)
.git_global(false)
.git_exclude(false)
.ignore(false)
.parents(false)
.hidden(false)
.follow_links(false)
.same_file_system(options.one_file_system);
builder.threads(walker_threads(options.jobs));
builder
}
fn walker_threads(jobs: Option<usize>) -> usize {
jobs.map_or_else(crate::limits::default_width, |jobs| jobs.max(1))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::classify::{ArtifactId, Provenance, Selection};
use crate::testing::tree;
fn classifier() -> Classifier {
Classifier::new(Selection::all()).expect("the catalog compiles")
}
#[test]
fn should_drop_a_finding_a_retry_re_emitted() {
let finding = |path: &str| Finding {
path: PathBuf::from(path),
provenance: Provenance::Included {
pattern: "x".to_owned(),
},
};
let mut scan = Scan::default();
scan.findings.push(finding("/root/target"));
let from = scan.findings.len();
scan.findings.push(finding("/root/target"));
scan.findings.push(finding("/root/target-twin"));
dedupe_findings(&mut scan, from);
let paths: Vec<_> = scan
.findings
.iter()
.map(|finding| finding.path.display().to_string())
.collect();
assert_eq!(paths, ["/root/target", "/root/target-twin"]);
}
fn found(scan: &Scan, root: &Path) -> Vec<String> {
let mut paths: Vec<String> = scan
.findings
.iter()
.map(|finding| {
finding
.path
.strip_prefix(root)
.unwrap_or(&finding.path)
.display()
.to_string()
.replace(std::path::MAIN_SEPARATOR, "/")
})
.collect();
paths.sort();
paths
}
fn run(root: &Path, options: &ScanOptions) -> Scan {
scan(&[root.to_path_buf()], &classifier(), options)
}
fn verbose() -> ScanOptions {
ScanOptions {
collect_skips: true,
..ScanOptions::default()
}
}
#[test]
fn should_take_an_included_path_that_no_marker_proves() {
let fixture = tree(&["junk/leftovers.o"]);
let options = PatternSet::new([format!("{}/junk", fixture.path().display())])
.map(|include| ScanOptions { include, ..verbose() })
.expect("the include compiles");
let scan = run(fixture.path(), &options);
assert_eq!(found(&scan, fixture.path()), vec!["junk"]);
assert!(
matches!(&scan.findings[0].provenance, Provenance::Included { pattern } if pattern.ends_with("junk")),
"the pattern is carried so the report can say what asked for this"
);
}
#[test]
fn should_let_exclude_beat_include_on_the_same_path() {
let fixture = tree(&["junk/leftovers.o"]);
let pattern = format!("{}/junk", fixture.path().display());
let options = ScanOptions {
exclude: PatternSet::new([pattern.clone()]).expect("it compiles"),
include: PatternSet::new([pattern]).expect("it compiles"),
..verbose()
};
let scan = run(fixture.path(), &options);
assert!(scan.findings.is_empty(), "exclude wins, whatever else names the path");
}
#[test]
fn should_let_an_include_reach_into_a_dependency_directory() {
let fixture = tree(&["node_modules/left-pad/index.js"]);
let options = PatternSet::new([format!("{}/node_modules", fixture.path().display())])
.map(|include| ScanOptions { include, ..verbose() })
.expect("the include compiles");
let scan = run(fixture.path(), &options);
assert_eq!(found(&scan, fixture.path()), vec!["node_modules"]);
}
#[test]
fn should_let_an_include_name_a_tool_cache_outright() {
let fixture = tree(&[".npm/_cacache/pkg/leftovers.o"]);
let options = ScanOptions {
caches: CacheRoots::under(fixture.path(), fixture.path()),
include: PatternSet::new([format!("{}/.npm", fixture.path().display())]).expect("the include compiles"),
..verbose()
};
let scan = run(fixture.path(), &options);
assert_eq!(found(&scan, fixture.path()), vec![".npm"]);
}
#[test]
fn should_not_let_an_include_reach_inside_a_pruned_directory() {
let fixture = tree(&[".npm/_cacache/pkg/leftovers.o"]);
let options = ScanOptions {
caches: CacheRoots::under(fixture.path(), fixture.path()),
include: PatternSet::new([format!("{}/.npm/_cacache/pkg", fixture.path().display())])
.expect("the include compiles"),
..verbose()
};
let scan = run(fixture.path(), &options);
assert!(
scan.findings.is_empty(),
"an include names a path, not an artifact to sweep"
);
assert!(fixture.path().join(".npm/_cacache/pkg/leftovers.o").exists());
}
#[test]
fn should_never_descend_into_a_tool_cache() {
let fixture = tree(&[
".npm/_cacache/pkg/package.json",
".npm/_cacache/pkg/dist/bundle.js",
"project/package.json",
"project/dist/bundle.js",
]);
let options = ScanOptions {
caches: CacheRoots::under(fixture.path(), fixture.path()),
..verbose()
};
let scan = run(fixture.path(), &options);
assert_eq!(found(&scan, fixture.path()), vec!["project/dist"]);
assert!(
scan.skips.iter().any(|skip| skip.reason == SkipReason::ToolCache),
"the skip is reported, not silent"
);
}
#[test]
fn should_take_a_named_cache_only_when_its_marker_is_inside_it() {
let fixture = tree(&[
".npm/_cacache/content-v2/blob",
".npm/_cacache/index-v5/entry",
".cargo/registry/src/crate.rs",
]);
let scan = scan(
&[fixture.path().to_path_buf()],
&classifier(),
&ScanOptions {
caches: CacheRoots::under_with(
fixture.path(),
fixture.path(),
&["npm".to_owned(), "cargo-registry".to_owned()],
),
..verbose()
},
);
assert_eq!(
found(&scan, fixture.path()),
vec![".npm/_cacache".to_owned()],
"the proven cache is taken and the unproven one is not"
);
assert!(
scan.skips.iter().any(|skipped| skipped.path.ends_with("registry")
&& matches!(skipped.reason, SkipReason::NoInnerMarker { .. })),
"and the unproven one says why: {:?}",
scan.skips
);
}
#[test]
fn should_not_walk_inside_an_unproven_inside_anchored_candidate() {
let fixture = tree(&[
".alef/blob",
".alef/nested/Cargo.toml",
".alef/nested/target/build.o",
]);
let scan = run(fixture.path(), &verbose());
assert!(
scan.findings.is_empty(),
"the interior of an unproven cache is not walked: {:?}",
found(&scan, fixture.path())
);
assert!(
scan.skips
.iter()
.any(|skipped| matches!(skipped.reason, SkipReason::NoInnerMarker { .. })),
"and the directory itself is reported, so the reader knows why: {:?}",
scan.skips
);
}
#[test]
fn should_not_walk_inside_a_proven_inside_anchored_artifact() {
let fixture = tree(&[
".alef/CACHEDIR.TAG",
".alef/nested/Cargo.toml",
".alef/nested/target/build.o",
]);
let scan = run(fixture.path(), &verbose());
assert_eq!(
found(&scan, fixture.path()),
vec![".alef".to_owned()],
"the artifact itself, and nothing from inside it"
);
}
#[test]
fn should_reach_a_named_cache_inside_a_pruned_ancestor() {
let fixture = tree(&[
".npm/_cacache/content-v2/blob",
".npm/_npx/installed/package.json",
".npm/_npx/installed/dist/bundle.js",
]);
let scan = scan(
&[fixture.path().to_path_buf()],
&classifier(),
&ScanOptions {
caches: CacheRoots::under_with(fixture.path(), fixture.path(), &["npm".to_owned()]),
..verbose()
},
);
assert_eq!(
found(&scan, fixture.path()),
vec![".npm/_cacache".to_owned()],
"the named cache, and nothing else inside the pruned ancestor"
);
}
#[test]
fn should_leave_every_cache_alone_when_none_is_named() {
let fixture = tree(&[".npm/_cacache/content-v2/blob", ".npm/_cacache/index-v5/entry"]);
let scan = scan(
&[fixture.path().to_path_buf()],
&classifier(),
&ScanOptions {
caches: CacheRoots::under_with(fixture.path(), fixture.path(), &[]),
..verbose()
},
);
assert!(scan.findings.is_empty(), "{:?}", found(&scan, fixture.path()));
}
#[test]
fn should_find_a_gitignored_target() {
let fixture = tree(&[".git/HEAD", ".gitignore", "Cargo.toml", "target/", "target/debug/"]);
std::fs::write(fixture.path().join(".gitignore"), b"/target\n").expect("a .gitignore");
let scan = run(fixture.path(), &ScanOptions::default());
assert_eq!(found(&scan, fixture.path()), vec!["target".to_owned()]);
}
#[test]
fn should_find_a_hidden_artifact() {
let fixture = tree(&["build.zig", ".zig-cache/"]);
let scan = run(fixture.path(), &ScanOptions::default());
assert_eq!(found(&scan, fixture.path()), vec![".zig-cache".to_owned()]);
}
#[test]
fn should_not_descend_into_a_matched_artifact() {
let fixture = tree(&[
"Cargo.toml",
"target/",
"target/nested/Cargo.toml",
"target/nested/target/",
]);
let scan = run(fixture.path(), &ScanOptions::default());
assert_eq!(found(&scan, fixture.path()), vec!["target".to_owned()]);
}
#[test]
fn should_not_descend_into_a_vcs_directory() {
let fixture = tree(&[".git/Cargo.toml", ".git/target/", "README.md"]);
let scan = run(fixture.path(), &ScanOptions::default());
assert!(
found(&scan, fixture.path()).is_empty(),
"nothing inside .git is voom's business"
);
}
#[test]
fn should_not_descend_into_a_dependency_directory() {
let fixture = tree(&[
"package.json",
"node_modules/pkg/Cargo.toml",
"node_modules/pkg/target/",
]);
let scan = run(fixture.path(), &ScanOptions::default());
assert!(
found(&scan, fixture.path()).is_empty(),
"a dependency directory is never a finding by default"
);
}
#[test]
fn should_prune_an_excluded_subtree() {
let fixture = tree(&["keep/Cargo.toml", "keep/target/", "drop/Cargo.toml", "drop/target/"]);
let exclude = PatternSet::new([format!("{}/drop/**", fixture.path().display())]).expect("a glob");
let options = ScanOptions { exclude, ..verbose() };
let scan = run(fixture.path(), &options);
assert_eq!(found(&scan, fixture.path()), vec!["keep/target".to_owned()]);
assert!(
scan.skips
.iter()
.any(|skip| matches!(skip.reason, SkipReason::Excluded { .. })),
"an exclusion must be reported, not silently applied"
);
}
#[test]
#[cfg(unix)]
fn should_report_a_symlinked_artifact_as_a_finding() {
let fixture = tree(&["Cargo.toml", "elsewhere/precious.txt"]);
std::os::unix::fs::symlink(fixture.path().join("elsewhere"), fixture.path().join("target")).expect("a symlink");
let scan = run(fixture.path(), &verbose());
assert_eq!(
found(&scan, fixture.path()),
vec!["target".to_owned()],
"the link is named rather than counted"
);
assert!(
!scan.skips.iter().any(|skip| skip.reason == SkipReason::Symlink),
"and is not also folded into the skips"
);
assert!(fixture.path().join("elsewhere/precious.txt").exists());
}
#[test]
#[cfg(unix)]
fn should_not_walk_inside_a_symlinked_artifact() {
let fixture = tree(&[
"Cargo.toml",
"elsewhere/nested/Cargo.toml",
"elsewhere/nested/target/app",
]);
std::os::unix::fs::symlink(fixture.path().join("elsewhere"), fixture.path().join("target")).expect("a symlink");
let scan = run(fixture.path(), &verbose());
assert_eq!(
found(&scan, fixture.path()),
vec!["elsewhere/nested/target".to_owned(), "target".to_owned()],
"the real nested artifact is found by its own path, and nothing through the link"
);
}
#[test]
fn should_reach_an_artifact_declared_inside_a_pruned_dependency_directory() {
let fixture = tree(&["Gemfile", "vendor/bundle/gems/rake.rb", "vendor/other/keep.rb"]);
let mut selection = Selection::all();
assert!(selection.set("ruby.vendor-bundle", true), "the spec exists");
let scan = scan(
&[fixture.path().to_path_buf()],
&Classifier::new(selection).expect("the catalog compiles"),
&ScanOptions::default(),
);
assert_eq!(found(&scan, fixture.path()), vec!["vendor/bundle".to_owned()]);
}
#[test]
fn should_still_not_walk_the_rest_of_a_dependency_directory() {
let fixture = tree(&[
"Gemfile",
"package.json",
"vendor/nested/Cargo.toml",
"vendor/nested/target/app",
]);
let scan = run(fixture.path(), &ScanOptions::default());
assert!(
found(&scan, fixture.path()).is_empty(),
"nothing else inside vendor/ is walked"
);
}
#[test]
fn should_not_enumerate_inside_an_enabled_dependency_directory() {
let fixture = tree(&[
"package.json",
"node_modules/pkg/Cargo.toml",
"node_modules/pkg/target/app",
]);
let mut selection = Selection::all();
assert!(selection.set("node.node_modules", true));
let classifier = Classifier::new(selection).expect("the catalog compiles");
let scan = scan(&[fixture.path().to_path_buf()], &classifier, &verbose());
assert_eq!(
found(&scan, fixture.path()),
vec!["node_modules".to_owned()],
"the directory itself is taken, and nothing inside it is even seen"
);
assert_eq!(
classifier.probe_count(),
1,
"one anchor listing — the project directory — and none below the prune"
);
}
#[test]
fn should_explain_a_dependency_directory_it_did_not_take() {
let fixture = tree(&["package.json", "node_modules/left-pad/index.js"]);
let scan = run(fixture.path(), &verbose());
assert!(
found(&scan, fixture.path()).is_empty(),
"a dependency directory is never a finding by default"
);
assert!(
scan.skips.iter().any(|skip| matches!(
&skip.reason,
SkipReason::NotEnabled { spec, note } if spec == "node.node_modules" && note.is_some()
)),
"a pruned dependency directory now explains its opt-in, got {:?}",
scan.skips
);
}
#[test]
fn should_not_report_a_nested_dependency_artifact_without_its_marker() {
let fixture = tree(&["vendor/bundle/gems/rake.rb"]);
let scan = run(fixture.path(), &verbose());
assert!(found(&scan, fixture.path()).is_empty(), "no Gemfile proves it");
assert!(fixture.path().join("vendor/bundle/gems/rake.rb").exists());
}
#[test]
fn should_scan_several_roots() {
let first = tree(&["Cargo.toml", "target/"]);
let second = tree(&["package.json", "dist/"]);
let roots = vec![first.path().to_path_buf(), second.path().to_path_buf()];
let scan = scan(&roots, &classifier(), &ScanOptions::default());
assert_eq!(scan.findings.len(), 2);
}
#[test]
fn should_count_skips_without_collecting_them_by_default() {
let fixture = tree(&["target/", "dist/"]);
let scan = run(fixture.path(), &ScanOptions::default());
assert!(scan.findings.is_empty(), "neither artifact is on by default");
assert_eq!(scan.skipped_count, 2);
assert!(scan.skips.is_empty(), "the default view keeps the report short");
}
#[test]
fn should_collect_skip_reasons_when_asked() {
let fixture = tree(&["target/"]);
let scan = run(fixture.path(), &verbose());
assert_eq!(scan.skips.len(), 1);
assert!(matches!(scan.skips[0].reason, SkipReason::NoMarker { .. }));
}
#[test]
fn should_find_the_root_artifact_but_never_the_root_itself() {
let fixture = tree(&["Cargo.toml", "target/"]);
let scan = run(fixture.path(), &ScanOptions::default());
assert!(scan.findings.iter().all(|finding| finding.path != fixture.path()));
}
#[test]
fn should_record_the_declaration_that_proved_each_finding() {
let fixture = tree(&["Cargo.toml", "target/"]);
let scan = run(fixture.path(), &ScanOptions::default());
let finding = &scan.findings[0];
let id: ArtifactId = finding.artifact().expect("a marker proved it");
assert_eq!(id.ecosystem().id, "rust");
assert!(matches!(
&finding.provenance,
Provenance::Anchored { marker_dir, .. } if marker_dir == fixture.path()
));
}
#[test]
fn an_empty_exclude_set_matches_nothing() {
let excludes = PatternSet::default();
assert!(excludes.is_empty());
assert_eq!(excludes.matched(Path::new("/anything")), None);
}
#[test]
#[cfg(unix)]
fn should_name_the_directory_it_cannot_read_and_call_it_inaccessible() {
use std::os::unix::fs::PermissionsExt;
let fixture = tree(&["locked/inside.txt", "open/inside.txt"]);
let locked = fixture.path().join("locked");
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).expect("a mode");
let readable = std::fs::read_dir(&locked).is_ok();
let scan = run(fixture.path(), &verbose());
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).expect("a mode");
if readable {
return; }
assert_eq!(scan.failures.len(), 1, "{:?}", scan.failures);
assert_eq!(scan.failures[0].path.as_deref(), Some(locked.as_path()));
assert!(scan.failures[0].inaccessible);
assert!(!scan.failures[0].transient);
}
#[test]
fn should_walk_a_directory_again_when_its_read_was_interrupted() {
let fixture = tree(&["app/Cargo.toml", "app/target/debug/out"]);
let root = fixture.path();
let mut scan = Scan {
failures: vec![WalkFailure {
path: Some(root.join("app")),
message: "Interrupted system call (os error 4)".to_owned(),
transient: true,
inaccessible: false,
}],
..Scan::default()
};
let visitor_root = root.to_path_buf();
retry_interrupted(&visitor_root, &classifier(), &verbose(), &mut scan, 0);
assert!(scan.failures.is_empty(), "{:?}", scan.failures);
assert_eq!(found(&scan, root), vec!["app/target"]);
}
#[test]
fn should_give_up_on_a_directory_that_stays_interrupted() {
let fixture = tree(&["app/Cargo.toml"]);
let gone = fixture.path().join("vanished");
let mut scan = Scan {
failures: vec![WalkFailure {
path: Some(gone.clone()),
message: "Interrupted system call (os error 4)".to_owned(),
transient: true,
inaccessible: false,
}],
..Scan::default()
};
retry_interrupted(fixture.path(), &classifier(), &verbose(), &mut scan, 0);
assert_eq!(scan.failures.len(), 1);
assert_eq!(scan.failures[0].path.as_deref(), Some(gone.as_path()));
}
}