use std::ffi::OsStr;
use std::path::{Path, PathBuf};
use std::sync::mpsc;
use globset::{Glob, GlobSet, GlobSetBuilder};
use ignore::{WalkBuilder, WalkState};
use crate::caches::CacheRoots;
use crate::classify::{Classifier, Finding, Provenance, SkipReason, Verdict, is_dependency_dir};
use crate::error::{Error, Result};
pub const NEVER_DESCEND: &[&str] = &[".git", ".hg", ".svn"];
#[derive(Debug, Default)]
pub struct PatternSet {
patterns: Vec<String>,
globs: Option<GlobSet>,
}
impl PatternSet {
pub fn new(patterns: impl IntoIterator<Item = String>) -> Result<Self> {
let patterns: Vec<String> = patterns.into_iter().collect();
if patterns.is_empty() {
return Ok(Self { patterns, globs: None });
}
let mut builder = GlobSetBuilder::new();
for pattern in &patterns {
let glob = Glob::new(pattern).map_err(|source| Error::CatalogPattern {
pattern: pattern.clone(),
source,
})?;
builder.add(glob);
}
let globs = builder.build().map_err(|source| Error::CatalogPattern {
pattern: "<exclude set>".to_owned(),
source,
})?;
Ok(Self {
patterns,
globs: Some(globs),
})
}
#[must_use]
pub fn matched(&self, path: &Path) -> Option<&str> {
let globs = self.globs.as_ref()?;
globs
.matches(path)
.first()
.and_then(|index| self.patterns.get(*index))
.map(String::as_str)
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.globs.is_none()
}
}
#[derive(Debug)]
pub struct ScanOptions {
pub jobs: Option<usize>,
pub one_file_system: bool,
pub collect_skips: bool,
pub exclude: PatternSet,
pub include: PatternSet,
pub caches: CacheRoots,
pub collect_repositories: bool,
}
impl Default for ScanOptions {
fn default() -> Self {
Self {
jobs: None,
one_file_system: true,
collect_skips: false,
exclude: PatternSet::default(),
include: PatternSet::default(),
caches: CacheRoots::default(),
collect_repositories: false,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Skipped {
pub path: PathBuf,
pub reason: SkipReason,
}
#[derive(Debug, Clone)]
pub struct WalkFailure {
pub path: Option<PathBuf>,
pub message: String,
pub transient: bool,
}
#[derive(Debug, Default)]
pub struct Scan {
pub findings: Vec<Finding>,
pub skips: Vec<Skipped>,
pub skipped_count: usize,
pub failures: Vec<WalkFailure>,
pub repositories: Vec<PathBuf>,
}
enum Message {
Found(Box<Finding>),
Skipped(Box<Skipped>),
SkippedCount,
Failure(Box<WalkFailure>),
Repository(PathBuf),
}
#[must_use]
pub fn scan(roots: &[PathBuf], classifier: &Classifier, options: &ScanOptions) -> Scan {
let mut scan = Scan::default();
for root in roots {
scan_root(root, classifier, options, &mut scan);
}
scan
}
fn scan_root(root: &Path, classifier: &Classifier, options: &ScanOptions, scan: &mut Scan) {
let (sender, receiver) = mpsc::channel();
let visitor = Visitor {
root,
classifier,
options,
};
configure(&mut WalkBuilder::new(root), options)
.build_parallel()
.run(|| {
let sender = sender.clone();
Box::new(move |result| visitor.visit(result, &sender))
});
drop(sender);
for message in receiver {
match message {
Message::Found(finding) => scan.findings.push(*finding),
Message::Skipped(skipped) => {
scan.skipped_count += 1;
scan.skips.push(*skipped);
}
Message::SkippedCount => scan.skipped_count += 1,
Message::Failure(failure) => scan.failures.push(*failure),
Message::Repository(path) => scan.repositories.push(path),
}
}
}
fn configure<'b>(builder: &'b mut WalkBuilder, options: &ScanOptions) -> &'b mut WalkBuilder {
builder
.git_ignore(false)
.git_global(false)
.git_exclude(false)
.ignore(false)
.parents(false)
.hidden(false)
.follow_links(false)
.same_file_system(options.one_file_system);
if let Some(jobs) = options.jobs {
builder.threads(jobs.max(1));
}
builder
}
#[derive(Clone, Copy)]
struct Visitor<'a> {
root: &'a Path,
classifier: &'a Classifier,
options: &'a ScanOptions,
}
impl Visitor<'_> {
fn visit(
&self,
result: std::result::Result<ignore::DirEntry, ignore::Error>,
sender: &mpsc::Sender<Message>,
) -> WalkState {
let entry = match result {
Ok(entry) => entry,
Err(error) => {
let path = match &error {
ignore::Error::WithPath { path, .. } => Some(path.clone()),
_ => None,
};
let transient = error
.io_error()
.is_some_and(|io| io.kind() == std::io::ErrorKind::Interrupted);
let failure = WalkFailure {
path,
message: error.to_string(),
transient,
};
let _ = sender.send(Message::Failure(Box::new(failure)));
return WalkState::Continue;
}
};
if entry.depth() == 0 {
return WalkState::Continue;
}
let path = entry.path();
let file_type = entry.file_type();
let is_dir = file_type.is_some_and(|file_type| file_type.is_dir());
let is_symlink = file_type.is_some_and(|file_type| file_type.is_symlink());
if let Some(pattern) = self.options.exclude.matched(path) {
self.note_skip(
sender,
path,
SkipReason::Excluded {
pattern: pattern.to_owned(),
},
);
return WalkState::Skip;
}
if let Some(pattern) = self.options.include.matched(path) {
let finding = Finding {
path: path.to_path_buf(),
provenance: Provenance::Included {
pattern: pattern.to_owned(),
},
};
let _ = sender.send(Message::Found(Box::new(finding)));
return WalkState::Skip;
}
if is_dir && self.options.caches.contains(path) {
self.note_skip(sender, path, SkipReason::ToolCache);
return WalkState::Skip;
}
if self.options.collect_repositories
&& let Some(work_tree) = crate::git::repository_at(entry.file_name(), path, is_symlink)
{
let _ = sender.send(Message::Repository(work_tree.to_path_buf()));
}
if is_dir && is_never_descended(entry.file_name()) {
return WalkState::Skip;
}
if is_dir && is_dependency_dir(entry.file_name()) {
self.visit_dependency_dir(entry.file_name(), path, sender);
return WalkState::Skip;
}
match self.classifier.classify(self.root, path, is_dir || is_symlink) {
Verdict::Artifact(finding) => {
let _ = sender.send(Message::Found(Box::new(finding)));
if is_dir || is_symlink {
return WalkState::Skip;
}
}
Verdict::Skip(reason) => self.note_skip(sender, path, reason),
Verdict::NotACandidate => {}
}
WalkState::Continue
}
fn visit_dependency_dir(&self, name: &OsStr, path: &Path, sender: &mpsc::Sender<Message>) {
self.check_inside_dependency(name, path, sender);
match self.classifier.classify(self.root, path, true) {
Verdict::Artifact(finding) => {
let _ = sender.send(Message::Found(Box::new(finding)));
}
Verdict::Skip(reason) => self.note_skip(sender, path, reason),
Verdict::NotACandidate => {}
}
}
fn check_inside_dependency(&self, name: &OsStr, dir: &Path, sender: &mpsc::Sender<Message>) {
for id in self.classifier.artifacts_inside(name) {
let artifact = id.artifact();
let relative: PathBuf = artifact.segments().skip(1).collect();
let candidate = dir.join(relative);
let Ok(metadata) = std::fs::symlink_metadata(&candidate) else {
continue;
};
let is_dir = metadata.is_dir() || metadata.file_type().is_symlink();
match self.classifier.classify(self.root, &candidate, is_dir) {
Verdict::Artifact(finding) => {
let _ = sender.send(Message::Found(Box::new(finding)));
}
Verdict::Skip(reason) => self.note_skip(sender, &candidate, reason),
Verdict::NotACandidate => {}
}
}
}
fn note_skip(&self, sender: &mpsc::Sender<Message>, path: &Path, reason: SkipReason) {
let message = if self.options.collect_skips {
Message::Skipped(Box::new(Skipped {
path: path.to_path_buf(),
reason,
}))
} else {
Message::SkippedCount
};
let _ = sender.send(message);
}
}
fn is_never_descended(name: &OsStr) -> bool {
NEVER_DESCEND.iter().any(|never| name == OsStr::new(never))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::classify::{ArtifactId, Selection};
use crate::testing::tree;
fn classifier() -> Classifier {
Classifier::new(Selection::all()).expect("the catalog compiles")
}
fn found(scan: &Scan, root: &Path) -> Vec<String> {
let mut paths: Vec<String> = scan
.findings
.iter()
.map(|finding| {
finding
.path
.strip_prefix(root)
.unwrap_or(&finding.path)
.display()
.to_string()
.replace(std::path::MAIN_SEPARATOR, "/")
})
.collect();
paths.sort();
paths
}
fn run(root: &Path, options: &ScanOptions) -> Scan {
scan(&[root.to_path_buf()], &classifier(), options)
}
fn verbose() -> ScanOptions {
ScanOptions {
collect_skips: true,
..ScanOptions::default()
}
}
#[test]
fn should_take_an_included_path_that_no_marker_proves() {
let fixture = tree(&["junk/leftovers.o"]);
let options = PatternSet::new([format!("{}/junk", fixture.path().display())])
.map(|include| ScanOptions { include, ..verbose() })
.expect("the include compiles");
let scan = run(fixture.path(), &options);
assert_eq!(found(&scan, fixture.path()), vec!["junk"]);
assert!(
matches!(&scan.findings[0].provenance, Provenance::Included { pattern } if pattern.ends_with("junk")),
"the pattern is carried so the report can say what asked for this"
);
}
#[test]
fn should_let_exclude_beat_include_on_the_same_path() {
let fixture = tree(&["junk/leftovers.o"]);
let pattern = format!("{}/junk", fixture.path().display());
let options = ScanOptions {
exclude: PatternSet::new([pattern.clone()]).expect("it compiles"),
include: PatternSet::new([pattern]).expect("it compiles"),
..verbose()
};
let scan = run(fixture.path(), &options);
assert!(scan.findings.is_empty(), "exclude wins, whatever else names the path");
}
#[test]
fn should_let_an_include_reach_into_a_dependency_directory() {
let fixture = tree(&["node_modules/left-pad/index.js"]);
let options = PatternSet::new([format!("{}/node_modules", fixture.path().display())])
.map(|include| ScanOptions { include, ..verbose() })
.expect("the include compiles");
let scan = run(fixture.path(), &options);
assert_eq!(found(&scan, fixture.path()), vec!["node_modules"]);
}
#[test]
fn should_let_an_include_name_a_tool_cache_outright() {
let fixture = tree(&[".npm/_cacache/pkg/leftovers.o"]);
let options = ScanOptions {
caches: CacheRoots::under(fixture.path(), fixture.path()),
include: PatternSet::new([format!("{}/.npm", fixture.path().display())]).expect("the include compiles"),
..verbose()
};
let scan = run(fixture.path(), &options);
assert_eq!(found(&scan, fixture.path()), vec![".npm"]);
}
#[test]
fn should_not_let_an_include_reach_inside_a_pruned_directory() {
let fixture = tree(&[".npm/_cacache/pkg/leftovers.o"]);
let options = ScanOptions {
caches: CacheRoots::under(fixture.path(), fixture.path()),
include: PatternSet::new([format!("{}/.npm/_cacache/pkg", fixture.path().display())])
.expect("the include compiles"),
..verbose()
};
let scan = run(fixture.path(), &options);
assert!(scan.findings.is_empty());
assert!(fixture.path().join(".npm/_cacache/pkg/leftovers.o").exists());
}
#[test]
fn should_never_descend_into_a_tool_cache() {
let fixture = tree(&[
".npm/_cacache/pkg/package.json",
".npm/_cacache/pkg/dist/bundle.js",
"project/package.json",
"project/dist/bundle.js",
]);
let options = ScanOptions {
caches: CacheRoots::under(fixture.path(), fixture.path()),
..verbose()
};
let scan = run(fixture.path(), &options);
assert_eq!(found(&scan, fixture.path()), vec!["project/dist"]);
assert!(
scan.skips.iter().any(|skip| skip.reason == SkipReason::ToolCache),
"the skip is reported, not silent"
);
}
#[test]
fn should_descend_into_a_tool_cache_when_asked_to() {
let fixture = tree(&[
".npm/_cacache/pkg/package.json",
".npm/_cacache/pkg/dist/bundle.js",
"project/package.json",
"project/dist/bundle.js",
]);
let options = ScanOptions {
caches: CacheRoots::for_root(fixture.path(), true),
..verbose()
};
let scan = run(fixture.path(), &options);
assert_eq!(
found(&scan, fixture.path()),
vec![".npm/_cacache/pkg/dist", "project/dist"]
);
}
#[test]
fn should_find_a_gitignored_target() {
let fixture = tree(&[".git/HEAD", ".gitignore", "Cargo.toml", "target/", "target/debug/"]);
std::fs::write(fixture.path().join(".gitignore"), b"/target\n").expect("a .gitignore");
let scan = run(fixture.path(), &ScanOptions::default());
assert_eq!(found(&scan, fixture.path()), vec!["target".to_owned()]);
}
#[test]
fn should_find_a_hidden_artifact() {
let fixture = tree(&["build.zig", ".zig-cache/"]);
let scan = run(fixture.path(), &ScanOptions::default());
assert_eq!(found(&scan, fixture.path()), vec![".zig-cache".to_owned()]);
}
#[test]
fn should_not_descend_into_a_matched_artifact() {
let fixture = tree(&[
"Cargo.toml",
"target/",
"target/nested/Cargo.toml",
"target/nested/target/",
]);
let scan = run(fixture.path(), &ScanOptions::default());
assert_eq!(found(&scan, fixture.path()), vec!["target".to_owned()]);
}
#[test]
fn should_not_descend_into_a_vcs_directory() {
let fixture = tree(&[".git/Cargo.toml", ".git/target/", "README.md"]);
let scan = run(fixture.path(), &ScanOptions::default());
assert!(
found(&scan, fixture.path()).is_empty(),
"nothing inside .git is voom's business"
);
}
#[test]
fn should_not_descend_into_a_dependency_directory() {
let fixture = tree(&[
"package.json",
"node_modules/pkg/Cargo.toml",
"node_modules/pkg/target/",
]);
let scan = run(fixture.path(), &ScanOptions::default());
assert!(found(&scan, fixture.path()).is_empty());
}
#[test]
fn should_prune_an_excluded_subtree() {
let fixture = tree(&["keep/Cargo.toml", "keep/target/", "drop/Cargo.toml", "drop/target/"]);
let exclude = PatternSet::new([format!("{}/drop/**", fixture.path().display())]).expect("a glob");
let options = ScanOptions { exclude, ..verbose() };
let scan = run(fixture.path(), &options);
assert_eq!(found(&scan, fixture.path()), vec!["keep/target".to_owned()]);
assert!(
scan.skips
.iter()
.any(|skip| matches!(skip.reason, SkipReason::Excluded { .. })),
"an exclusion must be reported, not silently applied"
);
}
#[test]
#[cfg(unix)]
fn should_report_a_symlinked_artifact_as_a_finding() {
let fixture = tree(&["Cargo.toml", "elsewhere/precious.txt"]);
std::os::unix::fs::symlink(fixture.path().join("elsewhere"), fixture.path().join("target")).expect("a symlink");
let scan = run(fixture.path(), &verbose());
assert_eq!(
found(&scan, fixture.path()),
vec!["target".to_owned()],
"the link is named rather than counted"
);
assert!(
!scan.skips.iter().any(|skip| skip.reason == SkipReason::Symlink),
"and is not also folded into the skips"
);
assert!(fixture.path().join("elsewhere/precious.txt").exists());
}
#[test]
#[cfg(unix)]
fn should_not_walk_inside_a_symlinked_artifact() {
let fixture = tree(&[
"Cargo.toml",
"elsewhere/nested/Cargo.toml",
"elsewhere/nested/target/app",
]);
std::os::unix::fs::symlink(fixture.path().join("elsewhere"), fixture.path().join("target")).expect("a symlink");
let scan = run(fixture.path(), &verbose());
assert_eq!(
found(&scan, fixture.path()),
vec!["elsewhere/nested/target".to_owned(), "target".to_owned()],
"the real nested artifact is found by its own path, and nothing through the link"
);
}
#[test]
fn should_reach_an_artifact_declared_inside_a_pruned_dependency_directory() {
let fixture = tree(&["Gemfile", "vendor/bundle/gems/rake.rb", "vendor/other/keep.rb"]);
let scan = run(fixture.path(), &ScanOptions::default());
assert_eq!(found(&scan, fixture.path()), vec!["vendor/bundle".to_owned()]);
}
#[test]
fn should_still_not_walk_the_rest_of_a_dependency_directory() {
let fixture = tree(&[
"Gemfile",
"package.json",
"vendor/nested/Cargo.toml",
"vendor/nested/target/app",
]);
let scan = run(fixture.path(), &ScanOptions::default());
assert!(
found(&scan, fixture.path()).is_empty(),
"nothing else inside vendor/ is walked"
);
}
#[test]
fn should_not_enumerate_inside_an_enabled_dependency_directory() {
let fixture = tree(&[
"package.json",
"node_modules/pkg/Cargo.toml",
"node_modules/pkg/target/app",
]);
let mut selection = Selection::all();
assert!(selection.set("node.node_modules", true));
let classifier = Classifier::new(selection).expect("the catalog compiles");
let scan = scan(&[fixture.path().to_path_buf()], &classifier, &verbose());
assert_eq!(
found(&scan, fixture.path()),
vec!["node_modules".to_owned()],
"the directory itself is taken, and nothing inside it is even seen"
);
assert_eq!(
classifier.probe_count(),
1,
"one anchor listing — the project directory — and none below the prune"
);
}
#[test]
fn should_explain_a_dependency_directory_it_did_not_take() {
let fixture = tree(&["package.json", "node_modules/left-pad/index.js"]);
let scan = run(fixture.path(), &verbose());
assert!(found(&scan, fixture.path()).is_empty());
assert!(
scan.skips.iter().any(|skip| matches!(
&skip.reason,
SkipReason::NotEnabled { spec, note } if spec == "node.node_modules" && note.is_some()
)),
"a pruned dependency directory now explains its opt-in, got {:?}",
scan.skips
);
}
#[test]
fn should_not_report_a_nested_dependency_artifact_without_its_marker() {
let fixture = tree(&["vendor/bundle/gems/rake.rb"]);
let scan = run(fixture.path(), &verbose());
assert!(found(&scan, fixture.path()).is_empty(), "no Gemfile proves it");
assert!(fixture.path().join("vendor/bundle/gems/rake.rb").exists());
}
#[test]
fn should_scan_several_roots() {
let first = tree(&["Cargo.toml", "target/"]);
let second = tree(&["package.json", "dist/"]);
let roots = vec![first.path().to_path_buf(), second.path().to_path_buf()];
let scan = scan(&roots, &classifier(), &ScanOptions::default());
assert_eq!(scan.findings.len(), 2);
}
#[test]
fn should_count_skips_without_collecting_them_by_default() {
let fixture = tree(&["target/", "dist/"]);
let scan = run(fixture.path(), &ScanOptions::default());
assert!(scan.findings.is_empty());
assert_eq!(scan.skipped_count, 2);
assert!(scan.skips.is_empty(), "the default view keeps the report short");
}
#[test]
fn should_collect_skip_reasons_when_asked() {
let fixture = tree(&["target/"]);
let scan = run(fixture.path(), &verbose());
assert_eq!(scan.skips.len(), 1);
assert!(matches!(scan.skips[0].reason, SkipReason::NoMarker { .. }));
}
#[test]
fn should_find_the_root_artifact_but_never_the_root_itself() {
let fixture = tree(&["Cargo.toml", "target/"]);
let scan = run(fixture.path(), &ScanOptions::default());
assert!(scan.findings.iter().all(|finding| finding.path != fixture.path()));
}
#[test]
fn should_record_the_declaration_that_proved_each_finding() {
let fixture = tree(&["Cargo.toml", "target/"]);
let scan = run(fixture.path(), &ScanOptions::default());
let finding = &scan.findings[0];
let id: ArtifactId = finding.artifact().expect("a marker proved it");
assert_eq!(id.ecosystem().id, "rust");
assert!(matches!(
&finding.provenance,
Provenance::Anchored { marker_dir, .. } if marker_dir == fixture.path()
));
}
#[test]
fn an_empty_exclude_set_matches_nothing() {
let excludes = PatternSet::default();
assert!(excludes.is_empty());
assert_eq!(excludes.matched(Path::new("/anything")), None);
}
}