use std::collections::HashMap;
use std::ffi::OsStr;
use std::fmt;
use std::path::{Path, PathBuf};
use std::sync::RwLock;
use std::sync::atomic::{AtomicUsize, Ordering};
use crate::catalog::{self, Artifact, CATALOG, Ecosystem};
use crate::error::Result;
use crate::names::NameMatcher;
pub const CONFIG_FILE_NAME: &str = "voom.toml";
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ArtifactId {
ecosystem: u8,
artifact: u8,
}
impl ArtifactId {
#[must_use]
pub fn ecosystem(self) -> &'static Ecosystem {
&CATALOG[self.ecosystem as usize]
}
#[must_use]
pub fn artifact(self) -> &'static Artifact {
&self.ecosystem().artifacts[self.artifact as usize]
}
#[must_use]
pub fn spec(self) -> String {
format!("{}.{}", self.ecosystem().id, self.artifact().slug())
}
#[must_use]
pub fn from_spec(spec: &str) -> Option<Self> {
let (ecosystem_id, artifact_slug) = spec.split_once('.')?;
let index = CATALOG.iter().position(|ecosystem| ecosystem.id == ecosystem_id)?;
let offset = CATALOG[index]
.artifacts
.iter()
.position(|artifact| artifact.slug() == artifact_slug)?;
Some(Self {
ecosystem: u8::try_from(index).ok()?,
artifact: u8::try_from(offset).ok()?,
})
}
}
#[derive(Debug, Clone)]
pub struct Selection {
ecosystems: u32,
overrides: HashMap<ArtifactId, bool>,
permissive: bool,
}
impl Default for Selection {
fn default() -> Self {
Self::all()
}
}
impl Selection {
#[must_use]
pub fn all() -> Self {
let ecosystems = if CATALOG.len() >= u32::BITS as usize {
u32::MAX
} else {
(1 << CATALOG.len()) - 1
};
Self {
ecosystems,
overrides: HashMap::new(),
permissive: false,
}
}
#[must_use]
pub fn only<'a>(ids: impl IntoIterator<Item = &'a str>) -> (Self, Vec<String>) {
let mut selection = Self {
ecosystems: 0,
overrides: HashMap::new(),
permissive: false,
};
let mut unknown = Vec::new();
for id in ids {
match CATALOG.iter().position(|ecosystem| ecosystem.id == id) {
Some(index) => selection.ecosystems |= 1 << index,
None => unknown.push(id.to_owned()),
}
}
(selection, unknown)
}
#[must_use]
pub fn permissive(mut self) -> Self {
self.permissive = true;
self
}
pub fn set(&mut self, spec: &str, enabled: bool) -> bool {
let (ecosystem_id, artifact_slug) = match spec.split_once('.') {
Some((ecosystem_id, artifact_slug)) => (ecosystem_id, Some(artifact_slug)),
None => (spec, None),
};
let Some(index) = CATALOG.iter().position(|ecosystem| ecosystem.id == ecosystem_id) else {
return false;
};
let ecosystem = &CATALOG[index];
let mut matched = false;
for (offset, artifact) in ecosystem.artifacts.iter().enumerate() {
if artifact_slug.is_some_and(|slug| artifact.slug() != slug) {
continue;
}
let Ok(ecosystem_index) = u8::try_from(index) else {
continue;
};
let Ok(artifact_index) = u8::try_from(offset) else {
continue;
};
self.overrides.insert(
ArtifactId {
ecosystem: ecosystem_index,
artifact: artifact_index,
},
enabled,
);
matched = true;
}
if matched && enabled {
self.ecosystems |= 1 << index;
}
matched
}
#[must_use]
pub fn ecosystem_enabled(&self, id: ArtifactId) -> bool {
self.ecosystems & (1 << id.ecosystem) != 0
}
#[must_use]
pub fn is_enabled(&self, id: ArtifactId) -> bool {
if !self.ecosystem_enabled(id) {
return false;
}
self.overrides
.get(&id)
.copied()
.unwrap_or_else(|| self.permissive || id.artifact().default_on)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct DirFacts {
markers: u32,
readable: bool,
}
impl DirFacts {
const UNREADABLE: Self = Self {
markers: 0,
readable: false,
};
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum Verdict {
Artifact(Finding),
Skip(SkipReason),
NotACandidate,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum Provenance {
Anchored {
artifact: ArtifactId,
marker_dir: PathBuf,
},
Included {
pattern: String,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Finding {
pub path: PathBuf,
pub provenance: Provenance,
}
impl Finding {
#[must_use]
pub fn artifact(&self) -> Option<ArtifactId> {
match &self.provenance {
Provenance::Anchored { artifact, .. } => Some(*artifact),
Provenance::Included { .. } => None,
}
}
#[must_use]
pub fn ecosystem(&self) -> Option<&'static str> {
self.artifact().map(|id| id.ecosystem().id)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum SkipReason {
NoMarker {
ecosystem: &'static str,
markers: &'static [&'static str],
},
NotEnabled {
spec: String,
note: Option<&'static str>,
},
ToolCache,
Excluded {
pattern: String,
},
KeptByPolicy {
rule: &'static str,
detail: String,
},
FilesystemBoundary,
Symlink,
Unreadable,
}
impl SkipReason {
#[must_use]
pub fn code(&self) -> &'static str {
match self {
Self::NoMarker { .. } => "no_marker",
Self::NotEnabled { .. } => "not_enabled",
Self::ToolCache => "tool_cache",
Self::Excluded { .. } => "excluded",
Self::KeptByPolicy { .. } => "kept_by_policy",
Self::FilesystemBoundary => "filesystem_boundary",
Self::Symlink => "symlink",
Self::Unreadable => "unreadable",
}
}
}
impl fmt::Display for SkipReason {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NoMarker { ecosystem, markers } => {
write!(f, "no {ecosystem} marker ({}) proves it", markers.join(", "))
}
Self::NotEnabled { spec, note } => match note {
Some(note) => write!(f, "not enabled — {note} Enable with `{spec}`"),
None => write!(f, "not enabled — enable with `{spec}`"),
},
Self::ToolCache => write!(f, "a tool cache, not build output — `--caches` sweeps these too"),
Self::Excluded { pattern } => write!(f, "excluded by `{pattern}`"),
Self::KeptByPolicy { rule, detail } => write!(f, "kept by {rule} ({detail})"),
Self::FilesystemBoundary => write!(f, "on a different filesystem than the scan root"),
Self::Symlink => write!(f, "is a symlink"),
Self::Unreadable => write!(f, "its anchor directory could not be read"),
}
}
}
#[derive(Debug)]
pub struct Classifier {
artifacts: NameMatcher<ArtifactId>,
markers: NameMatcher<u8>,
selection: Selection,
nested_in_dependencies: Vec<(&'static str, ArtifactId)>,
facts: RwLock<HashMap<PathBuf, DirFacts>>,
probes: AtomicUsize,
}
impl Classifier {
pub fn new(selection: Selection) -> Result<Self> {
let mut artifact_patterns = Vec::new();
let mut marker_patterns = Vec::new();
let mut nested_in_dependencies = Vec::new();
for (index, ecosystem) in CATALOG.iter().enumerate() {
let Ok(ecosystem_index) = u8::try_from(index) else {
continue;
};
for marker in ecosystem.markers {
marker_patterns.push((*marker, ecosystem_index));
}
for (offset, artifact) in ecosystem.artifacts.iter().enumerate() {
let Ok(artifact_index) = u8::try_from(offset) else {
continue;
};
let last = artifact
.path
.trim_end_matches('/')
.rsplit('/')
.next()
.unwrap_or(artifact.path);
let id = ArtifactId {
ecosystem: ecosystem_index,
artifact: artifact_index,
};
artifact_patterns.push((last, id));
if artifact.depth() > 1
&& let Some(first) = artifact.segments().next()
&& let Some(dependency) = catalog::DEPENDENCY_DIRS.iter().find(|dir| **dir == first)
{
nested_in_dependencies.push((*dependency, id));
}
}
}
Ok(Self {
artifacts: NameMatcher::build(artifact_patterns)?,
markers: NameMatcher::build(marker_patterns)?,
selection,
nested_in_dependencies,
facts: RwLock::new(HashMap::new()),
probes: AtomicUsize::new(0),
})
}
#[must_use]
pub fn probe_count(&self) -> usize {
self.probes.load(Ordering::Relaxed)
}
#[must_use]
pub fn selection(&self) -> &Selection {
&self.selection
}
#[must_use]
pub fn artifacts_inside(&self, name: &OsStr) -> Vec<ArtifactId> {
self.nested_in_dependencies
.iter()
.filter(|(dir, _)| name == OsStr::new(*dir))
.map(|(_, id)| *id)
.collect()
}
pub fn classify(&self, root: &Path, path: &Path, is_dir: bool) -> Verdict {
let Some(name) = path.file_name() else {
return Verdict::NotACandidate;
};
let mut candidates: Vec<ArtifactId> = Vec::new();
self.artifacts.for_each_match(name, |id| candidates.push(id));
if candidates.is_empty() {
return Verdict::NotACandidate;
}
candidates.sort_unstable();
let mut best: Option<(u8, SkipReason)> = None;
let mut record = |rank: u8, reason: SkipReason| {
if best.as_ref().is_none_or(|(existing, _)| rank < *existing) {
best = Some((rank, reason));
}
};
for id in candidates {
let artifact = id.artifact();
if artifact.requires_dir() && !is_dir {
continue;
}
let Some(anchor_dir) = anchor_dir(path, artifact) else {
continue;
};
if !anchor_dir.starts_with(root) {
continue;
}
if !self.selection.ecosystem_enabled(id) {
record(
RANK_ECOSYSTEM_OFF,
SkipReason::NotEnabled {
spec: id.spec(),
note: artifact.note,
},
);
continue;
}
match (self.prove(root, anchor_dir, id), self.selection.is_enabled(id)) {
(Proof::Found(marker_dir), true) => {
return Verdict::Artifact(Finding {
path: path.to_path_buf(),
provenance: Provenance::Anchored {
artifact: id,
marker_dir,
},
});
}
(Proof::Found(_), false) => {
record(
RANK_PROVEN_OPT_IN,
SkipReason::NotEnabled {
spec: id.spec(),
note: artifact.note,
},
);
}
(Proof::Missing, true) => record(
RANK_NO_MARKER,
SkipReason::NoMarker {
ecosystem: id.ecosystem().id,
markers: id.ecosystem().markers,
},
),
(Proof::Missing, false) => record(
RANK_UNPROVEN_OPT_IN,
SkipReason::NotEnabled {
spec: id.spec(),
note: artifact.note,
},
),
(Proof::Unreadable, _) => record(RANK_UNREADABLE, SkipReason::Unreadable),
}
}
best.map_or(Verdict::NotACandidate, |(_, reason)| Verdict::Skip(reason))
}
fn prove(&self, root: &Path, anchor_dir: &Path, id: ArtifactId) -> Proof {
let bit = 1_u32 << id.ecosystem;
let levels = id.artifact().anchor_in(id.ecosystem()).levels();
let mut dir = anchor_dir;
let mut unreadable = false;
for _ in 0..=levels {
let facts = self.facts(dir);
if facts.markers & bit != 0 {
return Proof::Found(dir.to_path_buf());
}
unreadable |= !facts.readable;
if dir == root {
break;
}
match dir.parent() {
Some(parent) => dir = parent,
None => break,
}
}
if unreadable { Proof::Unreadable } else { Proof::Missing }
}
fn facts(&self, dir: &Path) -> DirFacts {
if let Ok(cache) = self.facts.read()
&& let Some(facts) = cache.get(dir)
{
return *facts;
}
let facts = self.probe(dir);
if let Ok(mut cache) = self.facts.write() {
cache.insert(dir.to_path_buf(), facts);
}
facts
}
fn probe(&self, dir: &Path) -> DirFacts {
self.probes.fetch_add(1, Ordering::Relaxed);
let Ok(entries) = std::fs::read_dir(dir) else {
return DirFacts::UNREADABLE;
};
let mut facts = DirFacts {
markers: 0,
readable: true,
};
for entry in entries.flatten() {
self.markers
.for_each_match(&entry.file_name(), |index| facts.markers |= 1_u32 << index);
}
facts
}
}
const RANK_PROVEN_OPT_IN: u8 = 0;
const RANK_NO_MARKER: u8 = 1;
const RANK_UNPROVEN_OPT_IN: u8 = 2;
const RANK_ECOSYSTEM_OFF: u8 = 3;
const RANK_UNREADABLE: u8 = 4;
enum Proof {
Found(PathBuf),
Missing,
Unreadable,
}
fn anchor_dir<'p>(path: &'p Path, artifact: &Artifact) -> Option<&'p Path> {
let mut dir = path.parent()?;
for segment in artifact.path.trim_end_matches('/').rsplit('/').skip(1) {
if dir.file_name()?.as_encoded_bytes() != segment.as_bytes() {
return None;
}
dir = dir.parent()?;
}
Some(dir)
}
#[must_use]
pub fn is_dependency_dir(name: &OsStr) -> bool {
catalog::DEPENDENCY_DIRS.iter().any(|dir| name == OsStr::new(dir))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::testing::tree;
fn classifier() -> Classifier {
Classifier::new(Selection::all()).unwrap()
}
fn verdict(classifier: &Classifier, root: &Path, relative: &str) -> Verdict {
let path = root.join(relative);
let is_dir = path.is_dir();
classifier.classify(root, &path, is_dir)
}
fn assert_artifact(verdict: &Verdict, ecosystem: &str, artifact: &str) {
match verdict {
Verdict::Artifact(finding) => {
let id = finding.artifact().expect("a marker proved it");
assert_eq!(id.ecosystem().id, ecosystem);
assert_eq!(id.artifact().path, artifact);
}
other => panic!("expected an artifact verdict, got {other:?}"),
}
}
#[test]
fn should_classify_target_as_rust_when_cargo_toml_is_beside_it() {
let fixture = tree(&["Cargo.toml", "target/"]);
let classifier = classifier();
let verdict = verdict(&classifier, fixture.path(), "target");
assert_artifact(&verdict, "rust", "target/");
match verdict {
Verdict::Artifact(finding) => assert!(
matches!(finding.provenance, Provenance::Anchored { marker_dir, .. } if marker_dir == fixture.path())
),
other => panic!("expected an artifact verdict, got {other:?}"),
}
}
#[test]
fn should_not_classify_target_without_a_marker() {
let fixture = tree(&["target/", "target/important.txt"]);
let classifier = classifier();
match verdict(&classifier, fixture.path(), "target") {
Verdict::Skip(SkipReason::NoMarker { ecosystem, .. }) => {
assert!(
["rust", "maven", "scala"].contains(&ecosystem),
"unexpected ecosystem {ecosystem}"
);
}
other => panic!("expected a no-marker skip, got {other:?}"),
}
}
#[test]
fn should_resolve_an_ambiguous_name_by_ecosystem_not_precedence() {
let rust = tree(&["Cargo.toml", "target/"]);
let maven = tree(&["pom.xml", "target/"]);
let classifier = classifier();
assert_artifact(&verdict(&classifier, rust.path(), "target"), "rust", "target/");
assert_artifact(&verdict(&classifier, maven.path(), "target"), "maven", "target/");
}
#[test]
fn should_report_a_missing_marker_rather_than_a_lesser_reason() {
let fixture = tree(&["bin/"]);
let classifier = classifier();
match verdict(&classifier, fixture.path(), "bin") {
Verdict::Skip(SkipReason::NoMarker { ecosystem, .. }) => assert_eq!(ecosystem, "dotnet"),
other => panic!("expected a no-marker skip, got {other:?}"),
}
}
#[test]
fn should_prove_dotnet_obj_through_an_ancestor_marker() {
for depth in 0..=3 {
let nesting = "src/".repeat(depth);
let fixture = tree(&["App.csproj", &format!("{nesting}obj/")]);
let classifier = classifier();
let verdict = verdict(&classifier, fixture.path(), &format!("{nesting}obj"));
assert_artifact(&verdict, "dotnet", "obj/");
}
}
#[test]
fn should_not_prove_dotnet_obj_beyond_the_ancestor_bound() {
let fixture = tree(&["App.csproj", "a/b/c/d/obj/"]);
let classifier = classifier();
match verdict(&classifier, fixture.path(), "a/b/c/d/obj") {
Verdict::Skip(SkipReason::NoMarker { ecosystem, .. }) => assert_eq!(ecosystem, "dotnet"),
other => panic!("expected a no-marker skip four levels down, got {other:?}"),
}
}
#[test]
fn should_not_climb_above_the_scan_root_for_a_marker() {
let fixture = tree(&["App.csproj", "project/nested/obj/"]);
let classifier = classifier();
let root = fixture.path().join("project");
let path = root.join("nested/obj");
match classifier.classify(&root, &path, true) {
Verdict::Skip(SkipReason::NoMarker { .. }) => {}
other => panic!("expected the climb to stop at the scan root, got {other:?}"),
}
}
#[test]
fn should_prove_pycache_one_level_below_the_project_root() {
let fixture = tree(&["pyproject.toml", "package/__pycache__/"]);
let classifier = classifier();
assert_artifact(
&verdict(&classifier, fixture.path(), "package/__pycache__"),
"python",
"__pycache__/",
);
}
#[test]
fn should_verify_leading_segments_of_a_multi_segment_artifact() {
let matching = tree(&["Gemfile", "vendor/bundle/"]);
let classifier = classifier();
assert_artifact(
&verdict(&classifier, matching.path(), "vendor/bundle"),
"ruby",
"vendor/bundle/",
);
let elsewhere = tree(&["Gemfile", "lib/bundle/"]);
assert_eq!(
verdict(&classifier, elsewhere.path(), "lib/bundle"),
Verdict::NotACandidate
);
}
#[test]
fn should_require_a_directory_for_a_slash_terminated_artifact() {
let fixture = tree(&["Cargo.toml", "target"]);
let classifier = classifier();
assert_eq!(verdict(&classifier, fixture.path(), "target"), Verdict::NotACandidate);
}
#[test]
fn should_match_a_file_artifact_that_is_not_slash_terminated() {
let fixture = tree(&["package.json", "app.tsbuildinfo"]);
let classifier = classifier();
assert_artifact(
&verdict(&classifier, fixture.path(), "app.tsbuildinfo"),
"node",
"*.tsbuildinfo",
);
}
#[test]
fn should_match_a_prefix_glob_artifact() {
let fixture = tree(&["CMakeLists.txt", "cmake-build-debug/"]);
let classifier = classifier();
assert_artifact(
&verdict(&classifier, fixture.path(), "cmake-build-debug"),
"cmake",
"cmake-build-*/",
);
}
#[test]
fn should_skip_an_off_by_default_artifact_and_say_how_to_enable_it() {
let fixture = tree(&["go.mod", "bin/"]);
let classifier = classifier();
match verdict(&classifier, fixture.path(), "bin") {
Verdict::Skip(SkipReason::NotEnabled { spec, note }) => {
assert_eq!(spec, "go.bin");
assert!(note.is_some(), "an opt-in artifact must explain itself");
}
other => panic!("expected an opt-in skip, got {other:?}"),
}
}
#[test]
fn should_remove_an_off_by_default_artifact_once_enabled() {
let fixture = tree(&["go.mod", "bin/"]);
let mut selection = Selection::all();
assert!(selection.set("go.bin", true));
let classifier = Classifier::new(selection).unwrap();
assert_artifact(&verdict(&classifier, fixture.path(), "bin"), "go", "bin/");
}
#[test]
fn should_not_classify_when_the_ecosystem_is_deselected() {
let fixture = tree(&["Cargo.toml", "target/"]);
let (selection, unknown) = Selection::only(["node"]);
assert!(unknown.is_empty());
let classifier = Classifier::new(selection).unwrap();
match verdict(&classifier, fixture.path(), "target") {
Verdict::Skip(SkipReason::NotEnabled { .. }) => {}
other => panic!("expected a deselected skip, got {other:?}"),
}
}
#[test]
fn should_report_an_unknown_ecosystem_rather_than_ignoring_it() {
let (_, unknown) = Selection::only(["rust", "rustlang"]);
assert_eq!(unknown, vec!["rustlang".to_owned()]);
}
#[test]
fn should_list_each_anchor_directory_exactly_once() {
let mut entries = vec!["Cargo.toml".to_owned()];
for index in 0..20 {
entries.push(format!("crate{index}/Cargo.toml"));
entries.push(format!("crate{index}/target/"));
}
let refs: Vec<&str> = entries.iter().map(String::as_str).collect();
let fixture = tree(&refs);
let classifier = classifier();
for index in 0..20 {
assert_artifact(
&verdict(&classifier, fixture.path(), &format!("crate{index}/target")),
"rust",
"target/",
);
}
assert_eq!(classifier.probe_count(), 20);
}
#[test]
fn should_leave_an_unreadable_anchor_unproven() {
let fixture = tree(&["Cargo.toml", "target/"]);
let classifier = classifier();
let missing = fixture.path().join("gone");
match classifier.classify(fixture.path(), &missing.join("target"), true) {
Verdict::Skip(SkipReason::Unreadable) => {}
other => panic!("expected an unreadable skip, got {other:?}"),
}
}
#[test]
fn should_treat_an_unrelated_name_as_not_a_candidate() {
let fixture = tree(&["Cargo.toml", "src/"]);
let classifier = classifier();
assert_eq!(verdict(&classifier, fixture.path(), "src"), Verdict::NotACandidate);
assert_eq!(
classifier.probe_count(),
0,
"a non-candidate must not cost a directory listing"
);
}
#[test]
fn should_recognise_dependency_directories() {
assert!(is_dependency_dir(OsStr::new("node_modules")));
assert!(is_dependency_dir(OsStr::new("vendor")));
assert!(!is_dependency_dir(OsStr::new("target")));
}
#[test]
fn skip_reason_codes_are_stable() {
assert_eq!(SkipReason::Symlink.code(), "symlink");
assert_eq!(SkipReason::FilesystemBoundary.code(), "filesystem_boundary");
assert_eq!(SkipReason::Unreadable.code(), "unreadable");
assert_eq!(
SkipReason::NoMarker {
ecosystem: "rust",
markers: &["Cargo.toml"]
}
.code(),
"no_marker"
);
}
}