use std::collections::{BTreeMap, HashSet};
use std::ffi::{OsStr, OsString};
use std::path::{Path, PathBuf};
use ignore::gitignore::{Gitignore, GitignoreBuilder};
use ignore::{WalkBuilder, WalkState};
use crate::caches::CacheRoots;
use crate::catalog::CATALOG;
use crate::classify::is_dependency_dir;
use crate::scan::NEVER_DESCEND;
use crate::size;
const MIN_OCCURRENCES: usize = 3;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Suggestion {
pub name: OsString,
pub paths: Vec<PathBuf>,
pub bytes: u64,
}
impl Suggestion {
#[must_use]
pub fn include_line(&self) -> String {
format!("include = [\"{}\"]", Path::new(&self.name).display())
}
}
#[must_use]
pub fn suggest(roots: &[PathBuf], one_file_system: bool, jobs: Option<usize>) -> Vec<Suggestion> {
let ignored = ignored_names(roots, one_file_system, jobs);
if ignored.is_empty() {
return Vec::new();
}
let mut rules = Rules::default();
let mut found: BTreeMap<OsString, Vec<PathBuf>> = BTreeMap::new();
for (name, path) in matching_directories(roots, &ignored, one_file_system, jobs) {
if rules.ignores(&path) {
found.entry(name).or_default().push(path);
}
}
let mut suggestions: Vec<Suggestion> = found
.into_iter()
.filter(|(_, paths)| paths.len() >= MIN_OCCURRENCES)
.map(|(name, mut paths)| {
paths.sort();
let bytes = paths.iter().map(|path| size::measure(path)).sum();
Suggestion { name, paths, bytes }
})
.collect();
suggestions.sort_by(|left, right| right.bytes.cmp(&left.bytes).then_with(|| left.name.cmp(&right.name)));
suggestions
}
fn ignored_names(roots: &[PathBuf], one_file_system: bool, jobs: Option<usize>) -> HashSet<OsString> {
let names = std::sync::Mutex::new(HashSet::new());
walk(roots, one_file_system, jobs, |entry, is_dir| {
if is_dir {
return WalkState::Continue;
}
if entry.file_name() != OsStr::new(".gitignore") {
return WalkState::Continue;
}
if let Ok(contents) = std::fs::read_to_string(entry.path())
&& let Ok(mut names) = names.lock()
{
names.extend(literal_directory_names(&contents));
}
WalkState::Continue
});
names.into_inner().unwrap_or_default()
}
fn literal_directory_names(contents: &str) -> Vec<OsString> {
contents
.lines()
.map(str::trim)
.filter(|line| !line.is_empty() && !line.starts_with('#') && !line.starts_with('!'))
.filter_map(|line| line.strip_suffix('/'))
.map(|line| line.trim_start_matches("**/").trim_start_matches('/'))
.filter(|name| !name.is_empty() && !name.contains('/') && !name.contains(['*', '?', '[']))
.map(OsString::from)
.collect()
}
fn matching_directories(
roots: &[PathBuf],
ignored: &HashSet<OsString>,
one_file_system: bool,
jobs: Option<usize>,
) -> Vec<(OsString, PathBuf)> {
let found = std::sync::Mutex::new(Vec::new());
walk(roots, one_file_system, jobs, |entry, is_dir| {
if !is_dir {
return WalkState::Continue;
}
let name = entry.file_name();
if !ignored.contains(name) || claimed_by_the_catalog(name) {
return WalkState::Continue;
}
if let Ok(mut found) = found.lock() {
found.push((name.to_os_string(), entry.path().to_path_buf()));
}
WalkState::Skip
});
found.into_inner().unwrap_or_default()
}
#[derive(Debug, Default)]
struct Rules {
by_dir: BTreeMap<PathBuf, Gitignore>,
}
impl Rules {
fn ignores(&mut self, path: &Path) -> bool {
let Some(parent) = path.parent() else {
return false;
};
if !self.by_dir.contains_key(parent) {
let rules = Self::build(parent);
self.by_dir.insert(parent.to_path_buf(), rules);
}
self.by_dir
.get(parent)
.is_some_and(|rules| rules.matched(path, true).is_ignore())
}
fn build(dir: &Path) -> Gitignore {
let mut chain = Vec::new();
let mut current = Some(dir);
let mut repository = None;
while let Some(at) = current {
chain.push(at.to_path_buf());
if at.join(".git").exists() {
repository = Some(at.to_path_buf());
break;
}
current = at.parent();
}
let Some(repository) = repository else {
return Gitignore::empty();
};
let mut builder = GitignoreBuilder::new(&repository);
for at in chain.iter().rev() {
builder.add(at.join(".gitignore"));
}
builder.build().unwrap_or_else(|_| Gitignore::empty())
}
}
fn claimed_by_the_catalog(name: &OsStr) -> bool {
CATALOG.iter().any(|ecosystem| {
ecosystem
.artifacts
.iter()
.any(|artifact| OsStr::new(artifact.path.trim_end_matches('/')) == name)
})
}
fn walk(
roots: &[PathBuf],
one_file_system: bool,
jobs: Option<usize>,
visit: impl Fn(&ignore::DirEntry, bool) -> WalkState + Send + Sync,
) {
for root in roots {
let caches = CacheRoots::for_root(root, &[]);
let mut builder = WalkBuilder::new(root);
builder
.git_ignore(false)
.git_global(false)
.git_exclude(false)
.ignore(false)
.parents(false)
.hidden(false)
.follow_links(false)
.same_file_system(one_file_system);
if let Some(jobs) = jobs {
builder.threads(jobs.max(1));
}
builder.build_parallel().run(|| {
Box::new(|result| {
let Ok(entry) = result else {
return WalkState::Continue;
};
let is_dir = entry.file_type().is_some_and(|kind| kind.is_dir());
let name = entry.file_name();
if is_dir
&& (NEVER_DESCEND.iter().any(|never| name == OsStr::new(never))
|| is_dependency_dir(name)
|| caches.contains(entry.path()))
{
return WalkState::Skip;
}
visit(&entry, is_dir)
})
});
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::testing::tree;
fn names(suggestions: &[Suggestion]) -> Vec<String> {
suggestions
.iter()
.map(|suggestion| suggestion.name.to_string_lossy().into_owned())
.collect()
}
fn run(root: &Path) -> Vec<Suggestion> {
suggest(&[root.to_path_buf()], true, Some(1))
}
fn repository(root: &Path, gitignore: &str) {
std::fs::create_dir_all(root.join(".git")).unwrap();
std::fs::write(root.join(".gitignore"), gitignore).unwrap();
}
#[test]
fn should_suggest_a_repeated_name_a_gitignore_lists() {
let fixture = tree(&["a/.mytool/blob", "b/.mytool/blob", "c/.mytool/blob"]);
repository(fixture.path(), ".mytool/\n");
assert_eq!(names(&run(fixture.path())), vec![".mytool".to_owned()]);
}
#[test]
fn should_not_suggest_a_name_no_gitignore_names() {
let fixture = tree(&["a/.mytool/blob", "b/.mytool/blob", "c/.mytool/blob"]);
repository(fixture.path(), "unrelated/\n");
assert!(run(fixture.path()).is_empty(), "no suggestion is warranted here");
}
#[test]
fn should_not_suggest_a_name_that_barely_repeats() {
let fixture = tree(&["a/.mytool/blob", "b/.mytool/blob"]);
repository(fixture.path(), ".mytool/\n");
assert!(run(fixture.path()).is_empty(), "no suggestion is warranted here");
}
#[test]
fn should_not_suggest_something_the_catalog_already_claims() {
let fixture = tree(&["a/target/o", "b/target/o", "c/target/o"]);
repository(fixture.path(), "target/\n");
assert!(run(fixture.path()).is_empty(), "no suggestion is warranted here");
}
#[test]
fn should_read_only_literal_directory_names_out_of_a_gitignore() {
let parsed =
literal_directory_names("# comment\n.alef/\n**/.basemind/\n/build/\nbuild-*/\n!keep/\nnotes.txt\n");
assert_eq!(
parsed,
vec![
OsString::from(".alef"),
OsString::from(".basemind"),
OsString::from("build")
]
);
}
#[test]
fn should_never_look_inside_a_dependency_directory() {
let fixture = tree(&[
"node_modules/a/.mytool/blob",
"node_modules/b/.mytool/blob",
"node_modules/c/.mytool/blob",
]);
repository(fixture.path(), ".mytool/\n");
assert!(run(fixture.path()).is_empty(), "no suggestion is warranted here");
}
#[test]
fn should_attribute_nested_candidates_to_the_outer_one() {
let fixture = tree(&["a/out/.mytool/blob", "b/out/.mytool/blob", "c/out/.mytool/blob"]);
repository(fixture.path(), "out/\n.mytool/\n");
assert_eq!(
names(&run(fixture.path())),
vec!["out".to_owned()],
"the outer name is suggested and the inner one is not counted again"
);
}
#[test]
fn should_not_suggest_a_directory_another_repository_happens_to_ignore() {
let fixture = tree(&[
"generated/a/lib/blob",
"generated/b/lib/blob",
"handwritten/x/lib/source.rs",
"handwritten/y/lib/source.rs",
"handwritten/z/lib/source.rs",
]);
repository(&fixture.path().join("generated"), "lib/\n");
repository(&fixture.path().join("handwritten"), "unrelated/\n");
let suggestions = run(fixture.path());
assert!(
suggestions.is_empty(),
"the two ignored `lib/`s are below the threshold and the three hand-written ones \
are not ignored at all — suggesting them would be advice to delete source: {:?}",
names(&suggestions)
);
}
#[test]
fn should_emit_a_pasteable_include_line() {
let suggestion = Suggestion {
name: OsString::from(".alef"),
paths: Vec::new(),
bytes: 0,
};
assert_eq!(suggestion.include_line(), "include = [\".alef\"]");
}
}