use std::collections::HashMap;
use std::mem;
use std::path::{Path, PathBuf};
use std::sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
};
use ignore::{WalkBuilder, WalkState};
use tracing::{debug, warn};
use crate::error::ScanError;
use crate::model::{CandidateDraft, ScanWarning};
use crate::path_util::{path_file_name, path_file_name_string};
use crate::rules;
use crate::user_rules::UserRuleSet;
use super::progress::ProgressCounters;
use super::safety::{apply_path_safety, is_skip_dir, is_skip_name};
use super::sizer::DirSizes;
use super::{IgnoreMatcher, ScanOptions, should_include};
type WalkOutput = (
HashMap<PathBuf, Vec<CandidateDraft>>,
DirSizes,
Vec<ScanWarning>,
);
pub(crate) struct WalkScratch {
drafts_by_project: Mutex<HashMap<PathBuf, Vec<CandidateDraft>>>,
sizes: Mutex<DirSizes>,
warnings: Mutex<Vec<ScanWarning>>,
poisoned: AtomicBool,
progress: Option<Arc<ProgressCounters>>,
}
impl WalkScratch {
#[cfg(test)]
pub(crate) fn new() -> Self {
Self::new_with_progress(None)
}
pub(crate) fn new_with_progress(progress: Option<Arc<ProgressCounters>>) -> Self {
Self {
drafts_by_project: Mutex::new(HashMap::new()),
sizes: Mutex::new(HashMap::new()),
warnings: Mutex::new(Vec::new()),
poisoned: AtomicBool::new(false),
progress,
}
}
pub(crate) fn into_inner(self) -> Result<WalkOutput, ScanError> {
let WalkScratch {
drafts_by_project,
sizes,
warnings,
poisoned,
progress: _,
} = self;
if poisoned.load(Ordering::SeqCst) {
return Err(poison_error("walk scratch accumulator"));
}
let mut drafts = drafts_by_project
.into_inner()
.map_err(|_| poison_error("walk scratch drafts mutex"))?;
for project_drafts in drafts.values_mut() {
project_drafts.sort_by(|a, b| a.path.cmp(&b.path));
}
let sizes = sizes
.into_inner()
.map_err(|_| poison_error("walk scratch sizes mutex"))?;
let warnings = warnings
.into_inner()
.map_err(|_| poison_error("walk scratch warnings mutex"))?;
Ok((drafts, sizes, warnings))
}
fn mark_poisoned(&self, lock_name: &'static str) {
if !self.poisoned.swap(true, Ordering::SeqCst) {
warn!(
lock = lock_name,
"walk scratch mutex poisoned; aborting scan result"
);
}
}
fn is_poisoned(&self) -> bool {
self.poisoned.load(Ordering::SeqCst)
}
}
struct WalkLocal<'a> {
scratch: &'a WalkScratch,
drafts_by_project: HashMap<PathBuf, Vec<CandidateDraft>>,
sizes: DirSizes,
warnings: Vec<ScanWarning>,
}
impl<'a> WalkLocal<'a> {
fn new(scratch: &'a WalkScratch) -> Self {
Self {
scratch,
drafts_by_project: HashMap::new(),
sizes: HashMap::new(),
warnings: Vec::new(),
}
}
fn add_file_size(&mut self, parent: &Path, bytes: u64) {
let entry = self.sizes.entry(parent.to_path_buf()).or_insert(0);
*entry = entry.saturating_add(bytes);
}
fn add_draft(&mut self, project_dir: &Path, draft: CandidateDraft) {
if let Some(counters) = &self.scratch.progress {
counters.add_candidates(1);
}
self.drafts_by_project
.entry(project_dir.to_path_buf())
.or_default()
.push(draft);
}
fn add_warning(&mut self, warning: ScanWarning) {
self.warnings.push(warning);
}
}
impl Drop for WalkLocal<'_> {
fn drop(&mut self) {
let local_sizes = mem::take(&mut self.sizes);
if !local_sizes.is_empty() {
match self.scratch.sizes.lock() {
Ok(mut sizes) => {
for (dir, bytes) in local_sizes {
let entry = sizes.entry(dir).or_insert(0);
*entry = entry.saturating_add(bytes);
}
}
Err(_) => self.scratch.mark_poisoned("sizes"),
}
}
let local_drafts = mem::take(&mut self.drafts_by_project);
if !local_drafts.is_empty() {
match self.scratch.drafts_by_project.lock() {
Ok(mut drafts) => {
for (project_dir, mut project_drafts) in local_drafts {
drafts
.entry(project_dir)
.or_default()
.append(&mut project_drafts);
}
}
Err(_) => self.scratch.mark_poisoned("drafts_by_project"),
}
}
let local_warnings = mem::take(&mut self.warnings);
if !local_warnings.is_empty() {
match self.scratch.warnings.lock() {
Ok(mut warnings) => warnings.extend(local_warnings),
Err(_) => self.scratch.mark_poisoned("warnings"),
}
}
}
}
pub(crate) fn walk_parallel(
root: &Path,
options: &ScanOptions,
matcher: &IgnoreMatcher,
user_rules: &UserRuleSet,
scratch: &WalkScratch,
) {
use std::thread::available_parallelism;
let threads = available_parallelism().map(|n| n.get()).unwrap_or(1).max(1);
let mut builder = WalkBuilder::new(root);
builder
.max_depth(Some(options.max_depth))
.threads(threads)
.follow_links(false)
.standard_filters(false)
.hidden(false);
builder.build_parallel().run(|| {
let mut local = WalkLocal::new(scratch);
Box::new(move |result| {
if scratch.is_poisoned() {
return WalkState::Quit;
}
let entry = match result {
Ok(e) => e,
Err(err) => {
debug!(error = %err, "walk error");
local.add_warning(ScanWarning::WalkError {
path: None,
error: err.to_string(),
});
return WalkState::Continue;
}
};
let path = entry.path().to_path_buf();
let file_type = match entry.file_type() {
Some(ft) => ft,
None => return WalkState::Continue,
};
if path == root {
return WalkState::Continue;
}
if file_type.is_dir() && (is_skip_dir(&path) || is_skip_name_path(&path)) {
return WalkState::Skip;
}
if file_type.is_file() {
match entry.metadata() {
Ok(metadata) => {
if let Some(parent) = path.parent() {
local.add_file_size(parent, metadata.len());
}
}
Err(err) => {
debug!(path = %path.display(), error = %err, "walk metadata error");
local.add_warning(ScanWarning::MetadataError {
path,
error: err.to_string(),
});
}
}
return WalkState::Continue;
}
let is_dir_like = file_type.is_dir() || file_type.is_symlink();
if !is_dir_like {
return WalkState::Continue;
}
let Some(parent) = path.parent() else {
return WalkState::Continue;
};
let Some(name) = path_file_name_string(&path) else {
return WalkState::Continue;
};
if rules::is_candidate_name(&name)
&& let Some(mut draft) = rules::classify_candidate(parent, &name, path.clone())
{
if matcher.is_ignored(&path, true) {
return WalkState::Skip;
}
apply_path_safety(root, &mut draft);
if should_include(&draft, options) {
local.add_draft(parent, draft);
}
return WalkState::Skip;
}
if !user_rules.is_empty()
&& let Some(mut draft) = user_rules.classify(&name, parent)
{
if matcher.is_ignored(&path, true) {
return WalkState::Skip;
}
apply_path_safety(root, &mut draft);
if should_include(&draft, options) {
local.add_draft(parent, draft);
}
return WalkState::Skip;
}
if matcher.is_ignored(&path, true) {
return WalkState::Skip;
}
WalkState::Continue
})
});
}
fn poison_error(lock_name: &str) -> ScanError {
ScanError::Generic(format!("{lock_name} poisoned during scan"))
}
fn is_skip_name_path(path: &Path) -> bool {
path_file_name(path).is_some_and(is_skip_name)
}
#[cfg(test)]
mod tests {
use std::panic::{self, AssertUnwindSafe};
use super::*;
#[test]
fn into_inner_reports_poisoned_scratch_without_panicking() {
let scratch = WalkScratch::new();
let poison_result = panic::catch_unwind(AssertUnwindSafe(|| {
let _guard = match scratch.sizes.lock() {
Ok(guard) => guard,
Err(err) => panic!("unexpected pre-existing poison: {err}"),
};
panic!("poison sizes");
}));
assert!(poison_result.is_err());
let err = match scratch.into_inner() {
Ok(_) => panic!("poisoned scratch must error"),
Err(err) => err,
};
assert!(
err.to_string()
.contains("walk scratch sizes mutex poisoned")
);
}
#[test]
fn local_drop_marks_poisoned_scratch_without_panicking() {
let scratch = WalkScratch::new();
let poison_result = panic::catch_unwind(AssertUnwindSafe(|| {
let _guard = match scratch.sizes.lock() {
Ok(guard) => guard,
Err(err) => panic!("unexpected pre-existing poison: {err}"),
};
panic!("poison sizes");
}));
assert!(poison_result.is_err());
let result = panic::catch_unwind(AssertUnwindSafe(|| {
let mut local = WalkLocal::new(&scratch);
local.add_file_size(Path::new("/tmp/project"), 7);
}));
assert!(result.is_ok());
assert!(scratch.is_poisoned());
let err = match scratch.into_inner() {
Ok(_) => panic!("poisoned scratch must error"),
Err(err) => err,
};
assert!(
err.to_string()
.contains("walk scratch accumulator poisoned")
);
}
#[test]
fn into_inner_reports_poisoned_drafts_without_panicking() {
let scratch = WalkScratch::new();
let poison_result = panic::catch_unwind(AssertUnwindSafe(|| {
let _guard = match scratch.drafts_by_project.lock() {
Ok(guard) => guard,
Err(err) => panic!("unexpected pre-existing poison: {err}"),
};
panic!("poison drafts");
}));
assert!(poison_result.is_err());
let err = match scratch.into_inner() {
Ok(_) => panic!("poisoned scratch must error"),
Err(err) => err,
};
assert!(
err.to_string()
.contains("walk scratch drafts mutex poisoned")
);
}
#[test]
fn local_drop_marks_poisoned_drafts_without_panicking() {
let scratch = WalkScratch::new();
let poison_result = panic::catch_unwind(AssertUnwindSafe(|| {
let _guard = match scratch.drafts_by_project.lock() {
Ok(guard) => guard,
Err(err) => panic!("unexpected pre-existing poison: {err}"),
};
panic!("poison drafts");
}));
assert!(poison_result.is_err());
let result = panic::catch_unwind(AssertUnwindSafe(|| {
let mut local = WalkLocal::new(&scratch);
let draft = CandidateDraft {
path: PathBuf::from("/tmp/project/node_modules"),
name: "node_modules".to_string(),
rule_id: "node.node_modules".to_string(),
category: crate::model::Category::Deps,
safety: crate::model::Safety::Safe,
reasons: Vec::new(),
warnings: Vec::new(),
restore_hint: "reinstall dependencies".to_string(),
};
local.add_draft(Path::new("/tmp/project"), draft);
}));
assert!(result.is_ok());
assert!(scratch.is_poisoned());
let err = match scratch.into_inner() {
Ok(_) => panic!("poisoned scratch must error"),
Err(err) => err,
};
assert!(
err.to_string()
.contains("walk scratch accumulator poisoned")
);
}
}