1#[cfg(any(windows, target_os = "macos"))]
2use crate::walk::walk_root_entries as walk_roots;
3#[cfg(not(any(windows, target_os = "macos")))]
4use crate::walk::walk_roots;
5use crate::{crossdev, walk};
6use anyhow::Context;
7use byte_unit::{Byte, Unit, UnitType};
8use serde::Deserialize;
9use std::collections::BTreeSet;
10use std::path::PathBuf;
11use std::sync::Arc;
12use std::sync::atomic::{AtomicBool, Ordering};
13use std::time::Duration;
14use std::{fmt, path::Path};
15
16#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)]
18pub enum ByteFormat {
19 #[serde(rename = "metric")]
21 Metric,
22 #[serde(rename = "binary")]
24 Binary,
25 #[serde(rename = "bytes")]
27 Bytes,
28 #[serde(rename = "gb")]
30 GB,
31 #[serde(rename = "gib")]
33 GiB,
34 #[serde(rename = "mb")]
36 MB,
37 #[serde(rename = "mib")]
39 MiB,
40}
41
42impl ByteFormat {
43 #[must_use]
45 pub fn width(self) -> usize {
46 use ByteFormat::{Binary, Bytes, MB, MiB};
47 match self {
48 Binary => 11,
49 Bytes | MB | MiB => 12,
50 _ => 10,
51 }
52 }
53 #[must_use]
55 pub fn total_width(self) -> usize {
56 use ByteFormat::{Binary, Bytes, GB, GiB, MB, Metric, MiB};
57 const THE_SPACE_BETWEEN_UNIT_AND_NUMBER: usize = 1;
58
59 self.width()
60 + match self {
61 Binary | MiB | GiB => 3,
62 Metric | MB | GB => 2,
63 Bytes => 1,
64 }
65 + THE_SPACE_BETWEEN_UNIT_AND_NUMBER
66 }
67 #[must_use]
69 pub fn display(self, bytes: u128) -> impl fmt::Display {
70 ByteFormatDisplay {
71 format: self,
72 bytes,
73 }
74 }
75}
76
77struct ByteFormatDisplay {
79 format: ByteFormat,
80 bytes: u128,
81}
82
83impl fmt::Display for ByteFormatDisplay {
84 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
85 use ByteFormat::{Binary, Bytes, GB, GiB, MB, Metric, MiB};
86
87 let bytes = Byte::from_u128(self.bytes).expect("supported byte count");
88 let adjusted = match self.format {
89 Bytes => return write!(f, "{} b", self.bytes),
90 Binary => bytes.get_appropriate_unit(UnitType::Binary),
91 Metric => bytes.get_appropriate_unit(UnitType::Decimal),
92 GB => bytes.get_adjusted_unit(Unit::GB),
93 GiB => bytes.get_adjusted_unit(Unit::GiB),
94 MB => bytes.get_adjusted_unit(Unit::MB),
95 MiB => bytes.get_adjusted_unit(Unit::MiB),
96 };
97 let b = format!("{adjusted:.2}");
98 let mut splits = b.split(' ');
99 match (splits.next(), splits.next()) {
100 (Some(bytes), Some(unit)) => write!(
101 f,
102 "{} {:>unit_width$}",
103 bytes,
104 unit,
105 unit_width = match self.format {
106 Binary => 3,
107 _ => 2,
108 }
109 ),
110 _ => f.write_str(&b),
111 }
112 }
113}
114
115#[derive(Debug)]
117pub(crate) struct Throttle {
118 trigger: Arc<AtomicBool>,
119}
120
121impl Throttle {
122 pub(crate) fn new(duration: Duration, initial_sleep: Option<Duration>) -> Self {
126 let instance = Self {
127 trigger: Arc::default(),
128 };
129
130 let trigger = Arc::downgrade(&instance.trigger);
131 std::thread::spawn(move || {
132 if let Some(duration) = initial_sleep {
133 std::thread::sleep(duration);
134 }
135 while let Some(t) = trigger.upgrade() {
136 t.store(true, Ordering::Relaxed);
137 std::thread::sleep(duration);
138 }
139 });
140
141 instance
142 }
143
144 pub(crate) fn can_update(&self) -> bool {
146 self.trigger.swap(false, Ordering::Relaxed)
147 }
148}
149
150#[derive(Clone, Debug)]
160pub struct IgnorePatterns {
161 search: gix::ignore::Search,
162}
163
164impl IgnorePatterns {
165 pub fn from_files(files: &[PathBuf]) -> anyhow::Result<Option<Self>> {
172 let mut search = gix::ignore::Search::default();
173 for file in files {
174 let buf = std::fs::read(file).with_context(|| {
175 format!("Failed to read ignore patterns from {}", file.display())
176 })?;
177 search.add_patterns_buffer(
178 &buf,
179 file.clone(),
180 None,
181 gix::ignore::search::Ignore::default(),
182 );
183 }
184 let pattern_count = search
185 .patterns
186 .iter()
187 .map(|list| list.patterns.len())
188 .sum::<usize>();
189 Ok(if pattern_count != 0 {
190 log::info!(
191 "Loaded {pattern_count} ignore pattern(s) from {file_count} file(s)",
192 file_count = files.len()
193 );
194 Some(Self { search })
195 } else {
196 None
197 })
198 }
199
200 #[must_use]
207 pub fn is_excluded(&self, relative_path: &Path, is_dir: bool) -> bool {
208 if relative_path.as_os_str().is_empty() {
209 return false;
210 }
211 let relative_path =
212 gix::path::to_unix_separators_on_windows(gix::path::into_bstr(relative_path));
213 self.search
214 .pattern_matching_relative_path(
215 relative_path.as_ref(),
216 Some(is_dir),
217 gix::ignore::glob::pattern::Case::Sensitive,
218 )
219 .is_some_and(|match_| !match_.pattern.is_negative())
220 }
221
222 #[must_use]
228 pub fn excludes_input_path(&self, path: &Path, cwd: &Path) -> bool {
229 pattern_relative_path(path, cwd, path)
230 .is_some_and(|relative_path| self.is_excluded(relative_path, path.is_dir()))
231 }
232}
233
234#[derive(Clone)]
236pub struct WalkOptions {
237 pub threads: usize,
239 pub count_hard_links: bool,
241 pub apparent_size: bool,
243 pub cross_filesystems: bool,
245 pub ignore_dirs: BTreeSet<PathBuf>,
247 pub ignore_patterns: Option<IgnorePatterns>,
250 pub metadata_options: crate::TraversalOptions,
252}
253
254type ExcludeEntry = Arc<dyn Fn(usize, &walk::Entry) -> bool + Send + Sync>;
256
257pub(crate) struct WalkRoot {
259 pub index: usize,
261 pub path: PathBuf,
263 #[cfg(any(windows, target_os = "macos"))]
265 pub entry: Option<walk::Entry>,
266 pub pattern_root: Option<PathBuf>,
273 pub device_id: u64,
276}
277
278impl WalkOptions {
279 #[must_use]
281 pub fn is_ignored_directory(&self, path: &Path, cwd: &Path) -> bool {
282 ignore_directory(path, &self.ignore_dirs, cwd)
283 }
284
285 pub(crate) fn iter_from_paths(
286 &self,
287 roots: Vec<WalkRoot>,
288 skip_root: bool,
289 order: walk::Order,
290 ) -> impl Iterator<Item = (usize, walk::RootEvent)> + use<> {
291 let num_roots = roots
292 .iter()
293 .map(|root| root.index)
294 .max()
295 .map_or(0, |idx| idx + 1);
296 let path_count = roots.len();
297 let (device_ids, root_paths, indexed_roots) = roots.into_iter().fold(
298 (
299 vec![0; num_roots],
300 vec![None; num_roots],
301 Vec::with_capacity(path_count),
302 ),
303 |(mut device_ids, mut root_paths, mut indexed_roots), root| {
304 device_ids[root.index] = root.device_id;
305 root_paths[root.index] = root.pattern_root;
306 #[cfg(any(windows, target_os = "macos"))]
307 indexed_roots.push((
308 root.index,
309 root.entry.map_or_else(
310 || walk::Entry::from_path(&root.path, self.metadata_options),
311 Ok,
312 ),
313 ));
314 #[cfg(not(any(windows, target_os = "macos")))]
315 indexed_roots.push((root.index, root.path));
316 (device_ids, root_paths, indexed_roots)
317 },
318 );
319 let ignore_dirs = self.ignore_dirs.clone();
320 let cwd = std::env::current_dir().unwrap_or_default();
321 let cross_filesystems = self.cross_filesystems;
322
323 let is_excluded: ExcludeEntry = {
327 let patterns = self.ignore_patterns.clone();
328 let cwd = cwd.clone();
329 Arc::new(move |root_idx: usize, entry: &walk::Entry| {
330 let Some((patterns, pattern_root)) =
331 patterns.as_ref().zip(root_paths[root_idx].as_deref())
332 else {
333 return false;
334 };
335 let path = entry.path();
336 pattern_relative_path(&path, &cwd, pattern_root).is_some_and(|relative_path| {
337 patterns.is_excluded(relative_path, entry.file_type.is_dir())
338 })
339 })
340 };
341 let is_excluded_while_walking = Arc::clone(&is_excluded);
342 let descend = move |root_idx: usize, entry: &walk::Entry| {
343 (cross_filesystems
344 || entry.metadata.as_ref().map_or(true, |metadata| {
345 crossdev::is_same_device(device_ids[root_idx], metadata)
346 }))
347 && (entry.depth == 0 || !ignore_directory(&entry.path(), &ignore_dirs, &cwd))
348 && !is_excluded_while_walking(root_idx, entry)
349 };
350
351 let walk = walk_roots(
352 indexed_roots,
353 self.threads,
354 order,
355 self.metadata_options,
356 descend,
357 );
358
359 walk.filter(move |(root_idx, event)| match event {
360 walk::RootEvent::Entry(Ok(entry)) => {
361 (!skip_root || entry.depth > 0) && !is_excluded(*root_idx, entry)
362 }
363 walk::RootEvent::Entry(Err(_)) | walk::RootEvent::Finished => true,
364 })
365 }
366}
367
368#[derive(Default)]
370pub struct WalkResult {
371 pub num_errors: u64,
373}
374
375impl WalkResult {
376 #[must_use]
380 pub fn to_exit_code(&self) -> i32 {
381 i32::from(self.num_errors > 0)
382 }
383}
384
385pub fn canonicalize_ignore_dirs(ignore_dirs: &[PathBuf]) -> BTreeSet<PathBuf> {
389 let dirs = ignore_dirs
390 .iter()
391 .map(gix::path::realpath)
392 .filter_map(Result::ok)
393 .collect();
394 log::info!("Ignoring canonicalized {dirs:?}");
395 dirs
396}
397
398fn pattern_relative_path<'a>(
405 path: &'a Path,
406 cwd: &Path,
407 traversal_root: &Path,
408) -> Option<&'a Path> {
409 if path.is_relative() {
410 return Some(path);
411 }
412 path.strip_prefix(cwd)
413 .or_else(|_| path.strip_prefix(traversal_root))
414 .ok()
415}
416
417fn ignore_directory(path: &Path, ignore_dirs: &BTreeSet<PathBuf>, cwd: &Path) -> bool {
418 if ignore_dirs.is_empty() {
419 return false;
420 }
421 let path = gix::path::realpath_opts(path, cwd, 32);
422 path.is_ok_and(|path| {
423 let ignored = ignore_dirs.contains(&path);
424 if ignored {
425 log::debug!("Ignored {}", path.display());
426 }
427 ignored
428 })
429}
430
431#[cfg(test)]
432mod tests {
433 use super::*;
434
435 #[test]
436 fn walk_options_identify_ignored_directories() {
437 let cwd = std::env::current_dir().unwrap();
438 let mut options = WalkOptions {
439 threads: 1,
440 count_hard_links: false,
441 apparent_size: false,
442 cross_filesystems: true,
443 ignore_dirs: BTreeSet::new(),
444 ignore_patterns: None,
445 metadata_options: crate::TraversalOptions::default(),
446 };
447 #[cfg(unix)]
448 let mut parameters = vec![
449 ("/usr", vec!["/usr"], true),
450 ("/usr/local", vec!["/usr"], false),
451 ("/smth", vec!["/usr"], false),
452 ("/usr/local/..", vec!["/usr/local/.."], true),
453 ("/usr", vec!["/usr/local/.."], true),
454 ("/usr/local/share/../..", vec!["/usr"], true),
455 ];
456
457 #[cfg(windows)]
458 let mut parameters = vec![
459 ("C:\\Windows", vec!["C:\\Windows"], true),
460 ("C:\\Windows\\System", vec!["C:\\Windows"], false),
461 ("C:\\Smth", vec!["C:\\Windows"], false),
462 (
463 "C:\\Windows\\System\\..",
464 vec!["C:\\Windows\\System\\.."],
465 true,
466 ),
467 ("C:\\Windows", vec!["C:\\Windows\\System\\.."], true),
468 (
469 "C:\\Windows\\System\\Speech\\..\\..",
470 vec!["C:\\Windows"],
471 true,
472 ),
473 ];
474
475 parameters.extend([
476 ("src", vec![], false),
477 ("src", vec!["src"], true),
478 ("src/interactive", vec!["src"], false),
479 ("src/interactive/..", vec!["src"], true),
480 ]);
481
482 for (path, ignore_dirs, expected_result) in parameters {
483 options.ignore_dirs = canonicalize_ignore_dirs(
484 &ignore_dirs.into_iter().map(Into::into).collect::<Vec<_>>(),
485 );
486 assert_eq!(
487 options.is_ignored_directory(path.as_ref(), &cwd),
488 expected_result,
489 "result='{expected_result}' for path='{path}' and ignore_dir='{:?}' ",
490 options.ignore_dirs
491 );
492 }
493 }
494
495 #[test]
496 fn explicitly_selected_ignored_root_is_traversed() {
497 let root = tempfile::tempdir().unwrap();
498 let child = root.path().join("child");
499 std::fs::create_dir(&child).unwrap();
500 let options = WalkOptions {
501 threads: 2,
502 count_hard_links: false,
503 apparent_size: false,
504 cross_filesystems: true,
505 ignore_dirs: canonicalize_ignore_dirs(&[root.path().to_owned()]),
506 ignore_patterns: None,
507 metadata_options: crate::TraversalOptions::default(),
508 };
509
510 let paths = options
511 .iter_from_paths(
512 vec![WalkRoot {
513 index: 0,
514 pattern_root: None,
515 path: root.path().to_owned(),
516 #[cfg(any(windows, target_os = "macos"))]
517 entry: None,
518 device_id: crossdev::init(root.path()).unwrap(),
519 }],
520 false,
521 walk::Order::Completion,
522 )
523 .filter_map(|(_, event)| match event {
524 walk::RootEvent::Entry(entry) => Some(entry.unwrap().path()),
525 walk::RootEvent::Finished => None,
526 })
527 .collect::<Vec<_>>();
528
529 assert!(paths.contains(&child));
530 }
531
532 #[test]
533 fn ignore_patterns_use_gitignore_semantics() {
534 let patterns = patterns_from(
535 "# a comment, and the blank line below, match nothing\n\
536 \n\
537 *.log\n\
538 !keep.log\n\
539 build/\n\
540 /anchored\n\
541 **/node_modules/\n",
542 );
543
544 for (path, is_dir, expected) in [
545 ("debug.log", false, true),
546 ("nested/debug.log", false, true),
547 ("keep.log", false, false),
548 ("build", true, true),
549 ("build", false, false),
551 ("anchored", true, true),
552 ("nested/anchored", true, false),
553 ("nested/deeply/node_modules", true, true),
554 ("src", true, false),
555 ("", true, false),
556 ] {
557 assert_eq!(
558 patterns.is_excluded(Path::new(path), is_dir),
559 expected,
560 "expected is_excluded({path:?}, is_dir={is_dir}) to be {expected}"
561 );
562 }
563 }
564
565 #[test]
566 fn later_ignore_files_win_over_earlier_ones() {
567 let dir = tempfile::tempdir().unwrap();
568 let (first, second) = (dir.path().join("first"), dir.path().join("second"));
569 std::fs::write(&first, "*.tmp\n").unwrap();
570 std::fs::write(&second, "!important.tmp\n").unwrap();
571
572 let patterns = IgnorePatterns::from_files(&[first, second])
573 .unwrap()
574 .unwrap();
575
576 assert!(patterns.is_excluded(Path::new("scratch.tmp"), false));
577 assert!(
578 !patterns.is_excluded(Path::new("important.tmp"), false),
579 "the negation in the second file overrides the first file"
580 );
581 }
582
583 #[test]
584 fn unreadable_ignore_files_are_an_error() {
585 let err = IgnorePatterns::from_files(&[PathBuf::from("does-not-exist")])
586 .expect_err("a missing pattern file must not be silently skipped");
587 assert!(err.to_string().contains("does-not-exist"));
588 }
589
590 #[test]
591 fn empty_ignore_files_produce_none() {
592 let file = tempfile::NamedTempFile::new().unwrap();
593 std::fs::write(file.path(), "# comment only\n").unwrap();
594 assert!(
595 IgnorePatterns::from_files(&[file.path().to_owned()])
596 .unwrap()
597 .is_none(),
598 "no patterns means no need to match anything"
599 );
600 }
601
602 #[test]
603 fn matching_entries_are_pruned_from_the_walk() {
604 let root = tempfile::tempdir().unwrap();
605 for dir in ["keep", "build", "keep/node_modules"] {
606 std::fs::create_dir(root.path().join(dir)).unwrap();
607 }
608 for file in [
609 "keep/main.rs",
610 "keep/debug.log",
611 "build/artifact",
612 "keep/node_modules/dep",
613 ] {
614 std::fs::write(root.path().join(file), b"x").unwrap();
615 }
616
617 assert_eq!(
618 walk_with_patterns(root.path(), "*.log\n**/node_modules/\n"),
619 [
620 PathBuf::new(),
621 PathBuf::from("build"),
622 PathBuf::from("build/artifact"),
623 PathBuf::from("keep"),
624 PathBuf::from("keep/main.rs"),
625 ],
626 "excluded directories are pruned along with everything below them, \
627 and excluded files never show up"
628 );
629 }
630
631 #[test]
632 fn patterns_match_the_path_dua_reports() {
633 let here = tempfile::tempdir().unwrap();
634 let elsewhere = tempfile::tempdir().unwrap();
635 let (cwd, outside) = (here.path(), elsewhere.path());
636
637 for dir in ["target", "nested", "nested/target"] {
638 std::fs::create_dir_all(here.path().join(dir)).unwrap();
639 }
640 assert_eq!(
641 walk_with_patterns(here.path(), "/target/\n"),
642 [
643 PathBuf::new(),
644 PathBuf::from("nested"),
645 PathBuf::from("nested/target"),
646 ],
647 "an anchored pattern excludes only the top-level target"
648 );
649
650 let (entry, root) = (cwd.join("a").join("b"), cwd.join("a"));
652 let reported = Path::new("a").join("b");
653 assert_eq!(
654 pattern_relative_path(&entry, cwd, &root),
655 Some(reported.as_path())
656 );
657
658 let entry = outside.join("syslog");
661 assert_eq!(
662 pattern_relative_path(&entry, cwd, outside),
663 Some(Path::new("syslog"))
664 );
665
666 assert_eq!(
668 pattern_relative_path(outside, cwd, outside),
669 Some(Path::new(""))
670 );
671 assert!(!patterns_from("*\n").is_excluded(Path::new(""), true));
672 }
673
674 #[test]
675 fn subtree_walk_keeps_the_original_pattern_root() {
676 let root = tempfile::tempdir().unwrap();
677 let nested = root.path().join("nested");
678 std::fs::create_dir(&nested).unwrap();
679 std::fs::write(nested.join("secret"), []).unwrap();
680 std::fs::write(nested.join("visible"), []).unwrap();
681 let options = WalkOptions {
682 threads: 1,
683 count_hard_links: false,
684 apparent_size: false,
685 cross_filesystems: true,
686 ignore_dirs: BTreeSet::default(),
687 ignore_patterns: Some(patterns_from("nested/secret\n")),
688 metadata_options: crate::TraversalOptions::default(),
689 };
690
691 let paths = options
692 .iter_from_paths(
693 vec![WalkRoot {
694 index: 0,
695 path: nested,
696 #[cfg(any(windows, target_os = "macos"))]
697 entry: None,
698 pattern_root: Some(root.path().to_owned()),
699 device_id: 0,
700 }],
701 false,
702 walk::Order::Completion,
703 )
704 .filter_map(|(_, event)| match event {
705 walk::RootEvent::Entry(entry) => Some(entry.unwrap().file_name),
706 walk::RootEvent::Finished => None,
707 })
708 .collect::<Vec<_>>();
709
710 assert!(!paths.iter().any(|path| path == "secret"));
711 assert!(paths.iter().any(|path| path == "visible"));
712 }
713
714 #[test]
715 fn excluded_input_paths_are_dropped_before_the_walk() {
716 let patterns = patterns_from("src/\n*.toml\n");
718 let cwd = std::env::current_dir().unwrap();
719
720 assert!(
721 patterns.excludes_input_path(Path::new("src"), &cwd),
722 "a directory pattern matches a directory given as input"
723 );
724 assert!(patterns.excludes_input_path(Path::new("Cargo.toml"), &cwd));
725 assert!(!patterns.excludes_input_path(Path::new("README.md"), &cwd));
726 }
727
728 fn patterns_from(contents: &str) -> IgnorePatterns {
729 let file = tempfile::NamedTempFile::new().unwrap();
730 std::fs::write(file.path(), contents).unwrap();
731 IgnorePatterns::from_files(&[file.path().to_owned()])
732 .unwrap()
733 .unwrap()
734 }
735
736 fn walk_with_patterns(root: &Path, contents: &str) -> Vec<PathBuf> {
737 let options = WalkOptions {
738 threads: 2,
739 count_hard_links: false,
740 apparent_size: false,
741 cross_filesystems: true,
742 ignore_dirs: BTreeSet::default(),
743 ignore_patterns: Some(patterns_from(contents)),
744 metadata_options: crate::TraversalOptions::default(),
745 };
746
747 let mut paths = options
748 .iter_from_paths(
749 vec![WalkRoot {
750 index: 0,
751 pattern_root: Some(root.to_owned()),
752 path: root.to_owned(),
753 #[cfg(any(windows, target_os = "macos"))]
754 entry: None,
755 device_id: crossdev::init(root).unwrap(),
756 }],
757 false,
758 walk::Order::Completion,
759 )
760 .filter_map(|(_, event)| match event {
761 walk::RootEvent::Entry(entry) => {
762 Some(entry.unwrap().path().strip_prefix(root).unwrap().to_owned())
763 }
764 walk::RootEvent::Finished => None,
765 })
766 .collect::<Vec<_>>();
767 paths.sort();
768 paths
769 }
770}