1#[cfg(any(windows, target_os = "macos"))]
2use crate::walk::walk_root_entries as walk_roots;
3#[cfg(not(any(windows, target_os = "macos")))]
4use crate::walk::walk_roots;
5use crate::{crossdev, walk};
6use anyhow::Context;
7use byte_unit::{Byte, Unit, UnitType};
8use serde::Deserialize;
9use std::collections::BTreeSet;
10use std::path::PathBuf;
11use std::sync::Arc;
12use std::sync::atomic::{AtomicBool, Ordering};
13use std::time::Duration;
14use std::{fmt, path::Path};
15
16#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)]
18pub enum ByteFormat {
19 #[serde(rename = "metric")]
21 Metric,
22 #[serde(rename = "binary")]
24 Binary,
25 #[serde(rename = "bytes")]
27 Bytes,
28 #[serde(rename = "gb")]
30 GB,
31 #[serde(rename = "gib")]
33 GiB,
34 #[serde(rename = "mb")]
36 MB,
37 #[serde(rename = "mib")]
39 MiB,
40}
41
42impl ByteFormat {
43 #[must_use]
45 pub fn width(self) -> usize {
46 use ByteFormat::{Binary, Bytes, MB, MiB};
47 match self {
48 Binary => 11,
49 Bytes | MB | MiB => 12,
50 _ => 10,
51 }
52 }
53 #[must_use]
55 pub fn total_width(self) -> usize {
56 use ByteFormat::{Binary, Bytes, GB, GiB, MB, Metric, MiB};
57 const THE_SPACE_BETWEEN_UNIT_AND_NUMBER: usize = 1;
58
59 self.width()
60 + match self {
61 Binary | MiB | GiB => 3,
62 Metric | MB | GB => 2,
63 Bytes => 1,
64 }
65 + THE_SPACE_BETWEEN_UNIT_AND_NUMBER
66 }
67 #[must_use]
69 pub fn display(self, bytes: u128) -> impl fmt::Display {
70 ByteFormatDisplay {
71 format: self,
72 bytes,
73 }
74 }
75}
76
77struct ByteFormatDisplay {
79 format: ByteFormat,
80 bytes: u128,
81}
82
83impl fmt::Display for ByteFormatDisplay {
84 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
85 use ByteFormat::{Binary, Bytes, GB, GiB, MB, Metric, MiB};
86
87 if self.format == Bytes {
88 return write!(f, "{} b", self.bytes);
89 }
90 let Some(bytes) = Byte::from_u128(self.bytes) else {
91 return write!(f, "{} b", self.bytes);
92 };
93 let adjusted = match self.format {
94 Bytes => unreachable!("handled above"),
95 Binary => bytes.get_appropriate_unit(UnitType::Binary),
96 Metric => bytes.get_appropriate_unit(UnitType::Decimal),
97 GB => bytes.get_adjusted_unit(Unit::GB),
98 GiB => bytes.get_adjusted_unit(Unit::GiB),
99 MB => bytes.get_adjusted_unit(Unit::MB),
100 MiB => bytes.get_adjusted_unit(Unit::MiB),
101 };
102 let b = format!("{adjusted:.2}");
103 let mut splits = b.split(' ');
104 match (splits.next(), splits.next()) {
105 (Some(bytes), Some(unit)) => write!(
106 f,
107 "{} {:>unit_width$}",
108 bytes,
109 unit,
110 unit_width = match self.format {
111 Binary => 3,
112 _ => 2,
113 }
114 ),
115 _ => f.write_str(&b),
116 }
117 }
118}
119
120#[derive(Debug)]
122pub(crate) struct Throttle {
123 trigger: Arc<AtomicBool>,
124}
125
126impl Throttle {
127 pub(crate) fn new(duration: Duration, initial_sleep: Option<Duration>) -> Self {
131 let instance = Self {
132 trigger: Arc::default(),
133 };
134
135 let trigger = Arc::downgrade(&instance.trigger);
136 std::thread::spawn(move || {
137 if let Some(duration) = initial_sleep {
138 std::thread::sleep(duration);
139 }
140 while let Some(t) = trigger.upgrade() {
141 t.store(true, Ordering::Relaxed);
142 std::thread::sleep(duration);
143 }
144 });
145
146 instance
147 }
148
149 pub(crate) fn can_update(&self) -> bool {
151 self.trigger.swap(false, Ordering::Relaxed)
152 }
153}
154
155#[derive(Clone, Debug)]
165pub struct IgnorePatterns {
166 search: gix::ignore::Search,
167}
168
169impl IgnorePatterns {
170 pub fn from_files(files: &[PathBuf]) -> anyhow::Result<Option<Self>> {
177 let mut search = gix::ignore::Search::default();
178 for file in files {
179 let buf = std::fs::read(file).with_context(|| {
180 format!("Failed to read ignore patterns from {}", file.display())
181 })?;
182 search.add_patterns_buffer(
183 &buf,
184 file.clone(),
185 None,
186 gix::ignore::search::Ignore::default(),
187 );
188 }
189 let pattern_count = search
190 .patterns
191 .iter()
192 .map(|list| list.patterns.len())
193 .sum::<usize>();
194 Ok(if pattern_count != 0 {
195 log::info!(
196 "Loaded {pattern_count} ignore pattern(s) from {file_count} file(s)",
197 file_count = files.len()
198 );
199 Some(Self { search })
200 } else {
201 None
202 })
203 }
204
205 #[must_use]
212 pub fn is_excluded(&self, relative_path: &Path, is_dir: bool) -> bool {
213 if relative_path.as_os_str().is_empty() {
214 return false;
215 }
216 let relative_path =
217 gix::path::to_unix_separators_on_windows(gix::path::into_bstr(relative_path));
218 self.search
219 .pattern_matching_relative_path(
220 relative_path.as_ref(),
221 Some(is_dir),
222 gix::ignore::glob::pattern::Case::Sensitive,
223 )
224 .is_some_and(|match_| !match_.pattern.is_negative())
225 }
226
227 #[must_use]
233 pub fn excludes_input_path(&self, path: &Path, cwd: &Path) -> bool {
234 pattern_relative_path(path, cwd, path)
235 .is_some_and(|relative_path| self.is_excluded(relative_path, path.is_dir()))
236 }
237}
238
239#[derive(Clone)]
241pub struct WalkOptions {
242 pub threads: usize,
244 pub count_hard_links: bool,
246 pub apparent_size: bool,
248 pub cross_filesystems: bool,
250 pub ignore_dirs: BTreeSet<PathBuf>,
252 pub ignore_patterns: Option<IgnorePatterns>,
255 pub metadata_options: crate::TraversalOptions,
257}
258
259type ExcludeEntry = Arc<dyn Fn(usize, &walk::Entry) -> bool + Send + Sync>;
261
262pub(crate) struct WalkRoot {
264 pub index: usize,
266 pub path: PathBuf,
268 #[cfg(any(windows, target_os = "macos"))]
270 pub entry: Option<walk::Entry>,
271 pub pattern_root: Option<PathBuf>,
278 pub device_id: u64,
281}
282
283impl WalkOptions {
284 #[must_use]
286 pub fn is_ignored_directory(&self, path: &Path, cwd: &Path) -> bool {
287 ignore_directory(path, &self.ignore_dirs, cwd)
288 }
289
290 pub(crate) fn iter_from_paths(
291 &self,
292 roots: Vec<WalkRoot>,
293 skip_root: bool,
294 order: walk::Order,
295 ) -> impl Iterator<Item = (usize, walk::RootEvent)> + use<> {
296 let num_roots = roots
297 .iter()
298 .map(|root| root.index)
299 .max()
300 .map_or(0, |idx| idx + 1);
301 let path_count = roots.len();
302 let (device_ids, root_paths, indexed_roots) = roots.into_iter().fold(
303 (
304 vec![0; num_roots],
305 vec![None; num_roots],
306 Vec::with_capacity(path_count),
307 ),
308 |(mut device_ids, mut root_paths, mut indexed_roots), root| {
309 device_ids[root.index] = root.device_id;
310 root_paths[root.index] = root.pattern_root;
311 #[cfg(any(windows, target_os = "macos"))]
312 indexed_roots.push((
313 root.index,
314 root.entry.map_or_else(
315 || walk::Entry::from_path(&root.path, self.metadata_options),
316 Ok,
317 ),
318 ));
319 #[cfg(not(any(windows, target_os = "macos")))]
320 indexed_roots.push((root.index, root.path));
321 (device_ids, root_paths, indexed_roots)
322 },
323 );
324 let ignore_dirs = self.ignore_dirs.clone();
325 let cwd = std::env::current_dir().unwrap_or_default();
326 let cross_filesystems = self.cross_filesystems;
327
328 let is_excluded: ExcludeEntry = {
332 let patterns = self.ignore_patterns.clone();
333 let cwd = cwd.clone();
334 Arc::new(move |root_idx: usize, entry: &walk::Entry| {
335 let Some((patterns, pattern_root)) =
336 patterns.as_ref().zip(root_paths[root_idx].as_deref())
337 else {
338 return false;
339 };
340 let path = entry.path();
341 pattern_relative_path(&path, &cwd, pattern_root).is_some_and(|relative_path| {
342 patterns.is_excluded(relative_path, entry.file_type.is_dir())
343 })
344 })
345 };
346 let is_excluded_while_walking = Arc::clone(&is_excluded);
347 let descend = move |root_idx: usize, entry: &walk::Entry| {
348 (cross_filesystems
349 || entry.metadata.as_ref().map_or(true, |metadata| {
350 crossdev::is_same_device(device_ids[root_idx], metadata)
351 }))
352 && (entry.depth == 0 || !ignore_directory(&entry.path(), &ignore_dirs, &cwd))
353 && !is_excluded_while_walking(root_idx, entry)
354 };
355
356 let walk = walk_roots(
357 indexed_roots,
358 self.threads,
359 order,
360 self.metadata_options,
361 descend,
362 );
363
364 walk.filter(move |(root_idx, event)| match event {
365 walk::RootEvent::Entry(Ok(entry)) => {
366 (!skip_root || entry.depth > 0) && !is_excluded(*root_idx, entry)
367 }
368 walk::RootEvent::Entry(Err(_)) | walk::RootEvent::Finished => true,
369 })
370 }
371}
372
373#[derive(Default)]
375pub struct WalkResult {
376 pub num_errors: u64,
378}
379
380impl WalkResult {
381 #[must_use]
385 pub fn to_exit_code(&self) -> i32 {
386 i32::from(self.num_errors > 0)
387 }
388}
389
390pub fn canonicalize_ignore_dirs(ignore_dirs: &[PathBuf]) -> BTreeSet<PathBuf> {
394 let dirs = ignore_dirs
395 .iter()
396 .map(gix::path::realpath)
397 .filter_map(Result::ok)
398 .collect();
399 log::info!("Ignoring canonicalized {dirs:?}");
400 dirs
401}
402
403fn pattern_relative_path<'a>(
410 path: &'a Path,
411 cwd: &Path,
412 traversal_root: &Path,
413) -> Option<&'a Path> {
414 if path.is_relative() {
415 return Some(path);
416 }
417 path.strip_prefix(cwd)
418 .or_else(|_| path.strip_prefix(traversal_root))
419 .ok()
420}
421
422fn ignore_directory(path: &Path, ignore_dirs: &BTreeSet<PathBuf>, cwd: &Path) -> bool {
423 if ignore_dirs.is_empty() {
424 return false;
425 }
426 let path = gix::path::realpath_opts(path, cwd, 32);
427 path.is_ok_and(|path| {
428 let ignored = ignore_dirs.contains(&path);
429 if ignored {
430 log::debug!("Ignored {}", path.display());
431 }
432 ignored
433 })
434}
435
436#[cfg(test)]
437mod tests {
438 use super::*;
439
440 #[test]
441 fn every_byte_format_handles_the_maximum_snapshot_size() {
442 for format in [
443 ByteFormat::Metric,
444 ByteFormat::Binary,
445 ByteFormat::Bytes,
446 ByteFormat::GB,
447 ByteFormat::GiB,
448 ByteFormat::MB,
449 ByteFormat::MiB,
450 ] {
451 assert_eq!(
452 format.display(u128::MAX).to_string(),
453 format!("{} b", u128::MAX)
454 );
455 }
456 }
457
458 #[test]
459 fn walk_options_identify_ignored_directories() {
460 let cwd = std::env::current_dir().unwrap();
461 let mut options = WalkOptions {
462 threads: 1,
463 count_hard_links: false,
464 apparent_size: false,
465 cross_filesystems: true,
466 ignore_dirs: BTreeSet::new(),
467 ignore_patterns: None,
468 metadata_options: crate::TraversalOptions::default(),
469 };
470 #[cfg(unix)]
471 let mut parameters = vec![
472 ("/usr", vec!["/usr"], true),
473 ("/usr/local", vec!["/usr"], false),
474 ("/smth", vec!["/usr"], false),
475 ("/usr/local/..", vec!["/usr/local/.."], true),
476 ("/usr", vec!["/usr/local/.."], true),
477 ("/usr/local/share/../..", vec!["/usr"], true),
478 ];
479
480 #[cfg(windows)]
481 let mut parameters = vec![
482 ("C:\\Windows", vec!["C:\\Windows"], true),
483 ("C:\\Windows\\System", vec!["C:\\Windows"], false),
484 ("C:\\Smth", vec!["C:\\Windows"], false),
485 (
486 "C:\\Windows\\System\\..",
487 vec!["C:\\Windows\\System\\.."],
488 true,
489 ),
490 ("C:\\Windows", vec!["C:\\Windows\\System\\.."], true),
491 (
492 "C:\\Windows\\System\\Speech\\..\\..",
493 vec!["C:\\Windows"],
494 true,
495 ),
496 ];
497
498 parameters.extend([
499 ("src", vec![], false),
500 ("src", vec!["src"], true),
501 ("src/interactive", vec!["src"], false),
502 ("src/interactive/..", vec!["src"], true),
503 ]);
504
505 for (path, ignore_dirs, expected_result) in parameters {
506 options.ignore_dirs = canonicalize_ignore_dirs(
507 &ignore_dirs.into_iter().map(Into::into).collect::<Vec<_>>(),
508 );
509 assert_eq!(
510 options.is_ignored_directory(path.as_ref(), &cwd),
511 expected_result,
512 "result='{expected_result}' for path='{path}' and ignore_dir='{:?}' ",
513 options.ignore_dirs
514 );
515 }
516 }
517
518 #[test]
519 fn explicitly_selected_ignored_root_is_traversed() {
520 let root = tempfile::tempdir().unwrap();
521 let child = root.path().join("child");
522 std::fs::create_dir(&child).unwrap();
523 let options = WalkOptions {
524 threads: 2,
525 count_hard_links: false,
526 apparent_size: false,
527 cross_filesystems: true,
528 ignore_dirs: canonicalize_ignore_dirs(&[root.path().to_owned()]),
529 ignore_patterns: None,
530 metadata_options: crate::TraversalOptions::default(),
531 };
532
533 let paths = options
534 .iter_from_paths(
535 vec![WalkRoot {
536 index: 0,
537 pattern_root: None,
538 path: root.path().to_owned(),
539 #[cfg(any(windows, target_os = "macos"))]
540 entry: None,
541 device_id: crossdev::init(root.path()).unwrap(),
542 }],
543 false,
544 walk::Order::Completion,
545 )
546 .filter_map(|(_, event)| match event {
547 walk::RootEvent::Entry(entry) => Some(entry.unwrap().path()),
548 walk::RootEvent::Finished => None,
549 })
550 .collect::<Vec<_>>();
551
552 assert!(paths.contains(&child));
553 }
554
555 #[test]
556 fn ignore_patterns_use_gitignore_semantics() {
557 let patterns = patterns_from(
558 "# a comment, and the blank line below, match nothing\n\
559 \n\
560 *.log\n\
561 !keep.log\n\
562 build/\n\
563 /anchored\n\
564 **/node_modules/\n",
565 );
566
567 for (path, is_dir, expected) in [
568 ("debug.log", false, true),
569 ("nested/debug.log", false, true),
570 ("keep.log", false, false),
571 ("build", true, true),
572 ("build", false, false),
574 ("anchored", true, true),
575 ("nested/anchored", true, false),
576 ("nested/deeply/node_modules", true, true),
577 ("src", true, false),
578 ("", true, false),
579 ] {
580 assert_eq!(
581 patterns.is_excluded(Path::new(path), is_dir),
582 expected,
583 "expected is_excluded({path:?}, is_dir={is_dir}) to be {expected}"
584 );
585 }
586 }
587
588 #[test]
589 fn later_ignore_files_win_over_earlier_ones() {
590 let dir = tempfile::tempdir().unwrap();
591 let (first, second) = (dir.path().join("first"), dir.path().join("second"));
592 std::fs::write(&first, "*.tmp\n").unwrap();
593 std::fs::write(&second, "!important.tmp\n").unwrap();
594
595 let patterns = IgnorePatterns::from_files(&[first, second])
596 .unwrap()
597 .unwrap();
598
599 assert!(patterns.is_excluded(Path::new("scratch.tmp"), false));
600 assert!(
601 !patterns.is_excluded(Path::new("important.tmp"), false),
602 "the negation in the second file overrides the first file"
603 );
604 }
605
606 #[test]
607 fn unreadable_ignore_files_are_an_error() {
608 let err = IgnorePatterns::from_files(&[PathBuf::from("does-not-exist")])
609 .expect_err("a missing pattern file must not be silently skipped");
610 assert!(err.to_string().contains("does-not-exist"));
611 }
612
613 #[test]
614 fn empty_ignore_files_produce_none() {
615 let file = tempfile::NamedTempFile::new().unwrap();
616 std::fs::write(file.path(), "# comment only\n").unwrap();
617 assert!(
618 IgnorePatterns::from_files(&[file.path().to_owned()])
619 .unwrap()
620 .is_none(),
621 "no patterns means no need to match anything"
622 );
623 }
624
625 #[test]
626 fn matching_entries_are_pruned_from_the_walk() {
627 let root = tempfile::tempdir().unwrap();
628 for dir in ["keep", "build", "keep/node_modules"] {
629 std::fs::create_dir(root.path().join(dir)).unwrap();
630 }
631 for file in [
632 "keep/main.rs",
633 "keep/debug.log",
634 "build/artifact",
635 "keep/node_modules/dep",
636 ] {
637 std::fs::write(root.path().join(file), b"x").unwrap();
638 }
639
640 assert_eq!(
641 walk_with_patterns(root.path(), "*.log\n**/node_modules/\n"),
642 [
643 PathBuf::new(),
644 PathBuf::from("build"),
645 PathBuf::from("build/artifact"),
646 PathBuf::from("keep"),
647 PathBuf::from("keep/main.rs"),
648 ],
649 "excluded directories are pruned along with everything below them, \
650 and excluded files never show up"
651 );
652 }
653
654 #[test]
655 fn patterns_match_the_path_dua_reports() {
656 let here = tempfile::tempdir().unwrap();
657 let elsewhere = tempfile::tempdir().unwrap();
658 let (cwd, outside) = (here.path(), elsewhere.path());
659
660 for dir in ["target", "nested", "nested/target"] {
661 std::fs::create_dir_all(here.path().join(dir)).unwrap();
662 }
663 assert_eq!(
664 walk_with_patterns(here.path(), "/target/\n"),
665 [
666 PathBuf::new(),
667 PathBuf::from("nested"),
668 PathBuf::from("nested/target"),
669 ],
670 "an anchored pattern excludes only the top-level target"
671 );
672
673 let (entry, root) = (cwd.join("a").join("b"), cwd.join("a"));
675 let reported = Path::new("a").join("b");
676 assert_eq!(
677 pattern_relative_path(&entry, cwd, &root),
678 Some(reported.as_path())
679 );
680
681 let entry = outside.join("syslog");
684 assert_eq!(
685 pattern_relative_path(&entry, cwd, outside),
686 Some(Path::new("syslog"))
687 );
688
689 assert_eq!(
691 pattern_relative_path(outside, cwd, outside),
692 Some(Path::new(""))
693 );
694 assert!(!patterns_from("*\n").is_excluded(Path::new(""), true));
695 }
696
697 #[test]
698 fn subtree_walk_keeps_the_original_pattern_root() {
699 let root = tempfile::tempdir().unwrap();
700 let nested = root.path().join("nested");
701 std::fs::create_dir(&nested).unwrap();
702 std::fs::write(nested.join("secret"), []).unwrap();
703 std::fs::write(nested.join("visible"), []).unwrap();
704 let options = WalkOptions {
705 threads: 1,
706 count_hard_links: false,
707 apparent_size: false,
708 cross_filesystems: true,
709 ignore_dirs: BTreeSet::default(),
710 ignore_patterns: Some(patterns_from("nested/secret\n")),
711 metadata_options: crate::TraversalOptions::default(),
712 };
713
714 let paths = options
715 .iter_from_paths(
716 vec![WalkRoot {
717 index: 0,
718 path: nested,
719 #[cfg(any(windows, target_os = "macos"))]
720 entry: None,
721 pattern_root: Some(root.path().to_owned()),
722 device_id: 0,
723 }],
724 false,
725 walk::Order::Completion,
726 )
727 .filter_map(|(_, event)| match event {
728 walk::RootEvent::Entry(entry) => Some(entry.unwrap().file_name),
729 walk::RootEvent::Finished => None,
730 })
731 .collect::<Vec<_>>();
732
733 assert!(!paths.iter().any(|path| path == "secret"));
734 assert!(paths.iter().any(|path| path == "visible"));
735 }
736
737 #[test]
738 fn excluded_input_paths_are_dropped_before_the_walk() {
739 let patterns = patterns_from("src/\n*.toml\n");
741 let cwd = std::env::current_dir().unwrap();
742
743 assert!(
744 patterns.excludes_input_path(Path::new("src"), &cwd),
745 "a directory pattern matches a directory given as input"
746 );
747 assert!(patterns.excludes_input_path(Path::new("Cargo.toml"), &cwd));
748 assert!(!patterns.excludes_input_path(Path::new("README.md"), &cwd));
749 }
750
751 fn patterns_from(contents: &str) -> IgnorePatterns {
752 let file = tempfile::NamedTempFile::new().unwrap();
753 std::fs::write(file.path(), contents).unwrap();
754 IgnorePatterns::from_files(&[file.path().to_owned()])
755 .unwrap()
756 .unwrap()
757 }
758
759 fn walk_with_patterns(root: &Path, contents: &str) -> Vec<PathBuf> {
760 let options = WalkOptions {
761 threads: 2,
762 count_hard_links: false,
763 apparent_size: false,
764 cross_filesystems: true,
765 ignore_dirs: BTreeSet::default(),
766 ignore_patterns: Some(patterns_from(contents)),
767 metadata_options: crate::TraversalOptions::default(),
768 };
769
770 let mut paths = options
771 .iter_from_paths(
772 vec![WalkRoot {
773 index: 0,
774 pattern_root: Some(root.to_owned()),
775 path: root.to_owned(),
776 #[cfg(any(windows, target_os = "macos"))]
777 entry: None,
778 device_id: crossdev::init(root).unwrap(),
779 }],
780 false,
781 walk::Order::Completion,
782 )
783 .filter_map(|(_, event)| match event {
784 walk::RootEvent::Entry(entry) => {
785 Some(entry.unwrap().path().strip_prefix(root).unwrap().to_owned())
786 }
787 walk::RootEvent::Finished => None,
788 })
789 .collect::<Vec<_>>();
790 paths.sort();
791 paths
792 }
793}