1#[cfg(any(windows, target_os = "macos"))]
2use crate::walk::walk_root_entries as walk_roots;
3#[cfg(not(any(windows, target_os = "macos")))]
4use crate::walk::walk_roots;
5use crate::{crossdev, walk};
6use anyhow::Context;
7use byte_unit::{Byte, Unit, UnitType};
8use serde::Deserialize;
9use std::collections::BTreeSet;
10use std::path::PathBuf;
11use std::sync::Arc;
12use std::sync::atomic::{AtomicBool, Ordering};
13use std::time::Duration;
14use std::{fmt, path::Path};
15
16#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)]
18pub enum ByteFormat {
19 #[serde(rename = "metric")]
21 Metric,
22 #[serde(rename = "binary")]
24 Binary,
25 #[serde(rename = "bytes")]
27 Bytes,
28 #[serde(rename = "gb")]
30 GB,
31 #[serde(rename = "gib")]
33 GiB,
34 #[serde(rename = "mb")]
36 MB,
37 #[serde(rename = "mib")]
39 MiB,
40}
41
42impl ByteFormat {
43 #[must_use]
45 pub fn width(self) -> usize {
46 use ByteFormat::{Binary, Bytes, MB, MiB};
47 match self {
48 Binary => 11,
49 Bytes | MB | MiB => 12,
50 _ => 10,
51 }
52 }
53 #[must_use]
55 pub fn total_width(self) -> usize {
56 use ByteFormat::{Binary, Bytes, GB, GiB, MB, Metric, MiB};
57 const THE_SPACE_BETWEEN_UNIT_AND_NUMBER: usize = 1;
58
59 self.width()
60 + match self {
61 Binary | MiB | GiB => 3,
62 Metric | MB | GB => 2,
63 Bytes => 1,
64 }
65 + THE_SPACE_BETWEEN_UNIT_AND_NUMBER
66 }
67 #[must_use]
69 pub fn display(self, bytes: u128) -> impl fmt::Display {
70 ByteFormatDisplay {
71 format: self,
72 bytes,
73 }
74 }
75}
76
77struct ByteFormatDisplay {
79 format: ByteFormat,
80 bytes: u128,
81}
82
83impl fmt::Display for ByteFormatDisplay {
84 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
85 use ByteFormat::{Binary, Bytes, GB, GiB, MB, Metric, MiB};
86
87 let bytes = Byte::from_u128(self.bytes).expect("supported byte count");
88 let adjusted = match self.format {
89 Bytes => return write!(f, "{} b", self.bytes),
90 Binary => bytes.get_appropriate_unit(UnitType::Binary),
91 Metric => bytes.get_appropriate_unit(UnitType::Decimal),
92 GB => bytes.get_adjusted_unit(Unit::GB),
93 GiB => bytes.get_adjusted_unit(Unit::GiB),
94 MB => bytes.get_adjusted_unit(Unit::MB),
95 MiB => bytes.get_adjusted_unit(Unit::MiB),
96 };
97 let b = format!("{adjusted:.2}");
98 let mut splits = b.split(' ');
99 match (splits.next(), splits.next()) {
100 (Some(bytes), Some(unit)) => write!(
101 f,
102 "{} {:>unit_width$}",
103 bytes,
104 unit,
105 unit_width = match self.format {
106 Binary => 3,
107 _ => 2,
108 }
109 ),
110 _ => f.write_str(&b),
111 }
112 }
113}
114
115#[derive(Debug)]
117pub(crate) struct Throttle {
118 trigger: Arc<AtomicBool>,
119}
120
121impl Throttle {
122 pub(crate) fn new(duration: Duration, initial_sleep: Option<Duration>) -> Self {
126 let instance = Self {
127 trigger: Arc::default(),
128 };
129
130 let trigger = Arc::downgrade(&instance.trigger);
131 std::thread::spawn(move || {
132 if let Some(duration) = initial_sleep {
133 std::thread::sleep(duration);
134 }
135 while let Some(t) = trigger.upgrade() {
136 t.store(true, Ordering::Relaxed);
137 std::thread::sleep(duration);
138 }
139 });
140
141 instance
142 }
143
144 pub(crate) fn throttled<F>(&self, f: F)
146 where
147 F: FnOnce(),
148 {
149 if self.can_update() {
150 f();
151 }
152 }
153
154 pub(crate) fn can_update(&self) -> bool {
156 self.trigger.swap(false, Ordering::Relaxed)
157 }
158}
159
160#[derive(Clone, Debug)]
170pub struct IgnorePatterns {
171 search: gix::ignore::Search,
172}
173
174impl IgnorePatterns {
175 pub fn from_files(files: &[PathBuf]) -> anyhow::Result<Option<Self>> {
182 let mut search = gix::ignore::Search::default();
183 for file in files {
184 let buf = std::fs::read(file).with_context(|| {
185 format!("Failed to read ignore patterns from {}", file.display())
186 })?;
187 search.add_patterns_buffer(
188 &buf,
189 file.clone(),
190 None,
191 gix::ignore::search::Ignore::default(),
192 );
193 }
194 let pattern_count = search
195 .patterns
196 .iter()
197 .map(|list| list.patterns.len())
198 .sum::<usize>();
199 Ok(if pattern_count != 0 {
200 log::info!(
201 "Loaded {pattern_count} ignore pattern(s) from {file_count} file(s)",
202 file_count = files.len()
203 );
204 Some(Self { search })
205 } else {
206 None
207 })
208 }
209
210 #[must_use]
217 pub fn is_excluded(&self, relative_path: &Path, is_dir: bool) -> bool {
218 if relative_path.as_os_str().is_empty() {
219 return false;
220 }
221 let relative_path =
222 gix::path::to_unix_separators_on_windows(gix::path::into_bstr(relative_path));
223 self.search
224 .pattern_matching_relative_path(
225 relative_path.as_ref(),
226 Some(is_dir),
227 gix::ignore::glob::pattern::Case::Sensitive,
228 )
229 .is_some_and(|match_| !match_.pattern.is_negative())
230 }
231
232 #[must_use]
238 pub fn excludes_input_path(&self, path: &Path, cwd: &Path) -> bool {
239 pattern_relative_path(path, cwd, path)
240 .is_some_and(|relative_path| self.is_excluded(relative_path, path.is_dir()))
241 }
242}
243
244#[derive(Clone)]
246pub struct WalkOptions {
247 pub threads: usize,
249 pub count_hard_links: bool,
251 pub apparent_size: bool,
253 pub cross_filesystems: bool,
255 pub ignore_dirs: BTreeSet<PathBuf>,
257 pub ignore_patterns: Option<IgnorePatterns>,
260}
261
262type ExcludeEntry = Arc<dyn Fn(usize, &walk::Entry) -> bool + Send + Sync>;
264
265pub(crate) struct WalkRoot {
267 pub index: usize,
269 pub path: PathBuf,
271 #[cfg(any(windows, target_os = "macos"))]
273 pub entry: Option<walk::Entry>,
274 pub pattern_root: Option<PathBuf>,
281 pub device_id: u64,
284}
285
286impl WalkOptions {
287 pub(crate) fn iter_from_paths(
288 &self,
289 roots: Vec<WalkRoot>,
290 skip_root: bool,
291 order: walk::Order,
292 ) -> impl Iterator<Item = (usize, walk::RootEvent)> + use<> {
293 let num_roots = roots
294 .iter()
295 .map(|root| root.index)
296 .max()
297 .map_or(0, |idx| idx + 1);
298 let path_count = roots.len();
299 let (device_ids, root_paths, indexed_roots) = roots.into_iter().fold(
300 (
301 vec![0; num_roots],
302 vec![None; num_roots],
303 Vec::with_capacity(path_count),
304 ),
305 |(mut device_ids, mut root_paths, mut indexed_roots), root| {
306 device_ids[root.index] = root.device_id;
307 root_paths[root.index] = root.pattern_root;
308 #[cfg(any(windows, target_os = "macos"))]
309 indexed_roots.push((
310 root.index,
311 root.entry
312 .map_or_else(|| walk::Entry::from_path(&root.path), Ok),
313 ));
314 #[cfg(not(any(windows, target_os = "macos")))]
315 indexed_roots.push((root.index, root.path));
316 (device_ids, root_paths, indexed_roots)
317 },
318 );
319 let ignore_dirs = self.ignore_dirs.clone();
320 let cwd = std::env::current_dir().unwrap_or_default();
321 let cross_filesystems = self.cross_filesystems;
322
323 let is_excluded: ExcludeEntry = {
327 let patterns = self.ignore_patterns.clone();
328 let cwd = cwd.clone();
329 Arc::new(move |root_idx: usize, entry: &walk::Entry| {
330 let Some((patterns, pattern_root)) =
331 patterns.as_ref().zip(root_paths[root_idx].as_deref())
332 else {
333 return false;
334 };
335 let path = entry.path();
336 pattern_relative_path(&path, &cwd, pattern_root).is_some_and(|relative_path| {
337 patterns.is_excluded(relative_path, entry.file_type.is_dir())
338 })
339 })
340 };
341 let is_excluded_while_walking = Arc::clone(&is_excluded);
342
343 walk_roots(
344 indexed_roots,
345 self.threads,
346 order,
347 move |root_idx, entry| {
348 (cross_filesystems
349 || entry.metadata.as_ref().map_or(true, |metadata| {
350 crossdev::is_same_device(device_ids[root_idx], metadata)
351 }))
352 && (entry.depth == 0 || !ignore_directory(&entry.path(), &ignore_dirs, &cwd))
353 && !is_excluded_while_walking(root_idx, entry)
354 },
355 )
356 .filter(move |(root_idx, event)| match event {
357 walk::RootEvent::Entry(Ok(entry)) => {
358 (!skip_root || entry.depth > 0) && !is_excluded(*root_idx, entry)
359 }
360 walk::RootEvent::Entry(Err(_)) | walk::RootEvent::Finished => true,
361 })
362 }
363}
364
365#[derive(Default)]
367pub struct WalkResult {
368 pub num_errors: u64,
370}
371
372impl WalkResult {
373 #[must_use]
377 pub fn to_exit_code(&self) -> i32 {
378 i32::from(self.num_errors > 0)
379 }
380}
381
382pub fn canonicalize_ignore_dirs(ignore_dirs: &[PathBuf]) -> BTreeSet<PathBuf> {
386 let dirs = ignore_dirs
387 .iter()
388 .map(gix::path::realpath)
389 .filter_map(Result::ok)
390 .collect();
391 log::info!("Ignoring canonicalized {dirs:?}");
392 dirs
393}
394
395fn pattern_relative_path<'a>(
402 path: &'a Path,
403 cwd: &Path,
404 traversal_root: &Path,
405) -> Option<&'a Path> {
406 if path.is_relative() {
407 return Some(path);
408 }
409 path.strip_prefix(cwd)
410 .or_else(|_| path.strip_prefix(traversal_root))
411 .ok()
412}
413
414fn ignore_directory(path: &Path, ignore_dirs: &BTreeSet<PathBuf>, cwd: &Path) -> bool {
415 if ignore_dirs.is_empty() {
416 return false;
417 }
418 let path = gix::path::realpath_opts(path, cwd, 32);
419 path.is_ok_and(|path| {
420 let ignored = ignore_dirs.contains(&path);
421 if ignored {
422 log::debug!("Ignored {}", path.display());
423 }
424 ignored
425 })
426}
427
428#[cfg(test)]
429mod tests {
430 use super::*;
431
432 #[test]
433 fn test_ignore_directories() {
434 let cwd = std::env::current_dir().unwrap();
435 #[cfg(unix)]
436 let mut parameters = vec![
437 ("/usr", vec!["/usr"], true),
438 ("/usr/local", vec!["/usr"], false),
439 ("/smth", vec!["/usr"], false),
440 ("/usr/local/..", vec!["/usr/local/.."], true),
441 ("/usr", vec!["/usr/local/.."], true),
442 ("/usr/local/share/../..", vec!["/usr"], true),
443 ];
444
445 #[cfg(windows)]
446 let mut parameters = vec![
447 ("C:\\Windows", vec!["C:\\Windows"], true),
448 ("C:\\Windows\\System", vec!["C:\\Windows"], false),
449 ("C:\\Smth", vec!["C:\\Windows"], false),
450 (
451 "C:\\Windows\\System\\..",
452 vec!["C:\\Windows\\System\\.."],
453 true,
454 ),
455 ("C:\\Windows", vec!["C:\\Windows\\System\\.."], true),
456 (
457 "C:\\Windows\\System\\Speech\\..\\..",
458 vec!["C:\\Windows"],
459 true,
460 ),
461 ];
462
463 parameters.extend([
464 ("src", vec!["src"], true),
465 ("src/interactive", vec!["src"], false),
466 ("src/interactive/..", vec!["src"], true),
467 ]);
468
469 for (path, ignore_dirs, expected_result) in parameters {
470 let ignore_dirs = canonicalize_ignore_dirs(
471 &ignore_dirs.into_iter().map(Into::into).collect::<Vec<_>>(),
472 );
473 assert_eq!(
474 ignore_directory(path.as_ref(), &ignore_dirs, &cwd),
475 expected_result,
476 "result='{expected_result}' for path='{path}' and ignore_dir='{ignore_dirs:?}' "
477 );
478 }
479 }
480
481 #[test]
482 fn explicitly_selected_ignored_root_is_traversed() {
483 let root = tempfile::tempdir().unwrap();
484 let child = root.path().join("child");
485 std::fs::create_dir(&child).unwrap();
486 let options = WalkOptions {
487 threads: 2,
488 count_hard_links: false,
489 apparent_size: false,
490 cross_filesystems: true,
491 ignore_dirs: canonicalize_ignore_dirs(&[root.path().to_owned()]),
492 ignore_patterns: None,
493 };
494
495 let paths = options
496 .iter_from_paths(
497 vec![WalkRoot {
498 index: 0,
499 pattern_root: None,
500 path: root.path().to_owned(),
501 #[cfg(any(windows, target_os = "macos"))]
502 entry: None,
503 device_id: crossdev::init(root.path()).unwrap(),
504 }],
505 false,
506 walk::Order::Completion,
507 )
508 .filter_map(|(_, event)| match event {
509 walk::RootEvent::Entry(entry) => Some(entry.unwrap().path()),
510 walk::RootEvent::Finished => None,
511 })
512 .collect::<Vec<_>>();
513
514 assert!(paths.contains(&child));
515 }
516
517 #[test]
518 fn ignore_patterns_use_gitignore_semantics() {
519 let patterns = patterns_from(
520 "# a comment, and the blank line below, match nothing\n\
521 \n\
522 *.log\n\
523 !keep.log\n\
524 build/\n\
525 /anchored\n\
526 **/node_modules/\n",
527 );
528
529 for (path, is_dir, expected) in [
530 ("debug.log", false, true),
531 ("nested/debug.log", false, true),
532 ("keep.log", false, false),
533 ("build", true, true),
534 ("build", false, false),
536 ("anchored", true, true),
537 ("nested/anchored", true, false),
538 ("nested/deeply/node_modules", true, true),
539 ("src", true, false),
540 ("", true, false),
541 ] {
542 assert_eq!(
543 patterns.is_excluded(Path::new(path), is_dir),
544 expected,
545 "expected is_excluded({path:?}, is_dir={is_dir}) to be {expected}"
546 );
547 }
548 }
549
550 #[test]
551 fn later_ignore_files_win_over_earlier_ones() {
552 let dir = tempfile::tempdir().unwrap();
553 let (first, second) = (dir.path().join("first"), dir.path().join("second"));
554 std::fs::write(&first, "*.tmp\n").unwrap();
555 std::fs::write(&second, "!important.tmp\n").unwrap();
556
557 let patterns = IgnorePatterns::from_files(&[first, second])
558 .unwrap()
559 .unwrap();
560
561 assert!(patterns.is_excluded(Path::new("scratch.tmp"), false));
562 assert!(
563 !patterns.is_excluded(Path::new("important.tmp"), false),
564 "the negation in the second file overrides the first file"
565 );
566 }
567
568 #[test]
569 fn unreadable_ignore_files_are_an_error() {
570 let err = IgnorePatterns::from_files(&[PathBuf::from("does-not-exist")])
571 .expect_err("a missing pattern file must not be silently skipped");
572 assert!(err.to_string().contains("does-not-exist"));
573 }
574
575 #[test]
576 fn empty_ignore_files_produce_none() {
577 let file = tempfile::NamedTempFile::new().unwrap();
578 std::fs::write(file.path(), "# comment only\n").unwrap();
579 assert!(
580 IgnorePatterns::from_files(&[file.path().to_owned()])
581 .unwrap()
582 .is_none(),
583 "no patterns means no need to match anything"
584 );
585 }
586
587 #[test]
588 fn matching_entries_are_pruned_from_the_walk() {
589 let root = tempfile::tempdir().unwrap();
590 for dir in ["keep", "build", "keep/node_modules"] {
591 std::fs::create_dir(root.path().join(dir)).unwrap();
592 }
593 for file in [
594 "keep/main.rs",
595 "keep/debug.log",
596 "build/artifact",
597 "keep/node_modules/dep",
598 ] {
599 std::fs::write(root.path().join(file), b"x").unwrap();
600 }
601
602 assert_eq!(
603 walk_with_patterns(root.path(), "*.log\n**/node_modules/\n"),
604 [
605 PathBuf::new(),
606 PathBuf::from("build"),
607 PathBuf::from("build/artifact"),
608 PathBuf::from("keep"),
609 PathBuf::from("keep/main.rs"),
610 ],
611 "excluded directories are pruned along with everything below them, \
612 and excluded files never show up"
613 );
614 }
615
616 #[test]
617 fn patterns_match_the_path_dua_reports() {
618 let here = tempfile::tempdir().unwrap();
619 let elsewhere = tempfile::tempdir().unwrap();
620 let (cwd, outside) = (here.path(), elsewhere.path());
621
622 for dir in ["target", "nested", "nested/target"] {
623 std::fs::create_dir_all(here.path().join(dir)).unwrap();
624 }
625 assert_eq!(
626 walk_with_patterns(here.path(), "/target/\n"),
627 [
628 PathBuf::new(),
629 PathBuf::from("nested"),
630 PathBuf::from("nested/target"),
631 ],
632 "an anchored pattern excludes only the top-level target"
633 );
634
635 let (entry, root) = (cwd.join("a").join("b"), cwd.join("a"));
637 let reported = Path::new("a").join("b");
638 assert_eq!(
639 pattern_relative_path(&entry, cwd, &root),
640 Some(reported.as_path())
641 );
642
643 let entry = outside.join("syslog");
646 assert_eq!(
647 pattern_relative_path(&entry, cwd, outside),
648 Some(Path::new("syslog"))
649 );
650
651 assert_eq!(
653 pattern_relative_path(outside, cwd, outside),
654 Some(Path::new(""))
655 );
656 assert!(!patterns_from("*\n").is_excluded(Path::new(""), true));
657 }
658
659 #[test]
660 fn subtree_walk_keeps_the_original_pattern_root() {
661 let root = tempfile::tempdir().unwrap();
662 let nested = root.path().join("nested");
663 std::fs::create_dir(&nested).unwrap();
664 std::fs::write(nested.join("secret"), []).unwrap();
665 std::fs::write(nested.join("visible"), []).unwrap();
666 let options = WalkOptions {
667 threads: 1,
668 count_hard_links: false,
669 apparent_size: false,
670 cross_filesystems: true,
671 ignore_dirs: BTreeSet::default(),
672 ignore_patterns: Some(patterns_from("nested/secret\n")),
673 };
674
675 let paths = options
676 .iter_from_paths(
677 vec![WalkRoot {
678 index: 0,
679 path: nested,
680 #[cfg(any(windows, target_os = "macos"))]
681 entry: None,
682 pattern_root: Some(root.path().to_owned()),
683 device_id: 0,
684 }],
685 false,
686 walk::Order::Completion,
687 )
688 .filter_map(|(_, event)| match event {
689 walk::RootEvent::Entry(entry) => Some(entry.unwrap().file_name),
690 walk::RootEvent::Finished => None,
691 })
692 .collect::<Vec<_>>();
693
694 assert!(!paths.iter().any(|path| path == "secret"));
695 assert!(paths.iter().any(|path| path == "visible"));
696 }
697
698 #[test]
699 fn excluded_input_paths_are_dropped_before_the_walk() {
700 let patterns = patterns_from("src/\n*.toml\n");
702 let cwd = std::env::current_dir().unwrap();
703
704 assert!(
705 patterns.excludes_input_path(Path::new("src"), &cwd),
706 "a directory pattern matches a directory given as input"
707 );
708 assert!(patterns.excludes_input_path(Path::new("Cargo.toml"), &cwd));
709 assert!(!patterns.excludes_input_path(Path::new("README.md"), &cwd));
710 }
711
712 fn patterns_from(contents: &str) -> IgnorePatterns {
713 let file = tempfile::NamedTempFile::new().unwrap();
714 std::fs::write(file.path(), contents).unwrap();
715 IgnorePatterns::from_files(&[file.path().to_owned()])
716 .unwrap()
717 .unwrap()
718 }
719
720 fn walk_with_patterns(root: &Path, contents: &str) -> Vec<PathBuf> {
721 let options = WalkOptions {
722 threads: 2,
723 count_hard_links: false,
724 apparent_size: false,
725 cross_filesystems: true,
726 ignore_dirs: BTreeSet::default(),
727 ignore_patterns: Some(patterns_from(contents)),
728 };
729
730 let mut paths = options
731 .iter_from_paths(
732 vec![WalkRoot {
733 index: 0,
734 pattern_root: Some(root.to_owned()),
735 path: root.to_owned(),
736 #[cfg(any(windows, target_os = "macos"))]
737 entry: None,
738 device_id: crossdev::init(root).unwrap(),
739 }],
740 false,
741 walk::Order::Completion,
742 )
743 .filter_map(|(_, event)| match event {
744 walk::RootEvent::Entry(entry) => {
745 Some(entry.unwrap().path().strip_prefix(root).unwrap().to_owned())
746 }
747 walk::RootEvent::Finished => None,
748 })
749 .collect::<Vec<_>>();
750 paths.sort();
751 paths
752 }
753}