1use crate::{Throttle, WalkOptions, WalkRoot, crossdev, inodefilter::InodeFilter};
2
3use crossbeam::channel::Receiver;
4#[cfg(not(any(windows, target_os = "macos")))]
5use filesize::PathExt;
6use petgraph::{Directed, Direction, graph::NodeIndex, stable_graph::StableGraph};
7use std::time::Instant;
8use std::{
9 collections::HashMap,
10 fmt, io,
11 path::{Path, PathBuf},
12 sync::Arc,
13 time::{Duration, SystemTime, UNIX_EPOCH},
14};
15
16pub type TreeIndex = NodeIndex;
18pub type Tree = StableGraph<EntryData, (), Directed>;
20
21#[derive(Eq, PartialEq, Clone)]
23pub struct EntryData {
24 pub name: PathBuf,
26 pub size: u128,
29 pub mtime: SystemTime,
31 pub entry_count: Option<u64>,
33 pub metadata_io_error: bool,
35 pub is_dir: bool,
37}
38
39impl Default for EntryData {
40 fn default() -> EntryData {
41 EntryData {
42 name: PathBuf::default(),
43 size: u128::default(),
44 mtime: UNIX_EPOCH,
45 entry_count: None,
46 metadata_io_error: bool::default(),
47 is_dir: false,
48 }
49 }
50}
51
52impl fmt::Debug for EntryData {
53 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
54 f.debug_struct("EntryData")
55 .field("name", &self.name)
56 .field("size", &self.size)
57 .field("entry_count", &self.entry_count)
58 .field("metadata_io_error", &self.metadata_io_error)
60 .finish()
61 }
62}
63
64#[derive(Debug)]
66pub struct Traversal {
67 pub tree: Tree,
69 pub root_index: TreeIndex,
71 pub start_time: Instant,
73 pub cost: Option<Duration>,
75}
76
77impl Default for Traversal {
78 fn default() -> Self {
79 Self::new()
80 }
81}
82
83impl Traversal {
84 #[must_use]
86 pub fn new() -> Self {
87 let mut tree = Tree::new();
88 let root_index = tree.add_node(EntryData::default());
89 Self {
90 tree,
91 root_index,
92 start_time: Instant::now(),
93 cost: None,
94 }
95 }
96
97 #[must_use]
99 pub fn is_costly(&self) -> bool {
100 self.cost.is_none_or(|d| d.as_secs_f32() > 10.0)
101 }
102}
103
104#[derive(Clone, Copy)]
106pub struct TraversalStats {
107 pub entries_traversed: u64,
109 pub start: std::time::Instant,
111 pub elapsed: Option<std::time::Duration>,
113 pub io_errors: u64,
115 pub total_bytes: Option<u128>,
117}
118
119impl Default for TraversalStats {
120 fn default() -> Self {
121 Self {
122 entries_traversed: 0,
123 start: std::time::Instant::now(),
124 elapsed: None,
125 io_errors: 0,
126 total_bytes: None,
127 }
128 }
129}
130
131pub struct TraversalEntry(crate::walk::Entry);
133
134pub enum TraversalEvent {
136 Entry(io::Result<TraversalEntry>, Arc<PathBuf>, u64, usize),
144 RootError(Arc<PathBuf>, usize),
146 Finished,
148}
149
150pub struct BackgroundTraversal {
152 walk_options: WalkOptions,
153 pub root_idx: TreeIndex,
155 pub stats: TraversalStats,
157 pub(crate) root_nodes: Vec<Option<TreeIndex>>,
159 nodes_by_path: HashMap<(usize, PathBuf), TreeIndex>,
161 inodes: InodeFilter,
162 throttle: Option<Throttle>,
163 skip_root: bool,
164 use_root_path: bool,
165 retained_depth: Option<usize>,
166 pub event_rx: Receiver<TraversalEvent>,
168}
169
170impl BackgroundTraversal {
171 pub fn start(
174 root_idx: TreeIndex,
175 walk_options: &WalkOptions,
176 input: Vec<PathBuf>,
177 pattern_roots: Option<&[PathBuf]>,
178 skip_root: bool,
179 use_root_path: bool,
180 ) -> anyhow::Result<BackgroundTraversal> {
181 let num_roots = input.len();
182 let (entry_tx, entry_rx) = crossbeam::channel::bounded(100);
183 let pattern_roots = pattern_roots.map(<[PathBuf]>::to_owned);
184 std::thread::Builder::new()
185 .name("dua-fs-walk-dispatcher".to_string())
186 .spawn({
187 let walk_options = walk_options.clone();
188 move || {
189 let (mut root_paths, mut root_indices, mut device_ids, mut walk_roots) = (
190 Vec::with_capacity(input.len()),
191 Vec::with_capacity(input.len()),
192 Vec::with_capacity(input.len()),
193 Vec::with_capacity(input.len()),
194 );
195 for (root_idx, root_path) in input.into_iter().enumerate() {
196 log::info!("Walking {}", root_path.display());
197 let device_id = if walk_options.cross_filesystems {
198 0
199 } else {
200 let Ok(device_id) = crossdev::init(&root_path) else {
201 if entry_tx
202 .send(TraversalEvent::RootError(Arc::new(root_path), root_idx))
203 .is_err()
204 {
205 return;
206 }
207 continue;
208 };
209 device_id
210 };
211 let pattern_root = pattern_roots.as_deref().map(|pattern_roots| {
212 pattern_roots
213 .iter()
214 .filter(|candidate| root_path.starts_with(candidate))
215 .max_by_key(|candidate| candidate.components().count())
216 .cloned()
217 .unwrap_or_else(|| root_path.clone())
218 });
219 walk_roots.push(WalkRoot {
220 index: walk_roots.len(),
221 pattern_root,
222 path: root_path.clone(),
223 #[cfg(any(windows, target_os = "macos"))]
224 entry: None,
225 device_id,
226 });
227 root_indices.push(root_idx);
228 device_ids.push(device_id);
229 root_paths.push(Arc::new(root_path));
230 }
231
232 for (root, event) in walk_options.iter_from_paths(
233 walk_roots,
234 skip_root,
235 crate::walk::Order::ParentFirst,
236 ) {
237 let crate::walk::RootEvent::Entry(entry) = event else {
238 continue;
239 };
240 if entry_tx
241 .send(TraversalEvent::Entry(
242 entry.map(TraversalEntry),
243 Arc::clone(&root_paths[root]),
244 device_ids[root],
245 root_indices[root],
246 ))
247 .is_err()
248 {
249 return;
252 }
253 }
254 if entry_tx.send(TraversalEvent::Finished).is_err() {
255 log::error!("Failed to send TraversalEvents::Finished event");
256 }
257 }
258 })?;
259
260 Ok(Self {
261 walk_options: walk_options.clone(),
262 root_idx,
263 stats: TraversalStats::default(),
264 root_nodes: vec![None; num_roots],
265 nodes_by_path: HashMap::new(),
266 inodes: InodeFilter::default(),
267 throttle: Some(Throttle::new(Duration::from_millis(250), None)),
268 skip_root,
269 use_root_path,
270 retained_depth: None,
271 event_rx: entry_rx,
272 })
273 }
274
275 pub(crate) fn retain_depth(mut self, depth: Option<usize>) -> Self {
279 self.retained_depth = depth;
280 self
281 }
282
283 fn record_error_on_root(
284 &mut self,
285 traversal: &mut Traversal,
286 root_idx: usize,
287 root_path: &Path,
288 ) {
289 if self.skip_root {
290 return;
291 }
292 if let Some(root) = self.root_nodes[root_idx] {
294 traversal.tree[root].metadata_io_error = true;
295 return;
296 }
297 let name = if self.use_root_path {
298 root_path.to_owned()
299 } else {
300 root_path
301 .file_name()
302 .unwrap_or(root_path.as_os_str())
303 .into()
304 };
305 let node = traversal.tree.add_node(EntryData {
306 name,
307 metadata_io_error: true,
308 is_dir: true,
309 ..EntryData::default()
310 });
311 traversal.tree.add_edge(self.root_idx, node, ());
312 *traversal.tree[self.root_idx].entry_count.get_or_insert(0) += 1;
313 self.root_nodes[root_idx] = Some(node);
314 }
315
316 #[expect(
329 clippy::too_many_lines,
330 reason = "event integration keeps tree updates atomic"
331 )]
332 pub fn integrate_traversal_event(
333 &mut self,
334 traversal: &mut Traversal,
335 event: TraversalEvent,
336 ) -> Option<bool> {
337 match event {
338 TraversalEvent::Entry(entry, root_path, device_id, root_idx) => {
339 let root = Arc::as_ptr(&root_path) as usize;
340 self.stats.entries_traversed += 1;
341 let mut data = EntryData::default();
342 let Ok(TraversalEntry(entry)) = entry else {
343 self.stats.io_errors += 1;
344 self.record_error_on_root(traversal, root_idx, &root_path);
345 return self
346 .throttle
347 .as_ref()
348 .is_some_and(|t| t.can_update())
349 .then_some(false);
350 };
351 let walk_depth = entry.depth;
352 if self.skip_root {
353 data.name = entry.file_name.clone().into();
354 } else {
355 data.name = if walk_depth < 1 && self.use_root_path {
356 (*root_path).clone()
357 } else {
358 entry.file_name.clone().into()
359 }
360 }
361
362 let mut file_size = 0u128;
363 let mut mtime: SystemTime = UNIX_EPOCH;
364 data.is_dir = entry.file_type.is_dir();
365 if let Ok(m) = &entry.metadata {
366 if self.walk_options.count_hard_links
367 || self.inodes.add(&entry, m)
368 && (self.walk_options.cross_filesystems
369 || crossdev::is_same_device(device_id, m))
370 {
371 if self.walk_options.apparent_size {
372 file_size = u128::from(m.len());
373 } else {
374 file_size = u128::from(
375 size_on_disk(
376 &entry.parent_path,
377 &data.name,
378 m,
379 data.is_dir,
380 &self.walk_options,
381 &mut self.inodes,
382 )
383 .unwrap_or_else(|_| {
384 self.stats.io_errors += 1;
385 data.metadata_io_error = true;
386 0
387 }),
388 );
389 }
390 } else {
391 data.entry_count = Some(0);
392 }
393
394 if let Ok(modified) = m.modified() {
395 mtime = modified;
396 } else {
397 self.stats.io_errors += 1;
398 data.metadata_io_error = true;
399 }
400 } else {
401 self.stats.io_errors += 1;
402 data.metadata_io_error = true;
403 }
404
405 data.mtime = mtime;
406 data.size = file_size;
407 if data.is_dir {
408 data.entry_count = Some(1);
409 }
410 let entry_count = u64::from(data.is_dir || data.entry_count != Some(0));
411 let retain_entry = self.retained_depth.is_none_or(|depth| walk_depth <= depth);
412
413 let parent_index = if walk_depth == 0 {
414 self.root_idx
415 } else {
416 if self.skip_root {
417 self.nodes_by_path
418 .entry((root, (*root_path).clone()))
419 .or_insert(self.root_idx);
420 }
421 let mut parent_path = entry.parent_path.to_path_buf();
422 loop {
423 if let Some(index) = self.nodes_by_path.get(&(root, parent_path.clone())) {
424 break *index;
425 }
426 if !parent_path.pop() {
427 assert!(
428 !retain_entry,
429 "parent entries are emitted before their children"
430 );
431 break self.root_idx;
432 }
433 }
434 };
435 if retain_entry {
436 let entry_index = traversal.tree.add_node(data);
437 traversal.tree.add_edge(parent_index, entry_index, ());
438 if walk_depth == 0 {
439 self.root_nodes[root_idx] = Some(entry_index);
440 }
441 if traversal.tree[entry_index].is_dir {
442 self.nodes_by_path.insert((root, entry.path()), entry_index);
443 }
444 }
445
446 let mut ancestor = Some(parent_index);
447 while let Some(index) = ancestor {
448 ancestor = traversal
449 .tree
450 .neighbors_directed(index, Direction::Incoming)
451 .next();
452 let entry = &mut traversal.tree[index];
453 entry.size += file_size;
454 *entry.entry_count.get_or_insert(0) += entry_count;
455 }
456
457 if self.throttle.as_ref().is_some_and(|t| t.can_update()) {
458 return Some(false);
459 }
460 }
461 TraversalEvent::RootError(root_path, root_idx) => {
462 self.stats.io_errors += 1;
463 self.record_error_on_root(traversal, root_idx, &root_path);
464 }
465 TraversalEvent::Finished => {
466 self.throttle = None;
467 let root_size = traversal.tree[self.root_idx].size;
468 self.nodes_by_path = HashMap::new();
469 self.stats.total_bytes = Some(root_size);
470 self.stats.elapsed = Some(self.stats.start.elapsed());
471
472 return Some(true);
473 }
474 }
475 None
476 }
477}
478
479#[cfg(not(any(windows, target_os = "macos")))]
480fn size_on_disk(
482 _parent: &Path,
483 name: &Path,
484 meta: &crate::walk::Metadata,
485 _is_dir: bool,
486 _options: &WalkOptions,
487 _inodes: &mut InodeFilter,
488) -> io::Result<u64> {
489 name.size_on_disk_fast(meta)
490}
491
492#[cfg(target_os = "macos")]
493#[allow(clippy::unnecessary_wraps)]
495fn size_on_disk(
496 _parent: &Path,
497 _name: &Path,
498 meta: &crate::walk::Metadata,
499 _is_dir: bool,
500 options: &WalkOptions,
501 inodes: &mut InodeFilter,
502) -> io::Result<u64> {
503 Ok(if options.metadata_options.apfs_clone_metadata {
504 inodes.allocated_size(meta)
505 } else {
506 meta.allocated_size()
507 })
508}
509
510#[cfg(windows)]
511#[allow(clippy::unnecessary_wraps)]
513fn size_on_disk(
514 _parent: &Path,
515 _name: &Path,
516 meta: &crate::walk::Metadata,
517 is_dir: bool,
518 _options: &WalkOptions,
519 _inodes: &mut InodeFilter,
520) -> io::Result<u64> {
521 Ok(if is_dir { 0 } else { meta.allocated_size() })
522}
523
524#[cfg(test)]
525mod tests {
526 use super::*;
527
528 #[test]
529 fn ancestor_sizes_update_before_traversal_finishes() {
530 let dir = tempfile::tempdir().unwrap();
531 std::fs::create_dir(dir.path().join("nested")).unwrap();
532 std::fs::write(dir.path().join("nested/file"), b"content").unwrap();
533
534 let mut traversal = Traversal::new();
535 let mut background = BackgroundTraversal::start(
536 traversal.root_index,
537 &WalkOptions {
538 threads: 2,
539 count_hard_links: true,
540 apparent_size: true,
541 cross_filesystems: true,
542 ignore_dirs: std::collections::BTreeSet::default(),
543 ignore_patterns: None,
544 metadata_options: crate::TraversalOptions::default(),
545 },
546 vec![dir.path().to_owned()],
547 None,
548 false,
549 false,
550 )
551 .unwrap();
552
553 loop {
554 let event = background.event_rx.recv().unwrap();
555 let is_file = matches!(
556 &event,
557 TraversalEvent::Entry(Ok(TraversalEntry(entry)), _, _, _)
558 if entry.file_name == "file"
559 );
560 background.integrate_traversal_event(&mut traversal, event);
561 if is_file {
562 let root_size = traversal.tree[traversal.root_index].size;
563 assert!(
564 root_size >= 7,
565 "root size should include the 7-byte nested file, got {root_size}"
566 );
567 let nested_size = traversal
568 .tree
569 .node_weights()
570 .find(|entry| entry.name == Path::new("nested"))
571 .unwrap()
572 .size;
573 assert!(
574 nested_size >= 7,
575 "nested directory size should include its 7-byte file, got {nested_size}"
576 );
577 break;
578 }
579 }
580 }
581
582 #[test]
583 fn duplicate_roots_keep_their_own_children() {
584 let dir = tempfile::tempdir().unwrap();
585 std::fs::write(dir.path().join("file"), b"content").unwrap();
586 let mut traversal = Traversal::new();
587 let mut background = BackgroundTraversal::start(
588 traversal.root_index,
589 &WalkOptions {
590 threads: 1,
591 count_hard_links: true,
592 apparent_size: true,
593 cross_filesystems: true,
594 ignore_dirs: std::collections::BTreeSet::default(),
595 ignore_patterns: None,
596 metadata_options: crate::TraversalOptions::default(),
597 },
598 vec![dir.path().to_owned(), dir.path().to_owned()],
599 None,
600 false,
601 false,
602 )
603 .unwrap();
604
605 while !background
606 .integrate_traversal_event(&mut traversal, background.event_rx.recv().unwrap())
607 .unwrap_or(false)
608 {}
609
610 let roots = traversal
611 .tree
612 .neighbors_directed(traversal.root_index, Direction::Outgoing)
613 .collect::<Vec<_>>();
614 assert_eq!(roots.len(), 2);
615 for root in roots {
616 assert_eq!(
617 traversal
618 .tree
619 .neighbors_directed(root, Direction::Outgoing)
620 .count(),
621 1
622 );
623 }
624 }
625
626 #[test]
627 fn retained_depth_rolls_deeper_sizes_into_the_last_kept_node() {
628 let dir = tempfile::tempdir().unwrap();
629 std::fs::create_dir_all(dir.path().join("one/two")).unwrap();
630 std::fs::write(dir.path().join("one/two/file"), b"content").unwrap();
631 for (depth, expected_nodes) in [(0, 2), (1, 3)] {
632 let mut traversal = Traversal::new();
633 let mut background = BackgroundTraversal::start(
634 traversal.root_index,
635 &WalkOptions {
636 threads: 1,
637 count_hard_links: true,
638 apparent_size: true,
639 cross_filesystems: true,
640 ignore_dirs: std::collections::BTreeSet::default(),
641 ignore_patterns: None,
642 metadata_options: crate::TraversalOptions::default(),
643 },
644 vec![dir.path().to_owned()],
645 None,
646 false,
647 true,
648 )
649 .unwrap()
650 .retain_depth(Some(depth));
651
652 while !background
653 .integrate_traversal_event(&mut traversal, background.event_rx.recv().unwrap())
654 .unwrap_or(false)
655 {}
656
657 assert_eq!(traversal.tree.node_count(), expected_nodes);
658 assert!(traversal.tree[traversal.root_index].size >= 7);
659 let root = traversal
660 .tree
661 .neighbors_directed(traversal.root_index, Direction::Outgoing)
662 .next()
663 .unwrap();
664 let last_retained = if depth == 0 {
665 root
666 } else {
667 traversal
668 .tree
669 .neighbors_directed(root, Direction::Outgoing)
670 .next()
671 .unwrap()
672 };
673 assert!(traversal.tree[last_retained].size >= 7);
674 }
675 }
676
677 #[test]
678 fn descendant_entry_errors_mark_the_retained_root() {
679 let dir = tempfile::tempdir().unwrap();
680 let root_path = dir.path().to_owned();
681 let mut traversal = Traversal::new();
682 let mut background = BackgroundTraversal::start(
683 traversal.root_index,
684 &WalkOptions {
685 threads: 1,
686 count_hard_links: true,
687 apparent_size: true,
688 cross_filesystems: true,
689 ignore_dirs: std::collections::BTreeSet::default(),
690 ignore_patterns: None,
691 metadata_options: crate::TraversalOptions::default(),
692 },
693 vec![root_path.clone()],
694 None,
695 false,
696 true,
697 )
698 .unwrap()
699 .retain_depth(Some(0));
700
701 while background.root_nodes[0].is_none() {
702 let event = background.event_rx.recv().unwrap();
703 background.integrate_traversal_event(&mut traversal, event);
704 }
705 let root = background.root_nodes[0].unwrap();
706 background.integrate_traversal_event(
707 &mut traversal,
708 TraversalEvent::Entry(
709 Err(io::Error::other("unreadable descendant")),
710 Arc::new(root_path),
711 0,
712 0,
713 ),
714 );
715
716 assert_eq!(background.stats.io_errors, 1);
717 assert!(
718 traversal.tree[root].metadata_io_error,
719 "a path-less descendant error is reported on its retained root: {:?}",
720 traversal.tree[root]
721 );
722 }
723
724 #[cfg(target_os = "macos")]
725 #[test]
726 fn interactive_traversal_deduplicates_apfs_clones() {
727 use std::os::unix::fs::MetadataExt as _;
728
729 fn total(path: &Path, deduplicate: bool) -> u128 {
730 let mut traversal = Traversal::new();
731 let mut background = BackgroundTraversal::start(
732 traversal.root_index,
733 &WalkOptions {
734 threads: 2,
735 count_hard_links: false,
736 apparent_size: false,
737 cross_filesystems: true,
738 ignore_dirs: std::collections::BTreeSet::default(),
739 ignore_patterns: None,
740 metadata_options: crate::TraversalOptions {
741 apfs_clone_metadata: deduplicate,
742 },
743 },
744 vec![path.to_owned()],
745 None,
746 false,
747 false,
748 )
749 .unwrap();
750
751 while !background
752 .integrate_traversal_event(&mut traversal, background.event_rx.recv().unwrap())
753 .unwrap_or(false)
754 {}
755 traversal.tree[traversal.root_index].size
756 }
757
758 let directory = tempfile::tempdir().unwrap();
759 let original = directory.path().join("original");
760 let clone = directory.path().join("clone");
761 std::fs::write(&original, vec![1; 8192]).unwrap();
762 std::fs::copy(&original, clone).unwrap();
764 let data_fork_size = u128::from(std::fs::metadata(original).unwrap().blocks()) * 512;
765
766 assert_eq!(
767 total(directory.path(), false) - total(directory.path(), true),
768 data_fork_size
769 );
770 }
771
772 #[cfg(unix)]
773 #[test]
774 fn root_device_error_is_reported() {
775 use std::os::unix::fs::symlink;
776
777 let dir = tempfile::tempdir().unwrap();
778 let root = dir.path().join("dangling");
779 let valid = dir.path().join("valid");
780 symlink(dir.path().join("missing"), &root).unwrap();
781 std::fs::write(&valid, b"content").unwrap();
782 let mut traversal = Traversal::new();
783 let mut background = BackgroundTraversal::start(
784 traversal.root_index,
785 &WalkOptions {
786 threads: 1,
787 count_hard_links: true,
788 apparent_size: true,
789 cross_filesystems: false,
790 ignore_dirs: std::collections::BTreeSet::default(),
791 ignore_patterns: None,
792 metadata_options: crate::TraversalOptions::default(),
793 },
794 vec![root.clone(), valid.clone()],
795 None,
796 false,
797 false,
798 )
799 .unwrap();
800
801 while !background
802 .integrate_traversal_event(&mut traversal, background.event_rx.recv().unwrap())
803 .unwrap_or(false)
804 {}
805
806 assert_eq!(background.stats.io_errors, 1);
807 let roots = background
808 .root_nodes
809 .iter()
810 .copied()
811 .collect::<Option<Vec<_>>>()
812 .unwrap();
813 assert_eq!(roots.len(), 2, "one node per input root: {roots:?}");
814 assert_eq!(
815 traversal.tree[traversal.root_index].entry_count,
816 Some(2),
817 "the synthetic root counts both input roots"
818 );
819 assert!(
820 traversal.tree[roots[0]].metadata_io_error,
821 "the failed root records its I/O error: {:?}",
822 traversal.tree[roots[0]]
823 );
824 assert_eq!(
825 traversal.tree[roots[0]].name,
826 Path::new("dangling"),
827 "the failed root retains its display name"
828 );
829 assert!(
830 roots.iter().all(|root| {
831 traversal
832 .tree
833 .find_edge(traversal.root_index, *root)
834 .is_some()
835 }),
836 "all input roots are children of the synthetic root: {roots:?}"
837 );
838 }
839
840 #[test]
841 fn size_of_entry_data() {
842 assert!(
843 std::mem::size_of::<EntryData>() <= 80,
844 "the size of this ({}) should not exceed 80 as it affects overall memory consumption",
845 std::mem::size_of::<EntryData>()
846 );
847 }
848}