1use crate::{Throttle, WalkOptions, WalkRoot, crossdev, inodefilter::InodeFilter};
2
3use crossbeam::channel::Receiver;
4#[cfg(not(any(windows, target_os = "macos")))]
5use filesize::PathExt;
6use petgraph::{Directed, Direction, graph::NodeIndex, stable_graph::StableGraph};
7use std::time::Instant;
8use std::{
9 collections::HashMap,
10 fmt, io,
11 path::{Path, PathBuf},
12 sync::Arc,
13 time::{Duration, SystemTime, UNIX_EPOCH},
14};
15
16pub type TreeIndex = NodeIndex;
18pub type Tree = StableGraph<EntryData, (), Directed>;
20
21#[derive(Eq, PartialEq, Clone)]
23pub struct EntryData {
24 pub name: PathBuf,
26 pub size: u128,
29 pub mtime: SystemTime,
31 pub entry_count: Option<u64>,
33 pub metadata_io_error: bool,
35 pub is_dir: bool,
37}
38
39impl Default for EntryData {
40 fn default() -> EntryData {
41 EntryData {
42 name: PathBuf::default(),
43 size: u128::default(),
44 mtime: UNIX_EPOCH,
45 entry_count: None,
46 metadata_io_error: bool::default(),
47 is_dir: false,
48 }
49 }
50}
51
52impl fmt::Debug for EntryData {
53 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
54 f.debug_struct("EntryData")
55 .field("name", &self.name)
56 .field("size", &self.size)
57 .field("entry_count", &self.entry_count)
58 .field("metadata_io_error", &self.metadata_io_error)
60 .finish()
61 }
62}
63
64#[derive(Debug)]
66pub struct Traversal {
67 pub tree: Tree,
69 pub root_index: TreeIndex,
71 pub start_time: Instant,
73 pub cost: Option<Duration>,
75}
76
77impl Default for Traversal {
78 fn default() -> Self {
79 Self::new()
80 }
81}
82
83impl Traversal {
84 #[must_use]
86 pub fn new() -> Self {
87 let mut tree = Tree::new();
88 let root_index = tree.add_node(EntryData::default());
89 Self {
90 tree,
91 root_index,
92 start_time: Instant::now(),
93 cost: None,
94 }
95 }
96
97 #[must_use]
99 pub fn is_costly(&self) -> bool {
100 self.cost.is_none_or(|d| d.as_secs_f32() > 10.0)
101 }
102}
103
104#[derive(Clone, Copy)]
106pub struct TraversalStats {
107 pub entries_traversed: u64,
109 pub start: std::time::Instant,
111 pub elapsed: Option<std::time::Duration>,
113 pub io_errors: u64,
115 pub total_bytes: Option<u128>,
117}
118
119impl Default for TraversalStats {
120 fn default() -> Self {
121 Self {
122 entries_traversed: 0,
123 start: std::time::Instant::now(),
124 elapsed: None,
125 io_errors: 0,
126 total_bytes: None,
127 }
128 }
129}
130
131pub struct TraversalEntry(pub(crate) crate::walk::Entry);
133
134pub enum TraversalEvent {
136 Entry(io::Result<TraversalEntry>, Arc<PathBuf>, u64, usize),
144 RootError(Arc<PathBuf>, usize),
146 Finished,
148}
149
150pub struct BackgroundTraversal {
152 walk_options: WalkOptions,
153 pub root_idx: TreeIndex,
155 pub stats: TraversalStats,
157 pub(crate) root_nodes: Vec<Option<TreeIndex>>,
159 nodes_by_path: HashMap<(usize, PathBuf), TreeIndex>,
161 inodes: InodeFilter,
162 throttle: Option<Throttle>,
163 skip_root: bool,
164 use_root_path: bool,
165 retained_depth: Option<usize>,
166 pub event_rx: Receiver<TraversalEvent>,
168}
169
170impl BackgroundTraversal {
171 pub fn start(
174 root_idx: TreeIndex,
175 walk_options: &WalkOptions,
176 input: Vec<PathBuf>,
177 pattern_roots: Option<&[PathBuf]>,
178 skip_root: bool,
179 use_root_path: bool,
180 ) -> anyhow::Result<BackgroundTraversal> {
181 let num_roots = input.len();
182 let (entry_tx, entry_rx) = crossbeam::channel::bounded(100);
183 let pattern_roots = pattern_roots.map(<[PathBuf]>::to_owned);
184 std::thread::Builder::new()
185 .name("dua-fs-walk-dispatcher".to_string())
186 .spawn({
187 let walk_options = walk_options.clone();
188 move || {
189 let (mut root_paths, mut root_indices, mut device_ids, mut walk_roots) = (
190 Vec::with_capacity(input.len()),
191 Vec::with_capacity(input.len()),
192 Vec::with_capacity(input.len()),
193 Vec::with_capacity(input.len()),
194 );
195 for (root_idx, root_path) in input.into_iter().enumerate() {
196 log::info!("Walking {}", root_path.display());
197 let device_id = if walk_options.cross_filesystems {
198 0
199 } else {
200 let Ok(device_id) = crossdev::init(&root_path) else {
201 if entry_tx
202 .send(TraversalEvent::RootError(Arc::new(root_path), root_idx))
203 .is_err()
204 {
205 return;
206 }
207 continue;
208 };
209 device_id
210 };
211 let pattern_root = pattern_roots.as_deref().map(|pattern_roots| {
212 pattern_roots
213 .iter()
214 .filter(|candidate| root_path.starts_with(candidate))
215 .max_by_key(|candidate| candidate.components().count())
216 .cloned()
217 .unwrap_or_else(|| root_path.clone())
218 });
219 walk_roots.push(WalkRoot {
220 index: walk_roots.len(),
221 pattern_root,
222 path: root_path.clone(),
223 #[cfg(any(windows, target_os = "macos"))]
224 entry: None,
225 device_id,
226 });
227 root_indices.push(root_idx);
228 device_ids.push(device_id);
229 root_paths.push(Arc::new(root_path));
230 }
231
232 for (root, event) in walk_options.iter_from_paths(
233 walk_roots,
234 skip_root,
235 crate::walk::Order::ParentFirst,
236 ) {
237 let crate::walk::RootEvent::Entry(entry) = event else {
238 continue;
239 };
240 if entry_tx
241 .send(TraversalEvent::Entry(
242 entry.map(TraversalEntry),
243 Arc::clone(&root_paths[root]),
244 device_ids[root],
245 root_indices[root],
246 ))
247 .is_err()
248 {
249 return;
252 }
253 }
254 if entry_tx.send(TraversalEvent::Finished).is_err() {
255 log::error!("Failed to send TraversalEvents::Finished event");
256 }
257 }
258 })?;
259
260 Ok(Self {
261 walk_options: walk_options.clone(),
262 root_idx,
263 stats: TraversalStats::default(),
264 root_nodes: vec![None; num_roots],
265 nodes_by_path: HashMap::new(),
266 inodes: InodeFilter::default(),
267 throttle: Some(Throttle::new(Duration::from_millis(250), None)),
268 skip_root,
269 use_root_path,
270 retained_depth: None,
271 event_rx: entry_rx,
272 })
273 }
274
275 pub(crate) fn retain_depth(mut self, depth: Option<usize>) -> Self {
279 self.retained_depth = depth;
280 self
281 }
282
283 fn record_error_on_root(
284 &mut self,
285 traversal: &mut Traversal,
286 root_idx: usize,
287 root_path: &Path,
288 ) {
289 if self.skip_root {
290 return;
291 }
292 if let Some(root) = self.root_nodes[root_idx] {
294 traversal.tree[root].metadata_io_error = true;
295 return;
296 }
297 let name = if self.use_root_path {
298 root_path.to_owned()
299 } else {
300 root_path
301 .file_name()
302 .unwrap_or(root_path.as_os_str())
303 .into()
304 };
305 let node = traversal.tree.add_node(EntryData {
306 name,
307 metadata_io_error: true,
308 is_dir: true,
309 ..EntryData::default()
310 });
311 traversal.tree.add_edge(self.root_idx, node, ());
312 *traversal.tree[self.root_idx].entry_count.get_or_insert(0) += 1;
313 self.root_nodes[root_idx] = Some(node);
314 }
315
316 #[expect(
329 clippy::too_many_lines,
330 reason = "event integration keeps tree updates atomic"
331 )]
332 pub fn integrate_traversal_event(
333 &mut self,
334 traversal: &mut Traversal,
335 event: TraversalEvent,
336 ) -> Option<bool> {
337 match event {
338 TraversalEvent::Entry(entry, root_path, device_id, root_idx) => {
339 let root = Arc::as_ptr(&root_path) as usize;
340 self.stats.entries_traversed += 1;
341 let mut data = EntryData::default();
342 let Ok(TraversalEntry(entry)) = entry else {
343 self.stats.io_errors += 1;
344 self.record_error_on_root(traversal, root_idx, &root_path);
345 return self
346 .throttle
347 .as_ref()
348 .is_some_and(|t| t.can_update())
349 .then_some(false);
350 };
351 let walk_depth = entry.depth;
352 if self.skip_root {
353 data.name = entry.file_name.clone().into();
354 } else {
355 data.name = if walk_depth < 1 && self.use_root_path {
356 (*root_path).clone()
357 } else {
358 entry.file_name.clone().into()
359 }
360 }
361
362 let mut file_size = 0u128;
363 let mut mtime: SystemTime = UNIX_EPOCH;
364 data.is_dir = entry.file_type.is_dir();
365 if let Ok(m) = &entry.metadata {
366 if self.walk_options.count_hard_links
367 || self.inodes.add(&entry, m)
368 && (self.walk_options.cross_filesystems
369 || crossdev::is_same_device(device_id, m))
370 {
371 if self.walk_options.apparent_size {
372 file_size = u128::from(m.len());
373 } else {
374 file_size = u128::from(
375 size_on_disk(
376 &entry.parent_path,
377 &data.name,
378 m,
379 data.is_dir,
380 &self.walk_options,
381 &mut self.inodes,
382 )
383 .unwrap_or_else(|_| {
384 self.stats.io_errors += 1;
385 data.metadata_io_error = true;
386 0
387 }),
388 );
389 }
390 } else {
391 data.entry_count = Some(0);
392 }
393
394 if let Ok(modified) = m.modified() {
395 mtime = modified;
396 } else {
397 self.stats.io_errors += 1;
398 data.metadata_io_error = true;
399 }
400 } else {
401 self.stats.io_errors += 1;
402 data.metadata_io_error = true;
403 }
404
405 data.mtime = mtime;
406 data.size = file_size;
407 if data.is_dir {
408 data.entry_count = Some(1);
409 }
410 let entry_count = u64::from(data.is_dir || data.entry_count != Some(0));
411 let retain_entry = self.retained_depth.is_none_or(|depth| walk_depth <= depth);
412
413 let parent_index = if walk_depth == 0 {
414 self.root_idx
415 } else if self.retained_depth == Some(0) {
416 if self.skip_root {
417 self.root_idx
418 } else {
419 self.root_nodes[root_idx]
420 .expect("root entries are emitted before their children")
421 }
422 } else {
423 if self.skip_root {
424 self.nodes_by_path
425 .entry((root, (*root_path).clone()))
426 .or_insert(self.root_idx);
427 }
428 let mut parent_path = entry.parent_path.to_path_buf();
429 loop {
430 if let Some(index) = self.nodes_by_path.get(&(root, parent_path.clone())) {
431 break *index;
432 }
433 if !parent_path.pop() {
434 assert!(
435 !retain_entry,
436 "parent entries are emitted before their children"
437 );
438 break self.root_idx;
439 }
440 }
441 };
442 if retain_entry {
443 let entry_index = traversal.tree.add_node(data);
444 traversal.tree.add_edge(parent_index, entry_index, ());
445 if walk_depth == 0 {
446 self.root_nodes[root_idx] = Some(entry_index);
447 }
448 if traversal.tree[entry_index].is_dir {
449 self.nodes_by_path.insert((root, entry.path()), entry_index);
450 }
451 }
452
453 let mut ancestor = Some(parent_index);
454 while let Some(index) = ancestor {
455 ancestor = traversal
456 .tree
457 .neighbors_directed(index, Direction::Incoming)
458 .next();
459 let entry = &mut traversal.tree[index];
460 entry.size += file_size;
461 *entry.entry_count.get_or_insert(0) += entry_count;
462 }
463
464 if self.throttle.as_ref().is_some_and(|t| t.can_update()) {
465 return Some(false);
466 }
467 }
468 TraversalEvent::RootError(root_path, root_idx) => {
469 self.stats.io_errors += 1;
470 self.record_error_on_root(traversal, root_idx, &root_path);
471 }
472 TraversalEvent::Finished => {
473 self.throttle = None;
474 let root_size = traversal.tree[self.root_idx].size;
475 self.nodes_by_path = HashMap::new();
476 self.stats.total_bytes = Some(root_size);
477 self.stats.elapsed = Some(self.stats.start.elapsed());
478
479 return Some(true);
480 }
481 }
482 None
483 }
484}
485
486#[cfg(not(any(windows, target_os = "macos")))]
487fn size_on_disk(
489 _parent: &Path,
490 name: &Path,
491 meta: &crate::walk::Metadata,
492 _is_dir: bool,
493 _options: &WalkOptions,
494 _inodes: &mut InodeFilter,
495) -> io::Result<u64> {
496 name.size_on_disk_fast(meta)
497}
498
499#[cfg(target_os = "macos")]
500#[allow(clippy::unnecessary_wraps)]
502fn size_on_disk(
503 _parent: &Path,
504 _name: &Path,
505 meta: &crate::walk::Metadata,
506 _is_dir: bool,
507 options: &WalkOptions,
508 inodes: &mut InodeFilter,
509) -> io::Result<u64> {
510 Ok(if options.metadata_options.apfs_clone_metadata {
511 inodes.allocated_size(meta)
512 } else {
513 meta.allocated_size()
514 })
515}
516
517#[cfg(windows)]
518#[allow(clippy::unnecessary_wraps)]
520fn size_on_disk(
521 _parent: &Path,
522 _name: &Path,
523 meta: &crate::walk::Metadata,
524 is_dir: bool,
525 _options: &WalkOptions,
526 _inodes: &mut InodeFilter,
527) -> io::Result<u64> {
528 Ok(if is_dir { 0 } else { meta.allocated_size() })
529}
530
531#[cfg(test)]
532mod tests {
533 use super::*;
534
535 #[test]
536 fn ancestor_sizes_update_before_traversal_finishes() {
537 let dir = tempfile::tempdir().unwrap();
538 std::fs::create_dir(dir.path().join("nested")).unwrap();
539 std::fs::write(dir.path().join("nested/file"), b"content").unwrap();
540
541 let mut traversal = Traversal::new();
542 let mut background = BackgroundTraversal::start(
543 traversal.root_index,
544 &WalkOptions {
545 threads: 2,
546 count_hard_links: true,
547 apparent_size: true,
548 cross_filesystems: true,
549 ignore_dirs: std::collections::BTreeSet::default(),
550 ignore_patterns: None,
551 metadata_options: crate::TraversalOptions::default(),
552 },
553 vec![dir.path().to_owned()],
554 None,
555 false,
556 false,
557 )
558 .unwrap();
559
560 loop {
561 let event = background.event_rx.recv().unwrap();
562 let is_file = matches!(
563 &event,
564 TraversalEvent::Entry(Ok(TraversalEntry(entry)), _, _, _)
565 if entry.file_name == "file"
566 );
567 background.integrate_traversal_event(&mut traversal, event);
568 if is_file {
569 let root_size = traversal.tree[traversal.root_index].size;
570 assert!(
571 root_size >= 7,
572 "root size should include the 7-byte nested file, got {root_size}"
573 );
574 let nested_size = traversal
575 .tree
576 .node_weights()
577 .find(|entry| entry.name == Path::new("nested"))
578 .unwrap()
579 .size;
580 assert!(
581 nested_size >= 7,
582 "nested directory size should include its 7-byte file, got {nested_size}"
583 );
584 break;
585 }
586 }
587 }
588
589 #[test]
590 fn duplicate_roots_keep_their_own_children() {
591 let dir = tempfile::tempdir().unwrap();
592 std::fs::write(dir.path().join("file"), b"content").unwrap();
593 let mut traversal = Traversal::new();
594 let mut background = BackgroundTraversal::start(
595 traversal.root_index,
596 &WalkOptions {
597 threads: 1,
598 count_hard_links: true,
599 apparent_size: true,
600 cross_filesystems: true,
601 ignore_dirs: std::collections::BTreeSet::default(),
602 ignore_patterns: None,
603 metadata_options: crate::TraversalOptions::default(),
604 },
605 vec![dir.path().to_owned(), dir.path().to_owned()],
606 None,
607 false,
608 false,
609 )
610 .unwrap();
611
612 while !background
613 .integrate_traversal_event(&mut traversal, background.event_rx.recv().unwrap())
614 .unwrap_or(false)
615 {}
616
617 let roots = traversal
618 .tree
619 .neighbors_directed(traversal.root_index, Direction::Outgoing)
620 .collect::<Vec<_>>();
621 assert_eq!(roots.len(), 2);
622 for root in roots {
623 assert_eq!(
624 traversal
625 .tree
626 .neighbors_directed(root, Direction::Outgoing)
627 .count(),
628 1
629 );
630 }
631 }
632
633 #[test]
634 fn retained_depth_rolls_deeper_sizes_into_the_last_kept_node() {
635 let dir = tempfile::tempdir().unwrap();
636 std::fs::create_dir_all(dir.path().join("one/two")).unwrap();
637 std::fs::write(dir.path().join("one/two/file"), b"content").unwrap();
638 for (depth, expected_nodes) in [(0, 2), (1, 3)] {
639 let mut traversal = Traversal::new();
640 let mut background = BackgroundTraversal::start(
641 traversal.root_index,
642 &WalkOptions {
643 threads: 1,
644 count_hard_links: true,
645 apparent_size: true,
646 cross_filesystems: true,
647 ignore_dirs: std::collections::BTreeSet::default(),
648 ignore_patterns: None,
649 metadata_options: crate::TraversalOptions::default(),
650 },
651 vec![dir.path().to_owned()],
652 None,
653 false,
654 true,
655 )
656 .unwrap()
657 .retain_depth(Some(depth));
658
659 while !background
660 .integrate_traversal_event(&mut traversal, background.event_rx.recv().unwrap())
661 .unwrap_or(false)
662 {}
663
664 assert_eq!(traversal.tree.node_count(), expected_nodes);
665 assert!(traversal.tree[traversal.root_index].size >= 7);
666 let root = traversal
667 .tree
668 .neighbors_directed(traversal.root_index, Direction::Outgoing)
669 .next()
670 .unwrap();
671 let last_retained = if depth == 0 {
672 root
673 } else {
674 traversal
675 .tree
676 .neighbors_directed(root, Direction::Outgoing)
677 .next()
678 .unwrap()
679 };
680 assert!(traversal.tree[last_retained].size >= 7);
681 }
682 }
683
684 #[test]
685 fn descendant_entry_errors_mark_the_retained_root() {
686 let dir = tempfile::tempdir().unwrap();
687 let root_path = dir.path().to_owned();
688 let mut traversal = Traversal::new();
689 let mut background = BackgroundTraversal::start(
690 traversal.root_index,
691 &WalkOptions {
692 threads: 1,
693 count_hard_links: true,
694 apparent_size: true,
695 cross_filesystems: true,
696 ignore_dirs: std::collections::BTreeSet::default(),
697 ignore_patterns: None,
698 metadata_options: crate::TraversalOptions::default(),
699 },
700 vec![root_path.clone()],
701 None,
702 false,
703 true,
704 )
705 .unwrap()
706 .retain_depth(Some(0));
707
708 while background.root_nodes[0].is_none() {
709 let event = background.event_rx.recv().unwrap();
710 background.integrate_traversal_event(&mut traversal, event);
711 }
712 let root = background.root_nodes[0].unwrap();
713 background.integrate_traversal_event(
714 &mut traversal,
715 TraversalEvent::Entry(
716 Err(io::Error::other("unreadable descendant")),
717 Arc::new(root_path),
718 0,
719 0,
720 ),
721 );
722
723 assert_eq!(background.stats.io_errors, 1);
724 assert!(
725 traversal.tree[root].metadata_io_error,
726 "a path-less descendant error is reported on its retained root: {:?}",
727 traversal.tree[root]
728 );
729 }
730
731 #[cfg(target_os = "macos")]
732 #[test]
733 fn interactive_traversal_deduplicates_apfs_clones() {
734 use std::os::unix::fs::MetadataExt as _;
735
736 fn total(path: &Path, deduplicate: bool) -> u128 {
737 let mut traversal = Traversal::new();
738 let mut background = BackgroundTraversal::start(
739 traversal.root_index,
740 &WalkOptions {
741 threads: 2,
742 count_hard_links: false,
743 apparent_size: false,
744 cross_filesystems: true,
745 ignore_dirs: std::collections::BTreeSet::default(),
746 ignore_patterns: None,
747 metadata_options: crate::TraversalOptions {
748 apfs_clone_metadata: deduplicate,
749 },
750 },
751 vec![path.to_owned()],
752 None,
753 false,
754 false,
755 )
756 .unwrap();
757
758 while !background
759 .integrate_traversal_event(&mut traversal, background.event_rx.recv().unwrap())
760 .unwrap_or(false)
761 {}
762 traversal.tree[traversal.root_index].size
763 }
764
765 let directory = tempfile::tempdir().unwrap();
766 let original = directory.path().join("original");
767 let clone = directory.path().join("clone");
768 std::fs::write(&original, vec![1; 8192]).unwrap();
769 std::fs::copy(&original, clone).unwrap();
771 let data_fork_size = u128::from(std::fs::metadata(original).unwrap().blocks()) * 512;
772
773 assert_eq!(
774 total(directory.path(), false) - total(directory.path(), true),
775 data_fork_size
776 );
777 }
778
779 #[cfg(unix)]
780 #[test]
781 fn root_device_error_is_reported() {
782 use std::os::unix::fs::symlink;
783
784 let dir = tempfile::tempdir().unwrap();
785 let root = dir.path().join("dangling");
786 let valid = dir.path().join("valid");
787 symlink(dir.path().join("missing"), &root).unwrap();
788 std::fs::write(&valid, b"content").unwrap();
789 let mut traversal = Traversal::new();
790 let mut background = BackgroundTraversal::start(
791 traversal.root_index,
792 &WalkOptions {
793 threads: 1,
794 count_hard_links: true,
795 apparent_size: true,
796 cross_filesystems: false,
797 ignore_dirs: std::collections::BTreeSet::default(),
798 ignore_patterns: None,
799 metadata_options: crate::TraversalOptions::default(),
800 },
801 vec![root.clone(), valid.clone()],
802 None,
803 false,
804 false,
805 )
806 .unwrap();
807
808 while !background
809 .integrate_traversal_event(&mut traversal, background.event_rx.recv().unwrap())
810 .unwrap_or(false)
811 {}
812
813 assert_eq!(background.stats.io_errors, 1);
814 let roots = background
815 .root_nodes
816 .iter()
817 .copied()
818 .collect::<Option<Vec<_>>>()
819 .unwrap();
820 assert_eq!(roots.len(), 2, "one node per input root: {roots:?}");
821 assert_eq!(
822 traversal.tree[traversal.root_index].entry_count,
823 Some(2),
824 "the synthetic root counts both input roots"
825 );
826 assert!(
827 traversal.tree[roots[0]].metadata_io_error,
828 "the failed root records its I/O error: {:?}",
829 traversal.tree[roots[0]]
830 );
831 assert_eq!(
832 traversal.tree[roots[0]].name,
833 Path::new("dangling"),
834 "the failed root retains its display name"
835 );
836 assert!(
837 roots.iter().all(|root| {
838 traversal
839 .tree
840 .find_edge(traversal.root_index, *root)
841 .is_some()
842 }),
843 "all input roots are children of the synthetic root: {roots:?}"
844 );
845 }
846
847 #[test]
848 fn size_of_entry_data() {
849 assert!(
850 std::mem::size_of::<EntryData>() <= 80,
851 "the size of this ({}) should not exceed 80 as it affects overall memory consumption",
852 std::mem::size_of::<EntryData>()
853 );
854 }
855}