compare_dir/
file_hasher.rs

1use crate::{
2    CheckMode, Classification, ColumnFormatter, DirectoryComparer, FileComparer,
3    FileComparisonResult, FileHashCache, FileItem, FileIterator, OutputFormat, Progress,
4    ProgressBuilder, ProgressValue, SharedProgress,
5};
6use globset::GlobSet;
7use indicatif::FormattedDuration;
8use rayon::prelude::*;
9use simple_path::SimplePath;
10use std::{
11    collections::HashMap,
12    fs,
13    io::{self, Read, stdout},
14    path::{Path, PathBuf},
15    sync::{
16        Arc,
17        atomic::{self, AtomicUsize},
18        mpsc,
19    },
20    time,
21};
22
23type FileWithDirIndex = (FileItem, usize);
24
25#[derive(Debug, Clone)]
26enum DupEvent {
27    StartHashing,
28    Total(ProgressValue),
29    Result(FileItem, blake3::Hash),
30    Error,
31}
32
33#[derive(Debug)]
34enum CheckEvent {
35    StartChecking,
36    Total(ProgressValue),
37    Result(FileComparisonResult),
38    Error(FileItem),
39}
40
41enum DupState {
42    Single(FileItem, usize),
43    Hashing,
44}
45
46/// A tool for finding duplicated files in a directory.
47pub struct FileHasher {
48    dirs: Vec<PathBuf>,
49    pub buffer_size: usize,
50    cache: Option<Arc<FileHashCache>>,
51    num_hashed: AtomicUsize,
52    num_hash_looked_up: AtomicUsize,
53    pub exclude: Option<GlobSet>,
54    pub progress: Option<Arc<ProgressBuilder>>,
55    pub output_format: OutputFormat,
56    pub jobs: usize,
57}
58
59impl FileHasher {
60    const DEFAULT_JOBS: usize = DirectoryComparer::DEFAULT_JOBS;
61
62    /// Creates a new `FileHasher` for the given directories.
63    pub fn new<P: AsRef<Path>>(dirs: &[P]) -> anyhow::Result<Self> {
64        if dirs.is_empty() {
65            anyhow::bail!("At least one directory must be specified.");
66        }
67        Ok(Self {
68            dirs: dirs.iter().map(|p| p.as_ref().to_path_buf()).collect(),
69            buffer_size: FileComparer::DEFAULT_BUFFER_SIZE,
70            cache: None,
71            num_hashed: AtomicUsize::new(0),
72            num_hash_looked_up: AtomicUsize::new(0),
73            exclude: None,
74            progress: None,
75            output_format: OutputFormat::Default,
76            jobs: Self::DEFAULT_JOBS,
77        })
78    }
79
80    pub(crate) fn new_with_cache<P: AsRef<Path>>(dirs: &[P]) -> anyhow::Result<Self> {
81        let mut hasher = Self::new(dirs)?;
82        hasher.cache = Some(hasher.new_cache()?);
83        Ok(hasher)
84    }
85
86    fn new_cache(&self) -> anyhow::Result<Arc<FileHashCache>> {
87        let common_ancestor = crate::common_ancestor(&self.dirs)
88            .ok_or_else(|| anyhow::anyhow!("No common ancestor found"))?;
89        Ok(FileHashCache::find_or_new(&common_ancestor))
90    }
91
92    /// Gets the hash cache.
93    pub(crate) fn cache(&mut self) -> anyhow::Result<Arc<FileHashCache>> {
94        if self.cache.is_none() {
95            self.cache = Some(self.new_cache()?);
96        }
97        Ok(Arc::clone(self.cache.as_ref().unwrap()))
98    }
99
100    /// Remove a cache entry if it exists.
101    pub(crate) fn remove_cache_entry(&mut self, path: &Path) -> anyhow::Result<()> {
102        let cache = self.cache()?;
103        let relative = SimplePath::strip_prefix(path, cache.base_dir())?;
104        cache.remove(relative);
105        Ok(())
106    }
107
108    /// Save the hash cache if it is dirty.
109    pub fn save_cache(&self) -> anyhow::Result<()> {
110        log::info!(
111            "Hash stats for {:?}: {} computed, {} looked up",
112            self.dirs,
113            self.num_hashed.load(atomic::Ordering::Relaxed),
114            self.num_hash_looked_up.load(atomic::Ordering::Relaxed)
115        );
116        if let Some(cache) = &self.cache {
117            cache.save()?;
118        }
119        Ok(())
120    }
121
122    /// Clears the loaded hashes in the cache.
123    pub(crate) fn clear_cache(&mut self) -> anyhow::Result<()> {
124        let cache = self.cache()?;
125        for dir in &self.dirs {
126            let relative = SimplePath::strip_prefix(dir, cache.base_dir())?;
127            cache.clear(relative);
128        }
129        Ok(())
130    }
131
132    /// Executes the check/update process.
133    pub fn check(&self, mode: CheckMode) -> anyhow::Result<()> {
134        match self.output_format {
135            OutputFormat::Default | OutputFormat::Symbol => {}
136            _ => anyhow::bail!("Check mode only supports default or symbol output format."),
137        }
138        if self.dirs.len() > 1 {
139            anyhow::bail!("Check mode only supports one directory.");
140        }
141        let start_time = time::Instant::now();
142        if let Some(p) = &self.progress {
143            p.set_propagate();
144        }
145        let progress = self
146            .progress
147            .as_ref()
148            .map(|progress| progress.add_primary())
149            .unwrap_or_else(SharedProgress::none);
150        progress.use_bytes();
151        progress.set_message("Scanning directory...");
152        let mut num_new = 0;
153        let mut num_modified = 0;
154        let mut num_error = 0;
155        std::thread::scope(|scope| {
156            let (tx, rx) = mpsc::channel();
157            scope.spawn(|| {
158                if let Err(e) = self.check_streaming(tx, mode) {
159                    log::error!("Error during check: {}", e);
160                }
161            });
162            while let Ok(event) = rx.recv() {
163                match event {
164                    CheckEvent::StartChecking => {
165                        progress.set_message("Checking files...");
166                    }
167                    CheckEvent::Total(value) => {
168                        progress.set_length(value);
169                        progress.set_message("");
170                    }
171                    CheckEvent::Result(result) => {
172                        progress.suspend_for(stdout(), || {
173                            result.print(self.output_format, "cached", "current")
174                        });
175                        if result.classification == Classification::OnlyInDir2 {
176                            num_new += 1;
177                        } else if result.is_identical_content() == Some(false) {
178                            num_modified += 1;
179                        }
180                    }
181                    CheckEvent::Error(_file) => num_error += 1,
182                }
183            }
184        });
185        progress.finish();
186        self.print_check_summary(&start_time, num_new, num_modified, num_error)?;
187        Ok(())
188    }
189
190    fn print_check_summary(
191        &self,
192        start_time: &time::Instant,
193        num_new: usize,
194        num_modified: usize,
195        num_error: usize,
196    ) -> io::Result<()> {
197        let summary = [
198            ("Elapsed:", 0),
199            (
200                "Hash computed:",
201                self.num_hashed.load(atomic::Ordering::Relaxed),
202            ),
203            ("New files:", num_new),
204            ("Modified files:", num_modified),
205            ("Errors:", num_error),
206        ];
207        let formatter = ColumnFormatter::new(summary.iter().map(|(s, _)| *s));
208        let mut writer = std::io::stderr();
209        formatter.write_value(
210            &mut writer,
211            summary[0].0,
212            FormattedDuration(start_time.elapsed()),
213        )?;
214        formatter.write_values(&mut writer, &summary[1..])
215    }
216
217    fn check_streaming(&self, tx: mpsc::Sender<CheckEvent>, mode: CheckMode) -> anyhow::Result<()> {
218        assert_eq!(self.dirs.len(), 1);
219        let cache = self.new_cache()?;
220        let base_dir = &self.dirs[0];
221        let relative = SimplePath::strip_prefix(base_dir, cache.base_dir())?;
222        cache.set_remove_if_no_access(relative);
223        let cache_clone = Arc::clone(&cache);
224        std::thread::scope(|global_scope| {
225            let mut it = FileIterator::new(base_dir);
226            it.cache = Some(Arc::clone(&cache));
227            it.exclude = self.exclude.as_ref();
228            let it_rx = it.spawn_in_scope(global_scope);
229            tx.send(CheckEvent::StartChecking)?;
230            let pool = crate::build_thread_pool(self.jobs)?;
231            pool.scope(move |scope| -> anyhow::Result<()> {
232                let mut total = ProgressValue::default();
233                for file in it_rx {
234                    self.check_file(file, &cache, mode, &mut total, &tx, scope);
235                }
236                tx.send(CheckEvent::Total(total))?;
237                Ok(())
238            })
239        })?;
240        cache_clone.save()?;
241        Ok(())
242    }
243
244    fn check_file<'scope>(
245        &'scope self,
246        file: FileItem,
247        cache: &Arc<FileHashCache>,
248        mode: CheckMode,
249        total: &mut ProgressValue,
250        tx: &mpsc::Sender<CheckEvent>,
251        scope: &rayon::Scope<'scope>,
252    ) {
253        *total += ProgressValue::with_size(file.size());
254        let tx = tx.clone();
255        let cache = Arc::clone(cache);
256        scope.spawn(move |_| {
257            if let Err(error) = self._check_file(&file, cache, mode, &tx) {
258                log::error!("Failed to check file '{}': {}", file, error);
259                if tx.send(CheckEvent::Error(file)).is_err() {
260                    log::error!("Send failed");
261                }
262            }
263        });
264    }
265
266    fn _check_file(
267        &self,
268        file: &FileItem,
269        cache: Arc<FileHashCache>,
270        mode: CheckMode,
271        tx: &mpsc::Sender<CheckEvent>,
272    ) -> anyhow::Result<()> {
273        assert!(file.path().is_absolute());
274        // Create `Progress` here so that its `drop` can advance the parent when
275        // errors occur, or when `compute_hash` is skipped.
276        let progress = self.create_file_progress(file);
277        let path_in_cache = file.relative_path(cache.base_dir());
278        match cache.get_entry(path_in_cache) {
279            Some(cached) => {
280                if mode == CheckMode::Update && cached.eq(file) {
281                    // In update mode, consider the content isn't changed if the
282                    // metadata is the same.
283                    return Ok(());
284                }
285                let mut result =
286                    FileComparisonResult::new(file.path().into(), Classification::InBoth);
287                result.update_moodified(cached.modified, file.modified());
288                if cached.size != 0 {
289                    result.update_size(cached.size, file.size());
290                }
291                if mode == CheckMode::Check && cached.size != 0 && file.size() != cached.size {
292                    // In `Check` mode, hash isn't needed if the size is different.
293                    tx.send(CheckEvent::Result(result))?;
294                    return Ok(());
295                }
296                let hash = self.compute_hash_with_progress(file, progress)?;
297                result.is_content_same = Some(hash == cached.hash);
298                if hash == cached.hash {
299                    if cached.should_update(file, mode) {
300                        cache.insert(path_in_cache, file, hash);
301                    }
302                } else {
303                    if mode == CheckMode::Update || mode == CheckMode::UpdateAll {
304                        cache.insert(path_in_cache, file, hash);
305                    }
306                    tx.send(CheckEvent::Result(result))?;
307                }
308            }
309            None => {
310                if mode == CheckMode::Update || mode == CheckMode::UpdateAll {
311                    let hash = self.compute_hash_with_progress(file, progress)?;
312                    cache.insert(path_in_cache, file, hash);
313                }
314                tx.send(CheckEvent::Result(FileComparisonResult::new(
315                    file.path().into(),
316                    Classification::OnlyInDir2,
317                )))?;
318            }
319        }
320        Ok(())
321    }
322
323    /// Executes the duplicate file finding process and prints results.
324    pub fn run(&self) -> anyhow::Result<()> {
325        let start_time = time::Instant::now();
326        let mut duplicates = self.find_duplicates()?;
327        let mut total_wasted_space = 0;
328        if !duplicates.is_empty() {
329            duplicates.sort_by_key(|a| a.size);
330            total_wasted_space = self.print_duplicates_results(&duplicates)?;
331        }
332        self.print_duplicates_summary(&start_time, total_wasted_space)?;
333        Ok(())
334    }
335
336    fn print_duplicates_results(&self, duplicates: &Vec<DuplicatedFiles>) -> anyhow::Result<u64> {
337        let mut total_wasted_space = 0;
338        for dupes in duplicates {
339            dupes.print(self.output_format)?;
340            total_wasted_space += dupes.wasted_size();
341        }
342        Ok(total_wasted_space)
343    }
344
345    fn print_duplicates_summary(
346        &self,
347        start_time: &time::Instant,
348        total_wasted_space: u64,
349    ) -> io::Result<()> {
350        let elapsed = FormattedDuration(start_time.elapsed()).to_string();
351        let num_hashed = self.num_hashed.load(atomic::Ordering::Relaxed).to_string();
352        let total_wasted_space = crate::human_readable_size(total_wasted_space);
353        let summary = [
354            ("Elapsed:", elapsed),
355            ("Hash computed:", num_hashed),
356            ("Total wasted space:", total_wasted_space),
357        ];
358        let formatter = ColumnFormatter::new(summary.iter().map(|(s, _)| *s));
359        formatter.write_values(&mut io::stderr(), &summary)
360    }
361
362    /// Finds duplicated files and returns a list of duplicate groups.
363    pub fn find_duplicates(&self) -> anyhow::Result<Vec<DuplicatedFiles>> {
364        let progress = self
365            .progress
366            .as_ref()
367            .map(|progress| progress.add_primary())
368            .unwrap_or_else(SharedProgress::none);
369        progress.set_message("Scanning directories...");
370
371        let (tx, rx) = mpsc::channel();
372        let mut by_hash: HashMap<blake3::Hash, DuplicatedFiles> = HashMap::new();
373        std::thread::scope(|scope| {
374            scope.spawn(|| {
375                if let Err(e) = self.find_duplicates_streaming(tx) {
376                    log::error!("Error during duplicate finding: {}", e);
377                }
378            });
379
380            while let Ok(event) = rx.recv() {
381                match event {
382                    DupEvent::StartHashing => progress.set_message("Hashing files..."),
383                    DupEvent::Total(value) => progress.set_length(value),
384                    DupEvent::Result(file, hash) => {
385                        progress.inc(ProgressValue::with_size(file.size()));
386                        let entry = by_hash.entry(hash).or_insert_with(|| DuplicatedFiles {
387                            paths: Vec::new(),
388                            size: file.size(),
389                        });
390                        // Hash collisions shouldn't happen, but if they do, sizes shouldn't mismatch.
391                        assert_eq!(
392                            entry.size,
393                            file.size(),
394                            "Hash collision: sizes do not match"
395                        );
396                        entry.paths.push(file.into());
397                    }
398                    DupEvent::Error => {}
399                }
400            }
401        });
402        progress.finish();
403
404        let mut duplicates = Vec::new();
405        for (_, mut dupes) in by_hash {
406            if dupes.paths.len() > 1 {
407                dupes.paths.sort();
408                duplicates.push(dupes);
409            }
410        }
411        Ok(duplicates)
412    }
413
414    fn find_duplicates_streaming(&self, tx: mpsc::Sender<DupEvent>) -> anyhow::Result<()> {
415        std::thread::scope(|global_scope| {
416            let (it_rx, caches) = self.stream_file_items(global_scope)?;
417            let caches = &caches;
418            let pool = crate::build_thread_pool(self.jobs)?;
419            pool.scope(move |scope| -> anyhow::Result<()> {
420                let mut by_size: HashMap<u64, DupState> = HashMap::new();
421                let mut total = ProgressValue::default();
422                tx.send(DupEvent::StartHashing)?;
423                for (file, dir_index) in it_rx {
424                    let size = file.size();
425                    if size == 0 {
426                        continue;
427                    }
428                    let cache = &caches[dir_index];
429                    match by_size.entry(size) {
430                        std::collections::hash_map::Entry::Occupied(mut occ) => match occ.get_mut()
431                        {
432                            DupState::Single(file0, dir_index0) => {
433                                // We found a second file of identical size.
434                                // Time to start hashing both the *original* matching file and the *new* one!
435                                let cache0 = &caches[*dir_index0];
436                                self.send_hash(file0, cache0, &tx, scope);
437                                self.send_hash(&file, cache, &tx, scope);
438                                total += ProgressValue::with_size(file0.size());
439                                total += ProgressValue::with_size(file.size());
440
441                                // Modify the state to indicate we are now fully hashing this size bucket.
442                                *occ.get_mut() = DupState::Hashing;
443                            }
444                            DupState::Hashing => {
445                                // File size bucket already hashing; just dynamically spawn the new file immediately.
446                                self.send_hash(&file, cache, &tx, scope);
447                                total += ProgressValue::with_size(file.size());
448                            }
449                        },
450                        std::collections::hash_map::Entry::Vacant(vac) => {
451                            vac.insert(DupState::Single(file, dir_index));
452                        }
453                    }
454                }
455                tx.send(DupEvent::Total(total))?;
456                Ok(())
457            })?;
458            pool.install(|| caches.into_par_iter().try_for_each(|cache| cache.save()))?;
459            Ok::<(), anyhow::Error>(())
460        })?;
461        Ok(())
462    }
463
464    fn stream_file_items<'scope, 'env>(
465        &'env self,
466        scope: &'scope std::thread::Scope<'scope, 'env>,
467    ) -> anyhow::Result<(mpsc::Receiver<FileWithDirIndex>, Vec<Arc<FileHashCache>>)> {
468        let (it_tx, it_rx) = mpsc::channel();
469        let mut caches = Vec::with_capacity(self.dirs.len());
470        for (dir_index, dir) in self.dirs.iter().enumerate() {
471            let mut it = FileIterator::new(dir);
472            let cache = FileHashCache::find_or_new(dir);
473            it.cache = Some(Arc::clone(&cache));
474            it.exclude = self.exclude.as_ref();
475            let it_tx = it_tx.clone();
476            scope.spawn(move || it.send_to_as(it_tx, |path| (path, dir_index)));
477            caches.push(cache);
478        }
479        Ok((it_rx, caches))
480    }
481
482    fn send_hash<'scope>(
483        &'scope self,
484        file: &FileItem,
485        cache: &Arc<FileHashCache>,
486        tx: &mpsc::Sender<DupEvent>,
487        scope: &rayon::Scope<'scope>,
488    ) {
489        let (hash, relative) = self
490            .get_hash_from_cache(file, cache)
491            .expect("path should be in cache base_dir");
492        if let Some(hash) = hash {
493            let _ = tx.send(DupEvent::Result(file.clone(), hash));
494            return;
495        }
496
497        let file = file.clone();
498        let relative = relative.to_path_buf();
499        let tx = tx.clone();
500        let cache = Arc::clone(cache);
501        scope.spawn(move |_| {
502            if let Ok(hash) = self.compute_hash(&file) {
503                cache.insert(&relative, &file, hash);
504                let _ = tx.send(DupEvent::Result(file, hash));
505            } else {
506                log::error!("Failed to hash file: '{}'", file);
507                let _ = tx.send(DupEvent::Error);
508            }
509        });
510    }
511
512    /// Gets the hash of a file, using the cache if available.
513    pub fn get_hash(&self, file: &FileItem) -> anyhow::Result<blake3::Hash> {
514        let cache = self.cache.as_ref().expect("cache should be initialized");
515        let (hash, relative) = self.get_hash_from_cache(file, cache)?;
516        if let Some(hash) = hash {
517            return Ok(hash);
518        }
519
520        let hash = self.compute_hash(file)?;
521        cache.insert(relative, file, hash);
522        Ok(hash)
523    }
524
525    fn get_hash_from_cache<'a>(
526        &self,
527        file: &'a FileItem,
528        cache: &FileHashCache,
529    ) -> io::Result<(Option<blake3::Hash>, &'a Path)> {
530        let relative = file.relative_path(cache.base_dir());
531        if let Some(hash) = cache.get(relative, file) {
532            self.num_hash_looked_up
533                .fetch_add(1, atomic::Ordering::Relaxed);
534            return Ok((Some(hash), relative));
535        }
536        Ok((None, relative))
537    }
538
539    fn create_file_progress(&self, file: &FileItem) -> Progress {
540        self.progress
541            .as_ref()
542            .map(|progress| progress.add_file(file.path(), file.size()))
543            .unwrap_or_else(Progress::none)
544    }
545
546    fn compute_hash(&self, file: &FileItem) -> io::Result<blake3::Hash> {
547        self.compute_hash_with_progress(file, self.create_file_progress(file))
548    }
549
550    fn compute_hash_with_progress(
551        &self,
552        file: &FileItem,
553        mut progress: Progress,
554    ) -> io::Result<blake3::Hash> {
555        let start_time = time::Instant::now();
556        let mut f = fs::File::open(file.path())?;
557        let mut hasher = blake3::Hasher::new();
558        if self.buffer_size == 0 {
559            if file.size() > 0 {
560                let mmap = unsafe { memmap2::MmapOptions::new().map(&f)? };
561                hasher.update(&mmap[..]);
562                progress.inc(ProgressValue::with_size(file.size()));
563            }
564        } else {
565            let mut buf = vec![0u8; self.buffer_size];
566            loop {
567                let n = f.read(&mut buf)?;
568                if n == 0 {
569                    break;
570                }
571                hasher.update(&buf[..n]);
572                progress.inc_size(n as u64);
573            }
574            progress.inc_file(1);
575        }
576        progress.finish();
577        self.num_hashed.fetch_add(1, atomic::Ordering::Relaxed);
578        let hash = hasher.finalize();
579        log::trace!(
580            "Computed hash in {}: '{}'",
581            FormattedDuration(start_time.elapsed()),
582            file
583        );
584        Ok(hash)
585    }
586}
587
588/// A group of duplicated files and their size.
589#[derive(Clone, Debug)]
590pub struct DuplicatedFiles {
591    pub paths: Vec<PathBuf>,
592    pub size: u64,
593}
594
595impl DuplicatedFiles {
596    fn wasted_size(&self) -> u64 {
597        self.size * (self.paths.len() as u64 - 1)
598    }
599
600    fn print(&self, output_format: OutputFormat) -> anyhow::Result<()> {
601        match output_format {
602            OutputFormat::Default => self.write_human(stdout())?,
603            OutputFormat::PowerShell => self.write_pwsh(stdout())?,
604            OutputFormat::Shell => self.write_shell(stdout())?,
605            OutputFormat::Yaml | OutputFormat::Symbol => self.write_yaml(stdout())?,
606        }
607        Ok(())
608    }
609
610    fn write_human(&self, mut writer: impl io::Write) -> anyhow::Result<()> {
611        writeln!(
612            writer,
613            "Identical {} files of {}:",
614            self.paths.len(),
615            crate::human_readable_size(self.size)
616        )?;
617        for path in &self.paths {
618            writeln!(writer, "  {}", path.display())?;
619        }
620        Ok(())
621    }
622
623    fn write_yaml(&self, mut writer: impl io::Write) -> anyhow::Result<()> {
624        writeln!(writer, "- paths:")?;
625        for path in &self.paths {
626            writeln!(writer, "  - '{}'", Self::escape_quotes_by_double(path))?;
627        }
628        writeln!(writer, "  size: {}", self.size)?;
629        Ok(())
630    }
631
632    fn write_shell(&self, writer: impl io::Write) -> anyhow::Result<()> {
633        self.write_shell_with(writer, "cp", Self::escape_quotes_for_shell)
634    }
635
636    fn write_pwsh(&self, writer: impl io::Write) -> anyhow::Result<()> {
637        self.write_shell_with(
638            writer,
639            "Copy-Item -LiteralPath",
640            Self::escape_quotes_by_double,
641        )
642    }
643
644    fn write_shell_with(
645        &self,
646        mut writer: impl io::Write,
647        cmd: &str,
648        stringify: impl Fn(&Path) -> String,
649    ) -> anyhow::Result<()> {
650        let mut iter = self.paths.iter();
651        if let Some(path0) = iter.next() {
652            let path0 = stringify(path0);
653            for path in iter {
654                writeln!(writer, "{cmd} '{path0}' '{}'", stringify(path))?;
655            }
656        }
657        Ok(())
658    }
659
660    /// Escape single quotes for sh.
661    fn escape_quotes_for_shell(path: &Path) -> String {
662        path.to_string_lossy().replace('\'', "'\\''")
663    }
664
665    /// Escape single quotes by doubling them,
666    /// for single quoted strings in zsh, PowerShell, YAML, etc.
667    fn escape_quotes_by_double(path: &Path) -> String {
668        path.to_string_lossy().replace('\'', "''")
669    }
670}
671
672#[cfg(test)]
673mod tests {
674    use super::*;
675    use std::cmp::Ordering;
676
677    fn default_exclude() -> globset::GlobSet {
678        let mut builder = globset::GlobSetBuilder::new();
679        builder.add(
680            globset::GlobBuilder::new(".hash_cache")
681                .case_insensitive(true)
682                .build()
683                .unwrap(),
684        );
685        builder.build().unwrap()
686    }
687
688    #[test]
689    fn find_duplicates() -> anyhow::Result<()> {
690        let dir = tempfile::tempdir()?;
691
692        let file1_path = dir.path().join("same1.txt");
693        fs::write(&file1_path, "same content")?;
694
695        let file2_path = dir.path().join("same2.txt");
696        fs::write(&file2_path, "same content")?;
697
698        let diff_path = dir.path().join("diff.txt");
699        fs::write(&diff_path, "different content")?;
700
701        let mut hasher = FileHasher::new(&[dir.path()])?;
702        hasher.buffer_size = 8192;
703        let duplicates = hasher.find_duplicates()?;
704
705        assert_eq!(hasher.num_hashed.load(atomic::Ordering::Relaxed), 2);
706        assert_eq!(hasher.num_hash_looked_up.load(atomic::Ordering::Relaxed), 0);
707
708        assert_eq!(duplicates.len(), 1);
709        let group = &duplicates[0];
710        assert_eq!(group.paths.len(), 2);
711        assert_eq!(group.size, 12); // "same content" is 12 bytes
712
713        assert!(group.paths.contains(&file1_path));
714        assert!(group.paths.contains(&file2_path));
715
716        Ok(())
717    }
718
719    #[test]
720    fn find_duplicates_merge_cache() -> anyhow::Result<()> {
721        let dir = tempfile::tempdir()?;
722        let dir_path = dir.path();
723
724        let sub_dir = dir_path.join("a").join("a");
725        fs::create_dir_all(&sub_dir)?;
726
727        let file1_path = sub_dir.join("1");
728        fs::write(&file1_path, "same content")?;
729
730        let file2_path = sub_dir.join("2");
731        fs::write(&file2_path, "same content")?;
732
733        // Create empty cache file in a/a to force it to be the cache base
734        let cache_aa_path = sub_dir.join(FileHashCache::FILE_NAME);
735        fs::File::create(&cache_aa_path)?;
736
737        // Run find_duplicates on a/a
738        let hasher_aa = FileHasher::new(&[&sub_dir])?;
739        let duplicates_aa = hasher_aa.find_duplicates()?;
740        assert_eq!(duplicates_aa.len(), 1);
741        assert!(cache_aa_path.exists());
742        assert_eq!(hasher_aa.num_hashed.load(atomic::Ordering::Relaxed), 2);
743        assert_eq!(
744            hasher_aa.num_hash_looked_up.load(atomic::Ordering::Relaxed),
745            0
746        );
747
748        // Create empty cache file in a to force it to be the cache base
749        let root_a = dir_path.join("a");
750        let cache_a_path = root_a.join(FileHashCache::FILE_NAME);
751        fs::File::create(&cache_a_path)?;
752
753        // Run find_duplicates on a
754        let hasher_a = FileHasher::new(&[&root_a])?;
755        let duplicates_a = hasher_a.find_duplicates()?;
756        assert_eq!(duplicates_a.len(), 1);
757        assert_eq!(hasher_a.num_hashed.load(atomic::Ordering::Relaxed), 0);
758        assert_eq!(
759            hasher_a.num_hash_looked_up.load(atomic::Ordering::Relaxed),
760            2
761        );
762
763        // The merged child cache should be removed.
764        assert!(cache_a_path.exists());
765        assert!(!cache_aa_path.exists());
766
767        Ok(())
768    }
769
770    #[test]
771    fn find_duplicates_with_exclude() -> anyhow::Result<()> {
772        let dir = tempfile::tempdir()?;
773
774        let file1_path = dir.path().join("same1.txt");
775        fs::write(&file1_path, "same content")?;
776
777        let file2_path = dir.path().join("same2.txt");
778        fs::write(&file2_path, "same content")?;
779
780        let exclude_path = dir.path().join("exclude.txt");
781        fs::write(&exclude_path, "same content")?;
782
783        let mut hasher = FileHasher::new(&[dir.path()])?;
784        hasher.buffer_size = 8192;
785        let mut builder = globset::GlobSetBuilder::new();
786        builder.add(
787            globset::GlobBuilder::new("exclude.txt")
788                .case_insensitive(true)
789                .build()?,
790        );
791        let filter = builder.build()?;
792        hasher.exclude = Some(filter);
793
794        let duplicates = hasher.find_duplicates()?;
795        assert_eq!(duplicates.len(), 1);
796        let group = &duplicates[0];
797        assert_eq!(group.paths.len(), 2);
798        assert!(group.paths.contains(&file1_path));
799        assert!(group.paths.contains(&file2_path));
800        assert!(!group.paths.contains(&exclude_path));
801        Ok(())
802    }
803
804    #[derive(Default)]
805    struct CheckCollector {
806        start_seen: bool,
807        total_files: Option<u64>,
808        results: Vec<FileComparisonResult>,
809        num_error: usize,
810    }
811
812    impl CheckCollector {
813        fn collect(rx: mpsc::Receiver<CheckEvent>, base_dir: &Path) -> Self {
814            let mut collector = Self::default();
815            collector._collect(rx, base_dir);
816            collector
817        }
818
819        fn _collect(&mut self, rx: mpsc::Receiver<CheckEvent>, base_dir: &Path) {
820            while let Ok(event) = rx.recv() {
821                match event {
822                    CheckEvent::StartChecking => self.start_seen = true,
823                    CheckEvent::Total(total) => self.total_files = Some(total.num_files),
824                    CheckEvent::Result(mut result) => {
825                        result.relative_path = result
826                            .relative_path
827                            .strip_prefix(base_dir)
828                            .unwrap()
829                            .to_path_buf();
830                        self.results.push(result);
831                    }
832                    CheckEvent::Error(_) => self.num_error += 1,
833                }
834            }
835        }
836    }
837
838    #[test]
839    fn check_mode_empty_cache() -> anyhow::Result<()> {
840        let dir = tempfile::tempdir()?;
841        let dir_path = dir.path().to_path_buf();
842        println!("{:?}", dir_path);
843        let file1_path = dir.path().join("file1.txt");
844        fs::write(&file1_path, "content 1")?;
845        let file2_path = dir.path().join("file2.txt");
846        fs::write(&file2_path, "content 2")?;
847
848        let mut hasher = FileHasher::new(&[&dir_path])?;
849        hasher.exclude = Some(default_exclude());
850        let (tx, rx) = mpsc::channel();
851        hasher.check_streaming(tx, CheckMode::Check)?;
852        let collector = CheckCollector::collect(rx, &dir_path);
853        assert!(collector.start_seen);
854        assert_eq!(collector.total_files, Some(2));
855        assert_eq!(collector.num_error, 0);
856
857        let mut results = collector.results;
858        results.sort_by(|a, b| a.relative_path.cmp(&b.relative_path));
859        assert_eq!(results.len(), 2);
860        assert_eq!(results[0].relative_path, Path::new("file1.txt"));
861        assert_eq!(results[0].classification, Classification::OnlyInDir2);
862        assert_eq!(results[1].relative_path, Path::new("file2.txt"));
863        assert_eq!(results[1].classification, Classification::OnlyInDir2);
864
865        assert!(!dir.path().join(FileHashCache::FILE_NAME).exists());
866        Ok(())
867    }
868
869    #[test]
870    fn check_mode_with_cache() -> anyhow::Result<()> {
871        let dir = tempfile::tempdir()?;
872        let dir_path = dir.path().to_path_buf();
873        let file1_path = dir.path().join("file1.txt");
874        let file2_path = dir.path().join("file2.txt");
875        fs::write(&file1_path, "content 1")?;
876        fs::write(&file2_path, "content 2")?;
877        let file1 = FileItem::try_from(file1_path.as_path())?;
878        let file2 = FileItem::try_from(file2_path.as_path())?;
879
880        let mut hasher = FileHasher::new_with_cache(&[&dir_path])?;
881        hasher.exclude = Some(default_exclude());
882        let _hash1 = hasher.get_hash(&file1)?;
883        let _hash2 = hasher.get_hash(&file2)?;
884        hasher.save_cache()?;
885        assert!(dir.path().join(FileHashCache::FILE_NAME).exists());
886
887        let mut hasher = FileHasher::new(&[&dir_path])?;
888        hasher.exclude = Some(default_exclude());
889        let (tx, rx) = mpsc::channel();
890        hasher.check_streaming(tx, CheckMode::Check)?;
891        let collector = CheckCollector::collect(rx, &dir_path);
892        assert_eq!(collector.results.len(), 0);
893
894        fs::write(&file1_path, "content 1 modified")?;
895
896        let file2_meta_before = fs::metadata(&file2_path)?;
897        let mtime_before = file2_meta_before.modified()?;
898        std::thread::sleep(time::Duration::from_millis(10));
899        fs::write(&file2_path, "content 2")?;
900        let file2_meta_after = fs::metadata(&file2_path)?;
901        let mtime_after = file2_meta_after.modified()?;
902        assert!(mtime_after > mtime_before);
903
904        let mut hasher = FileHasher::new(&[&dir_path])?;
905        hasher.exclude = Some(default_exclude());
906        let (tx, rx) = mpsc::channel();
907        hasher.check_streaming(tx, CheckMode::Check)?;
908        let collector = CheckCollector::collect(rx, &dir_path);
909        assert_eq!(collector.results.len(), 1);
910        let results = collector.results;
911        assert_eq!(results[0].relative_path, Path::new("file1.txt"));
912        assert_eq!(results[0].modified_time_comparison, Some(Ordering::Less));
913        assert_eq!(results[0].size_comparison, Some(Ordering::Less));
914        assert_eq!(results[0].is_content_same, None);
915        Ok(())
916    }
917
918    #[test]
919    fn check_update_mode() -> anyhow::Result<()> {
920        let dir = tempfile::tempdir()?;
921        let dir_path = dir.path().to_path_buf();
922        let file1_path = dir.path().join("file1.txt");
923        fs::write(&file1_path, "content 1")?;
924
925        let mut hasher = FileHasher::new(&[&dir_path])?;
926        hasher.exclude = Some(default_exclude());
927        let (tx, rx) = mpsc::channel();
928        hasher.check_streaming(tx, CheckMode::UpdateAll)?;
929        let _ = CheckCollector::collect(rx, &dir_path);
930        hasher.save_cache()?;
931        assert!(dir.path().join(FileHashCache::FILE_NAME).exists());
932
933        let cache = FileHashCache::new(&dir_path);
934        let file1 = FileItem::try_from(file1_path.as_path())?;
935        let hash1 = cache.get(&PathBuf::from("file1.txt"), &file1);
936        assert!(hash1.is_some());
937
938        std::thread::sleep(time::Duration::from_millis(10));
939        fs::write(&file1_path, "content 1 modified")?;
940        let file1_mod = FileItem::try_from(file1_path.as_path())?;
941
942        let mut hasher = FileHasher::new(&[&dir_path])?;
943        hasher.exclude = Some(default_exclude());
944        let (tx, rx) = mpsc::channel();
945        hasher.check_streaming(tx, CheckMode::UpdateAll)?;
946        let _ = CheckCollector::collect(rx, &dir_path);
947        hasher.save_cache()?;
948
949        let cache = FileHashCache::new(&dir_path);
950        let hash_mod = cache.get(&PathBuf::from("file1.txt"), &file1_mod);
951        assert!(hash_mod.is_some());
952        assert_ne!(hash1, hash_mod);
953
954        std::thread::sleep(time::Duration::from_millis(10));
955        fs::write(&file1_path, "content 1 modified")?;
956        let file1_mod2 = FileItem::try_from(file1_path.as_path())?;
957        assert!(file1_mod2.modified() > file1_mod.modified());
958
959        assert!(
960            cache
961                .get(&PathBuf::from("file1.txt"), &file1_mod2)
962                .is_none()
963        );
964
965        let mut hasher = FileHasher::new(&[&dir_path])?;
966        hasher.exclude = Some(default_exclude());
967        let (tx, rx) = mpsc::channel();
968        hasher.check_streaming(tx, CheckMode::UpdateAll)?;
969        let _ = CheckCollector::collect(rx, &dir_path);
970        hasher.save_cache()?;
971
972        let cache = FileHashCache::new(&dir_path);
973        assert!(
974            cache
975                .get(&PathBuf::from("file1.txt"), &file1_mod2)
976                .is_some()
977        );
978        Ok(())
979    }
980
981    #[test]
982    fn check_cleanup_deleted_files() -> anyhow::Result<()> {
983        let dir = tempfile::tempdir()?;
984        let dir_path = dir.path().to_path_buf();
985        let file1_path = dir.path().join("file1.txt");
986        let file2_path = dir.path().join("file2.txt");
987        fs::write(&file1_path, "content 1")?;
988        fs::write(&file2_path, "content 2")?;
989        let file1 = FileItem::try_from(file1_path.as_path())?;
990        let file2 = FileItem::try_from(file2_path.as_path())?;
991
992        let mut hasher = FileHasher::new(&[&dir_path])?;
993        hasher.exclude = Some(default_exclude());
994        let (tx, rx) = mpsc::channel();
995        hasher.check_streaming(tx, CheckMode::UpdateAll)?;
996        let _ = CheckCollector::collect(rx, &dir_path);
997        hasher.save_cache()?;
998
999        // Verify both are in the cache
1000        let cache = FileHashCache::new(&dir_path);
1001        assert!(cache.get(&PathBuf::from("file1.txt"), &file1).is_some());
1002        assert!(cache.get(&PathBuf::from("file2.txt"), &file2).is_some());
1003
1004        // Now delete file2 from disk
1005        fs::remove_file(&file2_path)?;
1006
1007        // Run check and save again
1008        let mut hasher = FileHasher::new(&[&dir_path])?;
1009        hasher.exclude = Some(default_exclude());
1010        let (tx, rx) = mpsc::channel();
1011        hasher.check_streaming(tx, CheckMode::UpdateAll)?;
1012        let _ = CheckCollector::collect(rx, &dir_path);
1013        hasher.save_cache()?;
1014
1015        // Verify file2 is removed from cache, but file1 is still there
1016        let cache = FileHashCache::new(&dir_path);
1017        assert!(cache.get(&PathBuf::from("file2.txt"), &file2).is_none());
1018        assert!(cache.get(&PathBuf::from("file1.txt"), &file1).is_some());
1019        Ok(())
1020    }
1021
1022    #[test]
1023    fn find_duplicates_multiple_dirs() -> anyhow::Result<()> {
1024        let tmp = tempfile::tempdir()?;
1025        let dir1 = tmp.path().join("dir1");
1026        let dir2 = tmp.path().join("dir2");
1027        fs::create_dir(&dir1)?;
1028        fs::create_dir(&dir2)?;
1029        let file1_path = dir1.join("file1.txt");
1030        fs::write(&file1_path, "same content")?;
1031        let file2_path = dir2.join("file2.txt");
1032        fs::write(&file2_path, "same content")?;
1033        let hasher = FileHasher::new(&[&dir1, &dir2])?;
1034        let duplicates = hasher.find_duplicates()?;
1035        assert_eq!(duplicates.len(), 1);
1036        let group = &duplicates[0];
1037        assert_eq!(group.paths.len(), 2);
1038        assert_eq!(group.size, 12);
1039        assert!(group.paths.contains(&file1_path));
1040        assert!(group.paths.contains(&file2_path));
1041
1042        Ok(())
1043    }
1044
1045    #[test]
1046    fn check_fails_with_multiple_dirs() -> anyhow::Result<()> {
1047        let tmp = tempfile::tempdir()?;
1048        let dir1 = tmp.path().join("dir1");
1049        let dir2 = tmp.path().join("dir2");
1050        fs::create_dir(&dir1)?;
1051        fs::create_dir(&dir2)?;
1052        let hasher = FileHasher::new(&[&dir1, &dir2])?;
1053        assert!(hasher.check(CheckMode::Check).is_err());
1054        Ok(())
1055    }
1056
1057    #[test]
1058    fn check_update_mode_only_on_change() -> anyhow::Result<()> {
1059        let dir = tempfile::tempdir()?;
1060        let dir_path = dir.path().to_path_buf();
1061        let file1_path = dir.path().join("file1.txt");
1062        fs::write(&file1_path, "content 1")?;
1063
1064        // 1. Initial check under UpdateAll/Update (should compute hash and insert since new)
1065        let mut hasher = FileHasher::new(&[&dir_path])?;
1066        hasher.exclude = Some(default_exclude());
1067        let (tx, rx) = mpsc::channel();
1068        hasher.check_streaming(tx, CheckMode::Update)?;
1069        let _ = CheckCollector::collect(rx, &dir_path);
1070        hasher.save_cache()?;
1071        assert_eq!(hasher.num_hashed.load(atomic::Ordering::Relaxed), 1);
1072
1073        // 2. Second check under Update. Since metadata is unchanged, it should be a no-op (0 hashes computed).
1074        let mut hasher = FileHasher::new(&[&dir_path])?;
1075        hasher.exclude = Some(default_exclude());
1076        let (tx, rx) = mpsc::channel();
1077        hasher.check_streaming(tx, CheckMode::Update)?;
1078        let _ = CheckCollector::collect(rx, &dir_path);
1079        assert_eq!(hasher.num_hashed.load(atomic::Ordering::Relaxed), 0);
1080
1081        // 3. Sleep, modify file, and check again under Update. Metadata changed, so it should update (1 hash computed).
1082        std::thread::sleep(time::Duration::from_millis(10));
1083        fs::write(&file1_path, "content 1 modified")?;
1084
1085        let mut hasher = FileHasher::new(&[&dir_path])?;
1086        hasher.exclude = Some(default_exclude());
1087        let (tx, rx) = mpsc::channel();
1088        hasher.check_streaming(tx, CheckMode::Update)?;
1089        let _ = CheckCollector::collect(rx, &dir_path);
1090        assert_eq!(hasher.num_hashed.load(atomic::Ordering::Relaxed), 1);
1091
1092        Ok(())
1093    }
1094
1095    #[test]
1096    fn escape_shell() {
1097        let escape_shell = |p: &str| DuplicatedFiles::escape_quotes_for_shell(Path::new(p));
1098        assert_eq!(escape_shell(""), "");
1099        assert_eq!(escape_shell("abc"), "abc");
1100        assert_eq!(escape_shell("a'b"), "a'\\''b");
1101        assert_eq!(escape_shell("a'b'"), "a'\\''b'\\''");
1102
1103        let escape_shell_double = |p: &str| DuplicatedFiles::escape_quotes_by_double(Path::new(p));
1104        assert_eq!(escape_shell_double(""), "");
1105        assert_eq!(escape_shell_double("abc"), "abc");
1106        assert_eq!(escape_shell_double("a'b"), "a''b");
1107        assert_eq!(escape_shell_double("a'b'"), "a''b''");
1108    }
1109
1110    #[test]
1111    fn write_dups_shell_empty() -> anyhow::Result<()> {
1112        let dup_empty = DuplicatedFiles {
1113            paths: vec![],
1114            size: 100,
1115        };
1116        let mut buf = Vec::new();
1117        dup_empty.write_shell(&mut buf)?;
1118        assert_eq!(String::from_utf8(buf)?, "");
1119        Ok(())
1120    }
1121
1122    #[test]
1123    fn write_dups_shell_one() -> anyhow::Result<()> {
1124        let dup_one = DuplicatedFiles {
1125            paths: vec![PathBuf::from("a.txt")],
1126            size: 100,
1127        };
1128        let mut buf = Vec::new();
1129        dup_one.write_shell(&mut buf)?;
1130        assert_eq!(String::from_utf8(buf)?, "");
1131        Ok(())
1132    }
1133
1134    #[test]
1135    fn write_dups_shell_two() -> anyhow::Result<()> {
1136        let dup_multiple = DuplicatedFiles {
1137            paths: vec![PathBuf::from("a.txt"), PathBuf::from("b.txt")],
1138            size: 100,
1139        };
1140        let mut buf = Vec::new();
1141        dup_multiple.write_shell(&mut buf)?;
1142        assert_eq!(String::from_utf8(buf)?, "cp 'a.txt' 'b.txt'\n");
1143        Ok(())
1144    }
1145
1146    #[test]
1147    fn write_dups_shell_three() -> anyhow::Result<()> {
1148        let dup_multiple = DuplicatedFiles {
1149            paths: vec![
1150                PathBuf::from("a.txt"),
1151                PathBuf::from("b.txt"),
1152                PathBuf::from("c.txt"),
1153            ],
1154            size: 100,
1155        };
1156        let mut buf = Vec::new();
1157        dup_multiple.write_shell(&mut buf)?;
1158        assert_eq!(
1159            String::from_utf8(buf)?,
1160            "cp 'a.txt' 'b.txt'\ncp 'a.txt' 'c.txt'\n"
1161        );
1162        Ok(())
1163    }
1164
1165    #[test]
1166    fn write_dups_shell_quotes() -> anyhow::Result<()> {
1167        let dup_quotes = DuplicatedFiles {
1168            paths: vec![PathBuf::from("a'b.txt"), PathBuf::from("c'd.txt")],
1169            size: 100,
1170        };
1171        let mut buf = Vec::new();
1172        dup_quotes.write_shell(&mut buf)?;
1173        assert_eq!(String::from_utf8(buf)?, "cp 'a'\\''b.txt' 'c'\\''d.txt'\n");
1174
1175        let mut buf = Vec::new();
1176        dup_quotes.write_pwsh(&mut buf)?;
1177        assert_eq!(
1178            String::from_utf8(buf)?,
1179            "Copy-Item -LiteralPath 'a''b.txt' 'c''d.txt'\n"
1180        );
1181        Ok(())
1182    }
1183}
compare_dir/file_hasher.rs

compare_dir/
file_hasher.rs
