compare_dir/
file_hasher.rs

1use crate::{
2    Classification, ColumnFormatter, DirectoryComparer, FileComparer, FileComparisonResult,
3    FileHashCache, FileItem, FileIterator, OutputFormat, Progress, ProgressBuilder, ProgressValue,
4    SharedProgress,
5};
6use globset::GlobSet;
7use indicatif::FormattedDuration;
8use rayon::prelude::*;
9use simple_path::SimplePath;
10use std::{
11    collections::HashMap,
12    fs,
13    io::{self, Read, stdout},
14    path::{Path, PathBuf},
15    sync::{
16        Arc,
17        atomic::{self, AtomicUsize},
18        mpsc,
19    },
20    time,
21};
22
23type FileWithDirIndex = (FileItem, usize);
24
25#[derive(Debug, Clone)]
26enum DupEvent {
27    StartHashing,
28    Total(ProgressValue),
29    Result(FileItem, blake3::Hash),
30    Error,
31}
32
33#[derive(Debug)]
34enum CheckEvent {
35    StartChecking,
36    Total(ProgressValue),
37    Result(FileComparisonResult),
38    Progress(ProgressValue),
39    Error(FileItem),
40}
41
42enum DupState {
43    Single(FileItem, usize),
44    Hashing,
45}
46
47/// A tool for finding duplicated files in a directory.
48pub struct FileHasher {
49    dirs: Vec<PathBuf>,
50    pub buffer_size: usize,
51    cache: Option<Arc<FileHashCache>>,
52    num_hashed: AtomicUsize,
53    num_hash_looked_up: AtomicUsize,
54    pub exclude: Option<GlobSet>,
55    pub progress: Option<Arc<ProgressBuilder>>,
56    pub output_format: OutputFormat,
57    pub jobs: usize,
58}
59
60impl FileHasher {
61    const DEFAULT_JOBS: usize = DirectoryComparer::DEFAULT_JOBS;
62
63    /// Creates a new `FileHasher` for the given directories.
64    pub fn new<P: AsRef<Path>>(dirs: &[P]) -> anyhow::Result<Self> {
65        if dirs.is_empty() {
66            anyhow::bail!("At least one directory must be specified.");
67        }
68        Ok(Self {
69            dirs: dirs.iter().map(|p| p.as_ref().to_path_buf()).collect(),
70            buffer_size: FileComparer::DEFAULT_BUFFER_SIZE,
71            cache: None,
72            num_hashed: AtomicUsize::new(0),
73            num_hash_looked_up: AtomicUsize::new(0),
74            exclude: None,
75            progress: None,
76            output_format: OutputFormat::Default,
77            jobs: Self::DEFAULT_JOBS,
78        })
79    }
80
81    pub(crate) fn new_with_cache<P: AsRef<Path>>(dirs: &[P]) -> anyhow::Result<Self> {
82        let mut hasher = Self::new(dirs)?;
83        hasher.cache = Some(hasher.new_cache()?);
84        Ok(hasher)
85    }
86
87    fn new_cache(&self) -> anyhow::Result<Arc<FileHashCache>> {
88        let common_ancestor = crate::common_ancestor(&self.dirs)
89            .ok_or_else(|| anyhow::anyhow!("No common ancestor found"))?;
90        Ok(FileHashCache::find_or_new(&common_ancestor))
91    }
92
93    /// Gets the hash cache.
94    pub(crate) fn cache(&mut self) -> anyhow::Result<Arc<FileHashCache>> {
95        if self.cache.is_none() {
96            self.cache = Some(self.new_cache()?);
97        }
98        Ok(Arc::clone(self.cache.as_ref().unwrap()))
99    }
100
101    /// Remove a cache entry if it exists.
102    pub(crate) fn remove_cache_entry(&mut self, path: &Path) -> anyhow::Result<()> {
103        let cache = self.cache()?;
104        let relative = SimplePath::strip_prefix(path, cache.base_dir())?;
105        cache.remove(relative);
106        Ok(())
107    }
108
109    /// Save the hash cache if it is dirty.
110    pub fn save_cache(&self) -> anyhow::Result<()> {
111        log::info!(
112            "Hash stats for {:?}: {} computed, {} looked up",
113            self.dirs,
114            self.num_hashed.load(atomic::Ordering::Relaxed),
115            self.num_hash_looked_up.load(atomic::Ordering::Relaxed)
116        );
117        if let Some(cache) = &self.cache {
118            cache.save()?;
119        }
120        Ok(())
121    }
122
123    /// Clears the loaded hashes in the cache.
124    pub(crate) fn clear_cache(&mut self) -> anyhow::Result<()> {
125        let cache = self.cache()?;
126        for dir in &self.dirs {
127            let relative = SimplePath::strip_prefix(dir, cache.base_dir())?;
128            cache.clear(relative);
129        }
130        Ok(())
131    }
132
133    /// Executes the check/update process.
134    pub fn check(&self, update: bool) -> anyhow::Result<()> {
135        match self.output_format {
136            OutputFormat::Default | OutputFormat::Symbol => {}
137            _ => anyhow::bail!("Check mode only supports default or symbol output format."),
138        }
139        if self.dirs.len() > 1 {
140            anyhow::bail!("Check mode only supports one directory.");
141        }
142        let start_time = time::Instant::now();
143        if let Some(p) = &self.progress {
144            p.set_propagate();
145        }
146        let progress = self
147            .progress
148            .as_ref()
149            .map(|progress| progress.add_primary())
150            .unwrap_or_else(SharedProgress::none);
151        progress.use_bytes();
152        progress.set_message("Scanning directory...");
153        let mut num_new = 0;
154        let mut num_modified = 0;
155        let mut num_error = 0;
156        std::thread::scope(|scope| {
157            let (tx, rx) = mpsc::channel();
158            scope.spawn(|| {
159                if let Err(e) = self.check_streaming(tx, update) {
160                    log::error!("Error during check: {}", e);
161                }
162            });
163            while let Ok(event) = rx.recv() {
164                match event {
165                    CheckEvent::StartChecking => {
166                        progress.set_message("Checking files...");
167                    }
168                    CheckEvent::Total(value) => {
169                        progress.set_length(value);
170                        progress.set_message("");
171                    }
172                    CheckEvent::Result(result) => {
173                        progress.suspend_for(stdout(), || {
174                            result.print(self.output_format, "cached", "current")
175                        });
176                        if result.classification == Classification::OnlyInDir2 {
177                            num_new += 1;
178                        } else if result.is_identical_content() == Some(false) {
179                            num_modified += 1;
180                        }
181                    }
182                    CheckEvent::Progress(value) => progress.inc(value),
183                    CheckEvent::Error(file) => {
184                        progress.inc(ProgressValue::with_skip(file.size()));
185                        num_error += 1;
186                    }
187                }
188            }
189        });
190        progress.finish();
191        self.print_check_summary(&start_time, num_new, num_modified, num_error)?;
192        Ok(())
193    }
194
195    fn print_check_summary(
196        &self,
197        start_time: &time::Instant,
198        num_new: usize,
199        num_modified: usize,
200        num_error: usize,
201    ) -> io::Result<()> {
202        let summary = [
203            ("Elapsed:", 0),
204            (
205                "Hash computed:",
206                self.num_hashed.load(atomic::Ordering::Relaxed),
207            ),
208            ("New files:", num_new),
209            ("Modified files:", num_modified),
210            ("Errors:", num_error),
211        ];
212        let formatter = ColumnFormatter::new(summary.iter().map(|(s, _)| *s));
213        let mut writer = std::io::stderr();
214        formatter.write_value(
215            &mut writer,
216            summary[0].0,
217            FormattedDuration(start_time.elapsed()),
218        )?;
219        formatter.write_values(&mut writer, &summary[1..])
220    }
221
222    fn check_streaming(&self, tx: mpsc::Sender<CheckEvent>, update: bool) -> anyhow::Result<()> {
223        assert_eq!(self.dirs.len(), 1);
224        let cache = self.new_cache()?;
225        let base_dir = &self.dirs[0];
226        let relative = SimplePath::strip_prefix(base_dir, cache.base_dir())?;
227        cache.set_remove_if_no_access(relative);
228        let cache_clone = Arc::clone(&cache);
229        std::thread::scope(|global_scope| {
230            let mut it = FileIterator::new(base_dir);
231            it.cache = Some(Arc::clone(&cache));
232            it.exclude = self.exclude.as_ref();
233            let it_rx = it.spawn_in_scope(global_scope);
234            tx.send(CheckEvent::StartChecking)?;
235            let pool = crate::build_thread_pool(self.jobs)?;
236            pool.scope(move |scope| -> anyhow::Result<()> {
237                let mut total = ProgressValue::default();
238                for file in it_rx {
239                    self.check_file(file, &cache, update, &mut total, &tx, scope);
240                }
241                tx.send(CheckEvent::Total(total))?;
242                Ok(())
243            })
244        })?;
245        cache_clone.save()?;
246        Ok(())
247    }
248
249    fn check_file<'scope>(
250        &'scope self,
251        file: FileItem,
252        cache: &Arc<FileHashCache>,
253        update: bool,
254        total: &mut ProgressValue,
255        tx: &mpsc::Sender<CheckEvent>,
256        scope: &rayon::Scope<'scope>,
257    ) {
258        *total += ProgressValue::with_size(file.size());
259        let tx = tx.clone();
260        let cache = Arc::clone(cache);
261        scope.spawn(move |_| {
262            if let Err(error) = self._check_file(&file, cache, update, &tx) {
263                log::error!("Failed to check file '{}': {}", file, error);
264                if tx.send(CheckEvent::Error(file)).is_err() {
265                    log::error!("Send failed");
266                }
267            }
268        });
269    }
270
271    fn _check_file(
272        &self,
273        file: &FileItem,
274        cache: Arc<FileHashCache>,
275        update: bool,
276        tx: &mpsc::Sender<CheckEvent>,
277    ) -> anyhow::Result<()> {
278        assert!(file.path().is_absolute());
279        let path_in_cache = file.relative_path(cache.base_dir());
280        match cache.get_entry(path_in_cache) {
281            Some(cached) => {
282                let mut result =
283                    FileComparisonResult::new(file.path().into(), Classification::InBoth);
284                result.update_moodified(cached.modified, file.modified());
285                if cached.size != 0 {
286                    result.update_size(cached.size, file.size());
287                }
288                if !update && cached.size != 0 && file.size() != cached.size {
289                    tx.send(CheckEvent::Result(result))?;
290                    tx.send(CheckEvent::Progress(ProgressValue::with_skip(file.size())))?;
291                    return Ok(());
292                }
293                let hash = self.compute_hash(file)?;
294                result.is_content_same = Some(hash == cached.hash);
295                if hash == cached.hash {
296                    if cached.should_update(file, update) {
297                        cache.insert(path_in_cache, file, hash);
298                    }
299                } else {
300                    if update {
301                        cache.insert(path_in_cache, file, hash);
302                    }
303                    tx.send(CheckEvent::Result(result))?;
304                }
305            }
306            None => {
307                if update {
308                    let hash = self.compute_hash(file)?;
309                    cache.insert(path_in_cache, file, hash);
310                } else {
311                    tx.send(CheckEvent::Progress(ProgressValue::with_skip(file.size())))?;
312                }
313                tx.send(CheckEvent::Result(FileComparisonResult::new(
314                    file.path().into(),
315                    Classification::OnlyInDir2,
316                )))?;
317            }
318        }
319        Ok(())
320    }
321
322    /// Executes the duplicate file finding process and prints results.
323    pub fn run(&self) -> anyhow::Result<()> {
324        let start_time = time::Instant::now();
325        let mut duplicates = self.find_duplicates()?;
326        let mut total_wasted_space = 0;
327        if !duplicates.is_empty() {
328            duplicates.sort_by_key(|a| a.size);
329            total_wasted_space = self.print_duplicates_results(&duplicates)?;
330        }
331        self.print_duplicates_summary(&start_time, total_wasted_space)?;
332        Ok(())
333    }
334
335    fn print_duplicates_results(&self, duplicates: &Vec<DuplicatedFiles>) -> anyhow::Result<u64> {
336        let mut total_wasted_space = 0;
337        for dupes in duplicates {
338            dupes.print(self.output_format)?;
339            total_wasted_space += dupes.wasted_size();
340        }
341        Ok(total_wasted_space)
342    }
343
344    fn print_duplicates_summary(
345        &self,
346        start_time: &time::Instant,
347        total_wasted_space: u64,
348    ) -> io::Result<()> {
349        let elapsed = FormattedDuration(start_time.elapsed()).to_string();
350        let num_hashed = self.num_hashed.load(atomic::Ordering::Relaxed).to_string();
351        let total_wasted_space = crate::human_readable_size(total_wasted_space);
352        let summary = [
353            ("Elapsed:", elapsed),
354            ("Hash computed:", num_hashed),
355            ("Total wasted space:", total_wasted_space),
356        ];
357        let formatter = ColumnFormatter::new(summary.iter().map(|(s, _)| *s));
358        formatter.write_values(&mut io::stderr(), &summary)
359    }
360
361    /// Finds duplicated files and returns a list of duplicate groups.
362    pub fn find_duplicates(&self) -> anyhow::Result<Vec<DuplicatedFiles>> {
363        let progress = self
364            .progress
365            .as_ref()
366            .map(|progress| progress.add_primary())
367            .unwrap_or_else(SharedProgress::none);
368        progress.set_message("Scanning directories...");
369
370        let (tx, rx) = mpsc::channel();
371        let mut by_hash: HashMap<blake3::Hash, DuplicatedFiles> = HashMap::new();
372        std::thread::scope(|scope| {
373            scope.spawn(|| {
374                if let Err(e) = self.find_duplicates_streaming(tx) {
375                    log::error!("Error during duplicate finding: {}", e);
376                }
377            });
378
379            while let Ok(event) = rx.recv() {
380                match event {
381                    DupEvent::StartHashing => progress.set_message("Hashing files..."),
382                    DupEvent::Total(value) => progress.set_length(value),
383                    DupEvent::Result(file, hash) => {
384                        progress.inc(ProgressValue::with_size(file.size()));
385                        let entry = by_hash.entry(hash).or_insert_with(|| DuplicatedFiles {
386                            paths: Vec::new(),
387                            size: file.size(),
388                        });
389                        // Hash collisions shouldn't happen, but if they do, sizes shouldn't mismatch.
390                        assert_eq!(
391                            entry.size,
392                            file.size(),
393                            "Hash collision: sizes do not match"
394                        );
395                        entry.paths.push(file.into_path_buf());
396                    }
397                    DupEvent::Error => {}
398                }
399            }
400        });
401        progress.finish();
402
403        let mut duplicates = Vec::new();
404        for (_, mut dupes) in by_hash {
405            if dupes.paths.len() > 1 {
406                dupes.paths.sort();
407                duplicates.push(dupes);
408            }
409        }
410        Ok(duplicates)
411    }
412
413    fn find_duplicates_streaming(&self, tx: mpsc::Sender<DupEvent>) -> anyhow::Result<()> {
414        std::thread::scope(|global_scope| {
415            let (it_rx, caches) = self.stream_file_items(global_scope)?;
416            let caches = &caches;
417            let pool = crate::build_thread_pool(self.jobs)?;
418            pool.scope(move |scope| -> anyhow::Result<()> {
419                let mut by_size: HashMap<u64, DupState> = HashMap::new();
420                let mut total = ProgressValue::default();
421                tx.send(DupEvent::StartHashing)?;
422                for (file, dir_index) in it_rx {
423                    let size = file.size();
424                    if size == 0 {
425                        continue;
426                    }
427                    let cache = &caches[dir_index];
428                    match by_size.entry(size) {
429                        std::collections::hash_map::Entry::Occupied(mut occ) => match occ.get_mut()
430                        {
431                            DupState::Single(file0, dir_index0) => {
432                                // We found a second file of identical size.
433                                // Time to start hashing both the *original* matching file and the *new* one!
434                                let cache0 = &caches[*dir_index0];
435                                self.send_hash(file0, cache0, &tx, scope);
436                                self.send_hash(&file, cache, &tx, scope);
437                                total += ProgressValue::with_size(file0.size());
438                                total += ProgressValue::with_size(file.size());
439
440                                // Modify the state to indicate we are now fully hashing this size bucket.
441                                *occ.get_mut() = DupState::Hashing;
442                            }
443                            DupState::Hashing => {
444                                // File size bucket already hashing; just dynamically spawn the new file immediately.
445                                self.send_hash(&file, cache, &tx, scope);
446                                total += ProgressValue::with_size(file.size());
447                            }
448                        },
449                        std::collections::hash_map::Entry::Vacant(vac) => {
450                            vac.insert(DupState::Single(file, dir_index));
451                        }
452                    }
453                }
454                tx.send(DupEvent::Total(total))?;
455                Ok(())
456            })?;
457            pool.install(|| caches.into_par_iter().try_for_each(|cache| cache.save()))?;
458            Ok::<(), anyhow::Error>(())
459        })?;
460        Ok(())
461    }
462
463    fn stream_file_items<'scope, 'env>(
464        &'env self,
465        scope: &'scope std::thread::Scope<'scope, 'env>,
466    ) -> anyhow::Result<(mpsc::Receiver<FileWithDirIndex>, Vec<Arc<FileHashCache>>)> {
467        let (it_tx, it_rx) = mpsc::channel();
468        let mut caches = Vec::with_capacity(self.dirs.len());
469        for (dir_index, dir) in self.dirs.iter().enumerate() {
470            let mut it = FileIterator::new(dir);
471            let cache = FileHashCache::find_or_new(dir);
472            it.cache = Some(Arc::clone(&cache));
473            it.exclude = self.exclude.as_ref();
474            let it_tx = it_tx.clone();
475            scope.spawn(move || it.send_to_as(it_tx, |path| (path, dir_index)));
476            caches.push(cache);
477        }
478        Ok((it_rx, caches))
479    }
480
481    fn send_hash<'scope>(
482        &'scope self,
483        file: &FileItem,
484        cache: &Arc<FileHashCache>,
485        tx: &mpsc::Sender<DupEvent>,
486        scope: &rayon::Scope<'scope>,
487    ) {
488        let (hash, relative) = self
489            .get_hash_from_cache(file, cache)
490            .expect("path should be in cache base_dir");
491        if let Some(hash) = hash {
492            let _ = tx.send(DupEvent::Result(file.clone(), hash));
493            return;
494        }
495
496        let file = file.clone();
497        let relative = relative.to_path_buf();
498        let tx = tx.clone();
499        let cache = Arc::clone(cache);
500        scope.spawn(move |_| {
501            if let Ok(hash) = self.compute_hash(&file) {
502                cache.insert(&relative, &file, hash);
503                let _ = tx.send(DupEvent::Result(file, hash));
504            } else {
505                log::error!("Failed to hash file: '{}'", file);
506                let _ = tx.send(DupEvent::Error);
507            }
508        });
509    }
510
511    /// Gets the hash of a file, using the cache if available.
512    pub fn get_hash(&self, file: &FileItem) -> anyhow::Result<blake3::Hash> {
513        let cache = self.cache.as_ref().expect("cache should be initialized");
514        let (hash, relative) = self.get_hash_from_cache(file, cache)?;
515        if let Some(hash) = hash {
516            return Ok(hash);
517        }
518
519        let hash = self.compute_hash(file)?;
520        cache.insert(relative, file, hash);
521        Ok(hash)
522    }
523
524    fn get_hash_from_cache<'a>(
525        &self,
526        file: &'a FileItem,
527        cache: &FileHashCache,
528    ) -> io::Result<(Option<blake3::Hash>, &'a Path)> {
529        let relative = file.relative_path(cache.base_dir());
530        if let Some(hash) = cache.get(relative, file) {
531            self.num_hash_looked_up
532                .fetch_add(1, atomic::Ordering::Relaxed);
533            return Ok((Some(hash), relative));
534        }
535        Ok((None, relative))
536    }
537
538    fn compute_hash(&self, file: &FileItem) -> io::Result<blake3::Hash> {
539        let start_time = time::Instant::now();
540        let mut f = fs::File::open(file.path())?;
541        let mut progress = self
542            .progress
543            .as_ref()
544            .map(|progress| progress.add_file(file.path(), file.size()))
545            .unwrap_or_else(Progress::none);
546        let mut hasher = blake3::Hasher::new();
547        if self.buffer_size == 0 {
548            if file.size() > 0 {
549                let mmap = unsafe { memmap2::MmapOptions::new().map(&f)? };
550                hasher.update(&mmap[..]);
551                progress.inc(ProgressValue::with_size(file.size()));
552            }
553        } else {
554            let mut buf = vec![0u8; self.buffer_size];
555            loop {
556                let n = f.read(&mut buf)?;
557                if n == 0 {
558                    break;
559                }
560                hasher.update(&buf[..n]);
561                progress.inc_size(n as u64);
562            }
563            progress.inc_file(1);
564        }
565        progress.finish();
566        self.num_hashed.fetch_add(1, atomic::Ordering::Relaxed);
567        let hash = hasher.finalize();
568        log::trace!(
569            "Computed hash in {}: '{}'",
570            FormattedDuration(start_time.elapsed()),
571            file
572        );
573        Ok(hash)
574    }
575}
576
577/// A group of duplicated files and their size.
578#[derive(Clone, Debug)]
579pub struct DuplicatedFiles {
580    pub paths: Vec<PathBuf>,
581    pub size: u64,
582}
583
584impl DuplicatedFiles {
585    fn wasted_size(&self) -> u64 {
586        self.size * (self.paths.len() as u64 - 1)
587    }
588
589    fn print(&self, output_format: OutputFormat) -> anyhow::Result<()> {
590        match output_format {
591            OutputFormat::Default => self.write_human(stdout())?,
592            OutputFormat::PowerShell => self.write_pwsh(stdout())?,
593            OutputFormat::Shell => self.write_shell(stdout())?,
594            OutputFormat::Yaml | OutputFormat::Symbol => self.write_yaml(stdout())?,
595        }
596        Ok(())
597    }
598
599    fn write_human(&self, mut writer: impl io::Write) -> anyhow::Result<()> {
600        writeln!(
601            writer,
602            "Identical {} files of {}:",
603            self.paths.len(),
604            crate::human_readable_size(self.size)
605        )?;
606        for path in &self.paths {
607            writeln!(writer, "  {}", path.display())?;
608        }
609        Ok(())
610    }
611
612    fn write_yaml(&self, mut writer: impl io::Write) -> anyhow::Result<()> {
613        writeln!(writer, "- paths:")?;
614        for path in &self.paths {
615            writeln!(writer, "  - '{}'", Self::escape_quotes_by_double(path))?;
616        }
617        writeln!(writer, "  size: {}", self.size)?;
618        Ok(())
619    }
620
621    fn write_shell(&self, writer: impl io::Write) -> anyhow::Result<()> {
622        self.write_shell_with(writer, "cp", Self::escape_quotes_for_shell)
623    }
624
625    fn write_pwsh(&self, writer: impl io::Write) -> anyhow::Result<()> {
626        self.write_shell_with(
627            writer,
628            "Copy-Item -LiteralPath",
629            Self::escape_quotes_by_double,
630        )
631    }
632
633    fn write_shell_with(
634        &self,
635        mut writer: impl io::Write,
636        cmd: &str,
637        stringify: impl Fn(&Path) -> String,
638    ) -> anyhow::Result<()> {
639        let mut iter = self.paths.iter();
640        if let Some(path0) = iter.next() {
641            let path0 = stringify(path0);
642            for path in iter {
643                writeln!(writer, "{cmd} '{path0}' '{}'", stringify(path))?;
644            }
645        }
646        Ok(())
647    }
648
649    /// Escape single quotes for sh.
650    fn escape_quotes_for_shell(path: &Path) -> String {
651        path.to_string_lossy().replace('\'', "'\\''")
652    }
653
654    /// Escape single quotes by doubling them,
655    /// for single quoted strings in zsh, PowerShell, YAML, etc.
656    fn escape_quotes_by_double(path: &Path) -> String {
657        path.to_string_lossy().replace('\'', "''")
658    }
659}
660
661#[cfg(test)]
662mod tests {
663    use super::*;
664    use std::cmp::Ordering;
665
666    fn default_exclude() -> globset::GlobSet {
667        let mut builder = globset::GlobSetBuilder::new();
668        builder.add(
669            globset::GlobBuilder::new(".hash_cache")
670                .case_insensitive(true)
671                .build()
672                .unwrap(),
673        );
674        builder.build().unwrap()
675    }
676
677    #[test]
678    fn find_duplicates() -> anyhow::Result<()> {
679        let dir = tempfile::tempdir()?;
680
681        let file1_path = dir.path().join("same1.txt");
682        fs::write(&file1_path, "same content")?;
683
684        let file2_path = dir.path().join("same2.txt");
685        fs::write(&file2_path, "same content")?;
686
687        let diff_path = dir.path().join("diff.txt");
688        fs::write(&diff_path, "different content")?;
689
690        let mut hasher = FileHasher::new(&[dir.path()])?;
691        hasher.buffer_size = 8192;
692        let duplicates = hasher.find_duplicates()?;
693
694        assert_eq!(hasher.num_hashed.load(atomic::Ordering::Relaxed), 2);
695        assert_eq!(hasher.num_hash_looked_up.load(atomic::Ordering::Relaxed), 0);
696
697        assert_eq!(duplicates.len(), 1);
698        let group = &duplicates[0];
699        assert_eq!(group.paths.len(), 2);
700        assert_eq!(group.size, 12); // "same content" is 12 bytes
701
702        assert!(group.paths.contains(&file1_path));
703        assert!(group.paths.contains(&file2_path));
704
705        Ok(())
706    }
707
708    #[test]
709    fn find_duplicates_merge_cache() -> anyhow::Result<()> {
710        let dir = tempfile::tempdir()?;
711        let dir_path = dir.path();
712
713        let sub_dir = dir_path.join("a").join("a");
714        fs::create_dir_all(&sub_dir)?;
715
716        let file1_path = sub_dir.join("1");
717        fs::write(&file1_path, "same content")?;
718
719        let file2_path = sub_dir.join("2");
720        fs::write(&file2_path, "same content")?;
721
722        // Create empty cache file in a/a to force it to be the cache base
723        let cache_aa_path = sub_dir.join(FileHashCache::FILE_NAME);
724        fs::File::create(&cache_aa_path)?;
725
726        // Run find_duplicates on a/a
727        let hasher_aa = FileHasher::new(&[&sub_dir])?;
728        let duplicates_aa = hasher_aa.find_duplicates()?;
729        assert_eq!(duplicates_aa.len(), 1);
730        assert!(cache_aa_path.exists());
731        assert_eq!(hasher_aa.num_hashed.load(atomic::Ordering::Relaxed), 2);
732        assert_eq!(
733            hasher_aa.num_hash_looked_up.load(atomic::Ordering::Relaxed),
734            0
735        );
736
737        // Create empty cache file in a to force it to be the cache base
738        let root_a = dir_path.join("a");
739        let cache_a_path = root_a.join(FileHashCache::FILE_NAME);
740        fs::File::create(&cache_a_path)?;
741
742        // Run find_duplicates on a
743        let hasher_a = FileHasher::new(&[&root_a])?;
744        let duplicates_a = hasher_a.find_duplicates()?;
745        assert_eq!(duplicates_a.len(), 1);
746        assert_eq!(hasher_a.num_hashed.load(atomic::Ordering::Relaxed), 0);
747        assert_eq!(
748            hasher_a.num_hash_looked_up.load(atomic::Ordering::Relaxed),
749            2
750        );
751
752        // The merged child cache should be removed.
753        assert!(cache_a_path.exists());
754        assert!(!cache_aa_path.exists());
755
756        Ok(())
757    }
758
759    #[test]
760    fn find_duplicates_with_exclude() -> anyhow::Result<()> {
761        let dir = tempfile::tempdir()?;
762
763        let file1_path = dir.path().join("same1.txt");
764        fs::write(&file1_path, "same content")?;
765
766        let file2_path = dir.path().join("same2.txt");
767        fs::write(&file2_path, "same content")?;
768
769        let exclude_path = dir.path().join("exclude.txt");
770        fs::write(&exclude_path, "same content")?;
771
772        let mut hasher = FileHasher::new(&[dir.path()])?;
773        hasher.buffer_size = 8192;
774        let mut builder = globset::GlobSetBuilder::new();
775        builder.add(
776            globset::GlobBuilder::new("exclude.txt")
777                .case_insensitive(true)
778                .build()?,
779        );
780        let filter = builder.build()?;
781        hasher.exclude = Some(filter);
782
783        let duplicates = hasher.find_duplicates()?;
784        assert_eq!(duplicates.len(), 1);
785        let group = &duplicates[0];
786        assert_eq!(group.paths.len(), 2);
787        assert!(group.paths.contains(&file1_path));
788        assert!(group.paths.contains(&file2_path));
789        assert!(!group.paths.contains(&exclude_path));
790        Ok(())
791    }
792
793    #[derive(Default)]
794    struct CheckCollector {
795        start_seen: bool,
796        total_files: Option<u64>,
797        results: Vec<FileComparisonResult>,
798        file_done_count: u64,
799        num_error: usize,
800    }
801
802    impl CheckCollector {
803        fn collect(rx: mpsc::Receiver<CheckEvent>, base_dir: &Path) -> Self {
804            let mut collector = Self::default();
805            collector._collect(rx, base_dir);
806            collector
807        }
808
809        fn _collect(&mut self, rx: mpsc::Receiver<CheckEvent>, base_dir: &Path) {
810            while let Ok(event) = rx.recv() {
811                match event {
812                    CheckEvent::StartChecking => self.start_seen = true,
813                    CheckEvent::Total(total) => self.total_files = Some(total.num_files),
814                    CheckEvent::Result(mut result) => {
815                        result.relative_path = result
816                            .relative_path
817                            .strip_prefix(base_dir)
818                            .unwrap()
819                            .to_path_buf();
820                        self.results.push(result);
821                    }
822                    CheckEvent::Progress(progress_val) => {
823                        self.file_done_count += progress_val.num_files;
824                    }
825                    CheckEvent::Error(_) => {
826                        self.num_error += 1;
827                    }
828                }
829            }
830        }
831    }
832
833    #[test]
834    fn check_mode_empty_cache() -> anyhow::Result<()> {
835        let dir = tempfile::tempdir()?;
836        let dir_path = dir.path().to_path_buf();
837        println!("{:?}", dir_path);
838        let file1_path = dir.path().join("file1.txt");
839        fs::write(&file1_path, "content 1")?;
840        let file2_path = dir.path().join("file2.txt");
841        fs::write(&file2_path, "content 2")?;
842
843        let mut hasher = FileHasher::new(&[&dir_path])?;
844        hasher.exclude = Some(default_exclude());
845        let (tx, rx) = mpsc::channel();
846        hasher.check_streaming(tx, false)?;
847        let collector = CheckCollector::collect(rx, &dir_path);
848        assert!(collector.start_seen);
849        assert_eq!(collector.total_files, Some(2));
850        assert_eq!(collector.file_done_count, 2);
851        assert_eq!(collector.num_error, 0);
852
853        let mut results = collector.results;
854        results.sort_by(|a, b| a.relative_path.cmp(&b.relative_path));
855        assert_eq!(results.len(), 2);
856        assert_eq!(results[0].relative_path, Path::new("file1.txt"));
857        assert_eq!(results[0].classification, Classification::OnlyInDir2);
858        assert_eq!(results[1].relative_path, Path::new("file2.txt"));
859        assert_eq!(results[1].classification, Classification::OnlyInDir2);
860
861        assert!(!dir.path().join(FileHashCache::FILE_NAME).exists());
862        Ok(())
863    }
864
865    #[test]
866    fn check_mode_with_cache() -> anyhow::Result<()> {
867        let dir = tempfile::tempdir()?;
868        let dir_path = dir.path().to_path_buf();
869        let file1_path = dir.path().join("file1.txt");
870        let file2_path = dir.path().join("file2.txt");
871        fs::write(&file1_path, "content 1")?;
872        fs::write(&file2_path, "content 2")?;
873        let file1 = FileItem::try_from(file1_path.as_path())?;
874        let file2 = FileItem::try_from(file2_path.as_path())?;
875
876        let mut hasher = FileHasher::new_with_cache(&[&dir_path])?;
877        hasher.exclude = Some(default_exclude());
878        let _hash1 = hasher.get_hash(&file1)?;
879        let _hash2 = hasher.get_hash(&file2)?;
880        hasher.save_cache()?;
881        assert!(dir.path().join(FileHashCache::FILE_NAME).exists());
882
883        let mut hasher = FileHasher::new(&[&dir_path])?;
884        hasher.exclude = Some(default_exclude());
885        let (tx, rx) = mpsc::channel();
886        hasher.check_streaming(tx, false)?;
887        let collector = CheckCollector::collect(rx, &dir_path);
888        assert_eq!(collector.results.len(), 0);
889        assert_eq!(collector.file_done_count, 0);
890
891        fs::write(&file1_path, "content 1 modified")?;
892
893        let file2_meta_before = fs::metadata(&file2_path)?;
894        let mtime_before = file2_meta_before.modified()?;
895        std::thread::sleep(time::Duration::from_millis(10));
896        fs::write(&file2_path, "content 2")?;
897        let file2_meta_after = fs::metadata(&file2_path)?;
898        let mtime_after = file2_meta_after.modified()?;
899        assert!(mtime_after > mtime_before);
900
901        let mut hasher = FileHasher::new(&[&dir_path])?;
902        hasher.exclude = Some(default_exclude());
903        let (tx, rx) = mpsc::channel();
904        hasher.check_streaming(tx, false)?;
905        let collector = CheckCollector::collect(rx, &dir_path);
906        assert_eq!(collector.results.len(), 1);
907        let results = collector.results;
908        assert_eq!(results[0].relative_path, Path::new("file1.txt"));
909        assert_eq!(results[0].modified_time_comparison, Some(Ordering::Less));
910        assert_eq!(results[0].size_comparison, Some(Ordering::Less));
911        assert_eq!(results[0].is_content_same, None);
912        assert_eq!(collector.file_done_count, 1);
913        Ok(())
914    }
915
916    #[test]
917    fn check_update_mode() -> anyhow::Result<()> {
918        let dir = tempfile::tempdir()?;
919        let dir_path = dir.path().to_path_buf();
920        let file1_path = dir.path().join("file1.txt");
921        fs::write(&file1_path, "content 1")?;
922
923        let mut hasher = FileHasher::new(&[&dir_path])?;
924        hasher.exclude = Some(default_exclude());
925        let (tx, rx) = mpsc::channel();
926        hasher.check_streaming(tx, true)?;
927        let _ = CheckCollector::collect(rx, &dir_path);
928        hasher.save_cache()?;
929        assert!(dir.path().join(FileHashCache::FILE_NAME).exists());
930
931        let cache = FileHashCache::new(&dir_path);
932        let file1 = FileItem::try_from(file1_path.as_path())?;
933        let hash1 = cache.get(&PathBuf::from("file1.txt"), &file1);
934        assert!(hash1.is_some());
935
936        std::thread::sleep(time::Duration::from_millis(10));
937        fs::write(&file1_path, "content 1 modified")?;
938        let file1_mod = FileItem::try_from(file1_path.as_path())?;
939
940        let mut hasher = FileHasher::new(&[&dir_path])?;
941        hasher.exclude = Some(default_exclude());
942        let (tx, rx) = mpsc::channel();
943        hasher.check_streaming(tx, true)?;
944        let _ = CheckCollector::collect(rx, &dir_path);
945        hasher.save_cache()?;
946
947        let cache = FileHashCache::new(&dir_path);
948        let hash_mod = cache.get(&PathBuf::from("file1.txt"), &file1_mod);
949        assert!(hash_mod.is_some());
950        assert_ne!(hash1, hash_mod);
951
952        std::thread::sleep(time::Duration::from_millis(10));
953        fs::write(&file1_path, "content 1 modified")?;
954        let file1_mod2 = FileItem::try_from(file1_path.as_path())?;
955        assert!(file1_mod2.modified() > file1_mod.modified());
956
957        assert!(
958            cache
959                .get(&PathBuf::from("file1.txt"), &file1_mod2)
960                .is_none()
961        );
962
963        let mut hasher = FileHasher::new(&[&dir_path])?;
964        hasher.exclude = Some(default_exclude());
965        let (tx, rx) = mpsc::channel();
966        hasher.check_streaming(tx, true)?;
967        let _ = CheckCollector::collect(rx, &dir_path);
968        hasher.save_cache()?;
969
970        let cache = FileHashCache::new(&dir_path);
971        assert!(
972            cache
973                .get(&PathBuf::from("file1.txt"), &file1_mod2)
974                .is_some()
975        );
976        Ok(())
977    }
978
979    #[test]
980    fn check_cleanup_deleted_files() -> anyhow::Result<()> {
981        let dir = tempfile::tempdir()?;
982        let dir_path = dir.path().to_path_buf();
983        let file1_path = dir.path().join("file1.txt");
984        let file2_path = dir.path().join("file2.txt");
985        fs::write(&file1_path, "content 1")?;
986        fs::write(&file2_path, "content 2")?;
987        let file1 = FileItem::try_from(file1_path.as_path())?;
988        let file2 = FileItem::try_from(file2_path.as_path())?;
989
990        let mut hasher = FileHasher::new(&[&dir_path])?;
991        hasher.exclude = Some(default_exclude());
992        let (tx, rx) = mpsc::channel();
993        hasher.check_streaming(tx, true)?;
994        let _ = CheckCollector::collect(rx, &dir_path);
995        hasher.save_cache()?;
996
997        // Verify both are in the cache
998        let cache = FileHashCache::new(&dir_path);
999        assert!(cache.get(&PathBuf::from("file1.txt"), &file1).is_some());
1000        assert!(cache.get(&PathBuf::from("file2.txt"), &file2).is_some());
1001
1002        // Now delete file2 from disk
1003        fs::remove_file(&file2_path)?;
1004
1005        // Run check and save again
1006        let mut hasher = FileHasher::new(&[&dir_path])?;
1007        hasher.exclude = Some(default_exclude());
1008        let (tx, rx) = mpsc::channel();
1009        hasher.check_streaming(tx, true)?;
1010        let _ = CheckCollector::collect(rx, &dir_path);
1011        hasher.save_cache()?;
1012
1013        // Verify file2 is removed from cache, but file1 is still there
1014        let cache = FileHashCache::new(&dir_path);
1015        assert!(cache.get(&PathBuf::from("file2.txt"), &file2).is_none());
1016        assert!(cache.get(&PathBuf::from("file1.txt"), &file1).is_some());
1017        Ok(())
1018    }
1019
1020    #[test]
1021    fn find_duplicates_multiple_dirs() -> anyhow::Result<()> {
1022        let tmp = tempfile::tempdir()?;
1023        let dir1 = tmp.path().join("dir1");
1024        let dir2 = tmp.path().join("dir2");
1025        fs::create_dir(&dir1)?;
1026        fs::create_dir(&dir2)?;
1027        let file1_path = dir1.join("file1.txt");
1028        fs::write(&file1_path, "same content")?;
1029        let file2_path = dir2.join("file2.txt");
1030        fs::write(&file2_path, "same content")?;
1031        let hasher = FileHasher::new(&[&dir1, &dir2])?;
1032        let duplicates = hasher.find_duplicates()?;
1033        assert_eq!(duplicates.len(), 1);
1034        let group = &duplicates[0];
1035        assert_eq!(group.paths.len(), 2);
1036        assert_eq!(group.size, 12);
1037        assert!(group.paths.contains(&file1_path));
1038        assert!(group.paths.contains(&file2_path));
1039
1040        Ok(())
1041    }
1042
1043    #[test]
1044    fn check_fails_with_multiple_dirs() -> anyhow::Result<()> {
1045        let tmp = tempfile::tempdir()?;
1046        let dir1 = tmp.path().join("dir1");
1047        let dir2 = tmp.path().join("dir2");
1048        fs::create_dir(&dir1)?;
1049        fs::create_dir(&dir2)?;
1050        let hasher = FileHasher::new(&[&dir1, &dir2])?;
1051        assert!(hasher.check(false).is_err());
1052        Ok(())
1053    }
1054
1055    #[test]
1056    fn escape_shell() {
1057        let escape_shell = |p: &str| DuplicatedFiles::escape_quotes_for_shell(Path::new(p));
1058        assert_eq!(escape_shell(""), "");
1059        assert_eq!(escape_shell("abc"), "abc");
1060        assert_eq!(escape_shell("a'b"), "a'\\''b");
1061        assert_eq!(escape_shell("a'b'"), "a'\\''b'\\''");
1062
1063        let escape_shell_double = |p: &str| DuplicatedFiles::escape_quotes_by_double(Path::new(p));
1064        assert_eq!(escape_shell_double(""), "");
1065        assert_eq!(escape_shell_double("abc"), "abc");
1066        assert_eq!(escape_shell_double("a'b"), "a''b");
1067        assert_eq!(escape_shell_double("a'b'"), "a''b''");
1068    }
1069
1070    #[test]
1071    fn write_dups_shell_empty() -> anyhow::Result<()> {
1072        let dup_empty = DuplicatedFiles {
1073            paths: vec![],
1074            size: 100,
1075        };
1076        let mut buf = Vec::new();
1077        dup_empty.write_shell(&mut buf)?;
1078        assert_eq!(String::from_utf8(buf)?, "");
1079        Ok(())
1080    }
1081
1082    #[test]
1083    fn write_dups_shell_one() -> anyhow::Result<()> {
1084        let dup_one = DuplicatedFiles {
1085            paths: vec![PathBuf::from("a.txt")],
1086            size: 100,
1087        };
1088        let mut buf = Vec::new();
1089        dup_one.write_shell(&mut buf)?;
1090        assert_eq!(String::from_utf8(buf)?, "");
1091        Ok(())
1092    }
1093
1094    #[test]
1095    fn write_dups_shell_two() -> anyhow::Result<()> {
1096        let dup_multiple = DuplicatedFiles {
1097            paths: vec![PathBuf::from("a.txt"), PathBuf::from("b.txt")],
1098            size: 100,
1099        };
1100        let mut buf = Vec::new();
1101        dup_multiple.write_shell(&mut buf)?;
1102        assert_eq!(String::from_utf8(buf)?, "cp 'a.txt' 'b.txt'\n");
1103        Ok(())
1104    }
1105
1106    #[test]
1107    fn write_dups_shell_three() -> anyhow::Result<()> {
1108        let dup_multiple = DuplicatedFiles {
1109            paths: vec![
1110                PathBuf::from("a.txt"),
1111                PathBuf::from("b.txt"),
1112                PathBuf::from("c.txt"),
1113            ],
1114            size: 100,
1115        };
1116        let mut buf = Vec::new();
1117        dup_multiple.write_shell(&mut buf)?;
1118        assert_eq!(
1119            String::from_utf8(buf)?,
1120            "cp 'a.txt' 'b.txt'\ncp 'a.txt' 'c.txt'\n"
1121        );
1122        Ok(())
1123    }
1124
1125    #[test]
1126    fn write_dups_shell_quotes() -> anyhow::Result<()> {
1127        let dup_quotes = DuplicatedFiles {
1128            paths: vec![PathBuf::from("a'b.txt"), PathBuf::from("c'd.txt")],
1129            size: 100,
1130        };
1131        let mut buf = Vec::new();
1132        dup_quotes.write_shell(&mut buf)?;
1133        assert_eq!(String::from_utf8(buf)?, "cp 'a'\\''b.txt' 'c'\\''d.txt'\n");
1134
1135        let mut buf = Vec::new();
1136        dup_quotes.write_pwsh(&mut buf)?;
1137        assert_eq!(
1138            String::from_utf8(buf)?,
1139            "Copy-Item -LiteralPath 'a''b.txt' 'c''d.txt'\n"
1140        );
1141        Ok(())
1142    }
1143}
compare_dir/file_hasher.rs

compare_dir/
file_hasher.rs
