compare_dir/
file_hasher.rs

1use crate::{
2    Classification, ColumnFormatter, DirectoryComparer, FileComparer, FileComparisonResult,
3    FileHashCache, FileItem, FileIterator, OutputFormat, Progress, ProgressBuilder, ProgressValue,
4};
5use globset::GlobSet;
6use indicatif::FormattedDuration;
7use rayon::prelude::*;
8use simple_path::SimplePath;
9use std::{
10    collections::HashMap,
11    fs,
12    io::{self, Read, stdout},
13    path::{Path, PathBuf},
14    sync::{
15        Arc,
16        atomic::{self, AtomicUsize},
17        mpsc,
18    },
19    time,
20};
21
22type FileWithDirIndex = (FileItem, usize);
23
24#[derive(Debug, Clone)]
25enum DupEvent {
26    StartHashing,
27    Total(ProgressValue),
28    Result(FileItem, blake3::Hash),
29    Error,
30}
31
32#[derive(Debug)]
33enum CheckEvent {
34    StartChecking,
35    Total(ProgressValue),
36    Result(FileComparisonResult, ProgressValue),
37    Progress(ProgressValue),
38    Error(FileItem),
39}
40
41enum DupState {
42    Single(FileItem, usize),
43    Hashing,
44}
45
46/// A tool for finding duplicated files in a directory.
47pub struct FileHasher {
48    dirs: Vec<PathBuf>,
49    pub buffer_size: usize,
50    cache: Option<Arc<FileHashCache>>,
51    num_hashed: AtomicUsize,
52    num_hash_looked_up: AtomicUsize,
53    pub exclude: Option<GlobSet>,
54    pub progress: Option<Arc<ProgressBuilder>>,
55    pub output_format: OutputFormat,
56    pub jobs: usize,
57}
58
59impl FileHasher {
60    const DEFAULT_JOBS: usize = DirectoryComparer::DEFAULT_JOBS;
61
62    /// Creates a new `FileHasher` for the given directories.
63    pub fn new<P: AsRef<Path>>(dirs: &[P]) -> anyhow::Result<Self> {
64        if dirs.is_empty() {
65            anyhow::bail!("At least one directory must be specified.");
66        }
67        Ok(Self {
68            dirs: dirs.iter().map(|p| p.as_ref().to_path_buf()).collect(),
69            buffer_size: FileComparer::DEFAULT_BUFFER_SIZE,
70            cache: None,
71            num_hashed: AtomicUsize::new(0),
72            num_hash_looked_up: AtomicUsize::new(0),
73            exclude: None,
74            progress: None,
75            output_format: OutputFormat::Default,
76            jobs: Self::DEFAULT_JOBS,
77        })
78    }
79
80    pub(crate) fn new_with_cache<P: AsRef<Path>>(dirs: &[P]) -> anyhow::Result<Self> {
81        let mut hasher = Self::new(dirs)?;
82        hasher.cache = Some(hasher.new_cache()?);
83        Ok(hasher)
84    }
85
86    fn new_cache(&self) -> anyhow::Result<Arc<FileHashCache>> {
87        let common_ancestor = crate::common_ancestor(&self.dirs)
88            .ok_or_else(|| anyhow::anyhow!("No common ancestor found"))?;
89        Ok(FileHashCache::find_or_new(&common_ancestor))
90    }
91
92    /// Gets the hash cache.
93    pub(crate) fn cache(&mut self) -> anyhow::Result<Arc<FileHashCache>> {
94        if self.cache.is_none() {
95            self.cache = Some(self.new_cache()?);
96        }
97        Ok(Arc::clone(self.cache.as_ref().unwrap()))
98    }
99
100    /// Remove a cache entry if it exists.
101    pub(crate) fn remove_cache_entry(&mut self, path: &Path) -> anyhow::Result<()> {
102        let cache = self.cache()?;
103        let relative = SimplePath::strip_prefix(path, cache.base_dir())?;
104        cache.remove(relative);
105        Ok(())
106    }
107
108    /// Save the hash cache if it is dirty.
109    pub fn save_cache(&self) -> anyhow::Result<()> {
110        log::info!(
111            "Hash stats for {:?}: {} computed, {} looked up",
112            self.dirs,
113            self.num_hashed.load(atomic::Ordering::Relaxed),
114            self.num_hash_looked_up.load(atomic::Ordering::Relaxed)
115        );
116        if let Some(cache) = &self.cache {
117            cache.save()?;
118        }
119        Ok(())
120    }
121
122    /// Clears the loaded hashes in the cache.
123    pub(crate) fn clear_cache(&mut self) -> anyhow::Result<()> {
124        let cache = self.cache()?;
125        for dir in &self.dirs {
126            let relative = SimplePath::strip_prefix(dir, cache.base_dir())?;
127            cache.clear(relative);
128        }
129        Ok(())
130    }
131
132    /// Executes the check/update process.
133    pub fn check(&self, update: bool) -> anyhow::Result<()> {
134        match self.output_format {
135            OutputFormat::Default | OutputFormat::Symbol => {}
136            _ => anyhow::bail!("Check mode only supports default or symbol output format."),
137        }
138        if self.dirs.len() > 1 {
139            anyhow::bail!("Check mode only supports one directory.");
140        }
141        let start_time = time::Instant::now();
142        let mut progress = self
143            .progress
144            .as_ref()
145            .map(|progress| progress.add_spinner())
146            .unwrap_or_else(Progress::none);
147        progress.use_bytes();
148        progress.set_message("Scanning directory...");
149        let mut num_new = 0;
150        let mut num_modified = 0;
151        let mut num_error = 0;
152        std::thread::scope(|scope| {
153            let (tx, rx) = mpsc::channel();
154            scope.spawn(|| {
155                if let Err(e) = self.check_streaming(tx, update) {
156                    log::error!("Error during check: {}", e);
157                }
158            });
159            while let Ok(event) = rx.recv() {
160                match event {
161                    CheckEvent::StartChecking => {
162                        progress.set_message("Checking files...");
163                    }
164                    CheckEvent::Total(value) => {
165                        progress.set_length(value);
166                        progress.set_message("");
167                    }
168                    CheckEvent::Result(result, value) => {
169                        progress.inc(value);
170                        progress.suspend_for(stdout(), || {
171                            result.print(self.output_format, "cached", "current")
172                        });
173                        if result.classification == Classification::OnlyInDir2 {
174                            num_new += 1;
175                        } else if result.is_identical_content() == Some(false) {
176                            num_modified += 1;
177                        }
178                    }
179                    CheckEvent::Progress(value) => {
180                        progress.inc(value);
181                    }
182                    CheckEvent::Error(file) => {
183                        progress.inc(ProgressValue::with_skip(file.size()));
184                        num_error += 1;
185                    }
186                }
187            }
188        });
189        progress.finish();
190        self.print_check_summary(&start_time, num_new, num_modified, num_error)?;
191        Ok(())
192    }
193
194    fn print_check_summary(
195        &self,
196        start_time: &time::Instant,
197        num_new: usize,
198        num_modified: usize,
199        num_error: usize,
200    ) -> io::Result<()> {
201        let summary = [
202            ("Elapsed:", 0),
203            (
204                "Hash computed:",
205                self.num_hashed.load(atomic::Ordering::Relaxed),
206            ),
207            ("New files:", num_new),
208            ("Modified files:", num_modified),
209            ("Errors:", num_error),
210        ];
211        let formatter = ColumnFormatter::new(summary.iter().map(|(s, _)| *s));
212        let mut writer = std::io::stderr();
213        formatter.write_value(
214            &mut writer,
215            summary[0].0,
216            FormattedDuration(start_time.elapsed()),
217        )?;
218        formatter.write_values(&mut writer, &summary[1..])
219    }
220
221    fn check_streaming(&self, tx: mpsc::Sender<CheckEvent>, update: bool) -> anyhow::Result<()> {
222        assert_eq!(self.dirs.len(), 1);
223        let cache = self.new_cache()?;
224        let base_dir = &self.dirs[0];
225        let relative = SimplePath::strip_prefix(base_dir, cache.base_dir())?;
226        cache.set_remove_if_no_access(relative);
227        let cache_clone = Arc::clone(&cache);
228        std::thread::scope(|global_scope| {
229            let mut it = FileIterator::new(base_dir);
230            it.cache = Some(Arc::clone(&cache));
231            it.exclude = self.exclude.as_ref();
232            let it_rx = it.spawn_in_scope(global_scope);
233            tx.send(CheckEvent::StartChecking)?;
234            let pool = crate::build_thread_pool(self.jobs)?;
235            pool.scope(move |scope| -> anyhow::Result<()> {
236                let mut total = ProgressValue::default();
237                for file in it_rx {
238                    self.check_file(file, &cache, update, &mut total, &tx, scope);
239                }
240                tx.send(CheckEvent::Total(total))?;
241                Ok(())
242            })
243        })?;
244        cache_clone.save()?;
245        Ok(())
246    }
247
248    fn check_file<'scope>(
249        &'scope self,
250        file: FileItem,
251        cache: &Arc<FileHashCache>,
252        update: bool,
253        total: &mut ProgressValue,
254        tx: &mpsc::Sender<CheckEvent>,
255        scope: &rayon::Scope<'scope>,
256    ) {
257        *total += ProgressValue::with_size(file.size());
258        let tx = tx.clone();
259        let cache = Arc::clone(cache);
260        scope.spawn(move |_| {
261            if let Err(error) = self._check_file(&file, cache, update, &tx) {
262                log::error!("Failed to check file '{}': {}", file, error);
263                if tx.send(CheckEvent::Error(file)).is_err() {
264                    log::error!("Send failed");
265                }
266            }
267        });
268    }
269
270    fn _check_file(
271        &self,
272        file: &FileItem,
273        cache: Arc<FileHashCache>,
274        update: bool,
275        tx: &mpsc::Sender<CheckEvent>,
276    ) -> anyhow::Result<()> {
277        assert!(file.path().is_absolute());
278        let path_in_cache = file.relative_path(cache.base_dir());
279        match cache.get_entry(path_in_cache) {
280            Some(cached) => {
281                let mut result =
282                    FileComparisonResult::new(file.path().into(), Classification::InBoth);
283                result.update_moodified(cached.modified, file.modified());
284                if cached.size != 0 {
285                    result.update_size(cached.size, file.size());
286                }
287                if !update && cached.size != 0 && file.size() != cached.size {
288                    tx.send(CheckEvent::Result(
289                        result,
290                        ProgressValue::with_skip(file.size()),
291                    ))?;
292                    return Ok(());
293                }
294                let hash = self.compute_hash(file)?;
295                result.is_content_same = Some(hash == cached.hash);
296                if hash == cached.hash {
297                    if cached.should_update(file, update) {
298                        cache.insert(path_in_cache, file, hash);
299                    }
300                    tx.send(CheckEvent::Progress(ProgressValue::with_size(file.size())))?;
301                } else {
302                    if update {
303                        cache.insert(path_in_cache, file, hash);
304                    }
305                    tx.send(CheckEvent::Result(
306                        result,
307                        ProgressValue::with_size(file.size()),
308                    ))?;
309                }
310            }
311            None => {
312                if update {
313                    let hash = self.compute_hash(file)?;
314                    cache.insert(path_in_cache, file, hash);
315                }
316                tx.send(CheckEvent::Result(
317                    FileComparisonResult::new(file.path().into(), Classification::OnlyInDir2),
318                    ProgressValue::with_size(file.size()),
319                ))?;
320            }
321        }
322        Ok(())
323    }
324
325    /// Executes the duplicate file finding process and prints results.
326    pub fn run(&self) -> anyhow::Result<()> {
327        let start_time = time::Instant::now();
328        let mut duplicates = self.find_duplicates()?;
329        let mut total_wasted_space = 0;
330        if !duplicates.is_empty() {
331            duplicates.sort_by_key(|a| a.size);
332            total_wasted_space = self.print_duplicates_results(&duplicates)?;
333        }
334        self.print_duplicates_summary(&start_time, total_wasted_space)?;
335        Ok(())
336    }
337
338    fn print_duplicates_results(&self, duplicates: &Vec<DuplicatedFiles>) -> anyhow::Result<u64> {
339        let mut total_wasted_space = 0;
340        for dupes in duplicates {
341            dupes.print(self.output_format)?;
342            total_wasted_space += dupes.wasted_size();
343        }
344        Ok(total_wasted_space)
345    }
346
347    fn print_duplicates_summary(
348        &self,
349        start_time: &time::Instant,
350        total_wasted_space: u64,
351    ) -> io::Result<()> {
352        let elapsed = FormattedDuration(start_time.elapsed()).to_string();
353        let num_hashed = self.num_hashed.load(atomic::Ordering::Relaxed).to_string();
354        let total_wasted_space = crate::human_readable_size(total_wasted_space);
355        let summary = [
356            ("Elapsed:", elapsed),
357            ("Hash computed:", num_hashed),
358            ("Total wasted space:", total_wasted_space),
359        ];
360        let formatter = ColumnFormatter::new(summary.iter().map(|(s, _)| *s));
361        formatter.write_values(&mut io::stderr(), &summary)
362    }
363
364    /// Finds duplicated files and returns a list of duplicate groups.
365    pub fn find_duplicates(&self) -> anyhow::Result<Vec<DuplicatedFiles>> {
366        let mut progress = self
367            .progress
368            .as_ref()
369            .map(|progress| progress.add_spinner())
370            .unwrap_or_else(Progress::none);
371        progress.set_message("Scanning directories...");
372
373        let (tx, rx) = mpsc::channel();
374        let mut by_hash: HashMap<blake3::Hash, DuplicatedFiles> = HashMap::new();
375        std::thread::scope(|scope| {
376            scope.spawn(|| {
377                if let Err(e) = self.find_duplicates_streaming(tx) {
378                    log::error!("Error during duplicate finding: {}", e);
379                }
380            });
381
382            while let Ok(event) = rx.recv() {
383                match event {
384                    DupEvent::StartHashing => progress.set_message("Hashing files..."),
385                    DupEvent::Total(value) => progress.set_length(value),
386                    DupEvent::Result(file, hash) => {
387                        progress.inc(ProgressValue::with_size(file.size()));
388                        let entry = by_hash.entry(hash).or_insert_with(|| DuplicatedFiles {
389                            paths: Vec::new(),
390                            size: file.size(),
391                        });
392                        // Hash collisions shouldn't happen, but if they do, sizes shouldn't mismatch.
393                        assert_eq!(
394                            entry.size,
395                            file.size(),
396                            "Hash collision: sizes do not match"
397                        );
398                        entry.paths.push(file.into_path_buf());
399                    }
400                    DupEvent::Error => {}
401                }
402            }
403        });
404        progress.finish();
405
406        let mut duplicates = Vec::new();
407        for (_, mut dupes) in by_hash {
408            if dupes.paths.len() > 1 {
409                dupes.paths.sort();
410                duplicates.push(dupes);
411            }
412        }
413        Ok(duplicates)
414    }
415
416    fn find_duplicates_streaming(&self, tx: mpsc::Sender<DupEvent>) -> anyhow::Result<()> {
417        std::thread::scope(|global_scope| {
418            let (it_rx, caches) = self.stream_file_items(global_scope)?;
419            let caches = &caches;
420            let pool = crate::build_thread_pool(self.jobs)?;
421            pool.scope(move |scope| -> anyhow::Result<()> {
422                let mut by_size: HashMap<u64, DupState> = HashMap::new();
423                let mut total = ProgressValue::default();
424                tx.send(DupEvent::StartHashing)?;
425                for (file, dir_index) in it_rx {
426                    let size = file.size();
427                    if size == 0 {
428                        continue;
429                    }
430                    let cache = &caches[dir_index];
431                    match by_size.entry(size) {
432                        std::collections::hash_map::Entry::Occupied(mut occ) => match occ.get_mut()
433                        {
434                            DupState::Single(file0, dir_index0) => {
435                                // We found a second file of identical size.
436                                // Time to start hashing both the *original* matching file and the *new* one!
437                                let cache0 = &caches[*dir_index0];
438                                self.send_hash(file0, cache0, &tx, scope);
439                                self.send_hash(&file, cache, &tx, scope);
440                                total += ProgressValue::with_size(file0.size());
441                                total += ProgressValue::with_size(file.size());
442
443                                // Modify the state to indicate we are now fully hashing this size bucket.
444                                *occ.get_mut() = DupState::Hashing;
445                            }
446                            DupState::Hashing => {
447                                // File size bucket already hashing; just dynamically spawn the new file immediately.
448                                self.send_hash(&file, cache, &tx, scope);
449                                total += ProgressValue::with_size(file.size());
450                            }
451                        },
452                        std::collections::hash_map::Entry::Vacant(vac) => {
453                            vac.insert(DupState::Single(file, dir_index));
454                        }
455                    }
456                }
457                tx.send(DupEvent::Total(total))?;
458                Ok(())
459            })?;
460            pool.install(|| caches.into_par_iter().try_for_each(|cache| cache.save()))?;
461            Ok::<(), anyhow::Error>(())
462        })?;
463        Ok(())
464    }
465
466    fn stream_file_items<'scope, 'env>(
467        &'env self,
468        scope: &'scope std::thread::Scope<'scope, 'env>,
469    ) -> anyhow::Result<(mpsc::Receiver<FileWithDirIndex>, Vec<Arc<FileHashCache>>)> {
470        let (it_tx, it_rx) = mpsc::channel();
471        let mut caches = Vec::with_capacity(self.dirs.len());
472        for (dir_index, dir) in self.dirs.iter().enumerate() {
473            let mut it = FileIterator::new(dir);
474            let cache = FileHashCache::find_or_new(dir);
475            it.cache = Some(Arc::clone(&cache));
476            it.exclude = self.exclude.as_ref();
477            let it_tx = it_tx.clone();
478            scope.spawn(move || it.send_to_as(it_tx, |path| (path, dir_index)));
479            caches.push(cache);
480        }
481        Ok((it_rx, caches))
482    }
483
484    fn send_hash<'scope>(
485        &'scope self,
486        file: &FileItem,
487        cache: &Arc<FileHashCache>,
488        tx: &mpsc::Sender<DupEvent>,
489        scope: &rayon::Scope<'scope>,
490    ) {
491        let (hash, relative) = self
492            .get_hash_from_cache(file, cache)
493            .expect("path should be in cache base_dir");
494        if let Some(hash) = hash {
495            let _ = tx.send(DupEvent::Result(file.clone(), hash));
496            return;
497        }
498
499        let file = file.clone();
500        let relative = relative.to_path_buf();
501        let tx = tx.clone();
502        let cache = Arc::clone(cache);
503        scope.spawn(move |_| {
504            if let Ok(hash) = self.compute_hash(&file) {
505                cache.insert(&relative, &file, hash);
506                let _ = tx.send(DupEvent::Result(file, hash));
507            } else {
508                log::error!("Failed to hash file: '{}'", file);
509                let _ = tx.send(DupEvent::Error);
510            }
511        });
512    }
513
514    /// Gets the hash of a file, using the cache if available.
515    pub fn get_hash(&self, file: &FileItem) -> anyhow::Result<blake3::Hash> {
516        let cache = self.cache.as_ref().expect("cache should be initialized");
517        let (hash, relative) = self.get_hash_from_cache(file, cache)?;
518        if let Some(hash) = hash {
519            return Ok(hash);
520        }
521
522        let hash = self.compute_hash(file)?;
523        cache.insert(relative, file, hash);
524        Ok(hash)
525    }
526
527    fn get_hash_from_cache<'a>(
528        &self,
529        file: &'a FileItem,
530        cache: &FileHashCache,
531    ) -> io::Result<(Option<blake3::Hash>, &'a Path)> {
532        let relative = file.relative_path(cache.base_dir());
533        if let Some(hash) = cache.get(relative, file) {
534            self.num_hash_looked_up
535                .fetch_add(1, atomic::Ordering::Relaxed);
536            return Ok((Some(hash), relative));
537        }
538        Ok((None, relative))
539    }
540
541    fn compute_hash(&self, file: &FileItem) -> io::Result<blake3::Hash> {
542        let start_time = time::Instant::now();
543        let mut f = fs::File::open(file.path())?;
544        let mut progress = self
545            .progress
546            .as_ref()
547            .map(|progress| progress.add_file(file.path(), file.size()))
548            .unwrap_or_else(Progress::none);
549        let mut hasher = blake3::Hasher::new();
550        if self.buffer_size == 0 {
551            if file.size() > 0 {
552                let mmap = unsafe { memmap2::MmapOptions::new().map(&f)? };
553                hasher.update(&mmap[..]);
554                progress.inc(ProgressValue::with_size(file.size()));
555            }
556        } else {
557            let mut buf = vec![0u8; self.buffer_size];
558            loop {
559                let n = f.read(&mut buf)?;
560                if n == 0 {
561                    break;
562                }
563                hasher.update(&buf[..n]);
564                progress.inc(ProgressValue::with_size(n as u64));
565            }
566        }
567        progress.finish();
568        self.num_hashed.fetch_add(1, atomic::Ordering::Relaxed);
569        let hash = hasher.finalize();
570        log::trace!(
571            "Computed hash in {}: '{}'",
572            FormattedDuration(start_time.elapsed()),
573            file
574        );
575        Ok(hash)
576    }
577}
578
579/// A group of duplicated files and their size.
580#[derive(Clone, Debug)]
581pub struct DuplicatedFiles {
582    pub paths: Vec<PathBuf>,
583    pub size: u64,
584}
585
586impl DuplicatedFiles {
587    fn wasted_size(&self) -> u64 {
588        self.size * (self.paths.len() as u64 - 1)
589    }
590
591    fn print(&self, output_format: OutputFormat) -> anyhow::Result<()> {
592        match output_format {
593            OutputFormat::Default => self.write_human(stdout())?,
594            OutputFormat::PowerShell => self.write_pwsh(stdout())?,
595            OutputFormat::Shell => self.write_shell(stdout())?,
596            OutputFormat::Yaml | OutputFormat::Symbol => self.write_yaml(stdout())?,
597        }
598        Ok(())
599    }
600
601    fn write_human(&self, mut writer: impl io::Write) -> anyhow::Result<()> {
602        writeln!(
603            writer,
604            "Identical {} files of {}:",
605            self.paths.len(),
606            crate::human_readable_size(self.size)
607        )?;
608        for path in &self.paths {
609            writeln!(writer, "  {}", path.display())?;
610        }
611        Ok(())
612    }
613
614    fn write_yaml(&self, mut writer: impl io::Write) -> anyhow::Result<()> {
615        writeln!(writer, "- paths:")?;
616        for path in &self.paths {
617            writeln!(writer, "  - '{}'", Self::escape_quotes_by_double(path))?;
618        }
619        writeln!(writer, "  size: {}", self.size)?;
620        Ok(())
621    }
622
623    fn write_shell(&self, writer: impl io::Write) -> anyhow::Result<()> {
624        self.write_shell_with(writer, "cp", Self::escape_quotes_for_shell)
625    }
626
627    fn write_pwsh(&self, writer: impl io::Write) -> anyhow::Result<()> {
628        self.write_shell_with(
629            writer,
630            "Copy-Item -LiteralPath",
631            Self::escape_quotes_by_double,
632        )
633    }
634
635    fn write_shell_with(
636        &self,
637        mut writer: impl io::Write,
638        cmd: &str,
639        stringify: impl Fn(&Path) -> String,
640    ) -> anyhow::Result<()> {
641        let mut iter = self.paths.iter();
642        if let Some(path0) = iter.next() {
643            let path0 = stringify(path0);
644            for path in iter {
645                writeln!(writer, "{cmd} '{path0}' '{}'", stringify(path))?;
646            }
647        }
648        Ok(())
649    }
650
651    /// Escape single quotes for sh.
652    fn escape_quotes_for_shell(path: &Path) -> String {
653        path.to_string_lossy().replace('\'', "'\\''")
654    }
655
656    /// Escape single quotes by doubling them,
657    /// for single quoted strings in zsh, PowerShell, YAML, etc.
658    fn escape_quotes_by_double(path: &Path) -> String {
659        path.to_string_lossy().replace('\'', "''")
660    }
661}
662
663#[cfg(test)]
664mod tests {
665    use super::*;
666    use std::cmp::Ordering;
667
668    fn default_exclude() -> globset::GlobSet {
669        let mut builder = globset::GlobSetBuilder::new();
670        builder.add(
671            globset::GlobBuilder::new(".hash_cache")
672                .case_insensitive(true)
673                .build()
674                .unwrap(),
675        );
676        builder.build().unwrap()
677    }
678
679    #[test]
680    fn find_duplicates() -> anyhow::Result<()> {
681        let dir = tempfile::tempdir()?;
682
683        let file1_path = dir.path().join("same1.txt");
684        fs::write(&file1_path, "same content")?;
685
686        let file2_path = dir.path().join("same2.txt");
687        fs::write(&file2_path, "same content")?;
688
689        let diff_path = dir.path().join("diff.txt");
690        fs::write(&diff_path, "different content")?;
691
692        let mut hasher = FileHasher::new(&[dir.path()])?;
693        hasher.buffer_size = 8192;
694        let duplicates = hasher.find_duplicates()?;
695
696        assert_eq!(hasher.num_hashed.load(atomic::Ordering::Relaxed), 2);
697        assert_eq!(hasher.num_hash_looked_up.load(atomic::Ordering::Relaxed), 0);
698
699        assert_eq!(duplicates.len(), 1);
700        let group = &duplicates[0];
701        assert_eq!(group.paths.len(), 2);
702        assert_eq!(group.size, 12); // "same content" is 12 bytes
703
704        assert!(group.paths.contains(&file1_path));
705        assert!(group.paths.contains(&file2_path));
706
707        Ok(())
708    }
709
710    #[test]
711    fn find_duplicates_merge_cache() -> anyhow::Result<()> {
712        let dir = tempfile::tempdir()?;
713        let dir_path = dir.path();
714
715        let sub_dir = dir_path.join("a").join("a");
716        fs::create_dir_all(&sub_dir)?;
717
718        let file1_path = sub_dir.join("1");
719        fs::write(&file1_path, "same content")?;
720
721        let file2_path = sub_dir.join("2");
722        fs::write(&file2_path, "same content")?;
723
724        // Create empty cache file in a/a to force it to be the cache base
725        let cache_aa_path = sub_dir.join(FileHashCache::FILE_NAME);
726        fs::File::create(&cache_aa_path)?;
727
728        // Run find_duplicates on a/a
729        let hasher_aa = FileHasher::new(&[&sub_dir])?;
730        let duplicates_aa = hasher_aa.find_duplicates()?;
731        assert_eq!(duplicates_aa.len(), 1);
732        assert!(cache_aa_path.exists());
733        assert_eq!(hasher_aa.num_hashed.load(atomic::Ordering::Relaxed), 2);
734        assert_eq!(
735            hasher_aa.num_hash_looked_up.load(atomic::Ordering::Relaxed),
736            0
737        );
738
739        // Create empty cache file in a to force it to be the cache base
740        let root_a = dir_path.join("a");
741        let cache_a_path = root_a.join(FileHashCache::FILE_NAME);
742        fs::File::create(&cache_a_path)?;
743
744        // Run find_duplicates on a
745        let hasher_a = FileHasher::new(&[&root_a])?;
746        let duplicates_a = hasher_a.find_duplicates()?;
747        assert_eq!(duplicates_a.len(), 1);
748        assert_eq!(hasher_a.num_hashed.load(atomic::Ordering::Relaxed), 0);
749        assert_eq!(
750            hasher_a.num_hash_looked_up.load(atomic::Ordering::Relaxed),
751            2
752        );
753
754        // The merged child cache should be removed.
755        assert!(cache_a_path.exists());
756        assert!(!cache_aa_path.exists());
757
758        Ok(())
759    }
760
761    #[test]
762    fn find_duplicates_with_exclude() -> anyhow::Result<()> {
763        let dir = tempfile::tempdir()?;
764
765        let file1_path = dir.path().join("same1.txt");
766        fs::write(&file1_path, "same content")?;
767
768        let file2_path = dir.path().join("same2.txt");
769        fs::write(&file2_path, "same content")?;
770
771        let exclude_path = dir.path().join("exclude.txt");
772        fs::write(&exclude_path, "same content")?;
773
774        let mut hasher = FileHasher::new(&[dir.path()])?;
775        hasher.buffer_size = 8192;
776        let mut builder = globset::GlobSetBuilder::new();
777        builder.add(
778            globset::GlobBuilder::new("exclude.txt")
779                .case_insensitive(true)
780                .build()?,
781        );
782        let filter = builder.build()?;
783        hasher.exclude = Some(filter);
784
785        let duplicates = hasher.find_duplicates()?;
786        assert_eq!(duplicates.len(), 1);
787        let group = &duplicates[0];
788        assert_eq!(group.paths.len(), 2);
789        assert!(group.paths.contains(&file1_path));
790        assert!(group.paths.contains(&file2_path));
791        assert!(!group.paths.contains(&exclude_path));
792        Ok(())
793    }
794
795    #[derive(Default)]
796    struct CheckCollector {
797        start_seen: bool,
798        total_files: Option<u64>,
799        results: Vec<FileComparisonResult>,
800        file_done_count: u64,
801        num_error: usize,
802    }
803
804    impl CheckCollector {
805        fn collect(rx: mpsc::Receiver<CheckEvent>, base_dir: &Path) -> Self {
806            let mut collector = Self::default();
807            collector._collect(rx, base_dir);
808            collector
809        }
810
811        fn _collect(&mut self, rx: mpsc::Receiver<CheckEvent>, base_dir: &Path) {
812            while let Ok(event) = rx.recv() {
813                match event {
814                    CheckEvent::StartChecking => self.start_seen = true,
815                    CheckEvent::Total(total) => self.total_files = Some(total.num_files),
816                    CheckEvent::Result(mut result, _size) => {
817                        result.relative_path = result
818                            .relative_path
819                            .strip_prefix(base_dir)
820                            .unwrap()
821                            .to_path_buf();
822                        self.results.push(result);
823                    }
824                    CheckEvent::Progress(progress_val) => {
825                        self.file_done_count += progress_val.num_files;
826                    }
827                    CheckEvent::Error(_) => {
828                        self.num_error += 1;
829                    }
830                }
831            }
832        }
833    }
834
835    #[test]
836    fn check_mode_empty_cache() -> anyhow::Result<()> {
837        let dir = tempfile::tempdir()?;
838        let dir_path = dir.path().to_path_buf();
839        println!("{:?}", dir_path);
840        let file1_path = dir.path().join("file1.txt");
841        fs::write(&file1_path, "content 1")?;
842        let file2_path = dir.path().join("file2.txt");
843        fs::write(&file2_path, "content 2")?;
844
845        let mut hasher = FileHasher::new(&[&dir_path])?;
846        hasher.exclude = Some(default_exclude());
847        let (tx, rx) = mpsc::channel();
848        hasher.check_streaming(tx, false)?;
849        let collector = CheckCollector::collect(rx, &dir_path);
850        assert!(collector.start_seen);
851        assert_eq!(collector.total_files, Some(2));
852        assert_eq!(collector.file_done_count, 0);
853        assert_eq!(collector.num_error, 0);
854
855        let mut results = collector.results;
856        results.sort_by(|a, b| a.relative_path.cmp(&b.relative_path));
857        assert_eq!(results.len(), 2);
858        assert_eq!(results[0].relative_path, Path::new("file1.txt"));
859        assert_eq!(results[0].classification, Classification::OnlyInDir2);
860        assert_eq!(results[1].relative_path, Path::new("file2.txt"));
861        assert_eq!(results[1].classification, Classification::OnlyInDir2);
862
863        assert!(!dir.path().join(FileHashCache::FILE_NAME).exists());
864        Ok(())
865    }
866
867    #[test]
868    fn check_mode_with_cache() -> anyhow::Result<()> {
869        let dir = tempfile::tempdir()?;
870        let dir_path = dir.path().to_path_buf();
871        let file1_path = dir.path().join("file1.txt");
872        let file2_path = dir.path().join("file2.txt");
873        fs::write(&file1_path, "content 1")?;
874        fs::write(&file2_path, "content 2")?;
875        let file1 = FileItem::try_from(file1_path.as_path())?;
876        let file2 = FileItem::try_from(file2_path.as_path())?;
877
878        let mut hasher = FileHasher::new_with_cache(&[&dir_path])?;
879        hasher.exclude = Some(default_exclude());
880        let _hash1 = hasher.get_hash(&file1)?;
881        let _hash2 = hasher.get_hash(&file2)?;
882        hasher.save_cache()?;
883        assert!(dir.path().join(FileHashCache::FILE_NAME).exists());
884
885        let mut hasher = FileHasher::new(&[&dir_path])?;
886        hasher.exclude = Some(default_exclude());
887        let (tx, rx) = mpsc::channel();
888        hasher.check_streaming(tx, false)?;
889        let collector = CheckCollector::collect(rx, &dir_path);
890        assert_eq!(collector.results.len(), 0);
891        assert_eq!(collector.file_done_count, 2);
892
893        fs::write(&file1_path, "content 1 modified")?;
894
895        let file2_meta_before = fs::metadata(&file2_path)?;
896        let mtime_before = file2_meta_before.modified()?;
897        std::thread::sleep(time::Duration::from_millis(10));
898        fs::write(&file2_path, "content 2")?;
899        let file2_meta_after = fs::metadata(&file2_path)?;
900        let mtime_after = file2_meta_after.modified()?;
901        assert!(mtime_after > mtime_before);
902
903        let mut hasher = FileHasher::new(&[&dir_path])?;
904        hasher.exclude = Some(default_exclude());
905        let (tx, rx) = mpsc::channel();
906        hasher.check_streaming(tx, false)?;
907        let collector = CheckCollector::collect(rx, &dir_path);
908        assert_eq!(collector.results.len(), 1);
909        let results = collector.results;
910        assert_eq!(results[0].relative_path, Path::new("file1.txt"));
911        assert_eq!(results[0].modified_time_comparison, Some(Ordering::Less));
912        assert_eq!(results[0].size_comparison, Some(Ordering::Less));
913        assert_eq!(results[0].is_content_same, None);
914        assert_eq!(collector.file_done_count, 1);
915        Ok(())
916    }
917
918    #[test]
919    fn check_update_mode() -> anyhow::Result<()> {
920        let dir = tempfile::tempdir()?;
921        let dir_path = dir.path().to_path_buf();
922        let file1_path = dir.path().join("file1.txt");
923        fs::write(&file1_path, "content 1")?;
924
925        let mut hasher = FileHasher::new(&[&dir_path])?;
926        hasher.exclude = Some(default_exclude());
927        let (tx, rx) = mpsc::channel();
928        hasher.check_streaming(tx, true)?;
929        let _ = CheckCollector::collect(rx, &dir_path);
930        hasher.save_cache()?;
931        assert!(dir.path().join(FileHashCache::FILE_NAME).exists());
932
933        let cache = FileHashCache::new(&dir_path);
934        let file1 = FileItem::try_from(file1_path.as_path())?;
935        let hash1 = cache.get(&PathBuf::from("file1.txt"), &file1);
936        assert!(hash1.is_some());
937
938        std::thread::sleep(time::Duration::from_millis(10));
939        fs::write(&file1_path, "content 1 modified")?;
940        let file1_mod = FileItem::try_from(file1_path.as_path())?;
941
942        let mut hasher = FileHasher::new(&[&dir_path])?;
943        hasher.exclude = Some(default_exclude());
944        let (tx, rx) = mpsc::channel();
945        hasher.check_streaming(tx, true)?;
946        let _ = CheckCollector::collect(rx, &dir_path);
947        hasher.save_cache()?;
948
949        let cache = FileHashCache::new(&dir_path);
950        let hash_mod = cache.get(&PathBuf::from("file1.txt"), &file1_mod);
951        assert!(hash_mod.is_some());
952        assert_ne!(hash1, hash_mod);
953
954        std::thread::sleep(time::Duration::from_millis(10));
955        fs::write(&file1_path, "content 1 modified")?;
956        let file1_mod2 = FileItem::try_from(file1_path.as_path())?;
957        assert!(file1_mod2.modified() > file1_mod.modified());
958
959        assert!(
960            cache
961                .get(&PathBuf::from("file1.txt"), &file1_mod2)
962                .is_none()
963        );
964
965        let mut hasher = FileHasher::new(&[&dir_path])?;
966        hasher.exclude = Some(default_exclude());
967        let (tx, rx) = mpsc::channel();
968        hasher.check_streaming(tx, true)?;
969        let _ = CheckCollector::collect(rx, &dir_path);
970        hasher.save_cache()?;
971
972        let cache = FileHashCache::new(&dir_path);
973        assert!(
974            cache
975                .get(&PathBuf::from("file1.txt"), &file1_mod2)
976                .is_some()
977        );
978        Ok(())
979    }
980
981    #[test]
982    fn check_cleanup_deleted_files() -> anyhow::Result<()> {
983        let dir = tempfile::tempdir()?;
984        let dir_path = dir.path().to_path_buf();
985        let file1_path = dir.path().join("file1.txt");
986        let file2_path = dir.path().join("file2.txt");
987        fs::write(&file1_path, "content 1")?;
988        fs::write(&file2_path, "content 2")?;
989        let file1 = FileItem::try_from(file1_path.as_path())?;
990        let file2 = FileItem::try_from(file2_path.as_path())?;
991
992        let mut hasher = FileHasher::new(&[&dir_path])?;
993        hasher.exclude = Some(default_exclude());
994        let (tx, rx) = mpsc::channel();
995        hasher.check_streaming(tx, true)?;
996        let _ = CheckCollector::collect(rx, &dir_path);
997        hasher.save_cache()?;
998
999        // Verify both are in the cache
1000        let cache = FileHashCache::new(&dir_path);
1001        assert!(cache.get(&PathBuf::from("file1.txt"), &file1).is_some());
1002        assert!(cache.get(&PathBuf::from("file2.txt"), &file2).is_some());
1003
1004        // Now delete file2 from disk
1005        fs::remove_file(&file2_path)?;
1006
1007        // Run check and save again
1008        let mut hasher = FileHasher::new(&[&dir_path])?;
1009        hasher.exclude = Some(default_exclude());
1010        let (tx, rx) = mpsc::channel();
1011        hasher.check_streaming(tx, true)?;
1012        let _ = CheckCollector::collect(rx, &dir_path);
1013        hasher.save_cache()?;
1014
1015        // Verify file2 is removed from cache, but file1 is still there
1016        let cache = FileHashCache::new(&dir_path);
1017        assert!(cache.get(&PathBuf::from("file2.txt"), &file2).is_none());
1018        assert!(cache.get(&PathBuf::from("file1.txt"), &file1).is_some());
1019        Ok(())
1020    }
1021
1022    #[test]
1023    fn find_duplicates_multiple_dirs() -> anyhow::Result<()> {
1024        let tmp = tempfile::tempdir()?;
1025        let dir1 = tmp.path().join("dir1");
1026        let dir2 = tmp.path().join("dir2");
1027        fs::create_dir(&dir1)?;
1028        fs::create_dir(&dir2)?;
1029        let file1_path = dir1.join("file1.txt");
1030        fs::write(&file1_path, "same content")?;
1031        let file2_path = dir2.join("file2.txt");
1032        fs::write(&file2_path, "same content")?;
1033        let hasher = FileHasher::new(&[&dir1, &dir2])?;
1034        let duplicates = hasher.find_duplicates()?;
1035        assert_eq!(duplicates.len(), 1);
1036        let group = &duplicates[0];
1037        assert_eq!(group.paths.len(), 2);
1038        assert_eq!(group.size, 12);
1039        assert!(group.paths.contains(&file1_path));
1040        assert!(group.paths.contains(&file2_path));
1041
1042        Ok(())
1043    }
1044
1045    #[test]
1046    fn check_fails_with_multiple_dirs() -> anyhow::Result<()> {
1047        let tmp = tempfile::tempdir()?;
1048        let dir1 = tmp.path().join("dir1");
1049        let dir2 = tmp.path().join("dir2");
1050        fs::create_dir(&dir1)?;
1051        fs::create_dir(&dir2)?;
1052        let hasher = FileHasher::new(&[&dir1, &dir2])?;
1053        assert!(hasher.check(false).is_err());
1054        Ok(())
1055    }
1056
1057    #[test]
1058    fn escape_shell() {
1059        let escape_shell = |p: &str| DuplicatedFiles::escape_quotes_for_shell(Path::new(p));
1060        assert_eq!(escape_shell(""), "");
1061        assert_eq!(escape_shell("abc"), "abc");
1062        assert_eq!(escape_shell("a'b"), "a'\\''b");
1063        assert_eq!(escape_shell("a'b'"), "a'\\''b'\\''");
1064
1065        let escape_shell_double = |p: &str| DuplicatedFiles::escape_quotes_by_double(Path::new(p));
1066        assert_eq!(escape_shell_double(""), "");
1067        assert_eq!(escape_shell_double("abc"), "abc");
1068        assert_eq!(escape_shell_double("a'b"), "a''b");
1069        assert_eq!(escape_shell_double("a'b'"), "a''b''");
1070    }
1071
1072    #[test]
1073    fn write_dups_shell_empty() -> anyhow::Result<()> {
1074        let dup_empty = DuplicatedFiles {
1075            paths: vec![],
1076            size: 100,
1077        };
1078        let mut buf = Vec::new();
1079        dup_empty.write_shell(&mut buf)?;
1080        assert_eq!(String::from_utf8(buf)?, "");
1081        Ok(())
1082    }
1083
1084    #[test]
1085    fn write_dups_shell_one() -> anyhow::Result<()> {
1086        let dup_one = DuplicatedFiles {
1087            paths: vec![PathBuf::from("a.txt")],
1088            size: 100,
1089        };
1090        let mut buf = Vec::new();
1091        dup_one.write_shell(&mut buf)?;
1092        assert_eq!(String::from_utf8(buf)?, "");
1093        Ok(())
1094    }
1095
1096    #[test]
1097    fn write_dups_shell_two() -> anyhow::Result<()> {
1098        let dup_multiple = DuplicatedFiles {
1099            paths: vec![PathBuf::from("a.txt"), PathBuf::from("b.txt")],
1100            size: 100,
1101        };
1102        let mut buf = Vec::new();
1103        dup_multiple.write_shell(&mut buf)?;
1104        assert_eq!(String::from_utf8(buf)?, "cp 'a.txt' 'b.txt'\n");
1105        Ok(())
1106    }
1107
1108    #[test]
1109    fn write_dups_shell_three() -> anyhow::Result<()> {
1110        let dup_multiple = DuplicatedFiles {
1111            paths: vec![
1112                PathBuf::from("a.txt"),
1113                PathBuf::from("b.txt"),
1114                PathBuf::from("c.txt"),
1115            ],
1116            size: 100,
1117        };
1118        let mut buf = Vec::new();
1119        dup_multiple.write_shell(&mut buf)?;
1120        assert_eq!(
1121            String::from_utf8(buf)?,
1122            "cp 'a.txt' 'b.txt'\ncp 'a.txt' 'c.txt'\n"
1123        );
1124        Ok(())
1125    }
1126
1127    #[test]
1128    fn write_dups_shell_quotes() -> anyhow::Result<()> {
1129        let dup_quotes = DuplicatedFiles {
1130            paths: vec![PathBuf::from("a'b.txt"), PathBuf::from("c'd.txt")],
1131            size: 100,
1132        };
1133        let mut buf = Vec::new();
1134        dup_quotes.write_shell(&mut buf)?;
1135        assert_eq!(String::from_utf8(buf)?, "cp 'a'\\''b.txt' 'c'\\''d.txt'\n");
1136
1137        let mut buf = Vec::new();
1138        dup_quotes.write_pwsh(&mut buf)?;
1139        assert_eq!(
1140            String::from_utf8(buf)?,
1141            "Copy-Item -LiteralPath 'a''b.txt' 'c''d.txt'\n"
1142        );
1143        Ok(())
1144    }
1145}
compare_dir/file_hasher.rs

compare_dir/
file_hasher.rs
