compare_dir/
file_hasher.rs

1use crate::{
2    CheckMode, Classification, ColumnFormatter, DirectoryComparer, FileComparer,
3    FileComparisonResult, FileHashCache, FileItem, FileIterator, OutputFormat, Progress,
4    ProgressBuilder, ProgressValue, SharedProgress,
5};
6use globset::GlobSet;
7use indicatif::FormattedDuration;
8use rayon::prelude::*;
9use simple_path::SimplePath;
10use std::{
11    collections::HashMap,
12    fs,
13    io::{self, Read, stdout},
14    path::{Path, PathBuf},
15    sync::{
16        Arc,
17        atomic::{self, AtomicUsize},
18        mpsc,
19    },
20    time,
21};
22
23type FileWithDirIndex = (FileItem, usize);
24
25#[derive(Debug, Clone)]
26enum DupEvent {
27    StartHashing,
28    Total(ProgressValue),
29    Result(FileItem, blake3::Hash),
30    Error,
31}
32
33#[derive(Debug)]
34enum CheckEvent {
35    StartChecking,
36    Total(ProgressValue),
37    Result(FileComparisonResult),
38    Progress(ProgressValue),
39    Error(FileItem),
40}
41
42enum DupState {
43    Single(FileItem, usize),
44    Hashing,
45}
46
47/// A tool for finding duplicated files in a directory.
48pub struct FileHasher {
49    dirs: Vec<PathBuf>,
50    pub buffer_size: usize,
51    cache: Option<Arc<FileHashCache>>,
52    num_hashed: AtomicUsize,
53    num_hash_looked_up: AtomicUsize,
54    pub exclude: Option<GlobSet>,
55    pub progress: Option<Arc<ProgressBuilder>>,
56    pub output_format: OutputFormat,
57    pub jobs: usize,
58}
59
60impl FileHasher {
61    const DEFAULT_JOBS: usize = DirectoryComparer::DEFAULT_JOBS;
62
63    /// Creates a new `FileHasher` for the given directories.
64    pub fn new<P: AsRef<Path>>(dirs: &[P]) -> anyhow::Result<Self> {
65        if dirs.is_empty() {
66            anyhow::bail!("At least one directory must be specified.");
67        }
68        Ok(Self {
69            dirs: dirs.iter().map(|p| p.as_ref().to_path_buf()).collect(),
70            buffer_size: FileComparer::DEFAULT_BUFFER_SIZE,
71            cache: None,
72            num_hashed: AtomicUsize::new(0),
73            num_hash_looked_up: AtomicUsize::new(0),
74            exclude: None,
75            progress: None,
76            output_format: OutputFormat::Default,
77            jobs: Self::DEFAULT_JOBS,
78        })
79    }
80
81    pub(crate) fn new_with_cache<P: AsRef<Path>>(dirs: &[P]) -> anyhow::Result<Self> {
82        let mut hasher = Self::new(dirs)?;
83        hasher.cache = Some(hasher.new_cache()?);
84        Ok(hasher)
85    }
86
87    fn new_cache(&self) -> anyhow::Result<Arc<FileHashCache>> {
88        let common_ancestor = crate::common_ancestor(&self.dirs)
89            .ok_or_else(|| anyhow::anyhow!("No common ancestor found"))?;
90        Ok(FileHashCache::find_or_new(&common_ancestor))
91    }
92
93    /// Gets the hash cache.
94    pub(crate) fn cache(&mut self) -> anyhow::Result<Arc<FileHashCache>> {
95        if self.cache.is_none() {
96            self.cache = Some(self.new_cache()?);
97        }
98        Ok(Arc::clone(self.cache.as_ref().unwrap()))
99    }
100
101    /// Remove a cache entry if it exists.
102    pub(crate) fn remove_cache_entry(&mut self, path: &Path) -> anyhow::Result<()> {
103        let cache = self.cache()?;
104        let relative = SimplePath::strip_prefix(path, cache.base_dir())?;
105        cache.remove(relative);
106        Ok(())
107    }
108
109    /// Save the hash cache if it is dirty.
110    pub fn save_cache(&self) -> anyhow::Result<()> {
111        log::info!(
112            "Hash stats for {:?}: {} computed, {} looked up",
113            self.dirs,
114            self.num_hashed.load(atomic::Ordering::Relaxed),
115            self.num_hash_looked_up.load(atomic::Ordering::Relaxed)
116        );
117        if let Some(cache) = &self.cache {
118            cache.save()?;
119        }
120        Ok(())
121    }
122
123    /// Clears the loaded hashes in the cache.
124    pub(crate) fn clear_cache(&mut self) -> anyhow::Result<()> {
125        let cache = self.cache()?;
126        for dir in &self.dirs {
127            let relative = SimplePath::strip_prefix(dir, cache.base_dir())?;
128            cache.clear(relative);
129        }
130        Ok(())
131    }
132
133    /// Executes the check/update process.
134    pub fn check(&self, mode: CheckMode) -> anyhow::Result<()> {
135        match self.output_format {
136            OutputFormat::Default | OutputFormat::Symbol => {}
137            _ => anyhow::bail!("Check mode only supports default or symbol output format."),
138        }
139        if self.dirs.len() > 1 {
140            anyhow::bail!("Check mode only supports one directory.");
141        }
142        let start_time = time::Instant::now();
143        if let Some(p) = &self.progress {
144            p.set_propagate();
145        }
146        let progress = self
147            .progress
148            .as_ref()
149            .map(|progress| progress.add_primary())
150            .unwrap_or_else(SharedProgress::none);
151        progress.use_bytes();
152        progress.set_message("Scanning directory...");
153        let mut num_new = 0;
154        let mut num_modified = 0;
155        let mut num_error = 0;
156        std::thread::scope(|scope| {
157            let (tx, rx) = mpsc::channel();
158            scope.spawn(|| {
159                if let Err(e) = self.check_streaming(tx, mode) {
160                    log::error!("Error during check: {}", e);
161                }
162            });
163            while let Ok(event) = rx.recv() {
164                match event {
165                    CheckEvent::StartChecking => {
166                        progress.set_message("Checking files...");
167                    }
168                    CheckEvent::Total(value) => {
169                        progress.set_length(value);
170                        progress.set_message("");
171                    }
172                    CheckEvent::Result(result) => {
173                        progress.suspend_for(stdout(), || {
174                            result.print(self.output_format, "cached", "current")
175                        });
176                        if result.classification == Classification::OnlyInDir2 {
177                            num_new += 1;
178                        } else if result.is_identical_content() == Some(false) {
179                            num_modified += 1;
180                        }
181                    }
182                    CheckEvent::Progress(value) => progress.inc(value),
183                    CheckEvent::Error(file) => {
184                        progress.inc(ProgressValue::with_skip(file.size()));
185                        num_error += 1;
186                    }
187                }
188            }
189        });
190        progress.finish();
191        self.print_check_summary(&start_time, num_new, num_modified, num_error)?;
192        Ok(())
193    }
194
195    fn print_check_summary(
196        &self,
197        start_time: &time::Instant,
198        num_new: usize,
199        num_modified: usize,
200        num_error: usize,
201    ) -> io::Result<()> {
202        let summary = [
203            ("Elapsed:", 0),
204            (
205                "Hash computed:",
206                self.num_hashed.load(atomic::Ordering::Relaxed),
207            ),
208            ("New files:", num_new),
209            ("Modified files:", num_modified),
210            ("Errors:", num_error),
211        ];
212        let formatter = ColumnFormatter::new(summary.iter().map(|(s, _)| *s));
213        let mut writer = std::io::stderr();
214        formatter.write_value(
215            &mut writer,
216            summary[0].0,
217            FormattedDuration(start_time.elapsed()),
218        )?;
219        formatter.write_values(&mut writer, &summary[1..])
220    }
221
222    fn check_streaming(&self, tx: mpsc::Sender<CheckEvent>, mode: CheckMode) -> anyhow::Result<()> {
223        assert_eq!(self.dirs.len(), 1);
224        let cache = self.new_cache()?;
225        let base_dir = &self.dirs[0];
226        let relative = SimplePath::strip_prefix(base_dir, cache.base_dir())?;
227        cache.set_remove_if_no_access(relative);
228        let cache_clone = Arc::clone(&cache);
229        std::thread::scope(|global_scope| {
230            let mut it = FileIterator::new(base_dir);
231            it.cache = Some(Arc::clone(&cache));
232            it.exclude = self.exclude.as_ref();
233            let it_rx = it.spawn_in_scope(global_scope);
234            tx.send(CheckEvent::StartChecking)?;
235            let pool = crate::build_thread_pool(self.jobs)?;
236            pool.scope(move |scope| -> anyhow::Result<()> {
237                let mut total = ProgressValue::default();
238                for file in it_rx {
239                    self.check_file(file, &cache, mode, &mut total, &tx, scope);
240                }
241                tx.send(CheckEvent::Total(total))?;
242                Ok(())
243            })
244        })?;
245        cache_clone.save()?;
246        Ok(())
247    }
248
249    fn check_file<'scope>(
250        &'scope self,
251        file: FileItem,
252        cache: &Arc<FileHashCache>,
253        mode: CheckMode,
254        total: &mut ProgressValue,
255        tx: &mpsc::Sender<CheckEvent>,
256        scope: &rayon::Scope<'scope>,
257    ) {
258        *total += ProgressValue::with_size(file.size());
259        let tx = tx.clone();
260        let cache = Arc::clone(cache);
261        scope.spawn(move |_| {
262            if let Err(error) = self._check_file(&file, cache, mode, &tx) {
263                log::error!("Failed to check file '{}': {}", file, error);
264                if tx.send(CheckEvent::Error(file)).is_err() {
265                    log::error!("Send failed");
266                }
267            }
268        });
269    }
270
271    fn _check_file(
272        &self,
273        file: &FileItem,
274        cache: Arc<FileHashCache>,
275        mode: CheckMode,
276        tx: &mpsc::Sender<CheckEvent>,
277    ) -> anyhow::Result<()> {
278        assert!(file.path().is_absolute());
279        let path_in_cache = file.relative_path(cache.base_dir());
280        match cache.get_entry(path_in_cache) {
281            Some(cached) => {
282                if mode == CheckMode::Update && cached.eq(file) {
283                    // In update mode, consider the content isn't changed if the
284                    // metadata is the same,
285                    tx.send(CheckEvent::Progress(ProgressValue::with_skip(file.size())))?;
286                    return Ok(());
287                }
288                let mut result =
289                    FileComparisonResult::new(file.path().into(), Classification::InBoth);
290                result.update_moodified(cached.modified, file.modified());
291                if cached.size != 0 {
292                    result.update_size(cached.size, file.size());
293                }
294                if mode == CheckMode::Check && cached.size != 0 && file.size() != cached.size {
295                    // In `Check` mode, hash isn't needed if the size is different.
296                    tx.send(CheckEvent::Result(result))?;
297                    tx.send(CheckEvent::Progress(ProgressValue::with_skip(file.size())))?;
298                    return Ok(());
299                }
300                let hash = self.compute_hash(file)?;
301                result.is_content_same = Some(hash == cached.hash);
302                if hash == cached.hash {
303                    if cached.should_update(file, mode) {
304                        cache.insert(path_in_cache, file, hash);
305                    }
306                } else {
307                    if mode == CheckMode::Update || mode == CheckMode::UpdateAll {
308                        cache.insert(path_in_cache, file, hash);
309                    }
310                    tx.send(CheckEvent::Result(result))?;
311                }
312            }
313            None => {
314                if mode == CheckMode::Update || mode == CheckMode::UpdateAll {
315                    let hash = self.compute_hash(file)?;
316                    cache.insert(path_in_cache, file, hash);
317                } else {
318                    tx.send(CheckEvent::Progress(ProgressValue::with_skip(file.size())))?;
319                }
320                tx.send(CheckEvent::Result(FileComparisonResult::new(
321                    file.path().into(),
322                    Classification::OnlyInDir2,
323                )))?;
324            }
325        }
326        Ok(())
327    }
328
329    /// Executes the duplicate file finding process and prints results.
330    pub fn run(&self) -> anyhow::Result<()> {
331        let start_time = time::Instant::now();
332        let mut duplicates = self.find_duplicates()?;
333        let mut total_wasted_space = 0;
334        if !duplicates.is_empty() {
335            duplicates.sort_by_key(|a| a.size);
336            total_wasted_space = self.print_duplicates_results(&duplicates)?;
337        }
338        self.print_duplicates_summary(&start_time, total_wasted_space)?;
339        Ok(())
340    }
341
342    fn print_duplicates_results(&self, duplicates: &Vec<DuplicatedFiles>) -> anyhow::Result<u64> {
343        let mut total_wasted_space = 0;
344        for dupes in duplicates {
345            dupes.print(self.output_format)?;
346            total_wasted_space += dupes.wasted_size();
347        }
348        Ok(total_wasted_space)
349    }
350
351    fn print_duplicates_summary(
352        &self,
353        start_time: &time::Instant,
354        total_wasted_space: u64,
355    ) -> io::Result<()> {
356        let elapsed = FormattedDuration(start_time.elapsed()).to_string();
357        let num_hashed = self.num_hashed.load(atomic::Ordering::Relaxed).to_string();
358        let total_wasted_space = crate::human_readable_size(total_wasted_space);
359        let summary = [
360            ("Elapsed:", elapsed),
361            ("Hash computed:", num_hashed),
362            ("Total wasted space:", total_wasted_space),
363        ];
364        let formatter = ColumnFormatter::new(summary.iter().map(|(s, _)| *s));
365        formatter.write_values(&mut io::stderr(), &summary)
366    }
367
368    /// Finds duplicated files and returns a list of duplicate groups.
369    pub fn find_duplicates(&self) -> anyhow::Result<Vec<DuplicatedFiles>> {
370        let progress = self
371            .progress
372            .as_ref()
373            .map(|progress| progress.add_primary())
374            .unwrap_or_else(SharedProgress::none);
375        progress.set_message("Scanning directories...");
376
377        let (tx, rx) = mpsc::channel();
378        let mut by_hash: HashMap<blake3::Hash, DuplicatedFiles> = HashMap::new();
379        std::thread::scope(|scope| {
380            scope.spawn(|| {
381                if let Err(e) = self.find_duplicates_streaming(tx) {
382                    log::error!("Error during duplicate finding: {}", e);
383                }
384            });
385
386            while let Ok(event) = rx.recv() {
387                match event {
388                    DupEvent::StartHashing => progress.set_message("Hashing files..."),
389                    DupEvent::Total(value) => progress.set_length(value),
390                    DupEvent::Result(file, hash) => {
391                        progress.inc(ProgressValue::with_size(file.size()));
392                        let entry = by_hash.entry(hash).or_insert_with(|| DuplicatedFiles {
393                            paths: Vec::new(),
394                            size: file.size(),
395                        });
396                        // Hash collisions shouldn't happen, but if they do, sizes shouldn't mismatch.
397                        assert_eq!(
398                            entry.size,
399                            file.size(),
400                            "Hash collision: sizes do not match"
401                        );
402                        entry.paths.push(file.into_path_buf());
403                    }
404                    DupEvent::Error => {}
405                }
406            }
407        });
408        progress.finish();
409
410        let mut duplicates = Vec::new();
411        for (_, mut dupes) in by_hash {
412            if dupes.paths.len() > 1 {
413                dupes.paths.sort();
414                duplicates.push(dupes);
415            }
416        }
417        Ok(duplicates)
418    }
419
420    fn find_duplicates_streaming(&self, tx: mpsc::Sender<DupEvent>) -> anyhow::Result<()> {
421        std::thread::scope(|global_scope| {
422            let (it_rx, caches) = self.stream_file_items(global_scope)?;
423            let caches = &caches;
424            let pool = crate::build_thread_pool(self.jobs)?;
425            pool.scope(move |scope| -> anyhow::Result<()> {
426                let mut by_size: HashMap<u64, DupState> = HashMap::new();
427                let mut total = ProgressValue::default();
428                tx.send(DupEvent::StartHashing)?;
429                for (file, dir_index) in it_rx {
430                    let size = file.size();
431                    if size == 0 {
432                        continue;
433                    }
434                    let cache = &caches[dir_index];
435                    match by_size.entry(size) {
436                        std::collections::hash_map::Entry::Occupied(mut occ) => match occ.get_mut()
437                        {
438                            DupState::Single(file0, dir_index0) => {
439                                // We found a second file of identical size.
440                                // Time to start hashing both the *original* matching file and the *new* one!
441                                let cache0 = &caches[*dir_index0];
442                                self.send_hash(file0, cache0, &tx, scope);
443                                self.send_hash(&file, cache, &tx, scope);
444                                total += ProgressValue::with_size(file0.size());
445                                total += ProgressValue::with_size(file.size());
446
447                                // Modify the state to indicate we are now fully hashing this size bucket.
448                                *occ.get_mut() = DupState::Hashing;
449                            }
450                            DupState::Hashing => {
451                                // File size bucket already hashing; just dynamically spawn the new file immediately.
452                                self.send_hash(&file, cache, &tx, scope);
453                                total += ProgressValue::with_size(file.size());
454                            }
455                        },
456                        std::collections::hash_map::Entry::Vacant(vac) => {
457                            vac.insert(DupState::Single(file, dir_index));
458                        }
459                    }
460                }
461                tx.send(DupEvent::Total(total))?;
462                Ok(())
463            })?;
464            pool.install(|| caches.into_par_iter().try_for_each(|cache| cache.save()))?;
465            Ok::<(), anyhow::Error>(())
466        })?;
467        Ok(())
468    }
469
470    fn stream_file_items<'scope, 'env>(
471        &'env self,
472        scope: &'scope std::thread::Scope<'scope, 'env>,
473    ) -> anyhow::Result<(mpsc::Receiver<FileWithDirIndex>, Vec<Arc<FileHashCache>>)> {
474        let (it_tx, it_rx) = mpsc::channel();
475        let mut caches = Vec::with_capacity(self.dirs.len());
476        for (dir_index, dir) in self.dirs.iter().enumerate() {
477            let mut it = FileIterator::new(dir);
478            let cache = FileHashCache::find_or_new(dir);
479            it.cache = Some(Arc::clone(&cache));
480            it.exclude = self.exclude.as_ref();
481            let it_tx = it_tx.clone();
482            scope.spawn(move || it.send_to_as(it_tx, |path| (path, dir_index)));
483            caches.push(cache);
484        }
485        Ok((it_rx, caches))
486    }
487
488    fn send_hash<'scope>(
489        &'scope self,
490        file: &FileItem,
491        cache: &Arc<FileHashCache>,
492        tx: &mpsc::Sender<DupEvent>,
493        scope: &rayon::Scope<'scope>,
494    ) {
495        let (hash, relative) = self
496            .get_hash_from_cache(file, cache)
497            .expect("path should be in cache base_dir");
498        if let Some(hash) = hash {
499            let _ = tx.send(DupEvent::Result(file.clone(), hash));
500            return;
501        }
502
503        let file = file.clone();
504        let relative = relative.to_path_buf();
505        let tx = tx.clone();
506        let cache = Arc::clone(cache);
507        scope.spawn(move |_| {
508            if let Ok(hash) = self.compute_hash(&file) {
509                cache.insert(&relative, &file, hash);
510                let _ = tx.send(DupEvent::Result(file, hash));
511            } else {
512                log::error!("Failed to hash file: '{}'", file);
513                let _ = tx.send(DupEvent::Error);
514            }
515        });
516    }
517
518    /// Gets the hash of a file, using the cache if available.
519    pub fn get_hash(&self, file: &FileItem) -> anyhow::Result<blake3::Hash> {
520        let cache = self.cache.as_ref().expect("cache should be initialized");
521        let (hash, relative) = self.get_hash_from_cache(file, cache)?;
522        if let Some(hash) = hash {
523            return Ok(hash);
524        }
525
526        let hash = self.compute_hash(file)?;
527        cache.insert(relative, file, hash);
528        Ok(hash)
529    }
530
531    fn get_hash_from_cache<'a>(
532        &self,
533        file: &'a FileItem,
534        cache: &FileHashCache,
535    ) -> io::Result<(Option<blake3::Hash>, &'a Path)> {
536        let relative = file.relative_path(cache.base_dir());
537        if let Some(hash) = cache.get(relative, file) {
538            self.num_hash_looked_up
539                .fetch_add(1, atomic::Ordering::Relaxed);
540            return Ok((Some(hash), relative));
541        }
542        Ok((None, relative))
543    }
544
545    fn compute_hash(&self, file: &FileItem) -> io::Result<blake3::Hash> {
546        let start_time = time::Instant::now();
547        let mut f = fs::File::open(file.path())?;
548        let mut progress = self
549            .progress
550            .as_ref()
551            .map(|progress| progress.add_file(file.path(), file.size()))
552            .unwrap_or_else(Progress::none);
553        let mut hasher = blake3::Hasher::new();
554        if self.buffer_size == 0 {
555            if file.size() > 0 {
556                let mmap = unsafe { memmap2::MmapOptions::new().map(&f)? };
557                hasher.update(&mmap[..]);
558                progress.inc(ProgressValue::with_size(file.size()));
559            }
560        } else {
561            let mut buf = vec![0u8; self.buffer_size];
562            loop {
563                let n = f.read(&mut buf)?;
564                if n == 0 {
565                    break;
566                }
567                hasher.update(&buf[..n]);
568                progress.inc_size(n as u64);
569            }
570            progress.inc_file(1);
571        }
572        progress.finish();
573        self.num_hashed.fetch_add(1, atomic::Ordering::Relaxed);
574        let hash = hasher.finalize();
575        log::trace!(
576            "Computed hash in {}: '{}'",
577            FormattedDuration(start_time.elapsed()),
578            file
579        );
580        Ok(hash)
581    }
582}
583
584/// A group of duplicated files and their size.
585#[derive(Clone, Debug)]
586pub struct DuplicatedFiles {
587    pub paths: Vec<PathBuf>,
588    pub size: u64,
589}
590
591impl DuplicatedFiles {
592    fn wasted_size(&self) -> u64 {
593        self.size * (self.paths.len() as u64 - 1)
594    }
595
596    fn print(&self, output_format: OutputFormat) -> anyhow::Result<()> {
597        match output_format {
598            OutputFormat::Default => self.write_human(stdout())?,
599            OutputFormat::PowerShell => self.write_pwsh(stdout())?,
600            OutputFormat::Shell => self.write_shell(stdout())?,
601            OutputFormat::Yaml | OutputFormat::Symbol => self.write_yaml(stdout())?,
602        }
603        Ok(())
604    }
605
606    fn write_human(&self, mut writer: impl io::Write) -> anyhow::Result<()> {
607        writeln!(
608            writer,
609            "Identical {} files of {}:",
610            self.paths.len(),
611            crate::human_readable_size(self.size)
612        )?;
613        for path in &self.paths {
614            writeln!(writer, "  {}", path.display())?;
615        }
616        Ok(())
617    }
618
619    fn write_yaml(&self, mut writer: impl io::Write) -> anyhow::Result<()> {
620        writeln!(writer, "- paths:")?;
621        for path in &self.paths {
622            writeln!(writer, "  - '{}'", Self::escape_quotes_by_double(path))?;
623        }
624        writeln!(writer, "  size: {}", self.size)?;
625        Ok(())
626    }
627
628    fn write_shell(&self, writer: impl io::Write) -> anyhow::Result<()> {
629        self.write_shell_with(writer, "cp", Self::escape_quotes_for_shell)
630    }
631
632    fn write_pwsh(&self, writer: impl io::Write) -> anyhow::Result<()> {
633        self.write_shell_with(
634            writer,
635            "Copy-Item -LiteralPath",
636            Self::escape_quotes_by_double,
637        )
638    }
639
640    fn write_shell_with(
641        &self,
642        mut writer: impl io::Write,
643        cmd: &str,
644        stringify: impl Fn(&Path) -> String,
645    ) -> anyhow::Result<()> {
646        let mut iter = self.paths.iter();
647        if let Some(path0) = iter.next() {
648            let path0 = stringify(path0);
649            for path in iter {
650                writeln!(writer, "{cmd} '{path0}' '{}'", stringify(path))?;
651            }
652        }
653        Ok(())
654    }
655
656    /// Escape single quotes for sh.
657    fn escape_quotes_for_shell(path: &Path) -> String {
658        path.to_string_lossy().replace('\'', "'\\''")
659    }
660
661    /// Escape single quotes by doubling them,
662    /// for single quoted strings in zsh, PowerShell, YAML, etc.
663    fn escape_quotes_by_double(path: &Path) -> String {
664        path.to_string_lossy().replace('\'', "''")
665    }
666}
667
668#[cfg(test)]
669mod tests {
670    use super::*;
671    use std::cmp::Ordering;
672
673    fn default_exclude() -> globset::GlobSet {
674        let mut builder = globset::GlobSetBuilder::new();
675        builder.add(
676            globset::GlobBuilder::new(".hash_cache")
677                .case_insensitive(true)
678                .build()
679                .unwrap(),
680        );
681        builder.build().unwrap()
682    }
683
684    #[test]
685    fn find_duplicates() -> anyhow::Result<()> {
686        let dir = tempfile::tempdir()?;
687
688        let file1_path = dir.path().join("same1.txt");
689        fs::write(&file1_path, "same content")?;
690
691        let file2_path = dir.path().join("same2.txt");
692        fs::write(&file2_path, "same content")?;
693
694        let diff_path = dir.path().join("diff.txt");
695        fs::write(&diff_path, "different content")?;
696
697        let mut hasher = FileHasher::new(&[dir.path()])?;
698        hasher.buffer_size = 8192;
699        let duplicates = hasher.find_duplicates()?;
700
701        assert_eq!(hasher.num_hashed.load(atomic::Ordering::Relaxed), 2);
702        assert_eq!(hasher.num_hash_looked_up.load(atomic::Ordering::Relaxed), 0);
703
704        assert_eq!(duplicates.len(), 1);
705        let group = &duplicates[0];
706        assert_eq!(group.paths.len(), 2);
707        assert_eq!(group.size, 12); // "same content" is 12 bytes
708
709        assert!(group.paths.contains(&file1_path));
710        assert!(group.paths.contains(&file2_path));
711
712        Ok(())
713    }
714
715    #[test]
716    fn find_duplicates_merge_cache() -> anyhow::Result<()> {
717        let dir = tempfile::tempdir()?;
718        let dir_path = dir.path();
719
720        let sub_dir = dir_path.join("a").join("a");
721        fs::create_dir_all(&sub_dir)?;
722
723        let file1_path = sub_dir.join("1");
724        fs::write(&file1_path, "same content")?;
725
726        let file2_path = sub_dir.join("2");
727        fs::write(&file2_path, "same content")?;
728
729        // Create empty cache file in a/a to force it to be the cache base
730        let cache_aa_path = sub_dir.join(FileHashCache::FILE_NAME);
731        fs::File::create(&cache_aa_path)?;
732
733        // Run find_duplicates on a/a
734        let hasher_aa = FileHasher::new(&[&sub_dir])?;
735        let duplicates_aa = hasher_aa.find_duplicates()?;
736        assert_eq!(duplicates_aa.len(), 1);
737        assert!(cache_aa_path.exists());
738        assert_eq!(hasher_aa.num_hashed.load(atomic::Ordering::Relaxed), 2);
739        assert_eq!(
740            hasher_aa.num_hash_looked_up.load(atomic::Ordering::Relaxed),
741            0
742        );
743
744        // Create empty cache file in a to force it to be the cache base
745        let root_a = dir_path.join("a");
746        let cache_a_path = root_a.join(FileHashCache::FILE_NAME);
747        fs::File::create(&cache_a_path)?;
748
749        // Run find_duplicates on a
750        let hasher_a = FileHasher::new(&[&root_a])?;
751        let duplicates_a = hasher_a.find_duplicates()?;
752        assert_eq!(duplicates_a.len(), 1);
753        assert_eq!(hasher_a.num_hashed.load(atomic::Ordering::Relaxed), 0);
754        assert_eq!(
755            hasher_a.num_hash_looked_up.load(atomic::Ordering::Relaxed),
756            2
757        );
758
759        // The merged child cache should be removed.
760        assert!(cache_a_path.exists());
761        assert!(!cache_aa_path.exists());
762
763        Ok(())
764    }
765
766    #[test]
767    fn find_duplicates_with_exclude() -> anyhow::Result<()> {
768        let dir = tempfile::tempdir()?;
769
770        let file1_path = dir.path().join("same1.txt");
771        fs::write(&file1_path, "same content")?;
772
773        let file2_path = dir.path().join("same2.txt");
774        fs::write(&file2_path, "same content")?;
775
776        let exclude_path = dir.path().join("exclude.txt");
777        fs::write(&exclude_path, "same content")?;
778
779        let mut hasher = FileHasher::new(&[dir.path()])?;
780        hasher.buffer_size = 8192;
781        let mut builder = globset::GlobSetBuilder::new();
782        builder.add(
783            globset::GlobBuilder::new("exclude.txt")
784                .case_insensitive(true)
785                .build()?,
786        );
787        let filter = builder.build()?;
788        hasher.exclude = Some(filter);
789
790        let duplicates = hasher.find_duplicates()?;
791        assert_eq!(duplicates.len(), 1);
792        let group = &duplicates[0];
793        assert_eq!(group.paths.len(), 2);
794        assert!(group.paths.contains(&file1_path));
795        assert!(group.paths.contains(&file2_path));
796        assert!(!group.paths.contains(&exclude_path));
797        Ok(())
798    }
799
800    #[derive(Default)]
801    struct CheckCollector {
802        start_seen: bool,
803        total_files: Option<u64>,
804        results: Vec<FileComparisonResult>,
805        file_done_count: u64,
806        num_error: usize,
807    }
808
809    impl CheckCollector {
810        fn collect(rx: mpsc::Receiver<CheckEvent>, base_dir: &Path) -> Self {
811            let mut collector = Self::default();
812            collector._collect(rx, base_dir);
813            collector
814        }
815
816        fn _collect(&mut self, rx: mpsc::Receiver<CheckEvent>, base_dir: &Path) {
817            while let Ok(event) = rx.recv() {
818                match event {
819                    CheckEvent::StartChecking => self.start_seen = true,
820                    CheckEvent::Total(total) => self.total_files = Some(total.num_files),
821                    CheckEvent::Result(mut result) => {
822                        result.relative_path = result
823                            .relative_path
824                            .strip_prefix(base_dir)
825                            .unwrap()
826                            .to_path_buf();
827                        self.results.push(result);
828                    }
829                    CheckEvent::Progress(progress_val) => {
830                        self.file_done_count += progress_val.num_files;
831                    }
832                    CheckEvent::Error(_) => {
833                        self.num_error += 1;
834                    }
835                }
836            }
837        }
838    }
839
840    #[test]
841    fn check_mode_empty_cache() -> anyhow::Result<()> {
842        let dir = tempfile::tempdir()?;
843        let dir_path = dir.path().to_path_buf();
844        println!("{:?}", dir_path);
845        let file1_path = dir.path().join("file1.txt");
846        fs::write(&file1_path, "content 1")?;
847        let file2_path = dir.path().join("file2.txt");
848        fs::write(&file2_path, "content 2")?;
849
850        let mut hasher = FileHasher::new(&[&dir_path])?;
851        hasher.exclude = Some(default_exclude());
852        let (tx, rx) = mpsc::channel();
853        hasher.check_streaming(tx, CheckMode::Check)?;
854        let collector = CheckCollector::collect(rx, &dir_path);
855        assert!(collector.start_seen);
856        assert_eq!(collector.total_files, Some(2));
857        assert_eq!(collector.file_done_count, 2);
858        assert_eq!(collector.num_error, 0);
859
860        let mut results = collector.results;
861        results.sort_by(|a, b| a.relative_path.cmp(&b.relative_path));
862        assert_eq!(results.len(), 2);
863        assert_eq!(results[0].relative_path, Path::new("file1.txt"));
864        assert_eq!(results[0].classification, Classification::OnlyInDir2);
865        assert_eq!(results[1].relative_path, Path::new("file2.txt"));
866        assert_eq!(results[1].classification, Classification::OnlyInDir2);
867
868        assert!(!dir.path().join(FileHashCache::FILE_NAME).exists());
869        Ok(())
870    }
871
872    #[test]
873    fn check_mode_with_cache() -> anyhow::Result<()> {
874        let dir = tempfile::tempdir()?;
875        let dir_path = dir.path().to_path_buf();
876        let file1_path = dir.path().join("file1.txt");
877        let file2_path = dir.path().join("file2.txt");
878        fs::write(&file1_path, "content 1")?;
879        fs::write(&file2_path, "content 2")?;
880        let file1 = FileItem::try_from(file1_path.as_path())?;
881        let file2 = FileItem::try_from(file2_path.as_path())?;
882
883        let mut hasher = FileHasher::new_with_cache(&[&dir_path])?;
884        hasher.exclude = Some(default_exclude());
885        let _hash1 = hasher.get_hash(&file1)?;
886        let _hash2 = hasher.get_hash(&file2)?;
887        hasher.save_cache()?;
888        assert!(dir.path().join(FileHashCache::FILE_NAME).exists());
889
890        let mut hasher = FileHasher::new(&[&dir_path])?;
891        hasher.exclude = Some(default_exclude());
892        let (tx, rx) = mpsc::channel();
893        hasher.check_streaming(tx, CheckMode::Check)?;
894        let collector = CheckCollector::collect(rx, &dir_path);
895        assert_eq!(collector.results.len(), 0);
896        assert_eq!(collector.file_done_count, 0);
897
898        fs::write(&file1_path, "content 1 modified")?;
899
900        let file2_meta_before = fs::metadata(&file2_path)?;
901        let mtime_before = file2_meta_before.modified()?;
902        std::thread::sleep(time::Duration::from_millis(10));
903        fs::write(&file2_path, "content 2")?;
904        let file2_meta_after = fs::metadata(&file2_path)?;
905        let mtime_after = file2_meta_after.modified()?;
906        assert!(mtime_after > mtime_before);
907
908        let mut hasher = FileHasher::new(&[&dir_path])?;
909        hasher.exclude = Some(default_exclude());
910        let (tx, rx) = mpsc::channel();
911        hasher.check_streaming(tx, CheckMode::Check)?;
912        let collector = CheckCollector::collect(rx, &dir_path);
913        assert_eq!(collector.results.len(), 1);
914        let results = collector.results;
915        assert_eq!(results[0].relative_path, Path::new("file1.txt"));
916        assert_eq!(results[0].modified_time_comparison, Some(Ordering::Less));
917        assert_eq!(results[0].size_comparison, Some(Ordering::Less));
918        assert_eq!(results[0].is_content_same, None);
919        assert_eq!(collector.file_done_count, 1);
920        Ok(())
921    }
922
923    #[test]
924    fn check_update_mode() -> anyhow::Result<()> {
925        let dir = tempfile::tempdir()?;
926        let dir_path = dir.path().to_path_buf();
927        let file1_path = dir.path().join("file1.txt");
928        fs::write(&file1_path, "content 1")?;
929
930        let mut hasher = FileHasher::new(&[&dir_path])?;
931        hasher.exclude = Some(default_exclude());
932        let (tx, rx) = mpsc::channel();
933        hasher.check_streaming(tx, CheckMode::UpdateAll)?;
934        let _ = CheckCollector::collect(rx, &dir_path);
935        hasher.save_cache()?;
936        assert!(dir.path().join(FileHashCache::FILE_NAME).exists());
937
938        let cache = FileHashCache::new(&dir_path);
939        let file1 = FileItem::try_from(file1_path.as_path())?;
940        let hash1 = cache.get(&PathBuf::from("file1.txt"), &file1);
941        assert!(hash1.is_some());
942
943        std::thread::sleep(time::Duration::from_millis(10));
944        fs::write(&file1_path, "content 1 modified")?;
945        let file1_mod = FileItem::try_from(file1_path.as_path())?;
946
947        let mut hasher = FileHasher::new(&[&dir_path])?;
948        hasher.exclude = Some(default_exclude());
949        let (tx, rx) = mpsc::channel();
950        hasher.check_streaming(tx, CheckMode::UpdateAll)?;
951        let _ = CheckCollector::collect(rx, &dir_path);
952        hasher.save_cache()?;
953
954        let cache = FileHashCache::new(&dir_path);
955        let hash_mod = cache.get(&PathBuf::from("file1.txt"), &file1_mod);
956        assert!(hash_mod.is_some());
957        assert_ne!(hash1, hash_mod);
958
959        std::thread::sleep(time::Duration::from_millis(10));
960        fs::write(&file1_path, "content 1 modified")?;
961        let file1_mod2 = FileItem::try_from(file1_path.as_path())?;
962        assert!(file1_mod2.modified() > file1_mod.modified());
963
964        assert!(
965            cache
966                .get(&PathBuf::from("file1.txt"), &file1_mod2)
967                .is_none()
968        );
969
970        let mut hasher = FileHasher::new(&[&dir_path])?;
971        hasher.exclude = Some(default_exclude());
972        let (tx, rx) = mpsc::channel();
973        hasher.check_streaming(tx, CheckMode::UpdateAll)?;
974        let _ = CheckCollector::collect(rx, &dir_path);
975        hasher.save_cache()?;
976
977        let cache = FileHashCache::new(&dir_path);
978        assert!(
979            cache
980                .get(&PathBuf::from("file1.txt"), &file1_mod2)
981                .is_some()
982        );
983        Ok(())
984    }
985
986    #[test]
987    fn check_cleanup_deleted_files() -> anyhow::Result<()> {
988        let dir = tempfile::tempdir()?;
989        let dir_path = dir.path().to_path_buf();
990        let file1_path = dir.path().join("file1.txt");
991        let file2_path = dir.path().join("file2.txt");
992        fs::write(&file1_path, "content 1")?;
993        fs::write(&file2_path, "content 2")?;
994        let file1 = FileItem::try_from(file1_path.as_path())?;
995        let file2 = FileItem::try_from(file2_path.as_path())?;
996
997        let mut hasher = FileHasher::new(&[&dir_path])?;
998        hasher.exclude = Some(default_exclude());
999        let (tx, rx) = mpsc::channel();
1000        hasher.check_streaming(tx, CheckMode::UpdateAll)?;
1001        let _ = CheckCollector::collect(rx, &dir_path);
1002        hasher.save_cache()?;
1003
1004        // Verify both are in the cache
1005        let cache = FileHashCache::new(&dir_path);
1006        assert!(cache.get(&PathBuf::from("file1.txt"), &file1).is_some());
1007        assert!(cache.get(&PathBuf::from("file2.txt"), &file2).is_some());
1008
1009        // Now delete file2 from disk
1010        fs::remove_file(&file2_path)?;
1011
1012        // Run check and save again
1013        let mut hasher = FileHasher::new(&[&dir_path])?;
1014        hasher.exclude = Some(default_exclude());
1015        let (tx, rx) = mpsc::channel();
1016        hasher.check_streaming(tx, CheckMode::UpdateAll)?;
1017        let _ = CheckCollector::collect(rx, &dir_path);
1018        hasher.save_cache()?;
1019
1020        // Verify file2 is removed from cache, but file1 is still there
1021        let cache = FileHashCache::new(&dir_path);
1022        assert!(cache.get(&PathBuf::from("file2.txt"), &file2).is_none());
1023        assert!(cache.get(&PathBuf::from("file1.txt"), &file1).is_some());
1024        Ok(())
1025    }
1026
1027    #[test]
1028    fn find_duplicates_multiple_dirs() -> anyhow::Result<()> {
1029        let tmp = tempfile::tempdir()?;
1030        let dir1 = tmp.path().join("dir1");
1031        let dir2 = tmp.path().join("dir2");
1032        fs::create_dir(&dir1)?;
1033        fs::create_dir(&dir2)?;
1034        let file1_path = dir1.join("file1.txt");
1035        fs::write(&file1_path, "same content")?;
1036        let file2_path = dir2.join("file2.txt");
1037        fs::write(&file2_path, "same content")?;
1038        let hasher = FileHasher::new(&[&dir1, &dir2])?;
1039        let duplicates = hasher.find_duplicates()?;
1040        assert_eq!(duplicates.len(), 1);
1041        let group = &duplicates[0];
1042        assert_eq!(group.paths.len(), 2);
1043        assert_eq!(group.size, 12);
1044        assert!(group.paths.contains(&file1_path));
1045        assert!(group.paths.contains(&file2_path));
1046
1047        Ok(())
1048    }
1049
1050    #[test]
1051    fn check_fails_with_multiple_dirs() -> anyhow::Result<()> {
1052        let tmp = tempfile::tempdir()?;
1053        let dir1 = tmp.path().join("dir1");
1054        let dir2 = tmp.path().join("dir2");
1055        fs::create_dir(&dir1)?;
1056        fs::create_dir(&dir2)?;
1057        let hasher = FileHasher::new(&[&dir1, &dir2])?;
1058        assert!(hasher.check(CheckMode::Check).is_err());
1059        Ok(())
1060    }
1061
1062    #[test]
1063    fn check_update_mode_only_on_change() -> anyhow::Result<()> {
1064        let dir = tempfile::tempdir()?;
1065        let dir_path = dir.path().to_path_buf();
1066        let file1_path = dir.path().join("file1.txt");
1067        fs::write(&file1_path, "content 1")?;
1068
1069        // 1. Initial check under UpdateAll/Update (should compute hash and insert since new)
1070        let mut hasher = FileHasher::new(&[&dir_path])?;
1071        hasher.exclude = Some(default_exclude());
1072        let (tx, rx) = mpsc::channel();
1073        hasher.check_streaming(tx, CheckMode::Update)?;
1074        let _ = CheckCollector::collect(rx, &dir_path);
1075        hasher.save_cache()?;
1076        assert_eq!(hasher.num_hashed.load(atomic::Ordering::Relaxed), 1);
1077
1078        // 2. Second check under Update. Since metadata is unchanged, it should be a no-op (0 hashes computed).
1079        let mut hasher = FileHasher::new(&[&dir_path])?;
1080        hasher.exclude = Some(default_exclude());
1081        let (tx, rx) = mpsc::channel();
1082        hasher.check_streaming(tx, CheckMode::Update)?;
1083        let _ = CheckCollector::collect(rx, &dir_path);
1084        assert_eq!(hasher.num_hashed.load(atomic::Ordering::Relaxed), 0);
1085
1086        // 3. Sleep, modify file, and check again under Update. Metadata changed, so it should update (1 hash computed).
1087        std::thread::sleep(time::Duration::from_millis(10));
1088        fs::write(&file1_path, "content 1 modified")?;
1089
1090        let mut hasher = FileHasher::new(&[&dir_path])?;
1091        hasher.exclude = Some(default_exclude());
1092        let (tx, rx) = mpsc::channel();
1093        hasher.check_streaming(tx, CheckMode::Update)?;
1094        let _ = CheckCollector::collect(rx, &dir_path);
1095        assert_eq!(hasher.num_hashed.load(atomic::Ordering::Relaxed), 1);
1096
1097        Ok(())
1098    }
1099
1100    #[test]
1101    fn escape_shell() {
1102        let escape_shell = |p: &str| DuplicatedFiles::escape_quotes_for_shell(Path::new(p));
1103        assert_eq!(escape_shell(""), "");
1104        assert_eq!(escape_shell("abc"), "abc");
1105        assert_eq!(escape_shell("a'b"), "a'\\''b");
1106        assert_eq!(escape_shell("a'b'"), "a'\\''b'\\''");
1107
1108        let escape_shell_double = |p: &str| DuplicatedFiles::escape_quotes_by_double(Path::new(p));
1109        assert_eq!(escape_shell_double(""), "");
1110        assert_eq!(escape_shell_double("abc"), "abc");
1111        assert_eq!(escape_shell_double("a'b"), "a''b");
1112        assert_eq!(escape_shell_double("a'b'"), "a''b''");
1113    }
1114
1115    #[test]
1116    fn write_dups_shell_empty() -> anyhow::Result<()> {
1117        let dup_empty = DuplicatedFiles {
1118            paths: vec![],
1119            size: 100,
1120        };
1121        let mut buf = Vec::new();
1122        dup_empty.write_shell(&mut buf)?;
1123        assert_eq!(String::from_utf8(buf)?, "");
1124        Ok(())
1125    }
1126
1127    #[test]
1128    fn write_dups_shell_one() -> anyhow::Result<()> {
1129        let dup_one = DuplicatedFiles {
1130            paths: vec![PathBuf::from("a.txt")],
1131            size: 100,
1132        };
1133        let mut buf = Vec::new();
1134        dup_one.write_shell(&mut buf)?;
1135        assert_eq!(String::from_utf8(buf)?, "");
1136        Ok(())
1137    }
1138
1139    #[test]
1140    fn write_dups_shell_two() -> anyhow::Result<()> {
1141        let dup_multiple = DuplicatedFiles {
1142            paths: vec![PathBuf::from("a.txt"), PathBuf::from("b.txt")],
1143            size: 100,
1144        };
1145        let mut buf = Vec::new();
1146        dup_multiple.write_shell(&mut buf)?;
1147        assert_eq!(String::from_utf8(buf)?, "cp 'a.txt' 'b.txt'\n");
1148        Ok(())
1149    }
1150
1151    #[test]
1152    fn write_dups_shell_three() -> anyhow::Result<()> {
1153        let dup_multiple = DuplicatedFiles {
1154            paths: vec![
1155                PathBuf::from("a.txt"),
1156                PathBuf::from("b.txt"),
1157                PathBuf::from("c.txt"),
1158            ],
1159            size: 100,
1160        };
1161        let mut buf = Vec::new();
1162        dup_multiple.write_shell(&mut buf)?;
1163        assert_eq!(
1164            String::from_utf8(buf)?,
1165            "cp 'a.txt' 'b.txt'\ncp 'a.txt' 'c.txt'\n"
1166        );
1167        Ok(())
1168    }
1169
1170    #[test]
1171    fn write_dups_shell_quotes() -> anyhow::Result<()> {
1172        let dup_quotes = DuplicatedFiles {
1173            paths: vec![PathBuf::from("a'b.txt"), PathBuf::from("c'd.txt")],
1174            size: 100,
1175        };
1176        let mut buf = Vec::new();
1177        dup_quotes.write_shell(&mut buf)?;
1178        assert_eq!(String::from_utf8(buf)?, "cp 'a'\\''b.txt' 'c'\\''d.txt'\n");
1179
1180        let mut buf = Vec::new();
1181        dup_quotes.write_pwsh(&mut buf)?;
1182        assert_eq!(
1183            String::from_utf8(buf)?,
1184            "Copy-Item -LiteralPath 'a''b.txt' 'c''d.txt'\n"
1185        );
1186        Ok(())
1187    }
1188}
compare_dir/file_hasher.rs

compare_dir/
file_hasher.rs
