catfish 0.1.1

catfish is a CLI tool that compares two directories by hashing all files. It reports which files are in the 'right' folder but not in 'left', regardless of how things were moved or renamed. Great for making sure your 'left' folder has all the files from the 'right' one. 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

use clap::Parser;
use rayon::iter::{ParallelBridge, ParallelIterator};
use sha2::Digest;
use std::{
    collections::HashSet,
    fs::File,
    io::Read,
    path::{Path, PathBuf},
};
use walkdir::WalkDir;

/// Simple CLI to list files in `right` that are not in `left` (based on sha256 hash)
#[derive(Parser, Debug)]
#[command(author, version, about)]
struct Args {
    /// The "left" folder
    left: PathBuf,
    /// The "right" folder
    right: PathBuf,
    /// Only report the first file per hash in `right`, ignore duplicates
    #[arg(short = 'i', long = "ignore-duplicates")]
    ignore_duplicates: bool,
}

fn main() -> anyhow::Result<()> {
    let args = Args::parse();

    // Hash all files in left (in parallel)
    let left_entries = hash_all_files(&args.left)?;
    let left_hashes: HashSet<String> = left_entries.iter().map(|(h, _)| h.clone()).collect();

    let mut right_entries = hash_all_files(&args.right)?;

    // Optionally remove duplicates from 'right'
    if args.ignore_duplicates {
        right_entries = deduplicate_by_hash(right_entries);
    }

    println!("Files in {:?} but not in {:?}:", &args.right, &args.left);

    for (hash, path) in right_entries {
        if !left_hashes.contains(&hash) {
            println!("{hash} {}", path.display());
        }
    }

    Ok(())
}

fn hash_all_files(dir: &Path) -> anyhow::Result<Vec<(String, PathBuf)>> {
    let entries = WalkDir::new(dir)
        .into_iter()
        .filter_map(|e| e.ok())
        .filter(|e| e.file_type().is_file())
        .par_bridge()
        .filter_map(|entry: walkdir::DirEntry| {
            let path = entry.path().to_path_buf();
            let hash = match compute_sha256(&path) {
                Ok(hash) => hash,
                Err(err) => {
                    eprintln!("Error hashing {:?}: {}", path, err);
                    return None;
                }
            };
            Some((hash, path))
        })
        .collect::<Vec<(String, PathBuf)>>();

    Ok(entries)
}

fn compute_sha256(path: &Path) -> anyhow::Result<String> {
    let mut file = File::open(path)?;
    let mut hasher = sha2::Sha256::new();
    let mut buffer = [0u8; 8192];

    loop {
        let bytes_read = file.read(&mut buffer)?;
        if bytes_read == 0 {
            break;
        }
        hasher.update(&buffer[..bytes_read]);
    }

    Ok(format!("{:x}", hasher.finalize()))
}

/// If `--ignore-duplicates` is set, we only keep the first occurrence of each hash.
fn deduplicate_by_hash(entries: Vec<(String, PathBuf)>) -> Vec<(String, PathBuf)> {
    let mut seen = HashSet::new();
    let mut deduped = Vec::new();
    for (hash, path) in entries.into_iter() {
        if !seen.contains(&hash) {
            seen.insert(hash.clone());
            deduped.push((hash, path));
        }
    }
    deduped
}