rdupes 1.0.0

A rust port of pydupes. Super fast.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

pub mod types;
pub mod hashing;
pub mod traversal;
pub mod finder;

pub use types::*;
pub use hashing::*;
pub use traversal::*;
pub use finder::*;

#[cfg(test)]
mod tests {
    use super::*;
    use tokio::sync::mpsc;

    #[tokio::test]
    async fn hard_link_duplicates() {
        let temp = tempfile::tempdir().unwrap();
        let test_dir = temp.path();

        let f1 = test_dir.join("file1").to_string_lossy().to_string();
        let f1_hl = test_dir.join("file1_hl").to_string_lossy().to_string();
        let f2 = test_dir.join("file2").to_string_lossy().to_string();
        let f2_hl = test_dir.join("file2_hl").to_string_lossy().to_string();
        let f3 = test_dir.join("file3").to_string_lossy().to_string();

        std::fs::write(&f1, b"common content").unwrap();
        std::fs::hard_link(&f1, &f1_hl).unwrap();
        std::fs::write(&f2, b"common content").unwrap();
        std::fs::hard_link(&f2, &f2_hl).unwrap();
        std::fs::write(&f3, b"uncommon content").unwrap();

        let df = DupeFinder::new(DupeParams {
            min_file_size: 0,
            read_concurrency: 1,
            disable_mmap: true,
        });

        let size_map = df
            .traverse_paths(vec![test_dir.to_string_lossy().to_string()])
            .await
            .unwrap();
        let (tx, mut rx) = mpsc::channel(10);

        df.check_hashes_and_content(size_map, tx).await.unwrap();

        let mut all_paths = Vec::new();
        while let Some(ds) = rx.recv().await {
            for p in ds.paths {
                all_paths.push(p);
            }
        }
        all_paths.sort();

        let mut expected = vec![f1, f1_hl, f2, f2_hl];
        expected.sort();

        assert_eq!(all_paths, expected);

        std::fs::remove_dir_all(test_dir).unwrap();
    }

    #[test]
    fn sorting() {
        let input_paths = vec!["/asd/".into(), "/def/".into(), "/tmp/asd/".into()];
        let mut ds = DupeSet {
            paths: vec![
                "/asd/file1000.txt".to_string(),
                "/def/z".to_string(),
                "/tmp/asd/f123.mp4".to_string(),
                "/def/yyy".to_string(),
                "/def/xx".to_string(),
                "/asd/file1.txt".to_string(),
                "/def/1234".to_string(),
                "/asd/xile100.txt".to_string(),
            ],
            fsize: 0,
        };
        ds.sort_paths(&input_paths);
        assert_eq!(
            vec![
                "/asd/file1.txt".to_string(),
                "/asd/xile100.txt".to_string(),
                "/asd/file1000.txt".to_string(),
                "/def/z".to_string(),
                "/def/xx".to_string(),
                "/def/yyy".to_string(),
                "/def/1234".to_string(),
                "/tmp/asd/f123.mp4".to_string(),
            ],
            ds.paths
        )
    }
}