crcdir 0.3.0

A thin wrapper around crc32fast to checksum an entire directory
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

use crc32fast::Hasher;
use failure::*;
use filebuffer::FileBuffer;
#[cfg(feature = "progress")]
use indicatif::*;
#[cfg(feature = "progress")]
use lazy_static::lazy_static;
use rayon::prelude::*;
use std::path::Path;
use walkdir::WalkDir;

#[cfg(feature = "progress")]
lazy_static! {
    pub static ref PROGRESS_BAR: ProgressBar = {
        let progbar = ProgressBar::new(0);
        progbar.set_style(
            ProgressStyle::default_bar()
                .template("[{elapsed_precise}] [{bar:40.green/blue}] {pos}/{len} ({eta})"),
        );
        progbar
    };
}

#[cfg(feature = "progress")]
pub fn progress_init(length: u64) {
    PROGRESS_BAR.set_length(length);
    PROGRESS_BAR.set_draw_delta(length / 1000);
    PROGRESS_BAR.set_position(0);
}

pub fn hash_file<P: AsRef<Path>>(hasher: &mut Hasher, root: &Path, path: P) -> Result<(), Error> {
    let fbuffer = FileBuffer::open(&path)?;
    let len = fbuffer.len();
    fbuffer.prefetch(0, len);
    hasher.update(
        path.as_ref()
            .strip_prefix(root)?
            .to_str()
            .ok_or(format_err!(
                "path couldn't be read as str: {:?}",
                path.as_ref()
            ))?
            .as_bytes(),
    );
    hasher.update(&fbuffer);
    Ok(())
}

/// Hashes an individual file using `crc32`
pub fn hash_file_oneshot<P: AsRef<Path>>(root: &Path, path: P) -> Result<u32, Error> {
    let mut hasher = Hasher::new();
    hash_file(&mut hasher, root, path)?;
    Ok(hasher.finalize())
}

/// Walks a directory, hashing the files and summing the results.
pub fn sum_dir<P: AsRef<Path> + Send + Sync>(rootdir: P) -> Result<u32, Error> {
    WalkDir::new(&rootdir)
        .into_iter()
        .filter_map(Result::ok)
        .filter(|d| d.file_type().is_file())
        .par_bridge()
        .map(move |e| hash_file_oneshot(rootdir.as_ref(), e.path()))
        .reduce(|| Ok(0u32), |acc, i| Ok(acc?.wrapping_add(i?)))
}

#[cfg(feature = "progress")]
pub fn sum_dir_prog<P: AsRef<Path> + Send + Sync>(rootdir: P) -> Result<u32, Error> {
    let vec: Vec<_> = WalkDir::new(&rootdir)
        .into_iter()
        .filter_map(Result::ok)
        .filter(|d| d.file_type().is_file())
        .collect();

    progress_init(vec.len() as u64);

    vec.into_par_iter()
        .map(move |e| {
            let hash = hash_file_oneshot(rootdir.as_ref(), e.path());
            PROGRESS_BAR.inc(1);
            hash
        })
        .reduce(|| Ok(0u32), |acc, i| Ok(acc?.wrapping_add(i?)))
}

/// Walks a directory, hashing the files sequentially.
pub fn hash_dir<P: AsRef<Path>>(rootdir: P) -> Result<u32, Error> {
    let mut hasher = Hasher::new();
    let vec: Vec<_> = WalkDir::new(&rootdir)
        .into_iter()
        .filter_map(Result::ok)
        .filter(|d| d.file_type().is_file())
        .collect();

    for entry in vec.into_iter() {
        hash_file(&mut hasher, rootdir.as_ref(), entry.into_path())?;
    }

    Ok(hasher.finalize())
}

#[cfg(feature = "progress")]
pub fn hash_dir_prog<P: AsRef<Path>>(rootdir: P) -> Result<u32, Error> {
    let mut hasher = Hasher::new();
    let vec: Vec<_> = WalkDir::new(&rootdir)
        .into_iter()
        .filter_map(Result::ok)
        .filter(|d| d.file_type().is_file())
        .collect();

    progress_init(vec.len() as u64);

    for entry in vec.into_iter() {
        let hash = hash_file(&mut hasher, rootdir.as_ref(), entry.into_path())?;
        PROGRESS_BAR.inc(1);
        hash
    }

    Ok(hasher.finalize())
}