binsync 0.0.4

A library for syncing binary files between two locations.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

use std::{
    convert::TryInto,
    path::{Path, PathBuf},
    sync::{Arc, Mutex},
};

use fastcdc::FastCDC;
use serde::{Deserialize, Serialize};
use walkdir::WalkDir;

use crate::{
    chunk::{FileInfo, FileList},
    sync::ThreadPool,
};

use super::{Chunk, AVG_CHUNK, MAX_CHUNK, MIN_CHUNK};

/// Information about a file and which chunks it contains.
#[derive(Serialize, Deserialize, PartialEq, Debug)]
pub struct FileChunkInfo {
    pub path: PathBuf,
    pub chunks: Vec<Chunk>,
}

/// Holds a list of files and which chunks exist inside those files in which
/// order. The manifest is the source of the syncer allowing us to know what
/// we should be syncing to.
#[derive(Serialize, Deserialize, PartialEq, Debug)]
pub struct Manifest {
    pub files: Vec<FileChunkInfo>,
}

impl Manifest {
    pub fn new() -> Manifest {
        Manifest { files: Vec::new() }
    }

    /// Generates a manifest using the specified path as the root.
    pub fn from_path<P: AsRef<Path>>(path: P) -> Manifest {
        let mut list = FileList { files: Vec::new() };

        let prefix = path.as_ref().to_path_buf();

        // TODO: Handle empty folders.

        for entry in WalkDir::new(&prefix) {
            let info = entry.unwrap();

            if info.file_type().is_file() {
                let name = info.file_name().to_string_lossy().to_string();
                let directory = info.path().strip_prefix(&prefix).map_or_else(
                    |_| info.path().to_string_lossy().to_string(),
                    |p| p.to_string_lossy().to_string(),
                );

                list.files.push(FileInfo { name, directory });
            }
        }

        Manifest::from_file_list(path, &list)
    }

    /// Generates a manifest of specific files using the specified path as the
    /// base path. Use this if you want to filter only to specific files in the
    /// directory.
    pub fn from_file_list<P: AsRef<Path>>(path: P, file_list: &FileList) -> Manifest {
        let manifest = Arc::new(Mutex::new(Manifest::new()));
        let prefix = path.as_ref().to_path_buf();

        let pool = ThreadPool::new(4);

        for file_info in &file_list.files {
            let key = file_info.directory.clone();
            let path = prefix.join(Path::new(&file_info.directory));
            let manifest = Arc::clone(&manifest);

            pool.execute(move || {
                let contents = std::fs::read(path).unwrap();
                let chunker = FastCDC::new(&contents, MIN_CHUNK, AVG_CHUNK, MAX_CHUNK);

                let mut file_chunk_info = FileChunkInfo {
                    path: PathBuf::from(key),
                    chunks: Vec::new(),
                };

                for entry in chunker {
                    let end = entry.offset + entry.length;
                    let chunk = &contents[entry.offset..end];

                    let digest = md5::compute(chunk);
                    let hash = u64::from_le_bytes(digest[0..8].try_into().unwrap());

                    file_chunk_info.chunks.push(Chunk {
                        hash,
                        offset: entry.offset as u64,
                        length: entry.length as u64,
                    });
                }

                manifest.lock().unwrap().files.push(file_chunk_info);
            });
        }

        drop(pool);

        let mut manifest = Arc::try_unwrap(manifest).unwrap().into_inner().unwrap();

        manifest.files.sort_by_cached_key(|k| k.path.clone());

        manifest
    }
}

impl Default for Manifest {
    fn default() -> Self {
        Self::new()
    }
}