velesdb_memory/migration/
filesystem.rs

1use super::query_error;
2use sha2::{Digest, Sha256};
3use std::ffi::OsStr;
4use std::fs::File;
5use std::io::{BufReader, Read};
6use std::path::{Component, Path, PathBuf};
7
8const FINGERPRINT_DOMAIN: &[u8] = b"velesdb-migration-source-tree-v2\0";
9
10#[derive(Debug, Clone, Copy)]
11enum EntryKind {
12    Directory,
13    File { len: u64 },
14}
15
16#[derive(Debug)]
17struct TreeEntry {
18    relative_path: PathBuf,
19    kind: EntryKind,
20}
21
22/// A versioned SHA-256 digest of every directory, regular file and file byte.
23///
24/// Paths, entry kinds and lengths are length-delimited before they are hashed,
25/// so two different trees cannot become ambiguous through path concatenation.
26/// Symlinks and special files are refused: following one would let a migration
27/// fingerprint or copy data outside the source tree.
28///
29/// # Errors
30/// Returns [`crate::MemoryError`] if the tree cannot be walked or read, or if
31/// it contains anything other than directories and regular files.
32pub fn fingerprint(root: &Path) -> Result<String, crate::MemoryError> {
33    let entries = tree_entries(root)?;
34    let mut hash = Sha256::new();
35    hash.update(FINGERPRINT_DOMAIN);
36    hash.update(
37        u64::try_from(entries.len())
38            .unwrap_or(u64::MAX)
39            .to_le_bytes(),
40    );
41
42    for entry in entries {
43        match entry.kind {
44            EntryKind::Directory => hash.update([b'd']),
45            EntryKind::File { len } => {
46                hash.update([b'f']);
47                hash.update(len.to_le_bytes());
48            }
49        }
50        hash_relative_path(&mut hash, &entry.relative_path);
51        if let EntryKind::File { len } = entry.kind {
52            hash_file(root, &entry.relative_path, len, &mut hash)?;
53        }
54    }
55
56    Ok(format!("sha256-tree-v2:{}", encode_hex(&hash.finalize())))
57}
58
59/// Sum of every regular file's length under `root`.
60///
61/// # Errors
62/// Returns [`crate::MemoryError`] under the same conditions as [`fingerprint`].
63pub fn bytes_on_disk(root: &Path) -> Result<u64, crate::MemoryError> {
64    tree_entries(root)?
65        .into_iter()
66        .try_fold(0u64, |total, entry| {
67            let len = match entry.kind {
68                EntryKind::Directory => 0,
69                EntryKind::File { len } => len,
70            };
71            total.checked_add(len).ok_or_else(|| {
72                query_error(format!(
73                    "the byte count under {} exceeds u64",
74                    root.display()
75                ))
76            })
77        })
78}
79
80fn tree_entries(root: &Path) -> Result<Vec<TreeEntry>, crate::MemoryError> {
81    let metadata = std::fs::symlink_metadata(root)
82        .map_err(|err| query_error(format!("cannot inspect source {}: {err}", root.display())))?;
83    if !metadata.is_dir() || metadata.file_type().is_symlink() {
84        return Err(query_error(format!(
85            "migration source {} must be a real directory, not a symlink or special file",
86            root.display()
87        )));
88    }
89
90    let mut entries = Vec::new();
91    collect_entries(root, root, &mut entries)?;
92    entries.sort_unstable_by(|left, right| left.relative_path.cmp(&right.relative_path));
93    Ok(entries)
94}
95
96fn collect_entries(
97    root: &Path,
98    directory: &Path,
99    entries: &mut Vec<TreeEntry>,
100) -> Result<(), crate::MemoryError> {
101    let read = std::fs::read_dir(directory)
102        .map_err(|err| query_error(format!("cannot read {}: {err}", directory.display())))?;
103    for entry in read {
104        let entry = entry.map_err(|err| query_error(format!("cannot read an entry: {err}")))?;
105        collect_entry(root, &entry.path(), entries)?;
106    }
107    Ok(())
108}
109
110fn collect_entry(
111    root: &Path,
112    path: &Path,
113    entries: &mut Vec<TreeEntry>,
114) -> Result<(), crate::MemoryError> {
115    let metadata = std::fs::symlink_metadata(path)
116        .map_err(|err| query_error(format!("cannot inspect {}: {err}", path.display())))?;
117    let relative_path = path
118        .strip_prefix(root)
119        .map_err(|err| query_error(format!("cannot relativize {}: {err}", path.display())))?
120        .to_path_buf();
121    if metadata.file_type().is_symlink() {
122        return Err(query_error(format!(
123            "migration source contains symlink {}; refusing to follow data outside the tree",
124            path.display()
125        )));
126    }
127    if metadata.is_dir() {
128        entries.push(TreeEntry {
129            relative_path,
130            kind: EntryKind::Directory,
131        });
132        return collect_entries(root, path, entries);
133    }
134    if metadata.is_file() {
135        entries.push(TreeEntry {
136            relative_path,
137            kind: EntryKind::File {
138                len: metadata.len(),
139            },
140        });
141        return Ok(());
142    }
143    Err(query_error(format!(
144        "migration source contains special file {}; only directories and regular files are supported",
145        path.display()
146    )))
147}
148
149fn hash_relative_path(hash: &mut Sha256, path: &Path) {
150    let components: Vec<_> = path
151        .components()
152        .filter_map(|component| match component {
153            Component::Normal(part) => Some(part),
154            _ => None,
155        })
156        .collect();
157    hash.update(
158        u64::try_from(components.len())
159            .unwrap_or(u64::MAX)
160            .to_le_bytes(),
161    );
162    for component in components {
163        update_os_str(hash, component);
164    }
165}
166
167fn hash_file(
168    root: &Path,
169    relative_path: &Path,
170    expected_len: u64,
171    hash: &mut Sha256,
172) -> Result<(), crate::MemoryError> {
173    let path = root.join(relative_path);
174    let file = open_expected_file(&path, expected_len)?;
175    let bytes_read = hash_reader(&path, file, hash)?;
176    if bytes_read == expected_len {
177        return Ok(());
178    }
179    Err(query_error(format!(
180        "source changed while fingerprinting {}: expected {expected_len} bytes, read {bytes_read}",
181        path.display()
182    )))
183}
184
185fn open_expected_file(path: &Path, expected_len: u64) -> Result<File, crate::MemoryError> {
186    let file = File::open(path)
187        .map_err(|err| query_error(format!("cannot open {}: {err}", path.display())))?;
188    let before = file
189        .metadata()
190        .map_err(|err| query_error(format!("cannot inspect {}: {err}", path.display())))?;
191    if before.is_file() && before.len() == expected_len {
192        return Ok(file);
193    }
194    Err(query_error(format!(
195        "source changed while fingerprinting {}",
196        path.display()
197    )))
198}
199
200fn hash_reader(path: &Path, file: File, hash: &mut Sha256) -> Result<u64, crate::MemoryError> {
201    let mut reader = BufReader::new(file);
202    let mut buffer = vec![0u8; 64 * 1024].into_boxed_slice();
203    let mut bytes_read = 0u64;
204    loop {
205        let read = reader
206            .read(&mut buffer)
207            .map_err(|err| query_error(format!("cannot read {}: {err}", path.display())))?;
208        if read == 0 {
209            break;
210        }
211        hash.update(&buffer[..read]);
212        bytes_read = bytes_read
213            .checked_add(u64::try_from(read).unwrap_or(u64::MAX))
214            .ok_or_else(|| query_error(format!("file {} exceeds u64", path.display())))?;
215    }
216    Ok(bytes_read)
217}
218
219#[cfg(unix)]
220fn update_os_str(hash: &mut Sha256, value: &OsStr) {
221    use std::os::unix::ffi::OsStrExt;
222    let bytes = value.as_bytes();
223    hash.update(u64::try_from(bytes.len()).unwrap_or(u64::MAX).to_le_bytes());
224    hash.update(bytes);
225}
226
227#[cfg(windows)]
228fn update_os_str(hash: &mut Sha256, value: &OsStr) {
229    use std::os::windows::ffi::OsStrExt;
230
231    hash.update(
232        u64::try_from(value.encode_wide().count())
233            .unwrap_or(u64::MAX)
234            .to_le_bytes(),
235    );
236    for unit in value.encode_wide() {
237        hash.update(unit.to_le_bytes());
238    }
239}
240
241#[cfg(not(any(unix, windows)))]
242fn update_os_str(hash: &mut Sha256, value: &OsStr) {
243    let bytes = value.to_string_lossy();
244    hash.update(u64::try_from(bytes.len()).unwrap_or(u64::MAX).to_le_bytes());
245    hash.update(bytes.as_bytes());
246}
247
248pub(super) fn encode_hex(bytes: &[u8]) -> String {
249    const HEX: &[u8; 16] = b"0123456789abcdef";
250    let mut encoded = String::with_capacity(bytes.len() * 2);
251    for byte in bytes {
252        encoded.push(char::from(HEX[usize::from(byte >> 4)]));
253        encoded.push(char::from(HEX[usize::from(byte & 0x0f)]));
254    }
255    encoded
256}
257
258#[cfg(all(test, windows))]
259#[path = "filesystem_tests.rs"]
260mod windows_tests;
velesdb_memory/migration/filesystem.rs

velesdb_memory/migration/
filesystem.rs
