Skip to main content

edgestore_repl/
filesystem_remote_store.rs

1//! FilesystemRemoteStore — local-filesystem implementation of `RemoteStore`.
2//!
3//! Files are content-addressed: named `{hash_hex}.seg` where `hash_hex` is the
4//! 64-character lowercase hex encoding of the 32-byte BLAKE3 hash. Listing the
5//! directory is equivalent to listing stored segments.
6
7use std::path::PathBuf;
8
9use edgestore::error::EdgestoreError;
10use edgestore::RemoteStore;
11
12use crate::wire::hash_to_hex;
13
14/// Local-filesystem implementation of `RemoteStore`.
15///
16/// All operations are idempotent and atomic where applicable.
17/// `upload` uses a `.tmp` write + rename to prevent torn writes.
18pub struct FilesystemRemoteStore {
19    base_dir: PathBuf,
20}
21
22impl FilesystemRemoteStore {
23    /// Create a new `FilesystemRemoteStore` rooted at `base_dir`.
24    ///
25    /// Creates `base_dir` (and all parent directories) if it does not exist.
26    pub fn new(base_dir: PathBuf) -> Result<Self, EdgestoreError> {
27        std::fs::create_dir_all(&base_dir)
28            .map_err(|e| EdgestoreError::ReplicationError(e.to_string()))?;
29        Ok(Self { base_dir })
30    }
31
32    fn seg_path(&self, hash: &[u8; 32]) -> PathBuf {
33        self.base_dir.join(format!("{}.seg", hash_to_hex(hash)))
34    }
35
36    fn aux_path(&self, hash: &[u8; 32], ext: &str) -> Result<PathBuf, EdgestoreError> {
37        if ext.is_empty() || !ext.bytes().all(|b| b.is_ascii_lowercase()) {
38            return Err(EdgestoreError::InvalidOperation(format!(
39                "invalid sidecar ext {:?}: only lowercase ASCII letters allowed",
40                ext
41            )));
42        }
43        Ok(self
44            .base_dir
45            .join(format!("{}.{}", hash_to_hex(hash), ext)))
46    }
47}
48
49impl RemoteStore for FilesystemRemoteStore {
50    /// Store `data` under `hash`. Idempotent: if the file already exists, returns `Ok(())`.
51    ///
52    /// Writes to a `.tmp` file first, then renames atomically (T-04-10).
53    fn upload(&self, hash: &[u8; 32], data: &[u8]) -> Result<(), EdgestoreError> {
54        let dest = self.seg_path(hash);
55
56        // Content-addressed: if it already exists, nothing to do.
57        if dest.exists() {
58            return Ok(());
59        }
60
61        let tmp = self.base_dir.join(format!("{}.tmp", hash_to_hex(hash)));
62
63        std::fs::write(&tmp, data).map_err(|e| EdgestoreError::ReplicationError(e.to_string()))?;
64
65        std::fs::rename(&tmp, &dest)
66            .map_err(|e| EdgestoreError::ReplicationError(e.to_string()))?;
67
68        Ok(())
69    }
70
71    /// Atomically create the segment file only if it does not exist.
72    ///
73    /// Uses `O_CREAT | O_EXCL` (`create_new(true)`) to give true atomic
74    /// idempotent-create semantics without a TOCTOU race.  Returns `Ok(true)`
75    /// on first write, `Ok(false)` when the file already exists.
76    fn upload_if_absent(&self, hash: &[u8; 32], data: &[u8]) -> Result<bool, EdgestoreError> {
77        use std::io::Write as _;
78        let dest = self.seg_path(hash);
79        match std::fs::OpenOptions::new()
80            .write(true)
81            .create_new(true)
82            .open(&dest)
83        {
84            Ok(mut file) => {
85                file.write_all(data)
86                    .map_err(|e| EdgestoreError::ReplicationError(e.to_string()))?;
87                Ok(true)
88            }
89            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(false),
90            Err(e) => Err(EdgestoreError::ReplicationError(e.to_string())),
91        }
92    }
93
94    /// Download the segment bytes for `hash`.
95    ///
96    /// Returns `EdgestoreError::ReplicationError` if the segment is not present.
97    fn download(&self, hash: &[u8; 32]) -> Result<Vec<u8>, EdgestoreError> {
98        let path = self.seg_path(hash);
99        std::fs::read(&path).map_err(|e| {
100            if e.kind() == std::io::ErrorKind::NotFound {
101                EdgestoreError::ReplicationError(format!(
102                    "segment not found: {}",
103                    hash_to_hex(hash)
104                ))
105            } else {
106                EdgestoreError::ReplicationError(e.to_string())
107            }
108        })
109    }
110
111    /// List all stored segment hashes by scanning `{base_dir}/*.seg`.
112    ///
113    /// Filenames that are not exactly 64 lowercase hex characters followed by `.seg`
114    /// are silently skipped.
115    fn list(&self) -> Result<Vec<[u8; 32]>, EdgestoreError> {
116        let entries = std::fs::read_dir(&self.base_dir)
117            .map_err(|e| EdgestoreError::ReplicationError(e.to_string()))?;
118
119        let mut hashes = Vec::new();
120
121        for entry in entries.flatten() {
122            let file_name = entry.file_name();
123            let name = match file_name.to_str() {
124                Some(n) => n.to_owned(),
125                None => continue,
126            };
127
128            // Must end with ".seg"
129            if !name.ends_with(".seg") {
130                continue;
131            }
132
133            // Stem must be exactly 64 characters (32 bytes * 2 hex digits).
134            let stem = &name[..name.len() - 4]; // strip ".seg"
135            if stem.len() != 64 {
136                continue;
137            }
138
139            // Parse 64 hex chars → [u8; 32]
140            let parsed: Option<[u8; 32]> = (0..32)
141                .map(|i| u8::from_str_radix(&stem[i * 2..i * 2 + 2], 16).ok())
142                .collect::<Option<Vec<u8>>>()
143                .and_then(|v| v.try_into().ok());
144
145            if let Some(hash) = parsed {
146                hashes.push(hash);
147            }
148        }
149
150        Ok(hashes)
151    }
152
153    /// Remove the segment for `hash`. No-op if the segment does not exist (idempotent).
154    fn delete(&self, hash: &[u8; 32]) -> Result<(), EdgestoreError> {
155        let path = self.seg_path(hash);
156        match std::fs::remove_file(&path) {
157            Ok(()) => Ok(()),
158            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
159            Err(e) => Err(EdgestoreError::ReplicationError(e.to_string())),
160        }
161    }
162
163    fn upload_aux(&self, hash: &[u8; 32], ext: &str, data: &[u8]) -> Result<(), EdgestoreError> {
164        let dest = self.aux_path(hash, ext)?;
165        if dest.exists() {
166            return Ok(());
167        }
168        let tmp = self
169            .base_dir
170            .join(format!("{}.{}.tmp", hash_to_hex(hash), ext));
171        std::fs::write(&tmp, data).map_err(|e| EdgestoreError::ReplicationError(e.to_string()))?;
172        std::fs::rename(&tmp, &dest).map_err(|e| EdgestoreError::ReplicationError(e.to_string()))
173    }
174
175    fn download_aux(&self, hash: &[u8; 32], ext: &str) -> Result<Vec<u8>, EdgestoreError> {
176        let path = self.aux_path(hash, ext)?;
177        std::fs::read(&path).map_err(|e| {
178            if e.kind() == std::io::ErrorKind::NotFound {
179                EdgestoreError::ReplicationError(format!(
180                    "sidecar not found: {}.{}",
181                    hash_to_hex(hash),
182                    ext
183                ))
184            } else {
185                EdgestoreError::ReplicationError(e.to_string())
186            }
187        })
188    }
189}
190
191#[cfg(test)]
192mod tests {
193    use super::*;
194    use tempfile::TempDir;
195
196    fn make_store() -> (TempDir, FilesystemRemoteStore) {
197        let dir = TempDir::new().expect("tempdir");
198        let store = FilesystemRemoteStore::new(dir.path().to_path_buf())
199            .expect("FilesystemRemoteStore::new");
200        (dir, store)
201    }
202
203    #[test]
204    fn test_upload_download_roundtrip() {
205        let (_dir, store) = make_store();
206        let hash = [0x42u8; 32];
207        let data = b"hello edgestore";
208
209        store.upload(&hash, data).expect("upload");
210        let got = store.download(&hash).expect("download");
211        assert_eq!(got, data);
212    }
213
214    #[test]
215    fn test_upload_idempotent() {
216        let (_dir, store) = make_store();
217        let hash = [0x42u8; 32];
218        let data = b"original";
219
220        store.upload(&hash, data).expect("first upload");
221        // Second upload with same hash must succeed without error.
222        store
223            .upload(&hash, b"different")
224            .expect("second upload (idempotent)");
225
226        // File should still contain the original data (idempotent — skipped overwrite).
227        let got = store
228            .download(&hash)
229            .expect("download after idempotent upload");
230        assert_eq!(got, data);
231    }
232
233    #[test]
234    fn test_list_returns_uploaded_hashes() {
235        let (_dir, store) = make_store();
236        let hash1 = [0x01u8; 32];
237        let hash2 = [0x02u8; 32];
238        let hash3 = [0x03u8; 32];
239
240        store.upload(&hash1, b"a").expect("upload 1");
241        store.upload(&hash2, b"b").expect("upload 2");
242        store.upload(&hash3, b"c").expect("upload 3");
243
244        let mut listed = store.list().expect("list");
245        listed.sort();
246
247        let mut expected = vec![hash1, hash2, hash3];
248        expected.sort();
249
250        assert_eq!(listed, expected);
251    }
252
253    #[test]
254    fn test_delete_removes_file() {
255        let (_dir, store) = make_store();
256        let hash = [0x42u8; 32];
257
258        store.upload(&hash, b"segment data").expect("upload");
259        store.delete(&hash).expect("delete");
260
261        // Download must now fail.
262        let result = store.download(&hash);
263        assert!(result.is_err(), "download after delete should return Err");
264    }
265
266    #[test]
267    fn test_download_not_found() {
268        let (_dir, store) = make_store();
269        let hash = [0xFFu8; 32];
270
271        let result = store.download(&hash);
272        assert!(
273            result.is_err(),
274            "download of non-existent hash should return Err"
275        );
276    }
277
278    #[test]
279    fn test_upload_aux_download_aux_roundtrip() {
280        let (_dir, store) = make_store();
281        let hash = [0x77u8; 32];
282
283        store
284            .upload_aux(&hash, "idx", b"index-data")
285            .expect("upload_aux idx");
286        store
287            .upload_aux(&hash, "xf", b"filter-data")
288            .expect("upload_aux xf");
289        store
290            .upload_aux(&hash, "meta", b"meta-data")
291            .expect("upload_aux meta");
292
293        assert_eq!(store.download_aux(&hash, "idx").unwrap(), b"index-data");
294        assert_eq!(store.download_aux(&hash, "xf").unwrap(), b"filter-data");
295        assert_eq!(store.download_aux(&hash, "meta").unwrap(), b"meta-data");
296    }
297
298    #[test]
299    fn test_upload_aux_idempotent() {
300        let (_dir, store) = make_store();
301        let hash = [0x88u8; 32];
302
303        store
304            .upload_aux(&hash, "idx", b"v1")
305            .expect("first upload_aux");
306        store
307            .upload_aux(&hash, "idx", b"v2")
308            .expect("second upload_aux idempotent");
309
310        // Content-addressed: first upload wins.
311        assert_eq!(store.download_aux(&hash, "idx").unwrap(), b"v1");
312    }
313
314    #[test]
315    fn test_download_aux_not_found() {
316        let (_dir, store) = make_store();
317        let hash = [0x99u8; 32];
318
319        let result = store.download_aux(&hash, "idx");
320        assert!(
321            result.is_err(),
322            "download_aux of non-existent sidecar should Err"
323        );
324    }
325
326    #[test]
327    fn test_upload_if_absent_returns_true_first_time() {
328        let (_dir, store) = make_store();
329        let hash = [0x10u8; 32];
330        let result = store.upload_if_absent(&hash, b"data").unwrap();
331        assert!(result, "first upload_if_absent must return true");
332    }
333
334    #[test]
335    fn test_upload_if_absent_returns_false_when_present() {
336        let (_dir, store) = make_store();
337        let hash = [0x11u8; 32];
338        store.upload_if_absent(&hash, b"original").unwrap();
339        let result = store.upload_if_absent(&hash, b"other").unwrap();
340        assert!(!result, "second upload_if_absent must return false");
341        // Content must be unchanged (first write wins).
342        assert_eq!(store.download(&hash).unwrap(), b"original");
343    }
344
345    #[test]
346    fn test_upload_aux_rejects_path_traversal_ext() {
347        let (_dir, store) = make_store();
348        let hash = [0xAAu8; 32];
349
350        for bad_ext in &[
351            "../etc/passwd",
352            "idx/../../shadow",
353            "IDX",
354            "idx.bak",
355            "",
356            "idx\0evil",
357        ] {
358            let result = store.upload_aux(&hash, bad_ext, b"data");
359            assert!(result.is_err(), "upload_aux must reject ext {:?}", bad_ext);
360        }
361    }
362}