firecloud_storage/
manifest.rs

1//! File manifest storage using Sled
2//!
3//! Persists FileManifest to local storage, enabling:
4//! - Upload: Store manifest after chunking a file
5//! - Download: Retrieve manifest by file_id to reconstruct files
6//! - Share: Serve manifests to peers via P2P protocol
7
8use crate::{StorageError, StorageResult};
9use firecloud_core::{FileId, FileManifest};
10#[cfg(test)]
11use firecloud_core::{ChunkHash, FileMetadata};
12use sled::Db;
13use std::path::Path;
14use tracing::{debug, info, warn};
15
16/// Tree names for manifest data
17const TREE_MANIFESTS: &str = "manifests";
18const TREE_FILE_INDEX: &str = "file_index"; // name -> file_id mapping
19
20/// Local manifest store backed by Sled
21pub struct ManifestStore {
22    db: Db,
23}
24
25impl ManifestStore {
26    /// Open or create a manifest store at the given path
27    pub fn open<P: AsRef<Path>>(path: P) -> StorageResult<Self> {
28        let db = sled::open(path).map_err(|e| StorageError::Database(e.to_string()))?;
29        info!("Opened manifest store with Sled");
30        Ok(Self { db })
31    }
32
33    /// Store a file manifest
34    pub fn put(&self, manifest: &FileManifest) -> StorageResult<()> {
35        let manifests_tree = self
36            .db
37            .open_tree(TREE_MANIFESTS)
38            .map_err(|e| StorageError::Database(e.to_string()))?;
39        let index_tree = self
40            .db
41            .open_tree(TREE_FILE_INDEX)
42            .map_err(|e| StorageError::Database(e.to_string()))?;
43
44        // Use file_id UUID bytes as key
45        let key = manifest.metadata.id.0.as_bytes();
46
47        // Serialize and store manifest
48        let manifest_bytes =
49            bincode::serialize(manifest).map_err(|e| StorageError::Serialization(e.to_string()))?;
50        manifests_tree
51            .insert(key, manifest_bytes)
52            .map_err(|e| StorageError::Database(e.to_string()))?;
53
54        // Also index by filename for lookup
55        let name_key = manifest.metadata.name.as_bytes();
56        index_tree
57            .insert(name_key, key)
58            .map_err(|e| StorageError::Database(e.to_string()))?;
59
60        debug!(
61            "Stored manifest: {} ({})",
62            manifest.metadata.id, manifest.metadata.name
63        );
64        Ok(())
65    }
66
67    /// Retrieve a manifest by file ID
68    pub fn get(&self, file_id: &FileId) -> StorageResult<Option<FileManifest>> {
69        let manifests_tree = self
70            .db
71            .open_tree(TREE_MANIFESTS)
72            .map_err(|e| StorageError::Database(e.to_string()))?;
73
74        let key = file_id.0.as_bytes();
75
76        match manifests_tree
77            .get(key)
78            .map_err(|e| StorageError::Database(e.to_string()))?
79        {
80            Some(bytes) => {
81                let manifest: FileManifest = bincode::deserialize(&bytes)
82                    .map_err(|e| StorageError::Serialization(e.to_string()))?;
83                Ok(Some(manifest))
84            }
85            None => Ok(None),
86        }
87    }
88
89    /// Retrieve a manifest by file ID string (UUID format)
90    pub fn get_by_id_str(&self, file_id_str: &str) -> StorageResult<Option<FileManifest>> {
91        let uuid = uuid::Uuid::parse_str(file_id_str)
92            .map_err(|e| StorageError::InvalidId(format!("Invalid file ID: {}", e)))?;
93        let file_id = FileId::from_uuid(uuid);
94        self.get(&file_id)
95    }
96
97    /// Retrieve a manifest by filename
98    pub fn get_by_name(&self, name: &str) -> StorageResult<Option<FileManifest>> {
99        let index_tree = self
100            .db
101            .open_tree(TREE_FILE_INDEX)
102            .map_err(|e| StorageError::Database(e.to_string()))?;
103
104        let name_key = name.as_bytes();
105
106        match index_tree
107            .get(name_key)
108            .map_err(|e| StorageError::Database(e.to_string()))?
109        {
110            Some(file_id_bytes) => {
111                // file_id_bytes is UUID bytes
112                let uuid = uuid::Uuid::from_slice(&file_id_bytes)
113                    .map_err(|e| StorageError::Serialization(e.to_string()))?;
114                let file_id = FileId::from_uuid(uuid);
115                self.get(&file_id)
116            }
117            None => Ok(None),
118        }
119    }
120
121    /// Check if a manifest exists
122    pub fn contains(&self, file_id: &FileId) -> StorageResult<bool> {
123        let manifests_tree = self
124            .db
125            .open_tree(TREE_MANIFESTS)
126            .map_err(|e| StorageError::Database(e.to_string()))?;
127        Ok(manifests_tree
128            .contains_key(file_id.0.as_bytes())
129            .map_err(|e| StorageError::Database(e.to_string()))?)
130    }
131
132    /// Delete a manifest
133    pub fn delete(&self, file_id: &FileId) -> StorageResult<bool> {
134        let manifests_tree = self
135            .db
136            .open_tree(TREE_MANIFESTS)
137            .map_err(|e| StorageError::Database(e.to_string()))?;
138        let index_tree = self
139            .db
140            .open_tree(TREE_FILE_INDEX)
141            .map_err(|e| StorageError::Database(e.to_string()))?;
142
143        // Get manifest first to find filename for index removal
144        let key = file_id.0.as_bytes();
145        if let Some(bytes) = manifests_tree
146            .get(key)
147            .map_err(|e| StorageError::Database(e.to_string()))?
148        {
149            let manifest: FileManifest = bincode::deserialize(&bytes)
150                .map_err(|e| StorageError::Serialization(e.to_string()))?;
151
152            // Remove from name index
153            index_tree
154                .remove(manifest.metadata.name.as_bytes())
155                .map_err(|e| StorageError::Database(e.to_string()))?;
156        }
157
158        // Remove manifest
159        let existed = manifests_tree
160            .remove(key)
161            .map_err(|e| StorageError::Database(e.to_string()))?
162            .is_some();
163
164        if existed {
165            debug!("Deleted manifest: {}", file_id);
166        }
167        Ok(existed)
168    }
169
170    /// List all stored manifests (returns file_id, name, size, chunk_count)
171    pub fn list(&self) -> StorageResult<Vec<ManifestSummary>> {
172        let manifests_tree = self
173            .db
174            .open_tree(TREE_MANIFESTS)
175            .map_err(|e| StorageError::Database(e.to_string()))?;
176
177        let mut summaries = Vec::new();
178
179        for result in manifests_tree.iter() {
180            let (_, value) = result.map_err(|e| StorageError::Database(e.to_string()))?;
181            
182            // Try to deserialize manifest, skip corrupted entries
183            match bincode::deserialize::<FileManifest>(&value) {
184                Ok(manifest) => {
185                    summaries.push(ManifestSummary {
186                        file_id: manifest.metadata.id,
187                        name: manifest.metadata.name.clone(),
188                        size: manifest.metadata.size,
189                        chunk_count: manifest.chunks.len(),
190                        created_at: manifest.metadata.created_at,
191                    });
192                }
193                Err(e) => {
194                    // Log warning but continue iteration (old schema or corrupted data)
195                    warn!("Failed to deserialize manifest (skipping): {}", e);
196                    continue;
197                }
198            }
199        }
200
201        Ok(summaries)
202    }
203
204    /// Get total number of manifests stored
205    pub fn count(&self) -> StorageResult<usize> {
206        let manifests_tree = self
207            .db
208            .open_tree(TREE_MANIFESTS)
209            .map_err(|e| StorageError::Database(e.to_string()))?;
210        Ok(manifests_tree.len())
211    }
212
213    /// Flush all pending writes to disk
214    pub fn flush(&self) -> StorageResult<()> {
215        self.db
216            .flush()
217            .map_err(|e| StorageError::Database(e.to_string()))?;
218        Ok(())
219    }
220}
221
222/// Summary info for listing manifests
223#[derive(Debug, Clone)]
224pub struct ManifestSummary {
225    pub file_id: FileId,
226    pub name: String,
227    pub size: u64,
228    pub chunk_count: usize,
229    pub created_at: i64,
230}
231
232#[cfg(test)]
233mod tests {
234    use super::*;
235    use tempfile::TempDir;
236
237    fn create_test_manifest() -> FileManifest {
238        let metadata = FileMetadata {
239            id: FileId::new(),
240            name: "test_file.txt".to_string(),
241            mime_type: Some("text/plain".to_string()),
242            size: 1024,
243            created_at: chrono::Utc::now().timestamp_millis(),
244            modified_at: chrono::Utc::now().timestamp_millis(),
245            content_hash: ChunkHash::hash(b"test content"),
246        };
247
248        FileManifest {
249            metadata,
250            chunks: vec![
251                ChunkHash::hash(b"chunk1"),
252                ChunkHash::hash(b"chunk2"),
253            ],
254            encrypted_dek: Some(vec![0u8; 32]),
255            salt: Some(vec![0u8; 16]),  // Add salt field for Phase 3 compatibility
256            version: 1,
257        }
258    }
259
260    #[test]
261    fn test_manifest_store_put_get() {
262        let tmp_dir = TempDir::new().unwrap();
263        let store = ManifestStore::open(tmp_dir.path().join("manifests")).unwrap();
264
265        let manifest = create_test_manifest();
266        let file_id = manifest.metadata.id;
267
268        // Store
269        store.put(&manifest).unwrap();
270
271        // Retrieve by ID
272        let retrieved = store.get(&file_id).unwrap().unwrap();
273        assert_eq!(retrieved.metadata.name, "test_file.txt");
274        assert_eq!(retrieved.chunks.len(), 2);
275    }
276
277    #[test]
278    fn test_manifest_store_get_by_name() {
279        let tmp_dir = TempDir::new().unwrap();
280        let store = ManifestStore::open(tmp_dir.path().join("manifests")).unwrap();
281
282        let manifest = create_test_manifest();
283        store.put(&manifest).unwrap();
284
285        // Retrieve by name
286        let retrieved = store.get_by_name("test_file.txt").unwrap().unwrap();
287        assert_eq!(retrieved.metadata.id, manifest.metadata.id);
288    }
289
290    #[test]
291    fn test_manifest_store_list() {
292        let tmp_dir = TempDir::new().unwrap();
293        let store = ManifestStore::open(tmp_dir.path().join("manifests")).unwrap();
294
295        // Store multiple manifests
296        for i in 0..3 {
297            let mut manifest = create_test_manifest();
298            manifest.metadata.name = format!("file_{}.txt", i);
299            store.put(&manifest).unwrap();
300        }
301
302        let summaries = store.list().unwrap();
303        assert_eq!(summaries.len(), 3);
304    }
305}
firecloud_storage/manifest.rs

firecloud_storage/
manifest.rs
