hashtree_cli/storage/
upload.rs

1use super::*;
2use futures::io::AllowStdIo;
3use std::collections::HashMap;
4use std::io::Read;
5
6impl HashtreeStore {
7    /// Upload a file as raw plaintext and return its CID, with auto-pin
8    pub fn upload_file<P: AsRef<Path>>(&self, file_path: P) -> Result<String> {
9        self.upload_file_with_chunk_size(file_path, None)
10    }
11
12    /// Upload a file without pinning (for blossom uploads that can be evicted)
13    pub fn upload_file_no_pin<P: AsRef<Path>>(&self, file_path: P) -> Result<String> {
14        self.upload_file_internal(file_path, false, None)
15    }
16
17    pub fn upload_file_with_chunk_size<P: AsRef<Path>>(
18        &self,
19        file_path: P,
20        chunk_size: Option<usize>,
21    ) -> Result<String> {
22        self.upload_file_internal(file_path, true, chunk_size)
23    }
24
25    fn upload_file_internal<P: AsRef<Path>>(
26        &self,
27        file_path: P,
28        pin: bool,
29        chunk_size: Option<usize>,
30    ) -> Result<String> {
31        let file_path = file_path.as_ref();
32        let file = std::fs::File::open(file_path)
33            .with_context(|| format!("Failed to open file {}", file_path.display()))?;
34
35        // Store raw plaintext blobs without CHK encryption, streaming from disk.
36        let store = self.store_arc();
37        let mut config = HashTreeConfig::new(store).public();
38        if let Some(chunk_size) = chunk_size {
39            config = config.with_chunk_size(chunk_size);
40        }
41        let tree = HashTree::new(config);
42
43        let (cid, _size) = sync_block_on(async { tree.put_stream(AllowStdIo::new(file)).await })
44            .context("Failed to store file")?;
45
46        // Only pin if requested (htree add = pin, blossom upload = no pin)
47        if pin {
48            let mut wtxn = self.env.write_txn()?;
49            self.pins.put(&mut wtxn, cid.hash.as_slice(), &())?;
50            wtxn.commit()?;
51        }
52
53        Ok(to_hex(&cid.hash))
54    }
55
56    /// Upload a file from a stream with progress callbacks
57    pub fn upload_file_stream<R: Read, F>(
58        &self,
59        reader: R,
60        _file_name: impl Into<String>,
61        mut callback: F,
62    ) -> Result<String>
63    where
64        F: FnMut(&str),
65    {
66        // Use HashTree.put_stream for streaming upload without CHK encryption.
67        let store = self.store_arc();
68        let tree = HashTree::new(HashTreeConfig::new(store).public());
69
70        let (cid, _size) = sync_block_on(async { tree.put_stream(AllowStdIo::new(reader)).await })
71            .context("Failed to store file")?;
72
73        let root_hex = to_hex(&cid.hash);
74        callback(&root_hex);
75
76        // Auto-pin on upload
77        let mut wtxn = self.env.write_txn()?;
78        self.pins.put(&mut wtxn, cid.hash.as_slice(), &())?;
79        wtxn.commit()?;
80
81        Ok(root_hex)
82    }
83
84    /// Upload a directory and return its root hash (hex)
85    /// Respects .gitignore and ignores common OS junk files by default.
86    pub fn upload_dir<P: AsRef<Path>>(&self, dir_path: P) -> Result<String> {
87        self.upload_dir_with_options(dir_path, true)
88    }
89
90    /// Upload a directory with options as raw plaintext (no CHK encryption)
91    pub fn upload_dir_with_options<P: AsRef<Path>>(
92        &self,
93        dir_path: P,
94        respect_gitignore: bool,
95    ) -> Result<String> {
96        self.upload_dir_with_options_and_chunk_size(dir_path, respect_gitignore, None)
97    }
98
99    pub fn upload_dir_with_options_and_chunk_size<P: AsRef<Path>>(
100        &self,
101        dir_path: P,
102        respect_gitignore: bool,
103        chunk_size: Option<usize>,
104    ) -> Result<String> {
105        let dir_path = dir_path.as_ref();
106
107        let store = self.store_arc();
108        let mut config = HashTreeConfig::new(store).public();
109        if let Some(chunk_size) = chunk_size {
110            config = config.with_chunk_size(chunk_size);
111        }
112        let tree = HashTree::new(config);
113
114        let root_cid = sync_block_on(async {
115            self.upload_dir_recursive(&tree, dir_path, dir_path, respect_gitignore)
116                .await
117        })
118        .context("Failed to upload directory")?;
119
120        let root_hex = to_hex(&root_cid.hash);
121
122        let mut wtxn = self.env.write_txn()?;
123        self.pins.put(&mut wtxn, root_cid.hash.as_slice(), &())?;
124        wtxn.commit()?;
125
126        Ok(root_hex)
127    }
128
129    async fn upload_dir_recursive<S: Store>(
130        &self,
131        tree: &HashTree<S>,
132        _root_path: &Path,
133        current_path: &Path,
134        respect_gitignore: bool,
135    ) -> Result<Cid> {
136        // Build directory structure from flat file list - store full Cid with key
137        let mut dir_contents: HashMap<String, Vec<(String, Cid)>> = HashMap::new();
138        dir_contents.insert(String::new(), Vec::new()); // Root
139
140        let walker = crate::ignore_rules::build_content_walker(current_path, respect_gitignore);
141
142        for result in walker {
143            let entry = result?;
144            let path = entry.path();
145
146            // Skip the root directory itself
147            if path == current_path {
148                continue;
149            }
150
151            let relative = path.strip_prefix(current_path).unwrap_or(path);
152
153            if path.is_file() {
154                let file = std::fs::File::open(path)
155                    .with_context(|| format!("Failed to open file {}", path.display()))?;
156                let (cid, _size) = tree.put_stream(AllowStdIo::new(file)).await.map_err(|e| {
157                    anyhow::anyhow!("Failed to upload file {}: {}", path.display(), e)
158                })?;
159
160                // Get parent directory path and file name
161                let parent = relative
162                    .parent()
163                    .map(|p| p.to_string_lossy().to_string())
164                    .unwrap_or_default();
165                let name = relative
166                    .file_name()
167                    .map(|n| n.to_string_lossy().to_string())
168                    .unwrap_or_default();
169
170                dir_contents.entry(parent).or_default().push((name, cid));
171            } else if path.is_dir() {
172                // Ensure directory entry exists
173                let dir_path = relative.to_string_lossy().to_string();
174                dir_contents.entry(dir_path).or_default();
175            }
176        }
177
178        // Build directory tree bottom-up
179        self.build_directory_tree(tree, &mut dir_contents).await
180    }
181
182    async fn build_directory_tree<S: Store>(
183        &self,
184        tree: &HashTree<S>,
185        dir_contents: &mut HashMap<String, Vec<(String, Cid)>>,
186    ) -> Result<Cid> {
187        // Sort directories by depth (deepest first) to build bottom-up
188        let mut dirs: Vec<String> = dir_contents.keys().cloned().collect();
189        dirs.sort_by(|a, b| {
190            let depth_a = a.matches('/').count() + if a.is_empty() { 0 } else { 1 };
191            let depth_b = b.matches('/').count() + if b.is_empty() { 0 } else { 1 };
192            depth_b.cmp(&depth_a) // Deepest first
193        });
194
195        let mut dir_cids: HashMap<String, Cid> = HashMap::new();
196
197        for dir_path in dirs {
198            let files = dir_contents.get(&dir_path).cloned().unwrap_or_default();
199
200            let mut entries: Vec<hashtree_core::DirEntry> = files
201                .into_iter()
202                .map(|(name, cid)| hashtree_core::DirEntry::from_cid(name, &cid))
203                .collect();
204
205            // Add subdirectory entries
206            for (subdir_path, cid) in &dir_cids {
207                let parent = Path::new(subdir_path)
208                    .parent()
209                    .map(|p| p.to_string_lossy().to_string())
210                    .unwrap_or_default();
211
212                if parent == dir_path {
213                    let name = Path::new(subdir_path)
214                        .file_name()
215                        .map(|n| n.to_string_lossy().to_string())
216                        .unwrap_or_default();
217                    entries.push(hashtree_core::DirEntry::from_cid(name, cid));
218                }
219            }
220
221            let cid = tree
222                .put_directory(entries)
223                .await
224                .map_err(|e| anyhow::anyhow!("Failed to create directory node: {}", e))?;
225
226            dir_cids.insert(dir_path, cid);
227        }
228
229        // Return root Cid
230        dir_cids
231            .get("")
232            .cloned()
233            .ok_or_else(|| anyhow::anyhow!("No root directory"))
234    }
235
236    /// Upload a file with CHK encryption, returns CID in format "hash:key"
237    pub fn upload_file_encrypted<P: AsRef<Path>>(&self, file_path: P) -> Result<String> {
238        self.upload_file_encrypted_with_chunk_size(file_path, None)
239    }
240
241    pub fn upload_file_encrypted_with_chunk_size<P: AsRef<Path>>(
242        &self,
243        file_path: P,
244        chunk_size: Option<usize>,
245    ) -> Result<String> {
246        let file_path = file_path.as_ref();
247        let file = std::fs::File::open(file_path)
248            .with_context(|| format!("Failed to open file {}", file_path.display()))?;
249
250        // Use unified API with encryption enabled (default), streaming from disk.
251        let store = self.store_arc();
252        let mut config = HashTreeConfig::new(store);
253        if let Some(chunk_size) = chunk_size {
254            config = config.with_chunk_size(chunk_size);
255        }
256        let tree = HashTree::new(config);
257
258        let (cid, _size) = sync_block_on(async { tree.put_stream(AllowStdIo::new(file)).await })
259            .map_err(|e| anyhow::anyhow!("Failed to encrypt file: {}", e))?;
260
261        let cid_str = cid.to_string();
262
263        let mut wtxn = self.env.write_txn()?;
264        self.pins.put(&mut wtxn, cid.hash.as_slice(), &())?;
265        wtxn.commit()?;
266
267        Ok(cid_str)
268    }
269
270    /// Upload a directory with CHK encryption, returns CID
271    /// Respects .gitignore and ignores common OS junk files by default.
272    pub fn upload_dir_encrypted<P: AsRef<Path>>(&self, dir_path: P) -> Result<String> {
273        self.upload_dir_encrypted_with_options(dir_path, true)
274    }
275
276    /// Upload a directory with CHK encryption and options
277    /// Returns CID as "hash:key" format for encrypted directories
278    pub fn upload_dir_encrypted_with_options<P: AsRef<Path>>(
279        &self,
280        dir_path: P,
281        respect_gitignore: bool,
282    ) -> Result<String> {
283        self.upload_dir_encrypted_with_options_and_chunk_size(dir_path, respect_gitignore, None)
284    }
285
286    pub fn upload_dir_encrypted_with_options_and_chunk_size<P: AsRef<Path>>(
287        &self,
288        dir_path: P,
289        respect_gitignore: bool,
290        chunk_size: Option<usize>,
291    ) -> Result<String> {
292        let dir_path = dir_path.as_ref();
293        let store = self.store_arc();
294
295        // Use unified API with encryption enabled (default)
296        let mut config = HashTreeConfig::new(store);
297        if let Some(chunk_size) = chunk_size {
298            config = config.with_chunk_size(chunk_size);
299        }
300        let tree = HashTree::new(config);
301
302        let root_cid = sync_block_on(async {
303            self.upload_dir_recursive(&tree, dir_path, dir_path, respect_gitignore)
304                .await
305        })
306        .context("Failed to upload encrypted directory")?;
307
308        let cid_str = root_cid.to_string(); // Returns "hash:key" or "hash"
309
310        let mut wtxn = self.env.write_txn()?;
311        // Pin by hash only (the key is for decryption, not identification)
312        self.pins.put(&mut wtxn, root_cid.hash.as_slice(), &())?;
313        wtxn.commit()?;
314
315        Ok(cid_str)
316    }
317}
hashtree_cli/storage/upload.rs

hashtree_cli/storage/
upload.rs
