1use log::trace;
2use quick_cache::{sync::Cache, Weighter};
3use sha2::{Digest, Sha256};
4use std::path::{Path, PathBuf};
5use tokio::fs::{self, create_dir_all, File};
6use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader, BufWriter};
7
8use crate::errors::SyncError;
9
10const BINARY_CHUNK_SIZE: usize = 1_024 * 1_024; const BINARY_HASH_SIZE: usize = 32;
12const TEXT_HASH_SIZE: usize = 10;
13
14pub struct Chunker {
15 cache: InMemoryCache,
16 base_path: PathBuf,
17}
18
19type Result<T, E = SyncError> = std::result::Result<T, E>;
20
21impl Chunker {
22 pub fn new(cache: InMemoryCache, base_path: PathBuf) -> Chunker {
23 Chunker { cache, base_path }
24 }
25
26 fn full_path(&self, path: &str) -> PathBuf {
27 let mut base = self.base_path.clone();
28 base.push(path);
29 base
30 }
31
32 pub async fn hashify(&mut self, path: &str) -> Result<Vec<String>> {
33 let p = Path::new(path);
34
35 if is_text(p) {
37 self.hashify_text(path).await
38 } else if is_binary(p) {
39 self.hashify_binary(path).await
40 } else {
41 Err(SyncError::UnlistedFileFormat(path.to_string()))
42 }
43 }
44
45 async fn hashify_binary(&mut self, path: &str) -> Result<Vec<String>> {
46 let file = File::open(self.full_path(path))
47 .await
48 .map_err(|e| SyncError::from_io_error(path, e))?;
49 let mut reader = BufReader::new(file);
50 let mut hashes = Vec::new();
51 let mut buffer = vec![0u8; BINARY_CHUNK_SIZE];
52
53 loop {
54 let bytes_read = reader
55 .read(&mut buffer)
56 .await
57 .map_err(|e| SyncError::from_io_error(path, e))?;
58 if bytes_read == 0 {
59 break;
60 }
61
62 let data = &buffer[..bytes_read].to_vec();
63 let hash = self.hash(data, BINARY_HASH_SIZE);
64 self.save_chunk(&hash, data.to_vec())?;
65 hashes.push(hash);
66 }
67
68 Ok(hashes)
69 }
70
71 async fn hashify_text(&mut self, path: &str) -> Result<Vec<String>> {
72 let file = File::open(self.full_path(path))
73 .await
74 .map_err(|e| SyncError::from_io_error(path, e))?;
75 let mut reader = BufReader::new(file);
76 let mut buffer = Vec::new();
77 let mut hashes = Vec::new();
78
79 while reader
80 .read_until(b'\n', &mut buffer)
81 .await
82 .map_err(|e| SyncError::from_io_error(path, e))?
83 > 0
84 {
85 let data: Vec<u8> = buffer.clone();
86 let hash = self.hash(&data, TEXT_HASH_SIZE);
87 self.save_chunk(&hash, data)?;
88 hashes.push(hash);
89
90 buffer.clear();
92 }
93
94 Ok(hashes)
95 }
96
97 pub fn hash(&self, data: &Vec<u8>, size: usize) -> String {
98 let mut hasher = Sha256::new();
99
100 hasher.update(data);
101
102 let result = hasher.finalize();
103 let hex_string = format!("{:x}", result);
104
105 hex_string[0..size].to_string()
106 }
107
108 pub fn exists(&mut self, path: &str) -> bool {
109 let full_path = self.full_path(path);
110
111 full_path.exists()
112 }
113
114 pub async fn save(&mut self, path: &str, hashes: Vec<&str>) -> Result<()> {
116 trace!("saving {:?}", path);
117 let full_path = self.full_path(path);
118
119 if let Some(parent) = full_path.parent() {
120 create_dir_all(parent)
121 .await
122 .map_err(|e| SyncError::from_io_error(path, e))?;
123 }
124
125 let file = File::create(full_path)
126 .await
127 .map_err(|e| SyncError::from_io_error(path, e))?;
128 let mut writer = BufWriter::new(file);
129
130 for hash in hashes {
131 let chunk = self.cache.get(hash)?;
132
133 writer
134 .write_all(&chunk)
135 .await
136 .map_err(|e| SyncError::from_io_error(path, e))?;
137 }
138
139 writer
140 .flush()
141 .await
142 .map_err(|e| SyncError::from_io_error(path, e))?;
143
144 Ok(())
145 }
146
147 pub async fn delete(&mut self, path: &str) -> Result<()> {
148 trace!("deleting {:?}", path);
149 let full_path = self.full_path(path);
150
151 fs::remove_file(&full_path)
152 .await
153 .map_err(|e| SyncError::from_io_error(path, e))?;
154
155 let mut parent = full_path.parent();
170 while let Some(dir) = parent {
171 if dir == self.base_path {
172 break;
173 }
174 if let Err(e) = fs::remove_dir(dir).await {
175 trace!("stopping empty-dir cleanup at {:?}: {}", dir, e);
176 break;
177 }
178 parent = dir.parent();
179 }
180
181 Ok(())
182 }
183
184 pub fn read_chunk(&self, chunk_hash: &str) -> Result<Vec<u8>> {
185 self.cache.get(chunk_hash)
186 }
187
188 pub fn save_chunk(&mut self, chunk_hash: &str, content: Vec<u8>) -> Result<()> {
189 self.cache.set(chunk_hash, content)
190 }
191
192 pub fn check_chunk(&self, chunk_hash: &str) -> bool {
193 if chunk_hash.is_empty() {
194 true
195 } else {
196 self.cache.contains(chunk_hash)
197 }
198 }
199}
200
201#[derive(Clone)]
202pub struct BytesWeighter;
203
204impl Weighter<String, Vec<u8>> for BytesWeighter {
205 fn weight(&self, _key: &String, val: &Vec<u8>) -> u64 {
206 val.len().clamp(1, u64::MAX as usize) as u64
208 }
209}
210
211pub struct InMemoryCache {
212 cache: Cache<String, Vec<u8>, BytesWeighter>,
213}
214
215impl InMemoryCache {
216 pub fn new(total_keys: usize, total_weight: u64) -> InMemoryCache {
217 InMemoryCache {
218 cache: Cache::with_weighter(total_keys, total_weight, BytesWeighter),
219 }
220 }
221
222 fn get(&self, chunk_hash: &str) -> Result<Vec<u8>> {
223 if chunk_hash.is_empty() {
224 return Ok(vec![]);
225 }
226
227 match self.cache.get(chunk_hash) {
228 Some(content) => Ok(content.clone()),
229 None => Err(SyncError::GetFromCacheError),
230 }
231 }
232
233 fn set(&mut self, chunk_hash: &str, content: Vec<u8>) -> Result<()> {
234 self.cache.insert(chunk_hash.to_string(), content);
236 Ok(())
237 }
238
239 fn contains(&self, chunk_hash: &str) -> bool {
240 match self.cache.get(chunk_hash) {
241 Some(_content) => true,
242 None => false,
243 }
244 }
245}
246
247pub fn is_binary(p: &Path) -> bool {
248 if let Some(ext) = p.extension() {
249 let ext = ext.to_ascii_lowercase();
250
251 ext == "jpg" || ext == "jpeg" || ext == "png"
252 } else {
253 false
254 }
255}
256
257pub fn is_text(p: &Path) -> bool {
258 if let Some(file_name) = p.file_name() {
260 let file_name_str = file_name.to_string_lossy();
261 if file_name_str == ".shopping-list"
262 || file_name_str == ".shopping-checked"
263 || file_name_str == ".bookmarks"
264 {
265 return true;
266 }
267 }
268
269 if let Some(ext) = p.extension() {
271 let ext = ext.to_ascii_lowercase();
272
273 ext == "cook"
274 || ext == "conf"
275 || ext == "yaml"
276 || ext == "yml"
277 || ext == "md"
278 || ext == "menu"
279 || ext == "jinja"
280 || ext == "j2"
281 } else {
282 false
283 }
284}
285
286#[cfg(test)]
287mod tests {
288 use super::*;
289 use std::path::Path;
290 use tempfile::TempDir;
291 use tokio::fs::File;
292 use tokio::io::AsyncWriteExt;
293
294 #[test]
295 fn test_is_binary_with_jpg() {
296 let path = Path::new("image.jpg");
297 assert!(is_binary(path));
298 }
299
300 #[test]
301 fn test_is_binary_with_jpeg() {
302 let path = Path::new("image.JPEG");
303 assert!(is_binary(path));
304 }
305
306 #[test]
307 fn test_is_binary_with_png() {
308 let path = Path::new("image.png");
309 assert!(is_binary(path));
310 }
311
312 #[test]
313 fn test_is_binary_returns_false_for_text() {
314 let path = Path::new("recipe.cook");
315 assert!(!is_binary(path));
316 }
317
318 #[test]
319 fn test_is_text_with_cook_extension() {
320 let path = Path::new("recipe.cook");
321 assert!(is_text(path));
322 }
323
324 #[test]
325 fn test_is_text_with_md_extension() {
326 let path = Path::new("README.md");
327 assert!(is_text(path));
328 }
329
330 #[test]
331 fn test_is_text_with_yaml_extension() {
332 let path = Path::new("config.yaml");
333 assert!(is_text(path));
334 }
335
336 #[test]
337 fn test_is_text_with_yml_extension() {
338 let path = Path::new("config.yml");
339 assert!(is_text(path));
340 }
341
342 #[test]
343 fn test_is_text_with_special_filenames() {
344 assert!(is_text(Path::new(".shopping-list")));
345 assert!(is_text(Path::new(".shopping-checked")));
346 assert!(is_text(Path::new(".bookmarks")));
347 }
348
349 #[test]
350 fn test_is_text_returns_false_for_unknown() {
351 let path = Path::new("file.unknown");
352 assert!(!is_text(path));
353 }
354
355 #[test]
356 fn test_hash_consistency() {
357 let cache = InMemoryCache::new(100, 1000);
358 let chunker = Chunker::new(cache, PathBuf::from("/tmp"));
359
360 let data = b"Hello, World!".to_vec();
361 let hash1 = chunker.hash(&data, 10);
362 let hash2 = chunker.hash(&data, 10);
363
364 assert_eq!(hash1, hash2);
366 }
367
368 #[test]
369 fn test_hash_different_data_produces_different_hash() {
370 let cache = InMemoryCache::new(100, 1000);
371 let chunker = Chunker::new(cache, PathBuf::from("/tmp"));
372
373 let data1 = b"Hello, World!".to_vec();
374 let data2 = b"Goodbye, World!".to_vec();
375
376 let hash1 = chunker.hash(&data1, 10);
377 let hash2 = chunker.hash(&data2, 10);
378
379 assert_ne!(hash1, hash2);
381 }
382
383 #[test]
384 fn test_hash_respects_size_parameter() {
385 let cache = InMemoryCache::new(100, 1000);
386 let chunker = Chunker::new(cache, PathBuf::from("/tmp"));
387
388 let data = b"Hello, World!".to_vec();
389 let hash_short = chunker.hash(&data, 5);
390 let hash_long = chunker.hash(&data, 10);
391
392 assert_eq!(hash_short.len(), 5);
393 assert_eq!(hash_long.len(), 10);
394 assert!(hash_long.starts_with(&hash_short));
396 }
397
398 #[test]
399 fn test_inmemory_cache_set_and_get() {
400 let mut cache = InMemoryCache::new(100, 1000);
401
402 let hash = "testhash123";
403 let data = vec![1, 2, 3, 4, 5];
404
405 cache.set(hash, data.clone()).unwrap();
406 let retrieved = cache.get(hash).unwrap();
407
408 assert_eq!(data, retrieved);
409 }
410
411 #[test]
412 fn test_inmemory_cache_get_nonexistent() {
413 let cache = InMemoryCache::new(100, 1000);
414
415 let result = cache.get("nonexistent");
416 assert!(result.is_err());
417 }
418
419 #[test]
420 fn test_inmemory_cache_contains() {
421 let mut cache = InMemoryCache::new(100, 1000);
422
423 let hash = "testhash456";
424 let data = vec![1, 2, 3];
425
426 assert!(!cache.contains(hash));
427 cache.set(hash, data).unwrap();
428 assert!(cache.contains(hash));
429 }
430
431 #[test]
432 fn test_inmemory_cache_empty_hash() {
433 let cache = InMemoryCache::new(100, 1000);
434
435 let result = cache.get("").unwrap();
437 assert_eq!(result, Vec::<u8>::new());
438 }
439
440 #[test]
441 fn test_chunker_check_chunk_empty_hash() {
442 let cache = InMemoryCache::new(100, 1000);
443 let chunker = Chunker::new(cache, PathBuf::from("/tmp"));
444
445 assert!(chunker.check_chunk(""));
447 }
448
449 #[test]
450 fn test_chunker_check_chunk_existing() {
451 let mut cache = InMemoryCache::new(100, 1000);
452 cache.set("existinghash", vec![1, 2, 3]).unwrap();
453 let chunker = Chunker::new(cache, PathBuf::from("/tmp"));
454
455 assert!(chunker.check_chunk("existinghash"));
456 }
457
458 #[test]
459 fn test_chunker_check_chunk_nonexistent() {
460 let cache = InMemoryCache::new(100, 1000);
461 let chunker = Chunker::new(cache, PathBuf::from("/tmp"));
462
463 assert!(!chunker.check_chunk("nonexistent"));
464 }
465
466 #[tokio::test]
467 async fn test_chunker_hashify_text_round_trip() {
468 let temp_dir = TempDir::new().unwrap();
469 let cache = InMemoryCache::new(1000, 100000);
470 let mut chunker = Chunker::new(cache, temp_dir.path().to_path_buf());
471
472 let test_file = "test.cook";
474 let content = "Line 1\nLine 2\nLine 3\n";
475 let mut file = File::create(temp_dir.path().join(test_file)).await.unwrap();
476 file.write_all(content.as_bytes()).await.unwrap();
477 file.flush().await.unwrap();
478
479 let hashes = chunker.hashify(test_file).await.unwrap();
481
482 assert_eq!(hashes.len(), 3);
484
485 for hash in &hashes {
487 assert!(chunker.check_chunk(hash));
488 }
489 }
490
491 #[tokio::test]
492 async fn test_chunker_save_and_read() {
493 let temp_dir = TempDir::new().unwrap();
494 let cache = InMemoryCache::new(1000, 100000);
495 let mut chunker = Chunker::new(cache, temp_dir.path().to_path_buf());
496
497 let chunk1 = b"Hello ".to_vec();
499 let chunk2 = b"World!".to_vec();
500 let hash1 = chunker.hash(&chunk1, 10);
501 let hash2 = chunker.hash(&chunk2, 10);
502
503 chunker.save_chunk(&hash1, chunk1).unwrap();
504 chunker.save_chunk(&hash2, chunk2).unwrap();
505
506 let test_file = "output.txt";
508 chunker.save(test_file, vec![&hash1, &hash2]).await.unwrap();
509
510 assert!(chunker.exists(test_file));
512
513 let content = tokio::fs::read(temp_dir.path().join(test_file))
515 .await
516 .unwrap();
517 assert_eq!(content, b"Hello World!");
518 }
519
520 #[tokio::test]
521 async fn test_chunker_delete() {
522 let temp_dir = TempDir::new().unwrap();
523 let cache = InMemoryCache::new(1000, 100000);
524 let mut chunker = Chunker::new(cache, temp_dir.path().to_path_buf());
525
526 let test_file = "to_delete.txt";
528 let mut file = File::create(temp_dir.path().join(test_file)).await.unwrap();
529 file.write_all(b"test content").await.unwrap();
530 file.flush().await.unwrap();
531
532 assert!(chunker.exists(test_file));
533
534 chunker.delete(test_file).await.unwrap();
536
537 assert!(!chunker.exists(test_file));
539 }
540
541 #[test]
542 fn test_bytes_weighter() {
543 let weighter = BytesWeighter;
544
545 let key = "test".to_string();
546 let small_val = vec![1, 2, 3];
547 let large_val = vec![0u8; 1000];
548
549 assert_eq!(weighter.weight(&key, &small_val), 3);
550 assert_eq!(weighter.weight(&key, &large_val), 1000);
551 }
552
553 #[test]
554 fn test_bytes_weighter_empty_vec() {
555 let weighter = BytesWeighter;
556
557 let key = "test".to_string();
558 let empty_val = vec![];
559
560 assert_eq!(weighter.weight(&key, &empty_val), 1);
562 }
563
564 #[tokio::test]
565 async fn hashify_errors_on_missing_file() {
566 let temp = tempfile::TempDir::new().unwrap();
567 let cache = InMemoryCache::new(100, 10_000);
568 let mut chunker = Chunker::new(cache, temp.path().to_path_buf());
569
570 let err = chunker
571 .hashify("does_not_exist.cook")
572 .await
573 .expect_err("hashify on missing file should error");
574 assert!(
577 matches!(err, SyncError::IoError { .. }),
578 "expected SyncError::IoError for missing file, got {err:?}"
579 );
580 }
581
582 #[tokio::test]
583 async fn save_errors_when_referenced_chunk_is_missing() {
584 let temp = tempfile::TempDir::new().unwrap();
585 let cache = InMemoryCache::new(100, 10_000);
586 let mut chunker = Chunker::new(cache, temp.path().to_path_buf());
587
588 let phantom = "deadbeefdeadbeefdeadbeefdeadbeef";
590 let err = chunker
591 .save("out.cook", vec![phantom])
592 .await
593 .expect_err("save should fail when chunk is not available");
594 assert!(
596 matches!(err, SyncError::GetFromCacheError),
597 "expected SyncError::GetFromCacheError for unknown chunk, got {err:?}"
598 );
599 }
600
601 #[tokio::test]
602 async fn delete_removes_file_from_storage_dir() {
603 let temp = tempfile::TempDir::new().unwrap();
604 let cache = InMemoryCache::new(100, 10_000);
605 let mut chunker = Chunker::new(cache, temp.path().to_path_buf());
606
607 let path = "recipe.cook";
609 tokio::fs::write(temp.path().join(path), b"eggs\n")
610 .await
611 .unwrap();
612 assert!(temp.path().join(path).exists(), "precondition: file written");
613
614 chunker
615 .delete(path)
616 .await
617 .expect("Chunker::delete should succeed on existing file");
618
619 assert!(
620 !temp.path().join(path).exists(),
621 "file should be gone after Chunker::delete"
622 );
623 }
624
625 #[tokio::test]
626 async fn hashify_then_read_chunk_round_trip_via_cache() {
627 let temp = tempfile::TempDir::new().unwrap();
632 let cache = InMemoryCache::new(100, 10_000);
633 let mut chunker = Chunker::new(cache, temp.path().to_path_buf());
634
635 let path = "round.cook";
636 let body = b"alpha\nbeta\n"; tokio::fs::write(temp.path().join(path), body).await.unwrap();
638
639 let hashes = chunker.hashify(path).await.expect("hashify");
640 assert_eq!(hashes.len(), 2, "two-line file produces two chunks");
641
642 let mut recovered = Vec::new();
645 for h in &hashes {
646 assert!(
647 chunker.check_chunk(h),
648 "hashify must populate the cache for every hash it returns"
649 );
650 let bytes = chunker.read_chunk(h).expect("read_chunk on known hash");
651 recovered.extend_from_slice(&bytes);
652 }
653 assert_eq!(recovered, body, "chunks concatenate back to the original bytes");
654 }
655
656 #[tokio::test]
657 async fn hashify_text_file_with_multiple_lines_produces_multiple_chunks() {
658 let temp = tempfile::TempDir::new().unwrap();
659 let cache = InMemoryCache::new(1000, 100_000);
660 let mut chunker = Chunker::new(cache, temp.path().to_path_buf());
661
662 let content = "line-a\nline-b\nline-c\n";
663 tokio::fs::write(temp.path().join("multi.cook"), content.as_bytes())
664 .await
665 .unwrap();
666
667 let hashes = chunker.hashify("multi.cook").await.unwrap();
668 assert_eq!(
669 hashes.len(),
670 3,
671 "three newline-terminated lines should yield three text chunks; got {hashes:?}"
672 );
673 }
674
675 #[tokio::test]
676 async fn delete_removes_empty_parent_directories() {
677 let temp = tempfile::TempDir::new().unwrap();
681 let cache = InMemoryCache::new(100, 10_000);
682 let mut chunker = Chunker::new(cache, temp.path().to_path_buf());
683
684 let nested_dir = temp.path().join("a").join("b");
685 tokio::fs::create_dir_all(&nested_dir).await.unwrap();
686 tokio::fs::write(nested_dir.join("c.cook"), b"eggs\n")
687 .await
688 .unwrap();
689
690 chunker
691 .delete("a/b/c.cook")
692 .await
693 .expect("delete should succeed");
694
695 assert!(
696 !temp.path().join("a/b/c.cook").exists(),
697 "file should be removed"
698 );
699 assert!(
700 !temp.path().join("a/b").exists(),
701 "empty intermediate directory should be removed"
702 );
703 assert!(
704 !temp.path().join("a").exists(),
705 "empty grandparent directory should be removed"
706 );
707 assert!(
708 temp.path().exists(),
709 "storage root must never be removed"
710 );
711 }
712
713 #[tokio::test]
714 async fn delete_stops_at_first_non_empty_ancestor() {
715 let temp = tempfile::TempDir::new().unwrap();
721 let cache = InMemoryCache::new(100, 10_000);
722 let mut chunker = Chunker::new(cache, temp.path().to_path_buf());
723
724 let nested_dir = temp.path().join("a").join("b");
725 tokio::fs::create_dir_all(&nested_dir).await.unwrap();
726 tokio::fs::write(nested_dir.join("c.cook"), b"eggs\n")
727 .await
728 .unwrap();
729 tokio::fs::write(temp.path().join("a").join("sibling.cook"), b"flour\n")
730 .await
731 .unwrap();
732
733 chunker
734 .delete("a/b/c.cook")
735 .await
736 .expect("delete should succeed");
737
738 assert!(
739 !temp.path().join("a/b/c.cook").exists(),
740 "target file should be removed"
741 );
742 assert!(
743 !temp.path().join("a/b").exists(),
744 "empty intermediate directory should be removed"
745 );
746 assert!(
747 temp.path().join("a").exists(),
748 "non-empty ancestor must be preserved"
749 );
750 assert!(
751 temp.path().join("a/sibling.cook").exists(),
752 "sibling file must be preserved"
753 );
754 }
755
756 #[tokio::test]
757 async fn delete_does_not_remove_storage_root() {
758 let temp = tempfile::TempDir::new().unwrap();
763 let cache = InMemoryCache::new(100, 10_000);
764 let mut chunker = Chunker::new(cache, temp.path().to_path_buf());
765
766 tokio::fs::write(temp.path().join("only.cook"), b"sugar\n")
767 .await
768 .unwrap();
769
770 chunker
771 .delete("only.cook")
772 .await
773 .expect("delete should succeed");
774
775 assert!(
776 !temp.path().join("only.cook").exists(),
777 "file should be removed"
778 );
779 assert!(
780 temp.path().exists(),
781 "storage root must never be removed"
782 );
783 assert!(
784 temp.path().is_dir(),
785 "storage root must still be a directory"
786 );
787 }
788
789 #[tokio::test]
790 async fn delete_leaves_sibling_files_in_same_directory() {
791 let temp = tempfile::TempDir::new().unwrap();
796 let cache = InMemoryCache::new(100, 10_000);
797 let mut chunker = Chunker::new(cache, temp.path().to_path_buf());
798
799 let dir = temp.path().join("a");
800 tokio::fs::create_dir_all(&dir).await.unwrap();
801 tokio::fs::write(dir.join("x.cook"), b"salt\n").await.unwrap();
802 tokio::fs::write(dir.join("y.cook"), b"pepper\n").await.unwrap();
803
804 chunker
805 .delete("a/x.cook")
806 .await
807 .expect("delete should succeed");
808
809 assert!(
810 !temp.path().join("a/x.cook").exists(),
811 "target file should be removed"
812 );
813 assert!(
814 temp.path().join("a").exists(),
815 "directory with remaining files must be preserved"
816 );
817 assert!(
818 temp.path().join("a/y.cook").exists(),
819 "sibling file must be preserved"
820 );
821 }
822}