1use async_trait::async_trait;
7use parking_lot::RwLock;
8use std::collections::HashMap;
9use std::io;
10use std::ops::Range;
11use std::path::{Path, PathBuf};
12use std::sync::Arc;
13
14#[derive(Debug, Clone)]
16pub struct FileSlice {
17 data: OwnedBytes,
18 range: Range<usize>,
19}
20
21impl FileSlice {
22 pub fn new(data: OwnedBytes) -> Self {
23 let len = data.len();
24 Self {
25 data,
26 range: 0..len,
27 }
28 }
29
30 pub fn empty() -> Self {
31 Self {
32 data: OwnedBytes::empty(),
33 range: 0..0,
34 }
35 }
36
37 pub fn slice(&self, range: Range<usize>) -> Self {
38 let start = self.range.start + range.start;
39 let end = self.range.start + range.end;
40 Self {
41 data: self.data.clone(),
42 range: start..end,
43 }
44 }
45
46 pub fn len(&self) -> usize {
47 self.range.len()
48 }
49
50 pub fn is_empty(&self) -> bool {
51 self.range.is_empty()
52 }
53
54 pub async fn read_bytes(&self) -> io::Result<OwnedBytes> {
56 Ok(self.data.slice(self.range.clone()))
57 }
58
59 pub async fn read_bytes_range(&self, range: Range<usize>) -> io::Result<OwnedBytes> {
61 let start = self.range.start + range.start;
62 let end = self.range.start + range.end;
63 if end > self.range.end {
64 return Err(io::Error::new(
65 io::ErrorKind::InvalidInput,
66 "Range out of bounds",
67 ));
68 }
69 Ok(self.data.slice(start..end))
70 }
71}
72
73#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
75#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
76#[cfg(not(target_arch = "wasm32"))]
77pub trait AsyncFileRead: Send + Sync {
78 fn len(&self) -> usize;
80
81 fn is_empty(&self) -> bool {
83 self.len() == 0
84 }
85
86 async fn read_bytes_range(&self, range: Range<usize>) -> io::Result<OwnedBytes>;
88
89 async fn read_bytes(&self) -> io::Result<OwnedBytes> {
91 self.read_bytes_range(0..self.len()).await
92 }
93}
94
95#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
97#[cfg(target_arch = "wasm32")]
98pub trait AsyncFileRead {
99 fn len(&self) -> usize;
101
102 fn is_empty(&self) -> bool {
104 self.len() == 0
105 }
106
107 async fn read_bytes_range(&self, range: Range<usize>) -> io::Result<OwnedBytes>;
109
110 async fn read_bytes(&self) -> io::Result<OwnedBytes> {
112 self.read_bytes_range(0..self.len()).await
113 }
114}
115
116#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
117#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
118impl AsyncFileRead for FileSlice {
119 fn len(&self) -> usize {
120 self.range.len()
121 }
122
123 async fn read_bytes_range(&self, range: Range<usize>) -> io::Result<OwnedBytes> {
124 let start = self.range.start + range.start;
125 let end = self.range.start + range.end;
126 if end > self.range.end {
127 return Err(io::Error::new(
128 io::ErrorKind::InvalidInput,
129 "Range out of bounds",
130 ));
131 }
132 Ok(self.data.slice(start..end))
133 }
134}
135
136#[cfg(not(target_arch = "wasm32"))]
138pub type RangeReadFn = Arc<
139 dyn Fn(
140 Range<u64>,
141 )
142 -> std::pin::Pin<Box<dyn std::future::Future<Output = io::Result<OwnedBytes>> + Send>>
143 + Send
144 + Sync,
145>;
146
147#[cfg(target_arch = "wasm32")]
148pub type RangeReadFn = Arc<
149 dyn Fn(
150 Range<u64>,
151 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = io::Result<OwnedBytes>>>>,
152>;
153
154pub struct LazyFileHandle {
157 file_size: usize,
159 read_fn: RangeReadFn,
161}
162
163impl std::fmt::Debug for LazyFileHandle {
164 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
165 f.debug_struct("LazyFileHandle")
166 .field("file_size", &self.file_size)
167 .finish()
168 }
169}
170
171impl Clone for LazyFileHandle {
172 fn clone(&self) -> Self {
173 Self {
174 file_size: self.file_size,
175 read_fn: Arc::clone(&self.read_fn),
176 }
177 }
178}
179
180impl LazyFileHandle {
181 pub fn new(file_size: usize, read_fn: RangeReadFn) -> Self {
183 Self { file_size, read_fn }
184 }
185
186 pub fn file_size(&self) -> usize {
188 self.file_size
189 }
190
191 pub fn slice(&self, range: Range<usize>) -> LazyFileSlice {
193 LazyFileSlice {
194 handle: self.clone(),
195 offset: range.start,
196 len: range.end - range.start,
197 }
198 }
199}
200
201#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
202#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
203impl AsyncFileRead for LazyFileHandle {
204 fn len(&self) -> usize {
205 self.file_size
206 }
207
208 async fn read_bytes_range(&self, range: Range<usize>) -> io::Result<OwnedBytes> {
209 if range.end > self.file_size {
210 return Err(io::Error::new(
211 io::ErrorKind::InvalidInput,
212 format!(
213 "Range {:?} out of bounds (file size: {})",
214 range, self.file_size
215 ),
216 ));
217 }
218 (self.read_fn)(range.start as u64..range.end as u64).await
219 }
220}
221
222#[derive(Clone)]
224pub struct LazyFileSlice {
225 handle: LazyFileHandle,
226 offset: usize,
227 len: usize,
228}
229
230impl std::fmt::Debug for LazyFileSlice {
231 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
232 f.debug_struct("LazyFileSlice")
233 .field("offset", &self.offset)
234 .field("len", &self.len)
235 .finish()
236 }
237}
238
239impl LazyFileSlice {
240 pub fn slice(&self, range: Range<usize>) -> Self {
242 Self {
243 handle: self.handle.clone(),
244 offset: self.offset + range.start,
245 len: range.end - range.start,
246 }
247 }
248}
249
250#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
251#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
252impl AsyncFileRead for LazyFileSlice {
253 fn len(&self) -> usize {
254 self.len
255 }
256
257 async fn read_bytes_range(&self, range: Range<usize>) -> io::Result<OwnedBytes> {
258 if range.end > self.len {
259 return Err(io::Error::new(
260 io::ErrorKind::InvalidInput,
261 format!("Range {:?} out of bounds (slice len: {})", range, self.len),
262 ));
263 }
264 let abs_start = self.offset + range.start;
265 let abs_end = self.offset + range.end;
266 self.handle.read_bytes_range(abs_start..abs_end).await
267 }
268}
269
270#[derive(Debug, Clone)]
272pub struct OwnedBytes {
273 data: Arc<Vec<u8>>,
274 range: Range<usize>,
275}
276
277impl OwnedBytes {
278 pub fn new(data: Vec<u8>) -> Self {
279 let len = data.len();
280 Self {
281 data: Arc::new(data),
282 range: 0..len,
283 }
284 }
285
286 pub fn empty() -> Self {
287 Self {
288 data: Arc::new(Vec::new()),
289 range: 0..0,
290 }
291 }
292
293 pub fn len(&self) -> usize {
294 self.range.len()
295 }
296
297 pub fn is_empty(&self) -> bool {
298 self.range.is_empty()
299 }
300
301 pub fn slice(&self, range: Range<usize>) -> Self {
302 let start = self.range.start + range.start;
303 let end = self.range.start + range.end;
304 Self {
305 data: Arc::clone(&self.data),
306 range: start..end,
307 }
308 }
309
310 pub fn as_slice(&self) -> &[u8] {
311 &self.data[self.range.clone()]
312 }
313
314 pub fn to_vec(&self) -> Vec<u8> {
315 self.as_slice().to_vec()
316 }
317}
318
319impl AsRef<[u8]> for OwnedBytes {
320 fn as_ref(&self) -> &[u8] {
321 self.as_slice()
322 }
323}
324
325impl std::ops::Deref for OwnedBytes {
326 type Target = [u8];
327
328 fn deref(&self) -> &Self::Target {
329 self.as_slice()
330 }
331}
332
333#[cfg(not(target_arch = "wasm32"))]
335#[async_trait]
336pub trait Directory: Send + Sync + 'static {
337 async fn exists(&self, path: &Path) -> io::Result<bool>;
339
340 async fn file_size(&self, path: &Path) -> io::Result<u64>;
342
343 async fn open_read(&self, path: &Path) -> io::Result<FileSlice>;
345
346 async fn read_range(&self, path: &Path, range: Range<u64>) -> io::Result<OwnedBytes>;
348
349 async fn list_files(&self, prefix: &Path) -> io::Result<Vec<PathBuf>>;
351
352 async fn open_lazy(&self, path: &Path) -> io::Result<LazyFileHandle>;
355}
356
357#[cfg(target_arch = "wasm32")]
359#[async_trait(?Send)]
360pub trait Directory: 'static {
361 async fn exists(&self, path: &Path) -> io::Result<bool>;
363
364 async fn file_size(&self, path: &Path) -> io::Result<u64>;
366
367 async fn open_read(&self, path: &Path) -> io::Result<FileSlice>;
369
370 async fn read_range(&self, path: &Path, range: Range<u64>) -> io::Result<OwnedBytes>;
372
373 async fn list_files(&self, prefix: &Path) -> io::Result<Vec<PathBuf>>;
375
376 async fn open_lazy(&self, path: &Path) -> io::Result<LazyFileHandle>;
379}
380
381#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
383#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
384pub trait DirectoryWriter: Directory {
385 async fn write(&self, path: &Path, data: &[u8]) -> io::Result<()>;
387
388 async fn delete(&self, path: &Path) -> io::Result<()>;
390
391 async fn rename(&self, from: &Path, to: &Path) -> io::Result<()>;
393
394 async fn sync(&self) -> io::Result<()>;
396}
397
398#[derive(Debug, Default)]
400pub struct RamDirectory {
401 files: Arc<RwLock<HashMap<PathBuf, Arc<Vec<u8>>>>>,
402}
403
404impl Clone for RamDirectory {
405 fn clone(&self) -> Self {
406 Self {
407 files: Arc::clone(&self.files),
408 }
409 }
410}
411
412impl RamDirectory {
413 pub fn new() -> Self {
414 Self::default()
415 }
416}
417
418#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
419#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
420impl Directory for RamDirectory {
421 async fn exists(&self, path: &Path) -> io::Result<bool> {
422 Ok(self.files.read().contains_key(path))
423 }
424
425 async fn file_size(&self, path: &Path) -> io::Result<u64> {
426 self.files
427 .read()
428 .get(path)
429 .map(|data| data.len() as u64)
430 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found"))
431 }
432
433 async fn open_read(&self, path: &Path) -> io::Result<FileSlice> {
434 let files = self.files.read();
435 let data = files
436 .get(path)
437 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found"))?;
438
439 Ok(FileSlice::new(OwnedBytes {
440 data: Arc::clone(data),
441 range: 0..data.len(),
442 }))
443 }
444
445 async fn read_range(&self, path: &Path, range: Range<u64>) -> io::Result<OwnedBytes> {
446 let files = self.files.read();
447 let data = files
448 .get(path)
449 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found"))?;
450
451 let start = range.start as usize;
452 let end = range.end as usize;
453
454 if end > data.len() {
455 return Err(io::Error::new(
456 io::ErrorKind::InvalidInput,
457 "Range out of bounds",
458 ));
459 }
460
461 Ok(OwnedBytes {
462 data: Arc::clone(data),
463 range: start..end,
464 })
465 }
466
467 async fn list_files(&self, prefix: &Path) -> io::Result<Vec<PathBuf>> {
468 let files = self.files.read();
469 Ok(files
470 .keys()
471 .filter(|p| p.starts_with(prefix))
472 .cloned()
473 .collect())
474 }
475
476 async fn open_lazy(&self, path: &Path) -> io::Result<LazyFileHandle> {
477 let files = Arc::clone(&self.files);
478 let path = path.to_path_buf();
479
480 let file_size = {
481 let files_guard = files.read();
482 files_guard
483 .get(&path)
484 .map(|data| data.len())
485 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found"))?
486 };
487
488 let read_fn: RangeReadFn = Arc::new(move |range: Range<u64>| {
489 let files = Arc::clone(&files);
490 let path = path.clone();
491 Box::pin(async move {
492 let files_guard = files.read();
493 let data = files_guard
494 .get(&path)
495 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found"))?;
496
497 let start = range.start as usize;
498 let end = range.end as usize;
499 if end > data.len() {
500 return Err(io::Error::new(
501 io::ErrorKind::InvalidInput,
502 "Range out of bounds",
503 ));
504 }
505 Ok(OwnedBytes {
506 data: Arc::clone(data),
507 range: start..end,
508 })
509 })
510 });
511
512 Ok(LazyFileHandle::new(file_size, read_fn))
513 }
514}
515
516#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
517#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
518impl DirectoryWriter for RamDirectory {
519 async fn write(&self, path: &Path, data: &[u8]) -> io::Result<()> {
520 self.files
521 .write()
522 .insert(path.to_path_buf(), Arc::new(data.to_vec()));
523 Ok(())
524 }
525
526 async fn delete(&self, path: &Path) -> io::Result<()> {
527 self.files.write().remove(path);
528 Ok(())
529 }
530
531 async fn rename(&self, from: &Path, to: &Path) -> io::Result<()> {
532 let mut files = self.files.write();
533 if let Some(data) = files.remove(from) {
534 files.insert(to.to_path_buf(), data);
535 }
536 Ok(())
537 }
538
539 async fn sync(&self) -> io::Result<()> {
540 Ok(())
541 }
542}
543
544#[cfg(feature = "native")]
546#[derive(Debug, Clone)]
547pub struct FsDirectory {
548 root: PathBuf,
549}
550
551#[cfg(feature = "native")]
552impl FsDirectory {
553 pub fn new(root: impl AsRef<Path>) -> Self {
554 Self {
555 root: root.as_ref().to_path_buf(),
556 }
557 }
558
559 fn resolve(&self, path: &Path) -> PathBuf {
560 self.root.join(path)
561 }
562}
563
564#[cfg(feature = "native")]
565#[async_trait]
566impl Directory for FsDirectory {
567 async fn exists(&self, path: &Path) -> io::Result<bool> {
568 let full_path = self.resolve(path);
569 Ok(tokio::fs::try_exists(&full_path).await.unwrap_or(false))
570 }
571
572 async fn file_size(&self, path: &Path) -> io::Result<u64> {
573 let full_path = self.resolve(path);
574 let metadata = tokio::fs::metadata(&full_path).await?;
575 Ok(metadata.len())
576 }
577
578 async fn open_read(&self, path: &Path) -> io::Result<FileSlice> {
579 let full_path = self.resolve(path);
580 let data = tokio::fs::read(&full_path).await?;
581 Ok(FileSlice::new(OwnedBytes::new(data)))
582 }
583
584 async fn read_range(&self, path: &Path, range: Range<u64>) -> io::Result<OwnedBytes> {
585 use tokio::io::{AsyncReadExt, AsyncSeekExt};
586
587 let full_path = self.resolve(path);
588 let mut file = tokio::fs::File::open(&full_path).await?;
589
590 file.seek(std::io::SeekFrom::Start(range.start)).await?;
591
592 let len = (range.end - range.start) as usize;
593 let mut buffer = vec![0u8; len];
594 file.read_exact(&mut buffer).await?;
595
596 Ok(OwnedBytes::new(buffer))
597 }
598
599 async fn list_files(&self, prefix: &Path) -> io::Result<Vec<PathBuf>> {
600 let full_path = self.resolve(prefix);
601 let mut entries = tokio::fs::read_dir(&full_path).await?;
602 let mut files = Vec::new();
603
604 while let Some(entry) = entries.next_entry().await? {
605 if entry.file_type().await?.is_file() {
606 files.push(entry.path().strip_prefix(&self.root).unwrap().to_path_buf());
607 }
608 }
609
610 Ok(files)
611 }
612
613 async fn open_lazy(&self, path: &Path) -> io::Result<LazyFileHandle> {
614 let full_path = self.resolve(path);
615 let metadata = tokio::fs::metadata(&full_path).await?;
616 let file_size = metadata.len() as usize;
617
618 let read_fn: RangeReadFn = Arc::new(move |range: Range<u64>| {
619 let full_path = full_path.clone();
620 Box::pin(async move {
621 use tokio::io::{AsyncReadExt, AsyncSeekExt};
622
623 let mut file = tokio::fs::File::open(&full_path).await?;
624 file.seek(std::io::SeekFrom::Start(range.start)).await?;
625
626 let len = (range.end - range.start) as usize;
627 let mut buffer = vec![0u8; len];
628 file.read_exact(&mut buffer).await?;
629
630 Ok(OwnedBytes::new(buffer))
631 })
632 });
633
634 Ok(LazyFileHandle::new(file_size, read_fn))
635 }
636}
637
638#[cfg(feature = "native")]
639#[async_trait]
640impl DirectoryWriter for FsDirectory {
641 async fn write(&self, path: &Path, data: &[u8]) -> io::Result<()> {
642 let full_path = self.resolve(path);
643
644 if let Some(parent) = full_path.parent() {
646 tokio::fs::create_dir_all(parent).await?;
647 }
648
649 tokio::fs::write(&full_path, data).await
650 }
651
652 async fn delete(&self, path: &Path) -> io::Result<()> {
653 let full_path = self.resolve(path);
654 tokio::fs::remove_file(&full_path).await
655 }
656
657 async fn rename(&self, from: &Path, to: &Path) -> io::Result<()> {
658 let from_path = self.resolve(from);
659 let to_path = self.resolve(to);
660 tokio::fs::rename(&from_path, &to_path).await
661 }
662
663 async fn sync(&self) -> io::Result<()> {
664 let dir = std::fs::File::open(&self.root)?;
666 dir.sync_all()?;
667 Ok(())
668 }
669}
670
671pub struct CachingDirectory<D: Directory> {
673 inner: D,
674 cache: RwLock<HashMap<PathBuf, Arc<Vec<u8>>>>,
675 max_cached_bytes: usize,
676 current_bytes: RwLock<usize>,
677}
678
679impl<D: Directory> CachingDirectory<D> {
680 pub fn new(inner: D, max_cached_bytes: usize) -> Self {
681 Self {
682 inner,
683 cache: RwLock::new(HashMap::new()),
684 max_cached_bytes,
685 current_bytes: RwLock::new(0),
686 }
687 }
688
689 fn try_cache(&self, path: &Path, data: &[u8]) {
690 let mut current = self.current_bytes.write();
691 if *current + data.len() <= self.max_cached_bytes {
692 self.cache
693 .write()
694 .insert(path.to_path_buf(), Arc::new(data.to_vec()));
695 *current += data.len();
696 }
697 }
698}
699
700#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
701#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
702impl<D: Directory> Directory for CachingDirectory<D> {
703 async fn exists(&self, path: &Path) -> io::Result<bool> {
704 if self.cache.read().contains_key(path) {
705 return Ok(true);
706 }
707 self.inner.exists(path).await
708 }
709
710 async fn file_size(&self, path: &Path) -> io::Result<u64> {
711 if let Some(data) = self.cache.read().get(path) {
712 return Ok(data.len() as u64);
713 }
714 self.inner.file_size(path).await
715 }
716
717 async fn open_read(&self, path: &Path) -> io::Result<FileSlice> {
718 if let Some(data) = self.cache.read().get(path) {
720 return Ok(FileSlice::new(OwnedBytes {
721 data: Arc::clone(data),
722 range: 0..data.len(),
723 }));
724 }
725
726 let slice = self.inner.open_read(path).await?;
728 let bytes = slice.read_bytes().await?;
729
730 self.try_cache(path, bytes.as_slice());
731
732 Ok(FileSlice::new(bytes))
733 }
734
735 async fn read_range(&self, path: &Path, range: Range<u64>) -> io::Result<OwnedBytes> {
736 if let Some(data) = self.cache.read().get(path) {
738 let start = range.start as usize;
739 let end = range.end as usize;
740 return Ok(OwnedBytes {
741 data: Arc::clone(data),
742 range: start..end,
743 });
744 }
745
746 self.inner.read_range(path, range).await
747 }
748
749 async fn list_files(&self, prefix: &Path) -> io::Result<Vec<PathBuf>> {
750 self.inner.list_files(prefix).await
751 }
752
753 async fn open_lazy(&self, path: &Path) -> io::Result<LazyFileHandle> {
754 self.inner.open_lazy(path).await
756 }
757}
758
759#[cfg(test)]
760mod tests {
761 use super::*;
762
763 #[tokio::test]
764 async fn test_ram_directory() {
765 let dir = RamDirectory::new();
766
767 dir.write(Path::new("test.txt"), b"hello world")
769 .await
770 .unwrap();
771
772 assert!(dir.exists(Path::new("test.txt")).await.unwrap());
774 assert!(!dir.exists(Path::new("nonexistent.txt")).await.unwrap());
775
776 let slice = dir.open_read(Path::new("test.txt")).await.unwrap();
778 let data = slice.read_bytes().await.unwrap();
779 assert_eq!(data.as_slice(), b"hello world");
780
781 let range_data = dir.read_range(Path::new("test.txt"), 0..5).await.unwrap();
783 assert_eq!(range_data.as_slice(), b"hello");
784
785 dir.delete(Path::new("test.txt")).await.unwrap();
787 assert!(!dir.exists(Path::new("test.txt")).await.unwrap());
788 }
789
790 #[tokio::test]
791 async fn test_file_slice() {
792 let data = OwnedBytes::new(b"hello world".to_vec());
793 let slice = FileSlice::new(data);
794
795 assert_eq!(slice.len(), 11);
796
797 let sub_slice = slice.slice(0..5);
798 let bytes = sub_slice.read_bytes().await.unwrap();
799 assert_eq!(bytes.as_slice(), b"hello");
800
801 let sub_slice2 = slice.slice(6..11);
802 let bytes2 = sub_slice2.read_bytes().await.unwrap();
803 assert_eq!(bytes2.as_slice(), b"world");
804 }
805
806 #[tokio::test]
807 async fn test_owned_bytes() {
808 let bytes = OwnedBytes::new(vec![1, 2, 3, 4, 5]);
809
810 assert_eq!(bytes.len(), 5);
811 assert_eq!(bytes.as_slice(), &[1, 2, 3, 4, 5]);
812
813 let sliced = bytes.slice(1..4);
814 assert_eq!(sliced.as_slice(), &[2, 3, 4]);
815
816 assert_eq!(bytes.as_slice(), &[1, 2, 3, 4, 5]);
818 }
819}