1use async_trait::async_trait;
7use parking_lot::RwLock;
8use std::collections::HashMap;
9use std::io;
10use std::ops::Range;
11use std::path::{Path, PathBuf};
12use std::sync::Arc;
13
14#[derive(Debug, Clone)]
16pub struct FileSlice {
17 data: OwnedBytes,
18 range: Range<u64>,
19}
20
21impl FileSlice {
22 pub fn new(data: OwnedBytes) -> Self {
23 let len = data.len() as u64;
24 Self {
25 data,
26 range: 0..len,
27 }
28 }
29
30 pub fn empty() -> Self {
31 Self {
32 data: OwnedBytes::empty(),
33 range: 0..0,
34 }
35 }
36
37 pub fn slice(&self, range: Range<u64>) -> Self {
38 let start = self.range.start + range.start;
39 let end = self.range.start + range.end;
40 Self {
41 data: self.data.clone(),
42 range: start..end,
43 }
44 }
45
46 pub fn len(&self) -> u64 {
47 self.range.end - self.range.start
48 }
49
50 pub fn is_empty(&self) -> bool {
51 self.range.start == self.range.end
52 }
53
54 pub async fn read_bytes(&self) -> io::Result<OwnedBytes> {
56 Ok(self
57 .data
58 .slice(self.range.start as usize..self.range.end as usize))
59 }
60
61 pub async fn read_bytes_range(&self, range: Range<u64>) -> io::Result<OwnedBytes> {
63 let start = self.range.start + range.start;
64 let end = self.range.start + range.end;
65 if end > self.range.end {
66 return Err(io::Error::new(
67 io::ErrorKind::InvalidInput,
68 "Range out of bounds",
69 ));
70 }
71 Ok(self.data.slice(start as usize..end as usize))
72 }
73}
74
75#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
77#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
78#[cfg(not(target_arch = "wasm32"))]
79pub trait AsyncFileRead: Send + Sync {
80 fn len(&self) -> u64;
82
83 fn is_empty(&self) -> bool {
85 self.len() == 0
86 }
87
88 async fn read_bytes_range(&self, range: Range<u64>) -> io::Result<OwnedBytes>;
90
91 async fn read_bytes(&self) -> io::Result<OwnedBytes> {
93 self.read_bytes_range(0..self.len()).await
94 }
95}
96
97#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
99#[cfg(target_arch = "wasm32")]
100pub trait AsyncFileRead {
101 fn len(&self) -> u64;
103
104 fn is_empty(&self) -> bool {
106 self.len() == 0
107 }
108
109 async fn read_bytes_range(&self, range: Range<u64>) -> io::Result<OwnedBytes>;
111
112 async fn read_bytes(&self) -> io::Result<OwnedBytes> {
114 self.read_bytes_range(0..self.len()).await
115 }
116}
117
118#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
119#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
120impl AsyncFileRead for FileSlice {
121 fn len(&self) -> u64 {
122 self.range.end - self.range.start
123 }
124
125 async fn read_bytes_range(&self, range: Range<u64>) -> io::Result<OwnedBytes> {
126 let start = self.range.start + range.start;
127 let end = self.range.start + range.end;
128 if end > self.range.end {
129 return Err(io::Error::new(
130 io::ErrorKind::InvalidInput,
131 "Range out of bounds",
132 ));
133 }
134 Ok(self.data.slice(start as usize..end as usize))
135 }
136}
137
138#[cfg(not(target_arch = "wasm32"))]
140pub type RangeReadFn = Arc<
141 dyn Fn(
142 Range<u64>,
143 )
144 -> std::pin::Pin<Box<dyn std::future::Future<Output = io::Result<OwnedBytes>> + Send>>
145 + Send
146 + Sync,
147>;
148
149#[cfg(target_arch = "wasm32")]
150pub type RangeReadFn = Arc<
151 dyn Fn(
152 Range<u64>,
153 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = io::Result<OwnedBytes>>>>,
154>;
155
156pub struct LazyFileHandle {
159 file_size: u64,
161 read_fn: RangeReadFn,
163}
164
165impl std::fmt::Debug for LazyFileHandle {
166 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
167 f.debug_struct("LazyFileHandle")
168 .field("file_size", &self.file_size)
169 .finish()
170 }
171}
172
173impl Clone for LazyFileHandle {
174 fn clone(&self) -> Self {
175 Self {
176 file_size: self.file_size,
177 read_fn: Arc::clone(&self.read_fn),
178 }
179 }
180}
181
182impl LazyFileHandle {
183 pub fn new(file_size: u64, read_fn: RangeReadFn) -> Self {
185 Self { file_size, read_fn }
186 }
187
188 pub fn file_size(&self) -> u64 {
190 self.file_size
191 }
192
193 pub fn slice(&self, range: Range<u64>) -> LazyFileSlice {
195 LazyFileSlice {
196 handle: self.clone(),
197 offset: range.start,
198 len: range.end - range.start,
199 }
200 }
201}
202
203#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
204#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
205impl AsyncFileRead for LazyFileHandle {
206 fn len(&self) -> u64 {
207 self.file_size
208 }
209
210 async fn read_bytes_range(&self, range: Range<u64>) -> io::Result<OwnedBytes> {
211 if range.end > self.file_size {
212 return Err(io::Error::new(
213 io::ErrorKind::InvalidInput,
214 format!(
215 "Range {:?} out of bounds (file size: {})",
216 range, self.file_size
217 ),
218 ));
219 }
220 (self.read_fn)(range).await
221 }
222}
223
224#[derive(Clone)]
226pub struct LazyFileSlice {
227 handle: LazyFileHandle,
228 offset: u64,
229 len: u64,
230}
231
232impl std::fmt::Debug for LazyFileSlice {
233 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
234 f.debug_struct("LazyFileSlice")
235 .field("offset", &self.offset)
236 .field("len", &self.len)
237 .finish()
238 }
239}
240
241impl LazyFileSlice {
242 pub fn slice(&self, range: Range<u64>) -> Self {
244 Self {
245 handle: self.handle.clone(),
246 offset: self.offset + range.start,
247 len: range.end - range.start,
248 }
249 }
250}
251
252#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
253#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
254impl AsyncFileRead for LazyFileSlice {
255 fn len(&self) -> u64 {
256 self.len
257 }
258
259 async fn read_bytes_range(&self, range: Range<u64>) -> io::Result<OwnedBytes> {
260 if range.end > self.len {
261 return Err(io::Error::new(
262 io::ErrorKind::InvalidInput,
263 format!("Range {:?} out of bounds (slice len: {})", range, self.len),
264 ));
265 }
266 let abs_start = self.offset + range.start;
267 let abs_end = self.offset + range.end;
268 self.handle.read_bytes_range(abs_start..abs_end).await
269 }
270}
271
272#[derive(Debug, Clone)]
274pub struct OwnedBytes {
275 data: Arc<Vec<u8>>,
276 range: Range<usize>,
277}
278
279impl OwnedBytes {
280 pub fn new(data: Vec<u8>) -> Self {
281 let len = data.len();
282 Self {
283 data: Arc::new(data),
284 range: 0..len,
285 }
286 }
287
288 pub fn empty() -> Self {
289 Self {
290 data: Arc::new(Vec::new()),
291 range: 0..0,
292 }
293 }
294
295 pub fn len(&self) -> usize {
296 self.range.len()
297 }
298
299 pub fn is_empty(&self) -> bool {
300 self.range.is_empty()
301 }
302
303 pub fn slice(&self, range: Range<usize>) -> Self {
304 let start = self.range.start + range.start;
305 let end = self.range.start + range.end;
306 Self {
307 data: Arc::clone(&self.data),
308 range: start..end,
309 }
310 }
311
312 pub fn as_slice(&self) -> &[u8] {
313 &self.data[self.range.clone()]
314 }
315
316 pub fn to_vec(&self) -> Vec<u8> {
317 self.as_slice().to_vec()
318 }
319}
320
321impl AsRef<[u8]> for OwnedBytes {
322 fn as_ref(&self) -> &[u8] {
323 self.as_slice()
324 }
325}
326
327impl std::ops::Deref for OwnedBytes {
328 type Target = [u8];
329
330 fn deref(&self) -> &Self::Target {
331 self.as_slice()
332 }
333}
334
335#[cfg(not(target_arch = "wasm32"))]
337#[async_trait]
338pub trait Directory: Send + Sync + 'static {
339 async fn exists(&self, path: &Path) -> io::Result<bool>;
341
342 async fn file_size(&self, path: &Path) -> io::Result<u64>;
344
345 async fn open_read(&self, path: &Path) -> io::Result<FileSlice>;
347
348 async fn read_range(&self, path: &Path, range: Range<u64>) -> io::Result<OwnedBytes>;
350
351 async fn list_files(&self, prefix: &Path) -> io::Result<Vec<PathBuf>>;
353
354 async fn open_lazy(&self, path: &Path) -> io::Result<LazyFileHandle>;
357}
358
359#[cfg(target_arch = "wasm32")]
361#[async_trait(?Send)]
362pub trait Directory: 'static {
363 async fn exists(&self, path: &Path) -> io::Result<bool>;
365
366 async fn file_size(&self, path: &Path) -> io::Result<u64>;
368
369 async fn open_read(&self, path: &Path) -> io::Result<FileSlice>;
371
372 async fn read_range(&self, path: &Path, range: Range<u64>) -> io::Result<OwnedBytes>;
374
375 async fn list_files(&self, prefix: &Path) -> io::Result<Vec<PathBuf>>;
377
378 async fn open_lazy(&self, path: &Path) -> io::Result<LazyFileHandle>;
381}
382
383#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
385#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
386pub trait DirectoryWriter: Directory {
387 async fn write(&self, path: &Path, data: &[u8]) -> io::Result<()>;
389
390 async fn delete(&self, path: &Path) -> io::Result<()>;
392
393 async fn rename(&self, from: &Path, to: &Path) -> io::Result<()>;
395
396 async fn sync(&self) -> io::Result<()>;
398}
399
400#[derive(Debug, Default)]
402pub struct RamDirectory {
403 files: Arc<RwLock<HashMap<PathBuf, Arc<Vec<u8>>>>>,
404}
405
406impl Clone for RamDirectory {
407 fn clone(&self) -> Self {
408 Self {
409 files: Arc::clone(&self.files),
410 }
411 }
412}
413
414impl RamDirectory {
415 pub fn new() -> Self {
416 Self::default()
417 }
418}
419
420#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
421#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
422impl Directory for RamDirectory {
423 async fn exists(&self, path: &Path) -> io::Result<bool> {
424 Ok(self.files.read().contains_key(path))
425 }
426
427 async fn file_size(&self, path: &Path) -> io::Result<u64> {
428 self.files
429 .read()
430 .get(path)
431 .map(|data| data.len() as u64)
432 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found"))
433 }
434
435 async fn open_read(&self, path: &Path) -> io::Result<FileSlice> {
436 let files = self.files.read();
437 let data = files
438 .get(path)
439 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found"))?;
440
441 Ok(FileSlice::new(OwnedBytes {
442 data: Arc::clone(data),
443 range: 0..data.len(),
444 }))
445 }
446
447 async fn read_range(&self, path: &Path, range: Range<u64>) -> io::Result<OwnedBytes> {
448 let files = self.files.read();
449 let data = files
450 .get(path)
451 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found"))?;
452
453 let start = range.start as usize;
454 let end = range.end as usize;
455
456 if end > data.len() {
457 return Err(io::Error::new(
458 io::ErrorKind::InvalidInput,
459 "Range out of bounds",
460 ));
461 }
462
463 Ok(OwnedBytes {
464 data: Arc::clone(data),
465 range: start..end,
466 })
467 }
468
469 async fn list_files(&self, prefix: &Path) -> io::Result<Vec<PathBuf>> {
470 let files = self.files.read();
471 Ok(files
472 .keys()
473 .filter(|p| p.starts_with(prefix))
474 .cloned()
475 .collect())
476 }
477
478 async fn open_lazy(&self, path: &Path) -> io::Result<LazyFileHandle> {
479 let files = Arc::clone(&self.files);
480 let path = path.to_path_buf();
481
482 let file_size = {
483 let files_guard = files.read();
484 files_guard
485 .get(&path)
486 .map(|data| data.len() as u64)
487 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found"))?
488 };
489
490 let read_fn: RangeReadFn = Arc::new(move |range: Range<u64>| {
491 let files = Arc::clone(&files);
492 let path = path.clone();
493 Box::pin(async move {
494 let files_guard = files.read();
495 let data = files_guard
496 .get(&path)
497 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found"))?;
498
499 let start = range.start as usize;
500 let end = range.end as usize;
501 if end > data.len() {
502 return Err(io::Error::new(
503 io::ErrorKind::InvalidInput,
504 "Range out of bounds",
505 ));
506 }
507 Ok(OwnedBytes {
508 data: Arc::clone(data),
509 range: start..end,
510 })
511 })
512 });
513
514 Ok(LazyFileHandle::new(file_size, read_fn))
515 }
516}
517
518#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
519#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
520impl DirectoryWriter for RamDirectory {
521 async fn write(&self, path: &Path, data: &[u8]) -> io::Result<()> {
522 self.files
523 .write()
524 .insert(path.to_path_buf(), Arc::new(data.to_vec()));
525 Ok(())
526 }
527
528 async fn delete(&self, path: &Path) -> io::Result<()> {
529 self.files.write().remove(path);
530 Ok(())
531 }
532
533 async fn rename(&self, from: &Path, to: &Path) -> io::Result<()> {
534 let mut files = self.files.write();
535 if let Some(data) = files.remove(from) {
536 files.insert(to.to_path_buf(), data);
537 }
538 Ok(())
539 }
540
541 async fn sync(&self) -> io::Result<()> {
542 Ok(())
543 }
544}
545
546#[cfg(feature = "native")]
548#[derive(Debug, Clone)]
549pub struct FsDirectory {
550 root: PathBuf,
551}
552
553#[cfg(feature = "native")]
554impl FsDirectory {
555 pub fn new(root: impl AsRef<Path>) -> Self {
556 Self {
557 root: root.as_ref().to_path_buf(),
558 }
559 }
560
561 fn resolve(&self, path: &Path) -> PathBuf {
562 self.root.join(path)
563 }
564}
565
566#[cfg(feature = "native")]
567#[async_trait]
568impl Directory for FsDirectory {
569 async fn exists(&self, path: &Path) -> io::Result<bool> {
570 let full_path = self.resolve(path);
571 Ok(tokio::fs::try_exists(&full_path).await.unwrap_or(false))
572 }
573
574 async fn file_size(&self, path: &Path) -> io::Result<u64> {
575 let full_path = self.resolve(path);
576 let metadata = tokio::fs::metadata(&full_path).await?;
577 Ok(metadata.len())
578 }
579
580 async fn open_read(&self, path: &Path) -> io::Result<FileSlice> {
581 let full_path = self.resolve(path);
582 let data = tokio::fs::read(&full_path).await?;
583 Ok(FileSlice::new(OwnedBytes::new(data)))
584 }
585
586 async fn read_range(&self, path: &Path, range: Range<u64>) -> io::Result<OwnedBytes> {
587 use tokio::io::{AsyncReadExt, AsyncSeekExt};
588
589 let full_path = self.resolve(path);
590 let mut file = tokio::fs::File::open(&full_path).await?;
591
592 file.seek(std::io::SeekFrom::Start(range.start)).await?;
593
594 let len = (range.end - range.start) as usize;
595 let mut buffer = vec![0u8; len];
596 file.read_exact(&mut buffer).await?;
597
598 Ok(OwnedBytes::new(buffer))
599 }
600
601 async fn list_files(&self, prefix: &Path) -> io::Result<Vec<PathBuf>> {
602 let full_path = self.resolve(prefix);
603 let mut entries = tokio::fs::read_dir(&full_path).await?;
604 let mut files = Vec::new();
605
606 while let Some(entry) = entries.next_entry().await? {
607 if entry.file_type().await?.is_file() {
608 files.push(entry.path().strip_prefix(&self.root).unwrap().to_path_buf());
609 }
610 }
611
612 Ok(files)
613 }
614
615 async fn open_lazy(&self, path: &Path) -> io::Result<LazyFileHandle> {
616 let full_path = self.resolve(path);
617 let metadata = tokio::fs::metadata(&full_path).await?;
618 let file_size = metadata.len();
619
620 let read_fn: RangeReadFn = Arc::new(move |range: Range<u64>| {
621 let full_path = full_path.clone();
622 Box::pin(async move {
623 use tokio::io::{AsyncReadExt, AsyncSeekExt};
624
625 let mut file = tokio::fs::File::open(&full_path).await?;
626 file.seek(std::io::SeekFrom::Start(range.start)).await?;
627
628 let len = (range.end - range.start) as usize;
629 let mut buffer = vec![0u8; len];
630 file.read_exact(&mut buffer).await?;
631
632 Ok(OwnedBytes::new(buffer))
633 })
634 });
635
636 Ok(LazyFileHandle::new(file_size, read_fn))
637 }
638}
639
640#[cfg(feature = "native")]
641#[async_trait]
642impl DirectoryWriter for FsDirectory {
643 async fn write(&self, path: &Path, data: &[u8]) -> io::Result<()> {
644 let full_path = self.resolve(path);
645
646 if let Some(parent) = full_path.parent() {
648 tokio::fs::create_dir_all(parent).await?;
649 }
650
651 tokio::fs::write(&full_path, data).await
652 }
653
654 async fn delete(&self, path: &Path) -> io::Result<()> {
655 let full_path = self.resolve(path);
656 tokio::fs::remove_file(&full_path).await
657 }
658
659 async fn rename(&self, from: &Path, to: &Path) -> io::Result<()> {
660 let from_path = self.resolve(from);
661 let to_path = self.resolve(to);
662 tokio::fs::rename(&from_path, &to_path).await
663 }
664
665 async fn sync(&self) -> io::Result<()> {
666 let dir = std::fs::File::open(&self.root)?;
668 dir.sync_all()?;
669 Ok(())
670 }
671}
672
673pub struct CachingDirectory<D: Directory> {
675 inner: D,
676 cache: RwLock<HashMap<PathBuf, Arc<Vec<u8>>>>,
677 max_cached_bytes: usize,
678 current_bytes: RwLock<usize>,
679}
680
681impl<D: Directory> CachingDirectory<D> {
682 pub fn new(inner: D, max_cached_bytes: usize) -> Self {
683 Self {
684 inner,
685 cache: RwLock::new(HashMap::new()),
686 max_cached_bytes,
687 current_bytes: RwLock::new(0),
688 }
689 }
690
691 fn try_cache(&self, path: &Path, data: &[u8]) {
692 let mut current = self.current_bytes.write();
693 if *current + data.len() <= self.max_cached_bytes {
694 self.cache
695 .write()
696 .insert(path.to_path_buf(), Arc::new(data.to_vec()));
697 *current += data.len();
698 }
699 }
700}
701
702#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
703#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
704impl<D: Directory> Directory for CachingDirectory<D> {
705 async fn exists(&self, path: &Path) -> io::Result<bool> {
706 if self.cache.read().contains_key(path) {
707 return Ok(true);
708 }
709 self.inner.exists(path).await
710 }
711
712 async fn file_size(&self, path: &Path) -> io::Result<u64> {
713 if let Some(data) = self.cache.read().get(path) {
714 return Ok(data.len() as u64);
715 }
716 self.inner.file_size(path).await
717 }
718
719 async fn open_read(&self, path: &Path) -> io::Result<FileSlice> {
720 if let Some(data) = self.cache.read().get(path) {
722 return Ok(FileSlice::new(OwnedBytes {
723 data: Arc::clone(data),
724 range: 0..data.len(),
725 }));
726 }
727
728 let slice = self.inner.open_read(path).await?;
730 let bytes = slice.read_bytes().await?;
731
732 self.try_cache(path, bytes.as_slice());
733
734 Ok(FileSlice::new(bytes))
735 }
736
737 async fn read_range(&self, path: &Path, range: Range<u64>) -> io::Result<OwnedBytes> {
738 if let Some(data) = self.cache.read().get(path) {
740 let start = range.start as usize;
741 let end = range.end as usize;
742 return Ok(OwnedBytes {
743 data: Arc::clone(data),
744 range: start..end,
745 });
746 }
747
748 self.inner.read_range(path, range).await
749 }
750
751 async fn list_files(&self, prefix: &Path) -> io::Result<Vec<PathBuf>> {
752 self.inner.list_files(prefix).await
753 }
754
755 async fn open_lazy(&self, path: &Path) -> io::Result<LazyFileHandle> {
756 self.inner.open_lazy(path).await
758 }
759}
760
761#[cfg(test)]
762mod tests {
763 use super::*;
764
765 #[tokio::test]
766 async fn test_ram_directory() {
767 let dir = RamDirectory::new();
768
769 dir.write(Path::new("test.txt"), b"hello world")
771 .await
772 .unwrap();
773
774 assert!(dir.exists(Path::new("test.txt")).await.unwrap());
776 assert!(!dir.exists(Path::new("nonexistent.txt")).await.unwrap());
777
778 let slice = dir.open_read(Path::new("test.txt")).await.unwrap();
780 let data = slice.read_bytes().await.unwrap();
781 assert_eq!(data.as_slice(), b"hello world");
782
783 let range_data = dir.read_range(Path::new("test.txt"), 0..5).await.unwrap();
785 assert_eq!(range_data.as_slice(), b"hello");
786
787 dir.delete(Path::new("test.txt")).await.unwrap();
789 assert!(!dir.exists(Path::new("test.txt")).await.unwrap());
790 }
791
792 #[tokio::test]
793 async fn test_file_slice() {
794 let data = OwnedBytes::new(b"hello world".to_vec());
795 let slice = FileSlice::new(data);
796
797 assert_eq!(slice.len(), 11);
798
799 let sub_slice = slice.slice(0..5);
800 let bytes = sub_slice.read_bytes().await.unwrap();
801 assert_eq!(bytes.as_slice(), b"hello");
802
803 let sub_slice2 = slice.slice(6..11);
804 let bytes2 = sub_slice2.read_bytes().await.unwrap();
805 assert_eq!(bytes2.as_slice(), b"world");
806 }
807
808 #[tokio::test]
809 async fn test_owned_bytes() {
810 let bytes = OwnedBytes::new(vec![1, 2, 3, 4, 5]);
811
812 assert_eq!(bytes.len(), 5);
813 assert_eq!(bytes.as_slice(), &[1, 2, 3, 4, 5]);
814
815 let sliced = bytes.slice(1..4);
816 assert_eq!(sliced.as_slice(), &[2, 3, 4]);
817
818 assert_eq!(bytes.as_slice(), &[1, 2, 3, 4, 5]);
820 }
821}