Skip to main content

openreadout_core/
source.rs

1//! Where a reader's bytes come from: a local file, a buffer in memory, or a callback the host
2//! provides (a Python file-like object, a browser `File`, an S3 or HTTP range reader written by
3//! the host). The binary itself never opens a network connection (rule 5): remote sources are
4//! implemented by the program embedding the library and handed in as a [`CallbackSource`].
5//!
6//! Two traits:
7//!
8//! - [`ByteSource`] is one file's bytes: its size and positional reads. Implementations are
9//!   `Send + Sync` and read through `&self`, so several threads (and several dataset handles)
10//!   can share one source.
11//! - [`DirSource`] is a namespace of files addressed by path: open a file, list a directory,
12//!   ask whether a path exists and how large it is. Readers use it for the file they were asked
13//!   to open, for sibling files (a CZI's following parts, an OME-TIFF companion, an imzML's
14//!   `.ibd`) and for directory datasets (a Bruker `.d`).
15//!
16//! Readers do not use the traits directly. They get an [`Input`] (a path plus an [`Fs`], the
17//! cheap-to-clone handle on a `DirSource`) and use [`Fs`] like `std::fs`, and [`SourceFile`]
18//! like `std::fs::File` (it implements `Read` and `Seek`, and has `metadata().len()`).
19//!
20//! Local files are read with positional reads (`pread` on Unix, `seek_read` on Windows): one
21//! system call per read and no shared file position.
22//! Memory mapping would need `unsafe`, which the workspace forbids.
23//!
24//! ```
25//! use std::io::{Read, Seek, SeekFrom};
26//! use openreadout_core::source::Input;
27//!
28//! let input = Input::from_bytes("sample.bin", b"0123456789".to_vec());
29//! let mut f = input.open().unwrap();
30//! assert_eq!(f.metadata().unwrap().len(), 10);
31//! f.seek(SeekFrom::Start(4)).unwrap();
32//! let mut buf = [0u8; 3];
33//! f.read_exact(&mut buf).unwrap();
34//! assert_eq!(&buf, b"456");
35//! assert!(!input.is_local());
36//! ```
37
38use std::collections::{BTreeMap, HashMap, VecDeque};
39use std::ffi::OsString;
40use std::fmt;
41use std::fs::File;
42use std::io::{self, Read, Seek, SeekFrom};
43use std::ops::{Deref, Range};
44use std::path::{Component, Path, PathBuf};
45use std::sync::{Arc, Mutex, PoisonError};
46use std::time::SystemTime;
47
48use crate::error::{Error, Result};
49
50// ---------------------------------------------------------------------------------------------
51// ByteSource
52// ---------------------------------------------------------------------------------------------
53
54/// Random-access bytes of one file.
55///
56/// Only [`size`](ByteSource::size), [`read_at`](ByteSource::read_at) and
57/// [`name`](ByteSource::name) are required. `size` is asked again whenever a reader wants the
58/// length, so a source over a file that is still being written reports its current size.
59pub trait ByteSource: Send + Sync + fmt::Debug {
60    /// Current length in bytes.
61    fn size(&self) -> io::Result<u64>;
62
63    /// Read up to `buf.len()` bytes at `offset`; return how many were read (0 at or past the
64    /// end). Like `std::io::Read::read`, a short read is allowed anywhere.
65    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize>;
66
67    /// A name for messages: the path of a local file, the name a host gave its buffer.
68    fn name(&self) -> String;
69
70    /// Fill `buf` from `offset`, or fail with `UnexpectedEof` if the source ends first.
71    fn read_exact_at(&self, mut offset: u64, mut buf: &mut [u8]) -> io::Result<()> {
72        while !buf.is_empty() {
73            match self.read_at(offset, buf) {
74                Ok(0) => {
75                    return Err(io::Error::new(
76                        io::ErrorKind::UnexpectedEof,
77                        "source ends inside the read",
78                    ));
79                }
80                Ok(n) => {
81                    buf = &mut buf[n..];
82                    offset = offset.saturating_add(n as u64);
83                }
84                Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
85                Err(e) => return Err(e),
86            }
87        }
88        Ok(())
89    }
90
91    /// Up to `len` bytes at `offset` (fewer at the end of the source). In-memory sources return
92    /// a view of their buffer without copying; the default reads into a new buffer.
93    fn read_range(&self, offset: u64, len: usize) -> io::Result<SharedBytes> {
94        let size = self.size()?;
95        let n = usize::try_from(size.saturating_sub(offset))
96            .unwrap_or(usize::MAX)
97            .min(len);
98        let mut buf = vec![0u8; n];
99        self.read_exact_at(offset, &mut buf)?;
100        Ok(SharedBytes::from(buf))
101    }
102
103    /// The file on the local disk this source reads, if it is one.
104    fn local_path(&self) -> Option<&Path> {
105        None
106    }
107}
108
109/// Bytes shared without copying: a range of a reference-counted buffer. Dereferences to
110/// `[u8]`.
111#[derive(Clone)]
112pub struct SharedBytes {
113    buf: Arc<dyn AsRef<[u8]> + Send + Sync>,
114    range: Range<usize>,
115}
116
117impl SharedBytes {
118    /// A view of `range` of `buf`; the range is clamped to the buffer.
119    pub fn new(buf: Arc<dyn AsRef<[u8]> + Send + Sync>, range: Range<usize>) -> Self {
120        let len = (*buf).as_ref().len();
121        let end = range.end.min(len);
122        let start = range.start.min(end);
123        Self {
124            buf,
125            range: start..end,
126        }
127    }
128
129    /// Copy into an owned vector.
130    pub fn to_vec(&self) -> Vec<u8> {
131        self.deref().to_vec()
132    }
133}
134
135impl Deref for SharedBytes {
136    type Target = [u8];
137    fn deref(&self) -> &[u8] {
138        &(*self.buf).as_ref()[self.range.clone()]
139    }
140}
141
142impl AsRef<[u8]> for SharedBytes {
143    fn as_ref(&self) -> &[u8] {
144        self
145    }
146}
147
148impl From<Vec<u8>> for SharedBytes {
149    fn from(v: Vec<u8>) -> Self {
150        let n = v.len();
151        Self::new(Arc::new(v), 0..n)
152    }
153}
154
155impl fmt::Debug for SharedBytes {
156    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
157        f.debug_struct("SharedBytes")
158            .field("len", &self.range.len())
159            .finish_non_exhaustive()
160    }
161}
162
163/// A file on the local disk, read with positional reads.
164#[derive(Debug)]
165pub struct LocalFile {
166    #[cfg(any(unix, windows))]
167    file: File,
168    #[cfg(not(any(unix, windows)))]
169    file: Mutex<File>,
170    path: PathBuf,
171}
172
173impl LocalFile {
174    /// Open `path` for reading.
175    pub fn open(path: &Path) -> io::Result<Self> {
176        let file = File::open(path)?;
177        Ok(Self::from_file(file, path))
178    }
179
180    /// Wrap a file already open for reading; `path` names it in messages.
181    pub fn from_file(file: File, path: &Path) -> Self {
182        Self {
183            #[cfg(any(unix, windows))]
184            file,
185            #[cfg(not(any(unix, windows)))]
186            file: Mutex::new(file),
187            path: path.to_path_buf(),
188        }
189    }
190}
191
192impl ByteSource for LocalFile {
193    fn size(&self) -> io::Result<u64> {
194        #[cfg(any(unix, windows))]
195        let m = self.file.metadata()?;
196        #[cfg(not(any(unix, windows)))]
197        let m = self
198            .file
199            .lock()
200            .unwrap_or_else(PoisonError::into_inner)
201            .metadata()?;
202        Ok(m.len())
203    }
204
205    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
206        #[cfg(unix)]
207        {
208            std::os::unix::fs::FileExt::read_at(&self.file, buf, offset)
209        }
210        #[cfg(windows)]
211        {
212            std::os::windows::fs::FileExt::seek_read(&self.file, buf, offset)
213        }
214        #[cfg(not(any(unix, windows)))]
215        {
216            let mut f = self.file.lock().unwrap_or_else(PoisonError::into_inner);
217            f.seek(SeekFrom::Start(offset))?;
218            f.read(buf)
219        }
220    }
221
222    fn name(&self) -> String {
223        self.path.display().to_string()
224    }
225
226    fn local_path(&self) -> Option<&Path> {
227        Some(&self.path)
228    }
229}
230
231/// Bytes held in memory (a `Vec<u8>`, a host buffer that dereferences to bytes).
232#[derive(Clone)]
233pub struct MemSource {
234    data: Arc<dyn AsRef<[u8]> + Send + Sync>,
235    name: String,
236}
237
238impl MemSource {
239    /// Serve `data`, named `name` in messages.
240    pub fn new(name: impl Into<String>, data: impl AsRef<[u8]> + Send + Sync + 'static) -> Self {
241        Self {
242            data: Arc::new(data),
243            name: name.into(),
244        }
245    }
246
247    fn bytes(&self) -> &[u8] {
248        (*self.data).as_ref()
249    }
250}
251
252impl fmt::Debug for MemSource {
253    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
254        f.debug_struct("MemSource")
255            .field("name", &self.name)
256            .field("len", &self.bytes().len())
257            .finish_non_exhaustive()
258    }
259}
260
261impl ByteSource for MemSource {
262    fn size(&self) -> io::Result<u64> {
263        Ok(self.bytes().len() as u64)
264    }
265
266    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
267        let b = self.bytes();
268        let Ok(start) = usize::try_from(offset) else {
269            return Ok(0);
270        };
271        let Some(avail) = b.get(start..) else {
272            return Ok(0);
273        };
274        let n = avail.len().min(buf.len());
275        buf[..n].copy_from_slice(&avail[..n]);
276        Ok(n)
277    }
278
279    fn name(&self) -> String {
280        self.name.clone()
281    }
282
283    fn read_range(&self, offset: u64, len: usize) -> io::Result<SharedBytes> {
284        let size = self.bytes().len();
285        let start = usize::try_from(offset).unwrap_or(usize::MAX).min(size);
286        let end = start.saturating_add(len).min(size);
287        Ok(SharedBytes::new(Arc::clone(&self.data), start..end))
288    }
289}
290
291/// The function a [`CallbackSource`] calls: fill the buffer from the offset, return the number
292/// of bytes read (0 at the end).
293pub type ReadCallback = dyn Fn(u64, &mut [u8]) -> io::Result<usize> + Send + Sync;
294
295/// Bytes the host reads for us: a Python file-like object, a browser `Blob`, an object in S3
296/// or a URL fetched with HTTP range requests, all implemented by the host program. Each call
297/// is a round trip into the host, so wrap slow ones in a [`CachedSource`].
298pub struct CallbackSource {
299    size: u64,
300    name: String,
301    read: Box<ReadCallback>,
302}
303
304impl CallbackSource {
305    /// A source of `size` bytes named `name`, read through `read`.
306    pub fn new(
307        name: impl Into<String>,
308        size: u64,
309        read: impl Fn(u64, &mut [u8]) -> io::Result<usize> + Send + Sync + 'static,
310    ) -> Self {
311        Self {
312            size,
313            name: name.into(),
314            read: Box::new(read),
315        }
316    }
317}
318
319impl fmt::Debug for CallbackSource {
320    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
321        f.debug_struct("CallbackSource")
322            .field("name", &self.name)
323            .field("size", &self.size)
324            .finish_non_exhaustive()
325    }
326}
327
328impl ByteSource for CallbackSource {
329    fn size(&self) -> io::Result<u64> {
330        Ok(self.size)
331    }
332
333    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
334        if offset >= self.size || buf.is_empty() {
335            return Ok(0);
336        }
337        let n = usize::try_from(self.size - offset)
338            .unwrap_or(usize::MAX)
339            .min(buf.len());
340        let got = (self.read)(offset, &mut buf[..n])?;
341        if got > n {
342            return Err(io::Error::other(format!(
343                "the host read returned {got} bytes for a {n}-byte request"
344            )));
345        }
346        Ok(got)
347    }
348
349    fn name(&self) -> String {
350        self.name.clone()
351    }
352}
353
354/// A block cache in front of a slow source (a host callback, a network reader). Readers issue
355/// many small reads for headers; this turns them into a few block-sized reads of the inner
356/// source. Reads of several blocks or more bypass the cache.
357pub struct CachedSource {
358    inner: Arc<dyn ByteSource>,
359    block: u64,
360    capacity: usize,
361    state: Mutex<CacheState>,
362}
363
364#[derive(Default)]
365struct CacheState {
366    blocks: HashMap<u64, Arc<Vec<u8>>>,
367    order: VecDeque<u64>,
368}
369
370impl CachedSource {
371    /// Default block size (64 KiB).
372    pub const DEFAULT_BLOCK: u64 = 64 * 1024;
373    /// Default number of cached blocks (16 MiB with the default block size).
374    pub const DEFAULT_CAPACITY: usize = 256;
375
376    /// Cache `inner` in blocks of `block` bytes, keeping at most `capacity` blocks.
377    pub fn new(inner: Arc<dyn ByteSource>, block: u64, capacity: usize) -> Self {
378        Self {
379            inner,
380            block: block.max(512),
381            capacity: capacity.max(1),
382            state: Mutex::new(CacheState::default()),
383        }
384    }
385
386    /// Cache `inner` with the default block size and capacity.
387    pub fn with_defaults(inner: Arc<dyn ByteSource>) -> Self {
388        Self::new(inner, Self::DEFAULT_BLOCK, Self::DEFAULT_CAPACITY)
389    }
390
391    fn block_at(&self, index: u64) -> io::Result<Arc<Vec<u8>>> {
392        {
393            let st = self.state.lock().unwrap_or_else(PoisonError::into_inner);
394            if let Some(b) = st.blocks.get(&index) {
395                return Ok(Arc::clone(b));
396            }
397        }
398        let start = index.saturating_mul(self.block);
399        let size = self.inner.size()?;
400        let n = usize::try_from(size.saturating_sub(start).min(self.block)).unwrap_or(0);
401        let mut buf = vec![0u8; n];
402        let mut filled = 0;
403        while filled < n {
404            match self
405                .inner
406                .read_at(start + filled as u64, &mut buf[filled..])
407            {
408                Ok(0) => break,
409                Ok(k) => filled += k,
410                Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
411                Err(e) => return Err(e),
412            }
413        }
414        buf.truncate(filled);
415        let b = Arc::new(buf);
416        let mut st = self.state.lock().unwrap_or_else(PoisonError::into_inner);
417        if st.blocks.insert(index, Arc::clone(&b)).is_none() {
418            st.order.push_back(index);
419        }
420        while st.order.len() > self.capacity {
421            if let Some(old) = st.order.pop_front() {
422                st.blocks.remove(&old);
423            }
424        }
425        Ok(b)
426    }
427}
428
429impl fmt::Debug for CachedSource {
430    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
431        f.debug_struct("CachedSource")
432            .field("inner", &self.inner)
433            .field("block", &self.block)
434            .field("capacity", &self.capacity)
435            .finish_non_exhaustive()
436    }
437}
438
439impl ByteSource for CachedSource {
440    fn size(&self) -> io::Result<u64> {
441        self.inner.size()
442    }
443
444    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
445        if buf.len() as u64 >= self.block.saturating_mul(4) {
446            return self.inner.read_at(offset, buf);
447        }
448        let index = offset / self.block;
449        let b = self.block_at(index)?;
450        let within = usize::try_from(offset - index * self.block).unwrap_or(usize::MAX);
451        let Some(avail) = b.get(within..) else {
452            return Ok(0);
453        };
454        let n = avail.len().min(buf.len());
455        buf[..n].copy_from_slice(&avail[..n]);
456        Ok(n)
457    }
458
459    fn name(&self) -> String {
460        self.inner.name()
461    }
462
463    fn local_path(&self) -> Option<&Path> {
464        self.inner.local_path()
465    }
466}
467
468// ---------------------------------------------------------------------------------------------
469// SourceFile: a cursor, used by readers like `std::fs::File`
470// ---------------------------------------------------------------------------------------------
471
472/// A read cursor over a [`ByteSource`], used by readers like `std::fs::File`: it
473/// implements `Read` and `Seek`, and `metadata()?.len()` is the source's current size. Cloning
474/// it shares the source and copies the position.
475#[derive(Clone)]
476pub struct SourceFile {
477    src: Arc<dyn ByteSource>,
478    pos: u64,
479}
480
481impl SourceFile {
482    /// A cursor at the start of `src`.
483    pub fn new(src: Arc<dyn ByteSource>) -> Self {
484        Self { src, pos: 0 }
485    }
486
487    /// Open a local file (shorthand for `SourceFile::new(Arc::new(LocalFile::open(path)?))`).
488    pub fn open_local(path: &Path) -> io::Result<Self> {
489        Ok(Self::new(Arc::new(LocalFile::open(path)?)))
490    }
491
492    /// The size, like `std::fs::File::metadata` (only `len()` and the file-type questions are
493    /// meaningful).
494    pub fn metadata(&self) -> io::Result<EntryMeta> {
495        Ok(EntryMeta::file(self.src.size()?))
496    }
497
498    /// The source's current size.
499    pub fn size(&self) -> io::Result<u64> {
500        self.src.size()
501    }
502
503    /// The source this cursor reads.
504    pub fn source(&self) -> &Arc<dyn ByteSource> {
505        &self.src
506    }
507
508    /// Another cursor on the same source (like `File::try_clone`; never fails).
509    pub fn try_clone(&self) -> io::Result<Self> {
510        Ok(self.clone())
511    }
512
513    /// Positional read that leaves the cursor where it is.
514    pub fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
515        self.src.read_at(offset, buf)
516    }
517
518    /// Positional exact read that leaves the cursor where it is.
519    pub fn read_exact_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
520        self.src.read_exact_at(offset, buf)
521    }
522}
523
524impl fmt::Debug for SourceFile {
525    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
526        f.debug_struct("SourceFile")
527            .field("source", &self.src.name())
528            .field("pos", &self.pos)
529            .finish()
530    }
531}
532
533impl Read for SourceFile {
534    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
535        let n = self.src.read_at(self.pos, buf)?;
536        self.pos = self.pos.saturating_add(n as u64);
537        Ok(n)
538    }
539
540    fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
541        self.src.read_exact_at(self.pos, buf)?;
542        self.pos = self.pos.saturating_add(buf.len() as u64);
543        Ok(())
544    }
545}
546
547impl Seek for SourceFile {
548    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
549        let (base, delta) = match pos {
550            SeekFrom::Start(n) => {
551                self.pos = n;
552                return Ok(n);
553            }
554            SeekFrom::End(d) => (self.src.size()?, d),
555            SeekFrom::Current(d) => (self.pos, d),
556        };
557        let new = base.checked_add_signed(delta).ok_or_else(|| {
558            io::Error::new(
559                io::ErrorKind::InvalidInput,
560                "invalid seek to a negative or overflowing position",
561            )
562        })?;
563        self.pos = new;
564        Ok(new)
565    }
566
567    fn stream_position(&mut self) -> io::Result<u64> {
568        Ok(self.pos)
569    }
570}
571
572// ---------------------------------------------------------------------------------------------
573// DirSource: a namespace of files
574// ---------------------------------------------------------------------------------------------
575
576/// What [`DirSource::metadata`] knows about a path. Mirrors the parts of
577/// `std::fs::Metadata` readers use.
578#[derive(Debug, Clone, Copy, PartialEq, Eq)]
579pub struct EntryMeta {
580    len: u64,
581    dir: bool,
582    modified: Option<SystemTime>,
583}
584
585impl EntryMeta {
586    /// A file of `len` bytes.
587    pub fn file(len: u64) -> Self {
588        Self {
589            len,
590            dir: false,
591            modified: None,
592        }
593    }
594
595    /// A directory.
596    pub fn dir() -> Self {
597        Self {
598            len: 0,
599            dir: true,
600            modified: None,
601        }
602    }
603
604    /// With a modification time.
605    pub fn with_modified(mut self, t: SystemTime) -> Self {
606        self.modified = Some(t);
607        self
608    }
609
610    /// Size in bytes (0 for directories).
611    #[allow(clippy::len_without_is_empty)]
612    pub fn len(&self) -> u64 {
613        self.len
614    }
615
616    /// True for directories.
617    pub fn is_dir(&self) -> bool {
618        self.dir
619    }
620
621    /// True for files.
622    pub fn is_file(&self) -> bool {
623        !self.dir
624    }
625
626    /// Last modification time, when the source knows it.
627    pub fn modified(&self) -> io::Result<SystemTime> {
628        self.modified.ok_or_else(|| {
629            io::Error::new(
630                io::ErrorKind::Unsupported,
631                "this source does not record modification times",
632            )
633        })
634    }
635
636    fn from_std(m: &std::fs::Metadata) -> Self {
637        Self {
638            len: m.len(),
639            dir: m.is_dir(),
640            modified: m.modified().ok(),
641        }
642    }
643}
644
645/// One entry of a directory listing.
646#[derive(Debug)]
647pub struct DirEntry {
648    path: PathBuf,
649    kind: EntryKind,
650}
651
652#[derive(Debug)]
653enum EntryKind {
654    Local(Box<std::fs::DirEntry>),
655    Known(EntryMeta),
656}
657
658impl DirEntry {
659    /// An entry at `path` (the directory joined with the name) with known metadata.
660    pub fn new(path: PathBuf, meta: EntryMeta) -> Self {
661        Self {
662            path,
663            kind: EntryKind::Known(meta),
664        }
665    }
666
667    /// Full path: the listed directory joined with the entry's name.
668    pub fn path(&self) -> PathBuf {
669        self.path.clone()
670    }
671
672    /// The entry's name.
673    pub fn file_name(&self) -> OsString {
674        self.path.file_name().unwrap_or_default().to_os_string()
675    }
676
677    /// Size, type and modification time (follows symbolic links for local entries).
678    pub fn metadata(&self) -> io::Result<EntryMeta> {
679        match &self.kind {
680            EntryKind::Local(e) => {
681                // Like `std::fs::metadata`: a symbolic link reports its target.
682                std::fs::metadata(e.path()).map(|m| EntryMeta::from_std(&m))
683            }
684            EntryKind::Known(m) => Ok(*m),
685        }
686    }
687
688    /// The entry's type without following symbolic links where the platform reports it for
689    /// free (local listings); `is_dir()`/`is_file()` of the result are meaningful.
690    pub fn file_type(&self) -> io::Result<EntryMeta> {
691        match &self.kind {
692            EntryKind::Local(e) => {
693                let t = e.file_type()?;
694                Ok(if t.is_dir() {
695                    EntryMeta::dir()
696                } else {
697                    EntryMeta::file(0)
698                })
699            }
700            EntryKind::Known(m) => Ok(*m),
701        }
702    }
703}
704
705/// Iterator over a directory listing, like `std::fs::ReadDir`.
706#[derive(Debug)]
707pub struct ReadDir(std::vec::IntoIter<io::Result<DirEntry>>);
708
709impl Iterator for ReadDir {
710    type Item = io::Result<DirEntry>;
711    fn next(&mut self) -> Option<Self::Item> {
712        self.0.next()
713    }
714}
715
716/// A namespace of files: the local file system, or a set of files the host supplied.
717/// Paths are the ones the reader builds from its [`Input`]'s path (siblings with
718/// `with_file_name`, members of a directory dataset with `join`).
719pub trait DirSource: Send + Sync + fmt::Debug {
720    /// Open the file at `path`.
721    fn open(&self, path: &Path) -> io::Result<Arc<dyn ByteSource>>;
722
723    /// Size and type of `path` (`NotFound` if it does not exist).
724    fn metadata(&self, path: &Path) -> io::Result<EntryMeta>;
725
726    /// The entries of directory `dir`, in no particular order.
727    fn list(&self, dir: &Path) -> io::Result<Vec<DirEntry>>;
728
729    /// Does `path` exist?
730    fn exists(&self, path: &Path) -> bool {
731        self.metadata(path).is_ok()
732    }
733
734    /// True for the local file system: paths are real paths, and readers that only read local
735    /// files can open them directly.
736    fn is_local(&self) -> bool {
737        false
738    }
739}
740
741/// The local file system (`std::fs`).
742#[derive(Debug, Clone, Copy, Default)]
743pub struct LocalFs;
744
745impl DirSource for LocalFs {
746    fn open(&self, path: &Path) -> io::Result<Arc<dyn ByteSource>> {
747        Ok(Arc::new(LocalFile::open(path)?))
748    }
749
750    fn metadata(&self, path: &Path) -> io::Result<EntryMeta> {
751        std::fs::metadata(path).map(|m| EntryMeta::from_std(&m))
752    }
753
754    fn list(&self, dir: &Path) -> io::Result<Vec<DirEntry>> {
755        let mut out = Vec::new();
756        for e in std::fs::read_dir(dir)? {
757            let e = e?;
758            out.push(DirEntry {
759                path: e.path(),
760                kind: EntryKind::Local(Box::new(e)),
761            });
762        }
763        Ok(out)
764    }
765
766    fn exists(&self, path: &Path) -> bool {
767        path.exists()
768    }
769
770    fn is_local(&self) -> bool {
771        true
772    }
773}
774
775/// Files held by the host: a map from path to source. Directories exist implicitly (every
776/// ancestor of a file is one). One dropped file, the files of a dropped folder, or a Python
777/// buffer all become a `MemFs`.
778#[derive(Debug, Default, Clone)]
779pub struct MemFs {
780    files: BTreeMap<PathBuf, Arc<dyn ByteSource>>,
781}
782
783impl MemFs {
784    /// An empty namespace.
785    pub fn new() -> Self {
786        Self::default()
787    }
788
789    /// Add (or replace) the file at `path`.
790    pub fn insert(&mut self, path: impl AsRef<Path>, src: Arc<dyn ByteSource>) {
791        self.files.insert(normalize(path.as_ref()), src);
792    }
793
794    /// Builder form of [`MemFs::insert`].
795    pub fn with(mut self, path: impl AsRef<Path>, src: Arc<dyn ByteSource>) -> Self {
796        self.insert(path, src);
797        self
798    }
799
800    /// Paths of every file held, sorted.
801    pub fn paths(&self) -> Vec<PathBuf> {
802        self.files.keys().cloned().collect()
803    }
804
805    fn is_dir_path(&self, dir: &Path) -> bool {
806        let dir = normalize(dir);
807        self.files
808            .keys()
809            .any(|k| k.starts_with(&dir) && k.as_path() != dir.as_path())
810    }
811}
812
813/// Lexical normalization: drop `.` components and a leading `./`, resolve `..` where possible.
814fn normalize(p: &Path) -> PathBuf {
815    let mut out = PathBuf::new();
816    for c in p.components() {
817        match c {
818            Component::CurDir => {}
819            Component::ParentDir => {
820                if !out.pop() {
821                    out.push("..");
822                }
823            }
824            other => out.push(other.as_os_str()),
825        }
826    }
827    out
828}
829
830impl DirSource for MemFs {
831    fn open(&self, path: &Path) -> io::Result<Arc<dyn ByteSource>> {
832        let key = normalize(path);
833        self.files.get(&key).cloned().ok_or_else(|| {
834            let kind = if self.is_dir_path(&key) {
835                io::ErrorKind::IsADirectory
836            } else {
837                io::ErrorKind::NotFound
838            };
839            io::Error::new(kind, format!("{} was not provided", path.display()))
840        })
841    }
842
843    fn metadata(&self, path: &Path) -> io::Result<EntryMeta> {
844        let key = normalize(path);
845        if let Some(src) = self.files.get(&key) {
846            return Ok(EntryMeta::file(src.size()?));
847        }
848        if key.as_os_str().is_empty() || self.is_dir_path(&key) {
849            return Ok(EntryMeta::dir());
850        }
851        Err(io::Error::new(
852            io::ErrorKind::NotFound,
853            format!("{} was not provided", path.display()),
854        ))
855    }
856
857    fn list(&self, dir: &Path) -> io::Result<Vec<DirEntry>> {
858        let key = normalize(dir);
859        if self.files.contains_key(&key) {
860            return Err(io::Error::new(
861                io::ErrorKind::NotADirectory,
862                format!("{} is a file", dir.display()),
863            ));
864        }
865        let mut names: BTreeMap<OsString, EntryMeta> = BTreeMap::new();
866        for (k, src) in &self.files {
867            let Ok(rest) = k.strip_prefix(&key) else {
868                continue;
869            };
870            let mut comps = rest.components();
871            let Some(first) = comps.next() else {
872                continue;
873            };
874            let name = first.as_os_str().to_os_string();
875            if comps.next().is_some() {
876                names.insert(name, EntryMeta::dir());
877            } else {
878                names.insert(name, EntryMeta::file(src.size().unwrap_or(0)));
879            }
880        }
881        if names.is_empty() && !key.as_os_str().is_empty() {
882            return Err(io::Error::new(
883                io::ErrorKind::NotFound,
884                format!("{} was not provided", dir.display()),
885            ));
886        }
887        // Entries carry the directory as the caller spelled it, like `std::fs::read_dir`.
888        Ok(names
889            .into_iter()
890            .map(|(n, m)| DirEntry::new(dir.join(n), m))
891            .collect())
892    }
893}
894
895/// A cheap-to-clone handle on a [`DirSource`], with the `std::fs`-like calls readers use.
896#[derive(Clone)]
897pub struct Fs(Arc<dyn DirSource>);
898
899impl fmt::Debug for Fs {
900    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
901        self.0.fmt(f)
902    }
903}
904
905impl Default for Fs {
906    fn default() -> Self {
907        Self::local()
908    }
909}
910
911impl Fs {
912    /// The local file system.
913    pub fn local() -> Self {
914        Self(Arc::new(LocalFs))
915    }
916
917    /// Wrap any namespace.
918    pub fn new(dir: Arc<dyn DirSource>) -> Self {
919        Self(dir)
920    }
921
922    /// The namespace this handle wraps.
923    pub fn dir_source(&self) -> &Arc<dyn DirSource> {
924        &self.0
925    }
926
927    /// True for the local file system.
928    pub fn is_local(&self) -> bool {
929        self.0.is_local()
930    }
931
932    /// Open a file (like `std::fs::File::open`).
933    pub fn open(&self, path: &Path) -> io::Result<SourceFile> {
934        Ok(SourceFile::new(self.0.open(path)?))
935    }
936
937    /// The byte source of a file.
938    pub fn source(&self, path: &Path) -> io::Result<Arc<dyn ByteSource>> {
939        self.0.open(path)
940    }
941
942    /// Like `std::fs::metadata`.
943    pub fn metadata(&self, path: &Path) -> io::Result<EntryMeta> {
944        self.0.metadata(path)
945    }
946
947    /// Like `Path::exists`.
948    pub fn exists(&self, path: &Path) -> bool {
949        self.0.exists(path)
950    }
951
952    /// The entry of `dir` named `name` as the directory lists it: an exact match first, else
953    /// one that differs only in ASCII case. Case-insensitive file systems (macOS, Windows)
954    /// would open `dir/name` either way; resolving through the listing reports the name the
955    /// file really has, identically on every file system and byte source.
956    pub fn find_in_dir(&self, dir: &Path, name: &str) -> Option<PathBuf> {
957        let dir = if dir.as_os_str().is_empty() {
958            Path::new(".")
959        } else {
960            dir
961        };
962        let names: Vec<std::ffi::OsString> = self
963            .read_dir(dir)
964            .ok()?
965            .filter_map(std::result::Result::ok)
966            .map(|e| e.file_name())
967            .collect();
968        names
969            .iter()
970            .find(|n| n.to_string_lossy() == name)
971            .or_else(|| {
972                names
973                    .iter()
974                    .find(|n| n.to_string_lossy().eq_ignore_ascii_case(name))
975            })
976            .map(|n| dir.join(n))
977    }
978
979    /// Like `Path::is_file`.
980    pub fn is_file(&self, path: &Path) -> bool {
981        self.0.metadata(path).is_ok_and(|m| m.is_file())
982    }
983
984    /// Like `Path::is_dir`.
985    pub fn is_dir(&self, path: &Path) -> bool {
986        self.0.metadata(path).is_ok_and(|m| m.is_dir())
987    }
988
989    /// Like `std::fs::read_dir`.
990    pub fn read_dir(&self, dir: &Path) -> io::Result<ReadDir> {
991        let v: Vec<io::Result<DirEntry>> = self.0.list(dir)?.into_iter().map(Ok).collect();
992        Ok(ReadDir(v.into_iter()))
993    }
994
995    /// Like `std::fs::read`: the whole file.
996    pub fn read(&self, path: &Path) -> io::Result<Vec<u8>> {
997        let mut f = self.open(path)?;
998        let mut out = Vec::new();
999        f.read_to_end(&mut out)?;
1000        Ok(out)
1001    }
1002
1003    /// Like `std::fs::read_to_string`.
1004    pub fn read_to_string(&self, path: &Path) -> io::Result<String> {
1005        String::from_utf8(self.read(path)?)
1006            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
1007    }
1008}
1009
1010// ---------------------------------------------------------------------------------------------
1011// Input: what a reader opens
1012// ---------------------------------------------------------------------------------------------
1013
1014/// What a reader is asked to open: a path in a namespace of files. For a local file the path
1015/// is the real path and the namespace is the file system; for a buffer or a host source the
1016/// path is the name the host gave it (its extension still guides detection).
1017#[derive(Clone, Debug)]
1018pub struct Input {
1019    path: PathBuf,
1020    fs: Fs,
1021}
1022
1023impl Input {
1024    /// A path on the local file system.
1025    pub fn local(path: impl Into<PathBuf>) -> Self {
1026        Self {
1027            path: path.into(),
1028            fs: Fs::local(),
1029        }
1030    }
1031
1032    /// `path` inside the namespace `fs`.
1033    pub fn new(path: impl Into<PathBuf>, fs: Fs) -> Self {
1034        Self {
1035            path: path.into(),
1036            fs,
1037        }
1038    }
1039
1040    /// One file held in memory, named `name`.
1041    pub fn from_bytes(
1042        name: impl AsRef<Path>,
1043        data: impl AsRef<[u8]> + Send + Sync + 'static,
1044    ) -> Self {
1045        let name = name.as_ref();
1046        let src = MemSource::new(name.display().to_string(), data);
1047        Self::from_source(name, Arc::new(src))
1048    }
1049
1050    /// One file read through `src` (a host callback, a cached remote reader), named `name`.
1051    pub fn from_source(name: impl AsRef<Path>, src: Arc<dyn ByteSource>) -> Self {
1052        let name = normalize(name.as_ref());
1053        let fs = MemFs::new().with(&name, src);
1054        Self {
1055            path: name,
1056            fs: Fs::new(Arc::new(fs)),
1057        }
1058    }
1059
1060    /// The path (or host-given name) of what is opened.
1061    pub fn path(&self) -> &Path {
1062        &self.path
1063    }
1064
1065    /// The namespace siblings and directory members are looked up in.
1066    pub fn fs(&self) -> &Fs {
1067        &self.fs
1068    }
1069
1070    /// True when the input is on the local file system.
1071    pub fn is_local(&self) -> bool {
1072        self.fs.is_local()
1073    }
1074
1075    /// Another path in the same namespace (a sibling file, a member of a directory dataset).
1076    pub fn with_path(&self, path: impl Into<PathBuf>) -> Self {
1077        Self {
1078            path: path.into(),
1079            fs: self.fs.clone(),
1080        }
1081    }
1082
1083    /// Open the input's file; errors name its path (exit 5).
1084    pub fn open(&self) -> Result<SourceFile> {
1085        self.fs
1086            .open(&self.path)
1087            .map_err(|e| Error::io(&self.path, e))
1088    }
1089
1090    /// Size and type of the input's path.
1091    pub fn metadata(&self) -> Result<EntryMeta> {
1092        self.fs
1093            .metadata(&self.path)
1094            .map_err(|e| Error::io(&self.path, e))
1095    }
1096
1097    /// True when the input is a directory (a vendor folder format).
1098    pub fn is_dir(&self) -> bool {
1099        self.fs.is_dir(&self.path)
1100    }
1101}
1102
1103#[cfg(test)]
1104mod tests {
1105    use super::*;
1106    use std::sync::atomic::{AtomicUsize, Ordering};
1107
1108    #[test]
1109    fn source_file_reads_and_seeks_like_a_file() {
1110        let dir = tempfile::tempdir().unwrap();
1111        let path = dir.path().join("a.bin");
1112        std::fs::write(&path, b"hello world").unwrap();
1113        for input in [
1114            Input::local(&path),
1115            Input::from_bytes("a.bin", b"hello world".to_vec()),
1116        ] {
1117            let mut f = input.open().unwrap();
1118            assert_eq!(f.metadata().unwrap().len(), 11);
1119            f.seek(SeekFrom::End(-5)).unwrap();
1120            let mut s = String::new();
1121            f.read_to_string(&mut s).unwrap();
1122            assert_eq!(s, "world");
1123            f.seek(SeekFrom::Start(20)).unwrap();
1124            assert_eq!(f.read(&mut [0u8; 4]).unwrap(), 0);
1125            let mut b = [0u8; 4];
1126            let e = f.read_exact(&mut b).unwrap_err();
1127            assert_eq!(e.kind(), io::ErrorKind::UnexpectedEof);
1128            assert!(f.seek(SeekFrom::Current(-100)).is_err());
1129            let mut b = [0u8; 5];
1130            f.read_exact_at(0, &mut b).unwrap();
1131            assert_eq!(&b, b"hello");
1132        }
1133    }
1134
1135    #[test]
1136    fn mem_fs_lists_and_answers_like_a_directory() {
1137        let fs = MemFs::new()
1138            .with(
1139                "run.d/analysis.tdf",
1140                Arc::new(MemSource::new("t", vec![1u8; 3])),
1141            )
1142            .with(
1143                "run.d/sub/x.bin",
1144                Arc::new(MemSource::new("x", vec![2u8; 5])),
1145            )
1146            .with("./top.czi", Arc::new(MemSource::new("c", vec![0u8; 1])));
1147        let fs = Fs::new(Arc::new(fs));
1148        assert!(fs.is_dir(Path::new("run.d")));
1149        assert!(fs.is_file(Path::new("run.d/analysis.tdf")));
1150        assert!(fs.is_file(Path::new("top.czi")));
1151        assert!(!fs.exists(Path::new("run.d/missing")));
1152        assert_eq!(fs.metadata(Path::new("run.d/sub/x.bin")).unwrap().len(), 5);
1153        let mut names: Vec<(String, bool)> = fs
1154            .read_dir(Path::new("run.d"))
1155            .unwrap()
1156            .flatten()
1157            .map(|e| {
1158                (
1159                    e.file_name().to_string_lossy().into_owned(),
1160                    e.metadata().unwrap().is_dir(),
1161                )
1162            })
1163            .collect();
1164        names.sort();
1165        assert_eq!(
1166            names,
1167            vec![("analysis.tdf".into(), false), ("sub".into(), true)]
1168        );
1169        let root: Vec<_> = fs
1170            .read_dir(Path::new(""))
1171            .unwrap()
1172            .flatten()
1173            .map(|e| e.path())
1174            .collect();
1175        assert_eq!(root.len(), 2);
1176        assert_eq!(
1177            fs.open(Path::new("run.d")).unwrap_err().kind(),
1178            io::ErrorKind::IsADirectory
1179        );
1180        assert_eq!(
1181            fs.read(Path::new("run.d/sub/../analysis.tdf")).unwrap(),
1182            [1, 1, 1]
1183        );
1184    }
1185
1186    #[test]
1187    fn local_fs_lists_directories() {
1188        let dir = tempfile::tempdir().unwrap();
1189        std::fs::write(dir.path().join("a"), b"12").unwrap();
1190        std::fs::create_dir(dir.path().join("b")).unwrap();
1191        let fs = Fs::local();
1192        let mut v: Vec<(String, bool, bool)> = fs
1193            .read_dir(dir.path())
1194            .unwrap()
1195            .map(|e| {
1196                let e = e.unwrap();
1197                (
1198                    e.file_name().to_string_lossy().into_owned(),
1199                    e.file_type().unwrap().is_dir(),
1200                    e.metadata().unwrap().is_dir(),
1201                )
1202            })
1203            .collect();
1204        v.sort();
1205        assert_eq!(
1206            v,
1207            vec![("a".into(), false, false), ("b".into(), true, true)]
1208        );
1209        assert!(
1210            fs.metadata(&dir.path().join("a"))
1211                .unwrap()
1212                .modified()
1213                .is_ok()
1214        );
1215        assert!(fs.is_local());
1216    }
1217
1218    #[test]
1219    fn callback_source_is_bounded_and_cached() {
1220        let data: Arc<Vec<u8>> = Arc::new((0..=255u8).cycle().take(300_000).collect());
1221        let calls = Arc::new(AtomicUsize::new(0));
1222        let (d, c) = (Arc::clone(&data), Arc::clone(&calls));
1223        let cb = CallbackSource::new("remote.bin", data.len() as u64, move |off, buf| {
1224            c.fetch_add(1, Ordering::SeqCst);
1225            let off = off as usize;
1226            let n = buf.len().min(d.len() - off);
1227            buf[..n].copy_from_slice(&d[off..off + n]);
1228            Ok(n)
1229        });
1230        let cached = Arc::new(CachedSource::new(Arc::new(cb), 4096, 4));
1231        let mut f = SourceFile::new(cached);
1232        let mut small = [0u8; 16];
1233        for i in 0..100u64 {
1234            f.seek(SeekFrom::Start(i * 16)).unwrap();
1235            f.read_exact(&mut small).unwrap();
1236            assert_eq!(small[0], (i * 16 % 256) as u8);
1237        }
1238        assert_eq!(
1239            calls.load(Ordering::SeqCst),
1240            1,
1241            "one block serves 100 small reads"
1242        );
1243        // Straddling blocks and reading to the end.
1244        let mut v = Vec::new();
1245        f.seek(SeekFrom::Start(299_990)).unwrap();
1246        f.read_to_end(&mut v).unwrap();
1247        assert_eq!(v.len(), 10);
1248        let mut big = vec![0u8; 50_000];
1249        f.read_exact_at(1000, &mut big).unwrap();
1250        assert_eq!(&big[..], &data[1000..51_000]);
1251        // A host callback that claims more than it was asked for is an error, not a panic.
1252        let bad = CallbackSource::new("bad", 10, |_, buf| Ok(buf.len() + 1));
1253        assert!(bad.read_at(0, &mut [0u8; 4]).is_err());
1254    }
1255
1256    #[test]
1257    fn mem_source_ranges_share_the_buffer() {
1258        let s = MemSource::new("m", vec![9u8; 10]);
1259        let r = s.read_range(8, 100).unwrap();
1260        assert_eq!(&*r, &[9, 9]);
1261        assert!(s.read_range(20, 4).unwrap().is_empty());
1262        assert_eq!(s.read_at(u64::MAX, &mut [0u8; 4]).unwrap(), 0);
1263    }
1264}