#[cfg(not(feature = "std"))]
use alloc::{vec, vec::Vec};
use crate::convert::TryToUsize;
use crate::error::FormatError;
pub const DEFAULT_METADATA_CACHE_MAX_ENTRY_BYTES: usize = 64 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct MetadataCacheConfig {
max_bytes: usize,
max_entry_bytes: usize,
}
impl MetadataCacheConfig {
pub const fn new(max_bytes: usize) -> Self {
let max_entry_bytes = if max_bytes < DEFAULT_METADATA_CACHE_MAX_ENTRY_BYTES {
max_bytes
} else {
DEFAULT_METADATA_CACHE_MAX_ENTRY_BYTES
};
Self {
max_bytes,
max_entry_bytes,
}
}
pub const fn disabled() -> Self {
Self {
max_bytes: 0,
max_entry_bytes: 0,
}
}
pub const fn with_max_entry_bytes(mut self, max_entry_bytes: usize) -> Self {
self.max_entry_bytes = max_entry_bytes;
self
}
pub const fn max_bytes(&self) -> usize {
self.max_bytes
}
pub const fn max_entry_bytes(&self) -> usize {
self.max_entry_bytes
}
pub const fn is_enabled(&self) -> bool {
self.max_bytes > 0 && self.max_entry_bytes > 0
}
}
impl Default for MetadataCacheConfig {
fn default() -> Self {
Self::disabled()
}
}
pub trait FileSource {
fn len(&self) -> u64;
fn is_empty(&self) -> bool {
self.len() == 0
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<(), FormatError>;
fn read_exact_at(&self, offset: u64, len: usize) -> Result<Vec<u8>, FormatError> {
let end = offset
.checked_add(len as u64)
.ok_or(FormatError::OffsetOverflow {
offset,
length: len as u64,
})?;
if end > self.len() {
return Err(FormatError::UnexpectedEof {
expected: end.to_usize().unwrap_or(usize::MAX),
available: self.len().to_usize().unwrap_or(usize::MAX),
});
}
let mut buf = vec![0u8; len];
self.read_at(offset, &mut buf)?;
Ok(buf)
}
fn read_metadata_at(&self, offset: u64, len: usize) -> Result<Vec<u8>, FormatError> {
self.read_exact_at(offset, len)
}
}
impl<S: FileSource + ?Sized> FileSource for &S {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<(), FormatError> {
(**self).read_at(offset, buf)
}
fn read_exact_at(&self, offset: u64, len: usize) -> Result<Vec<u8>, FormatError> {
(**self).read_exact_at(offset, len)
}
fn read_metadata_at(&self, offset: u64, len: usize) -> Result<Vec<u8>, FormatError> {
(**self).read_metadata_at(offset, len)
}
}
#[cfg(feature = "std")]
impl<S: FileSource + ?Sized> FileSource for std::boxed::Box<S> {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<(), FormatError> {
(**self).read_at(offset, buf)
}
fn read_exact_at(&self, offset: u64, len: usize) -> Result<Vec<u8>, FormatError> {
(**self).read_exact_at(offset, len)
}
fn read_metadata_at(&self, offset: u64, len: usize) -> Result<Vec<u8>, FormatError> {
(**self).read_metadata_at(offset, len)
}
}
#[derive(Debug, Clone, Copy)]
pub struct BytesSource<T>(pub T);
impl<T: AsRef<[u8]>> BytesSource<T> {
pub fn new(bytes: T) -> Self {
BytesSource(bytes)
}
}
impl<T: AsRef<[u8]>> FileSource for BytesSource<T> {
fn len(&self) -> u64 {
self.0.as_ref().len() as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<(), FormatError> {
let bytes = self.0.as_ref();
let start = offset.to_usize()?;
let end = start
.checked_add(buf.len())
.ok_or(FormatError::OffsetOverflow {
offset,
length: buf.len() as u64,
})?;
if end > bytes.len() {
return Err(FormatError::UnexpectedEof {
expected: end,
available: bytes.len(),
});
}
buf.copy_from_slice(&bytes[start..end]);
Ok(())
}
}
#[cfg(feature = "std")]
struct CachedMetadataRead {
offset: u64,
len: usize,
bytes: Vec<u8>,
last_access: u64,
}
#[cfg(feature = "std")]
struct MetadataReadCache {
entries: Vec<CachedMetadataRead>,
current_bytes: usize,
tick: u64,
}
#[cfg(feature = "std")]
impl MetadataReadCache {
fn new() -> Self {
Self {
entries: Vec::new(),
current_bytes: 0,
tick: 0,
}
}
fn get(&mut self, offset: u64, len: usize) -> Option<Vec<u8>> {
self.tick = self.tick.wrapping_add(1);
let tick = self.tick;
for entry in &mut self.entries {
if entry.offset == offset && entry.len == len {
entry.last_access = tick;
return Some(entry.bytes.clone());
}
}
None
}
fn insert(&mut self, offset: u64, len: usize, bytes: Vec<u8>, max_bytes: usize) {
if len == 0 || bytes.len() > max_bytes {
return;
}
self.tick = self.tick.wrapping_add(1);
let tick = self.tick;
for entry in &mut self.entries {
if entry.offset == offset && entry.len == len {
self.current_bytes = self.current_bytes - entry.bytes.len() + bytes.len();
entry.bytes = bytes;
entry.last_access = tick;
self.evict_to_budget(max_bytes);
return;
}
}
self.current_bytes += bytes.len();
self.entries.push(CachedMetadataRead {
offset,
len,
bytes,
last_access: tick,
});
self.evict_to_budget(max_bytes);
}
fn evict_to_budget(&mut self, max_bytes: usize) {
while self.current_bytes > max_bytes && !self.entries.is_empty() {
let lru_idx = self
.entries
.iter()
.enumerate()
.min_by_key(|(_, entry)| entry.last_access)
.map(|(idx, _)| idx)
.unwrap();
let removed = self.entries.swap_remove(lru_idx);
self.current_bytes -= removed.bytes.len();
}
}
}
#[cfg(feature = "std")]
pub struct MetadataCachingSource<S> {
inner: S,
config: MetadataCacheConfig,
cache: std::sync::Mutex<MetadataReadCache>,
}
#[cfg(feature = "std")]
impl<S> MetadataCachingSource<S> {
pub fn new(inner: S, config: MetadataCacheConfig) -> Self {
Self {
inner,
config,
cache: std::sync::Mutex::new(MetadataReadCache::new()),
}
}
}
#[cfg(feature = "std")]
impl<S: FileSource> FileSource for MetadataCachingSource<S> {
fn len(&self) -> u64 {
self.inner.len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<(), FormatError> {
self.inner.read_at(offset, buf)
}
fn read_exact_at(&self, offset: u64, len: usize) -> Result<Vec<u8>, FormatError> {
self.inner.read_exact_at(offset, len)
}
fn read_metadata_at(&self, offset: u64, len: usize) -> Result<Vec<u8>, FormatError> {
if !self.config.is_enabled()
|| len == 0
|| len > self.config.max_entry_bytes
|| len > self.config.max_bytes
{
return self.inner.read_metadata_at(offset, len);
}
if let Some(bytes) = self
.cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(offset, len)
{
return Ok(bytes);
}
let bytes = self.inner.read_metadata_at(offset, len)?;
self.cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(offset, len, bytes.clone(), self.config.max_bytes);
Ok(bytes)
}
}
#[cfg(feature = "std")]
pub struct ReadSeekSource<R> {
inner: std::sync::Mutex<R>,
len: u64,
}
#[cfg(feature = "std")]
impl<R: std::io::Read + std::io::Seek> ReadSeekSource<R> {
pub fn new(mut reader: R) -> Result<Self, FormatError> {
let len = reader
.seek(std::io::SeekFrom::End(0))
.map_err(|e| FormatError::Source(format_io(&e)))?;
Ok(ReadSeekSource {
inner: std::sync::Mutex::new(reader),
len,
})
}
}
#[cfg(feature = "std")]
impl<R: std::io::Read + std::io::Seek> FileSource for ReadSeekSource<R> {
fn len(&self) -> u64 {
self.len
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<(), FormatError> {
let end = offset
.checked_add(buf.len() as u64)
.ok_or(FormatError::OffsetOverflow {
offset,
length: buf.len() as u64,
})?;
if end > self.len {
return Err(FormatError::UnexpectedEof {
expected: end.to_usize().unwrap_or(usize::MAX),
available: self.len.to_usize().unwrap_or(usize::MAX),
});
}
let mut guard = self
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
guard
.seek(std::io::SeekFrom::Start(offset))
.map_err(|e| FormatError::Source(format_io(&e)))?;
guard
.read_exact(buf)
.map_err(|e| FormatError::Source(format_io(&e)))?;
Ok(())
}
}
#[cfg(feature = "std")]
fn format_io(e: &std::io::Error) -> std::string::String {
std::format!("{e}")
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(not(feature = "std"))]
use alloc::vec;
#[test]
fn bytes_source_reads_and_reports_len() {
let data = (0u8..=255).collect::<Vec<u8>>();
let src = BytesSource::new(data.clone());
assert_eq!(src.len(), 256);
assert!(!src.is_empty());
let mut buf = [0u8; 4];
src.read_at(10, &mut buf).unwrap();
assert_eq!(buf, [10, 11, 12, 13]);
let owned = src.read_exact_at(250, 6).unwrap();
assert_eq!(owned, vec![250, 251, 252, 253, 254, 255]);
}
#[test]
fn bytes_source_short_read_is_eof() {
let src = BytesSource::new(vec![1u8, 2, 3]);
let mut buf = [0u8; 4];
let err = src.read_at(0, &mut buf).unwrap_err();
assert!(matches!(err, FormatError::UnexpectedEof { .. }));
let mut ok = [0u8; 3];
src.read_at(0, &mut ok).unwrap();
assert_eq!(ok, [1, 2, 3]);
}
#[test]
fn bytes_source_offset_past_end_is_eof() {
let src = BytesSource::new(vec![0u8; 8]);
let mut buf = [0u8; 1];
assert!(matches!(
src.read_at(8, &mut buf).unwrap_err(),
FormatError::UnexpectedEof { .. }
));
src.read_at(8, &mut []).unwrap();
}
#[test]
fn read_exact_at_rejects_oversized_len_without_allocating() {
let src = BytesSource::new(vec![1u8, 2, 3, 4]);
assert!(matches!(
src.read_exact_at(0, usize::MAX).unwrap_err(),
FormatError::UnexpectedEof { .. }
));
assert_eq!(src.read_exact_at(1, 3).unwrap(), vec![2, 3, 4]);
}
#[test]
fn empty_source() {
let src = BytesSource::new(Vec::<u8>::new());
assert_eq!(src.len(), 0);
assert!(src.is_empty());
}
#[test]
fn forwarding_through_reference() {
let src = BytesSource::new(vec![9u8, 8, 7]);
let r: &dyn FileSource = &src;
let mut buf = [0u8; 2];
r.read_at(1, &mut buf).unwrap();
assert_eq!(buf, [8, 7]);
}
#[test]
fn forwarding_through_reference_preserves_metadata_reads() {
use core::cell::Cell;
struct MetadataSource {
metadata_reads: Cell<usize>,
}
impl FileSource for MetadataSource {
fn len(&self) -> u64 {
16
}
fn read_at(&self, _offset: u64, buf: &mut [u8]) -> Result<(), FormatError> {
buf.fill(0);
Ok(())
}
fn read_metadata_at(&self, _offset: u64, len: usize) -> Result<Vec<u8>, FormatError> {
self.metadata_reads.set(self.metadata_reads.get() + 1);
Ok(vec![0xAB; len])
}
}
fn read_metadata_via_trait<T: FileSource>(source: T) -> Vec<u8> {
source.read_metadata_at(4, 3).unwrap()
}
let source = MetadataSource {
metadata_reads: Cell::new(0),
};
assert_eq!(read_metadata_via_trait(&source), vec![0xAB; 3]);
assert_eq!(source.metadata_reads.get(), 1);
}
#[cfg(feature = "std")]
#[test]
fn metadata_cache_caches_only_metadata_reads() {
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
struct CountingSource {
data: Vec<u8>,
reads: Arc<AtomicUsize>,
}
impl FileSource for CountingSource {
fn len(&self) -> u64 {
self.data.len() as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<(), FormatError> {
self.reads.fetch_add(1, Ordering::SeqCst);
BytesSource::new(&self.data).read_at(offset, buf)
}
}
let reads = Arc::new(AtomicUsize::new(0));
let source = MetadataCachingSource::new(
CountingSource {
data: (0u8..16).collect(),
reads: Arc::clone(&reads),
},
MetadataCacheConfig::new(16),
);
assert_eq!(source.read_metadata_at(4, 4).unwrap(), vec![4, 5, 6, 7]);
assert_eq!(source.read_metadata_at(4, 4).unwrap(), vec![4, 5, 6, 7]);
assert_eq!(reads.load(Ordering::SeqCst), 1);
assert_eq!(source.read_exact_at(4, 4).unwrap(), vec![4, 5, 6, 7]);
assert_eq!(source.read_exact_at(4, 4).unwrap(), vec![4, 5, 6, 7]);
assert_eq!(reads.load(Ordering::SeqCst), 3);
}
#[cfg(feature = "std")]
#[test]
fn read_seek_source_matches_in_memory() {
use std::io::Cursor;
let data = (0u8..200).collect::<Vec<u8>>();
let mem = BytesSource::new(data.clone());
let seek = ReadSeekSource::new(Cursor::new(data.clone())).unwrap();
assert_eq!(seek.len(), mem.len());
for &(off, len) in &[(0u64, 1usize), (5, 10), (199, 1), (100, 50)] {
let a = mem.read_exact_at(off, len).unwrap();
let b = seek.read_exact_at(off, len).unwrap();
assert_eq!(a, b, "mismatch at offset {off} len {len}");
}
}
#[cfg(feature = "std")]
#[test]
fn read_seek_source_past_end_is_error() {
use std::io::Cursor;
let seek = ReadSeekSource::new(Cursor::new(vec![1u8, 2, 3, 4])).unwrap();
let mut buf = [0u8; 3];
assert!(matches!(
seek.read_at(2, &mut buf).unwrap_err(),
FormatError::UnexpectedEof { .. }
));
}
#[cfg(feature = "std")]
#[test]
fn read_seek_source_is_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<ReadSeekSource<std::io::Cursor<Vec<u8>>>>();
}
}