use std::fs;
use std::io::{Read, Seek, SeekFrom, Write};
use crate::convert::TryToUsize;
use crate::error::{Error, FormatError};
use crate::source::{BytesSource, MetadataCacheConfig, MetadataReadCache, Source};
pub(crate) trait FileImage: Source + Send + Sync {
fn append(&mut self, bytes: &[u8]) -> Result<u64, Error>;
fn write_at(&mut self, offset: u64, bytes: &[u8]) -> Result<(), Error>;
fn truncate(&mut self, len: u64) -> Result<(), Error>;
fn sync_data(&mut self) -> Result<(), Error>;
fn sync_all(&mut self) -> Result<(), Error>;
fn as_slice(&self) -> Option<&[u8]> {
None
}
}
pub(crate) struct MirrorImage {
handle: fs::File,
data: Vec<u8>,
}
impl MirrorImage {
pub(crate) fn new(handle: fs::File, data: Vec<u8>) -> Self {
Self { handle, data }
}
}
impl Source for MirrorImage {
fn len(&self) -> u64 {
self.data.len() as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<(), FormatError> {
BytesSource::new(&self.data[..]).read_at(offset, buf)
}
}
impl FileImage for MirrorImage {
fn append(&mut self, bytes: &[u8]) -> Result<u64, Error> {
let addr = self.data.len() as u64;
self.handle.seek(SeekFrom::Start(addr)).map_err(Error::Io)?;
self.handle.write_all(bytes).map_err(Error::Io)?;
self.data.extend_from_slice(bytes);
Ok(addr)
}
fn write_at(&mut self, offset: u64, bytes: &[u8]) -> Result<(), Error> {
debug_assert!(
offset.saturating_add(bytes.len() as u64) <= self.len(),
"write_at past end-of-file: {offset}+{} > {}",
bytes.len(),
self.len()
);
let offset_usize = offset.to_usize()?;
self.handle
.seek(SeekFrom::Start(offset))
.map_err(Error::Io)?;
self.handle.write_all(bytes).map_err(Error::Io)?;
self.data[offset_usize..offset_usize + bytes.len()].copy_from_slice(bytes);
Ok(())
}
fn truncate(&mut self, len: u64) -> Result<(), Error> {
debug_assert!(
len <= self.len(),
"truncate would grow the image: {len} > {}",
self.len()
);
let len_usize = len.to_usize()?;
self.handle.set_len(len).map_err(Error::Io)?;
self.data.truncate(len_usize);
Ok(())
}
fn sync_data(&mut self) -> Result<(), Error> {
self.handle.flush().map_err(Error::Io)?;
self.handle.sync_data().map_err(Error::Io)?;
Ok(())
}
fn sync_all(&mut self) -> Result<(), Error> {
self.handle.flush().map_err(Error::Io)?;
self.handle.sync_all().map_err(Error::Io)?;
Ok(())
}
fn as_slice(&self) -> Option<&[u8]> {
Some(&self.data)
}
}
pub(crate) fn read_at_handle(
handle: &fs::File,
len: u64,
offset: u64,
buf: &mut [u8],
) -> Result<(), FormatError> {
let end = offset
.checked_add(buf.len() as u64)
.ok_or(FormatError::OffsetOverflow {
offset,
length: buf.len() as u64,
})?;
if end > len {
return Err(FormatError::UnexpectedEof {
expected: end.to_usize().unwrap_or(usize::MAX),
available: len.to_usize().unwrap_or(usize::MAX),
});
}
let mut h = handle;
h.seek(SeekFrom::Start(offset))
.map_err(|e| FormatError::Source(std::format!("{e}")))?;
h.read_exact(buf)
.map_err(|e| FormatError::Source(std::format!("{e}")))?;
Ok(())
}
pub(crate) struct HandleImage {
handle: fs::File,
len: u64,
metadata_cache: Option<(MetadataCacheConfig, std::sync::Mutex<MetadataReadCache>)>,
}
impl HandleImage {
pub(crate) fn new(handle: fs::File, len: u64, cache: MetadataCacheConfig) -> Self {
Self {
handle,
len,
metadata_cache: cache
.is_enabled()
.then(|| (cache, std::sync::Mutex::new(MetadataReadCache::new()))),
}
}
fn invalidate(&self, offset: u64, len: u64) {
let Some((_, cache)) = &self.metadata_cache else {
return;
};
cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.invalidate_overlapping(offset, len.to_usize().unwrap_or(usize::MAX));
}
}
impl Source for HandleImage {
fn len(&self) -> u64 {
self.len
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<(), FormatError> {
read_at_handle(&self.handle, self.len, offset, buf)
}
fn read_metadata_at(&self, offset: u64, len: usize) -> Result<Vec<u8>, FormatError> {
let Some((config, cache)) = &self.metadata_cache else {
return self.read_exact_at(offset, len);
};
if len == 0 || len > config.max_entry_bytes() || len > config.max_bytes() {
return self.read_exact_at(offset, len);
}
if let Some(bytes) = cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(offset, len)
{
return Ok(bytes);
}
let bytes = self.read_exact_at(offset, len)?;
cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(offset, len, bytes.clone(), config.max_bytes());
Ok(bytes)
}
}
impl FileImage for HandleImage {
fn append(&mut self, bytes: &[u8]) -> Result<u64, Error> {
let addr = self.len;
self.invalidate(addr, bytes.len() as u64);
self.handle.seek(SeekFrom::Start(addr)).map_err(Error::Io)?;
self.handle.write_all(bytes).map_err(Error::Io)?;
self.len += bytes.len() as u64;
Ok(addr)
}
fn write_at(&mut self, offset: u64, bytes: &[u8]) -> Result<(), Error> {
let end = offset
.checked_add(bytes.len() as u64)
.filter(|&e| e <= self.len)
.ok_or(Error::Format(FormatError::UnexpectedEof {
expected: offset.to_usize().unwrap_or(usize::MAX),
available: self.len.to_usize().unwrap_or(usize::MAX),
}))?;
debug_assert!(end <= self.len);
self.invalidate(offset, bytes.len() as u64);
self.handle
.seek(SeekFrom::Start(offset))
.map_err(Error::Io)?;
self.handle.write_all(bytes).map_err(Error::Io)?;
Ok(())
}
fn truncate(&mut self, len: u64) -> Result<(), Error> {
debug_assert!(
len <= self.len,
"truncate would grow the image: {len} > {}",
self.len
);
self.invalidate(len, self.len.saturating_sub(len));
self.handle.set_len(len).map_err(Error::Io)?;
self.len = len;
Ok(())
}
fn sync_data(&mut self) -> Result<(), Error> {
self.handle.flush().map_err(Error::Io)?;
self.handle.sync_data().map_err(Error::Io)?;
Ok(())
}
fn sync_all(&mut self) -> Result<(), Error> {
self.handle.flush().map_err(Error::Io)?;
self.handle.sync_all().map_err(Error::Io)?;
Ok(())
}
}
#[cfg(test)]
pub(crate) struct CountingImage {
inner: Box<dyn FileImage>,
read_bytes: std::sync::Arc<std::sync::atomic::AtomicU64>,
}
#[cfg(test)]
impl CountingImage {
pub(crate) fn new(
inner: Box<dyn FileImage>,
read_bytes: std::sync::Arc<std::sync::atomic::AtomicU64>,
) -> Self {
Self { inner, read_bytes }
}
}
#[cfg(test)]
impl Source for CountingImage {
fn len(&self) -> u64 {
self.inner.len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<(), FormatError> {
self.read_bytes
.fetch_add(buf.len() as u64, std::sync::atomic::Ordering::Relaxed);
self.inner.read_at(offset, buf)
}
}
#[cfg(test)]
impl FileImage for CountingImage {
fn append(&mut self, bytes: &[u8]) -> Result<u64, Error> {
self.inner.append(bytes)
}
fn write_at(&mut self, offset: u64, bytes: &[u8]) -> Result<(), Error> {
self.inner.write_at(offset, bytes)
}
fn truncate(&mut self, len: u64) -> Result<(), Error> {
self.inner.truncate(len)
}
fn sync_data(&mut self) -> Result<(), Error> {
self.inner.sync_data()
}
fn sync_all(&mut self) -> Result<(), Error> {
self.inner.sync_all()
}
fn as_slice(&self) -> Option<&[u8]> {
self.inner.as_slice()
}
}
#[cfg(test)]
pub(crate) struct SourceOnlyImage(MirrorImage);
#[cfg(test)]
impl SourceOnlyImage {
pub(crate) fn new(inner: MirrorImage) -> Self {
Self(inner)
}
}
#[cfg(test)]
impl Source for SourceOnlyImage {
fn len(&self) -> u64 {
self.0.len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<(), FormatError> {
self.0.read_at(offset, buf)
}
}
#[cfg(test)]
impl FileImage for SourceOnlyImage {
fn append(&mut self, bytes: &[u8]) -> Result<u64, Error> {
self.0.append(bytes)
}
fn write_at(&mut self, offset: u64, bytes: &[u8]) -> Result<(), Error> {
self.0.write_at(offset, bytes)
}
fn truncate(&mut self, len: u64) -> Result<(), Error> {
self.0.truncate(len)
}
fn sync_data(&mut self) -> Result<(), Error> {
self.0.sync_data()
}
fn sync_all(&mut self) -> Result<(), Error> {
self.0.sync_all()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(Clone, Copy, Debug)]
enum Backing {
Mirror,
Handle,
}
const BACKINGS: [Backing; 2] = [Backing::Mirror, Backing::Handle];
fn image(
dir: &std::path::Path,
initial: &[u8],
backing: Backing,
) -> (std::path::PathBuf, Box<dyn FileImage>) {
let path = dir.join(std::format!("{backing:?}.bin"));
std::fs::write(&path, initial).unwrap();
let handle = fs::OpenOptions::new()
.read(true)
.write(true)
.open(&path)
.unwrap();
let img: Box<dyn FileImage> = match backing {
Backing::Mirror => Box::new(MirrorImage::new(handle, initial.to_vec())),
Backing::Handle => Box::new(HandleImage::new(
handle,
initial.len() as u64,
MetadataCacheConfig::new(64 * 1024),
)),
};
(path, img)
}
fn bytes(img: &dyn FileImage) -> Vec<u8> {
let mut buf = vec![0u8; img.len().to_usize().unwrap()];
img.read_at(0, &mut buf).unwrap();
buf
}
fn assert_in_sync(path: &std::path::Path, img: &dyn FileImage, backing: Backing) {
let on_disk = std::fs::read(path).unwrap();
assert_eq!(
img.len(),
on_disk.len() as u64,
"{backing:?}: end-of-file disagrees with the file"
);
assert_eq!(
bytes(img),
on_disk,
"{backing:?}: reads disagree with the file"
);
if let Some(slice) = img.as_slice() {
assert_eq!(
slice,
&on_disk[..],
"{backing:?}: the slice disagrees with the file"
);
}
}
#[test]
fn append_returns_the_pre_append_end_and_extends_by_exactly_the_length() {
let dir = tempfile::tempdir().unwrap();
for backing in BACKINGS {
let (path, mut img) = image(dir.path(), b"abcd", backing);
let addr = img.append(b"XYZ").unwrap();
assert_eq!(addr, 4, "{backing:?}: append must report where it wrote");
assert_eq!(
img.len(),
7,
"{backing:?}: append must extend len by exactly bytes.len()"
);
assert_in_sync(&path, img.as_ref(), backing);
}
}
#[test]
fn write_at_overwrites_both_sides_in_place() {
let dir = tempfile::tempdir().unwrap();
for backing in BACKINGS {
let (path, mut img) = image(dir.path(), b"abcdef", backing);
img.write_at(2, b"ZZ").unwrap();
assert_eq!(bytes(img.as_ref()), b"abZZef");
assert_eq!(
img.len(),
6,
"{backing:?}: an in-place write must not move end-of-file"
);
assert_in_sync(&path, img.as_ref(), backing);
}
}
#[test]
fn truncate_shrinks_both_sides() {
let dir = tempfile::tempdir().unwrap();
for backing in BACKINGS {
let (path, mut img) = image(dir.path(), b"abcdef", backing);
img.truncate(2).unwrap();
assert_eq!(bytes(img.as_ref()), b"ab");
assert_eq!(img.len(), 2, "{backing:?}");
assert_in_sync(&path, img.as_ref(), backing);
}
}
#[test]
fn append_after_truncate_lands_at_the_new_end() {
let dir = tempfile::tempdir().unwrap();
for backing in BACKINGS {
let (path, mut img) = image(dir.path(), b"abcdef", backing);
img.truncate(3).unwrap();
let addr = img.append(b"Z").unwrap();
assert_eq!(addr, 3, "{backing:?}");
assert_eq!(bytes(img.as_ref()), b"abcZ");
assert_in_sync(&path, img.as_ref(), backing);
}
}
#[test]
fn reads_observe_writes_immediately() {
let dir = tempfile::tempdir().unwrap();
for backing in BACKINGS {
let (_path, mut img) = image(dir.path(), b"abcdef", backing);
img.write_at(0, b"ZY").unwrap();
img.append(b"!").unwrap();
let mut buf = [0u8; 3];
img.read_at(0, &mut buf).unwrap();
assert_eq!(&buf, b"ZYc", "{backing:?}");
img.read_at(6, &mut buf[..1]).unwrap();
assert_eq!(buf[0], b'!', "{backing:?}");
}
}
#[test]
fn cached_reads_observe_writes_immediately() {
let dir = tempfile::tempdir().unwrap();
for backing in BACKINGS {
let (_path, mut img) = image(dir.path(), b"abcdef", backing);
assert_eq!(img.read_metadata_at(0, 4).unwrap(), b"abcd", "{backing:?}");
img.write_at(1, b"ZZ").unwrap();
assert_eq!(
img.read_metadata_at(0, 4).unwrap(),
b"aZZd",
"{backing:?}: a cached read outlived the write that overwrote it"
);
}
}
#[test]
fn a_cached_read_does_not_survive_being_truncated_and_appended_over() {
let dir = tempfile::tempdir().unwrap();
for backing in BACKINGS {
let (_path, mut img) = image(dir.path(), b"abcdef", backing);
assert_eq!(img.read_metadata_at(4, 2).unwrap(), b"ef", "{backing:?}");
img.truncate(4).unwrap();
img.append(b"ZZ").unwrap();
assert_eq!(
img.read_metadata_at(4, 2).unwrap(),
b"ZZ",
"{backing:?}: a read cached before the truncate survived the append"
);
}
}
#[test]
fn reads_past_end_of_file_are_refused() {
let dir = tempfile::tempdir().unwrap();
for backing in BACKINGS {
let (_path, img) = image(dir.path(), b"abcd", backing);
let mut buf = [0u8; 2];
assert!(img.read_at(3, &mut buf).is_err(), "{backing:?}");
}
}
#[test]
fn only_the_mirror_offers_a_whole_file_slice() {
let dir = tempfile::tempdir().unwrap();
let (_p1, mirror) = image(dir.path(), b"abcdef", Backing::Mirror);
let (_p2, handle) = image(dir.path(), b"abcdef", Backing::Handle);
assert_eq!(mirror.as_slice(), Some(&b"abcdef"[..]));
assert!(handle.as_slice().is_none());
}
#[test]
fn source_only_withholds_the_slice_but_reads_the_same() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("source_only.bin");
std::fs::write(&path, b"abcdef").unwrap();
let handle = fs::OpenOptions::new()
.read(true)
.write(true)
.open(&path)
.unwrap();
let mut only = SourceOnlyImage::new(MirrorImage::new(handle, b"abcdef".to_vec()));
assert!(only.as_slice().is_none(), "the slice must be withheld");
only.write_at(1, b"Z").unwrap();
only.append(b"gh").unwrap();
assert_eq!(only.len(), 8);
let mut buf = [0u8; 8];
only.read_at(0, &mut buf).unwrap();
assert_eq!(&buf, b"aZcdefgh");
}
}