use std::io::Read;
use std::path::Path;
use crate::error::CoreError;
use crate::inode::{ContentHandle, Inode, SliceRef};
use crate::metadata::MetadataBlob;
use crate::slab_cache::CachedSlabStore;
use crate::slab_source::SlabSource;
use crate::slab_store::SlabStore;
use crate::{
parse_feature_flags_section, parse_manifest_header, parse_metadata_blob,
parse_metadata_reference, parse_slab_index, read_external_metadata, ManifestCursor,
};
#[derive(Clone, Debug)]
pub struct ReadConfig {
pub cache_bytes: usize,
pub cache_entries: usize,
pub parallel_decode: bool,
pub frame_cache_bytes: usize,
}
impl Default for ReadConfig {
fn default() -> Self {
Self {
cache_bytes: crate::slab_cache::DEFAULT_CACHE_BYTES,
cache_entries: crate::slab_cache::DEFAULT_CACHE_CAPACITY,
parallel_decode: false,
frame_cache_bytes: crate::slab_cache::DEFAULT_FRAME_CACHE_BYTES,
}
}
}
pub struct ImageReader {
blob: MetadataBlob,
root_inode_number: u64,
store: CachedSlabStore,
}
impl ImageReader {
pub fn open(path: &Path, config: ReadConfig) -> Result<Self, CoreError> {
let bytes = std::fs::read(path).map_err(|e| CoreError::Corrupt {
reason: format!("read {}: {e}", path.display()),
})?;
let image_dir = path.parent().unwrap_or_else(|| Path::new("."));
Self::from_parts(&bytes, image_dir, config)
}
pub fn from_parts(
manifest: &[u8],
image_dir: &Path,
config: ReadConfig,
) -> Result<Self, CoreError> {
let mut cursor = ManifestCursor::new(manifest);
parse_manifest_header(&mut cursor)?;
parse_feature_flags_section(&mut cursor)?;
let meta_ref = parse_metadata_reference(&mut cursor)?;
let blob_wire: Vec<u8> = match meta_ref.inline_metadata.as_deref() {
Some(inline) => inline.to_vec(),
None => read_external_metadata(&meta_ref, &image_dir.join("image.lim"))?,
};
let blob = parse_metadata_blob(&mut ManifestCursor::new(&blob_wire))?;
let slab_index = parse_slab_index(&mut cursor)?;
let mut slab_count: u64 = 0;
for entry in &slab_index.entries {
slab_count = slab_count.max(entry.slab_id.ordinal + 1);
}
let mut sources: Vec<Option<crate::slab_store::SlabSource>> =
(0..usize::try_from(slab_count).unwrap_or(0))
.map(|_| None)
.collect();
for entry in &slab_index.entries {
for locator in &entry.locators {
let name = crate::locator::local_sidecar_name(&locator.uri)?;
let path = image_dir.join(name);
if path.exists() {
let idx =
usize::try_from(entry.slab_id.ordinal).expect("slab ordinal fits usize");
let file = std::fs::File::open(&path).map_err(|e| CoreError::Corrupt {
reason: format!("open slab {}: {e}", path.display()),
})?;
#[allow(unsafe_code)]
let mmap =
unsafe { memmap2::Mmap::map(&file) }.map_err(|e| CoreError::Corrupt {
reason: format!("mmap slab {}: {e}", path.display()),
})?;
#[cfg(unix)]
let _ = mmap.advise(memmap2::Advice::Random);
sources[idx] = Some(crate::slab_store::SlabSource::Mapped(mmap));
break;
}
}
}
let sources: Vec<_> = sources
.into_iter()
.map(|s| s.unwrap_or_else(|| crate::slab_store::SlabSource::Memory(Vec::new())))
.collect();
let store = SlabStore::from_sources(sources)?;
let store = CachedSlabStore::with_frame_budget(
store,
config.cache_entries,
config.cache_bytes,
config.frame_cache_bytes,
);
let root_inode_number = blob.root_inode_number().ok_or_else(|| CoreError::Corrupt {
reason: "no unique root directory inode".into(),
})?;
Ok(Self {
blob,
root_inode_number,
store,
})
}
pub fn file(&self, path: &str) -> Result<FileReader<'_>, CoreError> {
let trimmed = path.trim_start_matches('/').trim_end_matches('/');
let mut current = self
.blob
.inode_by_number(self.root_inode_number)
.ok_or_else(|| CoreError::Corrupt {
reason: "root inode missing".into(),
})?;
if !trimmed.is_empty() {
for component in trimmed.split('/') {
if component.is_empty() || component.contains('\0') {
return Err(CoreError::Corrupt {
reason: format!("invalid path component in {path:?}"),
});
}
let hash = match ¤t.content_handle {
ContentHandle::Directory(h) => *h,
_ => {
return Err(CoreError::Corrupt {
reason: format!("{path:?}: not a directory"),
})
}
};
let node = self
.blob
.dir_node_by_hash(&hash)
.ok_or_else(|| CoreError::Corrupt {
reason: format!("{path:?}: directory node missing"),
})?;
let entry = node
.entries
.iter()
.find(|e| e.name == component)
.ok_or_else(|| CoreError::Corrupt {
reason: format!("{path:?}: no such file or directory"),
})?;
current = self
.blob
.inode_by_number(entry.inode_number)
.ok_or_else(|| CoreError::Corrupt {
reason: format!("{path:?}: inode {} missing", entry.inode_number),
})?;
}
}
if !current.is_regular() {
return Err(CoreError::Corrupt {
reason: format!("{path:?}: not a regular file"),
});
}
Ok(FileReader::new(current, &self.store))
}
#[must_use]
pub fn cache_stats(&self) -> crate::slab_cache::CacheStats {
self.store.cache_stats()
}
pub fn decoded(&self, drop_id: &[u8; 32]) -> Option<Result<std::sync::Arc<[u8]>, CoreError>> {
self.store.decoded(drop_id)
}
}
pub struct FileReader<'a> {
inode: &'a Inode,
store: &'a CachedSlabStore,
pos: u64,
}
impl<'a> FileReader<'a> {
fn new(inode: &'a Inode, store: &'a CachedSlabStore) -> Self {
Self {
inode,
store,
pos: 0,
}
}
#[must_use]
pub fn size(&self) -> u64 {
match &self.inode.content_handle {
ContentHandle::InlineData(d) => d.len() as u64,
ContentHandle::SliceMap(s) => s.last().map_or(0, |s| s.file_byte_end),
_ => 0,
}
}
pub fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<usize, CoreError> {
self.read_at_into(offset, buf)
}
pub fn read_at_into(&self, offset: u64, buf: &mut [u8]) -> Result<usize, CoreError> {
read_window_into(self.inode, Some(self.store), offset, buf)
}
}
impl Read for FileReader<'_> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self
.read_at(self.pos, buf)
.map_err(|e| std::io::Error::other(e.to_string()))?;
self.pos += n as u64;
Ok(n)
}
}
pub fn read_window_into(
inode: &Inode,
store: Option<&dyn SlabSource>,
offset: u64,
buf: &mut [u8],
) -> Result<usize, CoreError> {
if buf.is_empty() {
return Ok(0);
}
let window_end = offset.saturating_add(buf.len() as u64);
match &inode.content_handle {
ContentHandle::InlineData(d) => {
let start = usize::try_from(offset).unwrap_or(usize::MAX);
if start >= d.len() {
return Ok(0);
}
let end = (start + buf.len()).min(d.len());
buf[..end - start].copy_from_slice(&d[start..end]);
Ok(end - start)
}
ContentHandle::SliceMap(slices) => {
let store = store.ok_or_else(|| CoreError::Corrupt {
reason: "read_window_into: slice-backed file but no slab store provided".into(),
})?;
let mut filled = 0usize;
for slice in slices {
if slice.file_byte_end <= offset || slice.file_byte_start >= window_end {
continue;
}
let from_abs = offset.max(slice.file_byte_start);
let to_abs = window_end.min(slice.file_byte_end);
let want = (to_abs - from_abs) as usize;
if want == 0 {
continue;
}
let n = store
.decoded_range_into(
slice.drop_id.as_bytes(),
from_abs - slice.file_byte_start,
&mut buf[filled..filled + want],
)
.transpose()?
.ok_or_else(|| CoreError::Corrupt {
reason: "slice references a drop missing from every slab".into(),
})?;
filled += n;
if filled == buf.len() || slice.file_byte_end >= window_end {
break;
}
}
Ok(filled)
}
_ => Ok(0),
}
}
pub fn extract_file(
image_path: &Path,
file_path: &str,
writer: &mut dyn std::io::Write,
config: ReadConfig,
) -> Result<(), CoreError> {
let io_err = |e: std::io::Error| CoreError::Corrupt {
reason: format!("extract_file: {e}"),
};
let reader = ImageReader::open(image_path, config.clone())?;
let file = reader.file(file_path)?;
match &file.inode.content_handle {
ContentHandle::InlineData(d) => writer.write_all(d).map_err(io_err),
ContentHandle::SliceMap(slices) => {
if config.parallel_decode && slices.len() > 1 {
use rayon::prelude::*;
let store = file.store;
const BATCH: usize = 8;
for batch in slices.chunks(BATCH) {
let decoded: Vec<std::sync::Arc<[u8]>> = batch
.par_iter()
.map(|s| {
store
.decoded(s.drop_id.as_bytes())
.transpose()?
.ok_or_else(|| CoreError::Corrupt {
reason: "slice references a drop missing from every slab"
.into(),
})
})
.collect::<Result<_, CoreError>>()?;
for bytes in decoded {
writer.write_all(&bytes).map_err(io_err)?;
}
}
Ok(())
} else {
let mut file = file;
std::io::copy(&mut file, writer).map_err(io_err)?;
Ok(())
}
}
_ => Ok(()),
}
}
#[must_use]
pub fn slices_of(inode: &Inode) -> &[SliceRef] {
match &inode.content_handle {
ContentHandle::SliceMap(s) => s,
_ => &[],
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::PathBuf;
struct Fixture {
dir: std::path::PathBuf,
drops: usize,
}
impl Fixture {
fn pack(src: &Path) -> Fixture {
let art = limnifs_write::write_directory(src).expect("write_directory");
let dir = std::env::temp_dir().join(format!(
"limnifs-read-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.subsec_nanos())
));
std::fs::create_dir_all(&dir).expect("mkdir img");
let manifest_path = dir.join("image.lim");
std::fs::write(&manifest_path, &art.bytes).expect("manifest");
for slab in &art.slabs {
let name =
crate::locator::local_sidecar_name(&slab.locator).expect("flat slab locator");
std::fs::write(dir.join(name), &slab.bytes).expect("slab");
}
if let Some(side) = &art.metadata_sidecar {
let name = crate::locator::local_sidecar_name(&side.locator)
.expect("flat metadata locator");
std::fs::write(dir.join(name), &side.bytes).expect("metadata sidecar");
}
Fixture {
dir,
drops: art.drop_count,
}
}
fn image(&self) -> PathBuf {
self.dir.join("image.lim")
}
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.dir);
}
}
fn xorshift_file(bytes: usize) -> Vec<u8> {
let mut state = 0x0123_4567_89AB_CDEFu64;
let mut out = Vec::with_capacity(bytes);
while out.len() < bytes {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
out.extend_from_slice(&state.to_le_bytes());
}
out.truncate(bytes);
out
}
fn build_src(tag: &str) -> std::path::PathBuf {
let src = std::env::temp_dir().join(format!(
"limnifs-read-src-{tag}-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.subsec_nanos())
));
std::fs::create_dir_all(&src).expect("mkdir src");
src
}
#[test]
fn file_reader_random_windows_equal_sequential() {
let mut whole = Vec::new();
for i in 0..96 {
whole.extend_from_slice(&vec![(i % 251) as u8; 16 * 1024]);
}
let src = build_src("windows");
std::fs::write(src.join("multi.bin"), &whole).expect("write multi");
let fx = Fixture::pack(&src);
let reader = ImageReader::open(&fx.image(), ReadConfig::default()).expect("open");
let mut file = reader.file("/multi.bin").expect("file");
assert_eq!(file.size(), whole.len() as u64);
let mut state = 0xFEED_FACE_CAFE_BABEu64;
let mut next = || {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
state
};
let mut buf = vec![0u8; 8192];
for _ in 0..200 {
let off = (next() % (whole.len() as u64 + 1)) as usize;
let n = file.read_at(off as u64, &mut buf).expect("read_at");
assert_eq!(&buf[..n], &whole[off..off + n], "off={off}");
}
let mut seq = Vec::new();
file.read_to_end(&mut seq).expect("read_to_end");
assert_eq!(seq, whole);
assert!(
reader.cache_stats().hits + reader.cache_stats().misses > 0,
"drop cache should have seen traffic (got {:?})",
reader.cache_stats()
);
let _ = std::fs::remove_dir_all(&src);
}
#[test]
fn deep_window_decodes_only_covering_drops() {
let mut whole = Vec::with_capacity(96 * 16 * 1024);
let mut state = 0xD1CE_D1CE_D1CE_D1CEu64;
while whole.len() < 96 * 16 * 1024 {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
whole.push((state >> 56) as u8);
}
let src = build_src("deepwin");
std::fs::write(src.join("multi.bin"), &whole).expect("write multi");
let fx = Fixture::pack(&src);
assert!(
fx.drops >= 5,
"fixture must span many drops (got {})",
fx.drops
);
let reader = ImageReader::open(&fx.image(), ReadConfig::default()).expect("open");
let file = reader.file("/multi.bin").expect("file");
let before = reader.cache_stats().misses;
let mut last = [0u8; 1];
let n = file
.read_at(whole.len() as u64 - 1, &mut last)
.expect("read_at tail");
assert_eq!(n, 1);
assert_eq!(last[0], whole[whole.len() - 1]);
let decoded = reader.cache_stats().misses - before;
assert!(
decoded <= 2,
"one-byte tail window decoded {decoded} drops; only the covering one may decode"
);
let reader2 = ImageReader::open(&fx.image(), ReadConfig::default()).expect("open");
let file2 = reader2.file("/multi.bin").expect("file");
let before2 = reader2.cache_stats().misses;
let n = file2.read_at(0, &mut last).expect("read_at head");
assert_eq!(n, 1);
assert_eq!(last[0], whole[0]);
let decoded2 = reader2.cache_stats().misses - before2;
assert!(
decoded2 <= 1,
"one-byte head window decoded {decoded2} drops"
);
let _ = std::fs::remove_dir_all(&src);
let _ = std::fs::remove_dir_all(&fx.dir);
}
#[test]
fn extract_file_matches_content_both_modes() {
let whole = xorshift_file(900 * 1024);
let src = build_src("extract");
std::fs::write(src.join("data.bin"), &whole).expect("write data");
let fx = Fixture::pack(&src);
for parallel in [false, true] {
let mut out = Vec::new();
let cfg = ReadConfig {
parallel_decode: parallel,
..ReadConfig::default()
};
extract_file(&fx.image(), "/data.bin", &mut out, cfg).expect("extract");
assert_eq!(out, whole, "parallel={parallel}");
}
let _ = std::fs::remove_dir_all(&src);
}
#[test]
fn file_rejects_missing_and_directory_paths() {
let src = build_src("reject");
std::fs::write(src.join("a.txt"), b"hi").expect("write a");
std::fs::create_dir_all(src.join("sub")).expect("mkdir sub");
let fx = Fixture::pack(&src);
let reader = ImageReader::open(&fx.image(), ReadConfig::default()).expect("open");
assert!(reader.file("/nope.txt").is_err());
assert!(reader.file("/sub").is_err());
assert!(reader.file("/a.txt").is_ok());
let _ = std::fs::remove_dir_all(&src);
}
}