use crate::VoxelBlock;
use crate::engine::block::VolumeShape;
use crate::engine::endian::FileEndian;
use crate::mode::Voxel;
use crate::{Error, Header, Mode};
use std::borrow::Cow;
use std::path::Path;
use std::vec::Vec;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum CompressionType {
Plain,
#[cfg(feature = "gzip")]
Gzip,
#[cfg(feature = "bzip2")]
Bzip2,
}
#[doc(hidden)]
pub fn detect_compression_from_bytes(bytes: &[u8]) -> CompressionType {
if bytes.len() < 2 {
return CompressionType::Plain;
}
let magic = [bytes[0], bytes[1]];
#[cfg(feature = "gzip")]
if magic == [0x1f, 0x8b] {
return CompressionType::Gzip;
}
#[cfg(feature = "bzip2")]
if magic == [b'B', b'Z'] {
return CompressionType::Bzip2;
}
CompressionType::Plain
}
pub fn detect_compression<P: AsRef<Path>>(path: P) -> Result<CompressionType, Error> {
use std::fs::File;
use std::io::Read;
let mut file = File::open(path)?;
let mut buf = [0u8; 2];
let n = file.read(&mut buf)?;
Ok(detect_compression_from_bytes(&buf[..n]))
}
#[derive(Debug)]
enum DataSource {
Buffered {
data: Vec<u8>,
truncated: bool,
},
#[cfg(feature = "mmap")]
Mmap {
map: memmap2::Mmap,
data_offset: usize,
truncated: bool,
},
}
#[derive(Debug)]
pub struct Reader {
pub(crate) header: Header,
pub(crate) ext_header: Vec<u8>,
pub(crate) endian: FileEndian,
pub(crate) mode: Mode,
pub(crate) shape: VolumeShape,
source: DataSource,
}
impl Reader {
pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self, Error> {
Self::_open_detect(path.as_ref(), false).map(|(r, _)| r)
}
pub fn open_permissive<P: AsRef<std::path::Path>>(
path: P,
) -> Result<(Self, Vec<String>), Error> {
Self::_open_detect(path.as_ref(), true)
}
pub fn open_plain<P: AsRef<std::path::Path>>(path: P) -> Result<Self, Error> {
Self::_open_plain(path, false).map(|(r, _)| r)
}
pub fn from_reader<R: std::io::Read>(mut reader: R) -> Result<Self, Error> {
let mut buf = Vec::new();
reader.read_to_end(&mut buf)?;
Self::_read_from_buf(buf, false).map(|(r, _)| r)
}
pub fn from_reader_permissive<R: std::io::Read>(
mut reader: R,
) -> Result<(Self, Vec<String>), Error> {
let mut buf = Vec::new();
reader.read_to_end(&mut buf)?;
Self::_read_from_buf(buf, true)
}
pub fn from_bytes(data: Vec<u8>) -> Result<Self, Error> {
Self::_read_from_buf(data, false).map(|(r, _)| r)
}
pub fn from_bytes_permissive(data: Vec<u8>) -> Result<(Self, Vec<String>), Error> {
Self::_read_from_buf(data, true)
}
fn _open_detect(
path: &std::path::Path,
permissive: bool,
) -> Result<(Self, Vec<String>), Error> {
use std::io::{Read, Seek};
let mut file = std::fs::File::open(path)?;
let mut magic = [0u8; 2];
let n = file.read(&mut magic)?;
if n >= 2 {
match magic {
#[cfg(feature = "gzip")]
[0x1f, 0x8b] => {
let _ = file.seek(std::io::SeekFrom::Start(0));
return Self::_open_gzip_file(
file,
permissive,
crate::io::reader_common::DEFAULT_MAX_DECOMPRESSED_BYTES,
);
}
#[cfg(feature = "bzip2")]
[b'B', b'Z'] => {
let _ = file.seek(std::io::SeekFrom::Start(0));
return Self::_open_bzip2_file(
file,
permissive,
crate::io::reader_common::DEFAULT_MAX_DECOMPRESSED_BYTES,
);
}
_ => {}
}
}
#[cfg(feature = "mmap")]
{
drop(file);
if let Ok(result) = Self::_open_mmap_path(path, permissive) {
return Ok(result);
}
let file = std::fs::File::open(path)?;
Self::_open_plain_file(file, permissive)
}
#[cfg(not(feature = "mmap"))]
{
let _ = file.seek(std::io::SeekFrom::Start(0));
Self::_open_plain_file(file, permissive)
}
}
fn _open_plain<P: AsRef<std::path::Path>>(
path: P,
permissive: bool,
) -> Result<(Self, Vec<String>), Error> {
Self::_open_plain_file(std::fs::File::open(path)?, permissive)
}
fn _open_plain_file(
mut file: std::fs::File,
permissive: bool,
) -> Result<(Self, Vec<String>), Error> {
use std::io::Read;
let mut header_bytes = [0u8; 1024];
file.read_exact(&mut header_bytes)?;
let (header, warnings, _endian, data_size) =
crate::io::reader_common::parse_header(&header_bytes, permissive)?;
let ext_size = header.nsymbt as usize;
let mut ext_header = vec![0u8; ext_size];
if ext_size > 0 {
file.read_exact(&mut ext_header)?;
}
let mut data = vec![0u8; data_size];
file.read_exact(&mut data)?;
if !permissive {
let file_len = file.metadata()?.len() as usize;
let expected_len = header.data_offset() + data_size;
if file_len != expected_len {
return Err(Error::FileSizeMismatch {
expected: expected_len,
actual: file_len,
});
}
}
Self::_build(
header,
ext_header,
DataSource::Buffered {
data,
truncated: false,
},
warnings,
)
}
fn _read_from_buf(data: Vec<u8>, permissive: bool) -> Result<(Self, Vec<String>), Error> {
if data.len() < 1024 {
return Err(Error::InvalidHeader);
}
let mut header_bytes = [0u8; 1024];
header_bytes.copy_from_slice(&data[..1024]);
let (header, mut warnings, _endian, data_size) =
crate::io::reader_common::parse_header(&header_bytes, permissive)?;
let ext_size = header.nsymbt as usize;
let ext_end = (1024 + ext_size).min(data.len());
let ext_header = if ext_size > 0 && ext_end > 1024 {
if ext_end < 1024 + ext_size {
warnings.push(format!(
"Extended header truncated: expected {} bytes, got {}",
ext_size,
ext_end - 1024
));
}
data[1024..ext_end].to_vec()
} else {
Vec::new()
};
let data_offset = header.data_offset();
let voxel_data = if data_offset < data.len() {
let available = data.len() - data_offset;
let expected = data_size.min(available);
data[data_offset..data_offset + expected].to_vec()
} else {
Vec::new()
};
if !permissive && voxel_data.len() != data_size {
return Err(Error::FileSizeMismatch {
expected: header.data_offset() + data_size,
actual: data.len(),
});
}
let truncated = voxel_data.len() != data_size;
Self::_build(
header,
ext_header,
DataSource::Buffered {
data: voxel_data,
truncated,
},
warnings,
)
}
#[cfg(feature = "mmap")]
fn _open_mmap_path(
path: &std::path::Path,
permissive: bool,
) -> Result<(Self, Vec<String>), Error> {
use std::fs::File;
let file = File::open(path)?;
let mmap = unsafe {
memmap2::MmapOptions::new()
.map(&file)
.map_err(|_| Error::Mmap)?
};
if mmap.len() < 1024 {
return Err(Error::InvalidHeader);
}
let mut header_bytes = [0u8; 1024];
header_bytes.copy_from_slice(&mmap[..1024]);
let (header, warnings, _endian, data_size) =
crate::io::reader_common::parse_header(&header_bytes, permissive)?;
let expected_size = header
.data_offset()
.checked_add(data_size)
.ok_or(Error::InvalidHeader)?;
let truncated = if !permissive {
if mmap.len() != expected_size {
return Err(Error::FileSizeMismatch {
expected: expected_size,
actual: mmap.len(),
});
}
false
} else if mmap.len() < header.data_offset() {
return Err(Error::FileSizeMismatch {
expected: header.data_offset(),
actual: mmap.len(),
});
} else {
mmap.len() < expected_size
};
Self::_build(
header,
Vec::new(), DataSource::Mmap {
map: mmap,
data_offset: header.data_offset(),
truncated,
},
warnings,
)
}
fn _build(
header: Header,
ext_header: Vec<u8>,
source: DataSource,
warnings: Vec<String>,
) -> Result<(Self, Vec<String>), Error> {
let shape = VolumeShape::new(header.nx as usize, header.ny as usize, header.nz as usize);
let mode = Mode::from_i32(header.mode).ok_or(Error::UnsupportedMode)?;
let endian = header.detect_endian();
let mut warnings = warnings;
if mode == Mode::Int8 {
if let Some(imod) = header.detect_imod() {
if !imod.bytes_are_signed {
warnings.push(
"IMOD file with unsigned Mode 0 detected: use slices_mode0() \
or convert::<f32>() for correct values"
.into(),
);
}
}
}
Ok((
Self {
header,
ext_header,
endian,
mode,
shape,
source,
},
warnings,
))
}
pub(crate) fn _from_decompressed(
d: crate::io::reader_common::DecompressedMrc,
) -> Result<(Self, Vec<String>), Error> {
Self::_build(
d.header,
d.ext_header,
DataSource::Buffered {
data: d.data,
truncated: false,
},
d.warnings,
)
}
}
impl Reader {
pub fn shape(&self) -> VolumeShape {
self.shape
}
pub fn mode(&self) -> Mode {
self.mode
}
pub fn header(&self) -> &Header {
&self.header
}
pub fn endian(&self) -> FileEndian {
self.endian
}
pub fn data_bytes(&self) -> &[u8] {
match &self.source {
DataSource::Buffered { data, .. } => data,
#[cfg(feature = "mmap")]
DataSource::Mmap {
map, data_offset, ..
} => {
let data_size = self.header.data_size().unwrap_or(0);
let end = data_offset + data_size;
if end > map.len() {
&map[*data_offset..]
} else {
&map[*data_offset..end]
}
}
}
}
pub fn ext_header_bytes(&self) -> &[u8] {
if !self.ext_header.is_empty() {
return &self.ext_header;
}
let ext_size = self.header.nsymbt.max(0) as usize;
if ext_size == 0 {
return &[];
}
match &self.source {
#[cfg(feature = "mmap")]
DataSource::Mmap { map, .. } => {
if 1024 + ext_size <= map.len() {
&map[1024..1024 + ext_size]
} else {
&[]
}
}
_ => &[],
}
}
pub fn is_truncated(&self) -> bool {
match &self.source {
DataSource::Buffered { truncated, .. } => *truncated,
#[cfg(feature = "mmap")]
DataSource::Mmap { truncated, .. } => *truncated,
}
}
pub fn validate_header_stats(&self) -> Result<(), Error> {
crate::engine::stats::validate_header_stats(&self.header, self.data_bytes())
}
}
impl Reader {
pub fn read_block_bytes(
&self,
offset: [usize; 3],
shape: [usize; 3],
) -> Result<Vec<u8>, Error> {
let data = self._source_data();
let data_len = data.len();
crate::io::reader_common::validate_block_bounds(
self.shape,
self.mode(),
data_len,
offset,
shape,
)?;
Ok(crate::io::reader_common::gather_block_bytes(
data,
self.shape,
self.mode(),
offset,
shape,
))
}
pub fn slab_as<T: Voxel>(&self, z: usize, k: usize) -> Result<&[T], Error> {
#[cfg(feature = "mmap")]
if let DataSource::Mmap {
map, data_offset, ..
} = &self.source
{
return self._slab_as_mmap::<T>(map, *data_offset, z, k);
}
Err(Error::TypeMismatch {
expected: 0,
actual: 0,
})
}
#[cfg(feature = "mmap")]
fn _slab_as_mmap<'a, T: Voxel>(
&self,
map: &'a memmap2::Mmap,
data_offset: usize,
z: usize,
k: usize,
) -> Result<&'a [T], Error> {
if T::MODE != self.mode() {
return Err(Error::ModeMismatch {
file_mode: self.mode(),
requested_mode: T::MODE,
offset: None,
});
}
if !self.endian.is_native() {
return Err(Error::TypeMismatch {
expected: T::BYTE_SIZE,
actual: 0,
});
}
let b = T::BYTE_SIZE;
let [nx, ny, nz] = [self.shape.nx, self.shape.ny, self.shape.nz];
if k == 0 || z + k > nz {
return Err(Error::bounds_err());
}
let linear_start = z * nx * ny;
let byte_start = data_offset + linear_start * b;
let count = nx * ny * k;
let byte_end = byte_start + count * b;
if byte_end > map.len() {
return Err(Error::bounds_err());
}
if byte_start % core::mem::align_of::<T>() != 0 {
return Err(Error::TypeMismatch {
expected: core::mem::align_of::<T>(),
actual: byte_start % core::mem::align_of::<T>(),
});
}
unsafe {
let ptr = map.as_ptr().add(byte_start) as *const T;
Ok(core::slice::from_raw_parts(ptr, count))
}
}
pub(crate) fn read_block_bytes_cow<'a>(
&'a self,
offset: [usize; 3],
shape: [usize; 3],
) -> Result<Cow<'a, [u8]>, Error> {
match &self.source {
DataSource::Buffered { data, .. } => {
let data_len = data.len();
crate::io::reader_common::validate_block_bounds(
self.shape,
self.mode(),
data_len,
offset,
shape,
)?;
Ok(Cow::Owned(crate::io::reader_common::gather_block_bytes(
data,
self.shape,
self.mode(),
offset,
shape,
)))
}
#[cfg(feature = "mmap")]
DataSource::Mmap {
map, data_offset, ..
} => {
let [nx, ny, ..] = [self.shape.nx, self.shape.ny, self.shape.nz];
let [ox, oy, oz] = offset;
let [sx, sy, sz] = shape;
let data_len = map.len().saturating_sub(*data_offset);
crate::io::reader_common::validate_block_bounds(
self.shape,
self.mode(),
data_len,
offset,
shape,
)?;
if ox == 0 && sx == nx && oy == 0 && sy == ny {
let (start_offset, byte_len) = if self.mode == Mode::Packed4Bit {
let row_bytes = nx.div_ceil(2);
(data_offset + oz * ny * row_bytes, row_bytes * ny * sz)
} else {
let linear = oz * nx * ny;
let b = self.mode.byte_size();
(data_offset + linear * b, sx * sy * sz * b)
};
return Ok(Cow::Borrowed(&map[start_offset..start_offset + byte_len]));
}
Ok(Cow::Owned(crate::io::reader_common::gather_block_bytes(
&map[*data_offset..],
self.shape,
self.mode(),
offset,
shape,
)))
}
}
}
pub(crate) fn decode_block<T: Voxel>(&self, bytes: &[u8]) -> Result<Vec<T>, Error> {
crate::io::reader_common::decode_block(bytes, self.mode(), self.endian)
}
fn _source_data(&self) -> &[u8] {
match &self.source {
DataSource::Buffered { data, .. } => data,
#[cfg(feature = "mmap")]
DataSource::Mmap {
map, data_offset, ..
} => &map[*data_offset..],
}
}
pub fn slices<T: Voxel>(&self) -> crate::iter::RegionIter<'_, T, crate::iter::SliceStepper> {
crate::iter::RegionIter::with_stepper(
self,
self.shape(),
crate::iter::SliceStepper::default(),
)
}
pub fn slabs<T: Voxel>(
&self,
k: usize,
) -> crate::iter::RegionIter<'_, T, crate::iter::SlabStepper> {
crate::iter::RegionIter::with_stepper(self, self.shape(), crate::iter::SlabStepper::new(k))
}
pub fn tiles<T: Voxel>(
&self,
tile_shape: [usize; 3],
) -> Result<crate::iter::RegionIter<'_, T, crate::iter::TileStepper>, Error> {
Ok(crate::iter::RegionIter::with_stepper(
self,
self.shape(),
crate::iter::TileStepper::new(tile_shape)?,
))
}
pub fn volumes<T: Voxel>(
&self,
) -> Result<crate::iter::RegionIter<'_, T, crate::iter::SlabStepper>, Error> {
let mz = self.header().mz.max(0) as usize;
if !self.header().is_volume_stack() || mz == 0 {
return Err(Error::NotAVolumeStack {
ispg: self.header().ispg,
mz: self.header().mz,
});
}
Ok(self.slabs(mz))
}
pub fn subregion<T: Voxel>(
&self,
offset: [usize; 3],
shape: [usize; 3],
) -> Result<VoxelBlock<T>, Error> {
let bytes = self.read_block_bytes_cow(offset, shape)?;
let data = self.decode_block::<T>(&bytes)?;
Ok(VoxelBlock {
offset,
shape,
data,
})
}
pub fn read_volume<T: Voxel>(&self) -> Result<VoxelBlock<T>, Error> {
self.subregion([0, 0, 0], [self.shape.nx, self.shape.ny, self.shape.nz])
}
pub fn slices_u8(&self) -> crate::io::reader_common::VoxelIter<'_, u8> {
if self.mode() == Mode::Packed4Bit {
let shape = self.shape();
let nx = shape.nx;
let ny = shape.ny;
let nz = shape.nz;
return Box::new((0..nz).map(move |z| {
let bytes = self.read_block_bytes_cow([0, 0, z], [nx, ny, 1])?;
let data = crate::engine::convert::unpack_u4_bytes_to_u8(&bytes, nx, ny);
Ok(VoxelBlock {
offset: [0, 0, z],
shape: [nx, ny, 1],
data,
})
}));
}
if self.mode() != Mode::Uint16 {
return Box::new(std::iter::once(Err(Error::ModeMismatch {
file_mode: self.mode(),
requested_mode: Mode::Uint16,
offset: None,
})));
}
Box::new(self.slices::<u16>().map(|b| {
let b = b?;
Ok(VoxelBlock {
offset: b.offset,
shape: b.shape,
data: crate::engine::convert::convert_u16_slice_to_u8(&b.data)?,
})
}))
}
pub fn slabs_u8(&self, k: usize) -> crate::io::reader_common::VoxelIter<'_, u8> {
if self.mode() == Mode::Packed4Bit {
let volume_shape = self.shape();
let nx = volume_shape.nx;
let ny = volume_shape.ny;
let k = k.max(1);
let mut z = 0usize;
return Box::new(std::iter::from_fn(move || {
if z >= volume_shape.nz {
return None;
}
let start = z;
let sz = k.min(volume_shape.nz - z);
z += sz;
let bytes = match self.read_block_bytes_cow([0, 0, start], [nx, ny, sz]) {
Ok(b) => b,
Err(e) => return Some(Err(e)),
};
let data = crate::engine::convert::unpack_u4_bytes_to_u8(&bytes, nx, ny * sz);
Some(Ok(VoxelBlock {
offset: [0, 0, start],
shape: [nx, ny, sz],
data,
}))
}));
}
if self.mode() != Mode::Uint16 {
return Box::new(std::iter::once(Err(Error::ModeMismatch {
file_mode: self.mode(),
requested_mode: Mode::Uint16,
offset: None,
})));
}
let k = k.max(1);
Box::new(self.slabs::<u16>(k).map(|b| {
let b = b?;
Ok(VoxelBlock {
offset: b.offset,
shape: b.shape,
data: crate::engine::convert::convert_u16_slice_to_u8(&b.data)?,
})
}))
}
pub fn slices_mode0(
&self,
interp: crate::M0Interpretation,
) -> crate::io::reader_common::VoxelIter<'_, f32> {
if self.mode() != Mode::Int8 {
return Box::new(std::iter::once(Err(Error::ModeMismatch {
file_mode: self.mode(),
requested_mode: Mode::Int8,
offset: None,
})));
}
let volume_shape = self.shape();
Box::new((0..volume_shape.nz).map(move |z| {
let bytes =
self.read_block_bytes_cow([0, 0, z], [volume_shape.nx, volume_shape.ny, 1])?;
let data = crate::engine::convert::reinterpret_m0(&bytes, interp);
Ok(VoxelBlock {
offset: [0, 0, z],
shape: [volume_shape.nx, volume_shape.ny, 1],
data,
})
}))
}
pub fn slabs_mode0(
&self,
k: usize,
interp: crate::M0Interpretation,
) -> crate::io::reader_common::VoxelIter<'_, f32> {
if self.mode() != Mode::Int8 {
return Box::new(std::iter::once(Err(Error::ModeMismatch {
file_mode: self.mode(),
requested_mode: Mode::Int8,
offset: None,
})));
}
let volume_shape = self.shape();
let k = k.max(1);
let mut z = 0usize;
Box::new(std::iter::from_fn(move || {
if z >= volume_shape.nz {
return None;
}
let start = z;
let sz = k.min(volume_shape.nz - z);
z += sz;
let bytes = match self
.read_block_bytes_cow([0, 0, start], [volume_shape.nx, volume_shape.ny, sz])
{
Ok(b) => b,
Err(e) => return Some(Err(e)),
};
let data = crate::engine::convert::reinterpret_m0(&bytes, interp);
Some(Ok(VoxelBlock {
offset: [0, 0, start],
shape: [volume_shape.nx, volume_shape.ny, sz],
data,
}))
}))
}
pub fn convert<T>(&self) -> crate::io::reader_common::ConvertReader<'_, T>
where
T: Voxel + crate::engine::convert::ConvertFrom<f32>,
{
let m0_interp = if self.mode() == Mode::Int8 {
if let Some(imod) = self.header().detect_imod() {
if !imod.bytes_are_signed {
crate::M0Interpretation::Unsigned
} else {
crate::M0Interpretation::Signed
}
} else {
crate::M0Interpretation::Signed
}
} else {
crate::M0Interpretation::Signed
};
crate::io::reader_common::ConvertReader {
reader: self,
complex_strategy: crate::ComplexToRealStrategy::Magnitude,
m0_interp,
_target: std::marker::PhantomData,
}
}
#[cfg(feature = "ndarray")]
pub fn to_ndarray<T: Voxel>(&self) -> Result<ndarray::Array3<T>, Error> {
let block = self.read_volume::<T>()?;
ndarray::Array3::from_shape_vec([self.shape.nz, self.shape.ny, self.shape.nx], block.data)
.map_err(|_| Error::bounds_err())
}
pub fn read_volume_u8(&self) -> Result<VoxelBlock<u8>, Error> {
if self.mode() != Mode::Packed4Bit {
return Err(Error::ModeMismatch {
file_mode: self.mode(),
requested_mode: Mode::Packed4Bit,
offset: None,
});
}
let shape = self.shape();
let block_shape = [shape.nx, shape.ny, shape.nz];
let bytes = self.read_block_bytes_cow([0, 0, 0], block_shape)?;
let data =
crate::engine::convert::unpack_u4_bytes_to_u8(&bytes, shape.nx, shape.ny * shape.nz);
Ok(VoxelBlock {
offset: [0, 0, 0],
shape: block_shape,
data,
})
}
pub fn parse_extended_header(&self) -> crate::ExtHeaderData {
crate::ExtHeaderData::from_header(&self.header, self.ext_header_bytes())
}
pub fn fei1_metadata(&self) -> Option<Vec<crate::Fei1Metadata>> {
crate::parse_fei1_records(self.ext_header_bytes())
}
pub fn fei2_metadata(&self) -> Option<Vec<crate::Fei2Metadata>> {
crate::parse_fei2_records(self.ext_header_bytes())
}
pub fn ccp4_records(&self) -> Option<Vec<crate::Ccp4Record>> {
crate::parse_ccp4_records(self.ext_header_bytes())
}
pub fn mrco_records(&self) -> Option<Vec<crate::MrcoRecord>> {
crate::parse_mrco_records(self.ext_header_bytes())
}
pub fn seri_records(&self) -> Option<Vec<crate::SeriRecord>> {
crate::parse_seri_records(self.ext_header_bytes())
}
pub fn agar_records(&self) -> Option<Vec<crate::AgarRecord>> {
crate::parse_agar_records(self.ext_header_bytes())
}
pub fn imod_metadata(&self) -> Option<crate::ImodMetadata> {
crate::parse_imod_metadata(&self.header)
}
}