#![forbid(unsafe_code)]
use std::fs::File;
use std::io::BufWriter;
use std::path::Path;
use crate::data::{Dataset, Frames, Layout, Tomo, Volume};
use crate::error::{Error, Result};
use ndarray::{Array3, Axis};
use rayon::prelude::*;
use rust_hdf5::{ByteOrder, DatatypeMessage, H5Dataset, H5File, Hdf5Error, VarLenUnicode};
use tiff::encoder::{colortype::Gray32Float, TiffEncoder};
pub mod dxchange {
pub const DATA: &str = "/exchange/data";
pub const DATA_WHITE: &str = "/exchange/data_white";
pub const DATA_DARK: &str = "/exchange/data_dark";
pub const THETA: &str = "/exchange/theta";
}
pub trait DatasetReader {
fn read_sizes(&mut self) -> Result<(usize, usize, usize, usize, usize)>;
fn read_theta(&mut self) -> Result<Vec<f32>>;
fn read_all(&mut self) -> Result<Dataset<f32>>;
fn read_chunk(&mut self, _row0: usize, _row1: usize) -> Result<Dataset<f32>> {
Err(Error::Io(
"read_chunk: out-of-core reads not supported by this reader".into(),
))
}
fn read_chunk_into(
&mut self,
_row0: usize,
_row1: usize,
_data_out: &mut [f32],
) -> Result<ChunkAux> {
Err(Error::Io(
"read_chunk_into: out-of-core reads not supported by this reader".into(),
))
}
}
pub struct ChunkAux {
pub flat: Option<Frames<f32>>,
pub dark: Option<Frames<f32>>,
pub theta: Vec<f32>,
}
pub struct RowBandReader {
inner: Box<dyn DatasetReader>,
r0: usize,
r1: usize,
}
impl RowBandReader {
pub fn new(mut inner: Box<dyn DatasetReader>, r0: usize, r1: usize) -> Result<Self> {
let (_nproj, nz, _nx, _nflat, _ndark) = inner.read_sizes()?;
let r1 = r1.min(nz);
if r0 >= r1 {
return Err(Error::InvalidParam(format!(
"RowBandReader: empty row band [{r0}, {r1}) in a {nz}-row dataset"
)));
}
Ok(RowBandReader { inner, r0, r1 })
}
fn to_inner(&self, row0: usize, row1: usize) -> Result<(usize, usize)> {
let nz = self.r1 - self.r0;
if row0 > row1 || row1 > nz {
return Err(Error::InvalidParam(format!(
"RowBandReader: chunk [{row0}, {row1}) outside the band height {nz}"
)));
}
Ok((self.r0 + row0, self.r0 + row1))
}
}
impl DatasetReader for RowBandReader {
fn read_sizes(&mut self) -> Result<(usize, usize, usize, usize, usize)> {
let (nproj, _nz, nx, nflat, ndark) = self.inner.read_sizes()?;
Ok((nproj, self.r1 - self.r0, nx, nflat, ndark))
}
fn read_theta(&mut self) -> Result<Vec<f32>> {
self.inner.read_theta()
}
fn read_all(&mut self) -> Result<Dataset<f32>> {
self.inner.read_chunk(self.r0, self.r1)
}
fn read_chunk(&mut self, row0: usize, row1: usize) -> Result<Dataset<f32>> {
let (row0, row1) = self.to_inner(row0, row1)?;
self.inner.read_chunk(row0, row1)
}
fn read_chunk_into(
&mut self,
row0: usize,
row1: usize,
data_out: &mut [f32],
) -> Result<ChunkAux> {
let (row0, row1) = self.to_inner(row0, row1)?;
self.inner.read_chunk_into(row0, row1, data_out)
}
}
pub trait VolumeWriter {
fn reserve(&mut self, _total_nz: usize) -> Result<()> {
Ok(())
}
fn write_chunk(&mut self, vol: &Volume<f32>, start: usize, end: usize) -> Result<()>;
fn finalize(&mut self) -> Result<()> {
Ok(())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)]
pub enum SaveFormat {
#[default]
Tiff,
H5,
Zarr,
}
impl std::str::FromStr for SaveFormat {
type Err = Error;
fn from_str(s: &str) -> Result<Self> {
match s.trim().to_ascii_lowercase().as_str() {
"tiff" | "tif" => Ok(SaveFormat::Tiff),
"h5" | "hdf5" => Ok(SaveFormat::H5),
"zarr" => Ok(SaveFormat::Zarr),
other => Err(Error::InvalidParam(format!(
"unknown save format {other:?} (expected tiff | h5 | zarr)"
))),
}
}
}
#[derive(Debug, Default)]
pub struct InMemoryVolume {
total_nz: usize,
dims: Option<(usize, usize)>,
data: Vec<f32>,
}
impl InMemoryVolume {
pub fn dims(&self) -> Option<(usize, usize, usize)> {
self.dims.map(|(ny, nx)| (self.total_nz, ny, nx))
}
pub fn data(&self) -> &[f32] {
&self.data
}
pub fn to_volume(&self) -> Option<Volume<f32>> {
let (nz, ny, nx) = self.dims()?;
let array = Array3::from_shape_vec((nz, ny, nx), self.data.clone())
.expect("data length is kept at nz*ny*nx by write_chunk");
Some(Volume::new(array))
}
}
#[derive(Default)]
pub struct InMemoryWriter {
vol: std::sync::Arc<std::sync::Mutex<InMemoryVolume>>,
on_chunk: Option<Box<dyn FnMut(usize, usize) + Send>>,
}
impl InMemoryWriter {
pub fn new() -> Self {
Self::default()
}
pub fn buffer(&self) -> std::sync::Arc<std::sync::Mutex<InMemoryVolume>> {
std::sync::Arc::clone(&self.vol)
}
pub fn with_on_chunk(mut self, on_chunk: impl FnMut(usize, usize) + Send + 'static) -> Self {
self.on_chunk = Some(Box::new(on_chunk));
self
}
}
impl VolumeWriter for InMemoryWriter {
fn reserve(&mut self, total_nz: usize) -> Result<()> {
let mut vol = self.vol.lock().expect("InMemoryVolume lock poisoned");
vol.total_nz = total_nz;
Ok(())
}
fn write_chunk(&mut self, chunk: &Volume<f32>, start: usize, end: usize) -> Result<()> {
let (cz, ny, nx) = chunk.array.dim();
{
let mut vol = self.vol.lock().expect("InMemoryVolume lock poisoned");
if cz != end - start || end > vol.total_nz {
return Err(Error::ShapeMismatch {
expected: format!(
"chunk of {} slices within [0, {})",
end.saturating_sub(start),
vol.total_nz
),
found: format!("{cz} slices at [{start}, {end})"),
});
}
match vol.dims {
None => {
vol.dims = Some((ny, nx));
let total = vol.total_nz;
vol.data = vec![0.0; total * ny * nx];
}
Some(d) if d != (ny, nx) => {
return Err(Error::ShapeMismatch {
expected: format!("cross-section {:?} (from first chunk)", d),
found: format!("({ny}, {nx})"),
});
}
Some(_) => {}
}
let dst = &mut vol.data[start * ny * nx..end * ny * nx];
if let Some(src) = chunk.array.as_slice() {
dst.copy_from_slice(src);
} else {
for (d, s) in dst.iter_mut().zip(chunk.array.iter()) {
*d = *s;
}
}
}
if let Some(cb) = &mut self.on_chunk {
cb(start, end);
}
Ok(())
}
}
pub fn open_dxchange(path: &str) -> Result<Box<dyn DatasetReader>> {
Ok(Box::new(H5DxchangeReader::open(path)?))
}
pub fn read_h5_frame(
path: &str,
data_path: &str,
index: usize,
) -> Result<(usize, usize, Vec<f32>)> {
let file = H5File::open(path).map_err(|e| Error::Io(format!("open {path}: {e}")))?;
let key = data_path.strip_prefix('/').unwrap_or(data_path);
let ds = file
.dataset(key)
.map_err(|e| Error::Io(format!("dataset {data_path}: {e}")))?;
let shape = ds.shape();
let [n, ny, nx] = shape[..] else {
return Err(Error::ShapeMismatch {
expected: "[n, ny, nx] (3-D stack)".into(),
found: format!("{shape:?}"),
});
};
if index >= n {
return Err(Error::InvalidParam(format!(
"frame {index} out of range (stack has {n})"
)));
}
let data = read_f32_slice(&ds, &[index, 0, 0], &[1, ny, nx])?;
Ok((ny, nx, data))
}
pub fn read_h5_band(
path: &str,
data_path: &str,
z0: usize,
z1: usize,
) -> Result<(usize, usize, usize, Vec<f32>)> {
let file = H5File::open(path).map_err(|e| Error::Io(format!("open {path}: {e}")))?;
let key = data_path.strip_prefix('/').unwrap_or(data_path);
let ds = file
.dataset(key)
.map_err(|e| Error::Io(format!("dataset {data_path}: {e}")))?;
let shape = ds.shape();
let [n, ny, nx] = shape[..] else {
return Err(Error::ShapeMismatch {
expected: "[n, ny, nx] (3-D stack)".into(),
found: format!("{shape:?}"),
});
};
if z0 >= z1 || z1 > n {
return Err(Error::InvalidParam(format!(
"band [{z0}, {z1}) out of range (stack has {n} slices)"
)));
}
let band = z1 - z0;
let data = read_f32_slice(&ds, &[z0, 0, 0], &[band, ny, nx])?;
Ok((band, ny, nx, data))
}
pub fn list_h5_datasets(path: &str) -> Result<Vec<String>> {
let file = H5File::open(path).map_err(|e| Error::Io(format!("open {path}: {e}")))?;
Ok(file.dataset_names())
}
pub fn read_h5_sizes(path: &str, data_path: &str) -> Result<(usize, usize, usize)> {
let file = H5File::open(path).map_err(|e| Error::Io(format!("open {path}: {e}")))?;
let key = data_path.strip_prefix('/').unwrap_or(data_path);
let ds = file
.dataset(key)
.map_err(|e| Error::Io(format!("dataset {data_path}: {e}")))?;
let shape = ds.shape();
let [n, ny, nx] = shape[..] else {
return Err(Error::ShapeMismatch {
expected: "[n, ny, nx] (3-D stack)".into(),
found: format!("{shape:?}"),
});
};
Ok((n, ny, nx))
}
struct H5DxchangeReader {
file: H5File,
}
impl H5DxchangeReader {
fn open(path: &str) -> Result<Self> {
let file = H5File::open(path).map_err(|e| Error::Io(format!("open {path}: {e}")))?;
Ok(Self { file })
}
fn required(&self, path: &str) -> Result<H5Dataset> {
self.optional(path)?
.ok_or_else(|| Error::Io(format!("dataset {path}: not found")))
}
fn optional(&self, path: &str) -> Result<Option<H5Dataset>> {
let key = path.strip_prefix('/').unwrap_or(path);
match self.file.dataset(key) {
Ok(ds) => Ok(Some(ds)),
Err(Hdf5Error::NotFound(_)) => Ok(None),
Err(e) => Err(Error::Io(format!("dataset {path}: {e}"))),
}
}
fn data_shape(&self) -> Result<[usize; 3]> {
let shape = self.required(dxchange::DATA)?.shape();
match shape.as_slice() {
&[nproj, nz, nx] => Ok([nproj, nz, nx]),
other => Err(Error::ShapeMismatch {
expected: "3-D [nproj, nz, nx]".into(),
found: format!("{other:?}"),
}),
}
}
}
impl DatasetReader for H5DxchangeReader {
fn read_sizes(&mut self) -> Result<(usize, usize, usize, usize, usize)> {
let [nproj, nz, nx] = self.data_shape()?;
let nflat = match self.optional(dxchange::DATA_WHITE)? {
Some(ds) => ds.shape().first().copied().unwrap_or(0),
None => 0,
};
let ndark = match self.optional(dxchange::DATA_DARK)? {
Some(ds) => ds.shape().first().copied().unwrap_or(0),
None => 0,
};
Ok((nproj, nz, nx, nflat, ndark))
}
fn read_theta(&mut self) -> Result<Vec<f32>> {
if let Some(ds) = self.optional(dxchange::THETA)? {
let deg = read_f32_vec(&ds)?;
Ok(deg
.into_iter()
.map(|d| d / 180.0 * std::f32::consts::PI)
.collect())
} else {
let nproj = self.data_shape()?[0];
Ok(linspace_inclusive(nproj))
}
}
fn read_all(&mut self) -> Result<Dataset<f32>> {
let [nproj, nz, nx] = self.data_shape()?;
let data = read_f32_array(&self.required(dxchange::DATA)?, (nproj, nz, nx))?;
let flat = match self.optional(dxchange::DATA_WHITE)? {
Some(ds) => Some(Frames::new(read_frames(&ds, nz, nx)?)),
None => None,
};
let dark = match self.optional(dxchange::DATA_DARK)? {
Some(ds) => Some(Frames::new(read_frames(&ds, nz, nx)?)),
None => None,
};
let theta = self.read_theta()?;
Ok(Dataset {
data: Tomo::new(data, Layout::Projection),
flat,
dark,
theta,
})
}
fn read_chunk(&mut self, row0: usize, row1: usize) -> Result<Dataset<f32>> {
let [nproj, nz, nx] = self.data_shape()?;
if row0 > row1 || row1 > nz {
return Err(Error::InvalidParam(format!(
"read_chunk rows [{row0}, {row1}) out of range for nz={nz}"
)));
}
let rows = row1 - row0;
let slab = |ds: &H5Dataset, n: usize| -> Result<Array3<f32>> {
let v = read_f32_slice(ds, &[0, row0, 0], &[n, rows, nx])?;
Array3::from_shape_vec((n, rows, nx), v).map_err(|e| Error::ShapeMismatch {
expected: format!("[{n}, {rows}, {nx}]"),
found: e.to_string(),
})
};
let data = slab(&self.required(dxchange::DATA)?, nproj)?;
let flat = match self.optional(dxchange::DATA_WHITE)? {
Some(ds) => {
let nf = ds.shape().first().copied().unwrap_or(0);
Some(Frames::new(slab(&ds, nf)?))
}
None => None,
};
let dark = match self.optional(dxchange::DATA_DARK)? {
Some(ds) => {
let nd = ds.shape().first().copied().unwrap_or(0);
Some(Frames::new(slab(&ds, nd)?))
}
None => None,
};
let theta = self.read_theta()?;
Ok(Dataset {
data: Tomo::new(data, Layout::Projection),
flat,
dark,
theta,
})
}
fn read_chunk_into(
&mut self,
row0: usize,
row1: usize,
data_out: &mut [f32],
) -> Result<ChunkAux> {
let [nproj, nz, nx] = self.data_shape()?;
if row0 > row1 || row1 > nz {
return Err(Error::InvalidParam(format!(
"read_chunk_into rows [{row0}, {row1}) out of range for nz={nz}"
)));
}
let rows = row1 - row0;
let expected = nproj * rows * nx;
if data_out.len() != expected {
return Err(Error::ShapeMismatch {
expected: format!("{expected} elems ([{nproj}, {rows}, {nx}])"),
found: format!("{} elems", data_out.len()),
});
}
read_f32_slice_into(
&self.required(dxchange::DATA)?,
&[0, row0, 0],
&[nproj, rows, nx],
data_out,
)?;
let slab = |ds: &H5Dataset, n: usize| -> Result<Array3<f32>> {
let v = read_f32_slice(ds, &[0, row0, 0], &[n, rows, nx])?;
Array3::from_shape_vec((n, rows, nx), v).map_err(|e| Error::ShapeMismatch {
expected: format!("[{n}, {rows}, {nx}]"),
found: e.to_string(),
})
};
let flat = match self.optional(dxchange::DATA_WHITE)? {
Some(ds) => {
let nf = ds.shape().first().copied().unwrap_or(0);
Some(Frames::new(slab(&ds, nf)?))
}
None => None,
};
let dark = match self.optional(dxchange::DATA_DARK)? {
Some(ds) => {
let nd = ds.shape().first().copied().unwrap_or(0);
Some(Frames::new(slab(&ds, nd)?))
}
None => None,
};
let theta = self.read_theta()?;
Ok(ChunkAux { flat, dark, theta })
}
}
fn linspace_inclusive(n: usize) -> Vec<f32> {
match n {
0 => Vec::new(),
1 => vec![0.0],
_ => {
let step = std::f32::consts::PI / (n - 1) as f32;
(0..n).map(|i| i as f32 * step).collect()
}
}
}
fn read_frames(ds: &H5Dataset, nz: usize, nx: usize) -> Result<Array3<f32>> {
let nframe = ds.shape().first().copied().unwrap_or(0);
read_f32_array(ds, (nframe, nz, nx))
}
fn read_f32_array(ds: &H5Dataset, dims: (usize, usize, usize)) -> Result<Array3<f32>> {
let v = read_f32_vec(ds)?;
Array3::from_shape_vec(dims, v).map_err(|e| Error::ShapeMismatch {
expected: format!("{dims:?}"),
found: e.to_string(),
})
}
fn read_f32_vec(ds: &H5Dataset) -> Result<Vec<f32>> {
let dt = ds
.datatype()
.map_err(|e| Error::Io(format!("datatype: {e}")))?;
let raw = |e: Hdf5Error| Error::Io(format!("read: {e}"));
macro_rules! cast {
($t:ty) => {
ds.read_raw::<$t>()
.map_err(raw)?
.into_par_iter()
.map(|x| x as f32)
.collect()
};
}
let v: Vec<f32> = match dt {
DatatypeMessage::FloatingPoint {
size: 4,
byte_order,
..
} => {
ensure_le(byte_order)?;
ds.read_raw::<f32>().map_err(raw)?
}
DatatypeMessage::FloatingPoint {
size: 8,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(f64)
}
DatatypeMessage::FixedPoint {
size: 1,
signed: false,
..
} => cast!(u8),
DatatypeMessage::FixedPoint {
size: 1,
signed: true,
..
} => cast!(i8),
DatatypeMessage::FixedPoint {
size: 2,
signed: false,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(u16)
}
DatatypeMessage::FixedPoint {
size: 2,
signed: true,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(i16)
}
DatatypeMessage::FixedPoint {
size: 4,
signed: false,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(u32)
}
DatatypeMessage::FixedPoint {
size: 4,
signed: true,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(i32)
}
DatatypeMessage::FixedPoint {
size: 8,
signed: false,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(u64)
}
DatatypeMessage::FixedPoint {
size: 8,
signed: true,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(i64)
}
other => {
return Err(Error::Io(format!(
"unsupported HDF5 datatype for numeric read: {other:?}"
)))
}
};
Ok(v)
}
fn read_f32_slice(ds: &H5Dataset, starts: &[usize], counts: &[usize]) -> Result<Vec<f32>> {
let dt = ds
.datatype()
.map_err(|e| Error::Io(format!("datatype: {e}")))?;
let raw = |e: Hdf5Error| Error::Io(format!("read: {e}"));
macro_rules! cast {
($t:ty) => {
ds.read_slice::<$t>(starts, counts)
.map_err(raw)?
.into_par_iter()
.map(|x| x as f32)
.collect()
};
}
let v: Vec<f32> = match dt {
DatatypeMessage::FloatingPoint {
size: 4,
byte_order,
..
} => {
ensure_le(byte_order)?;
ds.read_slice::<f32>(starts, counts).map_err(raw)?
}
DatatypeMessage::FloatingPoint {
size: 8,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(f64)
}
DatatypeMessage::FixedPoint {
size: 1,
signed: false,
..
} => cast!(u8),
DatatypeMessage::FixedPoint {
size: 1,
signed: true,
..
} => cast!(i8),
DatatypeMessage::FixedPoint {
size: 2,
signed: false,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(u16)
}
DatatypeMessage::FixedPoint {
size: 2,
signed: true,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(i16)
}
DatatypeMessage::FixedPoint {
size: 4,
signed: false,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(u32)
}
DatatypeMessage::FixedPoint {
size: 4,
signed: true,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(i32)
}
DatatypeMessage::FixedPoint {
size: 8,
signed: false,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(u64)
}
DatatypeMessage::FixedPoint {
size: 8,
signed: true,
byte_order,
..
} => {
ensure_le(byte_order)?;
cast!(i64)
}
other => {
return Err(Error::Io(format!(
"unsupported HDF5 datatype for numeric read: {other:?}"
)))
}
};
Ok(v)
}
fn read_f32_slice_into(
ds: &H5Dataset,
starts: &[usize],
counts: &[usize],
out: &mut [f32],
) -> Result<()> {
let expected: usize = counts.iter().product();
if out.len() != expected {
return Err(Error::ShapeMismatch {
expected: format!("{expected} elems"),
found: format!("{} elems", out.len()),
});
}
let dt = ds
.datatype()
.map_err(|e| Error::Io(format!("datatype: {e}")))?;
match dt {
DatatypeMessage::FloatingPoint {
size: 4,
byte_order,
..
} => {
ensure_le(byte_order)?;
ds.read_slice_into::<f32>(out, starts, counts)
.map_err(|e| Error::Io(format!("read: {e}")))?;
Ok(())
}
_ => {
let v = read_f32_slice(ds, starts, counts)?;
out.copy_from_slice(&v);
Ok(())
}
}
}
fn ensure_le(byte_order: ByteOrder) -> Result<()> {
match byte_order {
ByteOrder::LittleEndian => Ok(()),
ByteOrder::BigEndian => Err(Error::Io(
"big-endian HDF5 datasets are not supported (read_raw does not byte-swap)".into(),
)),
}
}
pub fn create_writer(path: &str, format: SaveFormat) -> Result<Box<dyn VolumeWriter>> {
match format {
SaveFormat::Tiff => Ok(Box::new(TiffWriter::new(path)?)),
SaveFormat::H5 => Ok(Box::new(H5Writer::new(path)?)),
SaveFormat::Zarr => Ok(Box::new(ZarrWriter::new(path)?)),
}
}
struct TiffWriter {
prefix: String,
}
impl TiffWriter {
fn new(path: &str) -> Result<Self> {
if let Some(parent) = Path::new(path).parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent)
.map_err(|e| Error::Io(format!("create dir {}: {e}", parent.display())))?;
}
}
Ok(Self {
prefix: path.to_string(),
})
}
}
impl VolumeWriter for TiffWriter {
fn write_chunk(&mut self, vol: &Volume<f32>, start: usize, end: usize) -> Result<()> {
let (cz, ny, nx) = vol.dims();
if start > end {
return Err(Error::InvalidParam(format!(
"write_chunk: inverted global range [{start}, {end})"
)));
}
if cz != end - start {
return Err(Error::InvalidParam(format!(
"write_chunk: volume has {cz} slices but global range [{start}, {end}) expects {}",
end - start
)));
}
for local in 0..cz {
let slice = vol.array.index_axis(Axis(0), local);
let gathered;
let buf: &[f32] = match slice.as_slice() {
Some(s) => s,
None => {
gathered = slice.iter().copied().collect::<Vec<f32>>();
&gathered
}
};
let global = start + local;
let fname = format!("{}_{global:05}.tiff", self.prefix);
let file =
File::create(&fname).map_err(|e| Error::Io(format!("create {fname}: {e}")))?;
let mut enc = TiffEncoder::new(BufWriter::new(file))
.map_err(|e| Error::Io(format!("tiff encoder {fname}: {e}")))?;
enc.write_image::<Gray32Float>(nx as u32, ny as u32, buf)
.map_err(|e| Error::Io(format!("tiff write {fname}: {e}")))?;
}
Ok(())
}
}
struct H5Writer {
path: std::path::PathBuf,
total_nz: Option<usize>,
state: Option<H5WriteState>,
}
struct H5WriteState {
file: H5File,
dataset: H5Dataset,
dims: (usize, usize, usize),
}
impl H5Writer {
fn new(path: &str) -> Result<Self> {
let out = if path.ends_with(".h5") {
std::path::PathBuf::from(path)
} else {
std::path::PathBuf::from(format!("{path}.h5"))
};
if let Some(parent) = out.parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent)
.map_err(|e| Error::Io(format!("create dir {}: {e}", parent.display())))?;
}
}
Ok(Self {
path: out,
total_nz: None,
state: None,
})
}
fn create_dataset(&self, nz: usize, ny: usize, nx: usize) -> Result<H5WriteState> {
match std::fs::remove_file(&self.path) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => {
return Err(Error::Io(format!(
"remove stale {}: {e}",
self.path.display()
)))
}
}
let file = H5File::create(&self.path)
.map_err(|e| Error::Io(format!("create {}: {e}", self.path.display())))?;
let group = file
.create_group("exchange")
.map_err(|e| Error::Io(format!("create group exchange: {e}")))?;
let dataset = group
.new_dataset::<f32>()
.shape([nz, ny, nx])
.create("data")
.map_err(|e| Error::Io(format!("create dataset /exchange/data: {e}")))?;
for (name, value) in [
("axes", "z:y:x"),
("description", "ReconData"),
("units", "counts"),
] {
dataset
.new_attr::<VarLenUnicode>()
.shape(())
.create(name)
.map_err(|e| Error::Io(format!("create attr {name}: {e}")))?
.write_string(value)
.map_err(|e| Error::Io(format!("write attr {name}: {e}")))?;
}
Ok(H5WriteState {
file,
dataset,
dims: (nz, ny, nx),
})
}
}
impl VolumeWriter for H5Writer {
fn reserve(&mut self, total_nz: usize) -> Result<()> {
self.total_nz = Some(total_nz);
Ok(())
}
fn write_chunk(&mut self, vol: &Volume<f32>, start: usize, end: usize) -> Result<()> {
let (cz, ny, nx) = vol.dims();
if start > end {
return Err(Error::InvalidParam(format!(
"write_chunk: inverted global range [{start}, {end})"
)));
}
if cz != end - start {
return Err(Error::InvalidParam(format!(
"write_chunk: volume has {cz} slices but global range [{start}, {end}) expects {}",
end - start
)));
}
let total_nz = self.total_nz.ok_or_else(|| {
Error::InvalidParam(
"write_chunk: reserve(total_nz) must be called before writing H5 chunks".into(),
)
})?;
if end > total_nz {
return Err(Error::InvalidParam(format!(
"write_chunk: global range [{start}, {end}) exceeds the reserved {total_nz} slices"
)));
}
if self.state.is_none() {
self.state = Some(self.create_dataset(total_nz, ny, nx)?);
}
let state = self.state.as_ref().unwrap();
if state.dims != (total_nz, ny, nx) {
return Err(Error::InvalidParam(format!(
"write_chunk: chunk cross-section {:?} differs from the created dataset {:?}",
(ny, nx),
(state.dims.1, state.dims.2)
)));
}
if cz == 0 {
return Ok(());
}
let write = |slab: &[f32]| {
state
.dataset
.write_slice(&[start, 0, 0], &[cz, ny, nx], slab)
.map_err(|e| Error::Io(format!("write /exchange/data[{start}..{end}]: {e}")))
};
match vol.array.as_slice() {
Some(slab) => write(slab)?,
None => {
let slab: Vec<f32> = vol.array.iter().copied().collect();
write(&slab)?;
}
}
Ok(())
}
fn finalize(&mut self) -> Result<()> {
if let Some(state) = self.state.take() {
let H5WriteState { file, dataset, .. } = state;
drop(dataset);
file.close_no_sync()
.map_err(|e| Error::Io(format!("close {}: {e}", self.path.display())))?;
}
Ok(())
}
}
struct ZarrWriter {
root: std::path::PathBuf,
total_nz: Option<usize>,
dims: Option<(usize, usize, usize)>,
}
impl ZarrWriter {
fn new(path: &str) -> Result<Self> {
let root = if path.ends_with(".zarr") {
std::path::PathBuf::from(path)
} else {
std::path::PathBuf::from(format!("{path}.zarr"))
};
if let Some(parent) = root.parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent)
.map_err(|e| Error::Io(format!("create dir {}: {e}", parent.display())))?;
}
}
Ok(Self {
root,
total_nz: None,
dims: None,
})
}
fn init_store(&self, nz: usize, ny: usize, nx: usize) -> Result<()> {
let data_dir = self.root.join("exchange").join("data");
std::fs::create_dir_all(&data_dir)
.map_err(|e| Error::Io(format!("create dir {}: {e}", data_dir.display())))?;
let write = |rel: std::path::PathBuf, body: String| -> Result<()> {
std::fs::write(&rel, body)
.map_err(|e| Error::Io(format!("write {}: {e}", rel.display())))
};
let zgroup = "{\n \"zarr_format\": 2\n}".to_string();
write(self.root.join(".zgroup"), zgroup.clone())?;
write(self.root.join("exchange").join(".zgroup"), zgroup)?;
let zarray = format!(
"{{\n \"chunks\": [1, {ny}, {nx}],\n \"compressor\": null,\n \
\"dtype\": \"<f4\",\n \"fill_value\": 0.0,\n \"filters\": null,\n \
\"order\": \"C\",\n \"shape\": [{nz}, {ny}, {nx}],\n \
\"zarr_format\": 2\n}}"
);
write(data_dir.join(".zarray"), zarray)?;
let zattrs = "{\n \"axes\": \"z:y:x\",\n \"description\": \"ReconData\",\n \
\"units\": \"counts\"\n}"
.to_string();
write(data_dir.join(".zattrs"), zattrs)?;
Ok(())
}
}
impl VolumeWriter for ZarrWriter {
fn reserve(&mut self, total_nz: usize) -> Result<()> {
self.total_nz = Some(total_nz);
Ok(())
}
fn write_chunk(&mut self, vol: &Volume<f32>, start: usize, end: usize) -> Result<()> {
let (cz, ny, nx) = vol.dims();
if start > end {
return Err(Error::InvalidParam(format!(
"write_chunk: inverted global range [{start}, {end})"
)));
}
if cz != end - start {
return Err(Error::InvalidParam(format!(
"write_chunk: volume has {cz} slices but global range [{start}, {end}) expects {}",
end - start
)));
}
let total_nz = self.total_nz.ok_or_else(|| {
Error::InvalidParam(
"write_chunk: reserve(total_nz) must be called before writing Zarr chunks".into(),
)
})?;
if end > total_nz {
return Err(Error::InvalidParam(format!(
"write_chunk: global range [{start}, {end}) exceeds the reserved {total_nz} slices"
)));
}
if self.dims.is_none() {
self.init_store(total_nz, ny, nx)?;
self.dims = Some((total_nz, ny, nx));
}
let dims = self.dims.unwrap();
if dims != (total_nz, ny, nx) {
return Err(Error::InvalidParam(format!(
"write_chunk: chunk cross-section {:?} differs from the created store {:?}",
(ny, nx),
(dims.1, dims.2)
)));
}
let data_dir = self.root.join("exchange").join("data");
for local in 0..cz {
let slice = vol.array.index_axis(Axis(0), local);
let gathered;
let bytes: &[u8] = match slice.as_slice() {
Some(s) if cfg!(target_endian = "little") => bytemuck::cast_slice(s),
_ => {
gathered = slice
.iter()
.flat_map(|v| v.to_le_bytes())
.collect::<Vec<u8>>();
&gathered
}
};
let global = start + local;
let chunk = data_dir.join(format!("{global}.0.0"));
std::fs::write(&chunk, bytes)
.map_err(|e| Error::Io(format!("write {}: {e}", chunk.display())))?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use ndarray::Slice;
#[test]
fn dxchange_paths_are_stable() {
assert_eq!(dxchange::DATA, "/exchange/data");
}
#[test]
fn open_missing_file_is_io_error() {
assert!(matches!(
open_dxchange("definitely-not-a-real-file.h5"),
Err(Error::Io(_))
));
}
#[test]
fn read_h5_frame_reads_one_u16_frame() {
let path = std::env::temp_dir().join("tomoxide_read_h5_frame_test.h5");
let _ = std::fs::remove_file(&path);
{
let file = H5File::create(&path).unwrap();
let group = file.create_group("exchange").unwrap();
let ds = group
.new_dataset::<u16>()
.shape([2, 2, 3])
.create("data")
.unwrap();
let vals: Vec<u16> = (0..12).collect();
ds.write_raw(&vals).unwrap();
}
let p = path.to_str().unwrap();
let (ny, nx, f1) = read_h5_frame(p, "/exchange/data", 1).unwrap();
assert_eq!((ny, nx), (2, 3));
assert_eq!(f1, vec![6.0, 7.0, 8.0, 9.0, 10.0, 11.0]);
let (_, _, f0) = read_h5_frame(p, "exchange/data", 0).unwrap();
assert_eq!(f0, vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0]);
assert!(matches!(
read_h5_frame(p, "/exchange/data", 2),
Err(Error::InvalidParam(_))
));
assert_eq!(read_h5_sizes(p, "/exchange/data").unwrap(), (2, 2, 3));
assert_eq!(read_h5_sizes(p, "exchange/data").unwrap(), (2, 2, 3));
assert!(matches!(
read_h5_sizes(p, "/exchange/missing"),
Err(Error::Io(_))
));
let _ = std::fs::remove_file(&path);
}
#[test]
fn tiff_writer_streams_per_chunk_volumes_with_global_indices() {
let dir = std::env::temp_dir().join("tomoxide_tiff_stream_test");
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let prefix = dir.join("recon");
let prefix = prefix.to_str().unwrap();
let mut w = create_writer(prefix, SaveFormat::Tiff).unwrap();
let chunk0 = Volume::new(Array3::from_shape_fn((2, 2, 3), |(z, y, x)| {
(z * 100 + y * 10 + x) as f32
}));
let chunk1 = Volume::new(Array3::from_shape_fn((2, 2, 3), |(z, y, x)| {
((2 + z) * 100 + y * 10 + x) as f32
}));
w.write_chunk(&chunk0, 0, 2).unwrap();
w.write_chunk(&chunk1, 2, 4).unwrap();
for g in 0..4 {
let f = format!("{prefix}_{g:05}.tiff");
assert!(std::path::Path::new(&f).is_file(), "missing {f}");
}
let bad = Volume::new(Array3::<f32>::zeros((3, 2, 3)));
assert!(matches!(
w.write_chunk(&bad, 4, 6),
Err(Error::InvalidParam(_))
));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn zarr_writer_roundtrips_a_spec_compliant_store() {
let base = std::env::temp_dir().join("tomoxide_zarr_writer_roundtrip");
let root = base.with_extension("zarr");
let _ = std::fs::remove_dir_all(&root);
let arr = Array3::from_shape_fn((3, 2, 2), |(z, y, x)| (z * 100 + y * 10 + x) as f32);
let mut w = create_writer(base.to_str().unwrap(), SaveFormat::Zarr).unwrap();
w.reserve(3).unwrap();
let c0 = Volume::new(arr.slice_axis(Axis(0), Slice::from(0..2)).to_owned());
let c1 = Volume::new(arr.slice_axis(Axis(0), Slice::from(2..3)).to_owned());
w.write_chunk(&c0, 0, 2).unwrap();
w.write_chunk(&c1, 2, 3).unwrap();
assert!(root.join(".zgroup").is_file());
assert!(root.join("exchange").join(".zgroup").is_file());
let data = root.join("exchange").join("data");
let zarray = std::fs::read_to_string(data.join(".zarray")).unwrap();
assert!(zarray.contains("\"shape\": [3, 2, 2]"), "{zarray}");
assert!(zarray.contains("\"chunks\": [1, 2, 2]"), "{zarray}");
assert!(zarray.contains("\"dtype\": \"<f4\""), "{zarray}");
assert!(zarray.contains("\"compressor\": null"), "{zarray}");
assert!(data.join(".zattrs").is_file());
for z in 0..3 {
let bytes = std::fs::read(data.join(format!("{z}.0.0"))).unwrap();
assert_eq!(bytes.len(), 2 * 2 * 4, "chunk {z} byte length");
let got: Vec<f32> = bytes
.as_chunks::<4>()
.0
.iter()
.map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
.collect();
let want: Vec<f32> = arr.index_axis(Axis(0), z).iter().copied().collect();
assert_eq!(got, want, "slice {z} round-trip");
}
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn zarr_writer_rejects_changed_dims() {
let base = std::env::temp_dir().join("tomoxide_zarr_writer_dims");
let root = base.with_extension("zarr");
let _ = std::fs::remove_dir_all(&root);
let mut w = create_writer(base.to_str().unwrap(), SaveFormat::Zarr).unwrap();
w.reserve(2).unwrap();
let a = Volume::new(Array3::zeros((2, 4, 4)));
w.write_chunk(&a, 0, 2).unwrap();
let b = Volume::new(Array3::zeros((2, 8, 8)));
assert!(matches!(
w.write_chunk(&b, 0, 2),
Err(Error::InvalidParam(_))
));
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn h5_writer_streams_per_chunk_volumes_and_round_trips() {
let base = std::env::temp_dir().join(format!("tomoxide_h5_stream_{}", std::process::id()));
let base_str = base.to_str().unwrap();
let out = std::path::PathBuf::from(format!("{base_str}.h5"));
let _ = std::fs::remove_file(&out);
let arr = Array3::from_shape_fn((3, 2, 2), |(z, y, x)| (z * 100 + y * 10 + x) as f32);
let mut w = create_writer(base_str, SaveFormat::H5).unwrap();
w.reserve(3).unwrap();
let c0 = Volume::new(arr.slice_axis(Axis(0), Slice::from(0..2)).to_owned());
let c1 = Volume::new(arr.slice_axis(Axis(0), Slice::from(2..3)).to_owned());
w.write_chunk(&c0, 0, 2).unwrap();
w.write_chunk(&c1, 2, 3).unwrap();
drop(w);
let f = H5File::open(&out).unwrap();
let ds = f.dataset("exchange/data").unwrap();
let got = ds.read_raw::<f32>().unwrap();
let want: Vec<f32> = arr.iter().copied().collect();
assert_eq!(got, want, "h5 streamed round-trip");
let _ = std::fs::remove_file(&out);
}
#[test]
fn h5_and_zarr_reject_write_before_reserve() {
let v = Volume::new(Array3::<f32>::zeros((1, 2, 2)));
let zb =
std::env::temp_dir().join(format!("tomoxide_zarr_noreserve_{}", std::process::id()));
let mut zw = create_writer(zb.to_str().unwrap(), SaveFormat::Zarr).unwrap();
assert!(matches!(
zw.write_chunk(&v, 0, 1),
Err(Error::InvalidParam(_))
));
let _ = std::fs::remove_dir_all(zb.with_extension("zarr"));
let hb = std::env::temp_dir().join(format!("tomoxide_h5_noreserve_{}", std::process::id()));
let mut hw = create_writer(hb.to_str().unwrap(), SaveFormat::H5).unwrap();
assert!(matches!(
hw.write_chunk(&v, 0, 1),
Err(Error::InvalidParam(_))
));
let _ = std::fs::remove_file(format!("{}.h5", hb.to_str().unwrap()));
}
#[test]
fn linspace_inclusive_matches_numpy() {
assert_eq!(linspace_inclusive(0), Vec::<f32>::new());
assert_eq!(linspace_inclusive(1), vec![0.0]);
let a = linspace_inclusive(5);
assert_eq!(a.len(), 5);
assert_eq!(a[0], 0.0);
assert_eq!(a[4], std::f32::consts::PI);
assert!((a[2] - std::f32::consts::FRAC_PI_2).abs() < 1e-6);
}
}