use crate::data::{Dataset, Frames, Layout, Tomo, Volume};
use crate::error::Result;
use crate::geometry::Geometry;
use crate::params::{Algorithm, PhaseMethod, ReconParams, StripeMethod};
use ndarray::Array3;
use crate::engine::Engine;
#[derive(Clone, Copy, Debug, Default)]
pub struct PrepOptions {
pub stripe: StripeMethod,
pub phase: PhaseMethod,
}
pub fn reconstruct(
mut ds: Dataset<f32>,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
engine: &Engine,
) -> Result<Volume<f32>> {
let backend = engine.backend();
crate::prep::normalize_dataset(&mut ds, backend)?;
crate::prep::retrieve_phase(&mut ds.data, prep.phase, backend)?;
let mut sino = ds.data.to_layout(Layout::Sinogram);
crate::prep::remove_stripe(&mut sino, prep.stripe)?;
crate::recon::recon(&sino, geom, algorithm, params, backend)
}
pub struct ReconSteps {
pub chunk_rows: usize,
}
impl ReconSteps {
pub fn new(chunk_rows: usize) -> Self {
ReconSteps {
chunk_rows: chunk_rows.max(1),
}
}
#[allow(clippy::too_many_arguments)]
pub fn run(
&self,
reader: &mut dyn crate::io::DatasetReader,
writer: &mut dyn crate::io::VolumeWriter,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
engine: &Engine,
) -> Result<()> {
let backend = engine.backend();
let mut ds = reader.read_all()?;
crate::prep::normalize_dataset(&mut ds, backend)?;
crate::prep::retrieve_phase(&mut ds.data, prep.phase, backend)?;
let sino = ds.data.as_layout(Layout::Sinogram); let nz = sino.n_rows();
writer.reserve(nz)?;
let chunk = self.chunk_rows.max(1);
let mut r0 = 0;
let mut recon = None; while r0 < nz {
let r1 = (r0 + chunk).min(nz);
let sub = sino
.array
.slice_axis(ndarray::Axis(0), ndarray::Slice::from(r0..r1))
.to_owned();
let mut sub = Tomo::new(sub, Layout::Sinogram);
crate::prep::remove_stripe(&mut sub, prep.stripe)?;
let chunk_geom = chunk_geometry(geom, r0, r1);
let vol = recon_chunk_reusing(
&mut recon,
backend,
&sub,
geom,
&chunk_geom,
algorithm,
params,
)?;
writer.write_chunk(&vol, r0, r1)?;
r0 = r1;
}
Ok(())
}
}
impl ReconSteps {
#[allow(clippy::too_many_arguments)]
pub fn run_streaming(
&self,
reader: &mut dyn crate::io::DatasetReader,
writer: &mut dyn crate::io::VolumeWriter,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
engine: &Engine,
) -> Result<()> {
if prep.phase != PhaseMethod::None {
return Err(crate::error::Error::InvalidParam(
"ReconSteps::run_streaming does not support phase retrieval (row-coupled); \
use run() for a phase pipeline"
.into(),
));
}
let backend = engine.backend();
let (_nproj, nz, _nx, _nflat, _ndark) = reader.read_sizes()?;
writer.reserve(nz)?;
let chunk = self.chunk_rows.max(1);
let mut r0 = 0;
let mut recon = None; while r0 < nz {
let r1 = (r0 + chunk).min(nz);
let ds = reader.read_chunk(r0, r1)?;
let chunk_geom = chunk_geometry(geom, r0, r1);
let vol = chunk_to_volume(
&mut recon,
backend,
ds,
geom,
&chunk_geom,
algorithm,
params,
prep,
)?;
writer.write_chunk(&vol, r0, r1)?;
r0 = r1;
}
Ok(())
}
}
const PIPELINE_DEPTH: usize = 2;
impl ReconSteps {
#[allow(clippy::too_many_arguments)]
pub fn run_streaming_pipelined<RF, WF>(
&self,
make_reader: RF,
make_writer: WF,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
engine: &Engine,
) -> Result<()>
where
RF: FnOnce() -> Result<Box<dyn crate::io::DatasetReader>> + Send,
WF: FnOnce() -> Result<Box<dyn crate::io::VolumeWriter>> + Send,
{
self.run_streaming_pipelined_range(
0,
usize::MAX,
make_reader,
make_writer,
geom,
algorithm,
params,
prep,
engine,
)
}
#[allow(clippy::too_many_arguments)]
pub fn run_streaming_pipelined_range<RF, WF>(
&self,
z_start: usize,
z_end: usize,
make_reader: RF,
make_writer: WF,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
engine: &Engine,
) -> Result<()>
where
RF: FnOnce() -> Result<Box<dyn crate::io::DatasetReader>> + Send,
WF: FnOnce() -> Result<Box<dyn crate::io::VolumeWriter>> + Send,
{
if prep.phase != PhaseMethod::None {
return Err(crate::error::Error::InvalidParam(
"ReconSteps::run_streaming_pipelined does not support phase retrieval \
(row-coupled); use run() for a phase pipeline"
.into(),
));
}
let backend = engine.backend();
let chunk = self.chunk_rows.max(1);
let (read_tx, read_rx) =
std::sync::mpsc::sync_channel::<(usize, usize, ChunkMsg)>(PIPELINE_DEPTH);
let (write_tx, write_rx) =
std::sync::mpsc::sync_channel::<(usize, usize, Volume<f32>)>(PIPELINE_DEPTH);
let (nz_tx, nz_rx) = std::sync::mpsc::sync_channel::<usize>(1);
const POOL: usize = PIPELINE_DEPTH + 1;
let (free_tx, free_rx) =
std::sync::mpsc::sync_channel::<Box<dyn crate::backend::HostBuffer>>(POOL);
let free_tx_reader = free_tx.clone();
let (out_free_tx, out_free_rx) = std::sync::mpsc::channel::<Vec<f32>>();
std::thread::scope(|s| {
let reader_handle = s.spawn(move || -> Result<()> {
let mut reader = make_reader()?;
let (nproj, nz, nx, _nflat, _ndark) = reader.read_sizes()?;
let z0 = z_start.min(nz);
let z1 = z_end.min(nz);
let _ = nz_tx.send(nz);
let raw_mode = backend.wants_raw_chunks();
if raw_mode {
let max_len = nproj * chunk.min(nz.max(1)) * nx;
for _ in 0..POOL {
if free_tx_reader
.send(backend.alloc_host_buffer(max_len))
.is_err()
{
return Ok(());
}
}
let mut r0 = z0;
while r0 < z1 {
let r1 = (r0 + chunk).min(z1);
let rows = r1 - r0;
let need = nproj * rows * nx;
let mut buf = match free_rx.recv() {
Ok(b) => b,
Err(_) => break, };
let aux =
reader.read_chunk_into(r0, r1, &mut buf.as_mut_slice()[..need])?;
let msg = ChunkMsg::Raw {
data: buf,
dims: (nproj, rows, nx),
flat: aux.flat,
dark: aux.dark,
theta: aux.theta,
};
if read_tx.send((r0, r1, msg)).is_err() {
break;
}
r0 = r1;
}
} else {
let mut r0 = z0;
while r0 < z1 {
let r1 = (r0 + chunk).min(z1);
let ds = reader.read_chunk(r0, r1)?;
if read_tx.send((r0, r1, ChunkMsg::Host(ds))).is_err() {
break;
}
r0 = r1;
}
}
Ok(())
});
let writer_handle = s.spawn(move || -> Result<()> {
let mut writer = make_writer()?;
if let Ok(total_nz) = nz_rx.recv() {
writer.reserve(total_nz)?;
}
while let Ok((r0, r1, vol)) = write_rx.recv() {
writer.write_chunk(&vol, r0, r1)?;
let _ = out_free_tx.send(vol.array.into_raw_vec_and_offset().0);
}
Ok(())
});
let compute = compute_chunks(
read_rx,
write_tx,
free_tx,
out_free_rx,
backend,
geom,
algorithm,
params,
prep,
);
let read_res = reader_handle.join().expect("reader thread panicked");
let write_res = writer_handle.join().expect("writer thread panicked");
compute.and(read_res).and(write_res)
})
}
}
enum ChunkMsg {
Raw {
data: Box<dyn crate::backend::HostBuffer>,
dims: (usize, usize, usize),
flat: Option<Frames<f32>>,
dark: Option<Frames<f32>>,
theta: Vec<f32>,
},
Host(Dataset<f32>),
}
#[allow(clippy::too_many_arguments)]
fn compute_chunks(
read_rx: std::sync::mpsc::Receiver<(usize, usize, ChunkMsg)>,
write_tx: std::sync::mpsc::SyncSender<(usize, usize, Volume<f32>)>,
free_tx: std::sync::mpsc::SyncSender<Box<dyn crate::backend::HostBuffer>>,
out_free: std::sync::mpsc::Receiver<Vec<f32>>,
backend: &dyn crate::backend::Backend,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
) -> Result<()> {
let mut recon: Option<Option<Box<dyn crate::backend::StreamingAnalytic>>> = None;
while let Ok((r0, r1, msg)) = read_rx.recv() {
while let Ok(buf) = out_free.try_recv() {
if let Some(Some(r)) = recon.as_mut() {
r.give_reuse_buffer(buf);
}
}
let chunk_geom = chunk_geometry(geom, r0, r1);
let vol = match msg {
ChunkMsg::Raw {
data,
dims,
flat,
dark,
theta,
} => {
let vol = raw_chunk_to_volume(
&mut recon,
backend,
data.as_ref(),
dims,
flat,
dark,
theta,
geom,
&chunk_geom,
algorithm,
params,
prep,
)?;
let _ = free_tx.send(data);
vol
}
ChunkMsg::Host(ds) => chunk_to_volume(
&mut recon,
backend,
ds,
geom,
&chunk_geom,
algorithm,
params,
prep,
)?,
};
if write_tx.send((r0, r1, vol)).is_err() {
break;
}
}
Ok(())
}
fn chunk_geometry(geom: &Geometry, r0: usize, r1: usize) -> Geometry {
use crate::geometry::Center;
let center = match &geom.center {
Center::Scalar(c) => Center::Scalar(*c),
Center::PerRow(v) => Center::PerRow(v[r0..r1].to_vec()),
};
Geometry {
center,
..geom.clone()
}
}
#[allow(clippy::too_many_arguments)]
fn chunk_to_volume(
slot: &mut Option<Option<Box<dyn crate::backend::StreamingAnalytic>>>,
backend: &dyn crate::backend::Backend,
ds: Dataset<f32>,
geom: &Geometry,
chunk_geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
) -> Result<Volume<f32>> {
ensure_slot(
slot,
backend,
algorithm,
params,
geom,
ds.data.n_cols(),
ds.data.n_rows(),
)?;
if let Some(Some(s)) = slot.as_mut() {
let std = ds.data.array.as_standard_layout();
let raw = std.as_slice().ok_or_else(|| {
crate::error::Error::InvalidParam("non-contiguous projection chunk".into())
})?;
let dims = ds.data.array.dim();
if let Some(vol) = s.reconstruct_chunk_raw(
raw,
dims,
ds.flat.as_ref(),
ds.dark.as_ref(),
chunk_geom,
prep.stripe,
)? {
return Ok(vol);
}
}
host_path(slot, backend, ds, geom, chunk_geom, algorithm, params, prep)
}
#[allow(clippy::too_many_arguments)]
fn raw_chunk_to_volume(
slot: &mut Option<Option<Box<dyn crate::backend::StreamingAnalytic>>>,
backend: &dyn crate::backend::Backend,
data: &dyn crate::backend::HostBuffer,
dims: (usize, usize, usize),
flat: Option<Frames<f32>>,
dark: Option<Frames<f32>>,
theta: Vec<f32>,
geom: &Geometry,
chunk_geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
) -> Result<Volume<f32>> {
let (nproj, rows, nx) = dims;
let need = nproj * rows * nx;
let raw = &data.as_slice()[..need];
ensure_slot(slot, backend, algorithm, params, geom, nx, rows)?;
if let Some(Some(s)) = slot.as_mut() {
if let Some(vol) = s.reconstruct_chunk_raw(
raw,
dims,
flat.as_ref(),
dark.as_ref(),
chunk_geom,
prep.stripe,
)? {
return Ok(vol);
}
}
let array = Array3::from_shape_vec((nproj, rows, nx), raw.to_vec()).map_err(|e| {
crate::error::Error::ShapeMismatch {
expected: format!("[{nproj}, {rows}, {nx}]"),
found: e.to_string(),
}
})?;
let ds = Dataset {
data: Tomo::new(array, Layout::Projection),
flat,
dark,
theta,
};
host_path(slot, backend, ds, geom, chunk_geom, algorithm, params, prep)
}
fn ensure_slot(
slot: &mut Option<Option<Box<dyn crate::backend::StreamingAnalytic>>>,
backend: &dyn crate::backend::Backend,
algorithm: Algorithm,
params: &ReconParams,
geom: &Geometry,
ncols: usize,
nz: usize,
) -> Result<()> {
if slot.is_none() {
let built = match backend.analytic_reconstruct() {
Some(ar) => ar.streaming(algorithm, params, geom, ncols, nz)?,
None => None,
};
*slot = Some(built);
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn host_path(
slot: &mut Option<Option<Box<dyn crate::backend::StreamingAnalytic>>>,
backend: &dyn crate::backend::Backend,
mut ds: Dataset<f32>,
geom: &Geometry,
chunk_geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
) -> Result<Volume<f32>> {
crate::prep::normalize_dataset(&mut ds, backend)?;
let mut sino = ds.data.to_layout(Layout::Sinogram);
sino.array = sino.array.as_standard_layout().to_owned();
crate::prep::remove_stripe(&mut sino, prep.stripe)?;
recon_chunk_reusing(slot, backend, &sino, geom, chunk_geom, algorithm, params)
}
fn recon_chunk_reusing(
slot: &mut Option<Option<Box<dyn crate::backend::StreamingAnalytic>>>,
backend: &dyn crate::backend::Backend,
sino: &Tomo<f32>,
geom: &Geometry,
chunk_geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
) -> Result<Volume<f32>> {
if slot.is_none() {
let built = match backend.analytic_reconstruct() {
Some(ar) => ar.streaming(algorithm, params, geom, sino.n_cols(), sino.n_rows())?,
None => None,
};
*slot = Some(built);
}
match slot.as_mut().expect("slot built above").as_mut() {
Some(s) => s.reconstruct_chunk(sino, chunk_geom),
None => crate::recon::recon(sino, chunk_geom, algorithm, params, backend),
}
}