use crate::data::{Dataset, Frames, Layout, Tomo, Volume};
use crate::error::Result;
use crate::geometry::Geometry;
use crate::params::{Algorithm, PhaseMethod, ReconParams, StripeMethod};
use ndarray::Array3;
use std::borrow::Cow;
use crate::engine::Engine;
#[derive(Clone, Copy, Debug, Default)]
pub struct PrepOptions {
pub stripe: StripeMethod,
pub phase: PhaseMethod,
}
impl PrepOptions {
pub fn row_separable(&self) -> bool {
self.phase == PhaseMethod::None
}
}
pub fn reconstruct(
mut ds: Dataset<f32>,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
engine: &Engine,
) -> Result<Volume<f32>> {
let backend = engine.backend();
crate::prep::normalize_dataset(&mut ds, backend)?;
crate::prep::retrieve_phase(&mut ds.data, prep.phase, backend)?;
let mut sino = ds.data.to_layout(Layout::Sinogram);
crate::prep::remove_stripe(&mut sino, prep.stripe)?;
crate::recon::recon(&sino, geom, algorithm, params, backend)
}
pub fn prep_band(
reader: &mut dyn crate::io::DatasetReader,
z0: usize,
z1: usize,
prep: &PrepOptions,
backend: &dyn crate::backend::Backend,
) -> Result<Tomo<f32>> {
let (_nproj, nz, _nx, _nflat, _ndark) = reader.read_sizes()?;
let m = crate::prep::phase::margin_rows(&prep.phase);
let (r0, r1) = (z0.saturating_sub(m), (z1 + m).min(nz));
let mut ds = if (r0, r1) == (0, nz) {
reader.read_all()?
} else {
reader.read_chunk(r0, r1)?
};
crate::prep::normalize_dataset(&mut ds, backend)?;
crate::prep::retrieve_phase(&mut ds.data, prep.phase, backend)?;
let rows = match ds.data.layout {
Layout::Projection => ndarray::Axis(1),
Layout::Sinogram => ndarray::Axis(0),
};
ds.data
.array
.slice_axis_inplace(rows, ndarray::Slice::from(z0 - r0..z1 - r0));
let mut sino = ds.data.to_layout(Layout::Sinogram);
crate::prep::remove_stripe(&mut sino, prep.stripe)?;
Ok(sino)
}
pub fn reconstruct_lamino_streaming(
mut ds: Dataset<f32>,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
engine: &Engine,
on_tile: &mut crate::cuda::LaminoTileFn,
) -> Result<(usize, usize, usize)> {
let backend = engine.backend();
if algorithm == Algorithm::Fourierrec
&& prep.phase == PhaseMethod::None
&& prep.stripe == StripeMethod::None
{
let norm = crate::cuda::LamNorm::from_dataset(&ds)?;
return crate::cuda::reconstruct_lamino_streaming(
&ds.data,
geom,
algorithm,
params,
Some(&norm),
None,
on_tile,
);
}
crate::prep::normalize_dataset(&mut ds, backend)?;
lamino_streaming_normalized(
Cow::Owned(ds.data),
geom,
algorithm,
params,
prep,
backend,
on_tile,
)
}
pub(crate) fn lamino_streaming_normalized(
mut data: Cow<'_, Tomo<f32>>,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
backend: &dyn crate::backend::Backend,
on_tile: &mut crate::cuda::LaminoTileFn,
) -> Result<(usize, usize, usize)> {
if prep.phase != PhaseMethod::None {
crate::prep::retrieve_phase(data.to_mut(), prep.phase, backend)?;
}
if prep.stripe == StripeMethod::None {
crate::cuda::reconstruct_lamino_streaming(
&data, geom, algorithm, params, None, None, on_tile,
)
} else {
let mut sino = data.to_layout(Layout::Sinogram);
crate::prep::remove_stripe(&mut sino, prep.stripe)?;
crate::cuda::reconstruct_lamino_streaming(
&sino, geom, algorithm, params, None, None, on_tile,
)
}
}
#[derive(Clone, Debug, Default)]
pub struct CancelToken(std::sync::Arc<std::sync::atomic::AtomicBool>);
impl CancelToken {
pub fn new() -> Self {
Self::default()
}
pub fn cancel(&self) {
self.0.store(true, std::sync::atomic::Ordering::Relaxed);
}
pub fn is_cancelled(&self) -> bool {
self.0.load(std::sync::atomic::Ordering::Relaxed)
}
}
pub(crate) fn check_cancel(cancel: Option<&CancelToken>) -> Result<()> {
match cancel {
Some(c) if c.is_cancelled() => Err(crate::error::Error::Cancelled),
_ => Ok(()),
}
}
pub struct ReconSteps {
pub chunk_rows: usize,
cancel: CancelToken,
}
impl ReconSteps {
pub fn new(chunk_rows: usize) -> Self {
ReconSteps {
chunk_rows: chunk_rows.max(1),
cancel: CancelToken::new(),
}
}
pub fn with_cancel(mut self, cancel: CancelToken) -> Self {
self.cancel = cancel;
self
}
fn check_cancel(&self) -> Result<()> {
check_cancel(Some(&self.cancel))
}
#[allow(clippy::too_many_arguments)]
pub fn run(
&self,
reader: &mut dyn crate::io::DatasetReader,
writer: &mut dyn crate::io::VolumeWriter,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
engine: &Engine,
) -> Result<()> {
let backend = engine.backend();
self.check_cancel()?;
let mut ds = reader.read_all()?;
crate::prep::normalize_dataset(&mut ds, backend)?;
crate::prep::retrieve_phase(&mut ds.data, prep.phase, backend)?;
let sino = ds.data.as_layout(Layout::Sinogram); let nz = sino.n_rows();
writer.reserve(nz)?;
let chunk = self.chunk_rows.max(1);
let mut r0 = 0;
let mut recon = None; while r0 < nz {
self.check_cancel()?;
let r1 = (r0 + chunk).min(nz);
let sub = sino
.array
.slice_axis(ndarray::Axis(0), ndarray::Slice::from(r0..r1))
.to_owned();
let mut sub = Tomo::new(sub, Layout::Sinogram);
crate::prep::remove_stripe(&mut sub, prep.stripe)?;
let chunk_geom = chunk_geometry(geom, r0, r1);
let vol = recon_chunk_reusing(
&mut recon,
backend,
&sub,
geom,
&chunk_geom,
algorithm,
params,
)?;
writer.write_chunk(&vol, r0, r1)?;
r0 = r1;
}
writer.finalize()?;
Ok(())
}
}
impl ReconSteps {
#[allow(clippy::too_many_arguments)]
pub fn run_streaming(
&self,
reader: &mut dyn crate::io::DatasetReader,
writer: &mut dyn crate::io::VolumeWriter,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
engine: &Engine,
) -> Result<()> {
if !prep.row_separable() {
return Err(crate::error::Error::InvalidParam(
"ReconSteps::run_streaming does not support phase retrieval (row-coupled); \
use run() for a phase pipeline"
.into(),
));
}
let backend = engine.backend();
let (_nproj, nz, _nx, _nflat, _ndark) = reader.read_sizes()?;
writer.reserve(nz)?;
let chunk = self.chunk_rows.max(1);
let mut r0 = 0;
let mut recon = None; while r0 < nz {
self.check_cancel()?;
let r1 = (r0 + chunk).min(nz);
let ds = reader.read_chunk(r0, r1)?;
let chunk_geom = chunk_geometry(geom, r0, r1);
let vol = chunk_to_volume(
&mut recon,
backend,
ds,
geom,
&chunk_geom,
algorithm,
params,
prep,
)?;
writer.write_chunk(&vol, r0, r1)?;
r0 = r1;
}
writer.finalize()?;
Ok(())
}
}
const PIPELINE_DEPTH: usize = 2;
impl ReconSteps {
#[allow(clippy::too_many_arguments)]
pub fn run_streaming_pipelined<RF, WF>(
&self,
make_reader: RF,
make_writer: WF,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
engine: &Engine,
) -> Result<()>
where
RF: FnOnce() -> Result<Box<dyn crate::io::DatasetReader>> + Send,
WF: FnOnce() -> Result<Box<dyn crate::io::VolumeWriter>> + Send,
{
self.run_streaming_pipelined_range(
0,
usize::MAX,
make_reader,
make_writer,
geom,
algorithm,
params,
prep,
engine,
)
}
#[allow(clippy::too_many_arguments)]
pub fn run_streaming_pipelined_range<RF, WF>(
&self,
z_start: usize,
z_end: usize,
make_reader: RF,
make_writer: WF,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
engine: &Engine,
) -> Result<()>
where
RF: FnOnce() -> Result<Box<dyn crate::io::DatasetReader>> + Send,
WF: FnOnce() -> Result<Box<dyn crate::io::VolumeWriter>> + Send,
{
if !prep.row_separable() {
return Err(crate::error::Error::InvalidParam(
"ReconSteps::run_streaming_pipelined does not support phase retrieval \
(row-coupled); use run() for a phase pipeline"
.into(),
));
}
self.check_cancel()?;
let backend = engine.backend();
let chunk = self.chunk_rows.max(1);
let cancel_reader = self.cancel.clone();
let (read_tx, read_rx) =
std::sync::mpsc::sync_channel::<(usize, usize, ChunkMsg)>(PIPELINE_DEPTH);
let (write_tx, write_rx) =
std::sync::mpsc::sync_channel::<(usize, usize, Volume<f32>)>(PIPELINE_DEPTH);
let (nz_tx, nz_rx) = std::sync::mpsc::sync_channel::<usize>(1);
const POOL: usize = PIPELINE_DEPTH + 1;
let (free_tx, free_rx) =
std::sync::mpsc::sync_channel::<Box<dyn crate::backend::HostBuffer>>(POOL);
let free_tx_reader = free_tx.clone();
let (out_free_tx, out_free_rx) = std::sync::mpsc::channel::<Vec<f32>>();
std::thread::scope(|s| {
let reader_handle = s.spawn(move || -> Result<()> {
let mut reader = make_reader()?;
let (nproj, nz, nx, _nflat, _ndark) = reader.read_sizes()?;
let z0 = z_start.min(nz);
let z1 = z_end.min(nz);
let _ = nz_tx.send(nz);
let raw_mode = backend.wants_raw_chunks();
if raw_mode {
let max_len = nproj * chunk.min(nz.max(1)) * nx;
for _ in 0..POOL {
if free_tx_reader
.send(backend.alloc_host_buffer(max_len))
.is_err()
{
return Ok(());
}
}
let mut r0 = z0;
while r0 < z1 {
if cancel_reader.is_cancelled() {
break; }
let r1 = (r0 + chunk).min(z1);
let rows = r1 - r0;
let need = nproj * rows * nx;
let mut buf = match free_rx.recv() {
Ok(b) => b,
Err(_) => break, };
let aux =
reader.read_chunk_into(r0, r1, &mut buf.as_mut_slice()[..need])?;
let msg = ChunkMsg::Raw {
data: buf,
dims: (nproj, rows, nx),
flat: aux.flat,
dark: aux.dark,
theta: aux.theta,
};
if read_tx.send((r0, r1, msg)).is_err() {
break;
}
r0 = r1;
}
} else {
let mut r0 = z0;
while r0 < z1 {
if cancel_reader.is_cancelled() {
break; }
let r1 = (r0 + chunk).min(z1);
let ds = reader.read_chunk(r0, r1)?;
if read_tx.send((r0, r1, ChunkMsg::Host(ds))).is_err() {
break;
}
r0 = r1;
}
}
Ok(())
});
let writer_handle = s.spawn(move || -> Result<()> {
let mut writer = make_writer()?;
if let Ok(total_nz) = nz_rx.recv() {
writer.reserve(total_nz)?;
}
while let Ok((r0, r1, vol)) = write_rx.recv() {
writer.write_chunk(&vol, r0, r1)?;
let _ = out_free_tx.send(vol.array.into_raw_vec_and_offset().0);
}
writer.finalize()?;
Ok(())
});
let compute = compute_chunks(
read_rx,
write_tx,
free_tx,
out_free_rx,
&self.cancel,
backend,
geom,
algorithm,
params,
prep,
);
let read_res = reader_handle.join().expect("reader thread panicked");
let write_res = writer_handle.join().expect("writer thread panicked");
compute.and(read_res).and(write_res)
})
}
}
enum ChunkMsg {
Raw {
data: Box<dyn crate::backend::HostBuffer>,
dims: (usize, usize, usize),
flat: Option<Frames<f32>>,
dark: Option<Frames<f32>>,
theta: Vec<f32>,
},
Host(Dataset<f32>),
}
#[allow(clippy::too_many_arguments)]
fn compute_chunks(
read_rx: std::sync::mpsc::Receiver<(usize, usize, ChunkMsg)>,
write_tx: std::sync::mpsc::SyncSender<(usize, usize, Volume<f32>)>,
free_tx: std::sync::mpsc::SyncSender<Box<dyn crate::backend::HostBuffer>>,
out_free: std::sync::mpsc::Receiver<Vec<f32>>,
cancel: &CancelToken,
backend: &dyn crate::backend::Backend,
geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
) -> Result<()> {
let mut recon: Option<Option<Box<dyn crate::backend::StreamingAnalytic>>> = None;
while let Ok((r0, r1, msg)) = read_rx.recv() {
if cancel.is_cancelled() {
return Err(crate::error::Error::Cancelled);
}
while let Ok(buf) = out_free.try_recv() {
if let Some(Some(r)) = recon.as_mut() {
r.give_reuse_buffer(buf);
}
}
let chunk_geom = chunk_geometry(geom, r0, r1);
let vol = match msg {
ChunkMsg::Raw {
data,
dims,
flat,
dark,
theta,
} => {
let vol = raw_chunk_to_volume(
&mut recon,
backend,
data.as_ref(),
dims,
flat,
dark,
theta,
geom,
&chunk_geom,
algorithm,
params,
prep,
)?;
let _ = free_tx.send(data);
vol
}
ChunkMsg::Host(ds) => chunk_to_volume(
&mut recon,
backend,
ds,
geom,
&chunk_geom,
algorithm,
params,
prep,
)?,
};
if write_tx.send((r0, r1, vol)).is_err() {
break;
}
}
Ok(())
}
fn chunk_geometry(geom: &Geometry, r0: usize, r1: usize) -> Geometry {
use crate::geometry::Center;
let center = match &geom.center {
Center::Scalar(c) => Center::Scalar(*c),
Center::PerRow(v) => Center::PerRow(v[r0..r1].to_vec()),
};
Geometry {
center,
..geom.clone()
}
}
#[allow(clippy::too_many_arguments)]
fn chunk_to_volume(
slot: &mut Option<Option<Box<dyn crate::backend::StreamingAnalytic>>>,
backend: &dyn crate::backend::Backend,
ds: Dataset<f32>,
geom: &Geometry,
chunk_geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
) -> Result<Volume<f32>> {
ensure_slot(
slot,
backend,
algorithm,
params,
geom,
ds.data.n_cols(),
ds.data.n_rows(),
)?;
if let Some(Some(s)) = slot.as_mut() {
let std = ds.data.array.as_standard_layout();
let raw = std.as_slice().ok_or_else(|| {
crate::error::Error::InvalidParam("non-contiguous projection chunk".into())
})?;
let dims = ds.data.array.dim();
if let Some(vol) = s.reconstruct_chunk_raw(
raw,
dims,
ds.flat.as_ref(),
ds.dark.as_ref(),
chunk_geom,
prep.stripe,
)? {
return Ok(vol);
}
}
host_path(slot, backend, ds, geom, chunk_geom, algorithm, params, prep)
}
#[allow(clippy::too_many_arguments)]
fn raw_chunk_to_volume(
slot: &mut Option<Option<Box<dyn crate::backend::StreamingAnalytic>>>,
backend: &dyn crate::backend::Backend,
data: &dyn crate::backend::HostBuffer,
dims: (usize, usize, usize),
flat: Option<Frames<f32>>,
dark: Option<Frames<f32>>,
theta: Vec<f32>,
geom: &Geometry,
chunk_geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
) -> Result<Volume<f32>> {
let (nproj, rows, nx) = dims;
let need = nproj * rows * nx;
let raw = &data.as_slice()[..need];
ensure_slot(slot, backend, algorithm, params, geom, nx, rows)?;
if let Some(Some(s)) = slot.as_mut() {
if let Some(vol) = s.reconstruct_chunk_raw(
raw,
dims,
flat.as_ref(),
dark.as_ref(),
chunk_geom,
prep.stripe,
)? {
return Ok(vol);
}
}
let array = Array3::from_shape_vec((nproj, rows, nx), raw.to_vec()).map_err(|e| {
crate::error::Error::ShapeMismatch {
expected: format!("[{nproj}, {rows}, {nx}]"),
found: e.to_string(),
}
})?;
let ds = Dataset {
data: Tomo::new(array, Layout::Projection),
flat,
dark,
theta,
};
host_path(slot, backend, ds, geom, chunk_geom, algorithm, params, prep)
}
fn ensure_slot(
slot: &mut Option<Option<Box<dyn crate::backend::StreamingAnalytic>>>,
backend: &dyn crate::backend::Backend,
algorithm: Algorithm,
params: &ReconParams,
geom: &Geometry,
ncols: usize,
nz: usize,
) -> Result<()> {
if slot.is_none() {
let built = match backend.analytic_reconstruct() {
Some(ar) => ar.streaming(algorithm, params, geom, ncols, nz)?,
None => None,
};
*slot = Some(built);
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn host_path(
slot: &mut Option<Option<Box<dyn crate::backend::StreamingAnalytic>>>,
backend: &dyn crate::backend::Backend,
mut ds: Dataset<f32>,
geom: &Geometry,
chunk_geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
prep: &PrepOptions,
) -> Result<Volume<f32>> {
crate::prep::normalize_dataset(&mut ds, backend)?;
let mut sino = ds.data.to_layout(Layout::Sinogram);
sino.array = sino.array.as_standard_layout().to_owned();
crate::prep::remove_stripe(&mut sino, prep.stripe)?;
recon_chunk_reusing(slot, backend, &sino, geom, chunk_geom, algorithm, params)
}
fn recon_chunk_reusing(
slot: &mut Option<Option<Box<dyn crate::backend::StreamingAnalytic>>>,
backend: &dyn crate::backend::Backend,
sino: &Tomo<f32>,
geom: &Geometry,
chunk_geom: &Geometry,
algorithm: Algorithm,
params: &ReconParams,
) -> Result<Volume<f32>> {
if slot.is_none() {
let built = match backend.analytic_reconstruct() {
Some(ar) => ar.streaming(algorithm, params, geom, sino.n_cols(), sino.n_rows())?,
None => None,
};
*slot = Some(built);
}
match slot.as_mut().expect("slot built above").as_mut() {
Some(s) => s.reconstruct_chunk(sino, chunk_geom),
None => crate::recon::recon(sino, chunk_geom, algorithm, params, backend),
}
}