use crate::backend::{
make_fbp_filter, Elementwise, FbpFilter, Fft, FilteredBackproject, ForwardProject,
FourierReconstruct, LpRecReconstruct, RampShape, RankFilter,
};
use crate::data::{Frames, Layout, Tomo, Volume};
use crate::dtype::Complex32;
use crate::error::{Error, Result};
use crate::geometry::{Beam, Geometry};
use crate::params::FilterName;
use bytemuck::{Pod, Zeroable};
use ndarray::{Array3, Axis};
use crate::wgpu::shaders::{
BACKPROJECT_WGSL, BLUESTEIN_WGSL, ELEMENTWISE_WGSL, FBP_FILTER_WGSL, FFT_SHARED_WGSL,
FFT_TRANSPOSE_WGSL, FFT_WGSL, FOURIERREC_WGSL, LPREC_WGSL, MEDFILT3D_WGSL, PROJECT_WGSL,
};
use crate::wgpu::WgpuBackend;
const MEDFILT_MAX_WIN: usize = 343;
const SHARED_FFT_MAX: usize = 4096;
fn shared_fft_workgroup(n: usize) -> u32 {
((n / 2).clamp(1, 256)) as u32
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct DfParams {
n_elems: u32,
plane_size: u32,
_pad0: u32,
_pad1: u32,
}
impl Elementwise for WgpuBackend {
fn darkflat(&self, data: &mut Tomo<f32>, flat: &Frames<f32>, dark: &Frames<f32>) -> Result<()> {
let dark2d = dark
.array
.mean_axis(Axis(0))
.ok_or_else(|| Error::InvalidParam("empty dark stack".into()))?;
let flat2d = flat
.array
.mean_axis(Axis(0))
.ok_or_else(|| Error::InvalidParam("empty flat stack".into()))?;
let mut denom = &flat2d - &dark2d;
denom.mapv_inplace(|v| if v.abs() < 1e-6 { 1.0 } else { v });
let restore = data.layout == Layout::Sinogram;
if restore {
*data = data.to_layout(Layout::Projection);
}
let (np, nr, nc) = data.array.dim();
if dark2d.dim() != (nr, nc) {
return Err(Error::ShapeMismatch {
expected: format!("{:?}", (nr, nc)),
found: format!("{:?}", dark2d.dim()),
});
}
let plane = nr * nc;
let host: Vec<f32> = data.array.iter().copied().collect();
let dark_std = dark2d.as_standard_layout();
let denom_std = denom.as_standard_layout();
let data_buf = self.storage_rw("df_data", &host);
let dark_buf = self.storage_ro("df_dark", dark_std.as_slice().expect("standard layout"));
let denom_buf = self.storage_ro("df_denom", denom_std.as_slice().expect("standard layout"));
let params = DfParams {
n_elems: host.len() as u32,
plane_size: plane as u32,
_pad0: 0,
_pad1: 0,
};
let param_buf = self.uniform("df_params", ¶ms);
self.dispatch1d(
ELEMENTWISE_WGSL,
"darkflat",
&[&data_buf, &dark_buf, &denom_buf, ¶m_buf],
host.len() as u32,
);
let out = self.download_f32(&data_buf, host.len());
data.array = Array3::from_shape_vec((np, nr, nc), out).expect("len matches dims");
if restore {
*data = data.to_layout(Layout::Sinogram);
}
Ok(())
}
fn minus_log(&self, data: &mut Tomo<f32>) -> Result<()> {
let dims = data.array.dim();
let host: Vec<f32> = data.array.iter().copied().collect();
let buf = self.storage_rw("ml_data", &host);
self.dispatch1d(ELEMENTWISE_WGSL, "minus_log", &[&buf], host.len() as u32);
let out = self.download_f32(&buf, host.len());
data.array = Array3::from_shape_vec(dims, out).expect("len matches dims");
Ok(())
}
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct BpParams {
nproj: u32,
ncols: u32,
ny: u32,
nx: u32,
scale: f32,
_pad0: u32,
_pad1: u32,
_pad2: u32,
}
impl FilteredBackproject for WgpuBackend {
fn backproject(&self, sino: &Tomo<f32>, geom: &Geometry, out: &mut Volume<f32>) -> Result<()> {
if geom.beam != Beam::Parallel {
return Err(Error::InvalidParam(
"wgpu back-projection currently supports parallel beam only".into(),
));
}
let s = sino.as_layout(Layout::Sinogram); let nz = s.n_rows();
let nang = s.n_angles();
let ncols = s.n_cols();
let (oz, ny, nx) = out.dims();
if oz != nz {
return Err(Error::ShapeMismatch {
expected: format!("{nz} sinogram rows"),
found: oz.to_string(),
});
}
let angles = &geom.angles.0;
if angles.len() != nang {
return Err(Error::ShapeMismatch {
expected: format!("{nang} angles"),
found: angles.len().to_string(),
});
}
let sino_std = s.array.as_standard_layout();
let sino_buf = self.storage_ro("bp_sino", sino_std.as_slice().expect("standard layout"));
out.array = self.backproject_from_dev(&sino_buf, nz, nang, ncols, geom, (ny, nx), 1.0)?;
Ok(())
}
}
impl WgpuBackend {
#[allow(clippy::too_many_arguments)]
pub(crate) fn backproject_from_dev(
&self,
sino_buf: &wgpu::Buffer,
nz: usize,
nang: usize,
ncols: usize,
geom: &Geometry,
out_dims: (usize, usize),
gain: f32,
) -> Result<Array3<f32>> {
let (ny, nx) = out_dims;
let (cossin, center) = cossin_center(geom, nz);
let cossin_buf = self.storage_ro("bp_cossin", &cossin);
let center_buf = self.storage_ro("bp_center", ¢er);
let total = nz * ny * nx;
let vol_buf = self.storage_rw("bp_vol", &vec![0.0f32; total]);
let params = BpParams {
nproj: nang as u32,
ncols: ncols as u32,
ny: ny as u32,
nx: nx as u32,
scale: gain,
_pad0: 0,
_pad1: 0,
_pad2: 0,
};
let param_buf = self.uniform("bp_params", ¶ms);
self.dispatch1d(
BACKPROJECT_WGSL,
"backproject",
&[sino_buf, &cossin_buf, ¢er_buf, &vol_buf, ¶m_buf],
total as u32,
);
let result = self.download_f32(&vol_buf, total);
Ok(Array3::from_shape_vec((nz, ny, nx), result).expect("len matches dims"))
}
}
fn cossin_center(geom: &Geometry, nz: usize) -> (Vec<f32>, Vec<f32>) {
let mut cossin = Vec::with_capacity(geom.angles.0.len() * 2);
for &a in &geom.angles.0 {
let (sn, c) = a.sin_cos();
cossin.push(c);
cossin.push(sn);
}
let center = (0..nz).map(|r| geom.center.at(r)).collect();
(cossin, center)
}
fn transpose_images(buf: &mut [Complex32], d0: usize, d1: usize, batch: usize) {
let img = d0 * d1;
let mut tmp = vec![Complex32::default(); img];
for b in 0..batch {
let base = b * img;
for r in 0..d0 {
for c in 0..d1 {
tmp[c * d0 + r] = buf[base + r * d1 + c];
}
}
buf[base..base + img].copy_from_slice(&tmp);
}
}
impl ForwardProject for WgpuBackend {
fn project(&self, vol: &Volume<f32>, geom: &Geometry, out: &mut Tomo<f32>) -> Result<()> {
if geom.beam != Beam::Parallel {
return Err(Error::InvalidParam(
"wgpu forward projection currently supports parallel beam only".into(),
));
}
let (nz, ny, nx) = vol.dims();
let nang = geom.angles.0.len();
let ncols = geom.detector.width;
if nang == 0 || ncols == 0 {
return Err(Error::InvalidParam(
"geometry has no angles or zero detector width".into(),
));
}
let (cossin, center) = cossin_center(geom, nz);
let vol_std = vol.array.as_standard_layout();
let vol_buf = self.storage_ro("fp_vol", vol_std.as_slice().expect("standard layout"));
let cossin_buf = self.storage_ro("fp_cossin", &cossin);
let center_buf = self.storage_ro("fp_center", ¢er);
let total = nz * nang * ncols;
let sino_buf = self.storage_rw("fp_sino", &vec![0.0f32; total]);
let params = FpParams {
nproj: nang as u32,
ncols: ncols as u32,
ny: ny as u32,
nx: nx as u32,
_pad0: 0,
_pad1: 0,
_pad2: 0,
_pad3: 0,
};
let param_buf = self.uniform("fp_params", ¶ms);
let src = format!("{}{PROJECT_WGSL}", self.atomic_f32_decl("sino", 3));
self.dispatch1d(
&src,
"project",
&[&vol_buf, &cossin_buf, ¢er_buf, &sino_buf, ¶m_buf],
(nz * ny * nx) as u32,
);
let result = self.download_f32(&sino_buf, total);
let array = Array3::from_shape_vec((nz, nang, ncols), result).expect("len matches dims");
*out = Tomo::new(array, Layout::Sinogram);
Ok(())
}
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct FpParams {
nproj: u32,
ncols: u32,
ny: u32,
nx: u32,
_pad0: u32,
_pad1: u32,
_pad2: u32,
_pad3: u32,
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct MfParams {
dz: u32,
dy: u32,
dx: u32,
radius: u32,
threshold: f32,
_pad0: u32,
_pad1: u32,
_pad2: u32,
}
impl WgpuBackend {
fn medfilt3d_gpu(
&self,
arr: &Array3<f32>,
radius: usize,
threshold: f32,
) -> Result<Array3<f32>> {
let diameter = 2 * radius + 1;
let total = diameter * diameter * diameter;
if total > MEDFILT_MAX_WIN {
return Err(Error::InvalidParam(format!(
"wgpu median window {diameter}³={total} exceeds the GPU cap of \
{MEDFILT_MAX_WIN} (size ≤ 7); use the CPU backend for larger windows"
)));
}
let (dz, dy, dx) = arr.dim();
let host: Vec<f32> = arr.iter().copied().collect();
let n = host.len();
let inp = self.storage_ro("mf_inp", &host);
let outp = self.storage_rw("mf_outp", &vec![0.0f32; n]);
let params = MfParams {
dz: dz as u32,
dy: dy as u32,
dx: dx as u32,
radius: radius as u32,
threshold,
_pad0: 0,
_pad1: 0,
_pad2: 0,
};
let param_buf = self.uniform("mf_params", ¶ms);
self.dispatch1d(
MEDFILT3D_WGSL,
"medfilt3d",
&[&inp, &outp, ¶m_buf],
n as u32,
);
let out = self.download_f32(&outp, n);
Ok(Array3::from_shape_vec((dz, dy, dx), out).expect("len matches dims"))
}
}
impl RankFilter for WgpuBackend {
fn median3d(&self, vol: &mut Volume<f32>, size: usize) -> Result<()> {
let radius = (size.max(3) - 1) / 2;
vol.array = self.medfilt3d_gpu(&vol.array, radius, 0.0)?;
Ok(())
}
fn remove_outlier3d(&self, data: &mut Tomo<f32>, diff: f32, size: usize) -> Result<()> {
let radius = (size.max(3) - 1) / 2;
data.array = self.medfilt3d_gpu(&data.array, radius, diff)?;
Ok(())
}
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct FfltParams {
pad: u32,
ncols: u32,
pad_side: u32,
_pad0: u32,
scale: f32,
_pad1: f32,
_pad2: f32,
_pad3: f32,
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct FftParams {
n: u32,
logn: u32,
m: u32,
sign: f32,
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct BzParams {
m: u32,
_pad0: u32,
_pad1: u32,
_pad2: u32,
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct TParams {
rows: u32,
cols: u32,
_pad0: u32,
_pad1: u32,
}
impl WgpuBackend {
fn fft_passes(&self, data: &wgpu::Buffer, n: usize, lanes: usize, inverse: bool) {
let lim = self.device.limits();
let wg = shared_fft_workgroup(n);
let fits_shared = n <= SHARED_FFT_MAX
&& (n * std::mem::size_of::<[f32; 2]>()) as u32
<= lim.max_compute_workgroup_storage_size
&& wg <= lim.max_compute_invocations_per_workgroup;
if fits_shared {
self.fft_shared_pass(data, n, lanes, inverse, wg);
} else {
self.fft_global_passes(data, n, lanes, inverse);
}
}
fn fft_shared_pass(&self, data: &wgpu::Buffer, n: usize, lanes: usize, inverse: bool, wg: u32) {
let logn = n.trailing_zeros();
let sign = if inverse { 1.0f32 } else { -1.0f32 };
let params = self.uniform(
"fft_sh_p",
&FftParams {
n: n as u32,
logn,
m: 0,
sign,
},
);
let src = format!(
"const WG : u32 = {wg}u;\nconst NN : u32 = {n}u;\nconst LOGN : u32 = {logn}u;\n{FFT_SHARED_WGSL}"
);
let pipeline = self.cached_pipeline(&src, "fft_shared");
let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("fft_shared"),
layout: &pipeline.get_bind_group_layout(0),
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: data.as_entire_binding(),
},
wgpu::BindGroupEntry {
binding: 1,
resource: params.as_entire_binding(),
},
],
});
let mut enc = self
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("fft_shared"),
});
{
let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("fft_shared"),
timestamp_writes: None,
});
pass.set_pipeline(&pipeline);
pass.set_bind_group(0, &bind_group, &[]);
const MAX_DIM: u32 = 65535;
let lanes = lanes as u32;
let (wx, wy) = if lanes <= MAX_DIM {
(lanes, 1)
} else {
(MAX_DIM, lanes.div_ceil(MAX_DIM))
};
pass.dispatch_workgroups(wx, wy, 1);
}
self.queue.submit(Some(enc.finish()));
}
fn fft_global_passes(&self, data: &wgpu::Buffer, n: usize, lanes: usize, inverse: bool) {
let logn = n.trailing_zeros();
let sign = if inverse { 1.0f32 } else { -1.0f32 };
let p0 = self.uniform(
"fft_p",
&FftParams {
n: n as u32,
logn,
m: 0,
sign,
},
);
self.dispatch1d(FFT_WGSL, "bitrev", &[data, &p0], (n * lanes) as u32);
let mut m = 2u32;
for _ in 0..logn {
let p = self.uniform(
"fft_p",
&FftParams {
n: n as u32,
logn,
m,
sign,
},
);
self.dispatch1d(FFT_WGSL, "butterfly", &[data, &p], (n * lanes / 2) as u32);
m <<= 1;
}
}
fn fft_transpose(
&self,
src: &wgpu::Buffer,
dst: &wgpu::Buffer,
rows: usize,
cols: usize,
batch: usize,
) {
let params = TParams {
rows: rows as u32,
cols: cols as u32,
_pad0: 0,
_pad1: 0,
};
let p = self.uniform("fft_t", ¶ms);
self.dispatch1d(
FFT_TRANSPOSE_WGSL,
"transpose",
&[src, dst, &p],
(rows * cols * batch) as u32,
);
}
fn upload_complex(&self, label: &str, buf: &[Complex32]) -> wgpu::Buffer {
let host: Vec<f32> = buf.iter().flat_map(|c| [c.re, c.im]).collect();
self.storage_rw(label, &host)
}
fn download_complex(&self, data: &wgpu::Buffer, buf: &mut [Complex32], norm: f32) {
let out = self.download_f32(data, buf.len() * 2);
for (c, chunk) in buf.iter_mut().zip(out.as_chunks::<2>().0) {
c.re = chunk[0] * norm;
c.im = chunk[1] * norm;
}
}
fn fft_bluestein(&self, buf: &mut [Complex32], n: usize, lanes: usize, inverse: bool) {
let m = (2 * n - 1).next_power_of_two();
let s = if inverse { 1.0f32 } else { -1.0f32 };
let pi = std::f32::consts::PI;
let two_n = 2 * n as u64;
let p: Vec<Complex32> = (0..n)
.map(|j| {
let r = ((j as u64 * j as u64) % two_n) as f32;
let ang = s * pi * r / n as f32;
Complex32::new(ang.cos(), ang.sin())
})
.collect();
let mut a_host = vec![Complex32::new(0.0, 0.0); lanes * m];
for l in 0..lanes {
for j in 0..n {
a_host[l * m + j] = buf[l * n + j] * p[j];
}
}
let mut h_host = vec![Complex32::new(0.0, 0.0); m];
h_host[0] = p[0].conj();
for j in 1..n {
let hj = p[j].conj();
h_host[j] = hj;
h_host[m - j] = hj;
}
let a_buf = self.upload_complex("bz_a", &a_host);
let h_buf = self.upload_complex("bz_h", &h_host);
self.fft_passes(&h_buf, m, 1, false);
self.fft_passes(&a_buf, m, lanes, false);
let p_u = self.uniform(
"bz_p",
&BzParams {
m: m as u32,
_pad0: 0,
_pad1: 0,
_pad2: 0,
},
);
self.dispatch1d(
BLUESTEIN_WGSL,
"cmul",
&[&a_buf, &h_buf, &p_u],
(lanes * m) as u32,
);
self.fft_passes(&a_buf, m, lanes, true);
let out = self.download_f32(&a_buf, lanes * m * 2);
let conv_norm = 1.0 / m as f32;
let inv_norm = if inverse { 1.0 / n as f32 } else { 1.0 };
let scale = conv_norm * inv_norm;
for l in 0..lanes {
for k in 0..n {
let off = l * m + k;
let c = Complex32::new(out[2 * off], out[2 * off + 1]) * p[k];
buf[l * n + k] = Complex32::new(c.re * scale, c.im * scale);
}
}
}
}
impl Fft for WgpuBackend {
fn fft_1d(&self, buf: &mut [Complex32], len: usize, batch: usize, inverse: bool) -> Result<()> {
if len == 0 || batch == 0 {
return Ok(());
}
if buf.len() != len * batch {
return Err(Error::ShapeMismatch {
expected: (len * batch).to_string(),
found: buf.len().to_string(),
});
}
if len.is_power_of_two() {
let data = self.upload_complex("fft_1d", buf);
self.fft_passes(&data, len, batch, inverse);
let norm = if inverse { 1.0 / len as f32 } else { 1.0 };
self.download_complex(&data, buf, norm);
} else {
self.fft_bluestein(buf, len, batch, inverse);
}
Ok(())
}
fn fft_2d(
&self,
buf: &mut [Complex32],
rows: usize,
cols: usize,
batch: usize,
inverse: bool,
) -> Result<()> {
if rows == 0 || cols == 0 || batch == 0 {
return Ok(());
}
if buf.len() != rows * cols * batch {
return Err(Error::ShapeMismatch {
expected: (rows * cols * batch).to_string(),
found: buf.len().to_string(),
});
}
if !rows.is_power_of_two() || !cols.is_power_of_two() {
self.fft_1d(buf, cols, rows * batch, inverse)?;
transpose_images(buf, rows, cols, batch);
self.fft_1d(buf, rows, cols * batch, inverse)?;
transpose_images(buf, cols, rows, batch);
return Ok(());
}
let data = self.upload_complex("fft_2d", buf);
self.fft_passes(&data, cols, rows * batch, inverse);
let scratch = self.storage_rw("fft_2d_t", &vec![0.0f32; rows * cols * batch * 2]);
self.fft_transpose(&data, &scratch, rows, cols, batch);
self.fft_passes(&scratch, rows, cols * batch, inverse);
self.fft_transpose(&scratch, &data, cols, rows, batch);
let norm = if inverse {
1.0 / (rows * cols) as f32
} else {
1.0
};
self.download_complex(&data, buf, norm);
Ok(())
}
}
impl FbpFilter for WgpuBackend {
fn make_filter(&self, name: FilterName, n: usize) -> Result<Vec<f32>> {
make_fbp_filter(name, n, RampShape::Linear)
}
fn apply(&self, sino: &mut Tomo<f32>, filter: &[f32], geom: &Geometry) -> Result<()> {
let ncols = sino.n_cols();
let batch = sino.array.len() / ncols;
let out = self.filter_to_device(sino, filter, geom)?;
let host_out = self.download_f32(&out, batch * ncols);
for (l, mut lane) in sino.array.lanes_mut(Axis(2)).into_iter().enumerate() {
let base = l * ncols;
for (i, slot) in lane.iter_mut().enumerate() {
*slot = host_out[base + i];
}
}
Ok(())
}
}
impl WgpuBackend {
pub(crate) fn filter_to_device(
&self,
sino: &Tomo<f32>,
filter: &[f32],
geom: &Geometry,
) -> Result<wgpu::Buffer> {
let pad = filter.len();
if pad == 0 {
return Err(Error::InvalidParam("empty filter".into()));
}
let ncols = sino.n_cols();
if pad < ncols {
return Err(Error::ShapeMismatch {
expected: format!(">= {ncols} (n_cols)"),
found: pad.to_string(),
});
}
if !pad.is_power_of_two() {
return Err(Error::InvalidParam(format!(
"wgpu FBP filter requires a power-of-two length (got {pad}); use the CPU backend"
)));
}
let batch = sino.array.len() / ncols;
let pad_side = pad / 2 - ncols / 2;
let half = ncols as f32 / 2.0;
let d1 = sino.array.shape()[1];
let deltas: Vec<f32> = (0..batch)
.map(|l| {
let row = match sino.layout {
Layout::Sinogram => l / d1,
Layout::Projection => l % d1,
};
half - geom.center.at(row)
})
.collect();
let std = sino.array.as_standard_layout();
let sino_host = std
.as_slice()
.expect("as_standard_layout yields a contiguous slice");
let sino_buf = self.storage_ro("fbp_sino", sino_host);
let data = self.storage_empty("fbp_spectrum", batch * pad * 2);
let w = self.storage_ro("fbp_w", filter);
let deltas_buf = self.storage_ro("fbp_deltas", &deltas);
let p = self.uniform(
"fbp_p",
&FfltParams {
pad: pad as u32,
ncols: ncols as u32,
pad_side: pad_side as u32,
_pad0: 0,
scale: 1.0 / pad as f32,
_pad1: 0.0,
_pad2: 0.0,
_pad3: 0.0,
},
);
self.dispatch1d(
FBP_FILTER_WGSL,
"pack",
&[&sino_buf, &data, &p],
(batch * pad) as u32,
);
self.fft_passes(&data, pad, batch, false);
self.dispatch1d(
FBP_FILTER_WGSL,
"apply_filter",
&[&data, &w, &p, &deltas_buf],
(batch * pad) as u32,
);
self.fft_passes(&data, pad, batch, true);
let out = self.storage_empty("fbp_out", batch * ncols);
self.dispatch1d(
FBP_FILTER_WGSL,
"unpack",
&[&data, &out, &p],
(batch * ncols) as u32,
);
Ok(out)
}
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct FrParams {
nz: u32,
nang: u32,
nd: u32,
n: u32,
ng: u32,
nf: u32,
crop: u32,
_p0: u32,
mu: f32,
coeff0: f32,
coeff1: f32,
gscale: f32,
phi_sign: f32,
norm: f32,
_p1: f32,
_p2: f32,
}
impl FourierReconstruct for WgpuBackend {
fn reconstruct(&self, filtered: &Tomo<f32>, geom: &Geometry, n: usize) -> Result<Volume<f32>> {
let b = filtered.as_layout(Layout::Sinogram);
let nz = b.n_rows();
let nang = b.n_angles();
let nd = b.n_cols();
if !nd.is_power_of_two() {
return Ok(Volume::new(crate::recon::fourierrec::fourierrec(
filtered, geom, n, self,
)?));
}
let bdata = b
.array
.as_slice()
.expect("contiguous sinogram (as_layout yields a standard-layout copy)");
let sino_buf = self.storage_ro("fr_sino", bdata);
self.fourierrec_from_dev(&sino_buf, nz, nang, nd, geom, n)
}
}
impl WgpuBackend {
pub(crate) fn fourierrec_from_dev(
&self,
sino_buf: &wgpu::Buffer,
nz: usize,
nang: usize,
nd: usize,
geom: &Geometry,
n: usize,
) -> Result<Volume<f32>> {
const EPS: f64 = 1e-3;
let ndf = nd as f64;
let neg_log_eps = -EPS.ln();
let mu = neg_log_eps / (2.0 * ndf * ndf);
let inside = mu * neg_log_eps + (mu * ndf) * (mu * ndf) / 4.0;
let m = (2.0 * ndf / std::f64::consts::PI * inside.sqrt()).ceil() as usize;
let ng = 2 * nd + 2 * m;
let nf = 2 * nd;
let crop = (nd - n.min(nd)) / 2;
let mu32 = mu as f32;
let ndf32 = nd as f32;
let params = FrParams {
nz: nz as u32,
nang: nang as u32,
nd: nd as u32,
n: n as u32,
ng: ng as u32,
nf: nf as u32,
crop: crop as u32,
_p0: 0,
mu: mu32,
coeff0: std::f32::consts::PI / (mu32 * 4.0 * ndf32 * ndf32),
coeff1: -std::f32::consts::PI * std::f32::consts::PI / mu32,
gscale: 4.0 / ndf32,
phi_sign: 1.0 - (nd % 4) as f32,
norm: std::f32::consts::FRAC_PI_4,
_p1: 0.0,
_p2: 0.0,
};
let trig: Vec<f32> = geom
.angles
.0
.iter()
.flat_map(|&a| {
let (s, c) = a.sin_cos();
[c, s]
})
.collect();
let src = format!(
"const M : u32 = {m}u;\n{}{}{FOURIERREC_WGSL}",
self.atomic_f32_decl("ga_grid", 1),
self.atomic_f32_decl("wr_grid", 0),
);
let trig_buf = self.storage_ro("fr_trig", &trig);
let radial = self.storage_empty("fr_radial", nz * nang * nd * 2);
let grid = self.storage_empty("fr_grid", nz * ng * ng * 2);
let inner = self.storage_empty("fr_inner", nz * nf * nf * 2);
let scratch = self.storage_empty("fr_scratch", nz * nf * nf * 2);
let out = self.storage_empty("fr_out", nz * n * n);
let p = self.uniform("fr_p", ¶ms);
self.zero_buffer(&grid);
let radial_threads = (nz * nang * nd) as u32;
self.dispatch1d(
&src,
"build_radial",
&[sino_buf, &radial, &p],
radial_threads,
);
self.fft_passes(&radial, nd, nz * nang, false);
self.dispatch1d(&src, "postmod", &[&radial, &p], radial_threads);
self.dispatch1d(
&src,
"gather",
&[&radial, &grid, &trig_buf, &p],
radial_threads,
);
self.dispatch1d(&src, "wrap", &[&grid, &p], (nz * ng * ng) as u32);
self.dispatch1d(&src, "extract", &[&grid, &inner, &p], (nz * nf * nf) as u32);
self.fft_passes(&inner, nf, nf * nz, true);
self.fft_transpose(&inner, &scratch, nf, nf, nz);
self.fft_passes(&scratch, nf, nf * nz, true);
self.fft_transpose(&scratch, &inner, nf, nf, nz);
self.dispatch1d(&src, "fftshift", &[&inner, &p], (nz * nf * nf) as u32);
self.dispatch1d(&src, "deapodize", &[&inner, &out, &p], (nz * n * n) as u32);
let host = self.download_f32(&out, nz * n * n);
let arr = Array3::from_shape_vec((nz, n, n), host)
.expect("out buffer length nz*n*n matches the volume shape");
Ok(Volume::new(arr))
}
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct LpParams {
nz: u32,
nang: u32,
n: u32,
ntheta: u32,
nrho: u32,
npts: u32,
_p0: u32,
_p1: u32,
scale: f32,
_p2: f32,
_p3: f32,
_p4: f32,
}
struct LprecSpan {
xs_main: wgpu::Buffer,
ys_main: wgpu::Buffer,
wrap: Option<(wgpu::Buffer, wgpu::Buffer)>,
xs_c: wgpu::Buffer,
ys_c: wgpu::Buffer,
}
pub(crate) struct WgpuLprecGrids {
ntheta: usize,
nrho: usize,
n: usize,
nang: usize,
scale: f32,
lpids_len: usize,
wids_len: usize,
cids_len: usize,
kfull_buf: wgpu::Buffer,
lpids_buf: wgpu::Buffer,
wids_buf: wgpu::Buffer,
cids_buf: wgpu::Buffer,
spans: Vec<LprecSpan>,
}
impl WgpuBackend {
pub(crate) fn build_lprec_grids(&self, geom: &Geometry, n: usize) -> Result<WgpuLprecGrids> {
let nang = geom.angles.0.len();
if nang < 2 {
return Err(Error::InvalidParam(format!(
"lprec requires square geometry with detector width == grid size (got n={n})"
)));
}
let angles = &geom.angles.0;
let dth = (angles[1] - angles[0]).abs();
let nproj_test = (std::f32::consts::PI / dth).round() as usize;
if nproj_test != nang {
return Err(Error::InvalidParam(
"lprec requires equally spaced angles spanning [0, π)".into(),
));
}
let grids = crate::recon::lprec::build_grids(n, nang, &crate::cpu::CpuBackend::new())?;
let ntheta = grids.ntheta;
let nrho = grids.nrho;
let scale = 2.0 / (nrho as f32 * ntheta as f32);
let kfull_host: Vec<f32> = grids.kfull.iter().flat_map(|c| [c.re, c.im]).collect();
let kfull_buf = self.storage_ro("lp_kfull", &kfull_host);
let lpids: Vec<u32> = grids.lpids.iter().map(|&x| x as u32).collect();
let wids: Vec<u32> = grids.wids.iter().map(|&x| x as u32).collect();
let cids: Vec<u32> = grids.cids.iter().map(|&x| x as u32).collect();
let lpids_buf = self.storage_ro_u32("lp_lpids", &lpids);
let wids_buf = self.storage_ro_u32("lp_wids", &wids);
let cids_buf = self.storage_ro_u32("lp_cids", &cids);
let mut spans = Vec::with_capacity(grids.lp2p1.len());
for k in 0..grids.lp2p1.len() {
let xs_main = self.storage_ro("lp_xs", &grids.lp2p2[k]);
let ys_main = self.storage_ro("lp_ys", &grids.lp2p1[k]);
let wrap = if !wids.is_empty() {
Some((
self.storage_ro("lp_xsw", &grids.lp2p2w[k]),
self.storage_ro("lp_ysw", &grids.lp2p1w[k]),
))
} else {
None
};
let xs_c = self.storage_ro("lp_xsc", &grids.c2lp1[k]);
let ys_c = self.storage_ro("lp_ysc", &grids.c2lp2[k]);
spans.push(LprecSpan {
xs_main,
ys_main,
wrap,
xs_c,
ys_c,
});
}
Ok(WgpuLprecGrids {
ntheta,
nrho,
n,
nang,
scale,
lpids_len: lpids.len(),
wids_len: wids.len(),
cids_len: cids.len(),
kfull_buf,
lpids_buf,
wids_buf,
cids_buf,
spans,
})
}
pub(crate) fn lprec_run(
&self,
filtered: &Tomo<f32>,
grids: &WgpuLprecGrids,
) -> Result<Volume<f32>> {
let b = filtered.as_layout(Layout::Sinogram);
let nz = b.n_rows();
let nang = b.n_angles();
let nd = b.n_cols();
let n = grids.n;
if nd != n || nang != grids.nang {
return Err(Error::InvalidParam(format!(
"lprec chunk geometry {nang}×{nd} does not match prepared grids {}×{n}",
grids.nang
)));
}
let ntheta = grids.ntheta;
let nrho = grids.nrho;
let scale = grids.scale;
let bdata = b
.array
.as_slice()
.expect("contiguous sinogram (as_layout yields a standard-layout copy)");
let g = self.storage_rw("lp_g", bdata);
let fl = self.storage_empty("lp_fl", nz * nrho * ntheta);
let flc = self.storage_empty("lp_flc", nz * nrho * ntheta * 2);
let scratch = self.storage_empty("lp_scratch", nz * nrho * ntheta * 2);
let flcre = self.storage_empty("lp_flcre", nz * nrho * ntheta);
let f = self.storage_empty("lp_f", nz * n * n);
let mk = |npts: usize| {
self.uniform(
"lp_p",
&LpParams {
nz: nz as u32,
nang: nang as u32,
n: n as u32,
ntheta: ntheta as u32,
nrho: nrho as u32,
npts: npts as u32,
_p0: 0,
_p1: 0,
scale,
_p2: 0.0,
_p3: 0.0,
_p4: 0.0,
},
)
};
let p0 = mk(0);
let src = format!("{}{LPREC_WGSL}", self.atomic_f32_decl("ga_fl", 1));
self.dispatch1d(&src, "prefilter_rows", &[&g, &p0], (nz * nang) as u32);
self.dispatch1d(&src, "prefilter_cols", &[&g, &p0], (nz * n) as u32);
self.zero_buffer(&f);
for span in &grids.spans {
self.zero_buffer(&fl);
let p_main = mk(grids.lpids_len);
self.dispatch1d(
&src,
"gather",
&[
&g,
&fl,
&grids.lpids_buf,
&span.xs_main,
&span.ys_main,
&p_main,
],
(nz * grids.lpids_len) as u32,
);
if let Some((xs_w, ys_w)) = &span.wrap {
let p_w = mk(grids.wids_len);
self.dispatch1d(
&src,
"gather",
&[&g, &fl, &grids.wids_buf, xs_w, ys_w, &p_w],
(nz * grids.wids_len) as u32,
);
}
let grid_threads = (nz * nrho * ntheta) as u32;
self.dispatch1d(&src, "real_to_complex", &[&fl, &flc, &p0], grid_threads);
self.fft_passes(&flc, ntheta, nrho * nz, false);
self.fft_transpose(&flc, &scratch, nrho, ntheta, nz);
self.fft_passes(&scratch, nrho, ntheta * nz, false);
self.fft_transpose(&scratch, &flc, ntheta, nrho, nz);
self.dispatch1d(&src, "cmul", &[&flc, &grids.kfull_buf, &p0], grid_threads);
self.fft_passes(&flc, ntheta, nrho * nz, true);
self.fft_transpose(&flc, &scratch, nrho, ntheta, nz);
self.fft_passes(&scratch, nrho, ntheta * nz, true);
self.fft_transpose(&scratch, &flc, ntheta, nrho, nz);
self.dispatch1d(&src, "take_re", &[&flc, &flcre, &p0], grid_threads);
let p_c = mk(grids.cids_len);
self.dispatch1d(
&src,
"scatter",
&[&flcre, &f, &grids.cids_buf, &span.xs_c, &span.ys_c, &p_c],
(nz * grids.cids_len) as u32,
);
}
let host = self.download_f32(&f, nz * n * n);
let arr = Array3::from_shape_vec((nz, n, n), host)
.expect("out buffer length nz*n*n matches the volume shape");
Ok(Volume::new(arr))
}
}
impl LpRecReconstruct for WgpuBackend {
fn reconstruct(&self, filtered: &Tomo<f32>, geom: &Geometry, n: usize) -> Result<Volume<f32>> {
let grids = self.build_lprec_grids(geom, n)?;
self.lprec_run(filtered, &grids)
}
}
impl crate::backend::AnalyticReconstruct for WgpuBackend {
fn reconstruct(
&self,
sino: &Tomo<f32>,
geom: &Geometry,
algorithm: crate::params::Algorithm,
params: &crate::params::ReconParams,
) -> Result<Volume<f32>> {
use crate::params::Algorithm;
let ncols = sino.n_cols();
let n = params.num_gridx.unwrap_or(ncols);
let nz = sino.n_rows();
let nang = sino.n_angles();
let kernel = self.make_filter(params.filter_name, ncols)?;
if algorithm == Algorithm::Fourierrec && !ncols.is_power_of_two() {
let mut filtered = sino.clone();
<Self as FbpFilter>::apply(self, &mut filtered, &kernel, geom)?;
return Ok(Volume::new(crate::recon::fourierrec::fourierrec(
&filtered, geom, n, self,
)?));
}
let s = sino.as_layout(Layout::Sinogram);
let filt_dev = self.filter_to_device(&s, &kernel, geom)?;
match algorithm {
Algorithm::Fourierrec => self.fourierrec_from_dev(&filt_dev, nz, nang, ncols, geom, n),
Algorithm::Fbp | Algorithm::Linerec => {
let centered = Geometry {
center: crate::geometry::Center::Scalar(ncols as f32 / 2.0),
..geom.clone()
};
let arr = self.backproject_from_dev(
&filt_dev,
nz,
nang,
ncols,
¢ered,
(n, n),
std::f32::consts::PI / nang as f32, )?;
Ok(Volume::new(arr))
}
other => Err(Error::InvalidParam(format!(
"wgpu analytic_reconstruct does not handle {other:?}"
))),
}
}
fn streaming(
&self,
algorithm: crate::params::Algorithm,
params: &crate::params::ReconParams,
_geom: &Geometry,
_ncols: usize,
_max_nz: usize,
) -> Result<Option<Box<dyn crate::backend::StreamingAnalytic>>> {
use crate::params::Algorithm;
if !matches!(
algorithm,
Algorithm::Fbp | Algorithm::Linerec | Algorithm::Fourierrec | Algorithm::Lprec
) {
return Ok(None);
}
Ok(Some(Box::new(super::streaming::WgpuFbpStream::new(
self.share(),
algorithm,
params.clone(),
))))
}
}
impl WgpuBackend {
#[allow(clippy::too_many_arguments)]
pub(crate) fn forward_into_dev(
&self,
vol_buf: &wgpu::Buffer,
cossin_buf: &wgpu::Buffer,
center_buf: &wgpu::Buffer,
sino_buf: &wgpu::Buffer,
dims: (usize, usize, usize), nang: usize,
ncols: usize,
) {
let (nz, ny, nx) = dims;
self.zero_buffer(sino_buf);
let params = FpParams {
nproj: nang as u32,
ncols: ncols as u32,
ny: ny as u32,
nx: nx as u32,
_pad0: 0,
_pad1: 0,
_pad2: 0,
_pad3: 0,
};
let pbuf = self.uniform("fp_params", ¶ms);
let src = format!("{}{PROJECT_WGSL}", self.atomic_f32_decl("sino", 3));
self.dispatch1d(
&src,
"project",
&[vol_buf, cossin_buf, center_buf, sino_buf, &pbuf],
(nz * ny * nx) as u32,
);
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn backproject_into_dev(
&self,
sino_buf: &wgpu::Buffer,
cossin_buf: &wgpu::Buffer,
center_buf: &wgpu::Buffer,
vol_buf: &wgpu::Buffer,
dims: (usize, usize, usize), nang: usize,
ncols: usize,
) {
let (nz, ny, nx) = dims;
let params = BpParams {
nproj: nang as u32,
ncols: ncols as u32,
ny: ny as u32,
nx: nx as u32,
scale: 1.0, _pad0: 0,
_pad1: 0,
_pad2: 0,
};
let pbuf = self.uniform("bp_params", ¶ms);
self.dispatch1d(
BACKPROJECT_WGSL,
"backproject",
&[sino_buf, cossin_buf, center_buf, vol_buf, &pbuf],
(nz * ny * nx) as u32,
);
}
fn sirt_device(
&self,
sino: &Tomo<f32>,
geom: &Geometry,
params: &crate::params::ReconParams,
) -> Result<Volume<f32>> {
let s = sino.as_layout(Layout::Sinogram);
let nz = s.n_rows();
let nang = s.n_angles();
let ncols = s.n_cols();
let n = params.num_gridx.unwrap_or(ncols);
let nvol = nz * n * n;
let nsino = nz * nang * ncols;
let (cossin, center) = cossin_center(geom, nz);
let cossin_buf = self.storage_ro("sirt_cossin", &cossin);
let center_buf = self.storage_ro("sirt_center", ¢er);
let sino_std = s.array.as_standard_layout();
let b_buf = self.storage_ro(
"sirt_b",
sino_std
.as_slice()
.expect("as_standard_layout yields a contiguous slice"),
);
let vol_buf = match ¶ms.init {
Some(v) => {
if v.dims() != (nz, n, n) {
return Err(Error::ShapeMismatch {
expected: format!("init volume [{nz}, {n}, {n}]"),
found: format!("{:?}", v.dims()),
});
}
let vstd = v.array.as_standard_layout();
self.storage_rw(
"sirt_vol",
vstd.as_slice().expect("standard layout init volume"),
)
}
None => self.storage_rw("sirt_vol", &vec![0.0f32; nvol]),
};
let ax_buf = self.storage_empty("sirt_ax", nsino); let corr_buf = self.storage_empty("sirt_corr", nvol); let rw_buf = self.storage_empty("sirt_rw", nsino); let cw_buf = self.storage_empty("sirt_cw", nvol); let ones_v = self.storage_ro("sirt_ones_v", &vec![1.0f32; nvol]);
let ones_s = self.storage_ro("sirt_ones_s", &vec![1.0f32; nsino]);
let dims = (nz, n, n);
self.forward_into_dev(
&ones_v,
&cossin_buf,
¢er_buf,
&ax_buf,
dims,
nang,
ncols,
);
self.dispatch1d(
ELEMENTWISE_WGSL,
"iter_recip",
&[&ax_buf, &rw_buf],
nsino as u32,
);
self.backproject_into_dev(
&ones_s,
&cossin_buf,
¢er_buf,
&corr_buf,
dims,
nang,
ncols,
);
self.dispatch1d(
ELEMENTWISE_WGSL,
"iter_recip",
&[&corr_buf, &cw_buf],
nvol as u32,
);
for _ in 0..params.num_iter.max(1) {
self.forward_into_dev(
&vol_buf,
&cossin_buf,
¢er_buf,
&ax_buf,
dims,
nang,
ncols,
);
self.dispatch1d(
ELEMENTWISE_WGSL,
"iter_residual",
&[&ax_buf, &b_buf, &rw_buf],
nsino as u32,
);
self.backproject_into_dev(
&ax_buf,
&cossin_buf,
¢er_buf,
&corr_buf,
dims,
nang,
ncols,
);
self.dispatch1d(
ELEMENTWISE_WGSL,
"iter_update",
&[&vol_buf, &cw_buf, &corr_buf],
nvol as u32,
);
}
let host = self.download_f32(&vol_buf, nvol);
let array = Array3::from_shape_vec((nz, n, n), host)
.expect("out buffer length nz*n*n matches the volume shape");
Ok(Volume::new(array))
}
}
impl crate::backend::IterativeReconstruct for WgpuBackend {
fn solve(
&self,
sino: &Tomo<f32>,
geom: &Geometry,
algorithm: crate::params::Algorithm,
params: &crate::params::ReconParams,
) -> Result<Option<Volume<f32>>> {
if geom.beam != Beam::Parallel {
return Ok(None);
}
match algorithm {
crate::params::Algorithm::Sirt => self.sirt_device(sino, geom, params).map(Some),
_ => Ok(None),
}
}
}