tomoxide 0.14.0

GPU/CPU tomographic reconstruction toolkit (tomopy + tomocupy, in Rust)
Documentation
// Device-resident Gaussian-USFFT gridding — WGSL port of tomocupy's
// `cfunc_fourierrec` (kernels_fourierrec.cuh) that mirrors the CPU
// `recon::fourierrec` step-for-step so the result is bit-parity (modulo GPU
// transcendental ULPs). The whole chain runs on-device: only the filtered
// sinogram is uploaded and the reconstructed volume downloaded.
//
// `WG` (workgroup size) is injected by `dispatch1d`; `M` (Gaussian half-width)
// is injected by the host so the separable-kernel array `array<f32, K>` is sized
// exactly. Every entry recovers its flat thread index as
// `gid.y * nwg.x * WG + gid.x` (the 2-D dispatch fold used by all wgpu kernels).
//
// Float atomics: `gather`/`wrap` accumulate into the grid (two lanes per
// complex cell) through the injected `atom_add_ga_grid`/`atom_add_wr_grid`
// helpers (WgpuBackend::atomic_f32_decl) — native f32 atomicAdd on devices with
// SHADER_FLOAT32_ATOMIC, else the portable compare-exchange emulation on
// bit-cast u32 lanes. Kernels that only read the grid bind the same buffer as
// `array<f32>` (identical bytes either way).

const PI : f32 = 3.14159265358979;
const K : u32 = 2u * M + 1u; // separable Gaussian kernel length (2m+1)

struct FrParams {
    nz       : u32,
    nang     : u32,
    nd       : u32,
    n        : u32, // output slice size
    ng       : u32, // oversampled grid width  = 2*nd + 2*m
    nf       : u32, // centred inverse-FFT size = 2*nd
    crop     : u32, // central-crop offset      = (nd - min(n,nd)) / 2
    _p0      : u32,
    mu       : f32,
    coeff0   : f32, // gather amplitude   = PI / (mu * 4 * nd^2)
    coeff1   : f32, // Gaussian exponent  = -PI^2 / mu
    gscale   : f32, // gather input scale = 4 / nd
    phi_sign : f32, // divphi global sign = 1 - nd%4
    norm     : f32, // pi/4 = (1/nf^2 inverse-FFT normalisation) x (pi*nd^2 unified amplitude)
    _p1      : f32,
    _p2      : f32,
};

// Centred-DFT modulation sign 1 - 2*((i+1)%2): -1 on even, +1 on odd.
fn shift_sign(i : u32) -> f32 {
    return select(1.0, -1.0, (i & 1u) == 0u);
}

// Emulated f32 atomic add is inlined at each call site: naga (without the
// unrestricted-pointer-parameters extension) forbids passing a storage pointer
// into a helper, so the compare-exchange loop on the bit-cast integer lane is
// written out where it is used (`gather`, `wrap`).

// --- 1. build the complex radial buffer with the pre-FFT shift modulation -----
@group(0) @binding(0) var<storage, read>       br_sino   : array<f32>;
@group(0) @binding(1) var<storage, read_write> br_radial : array<vec2<f32>>;
@group(0) @binding(2) var<uniform>             br_p      : FrParams;

@compute @workgroup_size(WG)
fn build_radial(@builtin(global_invocation_id) gid : vec3<u32>,
                @builtin(num_workgroups) nwg : vec3<u32>) {
    let i = gid.y * nwg.x * WG + gid.x;
    if (i >= br_p.nz * br_p.nang * br_p.nd) { return; }
    let j = i % br_p.nd;
    br_radial[i] = vec2<f32>(br_sino[i] * shift_sign(j), 0.0);
}

// --- 2. post-FFT shift modulation (in place on the radial spectrum) ----------
@group(0) @binding(0) var<storage, read_write> pm_radial : array<vec2<f32>>;
@group(0) @binding(1) var<uniform>             pm_p      : FrParams;

@compute @workgroup_size(WG)
fn postmod(@builtin(global_invocation_id) gid : vec3<u32>,
           @builtin(num_workgroups) nwg : vec3<u32>) {
    let i = gid.y * nwg.x * WG + gid.x;
    if (i >= pm_p.nz * pm_p.nang * pm_p.nd) { return; }
    let j = i % pm_p.nd;
    pm_radial[i] = pm_radial[i] * shift_sign(j);
}

// --- 3. Gaussian gather (scatter each radial sample onto the Cartesian grid) --
// @binding(1) `ga_grid` + `atom_add_ga_grid` — injected (atomic_f32_decl).
@group(0) @binding(0) var<storage, read>       ga_radial : array<vec2<f32>>;
@group(0) @binding(2) var<storage, read>       ga_trig   : array<vec2<f32>>; // (cos, sin)
@group(0) @binding(3) var<uniform>             ga_p      : FrParams;

@compute @workgroup_size(WG)
fn gather(@builtin(global_invocation_id) gid : vec3<u32>,
          @builtin(num_workgroups) nwg : vec3<u32>) {
    let i = gid.y * nwg.x * WG + gid.x;
    let nd = ga_p.nd;
    let nang = ga_p.nang;
    if (i >= ga_p.nz * nang * nd) { return; }
    let td = i % nd;
    let ia = (i / nd) % nang;
    let z  = i / (nd * nang);

    let cs = ga_trig[ia];
    let ndf = f32(nd);
    var x0 = (f32(td) - ndf * 0.5) / ndf * cs.x;
    var y0 = (f32(td) - ndf * 0.5) / ndf * cs.y;
    if (x0 >= 0.5) { x0 = 0.5 - 1e-5; }
    if (y0 >= 0.5) { y0 = 0.5 - 1e-5; }
    let g0 = ga_radial[i] * ga_p.gscale;

    let base0 = i32(floor(2.0 * ndf * x0)) - i32(M);
    let base1 = i32(floor(2.0 * ndf * y0)) - i32(M);
    var kern0 : array<f32, K>;
    var kern1 : array<f32, K>;
    for (var i0 = 0u; i0 < K; i0 = i0 + 1u) {
        let w0 = f32(base0 + i32(i0)) / (2.0 * ndf) - x0;
        kern0[i0] = exp(ga_p.coeff1 * w0 * w0);
    }
    for (var i1 = 0u; i1 < K; i1 = i1 + 1u) {
        let w1 = f32(base1 + i32(i1)) / (2.0 * ndf) - y0;
        kern1[i1] = exp(ga_p.coeff1 * w1 * w1);
    }

    let ng = ga_p.ng;
    let col0 = i32(nd) + i32(M) + base0;
    let row0 = i32(nd) + i32(M) + base1;
    let zbase = z * ng * ng;
    for (var i1 = 0u; i1 < K; i1 = i1 + 1u) {
        let rr = u32(row0 + i32(i1));
        for (var i0 = 0u; i0 < K; i0 = i0 + 1u) {
            let w = ga_p.coeff0 * kern0[i0] * kern1[i1];
            let cc = u32(col0 + i32(i0));
            let cell = zbase + rr * ng + cc;
            atom_add_ga_grid(2u * cell, g0.x * w);
            atom_add_ga_grid(2u * cell + 1u, g0.y * w);
        }
    }
}

// --- 4. wrap the m-wide borders back into the interior (periodic over 2*nd) ---
// @binding(0) `wr_grid` + `atom_add_wr_grid`/`atom_load_wr_grid` — injected.
@group(0) @binding(1) var<uniform>             wr_p    : FrParams;

@compute @workgroup_size(WG)
fn wrap(@builtin(global_invocation_id) gid : vec3<u32>,
        @builtin(num_workgroups) nwg : vec3<u32>) {
    let i = gid.y * nwg.x * WG + gid.x;
    let ng = wr_p.ng;
    if (i >= wr_p.nz * ng * ng) { return; }
    let tx = i % ng;
    let ty = (i / ng) % ng;
    let z  = i / (ng * ng);
    let m = M;
    let twond = 2u * wr_p.nd;
    if (tx < m || tx >= twond + m || ty < m || ty >= twond + m) {
        let tx0 = (tx + twond - m) % twond;
        let ty0 = (ty + twond - m) % twond;
        let zbase = z * ng * ng;
        let src = zbase + ty * ng + tx;
        let dst = zbase + (ty0 + m) * ng + (tx0 + m);
        // Border cells receive no in-place accumulation after the gather, and
        // each border cell wraps to exactly one interior destination, so the
        // loads race with nothing; the adds still race with other border cells
        // mapping to the same interior cell.
        atom_add_wr_grid(2u * dst, atom_load_wr_grid(2u * src));
        atom_add_wr_grid(2u * dst + 1u, atom_load_wr_grid(2u * src + 1u));
    }
}

// --- 5. extract the 2*nd interior block + pre-inverse-FFT shift modulation ----
@group(0) @binding(0) var<storage, read>       ex_grid  : array<f32>; // grid bytes as f32
@group(0) @binding(1) var<storage, read_write> ex_inner : array<vec2<f32>>;
@group(0) @binding(2) var<uniform>             ex_p     : FrParams;

@compute @workgroup_size(WG)
fn extract(@builtin(global_invocation_id) gid : vec3<u32>,
           @builtin(num_workgroups) nwg : vec3<u32>) {
    let i = gid.y * nwg.x * WG + gid.x;
    let nf = ex_p.nf;
    if (i >= ex_p.nz * nf * nf) { return; }
    let rx = i % nf;
    let ry = (i / nf) % nf;
    let z  = i / (nf * nf);
    let ng = ex_p.ng;
    let m = M;
    let gcell = z * ng * ng + (ry + m) * ng + (rx + m);
    let v = vec2<f32>(ex_grid[2u * gcell], ex_grid[2u * gcell + 1u]);
    ex_inner[i] = v * (shift_sign(rx) * shift_sign(ry));
}

// --- 6. post-inverse-FFT shift modulation (in place) -------------------------
@group(0) @binding(0) var<storage, read_write> fs_inner : array<vec2<f32>>;
@group(0) @binding(1) var<uniform>             fs_p     : FrParams;

@compute @workgroup_size(WG)
fn fftshift(@builtin(global_invocation_id) gid : vec3<u32>,
            @builtin(num_workgroups) nwg : vec3<u32>) {
    let i = gid.y * nwg.x * WG + gid.x;
    let nf = fs_p.nf;
    if (i >= fs_p.nz * nf * nf) { return; }
    let rx = i % nf;
    let ry = (i / nf) % nf;
    fs_inner[i] = fs_inner[i] * (shift_sign(rx) * shift_sign(ry));
}

// --- 7. Gaussian deapodize (divphi) + central crop + unit-disk mask (circ) ----
@group(0) @binding(0) var<storage, read>       de_inner : array<vec2<f32>>;
@group(0) @binding(1) var<storage, read_write> de_out   : array<f32>;
@group(0) @binding(2) var<uniform>             de_p     : FrParams;

@compute @workgroup_size(WG)
fn deapodize(@builtin(global_invocation_id) gid : vec3<u32>,
             @builtin(num_workgroups) nwg : vec3<u32>) {
    let i = gid.y * nwg.x * WG + gid.x;
    let n = de_p.n;
    if (i >= de_p.nz * n * n) { return; }
    let ox = i % n;
    let oy = (i / n) % n;
    let z  = i / (n * n);
    let nd = de_p.nd;
    let nf = de_p.nf;
    let ty = oy + de_p.crop;
    let tx = ox + de_p.crop;
    let ndf = f32(nd);
    let dy = f32(ty) / ndf - 0.5;
    let dx = f32(tx) / ndf - 0.5;
    let phi = exp(de_p.mu * ndf * ndf * (dx * dx + dy * dy)) / f32(de_p.nang) * de_p.phi_sign;
    let inner_row = ty + nd / 2u;
    let inner_col = tx + nd / 2u;
    let icell = z * nf * nf + inner_row * nf + inner_col;
    let v = de_inner[icell].x * phi * de_p.norm;
    let my = (f32(ty) - ndf * 0.5) / ndf;
    let mx = (f32(tx) - ndf * 0.5) / ndf;
    let masked = select(0.0, v, (4.0 * mx * mx + 4.0 * my * my) < 1.0);
    de_out[z * n * n + oy * n + ox] = masked;
}