1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
//! Small reusable wgpu compute primitives shared by the WGSL kernel ports:
//! buffer upload, single-pass 1-D dispatch (auto bind-group layout, workgroup
//! size 256), and host readback. Only compiled under `gpu-wgpu`.
use bytemuck::Pod;
use wgpu::util::DeviceExt;
use crate::wgpu::WgpuBackend;
/// Threads per workgroup for the 1-D dispatch helper (must match the
/// `@workgroup_size(256)` in every kernel dispatched through [`WgpuBackend::dispatch1d`]).
pub(crate) const WORKGROUP: u32 = 256;
impl WgpuBackend {
/// A read/write storage buffer initialised from `data`, also usable as a
/// `copy` source so its contents can be read back with [`Self::download_f32`].
pub(crate) fn storage_rw(&self, label: &str, data: &[f32]) -> wgpu::Buffer {
self.device
.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some(label),
contents: bytemuck::cast_slice(data),
usage: wgpu::BufferUsages::STORAGE
| wgpu::BufferUsages::COPY_SRC
| wgpu::BufferUsages::COPY_DST,
})
}
/// A read-only storage buffer initialised from `data`.
pub(crate) fn storage_ro(&self, label: &str, data: &[f32]) -> wgpu::Buffer {
self.device
.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some(label),
contents: bytemuck::cast_slice(data),
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST,
})
}
/// A read-only storage buffer initialised from a `u32` index array (lprec
/// gather/scatter targets bind as `array<u32>`).
pub(crate) fn storage_ro_u32(&self, label: &str, data: &[u32]) -> wgpu::Buffer {
self.device
.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some(label),
contents: bytemuck::cast_slice(data),
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST,
})
}
/// An uninitialised read/write storage buffer of `len` f32s, usable as a
/// `copy` source (readback) and destination (clear / upload). The caller must
/// fully write it before reading, or [`Self::zero_buffer`] it first for
/// accumulation targets (the atomic gather/wrap grid).
pub(crate) fn storage_empty(&self, label: &str, len: usize) -> wgpu::Buffer {
self.device.create_buffer(&wgpu::BufferDescriptor {
label: Some(label),
size: (len * std::mem::size_of::<f32>()) as u64,
usage: wgpu::BufferUsages::STORAGE
| wgpu::BufferUsages::COPY_SRC
| wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
})
}
/// Zero a storage buffer on-device — cheaper than uploading a host zero
/// vector for the MB-scale accumulation grid.
pub(crate) fn zero_buffer(&self, buf: &wgpu::Buffer) {
let mut enc = self
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("zero_buffer"),
});
enc.clear_buffer(buf, 0, None);
self.queue.submit(Some(enc.finish()));
}
/// A uniform buffer holding a single `Pod` value (pad it to a multiple of 16
/// bytes so it satisfies the WGSL uniform layout rules).
pub(crate) fn uniform<T: Pod>(&self, label: &str, value: &T) -> wgpu::Buffer {
self.device
.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some(label),
contents: bytemuck::bytes_of(value),
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
})
}
/// WGSL header (binding declaration + accumulate/load helpers) for an
/// atomically-accumulated f32 storage buffer `name` at `@group(0)
/// @binding(binding)`. The scatter kernels (forward projection, fourierrec
/// gather/wrap, lprec gather) declare their accumulation target through this
/// so the *same kernel source* runs on both device classes:
///
/// - with [`wgpu::Features::SHADER_FLOAT32_ATOMIC`] (`self.f32_atomics`):
/// `array<atomic<f32>>` + native `atomicAdd` — one atomic per update;
/// - without: the portable emulation on `array<atomic<u32>>` — a
/// compare-exchange loop on the bit-cast lane, which costs a
/// read-modify-retry round per update (2–3× the memory traffic).
///
/// Both variants leave the buffer's raw bytes as IEEE f32, so zeroing,
/// readback ([`Self::download_f32`]) and non-atomic re-binding as
/// `array<f32>` in later kernels are identical. The generated helpers are
/// `atom_add_{name}(idx, v)` and `atom_load_{name}(idx) -> f32`.
pub(crate) fn atomic_f32_decl(&self, name: &str, binding: u32) -> String {
if self.f32_atomics {
format!(
"@group(0) @binding({binding}) var<storage, read_write> {name} : array<atomic<f32>>;\n\
fn atom_add_{name}(idx : u32, v : f32) {{ atomicAdd(&{name}[idx], v); }}\n\
fn atom_load_{name}(idx : u32) -> f32 {{ return atomicLoad(&{name}[idx]); }}\n"
)
} else {
format!(
"@group(0) @binding({binding}) var<storage, read_write> {name} : array<atomic<u32>>;\n\
fn atom_add_{name}(idx : u32, v : f32) {{\n\
var old = atomicLoad(&{name}[idx]);\n\
loop {{\n\
let r = atomicCompareExchangeWeak(&{name}[idx], old, bitcast<u32>(bitcast<f32>(old) + v));\n\
if (r.exchanged) {{ break; }}\n\
old = r.old_value;\n\
}}\n\
}}\n\
fn atom_load_{name}(idx : u32) -> f32 {{ return bitcast<f32>(atomicLoad(&{name}[idx])); }}\n"
)
}
}
/// Compile `wgsl`, then dispatch its `entry` compute function over
/// `n_threads` invocations, binding `buffers` at `@group(0)` in slice order.
/// Uses the pipeline's auto-deduced bind-group layout, so each buffer's
/// address space (`storage`/`uniform`, read vs read_write) is taken from the
/// shader declaration.
///
/// The dispatch launches `ceil(n_threads / WORKGROUP)` workgroups, so the
/// kernel's `@workgroup_size` MUST equal [`WORKGROUP`] or threads at the tail
/// go unlaunched. To make that impossible to get wrong, a `const WG` is
/// injected into the shader source from [`WORKGROUP`]; every 1-D kernel
/// declares `@workgroup_size(WG)` rather than a literal, so the size has a
/// single source of truth here.
///
/// WebGPU caps each dispatch dimension at 65535 workgroups, which a 1-D
/// dispatch blows past once `n_threads` exceeds `65535 * WORKGROUP` (≈16.8 M —
/// reached by a 512² back-projection's `nz·n·n` threads). So the workgroup
/// count is folded into a 2-D `(wx, wy)` grid with `wx ≤ 65535`, and every
/// kernel recovers its flat thread index as
/// `gid.y * num_workgroups.x * WG + gid.x` — a formula that also yields plain
/// `gid.x` in the common `wy == 1` case, so the same kernel source works for
/// both. Tail workgroups (the grid rounds up) are handled by each kernel's
/// existing `idx >= len` bounds check.
/// Fetch (or compile once and cache) the compute pipeline for `src`/`entry`.
///
/// Keyed by a hash of the full source (including any injected `const`s) plus
/// the entry name, so distinct kernels and size-specialized variants each get
/// their own cached pipeline while a repeated dispatch reuses the compiled
/// one. This turns the per-dispatch naga compile + driver pipeline build into
/// a one-time cost per unique kernel — the dominant wgpu overhead vs CUDA's
/// precompiled kernels (see the `pipelines` field doc on [`WgpuBackend`]).
pub(crate) fn cached_pipeline(
&self,
src: &str,
entry: &str,
) -> std::sync::Arc<wgpu::ComputePipeline> {
use std::hash::{Hash, Hasher};
let mut h = std::collections::hash_map::DefaultHasher::new();
src.hash(&mut h);
entry.hash(&mut h);
let key = h.finish();
if let Some(p) = self.pipelines.lock().unwrap().get(&key) {
return p.clone();
}
let module = self
.device
.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some(entry),
source: wgpu::ShaderSource::Wgsl(src.into()),
});
let pipeline = self
.device
.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: Some(entry),
layout: None,
module: &module,
entry_point: Some(entry),
compilation_options: wgpu::PipelineCompilationOptions::default(),
cache: None,
});
let arc = std::sync::Arc::new(pipeline);
self.pipelines.lock().unwrap().insert(key, arc.clone());
arc
}
pub(crate) fn dispatch1d(
&self,
wgsl: &str,
entry: &str,
buffers: &[&wgpu::Buffer],
n_threads: u32,
) {
let src = format!("const WG : u32 = {WORKGROUP}u;\n{wgsl}");
let pipeline = self.cached_pipeline(&src, entry);
let entries: Vec<wgpu::BindGroupEntry> = buffers
.iter()
.enumerate()
.map(|(i, b)| wgpu::BindGroupEntry {
binding: i as u32,
resource: b.as_entire_binding(),
})
.collect();
let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some(entry),
layout: &pipeline.get_bind_group_layout(0),
entries: &entries,
});
let mut enc = self
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some(entry) });
{
let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some(entry),
timestamp_writes: None,
});
pass.set_pipeline(&pipeline);
pass.set_bind_group(0, &bind_group, &[]);
// (pipeline is an Arc; &pipeline derefs to &ComputePipeline here.)
// Fold the workgroup count into a 2-D grid so neither dimension
// exceeds WebGPU's 65535 per-dimension cap (see method doc).
let wg = n_threads.div_ceil(WORKGROUP);
const MAX_DIM: u32 = 65535;
let (wx, wy) = if wg <= MAX_DIM {
(wg, 1)
} else {
(MAX_DIM, wg.div_ceil(MAX_DIM))
};
pass.dispatch_workgroups(wx, wy, 1);
}
self.queue.submit(Some(enc.finish()));
}
/// Copy `len` f32s back from a `COPY_SRC` storage buffer to the host, blocking
/// until the GPU work completes.
pub(crate) fn download_f32(&self, src: &wgpu::Buffer, len: usize) -> Vec<f32> {
let bytes = (len * std::mem::size_of::<f32>()) as wgpu::BufferAddress;
let staging = self.device.create_buffer(&wgpu::BufferDescriptor {
label: Some("staging"),
size: bytes,
usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
let mut enc = self
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("download"),
});
enc.copy_buffer_to_buffer(src, 0, &staging, 0, bytes);
self.queue.submit(Some(enc.finish()));
let slice = staging.slice(..);
let (tx, rx) = std::sync::mpsc::channel();
slice.map_async(wgpu::MapMode::Read, move |r| {
let _ = tx.send(r);
});
self.device
.poll(wgpu::PollType::wait_indefinitely())
.expect("wgpu device poll");
rx.recv().unwrap().unwrap();
let out = {
let view = slice
.get_mapped_range()
.expect("mapped range after successful map_async");
bytemuck::cast_slice(&view).to_vec()
};
staging.unmap();
out
}
}