use std::{
cell::{Cell, RefCell},
sync::{Arc, Mutex},
};
use super::vector::{Vec3u, Vec4f, vec3u, vec4f};
thread_local! {
static WORKGROUP_BARRIER: RefCell<Option<Arc<std::sync::Barrier>>> = const { RefCell::new(None) };
static QUAD_CONTEXT: RefCell<Option<Arc<QuadContext>>> = const { RefCell::new(None) };
static QUAD_INDEX: Cell<u8> = const { Cell::new(0) };
static COMPUTE_BUILTINS: RefCell<Option<ComputeBuiltins>> = const { RefCell::new(None) };
static DISCARDED: Cell<bool> = const { Cell::new(false) };
}
pub(crate) fn with_workgroup_barrier<R>(f: impl FnOnce(Option<&std::sync::Barrier>) -> R) -> R {
WORKGROUP_BARRIER.with(|b| {
let guard = b.borrow();
f(guard.as_ref().map(|arc| arc.as_ref()))
})
}
pub(crate) fn mark_discarded() {
DISCARDED.with(|d| d.set(true));
}
pub(crate) fn with_quad_context<R>(f: impl FnOnce(Option<(&QuadContext, u8)>) -> R) -> R {
QUAD_CONTEXT.with(|ctx| {
let guard = ctx.borrow();
match guard.as_ref() {
Some(qctx) => {
let idx = QUAD_INDEX.with(|i| i.get());
f(Some((qctx.as_ref(), idx)))
}
None => f(None),
}
})
}
#[derive(Clone, Copy, Debug)]
pub struct ComputeBuiltins {
pub global_invocation_id: Vec3u,
pub local_invocation_id: Vec3u,
pub local_invocation_index: u32,
pub workgroup_id: Vec3u,
pub num_workgroups: Vec3u,
}
pub fn dispatch_workgroups<F>(
workgroup_count: (u32, u32, u32),
workgroup_size: (u32, u32, u32),
shader_fn: F,
) where
F: Fn(ComputeBuiltins) + Send + Sync,
{
let (cx, cy, cz) = workgroup_count;
let (sx, sy, sz) = workgroup_size;
let invocations_per_workgroup = (sx * sy * sz) as usize;
let num_workgroups = vec3u(cx, cy, cz);
const MAX_INVOCATIONS: usize = 1024;
assert!(
invocations_per_workgroup <= MAX_INVOCATIONS,
"workgroup size {sx}x{sy}x{sz} = {invocations_per_workgroup} invocations exceeds the CPU \
dispatch runtime limit of {MAX_INVOCATIONS} threads per workgroup"
);
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(invocations_per_workgroup)
.build()
.expect("failed to build workgroup thread pool");
for wz in 0..cz {
for wy in 0..cy {
for wx in 0..cx {
let workgroup_id = vec3u(wx, wy, wz);
let barrier = Arc::new(std::sync::Barrier::new(invocations_per_workgroup));
pool.scope(|s| {
for lz in 0..sz {
for ly in 0..sy {
for lx in 0..sx {
let barrier = barrier.clone();
let local_id = vec3u(lx, ly, lz);
let local_index = lx + ly * sx + lz * sx * sy;
let global_id = vec3u(wx * sx + lx, wy * sy + ly, wz * sz + lz);
let builtins = ComputeBuiltins {
global_invocation_id: global_id,
local_invocation_id: local_id,
local_invocation_index: local_index,
workgroup_id,
num_workgroups,
};
let shader_fn = &shader_fn;
s.spawn(move |_| {
WORKGROUP_BARRIER.with(|b| {
*b.borrow_mut() = Some(barrier);
});
COMPUTE_BUILTINS.with(|b| {
*b.borrow_mut() = Some(builtins);
});
shader_fn(builtins);
WORKGROUP_BARRIER.with(|b| {
*b.borrow_mut() = None;
});
COMPUTE_BUILTINS.with(|b| {
*b.borrow_mut() = None;
});
});
}
}
}
});
}
}
}
}
pub struct QuadContext {
slots: [Mutex<[f32; 4]>; 4],
barrier: std::sync::Barrier,
}
impl QuadContext {
fn new() -> Self {
QuadContext {
slots: std::array::from_fn(|_| Mutex::new([0.0; 4])),
barrier: std::sync::Barrier::new(4),
}
}
pub fn dpdx_fine_f32(&self, quad_idx: u8, value: f32) -> f32 {
*self.slots[quad_idx as usize]
.lock()
.expect("quad slot poisoned") = [value, 0.0, 0.0, 0.0];
self.barrier.wait();
let left_idx = (quad_idx & !1) as usize; let right_idx = (quad_idx | 1) as usize; let left = self.slots[left_idx].lock().expect("quad slot poisoned")[0];
let right = self.slots[right_idx].lock().expect("quad slot poisoned")[0];
self.barrier.wait();
right - left
}
pub fn dpdx_coarse_f32(&self, quad_idx: u8, value: f32) -> f32 {
*self.slots[quad_idx as usize]
.lock()
.expect("quad slot poisoned") = [value, 0.0, 0.0, 0.0];
self.barrier.wait();
let left = self.slots[0].lock().expect("quad slot poisoned")[0];
let right = self.slots[1].lock().expect("quad slot poisoned")[0];
self.barrier.wait();
right - left
}
pub fn dpdy_fine_f32(&self, quad_idx: u8, value: f32) -> f32 {
*self.slots[quad_idx as usize]
.lock()
.expect("quad slot poisoned") = [value, 0.0, 0.0, 0.0];
self.barrier.wait();
let top_idx = (quad_idx & !2) as usize; let bottom_idx = (quad_idx | 2) as usize; let top = self.slots[top_idx].lock().expect("quad slot poisoned")[0];
let bottom = self.slots[bottom_idx].lock().expect("quad slot poisoned")[0];
self.barrier.wait();
bottom - top
}
pub fn dpdy_coarse_f32(&self, quad_idx: u8, value: f32) -> f32 {
*self.slots[quad_idx as usize]
.lock()
.expect("quad slot poisoned") = [value, 0.0, 0.0, 0.0];
self.barrier.wait();
let top = self.slots[0].lock().expect("quad slot poisoned")[0];
let bottom = self.slots[2].lock().expect("quad slot poisoned")[0];
self.barrier.wait();
bottom - top
}
pub fn dpdx_fine_components(&self, quad_idx: u8, components: &[f32], n: usize) -> [f32; 4] {
let mut slot = [0.0f32; 4];
slot[..n].copy_from_slice(&components[..n]);
*self.slots[quad_idx as usize]
.lock()
.expect("quad slot poisoned") = slot;
self.barrier.wait();
let left_idx = (quad_idx & !1) as usize;
let right_idx = (quad_idx | 1) as usize;
let left = *self.slots[left_idx].lock().expect("quad slot poisoned");
let right = *self.slots[right_idx].lock().expect("quad slot poisoned");
self.barrier.wait();
let mut result = [0.0f32; 4];
for i in 0..n {
result[i] = right[i] - left[i];
}
result
}
pub fn dpdx_coarse_components(&self, quad_idx: u8, components: &[f32], n: usize) -> [f32; 4] {
let mut slot = [0.0f32; 4];
slot[..n].copy_from_slice(&components[..n]);
*self.slots[quad_idx as usize]
.lock()
.expect("quad slot poisoned") = slot;
self.barrier.wait();
let left = *self.slots[0].lock().expect("quad slot poisoned");
let right = *self.slots[1].lock().expect("quad slot poisoned");
self.barrier.wait();
let mut result = [0.0f32; 4];
for i in 0..n {
result[i] = right[i] - left[i];
}
result
}
pub fn dpdy_fine_components(&self, quad_idx: u8, components: &[f32], n: usize) -> [f32; 4] {
let mut slot = [0.0f32; 4];
slot[..n].copy_from_slice(&components[..n]);
*self.slots[quad_idx as usize]
.lock()
.expect("quad slot poisoned") = slot;
self.barrier.wait();
let top_idx = (quad_idx & !2) as usize;
let bottom_idx = (quad_idx | 2) as usize;
let top = *self.slots[top_idx].lock().expect("quad slot poisoned");
let bottom = *self.slots[bottom_idx].lock().expect("quad slot poisoned");
self.barrier.wait();
let mut result = [0.0f32; 4];
for i in 0..n {
result[i] = bottom[i] - top[i];
}
result
}
pub fn dpdy_coarse_components(&self, quad_idx: u8, components: &[f32], n: usize) -> [f32; 4] {
let mut slot = [0.0f32; 4];
slot[..n].copy_from_slice(&components[..n]);
*self.slots[quad_idx as usize]
.lock()
.expect("quad slot poisoned") = slot;
self.barrier.wait();
let top = *self.slots[0].lock().expect("quad slot poisoned");
let bottom = *self.slots[2].lock().expect("quad slot poisoned");
self.barrier.wait();
let mut result = [0.0f32; 4];
for i in 0..n {
result[i] = bottom[i] - top[i];
}
result
}
}
#[derive(Clone, Copy, Debug)]
pub struct FragmentBuiltins {
pub position: Vec4f,
pub front_facing: bool,
pub sample_index: u32,
pub sample_mask: u32,
}
pub fn dispatch_fragments<I, O, F>(
width: u32,
height: u32,
input_fn: impl Fn(u32, u32) -> I + Send + Sync,
shader_fn: F,
) -> Vec<Vec<Option<O>>>
where
I: Send,
O: Send,
F: Fn(FragmentBuiltins, I) -> O + Send + Sync,
{
let padded_w = (width + 1) & !1;
let padded_h = (height + 1) & !1;
let grid: Vec<Vec<Arc<Mutex<Option<O>>>>> = (0..height)
.map(|_| (0..width).map(|_| Arc::new(Mutex::new(None))).collect())
.collect();
let quad_coords: Vec<(u32, u32)> = {
let mut coords = Vec::new();
let mut qy = 0;
while qy < padded_h {
let mut qx = 0;
while qx < padded_w {
coords.push((qx, qy));
qx += 2;
}
qy += 2;
}
coords
};
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(4)
.build()
.expect("failed to build fragment quad thread pool");
{
let shader_fn = &shader_fn;
let input_fn = &input_fn;
let grid = &grid;
for (qx, qy) in quad_coords {
let quad_ctx = Arc::new(QuadContext::new());
pool.scope(|s| {
for dy in 0..2u32 {
for dx in 0..2u32 {
let px = qx + dx;
let py = qy + dy;
let quad_idx = (dx + dy * 2) as u8;
let quad_ctx = quad_ctx.clone();
let is_helper = px >= width || py >= height;
let clamped_x = px.min(width.saturating_sub(1));
let clamped_y = py.min(height.saturating_sub(1));
let input = input_fn(clamped_x, clamped_y);
let grid_cell = if !is_helper {
Some(grid[py as usize][px as usize].clone())
} else {
None
};
s.spawn(move |_| {
QUAD_CONTEXT.with(|ctx| {
*ctx.borrow_mut() = Some(quad_ctx);
});
QUAD_INDEX.with(|i| i.set(quad_idx));
DISCARDED.with(|d| d.set(false));
let builtins = FragmentBuiltins {
position: vec4f(px as f32 + 0.5, py as f32 + 0.5, 0.0, 1.0),
front_facing: true,
sample_index: 0,
sample_mask: 0xFFFFFFFF,
};
let output = shader_fn(builtins, input);
if let Some(cell) = grid_cell {
let discarded = DISCARDED.with(|d| d.get());
if !discarded {
*cell.lock().expect("grid cell poisoned") = Some(output);
}
}
QUAD_CONTEXT.with(|ctx| {
*ctx.borrow_mut() = None;
});
});
}
}
});
}
}
grid.into_iter()
.map(|row| {
row.into_iter()
.map(|cell| {
Arc::try_unwrap(cell)
.unwrap_or_else(|_| panic!("all quad threads should have completed"))
.into_inner()
.unwrap_or_else(|_| panic!("grid cell poisoned"))
})
.collect()
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicU32, Ordering};
fn with_quad_test_pool<F: FnOnce(&rayon::Scope<'_>) + Send>(f: F) {
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(4)
.build()
.expect("failed to build quad test thread pool");
pool.scope(f);
}
#[test]
fn compute_dispatch_basic() {
let counter = AtomicU32::new(0);
dispatch_workgroups((2, 1, 1), (4, 1, 1), |_builtins| {
counter.fetch_add(1, Ordering::Relaxed);
});
assert_eq!(counter.load(Ordering::Relaxed), 8);
}
#[test]
fn compute_dispatch_3d() {
let counter = AtomicU32::new(0);
dispatch_workgroups((2, 3, 2), (2, 2, 2), |_builtins| {
counter.fetch_add(1, Ordering::Relaxed);
});
assert_eq!(counter.load(Ordering::Relaxed), 96);
}
#[test]
fn compute_builtins_correct() {
use std::sync::Mutex;
let results: Mutex<Vec<ComputeBuiltins>> = Mutex::new(Vec::new());
dispatch_workgroups((2, 1, 1), (2, 1, 1), |builtins| {
results.lock().unwrap().push(builtins);
});
let mut results = results.into_inner().unwrap();
results.sort_by_key(|b| {
let g = b.global_invocation_id.to_array();
(g[0], g[1], g[2])
});
assert_eq!(results.len(), 4);
let b = &results[0];
assert_eq!(b.global_invocation_id.to_array(), [0, 0, 0]);
assert_eq!(b.local_invocation_id.to_array(), [0, 0, 0]);
assert_eq!(b.local_invocation_index, 0);
assert_eq!(b.workgroup_id.to_array(), [0, 0, 0]);
let b = &results[1];
assert_eq!(b.global_invocation_id.to_array(), [1, 0, 0]);
assert_eq!(b.local_invocation_id.to_array(), [1, 0, 0]);
assert_eq!(b.local_invocation_index, 1);
assert_eq!(b.workgroup_id.to_array(), [0, 0, 0]);
let b = &results[2];
assert_eq!(b.global_invocation_id.to_array(), [2, 0, 0]);
assert_eq!(b.local_invocation_id.to_array(), [0, 0, 0]);
assert_eq!(b.workgroup_id.to_array(), [1, 0, 0]);
let b = &results[3];
assert_eq!(b.global_invocation_id.to_array(), [3, 0, 0]);
assert_eq!(b.local_invocation_id.to_array(), [1, 0, 0]);
assert_eq!(b.workgroup_id.to_array(), [1, 0, 0]);
}
#[test]
fn compute_dispatch_barrier_synchronizes() {
use std::sync::Mutex;
let workgroup_size = 4;
let shared = Arc::new(Mutex::new(vec![0u32; workgroup_size]));
let observed = Arc::new(Mutex::new(Vec::<Vec<u32>>::new()));
dispatch_workgroups((1, 1, 1), (workgroup_size as u32, 1, 1), |builtins| {
let idx = builtins.local_invocation_index as usize;
shared.lock().unwrap()[idx] = (idx as u32) + 1;
crate::std::workgroup_barrier();
let snapshot = shared.lock().unwrap().clone();
observed.lock().unwrap().push(snapshot);
});
let observed = observed.lock().unwrap();
assert_eq!(observed.len(), workgroup_size);
for snapshot in observed.iter() {
for (i, &val) in snapshot.iter().enumerate() {
assert_eq!(val, (i as u32) + 1, "barrier failed to synchronize");
}
}
}
#[test]
fn fragment_dispatch_basic() {
let results = dispatch_fragments(4, 4, |x, y| (x, y), |_builtins, input| input);
assert_eq!(results.len(), 4);
for (y, row) in results.iter().enumerate() {
assert_eq!(row.len(), 4);
for (x, cell) in row.iter().enumerate() {
let (rx, ry) = cell.expect("fragment should produce output");
assert_eq!(rx, x as u32);
assert_eq!(ry, y as u32);
}
}
}
#[test]
fn fragment_dispatch_odd_dimensions() {
let results = dispatch_fragments(3, 3, |x, y| (x, y), |_builtins, input| input);
assert_eq!(results.len(), 3);
for (y, row) in results.iter().enumerate() {
assert_eq!(row.len(), 3);
for (x, cell) in row.iter().enumerate() {
let (rx, ry) = cell.expect("fragment should produce output");
assert_eq!(rx, x as u32);
assert_eq!(ry, y as u32);
}
}
}
#[test]
fn fragment_dispatch_1x1() {
let results = dispatch_fragments(1, 1, |x, y| (x, y), |_builtins, input| input);
assert_eq!(results.len(), 1);
assert_eq!(results[0].len(), 1);
let (rx, ry) = results[0][0].expect("fragment should produce output");
assert_eq!(rx, 0);
assert_eq!(ry, 0);
}
#[test]
fn fragment_dispatch_position_builtins() {
let results = dispatch_fragments(2, 2, |_x, _y| (), |builtins, _| builtins.position);
for (y, row) in results.iter().enumerate() {
for (x, cell) in row.iter().enumerate() {
let pos = cell.expect("fragment should produce output");
let components = pos.to_array();
assert!(
(components[0] - (x as f32 + 0.5)).abs() < f32::EPSILON,
"position.x mismatch at ({x}, {y})"
);
assert!(
(components[1] - (y as f32 + 0.5)).abs() < f32::EPSILON,
"position.y mismatch at ({x}, {y})"
);
}
}
}
#[test]
fn fragment_dispatch_discard() {
let results = dispatch_fragments(
4,
4,
|x, y| (x, y),
|_builtins, (x, y)| {
if x % 2 == 0 {
mark_discarded();
}
(x, y)
},
);
assert_eq!(results.len(), 4);
for (y, row) in results.iter().enumerate() {
assert_eq!(row.len(), 4);
for (x, cell) in row.iter().enumerate() {
if x % 2 == 0 {
assert!(
cell.is_none(),
"discarded fragment at ({x}, {y}) should be None"
);
} else {
let (rx, ry) = cell.unwrap_or_else(|| {
panic!("non-discarded fragment at ({x}, {y}) should produce output")
});
assert_eq!(rx, x as u32);
assert_eq!(ry, y as u32);
}
}
}
}
#[test]
fn fragment_dispatch_discard_all() {
let results = dispatch_fragments(
2,
2,
|x, y| (x, y),
|_builtins, input| {
mark_discarded();
input
},
);
for (y, row) in results.iter().enumerate() {
for (x, cell) in row.iter().enumerate() {
assert!(
cell.is_none(),
"all-discarded fragment at ({x}, {y}) should be None"
);
}
}
}
#[test]
fn quad_context_dpdx_fine() {
let quad = Arc::new(QuadContext::new());
let values = [10.0f32, 30.0, 20.0, 40.0];
let results: Arc<Mutex<[f32; 4]>> = Arc::new(Mutex::new([0.0; 4]));
with_quad_test_pool(|s| {
for idx in 0..4u8 {
let quad = quad.clone();
let results = results.clone();
s.spawn(move |_| {
let r = quad.dpdx_fine_f32(idx, values[idx as usize]);
results.lock().unwrap()[idx as usize] = r;
});
}
});
let results = results.lock().unwrap();
assert_eq!(results[0], 20.0); assert_eq!(results[1], 20.0); assert_eq!(results[2], 20.0); assert_eq!(results[3], 20.0); }
#[test]
fn quad_context_dpdy_fine() {
let quad = Arc::new(QuadContext::new());
let values = [10.0f32, 30.0, 50.0, 70.0];
let results: Arc<Mutex<[f32; 4]>> = Arc::new(Mutex::new([0.0; 4]));
with_quad_test_pool(|s| {
for idx in 0..4u8 {
let quad = quad.clone();
let results = results.clone();
s.spawn(move |_| {
let r = quad.dpdy_fine_f32(idx, values[idx as usize]);
results.lock().unwrap()[idx as usize] = r;
});
}
});
let results = results.lock().unwrap();
assert_eq!(results[0], 40.0); assert_eq!(results[1], 40.0); assert_eq!(results[2], 40.0); assert_eq!(results[3], 40.0); }
#[test]
fn quad_context_dpdx_coarse() {
let quad = Arc::new(QuadContext::new());
let values = [10.0f32, 30.0, 20.0, 40.0];
let results: Arc<Mutex<[f32; 4]>> = Arc::new(Mutex::new([0.0; 4]));
with_quad_test_pool(|s| {
for idx in 0..4u8 {
let quad = quad.clone();
let results = results.clone();
s.spawn(move |_| {
let r = quad.dpdx_coarse_f32(idx, values[idx as usize]);
results.lock().unwrap()[idx as usize] = r;
});
}
});
let results = results.lock().unwrap();
for i in 0..4 {
assert_eq!(
results[i], 20.0,
"coarse dpdx should be 20.0 for all members"
);
}
}
#[test]
fn quad_context_dpdy_coarse() {
let quad = Arc::new(QuadContext::new());
let values = [10.0f32, 30.0, 50.0, 70.0];
let results: Arc<Mutex<[f32; 4]>> = Arc::new(Mutex::new([0.0; 4]));
with_quad_test_pool(|s| {
for idx in 0..4u8 {
let quad = quad.clone();
let results = results.clone();
s.spawn(move |_| {
let r = quad.dpdy_coarse_f32(idx, values[idx as usize]);
results.lock().unwrap()[idx as usize] = r;
});
}
});
let results = results.lock().unwrap();
for i in 0..4 {
assert_eq!(
results[i], 40.0,
"coarse dpdy should be 40.0 for all members"
);
}
}
#[test]
fn quad_context_multiple_derivative_calls() {
let quad = Arc::new(QuadContext::new());
let values1 = [1.0f32, 3.0, 2.0, 4.0];
let values2 = [10.0f32, 10.0, 20.0, 20.0];
let results: Arc<Mutex<[(f32, f32); 4]>> = Arc::new(Mutex::new([(0.0, 0.0); 4]));
with_quad_test_pool(|s| {
for idx in 0..4u8 {
let quad = quad.clone();
let results = results.clone();
s.spawn(move |_| {
let r1 = quad.dpdx_fine_f32(idx, values1[idx as usize]);
let r2 = quad.dpdy_fine_f32(idx, values2[idx as usize]);
results.lock().unwrap()[idx as usize] = (r1, r2);
});
}
});
let results = results.lock().unwrap();
for i in 0..4 {
assert_eq!(results[i].0, 2.0, "first derivative call");
assert_eq!(results[i].1, 10.0, "second derivative call");
}
}
#[test]
fn quad_context_components_2d() {
let quad = Arc::new(QuadContext::new());
let values: [[f32; 2]; 4] = [[1.0, 10.0], [3.0, 30.0], [2.0, 20.0], [4.0, 40.0]];
let results: Arc<Mutex<[[f32; 4]; 4]>> = Arc::new(Mutex::new([[0.0; 4]; 4]));
with_quad_test_pool(|s| {
for idx in 0..4u8 {
let quad = quad.clone();
let results = results.clone();
s.spawn(move |_| {
let r = quad.dpdx_fine_components(idx, &values[idx as usize], 2);
results.lock().unwrap()[idx as usize] = r;
});
}
});
let results = results.lock().unwrap();
for i in 0..4 {
assert_eq!(results[i][0], 2.0);
assert_eq!(results[i][1], 20.0);
}
}
}