use crate::RenderError;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct WorkgroupSize {
pub x: u32,
pub y: u32,
pub z: u32,
}
impl WorkgroupSize {
#[inline]
pub const fn new(x: u32, y: u32, z: u32) -> Self {
Self { x, y, z }
}
#[inline]
pub const fn linear(n: u32) -> Self {
Self::new(n, 1, 1)
}
#[inline]
pub const fn square(n: u32) -> Self {
Self::new(n, n, 1)
}
#[inline]
pub fn total(&self) -> u32 {
self.x.saturating_mul(self.y).saturating_mul(self.z)
}
#[inline]
pub fn dispatch_count_x(&self, n: u32) -> u32 {
n.div_ceil(self.x)
}
#[inline]
pub fn dispatch_count_y(&self, n: u32) -> u32 {
n.div_ceil(self.y)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WorkgroupProfile {
Mobile,
Balanced,
HighThroughput,
Custom,
}
impl WorkgroupProfile {
#[must_use]
pub fn default_size(&self) -> WorkgroupSize {
match self {
Self::Mobile => WorkgroupSize::linear(32),
Self::Balanced => WorkgroupSize::linear(64),
Self::HighThroughput => WorkgroupSize::linear(256),
Self::Custom => WorkgroupSize::linear(64), }
}
#[must_use]
pub const fn name(&self) -> &'static str {
match self {
Self::Mobile => "mobile",
Self::Balanced => "balanced",
Self::HighThroughput => "high_throughput",
Self::Custom => "custom",
}
}
#[must_use]
pub const fn description(&self) -> &'static str {
match self {
Self::Mobile => "Small workgroups (32 threads) for mobile/integrated GPUs",
Self::Balanced => "Balanced workgroups (64 threads) suitable for most GPUs",
Self::HighThroughput => {
"Large workgroups (256 threads) for powerful desktop/server GPUs"
}
Self::Custom => "User-defined workgroup configuration",
}
}
}
#[derive(Debug, Clone)]
pub struct WorkgroupConfig {
pub profile: WorkgroupProfile,
pub preprocess: WorkgroupSize,
pub sort: WorkgroupSize,
pub rasterize: WorkgroupSize,
pub backward: WorkgroupSize,
pub tile: WorkgroupSize,
}
impl WorkgroupConfig {
#[must_use]
pub fn from_profile(profile: WorkgroupProfile) -> Self {
let size = profile.default_size();
Self {
profile,
preprocess: size,
sort: size,
rasterize: size,
backward: size,
tile: Self::SHIPPED_TILE, }
}
pub const SHIPPED_TILE: WorkgroupSize = WorkgroupSize::square(16);
pub const SHIPPED_LINEAR: WorkgroupSize = WorkgroupSize::linear(256);
#[must_use]
pub fn shipped() -> Self {
Self::for_linear_size(Self::SHIPPED_LINEAR.x)
}
#[must_use]
pub fn for_linear_size(linear: u32) -> Self {
let linear = WorkgroupSize::linear(linear);
Self {
profile: WorkgroupProfile::HighThroughput,
preprocess: linear,
sort: linear,
rasterize: Self::SHIPPED_TILE,
backward: Self::SHIPPED_TILE,
tile: Self::SHIPPED_TILE,
}
}
#[must_use]
pub fn mobile() -> Self {
Self::from_profile(WorkgroupProfile::Mobile)
}
#[must_use]
pub fn balanced() -> Self {
Self::from_profile(WorkgroupProfile::Balanced)
}
#[must_use]
pub fn high_throughput() -> Self {
Self::from_profile(WorkgroupProfile::HighThroughput)
}
#[must_use]
pub fn adaptive(num_gaussians: usize) -> Self {
if num_gaussians < 10_000 {
Self::mobile()
} else if num_gaussians < 100_000 {
Self::balanced()
} else {
Self::high_throughput()
}
}
pub fn validate(&self) -> Result<(), RenderError> {
let passes = [
("preprocess", self.preprocess),
("sort", self.sort),
("rasterize", self.rasterize),
("backward", self.backward),
("tile", self.tile),
];
for (name, ws) in passes {
if ws.x == 0 || ws.y == 0 || ws.z == 0 {
return Err(RenderError::Rasterize(format!(
"workgroup '{}' has a zero dimension: {:?}",
name, ws
)));
}
let total = ws.total();
if total > 1024 {
return Err(RenderError::Rasterize(format!(
"workgroup '{}' total threads {} exceeds maximum 1024",
name, total
)));
}
if !is_power_of_two(ws.x) || !is_power_of_two(ws.y) || !is_power_of_two(ws.z) {
return Err(RenderError::Rasterize(format!(
"workgroup '{}' dimensions must be powers of two, got {:?}",
name, ws
)));
}
}
Ok(())
}
}
impl Default for WorkgroupConfig {
fn default() -> Self {
Self::balanced()
}
}
#[inline]
fn is_power_of_two(n: u32) -> bool {
n != 0 && n.is_power_of_two()
}
#[derive(Debug, Clone)]
pub struct WorkgroupBenchResult {
pub size: WorkgroupSize,
pub mean_duration_us: f64,
pub min_duration_us: f64,
pub samples: usize,
}
pub struct WorkgroupBenchmarker {
candidates: Vec<u32>,
warmup_rounds: usize,
measure_rounds: usize,
}
impl WorkgroupBenchmarker {
#[must_use]
pub fn new() -> Self {
Self {
candidates: vec![32, 64, 128, 256],
warmup_rounds: 3,
measure_rounds: 10,
}
}
#[must_use]
pub fn with_candidates(mut self, sizes: Vec<u32>) -> Self {
self.candidates = sizes;
self
}
#[must_use]
pub fn with_warmup(mut self, rounds: usize) -> Self {
self.warmup_rounds = rounds;
self
}
#[must_use]
pub fn with_measure(mut self, rounds: usize) -> Self {
self.measure_rounds = rounds;
self
}
pub fn benchmark<F>(&self, f: F) -> Vec<WorkgroupBenchResult>
where
F: Fn(WorkgroupSize) -> std::time::Duration,
{
let mut results = Vec::with_capacity(self.candidates.len());
for &size_n in &self.candidates {
let ws = WorkgroupSize::linear(size_n);
for _ in 0..self.warmup_rounds {
let _ = f(ws);
}
let mut durations_us: Vec<f64> = Vec::with_capacity(self.measure_rounds);
for _ in 0..self.measure_rounds {
let d = f(ws);
durations_us.push(duration_to_us(d));
}
let n = durations_us.len();
let mean = if n == 0 {
0.0
} else {
durations_us.iter().copied().sum::<f64>() / n as f64
};
let min = durations_us.iter().copied().fold(f64::INFINITY, f64::min);
results.push(WorkgroupBenchResult {
size: ws,
mean_duration_us: mean,
min_duration_us: if min.is_infinite() { 0.0 } else { min },
samples: n,
});
}
results
}
#[must_use]
pub fn best_of(&self, results: &[WorkgroupBenchResult]) -> Option<WorkgroupSize> {
results
.iter()
.min_by(|a, b| {
a.mean_duration_us
.partial_cmp(&b.mean_duration_us)
.unwrap_or(std::cmp::Ordering::Equal)
})
.map(|r| r.size)
}
#[must_use]
pub fn recommend<F>(&self, num_gaussians: usize, f: F) -> WorkgroupConfig
where
F: Fn(WorkgroupSize) -> std::time::Duration,
{
let results = self.benchmark(f);
self.config_from_results(&results, num_gaussians)
}
pub fn benchmark_dispatch(
&self,
device: &wgpu::Device,
queue: &wgpu::Queue,
num_elements: u32,
) -> Vec<WorkgroupBenchResult> {
let elements = num_elements.max(1);
let limits = device.limits();
let max_workgroup_size = limits
.max_compute_invocations_per_workgroup
.min(limits.max_compute_workgroup_size_x)
.max(1);
let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("workgroup_bench_bgl"),
entries: &[
wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: false },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 1,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
},
],
});
let data = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("workgroup_bench_data"),
size: u64::from(elements) * 4,
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
let params = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("workgroup_bench_params"),
size: 16,
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
queue.write_buffer(
¶ms,
0,
bytemuck::cast_slice(&[elements, 0u32, 0u32, 0u32]),
);
let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("workgroup_bench_bg"),
layout: &bgl,
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: data.as_entire_binding(),
},
wgpu::BindGroupEntry {
binding: 1,
resource: params.as_entire_binding(),
},
],
});
let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("workgroup_bench_layout"),
bind_group_layouts: &[Some(&bgl)],
immediate_size: 0,
});
let mut results = Vec::with_capacity(self.candidates.len());
for &size_n in &self.candidates {
let requested = size_n.max(1);
if requested > max_workgroup_size {
tracing::warn!(
candidate = requested,
max = max_workgroup_size,
"skipping workgroup candidate above the device's per-workgroup invocation limit"
);
continue;
}
let ws = WorkgroupSize::linear(requested);
let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("workgroup_bench_kernel"),
source: wgpu::ShaderSource::Wgsl(bench_kernel_source(ws.x).into()),
});
let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: Some("workgroup_bench_pipeline"),
layout: Some(&layout),
module: &module,
entry_point: Some("bench_kernel"),
compilation_options: Default::default(),
cache: None,
});
let dispatch = ws.dispatch_count_x(elements).max(1);
let run_once = || {
let start = std::time::Instant::now();
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("workgroup_bench"),
});
{
let mut cpass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("workgroup_bench"),
timestamp_writes: None,
});
cpass.set_pipeline(&pipeline);
cpass.set_bind_group(0, &bind_group, &[]);
cpass.dispatch_workgroups(dispatch, 1, 1);
}
queue.submit(std::iter::once(encoder.finish()));
let _ = device.poll(wgpu::PollType::Wait {
submission_index: None,
timeout: None,
});
start.elapsed()
};
for _ in 0..self.warmup_rounds {
let _ = run_once();
}
let mut durations_us: Vec<f64> = Vec::with_capacity(self.measure_rounds);
for _ in 0..self.measure_rounds {
durations_us.push(duration_to_us(run_once()));
}
let n = durations_us.len();
let mean = if n == 0 {
0.0
} else {
durations_us.iter().copied().sum::<f64>() / n as f64
};
let min = durations_us.iter().copied().fold(f64::INFINITY, f64::min);
results.push(WorkgroupBenchResult {
size: ws,
mean_duration_us: mean,
min_duration_us: if min.is_infinite() { 0.0 } else { min },
samples: n,
});
}
results
}
#[must_use]
pub fn recommend_on_device(
&self,
device: &wgpu::Device,
queue: &wgpu::Queue,
num_elements: u32,
) -> WorkgroupConfig {
let results = self.benchmark_dispatch(device, queue, num_elements);
self.config_from_results(&results, num_elements as usize)
}
fn config_from_results(
&self,
results: &[WorkgroupBenchResult],
num_gaussians: usize,
) -> WorkgroupConfig {
match self.best_of(results) {
Some(best_size) => WorkgroupConfig {
profile: WorkgroupProfile::Custom,
preprocess: best_size,
sort: best_size,
rasterize: best_size,
backward: best_size,
tile: WorkgroupConfig::SHIPPED_TILE,
},
None => WorkgroupConfig::adaptive(num_gaussians),
}
}
}
fn bench_kernel_source(workgroup_size: u32) -> String {
format!(
r#"@group(0) @binding(0) var<storage, read_write> data: array<u32>;
@group(0) @binding(1) var<uniform> params: vec4<u32>;
@compute @workgroup_size({workgroup_size})
fn bench_kernel(@builtin(global_invocation_id) gid: vec3<u32>) {{
let i = gid.x;
if i >= params.x {{
return;
}}
var v = data[i];
v = v * 1664525u + 1013904223u;
data[i] = v;
}}
"#
)
}
impl Default for WorkgroupBenchmarker {
fn default() -> Self {
Self::new()
}
}
#[inline]
fn duration_to_us(d: std::time::Duration) -> f64 {
d.as_secs_f64() * 1_000_000.0
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::{Duration, Instant};
#[test]
fn test_linear_total() {
assert_eq!(WorkgroupSize::linear(64).total(), 64);
}
#[test]
fn test_square_total() {
assert_eq!(WorkgroupSize::square(8).total(), 64);
}
#[test]
fn test_dispatch_count_x_rounds_up() {
let ws = WorkgroupSize::linear(64);
assert_eq!(ws.dispatch_count_x(65), 2);
assert_eq!(ws.dispatch_count_x(1), 1);
}
#[test]
fn test_dispatch_count_x_exact_multiple() {
let ws = WorkgroupSize::linear(64);
assert_eq!(ws.dispatch_count_x(128), 2);
}
#[test]
fn test_dispatch_count_y_rounds_up() {
let ws = WorkgroupSize::square(8);
assert_eq!(ws.dispatch_count_y(9), 2);
assert_eq!(ws.dispatch_count_y(8), 1);
}
#[test]
fn test_profile_mobile_total() {
assert_eq!(WorkgroupProfile::Mobile.default_size().total(), 32);
}
#[test]
fn test_profile_balanced_total() {
assert_eq!(WorkgroupProfile::Balanced.default_size().total(), 64);
}
#[test]
fn test_profile_high_throughput_total() {
assert_eq!(WorkgroupProfile::HighThroughput.default_size().total(), 256);
}
#[test]
fn test_profile_name() {
assert_eq!(WorkgroupProfile::Mobile.name(), "mobile");
assert_eq!(WorkgroupProfile::Balanced.name(), "balanced");
assert_eq!(WorkgroupProfile::HighThroughput.name(), "high_throughput");
assert_eq!(WorkgroupProfile::Custom.name(), "custom");
}
#[test]
fn test_profile_description_non_empty() {
for profile in [
WorkgroupProfile::Mobile,
WorkgroupProfile::Balanced,
WorkgroupProfile::HighThroughput,
WorkgroupProfile::Custom,
] {
assert!(!profile.description().is_empty());
}
}
#[test]
fn test_mobile_preprocess_total() {
assert_eq!(WorkgroupConfig::mobile().preprocess.total(), 32);
}
#[test]
fn test_balanced_profile_tag() {
assert_eq!(
WorkgroupConfig::balanced().profile,
WorkgroupProfile::Balanced
);
}
#[test]
fn test_adaptive_small() {
let cfg = WorkgroupConfig::adaptive(5_000);
assert_eq!(cfg.profile, WorkgroupProfile::Mobile);
}
#[test]
fn test_adaptive_medium() {
let cfg = WorkgroupConfig::adaptive(50_000);
assert_eq!(cfg.profile, WorkgroupProfile::Balanced);
}
#[test]
fn test_adaptive_large() {
let cfg = WorkgroupConfig::adaptive(500_000);
assert_eq!(cfg.profile, WorkgroupProfile::HighThroughput);
}
#[test]
fn test_default_is_balanced() {
let cfg = WorkgroupConfig::default();
assert_eq!(cfg.profile, WorkgroupProfile::Balanced);
}
#[test]
fn test_validate_balanced_ok() {
assert!(WorkgroupConfig::balanced().validate().is_ok());
}
#[test]
fn test_validate_mobile_ok() {
assert!(WorkgroupConfig::mobile().validate().is_ok());
}
#[test]
fn test_validate_high_throughput_ok() {
assert!(WorkgroupConfig::high_throughput().validate().is_ok());
}
#[test]
fn test_validate_zero_dimension_fails() {
let mut cfg = WorkgroupConfig::balanced();
cfg.preprocess = WorkgroupSize::new(0, 1, 1);
assert!(cfg.validate().is_err());
}
#[test]
fn test_validate_exceeds_max_threads_fails() {
let mut cfg = WorkgroupConfig::balanced();
cfg.sort = WorkgroupSize::new(32, 32, 2);
assert!(cfg.validate().is_err());
}
#[test]
fn test_validate_non_power_of_two_fails() {
let mut cfg = WorkgroupConfig::balanced();
cfg.backward = WorkgroupSize::new(48, 1, 1); assert!(cfg.validate().is_err());
}
#[test]
fn test_benchmarker_default_candidates() {
let b = WorkgroupBenchmarker::new();
assert!(!b.candidates.is_empty());
assert!(b.candidates.contains(&32));
assert!(b.candidates.contains(&64));
assert!(b.candidates.contains(&128));
assert!(b.candidates.contains(&256));
}
#[test]
fn test_benchmark_result_count() {
let b = WorkgroupBenchmarker::new().with_warmup(1).with_measure(2);
let results = b.benchmark(|_ws| Duration::from_nanos(100));
assert_eq!(results.len(), b.candidates.len());
}
#[test]
fn test_best_of_returns_lowest_mean() {
let b = WorkgroupBenchmarker::new();
let results = vec![
WorkgroupBenchResult {
size: WorkgroupSize::linear(32),
mean_duration_us: 200.0,
min_duration_us: 180.0,
samples: 5,
},
WorkgroupBenchResult {
size: WorkgroupSize::linear(64),
mean_duration_us: 50.0,
min_duration_us: 45.0,
samples: 5,
},
WorkgroupBenchResult {
size: WorkgroupSize::linear(128),
mean_duration_us: 120.0,
min_duration_us: 110.0,
samples: 5,
},
];
let best = b.best_of(&results);
assert_eq!(best, Some(WorkgroupSize::linear(64)));
}
#[test]
fn test_best_of_empty_returns_none() {
let b = WorkgroupBenchmarker::new();
assert_eq!(b.best_of(&[]), None);
}
#[test]
fn test_recommend_returns_config() {
let b = WorkgroupBenchmarker::new().with_warmup(1).with_measure(2);
let config = b.recommend(50_000, |_ws| Duration::from_nanos(1));
assert!(config.validate().is_ok());
}
#[test]
fn test_recommend_trivial_closure_works() {
let b = WorkgroupBenchmarker::new().with_warmup(1).with_measure(3);
let config = b.recommend(10_000, |ws| {
let start = Instant::now();
let _total = ws.total();
start.elapsed()
});
assert!(config.validate().is_ok());
assert!(config.preprocess.total() > 0);
}
#[test]
fn test_recommend_empty_candidates_fallback() {
let b = WorkgroupBenchmarker::new()
.with_candidates(vec![])
.with_warmup(0)
.with_measure(1);
let config = b.recommend(5_000, |_ws| Duration::from_nanos(1));
assert_eq!(config.profile, WorkgroupProfile::Mobile);
}
#[test]
fn test_custom_candidates_respected() {
let b = WorkgroupBenchmarker::new()
.with_candidates(vec![128])
.with_warmup(0)
.with_measure(2);
let results = b.benchmark(|_ws| Duration::from_nanos(50));
assert_eq!(results.len(), 1);
assert_eq!(results[0].size, WorkgroupSize::linear(128));
}
#[test]
fn test_duration_to_us_conversion() {
let d = std::time::Duration::from_micros(42);
assert!((duration_to_us(d) - 42.0).abs() < 0.01);
}
#[test]
fn test_is_power_of_two() {
assert!(is_power_of_two(1));
assert!(is_power_of_two(2));
assert!(is_power_of_two(64));
assert!(is_power_of_two(256));
assert!(!is_power_of_two(0));
assert!(!is_power_of_two(48));
assert!(!is_power_of_two(100));
}
#[test]
fn test_tile_workgroup_size() {
let cfg = WorkgroupConfig::balanced();
assert_eq!(cfg.tile, WorkgroupSize::square(16));
assert_eq!(cfg.tile.total(), 256);
}
#[test]
fn test_total_saturates_instead_of_overflowing() {
let ws = WorkgroupSize::new(u32::MAX, u32::MAX, u32::MAX);
assert_eq!(ws.total(), u32::MAX);
let mut cfg = WorkgroupConfig::balanced();
cfg.preprocess = ws;
assert!(cfg.validate().is_err());
}
#[test]
fn test_shipped_matches_compiled_shader_geometry() {
let cfg = WorkgroupConfig::shipped();
assert_eq!(cfg.preprocess, WorkgroupSize::linear(256));
assert_eq!(cfg.sort, WorkgroupSize::linear(256));
assert_eq!(cfg.rasterize, WorkgroupSize::square(16));
assert_eq!(cfg.backward, WorkgroupSize::square(16));
assert_eq!(cfg.tile, WorkgroupSize::square(16));
assert!(cfg.validate().is_ok());
}
#[test]
fn test_shipped_dispatch_counts_cover_every_element() {
let cfg = WorkgroupConfig::shipped();
assert_eq!(cfg.preprocess.dispatch_count_x(1000), 4);
assert_eq!(cfg.preprocess.dispatch_count_x(1024), 4);
assert_eq!(cfg.preprocess.dispatch_count_x(1025), 5);
assert_eq!(cfg.rasterize.dispatch_count_x(100), 7);
assert_eq!(cfg.rasterize.dispatch_count_y(70), 5);
}
#[test]
fn test_for_linear_size_retargets_only_the_one_dimensional_passes() {
let cfg = WorkgroupConfig::for_linear_size(64);
assert_eq!(cfg.preprocess, WorkgroupSize::linear(64));
assert_eq!(cfg.sort, WorkgroupSize::linear(64));
assert_eq!(cfg.rasterize, WorkgroupConfig::SHIPPED_TILE);
assert_eq!(cfg.backward, WorkgroupConfig::SHIPPED_TILE);
assert_eq!(cfg.tile, WorkgroupConfig::SHIPPED_TILE);
assert!(cfg.validate().is_ok());
assert_eq!(cfg.preprocess.dispatch_count_x(1000), 16);
}
#[test]
fn test_shipped_is_for_linear_size_of_the_shipped_attribute() {
let shipped = WorkgroupConfig::shipped();
let explicit = WorkgroupConfig::for_linear_size(WorkgroupConfig::SHIPPED_LINEAR.x);
assert_eq!(shipped.preprocess, explicit.preprocess);
assert_eq!(shipped.sort, explicit.sort);
assert_eq!(shipped.rasterize, explicit.rasterize);
assert_eq!(shipped.backward, explicit.backward);
assert_eq!(shipped.tile, explicit.tile);
assert_eq!(shipped.profile, explicit.profile);
}
#[test]
fn test_bench_kernel_source_substitutes_workgroup_size() {
let src = bench_kernel_source(128);
assert!(src.contains("@workgroup_size(128)"), "{src}");
assert!(src.contains("fn bench_kernel("), "{src}");
assert!(src.contains("if i >= params.x {"), "{src}");
assert!(!src.contains("{{"), "{src}");
}
}