use std::borrow::Cow;
use crate::config::RasterConfig;
use crate::RenderError;
const SHIPPED_LINEAR_ATTRIBUTE: &str = "@workgroup_size(256)";
fn retarget_linear_workgroup_size<'a>(label: &str, source: &'a str, size: u32) -> Cow<'a, str> {
if size == crate::workgroup::WorkgroupConfig::SHIPPED_LINEAR.x {
return Cow::Borrowed(source);
}
if !source.contains(SHIPPED_LINEAR_ATTRIBUTE) {
tracing::warn!(
shader = label,
expected = SHIPPED_LINEAR_ATTRIBUTE,
"shader does not declare the expected workgroup attribute; compiling it unchanged"
);
return Cow::Borrowed(source);
}
Cow::Owned(source.replace(
SHIPPED_LINEAR_ATTRIBUTE,
&format!("@workgroup_size({size})"),
))
}
pub struct RasterPipelines {
pub preprocess: wgpu::ComputePipeline,
pub prefix_sum: wgpu::ComputePipeline,
pub prefix_sum_add: wgpu::ComputePipeline,
pub tile_assign: wgpu::ComputePipeline,
pub tile_ranges: wgpu::ComputePipeline,
pub rasterize_fwd: wgpu::ComputePipeline,
pub rasterize_bwd: wgpu::ComputePipeline,
pub atomic_to_f32: wgpu::ComputePipeline,
pub preprocess_bwd: wgpu::ComputePipeline,
pub flame_binding_bwd: wgpu::ComputePipeline,
pub preprocess_bgl: wgpu::BindGroupLayout,
pub prefix_sum_bgl: wgpu::BindGroupLayout,
pub prefix_sum_add_bgl: wgpu::BindGroupLayout,
pub tile_assign_bgl: wgpu::BindGroupLayout,
pub tile_ranges_bgl: wgpu::BindGroupLayout,
pub rasterize_fwd_bgl: wgpu::BindGroupLayout,
pub rasterize_bwd_bgl: wgpu::BindGroupLayout,
pub atomic_to_f32_bgl: wgpu::BindGroupLayout,
pub preprocess_bwd_bgl: wgpu::BindGroupLayout,
pub flame_binding_bwd_bgl: wgpu::BindGroupLayout,
pub linear_workgroup_size: u32,
}
fn get_preprocess_shader_source(config: &RasterConfig) -> &'static str {
if config.sh_optimization && config.use_sh_variants {
match config.sh_degree {
0 => include_str!("../shaders/preprocess_sh0.wgsl"),
1 => include_str!("../shaders/preprocess_sh1.wgsl"),
2 => include_str!("../shaders/preprocess_sh2.wgsl"),
3 => include_str!("../shaders/preprocess_sh3.wgsl"),
_ => include_str!("../shaders/preprocess.wgsl"),
}
} else {
include_str!("../shaders/preprocess.wgsl")
}
}
fn get_preprocess_shader_label(config: &RasterConfig) -> &'static str {
if config.sh_optimization && config.use_sh_variants {
match config.sh_degree {
0 => "preprocess_sh0",
1 => "preprocess_sh1",
2 => "preprocess_sh2",
3 => "preprocess_sh3",
_ => "preprocess",
}
} else {
"preprocess"
}
}
impl RasterPipelines {
pub fn new(device: &wgpu::Device, config: &RasterConfig) -> Result<Self, RenderError> {
let preprocess_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("preprocess_bgl"),
entries: &[
uniform_entry(0), storage_ro_entry(1), storage_ro_entry(2), storage_ro_entry(3), storage_ro_entry(4), storage_rw_entry(5), storage_rw_entry(6), storage_rw_entry(7), storage_rw_entry(8), storage_rw_entry(9), storage_rw_entry(10), storage_ro_entry(11), storage_rw_entry(12), storage_rw_entry(13), ],
});
let preprocess_source = get_preprocess_shader_source(config);
let preprocess_label = get_preprocess_shader_label(config);
let linear_workgroup_size = config.effective_preprocess_wg_size();
let retarget = |label: &str, src: &'static str| {
retarget_linear_workgroup_size(label, src, linear_workgroup_size)
};
let preprocess = compile_pipeline(
device,
preprocess_label,
&retarget(preprocess_label, preprocess_source),
"preprocess",
&preprocess_bgl,
)?;
let prefix_sum_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("prefix_sum_bgl"),
entries: &[
storage_ro_entry(0), storage_rw_entry(1), uniform_entry(2), storage_rw_entry(3), ],
});
let prefix_sum = compile_pipeline(
device,
"prefix_sum",
include_str!("../shaders/prefix_sum.wgsl"),
"prefix_sum",
&prefix_sum_bgl,
)?;
let prefix_sum_add_bgl =
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("prefix_sum_add_bgl"),
entries: &[
storage_rw_entry(0), storage_ro_entry(1), uniform_entry(2), ],
});
let prefix_sum_add = compile_pipeline(
device,
"prefix_sum_add",
include_str!("../shaders/prefix_sum_add.wgsl"),
"prefix_sum_add",
&prefix_sum_add_bgl,
)?;
let tile_assign_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("tile_assign_bgl"),
entries: &[
uniform_entry(0), storage_ro_entry(1), storage_ro_entry(2), storage_ro_entry(3), storage_ro_entry(4), storage_rw_entry(5), storage_rw_entry(6), ],
});
let tile_assign = compile_pipeline(
device,
"tile_assign",
&retarget("tile_assign", include_str!("../shaders/tile_assign.wgsl")),
"tile_assign",
&tile_assign_bgl,
)?;
let tile_ranges_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("tile_ranges_bgl"),
entries: &[
storage_ro_entry(0), storage_rw_entry(1), uniform_entry(2), ],
});
let tile_ranges = compile_pipeline(
device,
"tile_ranges",
&retarget("tile_ranges", include_str!("../shaders/tile_ranges.wgsl")),
"tile_ranges_kernel",
&tile_ranges_bgl,
)?;
let rasterize_fwd_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("rasterize_fwd_bgl"),
entries: &[
uniform_entry(0), storage_ro_entry(1), storage_ro_entry(2), storage_ro_entry(3), storage_ro_entry(4), storage_ro_entry(5), storage_ro_entry(6), storage_ro_entry(7), storage_rw_entry(8), storage_rw_entry(9), storage_rw_entry(10), storage_rw_entry(11), storage_ro_entry(12), storage_rw_entry(13), ],
});
let rasterize_fwd = compile_pipeline(
device,
"rasterize_fwd",
include_str!("../shaders/rasterize_fwd.wgsl"),
"rasterize_forward",
&rasterize_fwd_bgl,
)?;
let rasterize_bwd_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("rasterize_bwd_bgl"),
entries: &[
uniform_entry(0), storage_ro_entry(1), storage_ro_entry(2), storage_ro_entry(3), storage_ro_entry(4), storage_ro_entry(5), storage_ro_entry(6), storage_ro_entry(7), storage_ro_entry(8), storage_ro_entry(9), storage_ro_entry(10), storage_rw_entry(11), storage_rw_entry(12), storage_rw_entry(13), storage_rw_entry(14), ],
});
let rasterize_bwd = compile_pipeline(
device,
"rasterize_bwd",
include_str!("../shaders/rasterize_bwd.wgsl"),
"rasterize_backward",
&rasterize_bwd_bgl,
)?;
let atomic_to_f32_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("atomic_to_f32_bgl"),
entries: &[
uniform_entry(0), storage_rw_entry(1), storage_rw_entry(2), ],
});
let atomic_to_f32 = compile_pipeline(
device,
"atomic_to_f32",
&retarget(
"atomic_to_f32",
include_str!("../shaders/atomic_to_f32.wgsl"),
),
"atomic_to_f32",
&atomic_to_f32_bgl,
)?;
let preprocess_bwd_bgl =
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("preprocess_bwd_bgl"),
entries: &[
uniform_entry(0), storage_ro_entry(1), storage_ro_entry(2), storage_ro_entry(3), storage_ro_entry(4), storage_ro_entry(5), storage_ro_entry(6), storage_ro_entry(7), storage_ro_entry(8), storage_ro_entry(9), storage_rw_entry(10), storage_rw_entry(11), storage_rw_entry(12), storage_rw_entry(13), ],
});
let preprocess_bwd = compile_pipeline(
device,
"preprocess_bwd",
&retarget(
"preprocess_bwd",
include_str!("../shaders/preprocess_bwd.wgsl"),
),
"preprocess_backward",
&preprocess_bwd_bgl,
)?;
let flame_binding_bwd_bgl =
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("flame_binding_bwd_bgl"),
entries: &[
uniform_entry(0), storage_ro_entry(1), storage_ro_entry(2), storage_ro_entry(3), storage_rw_entry(4), ],
});
let flame_binding_bwd = compile_pipeline(
device,
"flame_binding_bwd",
include_str!("../shaders/flame_binding_bwd.wgsl"),
"flame_binding_backward",
&flame_binding_bwd_bgl,
)?;
Ok(Self {
preprocess,
prefix_sum,
prefix_sum_add,
tile_assign,
tile_ranges,
rasterize_fwd,
rasterize_bwd,
atomic_to_f32,
preprocess_bwd,
flame_binding_bwd,
preprocess_bgl,
prefix_sum_bgl,
prefix_sum_add_bgl,
tile_assign_bgl,
tile_ranges_bgl,
rasterize_fwd_bgl,
rasterize_bwd_bgl,
atomic_to_f32_bgl,
preprocess_bwd_bgl,
flame_binding_bwd_bgl,
linear_workgroup_size,
})
}
}
fn poll_ready_now<F: std::future::Future>(fut: F) -> Option<F::Output> {
use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker};
fn no_op(_: *const ()) {}
fn clone_raw(_: *const ()) -> RawWaker {
RawWaker::new(std::ptr::null(), &VTABLE)
}
static VTABLE: RawWakerVTable = RawWakerVTable::new(clone_raw, no_op, no_op, no_op);
let raw_waker = RawWaker::new(std::ptr::null(), &VTABLE);
let waker = unsafe { Waker::from_raw(raw_waker) };
let mut cx = Context::from_waker(&waker);
let mut boxed = Box::pin(fut);
match boxed.as_mut().poll(&mut cx) {
Poll::Ready(v) => Some(v),
Poll::Pending => None,
}
}
fn compile_pipeline(
device: &wgpu::Device,
label: &str,
source: &str,
entry_point: &str,
bgl: &wgpu::BindGroupLayout,
) -> Result<wgpu::ComputePipeline, RenderError> {
let scope = device.push_error_scope(wgpu::ErrorFilter::Validation);
let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some(label),
source: wgpu::ShaderSource::Wgsl(source.into()),
});
if let Some(err) = poll_ready_now(scope.pop()).flatten() {
return Err(RenderError::ShaderCompilation {
shader_name: label.to_string(),
error: err.to_string(),
});
}
let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some(&format!("{label}_layout")),
bind_group_layouts: &[Some(bgl)],
immediate_size: 0,
});
let scope = device.push_error_scope(wgpu::ErrorFilter::Validation);
let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: Some(&format!("{label}_pipeline")),
layout: Some(&layout),
module: &module,
entry_point: Some(entry_point),
compilation_options: Default::default(),
cache: None,
});
if let Some(err) = poll_ready_now(scope.pop()).flatten() {
return Err(RenderError::ShaderValidation {
shader_name: label.to_string(),
error: err.to_string(),
});
}
Ok(pipeline)
}
fn uniform_entry(binding: u32) -> wgpu::BindGroupLayoutEntry {
wgpu::BindGroupLayoutEntry {
binding,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
}
}
fn storage_ro_entry(binding: u32) -> wgpu::BindGroupLayoutEntry {
wgpu::BindGroupLayoutEntry {
binding,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: true },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
}
}
fn storage_rw_entry(binding: u32) -> wgpu::BindGroupLayoutEntry {
wgpu::BindGroupLayoutEntry {
binding,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: false },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_get_preprocess_shader_label() {
let mut config = RasterConfig {
sh_optimization: true,
use_sh_variants: true,
..RasterConfig::default()
};
config.sh_degree = 0;
assert_eq!(get_preprocess_shader_label(&config), "preprocess_sh0");
config.sh_degree = 1;
assert_eq!(get_preprocess_shader_label(&config), "preprocess_sh1");
config.sh_degree = 2;
assert_eq!(get_preprocess_shader_label(&config), "preprocess_sh2");
config.sh_degree = 3;
assert_eq!(get_preprocess_shader_label(&config), "preprocess_sh3");
config.use_sh_variants = false;
config.sh_degree = 0;
assert_eq!(get_preprocess_shader_label(&config), "preprocess");
}
#[test]
fn test_retargetable_shaders_declare_the_expected_attribute() {
let retargetable: [(&str, &str); 9] = [
("preprocess", include_str!("../shaders/preprocess.wgsl")),
(
"preprocess_sh0",
include_str!("../shaders/preprocess_sh0.wgsl"),
),
(
"preprocess_sh1",
include_str!("../shaders/preprocess_sh1.wgsl"),
),
(
"preprocess_sh2",
include_str!("../shaders/preprocess_sh2.wgsl"),
),
(
"preprocess_sh3",
include_str!("../shaders/preprocess_sh3.wgsl"),
),
(
"preprocess_bwd",
include_str!("../shaders/preprocess_bwd.wgsl"),
),
("tile_assign", include_str!("../shaders/tile_assign.wgsl")),
("tile_ranges", include_str!("../shaders/tile_ranges.wgsl")),
(
"atomic_to_f32",
include_str!("../shaders/atomic_to_f32.wgsl"),
),
];
for (label, src) in retargetable {
assert_eq!(
src.matches(SHIPPED_LINEAR_ATTRIBUTE).count(),
1,
"{label} must declare exactly one {SHIPPED_LINEAR_ATTRIBUTE}"
);
let out = retarget_linear_workgroup_size(label, src, 64);
assert!(out.contains("@workgroup_size(64)"), "{label}");
assert!(!out.contains(SHIPPED_LINEAR_ATTRIBUTE), "{label}");
}
}
#[test]
fn test_retargetable_shaders_have_no_workgroup_local_state() {
let retargetable = [
include_str!("../shaders/preprocess.wgsl"),
include_str!("../shaders/preprocess_sh0.wgsl"),
include_str!("../shaders/preprocess_sh1.wgsl"),
include_str!("../shaders/preprocess_sh2.wgsl"),
include_str!("../shaders/preprocess_sh3.wgsl"),
include_str!("../shaders/preprocess_bwd.wgsl"),
include_str!("../shaders/tile_assign.wgsl"),
include_str!("../shaders/tile_ranges.wgsl"),
include_str!("../shaders/atomic_to_f32.wgsl"),
];
for src in retargetable {
let code: String = src
.lines()
.filter(|l| !l.trim_start().starts_with("//"))
.collect::<Vec<_>>()
.join("\n");
for forbidden in [
"local_invocation_id",
"workgroup_id",
"var<workgroup>",
"workgroupBarrier",
] {
assert!(
!code.contains(forbidden),
"a retargeted kernel must not use `{forbidden}`"
);
}
}
}
#[test]
fn test_non_retargetable_shaders_bake_in_their_thread_count() {
assert!(include_str!("../shaders/prefix_sum.wgsl").contains("var<workgroup>"));
let add = include_str!("../shaders/prefix_sum_add.wgsl");
assert!(
add.contains("512u"),
"prefix_sum_add hardcodes the block size"
);
assert!(
add.contains("256u"),
"prefix_sum_add hardcodes the half-block"
);
assert!(include_str!("../shaders/radix_scatter.wgsl").contains("var<workgroup>"));
assert!(include_str!("../shaders/radix_histogram.wgsl").contains("var<workgroup>"));
}
#[test]
fn test_retarget_is_a_no_op_at_the_shipped_size() {
let src = include_str!("../shaders/tile_assign.wgsl");
let out = retarget_linear_workgroup_size("tile_assign", src, 256);
assert!(matches!(out, Cow::Borrowed(_)));
assert_eq!(out, src);
assert_eq!(RasterConfig::default().effective_preprocess_wg_size(), 256);
}
#[test]
fn test_retarget_leaves_unknown_sources_alone() {
let src = "@compute @workgroup_size(16, 16)\nfn main() {}\n";
let out = retarget_linear_workgroup_size("tile_kernel", src, 64);
assert_eq!(out, src);
}
#[test]
#[ignore = "requires GPU"]
fn test_compile_pipeline_invalid_wgsl_returns_error_not_panic() {
let instance = wgpu::Instance::new(wgpu::InstanceDescriptor {
backends: wgpu::Backends::all(),
..wgpu::InstanceDescriptor::new_without_display_handle()
});
let adapter =
match pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {
power_preference: wgpu::PowerPreference::HighPerformance,
compatible_surface: None,
force_fallback_adapter: false,
apply_limit_buckets: false,
})) {
Ok(a) => a,
Err(_) => {
eprintln!("No GPU adapter available, skipping GPU test");
return;
}
};
let (device, _queue) =
pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor {
label: Some("pipeline_test_device"),
required_features: wgpu::Features::empty(),
required_limits: wgpu::Limits::default(),
memory_hints: wgpu::MemoryHints::Performance,
experimental_features: wgpu::ExperimentalFeatures::default(),
trace: wgpu::Trace::Off,
}))
.expect("failed to create device");
let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("test_bgl"),
entries: &[uniform_entry(0)],
});
let result = compile_pipeline(
&device,
"broken_shader",
"this is not valid wgsl {{{",
"main",
&bgl,
);
assert!(
matches!(
result,
Err(RenderError::ShaderCompilation { .. })
| Err(RenderError::ShaderValidation { .. })
),
"expected a ShaderCompilation or ShaderValidation error, got {result:?}"
);
}
}