mod bind_groups;
mod limits;
use limits::{check_linear_workgroup_size, check_rasterizer_limits};
pub use limits::{
rasterizer_device_limits, RASTERIZER_STORAGE_BUFFERS_PER_STAGE,
RASTERIZE_FWD_WORKGROUP_STORAGE_BYTES, RASTERIZE_TILE_SIZE,
};
use crate::binding::{build_face_frames, FaceFrame, FlameBindingBuffers};
use crate::buffers::{
GaussianBuffers, GradientBuffers, IntermediateBuffers, OutputBuffers, UniformBuffer, Uniforms,
};
use crate::config::RasterConfig;
use crate::gaussian::GaussianModel;
use crate::pipeline::RasterPipelines;
use crate::pool::{BufferPool, PoolStats, PooledBuffer};
use crate::profiler::GpuTimestampProfiler;
use crate::sort::RadixSorter;
use crate::workgroup::WorkgroupConfig;
use crate::RenderError;
pub struct RenderOutput {
pub color_data: Vec<f32>,
pub depth_data: Vec<f32>,
pub normals: Option<Vec<[f32; 3]>>,
pub width: u32,
pub height: u32,
}
#[derive(Debug)]
pub struct GaussianGradients {
pub grad_positions: Vec<[f32; 3]>,
pub grad_rotations: Vec<[f32; 4]>,
pub grad_scales: Vec<[f32; 3]>,
pub grad_opacities: Vec<f32>,
pub grad_sh_coeffs: Vec<f32>,
pub grad_local_offsets: Vec<[f32; 3]>,
}
#[derive(Debug, Clone)]
pub struct RenderCamera {
pub view_matrix: [f32; 16],
pub proj_matrix: [f32; 16],
pub position: [f32; 3],
pub focal: [f32; 2],
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct UploadedModel {
ptr: usize,
len: usize,
sh_len: usize,
sh_degree: u32,
}
impl UploadedModel {
fn of(model: &GaussianModel) -> Self {
Self {
ptr: model.gaussians.as_ptr() as usize,
len: model.gaussians.len(),
sh_len: model.sh_coeffs.len(),
sh_degree: model.sh_degree,
}
}
fn describes(&self, model: &GaussianModel) -> bool {
self.ptr == model.gaussians.as_ptr() as usize && self.describes_allocation(model)
}
fn describes_allocation(&self, model: &GaussianModel) -> bool {
self.len == model.gaussians.len()
&& self.sh_len == model.sh_coeffs.len()
&& self.sh_degree == model.sh_degree
}
}
struct FrameParams {
scan_count: wgpu::Buffer,
scan_l1: wgpu::Buffer,
scan_l2: wgpu::Buffer,
tile_ranges: wgpu::Buffer,
num_elements_means2d: wgpu::Buffer,
num_elements_conics: wgpu::Buffer,
num_elements_colors: wgpu::Buffer,
}
struct FrameBindGroups {
preprocess: wgpu::BindGroup,
scan_l0: wgpu::BindGroup,
scan_l1: wgpu::BindGroup,
scan_l2: wgpu::BindGroup,
add_l1: wgpu::BindGroup,
add_l0: wgpu::BindGroup,
tile_assign: wgpu::BindGroup,
tile_ranges: wgpu::BindGroup,
rasterize_fwd: wgpu::BindGroup,
}
struct BackwardBindGroups {
rasterize_bwd: wgpu::BindGroup,
atomic_means2d: wgpu::BindGroup,
atomic_conics: wgpu::BindGroup,
atomic_colors: wgpu::BindGroup,
preprocess_bwd: wgpu::BindGroup,
}
enum StagingBuffer {
Pooled(PooledBuffer),
Direct(wgpu::Buffer),
}
impl std::ops::Deref for StagingBuffer {
type Target = wgpu::Buffer;
fn deref(&self) -> &wgpu::Buffer {
match self {
Self::Pooled(b) => b,
Self::Direct(b) => b,
}
}
}
struct PendingReadback {
staging: StagingBuffer,
count: usize,
}
const STAGING_READ_LABEL: &str = "staging_read";
fn staging_size_for(byte_size: u64) -> u64 {
byte_size
.max(wgpu::MAP_ALIGNMENT)
.next_multiple_of(wgpu::MAP_ALIGNMENT)
}
fn check_staging_fits(staging_size: u64, max_buffer_size: u64) -> Result<(), RenderError> {
if staging_size > max_buffer_size {
return Err(RenderError::BufferOverflow {
buffer_name: STAGING_READ_LABEL.to_string(),
max_size: max_buffer_size,
requested: staging_size,
});
}
Ok(())
}
pub struct Rasterizer {
device: wgpu::Device,
queue: wgpu::Queue,
config: RasterConfig,
workgroups: WorkgroupConfig,
pipelines: RasterPipelines,
sorter: RadixSorter,
uniform_buf: UniformBuffer,
gaussian_bufs: Option<GaussianBuffers>,
intermediate_bufs: Option<IntermediateBuffers>,
output_bufs: Option<OutputBuffers>,
gradient_bufs: Option<GradientBuffers>,
buffer_pool: Option<BufferPool>,
uploaded: Option<UploadedModel>,
params: FrameParams,
dummy_block_sums: wgpu::Buffer,
dummy_out_normals: wgpu::Buffer,
grad_output_buf: wgpu::Buffer,
frame_bgs: Option<FrameBindGroups>,
bwd_bgs: Option<BackwardBindGroups>,
binding_frames: Option<Vec<FaceFrame>>,
flame_bufs: Option<FlameBindingBuffers>,
gpu_timestamps: Option<GpuTimestampProfiler>,
}
impl Rasterizer {
pub fn from_device(
device: wgpu::Device,
queue: wgpu::Queue,
config: RasterConfig,
) -> Result<Self, RenderError> {
config.validate_for_rasterizer()?;
check_rasterizer_limits(&device.limits(), "device")?;
let linear_workgroup_size = config.effective_preprocess_wg_size();
check_linear_workgroup_size(&device.limits(), linear_workgroup_size)?;
let workgroups = WorkgroupConfig::for_linear_size(linear_workgroup_size);
workgroups.validate()?;
let pipelines = RasterPipelines::new(&device, &config)?;
debug_assert_eq!(pipelines.linear_workgroup_size, linear_workgroup_size);
let sorter = RadixSorter::new(&device, 1024)?;
let uniform_buf = UniformBuffer::new(&device);
let params = {
let uniform_usage = wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST;
let small_uniform = |label: &'static str| {
device.create_buffer(&wgpu::BufferDescriptor {
label: Some(label),
size: 16, usage: uniform_usage,
mapped_at_creation: false,
})
};
FrameParams {
scan_count: small_uniform("prefix_sum_params"),
scan_l1: small_uniform("prefix_sum_l1_params"),
scan_l2: small_uniform("prefix_sum_l2_params"),
tile_ranges: small_uniform("tile_ranges_params"),
num_elements_means2d: small_uniform("num_elements_means2d"),
num_elements_conics: small_uniform("num_elements_conics"),
num_elements_colors: small_uniform("num_elements_colors"),
}
};
let dummy_block_sums = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("dummy_block_sums"),
size: 16,
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
let dummy_out_normals = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("dummy_out_normals"),
size: 16, usage: wgpu::BufferUsages::STORAGE,
mapped_at_creation: false,
});
let grad_output_buf = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("grad_output"),
size: (u64::from(config.num_pixels().max(1))) * 4 * 4, usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
let buffer_pool = if config.enable_buffer_pooling && config.max_gpu_memory_mb > 0 {
let budget_bytes = config.memory_budget_bytes();
tracing::info!(
budget_mb = config.max_gpu_memory_mb,
"Buffer pooling enabled"
);
Some(BufferPool::new(budget_bytes))
} else {
tracing::info!("Buffer pooling disabled");
None
};
Ok(Self {
device,
queue,
config,
workgroups,
pipelines,
sorter,
uniform_buf,
gaussian_bufs: None,
intermediate_bufs: None,
output_bufs: None,
gradient_bufs: None,
buffer_pool,
uploaded: None,
params,
dummy_block_sums,
dummy_out_normals,
grad_output_buf,
frame_bgs: None,
bwd_bgs: None,
binding_frames: None,
flame_bufs: None,
gpu_timestamps: None,
})
}
pub async fn new(config: RasterConfig) -> Result<Self, RenderError> {
#[cfg(feature = "gpu_debug")]
let instance = {
tracing::info!("GPU debug mode enabled - validation layers active");
wgpu::Instance::new(wgpu::InstanceDescriptor {
backends: wgpu::Backends::all(),
flags: wgpu::InstanceFlags::VALIDATION | wgpu::InstanceFlags::DEBUG,
..wgpu::InstanceDescriptor::new_without_display_handle()
})
};
#[cfg(not(feature = "gpu_debug"))]
let instance = wgpu::Instance::new(wgpu::InstanceDescriptor {
backends: wgpu::Backends::all(),
..wgpu::InstanceDescriptor::new_without_display_handle()
});
let adapter = instance
.request_adapter(&wgpu::RequestAdapterOptions {
power_preference: wgpu::PowerPreference::HighPerformance,
compatible_surface: None,
force_fallback_adapter: false,
apply_limit_buckets: false,
})
.await
.map_err(|e| RenderError::GpuInit(format!("No suitable GPU adapter: {e}")))?;
tracing::info!(
adapter = adapter.get_info().name,
backend = ?adapter.get_info().backend,
"Selected GPU adapter"
);
check_rasterizer_limits(&adapter.limits(), &adapter.get_info().name)?;
let required_limits = rasterizer_device_limits(wgpu::Limits::default());
let required_features =
adapter.features() & crate::profiler::GpuTimestampProfiler::REQUIRED_FEATURES;
let (device, queue) = adapter
.request_device(&wgpu::DeviceDescriptor {
label: Some("oxigaf_render"),
required_features,
required_limits,
memory_hints: wgpu::MemoryHints::Performance,
experimental_features: wgpu::ExperimentalFeatures::default(),
trace: wgpu::Trace::Off,
})
.await
.map_err(|e| RenderError::GpuInit(e.to_string()))?;
Self::from_device(device, queue, config)
}
pub fn upload_gaussians(&mut self, model: &GaussianModel) {
let n = model.len() as u32;
if let Err(e) = IntermediateBuffers::validate_gaussian_count(n) {
tracing::error!(num_gaussians = n, "{e}");
}
if let Err(e) = model.validate() {
tracing::error!(num_gaussians = n, "upload_gaussians: {e}");
}
let gauss = GaussianBuffers::from_model(&self.device, model);
let inter = IntermediateBuffers::allocate(&self.device, n, &self.config);
let output = OutputBuffers::allocate(&self.device, &self.config);
let grads =
GradientBuffers::allocate(&self.device, n, n * self.config.sh_coeffs_per_gaussian());
self.sorter.prepare(
&self.device,
&inter.sort_keys,
&inter.sort_values,
inter.max_pairs,
);
let frame_bgs = self.build_frame_bind_groups(&gauss, &inter, &output);
let bwd_bgs = self.build_backward_bind_groups(&gauss, &inter, &output, &grads);
self.gaussian_bufs = Some(gauss);
self.intermediate_bufs = Some(inter);
self.output_bufs = Some(output);
self.gradient_bufs = Some(grads);
self.frame_bgs = Some(frame_bgs);
self.bwd_bgs = Some(bwd_bgs);
self.uploaded = Some(UploadedModel::of(model));
tracing::debug!(n_gaussians = n, "Uploaded Gaussians to GPU");
}
pub fn update_gaussians(&mut self, model: &GaussianModel) {
let same_allocation = self.uploaded.is_some_and(|u| u.describes_allocation(model))
&& self.gaussian_bufs.is_some();
if !same_allocation {
self.upload_gaussians(model);
return;
}
if let Err(e) = model.validate() {
tracing::error!(num_gaussians = model.len(), "update_gaussians: {e}");
}
let write = self
.gaussian_bufs
.as_ref()
.map(|gauss| gauss.write_model(&self.queue, model));
match write {
Some(Ok(())) => {
self.uploaded = Some(UploadedModel::of(model));
tracing::trace!(n_gaussians = model.len(), "Updated Gaussians in place");
}
Some(Err(e)) => {
tracing::warn!("update_gaussians fell back to a full upload: {e}");
self.upload_gaussians(model);
}
None => self.upload_gaussians(model),
}
}
pub fn invalidate_gaussians(&mut self) {
self.uploaded = None;
}
pub fn workgroups(&self) -> &WorkgroupConfig {
&self.workgroups
}
pub fn set_binding_mesh(&mut self, mesh: &oxigaf_flame::Mesh) {
self.set_binding_frames(build_face_frames(mesh));
}
pub fn set_binding_frames(&mut self, frames: Vec<FaceFrame>) {
let changed = self
.binding_frames
.as_ref()
.is_none_or(|old| old.len() != frames.len());
self.binding_frames = Some(frames);
if changed {
self.flame_bufs = None;
}
}
pub fn clear_binding_frames(&mut self) {
self.binding_frames = None;
self.flame_bufs = None;
}
pub fn has_binding_frames(&self) -> bool {
self.binding_frames.is_some()
}
pub fn enable_gpu_timestamps(&mut self) -> Result<(), RenderError> {
if self.gpu_timestamps.is_some() {
return Ok(());
}
self.gpu_timestamps = Some(GpuTimestampProfiler::new(
&self.device,
&self.queue,
GpuTimestampProfiler::DEFAULT_MAX_PASSES,
)?);
Ok(())
}
pub fn disable_gpu_timestamps(&mut self) {
self.gpu_timestamps = None;
}
pub fn gpu_timestamps(&self) -> Option<&GpuTimestampProfiler> {
self.gpu_timestamps.as_ref()
}
fn timestamps_for(&self, pass_name: &str) -> Option<wgpu::ComputePassTimestampWrites<'_>> {
self.gpu_timestamps
.as_ref()
.and_then(|p| p.pass_writes(pass_name))
}
fn collect_timestamps(&self) {
if let Some(profiler) = self.gpu_timestamps.as_ref() {
if let Err(e) = profiler.collect(&self.device) {
tracing::warn!("GPU timestamp readback failed: {e}");
}
}
}
fn discard_timestamps(&self) {
if let Some(profiler) = self.gpu_timestamps.as_ref() {
profiler.discard();
}
}
pub fn forward(
&mut self,
model: &GaussianModel,
camera: &RenderCamera,
) -> Result<RenderOutput, RenderError> {
if !self.uploaded.is_some_and(|u| u.describes(model)) {
self.upload_gaussians(model);
}
let gauss = self.gaussian_bufs.as_ref().ok_or_else(|| {
RenderError::Rasterize("gaussian_bufs not initialized after upload_gaussians".into())
})?;
let inter = self.intermediate_bufs.as_ref().ok_or_else(|| {
RenderError::Rasterize(
"intermediate_bufs not initialized after upload_gaussians".into(),
)
})?;
let output = self.output_bufs.as_ref().ok_or_else(|| {
RenderError::Rasterize("output_bufs not initialized after upload_gaussians".into())
})?;
let bgs = self.frame_bgs.as_ref().ok_or_else(|| {
RenderError::Rasterize(
"frame bind groups not initialized after upload_gaussians".into(),
)
})?;
let n = gauss.count;
IntermediateBuffers::validate_gaussian_count(n)?;
let uniforms = Uniforms {
view: camera.view_matrix,
proj: camera.proj_matrix,
cam_pos: camera.position,
_pad0: 0.0,
focal: camera.focal,
viewport: [
self.config.image_width as f32,
self.config.image_height as f32,
],
tile_grid: [self.config.tiles_x(), self.config.tiles_y()],
num_gaussians: n,
sh_degree: self.config.sh_degree,
near_plane: self.config.near_plane,
far_plane: self.config.far_plane,
_pad_bg: [0.0, 0.0],
background: self.config.background,
output_flags: self.config.output_flags(),
transmittance_threshold: self.config.transmittance_threshold,
tile_size: self.config.tile_size,
_pad1: [0, 0],
};
self.uniform_buf.update(&self.queue, &uniforms);
let plan = inter.scan_plan;
self.queue.write_buffer(
&self.params.scan_count,
0,
bytemuck::cast_slice(&[n, 0u32, 0u32, 0u32]),
);
self.queue.write_buffer(
&self.params.scan_l1,
0,
bytemuck::cast_slice(&[plan.level0_workgroups, 0u32, 0u32, 0u32]),
);
self.queue.write_buffer(
&self.params.scan_l2,
0,
bytemuck::cast_slice(&[plan.level1_workgroups, 0u32, 0u32, 0u32]),
);
let mut encoder = self
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("forward_scan"),
});
if n > 0 {
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("preprocess"),
timestamp_writes: self.timestamps_for("preprocess"),
});
pass.set_pipeline(&self.pipelines.preprocess);
pass.set_bind_group(0, &bgs.preprocess, &[]);
pass.dispatch_workgroups(self.workgroups.preprocess.dispatch_count_x(n), 1, 1);
}
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("prefix_sum_l0"),
timestamp_writes: self.timestamps_for("prefix_sum_l0"),
});
pass.set_pipeline(&self.pipelines.prefix_sum);
pass.set_bind_group(0, &bgs.scan_l0, &[]);
pass.dispatch_workgroups(plan.level0_workgroups, 1, 1);
}
if plan.level0_workgroups > 1 {
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("prefix_sum_l1"),
timestamp_writes: self.timestamps_for("prefix_sum_l1"),
});
pass.set_pipeline(&self.pipelines.prefix_sum);
pass.set_bind_group(0, &bgs.scan_l1, &[]);
pass.dispatch_workgroups(plan.level1_workgroups, 1, 1);
}
if plan.level1_workgroups > 1 {
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("prefix_sum_l2"),
timestamp_writes: self.timestamps_for("prefix_sum_l2"),
});
pass.set_pipeline(&self.pipelines.prefix_sum);
pass.set_bind_group(0, &bgs.scan_l2, &[]);
pass.dispatch_workgroups(plan.level2_workgroups, 1, 1);
}
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("prefix_sum_add_l1"),
timestamp_writes: self.timestamps_for("prefix_sum_add_l1"),
});
pass.set_pipeline(&self.pipelines.prefix_sum_add);
pass.set_bind_group(0, &bgs.add_l1, &[]);
pass.dispatch_workgroups(plan.level1_workgroups, 1, 1);
}
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("prefix_sum_add_l0"),
timestamp_writes: self.timestamps_for("prefix_sum_add_l0"),
});
pass.set_pipeline(&self.pipelines.prefix_sum_add);
pass.set_bind_group(0, &bgs.add_l0, &[]);
pass.dispatch_workgroups(plan.level0_workgroups, 1, 1);
}
self.queue.submit(std::iter::once(encoder.finish()));
let actual_pairs = match inter.verify_pair_capacity(&self.device, &self.queue, n) {
Ok(pairs) => pairs,
Err(e) => {
self.discard_timestamps();
return Err(e);
}
};
let num_tiles = self.config.num_tiles();
self.queue.write_buffer(
&self.params.tile_ranges,
0,
bytemuck::cast_slice(&[actual_pairs, num_tiles, 0u32, 0u32]),
);
let mut encoder = self
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("forward_raster"),
});
encoder.clear_buffer(&inter.tile_ranges, 0, None);
if n > 0 && actual_pairs > 0 {
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("tile_assign"),
timestamp_writes: self.timestamps_for("tile_assign"),
});
pass.set_pipeline(&self.pipelines.tile_assign);
pass.set_bind_group(0, &bgs.tile_assign, &[]);
pass.dispatch_workgroups(self.workgroups.preprocess.dispatch_count_x(n), 1, 1);
}
if let Err(e) = self
.sorter
.sort(&mut encoder, &self.queue, actual_pairs, num_tiles)
{
self.discard_timestamps();
return Err(e);
}
if actual_pairs > 0 {
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("tile_ranges"),
timestamp_writes: self.timestamps_for("tile_ranges"),
});
pass.set_pipeline(&self.pipelines.tile_ranges);
pass.set_bind_group(0, &bgs.tile_ranges, &[]);
pass.dispatch_workgroups(self.workgroups.sort.dispatch_count_x(actual_pairs), 1, 1);
}
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("rasterize_fwd"),
timestamp_writes: self.timestamps_for("rasterize_fwd"),
});
pass.set_pipeline(&self.pipelines.rasterize_fwd);
pass.set_bind_group(0, &bgs.rasterize_fwd, &[]);
pass.dispatch_workgroups(self.config.tiles_x(), self.config.tiles_y(), 1);
}
let npx = self.config.num_pixels() as usize;
let want_normals = self.config.output_normals && output.normals.is_some();
let pending = match self.stage_forward_readbacks(&mut encoder, output, npx, want_normals) {
Ok(pending) => pending,
Err(e) => {
self.discard_timestamps();
return Err(e);
}
};
if let Some(profiler) = self.gpu_timestamps.as_ref() {
profiler.resolve(&mut encoder);
}
self.queue.submit(std::iter::once(encoder.finish()));
let readback_result = self.map_and_collect(pending);
self.collect_timestamps();
let mut readbacks = readback_result?.into_iter();
let color_data = readbacks.next().unwrap_or_default();
let depth_data = readbacks.next().unwrap_or_default();
let normals = readbacks.next().map(|raw| {
raw.chunks_exact(4)
.map(|c| [c[0], c[1], c[2]])
.collect::<Vec<[f32; 3]>>()
});
Ok(RenderOutput {
color_data,
depth_data,
normals,
width: self.config.image_width,
height: self.config.image_height,
})
}
pub fn backward(
&mut self,
model: &GaussianModel,
grad_image: &[f32],
) -> Result<GaussianGradients, RenderError> {
let uploaded = self.uploaded.ok_or_else(|| {
RenderError::Rasterize("backward() called before upload_gaussians()".into())
})?;
if uploaded.len != model.len()
|| uploaded.sh_len != model.sh_coeffs.len()
|| uploaded.sh_degree != model.sh_degree
{
return Err(RenderError::MismatchedBufferSizes {
expected: uploaded.len,
actual: model.len(),
});
}
let binding_active = self.prepare_binding_buffers(model)?;
let gauss = self.gaussian_bufs.as_ref().ok_or_else(|| {
RenderError::Rasterize("backward() called before upload_gaussians()".into())
})?;
let grads = self
.gradient_bufs
.as_ref()
.ok_or_else(|| RenderError::Rasterize("backward() called before forward()".into()))?;
let bgs = self
.bwd_bgs
.as_ref()
.ok_or_else(|| RenderError::Rasterize("backward() called before forward()".into()))?;
let n = gauss.count;
let expected = self.config.num_pixels() as usize * 4;
if grad_image.len() > expected {
return Err(RenderError::MismatchedBufferSizes {
expected,
actual: grad_image.len(),
});
}
if grad_image.len() == expected {
self.queue
.write_buffer(&self.grad_output_buf, 0, bytemuck::cast_slice(grad_image));
} else {
let mut padded = vec![0.0f32; expected];
padded[..grad_image.len()].copy_from_slice(grad_image);
self.queue
.write_buffer(&self.grad_output_buf, 0, bytemuck::cast_slice(&padded));
}
self.queue.write_buffer(
&self.params.num_elements_means2d,
0,
bytemuck::cast_slice(&[n * 2, 0u32, 0u32, 0u32]),
);
self.queue.write_buffer(
&self.params.num_elements_conics,
0,
bytemuck::cast_slice(&[n * 3, 0u32, 0u32, 0u32]),
);
self.queue.write_buffer(
&self.params.num_elements_colors,
0,
bytemuck::cast_slice(&[n * 3, 0u32, 0u32, 0u32]),
);
let mut encoder = self
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("backward_pass"),
});
encoder.clear_buffer(&grads.grad_positions, 0, None);
encoder.clear_buffer(&grads.grad_rotations, 0, None);
encoder.clear_buffer(&grads.grad_scales, 0, None);
encoder.clear_buffer(&grads.grad_opacities, 0, None);
encoder.clear_buffer(&grads.grad_sh_coeffs, 0, None);
encoder.clear_buffer(&grads.grad_means2d_atomic, 0, None);
encoder.clear_buffer(&grads.grad_conics_atomic, 0, None);
encoder.clear_buffer(&grads.grad_colors_atomic, 0, None);
encoder.clear_buffer(&grads.grad_means2d, 0, None);
encoder.clear_buffer(&grads.grad_conics, 0, None);
encoder.clear_buffer(&grads.grad_colors, 0, None);
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("rasterize_bwd"),
timestamp_writes: self.timestamps_for("rasterize_bwd"),
});
pass.set_pipeline(&self.pipelines.rasterize_bwd);
pass.set_bind_group(0, &bgs.rasterize_bwd, &[]);
pass.dispatch_workgroups(self.config.tiles_x(), self.config.tiles_y(), 1);
}
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("atomic_to_f32_means2d"),
timestamp_writes: self.timestamps_for("atomic_to_f32_means2d"),
});
pass.set_pipeline(&self.pipelines.atomic_to_f32);
pass.set_bind_group(0, &bgs.atomic_means2d, &[]);
pass.dispatch_workgroups(self.workgroups.sort.dispatch_count_x(n * 2), 1, 1);
}
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("atomic_to_f32_conics"),
timestamp_writes: self.timestamps_for("atomic_to_f32_conics"),
});
pass.set_pipeline(&self.pipelines.atomic_to_f32);
pass.set_bind_group(0, &bgs.atomic_conics, &[]);
pass.dispatch_workgroups(self.workgroups.sort.dispatch_count_x(n * 3), 1, 1);
}
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("atomic_to_f32_colors"),
timestamp_writes: self.timestamps_for("atomic_to_f32_colors"),
});
pass.set_pipeline(&self.pipelines.atomic_to_f32);
pass.set_bind_group(0, &bgs.atomic_colors, &[]);
pass.dispatch_workgroups(self.workgroups.sort.dispatch_count_x(n * 3), 1, 1);
}
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("preprocess_bwd"),
timestamp_writes: self.timestamps_for("preprocess_bwd"),
});
pass.set_pipeline(&self.pipelines.preprocess_bwd);
pass.set_bind_group(0, &bgs.preprocess_bwd, &[]);
pass.dispatch_workgroups(self.workgroups.preprocess.dispatch_count_x(n), 1, 1);
}
let flame = if binding_active {
self.flame_bufs.as_ref()
} else {
None
};
if let Some(bufs) = flame {
if let Err(e) = bufs.copy_position_gradients_from(&mut encoder, &grads.grad_positions) {
self.discard_timestamps();
return Err(e);
}
bufs.record_backward_with_timestamps(
&mut encoder,
&self.pipelines.flame_binding_bwd,
self.timestamps_for("flame_binding_bwd"),
);
}
let sh_total = uploaded.sh_len.max(1);
let pending = match self.stage_backward_readbacks(&mut encoder, grads, flame, n, sh_total) {
Ok(pending) => pending,
Err(e) => {
self.discard_timestamps();
return Err(e);
}
};
if let Some(profiler) = self.gpu_timestamps.as_ref() {
profiler.resolve(&mut encoder);
}
self.queue.submit(std::iter::once(encoder.finish()));
let readback_result = self.map_and_collect(pending);
self.collect_timestamps();
let mut readbacks = readback_result?.into_iter();
let grad_positions = readbacks.next().unwrap_or_default();
let grad_rotations = readbacks.next().unwrap_or_default();
let grad_scales = readbacks.next().unwrap_or_default();
let grad_opacities = readbacks.next().unwrap_or_default();
let grad_sh_coeffs = readbacks.next().unwrap_or_default();
let grad_local_offsets_raw = readbacks.next().unwrap_or_default();
#[cfg(feature = "gpu_debug")]
{
use crate::debug_readback::{validate_buffer_count, validate_no_nan_inf};
validate_no_nan_inf(&grad_positions, "grad_positions")?;
validate_no_nan_inf(&grad_rotations, "grad_rotations")?;
validate_no_nan_inf(&grad_scales, "grad_scales")?;
validate_no_nan_inf(&grad_opacities, "grad_opacities")?;
validate_no_nan_inf(&grad_sh_coeffs, "grad_sh_coeffs")?;
validate_buffer_count(grad_opacities.len(), n as usize, "grad_opacities")?;
validate_buffer_count(grad_sh_coeffs.len(), sh_total, "grad_sh_coeffs")?;
}
let unpad3 = |data: &[f32]| -> Vec<[f32; 3]> {
data.chunks_exact(4).map(|c| [c[0], c[1], c[2]]).collect()
};
let unpad4 = |data: &[f32]| -> Vec<[f32; 4]> {
data.chunks_exact(4)
.map(|c| [c[0], c[1], c[2], c[3]])
.collect()
};
Ok(GaussianGradients {
grad_positions: unpad3(&grad_positions),
grad_rotations: unpad4(&grad_rotations),
grad_scales: unpad3(&grad_scales),
grad_opacities,
grad_sh_coeffs,
grad_local_offsets: unpad3(&grad_local_offsets_raw),
})
}
fn prepare_binding_buffers(&mut self, model: &GaussianModel) -> Result<bool, RenderError> {
let table_len = match self.binding_frames.as_ref() {
Some(frames) => frames.len(),
None => return Ok(false),
};
let n = model.len();
if n == 0 || table_len == 0 {
return Ok(false);
}
if model.face_indices.len() != n {
return Err(RenderError::MismatchedBufferSizes {
expected: n,
actual: model.face_indices.len(),
});
}
let needs_alloc = self
.flame_bufs
.as_ref()
.is_none_or(|b| b.num_gaussians() != n || b.num_vertices() != table_len);
if needs_alloc {
self.flame_bufs = Some(FlameBindingBuffers::new(
&self.device,
&self.pipelines.flame_binding_bwd_bgl,
n,
table_len,
));
}
let bufs = self.flame_bufs.as_ref().ok_or_else(|| {
RenderError::Rasterize("flame binding buffers missing after allocation".into())
})?;
let frames = self.binding_frames.as_ref().ok_or_else(|| {
RenderError::Rasterize("binding frame table missing after allocation".into())
})?;
bufs.update_binding_info(&self.queue, &model.face_indices)?;
bufs.update_tbn_frames(&self.queue, frames)?;
Ok(true)
}
pub fn download_image(&self, output: &RenderOutput) -> image::RgbaImage {
let w = output.width;
let h = output.height;
let buf = color_to_rgba_bytes(&output.color_data, w, h);
image::RgbaImage::from_raw(w, h, buf).unwrap_or_else(|| {
tracing::warn!(
width = w,
height = h,
"download_image: could not build an image buffer; returning a blank image"
);
image::RgbaImage::new(w, h)
})
}
pub fn config(&self) -> &RasterConfig {
&self.config
}
pub fn buffer_pool(&self) -> Option<&BufferPool> {
self.buffer_pool.as_ref()
}
#[cfg(feature = "gpu_debug")]
pub fn debug_snapshot(
&self,
) -> Result<crate::debug_readback::RasterizationSnapshot, RenderError> {
let inter = self.intermediate_bufs.as_ref().ok_or_else(|| {
RenderError::Rasterize("debug_snapshot() called before upload_gaussians()".into())
})?;
let n = self.gaussian_bufs.as_ref().map_or(0, |g| g.count);
crate::debug_readback::DebugReadbackBuilder::new(
self.config.image_width,
self.config.image_height,
)
.with_tile_size(self.config.tile_size)
.read_from_gpu(
&self.device,
&self.queue,
&inter.tile_ranges,
&inter.depths,
&inter.radii,
n,
)
}
pub fn pool_stats(&self) -> Option<PoolStats> {
self.buffer_pool.as_ref().map(|p| p.stats())
}
pub fn log_memory_usage(&self) {
if let Some(ref pool) = self.buffer_pool {
pool.log_usage();
} else {
tracing::debug!("Buffer pooling disabled, no pool stats available");
}
}
pub fn clear_buffer_pool(&self) {
if let Some(ref pool) = self.buffer_pool {
pool.clear();
}
}
pub fn set_pool_budget(&self, max_bytes: u64) {
if let Some(ref pool) = self.buffer_pool {
pool.set_budget(max_bytes);
}
}
fn stage_readback(
&self,
encoder: &mut wgpu::CommandEncoder,
src: &wgpu::Buffer,
count: usize,
) -> Result<PendingReadback, RenderError> {
let byte_size = (count as u64).saturating_mul(4).min(src.size());
let count = (byte_size / 4) as usize;
let staging_size = staging_size_for(byte_size);
let usage = wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST;
let pooled = self
.buffer_pool
.as_ref()
.and_then(|pool| pool.acquire(&self.device, staging_size, usage, STAGING_READ_LABEL));
let staging = match pooled {
Some(pooled) => StagingBuffer::Pooled(pooled),
None => {
check_staging_fits(staging_size, self.device.limits().max_buffer_size)?;
StagingBuffer::Direct(self.device.create_buffer(&wgpu::BufferDescriptor {
label: Some(STAGING_READ_LABEL),
size: staging_size,
usage,
mapped_at_creation: false,
}))
}
};
if byte_size > 0 {
encoder.copy_buffer_to_buffer(src, 0, &staging, 0, byte_size);
}
Ok(PendingReadback { staging, count })
}
fn stage_forward_readbacks(
&self,
encoder: &mut wgpu::CommandEncoder,
output: &OutputBuffers,
npx: usize,
want_normals: bool,
) -> Result<Vec<PendingReadback>, RenderError> {
let mut pending = Vec::with_capacity(3);
pending.push(self.stage_readback(encoder, &output.color, npx * 4)?);
pending.push(self.stage_readback(encoder, &output.depth, npx)?);
if want_normals {
if let Some(normals_buf) = output.normals.as_ref() {
pending.push(self.stage_readback(encoder, normals_buf, npx * 4)?);
}
}
Ok(pending)
}
fn stage_backward_readbacks(
&self,
encoder: &mut wgpu::CommandEncoder,
grads: &GradientBuffers,
flame: Option<&FlameBindingBuffers>,
n: u32,
sh_total: usize,
) -> Result<Vec<PendingReadback>, RenderError> {
let n = n as usize;
let mut pending = Vec::with_capacity(6);
pending.push(self.stage_readback(encoder, &grads.grad_positions, n * 4)?);
pending.push(self.stage_readback(encoder, &grads.grad_rotations, n * 4)?);
pending.push(self.stage_readback(encoder, &grads.grad_scales, n * 4)?);
pending.push(self.stage_readback(encoder, &grads.grad_opacities, n)?);
pending.push(self.stage_readback(encoder, &grads.grad_sh_coeffs, sh_total)?);
if let Some(bufs) = flame {
pending.push(self.stage_readback(encoder, &bufs.offset_grads, n * 4)?);
}
Ok(pending)
}
fn map_and_collect(&self, pending: Vec<PendingReadback>) -> Result<Vec<Vec<f32>>, RenderError> {
let mut receivers = Vec::with_capacity(pending.len());
for item in &pending {
let (tx, rx) = std::sync::mpsc::channel();
item.staging
.slice(..)
.map_async(wgpu::MapMode::Read, move |result| {
tx.send(result).ok();
});
receivers.push(rx);
}
let _ = self.device.poll(wgpu::PollType::Wait {
submission_index: None,
timeout: None,
});
let mut out = Vec::with_capacity(pending.len());
let mut first_error: Option<RenderError> = None;
for (item, rx) in pending.iter().zip(receivers) {
let mapped = rx
.recv()
.map_err(|e| RenderError::Rasterize(format!("Channel recv failed: {e}")))
.and_then(|r| {
r.map_err(|e| RenderError::Rasterize(format!("Buffer map failed: {e}")))
});
if let Err(e) = mapped {
if first_error.is_none() {
first_error = Some(e);
}
continue;
}
let slice = item.staging.slice(..);
match slice.get_mapped_range() {
Ok(data) => {
let all_floats: &[f32] = bytemuck::cast_slice(&data);
let end = item.count.min(all_floats.len());
out.push(all_floats[..end].to_vec());
drop(data);
}
Err(e) => {
if first_error.is_none() {
first_error =
Some(RenderError::Rasterize(format!("Mapped range failed: {e}")));
}
}
}
item.staging.unmap();
}
match first_error {
Some(e) => Err(e),
None => Ok(out),
}
}
}
fn color_to_rgba_bytes(color_data: &[f32], width: u32, height: u32) -> Vec<u8> {
let needed = (width as usize)
.saturating_mul(height as usize)
.saturating_mul(4);
let mut out: Vec<u8> = Vec::with_capacity(needed);
out.extend(
color_data
.iter()
.take(needed)
.map(|v| (v.clamp(0.0, 1.0) * 255.0) as u8),
);
out.resize(needed, 0);
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::buffers::{ScanPlan, PREFIX_SUM_BLOCK};
use crate::gaussian::GaussianAttributes;
const BLOCK: usize = PREFIX_SUM_BLOCK as usize;
fn scan_blocks(input: &[u32]) -> (Vec<u32>, Vec<u32>) {
let mut scanned = Vec::with_capacity(input.len());
let mut totals = Vec::new();
for chunk in input.chunks(BLOCK) {
let mut acc = 0u32;
for &v in chunk {
acc += v;
scanned.push(acc);
}
totals.push(acc);
}
if totals.is_empty() {
totals.push(0);
}
(scanned, totals)
}
fn add_block_offsets(data: &mut [u32], block_offsets: &[u32]) {
for (b, chunk) in data.chunks_mut(BLOCK).enumerate() {
let offset = if b == 0 {
0
} else {
block_offsets.get(b - 1).copied().unwrap_or(0)
};
for v in chunk.iter_mut() {
*v += offset;
}
}
}
fn hierarchical_scan(input: &[u32]) -> Vec<u32> {
let (mut scanned, l1_totals) = scan_blocks(input);
if l1_totals.len() > 1 {
let (mut l1_scanned, l2_totals) = scan_blocks(&l1_totals);
if l2_totals.len() > 1 {
let (l2_scanned, _) = scan_blocks(&l2_totals);
add_block_offsets(&mut l1_scanned, &l2_scanned);
}
add_block_offsets(&mut scanned, &l1_scanned);
}
scanned
}
#[test]
fn test_hierarchical_scan_matches_plain_inclusive_scan() {
for n in [1usize, 511, 512, 513, 600, 2000, 262_145] {
let input: Vec<u32> = (0..n).map(|i| (i % 7) as u32).collect();
let mut expected = Vec::with_capacity(n);
let mut acc = 0u32;
for &v in &input {
acc += v;
expected.push(acc);
}
assert_eq!(hierarchical_scan(&input), expected, "n = {n}");
}
}
#[test]
fn test_block_offset_add_is_exclusive_not_inclusive() {
let input: Vec<u32> = vec![1; 600]; let scanned = hierarchical_scan(&input);
assert_eq!(scanned[0], 1, "block 0 must not be offset at all");
assert_eq!(scanned[511], 512);
assert_eq!(
scanned[512], 513,
"block 1 continues from block 0's total, it does not double it"
);
assert_eq!(scanned[599], 600);
}
#[test]
fn test_scan_plan_levels_match_the_model() {
assert_eq!(ScanPlan::new(500).levels, 1);
assert_eq!(ScanPlan::new(600).levels, 2);
assert_eq!(ScanPlan::new(262_145).levels, 3);
assert!(ScanPlan::new(262_145).is_supported());
}
fn make_model(n: usize) -> GaussianModel {
GaussianModel {
gaussians: vec![
GaussianAttributes {
position: [0.0, 0.0, 1.0],
_pad0: 0.0,
rotation: [0.0, 0.0, 0.0, 1.0],
scale: [-2.0, -2.0, -2.0],
opacity: 0.0,
};
n
],
sh_coeffs: vec![0.0; n * 3],
sh_degree: 0,
face_indices: vec![0u32; n],
barycentric: vec![[1.0, 0.0, 0.0]; n],
local_offsets: vec![[0.0, 0.0, 0.0]; n],
is_rigid: vec![false; n],
}
}
#[test]
fn test_color_to_rgba_bytes_matches_per_pixel_conversion() {
let color_data: Vec<f32> = vec![
0.0, 0.5, 1.0, 1.0, -0.25, 0.25, 2.0, 0.0, ];
let bytes = color_to_rgba_bytes(&color_data, 2, 1);
assert_eq!(bytes.len(), 2 * 4); let expected: Vec<u8> = color_data
.iter()
.map(|v| (v.clamp(0.0, 1.0) * 255.0) as u8)
.collect();
assert_eq!(bytes, expected);
}
#[test]
fn test_color_to_rgba_bytes_pads_short_input() {
let bytes = color_to_rgba_bytes(&[1.0, 1.0, 1.0, 1.0], 4, 4);
assert_eq!(bytes.len(), 4 * 4 * 4);
assert_eq!(&bytes[..4], &[255, 255, 255, 255]);
assert!(bytes[4..].iter().all(|&b| b == 0));
}
#[test]
fn test_color_to_rgba_bytes_truncates_long_input() {
let bytes = color_to_rgba_bytes(&vec![1.0f32; 1024], 2, 2);
assert_eq!(bytes.len(), 2 * 2 * 4);
}
#[test]
fn test_rasterize_tile_size_matches_config_default() {
assert_eq!(RasterConfig::default().tile_size, RASTERIZE_TILE_SIZE);
}
#[test]
fn test_uploaded_model_identity_distinguishes_models() {
let a = make_model(4);
let b = make_model(4);
let id_a = UploadedModel::of(&a);
assert_eq!(id_a, UploadedModel::of(&a));
assert!(id_a.describes(&a));
assert_ne!(id_a, UploadedModel::of(&b));
assert!(
!id_a.describes(&b),
"a second model of the same size must still force a re-upload"
);
}
#[test]
fn test_uploaded_model_identity_detects_sh_degree_change() {
let mut model = make_model(4);
let id = UploadedModel::of(&model);
model.sh_degree = 1;
assert!(!id.describes(&model));
}
#[test]
fn test_uploaded_model_identity_tracks_gaussian_count() {
let small = make_model(4);
let large = make_model(8);
assert_ne!(UploadedModel::of(&small).len, UploadedModel::of(&large).len);
assert_eq!(UploadedModel::of(&small).len, 4);
assert_eq!(UploadedModel::of(&large).sh_len, 8 * 3);
}
#[test]
fn test_allocation_identity_ignores_pointer_but_tracks_shape() {
let a = make_model(4);
let b = make_model(4);
let id = UploadedModel::of(&a);
assert!(id.describes_allocation(&b));
assert!(!id.describes(&b));
let bigger = make_model(8);
assert!(!id.describes_allocation(&bigger));
let mut degree_changed = make_model(4);
degree_changed.sh_degree = 2;
assert!(!id.describes_allocation(°ree_changed));
let mut sh_changed = make_model(4);
sh_changed.sh_coeffs.push(0.0);
assert!(!id.describes_allocation(&sh_changed));
}
#[test]
fn test_check_staging_fits_refuses_sizes_past_the_device_limit() {
const MAX: u64 = 256 * 1024 * 1024;
assert!(
check_staging_fits(MAX, MAX).is_ok(),
"exactly at the limit is allowed"
);
assert!(check_staging_fits(wgpu::MAP_ALIGNMENT, MAX).is_ok());
match check_staging_fits(MAX + wgpu::MAP_ALIGNMENT, MAX) {
Err(RenderError::BufferOverflow {
buffer_name,
max_size,
requested,
}) => {
assert_eq!(buffer_name, STAGING_READ_LABEL);
assert_eq!(max_size, MAX);
assert_eq!(requested, MAX + wgpu::MAP_ALIGNMENT);
}
other => panic!("expected BufferOverflow, got {other:?}"),
}
}
#[test]
fn test_staging_size_only_exceeds_a_misaligned_device_limit() {
const MAX: u64 = 256 * 1024 * 1024;
assert!(MAX.is_multiple_of(wgpu::MAP_ALIGNMENT));
for byte_size in [0, 4, 7, 8, 4095, MAX - 4, MAX] {
let staging = staging_size_for(byte_size);
assert!(staging >= byte_size.max(wgpu::MAP_ALIGNMENT));
assert!(staging.is_multiple_of(wgpu::MAP_ALIGNMENT));
assert!(
check_staging_fits(staging, MAX).is_ok(),
"byte_size {byte_size} rounds to {staging}, still within {MAX}"
);
}
let misaligned = MAX + 4;
assert!(!misaligned.is_multiple_of(wgpu::MAP_ALIGNMENT));
assert!(
check_staging_fits(staging_size_for(misaligned), misaligned).is_err(),
"rounding a legal misaligned size up must be refused, not created"
);
}
}