use super::Binder;
use crate::Scene;
use crate::content::identity::MaterialId;
use crate::shading::culling::{
CANDIDATE_STATISTICS_WORDS, CULL_CAMERA, CULL_CANDIDATE_STATISTICS, CULL_DISPATCH_WORDS,
CULL_FRUSTUM, CULL_LOD, CULL_NEAR, CULL_PAIRED, CULL_WORKGROUP, CullListsHeader,
CullStatistics, CullView, DrawCommand, PAIRED_INDICES, dispatch_side, late_entries, words,
};
use crate::shading::vertex::{DRAW_INSTANCE_SLOT, DrawInstance};
use crate::view::View;
use crate::view::culling::Frustum;
use crate::view::pipelines::{Alpha, GeometryPass, GeometryPipelines, Variant};
use crate::view::population::camera_variant;
use glam::{DMat4, Mat4};
use std::ops::Range;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Phase {
Early,
Late,
}
const COMMAND_KINDS: u32 = 3;
struct DrawnSet {
index: u32,
region: Range<u32>,
variant: Variant,
material: MaterialId,
positions: u32,
}
pub(crate) struct GpuList {
uniform: wgpu::Buffer,
lists: wgpu::Buffer,
dispatch: wgpu::Buffer,
regions: wgpu::Buffer,
late_regions: wgpu::Buffer,
draws: wgpu::Buffer,
reset: wgpu::Buffer,
commands: u32,
late: bool,
paired: bool,
paired_region: u32,
paired_indices: wgpu::Buffer,
candidates: u32,
candidate_statistics: bool,
drawn: Vec<DrawnSet>,
}
fn grown(
device: &wgpu::Device,
buffer: &mut wgpu::Buffer,
size: u64,
label: &str,
usage: wgpu::BufferUsages,
) -> bool {
if buffer.size() >= size {
return false;
}
let limits = device.limits();
let mut most = limits.max_buffer_size;
if usage.contains(wgpu::BufferUsages::STORAGE) {
most = most.min(limits.max_storage_buffer_binding_size);
}
*buffer = crate::counters::buffer(
device,
&wgpu::BufferDescriptor {
label: Some(label),
size: size.next_power_of_two().min(most).max(size),
usage,
mapped_at_creation: false,
},
);
true
}
fn bytes_of<T>(count: u32) -> u64 {
u64::from(count.max(1)) * std::mem::size_of::<T>() as u64
}
const STORAGE: wgpu::BufferUsages = wgpu::BufferUsages::STORAGE;
impl GpuList {
pub fn new(device: &wgpu::Device) -> Self {
let buffer = |label, size, usage| {
crate::counters::buffer(
device,
&wgpu::BufferDescriptor {
label: Some(label),
size,
usage,
mapped_at_creation: false,
},
)
};
Self {
uniform: buffer(
"cull view",
std::mem::size_of::<CullView>() as u64,
wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
),
lists: buffer("cull lists", 16, STORAGE),
dispatch: buffer(
"cull dispatch",
u64::from(CULL_DISPATCH_WORDS) * 4,
STORAGE | wgpu::BufferUsages::INDIRECT,
),
regions: buffer("cluster list", 16, STORAGE),
late_regions: buffer("late cluster list", 16, STORAGE),
draws: buffer("view draws", 16, STORAGE),
reset: buffer("view draws reset", 16, wgpu::BufferUsages::COPY_SRC),
commands: 0,
late: false,
paired: false,
paired_region: 0,
paired_indices: crate::counters::buffer_init(
device,
&wgpu::util::BufferInitDescriptor {
label: Some("paired indices"),
contents: bytemuck::cast_slice(&PAIRED_INDICES),
usage: wgpu::BufferUsages::INDEX,
},
),
candidates: 0,
candidate_statistics: false,
drawn: Vec::new(),
}
}
pub fn prepare(
&mut self,
device: &wgpu::Device,
queue: &wgpu::Queue,
scene: &Scene,
mut view: CullView,
late: bool,
) {
let candidates = scene.candidates.end();
let sets = &scene.candidates.sets;
let statistics = view.flags & CULL_CANDIDATE_STATISTICS != 0;
let queue_capacity = if late { sets.region_end() } else { 0 };
let entries = if late {
late_entries(candidates, queue_capacity)
} else {
u64::from(candidates)
};
grown(
device,
&mut self.lists,
std::mem::size_of::<CullListsHeader>() as u64
+ entries.max(1) * std::mem::size_of::<[u32; 2]>() as u64,
"cull lists",
STORAGE | wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::COPY_SRC,
);
let regions_usage = STORAGE | wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_SRC;
let regions = bytes_of::<DrawInstance>(sets.region_end());
let paired = view.flags & CULL_PAIRED != 0;
grown(
device,
&mut self.regions,
if paired { 2 * regions } else { regions },
"cluster list",
regions_usage,
);
if late {
grown(
device,
&mut self.late_regions,
regions,
"late cluster list",
regions_usage,
);
}
let commands = sets.end().max(1).next_power_of_two().max(self.commands);
let statistics_words = if statistics {
candidates.max(1).next_power_of_two() * CANDIDATE_STATISTICS_WORDS
} else {
0
};
let draws = std::mem::size_of::<CullStatistics>() as u64
+ u64::from(COMMAND_KINDS * commands) * std::mem::size_of::<DrawCommand>() as u64
+ u64::from(statistics_words) * 4;
let usage = STORAGE
| wgpu::BufferUsages::INDIRECT
| wgpu::BufferUsages::COPY_DST
| wgpu::BufferUsages::COPY_SRC;
let reallocated = grown(device, &mut self.draws, draws, "view draws", usage);
if reallocated || commands != self.commands {
self.reset = crate::counters::buffer(
device,
&wgpu::BufferDescriptor {
label: Some("view draws reset"),
size: self.draws.size(),
usage: wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
},
);
let mut reset = vec![0u32; (self.draws.size() / 4) as usize];
let first = words::<CullStatistics>() as usize;
let kinds = COMMAND_KINDS * commands * words::<DrawCommand>();
for command in reset[first..first + kinds as usize]
.chunks_exact_mut(words::<DrawCommand>() as usize)
{
command.copy_from_slice(bytemuck::cast_slice(&[DrawCommand::RESET]));
}
crate::counters::write_buffer(queue, &self.reset, 0, bytemuck::cast_slice(&reset));
self.commands = commands;
}
self.candidates = candidates;
self.candidate_statistics = statistics;
self.late = late;
self.paired = paired;
self.paired_region = if paired { sets.region_end().max(1) } else { 0 };
let workgroups = candidates.div_ceil(CULL_WORKGROUP);
view.candidates = candidates;
view.candidate_side = dispatch_side(workgroups)[0];
view.candidate_statistics =
words::<CullStatistics>() + COMMAND_KINDS * commands * words::<DrawCommand>();
view.late_command = commands;
view.queue_capacity = queue_capacity;
view.paired_command = 2 * commands;
view.paired_region = self.paired_region;
crate::counters::write_buffer(queue, &self.uniform, 0, bytemuck::bytes_of(&view));
let camera = view.flags & CULL_CAMERA != 0;
self.drawn.clear();
for (index, key, region) in sets.iter() {
let material = scene.drawn_material(key.material);
let shown = if camera {
material.enabled(Some(view.visibility_mask))
} else {
material.casts_directional_shadow(view.visibility_mask)
};
if !shown {
continue;
}
self.drawn.push(DrawnSet {
index,
region,
variant: camera_variant(&material.values, key.mirrored, key.deforms),
material: key.material,
positions: key.positions,
});
}
}
pub fn encode_reset(&self, encoder: &mut wgpu::CommandEncoder) {
encoder.copy_buffer_to_buffer(&self.reset, 0, &self.draws, 0, self.reset.size());
encoder.clear_buffer(
&self.lists,
0,
Some(std::mem::size_of::<CullListsHeader>() as u64),
);
}
pub fn instance_workgroups(&self) -> [u32; 2] {
dispatch_side(self.candidates.div_ceil(CULL_WORKGROUP))
}
pub fn culls(&self) -> bool {
self.candidates > 0
}
pub fn late(&self) -> bool {
self.late
}
pub fn buffers(&self) -> CullBuffers<'_> {
CullBuffers {
view: &self.uniform,
lists: &self.lists,
dispatch: &self.dispatch,
regions: &self.regions,
late_regions: &self.late_regions,
draws: &self.draws,
}
}
pub fn statistics(&self) -> (&wgpu::Buffer, Option<(u64, u32)>) {
let at = u64::from(
words::<CullStatistics>() + COMMAND_KINDS * self.commands * words::<DrawCommand>(),
) * 4;
(
&self.draws,
self.candidate_statistics.then_some((at, self.candidates)),
)
}
fn phases(&self) -> &'static [Phase] {
if self.late {
&[Phase::Early, Phase::Late]
} else {
&[Phase::Early]
}
}
#[cfg(any(test, feature = "diagnostics"))]
pub fn draws(&self) -> usize {
let paired = self
.drawn
.iter()
.filter(|set| self.paired && set.variant.alpha == Alpha::Opaque)
.count();
self.drawn.len() * self.phases().len() + paired
}
pub fn draw(
&self,
scene: &Scene,
pipelines: &GeometryPipelines,
pass: &mut wgpu::RenderPass<'_>,
kind: GeometryPass,
) -> usize {
self.issue(scene, pipelines, pass, kind, self.phases(), &|_| true)
}
pub fn draw_materials(
&self,
scene: &Scene,
pipelines: &GeometryPipelines,
pass: &mut wgpu::RenderPass<'_>,
kind: GeometryPass,
drawn: &dyn Fn(MaterialId) -> bool,
) -> usize {
self.issue(scene, pipelines, pass, kind, self.phases(), drawn)
}
pub fn draw_phase(
&self,
scene: &Scene,
pipelines: &GeometryPipelines,
pass: &mut wgpu::RenderPass<'_>,
kind: GeometryPass,
phase: Phase,
) -> usize {
if phase == Phase::Late && !self.late {
return 0;
}
self.issue(scene, pipelines, pass, kind, &[phase], &|_| true)
}
fn issue(
&self,
scene: &Scene,
pipelines: &GeometryPipelines,
pass: &mut wgpu::RenderPass<'_>,
kind: GeometryPass,
phases: &[Phase],
drawn: &dyn Fn(MaterialId) -> bool,
) -> usize {
if self.drawn.is_empty() {
return 0;
}
pass.set_bind_group(1, &scene.scene_group, &[]);
let paired = self.paired && phases.contains(&Phase::Early);
debug_assert!(!paired || kind == GeometryPass::DirectionalShadow);
if paired {
pass.set_index_buffer(self.paired_indices.slice(..), wgpu::IndexFormat::Uint32);
}
let mut binder = Binder::new(pipelines, kind);
let stride = std::mem::size_of::<DrawInstance>() as u64;
let command = std::mem::size_of::<DrawCommand>() as u64;
let first = std::mem::size_of::<CullStatistics>() as u64;
let mut positions = None;
let mut issued = 0;
for set in self.drawn.iter().filter(|set| drawn(set.material)) {
binder.bind(pass, scene, set.variant, set.material);
if kind.binds_caster_positions() && positions != Some(set.positions) {
pass.set_bind_group(3, scene.geometry.positions_group(set.positions), &[]);
positions = Some(set.positions);
}
let region = |at: u32| {
u64::from(at + set.region.start) * stride..u64::from(at + set.region.end) * stride
};
for phase in phases {
let (regions, index) = match phase {
Phase::Early => (&self.regions, set.index),
Phase::Late => (&self.late_regions, self.commands + set.index),
};
pass.set_vertex_buffer(DRAW_INSTANCE_SLOT, regions.slice(region(0)));
pass.draw_indirect(&self.draws, first + u64::from(index) * command);
issued += 1;
}
if paired && set.variant.alpha == Alpha::Opaque {
pass.set_pipeline(pipelines.get(GeometryPass::PairedShadow, set.variant));
binder.forget_pipeline();
let index = 2 * self.commands + set.index;
pass.set_vertex_buffer(
DRAW_INSTANCE_SLOT,
self.regions.slice(region(self.paired_region)),
);
pass.draw_indexed_indirect(&self.draws, first + u64::from(index) * command);
issued += 1;
}
}
issued
}
}
#[derive(Clone, Copy)]
pub(crate) struct CullBuffers<'a> {
pub view: &'a wgpu::Buffer,
pub lists: &'a wgpu::Buffer,
pub dispatch: &'a wgpu::Buffer,
pub regions: &'a wgpu::Buffer,
pub late_regions: &'a wgpu::Buffer,
pub draws: &'a wgpu::Buffer,
}
pub(crate) fn camera_cull(view: &View, size: [u32; 2], mask: u32, culling: bool) -> CullView {
let camera = Mat4::from_cols_array_2d(&view.uniform.view);
let projection = Mat4::from_cols_array_2d(&view.uniform.projection);
let (planes, plane_errors) = Frustum::planes(camera, projection, view.uniform.jitter);
let lod = !size.contains(&0) && camera.is_finite() && projection.is_finite();
let absolute = |m: Mat4| DMat4::from_cols_array(&m.as_dmat4().to_cols_array().map(f64::abs));
let flags = CULL_CAMERA
| CULL_NEAR
| if culling { CULL_FRUSTUM } else { 0 }
| if lod { CULL_LOD } else { 0 }
| if cfg!(feature = "diagnostics") {
CULL_CANDIDATE_STATISTICS
} else {
0
};
CullView {
planes,
plane_errors,
lod_clip_from_world: (projection.as_dmat4() * camera.as_dmat4())
.as_mat4()
.to_cols_array_2d(),
lod_magnitude: (absolute(projection) * absolute(camera))
.as_mat4()
.to_cols_array_2d(),
lod_size: size.map(|side| side as f32),
visibility_mask: mask,
flags,
..bytemuck::Zeroable::zeroed()
}
}
pub(crate) fn cascade_cull(view: &View, mask: u32) -> CullView {
let (planes, plane_errors) = Frustum::planes(
Mat4::from_cols_array_2d(&view.uniform.view),
Mat4::from_cols_array_2d(&view.uniform.projection),
view.uniform.jitter,
);
CullView {
planes,
plane_errors,
visibility_mask: mask,
flags: CULL_FRUSTUM | CULL_PAIRED,
..bytemuck::Zeroable::zeroed()
}
}
#[cfg(all(test, not(target_arch = "wasm32")))]
mod read;
#[cfg(all(test, not(target_arch = "wasm32")))]
pub(crate) use read::{read_early, read_phases};