mod buffer;
mod metadata;
mod range;
mod stateless;
mod texture;
use crate::{
binding_model, command, conv,
hal_api::HalApi,
id,
lock::{rank, Mutex, RwLock},
pipeline, resource,
snatch::SnatchGuard,
};
use std::{fmt, ops, sync::Arc};
use thiserror::Error;
pub(crate) use buffer::{BufferBindGroupState, BufferTracker, BufferUsageScope};
use metadata::{ResourceMetadata, ResourceMetadataProvider};
pub(crate) use stateless::{StatelessBindGroupSate, StatelessTracker};
pub(crate) use texture::{
TextureBindGroupState, TextureSelector, TextureTracker, TextureUsageScope,
};
use wgt::strict_assert_ne;
#[repr(transparent)]
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub(crate) struct TrackerIndex(u32);
impl TrackerIndex {
pub const INVALID: Self = TrackerIndex(u32::MAX);
pub fn as_usize(self) -> usize {
debug_assert!(self != Self::INVALID);
self.0 as usize
}
}
struct TrackerIndexAllocator {
unused: Vec<TrackerIndex>,
next_index: TrackerIndex,
}
impl TrackerIndexAllocator {
pub fn new() -> Self {
TrackerIndexAllocator {
unused: Vec::new(),
next_index: TrackerIndex(0),
}
}
pub fn alloc(&mut self) -> TrackerIndex {
if let Some(index) = self.unused.pop() {
return index;
}
let index = self.next_index;
self.next_index.0 += 1;
index
}
pub fn free(&mut self, index: TrackerIndex) {
self.unused.push(index);
}
pub fn size(&self) -> usize {
self.next_index.0 as usize
}
}
impl std::fmt::Debug for TrackerIndexAllocator {
fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
Ok(())
}
}
#[derive(Debug)]
pub(crate) struct SharedTrackerIndexAllocator {
inner: Mutex<TrackerIndexAllocator>,
}
impl SharedTrackerIndexAllocator {
pub fn new() -> Self {
SharedTrackerIndexAllocator {
inner: Mutex::new(
rank::SHARED_TRACKER_INDEX_ALLOCATOR_INNER,
TrackerIndexAllocator::new(),
),
}
}
pub fn alloc(&self) -> TrackerIndex {
self.inner.lock().alloc()
}
pub fn free(&self, index: TrackerIndex) {
self.inner.lock().free(index);
}
pub fn size(&self) -> usize {
self.inner.lock().size()
}
}
pub(crate) struct TrackerIndexAllocators {
pub buffers: Arc<SharedTrackerIndexAllocator>,
pub staging_buffers: Arc<SharedTrackerIndexAllocator>,
pub textures: Arc<SharedTrackerIndexAllocator>,
pub texture_views: Arc<SharedTrackerIndexAllocator>,
pub samplers: Arc<SharedTrackerIndexAllocator>,
pub bind_groups: Arc<SharedTrackerIndexAllocator>,
pub bind_group_layouts: Arc<SharedTrackerIndexAllocator>,
pub compute_pipelines: Arc<SharedTrackerIndexAllocator>,
pub render_pipelines: Arc<SharedTrackerIndexAllocator>,
pub pipeline_layouts: Arc<SharedTrackerIndexAllocator>,
pub bundles: Arc<SharedTrackerIndexAllocator>,
pub query_sets: Arc<SharedTrackerIndexAllocator>,
}
impl TrackerIndexAllocators {
pub fn new() -> Self {
TrackerIndexAllocators {
buffers: Arc::new(SharedTrackerIndexAllocator::new()),
staging_buffers: Arc::new(SharedTrackerIndexAllocator::new()),
textures: Arc::new(SharedTrackerIndexAllocator::new()),
texture_views: Arc::new(SharedTrackerIndexAllocator::new()),
samplers: Arc::new(SharedTrackerIndexAllocator::new()),
bind_groups: Arc::new(SharedTrackerIndexAllocator::new()),
bind_group_layouts: Arc::new(SharedTrackerIndexAllocator::new()),
compute_pipelines: Arc::new(SharedTrackerIndexAllocator::new()),
render_pipelines: Arc::new(SharedTrackerIndexAllocator::new()),
pipeline_layouts: Arc::new(SharedTrackerIndexAllocator::new()),
bundles: Arc::new(SharedTrackerIndexAllocator::new()),
query_sets: Arc::new(SharedTrackerIndexAllocator::new()),
}
}
}
#[derive(Debug, PartialEq)]
pub(crate) struct PendingTransition<S: ResourceUses> {
pub id: u32,
pub selector: S::Selector,
pub usage: ops::Range<S>,
}
pub(crate) type PendingTransitionList = Vec<PendingTransition<hal::TextureUses>>;
impl PendingTransition<hal::BufferUses> {
pub fn into_hal<'a, A: HalApi>(
self,
buf: &'a resource::Buffer<A>,
snatch_guard: &'a SnatchGuard<'a>,
) -> hal::BufferBarrier<'a, A> {
let buffer = buf.raw.get(snatch_guard).expect("Buffer is destroyed");
hal::BufferBarrier {
buffer,
usage: self.usage,
}
}
}
impl PendingTransition<hal::TextureUses> {
pub fn into_hal<'a, A: HalApi>(self, texture: &'a A::Texture) -> hal::TextureBarrier<'a, A> {
strict_assert_ne!(self.usage.start, hal::TextureUses::UNKNOWN);
strict_assert_ne!(self.usage.end, hal::TextureUses::UNKNOWN);
let mip_count = self.selector.mips.end - self.selector.mips.start;
strict_assert_ne!(mip_count, 0);
let layer_count = self.selector.layers.end - self.selector.layers.start;
strict_assert_ne!(layer_count, 0);
hal::TextureBarrier {
texture,
range: wgt::ImageSubresourceRange {
aspect: wgt::TextureAspect::All,
base_mip_level: self.selector.mips.start,
mip_level_count: Some(mip_count),
base_array_layer: self.selector.layers.start,
array_layer_count: Some(layer_count),
},
usage: self.usage,
}
}
}
pub(crate) trait ResourceUses:
fmt::Debug + ops::BitAnd<Output = Self> + ops::BitOr<Output = Self> + PartialEq + Sized + Copy
{
const EXCLUSIVE: Self;
type Selector: fmt::Debug;
fn bits(self) -> u16;
fn all_ordered(self) -> bool;
fn any_exclusive(self) -> bool;
}
fn invalid_resource_state<T: ResourceUses>(state: T) -> bool {
state.any_exclusive() && !conv::is_power_of_two_u16(state.bits())
}
fn skip_barrier<T: ResourceUses>(old_state: T, new_state: T) -> bool {
old_state == new_state && old_state.all_ordered()
}
#[derive(Clone, Debug, Error, Eq, PartialEq)]
pub enum UsageConflict {
#[error("Attempted to use invalid buffer")]
BufferInvalid { id: id::BufferId },
#[error("Attempted to use invalid texture")]
TextureInvalid { id: id::TextureId },
#[error("Attempted to use buffer with {invalid_use}.")]
Buffer {
id: id::BufferId,
invalid_use: InvalidUse<hal::BufferUses>,
},
#[error("Attempted to use a texture (mips {mip_levels:?} layers {array_layers:?}) with {invalid_use}.")]
Texture {
id: id::TextureId,
mip_levels: ops::Range<u32>,
array_layers: ops::Range<u32>,
invalid_use: InvalidUse<hal::TextureUses>,
},
}
impl UsageConflict {
fn from_buffer(
id: id::BufferId,
current_state: hal::BufferUses,
new_state: hal::BufferUses,
) -> Self {
Self::Buffer {
id,
invalid_use: InvalidUse {
current_state,
new_state,
},
}
}
fn from_texture(
id: id::TextureId,
selector: TextureSelector,
current_state: hal::TextureUses,
new_state: hal::TextureUses,
) -> Self {
Self::Texture {
id,
mip_levels: selector.mips,
array_layers: selector.layers,
invalid_use: InvalidUse {
current_state,
new_state,
},
}
}
}
impl crate::error::PrettyError for UsageConflict {
fn fmt_pretty(&self, fmt: &mut crate::error::ErrorFormatter) {
fmt.error(self);
match *self {
Self::BufferInvalid { id } => {
fmt.buffer_label(&id);
}
Self::TextureInvalid { id } => {
fmt.texture_label(&id);
}
Self::Buffer { id, .. } => {
fmt.buffer_label(&id);
}
Self::Texture { id, .. } => {
fmt.texture_label(&id);
}
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct InvalidUse<T> {
current_state: T,
new_state: T,
}
impl<T: ResourceUses> fmt::Display for InvalidUse<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let current = self.current_state;
let new = self.new_state;
let current_exclusive = current & T::EXCLUSIVE;
let new_exclusive = new & T::EXCLUSIVE;
let exclusive = current_exclusive | new_exclusive;
write!(
f,
"conflicting usages. Current usage {current:?} and new usage {new:?}. \
{exclusive:?} is an exclusive usage and cannot be used with any other \
usages within the usage scope (renderpass or compute dispatch)"
)
}
}
#[derive(Debug)]
pub(crate) struct BindGroupStates<A: HalApi> {
pub buffers: BufferBindGroupState<A>,
pub textures: TextureBindGroupState<A>,
pub views: StatelessBindGroupSate<resource::TextureView<A>>,
pub samplers: StatelessBindGroupSate<resource::Sampler<A>>,
}
impl<A: HalApi> BindGroupStates<A> {
pub fn new() -> Self {
Self {
buffers: BufferBindGroupState::new(),
textures: TextureBindGroupState::new(),
views: StatelessBindGroupSate::new(),
samplers: StatelessBindGroupSate::new(),
}
}
pub fn optimize(&mut self) {
self.buffers.optimize();
self.textures.optimize();
self.views.optimize();
self.samplers.optimize();
}
}
#[derive(Debug)]
pub(crate) struct RenderBundleScope<A: HalApi> {
pub buffers: RwLock<BufferUsageScope<A>>,
pub textures: RwLock<TextureUsageScope<A>>,
pub bind_groups: RwLock<StatelessTracker<binding_model::BindGroup<A>>>,
pub render_pipelines: RwLock<StatelessTracker<pipeline::RenderPipeline<A>>>,
pub query_sets: RwLock<StatelessTracker<resource::QuerySet<A>>>,
}
impl<A: HalApi> RenderBundleScope<A> {
pub fn new() -> Self {
Self {
buffers: RwLock::new(
rank::RENDER_BUNDLE_SCOPE_BUFFERS,
BufferUsageScope::default(),
),
textures: RwLock::new(
rank::RENDER_BUNDLE_SCOPE_TEXTURES,
TextureUsageScope::default(),
),
bind_groups: RwLock::new(
rank::RENDER_BUNDLE_SCOPE_BIND_GROUPS,
StatelessTracker::new(),
),
render_pipelines: RwLock::new(
rank::RENDER_BUNDLE_SCOPE_RENDER_PIPELINES,
StatelessTracker::new(),
),
query_sets: RwLock::new(
rank::RENDER_BUNDLE_SCOPE_QUERY_SETS,
StatelessTracker::new(),
),
}
}
pub unsafe fn merge_bind_group(
&mut self,
bind_group: &BindGroupStates<A>,
) -> Result<(), UsageConflict> {
unsafe { self.buffers.write().merge_bind_group(&bind_group.buffers)? };
unsafe {
self.textures
.write()
.merge_bind_group(&bind_group.textures)?
};
Ok(())
}
}
pub(crate) type UsageScopePool<A> = Mutex<Vec<(BufferUsageScope<A>, TextureUsageScope<A>)>>;
#[derive(Debug)]
pub(crate) struct UsageScope<'a, A: HalApi> {
pub pool: &'a UsageScopePool<A>,
pub buffers: BufferUsageScope<A>,
pub textures: TextureUsageScope<A>,
}
impl<'a, A: HalApi> Drop for UsageScope<'a, A> {
fn drop(&mut self) {
self.buffers.clear();
self.textures.clear();
self.pool.lock().push((
std::mem::take(&mut self.buffers),
std::mem::take(&mut self.textures),
));
}
}
impl<A: HalApi> UsageScope<'static, A> {
pub fn new_pooled<'d>(
pool: &'d UsageScopePool<A>,
tracker_indices: &TrackerIndexAllocators,
) -> UsageScope<'d, A> {
let pooled = pool.lock().pop().unwrap_or_default();
let mut scope = UsageScope::<'d, A> {
pool,
buffers: pooled.0,
textures: pooled.1,
};
scope.buffers.set_size(tracker_indices.buffers.size());
scope.textures.set_size(tracker_indices.textures.size());
scope
}
}
impl<'a, A: HalApi> UsageScope<'a, A> {
pub unsafe fn merge_bind_group(
&mut self,
bind_group: &BindGroupStates<A>,
) -> Result<(), UsageConflict> {
unsafe {
self.buffers.merge_bind_group(&bind_group.buffers)?;
self.textures.merge_bind_group(&bind_group.textures)?;
}
Ok(())
}
pub unsafe fn merge_render_bundle(
&mut self,
render_bundle: &RenderBundleScope<A>,
) -> Result<(), UsageConflict> {
self.buffers
.merge_usage_scope(&*render_bundle.buffers.read())?;
self.textures
.merge_usage_scope(&*render_bundle.textures.read())?;
Ok(())
}
}
pub(crate) trait ResourceTracker {
fn remove_abandoned(&mut self, index: TrackerIndex) -> bool;
}
pub(crate) struct Tracker<A: HalApi> {
pub buffers: BufferTracker<A>,
pub textures: TextureTracker<A>,
pub views: StatelessTracker<resource::TextureView<A>>,
pub samplers: StatelessTracker<resource::Sampler<A>>,
pub bind_groups: StatelessTracker<binding_model::BindGroup<A>>,
pub compute_pipelines: StatelessTracker<pipeline::ComputePipeline<A>>,
pub render_pipelines: StatelessTracker<pipeline::RenderPipeline<A>>,
pub bundles: StatelessTracker<command::RenderBundle<A>>,
pub query_sets: StatelessTracker<resource::QuerySet<A>>,
}
impl<A: HalApi> Tracker<A> {
pub fn new() -> Self {
Self {
buffers: BufferTracker::new(),
textures: TextureTracker::new(),
views: StatelessTracker::new(),
samplers: StatelessTracker::new(),
bind_groups: StatelessTracker::new(),
compute_pipelines: StatelessTracker::new(),
render_pipelines: StatelessTracker::new(),
bundles: StatelessTracker::new(),
query_sets: StatelessTracker::new(),
}
}
pub unsafe fn set_and_remove_from_usage_scope_sparse(
&mut self,
scope: &mut UsageScope<A>,
bind_group: &BindGroupStates<A>,
) {
unsafe {
self.buffers.set_and_remove_from_usage_scope_sparse(
&mut scope.buffers,
bind_group.buffers.used_tracker_indices(),
)
};
unsafe {
self.textures
.set_and_remove_from_usage_scope_sparse(&mut scope.textures, &bind_group.textures)
};
}
pub unsafe fn add_from_render_bundle(
&mut self,
render_bundle: &RenderBundleScope<A>,
) -> Result<(), UsageConflict> {
self.bind_groups
.add_from_tracker(&*render_bundle.bind_groups.read());
self.render_pipelines
.add_from_tracker(&*render_bundle.render_pipelines.read());
self.query_sets
.add_from_tracker(&*render_bundle.query_sets.read());
Ok(())
}
}