use std::{borrow::Cow, marker::PhantomData, sync::Arc};
use super::{PendingTransition, ResourceTracker, TrackerIndex};
use crate::{
hal_api::HalApi,
id::BufferId,
lock::{rank, Mutex},
resource::{Buffer, Resource},
snatch::SnatchGuard,
storage::Storage,
track::{
invalid_resource_state, skip_barrier, ResourceMetadata, ResourceMetadataProvider,
ResourceUses, UsageConflict,
},
};
use hal::{BufferBarrier, BufferUses};
use wgt::{strict_assert, strict_assert_eq};
impl ResourceUses for BufferUses {
const EXCLUSIVE: Self = Self::EXCLUSIVE;
type Selector = ();
fn bits(self) -> u16 {
Self::bits(&self)
}
fn all_ordered(self) -> bool {
Self::ORDERED.contains(self)
}
fn any_exclusive(self) -> bool {
self.intersects(Self::EXCLUSIVE)
}
}
#[derive(Debug)]
pub(crate) struct BufferBindGroupState<A: HalApi> {
buffers: Mutex<Vec<(Arc<Buffer<A>>, BufferUses)>>,
_phantom: PhantomData<A>,
}
impl<A: HalApi> BufferBindGroupState<A> {
pub fn new() -> Self {
Self {
buffers: Mutex::new(rank::BUFFER_BIND_GROUP_STATE_BUFFERS, Vec::new()),
_phantom: PhantomData,
}
}
#[allow(clippy::pattern_type_mismatch)]
pub(crate) fn optimize(&self) {
let mut buffers = self.buffers.lock();
buffers.sort_unstable_by_key(|(b, _)| b.as_info().tracker_index());
}
#[allow(clippy::pattern_type_mismatch)]
pub fn used_tracker_indices(&self) -> impl Iterator<Item = TrackerIndex> + '_ {
let buffers = self.buffers.lock();
buffers
.iter()
.map(|(ref b, _)| b.as_info().tracker_index())
.collect::<Vec<_>>()
.into_iter()
}
pub fn drain_resources(&self) -> impl Iterator<Item = Arc<Buffer<A>>> + '_ {
let mut buffers = self.buffers.lock();
buffers
.drain(..)
.map(|(buffer, _u)| buffer)
.collect::<Vec<_>>()
.into_iter()
}
pub fn add_single<'a>(
&self,
storage: &'a Storage<Buffer<A>>,
id: BufferId,
state: BufferUses,
) -> Option<&'a Arc<Buffer<A>>> {
let buffer = storage.get(id).ok()?;
let mut buffers = self.buffers.lock();
buffers.push((buffer.clone(), state));
Some(buffer)
}
}
#[derive(Debug)]
pub(crate) struct BufferUsageScope<A: HalApi> {
state: Vec<BufferUses>,
metadata: ResourceMetadata<Buffer<A>>,
}
impl<A: HalApi> Default for BufferUsageScope<A> {
fn default() -> Self {
Self {
state: Vec::new(),
metadata: ResourceMetadata::new(),
}
}
}
impl<A: HalApi> BufferUsageScope<A> {
fn tracker_assert_in_bounds(&self, index: usize) {
strict_assert!(index < self.state.len());
self.metadata.tracker_assert_in_bounds(index);
}
pub fn clear(&mut self) {
self.state.clear();
self.metadata.clear();
}
pub fn set_size(&mut self, size: usize) {
self.state.resize(size, BufferUses::empty());
self.metadata.set_size(size);
}
fn allow_index(&mut self, index: usize) {
if index >= self.state.len() {
self.set_size(index + 1);
}
}
pub fn drain_resources(&mut self) -> impl Iterator<Item = Arc<Buffer<A>>> + '_ {
let resources = self.metadata.drain_resources();
self.state.clear();
resources.into_iter()
}
pub unsafe fn merge_bind_group(
&mut self,
bind_group: &BufferBindGroupState<A>,
) -> Result<(), UsageConflict> {
let buffers = bind_group.buffers.lock();
for &(ref resource, state) in &*buffers {
let index = resource.as_info().tracker_index().as_usize();
unsafe {
insert_or_merge(
None,
&mut self.state,
&mut self.metadata,
index as _,
index,
BufferStateProvider::Direct { state },
ResourceMetadataProvider::Direct {
resource: Cow::Borrowed(resource),
},
)?
};
}
Ok(())
}
pub fn merge_usage_scope(&mut self, scope: &Self) -> Result<(), UsageConflict> {
let incoming_size = scope.state.len();
if incoming_size > self.state.len() {
self.set_size(incoming_size);
}
for index in scope.metadata.owned_indices() {
self.tracker_assert_in_bounds(index);
scope.tracker_assert_in_bounds(index);
unsafe {
insert_or_merge(
None,
&mut self.state,
&mut self.metadata,
index as u32,
index,
BufferStateProvider::Indirect {
state: &scope.state,
},
ResourceMetadataProvider::Indirect {
metadata: &scope.metadata,
},
)?;
};
}
Ok(())
}
pub fn merge_single<'a>(
&mut self,
storage: &'a Storage<Buffer<A>>,
id: BufferId,
new_state: BufferUses,
) -> Result<&'a Arc<Buffer<A>>, UsageConflict> {
let buffer = storage
.get(id)
.map_err(|_| UsageConflict::BufferInvalid { id })?;
self.insert_merge_single(buffer.clone(), new_state)
.map(|_| buffer)
}
pub fn insert_merge_single(
&mut self,
buffer: Arc<Buffer<A>>,
new_state: BufferUses,
) -> Result<(), UsageConflict> {
let index = buffer.info.tracker_index().as_usize();
self.allow_index(index);
self.tracker_assert_in_bounds(index);
unsafe {
insert_or_merge(
None,
&mut self.state,
&mut self.metadata,
index as _,
index,
BufferStateProvider::Direct { state: new_state },
ResourceMetadataProvider::Direct {
resource: Cow::Owned(buffer),
},
)?;
}
Ok(())
}
}
pub(crate) type SetSingleResult<A> =
Option<(Arc<Buffer<A>>, Option<PendingTransition<BufferUses>>)>;
pub(crate) struct BufferTracker<A: HalApi> {
start: Vec<BufferUses>,
end: Vec<BufferUses>,
metadata: ResourceMetadata<Buffer<A>>,
temp: Vec<PendingTransition<BufferUses>>,
}
impl<A: HalApi> ResourceTracker for BufferTracker<A> {
fn remove_abandoned(&mut self, index: TrackerIndex) -> bool {
let index = index.as_usize();
if index > self.metadata.size() {
return false;
}
self.tracker_assert_in_bounds(index);
unsafe {
if self.metadata.contains_unchecked(index) {
let existing_ref_count = self.metadata.get_ref_count_unchecked(index);
if existing_ref_count <= 2 {
self.metadata.remove(index);
return true;
}
return false;
}
}
true
}
}
impl<A: HalApi> BufferTracker<A> {
pub fn new() -> Self {
Self {
start: Vec::new(),
end: Vec::new(),
metadata: ResourceMetadata::new(),
temp: Vec::new(),
}
}
fn tracker_assert_in_bounds(&self, index: usize) {
strict_assert!(index < self.start.len());
strict_assert!(index < self.end.len());
self.metadata.tracker_assert_in_bounds(index);
}
pub fn set_size(&mut self, size: usize) {
self.start.resize(size, BufferUses::empty());
self.end.resize(size, BufferUses::empty());
self.metadata.set_size(size);
}
fn allow_index(&mut self, index: usize) {
if index >= self.start.len() {
self.set_size(index + 1);
}
}
pub fn used_resources(&self) -> impl Iterator<Item = Arc<Buffer<A>>> + '_ {
self.metadata.owned_resources()
}
pub fn drain_transitions<'a, 'b: 'a>(
&'b mut self,
snatch_guard: &'a SnatchGuard<'a>,
) -> impl Iterator<Item = BufferBarrier<'a, A>> {
let buffer_barriers = self.temp.drain(..).map(|pending| {
let buf = unsafe { self.metadata.get_resource_unchecked(pending.id as _) };
pending.into_hal(buf, snatch_guard)
});
buffer_barriers
}
pub fn insert_single(&mut self, resource: Arc<Buffer<A>>, state: BufferUses) {
let index = resource.info.tracker_index().as_usize();
self.allow_index(index);
self.tracker_assert_in_bounds(index);
unsafe {
let currently_owned = self.metadata.contains_unchecked(index);
if currently_owned {
panic!("Tried to insert buffer already tracked");
}
insert(
Some(&mut self.start),
&mut self.end,
&mut self.metadata,
index,
BufferStateProvider::Direct { state },
None,
ResourceMetadataProvider::Direct {
resource: Cow::Owned(resource),
},
)
}
}
pub fn set_single(&mut self, buffer: &Arc<Buffer<A>>, state: BufferUses) -> SetSingleResult<A> {
let index: usize = buffer.as_info().tracker_index().as_usize();
self.allow_index(index);
self.tracker_assert_in_bounds(index);
unsafe {
insert_or_barrier_update(
Some(&mut self.start),
&mut self.end,
&mut self.metadata,
index,
BufferStateProvider::Direct { state },
None,
ResourceMetadataProvider::Direct {
resource: Cow::Owned(buffer.clone()),
},
&mut self.temp,
)
};
strict_assert!(self.temp.len() <= 1);
Some((buffer.clone(), self.temp.pop()))
}
pub fn set_from_tracker(&mut self, tracker: &Self) {
let incoming_size = tracker.start.len();
if incoming_size > self.start.len() {
self.set_size(incoming_size);
}
for index in tracker.metadata.owned_indices() {
self.tracker_assert_in_bounds(index);
tracker.tracker_assert_in_bounds(index);
unsafe {
insert_or_barrier_update(
Some(&mut self.start),
&mut self.end,
&mut self.metadata,
index,
BufferStateProvider::Indirect {
state: &tracker.start,
},
Some(BufferStateProvider::Indirect {
state: &tracker.end,
}),
ResourceMetadataProvider::Indirect {
metadata: &tracker.metadata,
},
&mut self.temp,
)
}
}
}
pub fn set_from_usage_scope(&mut self, scope: &BufferUsageScope<A>) {
let incoming_size = scope.state.len();
if incoming_size > self.start.len() {
self.set_size(incoming_size);
}
for index in scope.metadata.owned_indices() {
self.tracker_assert_in_bounds(index);
scope.tracker_assert_in_bounds(index);
unsafe {
insert_or_barrier_update(
Some(&mut self.start),
&mut self.end,
&mut self.metadata,
index,
BufferStateProvider::Indirect {
state: &scope.state,
},
None,
ResourceMetadataProvider::Indirect {
metadata: &scope.metadata,
},
&mut self.temp,
)
}
}
}
pub unsafe fn set_and_remove_from_usage_scope_sparse(
&mut self,
scope: &mut BufferUsageScope<A>,
index_source: impl IntoIterator<Item = TrackerIndex>,
) {
let incoming_size = scope.state.len();
if incoming_size > self.start.len() {
self.set_size(incoming_size);
}
for index in index_source {
let index = index.as_usize();
scope.tracker_assert_in_bounds(index);
if unsafe { !scope.metadata.contains_unchecked(index) } {
continue;
}
unsafe {
insert_or_barrier_update(
Some(&mut self.start),
&mut self.end,
&mut self.metadata,
index,
BufferStateProvider::Indirect {
state: &scope.state,
},
None,
ResourceMetadataProvider::Indirect {
metadata: &scope.metadata,
},
&mut self.temp,
)
};
unsafe { scope.metadata.remove(index) };
}
}
#[allow(dead_code)]
pub fn get(&self, index: TrackerIndex) -> Option<&Arc<Buffer<A>>> {
let index = index.as_usize();
if index > self.metadata.size() {
return None;
}
self.tracker_assert_in_bounds(index);
unsafe {
if self.metadata.contains_unchecked(index) {
return Some(self.metadata.get_resource_unchecked(index));
}
}
None
}
}
#[derive(Debug, Clone)]
enum BufferStateProvider<'a> {
Direct { state: BufferUses },
Indirect { state: &'a [BufferUses] },
}
impl BufferStateProvider<'_> {
#[inline(always)]
unsafe fn get_state(&self, index: usize) -> BufferUses {
match *self {
BufferStateProvider::Direct { state } => state,
BufferStateProvider::Indirect { state } => {
strict_assert!(index < state.len());
*unsafe { state.get_unchecked(index) }
}
}
}
}
#[inline(always)]
unsafe fn insert_or_merge<A: HalApi>(
start_states: Option<&mut [BufferUses]>,
current_states: &mut [BufferUses],
resource_metadata: &mut ResourceMetadata<Buffer<A>>,
index32: u32,
index: usize,
state_provider: BufferStateProvider<'_>,
metadata_provider: ResourceMetadataProvider<'_, Buffer<A>>,
) -> Result<(), UsageConflict> {
let currently_owned = unsafe { resource_metadata.contains_unchecked(index) };
if !currently_owned {
unsafe {
insert(
start_states,
current_states,
resource_metadata,
index,
state_provider,
None,
metadata_provider,
)
};
return Ok(());
}
unsafe {
merge(
current_states,
index32,
index,
state_provider,
metadata_provider,
)
}
}
#[inline(always)]
unsafe fn insert_or_barrier_update<A: HalApi>(
start_states: Option<&mut [BufferUses]>,
current_states: &mut [BufferUses],
resource_metadata: &mut ResourceMetadata<Buffer<A>>,
index: usize,
start_state_provider: BufferStateProvider<'_>,
end_state_provider: Option<BufferStateProvider<'_>>,
metadata_provider: ResourceMetadataProvider<'_, Buffer<A>>,
barriers: &mut Vec<PendingTransition<BufferUses>>,
) {
let currently_owned = unsafe { resource_metadata.contains_unchecked(index) };
if !currently_owned {
unsafe {
insert(
start_states,
current_states,
resource_metadata,
index,
start_state_provider,
end_state_provider,
metadata_provider,
)
};
return;
}
let update_state_provider = end_state_provider.unwrap_or_else(|| start_state_provider.clone());
unsafe { barrier(current_states, index, start_state_provider, barriers) };
unsafe { update(current_states, index, update_state_provider) };
}
#[inline(always)]
unsafe fn insert<A: HalApi>(
start_states: Option<&mut [BufferUses]>,
current_states: &mut [BufferUses],
resource_metadata: &mut ResourceMetadata<Buffer<A>>,
index: usize,
start_state_provider: BufferStateProvider<'_>,
end_state_provider: Option<BufferStateProvider<'_>>,
metadata_provider: ResourceMetadataProvider<'_, Buffer<A>>,
) {
let new_start_state = unsafe { start_state_provider.get_state(index) };
let new_end_state =
end_state_provider.map_or(new_start_state, |p| unsafe { p.get_state(index) });
strict_assert_eq!(invalid_resource_state(new_start_state), false);
strict_assert_eq!(invalid_resource_state(new_end_state), false);
log::trace!("\tbuf {index}: insert {new_start_state:?}..{new_end_state:?}");
unsafe {
if let Some(&mut ref mut start_state) = start_states {
*start_state.get_unchecked_mut(index) = new_start_state;
}
*current_states.get_unchecked_mut(index) = new_end_state;
let resource = metadata_provider.get_own(index);
resource_metadata.insert(index, resource);
}
}
#[inline(always)]
unsafe fn merge<A: HalApi>(
current_states: &mut [BufferUses],
index32: u32,
index: usize,
state_provider: BufferStateProvider<'_>,
metadata_provider: ResourceMetadataProvider<'_, Buffer<A>>,
) -> Result<(), UsageConflict> {
let current_state = unsafe { current_states.get_unchecked_mut(index) };
let new_state = unsafe { state_provider.get_state(index) };
let merged_state = *current_state | new_state;
if invalid_resource_state(merged_state) {
return Err(UsageConflict::from_buffer(
unsafe { metadata_provider.get_own(index).info.id() },
*current_state,
new_state,
));
}
log::trace!("\tbuf {index32}: merge {current_state:?} + {new_state:?}");
*current_state = merged_state;
Ok(())
}
#[inline(always)]
unsafe fn barrier(
current_states: &mut [BufferUses],
index: usize,
state_provider: BufferStateProvider<'_>,
barriers: &mut Vec<PendingTransition<BufferUses>>,
) {
let current_state = unsafe { *current_states.get_unchecked(index) };
let new_state = unsafe { state_provider.get_state(index) };
if skip_barrier(current_state, new_state) {
return;
}
barriers.push(PendingTransition {
id: index as _,
selector: (),
usage: current_state..new_state,
});
log::trace!("\tbuf {index}: transition {current_state:?} -> {new_state:?}");
}
#[inline(always)]
unsafe fn update(
current_states: &mut [BufferUses],
index: usize,
state_provider: BufferStateProvider<'_>,
) {
let current_state = unsafe { current_states.get_unchecked_mut(index) };
let new_state = unsafe { state_provider.get_state(index) };
*current_state = new_state;
}