use super::super::{
BufferHandle, ComputePipelineHandle, ContextHandle, DeviceHandle, PipelineHandle, RenderTargetHandle,
SamplerHandle, ShaderHandle, SurfaceHandle, TextureHandle,
};
use crate::timeline::SmallContextMap;
use crate::types::{DepthFormat, SamplerDesc, TextureFormat};
use rustc_hash::FxHashMap;
use std::collections::HashMap;
use std::sync::{
atomic::{AtomicU64, Ordering},
Arc, Mutex, RwLock,
};
use windows::Win32::Foundation::HANDLE;
use windows::Win32::Graphics::{Direct3D12, Dxgi};
#[derive(Clone, Copy)]
pub(crate) struct SendSyncHandle(pub HANDLE);
unsafe impl Send for SendSyncHandle {}
unsafe impl Sync for SendSyncHandle {}
#[allow(dead_code)]
pub const MAX_BINDLESS_CBV_SRV_UAV: u32 = 16384;
#[allow(dead_code)]
pub const MAX_BINDLESS_SAMPLERS: u32 = 2048;
pub use super::super::shared::{PushLayout, TOTAL_PUSH_BYTES};
#[derive(Default)]
pub(crate) struct ResourceRegistry {
cbv_srv_uav: super::super::shared::SlotAllocator,
sampler: super::super::shared::SlotAllocator,
pub buffer_offsets: HashMap<BufferHandle, u32>,
pub buffer_srv_offsets: HashMap<BufferHandle, u32>,
pub texture_offsets: HashMap<TextureHandle, u32>,
pub texture_uav_offsets: HashMap<TextureHandle, u32>,
pub sampler_offsets: HashMap<SamplerHandle, u32>,
}
#[allow(dead_code)]
impl ResourceRegistry {
pub fn new() -> Self {
Self::default()
}
pub fn register_buffer_cbv(&mut self, handle: BufferHandle) -> u32 {
let offset = self.cbv_srv_uav.alloc();
self.buffer_offsets.insert(handle, offset);
offset
}
pub fn register_buffer_srv(&mut self, handle: BufferHandle) -> u32 {
let offset = self.cbv_srv_uav.alloc();
self.buffer_srv_offsets.insert(handle, offset);
offset
}
pub fn register_buffer_uav(&mut self, handle: BufferHandle) -> u32 {
let offset = self.cbv_srv_uav.alloc();
self.buffer_offsets.insert(handle, offset);
offset
}
pub fn register_texture(&mut self, handle: TextureHandle) -> u32 {
let offset = self.cbv_srv_uav.alloc();
self.texture_offsets.insert(handle, offset);
offset
}
pub fn register_texture_uav(&mut self, handle: TextureHandle) -> u32 {
let offset = self.cbv_srv_uav.alloc();
self.texture_uav_offsets.insert(handle, offset);
offset
}
pub fn register_texture_srv(&mut self, _handle: TextureHandle) -> u32 {
self.cbv_srv_uav.alloc()
}
pub fn register_sampler(&mut self, handle: SamplerHandle) -> u32 {
let offset = self.sampler.alloc();
self.sampler_offsets.insert(handle, offset);
offset
}
pub fn ensure_cbv_start(&mut self, start: u32) {
self.cbv_srv_uav.ensure_minimum_next(start);
}
pub fn get_buffer_srv_offset(&self, handle: BufferHandle) -> Option<u32> {
self.buffer_srv_offsets.get(&handle).copied()
}
pub fn alloc_cbv_srv_uav_slot(&mut self) -> u32 {
self.cbv_srv_uav.alloc()
}
pub fn extract_buffer_slots(&mut self, handle: BufferHandle) -> Vec<u32> {
let mut slots = Vec::new();
if let Some(offset) = self.buffer_offsets.remove(&handle) {
slots.push(offset);
}
if let Some(offset) = self.buffer_srv_offsets.remove(&handle) {
slots.push(offset);
}
slots
}
pub fn extract_texture_slots(&mut self, handle: TextureHandle) -> Vec<u32> {
let mut slots = Vec::new();
if let Some(offset) = self.texture_offsets.remove(&handle) {
slots.push(offset);
}
if let Some(offset) = self.texture_uav_offsets.remove(&handle) {
slots.push(offset);
}
slots
}
pub fn free_cbv_srv_uav_slot(&mut self, slot: u32) {
self.cbv_srv_uav.free(slot);
}
pub fn extract_sampler_slots(&mut self, handle: SamplerHandle) -> Vec<DeferredSlot> {
if let Some(offset) = self.sampler_offsets.remove(&handle) {
vec![DeferredSlot::Sampler(offset)]
} else {
Vec::new()
}
}
pub fn free_sampler_slot(&mut self, slot: u32) {
self.sampler.free(slot);
}
pub fn free_deferred_slot(&mut self, slot: DeferredSlot) {
match slot {
DeferredSlot::CbvSrvUav(s) => self.cbv_srv_uav.free(s),
DeferredSlot::Sampler(s) => self.sampler.free(s),
}
}
pub fn unregister_sampler(&mut self, handle: SamplerHandle) {
if let Some(offset) = self.sampler_offsets.remove(&handle) {
self.sampler.free(offset);
}
}
pub fn available_slots(&self, category: crate::types::ResourceCategory) -> u32 {
match category {
crate::types::ResourceCategory::Sampler => MAX_BINDLESS_SAMPLERS.saturating_sub(self.sampler.live_count()),
_ => MAX_BINDLESS_CBV_SRV_UAV.saturating_sub(self.cbv_srv_uav.live_count()),
}
}
pub fn max_slots(category: crate::types::ResourceCategory) -> u32 {
match category {
crate::types::ResourceCategory::Sampler => MAX_BINDLESS_SAMPLERS,
_ => MAX_BINDLESS_CBV_SRV_UAV,
}
}
}
#[cfg(test)]
#[allow(clippy::items_after_test_module)]
mod registry_tests {
use super::*;
fn free_buffer_slots(reg: &mut ResourceRegistry, handle: BufferHandle) {
for slot in reg.extract_buffer_slots(handle) {
reg.free_cbv_srv_uav_slot(slot);
}
}
fn free_texture_slots(reg: &mut ResourceRegistry, handle: TextureHandle) {
for slot in reg.extract_texture_slots(handle) {
reg.free_cbv_srv_uav_slot(slot);
}
}
#[test]
fn buffer_slots_recycled_under_churn() {
let mut reg = ResourceRegistry::new();
for i in 0..50_000u64 {
let handle = i as BufferHandle;
reg.register_buffer_uav(handle);
free_buffer_slots(&mut reg, handle);
}
assert_eq!(
reg.cbv_srv_uav.next_fresh(),
1,
"UAV counter grew; slot recycling not working"
);
assert_eq!(reg.cbv_srv_uav.free_count(), 1);
}
#[test]
fn storage_buffer_dual_slot_recycled() {
let mut reg = ResourceRegistry::new();
for i in 0..1_000u64 {
let handle = i as BufferHandle;
reg.register_buffer_uav(handle);
reg.register_buffer_srv(handle);
free_buffer_slots(&mut reg, handle);
}
assert_eq!(
reg.cbv_srv_uav.next_fresh(),
2,
"counter should only have advanced twice (one UAV + one SRV slot ever minted)"
);
assert_eq!(reg.cbv_srv_uav.free_count(), 2, "both slots must be in the free list");
}
#[test]
fn texture_dual_slot_recycled() {
let mut reg = ResourceRegistry::new();
for i in 0..1_000u64 {
let handle = i as TextureHandle;
reg.register_texture(handle);
reg.register_texture_uav(handle);
free_texture_slots(&mut reg, handle);
}
assert_eq!(reg.cbv_srv_uav.next_fresh(), 2);
assert_eq!(reg.cbv_srv_uav.free_count(), 2);
}
#[test]
fn sampler_slots_recycled() {
let mut reg = ResourceRegistry::new();
for i in 0..5_000u64 {
let handle = i as SamplerHandle;
reg.register_sampler(handle);
reg.unregister_sampler(handle);
}
assert_eq!(reg.sampler.next_fresh(), 1);
assert_eq!(reg.sampler.free_count(), 1);
}
#[test]
fn live_resources_get_distinct_slots() {
let mut reg = ResourceRegistry::new();
const N: u64 = 64;
let mut slots: Vec<u32> = (0..N).map(|i| reg.register_buffer_uav(i as BufferHandle)).collect();
slots.sort_unstable();
slots.dedup();
assert_eq!(slots.len(), N as usize, "duplicate slots assigned to live resources");
}
#[test]
fn high_water_mark_bounded_by_live_count() {
let mut reg = ResourceRegistry::new();
const LIVE: u64 = 8;
const ROUNDS: u64 = 10_000;
for i in 0..LIVE {
reg.register_buffer_uav(i as BufferHandle);
}
for i in LIVE..LIVE + ROUNDS {
free_buffer_slots(&mut reg, (i - LIVE) as BufferHandle);
reg.register_buffer_uav(i as BufferHandle);
}
assert!(
reg.cbv_srv_uav.next_fresh() <= LIVE as u32,
"counter ({}) exceeded live count ({LIVE}); slot recycling broken",
reg.cbv_srv_uav.next_fresh()
);
}
#[test]
fn slot_deferred_until_context_retires() {
use crate::backend::ContextHandle;
let mut reg = ResourceRegistry::new();
let handle = 1u64 as BufferHandle;
let slot = reg.register_buffer_uav(handle);
const CTX_A: ContextHandle = 10;
const SEQ: u64 = 5;
let slots = reg.extract_buffer_slots(handle);
assert_eq!(slots, vec![slot]);
let mut pending = vec![PendingSlotReclamation {
slot: DeferredSlot::CbvSrvUav(slot),
requirements: vec![(CTX_A, SEQ)],
}];
assert_eq!(reg.cbv_srv_uav.free_count(), 0, "slot must not be freed yet");
let mut retired = HashMap::from([(CTX_A, 4u64)]);
let drain_pending = |retired: &HashMap<ContextHandle, u64>,
reg: &mut ResourceRegistry,
pending: &mut Vec<PendingSlotReclamation>| {
let mut i = 0;
while i < pending.len() {
let ready = pending[i]
.requirements
.iter()
.all(|(ctx, seq)| retired.get(ctx).copied().unwrap_or(0) >= *seq);
if ready {
let entry = pending.swap_remove(i);
reg.free_deferred_slot(entry.slot);
} else {
i += 1;
}
}
};
drain_pending(&retired, &mut reg, &mut pending);
assert_eq!(reg.cbv_srv_uav.free_count(), 0, "still in flight at seq 4");
retired.insert(CTX_A, SEQ);
drain_pending(&retired, &mut reg, &mut pending);
assert_eq!(reg.cbv_srv_uav.free_count(), 1, "slot freed after context retires");
}
#[test]
fn slot_waits_for_all_referencing_contexts() {
use crate::backend::ContextHandle;
let mut reg = ResourceRegistry::new();
let handle = 2u64 as BufferHandle;
let slot = reg.register_buffer_uav(handle);
const CTX_A: ContextHandle = 1;
const CTX_B: ContextHandle = 2;
let mut pending = vec![PendingSlotReclamation {
slot: DeferredSlot::CbvSrvUav(slot),
requirements: vec![(CTX_A, 3), (CTX_B, 7)],
}];
reg.extract_buffer_slots(handle);
let mut retired = HashMap::from([(CTX_A, 0u64), (CTX_B, 0u64)]);
let drain_pending = |retired: &HashMap<ContextHandle, u64>,
reg: &mut ResourceRegistry,
pending: &mut Vec<PendingSlotReclamation>| {
let mut i = 0;
while i < pending.len() {
let ready = pending[i]
.requirements
.iter()
.all(|(ctx, seq)| retired.get(ctx).copied().unwrap_or(0) >= *seq);
if ready {
let entry = pending.swap_remove(i);
reg.free_deferred_slot(entry.slot);
} else {
i += 1;
}
}
};
drain_pending(&retired, &mut reg, &mut pending);
assert_eq!(reg.cbv_srv_uav.free_count(), 0);
retired.insert(CTX_A, 3);
drain_pending(&retired, &mut reg, &mut pending);
assert_eq!(reg.cbv_srv_uav.free_count(), 0, "CTX_B still in flight");
retired.insert(CTX_B, 7);
drain_pending(&retired, &mut reg, &mut pending);
assert_eq!(reg.cbv_srv_uav.free_count(), 1);
}
#[test]
fn retained_pin_blocks_slot_free_until_unpin() {
let mut dr = DescriptorRegistry::new();
let handle = 3u64 as BufferHandle;
let slot = dr.resource_registry.register_buffer_uav(handle);
let deferred = DeferredSlot::CbvSrvUav(slot);
dr.pin_retained_slots([deferred]);
dr.queue_slot_reclamation(deferred);
dr.resource_registry.extract_buffer_slots(handle);
let fences: HashMap<ContextHandle, ContextFenceEntry> = HashMap::new();
dr.drain_ready_slot_reclamations(&fences);
assert_eq!(
dr.resource_registry.cbv_srv_uav.free_count(),
0,
"pinned slot must not free"
);
assert_eq!(dr.retained_user_count(deferred), 1);
dr.unpin_retained_slots([deferred]);
dr.drain_ready_slot_reclamations(&fences);
assert_eq!(
dr.resource_registry.cbv_srv_uav.free_count(),
1,
"unpin then drain frees"
);
}
#[test]
fn retained_pin_shared_slot_needs_two_unpins() {
let mut dr = DescriptorRegistry::new();
let slot = dr.resource_registry.register_buffer_uav(4);
let deferred = DeferredSlot::CbvSrvUav(slot);
dr.pin_retained_slots([deferred]);
dr.pin_retained_slots([deferred]);
dr.queue_slot_reclamation(deferred);
dr.resource_registry.buffer_offsets.remove(&4);
let fences: HashMap<ContextHandle, ContextFenceEntry> = HashMap::new();
dr.drain_ready_slot_reclamations(&fences);
assert_eq!(dr.resource_registry.cbv_srv_uav.free_count(), 0);
dr.unpin_retained_slots([deferred]);
dr.drain_ready_slot_reclamations(&fences);
assert_eq!(dr.resource_registry.cbv_srv_uav.free_count(), 0, "one pin remains");
dr.unpin_retained_slots([deferred]);
dr.drain_ready_slot_reclamations(&fences);
assert_eq!(dr.resource_registry.cbv_srv_uav.free_count(), 1);
}
#[test]
fn retained_pin_replace_frees_old_only_slots() {
let mut dr = DescriptorRegistry::new();
let slot_old = dr.resource_registry.register_buffer_uav(5);
let slot_new = dr.resource_registry.register_buffer_uav(6);
let old_deferred = DeferredSlot::CbvSrvUav(slot_old);
let new_deferred = DeferredSlot::CbvSrvUav(slot_new);
dr.pin_retained_slots([old_deferred]);
dr.pin_retained_slots([new_deferred]);
dr.unpin_retained_slots([old_deferred]);
dr.queue_slot_reclamation(old_deferred);
dr.queue_slot_reclamation(new_deferred);
dr.resource_registry.buffer_offsets.remove(&5);
dr.resource_registry.buffer_offsets.remove(&6);
let fences: HashMap<ContextHandle, ContextFenceEntry> = HashMap::new();
dr.drain_ready_slot_reclamations(&fences);
assert_eq!(dr.resource_registry.cbv_srv_uav.free_count(), 1, "old slot freed");
assert_eq!(dr.retained_user_count(new_deferred), 1);
assert_eq!(dr.retained_user_count(old_deferred), 0);
}
#[test]
fn retained_pin_blocks_gpu_release_readiness() {
let mut dr = DescriptorRegistry::new();
let slot = dr.resource_registry.register_buffer_uav(7);
let deferred = DeferredSlot::CbvSrvUav(slot);
dr.pin_retained_slots([deferred]);
assert!(!dr.retained_pins_clear(&[deferred]));
dr.unpin_retained_slots([deferred]);
assert!(dr.retained_pins_clear(&[deferred]));
}
}
#[allow(dead_code)]
#[derive(Clone)]
pub(crate) struct DxgiAdapterInfo {
pub adapter: Dxgi::IDXGIAdapter1,
pub desc: Dxgi::DXGI_ADAPTER_DESC1,
pub adapter_id: u32,
pub supports_reserved_buffers: bool,
}
#[allow(dead_code)]
pub(crate) struct ComputeAllocatorSlot {
pub allocator: Direct3D12::ID3D12CommandAllocator,
pub fence_value: u64,
pub command_list: Option<Direct3D12::ID3D12GraphicsCommandList>,
pub retained: bool,
pub pre_reset: bool,
pub in_recording: bool,
}
pub(crate) struct Dx12SubmissionContext {
pub device: super::DeviceHandle,
pub fence: Direct3D12::ID3D12Fence,
pub command_queue: Direct3D12::ID3D12CommandQueue,
pub queue_lock: Arc<Mutex<()>>,
pub last_submitted_seq: Arc<AtomicU64>,
pub signal_queue: std::sync::Arc<crate::signal::SignalQueue>,
pub fence_shutdown: std::sync::Arc<std::sync::atomic::AtomicBool>,
pub fence_thread: Option<std::thread::JoinHandle<()>>,
pub compute_allocator_pool: Vec<ComputeAllocatorSlot>,
pub allocator_recycle_hint: usize,
pub retained_graphs: HashMap<u64, RetainedGraph>,
pub staging_belt: super::staging::StagingBelt,
pub texture_staging_pool: super::staging::TextureStagingPool,
pub deletion_queue: DeletionQueue,
pub reclamation_context: Option<(std::thread::ThreadId, u64)>,
pub frame_table: SharedContextFrameTable,
pub pending_gpu_profiles: Vec<(u64, super::compute::Dx12GpuProfileResources)>,
}
pub(crate) struct RetainedGraph {
pub command_list: Direct3D12::ID3D12GraphicsCommandList,
pub slot_idx: usize,
pub on_device_queue: bool,
pub used_slots: Arc<[DeferredSlot]>,
pub frame_table_staging: Option<std::sync::Arc<[u32]>>,
pub frame_table_row: Option<u32>,
}
#[allow(dead_code)]
pub(crate) enum PendingDeletion {
Buffer {
buffer_handle: BufferHandle,
resource: Direct3D12::ID3D12Resource,
upload_buffer: Option<Direct3D12::ID3D12Resource>,
coherent_readback: Option<Direct3D12::ID3D12Resource>,
reserved_tiles: Option<Vec<Option<(Direct3D12::ID3D12Heap, u64)>>>,
},
ReplacedBufferGpu {
resource: Direct3D12::ID3D12Resource,
upload_buffer: Option<Direct3D12::ID3D12Resource>,
coherent_readback: Option<Direct3D12::ID3D12Resource>,
},
ReplacedReservedBufferGpu {
resource: Direct3D12::ID3D12Resource,
tiles: Vec<Option<(Direct3D12::ID3D12Heap, u64)>>,
upload_buffer: Option<Direct3D12::ID3D12Resource>,
coherent_readback: Option<Direct3D12::ID3D12Resource>,
},
BufferView { buffer_handle: BufferHandle },
Texture {
texture_handle: TextureHandle,
resource: Direct3D12::ID3D12Resource,
},
StandaloneResource(Direct3D12::ID3D12Resource),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(crate) enum DeferredSlot {
CbvSrvUav(u32),
Sampler(u32),
}
pub(crate) struct PendingSlotReclamation {
pub slot: DeferredSlot,
pub requirements: Vec<(super::ContextHandle, u64)>,
}
pub(crate) struct PendingBufferGpuRelease {
pub requirements: Vec<(super::ContextHandle, u64)>,
pub retained_slots: Vec<DeferredSlot>,
pub resource: Direct3D12::ID3D12Resource,
pub upload_buffer: Option<Direct3D12::ID3D12Resource>,
pub coherent_readback: Option<Direct3D12::ID3D12Resource>,
pub reserved_tiles: Option<Vec<Option<(Direct3D12::ID3D12Heap, u64)>>>,
}
pub(crate) type ContextFenceEntry = (DeviceHandle, Direct3D12::ID3D12Fence, Arc<AtomicU64>);
pub(crate) type SharedContextFences = Arc<std::sync::RwLock<HashMap<super::ContextHandle, ContextFenceEntry>>>;
fn slot_requirements_met(
requirements: &[(super::ContextHandle, u64)],
context_fences: &HashMap<super::ContextHandle, ContextFenceEntry>,
) -> bool {
requirements.iter().all(|(ctx_id, required_seq)| {
context_fences
.get(ctx_id)
.is_none_or(|(_, fence, _)| unsafe { fence.GetCompletedValue() >= *required_seq })
})
}
fn device_deletion_requirements_met(
requirements: &[(super::ContextHandle, u64)],
context_fences: &HashMap<super::ContextHandle, ContextFenceEntry>,
) -> bool {
if requirements.is_empty() {
return context_fences.is_empty();
}
slot_requirements_met(requirements, context_fences)
}
pub(crate) struct DeletionQueue {
inner: super::super::shared::DeferredQueue<u64, PendingDeletion>,
}
impl DeletionQueue {
pub fn new() -> Self {
Self {
inner: super::super::shared::DeferredQueue::new(),
}
}
pub fn queue(&mut self, fence_value: u64, resource: PendingDeletion) {
self.inner.push(fence_value, resource);
}
pub(crate) fn drain_up_to_completed(&mut self, completed: u64) -> Vec<PendingDeletion> {
self.inner.drain_up_to(completed)
}
pub(crate) fn drain_everything(&mut self) -> Vec<PendingDeletion> {
self.inner.flush_all().collect()
}
pub(crate) fn pending_len(&self) -> usize {
self.inner.len()
}
}
pub(crate) struct DeviceDeletionQueue {
inner: super::super::shared::DeferredQueue<Vec<(super::ContextHandle, u64)>, PendingDeletion>,
}
impl DeviceDeletionQueue {
pub fn new() -> Self {
Self {
inner: super::super::shared::DeferredQueue::new(),
}
}
pub fn queue(&mut self, requirements: Vec<(super::ContextHandle, u64)>, resource: PendingDeletion) {
self.inner.push(requirements, resource);
}
pub(crate) fn drain_ready(
&mut self,
context_fences: &HashMap<super::ContextHandle, ContextFenceEntry>,
) -> Vec<(Vec<(super::ContextHandle, u64)>, PendingDeletion)> {
self.inner
.drain_where_with_keys(|reqs| device_deletion_requirements_met(reqs, context_fences))
}
pub(crate) fn drain_everything(&mut self) -> Vec<(Vec<(super::ContextHandle, u64)>, PendingDeletion)> {
self.inner.drain_where_with_keys(|_| true)
}
pub(crate) fn pending_len(&self) -> usize {
self.inner.len()
}
}
pub(crate) struct DescriptorRegistry {
pub resource_registry: ResourceRegistry,
pub slot_last_seen: FxHashMap<DeferredSlot, SmallContextMap<u64>>,
pub pending_slot_reclamations: Vec<PendingSlotReclamation>,
retained_users: HashMap<DeferredSlot, u32>,
}
impl DescriptorRegistry {
pub(crate) fn new() -> Self {
Self {
resource_registry: ResourceRegistry::new(),
slot_last_seen: FxHashMap::default(),
pending_slot_reclamations: Vec::new(),
retained_users: HashMap::new(),
}
}
pub(crate) fn pin_retained_slots(&mut self, slots: impl IntoIterator<Item = DeferredSlot>) {
for slot in slots {
*self.retained_users.entry(slot).or_insert(0) += 1;
}
}
pub(crate) fn unpin_retained_slots(&mut self, slots: impl IntoIterator<Item = DeferredSlot>) {
for slot in slots {
if let Some(count) = self.retained_users.get_mut(&slot) {
*count = count.saturating_sub(1);
if *count == 0 {
self.retained_users.remove(&slot);
}
}
}
}
#[cfg(test)]
pub(crate) fn retained_user_count(&self, slot: DeferredSlot) -> u32 {
self.retained_users.get(&slot).copied().unwrap_or(0)
}
pub(crate) fn retained_pins_clear(&self, slots: &[DeferredSlot]) -> bool {
slots
.iter()
.all(|slot| self.retained_users.get(slot).copied().unwrap_or(0) == 0)
}
pub(crate) fn record_slot_usage(
&mut self,
ctx: super::ContextHandle,
seq: u64,
slots: impl IntoIterator<Item = DeferredSlot>,
) {
for slot in slots {
self.slot_last_seen.entry(slot).or_default().mark_max(ctx, seq);
}
}
pub(crate) fn queue_slot_reclamation(&mut self, slot: DeferredSlot) {
let requirements: Vec<_> = self
.slot_last_seen
.remove(&slot)
.map(|m| m.iter().collect())
.unwrap_or_default();
self.pending_slot_reclamations
.push(PendingSlotReclamation { slot, requirements });
}
pub(crate) fn reclaim_buffer_slots(&mut self, handle: BufferHandle) -> Vec<DeferredSlot> {
let slots = self.resource_registry.extract_buffer_slots(handle);
let deferred: Vec<DeferredSlot> = slots.iter().map(|&s| DeferredSlot::CbvSrvUav(s)).collect();
for slot in slots {
self.queue_slot_reclamation(DeferredSlot::CbvSrvUav(slot));
}
deferred
}
pub(crate) fn buffer_slot_keys(&self, handle: BufferHandle) -> Vec<DeferredSlot> {
let rr = &self.resource_registry;
let mut slots = Vec::new();
if let Some(&offset) = rr.buffer_offsets.get(&handle) {
slots.push(DeferredSlot::CbvSrvUav(offset));
}
if let Some(&offset) = rr.buffer_srv_offsets.get(&handle) {
slots.push(DeferredSlot::CbvSrvUav(offset));
}
slots
}
pub(crate) fn texture_slot_keys(&self, handle: TextureHandle) -> Vec<DeferredSlot> {
let rr = &self.resource_registry;
let mut slots = Vec::new();
if let Some(&offset) = rr.texture_offsets.get(&handle) {
slots.push(DeferredSlot::CbvSrvUav(offset));
}
if let Some(&offset) = rr.texture_uav_offsets.get(&handle) {
slots.push(DeferredSlot::CbvSrvUav(offset));
}
slots
}
pub(crate) fn reclaim_texture_slots(&mut self, handle: TextureHandle) {
let slots = self.resource_registry.extract_texture_slots(handle);
for slot in slots {
self.queue_slot_reclamation(DeferredSlot::CbvSrvUav(slot));
}
}
pub(crate) fn reclaim_sampler_slots(&mut self, handle: SamplerHandle) {
let slots = self.resource_registry.extract_sampler_slots(handle);
for slot in slots {
self.queue_slot_reclamation(slot);
}
}
pub(crate) fn drain_ready_slot_reclamations(&mut self, context_fences: &HashMap<ContextHandle, ContextFenceEntry>) {
let mut i = 0;
while i < self.pending_slot_reclamations.len() {
let slot = self.pending_slot_reclamations[i].slot;
let gpu_ready =
device_deletion_requirements_met(&self.pending_slot_reclamations[i].requirements, context_fences);
let pin_clear = self.retained_users.get(&slot).copied().unwrap_or(0) == 0;
if gpu_ready && pin_clear {
let entry = self.pending_slot_reclamations.swap_remove(i);
self.resource_registry.free_deferred_slot(entry.slot);
} else {
i += 1;
}
}
}
pub(crate) fn bindless_retirement_requirements_for_buffer(
&self,
handle: BufferHandle,
base: Vec<(super::ContextHandle, u64)>,
) -> Vec<(super::ContextHandle, u64)> {
let rr = &self.resource_registry;
let mut slots = Vec::new();
if let Some(&offset) = rr.buffer_offsets.get(&handle) {
slots.push(offset);
}
if let Some(&offset) = rr.buffer_srv_offsets.get(&handle) {
slots.push(offset);
}
self.merge_slot_requirements(&slots, base)
}
pub(crate) fn bindless_retirement_requirements_for_texture(
&self,
handle: TextureHandle,
base: Vec<(super::ContextHandle, u64)>,
) -> Vec<(super::ContextHandle, u64)> {
let rr = &self.resource_registry;
let mut slots = Vec::new();
if let Some(&offset) = rr.texture_offsets.get(&handle) {
slots.push(offset);
}
self.merge_slot_requirements(&slots, base)
}
fn merge_slot_requirements(
&self,
slots: &[u32],
base: Vec<(super::ContextHandle, u64)>,
) -> Vec<(super::ContextHandle, u64)> {
let mut merged: HashMap<super::ContextHandle, u64> = base.into_iter().collect();
for &slot in slots {
if let Some(map) = self.slot_last_seen.get(&DeferredSlot::CbvSrvUav(slot)) {
for (ctx, seq) in map.iter() {
merged.entry(ctx).and_modify(|v| *v = (*v).max(seq)).or_insert(seq);
}
}
}
merged.into_iter().collect()
}
}
pub(crate) struct PsoCache {
pub graphics_blobs: HashMap<u64, Vec<u8>>,
pub compute_blobs: HashMap<u64, Vec<u8>>,
pub dirty: bool,
}
impl PsoCache {
pub(crate) fn new(graphics_blobs: HashMap<u64, Vec<u8>>, compute_blobs: HashMap<u64, Vec<u8>>) -> Self {
Self {
graphics_blobs,
compute_blobs,
dirty: false,
}
}
}
pub(crate) struct DeviceDirectSlot {
pub allocator: Direct3D12::ID3D12CommandAllocator,
pub command_list: Direct3D12::ID3D12GraphicsCommandList,
pub fence_value: u64,
pub retained: bool,
}
#[allow(dead_code)]
pub(crate) struct LogicalDevice {
pub device: Direct3D12::ID3D12Device10,
pub adapter_id: u32,
pub command_queue: Direct3D12::ID3D12CommandQueue,
pub command_allocator: Direct3D12::ID3D12CommandAllocator,
pub rtv_heap: Direct3D12::ID3D12DescriptorHeap,
pub rtv_descriptor_size: u32,
pub dsv_heap: Direct3D12::ID3D12DescriptorHeap,
pub dsv_descriptor_size: u32,
pub cbv_srv_uav_heap: Direct3D12::ID3D12DescriptorHeap,
pub cbv_srv_uav_descriptor_size: u32,
pub sampler_heap: Direct3D12::ID3D12DescriptorHeap,
pub sampler_descriptor_size: u32,
pub fence: Direct3D12::ID3D12Fence,
pub timeline_next: Arc<AtomicU64>,
pub retired_floor: AtomicU64,
pub supports_reserved_buffers: bool,
pub tile_heap_pool: Mutex<Option<super::tiles::TileHeapPool>>,
pub bindless_root_signature: Option<Direct3D12::ID3D12RootSignature>,
pub compute_dispatch_indirect_signature: Option<Direct3D12::ID3D12CommandSignature>,
pub compute_batch_dispatch_signature: Option<Direct3D12::ID3D12CommandSignature>,
pub zero_buffer: Direct3D12::ID3D12Resource,
pub deletion_queue: Mutex<DeviceDeletionQueue>,
pub pending_buffer_gpu_releases: Mutex<Vec<PendingBufferGpuRelease>>,
pub device_removed: std::sync::Arc<std::sync::atomic::AtomicBool>,
pub descriptors: Arc<Mutex<DescriptorRegistry>>,
pub pso_cache: Arc<RwLock<PsoCache>>,
pub queue_lock: Arc<Mutex<()>>,
pub device_last_submitted_seq: std::sync::Arc<std::sync::atomic::AtomicU64>,
pub device_direct_pool: std::sync::Mutex<Vec<DeviceDirectSlot>>,
pub legacy_frame_table: Mutex<Option<SharedContextFrameTable>>,
pub submission_worker: std::sync::Arc<super::super::submission_worker::SubmissionWorker>,
}
pub(crate) type SharedLogicalDevice = Arc<LogicalDevice>;
pub(crate) type SharedSubmissionContext = Arc<Mutex<Dx12SubmissionContext>>;
pub(crate) type SharedContextMap = Arc<std::sync::RwLock<HashMap<super::ContextHandle, SharedSubmissionContext>>>;
pub(crate) type SharedContextFrameTable = Arc<super::frame_table::ContextFrameTable>;
impl LogicalDevice {
pub(crate) fn process_deletion_queue_up_to(&self, context_fences: &SharedContextFences) {
let batch = {
let fences = context_fences.read().unwrap();
self.deletion_queue.lock().unwrap().drain_ready(&fences)
};
if !batch.is_empty() {
let descriptors_arc = Arc::clone(&self.descriptors);
let mut registry = descriptors_arc.lock().unwrap();
for (requirements, resource) in batch {
destroy_pending_deletion(self, &mut registry, resource, requirements);
}
}
let ready = {
let fences = context_fences.read().unwrap();
let registry = self.descriptors.lock().unwrap();
self.take_ready_buffer_gpu_releases(&fences, ®istry)
};
for entry in ready {
release_buffer_gpu_resources(self, entry);
}
}
pub(crate) fn take_ready_buffer_gpu_releases(
&self,
context_fences: &HashMap<super::ContextHandle, ContextFenceEntry>,
registry: &DescriptorRegistry,
) -> Vec<PendingBufferGpuRelease> {
let mut pending = self.pending_buffer_gpu_releases.lock().unwrap();
let mut ready = Vec::new();
let mut i = 0;
while i < pending.len() {
let gpu_ready = device_deletion_requirements_met(&pending[i].requirements, context_fences);
let pin_clear = registry.retained_pins_clear(&pending[i].retained_slots);
if gpu_ready && pin_clear {
ready.push(pending.swap_remove(i));
} else {
i += 1;
}
}
ready
}
pub(crate) fn drain_pending_buffer_gpu_releases(
&self,
context_fences: &HashMap<super::ContextHandle, ContextFenceEntry>,
) {
let ready = {
let registry = self.descriptors.lock().unwrap();
self.take_ready_buffer_gpu_releases(context_fences, ®istry)
};
for entry in ready {
release_buffer_gpu_resources(self, entry);
}
}
pub(crate) fn flush_deletion_queue(&self, context_fences: &HashMap<super::ContextHandle, ContextFenceEntry>) {
let batch = self.deletion_queue.lock().unwrap().drain_everything();
if !batch.is_empty() {
let descriptors_arc = Arc::clone(&self.descriptors);
let mut registry = descriptors_arc.lock().unwrap();
for (requirements, resource) in batch {
destroy_pending_deletion(self, &mut registry, resource, requirements);
}
}
self.drain_pending_buffer_gpu_releases(context_fences);
}
}
fn release_buffer_gpu_resources(ld: &LogicalDevice, entry: PendingBufferGpuRelease) {
let PendingBufferGpuRelease {
requirements: _,
retained_slots: _,
resource,
upload_buffer,
coherent_readback,
reserved_tiles,
..
} = entry;
if let Some(tiles) = reserved_tiles {
{
let mut pool = ld.tile_heap_pool.lock().unwrap();
super::tiles::teardown_reserved_mappings(&ld.command_queue, &mut pool, &resource, &tiles);
}
let fv = ld.timeline_next.fetch_add(1, Ordering::Relaxed);
let signaled = super::utils::with_queue_lock(ld, || {
unsafe { ld.command_queue.Signal(&ld.fence, fv) }
.map_err(|e| anyhow::anyhow!("Failed to signal device fence for reserved buffer deletion: {e:?}"))
});
if signaled.is_ok() {
let _ = super::utils::wait_for_fence_on_device(&ld.fence, fv, Some(ld));
}
}
drop(resource);
drop(upload_buffer);
drop(coherent_readback);
}
pub(crate) fn destroy_pending_deletion(
ld: &LogicalDevice,
registry: &mut DescriptorRegistry,
resource: PendingDeletion,
queue_requirements: Vec<(super::ContextHandle, u64)>,
) {
match resource {
PendingDeletion::Buffer {
buffer_handle,
resource,
upload_buffer,
coherent_readback,
reserved_tiles,
} => {
let retained_slots = registry.reclaim_buffer_slots(buffer_handle);
let release = PendingBufferGpuRelease {
requirements: queue_requirements,
retained_slots,
resource,
upload_buffer,
coherent_readback,
reserved_tiles,
};
ld.pending_buffer_gpu_releases.lock().unwrap().push(release);
}
PendingDeletion::BufferView { buffer_handle } => {
registry.reclaim_buffer_slots(buffer_handle);
}
PendingDeletion::ReplacedBufferGpu {
resource,
upload_buffer,
coherent_readback,
} => {
let _ = queue_requirements;
drop(resource);
drop(upload_buffer);
drop(coherent_readback);
}
PendingDeletion::ReplacedReservedBufferGpu {
resource,
tiles,
upload_buffer,
coherent_readback,
} => {
let _ = queue_requirements;
{
let mut pool = ld.tile_heap_pool.lock().unwrap();
super::tiles::teardown_reserved_mappings(&ld.command_queue, &mut pool, &resource, &tiles);
}
let fv = ld.timeline_next.fetch_add(1, Ordering::Relaxed);
let signaled = super::utils::with_queue_lock(ld, || {
unsafe { ld.command_queue.Signal(&ld.fence, fv) }
.map_err(|e| anyhow::anyhow!("Failed to signal device fence for buffer deletion: {e:?}"))
});
if signaled.is_ok() {
let _ = super::utils::wait_for_fence(&ld.fence, fv);
}
drop(resource);
drop(upload_buffer);
drop(coherent_readback);
}
PendingDeletion::Texture {
texture_handle,
resource,
} => {
let _ = queue_requirements;
registry.reclaim_texture_slots(texture_handle);
drop(resource);
}
PendingDeletion::StandaloneResource(resource) => {
let _ = queue_requirements;
drop(resource);
}
}
}
#[derive(Clone)]
#[allow(dead_code)]
pub(crate) struct BufferState {
pub device_handle: DeviceHandle,
pub resource: Direct3D12::ID3D12Resource,
pub size: u64,
pub allocation_size: u64,
pub bindless_offset: Option<u32>,
pub bindless_srv_offset: Option<u32>,
pub is_storage: bool,
pub upload_buffer: Option<Direct3D12::ID3D12Resource>,
pub element_stride: Option<u32>,
pub is_view: bool,
pub coherent_readback: Option<Direct3D12::ID3D12Resource>,
pub coherent_readback_mapped: Option<usize>,
pub cpu_writable_upload_mapped: Option<usize>,
pub flags: crate::types::BufferFlags,
pub transient_placed: bool,
pub parent_for_view: Option<BufferHandle>,
pub view_byte_offset: Option<u64>,
pub is_reserved: bool,
pub tile_byte_size: u32,
pub reserved_tiles: Vec<Option<(Direct3D12::ID3D12Heap, u64)>>,
pub is_withdraw_staging: bool,
pub texture_copy_footprint: Option<crate::backend::TextureCopyFootprint>,
}
pub(crate) struct ShaderState {
pub device_handle: DeviceHandle,
pub slang_source: String,
pub search_paths: Vec<String>,
pub defines: Vec<(String, String)>,
pub optimization_level: crate::types::OptimizationLevel,
pub vertex_bytecode: Option<Vec<u8>>,
pub fragment_bytecode: Option<Vec<u8>>,
pub compute_bytecode: Option<Vec<u8>>,
pub reflection: Option<crate::slang::ShaderReflection>,
pub layout_checks: Vec<crate::slang::OwnedLayoutCheck>,
}
#[allow(dead_code)]
pub(crate) struct PipelineState {
pub device_handle: DeviceHandle,
pub pipeline_state: Direct3D12::ID3D12PipelineState,
pub root_signature: Direct3D12::ID3D12RootSignature,
pub vertex_stride: u32,
pub topology: crate::types::PrimitiveTopology,
pub parameter_block_layouts: Vec<crate::slang::ParameterBlockLayout>,
pub push_constant_categories: Vec<Option<crate::types::ResourceCategory>>,
pub push_constant_slot_kinds: Vec<Option<crate::types::BindlessSlotKind>>,
pub binding_element_strides: Vec<Option<u32>>,
pub shader_debug_name: String,
}
#[allow(dead_code)]
pub(crate) struct ComputePipelineState {
pub device_handle: DeviceHandle,
pub pipeline_state: Direct3D12::ID3D12PipelineState,
pub root_signature: Direct3D12::ID3D12RootSignature,
pub parameter_block_layouts: Vec<crate::slang::ParameterBlockLayout>,
pub push_constant_categories: Vec<Option<crate::types::ResourceCategory>>,
pub push_constant_slot_kinds: Vec<Option<crate::types::BindlessSlotKind>>,
pub binding_element_strides: Vec<Option<u32>>,
pub shader_debug_name: String,
}
#[allow(dead_code)]
pub(crate) struct RenderTargetState {
pub device_handle: DeviceHandle,
pub width: u32,
pub height: u32,
pub texture: Direct3D12::ID3D12Resource,
pub rtv_offset: u32,
pub depth_format: Option<DepthFormat>,
pub depth_texture: Option<Direct3D12::ID3D12Resource>,
pub dsv_offset: Option<u32>,
pub command_list: Direct3D12::ID3D12GraphicsCommandList7,
}
#[allow(dead_code)]
pub(crate) struct TextureState {
pub device_handle: DeviceHandle,
pub width: u32,
pub height: u32,
pub format: TextureFormat,
pub resource: Direct3D12::ID3D12Resource,
pub srv_offset: u32,
pub bindless_offset: Option<u32>,
pub sampled_bindless_offset: Option<u32>,
pub last_layout: Direct3D12::D3D12_BARRIER_LAYOUT,
pub is_storage: bool,
pub transient_placed: bool,
}
#[allow(dead_code)]
pub(crate) struct SamplerState {
pub device_handle: DeviceHandle,
pub sampler_offset: u32,
pub desc: SamplerDesc,
pub bindless_offset: Option<u32>,
}
pub const MAX_FRAMES_IN_FLIGHT: usize = 3;
#[allow(dead_code)]
pub(crate) struct FrameSync {
pub command_list: Direct3D12::ID3D12GraphicsCommandList7,
pub command_allocator: Direct3D12::ID3D12CommandAllocator,
pub fence_value: u64,
pub render_pass_submitted: bool,
}
pub(crate) struct SurfaceState {
pub device_handle: DeviceHandle,
pub swapchain: Dxgi::IDXGISwapChain3,
pub render_targets: Vec<Direct3D12::ID3D12Resource>,
pub rtv_offsets: Vec<u32>,
pub width: u32,
pub height: u32,
pub format: Dxgi::Common::DXGI_FORMAT,
pub depth_format: Option<DepthFormat>,
#[allow(dead_code)] pub depth_texture: Option<Direct3D12::ID3D12Resource>,
pub dsv_offset: Option<u32>,
pub current_frame: usize,
pub current_image_index: Option<u32>,
pub frame_sync: Vec<FrameSync>,
pub current_texture_handle: Option<super::TextureHandle>,
pub compute_scratch_textures: Vec<Option<super::TextureHandle>>,
pub present_mode: crate::types::PresentMode,
pub frame_latency_waitable: Option<SendSyncHandle>,
pub pending_frame_compute: Vec<crate::backend::GpuCommand>,
pub pending_acquire_count: u32,
pub pending_swapchain_returns: Vec<(u32, crate::timeline::TimelineValue)>,
}
macro_rules! handle_table {
($table:ident, $shared:ident, $handle:ty, $value:ty) => {
#[derive(Default)]
pub(crate) struct $table {
pub entries: HashMap<$handle, $value>,
pub next_handle: $handle,
}
impl $table {
pub fn new() -> Self {
Self {
entries: HashMap::new(),
next_handle: 1,
}
}
pub fn alloc_handle(&mut self) -> $handle {
let h = self.next_handle;
self.next_handle += 1;
h
}
}
pub(crate) type $shared = Arc<RwLock<$table>>;
};
}
handle_table!(BufferTable, SharedBufferTable, BufferHandle, BufferState);
handle_table!(ShaderTable, SharedShaderTable, ShaderHandle, ShaderState);
handle_table!(PipelineTable, SharedPipelineTable, PipelineHandle, PipelineState);
handle_table!(
ComputePipelineTable,
SharedComputePipelineTable,
ComputePipelineHandle,
ComputePipelineState
);
handle_table!(
RenderTargetTable,
SharedRenderTargetTable,
RenderTargetHandle,
RenderTargetState
);
handle_table!(TextureTable, SharedTextureTable, TextureHandle, TextureState);
handle_table!(SamplerTable, SharedSamplerTable, SamplerHandle, SamplerState);
pub(super) struct Dx12State {
pub factory: Dxgi::IDXGIFactory4,
pub allow_tearing: bool,
pub adapters: Vec<DxgiAdapterInfo>,
pub devices: HashMap<DeviceHandle, SharedLogicalDevice>,
pub next_device_handle: DeviceHandle,
pub contexts: SharedContextMap,
pub next_context_id: super::ContextHandle,
pub device_owner_handles: HashMap<DeviceHandle, super::ContextHandle>,
pub context_fences: std::sync::Arc<std::sync::RwLock<HashMap<ContextHandle, ContextFenceEntry>>>,
pub buffers: SharedBufferTable,
pub shaders: SharedShaderTable,
pub pipelines: SharedPipelineTable,
pub compute_pipelines: SharedComputePipelineTable,
pub render_targets: SharedRenderTargetTable,
pub surfaces: HashMap<SurfaceHandle, SurfaceState>,
pub next_surface_handle: SurfaceHandle,
pub textures: SharedTextureTable,
pub samplers: SharedSamplerTable,
pub next_rtv_offset: u32,
pub free_rtv_offsets: Vec<u32>,
pub next_dsv_offset: u32,
pub free_dsv_offsets: Vec<u32>,
pub slang_compiler: crate::slang::SlangCompiler,
pub device_removed: std::sync::Arc<std::sync::atomic::AtomicBool>,
}