gpu_allocator/d3d12/
mod.rs

1#[cfg(feature = "std")]
2use alloc::sync::Arc;
3use alloc::{boxed::Box, string::String, vec::Vec};
4use core::{
5    fmt,
6    // TODO: Remove when bumping MSRV to 1.80
7    mem::size_of_val,
8};
9#[cfg(feature = "std")]
10use std::backtrace::Backtrace;
11
12use log::{debug, warn, Level};
13use windows::Win32::{
14    Foundation::E_OUTOFMEMORY,
15    Graphics::{
16        Direct3D12::*,
17        Dxgi::{Common::DXGI_FORMAT, DXGI_ERROR_DEVICE_REMOVED},
18    },
19};
20
21#[cfg(feature = "visualizer")]
22mod visualizer;
23#[cfg(feature = "visualizer")]
24pub use visualizer::AllocatorVisualizer;
25
26use crate::{
27    allocator::{
28        AllocationType, AllocatorReport, DedicatedBlockAllocator, FreeListAllocator,
29        MemoryBlockReport, SubAllocator,
30    },
31    AllocationError, AllocationSizes, AllocatorDebugSettings, MemoryLocation, Result,
32};
33
34/// [`ResourceCategory`] is used for supporting [`D3D12_RESOURCE_HEAP_TIER_1`].
35/// [`ResourceCategory`] will be ignored if device supports [`D3D12_RESOURCE_HEAP_TIER_2`].
36#[derive(Clone, Copy, Debug, PartialEq, Eq)]
37pub enum ResourceCategory {
38    Buffer,
39    RtvDsvTexture,
40    OtherTexture,
41}
42
43#[derive(Clone, Copy, Debug, PartialEq, Eq)]
44pub enum ResourceStateOrBarrierLayout {
45    ResourceState(D3D12_RESOURCE_STATES),
46    BarrierLayout(D3D12_BARRIER_LAYOUT),
47}
48
49#[derive(Clone, Copy)]
50pub struct ResourceCreateDesc<'a> {
51    pub name: &'a str,
52    pub memory_location: MemoryLocation,
53    pub resource_category: ResourceCategory,
54    pub resource_desc: &'a D3D12_RESOURCE_DESC,
55    pub castable_formats: &'a [DXGI_FORMAT],
56    pub clear_value: Option<&'a D3D12_CLEAR_VALUE>,
57    pub initial_state_or_layout: ResourceStateOrBarrierLayout,
58    pub resource_type: &'a ResourceType<'a>,
59}
60
61#[derive(Clone, Copy, Debug, PartialEq, Eq)]
62enum HeapCategory {
63    All,
64    Buffer,
65    RtvDsvTexture,
66    OtherTexture,
67}
68
69impl From<ResourceCategory> for HeapCategory {
70    fn from(resource_category: ResourceCategory) -> Self {
71        match resource_category {
72            ResourceCategory::Buffer => Self::Buffer,
73            ResourceCategory::RtvDsvTexture => Self::RtvDsvTexture,
74            ResourceCategory::OtherTexture => Self::OtherTexture,
75        }
76    }
77}
78
79impl From<&D3D12_RESOURCE_DESC> for ResourceCategory {
80    fn from(desc: &D3D12_RESOURCE_DESC) -> Self {
81        if desc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER {
82            Self::Buffer
83        } else if (desc.Flags
84            & (D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL))
85            != D3D12_RESOURCE_FLAG_NONE
86        {
87            Self::RtvDsvTexture
88        } else {
89            Self::OtherTexture
90        }
91    }
92}
93
94#[derive(Clone, Debug)]
95pub struct AllocationCreateDesc<'a> {
96    /// Name of the allocation, for tracking and debugging purposes
97    pub name: &'a str,
98    /// Location where the memory allocation should be stored
99    pub location: MemoryLocation,
100
101    /// Size of allocation, should be queried using [`ID3D12Device::GetResourceAllocationInfo()`]
102    pub size: u64,
103    /// Alignment of allocation, should be queried using [`ID3D12Device::GetResourceAllocationInfo()`]
104    pub alignment: u64,
105    /// Resource category based on resource dimension and flags. Can be created from a [`D3D12_RESOURCE_DESC`]
106    /// using the [helper `into()` function]. The resource category is ignored when Resource Heap Tier 2 or higher
107    /// is supported.
108    ///
109    /// [helper `into()` function]: ResourceCategory::from()
110    pub resource_category: ResourceCategory,
111}
112
113impl<'a> AllocationCreateDesc<'a> {
114    /// Helper function to construct an [`AllocationCreateDesc`] from an existing
115    /// [`D3D12_RESOURCE_DESC`] utilizing [`ID3D12Device::GetResourceAllocationInfo()`].
116    pub fn from_d3d12_resource_desc(
117        device: &ID3D12Device,
118        desc: &D3D12_RESOURCE_DESC,
119        name: &'a str,
120        location: MemoryLocation,
121    ) -> Self {
122        // SAFETY: `device` is a valid device handle, and no arguments (like pointers) are passed
123        // that could induce UB.
124        let allocation_info =
125            unsafe { device.GetResourceAllocationInfo(0, core::slice::from_ref(desc)) };
126        let resource_category: ResourceCategory = desc.into();
127
128        AllocationCreateDesc {
129            name,
130            location,
131            size: allocation_info.SizeInBytes,
132            alignment: allocation_info.Alignment,
133            resource_category,
134        }
135    }
136}
137
138#[derive(Clone, Debug)]
139pub enum ID3D12DeviceVersion {
140    /// Basic device compatible with legacy barriers only, i.e. can only be used in conjunction
141    /// with [`ResourceStateOrBarrierLayout::ResourceState`].
142    Device(ID3D12Device),
143    /// Required for enhanced barrier support, i.e. when using
144    /// [`ResourceStateOrBarrierLayout::BarrierLayout`].
145    Device10(ID3D12Device10),
146    /// Required for castable formats support, implies use of enhanced barriers
147    Device12(ID3D12Device12),
148}
149
150impl core::ops::Deref for ID3D12DeviceVersion {
151    type Target = ID3D12Device;
152
153    fn deref(&self) -> &Self::Target {
154        match self {
155            Self::Device(device) => device,
156            Self::Device10(device10) => device10.into(),
157            Self::Device12(device12) => device12.into(),
158        }
159    }
160}
161
162#[derive(Debug)]
163pub struct AllocatorCreateDesc {
164    pub device: ID3D12DeviceVersion,
165    pub debug_settings: AllocatorDebugSettings,
166    pub allocation_sizes: AllocationSizes,
167}
168
169pub enum ResourceType<'a> {
170    /// Create a D3D12 [`CommittedResource`].
171    ///
172    /// [`CommittedResource`]: https://learn.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device-createcommittedresource
173    Committed {
174        heap_properties: &'a D3D12_HEAP_PROPERTIES,
175        heap_flags: D3D12_HEAP_FLAGS,
176    },
177    /// Create a D3D12 [`PlacedResource`].
178    ///
179    /// [`PlacedResource`]: https://learn.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device-createplacedresource
180    Placed,
181}
182
183#[derive(Debug)]
184pub struct Resource {
185    name: String,
186    pub allocation: Option<Allocation>,
187    resource: Option<ID3D12Resource>,
188    pub memory_location: MemoryLocation,
189    memory_type_index: Option<usize>,
190    pub size: u64,
191}
192
193impl Resource {
194    pub fn resource(&self) -> &ID3D12Resource {
195        self.resource.as_ref().expect("Resource was already freed.")
196    }
197}
198
199impl Drop for Resource {
200    fn drop(&mut self) {
201        if self.resource.is_some() {
202            warn!("Dropping resource `{}` that was not freed. Call `Allocator::free_resource(resource)` instead.", self.name);
203        }
204    }
205}
206
207#[derive(Debug)]
208pub struct CommittedAllocationStatistics {
209    pub num_allocations: usize,
210    pub total_size: u64,
211}
212
213#[derive(Debug)]
214pub struct Allocation {
215    chunk_id: Option<core::num::NonZeroU64>,
216    offset: u64,
217    size: u64,
218    memory_block_index: usize,
219    memory_type_index: usize,
220    heap: ID3D12Heap,
221
222    name: Option<Box<str>>,
223}
224
225impl Allocation {
226    pub fn chunk_id(&self) -> Option<core::num::NonZeroU64> {
227        self.chunk_id
228    }
229
230    /// Returns the [`ID3D12Heap`] object that is backing this allocation.
231    ///
232    /// This heap object can be shared with multiple other allocations and shouldn't be allocated from
233    /// without this library, because that will lead to undefined behavior.
234    ///
235    /// # Safety
236    /// The result of this function can safely be passed into [`ID3D12Device::CreatePlacedResource()`].
237    /// It is exposed for this reason.  Keep in mind to also pass [`Self::offset()`] along to it.
238    ///
239    /// Also, this [`Allocation`] must not be [`Allocator::free()`]d while such a created resource
240    /// on this [`ID3D12Heap`] is still live.
241    pub unsafe fn heap(&self) -> &ID3D12Heap {
242        &self.heap
243    }
244
245    /// Returns the offset of the allocation on the [`ID3D12Heap`].
246    /// When creating a placed resource, this offset needs to be supplied as well.
247    pub fn offset(&self) -> u64 {
248        self.offset
249    }
250
251    /// Returns the size of the allocation
252    pub fn size(&self) -> u64 {
253        self.size
254    }
255
256    pub fn is_null(&self) -> bool {
257        self.chunk_id.is_none()
258    }
259}
260
261#[derive(Debug)]
262struct MemoryBlock {
263    heap: ID3D12Heap,
264    size: u64,
265    sub_allocator: Box<dyn SubAllocator>,
266}
267impl MemoryBlock {
268    fn new(
269        device: &ID3D12Device,
270        size: u64,
271        heap_properties: &D3D12_HEAP_PROPERTIES,
272        heap_category: HeapCategory,
273        dedicated: bool,
274    ) -> Result<Self> {
275        let heap = {
276            let mut desc = D3D12_HEAP_DESC {
277                SizeInBytes: size,
278                Properties: *heap_properties,
279                Alignment: D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT as u64,
280                ..Default::default()
281            };
282            desc.Flags = match heap_category {
283                HeapCategory::All => D3D12_HEAP_FLAG_NONE,
284                HeapCategory::Buffer => D3D12_HEAP_FLAG_ALLOW_ONLY_BUFFERS,
285                HeapCategory::RtvDsvTexture => D3D12_HEAP_FLAG_ALLOW_ONLY_RT_DS_TEXTURES,
286                HeapCategory::OtherTexture => D3D12_HEAP_FLAG_ALLOW_ONLY_NON_RT_DS_TEXTURES,
287            };
288
289            let mut heap = None;
290            let hr = unsafe { device.CreateHeap(&desc, &mut heap) };
291            match hr {
292                Err(e) if e.code() == E_OUTOFMEMORY => Err(AllocationError::OutOfMemory),
293                Err(e) => Err(AllocationError::Internal(format!(
294                    "ID3D12Device::CreateHeap failed: {e}"
295                ))),
296                Ok(()) => heap.ok_or_else(|| {
297                    AllocationError::Internal(
298                        "ID3D12Heap pointer is null, but should not be.".into(),
299                    )
300                }),
301            }?
302        };
303
304        let sub_allocator: Box<dyn SubAllocator> = if dedicated {
305            Box::new(DedicatedBlockAllocator::new(size))
306        } else {
307            Box::new(FreeListAllocator::new(size))
308        };
309
310        Ok(Self {
311            heap,
312            size,
313            sub_allocator,
314        })
315    }
316}
317
318#[derive(Debug)]
319struct MemoryType {
320    memory_blocks: Vec<Option<MemoryBlock>>,
321    committed_allocations: CommittedAllocationStatistics,
322    memory_location: MemoryLocation,
323    heap_category: HeapCategory,
324    heap_properties: D3D12_HEAP_PROPERTIES,
325    memory_type_index: usize,
326    active_general_blocks: usize,
327}
328
329impl MemoryType {
330    fn allocate(
331        &mut self,
332        device: &ID3D12DeviceVersion,
333        desc: &AllocationCreateDesc<'_>,
334        #[cfg(feature = "std")] backtrace: Arc<Backtrace>,
335        allocation_sizes: &AllocationSizes,
336    ) -> Result<Allocation> {
337        let allocation_type = AllocationType::Linear;
338
339        let is_host = self.heap_properties.Type != D3D12_HEAP_TYPE_DEFAULT;
340        let memblock_size = allocation_sizes.get_memblock_size(is_host, self.active_general_blocks);
341
342        let size = desc.size;
343        let alignment = desc.alignment;
344
345        // Create a dedicated block for large memory allocations
346        if size > memblock_size {
347            let mem_block = MemoryBlock::new(
348                device,
349                size,
350                &self.heap_properties,
351                self.heap_category,
352                true,
353            )?;
354
355            let block_index = self.memory_blocks.iter().position(|block| block.is_none());
356            let block_index = match block_index {
357                Some(i) => {
358                    self.memory_blocks[i].replace(mem_block);
359                    i
360                }
361                None => {
362                    self.memory_blocks.push(Some(mem_block));
363                    self.memory_blocks.len() - 1
364                }
365            };
366
367            let mem_block = self.memory_blocks[block_index]
368                .as_mut()
369                .ok_or_else(|| AllocationError::Internal("Memory block must be Some".into()))?;
370
371            let (offset, chunk_id) = mem_block.sub_allocator.allocate(
372                size,
373                alignment,
374                allocation_type,
375                1,
376                desc.name,
377                #[cfg(feature = "std")]
378                backtrace,
379            )?;
380
381            return Ok(Allocation {
382                chunk_id: Some(chunk_id),
383                size,
384                offset,
385                memory_block_index: block_index,
386                memory_type_index: self.memory_type_index,
387                heap: mem_block.heap.clone(),
388                name: Some(desc.name.into()),
389            });
390        }
391
392        let mut empty_block_index = None;
393        for (mem_block_i, mem_block) in self.memory_blocks.iter_mut().enumerate().rev() {
394            if let Some(mem_block) = mem_block {
395                let allocation = mem_block.sub_allocator.allocate(
396                    size,
397                    alignment,
398                    allocation_type,
399                    1,
400                    desc.name,
401                    #[cfg(feature = "std")]
402                    backtrace.clone(),
403                );
404
405                match allocation {
406                    Ok((offset, chunk_id)) => {
407                        return Ok(Allocation {
408                            chunk_id: Some(chunk_id),
409                            offset,
410                            size,
411                            memory_block_index: mem_block_i,
412                            memory_type_index: self.memory_type_index,
413                            heap: mem_block.heap.clone(),
414                            name: Some(desc.name.into()),
415                        });
416                    }
417                    Err(AllocationError::OutOfMemory) => {} // Block is full, continue search.
418                    Err(err) => return Err(err),            // Unhandled error, return.
419                }
420            } else if empty_block_index.is_none() {
421                empty_block_index = Some(mem_block_i);
422            }
423        }
424
425        let new_memory_block = MemoryBlock::new(
426            device,
427            memblock_size,
428            &self.heap_properties,
429            self.heap_category,
430            false,
431        )?;
432
433        let new_block_index = if let Some(block_index) = empty_block_index {
434            self.memory_blocks[block_index] = Some(new_memory_block);
435            block_index
436        } else {
437            self.memory_blocks.push(Some(new_memory_block));
438            self.memory_blocks.len() - 1
439        };
440
441        self.active_general_blocks += 1;
442
443        let mem_block = self.memory_blocks[new_block_index]
444            .as_mut()
445            .ok_or_else(|| AllocationError::Internal("Memory block must be Some".into()))?;
446        let allocation = mem_block.sub_allocator.allocate(
447            size,
448            alignment,
449            allocation_type,
450            1,
451            desc.name,
452            #[cfg(feature = "std")]
453            backtrace,
454        );
455        let (offset, chunk_id) = match allocation {
456            Err(AllocationError::OutOfMemory) => Err(AllocationError::Internal(
457                "Allocation that must succeed failed. This is a bug in the allocator.".into(),
458            )),
459            a => a,
460        }?;
461
462        Ok(Allocation {
463            chunk_id: Some(chunk_id),
464            offset,
465            size,
466            memory_block_index: new_block_index,
467            memory_type_index: self.memory_type_index,
468            heap: mem_block.heap.clone(),
469            name: Some(desc.name.into()),
470        })
471    }
472
473    #[allow(clippy::needless_pass_by_value)]
474    fn free(&mut self, allocation: Allocation) -> Result<()> {
475        let block_idx = allocation.memory_block_index;
476
477        let mem_block = self.memory_blocks[block_idx]
478            .as_mut()
479            .ok_or_else(|| AllocationError::Internal("Memory block must be Some.".into()))?;
480
481        mem_block.sub_allocator.free(allocation.chunk_id)?;
482
483        // We only want to destroy this now-empty block if it is either a dedicated/personal
484        // allocation, or a block supporting sub-allocations that is not the last one (ensuring
485        // there's always at least one block/allocator readily available).
486        let is_dedicated_or_not_last_general_block =
487            !mem_block.sub_allocator.supports_general_allocations()
488                || self.active_general_blocks > 1;
489        if mem_block.sub_allocator.is_empty() && is_dedicated_or_not_last_general_block {
490            let block = self.memory_blocks[block_idx]
491                .take()
492                .ok_or_else(|| AllocationError::Internal("Memory block must be Some.".into()))?;
493
494            if block.sub_allocator.supports_general_allocations() {
495                self.active_general_blocks -= 1;
496            }
497
498            // Note that `block` will be destroyed on `drop` here
499        }
500
501        Ok(())
502    }
503}
504
505pub struct Allocator {
506    device: ID3D12DeviceVersion,
507    debug_settings: AllocatorDebugSettings,
508    memory_types: Vec<MemoryType>,
509    allocation_sizes: AllocationSizes,
510}
511
512impl Allocator {
513    pub fn device(&self) -> &ID3D12DeviceVersion {
514        &self.device
515    }
516
517    pub fn new(desc: &AllocatorCreateDesc) -> Result<Self> {
518        // Perform AddRef on the device
519        let device = desc.device.clone();
520
521        // Query device for feature level
522        let mut options = Default::default();
523        unsafe {
524            device.CheckFeatureSupport(
525                D3D12_FEATURE_D3D12_OPTIONS,
526                <*mut D3D12_FEATURE_DATA_D3D12_OPTIONS>::cast(&mut options),
527                size_of_val(&options) as u32,
528            )
529        }
530        .map_err(|e| {
531            AllocationError::Internal(format!("ID3D12Device::CheckFeatureSupport failed: {e}"))
532        })?;
533
534        let is_heap_tier1 = options.ResourceHeapTier == D3D12_RESOURCE_HEAP_TIER_1;
535
536        let heap_types = [
537            (
538                MemoryLocation::GpuOnly,
539                D3D12_HEAP_PROPERTIES {
540                    Type: D3D12_HEAP_TYPE_DEFAULT,
541                    ..Default::default()
542                },
543            ),
544            (
545                MemoryLocation::CpuToGpu,
546                D3D12_HEAP_PROPERTIES {
547                    Type: D3D12_HEAP_TYPE_CUSTOM,
548                    CPUPageProperty: D3D12_CPU_PAGE_PROPERTY_WRITE_COMBINE,
549                    MemoryPoolPreference: D3D12_MEMORY_POOL_L0,
550                    ..Default::default()
551                },
552            ),
553            (
554                MemoryLocation::GpuToCpu,
555                D3D12_HEAP_PROPERTIES {
556                    Type: D3D12_HEAP_TYPE_CUSTOM,
557                    CPUPageProperty: D3D12_CPU_PAGE_PROPERTY_WRITE_BACK,
558                    MemoryPoolPreference: D3D12_MEMORY_POOL_L0,
559                    ..Default::default()
560                },
561            ),
562        ];
563
564        let heap_types = if is_heap_tier1 {
565            heap_types
566                .iter()
567                .flat_map(|(memory_location, heap_properties)| {
568                    [
569                        (HeapCategory::Buffer, *memory_location, *heap_properties),
570                        (
571                            HeapCategory::RtvDsvTexture,
572                            *memory_location,
573                            *heap_properties,
574                        ),
575                        (
576                            HeapCategory::OtherTexture,
577                            *memory_location,
578                            *heap_properties,
579                        ),
580                    ]
581                    .to_vec()
582                })
583                .collect::<Vec<_>>()
584        } else {
585            heap_types
586                .iter()
587                .map(|(memory_location, heap_properties)| {
588                    (HeapCategory::All, *memory_location, *heap_properties)
589                })
590                .collect::<Vec<_>>()
591        };
592
593        let memory_types = heap_types
594            .iter()
595            .enumerate()
596            .map(
597                |(i, &(heap_category, memory_location, heap_properties))| MemoryType {
598                    memory_blocks: Vec::default(),
599                    memory_location,
600                    heap_category,
601                    heap_properties,
602                    memory_type_index: i,
603                    active_general_blocks: 0,
604                    committed_allocations: CommittedAllocationStatistics {
605                        num_allocations: 0,
606                        total_size: 0,
607                    },
608                },
609            )
610            .collect::<Vec<_>>();
611
612        Ok(Self {
613            memory_types,
614            device,
615            debug_settings: desc.debug_settings,
616            allocation_sizes: desc.allocation_sizes,
617        })
618    }
619
620    pub fn allocate(&mut self, desc: &AllocationCreateDesc<'_>) -> Result<Allocation> {
621        let size = desc.size;
622        let alignment = desc.alignment;
623
624        #[cfg(feature = "std")]
625        let backtrace = Arc::new(if self.debug_settings.store_stack_traces {
626            Backtrace::force_capture()
627        } else {
628            Backtrace::disabled()
629        });
630
631        if self.debug_settings.log_allocations {
632            debug!(
633                "Allocating `{}` of {} bytes with an alignment of {}.",
634                &desc.name, size, alignment
635            );
636            #[cfg(feature = "std")]
637            if self.debug_settings.log_stack_traces {
638                let backtrace = Backtrace::force_capture();
639                debug!("Allocation stack trace: {backtrace}");
640            }
641        }
642
643        if size == 0 || !alignment.is_power_of_two() {
644            return Err(AllocationError::InvalidAllocationCreateDesc);
645        }
646
647        // Find memory type
648        let memory_type = self
649            .memory_types
650            .iter_mut()
651            .find(|memory_type| {
652                let is_location_compatible = desc.location == MemoryLocation::Unknown
653                    || desc.location == memory_type.memory_location;
654
655                let is_category_compatible = memory_type.heap_category == HeapCategory::All
656                    || memory_type.heap_category == desc.resource_category.into();
657
658                is_location_compatible && is_category_compatible
659            })
660            .ok_or(AllocationError::NoCompatibleMemoryTypeFound)?;
661
662        memory_type.allocate(
663            &self.device,
664            desc,
665            #[cfg(feature = "std")]
666            backtrace,
667            &self.allocation_sizes,
668        )
669    }
670
671    pub fn free(&mut self, allocation: Allocation) -> Result<()> {
672        if self.debug_settings.log_frees {
673            let name = allocation.name.as_deref().unwrap_or("<null>");
674            debug!("Freeing `{name}`.");
675            #[cfg(feature = "std")]
676            if self.debug_settings.log_stack_traces {
677                let backtrace = Backtrace::force_capture();
678                debug!("Free stack trace: {backtrace}");
679            }
680        }
681
682        if allocation.is_null() {
683            return Ok(());
684        }
685
686        self.memory_types[allocation.memory_type_index].free(allocation)?;
687
688        Ok(())
689    }
690
691    pub fn rename_allocation(&mut self, allocation: &mut Allocation, name: &str) -> Result<()> {
692        allocation.name = Some(name.into());
693
694        if allocation.is_null() {
695            return Ok(());
696        }
697
698        let mem_type = &mut self.memory_types[allocation.memory_type_index];
699        let mem_block = mem_type.memory_blocks[allocation.memory_block_index]
700            .as_mut()
701            .ok_or_else(|| AllocationError::Internal("Memory block must be Some.".into()))?;
702
703        mem_block
704            .sub_allocator
705            .rename_allocation(allocation.chunk_id, name)?;
706
707        Ok(())
708    }
709
710    pub fn report_memory_leaks(&self, log_level: Level) {
711        for (mem_type_i, mem_type) in self.memory_types.iter().enumerate() {
712            for (block_i, mem_block) in mem_type.memory_blocks.iter().enumerate() {
713                if let Some(mem_block) = mem_block {
714                    mem_block
715                        .sub_allocator
716                        .report_memory_leaks(log_level, mem_type_i, block_i);
717                }
718            }
719        }
720    }
721
722    fn d3d12_resource_desc_1(desc: &D3D12_RESOURCE_DESC) -> D3D12_RESOURCE_DESC1 {
723        D3D12_RESOURCE_DESC1 {
724            Dimension: desc.Dimension,
725            Alignment: desc.Alignment,
726            Width: desc.Width,
727            Height: desc.Height,
728            DepthOrArraySize: desc.DepthOrArraySize,
729            MipLevels: desc.MipLevels,
730            Format: desc.Format,
731            SampleDesc: desc.SampleDesc,
732            Layout: desc.Layout,
733            Flags: desc.Flags,
734            // TODO: This is the only new field
735            SamplerFeedbackMipRegion: D3D12_MIP_REGION::default(),
736        }
737    }
738
739    fn resource_allocation_info(
740        device: &ID3D12DeviceVersion,
741        desc: &ResourceCreateDesc<'_>,
742    ) -> D3D12_RESOURCE_ALLOCATION_INFO {
743        match device {
744            ID3D12DeviceVersion::Device(device) => unsafe {
745                device.GetResourceAllocationInfo(0, &[*desc.resource_desc])
746            },
747            ID3D12DeviceVersion::Device10(device) => unsafe {
748                device.GetResourceAllocationInfo(0, &[*desc.resource_desc])
749            },
750            ID3D12DeviceVersion::Device12(device) => unsafe {
751                let resource_desc1 = Self::d3d12_resource_desc_1(desc.resource_desc);
752
753                let resource_descs = &[resource_desc1];
754
755                // We always have one resource desc, hence we only have one mapping castable format array
756                let num_castable_formats = desc.castable_formats.len() as u32;
757                let num_castable_formats_array = &[num_castable_formats];
758
759                let castable_formats_array = &[desc.castable_formats.as_ptr()];
760
761                let (num_castable_formats_opt, castable_formats_opt) = if num_castable_formats > 0 {
762                    (
763                        Some(num_castable_formats_array.as_ptr()),
764                        Some(castable_formats_array.as_ptr()),
765                    )
766                } else {
767                    (None, None)
768                };
769
770                device.GetResourceAllocationInfo3(
771                    0,
772                    resource_descs.len() as u32,
773                    resource_descs.as_ptr(),
774                    num_castable_formats_opt,
775                    castable_formats_opt,
776                    None,
777                )
778            },
779        }
780    }
781
782    /// Create a resource according to the provided parameters.
783    /// Created resources should be freed at the end of their lifetime by calling [`Self::free_resource()`].
784    pub fn create_resource(&mut self, desc: &ResourceCreateDesc<'_>) -> Result<Resource> {
785        match desc.resource_type {
786            ResourceType::Committed {
787                heap_properties,
788                heap_flags,
789            } => {
790                let mut result: Option<ID3D12Resource> = None;
791
792                let clear_value: Option<*const D3D12_CLEAR_VALUE> =
793                    desc.clear_value.map(|v| -> *const _ { v });
794
795                if let Err(e) = unsafe {
796                    match (&self.device, desc.initial_state_or_layout) {
797                        (_, ResourceStateOrBarrierLayout::ResourceState(_))
798                            if !desc.castable_formats.is_empty() =>
799                        {
800                            return Err(AllocationError::CastableFormatsRequiresEnhancedBarriers)
801                        }
802                        (
803                            ID3D12DeviceVersion::Device12(device),
804                            ResourceStateOrBarrierLayout::BarrierLayout(initial_layout),
805                        ) => {
806                            let resource_desc1 = Self::d3d12_resource_desc_1(desc.resource_desc);
807                            device.CreateCommittedResource3(
808                                *heap_properties,
809                                *heap_flags,
810                                &resource_desc1,
811                                initial_layout,
812                                clear_value,
813                                None, // TODO
814                                Some(desc.castable_formats),
815                                &mut result,
816                            )
817                        }
818                        (_, ResourceStateOrBarrierLayout::BarrierLayout(_))
819                            if !desc.castable_formats.is_empty() =>
820                        {
821                            return Err(AllocationError::CastableFormatsRequiresAtLeastDevice12)
822                        }
823                        (
824                            ID3D12DeviceVersion::Device10(device),
825                            ResourceStateOrBarrierLayout::BarrierLayout(initial_layout),
826                        ) => {
827                            let resource_desc1 = Self::d3d12_resource_desc_1(desc.resource_desc);
828
829                            device.CreateCommittedResource3(
830                                *heap_properties,
831                                *heap_flags,
832                                &resource_desc1,
833                                initial_layout,
834                                clear_value,
835                                None, // TODO
836                                None,
837                                &mut result,
838                            )
839                        }
840                        (_, ResourceStateOrBarrierLayout::BarrierLayout(_)) => {
841                            return Err(AllocationError::BarrierLayoutNeedsDevice10)
842                        }
843                        (device, ResourceStateOrBarrierLayout::ResourceState(initial_state)) => {
844                            device.CreateCommittedResource(
845                                *heap_properties,
846                                *heap_flags,
847                                desc.resource_desc,
848                                initial_state,
849                                clear_value,
850                                &mut result,
851                            )
852                        }
853                    }
854                } {
855                    if e.code() == DXGI_ERROR_DEVICE_REMOVED {
856                        return Err(AllocationError::Internal(format!(
857                            "ID3D12Device::CreateCommittedResource DEVICE_REMOVED: {:?}",
858                            unsafe { self.device.GetDeviceRemovedReason() }
859                        )));
860                    }
861                    return Err(AllocationError::Internal(format!(
862                        "ID3D12Device::CreateCommittedResource failed: {e}"
863                    )));
864                }
865
866                let resource = result.expect("Allocation succeeded but no resource was returned?");
867
868                let allocation_info = Self::resource_allocation_info(&self.device, desc);
869
870                let memory_type = self
871                    .memory_types
872                    .iter_mut()
873                    .find(|memory_type| {
874                        let is_location_compatible = desc.memory_location
875                            == MemoryLocation::Unknown
876                            || desc.memory_location == memory_type.memory_location;
877
878                        let is_category_compatible = memory_type.heap_category == HeapCategory::All
879                            || memory_type.heap_category == desc.resource_category.into();
880
881                        is_location_compatible && is_category_compatible
882                    })
883                    .ok_or(AllocationError::NoCompatibleMemoryTypeFound)?;
884
885                memory_type.committed_allocations.num_allocations += 1;
886                memory_type.committed_allocations.total_size += allocation_info.SizeInBytes;
887
888                Ok(Resource {
889                    name: desc.name.into(),
890                    allocation: None,
891                    resource: Some(resource),
892                    size: allocation_info.SizeInBytes,
893                    memory_location: desc.memory_location,
894                    memory_type_index: Some(memory_type.memory_type_index),
895                })
896            }
897            ResourceType::Placed => {
898                let allocation_desc = {
899                    let allocation_info = Self::resource_allocation_info(&self.device, desc);
900
901                    AllocationCreateDesc {
902                        name: desc.name,
903                        location: desc.memory_location,
904                        size: allocation_info.SizeInBytes,
905                        alignment: allocation_info.Alignment,
906                        resource_category: desc.resource_category,
907                    }
908                };
909
910                let allocation = self.allocate(&allocation_desc)?;
911
912                let mut result: Option<ID3D12Resource> = None;
913                if let Err(e) = unsafe {
914                    match (&self.device, desc.initial_state_or_layout) {
915                        (_, ResourceStateOrBarrierLayout::ResourceState(_))
916                            if !desc.castable_formats.is_empty() =>
917                        {
918                            return Err(AllocationError::CastableFormatsRequiresEnhancedBarriers)
919                        }
920                        (
921                            ID3D12DeviceVersion::Device12(device),
922                            ResourceStateOrBarrierLayout::BarrierLayout(initial_layout),
923                        ) => {
924                            let resource_desc1 = Self::d3d12_resource_desc_1(desc.resource_desc);
925                            device.CreatePlacedResource2(
926                                allocation.heap(),
927                                allocation.offset(),
928                                &resource_desc1,
929                                initial_layout,
930                                None,
931                                Some(desc.castable_formats),
932                                &mut result,
933                            )
934                        }
935                        (_, ResourceStateOrBarrierLayout::BarrierLayout(_))
936                            if !desc.castable_formats.is_empty() =>
937                        {
938                            return Err(AllocationError::CastableFormatsRequiresAtLeastDevice12)
939                        }
940                        (
941                            ID3D12DeviceVersion::Device10(device),
942                            ResourceStateOrBarrierLayout::BarrierLayout(initial_layout),
943                        ) => {
944                            let resource_desc1 = Self::d3d12_resource_desc_1(desc.resource_desc);
945                            device.CreatePlacedResource2(
946                                allocation.heap(),
947                                allocation.offset(),
948                                &resource_desc1,
949                                initial_layout,
950                                None,
951                                None,
952                                &mut result,
953                            )
954                        }
955                        (_, ResourceStateOrBarrierLayout::BarrierLayout(_)) => {
956                            return Err(AllocationError::BarrierLayoutNeedsDevice10)
957                        }
958                        (device, ResourceStateOrBarrierLayout::ResourceState(initial_state)) => {
959                            device.CreatePlacedResource(
960                                allocation.heap(),
961                                allocation.offset(),
962                                desc.resource_desc,
963                                initial_state,
964                                None,
965                                &mut result,
966                            )
967                        }
968                    }
969                } {
970                    if e.code() == DXGI_ERROR_DEVICE_REMOVED {
971                        return Err(AllocationError::Internal(format!(
972                            "ID3D12Device::CreatePlacedResource DEVICE_REMOVED: {:?}",
973                            unsafe { self.device.GetDeviceRemovedReason() }
974                        )));
975                    }
976                    return Err(AllocationError::Internal(format!(
977                        "ID3D12Device::CreatePlacedResource failed: {e}"
978                    )));
979                }
980
981                let resource = result.expect("Allocation succeeded but no resource was returned?");
982                let size = allocation.size();
983                Ok(Resource {
984                    name: desc.name.into(),
985                    allocation: Some(allocation),
986                    resource: Some(resource),
987                    size,
988                    memory_location: desc.memory_location,
989                    memory_type_index: None,
990                })
991            }
992        }
993    }
994
995    /// Free a resource and its memory.
996    pub fn free_resource(&mut self, mut resource: Resource) -> Result<()> {
997        // Explicitly drop the resource (which is backed by a refcounted COM object)
998        // before freeing allocated memory. Windows-rs performs a Release() on drop().
999        let _ = resource
1000            .resource
1001            .take()
1002            .expect("Resource was already freed.");
1003
1004        if let Some(allocation) = resource.allocation.take() {
1005            self.free(allocation)
1006        } else {
1007            // Dx12 CommittedResources do not have an application managed allocation.
1008            // We only have to update the tracked allocation count and memory usage.
1009            if let Some(memory_type_index) = resource.memory_type_index {
1010                let memory_type = &mut self.memory_types[memory_type_index];
1011
1012                memory_type.committed_allocations.num_allocations -= 1;
1013                memory_type.committed_allocations.total_size -= resource.size;
1014            }
1015            Ok(())
1016        }
1017    }
1018
1019    pub fn generate_report(&self) -> AllocatorReport {
1020        let mut allocations = vec![];
1021        let mut blocks = vec![];
1022        let mut total_capacity_bytes = 0;
1023
1024        for memory_type in &self.memory_types {
1025            for block in memory_type.memory_blocks.iter().flatten() {
1026                total_capacity_bytes += block.size;
1027                let first_allocation = allocations.len();
1028                allocations.extend(block.sub_allocator.report_allocations());
1029                blocks.push(MemoryBlockReport {
1030                    size: block.size,
1031                    allocations: first_allocation..allocations.len(),
1032                });
1033            }
1034        }
1035
1036        let total_allocated_bytes = allocations.iter().map(|report| report.size).sum();
1037
1038        AllocatorReport {
1039            allocations,
1040            blocks,
1041            total_allocated_bytes,
1042            total_capacity_bytes,
1043        }
1044    }
1045
1046    /// Current total capacity of memory blocks allocated on the device, in bytes
1047    pub fn capacity(&self) -> u64 {
1048        let mut total_capacity_bytes = 0;
1049
1050        for memory_type in &self.memory_types {
1051            for block in memory_type.memory_blocks.iter().flatten() {
1052                total_capacity_bytes += block.size;
1053            }
1054        }
1055
1056        total_capacity_bytes
1057    }
1058}
1059
1060impl fmt::Debug for Allocator {
1061    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1062        self.generate_report().fmt(f)
1063    }
1064}
1065
1066impl Drop for Allocator {
1067    fn drop(&mut self) {
1068        if self.debug_settings.log_leaks_on_shutdown {
1069            self.report_memory_leaks(Level::Warn);
1070        }
1071
1072        // Because Rust drop rules drop members in source-code order (that would be the
1073        // ID3D12Device before the ID3D12Heaps nested in these memory blocks), free
1074        // all remaining memory blocks manually first by dropping.
1075        for mem_type in self.memory_types.iter_mut() {
1076            mem_type.memory_blocks.clear();
1077        }
1078    }
1079}
gpu_allocator/d3d12/mod.rs

gpu_allocator/d3d12/
mod.rs
