gpu_allocator/d3d12/
mod.rs

1use std::{backtrace::Backtrace, fmt, sync::Arc};
2
3use log::{debug, warn, Level};
4use windows::Win32::{
5    Foundation::E_OUTOFMEMORY,
6    Graphics::{Direct3D12::*, Dxgi::Common::DXGI_FORMAT},
7};
8
9#[cfg(feature = "public-winapi")]
10mod public_winapi {
11    pub use winapi::um::d3d12 as winapi_d3d12;
12
13    use super::*;
14
15    /// Trait similar to [`AsRef`]/[`AsMut`],
16    pub trait ToWinapi<T> {
17        fn as_winapi(&self) -> *const T;
18        fn as_winapi_mut(&mut self) -> *mut T;
19    }
20
21    /// [`windows`] types hold their pointer internally and provide drop semantics. As such this trait
22    /// is usually implemented on the _pointer type_ (`*const`, `*mut`) of the [`winapi`] object so that
23    /// a **borrow of** that pointer becomes a borrow of the [`windows`] type.
24    pub trait ToWindows<T> {
25        fn as_windows(&self) -> &T;
26    }
27
28    impl ToWinapi<winapi_d3d12::ID3D12Resource> for ID3D12Resource {
29        fn as_winapi(&self) -> *const winapi_d3d12::ID3D12Resource {
30            unsafe { std::mem::transmute_copy(self) }
31        }
32
33        fn as_winapi_mut(&mut self) -> *mut winapi_d3d12::ID3D12Resource {
34            unsafe { std::mem::transmute_copy(self) }
35        }
36    }
37
38    impl ToWinapi<winapi_d3d12::ID3D12Device> for ID3D12Device {
39        fn as_winapi(&self) -> *const winapi_d3d12::ID3D12Device {
40            unsafe { std::mem::transmute_copy(self) }
41        }
42
43        fn as_winapi_mut(&mut self) -> *mut winapi_d3d12::ID3D12Device {
44            unsafe { std::mem::transmute_copy(self) }
45        }
46    }
47
48    impl ToWindows<ID3D12Device> for *const winapi_d3d12::ID3D12Device {
49        fn as_windows(&self) -> &ID3D12Device {
50            unsafe { std::mem::transmute(self) }
51        }
52    }
53
54    impl ToWindows<ID3D12Device> for *mut winapi_d3d12::ID3D12Device {
55        fn as_windows(&self) -> &ID3D12Device {
56            unsafe { std::mem::transmute(self) }
57        }
58    }
59
60    impl ToWindows<ID3D12Device> for &mut winapi_d3d12::ID3D12Device {
61        fn as_windows(&self) -> &ID3D12Device {
62            unsafe { std::mem::transmute(self) }
63        }
64    }
65
66    impl ToWinapi<winapi_d3d12::ID3D12Heap> for ID3D12Heap {
67        fn as_winapi(&self) -> *const winapi_d3d12::ID3D12Heap {
68            unsafe { std::mem::transmute_copy(self) }
69        }
70
71        fn as_winapi_mut(&mut self) -> *mut winapi_d3d12::ID3D12Heap {
72            unsafe { std::mem::transmute_copy(self) }
73        }
74    }
75}
76
77#[cfg(feature = "public-winapi")]
78pub use public_winapi::*;
79
80#[cfg(feature = "visualizer")]
81mod visualizer;
82#[cfg(feature = "visualizer")]
83pub use visualizer::AllocatorVisualizer;
84
85use super::{allocator, allocator::AllocationType};
86use crate::{
87    allocator::{AllocatorReport, MemoryBlockReport},
88    AllocationError, AllocationSizes, AllocatorDebugSettings, MemoryLocation, Result,
89};
90
91/// [`ResourceCategory`] is used for supporting [`D3D12_RESOURCE_HEAP_TIER_1`].
92/// [`ResourceCategory`] will be ignored if device supports [`D3D12_RESOURCE_HEAP_TIER_2`].
93#[derive(Clone, Copy, Debug, PartialEq, Eq)]
94pub enum ResourceCategory {
95    Buffer,
96    RtvDsvTexture,
97    OtherTexture,
98}
99
100#[derive(Clone, Copy, Debug, PartialEq, Eq)]
101pub enum ResourceStateOrBarrierLayout {
102    ResourceState(D3D12_RESOURCE_STATES),
103    BarrierLayout(D3D12_BARRIER_LAYOUT),
104}
105
106#[derive(Clone, Copy)]
107pub struct ResourceCreateDesc<'a> {
108    pub name: &'a str,
109    pub memory_location: MemoryLocation,
110    pub resource_category: ResourceCategory,
111    pub resource_desc: &'a D3D12_RESOURCE_DESC,
112    pub castable_formats: &'a [DXGI_FORMAT],
113    pub clear_value: Option<&'a D3D12_CLEAR_VALUE>,
114    pub initial_state_or_layout: ResourceStateOrBarrierLayout,
115    pub resource_type: &'a ResourceType<'a>,
116}
117
118#[derive(Clone, Copy, Debug, PartialEq, Eq)]
119enum HeapCategory {
120    All,
121    Buffer,
122    RtvDsvTexture,
123    OtherTexture,
124}
125
126impl From<ResourceCategory> for HeapCategory {
127    fn from(resource_category: ResourceCategory) -> Self {
128        match resource_category {
129            ResourceCategory::Buffer => Self::Buffer,
130            ResourceCategory::RtvDsvTexture => Self::RtvDsvTexture,
131            ResourceCategory::OtherTexture => Self::OtherTexture,
132        }
133    }
134}
135
136impl From<&D3D12_RESOURCE_DESC> for ResourceCategory {
137    fn from(desc: &D3D12_RESOURCE_DESC) -> Self {
138        if desc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER {
139            Self::Buffer
140        } else if (desc.Flags
141            & (D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL))
142            != D3D12_RESOURCE_FLAG_NONE
143        {
144            Self::RtvDsvTexture
145        } else {
146            Self::OtherTexture
147        }
148    }
149}
150
151#[cfg(feature = "public-winapi")]
152impl From<&winapi_d3d12::D3D12_RESOURCE_DESC> for ResourceCategory {
153    fn from(desc: &winapi_d3d12::D3D12_RESOURCE_DESC) -> Self {
154        if desc.Dimension == winapi_d3d12::D3D12_RESOURCE_DIMENSION_BUFFER {
155            Self::Buffer
156        } else if (desc.Flags
157            & (winapi_d3d12::D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET
158                | winapi_d3d12::D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL))
159            != 0
160        {
161            Self::RtvDsvTexture
162        } else {
163            Self::OtherTexture
164        }
165    }
166}
167
168#[derive(Clone, Debug)]
169pub struct AllocationCreateDesc<'a> {
170    /// Name of the allocation, for tracking and debugging purposes
171    pub name: &'a str,
172    /// Location where the memory allocation should be stored
173    pub location: MemoryLocation,
174
175    /// Size of allocation, should be queried using [`ID3D12Device::GetResourceAllocationInfo()`]
176    pub size: u64,
177    /// Alignment of allocation, should be queried using [`ID3D12Device::GetResourceAllocationInfo()`]
178    pub alignment: u64,
179    /// Resource category based on resource dimension and flags. Can be created from a [`D3D12_RESOURCE_DESC`]
180    /// using the helper into function. The resource category is ignored when Resource Heap Tier 2 or higher
181    /// is supported.
182    pub resource_category: ResourceCategory,
183}
184
185impl<'a> AllocationCreateDesc<'a> {
186    /// Helper conversion function utilizing [`winapi`] types.
187    ///
188    /// This function is also available for [`windows::Win32::Graphics::Direct3D12`]
189    /// types as [`from_d3d12_resource_desc()`][Self::from_d3d12_resource_desc()].
190    #[cfg(feature = "public-winapi")]
191    pub fn from_winapi_d3d12_resource_desc(
192        device: *const winapi_d3d12::ID3D12Device,
193        desc: &winapi_d3d12::D3D12_RESOURCE_DESC,
194        name: &'a str,
195        location: MemoryLocation,
196    ) -> Self {
197        let device = device.as_windows();
198        // Raw structs are binary-compatible
199        let desc = unsafe {
200            std::mem::transmute::<&winapi_d3d12::D3D12_RESOURCE_DESC, &D3D12_RESOURCE_DESC>(desc)
201        };
202        let allocation_info =
203            unsafe { device.GetResourceAllocationInfo(0, std::slice::from_ref(desc)) };
204        let resource_category: ResourceCategory = desc.into();
205
206        AllocationCreateDesc {
207            name,
208            location,
209            size: allocation_info.SizeInBytes,
210            alignment: allocation_info.Alignment,
211            resource_category,
212        }
213    }
214
215    /// Helper conversion function utilizing [`windows::Win32::Graphics::Direct3D12`] types.
216    ///
217    /// This function is also available for `winapi` types as `from_winapi_d3d12_resource_desc()`
218    /// when the `public-winapi` feature is enabled.
219    pub fn from_d3d12_resource_desc(
220        device: &ID3D12Device,
221        desc: &D3D12_RESOURCE_DESC,
222        name: &'a str,
223        location: MemoryLocation,
224    ) -> Self {
225        let allocation_info =
226            unsafe { device.GetResourceAllocationInfo(0, std::slice::from_ref(desc)) };
227        let resource_category: ResourceCategory = desc.into();
228
229        AllocationCreateDesc {
230            name,
231            location,
232            size: allocation_info.SizeInBytes,
233            alignment: allocation_info.Alignment,
234            resource_category,
235        }
236    }
237}
238
239#[derive(Clone, Debug)]
240pub enum ID3D12DeviceVersion {
241    /// Basic device compatible with legacy barriers only, i.e. can only be used in conjunction
242    /// with [`ResourceStateOrBarrierLayout::ResourceState`].
243    Device(ID3D12Device),
244    /// Required for enhanced barrier support, i.e. when using
245    /// [`ResourceStateOrBarrierLayout::BarrierLayout`].
246    Device10(ID3D12Device10),
247    /// Required for castable formats support, implies use of enhanced barriers
248    Device12(ID3D12Device12),
249}
250
251impl std::ops::Deref for ID3D12DeviceVersion {
252    type Target = ID3D12Device;
253
254    fn deref(&self) -> &Self::Target {
255        match self {
256            Self::Device(device) => device,
257            Self::Device10(device10) => device10.into(),
258            Self::Device12(device12) => device12.into(),
259        }
260    }
261}
262
263#[derive(Debug)]
264pub struct AllocatorCreateDesc {
265    pub device: ID3D12DeviceVersion,
266    pub debug_settings: AllocatorDebugSettings,
267    pub allocation_sizes: AllocationSizes,
268}
269
270pub enum ResourceType<'a> {
271    /// Create a D3D12 [`CommittedResource`].
272    ///
273    /// [`CommittedResource`]: https://learn.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device-createcommittedresource
274    Committed {
275        heap_properties: &'a D3D12_HEAP_PROPERTIES,
276        heap_flags: D3D12_HEAP_FLAGS,
277    },
278    /// Create a D3D12 [`PlacedResource`].
279    ///
280    /// [`PlacedResource`]: https://learn.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device-createplacedresource
281    Placed,
282}
283
284#[derive(Debug)]
285pub struct Resource {
286    name: String,
287    pub allocation: Option<Allocation>,
288    resource: Option<ID3D12Resource>,
289    pub memory_location: MemoryLocation,
290    memory_type_index: Option<usize>,
291    pub size: u64,
292}
293
294impl Resource {
295    pub fn resource(&self) -> &ID3D12Resource {
296        self.resource.as_ref().expect("Resource was already freed.")
297    }
298}
299
300impl Drop for Resource {
301    fn drop(&mut self) {
302        if self.resource.is_some() {
303            warn!("Dropping resource `{}` that was not freed. Call `Allocator::free_resource(resource)` instead.", self.name);
304        }
305    }
306}
307
308#[derive(Debug)]
309pub struct CommittedAllocationStatistics {
310    pub num_allocations: usize,
311    pub total_size: u64,
312}
313
314#[derive(Debug)]
315pub struct Allocation {
316    chunk_id: Option<std::num::NonZeroU64>,
317    offset: u64,
318    size: u64,
319    memory_block_index: usize,
320    memory_type_index: usize,
321    heap: ID3D12Heap,
322
323    name: Option<Box<str>>,
324}
325
326impl Allocation {
327    pub fn chunk_id(&self) -> Option<std::num::NonZeroU64> {
328        self.chunk_id
329    }
330
331    /// Returns the [`ID3D12Heap`] object that is backing this allocation.
332    /// This heap object can be shared with multiple other allocations and shouldn't be freed (or allocated from)
333    /// without this library, because that will lead to undefined behavior.
334    ///
335    /// # Safety
336    /// The result of this function be safely passed into [`ID3D12Device::CreatePlacedResource()`].
337    /// It is exposed for this reason. Keep in mind to also pass [`Self::offset()`] along to it.
338    pub unsafe fn heap(&self) -> &ID3D12Heap {
339        &self.heap
340    }
341
342    /// Returns the offset of the allocation on the [`ID3D12Heap`].
343    /// When creating a placed resources, this offset needs to be supplied as well.
344    pub fn offset(&self) -> u64 {
345        self.offset
346    }
347
348    /// Returns the size of the allocation
349    pub fn size(&self) -> u64 {
350        self.size
351    }
352
353    pub fn is_null(&self) -> bool {
354        self.chunk_id.is_none()
355    }
356}
357
358#[derive(Debug)]
359struct MemoryBlock {
360    heap: ID3D12Heap,
361    size: u64,
362    sub_allocator: Box<dyn allocator::SubAllocator>,
363}
364impl MemoryBlock {
365    fn new(
366        device: &ID3D12Device,
367        size: u64,
368        heap_properties: &D3D12_HEAP_PROPERTIES,
369        heap_category: HeapCategory,
370        dedicated: bool,
371    ) -> Result<Self> {
372        let heap = {
373            let mut desc = D3D12_HEAP_DESC {
374                SizeInBytes: size,
375                Properties: *heap_properties,
376                Alignment: D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT as u64,
377                ..Default::default()
378            };
379            desc.Flags = match heap_category {
380                HeapCategory::All => D3D12_HEAP_FLAG_NONE,
381                HeapCategory::Buffer => D3D12_HEAP_FLAG_ALLOW_ONLY_BUFFERS,
382                HeapCategory::RtvDsvTexture => D3D12_HEAP_FLAG_ALLOW_ONLY_RT_DS_TEXTURES,
383                HeapCategory::OtherTexture => D3D12_HEAP_FLAG_ALLOW_ONLY_NON_RT_DS_TEXTURES,
384            };
385
386            let mut heap = None;
387            let hr = unsafe { device.CreateHeap(&desc, &mut heap) };
388            match hr {
389                Err(e) if e.code() == E_OUTOFMEMORY => Err(AllocationError::OutOfMemory),
390                Err(e) => Err(AllocationError::Internal(format!(
391                    "ID3D12Device::CreateHeap failed: {}",
392                    e
393                ))),
394                Ok(()) => heap.ok_or_else(|| {
395                    AllocationError::Internal(
396                        "ID3D12Heap pointer is null, but should not be.".into(),
397                    )
398                }),
399            }?
400        };
401
402        let sub_allocator: Box<dyn allocator::SubAllocator> = if dedicated {
403            Box::new(allocator::DedicatedBlockAllocator::new(size))
404        } else {
405            Box::new(allocator::FreeListAllocator::new(size))
406        };
407
408        Ok(Self {
409            heap,
410            size,
411            sub_allocator,
412        })
413    }
414}
415
416#[derive(Debug)]
417struct MemoryType {
418    memory_blocks: Vec<Option<MemoryBlock>>,
419    committed_allocations: CommittedAllocationStatistics,
420    memory_location: MemoryLocation,
421    heap_category: HeapCategory,
422    heap_properties: D3D12_HEAP_PROPERTIES,
423    memory_type_index: usize,
424    active_general_blocks: usize,
425}
426
427impl MemoryType {
428    fn allocate(
429        &mut self,
430        device: &ID3D12DeviceVersion,
431        desc: &AllocationCreateDesc<'_>,
432        backtrace: Arc<Backtrace>,
433        allocation_sizes: &AllocationSizes,
434    ) -> Result<Allocation> {
435        let allocation_type = AllocationType::Linear;
436
437        let memblock_size = if self.heap_properties.Type == D3D12_HEAP_TYPE_DEFAULT {
438            allocation_sizes.device_memblock_size
439        } else {
440            allocation_sizes.host_memblock_size
441        };
442
443        let size = desc.size;
444        let alignment = desc.alignment;
445
446        // Create a dedicated block for large memory allocations
447        if size > memblock_size {
448            let mem_block = MemoryBlock::new(
449                device,
450                size,
451                &self.heap_properties,
452                self.heap_category,
453                true,
454            )?;
455
456            let block_index = self.memory_blocks.iter().position(|block| block.is_none());
457            let block_index = match block_index {
458                Some(i) => {
459                    self.memory_blocks[i].replace(mem_block);
460                    i
461                }
462                None => {
463                    self.memory_blocks.push(Some(mem_block));
464                    self.memory_blocks.len() - 1
465                }
466            };
467
468            let mem_block = self.memory_blocks[block_index]
469                .as_mut()
470                .ok_or_else(|| AllocationError::Internal("Memory block must be Some".into()))?;
471
472            let (offset, chunk_id) = mem_block.sub_allocator.allocate(
473                size,
474                alignment,
475                allocation_type,
476                1,
477                desc.name,
478                backtrace,
479            )?;
480
481            return Ok(Allocation {
482                chunk_id: Some(chunk_id),
483                size,
484                offset,
485                memory_block_index: block_index,
486                memory_type_index: self.memory_type_index,
487                heap: mem_block.heap.clone(),
488                name: Some(desc.name.into()),
489            });
490        }
491
492        let mut empty_block_index = None;
493        for (mem_block_i, mem_block) in self.memory_blocks.iter_mut().enumerate().rev() {
494            if let Some(mem_block) = mem_block {
495                let allocation = mem_block.sub_allocator.allocate(
496                    size,
497                    alignment,
498                    allocation_type,
499                    1,
500                    desc.name,
501                    backtrace.clone(),
502                );
503
504                match allocation {
505                    Ok((offset, chunk_id)) => {
506                        return Ok(Allocation {
507                            chunk_id: Some(chunk_id),
508                            offset,
509                            size,
510                            memory_block_index: mem_block_i,
511                            memory_type_index: self.memory_type_index,
512                            heap: mem_block.heap.clone(),
513                            name: Some(desc.name.into()),
514                        });
515                    }
516                    Err(AllocationError::OutOfMemory) => {} // Block is full, continue search.
517                    Err(err) => return Err(err),            // Unhandled error, return.
518                }
519            } else if empty_block_index.is_none() {
520                empty_block_index = Some(mem_block_i);
521            }
522        }
523
524        let new_memory_block = MemoryBlock::new(
525            device,
526            memblock_size,
527            &self.heap_properties,
528            self.heap_category,
529            false,
530        )?;
531
532        let new_block_index = if let Some(block_index) = empty_block_index {
533            self.memory_blocks[block_index] = Some(new_memory_block);
534            block_index
535        } else {
536            self.memory_blocks.push(Some(new_memory_block));
537            self.memory_blocks.len() - 1
538        };
539
540        self.active_general_blocks += 1;
541
542        let mem_block = self.memory_blocks[new_block_index]
543            .as_mut()
544            .ok_or_else(|| AllocationError::Internal("Memory block must be Some".into()))?;
545        let allocation = mem_block.sub_allocator.allocate(
546            size,
547            alignment,
548            allocation_type,
549            1,
550            desc.name,
551            backtrace,
552        );
553        let (offset, chunk_id) = match allocation {
554            Err(AllocationError::OutOfMemory) => Err(AllocationError::Internal(
555                "Allocation that must succeed failed. This is a bug in the allocator.".into(),
556            )),
557            a => a,
558        }?;
559
560        Ok(Allocation {
561            chunk_id: Some(chunk_id),
562            offset,
563            size,
564            memory_block_index: new_block_index,
565            memory_type_index: self.memory_type_index,
566            heap: mem_block.heap.clone(),
567            name: Some(desc.name.into()),
568        })
569    }
570
571    #[allow(clippy::needless_pass_by_value)]
572    fn free(&mut self, allocation: Allocation) -> Result<()> {
573        let block_idx = allocation.memory_block_index;
574
575        let mem_block = self.memory_blocks[block_idx]
576            .as_mut()
577            .ok_or_else(|| AllocationError::Internal("Memory block must be Some.".into()))?;
578
579        mem_block.sub_allocator.free(allocation.chunk_id)?;
580
581        if mem_block.sub_allocator.is_empty() {
582            if mem_block.sub_allocator.supports_general_allocations() {
583                if self.active_general_blocks > 1 {
584                    let block = self.memory_blocks[block_idx].take();
585                    if block.is_none() {
586                        return Err(AllocationError::Internal(
587                            "Memory block must be Some.".into(),
588                        ));
589                    }
590                    // Note that `block` will be destroyed on `drop` here
591
592                    self.active_general_blocks -= 1;
593                }
594            } else {
595                let block = self.memory_blocks[block_idx].take();
596                if block.is_none() {
597                    return Err(AllocationError::Internal(
598                        "Memory block must be Some.".into(),
599                    ));
600                }
601                // Note that `block` will be destroyed on `drop` here
602            }
603        }
604
605        Ok(())
606    }
607}
608
609pub struct Allocator {
610    device: ID3D12DeviceVersion,
611    debug_settings: AllocatorDebugSettings,
612    memory_types: Vec<MemoryType>,
613    allocation_sizes: AllocationSizes,
614}
615
616impl Allocator {
617    pub fn device(&self) -> &ID3D12DeviceVersion {
618        &self.device
619    }
620
621    pub fn new(desc: &AllocatorCreateDesc) -> Result<Self> {
622        // Perform AddRef on the device
623        let device = desc.device.clone();
624
625        // Query device for feature level
626        let mut options = Default::default();
627        unsafe {
628            device.CheckFeatureSupport(
629                D3D12_FEATURE_D3D12_OPTIONS,
630                <*mut D3D12_FEATURE_DATA_D3D12_OPTIONS>::cast(&mut options),
631                std::mem::size_of_val(&options) as u32,
632            )
633        }
634        .map_err(|e| {
635            AllocationError::Internal(format!("ID3D12Device::CheckFeatureSupport failed: {}", e))
636        })?;
637
638        let is_heap_tier1 = options.ResourceHeapTier == D3D12_RESOURCE_HEAP_TIER_1;
639
640        let heap_types = [
641            (
642                MemoryLocation::GpuOnly,
643                D3D12_HEAP_PROPERTIES {
644                    Type: D3D12_HEAP_TYPE_DEFAULT,
645                    ..Default::default()
646                },
647            ),
648            (
649                MemoryLocation::CpuToGpu,
650                D3D12_HEAP_PROPERTIES {
651                    Type: D3D12_HEAP_TYPE_CUSTOM,
652                    CPUPageProperty: D3D12_CPU_PAGE_PROPERTY_WRITE_COMBINE,
653                    MemoryPoolPreference: D3D12_MEMORY_POOL_L0,
654                    ..Default::default()
655                },
656            ),
657            (
658                MemoryLocation::GpuToCpu,
659                D3D12_HEAP_PROPERTIES {
660                    Type: D3D12_HEAP_TYPE_CUSTOM,
661                    CPUPageProperty: D3D12_CPU_PAGE_PROPERTY_WRITE_BACK,
662                    MemoryPoolPreference: D3D12_MEMORY_POOL_L0,
663                    ..Default::default()
664                },
665            ),
666        ];
667
668        let heap_types = if is_heap_tier1 {
669            heap_types
670                .iter()
671                .flat_map(|(memory_location, heap_properties)| {
672                    [
673                        (HeapCategory::Buffer, *memory_location, *heap_properties),
674                        (
675                            HeapCategory::RtvDsvTexture,
676                            *memory_location,
677                            *heap_properties,
678                        ),
679                        (
680                            HeapCategory::OtherTexture,
681                            *memory_location,
682                            *heap_properties,
683                        ),
684                    ]
685                    .to_vec()
686                })
687                .collect::<Vec<_>>()
688        } else {
689            heap_types
690                .iter()
691                .map(|(memory_location, heap_properties)| {
692                    (HeapCategory::All, *memory_location, *heap_properties)
693                })
694                .collect::<Vec<_>>()
695        };
696
697        let memory_types = heap_types
698            .iter()
699            .enumerate()
700            .map(
701                |(i, &(heap_category, memory_location, heap_properties))| MemoryType {
702                    memory_blocks: Vec::default(),
703                    memory_location,
704                    heap_category,
705                    heap_properties,
706                    memory_type_index: i,
707                    active_general_blocks: 0,
708                    committed_allocations: CommittedAllocationStatistics {
709                        num_allocations: 0,
710                        total_size: 0,
711                    },
712                },
713            )
714            .collect::<Vec<_>>();
715
716        Ok(Self {
717            memory_types,
718            device,
719            debug_settings: desc.debug_settings,
720            allocation_sizes: desc.allocation_sizes,
721        })
722    }
723
724    pub fn allocate(&mut self, desc: &AllocationCreateDesc<'_>) -> Result<Allocation> {
725        let size = desc.size;
726        let alignment = desc.alignment;
727
728        let backtrace = Arc::new(if self.debug_settings.store_stack_traces {
729            Backtrace::force_capture()
730        } else {
731            Backtrace::disabled()
732        });
733
734        if self.debug_settings.log_allocations {
735            debug!(
736                "Allocating `{}` of {} bytes with an alignment of {}.",
737                &desc.name, size, alignment
738            );
739            if self.debug_settings.log_stack_traces {
740                let backtrace = Backtrace::force_capture();
741                debug!("Allocation stack trace: {}", backtrace);
742            }
743        }
744
745        if size == 0 || !alignment.is_power_of_two() {
746            return Err(AllocationError::InvalidAllocationCreateDesc);
747        }
748
749        // Find memory type
750        let memory_type = self
751            .memory_types
752            .iter_mut()
753            .find(|memory_type| {
754                let is_location_compatible = desc.location == MemoryLocation::Unknown
755                    || desc.location == memory_type.memory_location;
756
757                let is_category_compatible = memory_type.heap_category == HeapCategory::All
758                    || memory_type.heap_category == desc.resource_category.into();
759
760                is_location_compatible && is_category_compatible
761            })
762            .ok_or(AllocationError::NoCompatibleMemoryTypeFound)?;
763
764        memory_type.allocate(&self.device, desc, backtrace, &self.allocation_sizes)
765    }
766
767    pub fn free(&mut self, allocation: Allocation) -> Result<()> {
768        if self.debug_settings.log_frees {
769            let name = allocation.name.as_deref().unwrap_or("<null>");
770            debug!("Freeing `{}`.", name);
771            if self.debug_settings.log_stack_traces {
772                let backtrace = Backtrace::force_capture();
773                debug!("Free stack trace: {}", backtrace);
774            }
775        }
776
777        if allocation.is_null() {
778            return Ok(());
779        }
780
781        self.memory_types[allocation.memory_type_index].free(allocation)?;
782
783        Ok(())
784    }
785
786    pub fn rename_allocation(&mut self, allocation: &mut Allocation, name: &str) -> Result<()> {
787        allocation.name = Some(name.into());
788
789        if allocation.is_null() {
790            return Ok(());
791        }
792
793        let mem_type = &mut self.memory_types[allocation.memory_type_index];
794        let mem_block = mem_type.memory_blocks[allocation.memory_block_index]
795            .as_mut()
796            .ok_or_else(|| AllocationError::Internal("Memory block must be Some.".into()))?;
797
798        mem_block
799            .sub_allocator
800            .rename_allocation(allocation.chunk_id, name)?;
801
802        Ok(())
803    }
804
805    pub fn report_memory_leaks(&self, log_level: Level) {
806        for (mem_type_i, mem_type) in self.memory_types.iter().enumerate() {
807            for (block_i, mem_block) in mem_type.memory_blocks.iter().enumerate() {
808                if let Some(mem_block) = mem_block {
809                    mem_block
810                        .sub_allocator
811                        .report_memory_leaks(log_level, mem_type_i, block_i);
812                }
813            }
814        }
815    }
816
817    fn d3d12_resource_desc_1(desc: &D3D12_RESOURCE_DESC) -> D3D12_RESOURCE_DESC1 {
818        D3D12_RESOURCE_DESC1 {
819            Dimension: desc.Dimension,
820            Alignment: desc.Alignment,
821            Width: desc.Width,
822            Height: desc.Height,
823            DepthOrArraySize: desc.DepthOrArraySize,
824            MipLevels: desc.MipLevels,
825            Format: desc.Format,
826            SampleDesc: desc.SampleDesc,
827            Layout: desc.Layout,
828            Flags: desc.Flags,
829            // TODO: This is the only new field
830            SamplerFeedbackMipRegion: D3D12_MIP_REGION::default(),
831        }
832    }
833
834    fn resource_allocation_info(
835        device: &ID3D12DeviceVersion,
836        desc: &ResourceCreateDesc<'_>,
837    ) -> D3D12_RESOURCE_ALLOCATION_INFO {
838        match device {
839            ID3D12DeviceVersion::Device(device) => unsafe {
840                device.GetResourceAllocationInfo(0, &[*desc.resource_desc])
841            },
842            ID3D12DeviceVersion::Device10(device) => unsafe {
843                device.GetResourceAllocationInfo(0, &[*desc.resource_desc])
844            },
845            ID3D12DeviceVersion::Device12(device) => unsafe {
846                let resource_desc1 = Self::d3d12_resource_desc_1(desc.resource_desc);
847
848                let resource_descs = &[resource_desc1];
849
850                // We always have one resource desc, hence we only have one mapping castable format array
851                let num_castable_formats = desc.castable_formats.len() as u32;
852                let num_castable_formats_array = &[num_castable_formats];
853
854                let castable_formats_array = &[desc.castable_formats.as_ptr()];
855
856                let (num_castable_formats_opt, castable_formats_opt) = if num_castable_formats > 0 {
857                    (
858                        Some(num_castable_formats_array.as_ptr()),
859                        Some(castable_formats_array.as_ptr()),
860                    )
861                } else {
862                    (None, None)
863                };
864
865                device.GetResourceAllocationInfo3(
866                    0,
867                    resource_descs.len() as u32,
868                    resource_descs.as_ptr(),
869                    num_castable_formats_opt,
870                    castable_formats_opt,
871                    None,
872                )
873            },
874        }
875    }
876
877    /// Create a resource according to the provided parameters.
878    /// Created resources should be freed at the end of their lifetime by calling [`Self::free_resource()`].
879    pub fn create_resource(&mut self, desc: &ResourceCreateDesc<'_>) -> Result<Resource> {
880        match desc.resource_type {
881            ResourceType::Committed {
882                heap_properties,
883                heap_flags,
884            } => {
885                let mut result: Option<ID3D12Resource> = None;
886
887                let clear_value: Option<*const D3D12_CLEAR_VALUE> =
888                    desc.clear_value.map(|v| -> *const _ { v });
889
890                if let Err(e) = unsafe {
891                    match (&self.device, desc.initial_state_or_layout) {
892                        (_, ResourceStateOrBarrierLayout::ResourceState(_))
893                            if !desc.castable_formats.is_empty() =>
894                        {
895                            return Err(AllocationError::CastableFormatsRequiresEnhancedBarriers)
896                        }
897                        (
898                            ID3D12DeviceVersion::Device12(device),
899                            ResourceStateOrBarrierLayout::BarrierLayout(initial_layout),
900                        ) => {
901                            let resource_desc1 = Self::d3d12_resource_desc_1(desc.resource_desc);
902                            device.CreateCommittedResource3(
903                                *heap_properties,
904                                *heap_flags,
905                                &resource_desc1,
906                                initial_layout,
907                                clear_value,
908                                None, // TODO
909                                Some(desc.castable_formats),
910                                &mut result,
911                            )
912                        }
913                        (_, ResourceStateOrBarrierLayout::BarrierLayout(_))
914                            if !desc.castable_formats.is_empty() =>
915                        {
916                            return Err(AllocationError::CastableFormatsRequiresAtLeastDevice12)
917                        }
918                        (
919                            ID3D12DeviceVersion::Device10(device),
920                            ResourceStateOrBarrierLayout::BarrierLayout(initial_layout),
921                        ) => {
922                            let resource_desc1 = Self::d3d12_resource_desc_1(desc.resource_desc);
923
924                            device.CreateCommittedResource3(
925                                *heap_properties,
926                                *heap_flags,
927                                &resource_desc1,
928                                initial_layout,
929                                clear_value,
930                                None, // TODO
931                                None,
932                                &mut result,
933                            )
934                        }
935                        (_, ResourceStateOrBarrierLayout::BarrierLayout(_)) => {
936                            return Err(AllocationError::BarrierLayoutNeedsDevice10)
937                        }
938                        (device, ResourceStateOrBarrierLayout::ResourceState(initial_state)) => {
939                            device.CreateCommittedResource(
940                                *heap_properties,
941                                *heap_flags,
942                                desc.resource_desc,
943                                initial_state,
944                                clear_value,
945                                &mut result,
946                            )
947                        }
948                    }
949                } {
950                    return Err(AllocationError::Internal(format!(
951                        "ID3D12Device::CreateCommittedResource failed: {}",
952                        e
953                    )));
954                }
955
956                let resource = result.expect("Allocation succeeded but no resource was returned?");
957
958                let allocation_info = Self::resource_allocation_info(&self.device, desc);
959
960                let memory_type = self
961                    .memory_types
962                    .iter_mut()
963                    .find(|memory_type| {
964                        let is_location_compatible = desc.memory_location
965                            == MemoryLocation::Unknown
966                            || desc.memory_location == memory_type.memory_location;
967
968                        let is_category_compatible = memory_type.heap_category == HeapCategory::All
969                            || memory_type.heap_category == desc.resource_category.into();
970
971                        is_location_compatible && is_category_compatible
972                    })
973                    .ok_or(AllocationError::NoCompatibleMemoryTypeFound)?;
974
975                memory_type.committed_allocations.num_allocations += 1;
976                memory_type.committed_allocations.total_size += allocation_info.SizeInBytes;
977
978                Ok(Resource {
979                    name: desc.name.into(),
980                    allocation: None,
981                    resource: Some(resource),
982                    size: allocation_info.SizeInBytes,
983                    memory_location: desc.memory_location,
984                    memory_type_index: Some(memory_type.memory_type_index),
985                })
986            }
987            ResourceType::Placed => {
988                let allocation_desc = {
989                    let allocation_info = Self::resource_allocation_info(&self.device, desc);
990
991                    AllocationCreateDesc {
992                        name: desc.name,
993                        location: desc.memory_location,
994                        size: allocation_info.SizeInBytes,
995                        alignment: allocation_info.Alignment,
996                        resource_category: desc.resource_category,
997                    }
998                };
999
1000                let allocation = self.allocate(&allocation_desc)?;
1001
1002                let mut result: Option<ID3D12Resource> = None;
1003                if let Err(e) = unsafe {
1004                    match (&self.device, desc.initial_state_or_layout) {
1005                        (_, ResourceStateOrBarrierLayout::ResourceState(_))
1006                            if !desc.castable_formats.is_empty() =>
1007                        {
1008                            return Err(AllocationError::CastableFormatsRequiresEnhancedBarriers)
1009                        }
1010                        (
1011                            ID3D12DeviceVersion::Device12(device),
1012                            ResourceStateOrBarrierLayout::BarrierLayout(initial_layout),
1013                        ) => {
1014                            let resource_desc1 = Self::d3d12_resource_desc_1(desc.resource_desc);
1015                            device.CreatePlacedResource2(
1016                                allocation.heap(),
1017                                allocation.offset(),
1018                                &resource_desc1,
1019                                initial_layout,
1020                                None,
1021                                Some(desc.castable_formats),
1022                                &mut result,
1023                            )
1024                        }
1025                        (_, ResourceStateOrBarrierLayout::BarrierLayout(_))
1026                            if !desc.castable_formats.is_empty() =>
1027                        {
1028                            return Err(AllocationError::CastableFormatsRequiresAtLeastDevice12)
1029                        }
1030                        (
1031                            ID3D12DeviceVersion::Device10(device),
1032                            ResourceStateOrBarrierLayout::BarrierLayout(initial_layout),
1033                        ) => {
1034                            let resource_desc1 = Self::d3d12_resource_desc_1(desc.resource_desc);
1035                            device.CreatePlacedResource2(
1036                                allocation.heap(),
1037                                allocation.offset(),
1038                                &resource_desc1,
1039                                initial_layout,
1040                                None,
1041                                None,
1042                                &mut result,
1043                            )
1044                        }
1045                        (_, ResourceStateOrBarrierLayout::BarrierLayout(_)) => {
1046                            return Err(AllocationError::BarrierLayoutNeedsDevice10)
1047                        }
1048                        (device, ResourceStateOrBarrierLayout::ResourceState(initial_state)) => {
1049                            device.CreatePlacedResource(
1050                                allocation.heap(),
1051                                allocation.offset(),
1052                                desc.resource_desc,
1053                                initial_state,
1054                                None,
1055                                &mut result,
1056                            )
1057                        }
1058                    }
1059                } {
1060                    return Err(AllocationError::Internal(format!(
1061                        "ID3D12Device::CreatePlacedResource failed: {}",
1062                        e
1063                    )));
1064                }
1065
1066                let resource = result.expect("Allocation succeeded but no resource was returned?");
1067                let size = allocation.size();
1068                Ok(Resource {
1069                    name: desc.name.into(),
1070                    allocation: Some(allocation),
1071                    resource: Some(resource),
1072                    size,
1073                    memory_location: desc.memory_location,
1074                    memory_type_index: None,
1075                })
1076            }
1077        }
1078    }
1079
1080    /// Free a resource and its memory.
1081    pub fn free_resource(&mut self, mut resource: Resource) -> Result<()> {
1082        // Explicitly drop the resource (which is backed by a refcounted COM object)
1083        // before freeing allocated memory. Windows-rs performs a Release() on drop().
1084        let _ = resource
1085            .resource
1086            .take()
1087            .expect("Resource was already freed.");
1088
1089        if let Some(allocation) = resource.allocation.take() {
1090            self.free(allocation)
1091        } else {
1092            // Dx12 CommittedResources do not have an application managed allocation.
1093            // We only have to update the tracked allocation count and memory usage.
1094            if let Some(memory_type_index) = resource.memory_type_index {
1095                let memory_type = &mut self.memory_types[memory_type_index];
1096
1097                memory_type.committed_allocations.num_allocations -= 1;
1098                memory_type.committed_allocations.total_size -= resource.size;
1099            }
1100            Ok(())
1101        }
1102    }
1103
1104    pub fn generate_report(&self) -> AllocatorReport {
1105        let mut allocations = vec![];
1106        let mut blocks = vec![];
1107        let mut total_reserved_bytes = 0;
1108
1109        for memory_type in &self.memory_types {
1110            for block in memory_type.memory_blocks.iter().flatten() {
1111                total_reserved_bytes += block.size;
1112                let first_allocation = allocations.len();
1113                allocations.extend(block.sub_allocator.report_allocations());
1114                blocks.push(MemoryBlockReport {
1115                    size: block.size,
1116                    allocations: first_allocation..allocations.len(),
1117                });
1118            }
1119        }
1120
1121        let total_allocated_bytes = allocations.iter().map(|report| report.size).sum();
1122
1123        AllocatorReport {
1124            allocations,
1125            blocks,
1126            total_allocated_bytes,
1127            total_reserved_bytes,
1128        }
1129    }
1130}
1131
1132impl fmt::Debug for Allocator {
1133    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1134        self.generate_report().fmt(f)
1135    }
1136}
1137
1138impl Drop for Allocator {
1139    fn drop(&mut self) {
1140        if self.debug_settings.log_leaks_on_shutdown {
1141            self.report_memory_leaks(Level::Warn);
1142        }
1143
1144        // Because Rust drop rules drop members in source-code order (that would be the
1145        // ID3D12Device before the ID3D12Heaps nested in these memory blocks), free
1146        // all remaining memory blocks manually first by dropping.
1147        for mem_type in self.memory_types.iter_mut() {
1148            mem_type.memory_blocks.clear();
1149        }
1150    }
1151}
gpu_allocator/d3d12/mod.rs

gpu_allocator/d3d12/
mod.rs
