vulkan_malloc/
lib.rs

1//! This crate provides an implementation of a general purpose device memory allocator that should
2//! cover common use cases for Vulkan-based applications, while minimising the frequency of
3//! actual allocations of memory blocks from the Vulkan runtime.
4//!
5//! This crate is heavily based on the C++ library
6//! [VulkanMemoryAllocator](https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator)
7//! from AMD.
8//!
9//! For more details about the rationale and implementation of this library, please see
10//! [the documentation of VulkanMemoryAllocator](https://gpuopen-librariesandsdks.github.io/VulkanMemoryAllocator/html/).
11//!
12//! `Allocator` itself is thread-safe - it is both `Send` and `Sync`.
13
14extern crate dacite;
15extern crate array_ext;
16extern crate option_filter;
17
18use std::collections::HashMap;
19use std::sync::Mutex;
20use std::sync::atomic::{AtomicBool, ATOMIC_BOOL_INIT, Ordering as MemOrdering};
21
22mod sync_key;
23use sync_key::{SyncKey, SyncKeyId, SyncLock};
24
25mod list;
26use list::{List, ListItem};
27
28mod allocation;
29use allocation::{Allocation, OwnAllocation, SuballocationType};
30
31use dacite::core::{PhysicalDevice, Device, PhysicalDeviceProperties,
32                   PhysicalDeviceMemoryProperties, Buffer, BufferCreateInfo, Image,
33                   ImageCreateInfo, ImageTiling, MappedMemoryRange, MappedMemory,
34                   MAX_MEMORY_TYPES, MemoryRequirements, MemoryPropertyFlags, MemoryAllocateInfo,
35                   OptionalDeviceSize, Error};
36use dacite::VulkanObject;
37
38use array_ext::Array;
39
40const MB: u64 = 1024 * 1024;
41
42const SMALL_HEAP_MAX_SIZE: u64 = 512 * MB;
43
44/// Used to construct an `Allocator` using the builder pattern.
45pub struct AllocatorBuilder {
46    device: Device,
47    physical: PhysicalDevice,
48    large_heap_block_size: u64,
49    small_heap_block_size: u64,
50}
51
52impl AllocatorBuilder {
53    /// Overrides the default block size for use on "large" heaps, in bytes. The default is
54    /// 256MB.
55    pub fn large_heap_block_size(mut self, size: u64) -> AllocatorBuilder {
56        self.large_heap_block_size = size;
57        self
58    }
59
60    /// Overrides the default block size for use on "small" heaps, in bytes. The default is
61    /// 64MB.
62    pub fn small_heap_block_size(mut self, size: u64) -> AllocatorBuilder {
63        self.small_heap_block_size = size;
64        self
65    }
66
67    /// Finalise and build an `Allocator` instance.
68    pub fn build(self) -> Allocator {
69        Allocator {
70            small_heap_block_size: self.small_heap_block_size,
71            large_heap_block_size: self.large_heap_block_size,
72            device_properties: self.physical.get_properties(),
73            memory_properties: self.physical.get_memory_properties(),
74            allocations: Array::from_fn(|_| Mutex::new(Vec::new())),
75            has_empty_allocation: Array::from_fn(|_| ATOMIC_BOOL_INIT),
76            own_allocations: Array::from_fn(|_| Mutex::new(Vec::new())),
77            buffers_to_memory: Mutex::new(HashMap::new()),
78            images_to_memory: Mutex::new(HashMap::new()),
79            device: self.device,
80        }
81    }
82}
83
84/// Thread-safe device memory allocator
85///
86/// See top-level crate documentation for more detail and examples.
87pub struct Allocator {
88    device: Device,
89    small_heap_block_size: u64,
90    large_heap_block_size: u64,
91    device_properties: PhysicalDeviceProperties,
92    memory_properties: PhysicalDeviceMemoryProperties,
93    allocations: [Mutex<Vec<Allocation>>; MAX_MEMORY_TYPES],
94    has_empty_allocation: [AtomicBool; MAX_MEMORY_TYPES],
95    own_allocations: [Mutex<Vec<OwnAllocation>>; MAX_MEMORY_TYPES],
96    buffers_to_memory: Mutex<HashMap<u64, MappedMemoryRange>>,
97    images_to_memory: Mutex<HashMap<u64, MappedMemoryRange>>,
98}
99
100#[test]
101fn allocator_send_sync_test() {
102    fn foo<T: Send + Sync>() {}
103    foo::<Allocator>();
104}
105
106/// Specifies how memory will be used with respect to transfers between the device and the host.
107#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
108pub enum MemoryUsage {
109    /// No intended memory usage specified.
110    Unknown,
111    /// Memory will be used on the device only, no need to be mapped on host.
112    GpuOnly,
113    /// Memory will be mapped on host. Could be used for transfer to device.
114    CpuOnly,
115    /// Memory will be used for frequent (dynamic) updates from host and reads on device.
116    CpuToGpu,
117    /// Memory will be used for writing on device and readback on host.
118    GpuToCpu,
119}
120
121
122/// In addition to normal `MemoryRequirements`, this struct provides additional details affecting
123/// how the allocator chooses memory to allocate.
124#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
125pub struct AllocatorMemoryRequirements {
126    /// Set to true if this allocation should have its own memory block.
127    ///
128    /// Use it for special, big resources, like fullscreen images used as attachments.
129    ///
130    /// This flag must also be used for host visible resources that you want to map
131    /// simultaneously because otherwise they might end up as regions of the same
132    /// `DeviceMemory`, and mapping the same `DeviceMemory` multiple times is illegal.
133    pub own_memory: bool,
134    /// Intended usage of the allocated memory. If you specify `required_flags` as non-empty,
135    /// you can (but do not have to) leave this set to `MemoryUsage::Unknown`.
136    pub usage: MemoryUsage,
137    /// Flags that must be satisfied by the memory type used for allocation. Can be left empty if
138    /// `usage` is not `MemoryUsage::Unknown`.
139    pub required_flags: MemoryPropertyFlags,
140    /// Flags that determine which memory types should be chosen preferentially over others. If
141    /// this is not empty, it must be a superset of `required_flags`.
142    pub preferred_flags: Option<MemoryPropertyFlags>,
143    /// Set this flag to only try to allocate from existing device memory blocks and never create new blocks.
144    ///
145    /// If the new allocation cannot be placed in any of the existing blocks, allocation
146    /// fails with `Error::OutOfDeviceMemory`.
147    ///
148    /// It makes no sense to set `own_memory` and `never_allocate` at the same time.
149    pub never_allocate: bool,
150}
151
152impl Allocator {
153    /// Creates a builder for creating an `Allocator.`
154    ///
155    /// Simple usage:
156    ///
157    /// ```ignore
158    /// let allocator = Allocator::builder(device, physical_device).build();
159    /// ```
160    pub fn builder(device: Device, physical_device: PhysicalDevice) -> AllocatorBuilder {
161        AllocatorBuilder {
162            device: device,
163            physical: physical_device,
164            large_heap_block_size: 256 * MB,
165            small_heap_block_size: 64 * MB,
166        }
167    }
168
169    /// This algorithm tries to find a memory type that:
170    ///
171    /// - Is allowed by memoryTypeBits.
172    /// - Contains all the flags from pMemoryRequirements->requiredFlags.
173    /// - Matches intended usage.
174    /// - Has as many flags from pMemoryRequirements->preferredFlags as possible.
175    ///
176    /// Returns `Error::FeatureNotPresent` if not found. Receiving such result
177    /// from this function or any other allocating function probably means that your
178    /// device doesn't support any memory type with requested features for the specific
179    /// type of resource you want to use it for. Please check parameters of your
180    /// resource, like image layout (OPTIMAL versus LINEAR) or mip level count.
181    pub fn find_memory_type_index(
182        &self,
183        memory_type_bits: u32,
184        reqs: &AllocatorMemoryRequirements,
185    ) -> Result<u32, Error> {
186        let mut required_flags = reqs.required_flags;
187        let mut preferred_flags = reqs.preferred_flags.unwrap_or(required_flags);
188        if !(required_flags & !preferred_flags).is_empty() {
189            panic!("Preferred flags must be a superset of required flags")
190        }
191        match reqs.usage {
192            MemoryUsage::Unknown => {}
193            MemoryUsage::GpuOnly => {
194                preferred_flags.insert(dacite::core::MemoryPropertyFlags::DEVICE_LOCAL)
195            }
196            MemoryUsage::CpuOnly => {
197                required_flags.insert(
198                    dacite::core::MemoryPropertyFlags::HOST_VISIBLE |
199                        dacite::core::MemoryPropertyFlags::HOST_COHERENT,
200                )
201            }
202            MemoryUsage::CpuToGpu => {
203                required_flags.insert(dacite::core::MemoryPropertyFlags::HOST_VISIBLE);
204                preferred_flags.insert(dacite::core::MemoryPropertyFlags::DEVICE_LOCAL);
205            }
206            MemoryUsage::GpuToCpu => {
207                required_flags.insert(dacite::core::MemoryPropertyFlags::HOST_VISIBLE);
208                preferred_flags.insert(
209                    dacite::core::MemoryPropertyFlags::HOST_COHERENT |
210                        dacite::core::MemoryPropertyFlags::HOST_CACHED,
211                );
212            }
213        }
214
215        let mut memory_type_index = std::u32::MAX;
216        let mut min_cost = std::u32::MAX;
217        let mut memory_type_bit = 1;
218        for index in 0..self.memory_type_count() {
219            if (memory_type_bit & memory_type_bits) != 0 {
220                let current_flags = self.memory_properties.memory_types[index as usize]
221                    .property_flags;
222                if (required_flags & !current_flags).is_empty() {
223                    let current_cost = (preferred_flags & !current_flags).bits().count_ones();
224                    if current_cost < min_cost {
225                        if current_cost == 0 {
226                            return Ok(index);
227                        } else {
228                            memory_type_index = index;
229                            min_cost = current_cost;
230                        }
231                    }
232                }
233            }
234            memory_type_bit <<= 1;
235        }
236        if memory_type_index != std::u32::MAX {
237            Ok(memory_type_index)
238        } else {
239            Err(Error::FeatureNotPresent)
240        }
241    }
242
243    fn memory_type_count(&self) -> u32 {
244        self.memory_properties.memory_types.len() as u32
245    }
246
247    fn preferred_block_size(&self, index: u32) -> u64 {
248        let heap_index = self.memory_properties.memory_types[index as usize].heap_index;
249        let heap_size = self.memory_properties.memory_heaps[heap_index as usize].size;
250        if heap_size <= SMALL_HEAP_MAX_SIZE {
251            self.small_heap_block_size
252        } else {
253            self.large_heap_block_size
254        }
255    }
256
257    fn buffer_image_granularity(&self) -> u64 {
258        self.device_properties.limits.buffer_image_granularity
259    }
260
261    fn allocate_memory_of_type(
262        &self,
263        vulkan_reqs: &MemoryRequirements,
264        other_reqs: &AllocatorMemoryRequirements,
265        memory_type_index: u32,
266        suballoc_type: SuballocationType,
267    ) -> Result<MappedMemoryRange, Error> {
268        let preferred_block_size = self.preferred_block_size(memory_type_index);
269        let own_memory = other_reqs.own_memory ||
270            (!other_reqs.never_allocate && vulkan_reqs.size > preferred_block_size / 2);
271
272        if own_memory {
273            if other_reqs.never_allocate {
274                Err(Error::OutOfDeviceMemory)
275            } else {
276                self.allocate_own_memory(vulkan_reqs.size, suballoc_type, memory_type_index)
277            }
278        } else {
279            let mut allocation_vector =
280                self.allocations[memory_type_index as usize].lock().unwrap();
281            for alloc in allocation_vector.iter_mut() {
282                if let Some(request) = alloc.create_allocation_request(
283                    self.buffer_image_granularity(),
284                    vulkan_reqs.size,
285                    vulkan_reqs.alignment,
286                    suballoc_type,
287                )
288                {
289                    if alloc.is_empty() {
290                        self.has_empty_allocation[memory_type_index as usize]
291                            .store(false, MemOrdering::SeqCst);
292                    }
293                    alloc.alloc(&request, suballoc_type, vulkan_reqs.size);
294                    return Ok(MappedMemoryRange {
295                        memory: alloc.memory().clone(),
296                        offset: request.offset,
297                        size: OptionalDeviceSize::Size(vulkan_reqs.size),
298                        chain: None,
299                    });
300                }
301            }
302
303            if other_reqs.never_allocate {
304                Err(Error::OutOfDeviceMemory)
305            } else {
306                let mut alloc_info = MemoryAllocateInfo {
307                    allocation_size: preferred_block_size,
308                    memory_type_index: memory_type_index,
309                    chain: None,
310                };
311                let mut result = self.device.allocate_memory(&alloc_info, None);
312                if result.is_err() {
313                    alloc_info.allocation_size /= 2;
314                    if alloc_info.allocation_size >= vulkan_reqs.size {
315                        result = self.device.allocate_memory(&alloc_info, None);
316                        if result.is_err() {
317                            alloc_info.allocation_size /= 2;
318                            if alloc_info.allocation_size >= vulkan_reqs.size {
319                                result = self.device.allocate_memory(&alloc_info, None);
320                            }
321                        }
322                    }
323                }
324                let memory = if let Ok(memory) = result {
325                    memory
326                } else {
327                    return self.allocate_own_memory(
328                        vulkan_reqs.size,
329                        suballoc_type,
330                        memory_type_index,
331                    );
332                };
333
334                let (mut alloc, request) = Allocation::new_with_request(
335                    memory.clone(),
336                    alloc_info.allocation_size,
337                    vulkan_reqs.size,
338                );
339                alloc.alloc(&request, suballoc_type, vulkan_reqs.size);
340
341                allocation_vector.push(alloc);
342                Ok(MappedMemoryRange {
343                    memory: memory,
344                    offset: 0,
345                    size: OptionalDeviceSize::Size(vulkan_reqs.size),
346                    chain: None,
347                })
348            }
349        }
350    }
351
352    fn allocate_impl(
353        &self,
354        vulkan_reqs: &MemoryRequirements,
355        other_reqs: &AllocatorMemoryRequirements,
356        suballoc_type: SuballocationType,
357    ) -> Result<MappedMemoryRange, Error> {
358        if other_reqs.own_memory && other_reqs.never_allocate {
359            return Err(Error::OutOfDeviceMemory);
360        }
361        let mut memory_type_bits = vulkan_reqs.memory_type_bits;
362        let mut memory_type_index = self.find_memory_type_index(memory_type_bits, other_reqs)?;
363        if let Ok(memory_range) = self.allocate_memory_of_type(
364            vulkan_reqs,
365            other_reqs,
366            memory_type_index,
367            suballoc_type,
368        )
369        {
370            Ok(memory_range)
371        } else {
372            loop {
373                memory_type_bits &= !(1 << memory_type_index);
374                if let Ok(i) = self.find_memory_type_index(memory_type_bits, other_reqs) {
375                    memory_type_index = i;
376                } else {
377                    return Err(Error::OutOfDeviceMemory);
378                }
379                if let Ok(memory_range) = self.allocate_memory_of_type(
380                    vulkan_reqs,
381                    other_reqs,
382                    memory_type_index,
383                    suballoc_type,
384                )
385                {
386                    return Ok(memory_range);
387                }
388            }
389        }
390    }
391
392    /// Frees memory previously allocated using `allocate` or `allocate_for_buffer` or `allocate_for_image`.
393    pub fn free(&self, mem_range: &MappedMemoryRange) {
394        let mut found = false;
395        let mut allocation_to_delete = None;
396        for memory_type_index in 0..self.memory_type_count() {
397            let mut allocation_vector =
398                self.allocations[memory_type_index as usize].lock().unwrap();
399            if let Some(alloc_index) = allocation::vector_free(&mut allocation_vector, mem_range) {
400                found = true;
401                if allocation_vector[alloc_index].is_empty() &&
402                    self.has_empty_allocation[memory_type_index as usize]
403                        .compare_and_swap(false, true, MemOrdering::SeqCst)
404                {
405                    allocation_to_delete = Some(allocation_vector.remove(alloc_index));
406                    break;
407                }
408                allocation::incrementally_sort_allocations(&mut allocation_vector);
409                break;
410            }
411        }
412
413        if found {
414            if let Some(allocation) = allocation_to_delete {
415                drop(allocation);
416            }
417            return;
418        }
419
420        if self.free_own_memory(mem_range) {
421            return;
422        }
423
424        panic!("Attempted to free memory not allocated by this allocator.");
425    }
426
427    fn free_own_memory(&self, mem_range: &MappedMemoryRange) -> bool {
428        let id = mem_range.memory.id();
429        let mut memory = None;
430
431        for own_allocations in &self.own_allocations[..self.memory_type_count() as usize] {
432            let mut own_allocations = own_allocations.lock().unwrap();
433            if let Ok(index) = own_allocations.binary_search_by(
434                |allocation| allocation.memory.id().cmp(&id),
435            )
436            {
437                assert_eq!(
438                    mem_range.size,
439                    OptionalDeviceSize::Size(own_allocations[index].size)
440                );
441                assert_eq!(mem_range.offset, 0);
442                memory = Some(own_allocations.remove(index).memory);
443                break;
444            }
445        }
446        if let Some(memory) = memory {
447            drop(memory);
448            true
449        } else {
450            false
451        }
452    }
453
454    /// General purpose memory allocation.
455    ///
456    /// Memory allocated with this function should be freed using `free`.
457    ///
458    /// It is recommended to use `allocate_for_buffer`, `allocate_for_image`,
459    /// `create_buffer`, `create_image` instead whenever possible.
460    pub fn allocate(
461        &self,
462        vulkan_reqs: &MemoryRequirements,
463        other_reqs: &AllocatorMemoryRequirements,
464    ) -> Result<MappedMemoryRange, Error> {
465        self.allocate_impl(vulkan_reqs, other_reqs, SuballocationType::Unknown)
466    }
467
468    /// Memory allocated with this function should be freed using `free`.
469    pub fn allocate_for_image(
470        &self,
471        image: Image,
472        reqs: &AllocatorMemoryRequirements,
473    ) -> Result<MappedMemoryRange, Error> {
474        let vulkan_reqs = image.get_memory_requirements();
475        self.allocate_impl(&vulkan_reqs, reqs, SuballocationType::ImageUnknown)
476    }
477
478    /// Memory allocated with this function should be freed using `free`.
479    pub fn allocate_for_buffer(
480        &self,
481        buffer: Buffer,
482        reqs: &AllocatorMemoryRequirements,
483    ) -> Result<MappedMemoryRange, Error> {
484        let vulkan_reqs = buffer.get_memory_requirements();
485        self.allocate_impl(&vulkan_reqs, reqs, SuballocationType::Buffer)
486    }
487
488    /// This function automatically:
489    ///
490    /// - Creates a buffer.
491    /// - Allocates appropriate memory for it.
492    /// - Binds the buffer with the memory.
493    ///
494    /// Make sure to call `free_buffer` when finished with the returned buffer. Do not use `free`.
495    pub fn create_buffer(
496        &self,
497        create_info: &BufferCreateInfo,
498        reqs: &AllocatorMemoryRequirements,
499    ) -> Result<(Buffer, MappedMemoryRange), Error> {
500        let buffer = self.device.create_buffer(create_info, None)?;
501        let vulkan_reqs = buffer.get_memory_requirements();
502        let mem_range = self.allocate_impl(
503            &vulkan_reqs,
504            reqs,
505            SuballocationType::Buffer,
506        )?;
507        match buffer.bind_memory(mem_range.memory.clone(), mem_range.offset) {
508            Ok(_) => {
509                self.buffers_to_memory.lock().unwrap().insert(
510                    buffer.id(),
511                    mem_range.clone(),
512                );
513                Ok((buffer, mem_range))
514            }
515            Err(e) => {
516                self.free(&mem_range);
517                Err(e)
518            }
519        }
520    }
521
522    ///Frees internal resources and memory for a buffer created by `create_buffer`.
523    pub fn free_buffer(&self, buffer: Buffer) {
524        let mut buffers_to_memory = self.buffers_to_memory.lock().unwrap();
525        let mem_range = buffers_to_memory.remove(&buffer.id()).expect(
526            "Tried to free buffer not created by allocator.",
527        );
528        self.free(&mem_range);
529    }
530
531    /// This function automatically:
532    ///
533    /// - Creates an image.
534    /// - Allocates appropriate memory for it.
535    /// - Binds the image with the memory.
536    ///
537    /// Make sure to call `free_image` when finished with the returned image. Do not use `free`.
538    pub fn create_image(
539        &self,
540        create_info: &ImageCreateInfo,
541        reqs: &AllocatorMemoryRequirements,
542    ) -> Result<(Image, MappedMemoryRange), Error> {
543        let image = self.device.create_image(create_info, None)?;
544        let vulkan_reqs = image.get_memory_requirements();
545        let mem_range = self.allocate_impl(
546            &vulkan_reqs,
547            reqs,
548            if create_info.tiling == ImageTiling::Optimal {
549                SuballocationType::ImageOptimal
550            } else {
551                SuballocationType::ImageLinear
552            },
553        )?;
554        match image.bind_memory(mem_range.memory.clone(), mem_range.offset) {
555            Ok(_) => {
556                self.images_to_memory.lock().unwrap().insert(
557                    image.id(),
558                    mem_range.clone(),
559                );
560                Ok((image, mem_range))
561            }
562            Err(e) => {
563                self.free(&mem_range);
564                Err(e)
565            }
566        }
567    }
568
569    ///Frees internal resources and memory for an image created by `create_image`.
570    pub fn free_image(&self, image: Image) {
571        let mut images_to_memory = self.images_to_memory.lock().unwrap();
572        let mem_range = images_to_memory.remove(&image.id()).expect(
573            "Tried to free image not created by allocator.",
574        );
575        self.free(&mem_range);
576    }
577
578    /// Feel free to use `DeviceMemory::map` on your own if you want, but
579    /// just for convenience and to make sure correct offset and size is always
580    /// specified, usage of `map_memory` is recommended.
581    pub fn map_memory(range: &MappedMemoryRange) -> Result<MappedMemory, Error> {
582        range.memory.map(
583            range.offset,
584            range.size,
585            dacite::core::MemoryMapFlags::empty(),
586        )
587    }
588
589    fn allocate_own_memory(
590        &self,
591        size: u64,
592        suballoc_type: SuballocationType,
593        memory_type_index: u32,
594    ) -> Result<MappedMemoryRange, Error> {
595        let allocation = OwnAllocation {
596            memory: self.device.allocate_memory(
597                &MemoryAllocateInfo {
598                    allocation_size: size,
599                    memory_type_index: memory_type_index,
600                    chain: None,
601                },
602                None,
603            )?,
604            size: size,
605            type_: suballoc_type,
606        };
607        let memory = allocation.memory().clone();
608        let alloc_id = memory.id();
609        {
610            let mut own_allocations = self.own_allocations[memory_type_index as usize]
611                .lock()
612                .unwrap();
613            let insert_index = own_allocations
614                .binary_search_by(|a| a.memory().id().cmp(&alloc_id))
615                .unwrap_or_else(|e| e);
616            own_allocations.insert(insert_index, allocation);
617        }
618        Ok(MappedMemoryRange {
619            memory: memory,
620            offset: 0,
621            size: OptionalDeviceSize::Size(size),
622            chain: None,
623        })
624    }
625}
vulkan_malloc/lib.rs

vulkan_malloc/
lib.rs
