vk_graph/driver/
buffer.rs

1//! Buffer resource types
2
3use {
4    super::{DriverError, SharingMode, device::Device, pipeline_stage_access_flags},
5    ash::vk,
6    derive_builder::Builder,
7    gpu_allocator::{
8        MemoryLocation,
9        vulkan::{Allocation, AllocationCreateDesc, AllocationScheme},
10    },
11    log::trace,
12    log::warn,
13    smallvec::{SmallVec, smallvec},
14    std::{
15        fmt::{Debug, Formatter},
16        iter::once,
17        mem::{ManuallyDrop, take},
18        ops::{DerefMut, Range},
19        sync::atomic::{AtomicU8, AtomicU64, Ordering},
20        thread::panicking,
21    },
22    vk_sync::AccessType,
23};
24
25#[cfg(feature = "parking_lot")]
26use parking_lot::{Mutex, MutexGuard};
27
28#[cfg(not(feature = "parking_lot"))]
29use std::sync::{Mutex, MutexGuard};
30
31type AccessRuns = RunMap<AccessType>;
32
33/// Smart pointer handle to a [buffer] object.
34///
35/// Also contains information about the object.
36///
37/// ```no_run
38/// # use ash::vk;
39/// # use vk_graph::driver::DriverError;
40/// # use vk_graph::driver::device::{Device, DeviceInfo};
41/// # use vk_graph::driver::buffer::{Buffer, BufferInfo};
42/// # fn main() -> Result<(), DriverError> {
43/// # let device = Device::create(DeviceInfo::default())?;
44/// let info = BufferInfo::device_mem(1_024, vk::BufferUsageFlags::STORAGE_BUFFER);
45/// let my_buf = Buffer::create(&device, info)?;
46///
47/// assert_eq!(my_buf.info, info);
48/// assert_ne!(my_buf.handle, vk::Buffer::null());
49/// # Ok(()) }
50/// ```
51///
52/// [buffer]: https://registry.khronos.org/vulkan/specs/latest/man/html/VkBuffer.html
53#[read_only::cast]
54pub struct Buffer {
55    access_runs: Mutex<AccessRuns>,
56    allocation: ManuallyDrop<Allocation>,
57
58    /// The device which owns this buffer resource.
59    ///
60    /// _Note:_ This field is read-only.
61    #[readonly]
62    pub device: Device,
63
64    /// The native Vulkan resource handle of this buffer.
65    ///
66    /// _Note:_ This field is read-only.
67    #[readonly]
68    pub handle: vk::Buffer,
69
70    /// Information used to create this resource.
71    ///
72    /// _Note:_ This field is read-only.
73    #[readonly]
74    pub info: BufferInfo,
75
76    sharing: Sharing,
77}
78
79impl Buffer {
80    /// Creates a new buffer on the given device.
81    ///
82    /// # Examples
83    ///
84    /// Basic usage:
85    ///
86    /// ```no_run
87    /// # use std::sync::Arc;
88    /// # use ash::vk;
89    /// # use vk_graph::driver::DriverError;
90    /// # use vk_graph::driver::device::{Device, DeviceInfo};
91    /// # use vk_graph::driver::buffer::{Buffer, BufferInfo};
92    /// # fn main() -> Result<(), DriverError> {
93    /// # let device = Device::create(DeviceInfo::default())?;
94    /// const SIZE: vk::DeviceSize = 1024;
95    /// let info = BufferInfo::host_mem(SIZE, vk::BufferUsageFlags::UNIFORM_BUFFER);
96    /// let buf = Buffer::create(&device, info)?;
97    ///
98    /// assert_ne!(buf.handle, vk::Buffer::null());
99    /// assert_eq!(buf.info.size, SIZE);
100    /// # Ok(()) }
101    /// ```
102    #[profiling::function]
103    pub fn create(device: &Device, info: impl Into<BufferInfo>) -> Result<Self, DriverError> {
104        let info = info.into();
105
106        trace!("create: {:?}", info);
107
108        debug_assert_ne!(info.size, 0, "Size must be non-zero");
109
110        let device = device.clone();
111        let buffer_info = vk::BufferCreateInfo::default()
112            .size(info.size)
113            .usage(info.usage)
114            .sharing_mode(info.sharing_mode);
115
116        let buffer_info = if info.sharing_mode == vk::SharingMode::CONCURRENT {
117            buffer_info.queue_family_indices(&device.physical.queue_family_indices)
118        } else {
119            buffer_info
120        };
121        let handle = unsafe {
122            device.create_buffer(&buffer_info, None).map_err(|err| {
123                warn!("unable to create buffer: {err}");
124
125                DriverError::Unsupported
126            })?
127        };
128        let mut requirements = unsafe { device.get_buffer_memory_requirements(handle) };
129        requirements.alignment = requirements.alignment.max(info.alignment);
130
131        let allocation_scheme = if info.alloc_dedicated {
132            AllocationScheme::DedicatedBuffer(handle)
133        } else {
134            AllocationScheme::GpuAllocatorManaged
135        };
136        let location = if info.host_writable {
137            MemoryLocation::CpuToGpu
138        } else if info.host_readable {
139            MemoryLocation::GpuToCpu
140        } else {
141            MemoryLocation::GpuOnly
142        };
143        let allocation = {
144            profiling::scope!("allocate");
145
146            Device::with_allocator(&device, |allocator| {
147                allocator
148                    .allocate(&AllocationCreateDesc {
149                        name: "buffer",
150                        requirements,
151                        location,
152                        linear: true, // Buffers are always linear
153                        allocation_scheme,
154                    })
155                    .map_err(|err| {
156                        warn!("unable to allocate buffer memory: {err}");
157
158                        unsafe {
159                            device.destroy_buffer(handle, None);
160                        }
161
162                        DriverError::from_alloc_err(err)
163                    })
164                    .and_then(|allocation| {
165                        if let Err(err) = unsafe {
166                            device.bind_buffer_memory(
167                                handle,
168                                allocation.memory(),
169                                allocation.offset(),
170                            )
171                        } {
172                            warn!("unable to bind buffer memory: {err}");
173
174                            if let Err(err) = allocator.free(allocation) {
175                                warn!("unable to free buffer allocation: {err}")
176                            }
177
178                            unsafe {
179                                device.destroy_buffer(handle, None);
180                            }
181
182                            Err(DriverError::OutOfMemory)
183                        } else {
184                            Ok(allocation)
185                        }
186                    })
187            })
188        }?;
189
190        debug_assert_ne!(handle, vk::Buffer::null());
191
192        Ok(Self {
193            access_runs: Mutex::new(AccessRuns::new(info.size, AccessType::Nothing)),
194            allocation: ManuallyDrop::new(allocation),
195            device,
196            handle,
197            info,
198            sharing: Sharing::new(info.size, info.sharing_mode),
199        })
200    }
201
202    /// Creates a new mappable buffer on the given device and fills it with the data in `slice`.
203    ///
204    /// # Examples
205    ///
206    /// Basic usage:
207    ///
208    /// ```no_run
209    /// # use std::sync::Arc;
210    /// # use ash::vk;
211    /// # use vk_graph::driver::DriverError;
212    /// # use vk_graph::driver::device::{Device, DeviceInfo};
213    /// # use vk_graph::driver::buffer::{Buffer, BufferInfo};
214    /// # fn main() -> Result<(), DriverError> {
215    /// # let device = Device::create(DeviceInfo::default())?;
216    /// const DATA: [u8; 4] = [0xfe, 0xed, 0xbe, 0xef];
217    /// let buf = Buffer::create_from_slice(&device, vk::BufferUsageFlags::UNIFORM_BUFFER, &DATA)?;
218    ///
219    /// assert_ne!(buf.handle, vk::Buffer::null());
220    /// assert_eq!(buf.info.size, 4);
221    /// assert_eq!(Buffer::mapped_slice(&buf), &DATA);
222    /// # Ok(()) }
223    /// ```
224    #[profiling::function]
225    pub fn create_from_slice(
226        device: &Device,
227        usage: vk::BufferUsageFlags,
228        data: &[u8],
229    ) -> Result<Self, DriverError> {
230        let info = BufferInfo::host_mem(data.len() as _, usage);
231        let mut buffer = Self::create(device, info)?;
232
233        Self::copy_from_slice(&mut buffer, 0, data);
234
235        Ok(buffer)
236    }
237
238    /// Updates a mappable buffer starting at `offset` with the data in `slice`.
239    ///
240    /// # Panics
241    ///
242    /// Panics if the buffer was not created with host-writable memory.
243    ///
244    /// # Examples
245    ///
246    /// Basic usage:
247    ///
248    /// ```no_run
249    /// # use std::sync::Arc;
250    /// # use ash::vk;
251    /// # use vk_graph::driver::DriverError;
252    /// # use vk_graph::driver::device::{Device, DeviceInfo};
253    /// # use vk_graph::driver::buffer::{Buffer, BufferInfo};
254    /// # fn main() -> Result<(), DriverError> {
255    /// # let device = Device::create(DeviceInfo::default())?;
256    /// # let info = BufferInfo::host_mem(4, vk::BufferUsageFlags::empty());
257    /// # let mut my_buf = Buffer::create(&device, info)?;
258    /// const DATA: [u8; 4] = [0xde, 0xad, 0xc0, 0xde];
259    /// Buffer::copy_from_slice(&mut my_buf, 0, &DATA);
260    ///
261    /// assert_eq!(Buffer::mapped_slice(&my_buf), &DATA);
262    /// # Ok(()) }
263    /// ```
264    #[profiling::function]
265    pub fn copy_from_slice(&mut self, offset: vk::DeviceSize, data: &[u8]) {
266        let range = offset as _..offset as usize + data.len();
267        let mapped_data = self.mapped_slice_mut();
268
269        mapped_data[range].copy_from_slice(data);
270    }
271
272    /// Returns the device address of this object.
273    ///
274    /// # Panics
275    ///
276    /// Panics if the buffer was not created with the `SHADER_DEVICE_ADDRESS` usage flag.
277    ///
278    /// # Examples
279    ///
280    /// Basic usage:
281    ///
282    /// ```no_run
283    /// # use std::sync::Arc;
284    /// # use ash::vk;
285    /// # use vk_graph::driver::DriverError;
286    /// # use vk_graph::driver::device::{Device, DeviceInfo};
287    /// # use vk_graph::driver::buffer::{Buffer, BufferInfo};
288    /// # fn main() -> Result<(), DriverError> {
289    /// # let device = Device::create(DeviceInfo::default())?;
290    /// # let info = BufferInfo::host_mem(4, vk::BufferUsageFlags::SHADER_DEVICE_ADDRESS);
291    /// # let my_buf = Buffer::create(&device, info)?;
292    /// let addr = my_buf.device_address();
293    ///
294    /// assert_ne!(addr, 0);
295    /// # Ok(()) }
296    /// ```
297    #[profiling::function]
298    pub fn device_address(&self) -> vk::DeviceAddress {
299        #[cfg(feature = "checked")]
300        assert!(
301            self.info
302                .usage
303                .contains(vk::BufferUsageFlags::SHADER_DEVICE_ADDRESS)
304        );
305
306        unsafe {
307            self.device.get_buffer_device_address(
308                &vk::BufferDeviceAddressInfo::default().buffer(self.handle),
309            )
310        }
311    }
312
313    fn lock_access_runs(&self) -> MutexGuard<'_, AccessRuns> {
314        let access_runs = self.access_runs.lock();
315
316        #[cfg(not(feature = "parking_lot"))]
317        let access_runs = access_runs.expect("poisoned buffer access lock");
318
319        access_runs
320    }
321
322    /// Returns a mapped slice.
323    ///
324    /// # Panics
325    ///
326    /// Panics if the buffer was not created with host-readable memory.
327    ///
328    /// # Examples
329    ///
330    /// Basic usage:
331    ///
332    /// ```no_run
333    /// # use std::sync::Arc;
334    /// # use ash::vk;
335    /// # use vk_graph::driver::DriverError;
336    /// # use vk_graph::driver::device::{Device, DeviceInfo};
337    /// # use vk_graph::driver::buffer::{Buffer, BufferInfo};
338    /// # fn main() -> Result<(), DriverError> {
339    /// # let device = Device::create(DeviceInfo::default())?;
340    /// # const DATA: [u8; 4] = [0; 4];
341    /// # let my_buf = Buffer::create_from_slice(&device, vk::BufferUsageFlags::empty(), &DATA)?;
342    /// // my_buf is mappable and filled with four zeroes
343    /// let data = Buffer::mapped_slice(&my_buf);
344    ///
345    /// assert_eq!(data.len(), 4);
346    /// assert_eq!(data[0], 0x00);
347    /// # Ok(()) }
348    /// ```
349    #[profiling::function]
350    pub fn mapped_slice(&self) -> &[u8] {
351        #[cfg(feature = "checked")]
352        assert!(
353            self.info.host_readable,
354            "Buffer is not readable - create using host_readable flag"
355        );
356
357        &self
358            .allocation
359            .mapped_slice()
360            .expect("missing mapped buffer memory")[0..self.info.size as usize]
361    }
362
363    /// Returns a mapped mutable slice.
364    ///
365    /// # Panics
366    ///
367    /// Panics if the buffer was not created with host-writable memory.
368    ///
369    /// # Examples
370    ///
371    /// Basic usage:
372    ///
373    /// ```no_run
374    /// # use std::sync::Arc;
375    /// # use ash::vk;
376    /// # use glam::Mat4;
377    /// # use vk_graph::driver::DriverError;
378    /// # use vk_graph::driver::device::{Device, DeviceInfo};
379    /// # use vk_graph::driver::buffer::{Buffer, BufferInfo};
380    /// # fn main() -> Result<(), DriverError> {
381    /// # let device = Device::create(DeviceInfo::default())?;
382    /// # const DATA: [u8; 4] = [0; 4];
383    /// # let mut my_buf = Buffer::create_from_slice(
384    /// #     &device,
385    /// #     vk::BufferUsageFlags::empty(),
386    /// #     &DATA,
387    /// # )?;
388    /// let mut data = Buffer::mapped_slice_mut(&mut my_buf);
389    /// data.copy_from_slice(&42f32.to_be_bytes());
390    ///
391    /// assert_eq!(data.len(), 4);
392    /// assert_eq!(data[0], 0x42);
393    /// # Ok(()) }
394    /// ```
395    #[profiling::function]
396    pub fn mapped_slice_mut(&mut self) -> &mut [u8] {
397        #[cfg(feature = "checked")]
398        assert!(
399            self.info.host_writable,
400            "Buffer is not writable - create using host_writable flag"
401        );
402
403        &mut self
404            .allocation
405            .mapped_slice_mut()
406            .expect("missing mapped buffer memory")[0..self.info.size as usize]
407    }
408
409    /// Sets the debugging name assigned to this buffer.
410    pub fn set_debug_name(&self, name: impl AsRef<str>) {
411        Device::try_set_debug_utils_object_name(&self.device, self.handle, &name);
412        Device::try_set_private_data_object_name(
413            &self.device,
414            vk::ObjectType::BUFFER,
415            self.handle,
416            &name,
417        );
418    }
419
420    pub(crate) fn set_sharing_ranges(
421        &self,
422        sharing: SharingMode,
423        sharing_ranges: &[BufferSubresourceRange],
424    ) {
425        if sharing_ranges
426            .iter()
427            .all(|range| range.end != vk::WHOLE_SIZE)
428        {
429            self.sharing
430                .set_ranges(self.info.size, sharing, sharing_ranges.iter().copied());
431
432            return;
433        }
434
435        self.sharing.set_ranges(
436            self.info.size,
437            sharing,
438            sharing_ranges
439                .iter()
440                .copied()
441                .map(|range| range.resolve_whole(self.info.size)),
442        );
443    }
444
445    /// Keeps track of some `next_access` which affects this object.
446    ///
447    /// Returns the previous access for which a pipeline barrier should be used to prevent data
448    /// corruption.
449    #[profiling::function]
450    pub(crate) fn swap_access(
451        &self,
452        next_access: AccessType,
453        access_range: impl Into<BufferSubresourceRange>,
454    ) -> impl Iterator<Item = (AccessType, BufferSubresourceRange)> + '_ {
455        let mut access_range: BufferSubresourceRange = access_range.into();
456
457        if access_range.end == vk::WHOLE_SIZE {
458            access_range.end = self.info.size;
459        }
460
461        RunMapIter::new(self.lock_access_runs(), next_access, access_range)
462    }
463
464    pub(crate) fn swap_accesses<'a, I>(
465        &'a self,
466        accesses: I,
467    ) -> impl Iterator<Item = (AccessType, AccessType, BufferSubresourceRange)> + 'a
468    where
469        I: IntoIterator<Item = (AccessType, BufferSubresourceRange)>,
470        I::IntoIter: 'a,
471    {
472        struct Iter<'a, I>
473        where
474            I: Iterator<Item = (AccessType, BufferSubresourceRange)>,
475        {
476            access_runs: MutexGuard<'a, AccessRuns>,
477            accesses: I,
478            current: Option<(AccessType, RunMapCursor)>,
479            size: vk::DeviceSize,
480        }
481
482        impl<'a, I> Iter<'a, I>
483        where
484            I: Iterator<Item = (AccessType, BufferSubresourceRange)>,
485        {
486            fn new(
487                access_runs: MutexGuard<'a, AccessRuns>,
488                accesses: I,
489                size: vk::DeviceSize,
490            ) -> Self {
491                Self {
492                    access_runs,
493                    accesses,
494                    current: None,
495                    size,
496                }
497            }
498        }
499
500        impl<I> Iterator for Iter<'_, I>
501        where
502            I: Iterator<Item = (AccessType, BufferSubresourceRange)>,
503        {
504            type Item = (AccessType, AccessType, BufferSubresourceRange);
505
506            fn next(&mut self) -> Option<Self::Item> {
507                loop {
508                    if let Some((next_access, cursor)) = &mut self.current {
509                        if let Some((prev_access, range)) =
510                            cursor.next(&mut self.access_runs, *next_access)
511                        {
512                            return Some((*next_access, prev_access, range));
513                        }
514
515                        self.current = None;
516                    }
517
518                    let (next_access, mut access_range) = self.accesses.next()?;
519                    if access_range.end == vk::WHOLE_SIZE {
520                        access_range.end = self.size;
521                    }
522
523                    self.current = Some((
524                        next_access,
525                        RunMapCursor::new(&self.access_runs, access_range),
526                    ));
527                }
528            }
529        }
530
531        impl<I> Drop for Iter<'_, I>
532        where
533            I: Iterator<Item = (AccessType, BufferSubresourceRange)>,
534        {
535            fn drop(&mut self) {
536                while self.next().is_some() {}
537            }
538        }
539
540        let accesses = accesses.into_iter();
541        let (min_accesses, _) = accesses.size_hint();
542        let mut access_runs = self.lock_access_runs();
543        access_runs.runs.reserve(min_accesses.saturating_mul(2));
544
545        Iter::new(access_runs, accesses, self.info.size)
546    }
547
548    /// Returns compact synchronization information for the buffer's current access ranges.
549    pub fn sync_info(&self) -> BufferSyncInfo {
550        let ranges = self
551            .sync_info_with_sharing()
552            .map(|(range, sharing)| range.into_public(sharing))
553            .collect();
554
555        BufferSyncInfo { ranges }
556    }
557
558    pub(crate) fn sync_info_with_sharing(
559        &self,
560    ) -> impl Iterator<Item = (BufferSubresourceSyncInfo, SharingMode)> + '_ {
561        self.sync_info_with_sharing_range(BufferSubresourceRange {
562            start: 0,
563            end: self.info.size,
564        })
565    }
566
567    pub(crate) fn sync_info_with_sharing_range(
568        &self,
569        query_range: BufferSubresourceRange,
570    ) -> impl Iterator<Item = (BufferSubresourceSyncInfo, SharingMode)> + '_ {
571        struct SyncInfoIter<'a> {
572            access_runs: MutexGuard<'a, AccessRuns>,
573            access_run_idx: usize,
574            query_range: BufferSubresourceRange,
575            sharing_run: Option<(SharingMode, BufferSubresourceRange)>,
576            sharing_runs: SharingRunIter<'a>,
577        }
578
579        impl Iterator for SyncInfoIter<'_> {
580            type Item = (BufferSubresourceSyncInfo, SharingMode);
581
582            fn next(&mut self) -> Option<Self::Item> {
583                while self.access_run_idx < self.access_runs.runs.len() {
584                    let (access, start) = self.access_runs.runs[self.access_run_idx];
585                    let end = self
586                        .access_runs
587                        .runs
588                        .get(self.access_run_idx + 1)
589                        .map(|(_, next_start)| *next_start)
590                        .unwrap_or(self.access_runs.size);
591                    let access_range = BufferSubresourceRange { start, end };
592
593                    let Some(access_range) = access_range.intersection(self.query_range) else {
594                        if end <= self.query_range.start {
595                            self.access_run_idx += 1;
596
597                            continue;
598                        }
599
600                        return None;
601                    };
602
603                    let (sharing, sharing_run_range) = self.sharing_run?;
604
605                    let Some(range) = access_range.intersection(sharing_run_range) else {
606                        if sharing_run_range.end <= access_range.start {
607                            self.sharing_run = self.sharing_runs.next();
608                        } else {
609                            self.access_run_idx += 1;
610                        }
611
612                        continue;
613                    };
614
615                    if sharing_run_range.end <= access_range.end {
616                        self.sharing_run = self.sharing_runs.next();
617                    }
618
619                    if access_range.end <= sharing_run_range.end {
620                        self.access_run_idx += 1;
621                    }
622
623                    return Some((
624                        BufferSubresourceSyncInfo::from_access(access, range),
625                        sharing,
626                    ));
627                }
628
629                None
630            }
631        }
632
633        let access_runs = self.access_runs.lock();
634
635        #[cfg(not(feature = "parking_lot"))]
636        let access_runs = access_runs.expect("poisoned buffer access lock");
637
638        let query_range = query_range.resolve_whole(self.info.size);
639        let access_run_idx = access_runs.run_index_at(query_range.start);
640        let mut sharing_runs = self.sharing.ranges_in(query_range);
641        let sharing_run = sharing_runs.next();
642
643        SyncInfoIter {
644            access_runs,
645            access_run_idx,
646            query_range,
647            sharing_run,
648            sharing_runs,
649        }
650    }
651
652    /// Sets the debugging name assigned to this buffer.
653    pub fn with_debug_name(self, name: impl AsRef<str>) -> Self {
654        self.set_debug_name(name);
655
656        self
657    }
658}
659
660impl Debug for Buffer {
661    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
662        let mut res = f.debug_struct(stringify!(Buffer));
663
664        if let Some(debug_name) =
665            &Device::private_data_object_name(&self.device, vk::ObjectType::BUFFER, self.handle)
666        {
667            res.field("debug_name", debug_name);
668        }
669
670        res.field("handle", &self.handle).finish_non_exhaustive()
671    }
672}
673
674impl Drop for Buffer {
675    #[profiling::function]
676    fn drop(&mut self) {
677        if panicking() {
678            return;
679        }
680
681        {
682            profiling::scope!("deallocate");
683
684            Device::with_allocator(&self.device, |allocator| {
685                allocator.free(unsafe { ManuallyDrop::take(&mut self.allocation) })
686            })
687        }
688        .unwrap_or_else(|err| warn!("unable to free buffer allocation: {err}"));
689
690        Device::try_clear_private_data_object_name(
691            &self.device,
692            vk::ObjectType::BUFFER,
693            self.handle,
694        );
695
696        unsafe {
697            self.device.destroy_buffer(self.handle, None);
698        }
699    }
700}
701
702impl Eq for Buffer {}
703
704impl PartialEq for Buffer {
705    fn eq(&self, other: &Self) -> bool {
706        self.handle == other.handle
707    }
708}
709
710/// Information used to create a [`Buffer`] instance.
711///
712/// See [`VkBufferCreateInfo`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkBufferCreateInfo.html).
713#[derive(Builder, Clone, Copy, Debug, Eq, Hash, PartialEq)]
714#[builder(
715    build_fn(private, name = "fallible_build"),
716    derive(Clone, Copy, Debug),
717    pattern = "owned"
718)]
719pub struct BufferInfo {
720    /// Byte alignment of the base device address of the buffer.
721    ///
722    /// Must be a power of two.
723    #[builder(default = "1")]
724    pub alignment: vk::DeviceSize,
725
726    /// Specifies a dedicated memory allocation managed by the Vulkan driver and not by the
727    /// internal memory allocation pool transient resources share.
728    ///
729    /// The driver may optimize access to dedicated buffers.
730    #[builder(default)]
731    pub alloc_dedicated: bool,
732
733    /// Specifies a buffer whose memory is host-visible and may be mapped for reads.
734    ///
735    /// Memory optimal for CPU readback of data may be used.
736    ///
737    #[builder(default)]
738    pub host_readable: bool,
739
740    /// Specifies a buffer whose memory is host-visible and may be mapped for writes.
741    ///
742    /// Memory optimal for uploading data to the GPU and potentially for constant buffers may be
743    /// used.
744    ///
745    #[builder(default)]
746    pub host_writable: bool,
747
748    /// Controls whether the buffer is accessible from a single queue family (`EXCLUSIVE`) or
749    /// from all queues (`CONCURRENT`).
750    #[builder(default = "vk::SharingMode::EXCLUSIVE")]
751    pub sharing_mode: vk::SharingMode,
752
753    /// Size in bytes of the buffer to be created.
754    #[builder(default)]
755    pub size: vk::DeviceSize,
756
757    /// A bitmask specifying the allowed usages of the buffer.
758    ///
759    /// See [`VkBufferUsageFlagBits`](https://registry.khronos.org/vulkan/specs/latest/man/html/VkBufferUsageFlagBits.html).
760    #[builder(default)]
761    pub usage: vk::BufferUsageFlags,
762}
763
764impl BufferInfo {
765    /// Creates a default `BufferInfoBuilder`.
766    pub fn builder() -> BufferInfoBuilder {
767        Default::default()
768    }
769
770    /// Specifies a non-mappable buffer with the given `size` and `usage` values.
771    ///
772    /// Device-local memory (located on the GPU) is used.
773    #[inline(always)]
774    pub const fn device_mem(size: vk::DeviceSize, usage: vk::BufferUsageFlags) -> BufferInfo {
775        BufferInfo {
776            alignment: 1,
777            alloc_dedicated: false,
778            host_readable: false,
779            host_writable: false,
780            sharing_mode: vk::SharingMode::EXCLUSIVE,
781            size,
782            usage,
783        }
784    }
785
786    /// Specifies a mappable buffer with the given `size` and `usage` values.
787    ///
788    /// Host-local memory (located in CPU-accessible RAM) is used.
789    ///
790    /// # Note
791    ///
792    /// For convenience the given usage value will be bitwise OR'd with
793    /// `TRANSFER_DST | TRANSFER_SRC`.
794    #[inline(always)]
795    pub const fn host_mem(size: vk::DeviceSize, usage: vk::BufferUsageFlags) -> BufferInfo {
796        let usage = vk::BufferUsageFlags::from_raw(
797            usage.as_raw()
798                | vk::BufferUsageFlags::TRANSFER_DST.as_raw()
799                | vk::BufferUsageFlags::TRANSFER_SRC.as_raw(),
800        );
801
802        BufferInfo {
803            alignment: 1,
804            alloc_dedicated: false,
805            host_readable: true,
806            host_writable: true,
807            sharing_mode: vk::SharingMode::EXCLUSIVE,
808            size,
809            usage,
810        }
811    }
812
813    /// Converts a `BufferInfo` into a `BufferInfoBuilder`.
814    pub fn into_builder(self) -> BufferInfoBuilder {
815        BufferInfoBuilder {
816            alignment: Some(self.alignment),
817            alloc_dedicated: Some(self.alloc_dedicated),
818            host_readable: Some(self.host_readable),
819            host_writable: Some(self.host_writable),
820            sharing_mode: Some(self.sharing_mode),
821            size: Some(self.size),
822            usage: Some(self.usage),
823        }
824    }
825
826    /// Returns `true` if this information specifies host-visible memory.
827    pub fn is_host_visible(&self) -> bool {
828        self.host_readable | self.host_writable
829    }
830}
831
832impl From<BufferInfoBuilder> for BufferInfo {
833    fn from(info: BufferInfoBuilder) -> Self {
834        info.build()
835    }
836}
837
838impl BufferInfoBuilder {
839    /// Builds a new `BufferInfo`.
840    ///
841    /// If `alignment` is not a power of two and the `checked` feature is active this function will
842    /// panic.
843    #[inline(always)]
844    pub fn build(self) -> BufferInfo {
845        let res = self.fallible_build().expect("all fields have defaults");
846
847        #[cfg(feature = "checked")]
848        assert!(
849            res.alignment.is_power_of_two(),
850            "Alignment must be a power of two"
851        );
852
853        res
854    }
855}
856
857/// Specifies a range of buffer data.
858#[derive(Clone, Copy, Debug, Eq, PartialEq)]
859pub struct BufferSubresourceRange {
860    /// The start of range.
861    pub start: vk::DeviceSize,
862
863    /// The exclusive end of the range.
864    pub end: vk::DeviceSize,
865}
866
867impl BufferSubresourceRange {
868    pub(crate) fn contains(self, other: Self) -> bool {
869        self.start <= other.start && self.end >= other.end
870    }
871
872    pub(crate) fn intersection(self, other: Self) -> Option<Self> {
873        let start = self.start.max(other.start);
874        let end = self.end.min(other.end);
875
876        (start < end).then_some(Self { start, end })
877    }
878
879    #[cfg(test)]
880    pub(crate) fn intersects(self, other: Self) -> bool {
881        self.start < other.end && self.end > other.start
882    }
883
884    pub(crate) fn resolve_whole(mut self, size: vk::DeviceSize) -> Self {
885        if self.end == vk::WHOLE_SIZE {
886            self.end = size;
887        }
888
889        self
890    }
891}
892
893impl From<BufferInfo> for BufferSubresourceRange {
894    fn from(info: BufferInfo) -> Self {
895        Self {
896            start: 0,
897            end: info.size,
898        }
899    }
900}
901
902impl From<Range<vk::DeviceSize>> for BufferSubresourceRange {
903    fn from(range: Range<vk::DeviceSize>) -> Self {
904        Self {
905            start: range.start,
906            end: range.end,
907        }
908    }
909}
910
911impl From<BufferSubresourceRange> for Range<vk::DeviceSize> {
912    fn from(range: BufferSubresourceRange) -> Self {
913        range.start..range.end
914    }
915}
916
917/// Synchronization information for one accessed buffer range.
918#[derive(Clone, Copy, Debug, Eq, PartialEq)]
919pub struct BufferSubresourceSyncInfo {
920    /// Access types performed by `stage_mask`.
921    pub access_mask: vk::AccessFlags,
922
923    /// Queue-family ownership for this range, when exclusive ownership is known.
924    pub queue_family_index: Option<u32>,
925
926    /// The tracked buffer range.
927    pub range: BufferSubresourceRange,
928
929    /// Pipeline stages that access `range`.
930    pub stage_mask: vk::PipelineStageFlags,
931}
932
933impl BufferSubresourceSyncInfo {
934    fn can_merge(self, other: Self) -> bool {
935        self.stage_mask == other.stage_mask
936            && self.access_mask == other.access_mask
937            && self.queue_family_index == other.queue_family_index
938            && self.range.end == other.range.start
939    }
940
941    fn from_access(access: AccessType, range: BufferSubresourceRange) -> Self {
942        let (stage_mask, access_mask) = pipeline_stage_access_flags(access);
943
944        Self {
945            access_mask,
946            queue_family_index: None,
947            range,
948            stage_mask,
949        }
950    }
951
952    fn into_public(self, sharing: SharingMode) -> Self {
953        Self {
954            queue_family_index: match sharing {
955                SharingMode::Concurrent | SharingMode::Exclusive(None) => None,
956                SharingMode::Exclusive(Some((queue_family_index, _))) => Some(queue_family_index),
957            },
958            ..self
959        }
960    }
961
962    fn merge(&mut self, other: Self) {
963        self.range.end = other.range.end;
964    }
965}
966
967/// Synchronization information for a buffer.
968#[derive(Clone, Debug, Eq, PartialEq)]
969pub struct BufferSyncInfo {
970    /// Access state for the tracked buffer ranges.
971    pub ranges: Box<[BufferSubresourceSyncInfo]>,
972}
973
974impl BufferSyncInfo {
975    /// Compacts adjacent ranges with identical synchronization requirements.
976    ///
977    /// Runs in linear time over `ranges`. The implementation reuses the existing range storage by
978    /// converting the boxed slice into a vector, compacting entries in place, and converting it back
979    /// into a boxed slice.
980    pub fn compact(&mut self) {
981        let ranges = take(&mut self.ranges);
982        let mut ranges = ranges.into_vec();
983        let mut compacted_len = 0;
984
985        for idx in 0..ranges.len() {
986            let sync_info = ranges[idx];
987
988            if compacted_len > 0 && ranges[compacted_len - 1].can_merge(sync_info) {
989                ranges[compacted_len - 1].merge(sync_info);
990            } else {
991                ranges[compacted_len] = sync_info;
992                compacted_len += 1;
993            }
994        }
995
996        ranges.truncate(compacted_len);
997        self.ranges = ranges.into_boxed_slice();
998    }
999
1000    /// Returns a compacted copy of this synchronization snapshot.
1001    ///
1002    /// This has the same linear-time and in-place storage characteristics as [`Self::compact`], but
1003    /// consumes and returns the snapshot for use in iterator chains or expression-oriented code.
1004    pub fn into_compacted(mut self) -> Self {
1005        self.compact();
1006        self
1007    }
1008}
1009
1010#[derive(Debug)]
1011struct ExclusiveSharing {
1012    sharing_runs: Mutex<RunMap<SharingMode>>,
1013    sharing_runs_state: AtomicU8,
1014    uniform: AtomicU64,
1015}
1016
1017impl ExclusiveSharing {
1018    fn new(size: vk::DeviceSize) -> Self {
1019        let sharing = SharingMode::Exclusive(None);
1020
1021        Self {
1022            sharing_runs: Mutex::new(RunMap::new(size, sharing)),
1023            sharing_runs_state: AtomicU8::new(RunTrackingState::Uniform as _),
1024            uniform: AtomicU64::new(sharing.encode()),
1025        }
1026    }
1027
1028    fn is_sharing_runs_active(&self) -> bool {
1029        self.sharing_runs_state() == RunTrackingState::Dense
1030    }
1031
1032    fn is_promoting(&self) -> bool {
1033        self.sharing_runs_state() == RunTrackingState::Promoting
1034    }
1035
1036    fn promote_and_set_ranges<I>(
1037        &self,
1038        size: vk::DeviceSize,
1039        sharing: SharingMode,
1040        sharing_ranges: I,
1041    ) where
1042        I: Iterator<Item = BufferSubresourceRange>,
1043    {
1044        let sharing_runs = self.sharing_runs.lock();
1045
1046        #[cfg(not(feature = "parking_lot"))]
1047        let mut sharing_runs = sharing_runs.expect("poisoned buffer sharing lock");
1048
1049        #[cfg(feature = "parking_lot")]
1050        let mut sharing_runs = sharing_runs;
1051
1052        let (min_ranges, _) = sharing_ranges.size_hint();
1053        sharing_runs.runs.reserve(min_ranges.saturating_mul(2));
1054
1055        if self.is_sharing_runs_active() {
1056            for sharing_range in sharing_ranges {
1057                RunMapIter::new(&mut *sharing_runs, sharing, sharing_range).finish();
1058            }
1059
1060            return;
1061        }
1062
1063        self.set_promoting();
1064        let current = SharingMode::decode(self.uniform.load(Ordering::Acquire));
1065        *sharing_runs = RunMap::new(size, current);
1066        sharing_runs.runs.reserve(min_ranges.saturating_mul(2));
1067
1068        for sharing_range in sharing_ranges {
1069            RunMapIter::new(&mut *sharing_runs, sharing, sharing_range).finish();
1070        }
1071
1072        self.set_dense();
1073    }
1074
1075    fn ranges_in(&self, query_range: BufferSubresourceRange) -> SharingRunIter<'_> {
1076        if !self.uses_sharing_runs() {
1077            let sharing = SharingMode::decode(self.uniform.load(Ordering::Acquire));
1078
1079            return SharingRunIter::Constant(Some((sharing, query_range)));
1080        }
1081
1082        let sharing_runs = self.sharing_runs.lock();
1083
1084        #[cfg(not(feature = "parking_lot"))]
1085        let sharing_runs = sharing_runs.expect("poisoned buffer sharing lock");
1086
1087        let run_idx = sharing_runs.run_index_at(query_range.start);
1088
1089        SharingRunIter::Dense {
1090            query_range,
1091            run_idx,
1092            sharing_runs,
1093        }
1094    }
1095
1096    fn set_promoting(&self) {
1097        self.sharing_runs_state
1098            .store(RunTrackingState::Promoting as _, Ordering::Release);
1099    }
1100
1101    fn set_dense(&self) {
1102        self.sharing_runs_state
1103            .store(RunTrackingState::Dense as _, Ordering::Release);
1104    }
1105
1106    fn set_range(
1107        &self,
1108        size: vk::DeviceSize,
1109        sharing: SharingMode,
1110        sharing_range: BufferSubresourceRange,
1111    ) {
1112        if sharing_range.start == 0 && sharing_range.end == size {
1113            self.set_uniform_or_dense(sharing, sharing_range);
1114            return;
1115        }
1116
1117        let sharing_runs = self.sharing_runs.lock();
1118
1119        #[cfg(not(feature = "parking_lot"))]
1120        let mut sharing_runs = sharing_runs.expect("poisoned buffer sharing lock");
1121
1122        #[cfg(feature = "parking_lot")]
1123        let mut sharing_runs = sharing_runs;
1124
1125        if self.is_sharing_runs_active() {
1126            RunMapIter::new(sharing_runs, sharing, sharing_range).finish();
1127
1128            return;
1129        }
1130
1131        self.set_promoting();
1132        let current = SharingMode::decode(self.uniform.load(Ordering::Acquire));
1133        *sharing_runs = RunMap::new(size, current);
1134        RunMapIter::new(sharing_runs, sharing, sharing_range).finish();
1135        self.set_dense();
1136    }
1137
1138    fn set_ranges<I>(&self, size: vk::DeviceSize, sharing: SharingMode, sharing_ranges: I)
1139    where
1140        I: IntoIterator<Item = BufferSubresourceRange>,
1141    {
1142        let mut sharing_ranges = sharing_ranges.into_iter();
1143        let Some(first) = sharing_ranges.next() else {
1144            return;
1145        };
1146
1147        let Some(second) = sharing_ranges.next() else {
1148            self.set_range(size, sharing, first);
1149
1150            return;
1151        };
1152
1153        self.promote_and_set_ranges(
1154            size,
1155            sharing,
1156            once(first).chain(once(second)).chain(sharing_ranges),
1157        );
1158    }
1159
1160    fn set_uniform_or_dense(&self, sharing: SharingMode, sharing_range: BufferSubresourceRange) {
1161        let encoded_sharing = sharing.encode();
1162
1163        loop {
1164            if self.uses_sharing_runs() {
1165                let sharing_runs = self.sharing_runs.lock();
1166
1167                #[cfg(not(feature = "parking_lot"))]
1168                let mut sharing_runs = sharing_runs.expect("poisoned buffer sharing lock");
1169
1170                #[cfg(feature = "parking_lot")]
1171                let sharing_runs = sharing_runs;
1172
1173                RunMapIter::new(sharing_runs, sharing, sharing_range).finish();
1174
1175                return;
1176            }
1177
1178            let current = self.uniform.load(Ordering::Acquire);
1179            if self
1180                .uniform
1181                .compare_exchange(
1182                    current,
1183                    encoded_sharing,
1184                    Ordering::AcqRel,
1185                    Ordering::Acquire,
1186                )
1187                .is_ok()
1188            {
1189                if self.is_promoting() {
1190                    let sharing_runs = self.sharing_runs.lock();
1191
1192                    #[cfg(not(feature = "parking_lot"))]
1193                    let mut sharing_runs = sharing_runs.expect("poisoned buffer sharing lock");
1194
1195                    #[cfg(feature = "parking_lot")]
1196                    let sharing_runs = sharing_runs;
1197
1198                    RunMapIter::new(sharing_runs, sharing, sharing_range).finish();
1199                }
1200
1201                return;
1202            }
1203        }
1204    }
1205
1206    fn sharing_runs_state(&self) -> RunTrackingState {
1207        match self.sharing_runs_state.load(Ordering::Acquire) {
1208            0 => RunTrackingState::Uniform,
1209            1 => RunTrackingState::Promoting,
1210            2 => RunTrackingState::Dense,
1211            _ => unreachable!("invalid buffer sharing_runs_state"),
1212        }
1213    }
1214
1215    fn uses_sharing_runs(&self) -> bool {
1216        self.sharing_runs_state() != RunTrackingState::Uniform
1217    }
1218}
1219
1220#[derive(Debug)]
1221struct RunMap<V> {
1222    runs: SmallVec<[(V, vk::DeviceSize); 4]>,
1223    size: vk::DeviceSize,
1224}
1225
1226impl<V> RunMap<V> {
1227    fn new(size: vk::DeviceSize, value: V) -> Self {
1228        Self {
1229            runs: smallvec![(value, 0)],
1230            size,
1231        }
1232    }
1233
1234    fn run_index_at(&self, offset: vk::DeviceSize) -> usize {
1235        let needle = (offset << 1) | 1;
1236        let run_idx = self
1237            .runs
1238            .binary_search_by(|(_, probe)| (probe << 1).cmp(&needle));
1239
1240        debug_assert!(run_idx.is_err());
1241
1242        let run_idx = {
1243            #[cfg(feature = "checked")]
1244            {
1245                run_idx.unwrap_err()
1246            }
1247
1248            #[cfg(not(feature = "checked"))]
1249            unsafe {
1250                run_idx.unwrap_err_unchecked()
1251            }
1252        };
1253
1254        run_idx.saturating_sub(1)
1255    }
1256}
1257
1258struct RunMapCursor {
1259    run_idx: usize,
1260    remaining_range: BufferSubresourceRange,
1261}
1262
1263impl RunMapCursor {
1264    fn new<V>(map: &RunMap<V>, remaining_range: BufferSubresourceRange) -> Self
1265    where
1266        V: Copy + PartialEq + Debug,
1267    {
1268        debug_assert!(remaining_range.start < remaining_range.end);
1269        debug_assert!(remaining_range.end <= map.size);
1270
1271        #[cfg(feature = "checked")]
1272        {
1273            let run_start = |(_, start): &(V, vk::DeviceSize)| *start;
1274
1275            assert_eq!(map.runs.first().map(run_start), Some(0));
1276            assert!(map.runs.last().map(run_start).unwrap() < map.size);
1277
1278            // Custom is-sorted-by key to additionally check that all run starts are unique
1279            let (mut prev_value, mut prev_start) = map.runs.first().copied().unwrap();
1280            for (next_value, next_start) in map.runs.iter().skip(1).copied() {
1281                debug_assert_ne!(prev_value, next_value);
1282                debug_assert!(prev_start < next_start);
1283
1284                prev_value = next_value;
1285                prev_start = next_start;
1286            }
1287        };
1288
1289        // The needle will always be odd, and the probe always even, the result will always be err
1290        let needle = (remaining_range.start << 1) | 1;
1291        let run_idx = map
1292            .runs
1293            .binary_search_by(|(_, probe)| (probe << 1).cmp(&needle));
1294
1295        debug_assert!(run_idx.is_err());
1296
1297        let mut run_idx = {
1298            #[cfg(feature = "checked")]
1299            {
1300                run_idx.unwrap_err()
1301            }
1302
1303            #[cfg(not(feature = "checked"))]
1304            unsafe {
1305                run_idx.unwrap_err_unchecked()
1306            }
1307        };
1308
1309        // The first access will always be at start == 0, which is even, so run_idx cannot be 0
1310        debug_assert_ne!(run_idx, 0);
1311
1312        run_idx -= 1;
1313
1314        Self {
1315            remaining_range,
1316            run_idx,
1317        }
1318    }
1319
1320    fn next<V>(&mut self, map: &mut RunMap<V>, new_value: V) -> Option<(V, BufferSubresourceRange)>
1321    where
1322        V: Copy + PartialEq + Debug,
1323    {
1324        debug_assert!(self.remaining_range.start <= self.remaining_range.end);
1325        debug_assert!(self.remaining_range.end <= map.size);
1326
1327        if self.remaining_range.start == self.remaining_range.end {
1328            return None;
1329        }
1330
1331        debug_assert!(map.runs.get(self.run_idx).is_some());
1332
1333        let (old_value, old_start) = unsafe { *map.runs.get_unchecked(self.run_idx) };
1334        let old_end = map
1335            .runs
1336            .get(self.run_idx + 1)
1337            .map(|(_, start)| *start)
1338            .unwrap_or(map.size);
1339        let mut remaining_range = self.remaining_range;
1340
1341        remaining_range.end = remaining_range.end.min(old_end);
1342        self.remaining_range.start = remaining_range.end;
1343
1344        if old_value == new_value {
1345            self.run_idx += 1;
1346        } else if old_start < remaining_range.start {
1347            if let Some((_, start)) = map
1348                .runs
1349                .get_mut(self.run_idx + 1)
1350                .filter(|(value, _)| *value == new_value && old_end == remaining_range.end)
1351            {
1352                *start = remaining_range.start;
1353                self.run_idx += 1;
1354            } else {
1355                self.run_idx += 1;
1356                map.runs
1357                    .insert(self.run_idx, (new_value, remaining_range.start));
1358
1359                if old_end > remaining_range.end {
1360                    map.runs
1361                        .insert(self.run_idx + 1, (old_value, remaining_range.end));
1362                }
1363
1364                self.run_idx += 1;
1365            }
1366        } else if self.run_idx > 0 {
1367            if map
1368                .runs
1369                .get(self.run_idx - 1)
1370                .filter(|(value, _)| *value == new_value)
1371                .is_some()
1372            {
1373                if old_end == remaining_range.end {
1374                    map.runs.remove(self.run_idx);
1375
1376                    if map
1377                        .runs
1378                        .get(self.run_idx)
1379                        .filter(|(value, _)| *value == new_value)
1380                        .is_some()
1381                    {
1382                        map.runs.remove(self.run_idx);
1383                        self.run_idx -= 1;
1384                    }
1385                } else {
1386                    debug_assert!(map.runs.get(self.run_idx).is_some());
1387
1388                    let (_, start) = unsafe { map.runs.get_unchecked_mut(self.run_idx) };
1389                    *start = remaining_range.end;
1390                }
1391            } else if old_end == remaining_range.end {
1392                debug_assert!(map.runs.get(self.run_idx).is_some());
1393
1394                let (value, _) = unsafe { map.runs.get_unchecked_mut(self.run_idx) };
1395                *value = new_value;
1396
1397                if map
1398                    .runs
1399                    .get(self.run_idx + 1)
1400                    .filter(|(value, _)| *value == new_value)
1401                    .is_some()
1402                {
1403                    map.runs.remove(self.run_idx + 1);
1404                } else {
1405                    self.run_idx += 1;
1406                }
1407            } else {
1408                if let Some((_, start)) = map.runs.get_mut(self.run_idx) {
1409                    *start = remaining_range.end;
1410                }
1411
1412                map.runs
1413                    .insert(self.run_idx, (new_value, remaining_range.start));
1414                self.run_idx += 2;
1415            }
1416        } else if let Some((_, start)) = map
1417            .runs
1418            .get_mut(1)
1419            .filter(|(value, _)| *value == new_value && old_end == remaining_range.end)
1420        {
1421            *start = 0;
1422            map.runs.remove(0);
1423        } else if old_end > remaining_range.end {
1424            map.runs.insert(0, (new_value, 0));
1425
1426            debug_assert!(map.runs.get(1).is_some());
1427
1428            let (_, start) = unsafe { map.runs.get_unchecked_mut(1) };
1429            *start = remaining_range.end;
1430        } else {
1431            debug_assert!(!map.runs.is_empty());
1432
1433            let (value, _) = unsafe { map.runs.get_unchecked_mut(0) };
1434            *value = new_value;
1435
1436            if map
1437                .runs
1438                .get(1)
1439                .filter(|(value, _)| *value == new_value)
1440                .is_some()
1441            {
1442                map.runs.remove(1);
1443            } else {
1444                self.run_idx += 1;
1445            }
1446        }
1447
1448        Some((old_value, remaining_range))
1449    }
1450}
1451
1452struct RunMapIter<M, V>
1453where
1454    M: DerefMut<Target = RunMap<V>>,
1455    V: Copy + PartialEq + Debug,
1456{
1457    cursor: RunMapCursor,
1458    map: M,
1459    new_value: V,
1460}
1461
1462impl<M, V> RunMapIter<M, V>
1463where
1464    M: DerefMut<Target = RunMap<V>>,
1465    V: Copy + PartialEq + Debug,
1466{
1467    fn new(map: M, new_value: V, remaining_range: BufferSubresourceRange) -> Self {
1468        let cursor = RunMapCursor::new(&map, remaining_range);
1469
1470        Self {
1471            cursor,
1472            map,
1473            new_value,
1474        }
1475    }
1476
1477    fn finish(self) {}
1478}
1479
1480impl<M, V> Iterator for RunMapIter<M, V>
1481where
1482    M: DerefMut<Target = RunMap<V>>,
1483    V: Copy + PartialEq + Debug,
1484{
1485    type Item = (V, BufferSubresourceRange);
1486
1487    fn next(&mut self) -> Option<Self::Item> {
1488        self.cursor.next(&mut self.map, self.new_value)
1489    }
1490}
1491
1492impl<M, V> Drop for RunMapIter<M, V>
1493where
1494    M: DerefMut<Target = RunMap<V>>,
1495    V: Copy + PartialEq + Debug,
1496{
1497    fn drop(&mut self) {
1498        while self.next().is_some() {}
1499    }
1500}
1501
1502#[repr(u8)]
1503#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1504enum RunTrackingState {
1505    Uniform = 0,
1506    Promoting = 1,
1507    Dense = 2,
1508}
1509
1510#[derive(Debug)]
1511enum Sharing {
1512    Concurrent,
1513    Exclusive(ExclusiveSharing),
1514}
1515
1516impl Sharing {
1517    fn new(size: vk::DeviceSize, sharing_mode: vk::SharingMode) -> Self {
1518        if sharing_mode == vk::SharingMode::CONCURRENT {
1519            Self::Concurrent
1520        } else {
1521            Self::Exclusive(ExclusiveSharing::new(size))
1522        }
1523    }
1524
1525    fn ranges_in(&self, range: BufferSubresourceRange) -> SharingRunIter<'_> {
1526        match self {
1527            Self::Concurrent => SharingRunIter::Constant(Some((SharingMode::Concurrent, range))),
1528            Self::Exclusive(sharing) => sharing.ranges_in(range),
1529        }
1530    }
1531
1532    fn set_ranges<I>(&self, size: vk::DeviceSize, sharing: SharingMode, sharing_ranges: I)
1533    where
1534        I: IntoIterator<Item = BufferSubresourceRange>,
1535    {
1536        if let Self::Exclusive(exclusive) = self {
1537            exclusive.set_ranges(size, sharing, sharing_ranges);
1538        }
1539    }
1540}
1541
1542enum SharingRunIter<'a> {
1543    Constant(Option<(SharingMode, BufferSubresourceRange)>),
1544    Dense {
1545        query_range: BufferSubresourceRange,
1546        run_idx: usize,
1547        sharing_runs: MutexGuard<'a, RunMap<SharingMode>>,
1548    },
1549}
1550
1551impl Iterator for SharingRunIter<'_> {
1552    type Item = (SharingMode, BufferSubresourceRange);
1553
1554    fn next(&mut self) -> Option<Self::Item> {
1555        match self {
1556            Self::Constant(range) => range.take(),
1557            Self::Dense {
1558                query_range,
1559                run_idx,
1560                sharing_runs,
1561            } => {
1562                let &(sharing, start) = sharing_runs.runs.get(*run_idx)?;
1563                if start >= query_range.end {
1564                    return None;
1565                }
1566
1567                let end = sharing_runs
1568                    .runs
1569                    .get(*run_idx + 1)
1570                    .map(|(_, next_start)| *next_start)
1571                    .unwrap_or(sharing_runs.size);
1572
1573                *run_idx += 1;
1574
1575                let range = BufferSubresourceRange { start, end }.intersection(*query_range)?;
1576
1577                Some((sharing, range))
1578            }
1579        }
1580    }
1581}
1582
1583#[cfg(test)]
1584mod test {
1585    use {
1586        super::*,
1587        rand::{Rng, SeedableRng, rngs::SmallRng},
1588    };
1589
1590    type Info = BufferInfo;
1591    type Builder = BufferInfoBuilder;
1592
1593    const FUZZ_COUNT: usize = 100_000;
1594
1595    fn buffer_sync_info(range: Range<vk::DeviceSize>) -> BufferSubresourceSyncInfo {
1596        BufferSubresourceSyncInfo {
1597            access_mask: vk::AccessFlags::SHADER_READ,
1598            queue_family_index: None,
1599            range: buffer_subresource_range(range),
1600            stage_mask: vk::PipelineStageFlags::COMPUTE_SHADER,
1601        }
1602    }
1603
1604    fn assert_access_runs_eq(access_runs: &AccessRuns, expected: &[(AccessType, vk::DeviceSize)]) {
1605        assert_eq!(access_runs.runs.as_slice(), expected);
1606    }
1607
1608    #[test]
1609    pub fn buffer_access() {
1610        let mut access_runs = AccessRuns::new(100, AccessType::Nothing);
1611
1612        {
1613            let mut accesses = RunMapIter::new(
1614                &mut access_runs,
1615                AccessType::TransferWrite,
1616                buffer_subresource_range(0..10),
1617            );
1618
1619            assert_access_runs_eq(accesses.map, &[(AccessType::Nothing, 0)]);
1620            assert_eq!(
1621                accesses.next().unwrap(),
1622                (AccessType::Nothing, buffer_subresource_range(0..10))
1623            );
1624            assert_access_runs_eq(
1625                accesses.map,
1626                &[(AccessType::TransferWrite, 0), (AccessType::Nothing, 10)],
1627            );
1628            assert!(accesses.next().is_none());
1629        }
1630
1631        {
1632            let mut accesses = RunMapIter::new(
1633                &mut access_runs,
1634                AccessType::TransferRead,
1635                buffer_subresource_range(5..15),
1636            );
1637
1638            assert_access_runs_eq(
1639                accesses.map,
1640                &[(AccessType::TransferWrite, 0), (AccessType::Nothing, 10)],
1641            );
1642            assert_eq!(
1643                accesses.next().unwrap(),
1644                (AccessType::TransferWrite, buffer_subresource_range(5..10))
1645            );
1646            assert_access_runs_eq(
1647                accesses.map,
1648                &[
1649                    (AccessType::TransferWrite, 0),
1650                    (AccessType::TransferRead, 5),
1651                    (AccessType::Nothing, 10),
1652                ],
1653            );
1654            assert_eq!(
1655                accesses.next().unwrap(),
1656                (AccessType::Nothing, buffer_subresource_range(10..15))
1657            );
1658            assert_access_runs_eq(
1659                accesses.map,
1660                &[
1661                    (AccessType::TransferWrite, 0),
1662                    (AccessType::TransferRead, 5),
1663                    (AccessType::Nothing, 15),
1664                ],
1665            );
1666            assert!(accesses.next().is_none());
1667        }
1668
1669        {
1670            let mut accesses = RunMapIter::new(
1671                &mut access_runs,
1672                AccessType::HostRead,
1673                buffer_subresource_range(0..100),
1674            );
1675
1676            assert_access_runs_eq(
1677                accesses.map,
1678                &[
1679                    (AccessType::TransferWrite, 0),
1680                    (AccessType::TransferRead, 5),
1681                    (AccessType::Nothing, 15),
1682                ],
1683            );
1684            assert_eq!(
1685                accesses.next().unwrap(),
1686                (AccessType::TransferWrite, buffer_subresource_range(0..5))
1687            );
1688            assert_access_runs_eq(
1689                accesses.map,
1690                &[
1691                    (AccessType::HostRead, 0),
1692                    (AccessType::TransferRead, 5),
1693                    (AccessType::Nothing, 15),
1694                ],
1695            );
1696            assert_eq!(
1697                accesses.next().unwrap(),
1698                (AccessType::TransferRead, buffer_subresource_range(5..15))
1699            );
1700            assert_access_runs_eq(
1701                accesses.map,
1702                &[(AccessType::HostRead, 0), (AccessType::Nothing, 15)],
1703            );
1704            assert_eq!(
1705                accesses.next().unwrap(),
1706                (AccessType::Nothing, buffer_subresource_range(15..100))
1707            );
1708            assert_access_runs_eq(accesses.map, &[(AccessType::HostRead, 0)]);
1709            assert!(accesses.next().is_none());
1710        }
1711
1712        {
1713            let mut accesses = RunMapIter::new(
1714                &mut access_runs,
1715                AccessType::HostWrite,
1716                buffer_subresource_range(0..100),
1717            );
1718
1719            assert_access_runs_eq(accesses.map, &[(AccessType::HostRead, 0)]);
1720            assert_eq!(
1721                accesses.next().unwrap(),
1722                (AccessType::HostRead, buffer_subresource_range(0..100))
1723            );
1724            assert_access_runs_eq(accesses.map, &[(AccessType::HostWrite, 0)]);
1725            assert!(accesses.next().is_none());
1726        }
1727
1728        {
1729            let mut accesses = RunMapIter::new(
1730                &mut access_runs,
1731                AccessType::HostWrite,
1732                buffer_subresource_range(0..100),
1733            );
1734
1735            assert_access_runs_eq(accesses.map, &[(AccessType::HostWrite, 0)]);
1736            assert_eq!(
1737                accesses.next().unwrap(),
1738                (AccessType::HostWrite, buffer_subresource_range(0..100))
1739            );
1740            assert_access_runs_eq(accesses.map, &[(AccessType::HostWrite, 0)]);
1741            assert!(accesses.next().is_none());
1742        }
1743
1744        {
1745            let mut accesses = RunMapIter::new(
1746                &mut access_runs,
1747                AccessType::HostWrite,
1748                buffer_subresource_range(1..99),
1749            );
1750
1751            assert_access_runs_eq(accesses.map, &[(AccessType::HostWrite, 0)]);
1752            assert_eq!(
1753                accesses.next().unwrap(),
1754                (AccessType::HostWrite, buffer_subresource_range(1..99))
1755            );
1756            assert_access_runs_eq(accesses.map, &[(AccessType::HostWrite, 0)]);
1757            assert!(accesses.next().is_none());
1758        }
1759
1760        {
1761            let mut accesses = RunMapIter::new(
1762                &mut access_runs,
1763                AccessType::HostRead,
1764                buffer_subresource_range(1..99),
1765            );
1766
1767            assert_access_runs_eq(accesses.map, &[(AccessType::HostWrite, 0)]);
1768            assert_eq!(
1769                accesses.next().unwrap(),
1770                (AccessType::HostWrite, buffer_subresource_range(1..99))
1771            );
1772            assert_access_runs_eq(
1773                accesses.map,
1774                &[
1775                    (AccessType::HostWrite, 0),
1776                    (AccessType::HostRead, 1),
1777                    (AccessType::HostWrite, 99),
1778                ],
1779            );
1780            assert!(accesses.next().is_none());
1781        }
1782
1783        {
1784            let mut accesses = RunMapIter::new(
1785                &mut access_runs,
1786                AccessType::Nothing,
1787                buffer_subresource_range(0..100),
1788            );
1789
1790            assert_eq!(
1791                accesses.next().unwrap(),
1792                (AccessType::HostWrite, buffer_subresource_range(0..1))
1793            );
1794            assert_eq!(
1795                accesses.next().unwrap(),
1796                (AccessType::HostRead, buffer_subresource_range(1..99))
1797            );
1798            assert_eq!(
1799                accesses.next().unwrap(),
1800                (AccessType::HostWrite, buffer_subresource_range(99..100))
1801            );
1802            assert!(accesses.next().is_none());
1803        }
1804
1805        {
1806            let mut accesses = RunMapIter::new(
1807                &mut access_runs,
1808                AccessType::AnyShaderWrite,
1809                buffer_subresource_range(0..100),
1810            );
1811
1812            assert_eq!(
1813                accesses.next().unwrap(),
1814                (AccessType::Nothing, buffer_subresource_range(0..100))
1815            );
1816            assert!(accesses.next().is_none());
1817        }
1818
1819        {
1820            let mut accesses = RunMapIter::new(
1821                &mut access_runs,
1822                AccessType::AnyShaderReadOther,
1823                buffer_subresource_range(1..2),
1824            );
1825
1826            assert_eq!(
1827                accesses.next().unwrap(),
1828                (AccessType::AnyShaderWrite, buffer_subresource_range(1..2))
1829            );
1830            assert!(accesses.next().is_none());
1831        }
1832
1833        {
1834            let mut accesses = RunMapIter::new(
1835                &mut access_runs,
1836                AccessType::AnyShaderReadOther,
1837                buffer_subresource_range(3..4),
1838            );
1839
1840            assert_eq!(
1841                accesses.next().unwrap(),
1842                (AccessType::AnyShaderWrite, buffer_subresource_range(3..4))
1843            );
1844            assert!(accesses.next().is_none());
1845        }
1846
1847        {
1848            let mut accesses = RunMapIter::new(
1849                &mut access_runs,
1850                AccessType::Nothing,
1851                buffer_subresource_range(0..5),
1852            );
1853
1854            assert_eq!(
1855                accesses.next().unwrap(),
1856                (AccessType::AnyShaderWrite, buffer_subresource_range(0..1))
1857            );
1858            assert_eq!(
1859                accesses.next().unwrap(),
1860                (
1861                    AccessType::AnyShaderReadOther,
1862                    buffer_subresource_range(1..2)
1863                )
1864            );
1865            assert_eq!(
1866                accesses.next().unwrap(),
1867                (AccessType::AnyShaderWrite, buffer_subresource_range(2..3))
1868            );
1869            assert_eq!(
1870                accesses.next().unwrap(),
1871                (
1872                    AccessType::AnyShaderReadOther,
1873                    buffer_subresource_range(3..4)
1874                )
1875            );
1876            assert_eq!(
1877                accesses.next().unwrap(),
1878                (AccessType::AnyShaderWrite, buffer_subresource_range(4..5))
1879            );
1880            assert!(accesses.next().is_none());
1881        }
1882    }
1883
1884    #[test]
1885    pub fn buffer_access_basic() {
1886        let mut access_runs = AccessRuns::new(5, AccessType::Nothing);
1887
1888        access_runs.runs = smallvec![
1889            (AccessType::ColorAttachmentRead, 0),
1890            (AccessType::AnyShaderWrite, 4),
1891        ];
1892
1893        {
1894            let mut accesses = RunMapIter::new(
1895                &mut access_runs,
1896                AccessType::AnyShaderWrite,
1897                buffer_subresource_range(0..2),
1898            );
1899
1900            assert_eq!(
1901                accesses.next().unwrap(),
1902                (
1903                    AccessType::ColorAttachmentRead,
1904                    buffer_subresource_range(0..2)
1905                )
1906            );
1907            assert!(accesses.next().is_none());
1908        }
1909
1910        {
1911            let mut accesses = RunMapIter::new(
1912                &mut access_runs,
1913                AccessType::HostWrite,
1914                buffer_subresource_range(0..5),
1915            );
1916
1917            assert_eq!(
1918                accesses.next().unwrap(),
1919                (AccessType::AnyShaderWrite, buffer_subresource_range(0..2))
1920            );
1921            assert_eq!(
1922                accesses.next().unwrap(),
1923                (
1924                    AccessType::ColorAttachmentRead,
1925                    buffer_subresource_range(2..4)
1926                )
1927            );
1928            assert_eq!(
1929                accesses.next().unwrap(),
1930                (AccessType::AnyShaderWrite, buffer_subresource_range(4..5))
1931            );
1932
1933            assert!(accesses.next().is_none());
1934        }
1935    }
1936
1937    #[test]
1938    pub fn buffer_sharing_ranges_in_clips_dense_runs_to_query_range() {
1939        let sharing = ExclusiveSharing::new(16);
1940        let owner_a = SharingMode::Exclusive(Some((1, 0)));
1941        let owner_b = SharingMode::Exclusive(Some((2, 0)));
1942        let range_a = buffer_subresource_range(0..8);
1943        let range_b = buffer_subresource_range(8..16);
1944        let query_range = buffer_subresource_range(4..12);
1945
1946        sharing.set_ranges(16, owner_a, [range_a]);
1947        sharing.set_ranges(16, owner_b, [range_b]);
1948
1949        let ranges = sharing.ranges_in(query_range).collect::<Vec<_>>();
1950
1951        assert_eq!(
1952            ranges,
1953            vec![
1954                (owner_a, buffer_subresource_range(4..8)),
1955                (owner_b, buffer_subresource_range(8..12)),
1956            ]
1957        );
1958    }
1959
1960    fn buffer_access_fuzz(buffer_size: vk::DeviceSize) {
1961        static ACCESS_TYPES: &[AccessType] = &[
1962            AccessType::AnyShaderReadOther,
1963            AccessType::AnyShaderWrite,
1964            AccessType::ColorAttachmentRead,
1965            AccessType::ColorAttachmentWrite,
1966            AccessType::HostRead,
1967            AccessType::HostWrite,
1968            AccessType::Nothing,
1969        ];
1970
1971        let mut rng = SmallRng::seed_from_u64(42);
1972        let mut access_runs = AccessRuns::new(buffer_size, AccessType::Nothing);
1973        let mut data = vec![AccessType::Nothing; buffer_size as usize];
1974
1975        for _ in 0..FUZZ_COUNT {
1976            let access = ACCESS_TYPES[rng.random_range(..ACCESS_TYPES.len())];
1977            let access_start = rng.random_range(..buffer_size);
1978            let access_end = rng.random_range(access_start + 1..=buffer_size);
1979
1980            // println!("{access:?} {access_start}..{access_end}");
1981
1982            let accesses = RunMapIter::new(
1983                &mut access_runs,
1984                access,
1985                buffer_subresource_range(access_start..access_end),
1986            );
1987
1988            for (access, access_range) in accesses {
1989                // println!("\t{access:?} {}..{}", access_range.start, access_range.end);
1990                assert!(
1991                    data[access_range.start as usize..access_range.end as usize]
1992                        .iter()
1993                        .all(|data| *data == access),
1994                    "{:?}",
1995                    &data[access_range.start as usize..access_range.end as usize]
1996                );
1997            }
1998
1999            for data in &mut data[access_start as usize..access_end as usize] {
2000                *data = access;
2001            }
2002        }
2003    }
2004
2005    #[test]
2006    pub fn buffer_access_fuzz_small() {
2007        buffer_access_fuzz(5);
2008    }
2009
2010    #[test]
2011    pub fn buffer_access_fuzz_medium() {
2012        buffer_access_fuzz(101);
2013    }
2014
2015    #[test]
2016    pub fn buffer_access_fuzz_large() {
2017        buffer_access_fuzz(10_000);
2018    }
2019
2020    #[test]
2021    pub fn buffer_sync_info_compact_merges_adjacent_equal_ranges() {
2022        let mut sync_info = BufferSyncInfo {
2023            ranges: vec![
2024                buffer_sync_info(0..4),
2025                buffer_sync_info(4..8),
2026                BufferSubresourceSyncInfo {
2027                    access_mask: vk::AccessFlags::SHADER_WRITE,
2028                    queue_family_index: None,
2029                    range: buffer_subresource_range(8..12),
2030                    stage_mask: vk::PipelineStageFlags::COMPUTE_SHADER,
2031                },
2032            ]
2033            .into_boxed_slice(),
2034        };
2035
2036        sync_info.compact();
2037
2038        assert_eq!(sync_info.ranges.len(), 2);
2039        assert_eq!(sync_info.ranges[0], buffer_sync_info(0..8));
2040        assert_eq!(
2041            sync_info.ranges[1],
2042            BufferSubresourceSyncInfo {
2043                access_mask: vk::AccessFlags::SHADER_WRITE,
2044                queue_family_index: None,
2045                range: buffer_subresource_range(8..12),
2046                stage_mask: vk::PipelineStageFlags::COMPUTE_SHADER,
2047            }
2048        );
2049    }
2050
2051    #[test]
2052    pub fn buffer_sync_info_into_compacted_preserves_non_adjacent_ranges() {
2053        let sync_info = BufferSyncInfo {
2054            ranges: vec![
2055                BufferSubresourceSyncInfo {
2056                    queue_family_index: Some(3),
2057                    ..buffer_sync_info(0..4)
2058                },
2059                BufferSubresourceSyncInfo {
2060                    queue_family_index: Some(3),
2061                    ..buffer_sync_info(5..9)
2062                },
2063            ]
2064            .into_boxed_slice(),
2065        };
2066
2067        let sync_info = sync_info.into_compacted();
2068
2069        assert_eq!(sync_info.ranges.len(), 2);
2070        assert_eq!(sync_info.ranges[0].queue_family_index, Some(3));
2071        assert_eq!(sync_info.ranges[1].queue_family_index, Some(3));
2072        assert_eq!(sync_info.ranges[0].range, buffer_subresource_range(0..4));
2073        assert_eq!(sync_info.ranges[1].range, buffer_subresource_range(5..9));
2074    }
2075
2076    #[test]
2077    pub fn buffer_info() {
2078        let info = Info::device_mem(0, vk::BufferUsageFlags::empty());
2079        let builder = info.into_builder().build();
2080
2081        assert_eq!(info, builder);
2082    }
2083
2084    #[test]
2085    pub fn buffer_info_alignment() {
2086        let info = Info::device_mem(0, vk::BufferUsageFlags::empty());
2087
2088        assert_eq!(info.alignment, 1);
2089    }
2090
2091    #[test]
2092    pub fn buffer_info_builder() {
2093        let info = Info::device_mem(0, vk::BufferUsageFlags::empty());
2094        let builder = Builder::default().size(0).build();
2095
2096        assert_eq!(info, builder);
2097    }
2098
2099    #[test]
2100    #[should_panic(expected = "Alignment must be a power of two")]
2101    pub fn buffer_info_builder_alignment_0() {
2102        Builder::default().size(0).alignment(0).build();
2103    }
2104
2105    #[test]
2106    #[should_panic(expected = "Alignment must be a power of two")]
2107    pub fn buffer_info_builder_alignment_42() {
2108        Builder::default().size(0).alignment(42).build();
2109    }
2110
2111    #[test]
2112    pub fn buffer_info_builder_alignment_256() {
2113        let mut info = Info::device_mem(42, vk::BufferUsageFlags::empty());
2114        info.alignment = 256;
2115
2116        let builder = Builder::default().size(42).alignment(256).build();
2117
2118        assert_eq!(info, builder);
2119    }
2120
2121    #[test]
2122    pub fn buffer_info_builder_default_size() {
2123        assert_eq!(
2124            Builder::default().build(),
2125            Info::device_mem(0, vk::BufferUsageFlags::empty())
2126        );
2127    }
2128
2129    fn buffer_subresource_range(
2130        Range { start, end }: Range<vk::DeviceSize>,
2131    ) -> BufferSubresourceRange {
2132        BufferSubresourceRange { start, end }
2133    }
2134
2135    #[test]
2136    pub fn buffer_subresource_range_intersects() {
2137        use BufferSubresourceRange as B;
2138
2139        assert!(!B { start: 10, end: 20 }.intersects(B { start: 0, end: 5 }));
2140        assert!(!B { start: 10, end: 20 }.intersects(B { start: 5, end: 10 }));
2141        assert!(B { start: 10, end: 20 }.intersects(B { start: 10, end: 15 }));
2142        assert!(B { start: 10, end: 20 }.intersects(B { start: 15, end: 20 }));
2143        assert!(!B { start: 10, end: 20 }.intersects(B { start: 20, end: 25 }));
2144        assert!(!B { start: 10, end: 20 }.intersects(B { start: 25, end: 30 }));
2145
2146        assert!(!B { start: 5, end: 10 }.intersects(B { start: 10, end: 20 }));
2147        assert!(B { start: 5, end: 25 }.intersects(B { start: 10, end: 20 }));
2148        assert!(B { start: 5, end: 15 }.intersects(B { start: 10, end: 20 }));
2149        assert!(B { start: 10, end: 20 }.intersects(B { start: 10, end: 20 }));
2150        assert!(B { start: 11, end: 19 }.intersects(B { start: 10, end: 20 }));
2151        assert!(B { start: 15, end: 25 }.intersects(B { start: 10, end: 20 }));
2152        assert!(!B { start: 20, end: 25 }.intersects(B { start: 10, end: 20 }));
2153    }
2154}
vk_graph/driver/buffer.rs

vk_graph/driver/
buffer.rs
