use crate::buffer::sys::BufferCreationError;
use crate::buffer::sys::UnsafeBuffer;
use crate::buffer::traits::BufferAccess;
use crate::buffer::traits::BufferInner;
use crate::buffer::traits::TypedBufferAccess;
use crate::buffer::BufferUsage;
use crate::device::Device;
use crate::device::DeviceOwned;
use crate::device::Queue;
use crate::image::ImageAccess;
use crate::instance::QueueFamily;
use crate::memory::pool::AllocFromRequirementsFilter;
use crate::memory::pool::AllocLayout;
use crate::memory::pool::MappingRequirement;
use crate::memory::pool::MemoryPool;
use crate::memory::pool::MemoryPoolAlloc;
use crate::memory::pool::PotentialDedicatedAllocation;
use crate::memory::pool::StdMemoryPoolAlloc;
use crate::memory::{DedicatedAlloc, MemoryRequirements};
use crate::memory::{DeviceMemoryAllocError, ExternalMemoryHandleType};
use crate::sync::AccessError;
use crate::sync::Sharing;
use smallvec::SmallVec;
use std::fs::File;
use std::hash::Hash;
use std::hash::Hasher;
use std::marker::PhantomData;
use std::mem;
use std::sync::Arc;
use std::sync::Mutex;
#[derive(Debug)]
pub struct DeviceLocalBuffer<T: ?Sized, A = PotentialDedicatedAllocation<StdMemoryPoolAlloc>> {
inner: UnsafeBuffer,
memory: A,
queue_families: SmallVec<[u32; 4]>,
gpu_lock: Mutex<GpuAccess>,
marker: PhantomData<Box<T>>,
}
#[derive(Debug, Copy, Clone)]
enum GpuAccess {
None,
NonExclusive { num: u32 },
Exclusive { num: u32 },
}
impl<T> DeviceLocalBuffer<T> {
#[inline]
pub fn new<'a, I>(
device: Arc<Device>,
usage: BufferUsage,
queue_families: I,
) -> Result<Arc<DeviceLocalBuffer<T>>, DeviceMemoryAllocError>
where
I: IntoIterator<Item = QueueFamily<'a>>,
{
unsafe { DeviceLocalBuffer::raw(device, mem::size_of::<T>(), usage, queue_families) }
}
}
impl<T> DeviceLocalBuffer<[T]> {
#[inline]
pub fn array<'a, I>(
device: Arc<Device>,
len: usize,
usage: BufferUsage,
queue_families: I,
) -> Result<Arc<DeviceLocalBuffer<[T]>>, DeviceMemoryAllocError>
where
I: IntoIterator<Item = QueueFamily<'a>>,
{
unsafe { DeviceLocalBuffer::raw(device, len * mem::size_of::<T>(), usage, queue_families) }
}
}
impl<T: ?Sized> DeviceLocalBuffer<T> {
pub unsafe fn raw<'a, I>(
device: Arc<Device>,
size: usize,
usage: BufferUsage,
queue_families: I,
) -> Result<Arc<DeviceLocalBuffer<T>>, DeviceMemoryAllocError>
where
I: IntoIterator<Item = QueueFamily<'a>>,
{
let queue_families = queue_families
.into_iter()
.map(|f| f.id())
.collect::<SmallVec<[u32; 4]>>();
let (buffer, mem_reqs) = Self::build_buffer(&device, size, usage, &queue_families)?;
let mem = MemoryPool::alloc_from_requirements(
&Device::standard_pool(&device),
&mem_reqs,
AllocLayout::Linear,
MappingRequirement::DoNotMap,
DedicatedAlloc::Buffer(&buffer),
|t| {
if t.is_device_local() {
AllocFromRequirementsFilter::Preferred
} else {
AllocFromRequirementsFilter::Allowed
}
},
)?;
debug_assert!((mem.offset() % mem_reqs.alignment) == 0);
buffer.bind_memory(mem.memory(), mem.offset())?;
Ok(Arc::new(DeviceLocalBuffer {
inner: buffer,
memory: mem,
queue_families: queue_families,
gpu_lock: Mutex::new(GpuAccess::None),
marker: PhantomData,
}))
}
#[cfg(target_os = "linux")]
pub unsafe fn raw_with_exportable_fd<'a, I>(
device: Arc<Device>,
size: usize,
usage: BufferUsage,
queue_families: I,
) -> Result<Arc<DeviceLocalBuffer<T>>, DeviceMemoryAllocError>
where
I: IntoIterator<Item = QueueFamily<'a>>,
{
assert!(device.loaded_extensions().khr_external_memory_fd);
assert!(device.loaded_extensions().khr_external_memory);
let queue_families = queue_families
.into_iter()
.map(|f| f.id())
.collect::<SmallVec<[u32; 4]>>();
let (buffer, mem_reqs) = Self::build_buffer(&device, size, usage, &queue_families)?;
let mem = MemoryPool::alloc_from_requirements_with_exportable_fd(
&Device::standard_pool(&device),
&mem_reqs,
AllocLayout::Linear,
MappingRequirement::DoNotMap,
DedicatedAlloc::Buffer(&buffer),
|t| {
if t.is_device_local() {
AllocFromRequirementsFilter::Preferred
} else {
AllocFromRequirementsFilter::Allowed
}
},
)?;
debug_assert!((mem.offset() % mem_reqs.alignment) == 0);
buffer.bind_memory(mem.memory(), mem.offset())?;
Ok(Arc::new(DeviceLocalBuffer {
inner: buffer,
memory: mem,
queue_families: queue_families,
gpu_lock: Mutex::new(GpuAccess::None),
marker: PhantomData,
}))
}
unsafe fn build_buffer(
device: &Arc<Device>,
size: usize,
usage: BufferUsage,
queue_families: &SmallVec<[u32; 4]>,
) -> Result<(UnsafeBuffer, MemoryRequirements), DeviceMemoryAllocError> {
let (buffer, mem_reqs) = {
let sharing = if queue_families.len() >= 2 {
Sharing::Concurrent(queue_families.iter().cloned())
} else {
Sharing::Exclusive
};
match UnsafeBuffer::new(device.clone(), size, usage, sharing, None) {
Ok(b) => b,
Err(BufferCreationError::AllocError(err)) => return Err(err),
Err(_) => unreachable!(),
}
};
Ok((buffer, mem_reqs))
}
#[cfg(target_os = "linux")]
pub fn export_posix_fd(&self) -> Result<File, DeviceMemoryAllocError> {
self.memory
.memory()
.export_fd(ExternalMemoryHandleType::posix())
}
}
impl<T: ?Sized, A> DeviceLocalBuffer<T, A> {
#[inline]
pub fn queue_families(&self) -> Vec<QueueFamily> {
self.queue_families
.iter()
.map(|&num| {
self.device()
.physical_device()
.queue_family_by_id(num)
.unwrap()
})
.collect()
}
}
unsafe impl<T: ?Sized, A> DeviceOwned for DeviceLocalBuffer<T, A> {
#[inline]
fn device(&self) -> &Arc<Device> {
self.inner.device()
}
}
unsafe impl<T: ?Sized, A> BufferAccess for DeviceLocalBuffer<T, A>
where
T: 'static + Send + Sync,
{
#[inline]
fn inner(&self) -> BufferInner {
BufferInner {
buffer: &self.inner,
offset: 0,
}
}
#[inline]
fn size(&self) -> usize {
self.inner.size()
}
#[inline]
fn conflicts_buffer(&self, other: &dyn BufferAccess) -> bool {
self.conflict_key() == other.conflict_key()
}
#[inline]
fn conflicts_image(&self, other: &dyn ImageAccess) -> bool {
false
}
#[inline]
fn conflict_key(&self) -> (u64, usize) {
(self.inner.key(), 0)
}
#[inline]
fn try_gpu_lock(&self, exclusive: bool, _: &Queue) -> Result<(), AccessError> {
let mut lock = self.gpu_lock.lock().unwrap();
match &mut *lock {
a @ &mut GpuAccess::None => {
if exclusive {
*a = GpuAccess::Exclusive { num: 1 };
} else {
*a = GpuAccess::NonExclusive { num: 1 };
}
Ok(())
}
&mut GpuAccess::NonExclusive { ref mut num } => {
if exclusive {
Err(AccessError::AlreadyInUse)
} else {
*num += 1;
Ok(())
}
}
&mut GpuAccess::Exclusive { .. } => Err(AccessError::AlreadyInUse),
}
}
#[inline]
unsafe fn increase_gpu_lock(&self) {
let mut lock = self.gpu_lock.lock().unwrap();
match *lock {
GpuAccess::None => panic!(),
GpuAccess::NonExclusive { ref mut num } => {
debug_assert!(*num >= 1);
*num += 1;
}
GpuAccess::Exclusive { ref mut num } => {
debug_assert!(*num >= 1);
*num += 1;
}
}
}
#[inline]
unsafe fn unlock(&self) {
let mut lock = self.gpu_lock.lock().unwrap();
match *lock {
GpuAccess::None => panic!("Tried to unlock a buffer that isn't locked"),
GpuAccess::NonExclusive { ref mut num } => {
assert!(*num >= 1);
*num -= 1;
if *num >= 1 {
return;
}
}
GpuAccess::Exclusive { ref mut num } => {
assert!(*num >= 1);
*num -= 1;
if *num >= 1 {
return;
}
}
};
*lock = GpuAccess::None;
}
}
unsafe impl<T: ?Sized, A> TypedBufferAccess for DeviceLocalBuffer<T, A>
where
T: 'static + Send + Sync,
{
type Content = T;
}
impl<T: ?Sized, A> PartialEq for DeviceLocalBuffer<T, A>
where
T: 'static + Send + Sync,
{
#[inline]
fn eq(&self, other: &Self) -> bool {
self.inner() == other.inner() && self.size() == other.size()
}
}
impl<T: ?Sized, A> Eq for DeviceLocalBuffer<T, A> where T: 'static + Send + Sync {}
impl<T: ?Sized, A> Hash for DeviceLocalBuffer<T, A>
where
T: 'static + Send + Sync,
{
#[inline]
fn hash<H: Hasher>(&self, state: &mut H) {
self.inner().hash(state);
self.size().hash(state);
}
}