use super::{
sys::UnsafeBuffer, BufferAccess, BufferAccessObject, BufferContents, BufferInner, BufferUsage,
CpuAccessibleBuffer,
};
use crate::{
buffer::{sys::UnsafeBufferCreateInfo, BufferCreationError, TypedBufferAccess},
command_buffer::{
AutoCommandBufferBuilder, CommandBufferExecFuture, CommandBufferUsage,
PrimaryAutoCommandBuffer, PrimaryCommandBuffer,
},
device::{physical::QueueFamily, Device, DeviceOwned, Queue},
memory::{
pool::{
AllocFromRequirementsFilter, AllocLayout, MappingRequirement, MemoryPoolAlloc,
PotentialDedicatedAllocation, StdMemoryPoolAlloc,
},
DedicatedAllocation, DeviceMemoryAllocationError, MemoryPool,
},
sync::{AccessError, NowFuture, Sharing},
DeviceSize,
};
use smallvec::SmallVec;
use std::{
hash::{Hash, Hasher},
marker::PhantomData,
mem::size_of,
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
};
#[derive(Debug)]
pub struct ImmutableBuffer<T, A = PotentialDedicatedAllocation<StdMemoryPoolAlloc>>
where
T: BufferContents + ?Sized,
{
inner: UnsafeBuffer,
memory: A,
initialized: AtomicBool,
queue_families: SmallVec<[u32; 4]>,
marker: PhantomData<Box<T>>,
}
type ImmutableBufferFromBufferFuture = CommandBufferExecFuture<NowFuture, PrimaryAutoCommandBuffer>;
impl<T> ImmutableBuffer<T>
where
T: BufferContents + ?Sized,
{
pub fn from_buffer<B>(
source: Arc<B>,
usage: BufferUsage,
queue: Arc<Queue>,
) -> Result<
(Arc<ImmutableBuffer<T>>, ImmutableBufferFromBufferFuture),
DeviceMemoryAllocationError,
>
where
B: TypedBufferAccess<Content = T> + 'static,
{
unsafe {
let actual_usage = BufferUsage {
transfer_destination: true,
..usage
};
let (buffer, init) = ImmutableBuffer::raw(
source.device().clone(),
source.size(),
actual_usage,
source.device().active_queue_families(),
)?;
let mut cbb = AutoCommandBufferBuilder::primary(
source.device().clone(),
queue.family(),
CommandBufferUsage::MultipleSubmit,
)?;
cbb.copy_buffer(source, init).unwrap(); let cb = cbb.build().unwrap();
let future = match cb.execute(queue) {
Ok(f) => f,
Err(_) => unreachable!(),
};
Ok((buffer, future))
}
}
}
impl<T> ImmutableBuffer<T>
where
T: BufferContents,
{
pub fn from_data(
data: T,
usage: BufferUsage,
queue: Arc<Queue>,
) -> Result<
(Arc<ImmutableBuffer<T>>, ImmutableBufferFromBufferFuture),
DeviceMemoryAllocationError,
> {
let source = CpuAccessibleBuffer::from_data(
queue.device().clone(),
BufferUsage::transfer_source(),
false,
data,
)?;
ImmutableBuffer::from_buffer(source, usage, queue)
}
#[inline]
pub unsafe fn uninitialized(
device: Arc<Device>,
usage: BufferUsage,
) -> Result<
(
Arc<ImmutableBuffer<T>>,
Arc<ImmutableBufferInitialization<T>>,
),
DeviceMemoryAllocationError,
> {
ImmutableBuffer::raw(
device.clone(),
size_of::<T>() as DeviceSize,
usage,
device.active_queue_families(),
)
}
}
impl<T> ImmutableBuffer<[T]>
where
[T]: BufferContents,
{
pub fn from_iter<D>(
data: D,
usage: BufferUsage,
queue: Arc<Queue>,
) -> Result<
(Arc<ImmutableBuffer<[T]>>, ImmutableBufferFromBufferFuture),
DeviceMemoryAllocationError,
>
where
D: IntoIterator<Item = T>,
D::IntoIter: ExactSizeIterator,
{
let source = CpuAccessibleBuffer::from_iter(
queue.device().clone(),
BufferUsage::transfer_source(),
false,
data,
)?;
ImmutableBuffer::from_buffer(source, usage, queue)
}
#[inline]
pub unsafe fn uninitialized_array(
device: Arc<Device>,
len: DeviceSize,
usage: BufferUsage,
) -> Result<
(
Arc<ImmutableBuffer<[T]>>,
Arc<ImmutableBufferInitialization<[T]>>,
),
DeviceMemoryAllocationError,
> {
ImmutableBuffer::raw(
device.clone(),
len * size_of::<T>() as DeviceSize,
usage,
device.active_queue_families(),
)
}
}
impl<T> ImmutableBuffer<T>
where
T: BufferContents + ?Sized,
{
#[inline]
pub unsafe fn raw<'a, I>(
device: Arc<Device>,
size: DeviceSize,
usage: BufferUsage,
queue_families: I,
) -> Result<
(
Arc<ImmutableBuffer<T>>,
Arc<ImmutableBufferInitialization<T>>,
),
DeviceMemoryAllocationError,
>
where
I: IntoIterator<Item = QueueFamily<'a>>,
{
let queue_families = queue_families.into_iter().map(|f| f.id()).collect();
ImmutableBuffer::raw_impl(device, size, usage, queue_families)
}
unsafe fn raw_impl(
device: Arc<Device>,
size: DeviceSize,
usage: BufferUsage,
queue_families: SmallVec<[u32; 4]>,
) -> Result<
(
Arc<ImmutableBuffer<T>>,
Arc<ImmutableBufferInitialization<T>>,
),
DeviceMemoryAllocationError,
> {
let buffer = match UnsafeBuffer::new(
device.clone(),
UnsafeBufferCreateInfo {
sharing: if queue_families.len() >= 2 {
Sharing::Concurrent(queue_families.clone())
} else {
Sharing::Exclusive
},
size,
usage,
..Default::default()
},
) {
Ok(b) => b,
Err(BufferCreationError::AllocError(err)) => return Err(err),
Err(_) => unreachable!(), };
let mem_reqs = buffer.memory_requirements();
let mem = MemoryPool::alloc_from_requirements(
&Device::standard_pool(&device),
&mem_reqs,
AllocLayout::Linear,
MappingRequirement::DoNotMap,
Some(DedicatedAllocation::Buffer(&buffer)),
|t| {
if t.is_device_local() {
AllocFromRequirementsFilter::Preferred
} else {
AllocFromRequirementsFilter::Allowed
}
},
)?;
debug_assert!((mem.offset() % mem_reqs.alignment) == 0);
buffer.bind_memory(mem.memory(), mem.offset())?;
let final_buf = Arc::new(ImmutableBuffer {
inner: buffer,
memory: mem,
queue_families: queue_families,
initialized: AtomicBool::new(false),
marker: PhantomData,
});
let initialization = Arc::new(ImmutableBufferInitialization {
buffer: final_buf.clone(),
used: Arc::new(AtomicBool::new(false)),
});
Ok((final_buf, initialization))
}
}
impl<T, A> ImmutableBuffer<T, A>
where
T: BufferContents + ?Sized,
{
#[inline]
pub fn device(&self) -> &Arc<Device> {
self.inner.device()
}
#[inline]
pub fn queue_families(&self) -> Vec<QueueFamily> {
self.queue_families
.iter()
.map(|&num| {
self.device()
.physical_device()
.queue_family_by_id(num)
.unwrap()
})
.collect()
}
}
unsafe impl<T, A> BufferAccess for ImmutableBuffer<T, A>
where
T: BufferContents + ?Sized,
A: Send + Sync,
{
#[inline]
fn inner(&self) -> BufferInner {
BufferInner {
buffer: &self.inner,
offset: 0,
}
}
#[inline]
fn size(&self) -> DeviceSize {
self.inner.size()
}
#[inline]
fn conflict_key(&self) -> (u64, u64) {
(self.inner.key(), 0)
}
#[inline]
fn try_gpu_lock(&self, exclusive_access: bool, _: &Queue) -> Result<(), AccessError> {
if exclusive_access {
return Err(AccessError::ExclusiveDenied);
}
if !self.initialized.load(Ordering::Relaxed) {
return Err(AccessError::BufferNotInitialized);
}
Ok(())
}
#[inline]
unsafe fn increase_gpu_lock(&self) {}
#[inline]
unsafe fn unlock(&self) {}
}
impl<T, A> BufferAccessObject for Arc<ImmutableBuffer<T, A>>
where
T: BufferContents + ?Sized,
A: Send + Sync + 'static,
{
#[inline]
fn as_buffer_access_object(&self) -> Arc<dyn BufferAccess> {
self.clone()
}
}
unsafe impl<T, A> TypedBufferAccess for ImmutableBuffer<T, A>
where
T: BufferContents + ?Sized,
A: Send + Sync,
{
type Content = T;
}
unsafe impl<T, A> DeviceOwned for ImmutableBuffer<T, A>
where
T: BufferContents + ?Sized,
{
#[inline]
fn device(&self) -> &Arc<Device> {
self.inner.device()
}
}
impl<T, A> PartialEq for ImmutableBuffer<T, A>
where
T: BufferContents + ?Sized,
A: Send + Sync,
{
#[inline]
fn eq(&self, other: &Self) -> bool {
self.inner() == other.inner() && self.size() == other.size()
}
}
impl<T, A> Eq for ImmutableBuffer<T, A>
where
T: BufferContents + ?Sized,
A: Send + Sync,
{
}
impl<T, A> Hash for ImmutableBuffer<T, A>
where
T: BufferContents + ?Sized,
A: Send + Sync,
{
#[inline]
fn hash<H: Hasher>(&self, state: &mut H) {
self.inner().hash(state);
self.size().hash(state);
}
}
#[derive(Debug)]
pub struct ImmutableBufferInitialization<T, A = PotentialDedicatedAllocation<StdMemoryPoolAlloc>>
where
T: BufferContents + ?Sized,
{
buffer: Arc<ImmutableBuffer<T, A>>,
used: Arc<AtomicBool>,
}
unsafe impl<T, A> BufferAccess for ImmutableBufferInitialization<T, A>
where
T: BufferContents + ?Sized,
A: Send + Sync,
{
#[inline]
fn inner(&self) -> BufferInner {
self.buffer.inner()
}
#[inline]
fn size(&self) -> DeviceSize {
self.buffer.size()
}
#[inline]
fn conflict_key(&self) -> (u64, u64) {
(self.buffer.inner.key(), 0)
}
#[inline]
fn try_gpu_lock(&self, _: bool, _: &Queue) -> Result<(), AccessError> {
if self.buffer.initialized.load(Ordering::Relaxed) {
return Err(AccessError::AlreadyInUse);
}
if !self
.used
.compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed)
.unwrap_or_else(|e| e)
{
Ok(())
} else {
Err(AccessError::AlreadyInUse)
}
}
#[inline]
unsafe fn increase_gpu_lock(&self) {
debug_assert!(self.used.load(Ordering::Relaxed));
}
#[inline]
unsafe fn unlock(&self) {
self.buffer.initialized.store(true, Ordering::Relaxed);
}
}
impl<T, A> BufferAccessObject for Arc<ImmutableBufferInitialization<T, A>>
where
T: BufferContents + ?Sized,
A: Send + Sync + 'static,
{
#[inline]
fn as_buffer_access_object(&self) -> Arc<dyn BufferAccess> {
self.clone()
}
}
unsafe impl<T, A> TypedBufferAccess for ImmutableBufferInitialization<T, A>
where
T: BufferContents + ?Sized,
A: Send + Sync,
{
type Content = T;
}
unsafe impl<T, A> DeviceOwned for ImmutableBufferInitialization<T, A>
where
T: BufferContents + ?Sized,
{
#[inline]
fn device(&self) -> &Arc<Device> {
self.buffer.inner.device()
}
}
impl<T, A> Clone for ImmutableBufferInitialization<T, A>
where
T: BufferContents + ?Sized,
{
#[inline]
fn clone(&self) -> ImmutableBufferInitialization<T, A> {
ImmutableBufferInitialization {
buffer: self.buffer.clone(),
used: self.used.clone(),
}
}
}
impl<T, A> PartialEq for ImmutableBufferInitialization<T, A>
where
T: BufferContents + ?Sized,
A: Send + Sync,
{
#[inline]
fn eq(&self, other: &Self) -> bool {
self.inner() == other.inner() && self.size() == other.size()
}
}
impl<T, A> Eq for ImmutableBufferInitialization<T, A>
where
T: BufferContents + ?Sized,
A: Send + Sync,
{
}
impl<T, A> Hash for ImmutableBufferInitialization<T, A>
where
T: BufferContents + ?Sized,
A: Send + Sync,
{
#[inline]
fn hash<H: Hasher>(&self, state: &mut H) {
self.inner().hash(state);
self.size().hash(state);
}
}
#[cfg(test)]
mod tests {
use crate::buffer::cpu_access::CpuAccessibleBuffer;
use crate::buffer::immutable::ImmutableBuffer;
use crate::buffer::BufferUsage;
use crate::command_buffer::AutoCommandBufferBuilder;
use crate::command_buffer::CommandBufferUsage;
use crate::command_buffer::PrimaryCommandBuffer;
use crate::sync::GpuFuture;
#[test]
fn from_data_working() {
let (device, queue) = gfx_dev_and_queue!();
let (buffer, _) =
ImmutableBuffer::from_data(12u32, BufferUsage::all(), queue.clone()).unwrap();
let destination =
CpuAccessibleBuffer::from_data(device.clone(), BufferUsage::all(), false, 0).unwrap();
let mut cbb = AutoCommandBufferBuilder::primary(
device.clone(),
queue.family(),
CommandBufferUsage::MultipleSubmit,
)
.unwrap();
cbb.copy_buffer(buffer, destination.clone()).unwrap();
let _ = cbb
.build()
.unwrap()
.execute(queue.clone())
.unwrap()
.then_signal_fence_and_flush()
.unwrap();
let destination_content = destination.read().unwrap();
assert_eq!(*destination_content, 12);
}
#[test]
fn from_iter_working() {
let (device, queue) = gfx_dev_and_queue!();
let (buffer, _) = ImmutableBuffer::from_iter(
(0..512u32).map(|n| n * 2),
BufferUsage::all(),
queue.clone(),
)
.unwrap();
let destination = CpuAccessibleBuffer::from_iter(
device.clone(),
BufferUsage::all(),
false,
(0..512).map(|_| 0u32),
)
.unwrap();
let mut cbb = AutoCommandBufferBuilder::primary(
device.clone(),
queue.family(),
CommandBufferUsage::MultipleSubmit,
)
.unwrap();
cbb.copy_buffer(buffer, destination.clone()).unwrap();
let _ = cbb
.build()
.unwrap()
.execute(queue.clone())
.unwrap()
.then_signal_fence_and_flush()
.unwrap();
let destination_content = destination.read().unwrap();
for (n, &v) in destination_content.iter().enumerate() {
assert_eq!(n * 2, v as usize);
}
}
#[test]
fn writing_forbidden() {
let (device, queue) = gfx_dev_and_queue!();
let (buffer, _) =
ImmutableBuffer::from_data(12u32, BufferUsage::all(), queue.clone()).unwrap();
assert_should_panic!({
let mut cbb = AutoCommandBufferBuilder::primary(
device.clone(),
queue.family(),
CommandBufferUsage::MultipleSubmit,
)
.unwrap();
cbb.fill_buffer(buffer, 50).unwrap();
let _ = cbb
.build()
.unwrap()
.execute(queue.clone())
.unwrap()
.then_signal_fence_and_flush()
.unwrap();
});
}
#[test]
fn read_uninitialized_forbidden() {
let (device, queue) = gfx_dev_and_queue!();
let (buffer, _) = unsafe {
ImmutableBuffer::<u32>::uninitialized(device.clone(), BufferUsage::all()).unwrap()
};
let source =
CpuAccessibleBuffer::from_data(device.clone(), BufferUsage::all(), false, 0).unwrap();
assert_should_panic!({
let mut cbb = AutoCommandBufferBuilder::primary(
device.clone(),
queue.family(),
CommandBufferUsage::MultipleSubmit,
)
.unwrap();
cbb.copy_buffer(source, buffer).unwrap();
let _ = cbb
.build()
.unwrap()
.execute(queue.clone())
.unwrap()
.then_signal_fence_and_flush()
.unwrap();
});
}
#[test]
fn init_then_read_same_cb() {
let (device, queue) = gfx_dev_and_queue!();
let (buffer, init) = unsafe {
ImmutableBuffer::<u32>::uninitialized(device.clone(), BufferUsage::all()).unwrap()
};
let source =
CpuAccessibleBuffer::from_data(device.clone(), BufferUsage::all(), false, 0).unwrap();
let mut cbb = AutoCommandBufferBuilder::primary(
device.clone(),
queue.family(),
CommandBufferUsage::MultipleSubmit,
)
.unwrap();
cbb.copy_buffer(source.clone(), init)
.unwrap()
.copy_buffer(buffer, source.clone())
.unwrap();
let _ = cbb
.build()
.unwrap()
.execute(queue.clone())
.unwrap()
.then_signal_fence_and_flush()
.unwrap();
}
#[test]
#[ignore] fn init_then_read_same_future() {
let (device, queue) = gfx_dev_and_queue!();
let (buffer, init) = unsafe {
ImmutableBuffer::<u32>::uninitialized(device.clone(), BufferUsage::all()).unwrap()
};
let source =
CpuAccessibleBuffer::from_data(device.clone(), BufferUsage::all(), false, 0).unwrap();
let mut cbb = AutoCommandBufferBuilder::primary(
device.clone(),
queue.family(),
CommandBufferUsage::MultipleSubmit,
)
.unwrap();
cbb.copy_buffer(source.clone(), init).unwrap();
let cb1 = cbb.build().unwrap();
let mut cbb = AutoCommandBufferBuilder::primary(
device.clone(),
queue.family(),
CommandBufferUsage::MultipleSubmit,
)
.unwrap();
cbb.copy_buffer(buffer, source.clone()).unwrap();
let cb2 = cbb.build().unwrap();
let _ = cb1
.execute(queue.clone())
.unwrap()
.then_execute(queue.clone(), cb2)
.unwrap()
.then_signal_fence_and_flush()
.unwrap();
}
#[test]
#[allow(unused)]
fn create_buffer_zero_size_data() {
let (device, queue) = gfx_dev_and_queue!();
assert_should_panic!({
ImmutableBuffer::from_data((), BufferUsage::all(), queue.clone()).unwrap();
});
}
}