use {
crate::{align_down, align_up, error::MapError},
core::{
convert::TryFrom as _,
ptr::{copy_nonoverlapping, NonNull},
sync::atomic::{AtomicU8, Ordering::*},
},
gpu_alloc_types::{MappedMemoryRange, MemoryDevice, MemoryPropertyFlags},
};
#[derive(Debug)]
struct Relevant;
impl Drop for Relevant {
#[cfg(feature = "tracing")]
fn drop(&mut self) {
tracing::error!("Memory block wasn't deallocated");
}
#[cfg(all(not(feature = "tracing"), feature = "std"))]
fn drop(&mut self) {
eprintln!("Memory block wasn't deallocated")
}
#[cfg(all(not(feature = "tracing"), not(feature = "std")))]
fn drop(&mut self) {
panic!("Memory block wasn't deallocated")
}
}
const MAPPING_STATE_UNMAPPED: u8 = 0;
const MAPPING_STATE_MAPPED: u8 = 1;
const MAPPING_STATE_UNMAPPING: u8 = 2;
#[derive(Debug)]
pub struct MemoryBlock<M> {
memory: M,
memory_type: u32,
props: MemoryPropertyFlags,
offset: u64,
size: u64,
map_mask: u64,
mapped: AtomicU8,
flavor: MemoryBlockFlavor,
relevant: Relevant,
}
impl<M> MemoryBlock<M> {
pub(crate) fn new(
memory: M,
memory_type: u32,
props: MemoryPropertyFlags,
offset: u64,
size: u64,
map_mask: u64,
flavor: MemoryBlockFlavor,
) -> Self {
MemoryBlock {
memory,
memory_type,
props,
offset,
size,
map_mask,
flavor,
mapped: AtomicU8::new(MAPPING_STATE_UNMAPPED),
relevant: Relevant,
}
}
pub(crate) fn deallocate(self) -> (M, MemoryBlockFlavor) {
core::mem::forget(self.relevant);
(self.memory, self.flavor)
}
}
unsafe impl<M> Sync for MemoryBlock<M> where M: Sync {}
unsafe impl<M> Send for MemoryBlock<M> where M: Send {}
#[derive(Debug)]
pub(crate) enum MemoryBlockFlavor {
Dedicated,
Linear {
chunk: u64,
ptr: Option<NonNull<u8>>,
},
Buddy {
chunk: usize,
index: usize,
ptr: Option<NonNull<u8>>,
},
}
impl<M> MemoryBlock<M> {
#[inline(always)]
pub fn memory(&self) -> &M {
&self.memory
}
#[inline(always)]
pub fn offset(&self) -> u64 {
self.offset
}
#[inline(always)]
pub fn size(&self) -> u64 {
self.size
}
#[inline(always)]
pub fn props(&self) -> MemoryPropertyFlags {
self.props
}
#[inline(always)]
pub fn memory_type(&self) -> u32 {
self.memory_type
}
#[inline(always)]
pub unsafe fn map(
&self,
device: &impl MemoryDevice<M>,
offset: u64,
size: usize,
) -> Result<NonNull<u8>, MapError> {
let size_u64 = u64::try_from(size)
.expect("`size` doesn't fit device address space");
let size = align_up(size_u64, self.map_mask)
.expect("aligned `size` doesn't fit device address space");
let aligned_offset = align_down(offset, self.map_mask);
assert!(offset < self.size, "`offset` is out of memory block bounds");
assert!(
size_u64 <= self.size - offset,
"`offset + size` is out of memory block bounds"
);
let ptr = match self.flavor {
MemoryBlockFlavor::Dedicated => {
let offset_align_shift = offset - aligned_offset;
let offset_align_shift = isize::try_from(offset_align_shift)
.expect("`non_coherent_atom_size` is too large");
if !self.acquire_mapping() {
return Err(MapError::AlreadyMapped);
}
let aligned_size = offset + size - aligned_offset;
let result = device.map_memory(
&self.memory,
self.offset + aligned_offset,
aligned_size,
);
match result {
Ok(ptr) => ptr.as_ptr().offset(offset_align_shift),
Err(err) => {
self.mapping_failed();
return Err(err.into());
}
}
}
MemoryBlockFlavor::Linear { ptr: Some(ptr), .. }
| MemoryBlockFlavor::Buddy { ptr: Some(ptr), .. } => {
if !self.acquire_mapping() {
return Err(MapError::AlreadyMapped);
}
let offset_isize = isize::try_from(offset).expect(
"Buddy and linear block should fit host address space",
);
ptr.as_ptr().offset(offset_isize)
}
_ => return Err(MapError::NonHostVisible),
};
Ok(NonNull::new_unchecked(ptr))
}
#[inline(always)]
pub unsafe fn unmap(&self, device: &impl MemoryDevice<M>) -> bool {
if !self.start_unmapping() {
return false;
}
match self.flavor {
MemoryBlockFlavor::Dedicated => {
device.unmap_memory(&self.memory);
}
MemoryBlockFlavor::Linear { .. } => {}
MemoryBlockFlavor::Buddy { .. } => {}
}
self.end_unmapping();
true
}
#[inline(always)]
pub unsafe fn write_bytes(
&self,
device: &impl MemoryDevice<M>,
offset: u64,
data: &[u8],
) -> Result<(), MapError> {
let size = data.len();
let ptr = self.map(device, offset, size)?;
copy_nonoverlapping(data.as_ptr(), ptr.as_ptr(), size);
let result = if !self.coherent() {
let aligned_offset = align_down(offset, self.map_mask);
let size = align_up(data.len() as u64, self.map_mask).unwrap();
device.flush_memory_ranges(&[MappedMemoryRange {
memory: &self.memory,
offset: aligned_offset,
size,
}])
} else {
Ok(())
};
self.unmap(device);
result.map_err(Into::into)
}
#[inline(always)]
pub unsafe fn read_bytes(
&self,
device: &impl MemoryDevice<M>,
offset: u64,
data: &mut [u8],
) -> Result<(), MapError> {
#[cfg(feature = "tracing")]
{
if !self.cached() {
tracing::warn!("Reading from non-cached memory may be slow. Consider allocating HOST_CACHED memory block for host reads.")
}
}
let size = data.len();
let ptr = self.map(device, offset, size)?;
let result = if !self.coherent() {
let aligned_offset = align_down(offset, self.map_mask);
let size = align_up(data.len() as u64, self.map_mask).unwrap();
device.invalidate_memory_ranges(&[MappedMemoryRange {
memory: &self.memory,
offset: aligned_offset,
size,
}])
} else {
Ok(())
};
if result.is_ok() {
copy_nonoverlapping(ptr.as_ptr(), data.as_mut_ptr(), size);
}
self.unmap(device);
result.map_err(Into::into)
}
fn acquire_mapping(&self) -> bool {
self.mapped
.compare_exchange(
MAPPING_STATE_UNMAPPED,
MAPPING_STATE_MAPPED,
Acquire,
Relaxed,
)
.is_ok()
}
fn mapping_failed(&self) {
self.mapped.store(MAPPING_STATE_UNMAPPED, Relaxed);
}
fn start_unmapping(&self) -> bool {
self.mapped
.compare_exchange(
MAPPING_STATE_MAPPED,
MAPPING_STATE_UNMAPPING,
Release,
Relaxed,
)
.is_ok()
}
fn end_unmapping(&self) {
self.mapped.store(MAPPING_STATE_UNMAPPED, Relaxed);
}
fn coherent(&self) -> bool {
self.props.contains(MemoryPropertyFlags::HOST_COHERENT)
}
#[cfg(feature = "tracing")]
fn cached(&self) -> bool {
self.props.contains(MemoryPropertyFlags::HOST_CACHED)
}
}