#![deny(unsafe_op_in_unsafe_fn)]
use std::cell::RefCell;
use std::collections::HashMap;
use std::ops::Deref;
use std::rc::{Rc, Weak};
use objc2::Message as _;
use objc2::rc::Retained;
use objc2::runtime::ProtocolObject;
use objc2_metal::{
MTLBuffer, MTLCPUCacheMode, MTLDevice as _, MTLHazardTrackingMode, MTLHeap, MTLHeapDescriptor,
MTLHeapType, MTLResourceOptions, MTLStorageMode, MTLTexture, MTLTextureDescriptor,
};
use crate::suballoc::block_alloc::{BlockAllocator, Placement};
const MAX_BLOCK_BYTES: u64 = 64 * 1024 * 1024;
const FIRST_BLOCK_BYTES: u64 = 8 * 1024 * 1024;
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
struct PoolKey {
storage: MTLStorageMode,
cache: MTLCPUCacheMode,
}
impl PoolKey {
fn options(self) -> MTLResourceOptions {
MTLResourceOptions(
(self.storage.0 << MTL_RESOURCE_STORAGE_MODE_SHIFT)
| (self.cache.0 << MTL_RESOURCE_CPU_CACHE_MODE_SHIFT),
)
}
fn check_heap_backed(self) -> Result<Self, String> {
if self.storage == MTLStorageMode::Shared || self.storage == MTLStorageMode::Private {
Ok(self)
} else {
Err(format!(
"allocator: storage mode {} cannot back a heap",
self.storage.0
))
}
}
}
const MTL_RESOURCE_CPU_CACHE_MODE_SHIFT: usize = 0;
const MTL_RESOURCE_STORAGE_MODE_SHIFT: usize = 4;
fn pool_key(options: MTLResourceOptions) -> Result<PoolKey, String> {
PoolKey {
storage: MTLStorageMode((options.0 >> MTL_RESOURCE_STORAGE_MODE_SHIFT) & 0xf),
cache: MTLCPUCacheMode((options.0 >> MTL_RESOURCE_CPU_CACHE_MODE_SHIFT) & 0xf),
}
.check_heap_backed()
}
struct Pool {
placement: BlockAllocator,
heaps: Vec<Option<Retained<ProtocolObject<dyn MTLHeap>>>>,
}
impl Pool {
fn new() -> Self {
Self {
placement: BlockAllocator::new(MAX_BLOCK_BYTES),
heaps: Vec::new(),
}
}
fn next_block_bytes(&self, size: u64, align: u64) -> u64 {
let grown = FIRST_BLOCK_BYTES
.saturating_mul(1 << self.placement.block_count().min(3))
.min(MAX_BLOCK_BYTES);
let needed = size.saturating_add(align.max(1).saturating_sub(1));
needed.max(grown)
}
}
struct Inner {
pools: HashMap<PoolKey, Pool>,
frame: u64,
retire_depth: u64,
}
impl Inner {
fn free(&mut self, key: PoolKey, placement: Placement, size: u64) {
let retire = self.frame + self.retire_depth;
if let Some(pool) = self.pools.get_mut(&key) {
pool.placement.free(placement, size, retire);
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(in crate::metal) struct AllocatorStats {
pub(in crate::metal) reserved_bytes: u64,
pub(in crate::metal) in_use_bytes: u64,
pub(in crate::metal) block_count: usize,
}
struct Reservation {
heap: Retained<ProtocolObject<dyn MTLHeap>>,
key: PoolKey,
placement: Placement,
size: u64,
}
struct Lease {
owner: Weak<RefCell<Inner>>,
key: PoolKey,
placement: Placement,
size: u64,
}
impl Drop for Lease {
fn drop(&mut self) {
if let Some(inner) = self.owner.upgrade() {
inner.borrow_mut().free(self.key, self.placement, self.size);
}
}
}
pub(in crate::metal) struct PooledBuffer {
buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
_lease: Lease,
}
impl PooledBuffer {
pub(in crate::metal) fn retained(&self) -> Retained<ProtocolObject<dyn MTLBuffer>> {
self.buffer.clone()
}
}
impl Deref for PooledBuffer {
type Target = ProtocolObject<dyn MTLBuffer>;
fn deref(&self) -> &Self::Target {
&self.buffer
}
}
impl AsRef<ProtocolObject<dyn MTLBuffer>> for PooledBuffer {
fn as_ref(&self) -> &ProtocolObject<dyn MTLBuffer> {
&self.buffer
}
}
pub(in crate::metal) struct PooledTexture {
texture: Retained<ProtocolObject<dyn MTLTexture>>,
_lease: Lease,
}
impl Deref for PooledTexture {
type Target = ProtocolObject<dyn MTLTexture>;
fn deref(&self) -> &Self::Target {
&self.texture
}
}
impl AsRef<ProtocolObject<dyn MTLTexture>> for PooledTexture {
fn as_ref(&self) -> &ProtocolObject<dyn MTLTexture> {
&self.texture
}
}
pub(in crate::metal) struct DeviceAllocator {
device: Retained<ProtocolObject<dyn objc2_metal::MTLDevice>>,
inner: Rc<RefCell<Inner>>,
}
impl DeviceAllocator {
pub(in crate::metal) fn new(
device: &ProtocolObject<dyn objc2_metal::MTLDevice>,
frames_in_flight: usize,
) -> Self {
Self {
device: device.retain(),
inner: Rc::new(RefCell::new(Inner {
pools: HashMap::new(),
frame: 0,
retire_depth: frames_in_flight as u64 + 1,
})),
}
}
pub(in crate::metal) fn device(&self) -> &ProtocolObject<dyn objc2_metal::MTLDevice> {
&self.device
}
pub(in crate::metal) fn alloc_buffer(
&self,
len: usize,
options: MTLResourceOptions,
) -> Result<PooledBuffer, String> {
let key = pool_key(options)?;
let len = len.max(1);
let sizing = self
.device
.heapBufferSizeAndAlignWithLength_options(len, key.options());
let reservation = self.reserve(key, sizing.size as u64, sizing.align as u64)?;
let placed = unsafe {
reservation.heap.newBufferWithLength_options_offset(
len,
key.options(),
reservation.placement.offset as usize,
)
};
match placed {
Some(buffer) => Ok(PooledBuffer {
buffer,
_lease: self.lease(reservation),
}),
None => {
self.release(reservation);
Err(format!("allocator: failed to place {len}-byte buffer"))
}
}
}
pub(in crate::metal) fn alloc_buffer_with_bytes(
&self,
src: &[u8],
options: MTLResourceOptions,
) -> Result<PooledBuffer, String> {
if pool_key(options)?.storage != MTLStorageMode::Shared {
return Err("allocator: initialised buffers need shared storage".to_string());
}
let buffer = self.alloc_buffer(src.len(), options)?;
super::context::write_buffer_region(&buffer, 0, src)?;
Ok(buffer)
}
pub(in crate::metal) fn alloc_texture(
&self,
desc: &MTLTextureDescriptor,
) -> Result<PooledTexture, String> {
let key = PoolKey {
storage: desc.storageMode(),
cache: desc.cpuCacheMode(),
}
.check_heap_backed()?;
let sizing = self.device.heapTextureSizeAndAlignWithDescriptor(desc);
let reservation = self.reserve(key, sizing.size as u64, sizing.align as u64)?;
let placed = unsafe {
reservation
.heap
.newTextureWithDescriptor_offset(desc, reservation.placement.offset as usize)
};
match placed {
Some(texture) => Ok(PooledTexture {
texture,
_lease: self.lease(reservation),
}),
None => {
self.release(reservation);
Err(format!(
"allocator: failed to place {}-byte texture",
sizing.size
))
}
}
}
pub(in crate::metal) fn begin_frame(&self) {
let mut inner = self.inner.borrow_mut();
inner.frame += 1;
let frame = inner.frame;
for pool in inner.pools.values_mut() {
pool.placement.reclaim(frame);
for index in pool.placement.take_empty_blocks() {
if let Some(slot) = pool.heaps.get_mut(index) {
*slot = None;
}
}
}
}
pub(in crate::metal) fn stats(&self) -> AllocatorStats {
let inner = self.inner.borrow();
let mut stats = AllocatorStats::default();
for pool in inner.pools.values() {
stats.reserved_bytes += pool.placement.reserved_bytes();
stats.in_use_bytes += pool.placement.in_use_bytes();
stats.block_count += pool.placement.block_count();
}
stats
}
fn reserve(&self, key: PoolKey, size: u64, align: u64) -> Result<Reservation, String> {
let mut inner = self.inner.borrow_mut();
let pool = inner.pools.entry(key).or_insert_with(Pool::new);
if let Some(placement) = pool.placement.alloc(size, align) {
let heap = pool.heaps[placement.block]
.clone()
.ok_or("allocator: placement named a released heap")?;
return Ok(Reservation {
heap,
key,
placement,
size,
});
}
let block_bytes = pool.next_block_bytes(size, align);
let heap = new_heap(&self.device, key, block_bytes)?;
let index = pool.placement.add_block(block_bytes);
if index == pool.heaps.len() {
pool.heaps.push(Some(heap.clone()));
} else {
pool.heaps[index] = Some(heap.clone());
}
let placement = pool
.placement
.alloc_in(index, size, align)
.ok_or("allocator: a block sized for a request failed to host it")?;
Ok(Reservation {
heap,
key,
placement,
size,
})
}
fn lease(&self, reservation: Reservation) -> Lease {
Lease {
owner: Rc::downgrade(&self.inner),
key: reservation.key,
placement: reservation.placement,
size: reservation.size,
}
}
fn release(&self, reservation: Reservation) {
let mut inner = self.inner.borrow_mut();
if let Some(pool) = inner.pools.get_mut(&reservation.key) {
pool.placement
.free(reservation.placement, reservation.size, 0);
}
}
}
fn new_heap(
device: &ProtocolObject<dyn objc2_metal::MTLDevice>,
key: PoolKey,
size: u64,
) -> Result<Retained<ProtocolObject<dyn MTLHeap>>, String> {
let desc = MTLHeapDescriptor::new();
desc.setType(MTLHeapType::Placement);
desc.setStorageMode(key.storage);
desc.setCpuCacheMode(key.cache);
desc.setHazardTrackingMode(MTLHazardTrackingMode::Tracked);
desc.setSize(size.max(1) as usize);
device
.newHeapWithDescriptor(&desc)
.ok_or_else(|| format!("allocator: failed to create a {size}-byte heap"))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::metal::descriptors::TextureDesc;
fn device() -> Option<Retained<ProtocolObject<dyn objc2_metal::MTLDevice>>> {
objc2_metal::MTLCreateSystemDefaultDevice()
}
fn shared_texture_desc(width: usize) -> Retained<MTLTextureDescriptor> {
TextureDesc {
width,
height: width,
storage: MTLStorageMode::Shared,
..Default::default()
}
.build()
}
#[test]
fn pool_key_round_trips_through_resource_options() {
let key = pool_key(MTLResourceOptions::StorageModeShared).expect("shared is heap-backed");
assert_eq!(key.storage, MTLStorageMode::Shared);
assert_eq!(key.cache, MTLCPUCacheMode::DefaultCache);
assert_eq!(key.options(), MTLResourceOptions::StorageModeShared);
let combined =
MTLResourceOptions::StorageModePrivate | MTLResourceOptions::CPUCacheModeWriteCombined;
let key = pool_key(combined).expect("private is heap-backed");
assert_eq!(key.storage, MTLStorageMode::Private);
assert_eq!(key.cache, MTLCPUCacheMode::WriteCombined);
assert_eq!(key.options(), combined);
}
#[test]
fn hazard_tracking_bits_do_not_split_a_pool() {
let tracked = pool_key(
MTLResourceOptions::StorageModeShared | MTLResourceOptions::HazardTrackingModeTracked,
)
.expect("shared is heap-backed");
let untracked = pool_key(
MTLResourceOptions::StorageModeShared | MTLResourceOptions::HazardTrackingModeUntracked,
)
.expect("shared is heap-backed");
assert_eq!(tracked, untracked);
assert_eq!(tracked.options(), MTLResourceOptions::StorageModeShared);
}
#[test]
fn storage_modes_a_heap_cannot_back_are_rejected() {
assert!(pool_key(MTLResourceOptions::StorageModeManaged).is_err());
assert!(pool_key(MTLResourceOptions::StorageModeMemoryless).is_err());
}
#[test]
fn blocks_grow_from_the_first_size_up_to_the_cap() {
let mut pool = Pool::new();
let mut sizes = Vec::new();
for _ in 0..5 {
let bytes = pool.next_block_bytes(1024, 256);
sizes.push(bytes);
pool.placement.add_block(bytes);
}
assert_eq!(
sizes,
vec![
FIRST_BLOCK_BYTES,
FIRST_BLOCK_BYTES * 2,
FIRST_BLOCK_BYTES * 4,
MAX_BLOCK_BYTES,
MAX_BLOCK_BYTES,
]
);
}
#[test]
fn an_oversized_request_sizes_its_own_block() {
let pool = Pool::new();
let huge = MAX_BLOCK_BYTES * 3;
assert_eq!(pool.next_block_bytes(huge, 256), huge + 255);
}
#[test]
fn many_buffers_share_few_heaps() {
let Some(device) = device() else {
return;
};
let alloc = DeviceAllocator::new(&device, 2);
let buffers: Vec<PooledBuffer> = (0..512)
.map(|_| {
alloc
.alloc_buffer(4096, MTLResourceOptions::StorageModeShared)
.expect("shared buffer places")
})
.collect();
let stats = alloc.stats();
assert_eq!(stats.block_count, 1, "{stats:?}");
assert!(stats.in_use_bytes >= 512 * 4096, "{stats:?}");
assert!(stats.reserved_bytes >= stats.in_use_bytes, "{stats:?}");
drop(buffers);
}
#[test]
fn placed_buffers_get_distinct_non_overlapping_storage() {
let Some(device) = device() else {
return;
};
let alloc = DeviceAllocator::new(&device, 2);
let a = alloc
.alloc_buffer_with_bytes(&[0xAAu8; 256], MTLResourceOptions::StorageModeShared)
.expect("shared buffer places");
let b = alloc
.alloc_buffer_with_bytes(&[0x55u8; 256], MTLResourceOptions::StorageModeShared)
.expect("shared buffer places");
let (a_bytes, b_bytes) = unsafe {
(
std::slice::from_raw_parts(a.contents().as_ptr() as *const u8, 256),
std::slice::from_raw_parts(b.contents().as_ptr() as *const u8, 256),
)
};
assert!(a_bytes.iter().all(|&x| x == 0xAA));
assert!(b_bytes.iter().all(|&x| x == 0x55));
}
#[test]
fn a_dropped_lease_is_withheld_until_its_retire_frame() {
let Some(device) = device() else {
return;
};
let alloc = DeviceAllocator::new(&device, 2);
let first = alloc
.alloc_buffer(4096, MTLResourceOptions::StorageModeShared)
.expect("shared buffer places");
let stats = alloc.stats();
assert_eq!(stats.block_count, 1);
drop(first);
assert_eq!(alloc.stats().in_use_bytes, 0);
for _ in 0..4 {
alloc.begin_frame();
}
assert_eq!(alloc.stats().block_count, 0, "emptied heap is released");
assert_eq!(alloc.stats().reserved_bytes, 0);
}
#[test]
fn textures_and_buffers_share_a_pool_when_their_modes_match() {
let Some(device) = device() else {
return;
};
let alloc = DeviceAllocator::new(&device, 2);
let _buffer = alloc
.alloc_buffer(4096, MTLResourceOptions::StorageModeShared)
.expect("shared buffer places");
let _texture = alloc
.alloc_texture(&shared_texture_desc(64))
.expect("shared texture places");
assert_eq!(alloc.stats().block_count, 1);
}
#[test]
fn a_texture_larger_than_the_cap_gets_its_own_heap() {
let Some(device) = device() else {
return;
};
let alloc = DeviceAllocator::new(&device, 2);
let big = alloc
.alloc_texture(&shared_texture_desc(4096))
.expect("oversized texture places");
let small = alloc
.alloc_texture(&shared_texture_desc(64))
.expect("small texture places");
let stats = alloc.stats();
assert_eq!(stats.block_count, 2, "{stats:?}");
drop((big, small));
}
}