use super::super::{
BufferHandle, ComputePipelineHandle, DeviceHandle, PipelineHandle, RenderTargetHandle, SamplerHandle, ShaderHandle,
SurfaceHandle, TextureHandle,
};
use crate::backend::BufferKind;
use crate::timeline::TimelineValue;
use crate::types::{DepthFormat, TextureFormat};
use std::collections::{HashMap, VecDeque};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Condvar, Mutex};
use ::metal as mtl;
use mtl::{
ArgumentEncoder, Buffer as MTLBuffer, CommandQueue, ComputePipelineState as MTLComputePipelineState,
DepthStencilState as MTLDepthStencilState, Device as MTLDevice, Heap, Library, MTLPrimitiveType,
MTLResourceOptions, RenderPipelineState, SamplerState, SharedEvent, Texture as MTLTexture,
};
pub const ARGUMENT_BUFFER_SIZE: u64 = 20 * 1024 * 8;
pub const RESOURCE_SLOT_BUFFER: u64 = 1;
pub const VERTEX_BUFFER_START_SLOT: u64 = 2;
pub use super::super::shared::PushLayout;
const MIN_HEAP_SIZE: u64 = 64 * 1024 * 1024;
const MIN_OVERFLOW_HEAP_SIZE: u64 = 16 * 1024 * 1024;
const MAX_OVERFLOW_HEAPS: usize = 16;
pub(super) const MAX_HEAP_SIZE: u64 = 1024 * 1024 * 1024;
pub(crate) struct HeapAllocator {
device: MTLDevice,
primary: Heap,
overflow: Vec<Heap>,
high_water_mark: u64,
primary_size: u64,
buffer_count: u32,
}
impl HeapAllocator {
pub fn new(device: MTLDevice, primary: Heap, primary_size: u64) -> Self {
Self {
device,
primary,
overflow: Vec::new(),
high_water_mark: 0,
primary_size,
buffer_count: 0,
}
}
pub fn allocate(&mut self, size: u64, options: MTLResourceOptions) -> Option<MTLBuffer> {
if size > MAX_HEAP_SIZE {
tracing::error!(
"Refusing buffer allocation of {}MB (cap={}MB); this usually indicates \
a stale bump counter or similar upstream corruption",
size / 1024 / 1024,
MAX_HEAP_SIZE / 1024 / 1024,
);
return None;
}
if let Some(buf) = self.primary.new_buffer(size, options) {
self.buffer_count += 1;
self.update_high_water_mark();
return Some(buf);
}
if tracing::enabled!(target: "goldy::diag::alloc", tracing::Level::INFO) {
tracing::info!(
target: "goldy::diag::alloc",
size_mb = size / (1024 * 1024),
primary_used_mb = self.primary.used_size() / (1024 * 1024),
primary_total_mb = self.primary_size / (1024 * 1024),
overflow_count = self.overflow.len(),
"heap.primary_full"
);
}
for (idx, heap) in self.overflow.iter().rev().enumerate() {
if let Some(buf) = heap.new_buffer(size, options) {
if tracing::enabled!(target: "goldy::diag::alloc", tracing::Level::INFO) {
tracing::info!(
target: "goldy::diag::alloc",
size_mb = size / (1024 * 1024),
heap_idx = self.overflow.len() - 1 - idx,
overflow_count = self.overflow.len(),
"heap.alloc_from_overflow"
);
}
self.buffer_count += 1;
self.update_high_water_mark();
return Some(buf);
}
}
if self.overflow.len() >= MAX_OVERFLOW_HEAPS {
return None;
}
let overflow_size = (size * 2).clamp(MIN_OVERFLOW_HEAP_SIZE, MAX_HEAP_SIZE);
let new_heap = self.create_heap(overflow_size);
tracing::info!(
target: "goldy::diag::alloc",
"Created overflow buffer heap (size={}MB, overflow_count={}, request_mb={}, primary_used_mb={}/{}MB, hwm_mb={})",
overflow_size / 1024 / 1024,
self.overflow.len() + 1,
size / (1024 * 1024),
self.primary.used_size() / (1024 * 1024),
self.primary_size / (1024 * 1024),
self.high_water_mark / (1024 * 1024),
);
let buf = new_heap.new_buffer(size, options);
self.overflow.push(new_heap);
if buf.is_some() {
self.buffer_count += 1;
self.update_high_water_mark();
}
buf
}
pub fn has_buffers(&self) -> bool {
self.buffer_count > 0
}
pub fn buffer_count(&self) -> u32 {
self.buffer_count
}
pub fn overflow_count(&self) -> usize {
self.overflow.len()
}
pub fn high_water_mark(&self) -> u64 {
self.high_water_mark
}
pub fn primary_size(&self) -> u64 {
self.primary_size
}
pub fn use_heaps_for_compute(&self, encoder: &mtl::ComputeCommandEncoderRef) {
if !self.has_buffers() {
return;
}
encoder.use_heap(&self.primary);
for heap in &self.overflow {
encoder.use_heap(heap);
}
}
pub fn use_heaps_for_render(&self, encoder: &mtl::RenderCommandEncoderRef, stages: mtl::MTLRenderStages) {
if !self.has_buffers() {
return;
}
encoder.use_heap_at(&self.primary, stages);
for heap in &self.overflow {
encoder.use_heap_at(heap, stages);
}
}
pub fn reset_for_frame(&mut self) {
if !self.overflow.is_empty() {
let recommended_max = self.device.recommended_max_working_set_size();
let new_size = (self.high_water_mark * 3 / 2)
.next_power_of_two()
.max(MIN_HEAP_SIZE)
.min(recommended_max / 2);
if new_size > self.primary_size {
let new_primary = self.create_heap(new_size);
tracing::info!(
target: "goldy::diag::alloc",
"Resized primary buffer heap: {}MB -> {}MB (high_water_mark={}MB)",
self.primary_size / 1024 / 1024,
new_size / 1024 / 1024,
self.high_water_mark / 1024 / 1024,
);
self.primary = new_primary;
self.primary_size = new_size;
}
let overflow_count = self.overflow.len();
self.overflow.clear();
if tracing::enabled!(target: "goldy::diag::alloc", tracing::Level::INFO) {
tracing::info!(
target: "goldy::diag::alloc",
overflow_cleared = overflow_count,
primary_size_mb = self.primary_size / (1024 * 1024),
"heap.reset_for_frame"
);
} else {
tracing::debug!("Cleared {} overflow buffer heaps", overflow_count);
}
}
self.high_water_mark = 0;
}
pub fn compact_overflow(&mut self) {
let before = self.overflow.len();
self.overflow.retain(|heap| heap.used_size() > 0);
let dropped = before - self.overflow.len();
if dropped > 0 {
if tracing::enabled!(target: "goldy::diag::alloc", tracing::Level::INFO) {
tracing::info!(
target: "goldy::diag::alloc",
freed = dropped,
overflow_remaining = self.overflow.len(),
"heap.compact"
);
} else {
tracing::debug!(
"Compacted {} empty overflow buffer heaps ({} remaining)",
dropped,
self.overflow.len()
);
}
}
}
pub fn ensure_primary_capacity(&mut self, min_capacity: u64) {
let min_capacity = min_capacity.min(MAX_HEAP_SIZE);
let recommended_max = self.device.recommended_max_working_set_size();
let target = min_capacity
.next_power_of_two()
.max(MIN_HEAP_SIZE)
.min(recommended_max / 2);
let too_small = self.primary_size < min_capacity;
let too_large = self.primary_size > target.saturating_mul(4);
if too_small || too_large {
let new_primary = self.create_heap(target);
tracing::info!(
target: "goldy::diag::alloc",
"{} primary buffer heap: {}MB -> {}MB (requested={}MB)",
if too_small { "Grew" } else { "Shrank" },
self.primary_size / 1024 / 1024,
target / 1024 / 1024,
min_capacity / 1024 / 1024,
);
self.primary = new_primary;
self.primary_size = target;
}
}
fn update_high_water_mark(&mut self) {
let mut total = self.primary.used_size();
for heap in &self.overflow {
total += heap.used_size();
}
self.high_water_mark = self.high_water_mark.max(total);
}
fn create_heap(&self, size: u64) -> Heap {
let desc = mtl::HeapDescriptor::new();
desc.set_size(size);
desc.set_storage_mode(mtl::MTLStorageMode::Shared);
desc.set_cpu_cache_mode(mtl::MTLCPUCacheMode::DefaultCache);
desc.set_heap_type(mtl::MTLHeapType::Automatic);
desc.set_hazard_tracking_mode(mtl::MTLHazardTrackingMode::Tracked);
self.device.new_heap(&desc)
}
}
pub(crate) struct TextureHeapAllocator {
device: MTLDevice,
primary: Heap,
overflow: Vec<Heap>,
#[allow(dead_code)]
primary_size: u64,
texture_count: u32,
}
impl TextureHeapAllocator {
pub fn new(device: MTLDevice, primary: Heap, primary_size: u64) -> Self {
Self {
device,
primary,
overflow: Vec::new(),
primary_size,
texture_count: 0,
}
}
pub fn allocate(&mut self, descriptor: &mtl::TextureDescriptorRef) -> Option<MTLTexture> {
if let Some(tex) = self.primary.new_texture(descriptor) {
self.texture_count += 1;
return Some(tex);
}
for heap in self.overflow.iter().rev() {
if let Some(tex) = heap.new_texture(descriptor) {
self.texture_count += 1;
return Some(tex);
}
}
if self.overflow.len() >= MAX_OVERFLOW_HEAPS {
return None;
}
let alloc_size = self.device.heap_texture_size_and_align(descriptor).size;
let overflow_size = (alloc_size * 2).max(MIN_OVERFLOW_HEAP_SIZE);
let new_heap = self.create_heap(overflow_size);
tracing::info!(
target: "goldy::diag::alloc",
"Created overflow texture heap (size={}MB, overflow_count={})",
overflow_size / 1024 / 1024,
self.overflow.len() + 1
);
let tex = new_heap.new_texture(descriptor);
self.overflow.push(new_heap);
if tex.is_some() {
self.texture_count += 1;
}
tex
}
pub fn compact_overflow(&mut self) {
let before = self.overflow.len();
self.overflow.retain(|heap| heap.used_size() > 0);
let dropped = before - self.overflow.len();
if dropped > 0 {
tracing::debug!(
"Compacted {} empty overflow texture heaps ({} remaining)",
dropped,
self.overflow.len()
);
}
}
pub fn has_textures(&self) -> bool {
self.texture_count > 0
}
pub fn texture_count(&self) -> u32 {
self.texture_count
}
pub fn overflow_count(&self) -> usize {
self.overflow.len()
}
pub fn use_heaps_for_compute(&self, encoder: &mtl::ComputeCommandEncoderRef) {
if !self.has_textures() {
return;
}
encoder.use_heap(&self.primary);
for heap in &self.overflow {
encoder.use_heap(heap);
}
}
pub fn use_heaps_for_render(&self, encoder: &mtl::RenderCommandEncoderRef, stages: mtl::MTLRenderStages) {
if !self.has_textures() {
return;
}
encoder.use_heap_at(&self.primary, stages);
for heap in &self.overflow {
encoder.use_heap_at(heap, stages);
}
}
fn create_heap(&self, size: u64) -> Heap {
let desc = mtl::HeapDescriptor::new();
desc.set_size(size);
desc.set_storage_mode(mtl::MTLStorageMode::Shared);
desc.set_cpu_cache_mode(mtl::MTLCPUCacheMode::DefaultCache);
desc.set_heap_type(mtl::MTLHeapType::Automatic);
desc.set_hazard_tracking_mode(mtl::MTLHazardTrackingMode::Tracked);
self.device.new_heap(&desc)
}
}
#[allow(dead_code)]
pub(crate) enum PendingDeletion {
Buffer {
buffer: MTLBuffer,
retained_slots: Vec<MetalSlotKey>,
},
Texture {
texture: MTLTexture,
},
Sampler {
sampler: SamplerState,
},
}
pub(crate) struct DeletionQueue {
inner: super::super::shared::DeferredQueue<TimelineValue, PendingDeletion>,
}
impl DeletionQueue {
pub fn new() -> Self {
Self {
inner: super::super::shared::DeferredQueue::new(),
}
}
pub fn queue(&mut self, barrier: TimelineValue, resource: PendingDeletion) {
self.inner.push(barrier, resource);
}
#[allow(dead_code)]
pub fn process_up_to(&mut self, signaled: TimelineValue) {
self.process_up_to_gated(signaled, |_| true);
}
pub fn process_up_to_gated<F>(&mut self, signaled: TimelineValue, can_drop: F)
where
F: Fn(&PendingDeletion) -> bool,
{
drop(self.inner.drain_up_to_filtered(signaled, can_drop));
}
pub fn flush_all(&mut self) {
drop(self.inner.flush_all().collect::<Vec<_>>());
}
pub(crate) fn pending_len(&self) -> usize {
self.inner.len()
}
}
#[derive(Clone)]
pub(crate) struct TimelineWaiter {
inner: Arc<(Mutex<u64>, Condvar)>,
signal_queue: Option<Arc<crate::signal::SignalQueue>>,
last_emitted: Arc<AtomicU64>,
}
impl TimelineWaiter {
pub fn new_with_signals(signal_queue: Arc<crate::signal::SignalQueue>) -> Self {
Self {
inner: Arc::new((Mutex::new(0), Condvar::new())),
signal_queue: Some(signal_queue),
last_emitted: Arc::new(AtomicU64::new(0)),
}
}
pub fn signal(&self, value: u64) {
if let Some(queue) = &self.signal_queue {
let mut last = self.last_emitted.load(Ordering::Acquire);
while last < value {
last += 1;
queue.push_boundary_crossed(last);
self.last_emitted.store(last, Ordering::Release);
}
}
let (lock, cvar) = &*self.inner;
let mut signaled = lock.lock().unwrap();
if value > *signaled {
*signaled = value;
}
cvar.notify_all();
}
pub fn completed_value(&self) -> u64 {
*self.inner.0.lock().unwrap()
}
pub fn wait_until(&self, target: u64, timeout: std::time::Duration) -> bool {
let (lock, cvar) = &*self.inner;
let mut signaled = lock.lock().unwrap();
if *signaled >= target {
return true;
}
let deadline = std::time::Instant::now() + timeout;
loop {
let remaining = deadline.saturating_duration_since(std::time::Instant::now());
if remaining.is_zero() {
return *signaled >= target;
}
let (guard, result) = cvar.wait_timeout(signaled, remaining).unwrap();
signaled = guard;
if *signaled >= target {
return true;
}
if result.timed_out() {
return *signaled >= target;
}
}
}
}
pub(crate) struct MetalSubmissionContext {
pub device: super::DeviceHandle,
pub timeline_event: SharedEvent,
pub timeline_waiter: TimelineWaiter,
pub signal_queue: std::sync::Arc<crate::signal::SignalQueue>,
pub last_submitted_seq: u64,
pub in_flight_command_buffers: VecDeque<(crate::timeline::TimelineValue, mtl::CommandBuffer)>,
pub reclamation_context: Option<(std::thread::ThreadId, u64)>,
pub pending_swapchain_returns: Arc<Mutex<Vec<(super::SurfaceHandle, u32)>>>,
pub last_committed_timeline: Option<crate::timeline::TimelineValue>,
pub staging_belt: super::staging::StagingBelt,
pub texture_staging_pool: super::staging::TextureStagingPool,
pub deletion_queue: DeletionQueue,
pub retained_graphs: std::collections::HashMap<u64, MetalRetainedGraph>,
}
pub(crate) struct MetalRetainedGraph {
pub commands: std::sync::Arc<[super::super::GraphCommand]>,
pub used_slots: Vec<MetalSlotKey>,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
pub(crate) enum MetalSlotKey {
StorageBuffer(u32),
UniformBuffer(u32),
Texture(u32),
StorageImage(u32),
}
impl MetalSlotKey {
pub(crate) fn from_buffer(access: BufferKind, local_index: u32) -> Self {
match access {
BufferKind::Scattered => Self::StorageBuffer(local_index),
BufferKind::Broadcast => Self::UniformBuffer(local_index),
}
}
}
pub(crate) struct PendingSlotReclamation {
pub slot: MetalSlotKey,
pub requirements: Vec<(super::ContextHandle, u64)>,
}
pub(crate) struct LogicalDevice {
pub device: MTLDevice,
pub command_queue: CommandQueue,
pub heap_allocator: Mutex<HeapAllocator>,
pub texture_heap: Mutex<TextureHeapAllocator>,
pub argument_buffer: MTLBuffer,
pub argument_encoder: ArgumentEncoder,
pub texture_encoder: ArgumentEncoder,
pub storage_image_encoder: ArgumentEncoder,
pub sampler_encoder: ArgumentEncoder,
pub frame_table: Mutex<super::frame_table::MetalFrameTable>,
pub descriptors: Arc<Mutex<DescriptorRegistry>>,
pub timeline_next: Arc<AtomicU64>,
pub timeline_scheduled_max: AtomicU64,
pub retired_floor: AtomicU64,
pub deletion_queue: Mutex<DeletionQueue>,
pub queue_lock: Arc<Mutex<()>>,
pub submission_worker: Arc<crate::backend::submission_worker::SubmissionWorker>,
}
impl LogicalDevice {
pub(crate) fn process_deletion_queue_up_to(
&self,
completed: u64,
completed_by_context: Option<&HashMap<super::ContextHandle, u64>>,
) {
{
let registry = self.descriptors.lock().unwrap();
self.deletion_queue
.lock()
.unwrap()
.process_up_to_gated(completed, |deletion| match deletion {
PendingDeletion::Buffer { retained_slots, .. } => registry.retained_pins_clear(retained_slots),
_ => true,
});
}
if let Some(map) = completed_by_context {
self.descriptors.lock().unwrap().drain_ready_slot_reclamations(map);
}
}
}
pub const MAX_RESOURCES_PER_CATEGORY: u32 = 4096;
use super::super::shared::SlotAllocator;
#[derive(Default)]
pub(crate) struct ResourceRegistry {
storage_buffer: SlotAllocator,
uniform_buffer: SlotAllocator,
texture: SlotAllocator,
storage_image: SlotAllocator,
sampler: SlotAllocator,
pending_free_storage_buffer_slots: Vec<(u32, TimelineValue)>,
#[cfg_attr(not(test), allow(dead_code))]
pending_free_uniform_buffer_slots: Vec<(u32, TimelineValue)>,
#[cfg_attr(not(test), allow(dead_code))]
pending_free_texture_slots: Vec<(u32, TimelineValue)>,
#[cfg_attr(not(test), allow(dead_code))]
pending_free_storage_image_slots: Vec<(u32, TimelineValue)>,
pub buffer_indices: HashMap<BufferHandle, (u32, BufferKind)>,
pub texture_indices: HashMap<TextureHandle, u32>,
pub sampler_indices: HashMap<SamplerHandle, u32>,
}
impl ResourceRegistry {
pub fn new() -> Self {
Self::default()
}
pub fn ensure_storage_start(&mut self, min: u32) {
self.storage_buffer.ensure_minimum_next(min);
}
pub fn register_storage_buffer(&mut self, handle: BufferHandle) -> u32 {
assert!(
self.storage_buffer.next_fresh() < MAX_RESOURCES_PER_CATEGORY || self.storage_buffer.free_count() > 0,
"storage-buffer bindless slots exhausted ({MAX_RESOURCES_PER_CATEGORY} max). \
next_index={} free={} pending_free={} live_indices={} \
(Scattered={}, Broadcast={}). \
Likely a per-frame leak in bind_map; check that all transient buffers \
(config_buf, scene_buf, indirect_buf, etc.) are explicitly freed via \
`recording.free_buffer(...)` and that `run_recording` evicts them at \
the end of the frame.",
self.storage_buffer.next_fresh(),
self.storage_buffer.free_count(),
self.pending_free_storage_buffer_slots.len(),
self.buffer_indices.len(),
self.buffer_indices
.values()
.filter(|(_, a)| *a == BufferKind::Scattered)
.count(),
self.buffer_indices
.values()
.filter(|(_, a)| *a == BufferKind::Broadcast)
.count(),
);
let local_index = self.storage_buffer.alloc();
self.buffer_indices.insert(handle, (local_index, BufferKind::Scattered));
local_index
}
pub fn register_uniform_buffer(&mut self, handle: BufferHandle) -> u32 {
assert!(
self.uniform_buffer.next_fresh() < MAX_RESOURCES_PER_CATEGORY || self.uniform_buffer.free_count() > 0,
"uniform-buffer bindless slots exhausted ({MAX_RESOURCES_PER_CATEGORY} max). \
Likely a per-frame leak in bind_map for Broadcast buffers."
);
let local_index = self.uniform_buffer.alloc();
self.buffer_indices.insert(handle, (local_index, BufferKind::Broadcast));
local_index
}
pub fn uniform_global_index(local_index: u32) -> u32 {
local_index + MAX_RESOURCES_PER_CATEGORY
}
pub fn register_texture(&mut self, handle: TextureHandle) -> u32 {
let local_index = self.texture.alloc();
self.texture_indices.insert(handle, local_index);
local_index
}
#[cfg_attr(not(test), allow(dead_code))]
pub fn release_texture_slot(&mut self, local_index: u32, barrier: Option<TimelineValue>, slot_pinned: bool) {
release_slot(
local_index,
barrier,
slot_pinned,
&mut self.pending_free_texture_slots,
&mut self.texture,
);
}
pub fn texture_global_index(local_index: u32) -> u32 {
local_index + 2 * MAX_RESOURCES_PER_CATEGORY
}
pub fn register_storage_image(&mut self, handle: TextureHandle) -> u32 {
let local_index = self.storage_image.alloc();
self.texture_indices.insert(handle, local_index);
local_index
}
pub fn reserve_storage_image_slot(&mut self) -> u32 {
self.storage_image.alloc()
}
pub fn bind_storage_image_slot(&mut self, handle: TextureHandle, local_index: u32) {
self.texture_indices.insert(handle, local_index);
}
pub fn storage_image_global_index(local_index: u32) -> u32 {
local_index + 3 * MAX_RESOURCES_PER_CATEGORY
}
pub fn register_sampler(&mut self, handle: SamplerHandle) -> u32 {
let local_index = self.sampler.alloc();
self.sampler_indices.insert(handle, local_index);
local_index
}
pub fn sampler_global_index(local_index: u32) -> u32 {
local_index + 4 * MAX_RESOURCES_PER_CATEGORY
}
#[cfg_attr(not(test), allow(dead_code))]
pub fn unregister_buffer(&mut self, handle: BufferHandle, barrier: Option<TimelineValue>, slot_pinned: bool) {
if let Some((local_index, access)) = self.buffer_indices.remove(&handle) {
match access {
BufferKind::Scattered => release_slot(
local_index,
barrier,
slot_pinned,
&mut self.pending_free_storage_buffer_slots,
&mut self.storage_buffer,
),
BufferKind::Broadcast => release_slot(
local_index,
barrier,
slot_pinned,
&mut self.pending_free_uniform_buffer_slots,
&mut self.uniform_buffer,
),
}
}
}
pub fn unregister_texture(&mut self, handle: TextureHandle) {
self.texture_indices.remove(&handle);
}
pub fn unregister_sampler(&mut self, handle: SamplerHandle) {
self.sampler_indices.remove(&handle);
}
pub fn buffer_slot_keys(&self, handle: BufferHandle) -> Vec<MetalSlotKey> {
self.buffer_indices
.get(&handle)
.map(|&(local_index, access)| vec![MetalSlotKey::from_buffer(access, local_index)])
.unwrap_or_default()
}
pub fn extract_buffer_slots(&mut self, handle: BufferHandle) -> Vec<MetalSlotKey> {
if let Some((local_index, access)) = self.buffer_indices.remove(&handle) {
vec![MetalSlotKey::from_buffer(access, local_index)]
} else {
Vec::new()
}
}
pub fn free_slot(&mut self, key: MetalSlotKey) {
match key {
MetalSlotKey::StorageBuffer(i) => self.storage_buffer.free(i),
MetalSlotKey::UniformBuffer(i) => self.uniform_buffer.free(i),
MetalSlotKey::Texture(i) => self.texture.free(i),
MetalSlotKey::StorageImage(i) => self.storage_image.free(i),
}
}
#[cfg_attr(not(test), allow(dead_code))]
pub fn drain_pending_slots_up_to<F>(&mut self, signaled: TimelineValue, can_free: F)
where
F: Fn(MetalSlotKey) -> bool,
{
macro_rules! drain_to_allocator {
($pending:expr, $alloc:expr, $key:expr) => {{
let mut i = 0;
while i < $pending.len() {
let (slot, barrier) = $pending[i];
if barrier <= signaled && can_free($key(slot)) {
$pending.swap_remove(i);
$alloc.free(slot);
} else {
i += 1;
}
}
}};
}
drain_to_allocator!(
self.pending_free_storage_buffer_slots,
self.storage_buffer,
MetalSlotKey::StorageBuffer
);
drain_to_allocator!(
self.pending_free_uniform_buffer_slots,
self.uniform_buffer,
MetalSlotKey::UniformBuffer
);
drain_to_allocator!(self.pending_free_texture_slots, self.texture, MetalSlotKey::Texture);
drain_to_allocator!(
self.pending_free_storage_image_slots,
self.storage_image,
MetalSlotKey::StorageImage
);
}
#[cfg(test)]
pub fn drain_pending_slots_up_to_unpinned(&mut self, signaled: TimelineValue) {
self.drain_pending_slots_up_to(signaled, |_| true);
}
#[cfg(test)]
pub fn drain_pending_slots(&mut self) {
self.drain_pending_slots_up_to_unpinned(TimelineValue::MAX);
}
pub fn available_slots(&self, category: crate::types::ResourceCategory) -> u32 {
let allocator = match category {
crate::types::ResourceCategory::Scattered => &self.storage_buffer,
crate::types::ResourceCategory::Broadcast => &self.uniform_buffer,
crate::types::ResourceCategory::Texture => &self.texture,
crate::types::ResourceCategory::StorageImage => &self.storage_image,
crate::types::ResourceCategory::Sampler => &self.sampler,
};
MAX_RESOURCES_PER_CATEGORY.saturating_sub(allocator.live_count())
}
#[cfg(test)]
pub fn pending_buffer_slot_count(&self) -> usize {
self.pending_free_storage_buffer_slots.len() + self.pending_free_uniform_buffer_slots.len()
}
#[cfg(test)]
pub fn free_storage_buffer_count(&self) -> usize {
self.storage_buffer.free_count()
}
#[cfg(test)]
pub fn free_uniform_buffer_count(&self) -> usize {
self.uniform_buffer.free_count()
}
}
#[cfg_attr(not(test), allow(dead_code))]
fn release_slot(
local_index: u32,
barrier: Option<TimelineValue>,
slot_pinned: bool,
pending: &mut Vec<(u32, TimelineValue)>,
alloc: &mut SlotAllocator,
) {
if slot_pinned || barrier.is_some() {
let b = barrier.unwrap_or(0);
pending.push((local_index, b));
} else {
alloc.free(local_index);
}
}
pub(crate) struct DescriptorRegistry {
pub resource_registry: ResourceRegistry,
pub slot_last_seen: HashMap<MetalSlotKey, HashMap<super::ContextHandle, u64>>,
pub pending_slot_reclamations: Vec<PendingSlotReclamation>,
retained_users: HashMap<MetalSlotKey, u32>,
}
impl DescriptorRegistry {
pub(crate) fn new() -> Self {
Self {
resource_registry: ResourceRegistry::new(),
slot_last_seen: HashMap::new(),
pending_slot_reclamations: Vec::new(),
retained_users: HashMap::new(),
}
}
pub(crate) fn pin_retained_slots(&mut self, slots: impl IntoIterator<Item = MetalSlotKey>) {
for slot in slots {
*self.retained_users.entry(slot).or_insert(0) += 1;
}
}
pub(crate) fn unpin_retained_slots(&mut self, slots: impl IntoIterator<Item = MetalSlotKey>) {
for slot in slots {
if let Some(count) = self.retained_users.get_mut(&slot) {
*count = count.saturating_sub(1);
if *count == 0 {
self.retained_users.remove(&slot);
}
}
}
}
#[cfg(test)]
pub(crate) fn retained_user_count(&self, slot: MetalSlotKey) -> u32 {
self.retained_users.get(&slot).copied().unwrap_or(0)
}
fn slot_pinned(&self, slot: MetalSlotKey) -> bool {
self.retained_users.get(&slot).copied().unwrap_or(0) > 0
}
pub(crate) fn retained_pins_clear(&self, slots: &[MetalSlotKey]) -> bool {
slots.iter().all(|slot| !self.slot_pinned(*slot))
}
pub(crate) fn record_slot_usage(
&mut self,
ctx: super::ContextHandle,
seq: u64,
slots: impl IntoIterator<Item = MetalSlotKey>,
) {
for slot in slots {
self.slot_last_seen
.entry(slot)
.or_default()
.entry(ctx)
.and_modify(|v| *v = (*v).max(seq))
.or_insert(seq);
}
}
pub(crate) fn queue_slot_reclamation(&mut self, slot: MetalSlotKey) {
let requirements: Vec<_> = self
.slot_last_seen
.remove(&slot)
.map(|m| m.into_iter().collect())
.unwrap_or_default();
self.pending_slot_reclamations
.push(PendingSlotReclamation { slot, requirements });
}
pub(crate) fn reclaim_buffer_slots(&mut self, handle: BufferHandle) -> Vec<MetalSlotKey> {
let slots = self.resource_registry.extract_buffer_slots(handle);
for slot in slots.iter().copied() {
self.queue_slot_reclamation(slot);
}
slots
}
pub(crate) fn reclaim_texture_slot(&mut self, key: MetalSlotKey) {
self.queue_slot_reclamation(key);
}
pub(crate) fn drain_ready_slot_reclamations(&mut self, completed_values: &HashMap<super::ContextHandle, u64>) {
let mut i = 0;
while i < self.pending_slot_reclamations.len() {
let slot = self.pending_slot_reclamations[i].slot;
let gpu_ready = self.pending_slot_reclamations[i]
.requirements
.iter()
.all(|(ctx_id, required_seq)| completed_values.get(ctx_id).is_none_or(|&v| v >= *required_seq));
let pin_clear = !self.slot_pinned(slot);
if gpu_ready && pin_clear {
let entry = self.pending_slot_reclamations.swap_remove(i);
self.resource_registry.free_slot(entry.slot);
} else {
i += 1;
}
}
}
pub(crate) fn bindless_retirement_requirements_for_buffer(
&self,
handle: BufferHandle,
base: Vec<(super::ContextHandle, u64)>,
) -> Vec<(super::ContextHandle, u64)> {
let slots = self.resource_registry.buffer_slot_keys(handle);
let mut merged: HashMap<super::ContextHandle, u64> = base.into_iter().collect();
for &slot in &slots {
if let Some(map) = self.slot_last_seen.get(&slot) {
for (ctx, seq) in map.iter() {
merged.entry(*ctx).and_modify(|v| *v = (*v).max(*seq)).or_insert(*seq);
}
}
}
merged.into_iter().collect()
}
pub(crate) fn drain_pending_slots(&mut self) {
self.drain_ready_slot_reclamations(&HashMap::new());
#[cfg(test)]
self.resource_registry.drain_pending_slots();
}
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn unregister_buffer(&mut self, handle: BufferHandle) {
let _ = self.reclaim_buffer_slots(handle);
}
pub(crate) fn unregister_texture(&mut self, handle: TextureHandle) {
self.resource_registry.unregister_texture(handle);
}
#[allow(dead_code)]
pub(crate) fn release_texture_slot(&mut self, local_index: u32) {
self.reclaim_texture_slot(MetalSlotKey::Texture(local_index));
}
pub(crate) fn release_storage_image_slot(&mut self, local_index: u32) {
self.reclaim_texture_slot(MetalSlotKey::StorageImage(local_index));
}
pub(crate) fn buffer_retained_slot_keys(&self, handle: BufferHandle) -> Vec<MetalSlotKey> {
self.resource_registry.buffer_slot_keys(handle)
}
}
#[derive(Clone)]
pub(crate) struct BufferState {
pub device_handle: DeviceHandle,
pub buffer: MTLBuffer,
pub size: u64,
pub allocation_size: u64,
pub is_device_allocated: bool,
pub arg_buffer_index: u32,
pub flags: crate::types::BufferFlags,
pub element_stride: Option<u32>,
pub parent_for_view: Option<BufferHandle>,
pub access: BufferKind,
pub view_byte_offset: Option<u64>,
pub is_withdraw_staging: bool,
pub texture_copy_footprint: Option<crate::backend::TextureCopyFootprint>,
}
pub(crate) struct ShaderState {
pub device_handle: DeviceHandle,
pub slang_source: String,
pub search_paths: Vec<String>,
pub defines: Vec<(String, String)>,
pub optimization_level: crate::types::OptimizationLevel,
pub vertex_library: Option<Library>,
pub fragment_library: Option<Library>,
pub compute_library: Option<Library>,
pub reflection: Option<crate::slang::ShaderReflection>,
pub layout_checks: Vec<crate::slang::OwnedLayoutCheck>,
}
pub(crate) struct PipelineState {
pub device_handle: DeviceHandle,
pub pipeline: RenderPipelineState,
pub depth_stencil: Option<MTLDepthStencilState>,
pub primitive_type: MTLPrimitiveType,
pub push_constant_categories: Vec<Option<crate::types::ResourceCategory>>,
pub binding_element_strides: Vec<Option<u32>>,
pub shader_debug_name: String,
}
pub(crate) struct ComputePipelineState {
pub device_handle: DeviceHandle,
pub pipeline: MTLComputePipelineState,
pub workgroup_size: [u32; 3],
pub push_constant_categories: Vec<Option<crate::types::ResourceCategory>>,
pub binding_element_strides: Vec<Option<u32>>,
pub shader_debug_name: String,
}
pub(crate) struct RenderTargetState {
pub device_handle: DeviceHandle,
pub width: u32,
pub height: u32,
pub texture: MTLTexture,
pub depth_texture: Option<MTLTexture>,
}
pub(crate) struct TextureState {
pub device_handle: DeviceHandle,
pub width: u32,
pub height: u32,
pub format: TextureFormat,
pub texture: MTLTexture,
pub arg_buffer_index: u32,
pub sampled_arg_buffer_index: Option<u32>,
pub is_storage_image: bool,
pub slot_owned_externally: bool,
#[allow(dead_code)]
pub is_heap_allocated: bool,
}
pub(crate) struct SamplerState_ {
pub device_handle: DeviceHandle,
#[allow(dead_code)]
pub sampler: SamplerState,
pub arg_buffer_index: u32,
}
pub const MAX_FRAMES_IN_FLIGHT: usize = 3;
pub(crate) struct SurfaceState {
pub device_handle: DeviceHandle,
pub width: u32,
pub height: u32,
pub format: TextureFormat,
pub depth_format: Option<DepthFormat>,
pub depth_texture: Option<MTLTexture>,
pub current_frame: usize,
pub layer: *mut std::ffi::c_void,
pub drawable_slots: [Option<*mut std::ffi::c_void>; MAX_FRAMES_IN_FLIGHT],
pub drawable_texture_handles: [Option<TextureHandle>; MAX_FRAMES_IN_FLIGHT],
pub current_texture_handle: Option<TextureHandle>,
pub bindless_storage_slots: [u32; MAX_FRAMES_IN_FLIGHT],
pub present_mode: crate::types::PresentMode,
pub frame_pending_gpu_commands: Vec<crate::backend::GpuCommand>,
pub pending_acquire_count: u32,
pub last_acquired_image_index: Option<u32>,
}
unsafe impl Send for SurfaceState {}
unsafe impl Sync for SurfaceState {}
pub(crate) struct MetalAdapterInfo {
pub device: mtl::Device,
pub adapter_id: u32,
}
pub(crate) type SharedLogicalDevice = Arc<LogicalDevice>;
pub(crate) type SharedMetalSubmissionContext = Arc<Mutex<MetalSubmissionContext>>;
pub(super) struct MetalState {
pub adapters: Vec<MetalAdapterInfo>,
pub device_lost: std::sync::Arc<std::sync::atomic::AtomicBool>,
pub devices: std::collections::HashMap<DeviceHandle, SharedLogicalDevice>,
pub next_device_handle: DeviceHandle,
pub contexts: std::collections::HashMap<super::ContextHandle, SharedMetalSubmissionContext>,
pub next_context_id: super::ContextHandle,
pub buffers: std::collections::HashMap<BufferHandle, BufferState>,
pub next_buffer_handle: BufferHandle,
pub shaders: std::collections::HashMap<ShaderHandle, ShaderState>,
pub next_shader_handle: ShaderHandle,
pub pipelines: std::collections::HashMap<PipelineHandle, PipelineState>,
pub next_pipeline_handle: PipelineHandle,
pub compute_pipelines: std::collections::HashMap<ComputePipelineHandle, ComputePipelineState>,
pub next_compute_pipeline_handle: ComputePipelineHandle,
pub render_targets: std::collections::HashMap<RenderTargetHandle, RenderTargetState>,
pub next_render_target_handle: RenderTargetHandle,
pub surfaces: std::collections::HashMap<SurfaceHandle, SurfaceState>,
pub next_surface_handle: SurfaceHandle,
pub textures: std::collections::HashMap<TextureHandle, TextureState>,
pub next_texture_handle: TextureHandle,
pub samplers: std::collections::HashMap<SamplerHandle, SamplerState_>,
pub next_sampler_handle: SamplerHandle,
pub slang_compiler: Option<crate::slang::SlangCompiler>,
}
impl MetalState {
#[inline]
pub(super) fn slang_compiler_mut_or_init(&mut self) -> anyhow::Result<&mut crate::slang::SlangCompiler> {
use anyhow::Context;
if self.slang_compiler.is_none() {
self.slang_compiler = Some(crate::slang::SlangCompiler::new().context("Failed to create Slang compiler")?);
}
Ok(self.slang_compiler.as_mut().expect("just set"))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn storage_buffer_slots_are_reused_after_unregister() {
let mut reg = ResourceRegistry::new();
let mut all_indices = Vec::new();
for handle in 0..(MAX_RESOURCES_PER_CATEGORY as u64 * 4) {
let idx = reg.register_storage_buffer(handle);
all_indices.push(idx);
reg.unregister_buffer(handle, None, false);
}
assert!(
all_indices.iter().all(|&i| i < MAX_RESOURCES_PER_CATEGORY),
"storage buffer slots escaped the 0..{} window: {:?}",
MAX_RESOURCES_PER_CATEGORY,
all_indices
.iter()
.filter(|&&i| i >= MAX_RESOURCES_PER_CATEGORY)
.collect::<Vec<_>>()
);
}
#[test]
fn uniform_buffer_slots_are_reused_after_unregister() {
let mut reg = ResourceRegistry::new();
let mut all_indices = Vec::new();
for handle in 0..(MAX_RESOURCES_PER_CATEGORY as u64 * 4) {
let idx = reg.register_uniform_buffer(handle);
all_indices.push(idx);
reg.unregister_buffer(handle, None, false);
}
assert!(
all_indices.iter().all(|&i| i < MAX_RESOURCES_PER_CATEGORY),
"uniform buffer slots escaped the 0..{} window: {:?}",
MAX_RESOURCES_PER_CATEGORY,
all_indices
.iter()
.filter(|&&i| i >= MAX_RESOURCES_PER_CATEGORY)
.collect::<Vec<_>>()
);
}
#[test]
fn unregister_routes_slot_to_correct_category() {
let mut reg = ResourceRegistry::new();
let h_uni: BufferHandle = 10;
let h_sto: BufferHandle = 20;
let _ = reg.register_uniform_buffer(h_uni);
let _ = reg.register_storage_buffer(h_sto);
reg.unregister_buffer(h_uni, None, false);
reg.unregister_buffer(h_sto, None, false);
assert_eq!(
reg.free_uniform_buffer_count(),
1,
"uniform free list should have reclaimed the uniform slot"
);
assert_eq!(
reg.free_storage_buffer_count(),
1,
"storage free list should have reclaimed the storage slot"
);
}
#[test]
fn freed_storage_buffer_slot_is_reused_lifo() {
let mut reg = ResourceRegistry::new();
let h0: BufferHandle = 1;
let h1: BufferHandle = 2;
let i0 = reg.register_storage_buffer(h0);
let i1 = reg.register_storage_buffer(h1);
assert_eq!(i0, 0);
assert_eq!(i1, 1);
reg.unregister_buffer(h0, None, false);
reg.unregister_buffer(h1, None, false);
let h2: BufferHandle = 3;
let i2 = reg.register_storage_buffer(h2);
assert_eq!(i2, 1, "expected LIFO reuse of freed slot");
}
#[test]
fn deferred_buffer_slot_is_not_reused_until_drain() {
let mut reg = ResourceRegistry::new();
let h0: BufferHandle = 1;
let i0 = reg.register_storage_buffer(h0);
assert_eq!(i0, 0);
reg.unregister_buffer(h0, Some(1), false);
assert_eq!(reg.pending_buffer_slot_count(), 1, "expected slot to land in pending");
let h1: BufferHandle = 2;
let i1 = reg.register_storage_buffer(h1);
assert_eq!(i1, 1, "register_storage_buffer must not recycle a still-pending slot");
reg.drain_pending_slots();
assert_eq!(reg.pending_buffer_slot_count(), 0);
reg.unregister_buffer(h1, None, false);
let h2: BufferHandle = 3;
let i2 = reg.register_storage_buffer(h2);
assert_eq!(i2, 1, "LIFO pick from free list after drain");
}
#[test]
fn drain_pending_routes_to_correct_category() {
let mut reg = ResourceRegistry::new();
let h_sto: BufferHandle = 10;
let h_uni: BufferHandle = 20;
let _ = reg.register_storage_buffer(h_sto);
let _ = reg.register_uniform_buffer(h_uni);
reg.unregister_buffer(h_sto, Some(1), false);
reg.unregister_buffer(h_uni, Some(1), false);
reg.drain_pending_slots();
assert_eq!(reg.free_storage_buffer_count(), 1);
assert_eq!(reg.free_uniform_buffer_count(), 1);
assert_eq!(reg.pending_buffer_slot_count(), 0);
}
#[test]
fn deferred_texture_slot_is_not_reused_until_drain() {
let mut reg = ResourceRegistry::new();
let h0: TextureHandle = 100;
let i0 = reg.register_texture(h0);
reg.release_texture_slot(i0, Some(1), false);
let h1: TextureHandle = 101;
let i1 = reg.register_texture(h1);
assert_ne!(i1, i0, "texture slot must not be recycled while still pending");
reg.drain_pending_slots();
reg.release_texture_slot(i1, None, false);
let h2: TextureHandle = 102;
let i2 = reg.register_texture(h2);
assert!(
i2 == i0 || i2 == i1,
"expected texture slot reuse after drain, got {i2}"
);
}
#[test]
fn retained_pin_blocks_drain_until_unpin() {
let mut dr = DescriptorRegistry::new();
let h0: BufferHandle = 1;
let i0 = dr.resource_registry.register_storage_buffer(h0);
let key = MetalSlotKey::StorageBuffer(i0);
dr.pin_retained_slots([key]);
dr.unregister_buffer(h0);
dr.drain_pending_slots();
assert_eq!(dr.resource_registry.free_storage_buffer_count(), 0, "pin blocks drain");
assert_eq!(dr.retained_user_count(key), 1);
dr.unpin_retained_slots([key]);
dr.drain_pending_slots();
assert_eq!(dr.resource_registry.free_storage_buffer_count(), 1);
}
#[test]
fn retained_pin_blocks_immediate_unregister() {
let mut dr = DescriptorRegistry::new();
let h0: BufferHandle = 2;
let i0 = dr.resource_registry.register_storage_buffer(h0);
let key = MetalSlotKey::StorageBuffer(i0);
dr.pin_retained_slots([key]);
dr.unregister_buffer(h0);
assert_eq!(dr.resource_registry.free_storage_buffer_count(), 0);
assert_eq!(dr.pending_slot_reclamations.len(), 1);
dr.unpin_retained_slots([key]);
dr.drain_pending_slots();
assert_eq!(dr.resource_registry.free_storage_buffer_count(), 1);
}
#[test]
fn retained_pin_unpin_then_lifo_reuse() {
let mut dr = DescriptorRegistry::new();
let h0: BufferHandle = 3;
let i0 = dr.resource_registry.register_storage_buffer(h0);
let key = MetalSlotKey::StorageBuffer(i0);
let ctx = 1u64;
dr.record_slot_usage(ctx, 1, [key]);
dr.pin_retained_slots([key]);
dr.unregister_buffer(h0);
dr.unpin_retained_slots([key]);
let mut completed = HashMap::new();
completed.insert(ctx, 1);
dr.drain_ready_slot_reclamations(&completed);
let h1: BufferHandle = 4;
let i1 = dr.resource_registry.register_storage_buffer(h1);
assert_eq!(i1, i0, "LIFO reuse after unpin + drain");
}
#[test]
fn slot_last_seen_gates_reclaim_until_context_retires() {
let mut dr = DescriptorRegistry::new();
let h0: BufferHandle = 5;
let i0 = dr.resource_registry.register_storage_buffer(h0);
let key = MetalSlotKey::StorageBuffer(i0);
let ctx_a = 10u64;
let ctx_b = 11u64;
dr.record_slot_usage(ctx_a, 5, [key]);
dr.record_slot_usage(ctx_b, 7, [key]);
dr.unregister_buffer(h0);
let mut completed = HashMap::new();
completed.insert(ctx_a, 5);
completed.insert(ctx_b, 6);
dr.drain_ready_slot_reclamations(&completed);
assert_eq!(dr.resource_registry.free_storage_buffer_count(), 0);
completed.insert(ctx_b, 7);
dr.drain_ready_slot_reclamations(&completed);
assert_eq!(dr.resource_registry.free_storage_buffer_count(), 1);
assert_eq!(
dr.resource_registry.register_storage_buffer(6),
i0,
"reclaimed slot must be reusable"
);
}
}