use std::alloc::{Layout, alloc, dealloc};
use std::fmt;
use std::mem::ManuallyDrop;
use std::ptr::NonNull;
use std::slice;
use std::sync::{Mutex, MutexGuard, PoisonError};
use thiserror::Error;
use crate::format::{COMMON_FILE, ExpertsLayout, FormatError, Manifest};
use crate::kv::KvCache;
use super::error::IoError;
use super::stream::{bytes_per_slot, slots_for_budget};
pub const SLOT_ALIGN: usize = 4096;
pub const MAX_POOL_BYTES: u64 = 4 << 30;
#[derive(Debug, Error, PartialEq, Eq)]
pub enum SlotError {
#[error("slot pool needs at least one slot per layer")]
ZeroSlotsPerLayer,
#[error("slot pool needs at least one layer")]
NoLayers,
#[error("slot pool layer count {n_layers} exceeds u32")]
TooManyLayers {
n_layers: usize,
},
#[error("layer {layer}: blob stride is zero")]
ZeroStride {
layer: u32,
},
#[error("slot pool needs {requested} bytes, over the {limit}-byte cap")]
PoolTooLarge {
requested: u128,
limit: u128,
},
#[error(
"slot pool needs {requested} bytes, at or over this process's {limit}-byte memory limit"
)]
ExceedsMemoryBudget {
requested: u128,
limit: u64,
},
#[error("slot pool allocation of {bytes} bytes failed")]
AllocFailed {
bytes: usize,
},
#[error("layer {layer} out of range ({n_layers} layers)")]
LayerOutOfRange {
layer: u32,
n_layers: u32,
},
#[error("layer {layer}: all {slots_per_layer} slots are in use")]
Exhausted {
layer: u32,
slots_per_layer: u32,
},
}
#[derive(Debug, Clone, Copy)]
struct LayerGeom {
base_offset: usize,
stride: usize,
pitch: usize,
}
#[derive(Debug)]
struct PoolState {
leased: Box<[bool]>,
free: Box<[Vec<u32>]>,
}
#[derive(Debug)]
pub struct SlotPool {
base: NonNull<u8>,
alloc_layout: Layout,
total_bytes: usize,
slots_per_layer: u32,
layers: Box<[LayerGeom]>,
state: Mutex<PoolState>,
}
unsafe impl Send for SlotPool {}
unsafe impl Sync for SlotPool {}
impl SlotPool {
pub fn new(slots_per_layer: u32, layer_strides: &[u64]) -> Result<Self, SlotError> {
Self::new_within(slots_per_layer, layer_strides, resident_limit())
}
fn new_within(
slots_per_layer: u32,
layer_strides: &[u64],
limit: Option<u64>,
) -> Result<Self, SlotError> {
if slots_per_layer == 0 {
return Err(SlotError::ZeroSlotsPerLayer);
}
if layer_strides.is_empty() {
return Err(SlotError::NoLayers);
}
let n_layers =
u32::try_from(layer_strides.len()).map_err(|_| SlotError::TooManyLayers {
n_layers: layer_strides.len(),
})?;
let cap = u128::from(MAX_POOL_BYTES).min(isize::MAX as u128);
let align = SLOT_ALIGN as u64;
let mut layers = Vec::with_capacity(layer_strides.len());
let mut total: u128 = 0;
for (index, &stride) in layer_strides.iter().enumerate() {
let layer = index as u32;
if stride == 0 {
return Err(SlotError::ZeroStride { layer });
}
let pitch = stride
.checked_next_multiple_of(align)
.ok_or(SlotError::PoolTooLarge {
requested: u128::from(stride),
limit: cap,
})?;
let base_offset = total;
total += u128::from(pitch) * u128::from(slots_per_layer);
if total > cap {
let mut needed = total;
for &rest in &layer_strides[layers.len() + 1..] {
let pitch = rest.checked_next_multiple_of(align).unwrap_or(rest);
needed = needed.saturating_add(
u128::from(pitch).saturating_mul(u128::from(slots_per_layer)),
);
}
return Err(SlotError::PoolTooLarge {
requested: needed,
limit: cap,
});
}
layers.push(LayerGeom {
base_offset: base_offset as usize,
stride: stride as usize,
pitch: pitch as usize,
});
}
let total_bytes = total as usize;
check_resident_budget(total, limit)?;
let alloc_layout = Layout::from_size_align(total_bytes, SLOT_ALIGN).map_err(|_| {
SlotError::PoolTooLarge {
requested: total,
limit: cap,
}
})?;
let raw = unsafe { alloc(alloc_layout) };
let base = NonNull::new(raw).ok_or(SlotError::AllocFailed { bytes: total_bytes })?;
unsafe { prefault(base, total_bytes) };
let n_slots = layers.len() * slots_per_layer as usize;
let free = (0..layers.len())
.map(|_| {
let mut stack = Vec::with_capacity(slots_per_layer as usize);
stack.extend((0..slots_per_layer).rev());
stack
})
.collect::<Vec<_>>()
.into_boxed_slice();
tracing::debug!(
n_layers,
slots_per_layer,
total_bytes,
align = SLOT_ALIGN,
"expert slot pool allocated and pre-faulted"
);
Ok(Self {
base,
alloc_layout,
total_bytes,
slots_per_layer,
layers: layers.into_boxed_slice(),
state: Mutex::new(PoolState {
leased: vec![false; n_slots].into_boxed_slice(),
free,
}),
})
}
pub fn slots_per_layer(&self) -> u32 {
self.slots_per_layer
}
pub fn n_layers(&self) -> u32 {
self.layers.len() as u32
}
#[track_caller]
pub fn stride(&self, layer: u32) -> usize {
self.layers[layer as usize].stride
}
#[track_caller]
pub fn pitch(&self, layer: u32) -> usize {
self.layers[layer as usize].pitch
}
pub fn max_stride(&self) -> usize {
self.layers.iter().map(|l| l.stride).max().unwrap_or(0)
}
pub fn total_bytes(&self) -> usize {
self.total_bytes
}
pub(crate) fn slot_offset(&self, layer: u32, slot: u32) -> Option<usize> {
let geom = self.layers.get(layer as usize)?;
if slot >= self.slots_per_layer {
return None;
}
Some(geom.base_offset + geom.pitch * slot as usize)
}
pub(crate) fn padded_layer(&self) -> Option<(u32, usize, usize)> {
self.layers
.iter()
.enumerate()
.find(|(_, geom)| geom.pitch != geom.stride)
.map(|(index, geom)| (index as u32, geom.stride, geom.pitch))
}
pub(crate) unsafe fn slab_base(&self) -> NonNull<u8> {
self.base
}
pub fn free_slots(&self, layer: u32) -> Option<u32> {
if layer >= self.n_layers() {
return None;
}
let state = self.lock();
Some(state.free[layer as usize].len() as u32)
}
pub fn acquire(&self, layer: u32) -> Result<SlotGuard<'_>, SlotError> {
let geom = *self
.layers
.get(layer as usize)
.ok_or(SlotError::LayerOutOfRange {
layer,
n_layers: self.n_layers(),
})?;
let slot = {
let mut state = self.lock();
let slot = state.free[layer as usize]
.pop()
.ok_or(SlotError::Exhausted {
layer,
slots_per_layer: self.slots_per_layer,
})?;
let flat = self.flat_index(layer, slot);
debug_assert!(!state.leased[flat], "free stack yielded a leased slot");
state.leased[flat] = true;
slot
};
let offset = geom.base_offset + geom.pitch * slot as usize;
debug_assert!(offset + geom.stride <= self.total_bytes);
let ptr = unsafe { NonNull::new_unchecked(self.base.as_ptr().add(offset)) };
Ok(SlotGuard {
pool: self,
layer,
slot,
ptr,
len: geom.stride,
})
}
fn release(&self, layer: u32, slot: u32) {
let flat = self.flat_index(layer, slot);
let mut state = self.lock();
if !state.leased[flat] {
tracing::error!(layer, slot, "slot released without an active lease");
return;
}
state.leased[flat] = false;
let stack = &mut state.free[layer as usize];
debug_assert!(
stack.len() < stack.capacity(),
"free stack would reallocate"
);
stack.push(slot);
}
fn flat_index(&self, layer: u32, slot: u32) -> usize {
layer as usize * self.slots_per_layer as usize + slot as usize
}
fn any_slot_leased(&self) -> bool {
self.lock().leased.iter().any(|&leased| leased)
}
fn lock(&self) -> MutexGuard<'_, PoolState> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}
}
impl Drop for SlotPool {
fn drop(&mut self) {
if self.any_slot_leased() {
tracing::error!(
total_bytes = self.total_bytes,
"expert slot pool dropped with slots still leased; leaking the \
slab so an unreaped O_DIRECT read cannot land in freed memory"
);
return;
}
unsafe { dealloc(self.base.as_ptr(), self.alloc_layout) };
}
}
#[derive(Debug)]
pub struct SlotGuard<'pool> {
pool: &'pool SlotPool,
layer: u32,
slot: u32,
ptr: NonNull<u8>,
len: usize,
}
unsafe impl Send for SlotGuard<'_> {}
unsafe impl Sync for SlotGuard<'_> {}
impl SlotGuard<'_> {
pub fn layer(&self) -> u32 {
self.layer
}
pub fn index(&self) -> u32 {
self.slot
}
pub fn len(&self) -> usize {
self.len
}
pub fn is_empty(&self) -> bool {
self.len == 0
}
pub fn as_slice(&self) -> &[u8] {
unsafe { slice::from_raw_parts(self.ptr.as_ptr(), self.len) }
}
pub fn as_mut_slice(&mut self) -> &mut [u8] {
unsafe { slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) }
}
pub unsafe fn as_mut_ptr(&mut self) -> *mut u8 {
self.ptr.as_ptr()
}
pub fn leak(self) {
let guard = ManuallyDrop::new(self);
tracing::warn!(
layer = guard.layer,
slot = guard.slot,
"expert slot leaked to protect an unreaped read; it will not be reused"
);
}
pub fn as_ptr(&self) -> *const u8 {
self.ptr.as_ptr().cast_const()
}
}
impl Drop for SlotGuard<'_> {
fn drop(&mut self) {
self.pool.release(self.layer, self.slot);
}
}
fn check_resident_budget(requested: u128, limit: Option<u64>) -> Result<(), SlotError> {
let Some(limit) = limit else {
return Ok(());
};
if requested >= u128::from(limit) {
return Err(SlotError::ExceedsMemoryBudget { requested, limit });
}
Ok(())
}
const MIB: u128 = 1 << 20;
pub const RUNTIME_ANON_BYTES: u64 = (1151 * 1024 * 1024) / 10;
#[derive(Debug, Error)]
pub enum FootprintError {
#[error(transparent)]
Kv(#[from] crate::kv::KvError),
#[error(transparent)]
Io(#[from] IoError),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Footprint {
pub common: u64,
pub experts: u64,
pub kv: u64,
pub runtime: u64,
pub context_cap: usize,
}
impl Footprint {
pub fn project(
manifest: &Manifest,
layout: &ExpertsLayout,
cache_bytes: u64,
context_cap: usize,
) -> Result<Self, FootprintError> {
let common = manifest
.files
.get(COMMON_FILE)
.ok_or_else(|| IoError::from(FormatError::MissingFileEntry(COMMON_FILE.to_owned())))?
.size;
let strides: Vec<u64> = layout.layers.iter().map(|layer| layer.stride).collect();
let n_experts: Vec<u32> = layout.layers.iter().map(|layer| layer.n_experts).collect();
let per_slot = bytes_per_slot(&strides)?;
let slots = slots_for_budget(cache_bytes, &strides, &n_experts, manifest.arch.top_k)?;
let experts =
u64::try_from(per_slot * u128::from(slots)).map_err(|_| IoError::TooLarge {
what: "expert slot pool bytes",
value: cache_bytes,
})?;
let arch = &manifest.arch;
let kv = KvCache::projected_bytes(
arch.n_layers as usize,
arch.n_kv_heads as usize,
arch.head_dim as usize,
context_cap,
)?;
Ok(Self {
common,
experts,
kv,
runtime: RUNTIME_ANON_BYTES,
context_cap,
})
}
pub fn total(&self) -> u128 {
u128::from(self.common)
+ u128::from(self.experts)
+ u128::from(self.kv)
+ u128::from(self.runtime)
}
pub fn fits_within(&self, limit: u64) -> bool {
self.total() <= u128::from(limit)
}
}
impl fmt::Display for Footprint {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"about {} MiB (common {} + experts {} + KV {} at {} tokens + runtime {})",
mib(self.total()),
mib(u128::from(self.common)),
mib(u128::from(self.experts)),
mib(u128::from(self.kv)),
self.context_cap,
mib(u128::from(self.runtime)),
)
}
}
pub(crate) fn mib(bytes: u128) -> String {
let whole = (bytes + MIB / 2) / MIB;
let digits = whole.to_string();
let mut out = String::with_capacity(digits.len() + digits.len() / 3);
for (index, digit) in digits.char_indices() {
if index > 0 && (digits.len() - index).is_multiple_of(3) {
out.push(',');
}
out.push(digit);
}
out
}
pub fn resident_ceiling() -> Option<u64> {
resident_limit()
}
#[cfg(target_os = "linux")]
fn resident_limit() -> Option<u64> {
let mut limit = std::fs::read_to_string("/proc/meminfo")
.ok()
.and_then(|meminfo| parse_mem_total(&meminfo));
if let Ok(contents) = std::fs::read_to_string("/proc/self/cgroup")
&& let Some(path) = parse_cgroup_v2_path(&contents)
{
for dir in cgroup_ancestors(path) {
let max = std::fs::read_to_string(format!("/sys/fs/cgroup{dir}/memory.max"))
.ok()
.and_then(|contents| parse_memory_max(&contents));
if let Some(max) = max {
limit = Some(limit.map_or(max, |current| current.min(max)));
}
}
}
limit
}
#[cfg(not(target_os = "linux"))]
fn resident_limit() -> Option<u64> {
None
}
#[cfg(target_os = "linux")]
fn parse_cgroup_v2_path(contents: &str) -> Option<&str> {
contents
.lines()
.find_map(|line| line.strip_prefix("0::"))
.map(str::trim_end)
}
#[cfg(target_os = "linux")]
fn cgroup_ancestors(path: &str) -> Vec<&str> {
let mut out = vec![path];
let mut rest = path;
while let Some(cut) = rest.rfind('/') {
rest = &rest[..cut];
out.push(rest);
}
out
}
#[cfg(target_os = "linux")]
fn parse_memory_max(contents: &str) -> Option<u64> {
let value = contents.trim();
if value == "max" {
return None;
}
value.parse().ok()
}
#[cfg(target_os = "linux")]
fn parse_mem_total(contents: &str) -> Option<u64> {
let line = contents
.lines()
.find_map(|line| line.strip_prefix("MemTotal:"))?;
let mut fields = line.split_whitespace();
let value: u64 = fields.next()?.parse().ok()?;
match fields.next() {
Some("kB") => value.checked_mul(1024),
None => Some(value),
_ => None,
}
}
unsafe fn prefault(base: NonNull<u8>, len: usize) {
let ptr = base.as_ptr();
unsafe { ptr.write_bytes(0, len) };
let mut offset = 0;
while offset < len {
unsafe { ptr.add(offset).write_volatile(0) };
offset += SLOT_ALIGN;
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashSet;
use std::sync::Arc;
const REAL_STRIDES: [u64; 2] = [3_059_712, 2_654_208];
fn small_pool(slots_per_layer: u32) -> SlotPool {
SlotPool::new(slots_per_layer, &[8192, 4096, 8192]).unwrap()
}
fn acquire_all(pool: &SlotPool) -> Vec<SlotGuard<'_>> {
let mut held = Vec::new();
for layer in 0..pool.n_layers() {
for _ in 0..pool.slots_per_layer() {
held.push(pool.acquire(layer).unwrap());
}
}
held
}
#[test]
fn pool_is_send_and_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<SlotPool>();
assert_send_sync::<SlotGuard<'_>>();
assert_send_sync::<SlotError>();
}
#[test]
fn every_slot_base_is_page_aligned() {
let pool = SlotPool::new(3, &[8192, 4096, 100, 3_059_712]).unwrap();
let held = acquire_all(&pool);
assert_eq!(held.len(), 12);
for guard in &held {
assert_eq!(
guard.as_ptr() as usize % SLOT_ALIGN,
0,
"layer {} slot {} base {:p} is not {SLOT_ALIGN}-aligned",
guard.layer(),
guard.index(),
guard.as_ptr()
);
}
}
#[test]
fn a_gapless_pool_reports_no_padding_and_offsets_that_match_its_guards() {
for strides in [&[8192u64, 4096][..], &REAL_STRIDES[..]] {
let pool = SlotPool::new(3, strides).unwrap();
assert_eq!(
pool.padded_layer(),
None,
"{strides:?} is 4096-aligned throughout"
);
let base = unsafe { pool.slab_base() }.as_ptr() as usize;
assert_eq!(base % SLOT_ALIGN, 0);
let mut expected = 0usize;
for (layer, &stride) in strides.iter().enumerate() {
for slot in 0..3 {
assert_eq!(
pool.slot_offset(layer as u32, slot),
Some(expected),
"layer {layer} slot {slot}"
);
let guard = pool.acquire(layer as u32).unwrap();
assert_eq!(
guard.as_ptr() as usize - base,
pool.slot_offset(layer as u32, guard.index()).unwrap()
);
expected += stride as usize;
}
}
assert_eq!(expected, pool.total_bytes());
assert_eq!(pool.slot_offset(strides.len() as u32, 0), None);
assert_eq!(pool.slot_offset(0, 3), None);
}
}
#[test]
fn a_padded_layer_is_named_with_its_stride_and_pitch() {
let pool = SlotPool::new(2, &[4096, 5000, 8192]).unwrap();
assert_eq!(pool.padded_layer(), Some((1, 5000, 8192)));
assert_eq!(pool.slot_offset(1, 0), Some(8192));
assert_eq!(pool.slot_offset(1, 1), Some(8192 + 8192));
assert_ne!(pool.slot_offset(1, 1), Some(8192 + 5000));
}
#[test]
fn strides_and_sizing_match_the_real_layout() {
let strides: Vec<u64> = (0..48).map(|i| REAL_STRIDES[i % 2]).collect();
for &stride in &REAL_STRIDES {
assert_eq!(stride % SLOT_ALIGN as u64, 0, "real stride is page-sized");
}
let pool = SlotPool::new(1, &strides).unwrap();
assert_eq!(pool.n_layers(), 48);
assert_eq!(pool.slots_per_layer(), 1);
assert_eq!(pool.max_stride(), 3_059_712);
for layer in 0..pool.n_layers() {
let expected = REAL_STRIDES[layer as usize % 2] as usize;
assert_eq!(pool.stride(layer), expected, "layer {layer} stride");
assert_eq!(pool.pitch(layer), expected, "layer {layer} pitch");
assert_eq!(pool.acquire(layer).unwrap().len(), expected);
}
let expected_total: usize = strides.iter().map(|&s| s as usize).sum();
assert_eq!(pool.total_bytes(), expected_total);
let per_slot_per_model = 24 * (3_059_712usize + 2_654_208);
assert_eq!(per_slot_per_model, 137_134_080);
assert_eq!(11 * per_slot_per_model / (1024 * 1024), 1438);
assert_eq!(12 * per_slot_per_model / (1024 * 1024), 1569);
assert!(24 * per_slot_per_model < MAX_POOL_BYTES as usize);
}
#[test]
fn unaligned_stride_is_padded_up_to_a_page_pitch() {
let pool = SlotPool::new(2, &[100, 4097]).unwrap();
assert_eq!(pool.stride(0), 100);
assert_eq!(pool.pitch(0), 4096);
assert_eq!(pool.stride(1), 4097);
assert_eq!(pool.pitch(1), 8192);
assert_eq!(pool.total_bytes(), 2 * 4096 + 2 * 8192);
let a = pool.acquire(1).unwrap();
let b = pool.acquire(1).unwrap();
assert_eq!(a.len(), 4097);
let gap = (b.as_ptr() as usize).abs_diff(a.as_ptr() as usize);
assert_eq!(gap, 8192, "consecutive slots are a pitch apart");
assert_eq!(b.as_ptr() as usize % SLOT_ALIGN, 0);
}
#[test]
fn slots_are_zeroed_and_writable_over_their_whole_length() {
let pool = SlotPool::new(1, &[3_059_712]).unwrap();
let mut guard = pool.acquire(0).unwrap();
assert!(
guard.as_slice().iter().all(|&b| b == 0),
"slab is zero-initialized at construction"
);
let len = guard.len();
let bytes = guard.as_mut_slice();
let mut offset = 0;
while offset < len {
bytes[offset] = 0xa5;
let end = (offset + SLOT_ALIGN - 1).min(len - 1);
bytes[end] = 0x5a;
offset += SLOT_ALIGN;
}
let bytes = guard.as_slice();
let mut offset = 0;
while offset < len {
assert_eq!(bytes[offset], 0xa5, "page at {offset} lost its write");
offset += SLOT_ALIGN;
}
}
#[cfg(target_os = "linux")]
fn resident_pages(base: *const u8, len: usize) -> Option<usize> {
use std::io::{Read, Seek, SeekFrom};
let pages = len / SLOT_ALIGN;
let mut file = std::fs::File::open("/proc/self/pagemap").ok()?;
let index = base as u64 / SLOT_ALIGN as u64;
file.seek(SeekFrom::Start(index * 8)).ok()?;
let mut buf = vec![0u8; pages * 8];
file.read_exact(&mut buf).ok()?;
Some(
buf.chunks_exact(8)
.filter(|e| u64::from_le_bytes((*e).try_into().unwrap()) >> 63 & 1 == 1)
.count(),
)
}
#[test]
#[cfg(target_os = "linux")]
fn construction_faults_in_every_page() {
const SLAB: usize = 512 * 1024;
let control = memmap2::MmapOptions::new().len(SLAB).map_anon().unwrap();
let control_resident = resident_pages(control.as_ptr(), SLAB);
drop(control);
let pool = SlotPool::new(8, &[16384; 4]).unwrap();
assert_eq!(pool.total_bytes(), SLAB);
let first = pool.acquire(0).unwrap();
let pool_resident = resident_pages(first.as_ptr(), SLAB);
drop(first);
let (Some(control_resident), Some(pool_resident)) = (control_resident, pool_resident)
else {
eprintln!("skipping residency assertions: /proc/self/pagemap unreadable");
return;
};
let pages = SLAB / SLOT_ALIGN;
assert_eq!(
pool_resident, pages,
"every page of the slab must be resident after new(); \
an untouched page makes btrfs silently fall back to buffered I/O"
);
assert!(
control_resident * 2 < pages,
"residency check is not meaningful: an untouched {SLAB}-byte block \
already reported {control_resident}/{pages} pages resident"
);
}
#[test]
fn simultaneously_held_slots_never_overlap() {
let pool = SlotPool::new(4, &[8192, 4096, 12288]).unwrap();
let held = acquire_all(&pool);
assert_eq!(held.len(), 12);
let ranges: Vec<(usize, usize)> = held
.iter()
.map(|g| {
let start = g.as_ptr() as usize;
(start, start + g.len())
})
.collect();
for (i, a) in ranges.iter().enumerate() {
for (j, b) in ranges.iter().enumerate().skip(i + 1) {
assert!(
a.1 <= b.0 || b.1 <= a.0,
"slots {i} {a:?} and {j} {b:?} overlap"
);
}
}
let base = ranges.iter().map(|r| r.0).min().unwrap();
let end = ranges.iter().map(|r| r.1).max().unwrap();
assert!(end - base <= pool.total_bytes());
}
#[test]
fn writes_through_one_slot_do_not_disturb_another() {
let pool = small_pool(2);
let mut held = acquire_all(&pool);
for (i, guard) in held.iter_mut().enumerate() {
guard.as_mut_slice().fill(i as u8 + 1);
}
for (i, guard) in held.iter().enumerate() {
assert!(
guard.as_slice().iter().all(|&b| b == i as u8 + 1),
"slot {i} was clobbered by a neighbour"
);
}
}
#[test]
fn exhaustion_is_a_typed_error_and_a_release_makes_a_slot_available() {
let pool = small_pool(2);
let a = pool.acquire(1).unwrap();
let b = pool.acquire(1).unwrap();
assert_ne!(a.index(), b.index(), "the same slot was leased twice");
assert_eq!(pool.free_slots(1), Some(0));
assert_eq!(
pool.acquire(1).unwrap_err(),
SlotError::Exhausted {
layer: 1,
slots_per_layer: 2
}
);
assert_eq!(pool.free_slots(0), Some(2));
pool.acquire(0).unwrap();
let freed = a.index();
drop(a);
assert_eq!(pool.free_slots(1), Some(1));
let c = pool.acquire(1).unwrap();
assert_eq!(c.index(), freed, "the released slot came back");
drop((b, c));
assert_eq!(pool.free_slots(1), Some(2));
}
#[test]
fn acquire_rejects_an_out_of_range_layer() {
let pool = small_pool(1);
assert_eq!(
pool.acquire(3).unwrap_err(),
SlotError::LayerOutOfRange {
layer: 3,
n_layers: 3
}
);
assert_eq!(pool.acquire(u32::MAX).unwrap_err(), {
SlotError::LayerOutOfRange {
layer: u32::MAX,
n_layers: 3,
}
});
assert_eq!(pool.free_slots(3), None);
}
#[test]
fn rejects_degenerate_geometry() {
assert_eq!(
SlotPool::new(0, &[4096]).unwrap_err(),
SlotError::ZeroSlotsPerLayer
);
assert_eq!(SlotPool::new(4, &[]).unwrap_err(), SlotError::NoLayers);
assert_eq!(
SlotPool::new(4, &[4096, 0, 4096]).unwrap_err(),
SlotError::ZeroStride { layer: 1 }
);
}
#[test]
fn rejects_oversized_and_overflowing_geometry() {
let err = SlotPool::new(12, &[u64::from(u32::MAX); 48]).unwrap_err();
assert!(
matches!(err, SlotError::PoolTooLarge { .. }),
"expected PoolTooLarge, got {err:?}"
);
let err = SlotPool::new(u32::MAX, &[u64::MAX / 2; 4]).unwrap_err();
assert!(
matches!(err, SlotError::PoolTooLarge { .. }),
"expected PoolTooLarge, got {err:?}"
);
let err = SlotPool::new(1, &[u64::MAX]).unwrap_err();
assert!(
matches!(err, SlotError::PoolTooLarge { .. }),
"expected PoolTooLarge, got {err:?}"
);
let err = SlotPool::new(1, &[MAX_POOL_BYTES + SLOT_ALIGN as u64]).unwrap_err();
assert!(
matches!(err, SlotError::PoolTooLarge { requested, limit }
if requested > limit),
"expected PoolTooLarge, got {err:?}"
);
}
#[test]
fn steady_state_acquire_and_release_do_not_allocate() {
let pool = small_pool(3);
let capacities: Vec<usize> = {
let state = pool.lock();
state.free.iter().map(Vec::capacity).collect()
};
for _ in 0..64 {
let held = acquire_all(&pool);
drop(held);
}
let after: Vec<usize> = {
let state = pool.lock();
state.free.iter().map(Vec::capacity).collect()
};
assert_eq!(
capacities, after,
"free stacks reallocated during steady state"
);
assert!(capacities.iter().all(|&c| c == 3));
}
#[test]
fn concurrent_acquire_never_hands_out_a_live_slot_twice() {
let pool = Arc::new(SlotPool::new(2, &[8192, 4096, 8192]).unwrap());
let live: Arc<Mutex<HashSet<usize>>> = Arc::new(Mutex::new(HashSet::new()));
let mut handles = Vec::new();
for t in 0..8u32 {
let pool = Arc::clone(&pool);
let live = Arc::clone(&live);
handles.push(std::thread::spawn(move || {
for i in 0..500u32 {
let layer = (t + i) % pool.n_layers();
let Ok(mut guard) = pool.acquire(layer) else {
continue; };
let addr = guard.as_ptr() as usize;
assert!(
live.lock().unwrap().insert(addr),
"address {addr:#x} was leased twice concurrently"
);
guard.as_mut_slice()[0] = t as u8;
assert!(live.lock().unwrap().remove(&addr));
}
}));
}
for handle in handles {
handle.join().unwrap();
}
assert!(live.lock().unwrap().is_empty());
for layer in 0..pool.n_layers() {
assert_eq!(
pool.free_slots(layer),
Some(2),
"layer {layer} leaked slots"
);
}
}
#[test]
fn a_dropped_guard_recycles_the_same_address() {
let pool = SlotPool::new(1, &[8192]).unwrap();
let first = pool.acquire(0).unwrap();
let addr = first.as_ptr() as usize;
drop(first);
let second = pool.acquire(0).unwrap();
assert_eq!(
second.as_ptr() as usize,
addr,
"a released slot comes back at the same address, so a guard \
dropped under a live read would alias it"
);
}
#[test]
fn a_leaked_guard_never_returns_to_the_free_stack() {
let pool = small_pool(2);
let leaked = pool.acquire(0).unwrap();
let addr = leaked.as_ptr() as usize;
leaked.leak();
assert_eq!(pool.free_slots(0), Some(1), "the slot stays leased");
let other = pool.acquire(0).unwrap();
assert_ne!(
other.as_ptr() as usize,
addr,
"the leaked slot was handed out again"
);
assert_eq!(
pool.acquire(0).unwrap_err(),
SlotError::Exhausted {
layer: 0,
slots_per_layer: 2
}
);
drop(other);
assert!(
pool.any_slot_leased(),
"the leaked slot is what makes Drop keep the slab"
);
drop(pool);
}
#[test]
fn a_pool_with_a_leaked_slot_does_not_free_its_slab() {
let pool = small_pool(2);
assert!(
!pool.any_slot_leased(),
"a fresh pool owes nothing to anyone"
);
let ordinary = pool.acquire(0).unwrap();
assert!(pool.any_slot_leased(), "a live guard holds its lease");
drop(ordinary);
assert!(
!pool.any_slot_leased(),
"a dropped guard hands its lease back"
);
pool.acquire(0).unwrap().leak();
assert!(pool.any_slot_leased());
let mut rest = vec![pool.acquire(0).unwrap()];
for layer in 1..pool.n_layers() {
for _ in 0..pool.slots_per_layer() {
rest.push(pool.acquire(layer).unwrap());
}
}
drop(rest);
assert!(
pool.any_slot_leased(),
"the leaked slot must still pin the slab after every other guard \
has been returned"
);
drop(pool);
}
#[test]
fn the_raw_write_destination_is_the_slot_itself() {
let pool = SlotPool::new(1, &[8192]).unwrap();
let mut guard = pool.acquire(0).unwrap();
let ptr = unsafe { guard.as_mut_ptr() };
assert_eq!(ptr as usize % SLOT_ALIGN, 0);
unsafe {
ptr.write(0xa5);
ptr.add(8191).write(0x5a);
}
assert_eq!(guard.as_slice()[0], 0xa5);
assert_eq!(guard.as_slice()[8191], 0x5a);
}
#[test]
fn a_slab_over_the_resident_budget_is_a_typed_error() {
assert_eq!(check_resident_budget(1 << 20, None), Ok(()));
assert_eq!(check_resident_budget(1 << 20, Some(2 << 20)), Ok(()));
assert_eq!(
check_resident_budget(u128::from(3u64 << 30), Some(3 << 30)),
Err(SlotError::ExceedsMemoryBudget {
requested: u128::from(3u64 << 30),
limit: 3 << 30,
})
);
assert!(check_resident_budget(u128::from(MAX_POOL_BYTES), Some(3 << 30)).is_err());
let strides = vec![6 << 20; 48];
let err = SlotPool::new_within(12, &strides, Some(3 << 30)).unwrap_err();
assert_eq!(
err,
SlotError::ExceedsMemoryBudget {
requested: 48 * 12 * (6 << 20),
limit: 3 << 30,
}
);
const { assert!(48 * 12 * (6u64 << 20) < MAX_POOL_BYTES) };
SlotPool::new_within(1, &[4096], Some(3 << 30)).expect("4 KiB fits in 3 GiB");
SlotPool::new_within(1, &[4096], None).expect("an unknown limit is not a refusal");
}
#[test]
fn the_benchmark_cgroup_binds_before_the_policy_cap() {
const CGROUP_3G: u64 = 3 << 30;
assert_eq!(CGROUP_3G, 3_221_225_472);
let per_slot: u128 = 24 * (3_059_712 + 2_654_208);
assert_eq!(per_slot, 137_134_080);
assert_eq!(24 * per_slot, 3_291_217_920);
assert!(24 * per_slot < u128::from(MAX_POOL_BYTES));
assert_eq!(
check_resident_budget(24 * per_slot, Some(CGROUP_3G)),
Err(SlotError::ExceedsMemoryBudget {
requested: 24 * per_slot,
limit: CGROUP_3G,
}),
"24 slots/layer does not fit a memory.max=3G cgroup"
);
assert_eq!(23 * per_slot, 3_154_083_840);
assert_eq!(
check_resident_budget(23 * per_slot, Some(CGROUP_3G)),
Ok(())
);
assert!(23 * per_slot * 100 / u128::from(CGROUP_3G) >= 97);
}
const CGROUP_3G: u64 = 3 << 30;
const CGROUP_4G: u64 = 4 << 30;
const V0_COMMON_BYTES: u64 = 1_073_051_648;
const V0_PER_SLOT: u64 = 24 * (REAL_STRIDES[0] + REAL_STRIDES[1]);
fn v0_install() -> (Manifest, ExpertsLayout) {
use crate::format::{FileEntry, LayerLayout, QuantInfo, RVMP_VERSION, SourceInfo};
use std::collections::BTreeMap;
let mut arch = crate::io::testutil::fixture_arch();
arch.n_layers = 48;
arch.n_experts = 128;
arch.top_k = 8;
arch.n_heads = 32;
arch.n_kv_heads = 4;
arch.head_dim = 128;
let manifest = Manifest {
rvmp_version: RVMP_VERSION,
model_id: "qwen3-30b-a3b-instruct-2507".to_owned(),
source: SourceInfo {
hf_repo: "test/qwen3".to_owned(),
revision: "0".repeat(40),
file: "qwen3-Q4_K_M.gguf".to_owned(),
sha256: "0".repeat(64),
},
arch,
quant: QuantInfo {
scheme: "gguf".to_owned(),
tensor_types: BTreeMap::new(),
},
common_tensors: BTreeMap::new(),
files: BTreeMap::from([(
COMMON_FILE.to_owned(),
FileEntry {
size: V0_COMMON_BYTES,
sha256: "0".repeat(64),
},
)]),
};
let layout = ExpertsLayout {
layers: (0..48)
.map(|layer| LayerLayout {
file: crate::format::layer_file_name(layer),
stride: REAL_STRIDES[usize::from(layer >= 24)],
n_experts: 128,
projections: Vec::new(),
})
.collect(),
};
(manifest, layout)
}
#[test]
fn the_shipped_default_fits_the_published_four_gigabyte_budget() {
let (manifest, layout) = v0_install();
let projected =
Footprint::project(&manifest, &layout, crate::model::DEFAULT_CACHE_BYTES, 4096)
.expect("the shipped dials project");
assert_eq!(projected.common, V0_COMMON_BYTES, "common core row");
assert_eq!(V0_PER_SLOT, 137_134_080);
assert_eq!(projected.experts, 19 * V0_PER_SLOT, "expert slot pool row");
assert_eq!(projected.experts, 2_605_547_520);
assert_eq!(projected.kv, 384 << 20, "KV cache row at 4K");
assert_eq!(projected.runtime, RUNTIME_ANON_BYTES, "anonymous row");
assert_eq!(projected.context_cap, 4096);
assert_eq!(mib(projected.total()), "4,007");
assert!(
projected.fits_within(CGROUP_4G),
"the shipped default must be accepted: {projected}"
);
let spare = u128::from(CGROUP_4G) - projected.total();
assert_eq!(mib(spare), "89");
let twenty = Footprint::project(&manifest, &layout, 20 * V0_PER_SLOT, 4096).unwrap();
assert!(!twenty.fits_within(CGROUP_4G), "{twenty}");
let three = Footprint::project(&manifest, &layout, 1440 << 20, 4096).unwrap();
assert_eq!(three.experts, 11 * V0_PER_SLOT);
assert_eq!(mib(three.total()), "2,961");
assert!(three.fits_within(CGROUP_3G), "{three}");
assert_eq!(mib(u128::from(CGROUP_3G) - three.total()), "111");
}
#[test]
fn the_kv_term_is_ninety_six_kibibytes_a_token_on_the_v0_geometry() {
let (manifest, layout) = v0_install();
for cap in [1usize, 512, 4096, 16_384, 131_072] {
let projected =
Footprint::project(&manifest, &layout, crate::model::DEFAULT_CACHE_BYTES, cap)
.expect("a sane cap projects");
assert_eq!(
projected.kv,
cap as u64 * 96 * 1024,
"kv term at a {cap}-token cap"
);
assert_eq!(projected.context_cap, cap);
assert_eq!(projected.common, V0_COMMON_BYTES);
assert_eq!(projected.experts, 19 * V0_PER_SLOT);
assert_eq!(projected.runtime, RUNTIME_ANON_BYTES);
}
assert_eq!(4096u64 * 96 * 1024, 384 << 20);
assert_eq!(16_384u64 * 96 * 1024, 1536 << 20);
}
#[test]
fn the_expert_term_is_the_slab_the_pool_would_actually_allocate() {
use crate::format::{FileEntry, LayerLayout, QuantInfo, RVMP_VERSION, SourceInfo};
use std::collections::BTreeMap;
let strides = [8192u64, 100, 4096];
let per_slot = 8192 + 4096 + 4096;
let mut arch = crate::io::testutil::fixture_arch();
arch.n_layers = 3;
arch.n_experts = 8;
arch.top_k = 2;
let manifest = Manifest {
rvmp_version: RVMP_VERSION,
model_id: "tiny".to_owned(),
source: SourceInfo {
hf_repo: "test/tiny".to_owned(),
revision: "0".repeat(40),
file: "tiny.gguf".to_owned(),
sha256: "0".repeat(64),
},
arch,
quant: QuantInfo {
scheme: "gguf".to_owned(),
tensor_types: BTreeMap::new(),
},
common_tensors: BTreeMap::new(),
files: BTreeMap::from([(
COMMON_FILE.to_owned(),
FileEntry {
size: 4096,
sha256: "0".repeat(64),
},
)]),
};
let layout = ExpertsLayout {
layers: strides
.iter()
.enumerate()
.map(|(layer, &stride)| LayerLayout {
file: crate::format::layer_file_name(layer as u32),
stride,
n_experts: 8,
projections: Vec::new(),
})
.collect(),
};
for budget in [3 * per_slot, 3 * per_slot + per_slot - 1] {
let projected = Footprint::project(&manifest, &layout, budget, 16).unwrap();
let pool = SlotPool::new(3, &strides).unwrap();
assert_eq!(
projected.experts,
pool.total_bytes() as u64,
"predicted slab for a {budget}-byte budget"
);
assert_eq!(projected.experts, 3 * per_slot);
}
}
#[test]
fn a_context_cap_near_usize_max_is_an_error_not_an_overflow() {
let (manifest, layout) = v0_install();
for cap in [usize::MAX, usize::MAX / 2, usize::MAX / 512 + 1] {
let err =
Footprint::project(&manifest, &layout, crate::model::DEFAULT_CACHE_BYTES, cap)
.expect_err("a {cap}-token context cannot be sized");
assert!(
matches!(
err,
FootprintError::Kv(crate::kv::KvError::InvalidDims { .. })
),
"cap {cap} gave {err}"
);
}
let err = Footprint::project(&manifest, &layout, crate::model::DEFAULT_CACHE_BYTES, 0)
.expect_err("a zero context cannot be sized");
assert!(
matches!(
err,
FootprintError::Kv(crate::kv::KvError::InvalidDims { .. })
),
"{err}"
);
}
#[test]
fn the_verdict_is_the_arithmetic_and_nothing_more() {
let footprint = Footprint {
common: 1000,
experts: 1000,
kv: 1000,
runtime: 1000,
context_cap: 8,
};
assert_eq!(footprint.total(), 4000);
assert!(footprint.fits_within(4001));
assert!(footprint.fits_within(4000), "an exact fit is a fit");
assert!(!footprint.fits_within(3999));
let huge = Footprint {
common: u64::MAX,
experts: u64::MAX,
kv: u64::MAX,
runtime: u64::MAX,
context_cap: usize::MAX,
};
assert_eq!(huge.total(), 4 * u128::from(u64::MAX));
assert!(!huge.fits_within(u64::MAX));
}
#[test]
fn the_breakdown_names_every_tenant_in_the_contract_s_units() {
let (manifest, layout) = v0_install();
let projected = Footprint::project(
&manifest,
&layout,
crate::model::DEFAULT_CACHE_BYTES,
16_384,
)
.unwrap();
assert_eq!(
projected.to_string(),
"about 5,159 MiB (common 1,023 + experts 2,485 + KV 1,536 \
at 16384 tokens + runtime 115)"
);
}
#[test]
fn mib_rounds_to_nearest_and_groups_by_threes() {
assert_eq!(mib(0), "0");
assert_eq!(mib(MIB / 2 - 1), "0");
assert_eq!(mib(MIB / 2), "1", "rounds to nearest, not down");
assert_eq!(mib(MIB), "1");
assert_eq!(mib(u128::from(V0_COMMON_BYTES)), "1,023");
assert_eq!(mib(u128::from(11 * V0_PER_SLOT)), "1,439");
assert_eq!(mib(u128::from(RUNTIME_ANON_BYTES)), "115");
assert_eq!(mib(u128::from(CGROUP_3G)), "3,072");
assert_eq!(mib(1 << 40), "1,048,576");
}
#[test]
fn the_anonymous_runtime_row_is_exp_012_s_measurement() {
assert_eq!(RUNTIME_ANON_BYTES, 120_691_097);
assert_eq!(mib(u128::from(RUNTIME_ANON_BYTES)), "115");
let hundredths = u128::from(RUNTIME_ANON_BYTES) * 100 / MIB;
assert_eq!(hundredths, 11_509);
}
#[test]
fn a_manifest_missing_its_common_entry_reports_rather_than_panics() {
let (mut manifest, layout) = v0_install();
manifest.files.clear();
let err = Footprint::project(&manifest, &layout, crate::model::DEFAULT_CACHE_BYTES, 4096)
.unwrap_err();
assert!(
matches!(
err,
FootprintError::Io(IoError::Format(
crate::format::FormatError::MissingFileEntry(ref name)
)) if name == COMMON_FILE
),
"{err}"
);
}
#[test]
fn the_public_ceiling_is_the_pool_s_own_probe() {
assert_eq!(resident_ceiling(), resident_limit());
}
#[test]
#[cfg(target_os = "linux")]
fn the_resident_limit_probe_reports_something_plausible() {
let limit = resident_limit().expect("Linux always reports MemTotal");
assert!(
limit >= 64 << 20,
"implausible resident limit {limit}; the probe is misreading the kernel"
);
}
#[test]
#[cfg(target_os = "linux")]
fn budget_probe_parsers_read_the_kernel_formats() {
assert_eq!(parse_memory_max("3221225472\n"), Some(3 << 30));
assert_eq!(parse_memory_max("max\n"), None);
assert_eq!(parse_memory_max("garbage"), None);
assert_eq!(
parse_mem_total("MemTotal: 15914637 kB\nMemFree: 12 kB\n"),
Some(16_296_588_288)
);
assert_eq!(parse_mem_total("MemFree: 12 kB\n"), None);
assert_eq!(parse_mem_total("MemTotal:\n"), None);
assert_eq!(
parse_cgroup_v2_path("0::/user.slice/session.scope\n"),
Some("/user.slice/session.scope")
);
assert_eq!(parse_cgroup_v2_path("7:memory:/foo\n"), None);
assert_eq!(
cgroup_ancestors("/user.slice/session.scope"),
["/user.slice/session.scope", "/user.slice", ""]
);
assert_eq!(cgroup_ancestors("/"), ["/", ""]);
}
#[test]
fn error_messages_name_the_problem() {
assert_eq!(
SlotError::Exhausted {
layer: 7,
slots_per_layer: 12
}
.to_string(),
"layer 7: all 12 slots are in use"
);
assert_eq!(
SlotError::ZeroStride { layer: 3 }.to_string(),
"layer 3: blob stride is zero"
);
}
}