use std::fmt;
use std::fs::File;
use std::io;
use std::os::unix::fs::FileExt;
use std::path::{Path, PathBuf};
use std::ptr::NonNull;
use std::time::{Duration, Instant};
#[cfg(feature = "io-uring")]
use std::os::fd::AsRawFd;
#[cfg(feature = "io-uring")]
use io_uring::{IoUring, opcode, types};
use crate::format::{ExpertsLayout, FormatError, Manifest, sha256_file};
use super::direct::{self, DirectFault, DirectSupport};
use super::sweep::{SweepError, check_aligned};
use super::{
CachePlan, CacheStats, ExpertReader, ExpertView, IoError, LayerCache, LoadOptions, MAX_SLOTS,
SLOT_ALIGN, SlotError, SlotGuard, SlotPool,
};
#[cfg(feature = "io-uring")]
const RING_ENTRIES: u32 = 8;
#[cfg(feature = "io-uring")]
const MAX_IDLE_PASSES: u32 = 1024;
#[cfg(feature = "io-uring")]
const MAX_PUSH_ATTEMPTS: u32 = 8;
#[cfg(feature = "io-uring")]
const MAX_EINTR_RETRIES: u32 = 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StreamMode {
ODirect,
Buffered,
Pread,
}
impl fmt::Display for StreamMode {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::ODirect => "io_uring+O_DIRECT",
Self::Buffered => "io_uring+buffered",
Self::Pread => "pread",
})
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
pub enum StreamPhase {
#[default]
Prefill,
Decode,
}
impl StreamPhase {
pub const ALL: [Self; 2] = [Self::Prefill, Self::Decode];
fn index(self) -> usize {
match self {
Self::Prefill => 0,
Self::Decode => 1,
}
}
}
impl fmt::Display for StreamPhase {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::Prefill => "prefill",
Self::Decode => "decode",
})
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct StreamStats {
pub hits: u64,
pub pending_hits: u64,
pub misses: u64,
pub cold_misses: u64,
pub eviction_misses: u64,
pub bytes_read: u64,
pub reads_submitted: u64,
pub read_retries: u64,
pub io_wait: Duration,
pub sweep_bytes_read: u64,
pub sweep_reads_submitted: u64,
pub sweep_read_retries: u64,
pub sweep_io_wait: Duration,
pub sweep_windows_read: u64,
pub sweep_windows_skipped: u64,
}
impl StreamStats {
pub fn accesses(&self) -> u64 {
self.hits + self.pending_hits + self.misses
}
pub fn sweep_windows(&self) -> u64 {
self.sweep_windows_read + self.sweep_windows_skipped
}
pub fn is_idle(&self) -> bool {
self.accesses() == 0 && self.sweep_windows() == 0
}
pub fn hit_rate(&self) -> f64 {
let accesses = self.accesses();
if accesses == 0 {
0.0
} else {
self.hits as f64 / accesses as f64
}
}
pub fn since(&self, earlier: &Self) -> Self {
Self {
hits: self.hits.saturating_sub(earlier.hits),
pending_hits: self.pending_hits.saturating_sub(earlier.pending_hits),
misses: self.misses.saturating_sub(earlier.misses),
cold_misses: self.cold_misses.saturating_sub(earlier.cold_misses),
eviction_misses: self.eviction_misses.saturating_sub(earlier.eviction_misses),
bytes_read: self.bytes_read.saturating_sub(earlier.bytes_read),
reads_submitted: self.reads_submitted.saturating_sub(earlier.reads_submitted),
read_retries: self.read_retries.saturating_sub(earlier.read_retries),
io_wait: self.io_wait.saturating_sub(earlier.io_wait),
sweep_bytes_read: self
.sweep_bytes_read
.saturating_sub(earlier.sweep_bytes_read),
sweep_reads_submitted: self
.sweep_reads_submitted
.saturating_sub(earlier.sweep_reads_submitted),
sweep_read_retries: self
.sweep_read_retries
.saturating_sub(earlier.sweep_read_retries),
sweep_io_wait: self.sweep_io_wait.saturating_sub(earlier.sweep_io_wait),
sweep_windows_read: self
.sweep_windows_read
.saturating_sub(earlier.sweep_windows_read),
sweep_windows_skipped: self
.sweep_windows_skipped
.saturating_sub(earlier.sweep_windows_skipped),
}
}
pub(super) fn plus(&self, other: &Self) -> Self {
Self {
hits: self.hits + other.hits,
pending_hits: self.pending_hits + other.pending_hits,
misses: self.misses + other.misses,
cold_misses: self.cold_misses + other.cold_misses,
eviction_misses: self.eviction_misses + other.eviction_misses,
bytes_read: self.bytes_read + other.bytes_read,
reads_submitted: self.reads_submitted + other.reads_submitted,
read_retries: self.read_retries + other.read_retries,
io_wait: self.io_wait + other.io_wait,
sweep_bytes_read: self.sweep_bytes_read + other.sweep_bytes_read,
sweep_reads_submitted: self.sweep_reads_submitted + other.sweep_reads_submitted,
sweep_read_retries: self.sweep_read_retries + other.sweep_read_retries,
sweep_io_wait: self.sweep_io_wait + other.sweep_io_wait,
sweep_windows_read: self.sweep_windows_read + other.sweep_windows_read,
sweep_windows_skipped: self.sweep_windows_skipped + other.sweep_windows_skipped,
}
}
}
#[derive(Debug, Clone, Copy, Default)]
struct IoStats {
bytes_read: u64,
reads_submitted: u64,
read_retries: u64,
io_wait: Duration,
stale_completions: u64,
sweep_bytes_read: u64,
sweep_reads_submitted: u64,
sweep_read_retries: u64,
sweep_io_wait: Duration,
sweep_windows_read: u64,
sweep_windows_skipped: u64,
in_sweep: bool,
}
impl IoStats {
fn add_bytes(&mut self, bytes: u64) {
if self.in_sweep {
self.sweep_bytes_read += bytes;
} else {
self.bytes_read += bytes;
}
}
fn add_submit(&mut self) {
if self.in_sweep {
self.sweep_reads_submitted += 1;
} else {
self.reads_submitted += 1;
}
}
fn add_retry(&mut self) {
if self.in_sweep {
self.sweep_read_retries += 1;
} else {
self.read_retries += 1;
}
}
fn add_wait(&mut self, elapsed: Duration) {
if self.in_sweep {
self.sweep_io_wait += elapsed;
} else {
self.io_wait += elapsed;
}
}
}
#[derive(Debug)]
struct LayerState {
path: PathBuf,
name: String,
stride: u64,
len: u32,
n_experts: u32,
expected_size: u64,
expected_sha256: String,
file: Option<File>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Dest {
Slot,
Arena(usize),
}
#[derive(Debug, Clone, Copy)]
struct Read {
user_data: u64,
dst: Dest,
layer: u32,
slot: u32,
expert: u32,
base: u64,
filled: u32,
remaining: u32,
attempts: u8,
restarts: u8,
done: bool,
}
#[derive(Debug)]
enum Reap {
Stale,
Done(usize),
Again(usize),
Failed(usize, io::Error),
}
#[derive(Debug, Default)]
struct Inflight {
reads: Vec<Read>,
outstanding: usize,
next_tag: u32,
align: u32,
}
impl Inflight {
fn with_capacity(top_k: usize) -> Self {
Self {
reads: Vec::with_capacity(top_k),
..Self::default()
}
}
fn track(&mut self, layer: u32, slot: u32, expert: u32, base: u64, len: u32) -> usize {
self.push(Dest::Slot, layer, slot, expert, base, len)
}
fn track_arena(
&mut self,
layer: u32,
buffer: u32,
first_expert: u32,
base: u64,
len: u32,
offset: usize,
) -> usize {
self.push(Dest::Arena(offset), layer, buffer, first_expert, base, len)
}
fn push(
&mut self,
dst: Dest,
layer: u32,
slot: u32,
expert: u32,
base: u64,
len: u32,
) -> usize {
let index = self.reads.len();
let user_data = self.token(index);
self.reads.push(Read {
user_data,
dst,
layer,
slot,
expert,
base,
filled: 0,
remaining: len,
attempts: 0,
restarts: 0,
done: false,
});
self.outstanding += 1;
index
}
fn retag(&mut self, index: usize) -> u64 {
let user_data = self.token(index);
self.reads[index].user_data = user_data;
user_data
}
fn token(&mut self, index: usize) -> u64 {
self.next_tag = self.next_tag.wrapping_add(1);
(u64::from(self.next_tag) << 32) | (index as u64 & 0xffff_ffff)
}
fn resolve(&mut self, user_data: u64, result: i32, io: &mut IoStats) -> Reap {
let index = (user_data & 0xffff_ffff) as usize;
let align = self.align;
let Some(read) = self.reads.get_mut(index) else {
return Reap::Stale;
};
if read.user_data != user_data || read.done {
return Reap::Stale;
}
if result > 0 {
let got = (result as u32).min(read.remaining);
read.filled += got;
read.remaining -= got;
io.add_bytes(u64::from(got));
if read.remaining == 0 {
read.done = true;
self.outstanding -= 1;
return Reap::Done(index);
}
io.add_retry();
if align != 0 && read.filled % align != 0 {
if read.restarts > 0 {
read.done = true;
self.outstanding -= 1;
let (base, filled) = (read.base, read.filled);
return Reap::Failed(
index,
io::Error::new(
io::ErrorKind::InvalidData,
format!(
"expert blob at {base}: short reads keep stopping off a \
{align}-byte boundary (last at +{filled}), which direct \
I/O cannot resume from"
),
),
);
}
read.restarts += 1;
tracing::warn!(
layer = read.layer,
expert = read.expert,
slot = read.slot,
filled = read.filled,
"short read stopped off a block boundary; restarting the blob"
);
read.remaining += read.filled;
read.filled = 0;
read.attempts = 0;
return Reap::Again(index);
}
return Reap::Again(index);
}
if result == 0 {
read.done = true;
self.outstanding -= 1;
let (base, filled, remaining) = (read.base, read.filled, read.remaining);
return Reap::Failed(
index,
io::Error::new(
io::ErrorKind::UnexpectedEof,
format!(
"expert blob at {base}: {remaining} of {} bytes past end of file",
filled + remaining
),
),
);
}
let errno = -result;
let error = io::Error::from_raw_os_error(errno);
if is_retryable(errno) && read.attempts == 0 {
read.attempts += 1;
io.add_retry();
tracing::warn!(
layer = read.layer,
expert = read.expert,
slot = read.slot,
%error,
"expert blob read failed, retrying once"
);
return Reap::Again(index);
}
read.done = true;
self.outstanding -= 1;
Reap::Failed(index, error)
}
fn abandon(&mut self, index: usize) {
if let Some(read) = self.reads.get_mut(index)
&& !read.done
{
read.done = true;
self.outstanding -= 1;
}
}
fn clear(&mut self) {
debug_assert_eq!(self.outstanding, 0, "clearing the table under live reads");
self.reads.clear();
}
}
fn is_retryable(errno: i32) -> bool {
matches!(
errno,
libc::EIO | libc::EAGAIN | libc::EINTR | libc::ECANCELED | libc::ENOMEM
)
}
struct SlotTable {
guards: Box<[Option<SlotGuard<'static>>]>,
live: Box<[Vec<u32>]>,
pool: NonNull<SlotPool>,
slots_per_layer: u32,
}
unsafe impl Send for SlotTable {}
unsafe impl Sync for SlotTable {}
impl fmt::Debug for SlotTable {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SlotTable")
.field("slots_per_layer", &self.slots_per_layer)
.field("total_bytes", &self.pool().total_bytes())
.finish()
}
}
impl Drop for SlotTable {
fn drop(&mut self) {
self.guards = Box::default();
drop(unsafe { Box::from_raw(self.pool.as_ptr()) });
}
}
impl SlotTable {
fn new(slots_per_layer: u32, strides: &[u64]) -> Result<Self, SlotError> {
let raw = Box::into_raw(Box::new(SlotPool::new(slots_per_layer, strides)?));
let pool: &'static SlotPool = unsafe { &*raw };
let n_layers = pool.n_layers();
let mut guards: Vec<Option<SlotGuard<'static>>> = (0..strides.len()
* slots_per_layer as usize)
.map(|_| None)
.collect();
let mut failure = None;
'lease: for layer in 0..n_layers {
for _ in 0..slots_per_layer {
match pool.acquire(layer) {
Ok(guard) => {
let flat =
layer as usize * slots_per_layer as usize + guard.index() as usize;
guards[flat] = Some(guard);
}
Err(error) => {
failure = Some(error);
break 'lease;
}
}
}
}
if let Some(error) = failure {
drop(guards);
drop(unsafe { Box::from_raw(raw) });
return Err(error);
}
Ok(Self {
guards: guards.into_boxed_slice(),
live: (0..n_layers)
.map(|_| (0..slots_per_layer).collect())
.collect(),
pool: unsafe { NonNull::new_unchecked(raw) },
slots_per_layer,
})
}
fn pool(&self) -> &SlotPool {
unsafe { self.pool.as_ref() }
}
fn usable(&self, layer: u32) -> u32 {
self.live
.get(layer as usize)
.map_or(0, |live| live.len() as u32)
}
fn flat(&self, layer: u32, slot: u32) -> Option<usize> {
let pool_slot = *self.live.get(layer as usize)?.get(slot as usize)? as usize;
Some(layer as usize * self.slots_per_layer as usize + pool_slot)
}
fn bytes(&self, layer: u32, slot: u32) -> Option<&[u8]> {
let flat = self.flat(layer, slot)?;
self.guards[flat].as_ref().map(SlotGuard::as_slice)
}
fn bytes_mut(&mut self, layer: u32, slot: u32) -> Option<&mut [u8]> {
let flat = self.flat(layer, slot)?;
self.guards[flat].as_mut().map(SlotGuard::as_mut_slice)
}
#[cfg(feature = "io-uring")]
unsafe fn write_ptr(&mut self, layer: u32, slot: u32) -> Option<*mut u8> {
let flat = self.flat(layer, slot)?;
self.guards[flat]
.as_mut()
.map(|guard| unsafe { guard.as_mut_ptr() })
}
fn retire(&mut self, layer: u32, slot: u32) -> bool {
let Some(flat) = self.flat(layer, slot) else {
return false;
};
if let Some(guard) = self.guards[flat].take() {
guard.leak();
}
self.live[layer as usize].remove(slot as usize);
true
}
fn total_bytes(&self) -> u64 {
self.pool().total_bytes() as u64
}
fn slot_offset(&self, layer: u32, slot: u32) -> Option<usize> {
let pool_slot = *self.live.get(layer as usize)?.get(slot as usize)?;
self.pool().slot_offset(layer, pool_slot)
}
fn padded_layer(&self) -> Option<(u32, usize, usize)> {
self.pool().padded_layer()
}
fn retired_within(&self, bytes: usize) -> Option<(u32, usize)> {
let per_layer = self.slots_per_layer as usize;
if per_layer == 0 {
return None;
}
self.guards
.iter()
.enumerate()
.filter(|(_, guard)| guard.is_none())
.find_map(|(flat, _)| {
let layer = (flat / per_layer) as u32;
let pool_slot = (flat % per_layer) as u32;
self.pool()
.slot_offset(layer, pool_slot)
.filter(|offset| *offset < bytes)
.map(|offset| (layer, offset))
})
}
unsafe fn slab_base(&mut self) -> NonNull<u8> {
unsafe { self.pool().slab_base() }
}
}
#[derive(Debug, Clone, Copy)]
struct Arena {
base: NonNull<u8>,
len: usize,
}
unsafe impl Send for Arena {}
#[derive(Debug, Default)]
struct Step {
active: bool,
layer: u32,
hits: Vec<(usize, u32)>,
misses: Vec<(usize, u32)>,
protected: Vec<u32>,
}
impl Step {
fn with_capacity(top_k: usize) -> Self {
Self {
hits: Vec::with_capacity(top_k),
misses: Vec::with_capacity(top_k),
protected: Vec::with_capacity(top_k),
..Self::default()
}
}
fn reset(&mut self, layer: u32) {
self.active = true;
self.layer = layer;
self.hits.clear();
self.misses.clear();
self.protected.clear();
}
}
pub struct ExpertStream {
reader: ExpertReader,
layers: Box<[LayerState]>,
caches: Box<[LayerCache]>,
plan: CachePlan,
step: Step,
inflight: Inflight,
#[cfg(feature = "io-uring")]
completions: Vec<(u64, i32)>,
#[cfg(feature = "io-uring")]
reaping: Vec<(u64, i32)>,
#[cfg(feature = "io-uring")]
ring: Option<IoUring>,
#[cfg(feature = "io-uring")]
ring_owner: std::thread::ThreadId,
#[cfg(feature = "io-uring")]
ring_rebinds: u32,
slots: SlotTable,
arena: Option<Arena>,
arena_poisoned: bool,
top_k: u32,
mode: StreamMode,
support: DirectSupport,
direct_open: bool,
hash_layers: bool,
io: IoStats,
retired: CacheStats,
phase: StreamPhase,
phase_start: StreamStats,
phase_totals: [StreamStats; 2],
#[cfg(all(test, feature = "io-uring"))]
stalled_enters: u32,
}
impl fmt::Debug for ExpertStream {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ExpertStream")
.field("mode", &self.mode)
.field("direct_io", &self.support)
.field("n_layers", &self.layers.len())
.field("slots_per_layer", &self.slots.slots_per_layer)
.field("cache_bytes", &self.slots.total_bytes())
.field("outstanding", &self.inflight.outstanding)
.finish()
}
}
impl ExpertStream {
pub fn new(
dir: &Path,
manifest: &Manifest,
layout: &ExpertsLayout,
cache_bytes: u64,
options: LoadOptions,
) -> Result<Self, IoError> {
let reader = ExpertReader::new(dir, manifest, layout, options)?;
let mut layers = Vec::with_capacity(layout.layers.len());
let mut strides = Vec::with_capacity(layout.layers.len());
for layer in &layout.layers {
let entry = manifest
.files
.get(&layer.file)
.ok_or_else(|| FormatError::MissingFileEntry(layer.file.clone()))?;
layers.push(LayerState {
path: dir.join(&layer.file),
name: layer.file.clone(),
stride: layer.stride,
len: u32::try_from(layer.stride).map_err(|_| IoError::TooLarge {
what: "expert blob stride",
value: layer.stride,
})?,
n_experts: layer.n_experts,
expected_size: entry.size,
expected_sha256: entry.sha256.clone(),
file: None,
});
strides.push(layer.stride);
}
let n_experts: Vec<u32> = layers.iter().map(|layer| layer.n_experts).collect();
let slots_per_layer =
slots_for_budget(cache_bytes, &strides, &n_experts, manifest.arch.top_k)?;
let slots = SlotTable::new(slots_per_layer, &strides)?;
let mut caches = Vec::with_capacity(layers.len());
for layer in &layers {
caches.push(LayerCache::new(slots_per_layer, layer.n_experts)?);
}
let top_k = manifest.arch.top_k.max(1) as usize;
let mut stream = Self {
reader,
layers: layers.into_boxed_slice(),
caches: caches.into_boxed_slice(),
plan: CachePlan::with_capacity(top_k),
step: Step::with_capacity(top_k),
inflight: Inflight::with_capacity(top_k),
#[cfg(feature = "io-uring")]
completions: Vec::with_capacity(completion_capacity(slots_per_layer)),
#[cfg(feature = "io-uring")]
reaping: Vec::with_capacity(completion_capacity(slots_per_layer)),
#[cfg(feature = "io-uring")]
ring: None,
#[cfg(feature = "io-uring")]
ring_owner: std::thread::current().id(),
#[cfg(feature = "io-uring")]
ring_rebinds: 0,
slots,
arena: None,
arena_poisoned: false,
top_k: top_k as u32,
mode: StreamMode::Pread,
support: DirectSupport::Unusable(DirectFault::Unsupported),
direct_open: false,
hash_layers: options.verify_layer_hashes && !options.skip_hashes,
io: IoStats::default(),
retired: CacheStats::default(),
phase: StreamPhase::default(),
phase_start: StreamStats::default(),
phase_totals: [StreamStats::default(); 2],
#[cfg(all(test, feature = "io-uring"))]
stalled_enters: 0,
};
stream.support = stream.probe_direct_io(&strides);
stream.direct_open = stream.support.is_usable();
stream.inflight.align = if stream.direct_open {
direct::DIO_ALIGN as u32
} else {
0
};
#[cfg(feature = "io-uring")]
{
stream.ring = build_ring(true);
stream.ring_owner = std::thread::current().id();
}
stream.mode = stream.pick_mode();
let bytes = stream.slots.total_bytes();
tracing::info!(
mode = %stream.mode,
direct_io = %stream.support,
slots_per_layer,
pool_bytes = bytes,
budget_bytes = cache_bytes,
n_layers = stream.layers.len(),
"expert stream ready"
);
if !stream.support.is_verified() {
let measured = stream.support.fault() == Some(DirectFault::PageCacheGrew);
tracing::warn!(
mode = %stream.mode,
direct_io = %stream.support,
verdict = if measured { "measured" } else { "unmeasurable" },
"{}; the memory contract does not hold in this mode",
if measured {
"expert reads are charged to the page cache"
} else {
"expert reads could not be shown to bypass the page cache"
}
);
} else if stream.mode != StreamMode::ODirect {
tracing::info!(
mode = %stream.mode,
"no io_uring on this host; expert reads are synchronous but \
still bypass the page cache, so the memory contract holds"
);
}
Ok(stream)
}
pub fn slots_per_layer(&self) -> u32 {
self.slots.slots_per_layer
}
pub fn usable_slots(&self, layer: u32) -> u32 {
self.slots.usable(layer)
}
pub fn cache_bytes(&self) -> u64 {
self.slots.total_bytes()
}
pub fn mode(&self) -> StreamMode {
self.mode
}
pub fn direct_io(&self) -> bool {
self.support.is_verified()
}
pub fn direct_support(&self) -> DirectSupport {
self.support
}
pub fn n_layers(&self) -> u32 {
self.layers.len() as u32
}
pub fn begin_layer(&mut self, layer: u32, experts: &[u32]) -> Result<(), IoError> {
let index = self.layer_index(layer)?;
if self.step.active {
return Err(self.protocol_error(
layer,
"begin_layer called with a step already open; end_layer first",
));
}
if self.inflight.outstanding > 0 {
return Err(self.protocol_error(
layer,
"reads from an earlier step were never awaited; call \
await_misses before beginning another layer",
));
}
self.ensure_open(index)?;
#[cfg(feature = "io-uring")]
self.rebind_ring();
{
let plan = &mut self.plan;
let cache = &mut self.caches[index];
cache.plan(experts, plan)?;
}
self.step.reset(layer);
for &(expert, slot) in self.plan.hits() {
let request = position_of(experts, expert);
self.step.hits.push((request, slot));
self.step.protected.push(slot);
}
for &(expert, slot) in self.plan.misses() {
let request = position_of(experts, expert);
self.step.misses.push((request, slot));
self.step.protected.push(slot);
}
self.inflight.clear();
#[cfg(feature = "io-uring")]
{
self.completions.clear();
}
let stride = self.layers[index].stride;
let len = self.layers[index].len;
let mut failure = None;
for i in 0..self.plan.misses().len() {
let (expert, slot) = self.plan.misses()[i];
let read = self
.inflight
.track(layer, slot, expert, u64::from(expert) * stride, len);
if let Err(error) = self.submit(read) {
self.inflight.abandon(read);
failure = Some(self.fail(read, flatten_io(error)));
break;
}
}
if let Some(error) = failure {
let _ = self.await_misses();
self.abandon_step();
return Err(error);
}
if let Err(error) = self.flush() {
self.strand_step();
return Err(error);
}
Ok(())
}
pub fn hits(&self) -> &[(usize, u32)] {
&self.step.hits
}
pub fn misses(&self) -> &[(usize, u32)] {
&self.step.misses
}
pub fn await_misses(&mut self) -> Result<(), IoError> {
if self.inflight.outstanding == 0 {
return Ok(());
}
let mut failure = None;
if let Err(error) = self.drive_reads(None, &mut failure) {
self.strand_step();
return Err(failure.unwrap_or(error));
}
match failure {
None => Ok(()),
Some(error) => {
self.abandon_step();
Err(error)
}
}
}
pub fn view(&self, layer: u32, slot: u32) -> Result<ExpertView<'_>, IoError> {
let index = self.layer_index(layer)?;
if !self.caches[index].is_ready(slot) {
return Err(self.protocol_error(
layer,
&format!("slot {slot} holds no completed expert blob"),
));
}
let bytes = self.slots.bytes(layer, slot).ok_or_else(|| {
self.protocol_error(layer, &format!("slot {slot} is not in the pool"))
})?;
self.reader.view_over(layer, bytes)
}
pub fn sweep_layer<'a>(
&'a mut self,
plan: &'a mut super::SweepPlan,
layer: u32,
routed: &[u32],
config: super::SweepConfig,
) -> Result<super::LayerSweep<'a>, SweepError> {
super::LayerSweep::begin(self, plan, layer, routed, config)
}
pub fn begin_prefill(
&mut self,
scratch_bytes: usize,
config: super::SweepConfig,
) -> Result<super::PrefillSession<'_>, SweepError> {
super::PrefillSession::begin(self, scratch_bytes, config)
}
pub(super) fn take_arena(&mut self, bytes: usize) -> Result<(), SweepError> {
if self.arena_poisoned {
return Err(SweepError::ArenaPoisoned);
}
if let Some(arena) = self.arena {
tracing::error!(
live_bytes = arena.len,
bytes,
"refusing a second prefill arena over a live one"
);
return Err(SweepError::ArenaOut { bytes: arena.len });
}
if self.inflight.outstanding > 0 {
return Err(SweepError::ReadsInFlight {
outstanding: self.inflight.outstanding,
});
}
if self.step.active {
return Err(SweepError::StepOpen {
layer: self.step.layer,
});
}
if let Some((layer, stride, pitch)) = self.slots.padded_layer() {
return Err(SweepError::PaddedSlots {
layer,
stride,
pitch,
});
}
let available = self.slots.total_bytes();
if bytes as u64 > available {
return Err(SweepError::ArenaTooSmall {
needed: bytes as u64,
available,
});
}
if let Some((layer, offset)) = self.slots.retired_within(bytes) {
tracing::error!(
layer,
offset,
bytes,
"refusing a prefill arena over a retired slot buffer"
);
return Err(SweepError::ArenaOverRetired {
layer,
offset,
bytes,
});
}
let base = unsafe { self.slots.slab_base() };
debug_assert!(
(base.as_ptr() as usize as u64).is_multiple_of(direct::DIO_ALIGN),
"the slot pool slab is allocated 4096-aligned"
);
self.invalidate_all_slots();
self.inflight.clear();
#[cfg(feature = "io-uring")]
self.completions.clear();
self.arena = Some(Arena { base, len: bytes });
self.io.in_sweep = true;
tracing::debug!(
arena_bytes = bytes,
pool_bytes = available,
"prefill arena taken; the expert cache is now empty"
);
Ok(())
}
pub(super) fn release_arena(&mut self) {
if self.arena.take().is_none() {
return;
}
if self.inflight.outstanding == 0 {
self.invalidate_all_slots();
self.inflight.clear();
} else {
tracing::error!(
outstanding = self.inflight.outstanding,
"the prefill arena was released with window reads still in \
flight; leaving slot occupancy alone, since it is already \
empty from the carve"
);
}
self.io.in_sweep = false;
}
pub(super) fn arena_len(&self) -> usize {
self.arena.map_or(0, |arena| arena.len)
}
pub(super) fn arena_base(&self) -> Option<NonNull<u8>> {
self.arena.map(|arena| arena.base)
}
pub(super) unsafe fn arena_bytes(&self, offset: usize, len: usize) -> Option<&[u8]> {
let arena = self.arena?;
if offset.checked_add(len)? > arena.len {
return None;
}
Some(unsafe { std::slice::from_raw_parts(arena.base.as_ptr().add(offset), len) })
}
pub(super) fn sweep_submit(
&mut self,
layer: u32,
buffer: u32,
first_expert: u32,
file_offset: u64,
len: u32,
arena_offset: usize,
) -> Result<usize, SweepError> {
let Some(arena) = self.arena else {
return Err(SweepError::NoArena);
};
check_aligned("sweep read offset", file_offset)?;
check_aligned("sweep read length", u64::from(len))?;
check_aligned(
"sweep read destination",
arena.base.as_ptr() as usize as u64 + arena_offset as u64,
)?;
self.ensure_open(self.layer_index(layer)?)?;
let index =
self.inflight
.track_arena(layer, buffer, first_expert, file_offset, len, arena_offset);
if let Err(error) = self.submit(index) {
self.inflight.abandon(index);
return Err(error.into());
}
if let Err(error) = self.flush() {
return Err(self.strand_arena_error(error));
}
Ok(index)
}
pub(super) fn sweep_await(&mut self, until: Option<usize>) -> Result<(), SweepError> {
let mut failure = None;
if let Err(error) = self.drive_reads(until, &mut failure) {
let source = failure.unwrap_or(error);
return Err(self.strand_arena_error(source));
}
match failure {
None => Ok(()),
Some(error) => Err(error.into()),
}
}
pub(super) fn reads_outstanding(&self) -> usize {
self.inflight.outstanding
}
pub(super) fn sweep_clear_reads(&mut self) {
if self.inflight.outstanding == 0 {
self.inflight.clear();
}
}
pub(super) fn count_window(&mut self, read: bool) {
if read {
self.io.sweep_windows_read += 1;
} else {
self.io.sweep_windows_skipped += 1;
}
}
pub(super) fn open_layer(&mut self, layer: u32) -> Result<(), IoError> {
let index = self.layer_index(layer)?;
self.ensure_open(index)
}
pub(super) fn layer_geometry(&self, layer: u32) -> Result<(u64, u32), IoError> {
let index = self.layer_index(layer)?;
let state = &self.layers[index];
Ok((state.stride, state.n_experts))
}
pub(super) fn expert_reader(&self) -> &ExpertReader {
&self.reader
}
fn invalidate_all_slots(&mut self) {
for cache in &mut self.caches {
unsafe { cache.reset_occupancy() };
}
}
#[must_use = "a layer left below top_k has to be reported at the cause"]
fn strand_arena(&mut self) -> Option<(u32, u32)> {
let len = self.arena_len();
for index in 0..self.inflight.reads.len() {
let read = self.inflight.reads[index];
if read.done {
continue;
}
tracing::error!(
layer = read.layer,
window = read.slot,
first_expert = read.expert,
"prefill window read cannot be reaped; the arena is given up"
);
self.inflight.abandon(index);
}
self.arena = None;
self.arena_poisoned = true;
self.io.in_sweep = false;
for layer in (0..self.n_layers()).rev() {
for slot in (0..self.slots.usable(layer)).rev() {
if self
.slots
.slot_offset(layer, slot)
.is_some_and(|offset| offset < len)
{
self.retire_slot(layer, slot);
}
}
}
tracing::error!(
arena_bytes = len,
"the prefill arena was lost to an unreapable read; every slot it \
covered is retired and this process will not sweep again"
);
let stranded = (0..self.n_layers())
.map(|layer| (layer, self.slots.usable(layer)))
.find(|&(_, slots)| slots < self.top_k);
if let Some((layer, slots)) = stranded {
tracing::error!(
layer,
slots,
top_k = self.top_k,
"the retirement left a layer below top_k; that layer can no \
longer serve a decode step"
);
}
stranded
}
fn strand_arena_error(&mut self, source: IoError) -> SweepError {
match self.strand_arena() {
Some((layer, slots)) => SweepError::CacheStranded {
layer,
slots,
top_k: self.top_k,
source,
},
None => SweepError::Io(source),
}
}
pub fn end_layer(&mut self, layer: u32) {
if !self.step.active && self.inflight.outstanding == 0 {
tracing::warn!(layer, "end_layer without an open step");
return;
}
if self.step.active && self.step.layer != layer {
tracing::error!(
asked = layer,
open = self.step.layer,
"end_layer for a different layer than the open step; \
releasing the open one"
);
}
if self.inflight.outstanding > 0 {
tracing::error!(
layer = self.step.layer,
outstanding = self.inflight.outstanding,
"end_layer with reads still in flight; await_misses was \
skipped, so their slots are retired and the layer shrinks"
);
self.strand_step();
return;
}
let index = self.step.layer as usize;
if let Some(cache) = self.caches.get_mut(index) {
for slot in self.step.protected.drain(..) {
cache.release(slot);
}
}
self.step.active = false;
if self.inflight.outstanding == 0 {
self.inflight.clear();
}
}
pub fn stats(&self) -> StreamStats {
let mut stats = StreamStats {
bytes_read: self.io.bytes_read,
reads_submitted: self.io.reads_submitted,
read_retries: self.io.read_retries,
io_wait: self.io.io_wait,
sweep_bytes_read: self.io.sweep_bytes_read,
sweep_reads_submitted: self.io.sweep_reads_submitted,
sweep_read_retries: self.io.sweep_read_retries,
sweep_io_wait: self.io.sweep_io_wait,
sweep_windows_read: self.io.sweep_windows_read,
sweep_windows_skipped: self.io.sweep_windows_skipped,
..StreamStats::default()
};
let mut fold = |cache: CacheStats| {
stats.hits += cache.hits;
stats.pending_hits += cache.pending_hits;
stats.misses += cache.misses;
stats.cold_misses += cache.cold_misses;
stats.eviction_misses += cache.eviction_misses;
};
for cache in &self.caches {
fold(cache.stats());
}
fold(self.retired);
stats
}
pub fn stats_in(&self, phase: StreamPhase) -> StreamStats {
let closed = self.phase_totals[phase.index()];
if phase == self.phase {
closed.plus(&self.stats().since(&self.phase_start))
} else {
closed
}
}
pub fn phase(&self) -> StreamPhase {
self.phase
}
pub fn set_phase(&mut self, phase: StreamPhase) {
if phase == self.phase {
return;
}
let now = self.stats();
let closed = &mut self.phase_totals[self.phase.index()];
*closed = closed.plus(&now.since(&self.phase_start));
self.phase_start = now;
self.phase = phase;
}
pub fn stale_completions(&self) -> u64 {
self.io.stale_completions
}
fn layer_index(&self, layer: u32) -> Result<usize, IoError> {
if (layer as usize) < self.layers.len() {
Ok(layer as usize)
} else {
Err(IoError::LayerOutOfRange {
layer,
n_layers: self.n_layers(),
})
}
}
#[cfg(feature = "io-uring")]
fn layer_error(&self, source: io::Error) -> IoError {
IoError::Io {
path: self
.layers
.get(self.step.layer as usize)
.map(|state| state.path.clone())
.unwrap_or_default(),
source,
}
}
fn protocol_error(&self, layer: u32, message: &str) -> IoError {
IoError::Io {
path: self
.layers
.get(layer as usize)
.map(|state| state.path.clone())
.unwrap_or_default(),
source: io::Error::new(io::ErrorKind::InvalidInput, message.to_owned()),
}
}
fn ensure_open(&mut self, index: usize) -> Result<(), IoError> {
if self.layers[index].file.is_some() {
return Ok(());
}
let path = self.layers[index].path.clone();
let (file, applied) =
direct::open(&path, self.direct_open).map_err(|e| IoError::io(&path, e))?;
let size = file.metadata().map_err(|e| IoError::io(&path, e))?.len();
let state = &self.layers[index];
if size != state.expected_size {
return Err(FormatError::SizeMismatch {
name: state.name.clone(),
expected: state.expected_size,
actual: size,
}
.into());
}
if self.hash_layers {
let actual = sha256_file(&path)?;
let state = &self.layers[index];
if !actual.eq_ignore_ascii_case(&state.expected_sha256) {
return Err(FormatError::HashMismatch {
name: state.name.clone(),
expected: state.expected_sha256.clone(),
actual,
}
.into());
}
}
if self.hash_layers {
if let Err(error) = direct::fadvise_dontneed(&file, 0, 0) {
tracing::debug!(path = %path.display(), %error, "could not drop the layer file's page cache");
}
}
if self.direct_open && !applied {
tracing::warn!(path = %path.display(), "O_DIRECT refused for this layer file");
}
tracing::debug!(path = %path.display(), bytes = size, direct = applied, "layer file open");
self.layers[index].file = Some(file);
Ok(())
}
fn probe_direct_io(&mut self, strides: &[u64]) -> DirectSupport {
if !strides.iter().all(|stride| direct::is_aligned(*stride)) {
return DirectSupport::Unusable(DirectFault::UnalignedGeometry);
}
let Some(path) = self.layers.first().map(|state| state.path.clone()) else {
return DirectSupport::Unusable(DirectFault::Unsupported);
};
let Some(buf) = self.slots.bytes_mut(0, 0) else {
return DirectSupport::Unusable(DirectFault::BadProbeBuffer);
};
direct::probe(&path, buf)
}
fn pick_mode(&self) -> StreamMode {
#[cfg(feature = "io-uring")]
if self.ring.is_some() {
return if self.support.is_verified() {
StreamMode::ODirect
} else {
StreamMode::Buffered
};
}
StreamMode::Pread
}
fn submit(&mut self, index: usize) -> Result<(), IoError> {
#[cfg(feature = "io-uring")]
if self.ring.is_some() {
return self.push_sqe(index);
}
let _ = index;
Ok(())
}
fn flush(&mut self) -> Result<(), IoError> {
#[cfg(feature = "io-uring")]
if self.ring.is_some() && self.inflight.outstanding > 0 {
self.enter(0)?;
}
Ok(())
}
fn drive_reads(
&mut self,
until: Option<usize>,
failure: &mut Option<IoError>,
) -> Result<(), IoError> {
if self.inflight.outstanding == 0 {
return Ok(());
}
#[cfg(feature = "io-uring")]
if self.ring.is_some() {
self.await_ring(until, failure)?;
} else {
self.await_pread(until, failure);
}
#[cfg(not(feature = "io-uring"))]
self.await_pread(until, failure);
Ok(())
}
fn reached(&self, until: Option<usize>) -> bool {
until.is_some_and(|index| self.inflight.reads[index].done)
}
fn await_pread(&mut self, until: Option<usize>, failure: &mut Option<IoError>) {
let start = Instant::now();
for index in 0..self.inflight.reads.len() {
if self.reached(until) {
break;
}
while !self.inflight.reads[index].done {
let result = self.pread_once(index);
let user_data = self.inflight.reads[index].user_data;
match self.inflight.resolve(user_data, result, &mut self.io) {
Reap::Done(index) => self.mark_ready(index),
Reap::Again(index) => {
self.inflight.retag(index);
}
Reap::Failed(index, error) => {
let error = self.fail(index, error);
failure.get_or_insert(error);
}
Reap::Stale => {
self.io.stale_completions += 1;
self.inflight.abandon(index);
break;
}
}
}
}
self.io.add_wait(start.elapsed());
}
fn pread_once(&mut self, index: usize) -> i32 {
let read = self.inflight.reads[index];
let Some(file) = self.layers[read.layer as usize].file.as_ref() else {
return -libc::EBADF;
};
let from = read.filled as usize;
let to = from + read.remaining as usize;
let offset = read.base + u64::from(read.filled);
match read.dst {
Dest::Slot => {
let Some(slot) = self.slots.bytes_mut(read.layer, read.slot) else {
return -libc::EFAULT;
};
let Some(dst) = slot.get_mut(from..to) else {
return -libc::EFAULT;
};
self.io.add_submit();
pread_result(file, dst, offset)
}
Dest::Arena(base) => {
let Some(arena) = self.arena else {
return -libc::EFAULT;
};
if base.checked_add(to).is_none_or(|end| end > arena.len) {
return -libc::EFAULT;
}
let dst = unsafe {
std::slice::from_raw_parts_mut(
arena.base.as_ptr().add(base + from),
read.remaining as usize,
)
};
self.io.add_submit();
pread_result(file, dst, offset)
}
}
}
fn mark_ready(&mut self, index: usize) {
let read = self.inflight.reads[index];
if read.dst != Dest::Slot {
return;
}
if let Some(cache) = self.caches.get_mut(read.layer as usize) {
cache.mark_ready(read.slot);
}
}
fn fail(&mut self, index: usize, error: io::Error) -> IoError {
let read = self.inflight.reads[index];
tracing::error!(
layer = read.layer,
expert = read.expert,
slot = read.slot,
%error,
"expert blob read failed"
);
if read.dst == Dest::Slot {
if let Some(cache) = self.caches.get_mut(read.layer as usize) {
unsafe { cache.invalidate(read.slot) };
}
self.step.protected.retain(|slot| *slot != read.slot);
}
let path = self
.layers
.get(read.layer as usize)
.map(|state| state.path.clone())
.unwrap_or_default();
IoError::Io {
path,
source: error,
}
}
fn strand_step(&mut self) {
if self.arena.is_some() {
let _ = self.strand_arena();
return;
}
let mut retire: Vec<(u32, u32)> = Vec::new();
for index in 0..self.inflight.reads.len() {
let read = self.inflight.reads[index];
if read.done {
continue;
}
tracing::error!(
layer = read.layer,
expert = read.expert,
slot = read.slot,
"expert read cannot be reaped; retiring its slot"
);
self.inflight.abandon(index);
retire.push((read.layer, read.slot));
self.step.protected.retain(|slot| *slot != read.slot);
}
self.abandon_step();
retire.sort_unstable_by(|a, b| b.cmp(a));
for (layer, slot) in retire {
self.retire_slot(layer, slot);
}
}
fn retire_slot(&mut self, layer: u32, slot: u32) {
if self.slots.flat(layer, slot).is_none() {
return;
}
let usable = self.slots.usable(layer).saturating_sub(1);
let Some(cache) = self.caches.get(layer as usize) else {
self.poison_arena_unretired(layer, slot, "the layer has no cache");
return;
};
let (carried, n_experts) = (cache.stats(), cache.n_experts());
let fresh = match LayerCache::new(usable, n_experts) {
Ok(fresh) => fresh,
Err(error) => {
debug_assert!(false, "rebuilding a shrunken layer cache: {error}");
let reason = error.to_string();
self.poison_arena_unretired(layer, slot, &reason);
return;
}
};
self.slots.retire(layer, slot);
self.caches[layer as usize] = fresh;
self.retired = add_cache_stats(self.retired, carried);
tracing::error!(
layer,
slot,
usable,
of = self.slots.slots_per_layer,
"expert slot retired; the layer's cache is rebuilt smaller and \
loses its ghost history"
);
}
fn poison_arena_unretired(&mut self, layer: u32, slot: u32, reason: &str) {
self.arena_poisoned = true;
tracing::error!(
layer,
slot,
reason,
"could not retire a slot after a read that cannot be reaped; the \
slot stays leased and the layer keeps its old capacity, so this \
process will not sweep again"
);
}
fn abandon_step(&mut self) {
if !self.step.active {
return;
}
let layer = self.step.layer;
self.end_layer(layer);
}
#[cfg(feature = "io-uring")]
fn rebind_ring(&mut self) {
let current = std::thread::current().id();
if self.ring.is_none() || self.ring_owner == current {
return;
}
if self.inflight.outstanding > 0 {
tracing::error!(
outstanding = self.inflight.outstanding,
"the driving thread changed with reads in flight; the ring \
cannot be rebuilt under live DMA"
);
return;
}
self.ring_rebinds += 1;
let single_issuer = self.ring_rebinds < 2;
tracing::info!(
rebinds = self.ring_rebinds,
single_issuer,
"expert stream moved to another thread; rebuilding the ring"
);
self.ring = None;
self.ring = build_ring(single_issuer);
self.ring_owner = current;
self.mode = self.pick_mode();
}
#[cfg(test)]
fn strand_open_step(&mut self) {
self.strand_step();
}
#[cfg(test)]
pub(super) fn strand_open_arena(&mut self) -> Option<(u32, u32)> {
self.strand_arena()
}
#[cfg(test)]
pub(super) fn strand_open_arena_error(&mut self, source: IoError) -> SweepError {
self.strand_arena_error(source)
}
#[cfg(test)]
fn force_padded_slots(&mut self) {
self.slots = SlotTable::new(2, &[4096, 5000]).expect("a tiny pool builds");
}
#[cfg(all(test, feature = "io-uring"))]
fn stall_enters(&mut self, count: u32) {
self.stalled_enters = count;
}
#[cfg(all(test, feature = "io-uring"))]
fn force_cq_backlog(&mut self, count: usize) {
let ring = self.ring.as_mut().expect("ring");
for i in 0..count {
let nop = opcode::Nop::new()
.build()
.user_data(0xdead_beef_0000_0000 | i as u64);
while unsafe { ring.submission().push(&nop) }.is_err() {
let _ = ring.submitter().submit();
}
let _ = ring.submitter().submit();
}
std::thread::sleep(Duration::from_millis(5));
}
#[cfg(all(test, feature = "io-uring"))]
fn force_tiny_ring(&mut self) {
self.ring = None;
self.ring = IoUring::builder().build(1).ok();
self.ring_owner = std::thread::current().id();
assert!(self.ring.is_some(), "a one-entry ring should always build");
}
#[cfg(feature = "io-uring")]
fn push_sqe(&mut self, index: usize) -> Result<(), IoError> {
let read = self.inflight.reads[index];
let Some(file) = self.layers[read.layer as usize].file.as_ref() else {
return Err(self.protocol_error(read.layer, "layer file is not open"));
};
let fd = file.as_raw_fd();
let base = match read.dst {
Dest::Slot => match unsafe { self.slots.write_ptr(read.layer, read.slot) } {
Some(base) => base,
None => return Err(self.protocol_error(read.layer, "slot is not in the pool")),
},
Dest::Arena(offset) => {
let Some(arena) = self.arena else {
return Err(self.protocol_error(read.layer, "no prefill arena is out"));
};
if offset
.checked_add(read.filled as usize + read.remaining as usize)
.is_none_or(|end| end > arena.len)
{
return Err(self.protocol_error(read.layer, "window is outside the arena"));
}
unsafe { arena.base.as_ptr().add(offset) }
}
};
let dst = unsafe { base.add(read.filled as usize) };
debug_assert!(
read.dst == Dest::Slot
|| ((dst as usize as u64).is_multiple_of(direct::DIO_ALIGN)
&& (read.base + u64::from(read.filled)).is_multiple_of(direct::DIO_ALIGN)
&& u64::from(read.remaining) % direct::DIO_ALIGN == 0),
"a sweep read must be 4096-aligned in offset, length and destination"
);
let entry = opcode::Read::new(types::Fd(fd), dst, read.remaining)
.offset(read.base + u64::from(read.filled))
.build()
.user_data(read.user_data);
for _ in 0..MAX_PUSH_ATTEMPTS {
let pushed = match self.ring.as_mut() {
Some(ring) => unsafe { ring.submission().push(&entry) }.is_ok(),
None => return Err(self.protocol_error(read.layer, "the io_uring ring is gone")),
};
if pushed {
self.io.add_submit();
return Ok(());
}
if self.enter(0)? == 0 {
self.drain_cq();
}
}
Err(self.protocol_error(
read.layer,
"submission queue stayed full after submitting and draining completions",
))
}
#[cfg(feature = "io-uring")]
fn drain_cq(&mut self) {
let Some(ring) = self.ring.as_mut() else {
return;
};
let sink = &mut self.completions;
let mut cq = ring.completion();
cq.sync();
for cqe in &mut cq {
sink.push((cqe.user_data(), cqe.result()));
}
}
#[cfg(feature = "io-uring")]
fn enter(&mut self, want: usize) -> Result<usize, IoError> {
#[cfg(test)]
if self.stalled_enters > 0 {
self.stalled_enters -= 1;
return Ok(0);
}
let Some(ring) = self.ring.as_ref() else {
return Err(self.protocol_error(self.step.layer, "the io_uring ring is gone"));
};
let mut interrupted = 0u32;
loop {
match ring.submitter().submit_and_wait(want) {
Ok(submitted) => return Ok(submitted),
Err(error) if error.kind() == io::ErrorKind::Interrupted => {
interrupted += 1;
if interrupted > MAX_EINTR_RETRIES {
tracing::error!(
retries = interrupted,
"io_uring_enter interrupted on every attempt"
);
return Err(self.layer_error(error));
}
continue;
}
Err(error) if error.raw_os_error() == Some(libc::EBUSY) => return Ok(0),
Err(error) if error.raw_os_error() == Some(libc::EEXIST) => {
tracing::error!(
"io_uring rejected a submission from a thread other than \
the one that created the ring; one thread must drive a \
step from begin_layer to end_layer"
);
return Err(self.layer_error(error));
}
Err(error) => {
return Err(self.layer_error(error));
}
}
}
}
#[cfg(feature = "io-uring")]
fn await_ring(
&mut self,
until: Option<usize>,
failure: &mut Option<IoError>,
) -> Result<(), IoError> {
let mut idle = 0;
while self.inflight.outstanding > 0 && !self.reached(until) {
if self.completions.is_empty() {
let start = Instant::now();
self.enter(1)?;
self.io.add_wait(start.elapsed());
self.drain_cq();
}
if self.completions.is_empty() {
idle += 1;
if idle > MAX_IDLE_PASSES {
return Err(self.protocol_error(
self.step.layer,
"no completions after repeated waits; the ring is wedged",
));
}
continue;
}
idle = 0;
std::mem::swap(&mut self.completions, &mut self.reaping);
for i in 0..self.reaping.len() {
let (user_data, result) = self.reaping[i];
match self.inflight.resolve(user_data, result, &mut self.io) {
Reap::Done(index) => self.mark_ready(index),
Reap::Again(index) => {
self.inflight.retag(index);
if let Err(error) = self.push_sqe(index) {
self.inflight.abandon(index);
let error = self.fail(index, flatten_io(error));
failure.get_or_insert(error);
}
}
Reap::Failed(index, error) => {
let error = self.fail(index, error);
failure.get_or_insert(error);
}
Reap::Stale => {
self.io.stale_completions += 1;
tracing::error!(
user_data,
result,
"stale io_uring completion: no live read carries this tag"
);
}
}
}
self.reaping.clear();
self.flush()?;
}
Ok(())
}
}
impl Drop for ExpertStream {
fn drop(&mut self) {
if self.inflight.outstanding == 0 {
return;
}
tracing::error!(
outstanding = self.inflight.outstanding,
"expert stream dropped with reads in flight; leaking their slots"
);
let arena = self.arena_len();
let sweeping = arena > 0
&& self
.inflight
.reads
.iter()
.any(|read| !read.done && matches!(read.dst, Dest::Arena(_)));
let mut stranded: Vec<(u32, u32)> = self
.inflight
.reads
.iter()
.filter(|read| !read.done && read.dst == Dest::Slot)
.map(|read| (read.layer, read.slot))
.collect();
if sweeping {
for layer in 0..self.slots.live.len() as u32 {
for slot in 0..self.slots.usable(layer) {
if self
.slots
.slot_offset(layer, slot)
.is_some_and(|offset| offset < arena)
{
stranded.push((layer, slot));
}
}
}
}
stranded.sort_unstable_by(|a, b| b.cmp(a));
stranded.dedup();
for (layer, slot) in stranded {
self.slots.retire(layer, slot);
}
}
}
fn flatten_io(error: IoError) -> io::Error {
match error {
IoError::Io { source, .. } => source,
other => io::Error::other(other.to_string()),
}
}
fn pread_result(file: &File, dst: &mut [u8], offset: u64) -> i32 {
match file.read_at(dst, offset) {
Ok(got) => i32::try_from(got).unwrap_or(i32::MAX),
Err(error) => -error.raw_os_error().unwrap_or(libc::EIO),
}
}
fn add_cache_stats(a: CacheStats, b: CacheStats) -> CacheStats {
CacheStats {
hits: a.hits + b.hits,
pending_hits: a.pending_hits + b.pending_hits,
misses: a.misses + b.misses,
cold_misses: a.cold_misses + b.cold_misses,
eviction_misses: a.eviction_misses + b.eviction_misses,
evictions: a.evictions + b.evictions,
}
}
#[cfg(feature = "io-uring")]
fn completion_capacity(slots_per_layer: u32) -> usize {
(2 * RING_ENTRIES as usize).max(2 * slots_per_layer as usize)
}
pub(super) fn bytes_per_slot(strides: &[u64]) -> Result<u128, IoError> {
if strides.is_empty() {
return Err(SlotError::NoLayers.into());
}
let mut per_slot: u128 = 0;
for &stride in strides {
let pitch =
stride
.checked_next_multiple_of(SLOT_ALIGN as u64)
.ok_or(IoError::TooLarge {
what: "expert blob stride",
value: stride,
})?;
per_slot += u128::from(pitch);
}
if per_slot == 0 {
return Err(SlotError::ZeroStride { layer: 0 }.into());
}
Ok(per_slot)
}
pub(super) fn slots_for_budget(
cache_bytes: u64,
strides: &[u64],
n_experts: &[u32],
top_k: u32,
) -> Result<u32, IoError> {
let per_slot = bytes_per_slot(strides)?;
let slots = u32::try_from(u128::from(cache_bytes) / per_slot).unwrap_or(u32::MAX);
if slots == 0 {
tracing::error!(
cache_bytes,
bytes_per_slot = %per_slot,
n_layers = strides.len(),
"expert cache budget cannot fit one slot per layer"
);
return Err(SlotError::ZeroSlotsPerLayer.into());
}
let usable = n_experts
.iter()
.copied()
.min()
.unwrap_or(MAX_SLOTS)
.clamp(1, MAX_SLOTS);
let chosen = if slots > usable {
tracing::info!(
budget_slots = slots,
slots_per_layer = usable,
"expert cache budget buys more slots than a layer can use; clamping"
);
usable
} else {
slots
};
let floor = top_k.max(1);
if chosen < floor {
let needed = per_slot * u128::from(floor);
tracing::error!(
cache_bytes,
bytes_per_slot = %per_slot,
slots = chosen,
top_k = floor,
minimum_bytes = %needed,
"expert cache budget cannot fit one slot per routed expert"
);
return Err(IoError::CacheBudgetTooSmall {
given: cache_bytes,
needed: u64::try_from(needed).unwrap_or(u64::MAX),
slots: chosen,
top_k: floor,
});
}
Ok(chosen)
}
fn position_of(experts: &[u32], expert: u32) -> usize {
experts.iter().position(|&id| id == expert).unwrap_or(0)
}
#[cfg(feature = "io-uring")]
fn build_ring(single_issuer: bool) -> Option<IoUring> {
let first = if single_issuer { 0 } else { 2 };
for level in first..4 {
let mut builder = IoUring::builder();
let flags = match level {
0 => {
builder.setup_single_issuer().setup_defer_taskrun();
"single_issuer+defer_taskrun"
}
1 => {
builder
.setup_single_issuer()
.setup_coop_taskrun()
.setup_taskrun_flag();
"single_issuer+coop_taskrun+taskrun_flag"
}
2 => {
builder.setup_coop_taskrun().setup_taskrun_flag();
"coop_taskrun+taskrun_flag"
}
_ => "none",
};
match builder.build(RING_ENTRIES) {
Ok(ring) => {
tracing::info!(entries = RING_ENTRIES, flags, "io_uring ready");
return Some(ring);
}
Err(error) => {
tracing::debug!(flags, %error, "io_uring setup refused, dropping flags");
}
}
}
tracing::warn!("io_uring unavailable; expert reads fall back to pread");
None
}
#[cfg(test)]
mod tests {
use super::super::testutil::{Fixture, build_install};
use super::*;
use crate::io::CacheError;
fn slot_row(fx: &Fixture) -> u64 {
fx.layout
.layers
.iter()
.map(|layer| layer.stride.next_multiple_of(SLOT_ALIGN as u64))
.sum()
}
fn open(fx: &Fixture, slots: u32) -> ExpertStream {
open_with(fx, slots, LoadOptions::default()).expect("stream opens")
}
fn open_with(fx: &Fixture, slots: u32, options: LoadOptions) -> Result<ExpertStream, IoError> {
ExpertStream::new(
&fx.root,
&fx.manifest,
&fx.layout,
slot_row(fx) * u64::from(slots),
options,
)
}
fn blob_on_disk(fx: &Fixture, layer: u32, expert: u32) -> Vec<u8> {
let layout = &fx.layout.layers[layer as usize];
let bytes = std::fs::read(fx.root.join(&layout.file)).unwrap();
let base = (u64::from(expert) * layout.stride) as usize;
bytes[base..base + layout.stride as usize].to_vec()
}
#[test]
fn stream_is_send() {
fn assert_send<T: Send>() {}
assert_send::<ExpertStream>();
}
#[test]
fn budget_dial_converts_bytes_to_slots() {
let fx = build_install("stream-budget");
let row = slot_row(&fx);
for slots in fx.manifest.arch.top_k..=3u32 {
let stream = open(&fx, slots);
assert_eq!(stream.slots_per_layer(), slots);
let extra = ExpertStream::new(
&fx.root,
&fx.manifest,
&fx.layout,
row * u64::from(slots) + row / 2,
LoadOptions::default(),
)
.unwrap();
assert_eq!(extra.slots_per_layer(), slots);
assert_eq!(stream.cache_bytes(), row * u64::from(slots));
assert!(stream.cache_bytes() <= row * u64::from(slots));
}
let err = ExpertStream::new(
&fx.root,
&fx.manifest,
&fx.layout,
row - 1,
LoadOptions::default(),
)
.unwrap_err();
assert!(
matches!(err, IoError::Slots(SlotError::ZeroSlotsPerLayer)),
"unexpected error: {err}"
);
let huge = open(&fx, 64);
assert_eq!(huge.slots_per_layer(), fx.layout.layers[0].n_experts);
}
#[test]
fn a_budget_below_top_k_is_rejected_at_construction() {
let fx = build_install("stream-budget-floor");
let row = slot_row(&fx);
let top_k = fx.manifest.arch.top_k;
assert!(top_k > 1, "fixture must route more than one expert");
for slots in 1..top_k {
let err = ExpertStream::new(
&fx.root,
&fx.manifest,
&fx.layout,
row * u64::from(slots),
LoadOptions::default(),
)
.unwrap_err();
match err {
IoError::CacheBudgetTooSmall {
given,
needed,
slots: bought,
top_k: k,
} => {
assert_eq!(given, row * u64::from(slots));
assert_eq!(bought, slots);
assert_eq!(k, top_k);
assert_eq!(needed, row * u64::from(top_k));
assert!(
err.to_string().contains(&needed.to_string()),
"the error must name the minimum budget: {err}"
);
}
other => panic!("unexpected error: {other}"),
}
}
let ok = open(&fx, top_k);
assert_eq!(ok.slots_per_layer(), top_k);
let err = ExpertStream::new(
&fx.root,
&fx.manifest,
&fx.layout,
row - 1,
LoadOptions::default(),
)
.unwrap_err();
assert!(
matches!(err, IoError::Slots(SlotError::ZeroSlotsPerLayer)),
"unexpected error: {err}"
);
}
#[test]
fn miss_reads_the_blob_and_a_second_step_hits() {
let fx = build_install("stream-miss-hit");
let mut stream = open(&fx, 2);
let request = [1u32, 3];
stream.begin_layer(0, &request).unwrap();
assert!(stream.hits().is_empty(), "cold cache reported a hit");
let misses = stream.misses().to_vec();
assert_eq!(misses.len(), 2);
stream.await_misses().unwrap();
for &(index, slot) in &misses {
let expert = request[index];
let view = stream.view(0, slot).unwrap();
let expected = blob_on_disk(&fx, 0, expert);
assert_eq!(view.blob(), expected, "expert {expert} blob mismatch");
for projection in &fx.layout.layers[0].projections {
let from = projection.offset_in_blob as usize;
let to = from + projection.len as usize;
assert_eq!(
view.slab(projection.name).bytes,
&expected[from..to],
"expert {expert} {:?} slab",
projection.name
);
}
}
stream.end_layer(0);
let after_first = stream.stats();
assert_eq!(after_first.misses, 2);
assert_eq!(after_first.cold_misses, 2);
assert_eq!(
after_first.bytes_read,
2 * fx.layout.layers[0].stride,
"read more or fewer bytes than the two blobs"
);
assert_eq!(after_first.reads_submitted, 2);
assert_eq!(after_first.read_retries, 0);
stream.begin_layer(0, &request).unwrap();
assert_eq!(stream.misses().len(), 0, "a resident expert was re-read");
assert_eq!(stream.hits().len(), 2);
stream.await_misses().unwrap();
for &(index, slot) in stream.hits().to_vec().iter() {
let view = stream.view(0, slot).unwrap();
assert_eq!(view.blob(), blob_on_disk(&fx, 0, request[index]));
}
stream.end_layer(0);
let after_second = stream.stats();
assert_eq!(after_second.hits, 2);
assert_eq!(after_second.misses, 2, "a hit was counted as a miss");
assert_eq!(after_second.bytes_read, after_first.bytes_read);
assert_eq!(after_second.reads_submitted, after_first.reads_submitted);
}
#[test]
fn every_slot_of_a_batch_gets_its_own_blob() {
let fx = build_install("stream-batch");
let mut stream = open(&fx, 4);
let request = [3u32, 0, 2, 1];
stream.begin_layer(1, &request).unwrap();
assert_eq!(stream.misses().len(), 4);
let misses = stream.misses().to_vec();
stream.await_misses().unwrap();
let mut slots_seen = Vec::new();
for &(index, slot) in &misses {
let expert = request[index];
assert_eq!(
stream.view(1, slot).unwrap().blob(),
blob_on_disk(&fx, 1, expert)
);
slots_seen.push(slot);
}
slots_seen.sort_unstable();
slots_seen.dedup();
assert_eq!(slots_seen.len(), 4, "two experts shared a slot");
stream.end_layer(1);
}
#[cfg(feature = "io-uring")]
#[test]
fn a_step_wider_than_the_submission_queue_still_completes() {
let fx = build_install("stream-tiny-ring");
let mut stream = open(&fx, 4);
stream.force_tiny_ring();
let request = [0u32, 1, 2, 3];
stream.begin_layer(1, &request).unwrap();
assert_eq!(stream.misses().len(), 4);
let misses = stream.misses().to_vec();
stream.await_misses().unwrap();
for &(index, slot) in &misses {
assert_eq!(
stream.view(1, slot).unwrap().blob(),
blob_on_disk(&fx, 1, request[index])
);
}
stream.end_layer(1);
assert_eq!(stream.stats().reads_submitted, 4);
assert_eq!(stream.stale_completions(), 0);
}
#[cfg(feature = "io-uring")]
#[test]
fn a_full_completion_queue_is_drained_rather_than_reported_as_fatal() {
const BACKLOG: usize = 6;
const STALLS: u32 = 3;
let fx = build_install("stream-cq-overflow");
let mut stream = open(&fx, 4);
stream.force_tiny_ring();
stream.force_cq_backlog(BACKLOG);
stream.stall_enters(STALLS);
let request = [0u32, 1, 2, 3];
stream.begin_layer(1, &request).expect("step submits");
assert_eq!(stream.misses().len(), 4);
let misses = stream.misses().to_vec();
stream.await_misses().expect("step completes");
for &(index, slot) in &misses {
assert_eq!(
stream.view(1, slot).unwrap().blob(),
blob_on_disk(&fx, 1, request[index]),
"expert {} came back wrong through a full CQ",
request[index]
);
}
stream.end_layer(1);
assert_eq!(stream.stats().reads_submitted, 4);
assert_eq!(stream.stale_completions(), BACKLOG as u64);
assert_eq!(stream.usable_slots(1), 4);
stream.begin_layer(1, &request).unwrap();
assert_eq!(stream.hits().len(), 4);
stream.await_misses().unwrap();
stream.end_layer(1);
}
#[test]
fn a_retired_slot_is_never_handed_out_again() {
let fx = build_install("stream-retire");
let mut stream = open(&fx, 4);
let before = stream.usable_slots(0);
assert_eq!(before, 4);
stream.begin_layer(0, &[3]).unwrap();
assert_eq!(stream.misses().len(), 1);
stream.strand_open_step();
assert_eq!(stream.usable_slots(0), before - 1);
assert_eq!(stream.usable_slots(1), before);
for step in 0..64u32 {
let request = [step % 4, (step / 4 + 1) % 4];
let request: Vec<u32> = if request[0] == request[1] {
vec![request[0]]
} else {
request.to_vec()
};
stream
.begin_layer(0, &request)
.unwrap_or_else(|e| panic!("step {step} refused after a retirement: {e}"));
stream.await_misses().unwrap();
for &(index, slot) in stream.misses().to_vec().iter() {
assert_eq!(
stream.view(0, slot).unwrap().blob(),
blob_on_disk(&fx, 0, request[index]),
"step {step} served the wrong blob after a retirement"
);
}
for &(index, slot) in stream.hits().to_vec().iter() {
assert_eq!(
stream.view(0, slot).unwrap().blob(),
blob_on_disk(&fx, 0, request[index]),
"step {step} hit the wrong blob after a retirement"
);
}
stream.end_layer(0);
}
let stats = stream.stats();
assert!(stats.misses >= 1, "the stranded step's miss was dropped");
assert_eq!(
stats.hits + stats.pending_hits + stats.misses,
1 + 64 * 2 - 16
);
}
#[test]
fn retiring_a_layer_down_to_nothing_is_a_typed_error_not_a_wedge() {
let fx = build_install("stream-retire-all");
let mut stream = open(&fx, 4);
for expert in 0..4u32 {
if stream.usable_slots(0) == 0 {
break;
}
stream.begin_layer(0, &[expert]).unwrap();
stream.strand_open_step();
}
assert_eq!(stream.usable_slots(0), 0);
let err = stream.begin_layer(0, &[0]).unwrap_err();
assert!(
matches!(
err,
IoError::Cache(CacheError::TooFewSlots { n_slots: 0, .. })
),
"unexpected error: {err}"
);
assert!(matches!(
stream.begin_layer(0, &[0]).unwrap_err(),
IoError::Cache(CacheError::TooFewSlots { n_slots: 0, .. })
));
stream.begin_layer(1, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(1);
}
#[test]
fn end_layer_without_awaiting_retires_rather_than_wedging() {
let fx = build_install("stream-end-without-await");
let mut stream = open(&fx, 4);
stream.begin_layer(1, &[2]).unwrap();
stream.end_layer(1);
assert_eq!(stream.usable_slots(1), 3);
for expert in [0u32, 1, 3] {
stream
.begin_layer(1, &[expert])
.expect("layer still serves");
stream.await_misses().unwrap();
let slot = stream.misses()[0].1;
assert_eq!(
stream.view(1, slot).unwrap().blob(),
blob_on_disk(&fx, 1, expert)
);
stream.end_layer(1);
}
}
#[test]
fn phase_split_separates_prefill_from_decode() {
let fx = build_install("stream-phases");
let mut stream = open(&fx, 4);
assert_eq!(stream.phase(), StreamPhase::Prefill);
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
let prefill = stream.stats_in(StreamPhase::Prefill);
assert_eq!(prefill.misses, 2);
assert_eq!(prefill.cold_misses, 2);
assert_eq!(prefill.hits, 0);
assert_eq!(stream.stats_in(StreamPhase::Decode), StreamStats::default());
stream.set_phase(StreamPhase::Decode);
assert_eq!(stream.phase(), StreamPhase::Decode);
stream.set_phase(StreamPhase::Decode);
for _ in 0..3 {
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
}
let decode = stream.stats_in(StreamPhase::Decode);
assert_eq!(decode.hits, 6);
assert_eq!(decode.misses, 0);
assert_eq!(decode.hit_rate(), 1.0);
assert_eq!(decode.bytes_read, 0, "decode read a resident expert");
assert_eq!(stream.stats_in(StreamPhase::Prefill), prefill);
let total = stream.stats();
assert_eq!(prefill.plus(&decode), total);
assert!(
total.hit_rate() < decode.hit_rate(),
"cumulative {} vs decode {}",
total.hit_rate(),
decode.hit_rate()
);
stream.set_phase(StreamPhase::Prefill);
stream.begin_layer(1, &[2]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(1);
assert_eq!(stream.stats_in(StreamPhase::Prefill).misses, 3);
assert_eq!(stream.stats_in(StreamPhase::Decode), decode);
}
#[test]
fn layers_are_independent() {
let fx = build_install("stream-layers");
let mut stream = open(&fx, 2);
for layer in 0..2u32 {
stream.begin_layer(layer, &[0, 1]).unwrap();
stream.await_misses().unwrap();
for &(index, slot) in stream.misses().to_vec().iter() {
let expert = [0u32, 1][index];
assert_eq!(
stream.view(layer, slot).unwrap().blob(),
blob_on_disk(&fx, layer, expert)
);
}
stream.end_layer(layer);
}
stream.begin_layer(0, &[0, 1]).unwrap();
assert_eq!(stream.misses().len(), 0);
stream.end_layer(0);
}
#[test]
fn end_layer_releases_everything_over_a_long_run() {
let fx = build_install("stream-longrun");
let mut stream = open(&fx, 2);
for step in 0..200u32 {
let a = step % 4;
let b = (step / 4 + 1) % 4;
let request = if a == b { vec![a] } else { vec![a, b] };
stream.begin_layer(step % 2, &request).unwrap();
stream.await_misses().unwrap();
for &(_, slot) in stream.misses().to_vec().iter() {
stream.view(step % 2, slot).unwrap();
}
stream.end_layer(step % 2);
}
let stats = stream.stats();
assert_eq!(stats.hits + stats.misses + stats.pending_hits, 350);
assert!(stats.hits > 0, "no hit in 200 steps over 4 experts");
assert_eq!(stats.pending_hits, 0, "a step left a read in flight");
assert_eq!(stats.read_retries, 0);
let small = fx.layout.layers[0].stride.min(fx.layout.layers[1].stride);
let large = fx.layout.layers[0].stride.max(fx.layout.layers[1].stride);
assert!((stats.misses * small..=stats.misses * large).contains(&stats.bytes_read));
assert_eq!(stats.reads_submitted, stats.misses);
}
#[test]
fn rejects_bad_layers_experts_and_call_order() {
let fx = build_install("stream-bounds");
let mut stream = open(&fx, 2);
assert!(matches!(
stream.begin_layer(7, &[0]).unwrap_err(),
IoError::LayerOutOfRange { layer: 7, .. }
));
assert!(matches!(
stream.begin_layer(0, &[9]).unwrap_err(),
IoError::Cache(CacheError::ExpertOutOfRange { expert: 9, .. })
));
assert!(matches!(
stream.begin_layer(0, &[0, 1, 2]).unwrap_err(),
IoError::Cache(CacheError::TooFewSlots { .. })
));
stream.begin_layer(0, &[0]).unwrap();
let err = stream.begin_layer(0, &[1]).unwrap_err();
assert!(matches!(err, IoError::Io { .. }), "unexpected error: {err}");
stream.await_misses().unwrap();
stream.end_layer(0);
assert!(stream.view(0, 1).is_err());
assert!(stream.view(9, 0).is_err());
}
#[test]
fn duplicate_ids_resolve_once() {
let fx = build_install("stream-dupes");
let mut stream = open(&fx, 2);
stream.begin_layer(0, &[2, 2]).unwrap();
assert_eq!(stream.misses().len(), 1);
assert_eq!(stream.misses()[0].0, 0, "the first occurrence should win");
stream.await_misses().unwrap();
stream.end_layer(0);
assert_eq!(stream.stats().misses, 1);
}
#[test]
fn short_file_is_an_error_not_a_silent_short_read() {
let fx = build_install("stream-eof");
let mut stream = open(&fx, 2);
stream.begin_layer(0, &[0]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
let path = fx.root.join(&fx.layout.layers[0].file);
let bytes = std::fs::read(&path).unwrap();
std::fs::write(&path, &bytes[..bytes.len() / 2]).unwrap();
stream.begin_layer(0, &[3]).unwrap();
let err = stream.await_misses().unwrap_err();
match &err {
IoError::Io { source, .. } => assert!(
source.kind() == io::ErrorKind::UnexpectedEof
|| source.raw_os_error().is_some_and(|e| e == libc::EINVAL),
"unexpected source: {source:?}"
),
other => panic!("unexpected error: {other}"),
}
stream.end_layer(0);
std::fs::write(&path, &bytes).unwrap();
stream.begin_layer(0, &[0]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
}
#[test]
fn layer_hashes_are_checked_only_when_asked_for() {
let fx = build_install("stream-hash");
let victim = fx.root.join(&fx.layout.layers[1].file);
let mut bytes = std::fs::read(&victim).unwrap();
bytes[4096] ^= 0xff;
std::fs::write(&victim, &bytes).unwrap();
let mut strict = open_with(
&fx,
2,
LoadOptions {
skip_hashes: false,
verify_layer_hashes: true,
common_memory: crate::io::CommonMemory::Mapped,
},
)
.unwrap();
assert!(matches!(
strict.begin_layer(1, &[0]).unwrap_err(),
IoError::Format(FormatError::HashMismatch { .. })
));
strict.begin_layer(0, &[0]).unwrap();
strict.await_misses().unwrap();
strict.end_layer(0);
let mut lax = open(&fx, 2);
lax.begin_layer(1, &[0]).unwrap();
lax.await_misses().unwrap();
lax.end_layer(1);
std::fs::write(&victim, &bytes[..bytes.len() - 4096]).unwrap();
let mut short = open(&fx, 2);
assert!(matches!(
short.begin_layer(1, &[0]).unwrap_err(),
IoError::Format(FormatError::SizeMismatch { .. })
));
}
#[test]
fn a_stream_can_be_driven_from_the_thread_it_was_moved_to() {
let fx = build_install("stream-moved");
let mut stream = open(&fx, 2);
for expert in [2u32, 1, 3] {
let expected = blob_on_disk(&fx, 1, expert);
let handle = std::thread::spawn(move || {
let mut stream = stream;
stream.begin_layer(1, &[expert]).unwrap();
stream.await_misses().unwrap();
let slot = stream.misses()[0].1;
let blob = stream.view(1, slot).unwrap().blob().to_vec();
stream.end_layer(1);
(stream, blob)
});
let (moved, blob) = handle.join().unwrap();
assert_eq!(blob, expected, "expert {expert} after a thread move");
stream = moved;
}
assert_eq!(stream.stats().misses, 3);
assert_eq!(stream.stale_completions(), 0);
}
#[test]
fn probe_reports_a_sane_mode() {
let fx = build_install("stream-mode");
let stream = open(&fx, 2);
let mode = stream.mode();
println!(
"mode={mode} direct_io={} support={}",
stream.direct_io(),
stream.direct_support()
);
assert_eq!(
mode == StreamMode::ODirect,
stream.direct_io() && cfg!(feature = "io-uring")
);
if cfg!(not(feature = "io-uring")) {
assert_eq!(mode, StreamMode::Pread);
}
assert_eq!(stream.direct_io(), stream.direct_support().is_verified());
assert_eq!(stream.stale_completions(), 0);
}
#[test]
fn the_per_step_buffers_are_preallocated() {
let fx = build_install("stream-prealloc");
let stream = open(&fx, 4);
let top_k = fx.manifest.arch.top_k as usize;
assert!(top_k >= 2, "the fixture must route more than one expert");
assert!(stream.step.hits.capacity() >= top_k);
assert!(stream.step.misses.capacity() >= top_k);
assert!(stream.step.protected.capacity() >= top_k);
assert!(stream.inflight.reads.capacity() >= top_k);
assert!(stream.plan.capacity() >= top_k);
assert_eq!(CachePlan::new().capacity(), 0);
let request: Vec<u32> = (0..top_k as u32).collect();
let mut stream = stream;
let before = (
stream.step.hits.capacity(),
stream.step.misses.capacity(),
stream.step.protected.capacity(),
stream.inflight.reads.capacity(),
);
let plan_before = stream.plan.capacity();
stream.begin_layer(0, &request).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
assert_eq!(
before,
(
stream.step.hits.capacity(),
stream.step.misses.capacity(),
stream.step.protected.capacity(),
stream.inflight.reads.capacity(),
),
"a top-k step reallocated a preallocated buffer"
);
assert_eq!(
stream.plan.capacity(),
plan_before,
"a top-k step regrew the cache plan"
);
}
#[test]
fn the_arena_carve_is_page_aligned_and_allocates_nothing() {
let fx = build_install("stream-arena-carve");
let mut stream = open(&fx, 4);
let pool_bytes = stream.cache_bytes();
let want = 2 * fx.layout.layers[0].stride as usize;
stream.take_arena(want).unwrap();
let arena = stream.arena.expect("the arena is out");
assert_eq!(arena.len, want);
assert!(direct::is_aligned(arena.base.as_ptr() as usize as u64));
assert!(direct::is_aligned(arena.len as u64));
assert!(arena.len as u64 <= pool_bytes, "the carve left the slab");
assert_eq!(stream.cache_bytes(), pool_bytes);
assert_eq!(stream.arena_len(), want);
stream.release_arena();
assert!(stream.arena.is_none());
assert_eq!(stream.arena_len(), 0);
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
let err = stream.take_arena(pool_bytes as usize + 4096).unwrap_err();
assert!(
matches!(err, SweepError::ArenaTooSmall { .. }),
"unexpected error: {err}"
);
}
#[test]
fn the_arena_is_refused_while_the_cache_is_busy() {
let fx = build_install("stream-arena-busy");
let mut stream = open(&fx, 4);
stream.begin_layer(0, &[0, 1]).unwrap();
assert_eq!(stream.misses().len(), 2, "the fixture must miss cold");
let err = stream.take_arena(4096).unwrap_err();
assert!(
matches!(err, SweepError::ReadsInFlight { outstanding: 2 }),
"unexpected error: {err}"
);
stream.await_misses().unwrap();
let err = stream.take_arena(4096).unwrap_err();
assert!(
matches!(err, SweepError::StepOpen { layer: 0 }),
"unexpected error: {err}"
);
stream.end_layer(0);
stream
.take_arena(4096)
.expect("a closed step lets it through");
stream.release_arena();
}
#[test]
fn taking_the_arena_empties_every_layers_occupancy() {
let fx = build_install("stream-arena-invalidate");
let mut stream = open(&fx, 4);
for layer in 0..2u32 {
stream.begin_layer(layer, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(layer);
}
for layer in 0..2u32 {
assert!(
(0..stream.usable_slots(layer))
.any(|slot| stream.caches[layer as usize].is_ready(slot)),
"layer {layer} should hold something before the arena"
);
}
let hits_before = stream.stats().hits;
stream.take_arena(4096).unwrap();
for layer in 0..2u32 {
for slot in 0..stream.usable_slots(layer) {
assert!(
!stream.caches[layer as usize].is_ready(slot),
"layer {layer} slot {slot} survived the arena"
);
}
}
stream.release_arena();
assert_eq!(stream.stats().hits, hits_before);
stream.begin_layer(0, &[0, 1]).unwrap();
assert!(stream.hits().is_empty());
stream.await_misses().unwrap();
stream.end_layer(0);
assert_eq!(stream.stats().eviction_misses, 2);
}
#[test]
fn a_padded_slot_pool_refuses_the_arena() {
let fx = build_install("stream-arena-padded");
let mut stream = open(&fx, 4);
stream.take_arena(4096).expect("the fixture is gapless");
stream.release_arena();
stream.force_padded_slots();
let err = stream.take_arena(4096).unwrap_err();
match err {
SweepError::PaddedSlots {
layer,
stride,
pitch,
} => {
assert_eq!((layer, stride, pitch), (1, 5000, 8192));
}
other => panic!("unexpected error: {other}"),
}
}
#[test]
fn a_lost_window_read_retires_the_slots_the_arena_covered() {
let fx = build_install("stream-arena-strand");
let mut stream = open(&fx, 4);
let stride = fx.layout.layers[0].stride as usize;
assert_eq!(stream.usable_slots(0), 4);
assert_eq!(stream.usable_slots(1), 4);
stream.take_arena(2 * stride).unwrap();
assert_eq!(
stream.strand_arena(),
None,
"two of four slots is still top_k of 2"
);
assert_eq!(stream.usable_slots(0), 2, "the arena's slots stayed live");
assert_eq!(stream.usable_slots(1), 4, "an untouched layer shrank");
assert!(stream.arena.is_none());
let err = stream.take_arena(4096).unwrap_err();
assert!(
matches!(err, SweepError::ArenaPoisoned),
"unexpected error: {err}"
);
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
}
#[test]
fn the_arena_refuses_to_cover_a_retired_slot_buffer() {
let fx = build_install("stream-arena-over-retired");
let mut stream = open(&fx, 4);
let stride0 = fx.layout.layers[0].stride as usize;
stream.begin_layer(1, &[3]).unwrap();
stream.strand_open_step();
assert_eq!(stream.usable_slots(1), 3);
stream
.take_arena(2 * stride0)
.expect("layer 1's retirement is past this carve");
stream.release_arena();
stream.begin_layer(0, &[3]).unwrap();
stream.strand_open_step();
match stream.take_arena(2 * stride0).unwrap_err() {
SweepError::ArenaOverRetired {
layer,
offset,
bytes,
} => {
assert_eq!(layer, 0);
assert_eq!(bytes, 2 * stride0);
assert!(offset < bytes, "the refusal named a buffer outside it");
}
other => panic!("unexpected error: {other}"),
}
assert!(matches!(
stream.take_arena(4096).unwrap_err(),
SweepError::ArenaOverRetired { .. }
));
let mut plan = crate::io::SweepPlan::new();
assert!(matches!(
stream
.sweep_layer(&mut plan, 0, &[0], super::super::SweepConfig::default())
.unwrap_err(),
SweepError::ArenaOverRetired { .. }
));
let small = super::super::SweepConfig {
experts_per_window: 1,
windows_in_flight: 1,
};
assert!(matches!(
stream.begin_prefill(4096, small).unwrap_err(),
SweepError::ArenaOverRetired { .. }
));
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
}
#[test]
fn stranding_the_arena_below_top_k_is_named_at_its_cause() {
let fx = build_install("stream-arena-strand-top-k");
let top_k = fx.manifest.arch.top_k;
let stride0 = fx.layout.layers[0].stride as usize;
let lost = || IoError::io(Path::new("layer_00.bin"), io::Error::other("ring failed"));
let mut stream = open(&fx, 4);
stream.take_arena(4 * stride0).unwrap();
match stream.strand_open_arena_error(lost()) {
SweepError::CacheStranded {
layer,
slots,
top_k: k,
..
} => assert_eq!((layer, slots, k), (0, 0, top_k)),
other => panic!("unexpected error: {other}"),
}
assert_eq!(stream.usable_slots(0), 0);
assert_eq!(stream.usable_slots(1), 4, "an untouched layer shrank");
assert!(matches!(
stream.begin_layer(0, &[0]).unwrap_err(),
IoError::Cache(CacheError::TooFewSlots { .. })
));
let mut stream = open(&fx, 4);
stream.take_arena(2 * stride0).unwrap();
let err = stream.strand_open_arena_error(lost());
assert!(matches!(err, SweepError::Io(_)), "unexpected error: {err}");
assert_eq!(stream.usable_slots(0), 4 - 2);
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
}
fn tracked(inflight: &mut Inflight, slots: &[u32]) -> Vec<u64> {
slots
.iter()
.enumerate()
.map(|(i, &slot)| {
let index = inflight.track(0, slot, i as u32, u64::from(slot) * 4096, 4096);
inflight.reads[index].user_data
})
.collect()
}
#[test]
fn completions_are_dispatched_out_of_order_by_user_data() {
let mut io = IoStats::default();
let mut inflight = Inflight::default();
let tokens = tracked(&mut inflight, &[5, 2, 7]);
assert_eq!(inflight.outstanding, 3);
for &i in &[2usize, 0, 1] {
match inflight.resolve(tokens[i], 4096, &mut io) {
Reap::Done(index) => {
assert_eq!(index, i, "completion credited to the wrong read");
assert_eq!(inflight.reads[index].slot, [5u32, 2, 7][i]);
}
other => panic!("expected Done, got {other:?}"),
}
}
assert_eq!(inflight.outstanding, 0);
assert_eq!(io.bytes_read, 3 * 4096);
}
#[test]
fn stale_completions_are_rejected() {
let mut io = IoStats::default();
let mut inflight = Inflight::default();
let tokens = tracked(&mut inflight, &[1, 4]);
assert!(matches!(inflight.resolve(9, 4096, &mut io), Reap::Stale));
let forged = (tokens[0] & 0xffff_ffff) | (0xdead_u64 << 32);
assert!(matches!(
inflight.resolve(forged, 4096, &mut io),
Reap::Stale
));
assert!(matches!(
inflight.resolve(tokens[1], 4096, &mut io),
Reap::Done(1)
));
assert!(matches!(
inflight.resolve(tokens[1], 4096, &mut io),
Reap::Stale
));
assert!(matches!(
inflight.resolve(tokens[0], -libc::EIO, &mut io),
Reap::Again(0)
));
let reissued = inflight.retag(0);
assert_ne!(reissued, tokens[0]);
assert!(matches!(
inflight.resolve(tokens[0], 4096, &mut io),
Reap::Stale
));
assert_eq!(io.bytes_read, 4096, "a stale completion moved the counters");
assert_eq!(inflight.outstanding, 1);
assert!(matches!(
inflight.resolve(reissued, 4096, &mut io),
Reap::Done(0)
));
assert_eq!(inflight.outstanding, 0);
}
#[test]
fn eio_is_retried_exactly_once_then_reported() {
let mut io = IoStats::default();
let mut inflight = Inflight::default();
let tokens = tracked(&mut inflight, &[0]);
assert!(matches!(
inflight.resolve(tokens[0], -libc::EIO, &mut io),
Reap::Again(0)
));
assert_eq!(io.read_retries, 1);
let second = inflight.retag(0);
match inflight.resolve(second, -libc::EIO, &mut io) {
Reap::Failed(0, error) => {
assert_eq!(error.raw_os_error(), Some(libc::EIO));
}
other => panic!("expected Failed, got {other:?}"),
}
assert_eq!(io.read_retries, 1, "the second failure was retried too");
assert_eq!(inflight.outstanding, 0);
assert_eq!(io.bytes_read, 0);
}
#[test]
fn a_window_read_retries_like_a_blob_and_counts_in_the_sweep_half() {
let mut io = IoStats {
in_sweep: true,
..IoStats::default()
};
let mut inflight = Inflight {
align: direct::DIO_ALIGN as u32,
..Inflight::default()
};
let index = inflight.track_arena(0, 1, 8, 8 * 4096, 4 * 4096, 3 * 4096);
assert_eq!(inflight.reads[index].dst, Dest::Arena(3 * 4096));
let token = inflight.reads[index].user_data;
assert!(matches!(
inflight.resolve(token, 2 * 4096, &mut io),
Reap::Again(0)
));
assert_eq!(inflight.reads[0].filled, 2 * 4096);
assert_eq!(inflight.reads[0].remaining, 2 * 4096);
let second = inflight.retag(0);
assert!(matches!(
inflight.resolve(second, -libc::EIO, &mut io),
Reap::Again(0)
));
let third = inflight.retag(0);
assert!(matches!(
inflight.resolve(third, 2 * 4096, &mut io),
Reap::Done(0)
));
assert_eq!(inflight.outstanding, 0);
assert_eq!(io.sweep_bytes_read, 4 * 4096);
assert_eq!(io.sweep_read_retries, 2, "short read plus one EIO");
assert_eq!(io.bytes_read, 0, "sweep bytes leaked into the cache half");
assert_eq!(io.read_retries, 0);
let index = inflight.track_arena(0, 0, 0, 0, 4 * 4096, 0);
let token = inflight.reads[index].user_data;
assert!(matches!(
inflight.resolve(token, 4096 + 512, &mut io),
Reap::Again(1)
));
assert_eq!(inflight.reads[1].filled, 0, "the window was not restarted");
assert_eq!(inflight.reads[1].remaining, 4 * 4096);
}
#[test]
fn a_restart_gives_the_blob_a_fresh_retry_budget() {
let mut io = IoStats::default();
let mut inflight = Inflight {
align: direct::DIO_ALIGN as u32,
..Inflight::default()
};
let index = inflight.track(0, 0, 0, 0, 8192);
let token = inflight.reads[index].user_data;
assert!(matches!(
inflight.resolve(token, -libc::EIO, &mut io),
Reap::Again(0)
));
assert_eq!(inflight.reads[0].attempts, 1);
let token = inflight.retag(0);
assert!(matches!(
inflight.resolve(token, 1536, &mut io),
Reap::Again(0)
));
assert_eq!(inflight.reads[0].restarts, 1);
assert_eq!(inflight.reads[0].filled, 0);
assert_eq!(
inflight.reads[0].attempts, 0,
"a restarted blob kept a retry budget it had already spent"
);
let token = inflight.retag(0);
assert!(matches!(
inflight.resolve(token, -libc::EIO, &mut io),
Reap::Again(0)
));
let token = inflight.retag(0);
match inflight.resolve(token, -libc::EIO, &mut io) {
Reap::Failed(0, error) => assert_eq!(error.raw_os_error(), Some(libc::EIO)),
other => panic!("expected Failed, got {other:?}"),
}
assert_eq!(inflight.outstanding, 0);
}
#[test]
fn a_flattened_submission_failure_keeps_its_errno() {
let wrapped = IoError::io(
Path::new("experts/layer_00.bin"),
io::Error::from_raw_os_error(libc::ENOSPC),
);
let flat = flatten_io(wrapped);
assert_eq!(flat.raw_os_error(), Some(libc::ENOSPC));
let other = IoError::LayerOutOfRange {
layer: 9,
n_layers: 2,
};
let text = other.to_string();
assert_eq!(flatten_io(other).to_string(), text);
}
#[test]
fn permanent_errors_are_not_retried() {
let mut io = IoStats::default();
let mut inflight = Inflight::default();
let tokens = tracked(&mut inflight, &[0]);
match inflight.resolve(tokens[0], -libc::EINVAL, &mut io) {
Reap::Failed(0, error) => assert_eq!(error.raw_os_error(), Some(libc::EINVAL)),
other => panic!("expected Failed, got {other:?}"),
}
assert_eq!(io.read_retries, 0);
}
#[test]
fn short_reads_continue_and_eof_fails() {
let mut io = IoStats::default();
let mut inflight = Inflight::default();
let tokens = tracked(&mut inflight, &[0, 1]);
assert!(matches!(
inflight.resolve(tokens[0], 2048, &mut io),
Reap::Again(0)
));
assert_eq!(inflight.reads[0].filled, 2048);
assert_eq!(inflight.reads[0].remaining, 2048);
let next = inflight.retag(0);
assert!(matches!(
inflight.resolve(next, 2048, &mut io),
Reap::Done(0)
));
assert_eq!(io.bytes_read, 4096);
match inflight.resolve(tokens[1], 0, &mut io) {
Reap::Failed(1, error) => assert_eq!(error.kind(), io::ErrorKind::UnexpectedEof),
other => panic!("expected Failed, got {other:?}"),
}
assert_eq!(inflight.outstanding, 0);
}
#[test]
fn an_unaligned_short_read_restarts_the_blob_instead_of_being_resumed() {
let mut io = IoStats::default();
let mut inflight = Inflight {
align: direct::DIO_ALIGN as u32,
..Inflight::default()
};
let tokens = tracked(&mut inflight, &[0]);
assert!(matches!(
inflight.resolve(tokens[0], 1536, &mut io),
Reap::Again(0)
));
assert_eq!(
inflight.reads[0].filled, 0,
"reissue offset must be aligned"
);
assert_eq!(inflight.reads[0].remaining, 4096);
assert_eq!(inflight.reads[0].restarts, 1);
assert_eq!(io.bytes_read, 1536);
assert_eq!(io.read_retries, 1);
let next = inflight.retag(0);
match inflight.resolve(next, 1536, &mut io) {
Reap::Failed(0, error) => assert_eq!(error.kind(), io::ErrorKind::InvalidData),
other => panic!("expected Failed, got {other:?}"),
}
assert_eq!(inflight.outstanding, 0);
let mut aligned = Inflight {
align: direct::DIO_ALIGN as u32,
..Inflight::default()
};
let index = aligned.track(0, 0, 0, 0, 8192);
let token = aligned.reads[index].user_data;
assert!(matches!(
aligned.resolve(token, 4096, &mut io),
Reap::Again(0)
));
assert_eq!(aligned.reads[0].filled, 4096);
assert_eq!(aligned.reads[0].restarts, 0);
let mut buffered = Inflight::default();
let index = buffered.track(0, 0, 0, 0, 4096);
let token = buffered.reads[index].user_data;
assert!(matches!(
buffered.resolve(token, 1536, &mut io),
Reap::Again(0)
));
assert_eq!(buffered.reads[0].filled, 1536);
assert_eq!(buffered.reads[0].restarts, 0);
}
}