use std::fmt;
use std::ptr::NonNull;
use thiserror::Error;
use super::direct;
use super::{ExpertStream, ExpertView, IoError};
pub const DEFAULT_EXPERTS_PER_WINDOW: u32 = 8;
pub const DEFAULT_WINDOWS_IN_FLIGHT: u32 = 2;
pub const MAX_WINDOWS_IN_FLIGHT: u32 = 8;
#[derive(Debug, Error)]
pub enum SweepError {
#[error(transparent)]
Io(#[from] IoError),
#[error(
"sweep dials out of range: {experts_per_window} experts/window and \
{windows_in_flight} windows in flight (both must be >= 1, and windows \
in flight at most {max})"
)]
BadDials {
experts_per_window: u32,
windows_in_flight: u32,
max: u32,
},
#[error("routed expert {expert} out of range ({n_experts} experts)")]
ExpertOutOfRange {
expert: u32,
n_experts: u32,
},
#[error(
"a sweep window of {experts} experts x {stride} B is {bytes} B, past \
the {max} B one read can carry; lower experts-per-window"
)]
WindowTooLarge {
experts: u32,
stride: u64,
bytes: u128,
max: u64,
},
#[error("the prefill arena cannot be taken with {outstanding} read(s) in flight")]
ReadsInFlight {
outstanding: usize,
},
#[error("the prefill arena cannot be taken with a step open on layer {layer}")]
StepOpen {
layer: u32,
},
#[error(
"layer {layer}: blob stride {stride} B pads to a {pitch} B slot pitch, \
so the slot pool is not a gapless prefill arena"
)]
PaddedSlots {
layer: u32,
stride: usize,
pitch: usize,
},
#[error(
"the prefill arena needs {needed} B but the slot pool is {available} B; \
lower experts-per-window or raise the expert cache budget"
)]
ArenaTooSmall {
needed: u64,
available: u64,
},
#[error("a prefill arena of {bytes} B is already out")]
ArenaOut {
bytes: usize,
},
#[error(
"the prefill arena was given up by a window read that could not be \
reaped; this process cannot sweep again"
)]
ArenaPoisoned,
#[error("no prefill arena is out")]
NoArena,
#[error(
"the prefill arena of {bytes} B would cover layer {layer}'s retired \
slot buffer at slab offset {offset}; a read that could not be reaped \
may still be writing there"
)]
ArenaOverRetired {
layer: u32,
offset: usize,
bytes: usize,
},
#[error(
"a prefill window read could not be reaped: every slot the arena \
covered is retired, which leaves layer {layer} with {slots} slot(s) \
against a top_k of {top_k}, so that layer can no longer decode"
)]
CacheStranded {
layer: u32,
slots: u32,
top_k: u32,
#[source]
source: IoError,
},
#[error("this prefill sweep already failed and cannot be resumed: {reason}")]
Aborted {
reason: String,
},
#[error("{what} is {value}, not a multiple of {align}")]
Misaligned {
what: &'static str,
value: u64,
align: u64,
},
}
pub(super) fn check_aligned(what: &'static str, value: u64) -> Result<(), SweepError> {
if direct::is_aligned(value) {
Ok(())
} else {
Err(SweepError::Misaligned {
what,
value,
align: direct::DIO_ALIGN,
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SweepConfig {
pub experts_per_window: u32,
pub windows_in_flight: u32,
}
impl Default for SweepConfig {
fn default() -> Self {
Self {
experts_per_window: DEFAULT_EXPERTS_PER_WINDOW,
windows_in_flight: DEFAULT_WINDOWS_IN_FLIGHT,
}
}
}
impl SweepConfig {
pub fn validate(self) -> Result<Self, SweepError> {
if self.experts_per_window == 0
|| self.windows_in_flight == 0
|| self.windows_in_flight > MAX_WINDOWS_IN_FLIGHT
{
return Err(SweepError::BadDials {
experts_per_window: self.experts_per_window,
windows_in_flight: self.windows_in_flight,
max: MAX_WINDOWS_IN_FLIGHT,
});
}
Ok(self)
}
pub fn window_bytes(self, stride: u64) -> Result<u64, SweepError> {
let bytes = u128::from(self.experts_per_window) * u128::from(stride);
u64::try_from(bytes).map_err(|_| SweepError::WindowTooLarge {
experts: self.experts_per_window,
stride,
bytes,
max: u64::from(u32::MAX),
})
}
}
impl fmt::Display for SweepConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{} experts/window x {} in flight",
self.experts_per_window, self.windows_in_flight
)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SweepWindow {
pub first_expert: u32,
pub n_experts: u32,
pub file_offset: u64,
pub len: u64,
pub routed: bool,
}
#[derive(Debug, Clone, Default)]
pub struct SweepPlan {
windows: Vec<SweepWindow>,
routed: Vec<bool>,
n_read: usize,
stride: u64,
}
impl SweepPlan {
pub fn new() -> Self {
Self::default()
}
pub fn build(
&mut self,
n_experts: u32,
stride: u64,
config: SweepConfig,
routed: &[u32],
) -> Result<(), SweepError> {
let config = config.validate()?;
let bytes = config.window_bytes(stride)?;
if bytes > u64::from(u32::MAX) {
return Err(SweepError::WindowTooLarge {
experts: config.experts_per_window,
stride,
bytes: u128::from(bytes),
max: u64::from(u32::MAX),
});
}
self.stride = stride;
self.routed.clear();
self.routed.resize(n_experts as usize, false);
for &expert in routed {
let flag = self
.routed
.get_mut(expert as usize)
.ok_or(SweepError::ExpertOutOfRange { expert, n_experts })?;
*flag = true;
}
self.windows.clear();
self.n_read = 0;
let per = config.experts_per_window;
for first in (0..n_experts).step_by(per as usize) {
let covered = per.min(n_experts - first);
let is_routed = self.routed[first as usize..(first + covered) as usize]
.iter()
.any(|&flag| flag);
if is_routed {
self.n_read += 1;
}
self.windows.push(SweepWindow {
first_expert: first,
n_experts: covered,
file_offset: u64::from(first) * stride,
len: u64::from(covered) * stride,
routed: is_routed,
});
}
Ok(())
}
pub fn windows(&self) -> &[SweepWindow] {
&self.windows
}
pub fn is_routed(&self, expert: u32) -> bool {
self.routed.get(expert as usize).copied().unwrap_or(false)
}
pub fn windows_to_read(&self) -> usize {
self.n_read
}
pub fn windows_skipped(&self) -> usize {
self.windows.len() - self.n_read
}
pub fn stride(&self) -> u64 {
self.stride
}
}
#[derive(Debug)]
pub struct SweepExpert<'a> {
pub expert: u32,
pub bytes: &'a [u8],
pub view: ExpertView<'a>,
}
#[derive(Debug, Clone, Copy)]
struct Pending {
window: usize,
read: usize,
}
pub struct LayerSweep<'a> {
stream: &'a mut ExpertStream,
plan: &'a SweepPlan,
layer: u32,
stride: usize,
window_bytes: usize,
ring_base: usize,
in_flight: [Option<Pending>; MAX_WINDOWS_IN_FLIGHT as usize],
buffers: usize,
next_submit: usize,
next_consume: usize,
current: Option<(usize, usize)>,
cursor: u32,
counted: usize,
failed: Option<SweepError>,
owns_arena: bool,
finished: bool,
}
impl fmt::Debug for LayerSweep<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("LayerSweep")
.field("layer", &self.layer)
.field("windows", &self.plan.windows().len())
.field("to_read", &self.plan.windows_to_read())
.field("buffers", &self.buffers)
.field("consumed", &self.next_consume)
.finish()
}
}
impl<'a> LayerSweep<'a> {
pub(super) fn begin(
stream: &'a mut ExpertStream,
plan: &'a mut SweepPlan,
layer: u32,
routed: &[u32],
config: SweepConfig,
) -> Result<Self, SweepError> {
let config = config.validate()?;
let (stride, n_experts) = stream.layer_geometry(layer)?;
plan.build(n_experts, stride, config, routed)?;
let window_bytes = window_span(config, stride, n_experts)?;
let available = stream.cache_bytes();
let buffers = fit_buffers(
layer,
config,
window_bytes,
available,
plan.windows_to_read(),
)?;
stream.open_layer(layer)?;
stream.take_arena(buffers * window_bytes as usize)?;
Self::start(stream, plan, layer, stride, window_bytes, buffers, 0, true)
}
pub(super) fn begin_within(
stream: &'a mut ExpertStream,
plan: &'a mut SweepPlan,
layer: u32,
routed: &[u32],
config: SweepConfig,
ring_base: usize,
ring_bytes: u64,
) -> Result<Self, SweepError> {
let outstanding = stream.reads_outstanding();
if outstanding > 0 {
tracing::error!(
layer,
outstanding,
"refusing a sweep into a ring a previous sweep's windows are \
still writing into"
);
return Err(SweepError::ReadsInFlight { outstanding });
}
let config = config.validate()?;
let (stride, n_experts) = stream.layer_geometry(layer)?;
plan.build(n_experts, stride, config, routed)?;
let window_bytes = window_span(config, stride, n_experts)?;
let buffers = fit_buffers(
layer,
config,
window_bytes,
ring_bytes,
plan.windows_to_read(),
)?;
stream.open_layer(layer)?;
Self::start(
stream,
plan,
layer,
stride,
window_bytes,
buffers,
ring_base,
false,
)
}
#[allow(clippy::too_many_arguments)]
fn start(
stream: &'a mut ExpertStream,
plan: &'a SweepPlan,
layer: u32,
stride: u64,
window_bytes: u64,
buffers: usize,
ring_base: usize,
owns_arena: bool,
) -> Result<Self, SweepError> {
let mut sweep = Self {
stream,
plan,
layer,
stride: stride as usize,
window_bytes: window_bytes as usize,
ring_base,
in_flight: [None; MAX_WINDOWS_IN_FLIGHT as usize],
buffers,
next_submit: 0,
next_consume: 0,
current: None,
cursor: 0,
counted: 0,
failed: None,
owns_arena,
finished: false,
};
if let Err(error) = sweep.pump() {
let _ = sweep.teardown();
return Err(error);
}
Ok(sweep)
}
pub fn layer(&self) -> u32 {
self.layer
}
pub fn plan(&self) -> &SweepPlan {
self.plan
}
pub fn buffers(&self) -> usize {
self.buffers
}
pub fn next_expert(&mut self) -> Result<Option<SweepExpert<'_>>, SweepError> {
if let Some(error) = &self.failed {
return Err(SweepError::Aborted {
reason: error.to_string(),
});
}
let position = match self.advance() {
Ok(Some(position)) => position,
Ok(None) => return Ok(None),
Err(error) => return Err(self.latch(error)),
};
let (expert, offset) = position;
let Some(bytes) = (unsafe { self.stream.arena_bytes(offset, self.stride) }) else {
self.failed = Some(SweepError::NoArena);
return Err(SweepError::NoArena);
};
let view = match self.stream.expert_reader().view_over(self.layer, bytes) {
Ok(view) => view,
Err(error) => {
let error = SweepError::Io(error);
self.failed = Some(latchable(&error));
return Err(error);
}
};
Ok(Some(SweepExpert {
expert,
bytes,
view,
}))
}
pub fn is_aborted(&self) -> bool {
self.failed.is_some()
}
pub fn finish(mut self) -> Result<(), SweepError> {
self.teardown()
}
fn latch(&mut self, error: SweepError) -> SweepError {
if self.failed.is_none() {
self.failed = Some(latchable(&error));
}
error
}
fn advance(&mut self) -> Result<Option<(u32, usize)>, SweepError> {
loop {
if let Some((window, buffer)) = self.current {
let win = self.plan.windows()[window];
while self.cursor < win.n_experts {
let within = self.cursor;
self.cursor += 1;
let expert = win.first_expert + within;
if self.plan.is_routed(expert) {
let offset = self.ring_base
+ buffer * self.window_bytes
+ within as usize * self.stride;
return Ok(Some((expert, offset)));
}
}
self.in_flight[buffer] = None;
self.current = None;
self.next_consume = window + 1;
self.pump()?;
continue;
}
while self
.plan
.windows()
.get(self.next_consume)
.is_some_and(|win| !win.routed)
{
self.stream.count_window(false);
self.counted += 1;
self.next_consume += 1;
}
if self.next_consume >= self.plan.windows().len() {
return Ok(None);
}
let Some(buffer) = self.buffer_of(self.next_consume) else {
return Err(SweepError::NoArena);
};
let read = self.in_flight[buffer].map(|pending| pending.read);
self.stream.sweep_await(read)?;
self.stream.count_window(true);
self.counted += 1;
self.current = Some((self.next_consume, buffer));
self.cursor = 0;
}
}
fn pump(&mut self) -> Result<(), SweepError> {
loop {
while self
.plan
.windows()
.get(self.next_submit)
.is_some_and(|win| !win.routed)
{
self.next_submit += 1;
}
let Some(&win) = self.plan.windows().get(self.next_submit) else {
return Ok(());
};
let Some(buffer) = self.free_buffer() else {
return Ok(());
};
let arena_offset = self.ring_base + buffer * self.window_bytes;
check_aligned("sweep window file offset", win.file_offset)?;
check_aligned("sweep window length", win.len)?;
check_aligned("sweep window arena offset", arena_offset as u64)?;
let len = u32::try_from(win.len).map_err(|_| SweepError::WindowTooLarge {
experts: win.n_experts,
stride: self.stride as u64,
bytes: u128::from(win.len),
max: u64::from(u32::MAX),
})?;
let read = self.stream.sweep_submit(
self.layer,
buffer as u32,
win.first_expert,
win.file_offset,
len,
arena_offset,
)?;
self.in_flight[buffer] = Some(Pending {
window: self.next_submit,
read,
});
self.next_submit += 1;
}
}
fn free_buffer(&self) -> Option<usize> {
(0..self.buffers).find(|&buffer| self.in_flight[buffer].is_none())
}
fn buffer_of(&self, window: usize) -> Option<usize> {
(0..self.buffers).find(|&buffer| self.in_flight[buffer].is_some_and(|p| p.window == window))
}
fn teardown(&mut self) -> Result<(), SweepError> {
if self.finished {
return Ok(());
}
self.finished = true;
let windows = self.plan.windows().len();
if self.counted < windows {
tracing::warn!(
layer = self.layer,
covered = self.counted,
windows,
"prefill sweep torn down before it covered the layer; its \
window counters describe what it did, not what it planned"
);
}
let result = self.stream.sweep_await(None);
self.stream.sweep_clear_reads();
if self.owns_arena {
self.stream.release_arena();
}
match (result, self.failed.take()) {
(Err(error), _) => Err(error),
(Ok(()), Some(error)) => Err(error),
(Ok(()), None) => Ok(()),
}
}
}
impl Drop for LayerSweep<'_> {
fn drop(&mut self) {
if let Err(error) = self.teardown() {
tracing::error!(
layer = self.layer,
%error,
"prefill sweep torn down without completing the layer"
);
}
}
}
fn latchable(error: &SweepError) -> SweepError {
match error {
SweepError::Io(IoError::Io { path, source }) => SweepError::Io(IoError::Io {
path: path.clone(),
source: dup_os(source),
}),
SweepError::CacheStranded {
layer,
slots,
top_k,
source: IoError::Io { path, source },
} => SweepError::CacheStranded {
layer: *layer,
slots: *slots,
top_k: *top_k,
source: IoError::Io {
path: path.clone(),
source: dup_os(source),
},
},
other => SweepError::Aborted {
reason: other.to_string(),
},
}
}
fn dup_os(error: &std::io::Error) -> std::io::Error {
error.raw_os_error().map_or_else(
|| std::io::Error::new(error.kind(), error.to_string()),
std::io::Error::from_raw_os_error,
)
}
fn window_span(config: SweepConfig, stride: u64, n_experts: u32) -> Result<u64, SweepError> {
SweepConfig {
experts_per_window: config.experts_per_window.min(n_experts.max(1)),
..config
}
.window_bytes(stride)
}
fn fit_buffers(
layer: u32,
config: SweepConfig,
window_bytes: u64,
available: u64,
windows_to_read: usize,
) -> Result<usize, SweepError> {
if window_bytes > available {
return Err(SweepError::ArenaTooSmall {
needed: window_bytes,
available,
});
}
let buffers = (config.windows_in_flight as usize)
.min(windows_to_read.max(1))
.min((available / window_bytes) as usize)
.max(1);
if buffers < config.windows_in_flight as usize {
tracing::debug!(
layer,
asked = config.windows_in_flight,
buffers,
window_bytes,
ring_bytes = available,
"fewer prefill windows in flight than asked for"
);
}
Ok(buffers)
}
pub struct PrefillSession<'a> {
stream: &'a mut ExpertStream,
scratch: NonNull<u8>,
scratch_len: usize,
ring_base: usize,
ring_bytes: u64,
config: SweepConfig,
finished: bool,
}
impl fmt::Debug for PrefillSession<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("PrefillSession")
.field("scratch_bytes", &self.scratch_len)
.field("ring_base", &self.ring_base)
.field("ring_bytes", &self.ring_bytes)
.field("config", &self.config)
.finish()
}
}
impl<'a> PrefillSession<'a> {
pub(super) fn begin(
stream: &'a mut ExpertStream,
scratch_bytes: usize,
config: SweepConfig,
) -> Result<Self, SweepError> {
let config = config.validate()?;
let ring_bytes = ring_span(stream, config)?;
let align = direct::DIO_ALIGN as usize;
let ring_base =
scratch_bytes
.checked_next_multiple_of(align)
.ok_or(SweepError::ArenaTooSmall {
needed: u64::MAX,
available: stream.cache_bytes(),
})?;
let available = stream.cache_bytes();
let total = (ring_base as u64)
.checked_add(ring_bytes)
.filter(|total| *total <= available)
.ok_or(SweepError::ArenaTooSmall {
needed: (ring_base as u64).saturating_add(ring_bytes),
available,
})?;
stream.take_arena(total as usize)?;
let scratch = stream.arena_base().ok_or(SweepError::NoArena)?;
debug_assert!(
scratch_bytes <= ring_base,
"the scratch span and the sweep ring must not overlap"
);
tracing::debug!(
scratch_bytes,
ring_base,
ring_bytes,
pool_bytes = available,
"prefill session open"
);
Ok(Self {
stream,
scratch,
scratch_len: scratch_bytes,
ring_base,
ring_bytes,
config,
finished: false,
})
}
pub fn scratch_len(&self) -> usize {
self.scratch_len
}
pub fn ring_offset(&self) -> usize {
self.ring_base
}
pub fn config(&self) -> SweepConfig {
self.config
}
pub fn scratch(&mut self) -> &mut [u8] {
unsafe { std::slice::from_raw_parts_mut(self.scratch.as_ptr(), self.scratch_len) }
}
pub fn split<'s>(
&'s mut self,
plan: &'s mut SweepPlan,
layer: u32,
routed: &[u32],
) -> Result<(&'s mut [u8], LayerSweep<'s>), SweepError> {
let (scratch, scratch_len) = (self.scratch, self.scratch_len);
let (ring_base, ring_bytes, config) = (self.ring_base, self.ring_bytes, self.config);
let sweep = LayerSweep::begin_within(
&mut *self.stream,
plan,
layer,
routed,
config,
ring_base,
ring_bytes,
)?;
let staging = unsafe { std::slice::from_raw_parts_mut(scratch.as_ptr(), scratch_len) };
Ok((staging, sweep))
}
pub fn finish(mut self) -> Result<(), SweepError> {
self.teardown()
}
fn teardown(&mut self) -> Result<(), SweepError> {
if self.finished {
return Ok(());
}
self.finished = true;
let result = self.stream.sweep_await(None);
self.stream.sweep_clear_reads();
self.stream.release_arena();
result
}
}
impl Drop for PrefillSession<'_> {
fn drop(&mut self) {
if let Err(error) = self.teardown() {
tracing::error!(
%error,
"prefill session torn down with a window read still failing"
);
}
}
}
fn ring_span(stream: &ExpertStream, config: SweepConfig) -> Result<u64, SweepError> {
let mut widest = 0u64;
for layer in 0..stream.n_layers() {
let (stride, n_experts) = stream.layer_geometry(layer)?;
widest = widest.max(window_span(config, stride, n_experts)?);
}
widest
.checked_mul(u64::from(config.windows_in_flight))
.ok_or(SweepError::ArenaTooSmall {
needed: u64::MAX,
available: stream.cache_bytes(),
})
}
#[cfg(test)]
mod tests {
use super::super::testutil::{Fixture, N_EXPERTS, N_LAYERS, build_install};
use super::*;
use crate::io::{ExpertReader, LoadOptions, SLOT_ALIGN};
fn slot_row(fx: &Fixture) -> u64 {
fx.layout
.layers
.iter()
.map(|layer| layer.stride.next_multiple_of(SLOT_ALIGN as u64))
.sum()
}
fn open(fx: &Fixture, slots: u32) -> ExpertStream {
ExpertStream::new(
&fx.root,
&fx.manifest,
&fx.layout,
slot_row(fx) * u64::from(slots),
LoadOptions::default(),
)
.expect("stream opens")
}
fn all_experts(fx: &Fixture, layer: u32) -> Vec<u32> {
(0..fx.layout.layers[layer as usize].n_experts).collect()
}
fn sweep_all(
stream: &mut ExpertStream,
layer: u32,
routed: &[u32],
config: SweepConfig,
) -> Vec<(u32, Vec<u8>)> {
let mut plan = SweepPlan::new();
let mut sweep = stream
.sweep_layer(&mut plan, layer, routed, config)
.expect("sweep begins");
let mut out = Vec::new();
while let Some(expert) = sweep.next_expert().expect("window reads") {
out.push((expert.expert, expert.bytes.to_vec()));
}
sweep.finish().expect("sweep finishes");
out
}
#[test]
fn window_arithmetic_covers_both_stride_classes() {
const QWEN_STRIDES: [u64; 2] = [3_059_712, 2_654_208];
let config = SweepConfig::default();
let routed: Vec<u32> = (0..128).collect();
for stride in QWEN_STRIDES {
let mut plan = SweepPlan::new();
plan.build(128, stride, config, &routed).unwrap();
assert_eq!(plan.windows().len(), 16, "128 / 8 is 16 whole windows");
assert_eq!(plan.windows_to_read(), 16);
assert_eq!(plan.windows_skipped(), 0);
let mut expected_offset = 0;
for (index, window) in plan.windows().iter().enumerate() {
assert_eq!(window.first_expert, index as u32 * 8);
assert_eq!(window.n_experts, 8, "no ragged tail at 128/8");
assert_eq!(window.file_offset, expected_offset);
assert_eq!(window.len, 8 * stride);
assert!(window.routed);
assert!(direct::is_aligned(window.file_offset));
assert!(direct::is_aligned(window.len));
expected_offset += window.len;
}
assert_eq!(expected_offset, 128 * stride);
let mib = config.window_bytes(stride).unwrap() as f64 / (1024.0 * 1024.0);
assert!((16.0..=24.0).contains(&mib), "{mib} MiB out of band");
}
}
#[test]
fn the_last_window_is_short_when_the_dial_does_not_divide() {
let stride = 8192;
let config = SweepConfig {
experts_per_window: 8,
windows_in_flight: 2,
};
let routed: Vec<u32> = (0..130).collect();
let mut plan = SweepPlan::new();
plan.build(130, stride, config, &routed).unwrap();
assert_eq!(plan.windows().len(), 17);
let last = *plan.windows().last().unwrap();
assert_eq!(last.first_expert, 128);
assert_eq!(last.n_experts, 2, "130 - 16 * 8");
assert_eq!(last.len, 2 * stride);
assert_eq!(last.file_offset, 128 * stride);
for window in &plan.windows()[..16] {
assert_eq!(window.n_experts, 8);
}
assert_eq!(
last.file_offset + last.len,
130 * stride,
"the windows must tile the whole file"
);
plan.build(3, stride, config, &[0, 1, 2]).unwrap();
assert_eq!(plan.windows().len(), 1);
assert_eq!(plan.windows()[0].n_experts, 3);
}
#[test]
fn a_window_routing_nothing_is_skippable() {
let stride = 4096;
let config = SweepConfig {
experts_per_window: 4,
windows_in_flight: 2,
};
let mut plan = SweepPlan::new();
plan.build(16, stride, config, &[1, 15]).unwrap();
let routed: Vec<bool> = plan.windows().iter().map(|w| w.routed).collect();
assert_eq!(routed, vec![true, false, false, true]);
assert_eq!(plan.windows_to_read(), 2);
assert_eq!(plan.windows_skipped(), 2);
plan.build(16, stride, config, &[]).unwrap();
assert_eq!(plan.windows_to_read(), 0);
assert_eq!(plan.windows_skipped(), 4);
let all: Vec<u32> = (0..16).collect();
plan.build(16, stride, config, &all).unwrap();
assert_eq!(plan.windows_to_read(), 4);
assert_eq!(plan.windows_skipped(), 0);
}
#[test]
fn a_routed_id_past_the_layer_is_a_typed_error() {
let mut plan = SweepPlan::new();
let err = plan
.build(4, 4096, SweepConfig::default(), &[0, 4])
.unwrap_err();
assert!(
matches!(
err,
SweepError::ExpertOutOfRange {
expert: 4,
n_experts: 4
}
),
"unexpected error: {err}"
);
}
#[test]
fn degenerate_dials_are_refused() {
for config in [
SweepConfig {
experts_per_window: 0,
windows_in_flight: 2,
},
SweepConfig {
experts_per_window: 8,
windows_in_flight: 0,
},
SweepConfig {
experts_per_window: 8,
windows_in_flight: MAX_WINDOWS_IN_FLIGHT + 1,
},
] {
let err = config.validate().unwrap_err();
assert!(
matches!(err, SweepError::BadDials { .. }),
"unexpected error: {err}"
);
}
SweepConfig::default().validate().unwrap();
}
#[test]
fn a_window_past_u32_is_refused() {
let mut plan = SweepPlan::new();
let err = plan
.build(
64,
1 << 30,
SweepConfig {
experts_per_window: 8,
windows_in_flight: 2,
},
&[0],
)
.unwrap_err();
assert!(
matches!(err, SweepError::WindowTooLarge { .. }),
"unexpected error: {err}"
);
}
#[test]
fn alignment_checks_reject_each_quantity() {
check_aligned("sweep read offset", 0).unwrap();
check_aligned("sweep read offset", 4096).unwrap();
check_aligned("sweep read length", 3_059_712).unwrap();
for (what, value) in [
("sweep read offset", 4095u64),
("sweep read length", 1),
("sweep read destination", 0x1_0000_0800),
] {
let err = check_aligned(what, value).unwrap_err();
match err {
SweepError::Misaligned {
what: named,
value: got,
align,
} => {
assert_eq!(named, what);
assert_eq!(got, value);
assert_eq!(align, direct::DIO_ALIGN);
}
other => panic!("unexpected error: {other}"),
}
}
}
#[test]
fn a_sweep_yields_the_same_bytes_as_read_expert() {
let fx = build_install("sweep-bytes");
let mut stream = open(&fx, 4);
let reader = ExpertReader::new(&fx.root, &fx.manifest, &fx.layout, LoadOptions::default())
.expect("reader opens");
for layer in 0..fx.layout.layers.len() as u32 {
let routed = all_experts(&fx, layer);
let swept = sweep_all(&mut stream, layer, &routed, SweepConfig::default());
assert_eq!(swept.len(), routed.len(), "layer {layer} expert count");
let mut buf = Vec::new();
for (index, (expert, bytes)) in swept.iter().enumerate() {
assert_eq!(*expert, index as u32, "layer {layer} out of order");
let view = reader.read_expert(layer, *expert, &mut buf).unwrap();
assert_eq!(
bytes.as_slice(),
view.blob(),
"layer {layer} expert {expert} blob mismatch"
);
}
}
}
#[test]
fn a_swept_view_carves_the_layout_slabs() {
let fx = build_install("sweep-slabs");
let mut stream = open(&fx, 4);
let mut plan = SweepPlan::new();
let routed = all_experts(&fx, 0);
let mut sweep = stream
.sweep_layer(&mut plan, 0, &routed, SweepConfig::default())
.unwrap();
let file = std::fs::read(fx.root.join(&fx.layout.layers[0].file)).unwrap();
let stride = fx.layout.layers[0].stride as usize;
while let Some(expert) = sweep.next_expert().unwrap() {
let base = expert.expert as usize * stride;
for projection in &fx.layout.layers[0].projections {
let from = base + projection.offset_in_blob as usize;
let to = from + projection.len as usize;
assert_eq!(
expert.view.slab(projection.name).bytes,
&file[from..to],
"expert {} {:?} slab",
expert.expert,
projection.name
);
}
}
sweep.finish().unwrap();
}
#[test]
fn a_sparse_chunk_skips_whole_windows() {
let fx = build_install("sweep-skip");
let mut stream = open(&fx, 4);
let config = SweepConfig {
experts_per_window: 2,
windows_in_flight: 2,
};
let before = stream.stats();
let swept = sweep_all(&mut stream, 1, &[2, 3], config);
assert_eq!(
swept.iter().map(|(e, _)| *e).collect::<Vec<_>>(),
vec![2, 3]
);
let delta = stream.stats().since(&before);
assert_eq!(delta.sweep_windows_read, 1);
assert_eq!(delta.sweep_windows_skipped, 1);
assert_eq!(
delta.sweep_bytes_read,
2 * fx.layout.layers[1].stride,
"the skipped window's bytes were read anyway"
);
let reader = ExpertReader::new(&fx.root, &fx.manifest, &fx.layout, LoadOptions::default())
.expect("reader opens");
let mut buf = Vec::new();
for (expert, bytes) in &swept {
let view = reader.read_expert(1, *expert, &mut buf).unwrap();
assert_eq!(bytes.as_slice(), view.blob());
}
}
#[test]
fn a_chunk_routing_nothing_reads_nothing() {
let fx = build_install("sweep-empty");
let mut stream = open(&fx, 4);
let before = stream.stats();
assert!(sweep_all(&mut stream, 0, &[], SweepConfig::default()).is_empty());
let delta = stream.stats().since(&before);
assert_eq!(delta.sweep_windows_read, 0);
assert_eq!(delta.sweep_bytes_read, 0);
assert!(delta.sweep_windows_skipped > 0);
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
}
#[test]
fn every_dial_pair_reads_the_file_exactly_once() {
let fx = build_install("sweep-dials");
let stride = fx.layout.layers[0].stride;
let routed = all_experts(&fx, 0);
for experts_per_window in 1..=5u32 {
for windows_in_flight in 1..=3u32 {
let mut stream = open(&fx, 4);
let config = SweepConfig {
experts_per_window,
windows_in_flight,
};
let swept = sweep_all(&mut stream, 0, &routed, config);
assert_eq!(
swept.iter().map(|(e, _)| *e).collect::<Vec<_>>(),
routed,
"dials {config}"
);
let stats = stream.stats();
assert_eq!(
stats.sweep_bytes_read,
u64::from(fx.layout.layers[0].n_experts) * stride,
"dials {config} read the file a number of times other than once"
);
assert_eq!(stats.sweep_windows_skipped, 0);
assert_eq!(
stats.sweep_windows_read as usize,
(fx.layout.layers[0].n_experts as usize).div_ceil(experts_per_window as usize)
);
assert_eq!(stats.accesses(), 0);
assert_eq!(stats.bytes_read, 0);
}
}
}
#[test]
fn the_next_window_is_in_flight_before_this_one_is_consumed() {
let fx = build_install("sweep-overlap");
let mut stream = open(&fx, 4);
let mut plan = SweepPlan::new();
let routed = all_experts(&fx, 0);
let mut sweep = stream
.sweep_layer(
&mut plan,
0,
&routed,
SweepConfig {
experts_per_window: 1,
windows_in_flight: 2,
},
)
.unwrap();
assert_eq!(sweep.buffers(), 2);
assert_eq!(sweep.plan().windows().len(), 4);
{
let first = sweep.next_expert().unwrap().unwrap();
assert_eq!(first.expert, 0);
}
assert_eq!(sweep.stream.stats().sweep_windows_read, 1);
assert_eq!(
sweep.in_flight[1].map(|pending| pending.window),
Some(1),
"window 1 was not started before window 0 was handed over"
);
{
let second = sweep.next_expert().unwrap().unwrap();
assert_eq!(second.expert, 1);
}
assert_eq!(sweep.stream.stats().sweep_windows_read, 2);
assert_eq!(
sweep.in_flight[0].map(|pending| pending.window),
Some(2),
"the freed buffer was not refilled"
);
if sweep.stream.mode() != crate::io::StreamMode::Pread {
assert_eq!(sweep.stream.stats().sweep_reads_submitted, 3);
}
sweep.finish().unwrap();
}
#[test]
fn buffers_are_clamped_to_the_layer_and_the_pool() {
let fx = build_install("sweep-clamp");
let mut stream = open(&fx, 4);
let mut plan = SweepPlan::new();
let routed = all_experts(&fx, 0);
let sweep = stream
.sweep_layer(
&mut plan,
0,
&routed,
SweepConfig {
experts_per_window: 8,
windows_in_flight: 4,
},
)
.unwrap();
assert_eq!(sweep.buffers(), 1);
sweep.finish().unwrap();
}
#[test]
fn a_window_bigger_than_the_pool_is_refused() {
let fx = build_install("sweep-arena-small");
let mut stream = open(&fx, 2);
let mut plan = SweepPlan::new();
let routed = all_experts(&fx, 0);
let err = stream
.sweep_layer(
&mut plan,
0,
&routed,
SweepConfig {
experts_per_window: 8,
windows_in_flight: 1,
},
)
.unwrap_err();
assert!(
matches!(err, SweepError::ArenaTooSmall { .. }),
"unexpected error: {err}"
);
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
}
#[test]
fn a_sweep_invalidates_the_decode_cache() {
let fx = build_install("sweep-invalidates");
let mut stream = open(&fx, 4);
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
stream.begin_layer(0, &[0, 1]).unwrap();
assert_eq!(stream.hits().len(), 2);
stream.end_layer(0);
let routed = all_experts(&fx, 0);
sweep_all(&mut stream, 0, &routed, SweepConfig::default());
stream.begin_layer(0, &[0, 1]).unwrap();
assert!(stream.hits().is_empty(), "a slot survived the sweep");
assert_eq!(stream.misses().len(), 2);
stream.await_misses().unwrap();
for &(index, slot) in stream.misses().to_vec().iter() {
let expert = [0u32, 1][index];
let mut buf = Vec::new();
let reader =
ExpertReader::new(&fx.root, &fx.manifest, &fx.layout, LoadOptions::default())
.unwrap();
let expected = reader.read_expert(0, expert, &mut buf).unwrap();
assert_eq!(stream.view(0, slot).unwrap().blob(), expected.blob());
}
stream.end_layer(0);
}
#[test]
fn sweep_counters_split_by_phase() {
use crate::io::StreamPhase;
let fx = build_install("sweep-phases");
let mut stream = open(&fx, 4);
let stride = fx.layout.layers[0].stride;
let routed = all_experts(&fx, 0);
sweep_all(&mut stream, 0, &routed, SweepConfig::default());
stream.set_phase(StreamPhase::Decode);
stream.begin_layer(1, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(1);
let prefill = stream.stats_in(StreamPhase::Prefill);
let decode = stream.stats_in(StreamPhase::Decode);
let total = stream.stats();
assert_eq!(prefill.accesses(), 0);
assert!(!prefill.is_idle(), "the sweep phase must not look idle");
assert_eq!(
prefill.sweep_bytes_read,
u64::from(fx.layout.layers[0].n_experts) * stride
);
assert!(prefill.sweep_reads_submitted >= 1);
assert_eq!(prefill.bytes_read, 0);
assert_eq!(decode.accesses(), 2);
assert_eq!(decode.sweep_bytes_read, 0);
assert_eq!(decode.sweep_windows(), 0);
assert_eq!(decode.bytes_read, 2 * fx.layout.layers[1].stride);
assert_eq!(prefill.plus(&decode), total);
}
#[test]
fn the_arena_is_reusable_across_layers_and_sweeps() {
let fx = build_install("sweep-reuse");
let mut stream = open(&fx, 4);
let routed = all_experts(&fx, 0);
let first = sweep_all(&mut stream, 0, &routed, SweepConfig::default());
let second = sweep_all(&mut stream, 0, &routed, SweepConfig::default());
assert_eq!(first, second);
let other = sweep_all(&mut stream, 1, &all_experts(&fx, 1), SweepConfig::default());
assert_ne!(other[0].1, first[0].1, "layer 1 returned layer 0's bytes");
}
#[test]
fn dropping_a_sweep_releases_the_arena() {
let fx = build_install("sweep-drop");
let mut stream = open(&fx, 4);
let mut plan = SweepPlan::new();
let routed = all_experts(&fx, 0);
{
let mut sweep = stream
.sweep_layer(&mut plan, 0, &routed, SweepConfig::default())
.unwrap();
assert!(sweep.next_expert().unwrap().is_some());
}
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
}
#[test]
fn a_sweep_is_refused_while_reads_are_in_flight() {
let fx = build_install("sweep-inflight");
let mut stream = open(&fx, 4);
let mut plan = SweepPlan::new();
stream.begin_layer(0, &[0, 1]).unwrap();
assert_eq!(stream.misses().len(), 2, "the fixture must miss cold");
let routed = all_experts(&fx, 0);
let err = stream
.sweep_layer(&mut plan, 0, &routed, SweepConfig::default())
.unwrap_err();
assert!(
matches!(
err,
SweepError::ReadsInFlight { .. } | SweepError::StepOpen { .. }
),
"unexpected error: {err}"
);
stream.await_misses().unwrap();
let err = stream
.sweep_layer(&mut plan, 0, &routed, SweepConfig::default())
.unwrap_err();
assert!(
matches!(err, SweepError::StepOpen { layer: 0 }),
"unexpected error: {err}"
);
stream.end_layer(0);
stream
.sweep_layer(&mut plan, 0, &routed, SweepConfig::default())
.expect("a closed step lets the sweep start")
.finish()
.unwrap();
}
#[test]
fn a_failed_window_read_latches_instead_of_handing_out_unfilled_bytes() {
let fx = build_install("sweep-latch-read");
let mut stream = open(&fx, 4);
stream.begin_layer(0, &[0]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
let path = fx.root.join(&fx.layout.layers[0].file);
let whole = std::fs::read(&path).unwrap();
std::fs::write(&path, &whole[..whole.len() / 2]).unwrap();
let routed = all_experts(&fx, 0);
let config = SweepConfig {
experts_per_window: 1,
windows_in_flight: 1,
};
{
let mut plan = SweepPlan::new();
let mut sweep = stream.sweep_layer(&mut plan, 0, &routed, config).unwrap();
for expert in 0..2u32 {
assert_eq!(sweep.next_expert().unwrap().unwrap().expert, expert);
}
let first = sweep.next_expert().unwrap_err();
assert!(
matches!(first, SweepError::Io(_)),
"unexpected error: {first}"
);
assert!(sweep.is_aborted());
for _ in 0..3 {
match sweep.next_expert().unwrap_err() {
SweepError::Aborted { reason } => assert_eq!(reason, first.to_string()),
other => panic!("unexpected error: {other}"),
}
}
match sweep.finish().unwrap_err() {
SweepError::Io(IoError::Io { source, .. }) => {
assert_eq!(source.kind(), std::io::ErrorKind::UnexpectedEof);
}
other => panic!("unexpected error: {other}"),
}
}
std::fs::write(&path, &whole).unwrap();
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
}
#[test]
fn a_lost_arena_latches_instead_of_skipping_an_expert() {
let fx = build_install("sweep-latch-arena");
let mut stream = open(&fx, 4);
let routed = all_experts(&fx, 0);
let mut plan = SweepPlan::new();
let config = SweepConfig {
experts_per_window: 1,
windows_in_flight: 4,
};
let mut sweep = stream.sweep_layer(&mut plan, 0, &routed, config).unwrap();
assert_eq!(sweep.next_expert().unwrap().unwrap().expert, 0);
sweep.stream.strand_open_arena();
let err = sweep.next_expert().unwrap_err();
assert!(
matches!(err, SweepError::NoArena),
"unexpected error: {err}"
);
match sweep.next_expert().unwrap_err() {
SweepError::Aborted { reason } => assert_eq!(reason, SweepError::NoArena.to_string()),
other => panic!("unexpected error: {other}"),
}
let err = sweep.finish().unwrap_err();
assert!(
matches!(err, SweepError::NoArena),
"unexpected error: {err}"
);
}
#[test]
fn window_counters_follow_the_sweep_rather_than_the_plan() {
let fx = build_install("sweep-counters-abort");
let mut stream = open(&fx, 4);
let config = SweepConfig {
experts_per_window: 1,
windows_in_flight: 1,
};
let before = stream.stats();
{
let mut plan = SweepPlan::new();
let mut sweep = stream.sweep_layer(&mut plan, 0, &[0, 3], config).unwrap();
assert_eq!(sweep.next_expert().unwrap().unwrap().expert, 0);
}
let delta = stream.stats().since(&before);
assert_eq!(delta.sweep_windows_read, 1);
assert_eq!(
delta.sweep_windows_skipped, 0,
"windows the sweep never reached were counted as skipped"
);
let before = stream.stats();
assert_eq!(sweep_all(&mut stream, 0, &[0, 3], config).len(), 2);
let delta = stream.stats().since(&before);
assert_eq!(delta.sweep_windows_read, 2);
assert_eq!(delta.sweep_windows_skipped, 2);
assert_eq!(delta.sweep_windows(), 4);
}
#[test]
fn an_overflowing_window_is_refused_rather_than_panicking() {
let config = SweepConfig {
experts_per_window: 1 << 20,
windows_in_flight: 1,
};
match config.window_bytes(u64::MAX).unwrap_err() {
SweepError::WindowTooLarge {
experts,
stride,
bytes,
max,
} => {
assert_eq!(experts, 1 << 20);
assert_eq!(stride, u64::MAX);
assert_eq!(bytes, u128::from(u64::MAX) << 20);
assert_eq!(max, u64::from(u32::MAX));
}
other => panic!("unexpected error: {other}"),
}
assert_eq!(
SweepConfig::default().window_bytes(3_059_712).unwrap(),
8 * 3_059_712
);
let mut plan = SweepPlan::new();
assert!(matches!(
plan.build(4, u64::MAX, config, &[0]).unwrap_err(),
SweepError::WindowTooLarge { .. }
));
}
#[test]
fn a_prefill_session_hands_out_scratch_and_a_sweep_at_once() {
let fx = build_install("prefill-session");
let mut stream = open(&fx, 4);
let reader = ExpertReader::new(&fx.root, &fx.manifest, &fx.layout, LoadOptions::default())
.expect("reader opens");
let pool_bytes = stream.cache_bytes();
const SCRATCH: usize = 8192;
let config = SweepConfig {
experts_per_window: 2,
windows_in_flight: 2,
};
let mut plan = SweepPlan::new();
let mut session = stream
.begin_prefill(SCRATCH, config)
.expect("session opens");
assert_eq!(session.scratch_len(), SCRATCH);
assert!(
session.ring_offset() >= SCRATCH,
"the sweep ring overlaps the scratch span"
);
assert!(direct::is_aligned(session.ring_offset() as u64));
session.scratch().fill(0xa5);
for layer in 0..N_LAYERS {
let routed = all_experts(&fx, layer);
let (staging, mut sweep) = session
.split(&mut plan, layer, &routed)
.expect("the layer sweeps");
let mut seen = 0u32;
let mut buf = Vec::new();
while let Some(expert) = sweep.next_expert().expect("window reads") {
let expected = reader.read_expert(layer, expert.expert, &mut buf).unwrap();
assert_eq!(expert.bytes, expected.blob(), "layer {layer}");
staging[expert.expert as usize] = expert.expert as u8;
seen += 1;
}
sweep.finish().expect("the sweep finishes");
assert_eq!(seen, fx.layout.layers[layer as usize].n_experts);
}
let scratch = session.scratch();
for expert in 0..N_EXPERTS {
assert_eq!(scratch[expert as usize], expert as u8);
}
assert!(
scratch[N_EXPERTS as usize..].iter().all(|&b| b == 0xa5),
"a window read landed in the staging span"
);
session.finish().expect("the session finishes");
assert_eq!(stream.cache_bytes(), pool_bytes);
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
}
#[test]
fn a_prefill_session_carve_is_bounded_and_page_aligned() {
let fx = build_install("prefill-session-size");
let mut stream = open(&fx, 4);
let pool_bytes = stream.cache_bytes() as usize;
let config = SweepConfig {
experts_per_window: 1,
windows_in_flight: 1,
};
let err = stream.begin_prefill(pool_bytes, config).unwrap_err();
assert!(
matches!(err, SweepError::ArenaTooSmall { .. }),
"unexpected error: {err}"
);
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
let widest = fx
.layout
.layers
.iter()
.map(|layer| layer.stride)
.max()
.unwrap();
let session = stream.begin_prefill(4097, config).expect("session opens");
assert_eq!(session.scratch_len(), 4097);
assert_eq!(session.ring_offset(), 8192, "the ring must start on a page");
assert_eq!(session.config(), config);
session.finish().unwrap();
let session = stream.begin_prefill(0, config).expect("session opens");
assert_eq!(session.ring_offset(), 0);
drop(session);
let too_big = pool_bytes - widest as usize + 4096;
let err = stream.begin_prefill(too_big, config).unwrap_err();
assert!(
matches!(err, SweepError::ArenaTooSmall { .. }),
"unexpected error: {err}"
);
}
#[test]
fn a_leaked_sweep_cannot_start_a_second_one_over_its_ring() {
let fx = build_install("prefill-leaked-sweep");
let mut stream = open(&fx, 4);
let config = SweepConfig {
experts_per_window: 1,
windows_in_flight: 2,
};
let mut plan = SweepPlan::new();
let routed0 = all_experts(&fx, 0);
let routed1 = all_experts(&fx, 1);
let mut session = stream.begin_prefill(0, config).expect("session opens");
let (_staging, sweep) = session
.split(&mut plan, 0, &routed0)
.expect("layer 0 sweeps");
std::mem::forget(sweep);
let outstanding = session.stream.reads_outstanding();
assert!(outstanding > 0, "the leak left no windows on the wire");
match session.split(&mut plan, 1, &routed1) {
Err(SweepError::ReadsInFlight { outstanding: seen }) => assert_eq!(seen, outstanding),
Err(other) => panic!("unexpected error: {other}"),
Ok(_) => panic!("a leaked sweep's live windows started a second sweep"),
}
session.finish().expect("the session drains what was left");
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
}
#[test]
fn a_leaked_prefill_session_cannot_take_a_second_arena() {
let fx = build_install("prefill-leaked-session");
let mut stream = open(&fx, 4);
let config = SweepConfig {
experts_per_window: 1,
windows_in_flight: 1,
};
let session = stream.begin_prefill(4096, config).expect("session opens");
let scratch = session.scratch_len();
std::mem::forget(session);
match stream.begin_prefill(scratch, config).unwrap_err() {
SweepError::ArenaOut { bytes } => assert!(bytes >= scratch),
other => panic!("unexpected error: {other}"),
}
let mut plan = SweepPlan::new();
let routed = all_experts(&fx, 0);
match stream
.sweep_layer(&mut plan, 0, &routed, config)
.unwrap_err()
{
SweepError::ArenaOut { .. } => {}
other => panic!("unexpected error: {other}"),
}
stream.release_arena();
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
}
#[test]
fn a_latched_failure_keeps_its_errno() {
let path = std::path::Path::new("layer_00.bin");
let enospc = SweepError::Io(IoError::io(
path,
std::io::Error::from_raw_os_error(libc::ENOSPC),
));
match latchable(&enospc) {
SweepError::Io(IoError::Io { source, .. }) => {
assert_eq!(source.raw_os_error(), Some(libc::ENOSPC));
}
other => panic!("unexpected error: {other}"),
}
let stranded = SweepError::CacheStranded {
layer: 3,
slots: 0,
top_k: 8,
source: IoError::io(path, std::io::Error::from_raw_os_error(libc::EIO)),
};
match latchable(&stranded) {
SweepError::CacheStranded {
layer,
slots,
top_k,
source: IoError::Io { source, .. },
} => {
assert_eq!((layer, slots, top_k), (3, 0, 8));
assert_eq!(source.raw_os_error(), Some(libc::EIO));
}
other => panic!("unexpected error: {other}"),
}
let eof = SweepError::Io(IoError::io(
path,
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "short read"),
));
match latchable(&eof) {
SweepError::Io(IoError::Io { source, .. }) => {
assert_eq!(source.kind(), std::io::ErrorKind::UnexpectedEof);
assert_eq!(source.to_string(), "short read");
}
other => panic!("unexpected error: {other}"),
}
let plain = SweepError::NoArena;
match latchable(&plain) {
SweepError::Aborted { reason } => assert_eq!(reason, plain.to_string()),
other => panic!("unexpected error: {other}"),
}
}
#[test]
fn a_layer_past_the_layout_is_a_typed_error() {
let fx = build_install("sweep-layer-range");
let mut stream = open(&fx, 4);
let mut plan = SweepPlan::new();
let err = stream
.sweep_layer(&mut plan, 9, &[0], SweepConfig::default())
.unwrap_err();
assert!(
matches!(
err,
SweepError::Io(IoError::LayerOutOfRange {
layer: 9,
n_layers: 2
})
),
"unexpected error: {err}"
);
}
}