use std::marker::PhantomData;
use std::mem::{align_of, size_of};
use std::ops::Range;
use std::sync::Mutex;
use std::time::{Duration, Instant};
use crate::format::{ArchInfo, ExpertsLayout};
use crate::io::{PrefillSession, SweepConfig, SweepExpert, SweepPlan};
use crate::kernels::KernelError;
use crate::kernels::attention::{
AttentionError, attention_at_in, scratch_len as attention_scratch_len,
};
use crate::kernels::primitives::{
rmsnorm, rmsnorm_in_place, rope_neox_heads, softmax, swiglu_combine,
};
use crate::kernels::quants::{
BlockQ8_0, BlockQ8K, QuantFormat, quantize_row_q8_0, quantize_row_q8_k,
};
use crate::kernels::{gemv_q8_0_batched, gemv_q8_k_batched};
use crate::kv::{KvCache, KvError};
use crate::threads::ComputePool;
use super::forward::{ForwardError, ForwardState, dot_f32, pool_gemv_q8_k, record, taken};
use super::weights::Model;
pub const DEFAULT_PREFILL_CHUNK: usize = 512;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum PrefillMode {
#[default]
Sweep,
TokenMajor,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PrefillConfig {
pub mode: PrefillMode,
pub chunk: usize,
pub experts_per_window: u32,
pub windows_in_flight: u32,
}
impl Default for PrefillConfig {
fn default() -> Self {
let sweep = SweepConfig::default();
Self {
mode: PrefillMode::default(),
chunk: DEFAULT_PREFILL_CHUNK,
experts_per_window: sweep.experts_per_window,
windows_in_flight: sweep.windows_in_flight,
}
}
}
impl PrefillConfig {
#[must_use]
pub fn from_env() -> Self {
let mut config = Self::default();
if let Ok(raw) = std::env::var("RAMVAMP_PREFILL") {
match raw.trim().to_ascii_lowercase().as_str() {
"sweep" => config.mode = PrefillMode::Sweep,
"token" | "token-major" | "token_major" => config.mode = PrefillMode::TokenMajor,
other => tracing::warn!(value = other, "ignoring unknown RAMVAMP_PREFILL"),
}
}
if let Ok(raw) = std::env::var("RAMVAMP_PREFILL_CHUNK") {
match raw.trim().parse::<usize>() {
Ok(chunk) if chunk > 0 => config.chunk = chunk,
_ => tracing::warn!(value = raw, "ignoring unusable RAMVAMP_PREFILL_CHUNK"),
}
}
config
}
#[must_use]
pub fn sweep_config(self) -> SweepConfig {
SweepConfig {
experts_per_window: self.experts_per_window,
windows_in_flight: self.windows_in_flight,
}
}
pub fn validate(self) -> Result<Self, ForwardError> {
if self.chunk == 0 {
return Err(ForwardError::InvalidPrefillChunk { chunk: self.chunk });
}
self.sweep_config().validate()?;
Ok(self)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum Phase {
Attention,
Projections,
Router,
ExpertCompute,
ExpertIo,
Elementwise,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct PrefillTiming {
pub mode: Option<PrefillMode>,
pub tokens: u64,
pub total: Duration,
pub attention: Duration,
pub projections: Duration,
pub router: Duration,
pub expert_compute: Duration,
pub expert_io: Duration,
pub elementwise: Duration,
}
impl PrefillTiming {
#[must_use]
pub fn accounted(&self) -> Duration {
self.attention
+ self.projections
+ self.router
+ self.expert_compute
+ self.expert_io
+ self.elementwise
}
#[must_use]
pub fn other(&self) -> Duration {
self.total.saturating_sub(self.accounted())
}
#[must_use]
pub fn ran(&self) -> bool {
self.mode.is_some()
}
#[must_use]
pub fn phases(&self) -> [(&'static str, Duration); 7] {
[
("attention", self.attention),
("expert compute", self.expert_compute),
("expert io", self.expert_io),
("projections", self.projections),
("router", self.router),
("elementwise", self.elementwise),
("other", self.other()),
]
}
pub(super) fn started(mode: PrefillMode, tokens: usize) -> Self {
Self {
mode: Some(mode),
tokens: tokens as u64,
..Self::default()
}
}
}
pub(super) struct PhaseClock<'a> {
timing: Option<&'a mut PrefillTiming>,
since: Instant,
whole: Option<Instant>,
}
impl<'a> PhaseClock<'a> {
pub(super) fn new(timing: &'a mut PrefillTiming, armed: bool) -> Self {
let now = Instant::now();
Self {
timing: armed.then_some(timing),
since: now,
whole: None,
}
}
pub(super) fn decoding(timing: &'a mut PrefillTiming) -> Self {
let now = Instant::now();
Self {
timing: Some(timing),
since: now,
whole: Some(now),
}
}
pub(super) fn close(self) {
let Some(whole) = self.whole else {
return;
};
if let Some(timing) = self.timing {
timing.total += whole.elapsed();
timing.tokens += 1;
}
}
#[inline]
pub(super) fn charge(&mut self, phase: Phase) {
let Some(timing) = self.timing.as_deref_mut() else {
return;
};
let now = Instant::now();
let slot = match phase {
Phase::Attention => &mut timing.attention,
Phase::Projections => &mut timing.projections,
Phase::Router => &mut timing.router,
Phase::ExpertCompute => &mut timing.expert_compute,
Phase::ExpertIo => &mut timing.expert_io,
Phase::Elementwise => &mut timing.elementwise,
};
*slot += now.saturating_duration_since(self.since);
self.since = now;
}
}
pub type PrefillRouteSink<'a> = &'a mut dyn FnMut(usize, u32, &[(u32, f32)]);
pub type PrefillProgressSink<'a> = &'a mut dyn FnMut(usize, usize);
unsafe trait ScratchPod: Copy {}
unsafe impl ScratchPod for f32 {}
unsafe impl ScratchPod for u32 {}
unsafe impl ScratchPod for BlockQ8K {}
unsafe impl ScratchPod for BlockQ8_0 {}
unsafe impl ScratchPod for TopkEntry {}
unsafe impl ScratchPod for RowSlot {}
#[derive(Debug, Clone, Copy, PartialEq, Default)]
#[repr(C)]
struct TopkEntry {
expert: u32,
weight: f32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[repr(C)]
struct RowSlot {
row: u32,
slot: u32,
}
struct Carver<'a> {
rest: &'a mut [u8],
used: usize,
}
impl<'a> Carver<'a> {
fn new(bytes: &'a mut [u8]) -> Self {
Self {
rest: bytes,
used: 0,
}
}
fn used(&self) -> usize {
self.used
}
fn take<T: ScratchPod>(&mut self, len: usize) -> Result<&'a mut [T], ForwardError> {
let align = align_of::<T>();
let pad = self.rest.as_ptr().align_offset(align);
if pad == usize::MAX {
return Err(ForwardError::PrefillScratchAlign { align });
}
let want = len
.checked_mul(size_of::<T>())
.and_then(|bytes| bytes.checked_add(pad))
.ok_or(ForwardError::PrefillScratch {
needed: u64::MAX,
available: self.rest.len() as u64,
})?;
if want > self.rest.len() {
return Err(ForwardError::PrefillScratch {
needed: (self.used + want) as u64,
available: (self.used + self.rest.len()) as u64,
});
}
let rest = std::mem::take(&mut self.rest);
let (head, tail) = rest.split_at_mut(want);
self.rest = tail;
self.used += want;
let ptr = head.as_mut_ptr();
Ok(unsafe { std::slice::from_raw_parts_mut(ptr.add(pad).cast::<T>(), len) })
}
}
struct Scratch<'a> {
residual: &'a mut [f32],
normed: &'a mut [f32],
q: &'a mut [f32],
attn_out: &'a mut [f32],
k: &'a mut [f32],
v: &'a mut [f32],
tmat: &'a mut [f32],
staged: &'a mut [f32],
gate: &'a mut [f32],
up: &'a mut [f32],
moe_t: &'a mut [f32],
attn_shards: &'a mut [f32],
router_logits: &'a mut [f32],
router_probs: &'a mut [f32],
acts_hidden: &'a mut [BlockQ8K],
acts_gather: &'a mut [BlockQ8K],
acts_attn: &'a mut [BlockQ8K],
acts_moe: &'a mut [BlockQ8K],
acts_q80: &'a mut [BlockQ8_0],
topk: &'a mut [TopkEntry],
index: &'a mut [RowSlot],
index_start: &'a mut [u32],
index_cursor: &'a mut [u32],
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct PrefillDims {
hidden: usize,
q_dim: usize,
kv_dim: usize,
moe: usize,
top_k: usize,
n_experts: usize,
n_heads: usize,
n_kv_heads: usize,
head_dim: usize,
hidden_q8k: usize,
hidden_q80: usize,
attn_q8k: usize,
moe_q8k: usize,
wide: usize,
shards: usize,
max_positions: usize,
}
const QK_K: usize = 256;
const QK8_0: usize = 32;
impl PrefillDims {
fn attn_shard(&self) -> usize {
attention_scratch_len(
self.n_heads,
self.n_kv_heads,
self.head_dim,
self.max_positions,
)
}
fn attn_scratch(&self) -> usize {
self.shards.saturating_mul(self.attn_shard())
}
fn new(arch: &ArchInfo, shards: usize, max_positions: usize) -> Result<Self, ForwardError> {
let hidden = arch.hidden as usize;
let q_dim = arch.n_heads as usize * arch.head_dim as usize;
let kv_dim = arch.n_kv_heads as usize * arch.head_dim as usize;
let moe = arch.moe_intermediate as usize;
let top_k = arch.top_k as usize;
let n_experts = arch.n_experts as usize;
for (what, dim, block) in [
("hidden", hidden, QK_K),
("hidden", hidden, QK8_0),
("moe_intermediate", moe, QK_K),
("n_heads * head_dim", q_dim, QK_K),
] {
if dim == 0 || dim % block != 0 {
return Err(ForwardError::UnsupportedDim { what, dim, block });
}
}
if top_k == 0 || top_k > n_experts {
return Err(ForwardError::InvalidTopK { top_k, n_experts });
}
if kv_dim == 0 || arch.head_dim == 0 {
return Err(ForwardError::UnsupportedDim {
what: "n_kv_heads * head_dim",
dim: kv_dim,
block: 1,
});
}
Ok(Self {
hidden,
q_dim,
kv_dim,
moe,
top_k,
n_experts,
n_heads: arch.n_heads as usize,
n_kv_heads: arch.n_kv_heads as usize,
head_dim: arch.head_dim as usize,
hidden_q8k: hidden / QK_K,
hidden_q80: hidden / QK8_0,
attn_q8k: q_dim / QK_K,
moe_q8k: moe / QK_K,
wide: q_dim.max(hidden),
shards: shards.max(1),
max_positions,
})
}
}
fn accumulate(total: &mut usize, count: usize, unit: usize) -> Result<(), ForwardError> {
let bytes = count
.checked_mul(unit)
.and_then(|bytes| total.checked_add(bytes))
.ok_or(ForwardError::PrefillScratch {
needed: u64::MAX,
available: *total as u64,
})?;
*total = bytes;
Ok(())
}
fn scratch_bytes(dims: &PrefillDims, rows: usize) -> Result<usize, ForwardError> {
let f32_bytes = size_of::<f32>();
let mut total = 0usize;
for width in [
dims.hidden,
dims.hidden,
dims.q_dim,
dims.q_dim,
dims.kv_dim,
dims.kv_dim,
dims.wide,
dims.top_k * dims.hidden,
dims.moe,
dims.moe,
dims.moe,
] {
accumulate(&mut total, rows.saturating_mul(width), f32_bytes)?;
}
accumulate(&mut total, dims.attn_scratch(), f32_bytes)?;
accumulate(&mut total, rows.saturating_mul(dims.n_experts), f32_bytes)?;
accumulate(&mut total, rows.saturating_mul(dims.n_experts), f32_bytes)?;
let q8k = size_of::<BlockQ8K>();
for blocks in [
dims.hidden_q8k,
dims.hidden_q8k,
dims.attn_q8k,
dims.moe_q8k,
] {
accumulate(&mut total, rows.saturating_mul(blocks), q8k)?;
}
accumulate(
&mut total,
rows.saturating_mul(dims.hidden_q80),
size_of::<BlockQ8_0>(),
)?;
let pairs = rows.saturating_mul(dims.top_k);
accumulate(&mut total, pairs, size_of::<TopkEntry>())?;
accumulate(&mut total, pairs, size_of::<RowSlot>())?;
accumulate(&mut total, dims.n_experts + 1, size_of::<u32>())?;
accumulate(&mut total, dims.n_experts, size_of::<u32>())?;
Ok(total)
}
fn carve<'a>(
bytes: &'a mut [u8],
dims: &PrefillDims,
rows: usize,
) -> Result<(Scratch<'a>, usize), ForwardError> {
let mut c = Carver::new(bytes);
let scratch = Scratch {
residual: c.take(rows * dims.hidden)?,
normed: c.take(rows * dims.hidden)?,
q: c.take(rows * dims.q_dim)?,
attn_out: c.take(rows * dims.q_dim)?,
k: c.take(rows * dims.kv_dim)?,
v: c.take(rows * dims.kv_dim)?,
tmat: c.take(rows * dims.wide)?,
staged: c.take(rows * dims.top_k * dims.hidden)?,
gate: c.take(rows * dims.moe)?,
up: c.take(rows * dims.moe)?,
moe_t: c.take(rows * dims.moe)?,
attn_shards: c.take(dims.attn_scratch())?,
router_logits: c.take(rows * dims.n_experts)?,
router_probs: c.take(rows * dims.n_experts)?,
acts_hidden: c.take(rows * dims.hidden_q8k)?,
acts_gather: c.take(rows * dims.hidden_q8k)?,
acts_attn: c.take(rows * dims.attn_q8k)?,
acts_moe: c.take(rows * dims.moe_q8k)?,
acts_q80: c.take(rows * dims.hidden_q80)?,
topk: c.take(rows * dims.top_k)?,
index: c.take(rows * dims.top_k)?,
index_start: c.take(dims.n_experts + 1)?,
index_cursor: c.take(dims.n_experts)?,
};
Ok((scratch, c.used()))
}
struct Rows<'a, T> {
base: *mut T,
width: usize,
rows: usize,
_borrow: PhantomData<&'a mut [T]>,
}
impl<'a, T> Rows<'a, T> {
fn new(buf: &'a mut [T], width: usize, rows: usize) -> Result<Self, ForwardError> {
let need = width.checked_mul(rows).ok_or(KernelError::LengthMismatch {
what: "prefill rows: width * rows overflows",
left: width,
right: rows,
})?;
if buf.len() < need {
return Err(KernelError::LengthMismatch {
what: "prefill rows: buffer vs width * rows",
left: buf.len(),
right: need,
}
.into());
}
Ok(Self {
base: buf.as_mut_ptr(),
width,
rows,
_borrow: PhantomData,
})
}
#[allow(clippy::mut_from_ref)]
unsafe fn row(&self, r: usize) -> &mut [T] {
assert!(r < self.rows, "prefill rows: row {r} of {}", self.rows);
unsafe { std::slice::from_raw_parts_mut(self.base.add(r * self.width), self.width) }
}
}
unsafe impl<T: Send> Send for Rows<'_, T> {}
unsafe impl<T: Send> Sync for Rows<'_, T> {}
fn for_each_row<F>(pool: &mut ComputePool, n: usize, body: F) -> Result<(), ForwardError>
where
F: Fn(usize) -> Result<(), ForwardError> + Sync,
{
let failure: Mutex<Option<(usize, ForwardError)>> = Mutex::new(None);
pool.run_chunks(n, |chunk| {
for r in chunk.rows {
if let Err(err) = body(r) {
record(&failure, chunk.index, err);
return;
}
}
});
taken(failure)
}
pub(super) struct SendPtr(pub(super) *mut f32);
impl SendPtr {
pub(super) fn get(&self) -> *mut f32 {
self.0
}
}
unsafe impl Send for SendPtr {}
unsafe impl Sync for SendPtr {}
fn scatter_rows<F>(
pool: &mut ComputePool,
out: &mut [f32],
out_dim: usize,
n_acts: usize,
f: F,
) -> Result<(), KernelError>
where
F: Fn(Range<usize>, &mut [f32]) -> Result<(), KernelError> + Sync,
{
let expected = out_dim.saturating_mul(n_acts);
if out.len() != expected {
return Err(KernelError::LengthMismatch {
what: "prefill batched gemv: out vs out_dim * n_acts",
left: out.len(),
right: expected,
});
}
if n_acts == 0 || out_dim == 0 {
return Ok(());
}
let failure: Mutex<Option<(usize, KernelError)>> = Mutex::new(None);
let base = SendPtr(out.as_mut_ptr());
pool.run_chunks(out_dim, |chunk| {
let index = chunk.index;
let start = chunk.rows.start * n_acts;
let len = chunk.rows.len() * n_acts;
let dst = unsafe { std::slice::from_raw_parts_mut(base.get().add(start), len) };
if let Err(err) = f(chunk.rows, dst) {
record(&failure, index, err);
}
});
taken(failure)
}
#[allow(clippy::too_many_arguments)]
fn pool_batched_q8_k(
pool: &mut ComputePool,
format: QuantFormat,
weight: &[u8],
in_dim: usize,
out_dim: usize,
acts: &[BlockQ8K],
n_acts: usize,
out: &mut [f32],
) -> Result<(), KernelError> {
scatter_rows(pool, out, out_dim, n_acts, |rows, chunk| {
gemv_q8_k_batched(format, weight, in_dim, out_dim, acts, n_acts, rows, chunk)
})
}
fn pool_batched_q8_0(
pool: &mut ComputePool,
weight: &[u8],
in_dim: usize,
out_dim: usize,
acts: &[BlockQ8_0],
n_acts: usize,
out: &mut [f32],
) -> Result<(), KernelError> {
scatter_rows(pool, out, out_dim, n_acts, |rows, chunk| {
gemv_q8_0_batched(weight, in_dim, out_dim, acts, n_acts, rows, chunk)
})
}
fn attention_cost(rows: usize, start: usize) -> u128 {
let rows = rows as u128;
rows * start as u128 + rows * (rows + 1) / 2
}
fn attention_shard_range(rows: usize, start: usize, shards: usize, index: usize) -> Range<usize> {
if shards == 0 || index >= shards {
return 0..0;
}
if shards == 1 {
return 0..rows;
}
let total = attention_cost(rows, start);
let lo = attention_boundary(rows, start, total, shards, index);
let hi = attention_boundary(rows, start, total, shards, index + 1);
lo..hi
}
fn attention_boundary(
rows: usize,
start: usize,
total: u128,
shards: usize,
index: usize,
) -> usize {
if index == 0 {
return 0;
}
if index >= shards {
return rows;
}
let target = total * index as u128 / shards as u128;
let (mut lo, mut hi) = (0usize, rows);
while lo < hi {
let mid = lo + (hi - lo) / 2;
if attention_cost(mid, start) < target {
lo = mid + 1;
} else {
hi = mid;
}
}
lo
}
#[allow(clippy::too_many_arguments)]
fn scatter_attention(
pool: &mut ComputePool,
kv: &KvCache,
layer: usize,
start: usize,
rows: usize,
q_dim: usize,
scale: f32,
shard_scratch: usize,
q: &[f32],
scratch: &mut [f32],
out: &mut [f32],
) -> Result<(), ForwardError> {
if rows == 0 {
return Ok(());
}
let span = rows.saturating_mul(q_dim);
for (what, len) in [
("attention fan-out: q", q.len()),
("attention fan-out: out", out.len()),
] {
if len != span {
return Err(KernelError::LengthMismatch {
what,
left: len,
right: span,
}
.into());
}
}
let need = pool.shards().saturating_mul(shard_scratch);
if scratch.len() < need {
return Err(ForwardError::PrefillScratch {
needed: need as u64,
available: scratch.len() as u64,
});
}
let failure: Mutex<Option<(usize, AttentionError)>> = Mutex::new(None);
let out_base = SendPtr(out.as_mut_ptr());
let scratch_base = SendPtr(scratch.as_mut_ptr());
pool.run(rows, |shard| {
let index = shard.index;
let mine = attention_shard_range(rows, start, shard.count, index);
let slice = unsafe {
std::slice::from_raw_parts_mut(
scratch_base.get().add(index * shard_scratch),
shard_scratch,
)
};
for r in mine {
let row_out =
unsafe { std::slice::from_raw_parts_mut(out_base.get().add(r * q_dim), q_dim) };
if let Err(err) = attention_at_in(
&q[r * q_dim..(r + 1) * q_dim],
kv,
layer,
start + r + 1,
scale,
slice,
row_out,
) {
record(&failure, index, err);
return;
}
}
});
taken(failure)?;
Ok(())
}
fn transpose(
pool: &mut ComputePool,
src: &[f32],
rows: usize,
cols: usize,
dst: &mut [f32],
) -> Result<(), ForwardError> {
let expected = rows.checked_mul(cols).ok_or(KernelError::LengthMismatch {
what: "prefill transpose: rows * cols overflows",
left: rows,
right: cols,
})?;
for (what, len) in [
("prefill transpose: src vs rows * cols", src.len()),
("prefill transpose: dst vs rows * cols", dst.len()),
] {
if len != expected {
return Err(KernelError::LengthMismatch {
what,
left: len,
right: expected,
}
.into());
}
}
if expected == 0 {
return Ok(());
}
let out = Rows::new(dst, rows, cols)?;
for_each_row(pool, cols, |c| {
let out = unsafe { out.row(c) };
for (r, value) in out.iter_mut().enumerate() {
*value = src[r * cols + c];
}
Ok(())
})
}
fn ring_bytes(layout: &ExpertsLayout, experts_per_window: u32, windows_in_flight: u32) -> u64 {
let widest = layout
.layers
.iter()
.map(|layer| u64::from(experts_per_window.min(layer.n_experts.max(1))) * layer.stride)
.max()
.unwrap_or(0);
widest.saturating_mul(u64::from(windows_in_flight))
}
fn plan_arena(
layout: &ExpertsLayout,
dims: &PrefillDims,
config: PrefillConfig,
want_rows: usize,
pool_bytes: u64,
) -> Result<(usize, SweepConfig), ForwardError> {
const PAGE: u64 = 4096;
let config = config.validate()?;
let mut in_flight = config.windows_in_flight;
let mut smallest_need = u64::MAX;
loop {
let ring = ring_bytes(layout, config.experts_per_window, in_flight);
let budget = scratch_budget(pool_bytes, ring, PAGE);
let one = scratch_bytes(dims, 1)? as u64;
let two = scratch_bytes(dims, 2)? as u64;
let per_row = two.saturating_sub(one).max(1);
let fixed = one.saturating_sub(per_row);
let mut rows = want_rows
.max(1)
.min(usize::try_from(budget.saturating_sub(fixed) / per_row).unwrap_or(usize::MAX));
while rows > 0 && scratch_bytes(dims, rows)? as u64 > budget {
rows -= 1;
}
if rows > 0 {
if rows < want_rows || in_flight < config.windows_in_flight {
tracing::debug!(
asked_rows = want_rows,
rows,
asked_windows = config.windows_in_flight,
windows_in_flight = in_flight,
pool_bytes,
"prefill narrowed to fit the expert slot slab"
);
}
return Ok((
rows,
SweepConfig {
experts_per_window: config.experts_per_window,
windows_in_flight: in_flight,
},
));
}
smallest_need = smallest_need.min(one.saturating_add(ring));
if in_flight == 1 {
return Err(ForwardError::PrefillScratch {
needed: smallest_need,
available: pool_bytes,
});
}
in_flight -= 1;
}
}
fn scratch_budget(pool_bytes: u64, ring: u64, page: u64) -> u64 {
pool_bytes.saturating_sub(ring) / page * page
}
struct ChunkState<'a> {
kv: &'a mut KvCache,
pool: &'a mut ComputePool,
plan: &'a mut SweepPlan,
routed: &'a mut Vec<u32>,
sink_buf: &'a mut Vec<(u32, f32)>,
logits: &'a mut [f32],
}
pub fn prefill_prompt<'s>(
model: &Model,
state: &'s mut ForwardState,
tokens: &[u32],
on_route: Option<PrefillRouteSink<'_>>,
) -> Result<&'s [f32], ForwardError> {
prefill_prompt_with_progress(model, state, tokens, on_route, None)
}
pub fn prefill_prompt_with_progress<'s>(
model: &Model,
state: &'s mut ForwardState,
tokens: &[u32],
on_route: Option<PrefillRouteSink<'_>>,
on_progress: Option<PrefillProgressSink<'_>>,
) -> Result<&'s [f32], ForwardError> {
prefill(model, state, tokens, on_route, on_progress, None)
}
pub fn prefill_prompt_scored<'s>(
model: &Model,
state: &'s mut ForwardState,
tokens: &[u32],
first: usize,
out: &mut [f64],
) -> Result<&'s [f32], ForwardError> {
prefill_scored_in_blocks(model, state, tokens, first, out, MAX_SCORE_BLOCK)
}
fn prefill_scored_in_blocks<'s>(
model: &Model,
state: &'s mut ForwardState,
tokens: &[u32],
first: usize,
out: &mut [f64],
block: usize,
) -> Result<&'s [f32], ForwardError> {
let scored = tokens
.len()
.checked_sub(1)
.and_then(|last| last.checked_sub(first))
.filter(|&count| count > 0);
if scored != Some(out.len()) {
return Err(ForwardError::ScoreRange {
first,
prompt: tokens.len(),
out: out.len(),
});
}
let vocab = model.arch().vocab;
if let Some(&token) = tokens.iter().find(|&&t| t >= vocab) {
return Err(super::ModelError::TokenOutOfRange { token, vocab }.into());
}
let mut score = Score {
tokens,
first,
out,
block: block.clamp(1, MAX_SCORE_BLOCK),
};
prefill(model, state, tokens, None, None, Some(&mut score))
}
const MAX_SCORE_BLOCK: usize = 64;
const SCORE_TILE: usize = 4;
struct Score<'a> {
tokens: &'a [u32],
first: usize,
out: &'a mut [f64],
block: usize,
}
impl Score<'_> {
fn wants(&self, i: usize) -> bool {
i >= self.first && i + 1 < self.tokens.len()
}
}
fn prefill<'s>(
model: &Model,
state: &'s mut ForwardState,
tokens: &[u32],
on_route: Option<PrefillRouteSink<'_>>,
on_progress: Option<PrefillProgressSink<'_>>,
score: Option<&mut Score<'_>>,
) -> Result<&'s [f32], ForwardError> {
if tokens.is_empty() {
return Err(ForwardError::EmptyPrefill);
}
let mode = state.prefill_config().validate()?.mode;
state.arm_prefill_timing(mode, tokens.len());
let started = Instant::now();
let outcome = match mode {
PrefillMode::TokenMajor => {
prefill_token_major(model, state, tokens, on_route, on_progress, score)
}
PrefillMode::Sweep => prefill_sweep(model, state, tokens, on_route, on_progress, score),
};
state.close_prefill_timing(started.elapsed());
outcome?;
Ok(state.logits())
}
fn prefill_token_major(
model: &Model,
state: &mut ForwardState,
tokens: &[u32],
mut on_route: Option<PrefillRouteSink<'_>>,
mut on_progress: Option<PrefillProgressSink<'_>>,
mut score: Option<&mut Score<'_>>,
) -> Result<(), ForwardError> {
let start = state.seq_len()?;
let last = tokens.len() - 1;
for (i, &id) in tokens.iter().enumerate() {
let position = start + i;
let scored = score.as_deref().is_some_and(|score| score.wants(i));
let want_logits = i == last || scored;
match on_route.as_deref_mut() {
Some(sink) => {
let mut per_layer = |layer: u32, topk: &[(u32, f32)]| sink(position, layer, topk);
super::forward_token_traced(
model,
state,
id,
position,
want_logits,
Some(&mut per_layer),
)?;
}
None => {
super::forward_token(model, state, id, position, want_logits)?;
}
}
if scored && let Some(score) = score.as_deref_mut() {
score.out[i - score.first] = log_prob(state.logits(), tokens[i + 1])?;
}
if let Some(progress) = on_progress.as_deref_mut() {
progress(i + 1, tokens.len());
}
}
Ok(())
}
fn prefill_sweep(
model: &Model,
state: &mut ForwardState,
tokens: &[u32],
mut on_route: Option<PrefillRouteSink<'_>>,
mut on_progress: Option<PrefillProgressSink<'_>>,
mut score: Option<&mut Score<'_>>,
) -> Result<(), ForwardError> {
let arch = model.arch();
state.check_arch(arch)?;
let dims = PrefillDims::new(arch, state.shards(), state.context_cap())?;
let config = state.prefill_config().validate()?;
let start = state.seq_len()?;
let capacity = state.context_cap();
if start.saturating_add(tokens.len()) > capacity {
return Err(ForwardError::Kv(KvError::CapacityExceeded {
layer: 0,
capacity,
}));
}
let want_rows = config.chunk.min(tokens.len()).max(1);
let (rows, sweep_config) = plan_arena(
model.layout(),
&dims,
config,
want_rows,
state.cache_bytes(),
)?;
let bytes = scratch_bytes(&dims, rows)?;
tracing::debug!(
rows,
prompt = tokens.len(),
scratch_bytes = bytes,
sweep = %sweep_config,
"prefill sweep starting"
);
let parts = state.prefill_parts();
let timing = parts.timing;
let mut chunk_state = ChunkState {
kv: parts.kv,
pool: parts.pool,
plan: parts.sweep_plan,
routed: parts.routed,
sink_buf: parts.topk,
logits: parts.logits,
};
let mut session = parts.stream.begin_prefill(bytes, sweep_config)?;
let mut clock = PhaseClock::new(timing, true);
let mut position = start;
let mut rest = tokens;
while !rest.is_empty() {
let n = rows.min(rest.len());
let (chunk, tail) = rest.split_at(n);
run_chunk(
model,
arch,
&dims,
rows,
&mut session,
&mut chunk_state,
chunk,
position,
tail.is_empty(),
&mut on_route,
score.as_deref_mut().map(|score| (score, position - start)),
&mut clock,
)?;
position += n;
rest = tail;
if let Some(progress) = on_progress.as_deref_mut() {
progress(position - start, tokens.len());
}
}
session.finish()?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn run_chunk(
model: &Model,
arch: &ArchInfo,
dims: &PrefillDims,
rows: usize,
session: &mut PrefillSession<'_>,
state: &mut ChunkState<'_>,
tokens: &[u32],
start: usize,
want_logits: bool,
on_route: &mut Option<PrefillRouteSink<'_>>,
score: Option<(&mut Score<'_>, usize)>,
clock: &mut PhaseClock<'_>,
) -> Result<(), ForwardError> {
let n = tokens.len();
let eps = arch.rms_eps as f32;
let theta = arch.rope_theta as f32;
let scale = 1.0 / (dims.head_dim as f32).sqrt();
let (hidden, q_dim, kv_dim, top_k) = (dims.hidden, dims.q_dim, dims.kv_dim, dims.top_k);
{
let (s, _) = carve(session.scratch(), dims, rows)?;
for (r, &token) in tokens.iter().enumerate() {
model.embed(token, &mut s.residual[r * hidden..(r + 1) * hidden])?;
}
}
for layer in 0..model.n_layers() {
let lw = model.layer(layer)?;
let layer_idx = layer as usize;
{
let (mut s, _) = carve(session.scratch(), dims, rows)?;
{
let residual = &*s.residual;
let normed = Rows::new(&mut *s.normed, hidden, n)?;
let acts_k = Rows::new(&mut *s.acts_hidden, dims.hidden_q8k, n)?;
let acts_0 = Rows::new(&mut *s.acts_q80, dims.hidden_q80, n)?;
for_each_row(state.pool, n, |r| {
let (normed, acts_k, acts_0) =
unsafe { (normed.row(r), acts_k.row(r), acts_0.row(r)) };
rmsnorm(
&residual[r * hidden..(r + 1) * hidden],
lw.attn_norm,
eps,
normed,
)?;
quantize_row_q8_k(normed, acts_k)?;
quantize_row_q8_0(normed, acts_0)?;
Ok(())
})?;
}
clock.charge(Phase::Elementwise);
pool_batched_q8_k(
state.pool,
lw.attn_q.format,
lw.attn_q.bytes,
hidden,
q_dim,
&s.acts_hidden[..n * dims.hidden_q8k],
n,
&mut s.tmat[..q_dim * n],
)?;
clock.charge(Phase::Projections);
transpose(
state.pool,
&s.tmat[..q_dim * n],
q_dim,
n,
&mut s.q[..n * q_dim],
)?;
clock.charge(Phase::Elementwise);
pool_batched_q8_0(
state.pool,
lw.attn_k.bytes,
hidden,
kv_dim,
&s.acts_q80[..n * dims.hidden_q80],
n,
&mut s.tmat[..kv_dim * n],
)?;
clock.charge(Phase::Projections);
transpose(
state.pool,
&s.tmat[..kv_dim * n],
kv_dim,
n,
&mut s.k[..n * kv_dim],
)?;
clock.charge(Phase::Elementwise);
pool_batched_q8_k(
state.pool,
lw.attn_v.format,
lw.attn_v.bytes,
hidden,
kv_dim,
&s.acts_hidden[..n * dims.hidden_q8k],
n,
&mut s.tmat[..kv_dim * n],
)?;
clock.charge(Phase::Projections);
transpose(
state.pool,
&s.tmat[..kv_dim * n],
kv_dim,
n,
&mut s.v[..n * kv_dim],
)?;
{
let q_rows = Rows::new(&mut *s.q, q_dim, n)?;
let k_rows = Rows::new(&mut *s.k, kv_dim, n)?;
for_each_row(state.pool, n, |r| {
let position = start + r;
let rope_pos = u32::try_from(position)
.map_err(|_| ForwardError::PositionOverflow { position })?;
let (q, k) = unsafe { (q_rows.row(r), k_rows.row(r)) };
for head in q.chunks_exact_mut(dims.head_dim) {
rmsnorm_in_place(head, lw.attn_q_norm, eps)?;
}
rope_neox_heads(q, dims.n_heads, dims.head_dim, rope_pos, theta)?;
for head in k.chunks_exact_mut(dims.head_dim) {
rmsnorm_in_place(head, lw.attn_k_norm, eps)?;
}
rope_neox_heads(k, dims.n_kv_heads, dims.head_dim, rope_pos, theta)?;
Ok(())
})?;
}
for r in 0..n {
state.kv.append(
layer_idx,
&s.k[r * kv_dim..(r + 1) * kv_dim],
&s.v[r * kv_dim..(r + 1) * kv_dim],
)?;
}
clock.charge(Phase::Elementwise);
scatter_attention(
state.pool,
state.kv,
layer_idx,
start,
n,
q_dim,
scale,
dims.attn_shard(),
&s.q[..n * q_dim],
s.attn_shards,
&mut s.attn_out[..n * q_dim],
)?;
clock.charge(Phase::Attention);
{
let attn_out = &*s.attn_out;
let acts = Rows::new(&mut *s.acts_attn, dims.attn_q8k, n)?;
for_each_row(state.pool, n, |r| {
let acts = unsafe { acts.row(r) };
quantize_row_q8_k(&attn_out[r * q_dim..(r + 1) * q_dim], acts)?;
Ok(())
})?;
}
clock.charge(Phase::Elementwise);
pool_batched_q8_k(
state.pool,
lw.attn_output.format,
lw.attn_output.bytes,
q_dim,
hidden,
&s.acts_attn[..n * dims.attn_q8k],
n,
&mut s.tmat[..hidden * n],
)?;
clock.charge(Phase::Projections);
{
let tmat = &*s.tmat;
let residual_rows = Rows::new(&mut *s.residual, hidden, n)?;
let normed = Rows::new(&mut *s.normed, hidden, n)?;
let acts_k = Rows::new(&mut *s.acts_hidden, dims.hidden_q8k, n)?;
for_each_row(state.pool, n, |r| {
let (residual, normed, acts_k) =
unsafe { (residual_rows.row(r), normed.row(r), acts_k.row(r)) };
for (h, value) in residual.iter_mut().enumerate() {
*value += tmat[h * n + r];
}
rmsnorm(residual, lw.ffn_norm, eps, normed)?;
quantize_row_q8_k(normed, acts_k)?;
Ok(())
})?;
}
clock.charge(Phase::Elementwise);
{
let normed = &*s.normed;
let router = lw.router.data();
let logits_rows = Rows::new(&mut *s.router_logits, dims.n_experts, n)?;
let probs_rows = Rows::new(&mut *s.router_probs, dims.n_experts, n)?;
let topk_rows = Rows::new(&mut *s.topk, top_k, n)?;
for_each_row(state.pool, n, |r| {
let normed = &normed[r * hidden..(r + 1) * hidden];
let (logits, probs, topk) =
unsafe { (logits_rows.row(r), probs_rows.row(r), topk_rows.row(r)) };
for (row, logit) in router.chunks_exact(hidden).zip(logits.iter_mut()) {
*logit = dot_f32(row, normed);
}
probs.copy_from_slice(logits);
softmax(probs)?;
for entry in topk.iter_mut() {
let mut best_e = 0usize;
let mut best_p = f32::NEG_INFINITY;
for (e, &p) in probs.iter().enumerate() {
if p > best_p {
best_p = p;
best_e = e;
}
}
*entry = TopkEntry {
expert: best_e as u32,
weight: best_p,
};
probs[best_e] = f32::NEG_INFINITY;
}
if arch.norm_topk_prob {
let sum: f32 = topk.iter().map(|entry| entry.weight).sum();
for entry in topk.iter_mut() {
entry.weight /= sum;
}
}
Ok(())
})?;
}
if let Some(sink) = on_route.as_deref_mut() {
for r in 0..n {
let topk = &s.topk[r * top_k..(r + 1) * top_k];
state.sink_buf.clear();
state
.sink_buf
.extend(topk.iter().map(|entry| (entry.expert, entry.weight)));
sink(start + r, layer, state.sink_buf);
}
}
clock.charge(Phase::Router);
build_index(&mut s, dims, n)?;
state.routed.clear();
for expert in 0..dims.n_experts {
if s.index_start[expert + 1] > s.index_start[expert] {
state.routed.push(expert as u32);
}
}
clock.charge(Phase::Elementwise);
}
let covered = {
let (bytes, mut sweep) = session.split(state.plan, layer, state.routed)?;
let (mut s, _) = carve(bytes, dims, rows)?;
let mut covered = 0usize;
loop {
let next = sweep.next_expert()?;
clock.charge(Phase::ExpertIo);
let Some(expert) = next else { break };
covered += run_expert(state.pool, dims, &mut s, n, &expert)?;
clock.charge(Phase::ExpertCompute);
}
sweep.finish()?;
clock.charge(Phase::ExpertIo);
covered
};
if covered != n * top_k {
return Err(ForwardError::PrefillCoverage {
layer,
covered,
expected: n * top_k,
});
}
{
let (mut s, _) = carve(session.scratch(), dims, rows)?;
reduce_experts(state.pool, &mut s, dims, n)?;
}
clock.charge(Phase::Elementwise);
}
if let Some((score, offset)) = score {
score_rows(
model, dims, rows, session, state.pool, score, offset, n, clock,
)?;
}
if want_logits {
let (s, _) = carve(session.scratch(), dims, rows)?;
let last = n - 1;
let residual = &s.residual[last * hidden..(last + 1) * hidden];
let normed = &mut s.normed[last * hidden..(last + 1) * hidden];
rmsnorm(residual, model.final_norm(), eps, normed)?;
let acts = &mut s.acts_hidden[last * dims.hidden_q8k..(last + 1) * dims.hidden_q8k];
quantize_row_q8_k(normed, acts)?;
clock.charge(Phase::Elementwise);
let head = model.lm_head();
pool_gemv_q8_k(
state.pool,
head.format,
head.bytes,
head.in_dim,
head.out_dim,
acts,
state.logits,
)?;
clock.charge(Phase::Projections);
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn score_rows(
model: &Model,
dims: &PrefillDims,
rows: usize,
session: &mut PrefillSession<'_>,
pool: &mut ComputePool,
score: &mut Score<'_>,
offset: usize,
n: usize,
clock: &mut PhaseClock<'_>,
) -> Result<(), ForwardError> {
let lo = score.first.saturating_sub(offset).min(n);
let hi = (score.tokens.len() - 1).saturating_sub(offset).min(n);
if lo >= hi {
return Ok(());
}
let (hidden, hidden_q8k) = (dims.hidden, dims.hidden_q8k);
let eps = model.arch().rms_eps as f32;
let final_norm = model.final_norm();
let (s, _) = carve(session.scratch(), dims, rows)?;
{
let residual = &*s.residual;
let normed = Rows::new(&mut *s.normed, hidden, hi)?;
let acts = Rows::new(&mut *s.acts_hidden, hidden_q8k, hi)?;
for_each_row(pool, hi - lo, |j| {
let r = lo + j;
let (normed, acts) = unsafe { (normed.row(r), acts.row(r)) };
rmsnorm(
&residual[r * hidden..(r + 1) * hidden],
final_norm,
eps,
normed,
)?;
quantize_row_q8_k(normed, acts)?;
Ok(())
})?;
}
clock.charge(Phase::Elementwise);
let head = model.lm_head();
let vocab = head.out_dim;
let block = score_block(score.block, s.staged.len(), vocab)?;
let mut r = lo;
while r < hi {
let count = block.min(hi - r);
let logits = &mut s.staged[..vocab * count];
pool_batched_q8_k(
pool,
head.format,
head.bytes,
head.in_dim,
vocab,
&s.acts_hidden[r * hidden_q8k..(r + count) * hidden_q8k],
count,
logits,
)?;
clock.charge(Phase::Projections);
let at = offset + r;
let slots = at - score.first..at - score.first + count;
block_log_probs(
logits,
count,
&score.tokens[at + 1..at + 1 + count],
&mut score.out[slots],
)?;
clock.charge(Phase::Elementwise);
r += count;
}
Ok(())
}
fn score_block(requested: usize, staged: usize, vocab: usize) -> Result<usize, ForwardError> {
let block = requested.min(staged / vocab.max(1));
if block == 0 {
return Err(ForwardError::PrefillScratch {
needed: (vocab * size_of::<f32>()) as u64,
available: (staged * size_of::<f32>()) as u64,
});
}
Ok(if block >= SCORE_TILE {
block - block % SCORE_TILE
} else {
block
})
}
fn log_prob(logits: &[f32], target: u32) -> Result<f64, ForwardError> {
let t = target as usize;
let Some(&picked) = logits.get(t) else {
return Err(KernelError::LengthMismatch {
what: "log_prob: target vs vocab",
left: t,
right: logits.len(),
}
.into());
};
let mut max = logits[0];
for &l in &logits[1..] {
if max < l {
max = l;
}
}
let mut sum = 0.0f64;
for &l in logits {
sum += f64::from((l - max).exp());
}
Ok(f64::from(picked - max) - sum.ln())
}
fn block_log_probs(
block: &[f32],
n: usize,
targets: &[u32],
out: &mut [f64],
) -> Result<(), ForwardError> {
if n == 0
|| n > MAX_SCORE_BLOCK
|| block.is_empty()
|| !block.len().is_multiple_of(n)
|| targets.len() != n
|| out.len() != n
{
return Err(KernelError::LengthMismatch {
what: "block_log_probs: block, targets and out vs n",
left: block.len(),
right: n,
}
.into());
}
let vocab = block.len() / n;
let mut max = [0.0f32; MAX_SCORE_BLOCK];
let max = &mut max[..n];
max.copy_from_slice(&block[..n]);
for row in block.chunks_exact(n).skip(1) {
for (m, &l) in max.iter_mut().zip(row) {
if *m < l {
*m = l;
}
}
}
let mut sum = [0.0f64; MAX_SCORE_BLOCK];
let sum = &mut sum[..n];
for row in block.chunks_exact(n) {
for ((s, &l), &m) in sum.iter_mut().zip(row).zip(max.iter()) {
*s += f64::from((l - m).exp());
}
}
for (c, (&target, slot)) in targets.iter().zip(out.iter_mut()).enumerate() {
let t = target as usize;
if t >= vocab {
return Err(KernelError::LengthMismatch {
what: "block_log_probs: target vs vocab",
left: t,
right: vocab,
}
.into());
}
*slot = f64::from(block[t * n + c] - max[c]) - sum[c].ln();
}
Ok(())
}
fn build_index(s: &mut Scratch<'_>, dims: &PrefillDims, n: usize) -> Result<(), ForwardError> {
s.index_start.fill(0);
for entry in &s.topk[..n * dims.top_k] {
let expert = entry.expert as usize;
let slot =
s.index_start
.get_mut(expert + 1)
.ok_or(ForwardError::RoutedExpertOutOfRange {
expert: entry.expert,
n_experts: dims.n_experts,
})?;
*slot += 1;
}
for e in 0..dims.n_experts {
let count = s.index_start[e + 1] as usize;
if count > n {
return Err(ForwardError::RepeatedRoutedExpert {
expert: e as u32,
count,
rows: n,
});
}
s.index_start[e + 1] += s.index_start[e];
}
s.index_cursor
.copy_from_slice(&s.index_start[..dims.n_experts]);
for r in 0..n {
for slot in 0..dims.top_k {
let entry = s.topk[r * dims.top_k + slot];
let cursor = &mut s.index_cursor[entry.expert as usize];
let at = *cursor as usize;
s.index[at] = RowSlot {
row: r as u32,
slot: slot as u32,
};
*cursor += 1;
}
}
Ok(())
}
fn run_expert(
pool: &mut ComputePool,
dims: &PrefillDims,
s: &mut Scratch<'_>,
n_rows: usize,
expert: &SweepExpert<'_>,
) -> Result<usize, ForwardError> {
let e = expert.expert as usize;
let (lo, hi) = match (s.index_start.get(e), s.index_start.get(e + 1)) {
(Some(&lo), Some(&hi)) => (lo as usize, hi as usize),
_ => {
return Err(ForwardError::RoutedExpertOutOfRange {
expert: expert.expert,
n_experts: dims.n_experts,
});
}
};
if hi <= lo {
return Ok(0);
}
let count = hi - lo;
debug_assert!(count <= n_rows * dims.top_k);
let (hidden, moe) = (dims.hidden, dims.moe);
let blocks = dims.hidden_q8k;
let entries = &s.index[lo..hi];
let staged_rows = n_rows * dims.top_k;
for entry in entries {
let (row, slot) = (entry.row as usize, entry.slot as usize);
if row >= n_rows || slot >= dims.top_k {
return Err(KernelError::LengthMismatch {
what: "prefill expert scatter: staged row vs n_rows * top_k",
left: row * dims.top_k + slot,
right: staged_rows,
}
.into());
}
}
{
let acts_hidden = &*s.acts_hidden;
let gather = Rows::new(&mut *s.acts_gather, blocks, count)?;
for_each_row(pool, count, |j| {
let src = entries[j].row as usize * blocks;
let dst = unsafe { gather.row(j) };
dst.copy_from_slice(&acts_hidden[src..src + blocks]);
Ok(())
})?;
}
let acts = &s.acts_gather[..count * blocks];
let view = &expert.view;
let gate_slab = view.gate();
let up_slab = view.up();
let down_slab = view.down();
pool_batched_q8_k(
pool,
gate_slab.format,
gate_slab.bytes,
hidden,
moe,
acts,
count,
&mut s.gate[..moe * count],
)?;
pool_batched_q8_k(
pool,
up_slab.format,
up_slab.bytes,
hidden,
moe,
acts,
count,
&mut s.up[..moe * count],
)?;
{
let up = &s.up[..moe * count];
let failure: Mutex<Option<(usize, ForwardError)>> = Mutex::new(None);
pool.scatter_chunks(&mut s.gate[..moe * count], |chunk, gate| {
if let Err(err) = swiglu_combine(gate, &up[chunk.rows]) {
record(&failure, chunk.index, err.into());
}
});
taken(failure)?;
}
{
let gate = &s.gate[..moe * count];
let moe_t = Rows::new(&mut *s.moe_t, moe, count)?;
let acts_moe = Rows::new(&mut *s.acts_moe, dims.moe_q8k, count)?;
for_each_row(pool, count, |j| {
let (row, acts) = unsafe { (moe_t.row(j), acts_moe.row(j)) };
for (i, value) in row.iter_mut().enumerate() {
*value = gate[i * count + j];
}
quantize_row_q8_k(row, acts)?;
Ok(())
})?;
}
pool_batched_q8_k(
pool,
down_slab.format,
down_slab.bytes,
moe,
hidden,
&s.acts_moe[..count * dims.moe_q8k],
count,
&mut s.tmat[..hidden * count],
)?;
{
let tmat = &s.tmat[..hidden * count];
let staged = SendPtr(s.staged.as_mut_ptr());
pool.run_chunks(hidden, |chunk| {
for h in chunk.rows {
let src = &tmat[h * count..(h + 1) * count];
for (entry, &value) in entries.iter().zip(src) {
let at = (entry.row as usize * dims.top_k + entry.slot as usize) * hidden + h;
unsafe { staged.get().add(at).write(value) };
}
}
});
}
Ok(count)
}
fn reduce_experts(
pool: &mut ComputePool,
s: &mut Scratch<'_>,
dims: &PrefillDims,
n: usize,
) -> Result<(), ForwardError> {
let (hidden, top_k) = (dims.hidden, dims.top_k);
let (topk, staged) = (&*s.topk, &*s.staged);
let residual_rows = Rows::new(&mut *s.residual, hidden, n)?;
for_each_row(pool, n, |r| {
let residual = unsafe { residual_rows.row(r) };
let weights = &topk[r * top_k..(r + 1) * top_k];
let staged = &staged[r * top_k * hidden..(r + 1) * top_k * hidden];
for (h, value) in residual.iter_mut().enumerate() {
let mut acc = 0.0f32;
for (slot, entry) in weights.iter().enumerate() {
acc += entry.weight * staged[slot * hidden + h];
}
*value += acc;
}
Ok(())
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::{ExpertsLayout, LayerLayout};
use crate::io::testutil::{Fixture, build_install};
use crate::io::{LoadOptions, SweepError};
use crate::model::RuntimeConfig;
use crate::model::testsupport::temper_install;
const SKIP: LoadOptions = LoadOptions {
skip_hashes: true,
verify_layer_hashes: false,
common_memory: crate::io::CommonMemory::Mapped,
};
fn test_pool() -> crate::threads::ComputePool {
crate::threads::ComputePool::with_config(crate::threads::PoolConfig {
shards: Some(3),
placement: crate::threads::Placement::Off,
pin_caller: false,
..crate::threads::PoolConfig::default()
})
}
fn load_fixture(tag: &str) -> (Fixture, Model) {
let fx = build_install(tag);
temper_install(&fx);
let model = Model::load(&fx.root, SKIP).unwrap();
(fx, model)
}
fn state_with(model: &Model, cap: usize, prefill: PrefillConfig) -> ForwardState {
let mut st = ForwardState::with_config(model, cap, RuntimeConfig::testing()).unwrap();
st.set_prefill_config(prefill).unwrap();
st
}
fn sweep_config(chunk: usize) -> PrefillConfig {
PrefillConfig {
mode: PrefillMode::Sweep,
chunk,
experts_per_window: 1,
windows_in_flight: 1,
}
}
fn token_major_config() -> PrefillConfig {
PrefillConfig {
mode: PrefillMode::TokenMajor,
..sweep_config(1)
}
}
fn logit_bits(logits: &[f32]) -> Vec<u32> {
logits.iter().map(|v| v.to_bits()).collect()
}
mod wide {
use crate::io::testutil::{Fixture, Geometry, build_install_with};
use crate::model::testsupport::temper_install;
pub(super) const N_LAYERS: u32 = 2;
pub(super) const N_EXPERTS: u32 = 16;
pub(super) const TOP_K: u32 = 4;
pub(super) const HIDDEN: usize = 512;
pub(super) const MOE: usize = 256;
pub(super) const N_HEADS: usize = 12;
pub(super) const N_KV_HEADS: usize = 3;
pub(super) const HEAD_DIM: usize = 64;
pub(super) const Q_DIM: usize = N_HEADS * HEAD_DIM;
pub(super) const KV_DIM: usize = N_KV_HEADS * HEAD_DIM;
pub(super) const VOCAB: usize = 40;
fn norm_value(name: &str, index: usize) -> f32 {
1.0 + (((index * 7 + name.len()) % 13) as f32) / 64.0
}
fn router_value(_name: &str, index: usize) -> f32 {
((((index * 37 + 11) % 29) as f32) - 14.0) / 64.0
}
pub(super) const GEOMETRY: Geometry = Geometry {
n_layers: N_LAYERS,
n_experts: N_EXPERTS,
top_k: TOP_K,
hidden: HIDDEN,
moe_intermediate: MOE,
n_heads: N_HEADS,
n_kv_heads: N_KV_HEADS,
head_dim: HEAD_DIM,
vocab: VOCAB,
model_id: "fixture-moe-wide-2l",
hf_repo: "test/fixture-wide",
source_file: "fixture-wide-Q4_K_M.gguf",
norm_value,
router_value,
};
pub(super) fn build(tag: &str) -> Fixture {
let fx = build_install_with(tag, &GEOMETRY);
temper_install(&fx);
fx
}
}
fn run(model: &Model, prefill: PrefillConfig, ids: &[u32]) -> Vec<u32> {
let mut st = state_with(model, 64, prefill);
let logits = prefill_prompt(model, &mut st, ids, None).unwrap();
assert_eq!(logits.len(), crate::io::testutil::VOCAB);
logit_bits(logits)
}
#[test]
fn sweep_and_token_major_agree_bit_for_bit() {
let (_fx, model) = load_fixture("prefill-bitident");
let prompt: Vec<u32> = (0..12u32).map(|i| (i * 5 + 1) % 32).collect();
for &len in &[1usize, 2, 3, 4, 6, 7, 8, 12] {
let ids = &prompt[..len];
let want = run(&model, token_major_config(), ids);
for &chunk in &[1usize, 2, 3, 4, 8, 512] {
let got = run(&model, sweep_config(chunk), ids);
assert_eq!(
got,
want,
"prompt len {len}, chunk {chunk}: sweep prefill diverged \
(exact multiple: {})",
len % chunk == 0
);
}
}
}
fn wide_state(model: &Model, cap: usize, prefill: PrefillConfig) -> ForwardState {
let runtime = RuntimeConfig {
cache_bytes: 24 * 1024 * 1024,
threads: Some(2),
placement: crate::threads::Placement::Off,
};
let mut st = ForwardState::with_config(model, cap, runtime).unwrap();
st.set_prefill_config(prefill).unwrap();
st
}
fn wide_dials(
mode: PrefillMode,
chunk: usize,
experts_per_window: u32,
windows_in_flight: u32,
) -> PrefillConfig {
PrefillConfig {
mode,
chunk,
experts_per_window,
windows_in_flight,
}
}
fn load_wide(tag: &str) -> (Fixture, Model) {
let fx = wide::build(tag);
let model = Model::load(&fx.root, SKIP).unwrap();
(fx, model)
}
fn wide_run(model: &Model, prefill: PrefillConfig, ids: &[u32]) -> Vec<u32> {
let mut st = wide_state(model, 64, prefill);
let logits = prefill_prompt(model, &mut st, ids, None).unwrap();
assert_eq!(logits.len(), wide::VOCAB);
assert!(
logits.iter().all(|v| v.is_finite()),
"the wide fixture produced non-finite logits, so bit-identity \
between two NaN-producing paths would prove nothing"
);
logit_bits(logits)
}
#[test]
fn the_wide_geometry_separates_every_width() {
let (_fx, model) = load_wide("prefill-wide-geometry");
let dims = PrefillDims::new(model.arch(), 2, 64).unwrap();
assert_eq!((dims.q_dim, dims.kv_dim), (wide::Q_DIM, wide::KV_DIM));
let widths = [dims.hidden, dims.q_dim, dims.kv_dim, dims.moe];
for (i, a) in widths.iter().enumerate() {
for b in &widths[i + 1..] {
assert_ne!(a, b, "{widths:?} are not pairwise distinct");
}
}
assert_eq!(dims.wide, dims.q_dim);
assert!(dims.wide > dims.hidden, "`wide` is degenerate again");
assert_eq!(dims.hidden_q8k, 2);
assert_eq!(dims.attn_q8k, 3);
assert_eq!(dims.moe_q8k, 1);
assert_eq!(dims.hidden_q80, 16);
assert_eq!((dims.n_experts, dims.top_k), (16, 4));
let cache_bytes =
wide_state(&model, 8, wide_dials(PrefillMode::Sweep, 512, 8, 3)).cache_bytes();
for (experts_per_window, windows_in_flight) in [(1u32, 1u32), (4, 1), (4, 2), (8, 3)] {
let config = wide_dials(
PrefillMode::Sweep,
512,
experts_per_window,
windows_in_flight,
);
let (rows, planned) =
plan_arena(model.layout(), &dims, config, 9, cache_bytes).unwrap();
assert_eq!(rows, 9, "the chunk was narrowed at {config:?}");
assert_eq!(planned.experts_per_window, experts_per_window);
assert_eq!(
planned.windows_in_flight, windows_in_flight,
"read-ahead was given up at {config:?}"
);
}
}
#[test]
fn wide_sweep_and_token_major_agree_bit_for_bit() {
let (_fx, model) = load_wide("prefill-wide-bitident");
let vocab = wide::VOCAB as u32;
let prompt: Vec<u32> = (0..9u32).map(|i| (i * 11 + 3) % vocab).collect();
let dials: [(u32, u32); 4] = [(1, 1), (4, 1), (4, 2), (8, 3)];
for &len in &[1usize, 3, 4, 5, 8, 9] {
let ids = &prompt[..len];
let want = wide_run(&model, wide_dials(PrefillMode::TokenMajor, 1, 1, 1), ids);
for &chunk in &[2usize, 3, 4, 512] {
for &(experts_per_window, windows_in_flight) in &dials {
let got = wide_run(
&model,
wide_dials(
PrefillMode::Sweep,
chunk,
experts_per_window,
windows_in_flight,
),
ids,
);
assert_eq!(
got,
want,
"prompt len {len}, chunk {chunk}, {experts_per_window} experts/window, \
{windows_in_flight} in flight: sweep prefill diverged \
(exact multiple: {})",
len % chunk == 0
);
}
}
}
let dials = wide_dials(PrefillMode::Sweep, 4, 4, 2);
let eight = wide_run(&model, dials, &prompt[..8]);
assert_ne!(wide_run(&model, dials, &prompt), eight);
assert!(eight.iter().any(|&bits| bits != eight[0]));
}
fn scored(
model: &Model,
state: &mut ForwardState,
ids: &[u32],
first: usize,
block: usize,
) -> (Vec<u64>, Vec<u32>) {
let mut out = vec![0.0f64; ids.len() - 1 - first];
let logits = prefill_scored_in_blocks(model, state, ids, first, &mut out, block).unwrap();
let logits = logit_bits(logits);
assert!(
out.iter().all(|v| v.is_finite() && *v <= 0.0),
"not log-probabilities: {out:?}"
);
(out.iter().map(|v| v.to_bits()).collect(), logits)
}
fn prefix_scores(
model: &Model,
mut fresh: impl FnMut() -> ForwardState,
ids: &[u32],
) -> Vec<u64> {
(0..ids.len() - 1)
.map(|i| {
let mut st = fresh();
let logits = prefill_prompt(model, &mut st, &ids[..=i], None).unwrap();
log_prob(logits, ids[i + 1]).unwrap().to_bits()
})
.collect()
}
#[test]
fn scored_rows_match_a_prefill_that_ends_there() {
let (_fx, model) = load_fixture("prefill-scored");
let vocab = crate::io::testutil::VOCAB as u32;
let ids: Vec<u32> = (0..12u32).map(|i| (i * 7 + 3) % vocab).collect();
let want = prefix_scores(&model, || state_with(&model, 64, sweep_config(512)), &ids);
let last = run(&model, sweep_config(512), &ids);
assert!(
want.iter().any(|&bits| bits != want[0]),
"every position scored the same, so agreement would prove nothing"
);
let configs = [
token_major_config(),
sweep_config(1),
sweep_config(3),
sweep_config(4),
sweep_config(5),
sweep_config(512),
];
for &first in &[0usize, 1, 5, 10] {
for config in configs {
for &block in &[1usize, 3, 4, 5, MAX_SCORE_BLOCK] {
let mut st = state_with(&model, 64, config);
let (got, logits) = scored(&model, &mut st, &ids, first, block);
assert_eq!(
got,
want[first..],
"first {first}, {config:?}, block {block}"
);
assert_eq!(
logits, last,
"last logits moved: first {first}, {config:?}, block {block}"
);
assert_eq!(st.seq_len().unwrap(), ids.len());
}
}
}
}
#[test]
fn wide_scored_rows_match_a_prefill_that_ends_there() {
let (_fx, model) = load_wide("prefill-wide-scored");
let vocab = wide::VOCAB as u32;
let ids: Vec<u32> = (0..9u32).map(|i| (i * 11 + 3) % vocab).collect();
let token_major = wide_dials(PrefillMode::TokenMajor, 1, 1, 1);
let want = prefix_scores(&model, || wide_state(&model, 64, token_major), &ids);
for &first in &[0usize, 4] {
for &(chunk, experts_per_window, windows_in_flight) in
&[(2usize, 4u32, 2u32), (3, 8, 3), (512, 4, 1)]
{
let dials = wide_dials(
PrefillMode::Sweep,
chunk,
experts_per_window,
windows_in_flight,
);
for &block in &[1usize, 4, 5, MAX_SCORE_BLOCK] {
let mut st = wide_state(&model, 64, dials);
let (got, _) = scored(&model, &mut st, &ids, first, block);
assert_eq!(
got,
want[first..],
"first {first}, {dials:?}, block {block}"
);
}
}
}
}
#[test]
fn scored_prefill_continues_a_cache() {
let (_fx, model) = load_fixture("prefill-scored-continue");
let vocab = crate::io::testutil::VOCAB as u32;
let ids: Vec<u32> = (0..12u32).map(|i| (i * 7 + 3) % vocab).collect();
let want = prefix_scores(&model, || state_with(&model, 64, sweep_config(512)), &ids);
for config in [token_major_config(), sweep_config(3), sweep_config(512)] {
let mut st = state_with(&model, 64, config);
prefill_prompt(&model, &mut st, &ids[..4], None).unwrap();
let (got, _) = scored(&model, &mut st, &ids[4..], 1, 3);
assert_eq!(got, want[5..], "{config:?}");
assert_eq!(st.seq_len().unwrap(), ids.len());
}
}
#[test]
fn score_block_is_capped_by_the_staged_plane() {
let vocab = 151_936;
let staged = 512 * 8 * 2_048;
assert_eq!(score_block(MAX_SCORE_BLOCK, staged, vocab).unwrap(), 52);
assert_eq!(score_block(MAX_SCORE_BLOCK, 7 * vocab, vocab).unwrap(), 4);
assert_eq!(score_block(MAX_SCORE_BLOCK, 3 * vocab, vocab).unwrap(), 3);
assert_eq!(score_block(5, staged, vocab).unwrap(), 4);
assert_eq!(score_block(3, staged, vocab).unwrap(), 3);
assert!(matches!(
score_block(MAX_SCORE_BLOCK, 9 * 8 * 2_048, vocab),
Err(ForwardError::PrefillScratch { .. })
));
}
#[test]
fn block_log_probs_match_log_prob_per_column() {
let vocab = 37;
for n in [1usize, 2, 4, 5, MAX_SCORE_BLOCK] {
let block: Vec<f32> = (0..vocab * n)
.map(|i| (((i * 7919) % 1013) as f32 - 506.0) / 37.0)
.collect();
let targets: Vec<u32> = (0..n).map(|c| ((c * 13 + 5) % vocab) as u32).collect();
let mut out = vec![0.0; n];
block_log_probs(&block, n, &targets, &mut out).unwrap();
for c in 0..n {
let column: Vec<f32> = (0..vocab).map(|v| block[v * n + c]).collect();
assert_eq!(
out[c].to_bits(),
log_prob(&column, targets[c]).unwrap().to_bits(),
"n {n}, column {c}"
);
}
}
let mut out = [0.0; 2];
assert!(block_log_probs(&[0.0; 6], 2, &[0, 3], &mut out).is_err());
assert!(block_log_probs(&[0.0; 5], 2, &[0, 1], &mut out).is_err());
assert!(block_log_probs(&[0.0; 6], 2, &[0], &mut out).is_err());
}
#[test]
fn log_prob_is_a_log_softmax() {
let logits: Vec<f32> = (0..300)
.map(|i| ((i * 31 % 97) as f32 - 48.0) / 5.0)
.collect();
let max = logits
.iter()
.fold(f64::NEG_INFINITY, |m, &l| m.max(f64::from(l)));
let lse = max
+ logits
.iter()
.map(|&l| (f64::from(l) - max).exp())
.sum::<f64>()
.ln();
let mut total = 0.0;
for (t, &l) in logits.iter().enumerate() {
let got = log_prob(&logits, t as u32).unwrap();
assert!(
(got - (f64::from(l) - lse)).abs() < 1e-5,
"token {t}: {got}"
);
total += got.exp();
}
assert!((total - 1.0).abs() < 1e-5, "{total}");
assert!(log_prob(&logits, 300).is_err());
}
#[test]
fn scored_prefill_refuses_what_it_cannot_score() {
let (_fx, model) = load_fixture("prefill-scored-refusals");
let ids = [1u32, 2, 3, 4];
let mut st = state_with(&model, 64, sweep_config(512));
for (prompt, first, slots) in [
(&ids[..], 0usize, 2usize),
(&ids[..], 0, 4),
(&ids[..], 2, 0),
(&ids[..], 3, 0),
(&ids[..], 4, 0),
(&ids[..1], 0, 0),
(&ids[..0], 0, 0),
] {
let mut out = vec![0.0; slots];
let err = prefill_prompt_scored(&model, &mut st, prompt, first, &mut out).unwrap_err();
assert!(
matches!(err, ForwardError::ScoreRange { .. }),
"{} tokens, first {first}, {slots} slots: {err}",
prompt.len()
);
}
let bad = [1u32, 2, crate::io::testutil::VOCAB as u32];
let mut out = [0.0; 2];
let err = prefill_prompt_scored(&model, &mut st, &bad, 0, &mut out).unwrap_err();
assert!(
matches!(
err,
ForwardError::Model(crate::model::ModelError::TokenOutOfRange { .. })
),
"{err}"
);
assert_eq!(st.seq_len().unwrap(), 0, "a refused request ran anyway");
}
#[test]
fn wide_paths_leave_the_same_kv_cache() {
let (_fx, model) = load_wide("prefill-wide-kv");
let ids: Vec<u32> = (0..7u32)
.map(|i| (i * 5 + 2) % wide::VOCAB as u32)
.collect();
let mut token_major = wide_state(&model, 32, wide_dials(PrefillMode::TokenMajor, 1, 1, 1));
prefill_prompt(&model, &mut token_major, &ids, None).unwrap();
let mut sweep = wide_state(&model, 32, wide_dials(PrefillMode::Sweep, 3, 4, 2));
prefill_prompt(&model, &mut sweep, &ids, None).unwrap();
assert_eq!(sweep.seq_len().unwrap(), ids.len());
for layer in 0..model.n_layers() as usize {
assert_eq!(
sweep.kv_k_layer(layer).unwrap(),
token_major.kv_k_layer(layer).unwrap(),
"layer {layer} keys"
);
assert_eq!(
sweep.kv_v_layer(layer).unwrap(),
token_major.kv_v_layer(layer).unwrap(),
"layer {layer} values"
);
}
}
#[test]
fn the_wide_sweep_skips_windows_nothing_routed_to() {
use std::collections::BTreeSet;
let (_fx, model) = load_wide("prefill-wide-skips");
let ids: Vec<u32> = (0..6u32)
.map(|i| (i * 13 + 5) % wide::VOCAB as u32)
.collect();
for experts_per_window in [1u32, 2] {
let mut st = wide_state(
&model,
32,
wide_dials(PrefillMode::Sweep, 1, experts_per_window, 2),
);
let mut routed: BTreeSet<u32> = BTreeSet::new();
{
let mut sink = |_pos: usize, _layer: u32, topk: &[(u32, f32)]| {
routed.extend(topk.iter().map(|&(expert, _)| expert));
};
prefill_prompt(&model, &mut st, &ids, Some(&mut sink)).unwrap();
}
let stats = st.stream_stats();
assert_eq!(
stats.accesses(),
0,
"a swept prefill bypasses the expert cache entirely"
);
assert!(stats.sweep_windows_read > 0, "{experts_per_window}/window");
assert!(
stats.sweep_windows_skipped > 0,
"{experts_per_window} experts/window: no window went unrouted, \
so the skip path is still untested"
);
assert!(
routed.len() > model.arch().top_k as usize,
"the router picks the same experts for every row, so the skips \
above prove nothing about the driver: {routed:?}"
);
}
}
#[test]
fn a_wide_window_slices_experts_at_a_non_zero_offset() {
const EXPERTS_PER_WINDOW: u32 = 4;
let (_fx, model) = load_wide("prefill-wide-within");
let ids: Vec<u32> = (0..6u32)
.map(|i| (i * 13 + 5) % wide::VOCAB as u32)
.collect();
let mut st = wide_state(
&model,
32,
wide_dials(PrefillMode::Sweep, 1, EXPERTS_PER_WINDOW, 2),
);
let mut off_head = 0usize;
{
let mut sink = |_pos: usize, _layer: u32, topk: &[(u32, f32)]| {
off_head += topk
.iter()
.filter(|&&(expert, _)| expert % EXPERTS_PER_WINDOW != 0)
.count();
};
prefill_prompt(&model, &mut st, &ids, Some(&mut sink)).unwrap();
}
assert!(
off_head > 0,
"every routed expert led its window, so `within` was always 0 and \
the intra-window offset is still untested"
);
}
#[test]
fn both_paths_leave_the_same_kv_cache() {
let (_fx, model) = load_fixture("prefill-kv-agree");
let ids: Vec<u32> = (0..7u32).map(|i| (i * 3 + 2) % 32).collect();
let mut token_major = state_with(&model, 32, token_major_config());
prefill_prompt(&model, &mut token_major, &ids, None).unwrap();
let mut sweep = state_with(&model, 32, sweep_config(3));
prefill_prompt(&model, &mut sweep, &ids, None).unwrap();
assert_eq!(token_major.seq_len().unwrap(), ids.len());
assert_eq!(sweep.seq_len().unwrap(), ids.len());
for layer in 0..model.n_layers() as usize {
assert_eq!(
sweep.kv_k_layer(layer).unwrap(),
token_major.kv_k_layer(layer).unwrap(),
"layer {layer} keys"
);
assert_eq!(
sweep.kv_v_layer(layer).unwrap(),
token_major.kv_v_layer(layer).unwrap(),
"layer {layer} values"
);
}
}
#[test]
fn decode_continues_off_a_swept_prefill() {
let (_fx, model) = load_fixture("prefill-then-decode");
let ids = [1u32, 2, 3, 4, 5];
let mut a = state_with(&model, 32, token_major_config());
prefill_prompt(&model, &mut a, &ids, None).unwrap();
let want = logit_bits(
super::super::forward_token(&model, &mut a, 6, 5, true)
.unwrap()
.unwrap(),
);
let mut b = state_with(&model, 32, sweep_config(2));
prefill_prompt(&model, &mut b, &ids, None).unwrap();
assert_eq!(
b.seq_len().unwrap(),
5,
"the cache is uniform after a chunk"
);
let got = logit_bits(
super::super::forward_token(&model, &mut b, 6, 5, true)
.unwrap()
.unwrap(),
);
assert_eq!(got, want);
}
#[test]
fn a_row_never_attends_past_its_own_position() {
let (_fx, model) = load_fixture("prefill-causal");
let ids: Vec<u32> = (0..9u32).map(|i| (i * 7 + 3) % 32).collect();
let want = run(&model, sweep_config(1), &ids);
for chunk in [2usize, 3, 4, 9] {
assert_eq!(
run(&model, sweep_config(chunk), &ids),
want,
"chunk {chunk} let a row see the future"
);
}
let mut longer = ids.clone();
longer.push(11);
assert_ne!(run(&model, sweep_config(4), &longer), want);
}
#[test]
fn the_cache_is_ragged_mid_chunk_and_level_after() {
let (_fx, model) = load_fixture("prefill-ragged");
let n_layers = model.n_layers() as usize;
let arch = model.arch();
let mut kv = KvCache::new(
n_layers,
arch.n_kv_heads as usize,
arch.head_dim as usize,
8,
)
.unwrap();
let row = vec![0.25f32; arch.n_kv_heads as usize * arch.head_dim as usize];
for layer in 0..n_layers {
for _ in 0..3 {
kv.append(layer, &row, &row).unwrap();
}
assert_eq!(kv.len(layer).unwrap(), 3);
if layer + 1 < n_layers {
assert!(
matches!(kv.seq_len(), Err(KvError::RaggedLayers { .. })),
"layer {layer}: a half-swept chunk must read as ragged"
);
}
}
assert_eq!(kv.seq_len().unwrap(), 3);
for chunk in [1usize, 2, 3, 4, 5] {
let mut st = state_with(&model, 16, sweep_config(chunk));
prefill_prompt(&model, &mut st, &[1, 2, 3, 4, 5], None).unwrap();
assert_eq!(st.seq_len().unwrap(), 5, "chunk {chunk}");
for layer in 0..n_layers {
assert_eq!(st.kv_len(layer).unwrap(), 5, "chunk {chunk} layer {layer}");
}
}
let mut st = state_with(&model, 3, sweep_config(2));
assert!(prefill_prompt(&model, &mut st, &[1, 2, 3, 4], None).is_err());
assert_eq!(st.seq_len().unwrap(), 0);
for layer in 0..n_layers {
assert_eq!(st.kv_len(layer).unwrap(), 0);
}
}
#[test]
fn the_inverse_index_is_a_permutation() {
let dims = PrefillDims {
hidden: 256,
q_dim: 256,
kv_dim: 128,
moe: 256,
top_k: 3,
n_experts: 5,
n_heads: 4,
n_kv_heads: 2,
head_dim: 64,
hidden_q8k: 1,
hidden_q80: 8,
attn_q8k: 1,
moe_q8k: 1,
wide: 256,
shards: 2,
max_positions: 8,
};
let rows = 6usize;
let mut bytes = vec![0u8; scratch_bytes(&dims, rows).unwrap()];
let (mut s, _) = carve(&mut bytes, &dims, rows).unwrap();
for r in 0..rows {
for slot in 0..dims.top_k {
s.topk[r * dims.top_k + slot] = TopkEntry {
expert: ((r * dims.top_k + slot) % 4) as u32,
weight: 1.0,
};
}
}
build_index(&mut s, &dims, rows).unwrap();
assert_eq!(s.index_start[0], 0);
assert_eq!(
s.index_start[dims.n_experts] as usize,
rows * dims.top_k,
"the index covers every (row, slot) pair"
);
let mut seen = vec![false; rows * dims.top_k];
for expert in 0..dims.n_experts {
let (lo, hi) = (
s.index_start[expert] as usize,
s.index_start[expert + 1] as usize,
);
assert!(lo <= hi, "expert {expert} has a reversed run");
for entry in &s.index[lo..hi] {
let flat = entry.row as usize * dims.top_k + entry.slot as usize;
assert!(
!seen[flat],
"(row {}, slot {}) twice",
entry.row, entry.slot
);
seen[flat] = true;
assert_eq!(
s.topk[flat].expert as usize, expert,
"entry filed under the wrong expert"
);
}
}
assert!(seen.iter().all(|&hit| hit), "a (row, slot) pair was lost");
assert_eq!(s.index_start[4], s.index_start[5]);
}
#[test]
fn a_repeated_routed_expert_is_typed() {
let dims = PrefillDims {
top_k: 2,
..tiny_dims()
};
let mut bytes = vec![0u8; scratch_bytes(&dims, 1).unwrap()];
let (mut s, _) = carve(&mut bytes, &dims, 1).unwrap();
s.topk[0] = TopkEntry {
expert: 3,
weight: 0.5,
};
s.topk[1] = TopkEntry {
expert: 3,
weight: 0.5,
};
assert!(
matches!(
build_index(&mut s, &dims, 1).unwrap_err(),
ForwardError::RepeatedRoutedExpert {
expert: 3,
count: 2,
rows: 1
}
),
"a doubly-routed expert must be refused"
);
}
#[test]
fn an_out_of_range_routed_expert_is_typed() {
let dims = PrefillDims {
hidden: 256,
q_dim: 256,
kv_dim: 128,
moe: 256,
top_k: 2,
n_experts: 4,
n_heads: 4,
n_kv_heads: 2,
head_dim: 64,
hidden_q8k: 1,
hidden_q80: 8,
attn_q8k: 1,
moe_q8k: 1,
wide: 256,
shards: 2,
max_positions: 8,
};
let mut bytes = vec![0u8; scratch_bytes(&dims, 1).unwrap()];
let (mut s, _) = carve(&mut bytes, &dims, 1).unwrap();
s.topk[0] = TopkEntry {
expert: 4,
weight: 1.0,
};
s.topk[1] = TopkEntry {
expert: 0,
weight: 1.0,
};
assert!(matches!(
build_index(&mut s, &dims, 1).unwrap_err(),
ForwardError::RoutedExpertOutOfRange {
expert: 4,
n_experts: 4
}
));
}
#[test]
fn the_reduction_runs_in_top_k_order_not_arrival_order() {
let dims = PrefillDims {
hidden: 4,
q_dim: 4,
kv_dim: 4,
moe: 4,
top_k: 3,
n_experts: 3,
n_heads: 1,
n_kv_heads: 1,
head_dim: 4,
hidden_q8k: 1,
hidden_q80: 1,
attn_q8k: 1,
moe_q8k: 1,
wide: 4,
shards: 2,
max_positions: 8,
};
let mut bytes = vec![0u8; scratch_bytes(&dims, 1).unwrap()];
let (mut s, _) = carve(&mut bytes, &dims, 1).unwrap();
let big = 1.0f32;
let small = f32::EPSILON / 2.0;
s.topk[0] = TopkEntry {
expert: 2,
weight: 1.0,
};
s.topk[1] = TopkEntry {
expert: 0,
weight: 1.0,
};
s.topk[2] = TopkEntry {
expert: 1,
weight: 1.0,
};
let staged = [big, small, small];
for (slot, &value) in staged.iter().enumerate() {
for h in 0..dims.hidden {
s.staged[slot * dims.hidden + h] = value;
}
}
s.residual.fill(0.0);
reduce_experts(&mut test_pool(), &mut s, &dims, 1).unwrap();
let want_topk = ((0.0f32 + big) + small) + small;
let arrival = ((0.0f32 + small) + small) + big;
assert_ne!(
want_topk.to_bits(),
arrival.to_bits(),
"the fixture is not order-sensitive; the test proves nothing"
);
for (h, &value) in s.residual.iter().enumerate() {
assert_eq!(value.to_bits(), want_topk.to_bits(), "element {h}");
}
}
fn tiny_dims() -> PrefillDims {
PrefillDims {
hidden: 512,
q_dim: 768,
kv_dim: 256,
moe: 256,
top_k: 4,
n_experts: 9,
n_heads: 6,
n_kv_heads: 2,
head_dim: 128,
hidden_q8k: 2,
hidden_q80: 16,
attn_q8k: 3,
moe_q8k: 1,
wide: 768,
shards: 2,
max_positions: 16,
}
}
fn qwen3_30b_a3b_dims() -> PrefillDims {
PrefillDims {
hidden: 2048,
q_dim: 32 * 128,
kv_dim: 4 * 128,
moe: 768,
top_k: 8,
n_experts: 128,
n_heads: 32,
n_kv_heads: 4,
head_dim: 128,
hidden_q8k: 8,
hidden_q80: 64,
attn_q8k: 16,
moe_q8k: 3,
wide: 4096,
shards: 6,
max_positions: 4096,
}
}
#[test]
fn the_documented_v0_scratch_arithmetic_holds() {
let dims = qwen3_30b_a3b_dims();
assert_eq!(scratch_bytes(&dims, 512).unwrap(), 81_720_324);
assert_eq!(81_720_324 / 1024 / 1024, 77);
assert_eq!(512 * dims.top_k * dims.hidden * 4, 33_554_432);
assert_eq!(2 * 512 * dims.q_dim * 4, 16_777_216);
assert_eq!(dims.attn_shard(), 33_024);
assert_eq!(dims.attn_scratch() * 4, 792_576);
assert_eq!(80_927_748 + 792_576, 81_720_324);
let per_row = scratch_bytes(&dims, 2).unwrap() - scratch_bytes(&dims, 1).unwrap();
assert_eq!(per_row, 145_408 + 35 * 292 + 64 * 36 + 2 * 8 * 8);
assert_eq!(
scratch_bytes(&dims, 512).unwrap(),
scratch_bytes(&dims, 1).unwrap() + 511 * per_row
);
let shipped = layout_of(&[(3_059_712, 128), (2_654_208, 128)]);
let ring = ring_bytes(&shipped, 8, 2);
assert_eq!(ring, 48_955_392);
assert!(ring + 81_720_324 < 1_438 * 1024 * 1024);
}
#[test]
fn the_attention_split_partitions_every_row() {
for rows in [0usize, 1, 2, 5, 6, 7, 8, 63, 512, 1891] {
for start in [0usize, 1, 7, 512, 4095] {
for shards in [1usize, 2, 3, 6, 8, 16] {
let mut seen = vec![0u32; rows];
let mut next = 0usize;
for index in 0..shards {
let mine = attention_shard_range(rows, start, shards, index);
assert!(mine.start <= mine.end, "{mine:?} is reversed");
assert_eq!(mine.start, next, "shards {shards} index {index} not flush");
assert!(mine.end <= rows, "{mine:?} runs past {rows}");
next = mine.end;
for r in mine {
seen[r] += 1;
}
}
assert_eq!(next, rows, "rows {rows} shards {shards} left a tail");
assert!(
seen.iter().all(|&hits| hits == 1),
"rows {rows} start {start} shards {shards}: {seen:?}"
);
}
}
}
}
#[test]
fn the_attention_split_is_a_pure_function() {
for (rows, start, shards) in [(512usize, 0usize, 6usize), (355, 1536, 6), (17, 512, 3)] {
for index in 0..shards {
let once = attention_shard_range(rows, start, shards, index);
for _ in 0..4 {
assert_eq!(attention_shard_range(rows, start, shards, index), once);
}
}
}
}
#[test]
fn the_attention_split_balances_cost_not_rows() {
let (rows, start, shards) = (512usize, 0usize, 6usize);
let total = attention_cost(rows, start);
assert_eq!(total, 131_328);
let ideal = total / shards as u128;
let mut widest = 0u128;
for index in 0..shards {
let mine = attention_shard_range(rows, start, shards, index);
let cost = attention_cost(mine.end, start) - attention_cost(mine.start, start);
let slack = cost.abs_diff(ideal);
assert!(
slack <= (start + rows) as u128,
"shard {index} {mine:?} costs {cost}, ideal {ideal}"
);
widest = widest.max(cost);
}
let first = attention_shard_range(rows, start, shards, 0).len();
let last = attention_shard_range(rows, start, shards, shards - 1).len();
assert!(first > last * 2, "{first} rows against {last}");
let naive = attention_cost(rows, start) - attention_cost(rows - rows / shards, start);
assert!(
widest * 3 < naive * 2,
"cost balancing bought too little: {widest} vs {naive}"
);
}
#[test]
fn carve_consumes_exactly_the_documented_bytes() {
let dims = tiny_dims();
for rows in [1usize, 2, 7, 33] {
let total = scratch_bytes(&dims, rows).unwrap();
let mut bytes = vec![0u8; total];
let (_, used) = carve(&mut bytes, &dims, rows).unwrap();
assert_eq!(used, total, "rows {rows}");
}
}
#[test]
fn every_scratch_type_packs_without_padding() {
for (name, size, align) in [
("f32", size_of::<f32>(), align_of::<f32>()),
("u32", size_of::<u32>(), align_of::<u32>()),
("BlockQ8K", size_of::<BlockQ8K>(), align_of::<BlockQ8K>()),
("BlockQ8_0", size_of::<BlockQ8_0>(), align_of::<BlockQ8_0>()),
("TopkEntry", size_of::<TopkEntry>(), align_of::<TopkEntry>()),
("RowSlot", size_of::<RowSlot>(), align_of::<RowSlot>()),
] {
assert_eq!(align, 4, "{name} alignment");
assert_eq!(size % 4, 0, "{name} size {size}");
}
assert_eq!(size_of::<BlockQ8K>(), 292);
assert_eq!(size_of::<BlockQ8_0>(), 36);
}
#[test]
fn carved_buffers_do_not_overlap() {
let dims = tiny_dims();
let rows = 5usize;
let total = scratch_bytes(&dims, rows).unwrap();
let mut bytes = vec![0u8; total];
{
let (s, _) = carve(&mut bytes, &dims, rows).unwrap();
s.residual.fill(f32::from_bits(0x0101_0101));
s.normed.fill(f32::from_bits(0x0202_0202));
s.q.fill(f32::from_bits(0x0303_0303));
s.staged.fill(f32::from_bits(0x0404_0404));
s.attn_shards.fill(f32::from_bits(0x0808_0808));
s.index_start.fill(0x0606_0606);
s.index_cursor.fill(0x0707_0707);
}
let (s, _) = carve(&mut bytes, &dims, rows).unwrap();
assert!(s.residual.iter().all(|v| v.to_bits() == 0x0101_0101));
assert!(s.normed.iter().all(|v| v.to_bits() == 0x0202_0202));
assert!(s.q.iter().all(|v| v.to_bits() == 0x0303_0303));
assert!(s.staged.iter().all(|v| v.to_bits() == 0x0404_0404));
assert!(s.attn_shards.iter().all(|v| v.to_bits() == 0x0808_0808));
assert!(s.index_start.iter().all(|&v| v == 0x0606_0606));
assert!(s.index_cursor.iter().all(|&v| v == 0x0707_0707));
assert_eq!(s.residual.len(), rows * dims.hidden);
assert_eq!(s.staged.len(), rows * dims.top_k * dims.hidden);
assert_eq!(s.attn_shards.len(), dims.shards * dims.attn_shard());
assert_eq!(s.index_start.len(), dims.n_experts + 1);
}
#[test]
fn a_short_scratch_span_is_typed() {
let dims = tiny_dims();
let total = scratch_bytes(&dims, 3).unwrap();
let mut bytes = vec![0u8; total - 1];
assert!(matches!(
carve(&mut bytes, &dims, 3),
Err(ForwardError::PrefillScratch { .. })
));
let mut empty: [u8; 0] = [];
assert!(matches!(
carve(&mut empty, &dims, 1),
Err(ForwardError::PrefillScratch { .. })
));
}
#[test]
fn the_carve_aligns_from_an_offset_base() {
let dims = tiny_dims();
let rows = 2usize;
let total = scratch_bytes(&dims, rows).unwrap();
let mut bytes = vec![0u8; total + 8];
let (s, used) = carve(&mut bytes[3..], &dims, rows).unwrap();
assert!(used <= total + 3, "padding ran away: {used} vs {total}");
assert_eq!(s.residual.as_ptr() as usize % align_of::<f32>(), 0);
assert_eq!(s.acts_hidden.as_ptr() as usize % align_of::<BlockQ8K>(), 0);
}
fn layout_of(strides: &[(u64, u32)]) -> ExpertsLayout {
ExpertsLayout {
layers: strides
.iter()
.enumerate()
.map(|(i, &(stride, n_experts))| LayerLayout {
file: format!("experts/layer_{i:02}.bin"),
stride,
n_experts,
projections: Vec::new(),
})
.collect(),
}
}
#[test]
fn the_ring_estimate_caps_windows_at_the_expert_count() {
let shipped = layout_of(&[(3_059_712, 128), (2_654_208, 128)]);
assert_eq!(ring_bytes(&shipped, 8, 2), 2 * 8 * 3_059_712);
let tiny = layout_of(&[(4096, 3)]);
assert_eq!(ring_bytes(&tiny, 8, 2), 2 * 3 * 4096);
let empty = layout_of(&[(4096, 0)]);
assert_eq!(ring_bytes(&empty, 8, 1), 4096);
}
#[test]
fn the_arena_plan_narrows_before_it_refuses() {
let dims = tiny_dims();
let layout = layout_of(&[(65_536, 4)]);
let config = PrefillConfig {
mode: PrefillMode::Sweep,
chunk: 512,
experts_per_window: 8,
windows_in_flight: 2,
};
let (rows, sweep) = plan_arena(&layout, &dims, config, 512, 64 * 1024 * 1024).unwrap();
assert_eq!(rows, 512);
assert_eq!(sweep.windows_in_flight, 2);
let (rows, sweep) = plan_arena(&layout, &dims, config, 512, 1024 * 1024).unwrap();
assert!((1..512).contains(&rows), "rows {rows}");
assert_eq!(sweep.windows_in_flight, 2, "read-ahead is given up last");
let one_window = 4 * 65_536;
let (rows, sweep) = plan_arena(
&layout,
&dims,
config,
512,
one_window + scratch_bytes(&dims, 1).unwrap() as u64 + 8192,
)
.unwrap();
assert_eq!(rows, 1);
assert_eq!(sweep.windows_in_flight, 1);
let err = plan_arena(&layout, &dims, config, 512, one_window).unwrap_err();
assert!(
matches!(err, ForwardError::PrefillScratch { .. }),
"unexpected error: {err}"
);
}
#[test]
fn a_short_prompt_carves_for_the_prompt() {
let dims = tiny_dims();
let layout = layout_of(&[(65_536, 4)]);
let config = PrefillConfig {
mode: PrefillMode::Sweep,
chunk: 512,
experts_per_window: 8,
windows_in_flight: 2,
};
let (rows, _) = plan_arena(&layout, &dims, config, 3, 64 * 1024 * 1024).unwrap();
assert_eq!(rows, 3);
}
#[test]
fn transpose_moves_every_element() {
let src: Vec<f32> = (0..12).map(|i| i as f32).collect();
let mut dst = vec![0.0f32; 12];
transpose(&mut test_pool(), &src, 3, 4, &mut dst).unwrap();
for r in 0..3 {
for c in 0..4 {
assert_eq!(dst[c * 3 + r], src[r * 4 + c]);
}
}
let mut empty: [f32; 0] = [];
transpose(&mut test_pool(), &[], 0, 4, &mut empty).unwrap();
transpose(&mut test_pool(), &[], 3, 0, &mut empty).unwrap();
}
#[test]
fn a_mis_sized_transpose_is_typed() {
let src: Vec<f32> = (0..12).map(|i| i as f32).collect();
let mut dst = vec![0.0f32; 12];
assert!(matches!(
transpose(&mut test_pool(), &src[..8], 3, 4, &mut dst),
Err(ForwardError::Kernel(KernelError::LengthMismatch {
left: 8,
right: 12,
..
}))
));
assert!(matches!(
transpose(&mut test_pool(), &src, 3, 4, &mut dst[..11]),
Err(ForwardError::Kernel(KernelError::LengthMismatch {
left: 11,
right: 12,
..
}))
));
assert!(transpose(&mut test_pool(), &[], 0, 4, &mut dst).is_err());
assert!(transpose(&mut test_pool(), &src, usize::MAX, 2, &mut dst).is_err());
}
#[test]
fn every_prefill_dial_refuses_at_validate() {
let base = PrefillConfig::default();
assert!(base.validate().is_ok());
assert!(matches!(
PrefillConfig { chunk: 0, ..base }.validate().unwrap_err(),
ForwardError::InvalidPrefillChunk { chunk: 0 }
));
for bad in [
PrefillConfig {
experts_per_window: 0,
..base
},
PrefillConfig {
windows_in_flight: 0,
..base
},
PrefillConfig {
windows_in_flight: crate::io::MAX_WINDOWS_IN_FLIGHT + 1,
..base
},
] {
let err = bad.validate().unwrap_err();
assert!(
matches!(err, ForwardError::Sweep(SweepError::BadDials { .. })),
"unexpected error for {bad:?}: {err}"
);
}
let layout = layout_of(&[(65_536, 4)]);
assert!(matches!(
plan_arena(
&layout,
&tiny_dims(),
PrefillConfig {
experts_per_window: 0,
..base
},
4,
64 * 1024 * 1024,
)
.unwrap_err(),
ForwardError::Sweep(SweepError::BadDials { .. })
));
let (_fx, model) = load_fixture("prefill-dials");
let mut st = state_with(&model, 8, sweep_config(4));
assert!(matches!(
st.set_prefill_config(PrefillConfig {
windows_in_flight: crate::io::MAX_WINDOWS_IN_FLIGHT + 1,
..sweep_config(4)
})
.unwrap_err(),
ForwardError::Sweep(SweepError::BadDials { .. })
));
}
#[test]
fn an_empty_prompt_is_typed() {
let (_fx, model) = load_fixture("prefill-empty");
let mut st = state_with(&model, 8, sweep_config(4));
assert!(matches!(
prefill_prompt(&model, &mut st, &[], None).unwrap_err(),
ForwardError::EmptyPrefill
));
}
#[test]
fn a_zero_chunk_is_typed() {
let (_fx, model) = load_fixture("prefill-zero-chunk");
let mut st = state_with(&model, 8, sweep_config(4));
assert!(matches!(
st.set_prefill_config(PrefillConfig {
chunk: 0,
..sweep_config(4)
})
.unwrap_err(),
ForwardError::InvalidPrefillChunk { chunk: 0 }
));
}
#[test]
fn a_prompt_past_the_context_cap_is_typed() {
let (_fx, model) = load_fixture("prefill-cap");
let mut st = state_with(&model, 3, sweep_config(4));
let err = prefill_prompt(&model, &mut st, &[1, 2, 3, 4], None).unwrap_err();
assert!(
matches!(
err,
ForwardError::Kv(KvError::CapacityExceeded { capacity: 3, .. })
),
"unexpected error: {err}"
);
}
#[test]
fn the_route_sink_covers_every_position_and_layer() {
let (_fx, model) = load_fixture("prefill-sink");
let ids = [1u32, 2, 3, 4, 5];
let n_layers = model.n_layers();
let top_k = model.arch().top_k as usize;
let collect = |config: PrefillConfig| {
let mut st = state_with(&model, 16, config);
let mut seen: Vec<(usize, u32, Vec<u32>)> = Vec::new();
{
let mut sink = |pos: usize, layer: u32, topk: &[(u32, f32)]| {
seen.push((pos, layer, topk.iter().map(|&(e, _)| e).collect()));
};
prefill_prompt(&model, &mut st, &ids, Some(&mut sink)).unwrap();
}
seen.sort_by_key(|(pos, layer, _)| (*pos, *layer));
seen
};
let want = collect(token_major_config());
assert_eq!(want.len(), ids.len() * n_layers as usize);
assert!(want.iter().all(|(_, _, experts)| experts.len() == top_k));
assert_eq!(collect(sweep_config(2)), want);
assert_eq!(collect(sweep_config(5)), want);
}
#[test]
fn the_env_override_parses_or_is_ignored() {
let base = PrefillConfig::default();
assert_eq!(base.mode, PrefillMode::Sweep);
assert_eq!(base.chunk, DEFAULT_PREFILL_CHUNK);
assert_eq!(
base.sweep_config(),
SweepConfig {
experts_per_window: base.experts_per_window,
windows_in_flight: base.windows_in_flight,
}
);
assert!(base.validate().is_ok());
assert!(PrefillConfig { chunk: 0, ..base }.validate().is_err());
}
#[test]
fn the_unaccounted_remainder_closes_the_sum() {
let ms = Duration::from_millis;
let timing = PrefillTiming {
mode: Some(PrefillMode::Sweep),
tokens: 512,
total: ms(1000),
attention: ms(400),
projections: ms(30),
router: ms(5),
expert_compute: ms(450),
expert_io: ms(90),
elementwise: ms(20),
};
assert_eq!(timing.accounted(), ms(995));
assert_eq!(timing.other(), ms(5));
assert_eq!(
timing.accounted() + timing.other(),
timing.total,
"the phases and the remainder must be the whole run"
);
assert!(timing.ran());
let broken = PrefillTiming {
total: ms(1),
..timing
};
assert_eq!(broken.other(), Duration::ZERO);
let fresh = PrefillTiming::default();
assert!(!fresh.ran());
assert_eq!(fresh.accounted(), Duration::ZERO);
assert_eq!(fresh.other(), Duration::ZERO);
assert_eq!(fresh.phases().len(), 7);
}
#[test]
fn both_prefill_paths_report_phases_inside_their_total() {
let (_fx, model) = load_fixture("prefill-timing-split");
let ids = [1u32, 2, 3, 4, 5, 6];
for (want_mode, config) in [
(PrefillMode::Sweep, sweep_config(4)),
(PrefillMode::TokenMajor, token_major_config()),
] {
let mut st = state_with(&model, 32, config);
assert!(
!st.prefill_timing().ran(),
"{want_mode:?}: a fresh state has no prefill to report"
);
prefill_prompt(&model, &mut st, &ids, None).unwrap();
let timing = st.prefill_timing();
assert_eq!(timing.mode, Some(want_mode));
assert_eq!(timing.tokens, ids.len() as u64);
assert!(timing.ran());
assert!(
timing.accounted() <= timing.total,
"{want_mode:?}: phases {:?} exceed the total {:?} — a region is \
charged twice",
timing.accounted(),
timing.total
);
assert_eq!(
timing.accounted() + timing.other(),
timing.total,
"{want_mode:?}: the split must account for the whole run"
);
assert!(
timing.total > Duration::ZERO,
"{want_mode:?}: a prefill that ran took some time"
);
}
}
#[test]
fn every_prefill_rearms_the_timing() {
let (_fx, model) = load_fixture("prefill-timing-rearm");
let long: Vec<u32> = (0..12u32).map(|i| (i * 5 + 1) % 32).collect();
let mut st = state_with(&model, 64, sweep_config(4));
prefill_prompt(&model, &mut st, &long, None).unwrap();
assert_eq!(st.prefill_timing().tokens, long.len() as u64);
st.reset();
st.set_prefill_config(token_major_config()).unwrap();
prefill_prompt(&model, &mut st, &long[..1], None).unwrap();
let timing = st.prefill_timing();
assert_eq!(timing.tokens, 1, "the token count is per run, not summed");
assert_eq!(
timing.mode,
Some(PrefillMode::TokenMajor),
"the mode is the run's, not the state's history"
);
assert!(
timing.accounted() <= timing.total,
"phases {:?} exceed this run's total {:?}: the previous run leaked in",
timing.accounted(),
timing.total
);
}
#[test]
fn a_reset_drops_the_prefill_timing() {
let (_fx, model) = load_fixture("prefill-timing-reset");
let mut st = state_with(&model, 32, sweep_config(4));
prefill_prompt(&model, &mut st, &[1, 2, 3, 4, 5], None).unwrap();
assert!(st.prefill_timing().ran());
st.reset();
assert_eq!(
st.prefill_timing(),
PrefillTiming::default(),
"a reset state reports no prefill at all"
);
assert!(!st.prefill_timing().ran());
}
#[test]
fn decode_does_not_charge_the_prefill_timing() {
let (_fx, model) = load_fixture("prefill-timing-decode");
let ids = [1u32, 2, 3, 4, 5];
let mut st = state_with(&model, 32, sweep_config(4));
prefill_prompt(&model, &mut st, &ids, None).unwrap();
let after_prefill = st.prefill_timing();
for step in 0..3usize {
let position = ids.len() + step;
crate::model::forward_token(&model, &mut st, 7, position, true).unwrap();
}
assert_eq!(
st.prefill_timing(),
after_prefill,
"three decode tokens moved a prefill counter"
);
}
#[test]
fn a_token_major_prefill_charges_no_decode_gemvs() {
let (_fx, model) = load_fixture("prefill-gemv-subsplit");
let ids = [1u32, 2, 3, 4, 5];
let mut st = state_with(&model, 32, token_major_config());
prefill_prompt(&model, &mut st, &ids, None).unwrap();
let after_prefill = st.decode_gemv_split();
for &(label, own, wait, scatters) in &after_prefill {
assert_eq!(
(own, wait, scatters),
(Duration::ZERO, Duration::ZERO, 0),
"the prefill charged {label}: {after_prefill:?}"
);
}
for step in 0..2usize {
let position = ids.len() + step;
crate::model::forward_token(&model, &mut st, 7, position, true).unwrap();
}
for &(label, own, _, scatters) in &st.decode_gemv_split() {
assert!(scatters > 0, "{label} charged no fan-out after decoding");
assert!(own > Duration::ZERO, "{label} charged no own time");
}
}
}