use std::cell::Cell;
use std::fmt;
use std::ops::Range;
use std::sync::{Mutex, PoisonError};
use std::time::{Duration, Instant};
use crate::format::ArchInfo;
use crate::io::{
ExpertStream, Footprint, FootprintError, IoError, StreamMode, StreamPhase, StreamStats,
SweepError, SweepPlan, mib, resident_ceiling,
};
use crate::kernels::KernelError;
use crate::kernels::attention::{
AttentionError, decode_attention_in, decode_attention_kv_range_in,
scratch_len as attention_scratch_len,
};
use crate::kernels::primitives::{
rmsnorm, rmsnorm_in_place, rope_neox_heads, softmax, swiglu_combine, vec_add,
};
use crate::kernels::quants::{
BlockQ8_0, BlockQ8K, QuantFormat, quantize_row_q8_0, quantize_row_q8_k,
};
use crate::kernels::gemv::{FusedPart, fused_out_dim, fused_row_parts, gemv_q8_k_fused_rows};
use crate::kernels::{gemv_q8_0_rows, gemv_q8_k_rows};
use crate::kv::{KvCache, KvError};
use crate::threads::{ComputePool, Placement, PoolConfig};
use thiserror::Error;
use super::error::ModelError;
use super::prefill::{Phase, PhaseClock, PrefillConfig, PrefillMode, PrefillTiming, SendPtr};
use super::weights::Model;
pub const DEFAULT_CACHE_BYTES: u64 = 2485 * 1024 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RuntimeConfig {
pub cache_bytes: u64,
pub threads: Option<usize>,
pub placement: Placement,
}
impl Default for RuntimeConfig {
fn default() -> Self {
Self {
cache_bytes: DEFAULT_CACHE_BYTES,
threads: None,
placement: Placement::Shared,
}
}
}
impl RuntimeConfig {
#[cfg(test)]
pub(crate) fn testing() -> Self {
Self {
cache_bytes: 4 * 1024 * 1024,
threads: Some(2),
placement: Placement::Off,
}
}
}
#[derive(Debug, Error)]
pub enum ForwardError {
#[error(transparent)]
Model(#[from] ModelError),
#[error(transparent)]
Kernel(#[from] KernelError),
#[error(transparent)]
Attention(#[from] AttentionError),
#[error(transparent)]
Kv(#[from] KvError),
#[error(transparent)]
Io(#[from] IoError),
#[error("forward: {what} = {dim} is not a multiple of the {block}-weight activation block")]
UnsupportedDim {
what: &'static str,
dim: usize,
block: usize,
},
#[error("forward: top_k {top_k} invalid for {n_experts} experts")]
InvalidTopK {
top_k: usize,
n_experts: usize,
},
#[error(
"forward: layer {layer}: the expert stream plan is not a permutation \
of the {top_k} routed experts (bad or repeated index {index})"
)]
StreamPlanIndex {
layer: u32,
index: usize,
top_k: usize,
},
#[error(
"forward: layer {layer}: the expert stream plan covered {covered} of {top_k} routed experts"
)]
StreamPlanCoverage {
layer: u32,
covered: usize,
top_k: usize,
},
#[error("forward: position {position}, but the kv cache holds {expected} positions")]
PositionMismatch {
position: usize,
expected: usize,
},
#[error("forward: position {position} exceeds u32::MAX")]
PositionOverflow {
position: usize,
},
#[error(
"forward: this state was built for a model with {what} = {built}, \
but the model it was given has {what} = {given}"
)]
ArchMismatch {
what: &'static str,
built: u32,
given: u32,
},
#[error(transparent)]
Sweep(#[from] SweepError),
#[error(
"this configuration needs {footprint} but only {} MiB are available; \
reduce --context or --cache-bytes",
mib(u128::from(*limit))
)]
ConfigTooLarge {
footprint: Footprint,
limit: u64,
},
#[error(
"prefill: the expert slot pool holds {available} B, but the narrowest \
chunk needs {needed} B of scratch plus its sweep ring"
)]
PrefillScratch {
needed: u64,
available: u64,
},
#[error("prefill: the scratch span cannot be aligned to {align} B")]
PrefillScratchAlign {
align: usize,
},
#[error("prefill: chunk size {chunk} is not a usable number of positions")]
InvalidPrefillChunk {
chunk: usize,
},
#[error("prefill: no tokens to prefill")]
EmptyPrefill,
#[error("prefill: layer {layer}: the sweep covered {covered} of {expected} routed rows")]
PrefillCoverage {
layer: u32,
covered: usize,
expected: usize,
},
#[error("prefill: routed expert {expert} is outside the layer's {n_experts} experts")]
RoutedExpertOutOfRange {
expert: u32,
n_experts: usize,
},
#[error("prefill: expert {expert} is routed {count} times by only {rows} chunk rows")]
RepeatedRoutedExpert {
expert: u32,
count: usize,
rows: usize,
},
#[error(
"prefill: scoring from position {first} of a {prompt}-token prompt cannot fill {out} slots"
)]
ScoreRange {
first: usize,
prompt: usize,
out: usize,
},
}
impl From<FootprintError> for ForwardError {
fn from(error: FootprintError) -> Self {
match error {
FootprintError::Kv(error) => Self::Kv(error),
FootprintError::Io(error) => Self::Io(error),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ArchFingerprint {
n_layers: u32,
n_experts: u32,
top_k: u32,
hidden: u32,
moe_intermediate: u32,
n_heads: u32,
n_kv_heads: u32,
head_dim: u32,
vocab: u32,
}
impl ArchFingerprint {
fn of(arch: &ArchInfo) -> Self {
Self {
n_layers: arch.n_layers,
n_experts: arch.n_experts,
top_k: arch.top_k,
hidden: arch.hidden,
moe_intermediate: arch.moe_intermediate,
n_heads: arch.n_heads,
n_kv_heads: arch.n_kv_heads,
head_dim: arch.head_dim,
vocab: arch.vocab,
}
}
fn check(&self, arch: &ArchInfo) -> Result<(), ForwardError> {
let given = Self::of(arch);
for (what, built, given) in [
("n_layers", self.n_layers, given.n_layers),
("n_experts", self.n_experts, given.n_experts),
("top_k", self.top_k, given.top_k),
("hidden", self.hidden, given.hidden),
(
"moe_intermediate",
self.moe_intermediate,
given.moe_intermediate,
),
("n_heads", self.n_heads, given.n_heads),
("n_kv_heads", self.n_kv_heads, given.n_kv_heads),
("head_dim", self.head_dim, given.head_dim),
("vocab", self.vocab, given.vocab),
] {
if built != given {
return Err(ForwardError::ArchMismatch { what, built, given });
}
}
Ok(())
}
}
pub struct ForwardState {
arch: ArchFingerprint,
kv: KvCache,
attn_scratch: Vec<f32>,
hidden: Vec<f32>,
normed: Vec<f32>,
acts_q8k_hidden: Vec<BlockQ8K>,
acts_q8k_moe: Vec<BlockQ8K>,
acts_q8k_attn: Vec<BlockQ8K>,
acts_q8_0_hidden: Vec<BlockQ8_0>,
q: Vec<f32>,
k: Vec<f32>,
v: Vec<f32>,
attn_out: Vec<f32>,
o_proj: Vec<f32>,
router_logits: Vec<f32>,
router_probs: Vec<f32>,
topk: Vec<(u32, f32)>,
expert_ids: Vec<u32>,
gate_up: Vec<f32>,
expert_staged: Vec<f32>,
expert_done: Vec<bool>,
expert_acc: Vec<f32>,
logits: Vec<f32>,
pool: ComputePool,
stream: ExpertStream,
prefill: PrefillConfig,
sweep_plan: SweepPlan,
routed: Vec<u32>,
prefill_timing: PrefillTiming,
decode_timing: PrefillTiming,
decode_gemv: GemvSplit,
prefill_charging: bool,
}
impl fmt::Debug for ForwardState {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ForwardState")
.field("context_cap", &self.kv.capacity())
.field("top_k", &self.topk.capacity())
.field("shards", &self.pool.shards())
.field("slots_per_layer", &self.stream.slots_per_layer())
.field("cache_bytes", &self.stream.cache_bytes())
.field("stream_mode", &self.stream.mode())
.finish_non_exhaustive()
}
}
impl ForwardState {
pub fn new(model: &Model, context_cap: usize) -> Result<Self, ForwardError> {
Self::with_config(model, context_cap, RuntimeConfig::default())
}
pub fn with_config(
model: &Model,
context_cap: usize,
config: RuntimeConfig,
) -> Result<Self, ForwardError> {
let footprint = Footprint::project(
model.manifest(),
model.layout(),
config.cache_bytes,
context_cap,
)?;
let ceiling = resident_ceiling();
if let Some(limit) = ceiling
&& !footprint.fits_within(limit)
{
tracing::error!(
common = footprint.common,
experts = footprint.experts,
kv = footprint.kv,
runtime = footprint.runtime,
context_cap,
total = %footprint.total(),
limit,
"configuration does not fit this process's memory limit"
);
return Err(ForwardError::ConfigTooLarge { footprint, limit });
}
tracing::debug!(
common = footprint.common,
experts = footprint.experts,
kv = footprint.kv,
runtime = footprint.runtime,
context_cap,
total = %footprint.total(),
limit = ?ceiling,
"projected resident footprint"
);
let arch = model.arch();
let hidden = arch.hidden as usize;
let q_dim = arch.n_heads as usize * arch.head_dim as usize;
let kv_dim = arch.n_kv_heads as usize * arch.head_dim as usize;
let moe = arch.moe_intermediate as usize;
let n_experts = arch.n_experts as usize;
let top_k = arch.top_k as usize;
let vocab = arch.vocab as usize;
const QK_K: usize = 256;
const QK8_0: usize = 32;
for (what, dim, block) in [
("hidden", hidden, QK_K),
("hidden", hidden, QK8_0),
("moe_intermediate", moe, QK_K),
("n_heads * head_dim", q_dim, QK_K),
] {
if dim % block != 0 {
return Err(ForwardError::UnsupportedDim { what, dim, block });
}
}
if top_k == 0 || top_k > n_experts {
return Err(ForwardError::InvalidTopK { top_k, n_experts });
}
let window = top_k.min(MAX_FUSED_EXPERTS);
let kv = KvCache::new(
arch.n_layers as usize,
arch.n_kv_heads as usize,
arch.head_dim as usize,
context_cap,
)?;
let pool = ComputePool::with_config(PoolConfig {
shards: config.threads,
placement: config.placement,
pin_caller: true,
inline_caller: true,
..PoolConfig::default()
});
let stream = ExpertStream::new(
model.dir(),
model.manifest(),
model.layout(),
config.cache_bytes,
model.load_options(),
)?;
tracing::info!(
shards = pool.shards(),
slots_per_layer = stream.slots_per_layer(),
cache_bytes = stream.cache_bytes(),
mode = ?stream.mode(),
"decode runtime ready"
);
let attn_shard = attention_scratch_len(
arch.n_heads as usize,
arch.n_kv_heads as usize,
arch.head_dim as usize,
context_cap,
);
let attn_carves = (arch.n_kv_heads as usize).min(pool.shards()).max(1);
Ok(Self {
arch: ArchFingerprint::of(arch),
kv,
attn_scratch: vec![0.0; attn_shard.saturating_mul(attn_carves)],
hidden: vec![0.0; hidden],
normed: vec![0.0; hidden],
acts_q8k_hidden: vec![BlockQ8K::default(); hidden / QK_K],
acts_q8k_moe: vec![BlockQ8K::default(); window * (moe / QK_K)],
acts_q8k_attn: vec![BlockQ8K::default(); q_dim / QK_K],
acts_q8_0_hidden: vec![BlockQ8_0::default(); hidden / QK8_0],
q: vec![0.0; q_dim],
k: vec![0.0; kv_dim],
v: vec![0.0; kv_dim],
attn_out: vec![0.0; q_dim],
o_proj: vec![0.0; hidden],
router_logits: vec![0.0; n_experts],
router_probs: vec![0.0; n_experts],
topk: Vec::with_capacity(top_k),
expert_ids: Vec::with_capacity(top_k),
gate_up: vec![0.0; 2 * window * moe],
expert_staged: vec![0.0; top_k * hidden],
expert_done: vec![false; top_k],
expert_acc: vec![0.0; hidden],
logits: vec![0.0; vocab],
pool,
stream,
prefill: PrefillConfig::from_env(),
sweep_plan: SweepPlan::new(),
routed: Vec::with_capacity(n_experts),
prefill_timing: PrefillTiming::default(),
decode_timing: PrefillTiming::default(),
decode_gemv: GemvSplit::default(),
prefill_charging: false,
})
}
pub fn prefill_config(&self) -> PrefillConfig {
self.prefill
}
pub fn set_prefill_config(&mut self, config: PrefillConfig) -> Result<(), ForwardError> {
self.prefill = config.validate()?;
Ok(())
}
pub fn logits(&self) -> &[f32] {
&self.logits
}
pub fn kv_len(&self, layer: usize) -> Result<usize, ForwardError> {
Ok(self.kv.len(layer)?)
}
pub fn reset(&mut self) {
self.kv.clear();
self.prefill_timing = PrefillTiming::default();
self.decode_timing = PrefillTiming::default();
self.decode_gemv = GemvSplit::default();
self.prefill_charging = false;
}
pub fn truncate(&mut self, new_len: usize) -> Result<(), ForwardError> {
self.kv.truncate(new_len)?;
self.prefill_timing = PrefillTiming::default();
self.decode_timing = PrefillTiming::default();
self.decode_gemv = GemvSplit::default();
self.prefill_charging = false;
Ok(())
}
pub(super) fn prefill_parts(&mut self) -> PrefillParts<'_> {
PrefillParts {
kv: &mut self.kv,
topk: &mut self.topk,
logits: &mut self.logits,
pool: &mut self.pool,
stream: &mut self.stream,
sweep_plan: &mut self.sweep_plan,
routed: &mut self.routed,
timing: &mut self.prefill_timing,
}
}
pub(super) fn arm_prefill_timing(&mut self, mode: PrefillMode, tokens: usize) {
self.prefill_timing = PrefillTiming::started(mode, tokens);
self.decode_timing = PrefillTiming::default();
self.decode_gemv = GemvSplit::default();
self.prefill_charging = true;
}
pub(super) fn close_prefill_timing(&mut self, total: Duration) {
self.prefill_timing.total = total;
self.prefill_charging = false;
}
pub(super) fn check_arch(&self, arch: &ArchInfo) -> Result<(), ForwardError> {
self.arch.check(arch)
}
#[cfg(test)]
pub(super) fn kv_k_layer(&self, layer: usize) -> Result<&[u16], ForwardError> {
Ok(self.kv.k_layer(layer)?)
}
#[cfg(test)]
pub(super) fn kv_v_layer(&self, layer: usize) -> Result<&[u16], ForwardError> {
Ok(self.kv.v_layer(layer)?)
}
}
pub(super) struct PrefillParts<'a> {
pub(super) kv: &'a mut KvCache,
pub(super) topk: &'a mut Vec<(u32, f32)>,
pub(super) logits: &'a mut Vec<f32>,
pub(super) pool: &'a mut ComputePool,
pub(super) stream: &'a mut ExpertStream,
pub(super) sweep_plan: &'a mut SweepPlan,
pub(super) routed: &'a mut Vec<u32>,
pub(super) timing: &'a mut PrefillTiming,
}
impl ForwardState {
pub fn seq_len(&self) -> Result<usize, ForwardError> {
Ok(self.kv.seq_len()?)
}
pub fn context_cap(&self) -> usize {
self.kv.capacity()
}
pub fn shards(&self) -> usize {
self.pool.shards()
}
pub fn slots_per_layer(&self) -> u32 {
self.stream.slots_per_layer()
}
pub fn cache_bytes(&self) -> u64 {
self.stream.cache_bytes()
}
pub fn stream_mode(&self) -> StreamMode {
self.stream.mode()
}
pub fn stream_stats(&self) -> StreamStats {
self.stream.stats()
}
pub fn stream_stats_in(&self, phase: StreamPhase) -> StreamStats {
self.stream.stats_in(phase)
}
pub fn prefill_timing(&self) -> PrefillTiming {
self.prefill_timing
}
pub fn decode_timing(&self) -> PrefillTiming {
self.decode_timing
}
pub fn decode_gemv_split(&self) -> [(&'static str, Duration, Duration, u64); 4] {
self.decode_gemv.rows()
}
pub fn set_stream_phase(&mut self, phase: StreamPhase) {
self.stream.set_phase(phase);
}
}
pub type ExpertRouteSink<'a> = &'a mut dyn FnMut(u32, &[(u32, f32)]);
#[inline]
pub(super) fn dot_f32(a: &[f32], b: &[f32]) -> f32 {
let mut acc = 0.0f32;
for (&x, &y) in a.iter().zip(b) {
acc += x * y;
}
acc
}
pub(super) fn record<E>(slot: &Mutex<Option<(usize, E)>>, index: usize, err: E) {
let mut held = slot.lock().unwrap_or_else(PoisonError::into_inner);
match held.as_ref() {
Some((held_index, _)) if *held_index <= index => {}
_ => *held = Some((index, err)),
}
}
pub(super) fn taken<E>(slot: Mutex<Option<(usize, E)>>) -> Result<(), E> {
match slot.into_inner().unwrap_or_else(PoisonError::into_inner) {
Some((_, err)) => Err(err),
None => Ok(()),
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum GemvSite {
Projections,
Experts,
LmHead,
Router,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct GemvBucket {
own: Duration,
wait: Duration,
scatters: u64,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct GemvSplit {
projections: GemvBucket,
experts: GemvBucket,
lm_head: GemvBucket,
router: GemvBucket,
}
impl GemvSplit {
fn bucket(&mut self, site: GemvSite) -> &mut GemvBucket {
match site {
GemvSite::Projections => &mut self.projections,
GemvSite::Experts => &mut self.experts,
GemvSite::LmHead => &mut self.lm_head,
GemvSite::Router => &mut self.router,
}
}
fn rows(&self) -> [(&'static str, Duration, Duration, u64); 4] {
[
(
"projections",
self.projections.own,
self.projections.wait,
self.projections.scatters,
),
(
"experts",
self.experts.own,
self.experts.wait,
self.experts.scatters,
),
(
"lm_head",
self.lm_head.own,
self.lm_head.wait,
self.lm_head.scatters,
),
(
"router",
self.router.own,
self.router.wait,
self.router.scatters,
),
]
}
}
thread_local! {
static CALLER_DONE: Cell<Option<Instant>> = const { Cell::new(None) };
}
struct GemvClock<'a> {
split: Option<&'a mut GemvSplit>,
}
impl<'a> GemvClock<'a> {
fn new(split: &'a mut GemvSplit, armed: bool) -> Self {
Self {
split: armed.then_some(split),
}
}
fn disarmed() -> Self {
Self { split: None }
}
#[inline]
fn open(&self) -> Option<Instant> {
self.split.as_ref()?;
CALLER_DONE.set(None);
Some(Instant::now())
}
#[inline]
fn close(&mut self, site: GemvSite, opened: Option<Instant>) {
let (Some(split), Some(t0)) = (self.split.as_deref_mut(), opened) else {
return;
};
let t2 = Instant::now();
let t1 = CALLER_DONE.replace(None).unwrap_or(t2);
let bucket = split.bucket(site);
bucket.own += t1.saturating_duration_since(t0);
bucket.wait += t2.saturating_duration_since(t1);
bucket.scatters += 1;
}
}
pub(super) fn pool_gemv_q8_k(
pool: &mut ComputePool,
format: QuantFormat,
weight: &[u8],
in_dim: usize,
out_dim: usize,
acts: &[BlockQ8K],
out: &mut [f32],
) -> Result<(), KernelError> {
pool_gemv_q8_k_stamped(pool, format, weight, in_dim, out_dim, acts, out, false)
}
#[allow(clippy::too_many_arguments)]
fn pool_gemv_q8_k_stamped(
pool: &mut ComputePool,
format: QuantFormat,
weight: &[u8],
in_dim: usize,
out_dim: usize,
acts: &[BlockQ8K],
out: &mut [f32],
stamp: bool,
) -> Result<(), KernelError> {
let failure: Mutex<Option<(usize, KernelError)>> = Mutex::new(None);
pool.scatter_chunks(out, |chunk, dst| {
let (index, on_caller) = (chunk.index, chunk.on_caller);
if let Err(err) = gemv_q8_k_rows(format, weight, in_dim, out_dim, acts, chunk.rows, dst) {
record(&failure, index, err);
}
if stamp && on_caller {
CALLER_DONE.set(Some(Instant::now()));
}
});
taken(failure)
}
fn pool_gemv_q8_k_fused(
pool: &mut ComputePool,
parts: &[FusedPart<'_>],
out: &mut [f32],
stamp: bool,
) -> Result<(), KernelError> {
let total = fused_out_dim(parts);
if out.len() != total {
return Err(KernelError::LengthMismatch {
what: FUSED_OUT_VS_SPACE,
left: out.len(),
right: total,
});
}
let failure: Mutex<Option<(usize, KernelError)>> = Mutex::new(None);
pool.scatter_chunks(out, |chunk, dst| {
let (index, on_caller) = (chunk.index, chunk.on_caller);
if let Err(err) = gemv_q8_k_fused_rows(parts, chunk.rows, dst) {
record(&failure, index, err);
}
if stamp && on_caller {
CALLER_DONE.set(Some(Instant::now()));
}
});
taken(failure)
}
const FUSED_OUT_VS_SPACE: &str = "pool fused gemv: out vs fused out_dim";
const FUSED_SPAN_BOUND: &str = "pool fused gemv: destination span end vs dst";
const FUSED_SPAN_ORDER: &str = "pool fused gemv: destination span start vs previous end";
const FUSED_SPAN_COUNT: &str = "pool fused gemv: destination spans vs parts";
fn pool_gemv_q8_k_fused_at(
pool: &mut ComputePool,
parts: &[FusedPart<'_>],
at: &[usize],
dst: &mut [f32],
stamp: bool,
) -> Result<(), KernelError> {
if parts.len() != at.len() {
return Err(KernelError::LengthMismatch {
what: FUSED_SPAN_COUNT,
left: at.len(),
right: parts.len(),
});
}
let mut watermark = 0usize;
for (part, &start) in parts.iter().zip(at) {
let end = start.saturating_add(part.out_dim);
if start < watermark {
return Err(KernelError::LengthMismatch {
what: FUSED_SPAN_ORDER,
left: start,
right: watermark,
});
}
if end > dst.len() {
return Err(KernelError::LengthMismatch {
what: FUSED_SPAN_BOUND,
left: end,
right: dst.len(),
});
}
watermark = end;
}
let total = fused_out_dim(parts);
let failure: Mutex<Option<(usize, KernelError)>> = Mutex::new(None);
let base = SendPtr(dst.as_mut_ptr());
pool.run_chunks(total, |chunk| {
let index = chunk.index;
for (p, local) in fused_row_parts(parts, chunk.rows.clone()) {
let part = parts[p];
let start = at[p] + local.start;
let len = local.len();
let chunk = unsafe { std::slice::from_raw_parts_mut(base.get().add(start), len) };
if let Err(err) = gemv_q8_k_rows(
part.format,
part.weight,
part.in_dim,
part.out_dim,
part.acts,
local,
chunk,
) {
record(&failure, index, err);
break;
}
}
if stamp && chunk.on_caller {
CALLER_DONE.set(Some(Instant::now()));
}
});
taken(failure)
}
fn pool_gemv_q8_0(
pool: &mut ComputePool,
weight: &[u8],
in_dim: usize,
out_dim: usize,
acts: &[BlockQ8_0],
out: &mut [f32],
stamp: bool,
) -> Result<(), KernelError> {
let failure: Mutex<Option<(usize, KernelError)>> = Mutex::new(None);
pool.scatter_chunks(out, |chunk, dst| {
let (index, on_caller) = (chunk.index, chunk.on_caller);
if let Err(err) = gemv_q8_0_rows(weight, in_dim, out_dim, acts, chunk.rows, dst) {
record(&failure, index, err);
}
if stamp && on_caller {
CALLER_DONE.set(Some(Instant::now()));
}
});
taken(failure)
}
const ATTENTION_FANOUT_MIN_POSITIONS: usize = 8;
fn pool_decode_attention(
pool: &mut ComputePool,
kv: &KvCache,
layer: usize,
scale: f32,
q: &[f32],
scratch: &mut [f32],
out: &mut [f32],
) -> Result<(), ForwardError> {
let n_kv_heads = kv.n_kv_heads();
let units = pool.shards();
let positions = kv.len(layer).unwrap_or(0);
let group_dim = q.len().checked_div(n_kv_heads).unwrap_or(0);
if units < 2
|| n_kv_heads < 2
|| positions < ATTENTION_FANOUT_MIN_POSITIONS
|| out.len() != q.len()
|| group_dim == 0
|| group_dim * n_kv_heads != q.len()
{
decode_attention_in(q, kv, layer, scale, scratch, out)?;
return Ok(());
}
let reachable = n_kv_heads.min(units);
let shard_scratch = scratch.len() / reachable;
let failure: Mutex<Option<(usize, AttentionError)>> = Mutex::new(None);
let out_base = SendPtr(out.as_mut_ptr());
let scratch_base = SendPtr(scratch.as_mut_ptr());
pool.run(units, |shard| {
let kv_heads = kv_head_range(n_kv_heads, units, &shard.rows);
if kv_heads.is_empty() {
return;
}
let index = shard.index;
let lo = kv_heads.start * group_dim;
let len = kv_heads.len() * group_dim;
let mine_out = unsafe { std::slice::from_raw_parts_mut(out_base.get().add(lo), len) };
let mine_scratch = unsafe {
std::slice::from_raw_parts_mut(
scratch_base.get().add(index * shard_scratch),
shard_scratch,
)
};
if let Err(err) =
decode_attention_kv_range_in(q, kv, layer, scale, kv_heads, mine_scratch, mine_out)
{
record(&failure, index, err);
}
});
taken(failure)?;
Ok(())
}
fn kv_head_range(n_kv_heads: usize, units: usize, slots: &Range<usize>) -> Range<usize> {
if units == 0 {
return 0..0;
}
let (base, rem) = (n_kv_heads / units, n_kv_heads % units);
let first = |u: usize| {
let u = u.min(units);
u * base + u.min(rem)
};
let lo = first(slots.start);
lo..first(slots.end).max(lo)
}
#[derive(Clone, Copy, Debug)]
struct MoeDims {
hidden: usize,
moe: usize,
top_k: usize,
}
const FUSED_WINDOW_PARTS: &str = "expert window: experts vs MAX_FUSED_EXPERTS";
const FUSED_WINDOW_GATE_UP: &str = "expert window: gate_up vs 2 * experts * moe_intermediate";
const FUSED_WINDOW_ACTS: &str = "expert window: intermediate blocks vs experts";
const MAX_FUSED_EXPERTS: usize = 16;
struct FfnScratch<'a> {
acts_hidden: &'a [BlockQ8K],
acts_moe: &'a mut [BlockQ8K],
gate_up: &'a mut [f32],
}
struct MoeScratch<'a> {
ffn: FfnScratch<'a>,
staged: &'a mut [f32],
done: &'a mut [bool],
}
fn run_expert_window(
stream: &ExpertStream,
pool: &mut ComputePool,
layer: u32,
dims: MoeDims,
window: &[(usize, u32)],
scratch: &mut MoeScratch<'_>,
gemv: &mut GemvClock<'_>,
) -> Result<(), ForwardError> {
let MoeScratch { ffn, staged, .. } = scratch;
let n = window.len();
let blocks = dims.moe / QuantFormat::Q8_K.block_weights();
let acts_hidden: &[BlockQ8K] = ffn.acts_hidden;
let gate_up_len = 2usize.saturating_mul(n).saturating_mul(dims.moe);
for (what, have, need) in [
(FUSED_WINDOW_PARTS, MAX_FUSED_EXPERTS, n),
(FUSED_WINDOW_GATE_UP, ffn.gate_up.len(), gate_up_len),
(
FUSED_WINDOW_ACTS,
ffn.acts_moe.len(),
n.saturating_mul(blocks),
),
] {
if have < need {
return Err(KernelError::LengthMismatch {
what,
left: have,
right: need,
}
.into());
}
}
let mut gate_up = [FusedPart::empty(); 2 * MAX_FUSED_EXPERTS];
let mut down = [FusedPart::empty(); MAX_FUSED_EXPERTS];
for (j, &(_, slot)) in window.iter().enumerate() {
let view = stream.view(layer, slot)?;
let (gate_slab, up_slab, down_slab) = (view.gate(), view.up(), view.down());
gate_up[2 * j] = FusedPart {
format: gate_slab.format,
weight: gate_slab.bytes,
in_dim: dims.hidden,
out_dim: dims.moe,
acts: acts_hidden,
};
gate_up[2 * j + 1] = FusedPart {
format: up_slab.format,
weight: up_slab.bytes,
in_dim: dims.hidden,
out_dim: dims.moe,
acts: acts_hidden,
};
down[j] = FusedPart {
format: down_slab.format,
weight: down_slab.bytes,
in_dim: dims.moe,
out_dim: dims.hidden,
acts: &[],
};
}
let combined = &mut ffn.gate_up[..gate_up_len];
let opened = gemv.open();
pool_gemv_q8_k_fused(pool, &gate_up[..2 * n], combined, opened.is_some())?;
gemv.close(GemvSite::Experts, opened);
for j in 0..n {
let slot = &mut combined[j * 2 * dims.moe..(j + 1) * 2 * dims.moe];
let (gate, up) = slot.split_at_mut(dims.moe);
swiglu_combine(gate, up)?;
quantize_row_q8_k(gate, &mut ffn.acts_moe[j * blocks..(j + 1) * blocks])?;
}
let intermediates: &[BlockQ8K] = ffn.acts_moe;
let mut at = [0usize; MAX_FUSED_EXPERTS];
for (j, &(index, _)) in window.iter().enumerate() {
down[j].acts = &intermediates[j * blocks..(j + 1) * blocks];
at[j] = index * dims.hidden;
}
let opened = gemv.open();
pool_gemv_q8_k_fused_at(pool, &down[..n], &at[..n], staged, opened.is_some())?;
gemv.close(GemvSite::Experts, opened);
Ok(())
}
fn run_plan(
stream: &ExpertStream,
pool: &mut ComputePool,
layer: u32,
dims: MoeDims,
plan: &[(usize, u32)],
scratch: &mut MoeScratch<'_>,
gemv: &mut GemvClock<'_>,
) -> Result<(), ForwardError> {
for &(index, _) in plan {
if index >= dims.top_k || scratch.done[index] {
return Err(ForwardError::StreamPlanIndex {
layer,
index,
top_k: dims.top_k,
});
}
scratch.done[index] = true;
}
let mut window = [(0usize, 0u32); MAX_FUSED_EXPERTS];
let mut base = 0usize;
while base < dims.top_k {
let end = base.saturating_add(MAX_FUSED_EXPERTS).min(dims.top_k);
let mut n = 0usize;
for index in base..end {
if let Some(&(_, slot)) = plan.iter().find(|&&(i, _)| i == index) {
window[n] = (index, slot);
n += 1;
}
}
if n > 0 {
run_expert_window(stream, pool, layer, dims, &window[..n], scratch, gemv)?;
}
base = end;
}
Ok(())
}
fn stream_experts(
stream: &mut ExpertStream,
pool: &mut ComputePool,
layer: u32,
dims: MoeDims,
scratch: &mut MoeScratch<'_>,
clock: &mut PhaseClock<'_>,
gemv: &mut GemvClock<'_>,
) -> Result<(), ForwardError> {
let outcome = stage_expert_phases(stream, pool, layer, dims, scratch, clock, gemv);
if outcome.is_err()
&& let Err(drain) = stream.await_misses()
{
tracing::error!(
layer,
error = %drain,
"draining a failed expert phase also failed"
);
}
outcome
}
#[allow(clippy::too_many_arguments)]
fn stage_expert_phases(
stream: &mut ExpertStream,
pool: &mut ComputePool,
layer: u32,
dims: MoeDims,
scratch: &mut MoeScratch<'_>,
clock: &mut PhaseClock<'_>,
gemv: &mut GemvClock<'_>,
) -> Result<(), ForwardError> {
scratch.done.fill(false);
run_plan(stream, pool, layer, dims, stream.hits(), scratch, gemv)?;
clock.charge(Phase::ExpertCompute);
stream.await_misses()?;
clock.charge(Phase::ExpertIo);
run_plan(stream, pool, layer, dims, stream.misses(), scratch, gemv)?;
clock.charge(Phase::ExpertCompute);
let covered = scratch.done.iter().filter(|filled| **filled).count();
if covered != dims.top_k {
return Err(ForwardError::StreamPlanCoverage {
layer,
covered,
top_k: dims.top_k,
});
}
Ok(())
}
pub fn forward_token<'s>(
model: &Model,
state: &'s mut ForwardState,
token_id: u32,
position: usize,
want_logits: bool,
) -> Result<Option<&'s [f32]>, ForwardError> {
forward_token_traced(model, state, token_id, position, want_logits, None)
}
pub fn forward_token_traced<'s>(
model: &Model,
state: &'s mut ForwardState,
token_id: u32,
position: usize,
want_logits: bool,
mut on_route: Option<ExpertRouteSink<'_>>,
) -> Result<Option<&'s [f32]>, ForwardError> {
let arch = model.arch();
state.arch.check(arch)?;
let n_heads = arch.n_heads as usize;
let n_kv_heads = arch.n_kv_heads as usize;
let head_dim = arch.head_dim as usize;
let hidden = arch.hidden as usize;
let q_dim = n_heads * head_dim;
let kv_dim = n_kv_heads * head_dim;
let dims = MoeDims {
hidden,
moe: arch.moe_intermediate as usize,
top_k: arch.top_k as usize,
};
let eps = arch.rms_eps as f32;
let theta = arch.rope_theta as f32;
let scale = 1.0 / (head_dim as f32).sqrt();
let ForwardState {
arch: _,
kv,
attn_scratch,
hidden: residual,
normed,
acts_q8k_hidden,
acts_q8k_moe,
acts_q8k_attn,
acts_q8_0_hidden,
q,
k,
v,
attn_out,
o_proj,
router_logits,
router_probs,
topk,
expert_ids,
gate_up,
expert_staged,
expert_done,
expert_acc,
logits,
pool,
stream,
prefill: _,
sweep_plan: _,
routed: _,
prefill_timing,
decode_timing,
decode_gemv,
prefill_charging,
} = state;
let (mut clock, mut gemv) = if *prefill_charging {
(PhaseClock::new(prefill_timing, true), GemvClock::disarmed())
} else {
(
PhaseClock::decoding(decode_timing),
GemvClock::new(decode_gemv, true),
)
};
let expected = kv.seq_len()?;
if position != expected {
return Err(ForwardError::PositionMismatch { position, expected });
}
let rope_pos =
u32::try_from(position).map_err(|_| ForwardError::PositionOverflow { position })?;
model.embed(token_id, residual)?;
for layer in 0..model.n_layers() {
let lw = model.layer(layer)?;
let layer_idx = layer as usize;
rmsnorm(residual, lw.attn_norm, eps, normed)?;
quantize_row_q8_k(normed, acts_q8k_hidden)?;
quantize_row_q8_0(normed, acts_q8_0_hidden)?;
clock.charge(Phase::Elementwise);
let opened = gemv.open();
pool_gemv_q8_k_stamped(
pool,
lw.attn_q.format,
lw.attn_q.bytes,
hidden,
q_dim,
acts_q8k_hidden,
q,
opened.is_some(),
)?;
gemv.close(GemvSite::Projections, opened);
let opened = gemv.open();
pool_gemv_q8_0(
pool,
lw.attn_k.bytes,
hidden,
kv_dim,
acts_q8_0_hidden,
k,
opened.is_some(),
)?;
gemv.close(GemvSite::Projections, opened);
let opened = gemv.open();
pool_gemv_q8_k_stamped(
pool,
lw.attn_v.format,
lw.attn_v.bytes,
hidden,
kv_dim,
acts_q8k_hidden,
v,
opened.is_some(),
)?;
gemv.close(GemvSite::Projections, opened);
clock.charge(Phase::Projections);
for head in q.chunks_exact_mut(head_dim) {
rmsnorm_in_place(head, lw.attn_q_norm, eps)?;
}
for head in k.chunks_exact_mut(head_dim) {
rmsnorm_in_place(head, lw.attn_k_norm, eps)?;
}
rope_neox_heads(q, n_heads, head_dim, rope_pos, theta)?;
rope_neox_heads(k, n_kv_heads, head_dim, rope_pos, theta)?;
kv.append(layer_idx, k, v)?;
clock.charge(Phase::Elementwise);
pool_decode_attention(pool, kv, layer_idx, scale, q, attn_scratch, attn_out)?;
clock.charge(Phase::Attention);
quantize_row_q8_k(attn_out, acts_q8k_attn)?;
clock.charge(Phase::Elementwise);
let opened = gemv.open();
pool_gemv_q8_k_stamped(
pool,
lw.attn_output.format,
lw.attn_output.bytes,
q_dim,
hidden,
acts_q8k_attn,
o_proj,
opened.is_some(),
)?;
gemv.close(GemvSite::Projections, opened);
clock.charge(Phase::Projections);
vec_add(residual, o_proj)?;
rmsnorm(residual, lw.ffn_norm, eps, normed)?;
clock.charge(Phase::Elementwise);
let opened = gemv.open();
{
let router = lw.router.data();
let stamp = opened.is_some();
let normed = &*normed;
pool.scatter_chunks(router_logits, |chunk, logits| {
let rows = &router[chunk.rows.start * hidden..chunk.rows.end * hidden];
for (row, logit) in rows.chunks_exact(hidden).zip(logits.iter_mut()) {
*logit = dot_f32(row, normed);
}
if stamp && chunk.on_caller {
CALLER_DONE.set(Some(Instant::now()));
}
});
}
gemv.close(GemvSite::Router, opened);
router_probs.copy_from_slice(router_logits);
softmax(router_probs)?;
topk.clear();
for _ in 0..dims.top_k {
let mut best_e = 0usize;
let mut best_p = f32::NEG_INFINITY;
for (e, &p) in router_probs.iter().enumerate() {
if p > best_p {
best_p = p;
best_e = e;
}
}
topk.push((best_e as u32, best_p));
router_probs[best_e] = f32::NEG_INFINITY;
}
if arch.norm_topk_prob {
let sum: f32 = topk.iter().map(|&(_, w)| w).sum();
for (_, w) in topk.iter_mut() {
*w /= sum;
}
}
clock.charge(Phase::Router);
if let Some(sink) = on_route.as_deref_mut() {
sink(layer, topk);
}
quantize_row_q8_k(normed, acts_q8k_hidden)?;
expert_ids.clear();
expert_ids.extend(topk.iter().map(|&(expert, _)| expert));
clock.charge(Phase::Elementwise);
stream.begin_layer(layer, expert_ids)?;
clock.charge(Phase::ExpertIo);
let mut scratch = MoeScratch {
ffn: FfnScratch {
acts_hidden: acts_q8k_hidden,
acts_moe: acts_q8k_moe,
gate_up,
},
staged: expert_staged,
done: expert_done,
};
let phases = stream_experts(
stream,
pool,
layer,
dims,
&mut scratch,
&mut clock,
&mut gemv,
);
stream.end_layer(layer);
clock.charge(Phase::ExpertIo);
phases?;
expert_acc.fill(0.0);
for (i, &(_, weight)) in topk.iter().enumerate() {
let staged = &expert_staged[i * hidden..(i + 1) * hidden];
for (acc, &d) in expert_acc.iter_mut().zip(staged) {
*acc += weight * d;
}
}
vec_add(residual, expert_acc)?;
clock.charge(Phase::Elementwise);
}
if !want_logits {
clock.close();
return Ok(None);
}
rmsnorm(residual, model.final_norm(), eps, normed)?;
quantize_row_q8_k(normed, acts_q8k_hidden)?;
clock.charge(Phase::Elementwise);
let head = model.lm_head();
let opened = gemv.open();
pool_gemv_q8_k_stamped(
pool,
head.format,
head.bytes,
head.in_dim,
head.out_dim,
acts_q8k_hidden,
logits,
opened.is_some(),
)?;
gemv.close(GemvSite::LmHead, opened);
clock.charge(Phase::Projections);
clock.close();
Ok(Some(logits))
}
#[cfg(test)]
pub(crate) mod testsupport {
use std::collections::BTreeMap;
use std::path::Path;
use crate::format::{CommonTensor, LayerLayout};
use crate::io::parse_quant_format;
use crate::io::testutil::Fixture;
use crate::kernels::quants::{QuantFormat, f32_to_f16};
fn restamp_scales(bytes: &mut [u8], format: QuantFormat) {
let d = f32_to_f16(f32::powi(2.0, -12)).to_le_bytes();
for block in bytes.chunks_exact_mut(format.block_bytes()) {
match format {
QuantFormat::Q4_K | QuantFormat::Q5_K => {
block[0..2].copy_from_slice(&d);
block[2..4].copy_from_slice(&d);
}
QuantFormat::Q6_K => block[208..210].copy_from_slice(&d),
QuantFormat::Q8_0 => block[0..2].copy_from_slice(&d),
QuantFormat::Q8_K => unreachable!("activation-only format"),
}
}
}
pub(crate) fn temper_install(fx: &Fixture) {
temper_parts(&fx.root, &fx.manifest.common_tensors, &fx.layout.layers);
}
pub(crate) fn temper_parts(
root: &Path,
common_tensors: &BTreeMap<String, CommonTensor>,
layers: &[LayerLayout],
) {
let path = root.join("common.bin");
let mut bytes = std::fs::read(&path).unwrap();
for tensor in common_tensors.values() {
if let Some(format) = parse_quant_format(&tensor.dtype) {
let start = tensor.offset as usize;
restamp_scales(&mut bytes[start..start + tensor.len as usize], format);
}
}
std::fs::write(&path, bytes).unwrap();
for layer in layers {
let path = root.join(&layer.file);
let mut bytes = std::fs::read(&path).unwrap();
for expert in 0..layer.n_experts {
let base = expert as usize * layer.stride as usize;
for projection in &layer.projections {
let format = parse_quant_format(&projection.quant).unwrap();
let start = base + projection.offset_in_blob as usize;
restamp_scales(&mut bytes[start..start + projection.len as usize], format);
}
}
std::fs::write(&path, bytes).unwrap();
}
}
}
#[cfg(test)]
mod tests {
use super::testsupport::temper_install;
use super::*;
use crate::io::LoadOptions;
use crate::io::testutil::{Fixture, VOCAB, build_install};
const SKIP: LoadOptions = LoadOptions {
skip_hashes: true,
verify_layer_hashes: false,
common_memory: crate::io::CommonMemory::Mapped,
};
fn load_fixture(tag: &str) -> (Fixture, Model) {
let fx = build_install(tag);
temper_install(&fx);
let model = Model::load(&fx.root, SKIP).unwrap();
(fx, model)
}
pub(crate) fn state(model: &Model, context_cap: usize) -> ForwardState {
ForwardState::with_config(model, context_cap, RuntimeConfig::testing()).unwrap()
}
fn run(model: &Model, state: &mut ForwardState, ids: &[u32]) -> Vec<f32> {
let last = ids.len() - 1;
for (pos, &id) in ids.iter().enumerate() {
let want = pos == last;
let out = forward_token(model, state, id, pos, want).unwrap();
assert_eq!(out.is_some(), want);
if let Some(logits) = out {
return logits.to_vec();
}
}
unreachable!("last token always requests logits");
}
fn spread(i: usize) -> f32 {
((i * 37 % 101) as f32 - 50.0) / 64.0
}
#[test]
fn a_multi_shard_failure_reports_the_lowest_shard_index() {
let err = |layer| AttentionError::EmptyLayer { layer };
for arrival in [[3usize, 1, 2, 0], [0, 1, 2, 3], [1, 0, 3, 2]] {
let slot: Mutex<Option<(usize, AttentionError)>> = Mutex::new(None);
for index in arrival {
record(&slot, index, err(index));
}
assert_eq!(
taken(slot).unwrap_err(),
err(0),
"arrival order {arrival:?} must not change the reported error"
);
}
}
#[test]
fn the_kv_head_split_tiles_the_kv_heads() {
for (n_kv_heads, units) in [
(4usize, 6usize),
(4, 4),
(4, 32),
(4, 3),
(4, 2),
(3, 2),
(8, 6),
(1, 6),
(0, 6),
] {
let mut covered = vec![0u32; n_kv_heads];
let mut empty = 0usize;
for slot in 0..units {
let range = kv_head_range(n_kv_heads, units, &(slot..slot + 1));
assert!(range.start <= range.end, "kv {n_kv_heads} units {units}");
assert!(range.end <= n_kv_heads, "kv {n_kv_heads} units {units}");
if range.is_empty() {
empty += 1;
}
for head in range {
covered[head] += 1;
}
}
assert!(
covered.iter().all(|&n| n == 1),
"kv {n_kv_heads} units {units}: {covered:?}"
);
assert_eq!(
empty,
units - n_kv_heads.min(units),
"kv {n_kv_heads} units {units}: surplus slots must be empty no-ops"
);
assert_eq!(
kv_head_range(n_kv_heads, units, &(0..units)),
0..n_kv_heads,
"kv {n_kv_heads} units {units}: single-shard fallback"
);
}
}
#[test]
fn the_kv_range_split_is_bit_identical_to_the_whole_call() {
const GEOMETRIES: [(usize, usize, usize); 4] =
[(3, 4, 8), (1, 8, 8), (4, 1, 16), (2, 3, 4)];
const LENS: [usize; 4] = [1, 7, 8, 17];
struct Case {
label: String,
q: Vec<f32>,
kv: KvCache,
scale: f32,
carve: usize,
want: Vec<Vec<f32>>,
}
let cap = LENS.iter().copied().max().unwrap();
let cases: Vec<Case> = GEOMETRIES
.into_iter()
.map(|(n_kv_heads, group, head_dim)| {
let n_heads = n_kv_heads * group;
let (q_dim, kv_dim) = (n_heads * head_dim, n_kv_heads * head_dim);
let mut kv = KvCache::new(LENS.len(), n_kv_heads, head_dim, cap).unwrap();
for (layer, &len) in LENS.iter().enumerate() {
for t in 0..len {
let k: Vec<f32> = (0..kv_dim).map(|i| spread(t * 7 + i + layer)).collect();
let v: Vec<f32> = (0..kv_dim)
.map(|i| spread(t * 11 + i + 3 + layer * 5))
.collect();
kv.append(layer, &k, &v).unwrap();
}
}
let q: Vec<f32> = (0..q_dim).map(|i| spread(i * 5 + 1)).collect();
let scale = 1.0 / (head_dim as f32).sqrt();
let carve = attention_scratch_len(n_heads, n_kv_heads, head_dim, cap);
let want: Vec<Vec<f32>> = (0..LENS.len())
.map(|layer| {
let mut w = vec![0.0f32; q_dim];
decode_attention_in(&q, &kv, layer, scale, &mut vec![0.0; carve], &mut w)
.unwrap();
assert!(
w.iter().any(|v| *v != 0.0),
"the reference is degenerate at layer {layer}"
);
w
})
.collect();
Case {
label: format!("({n_kv_heads}, {group}, {head_dim})"),
q,
kv,
scale,
carve,
want,
}
})
.collect();
for shards in [1usize, 2, 3, 4, 5, 8] {
let mut pool = ComputePool::with_config(PoolConfig {
shards: Some(shards),
placement: Placement::Off,
pin_caller: false,
inline_caller: true,
..PoolConfig::default()
});
for case in &cases {
let reachable = case.kv.n_kv_heads().min(pool.shards()).max(1);
let mut scratch = vec![0.0f32; case.carve * reachable];
for layer in 0..LENS.len() {
let mut got = vec![f32::NAN; case.q.len()];
pool_decode_attention(
&mut pool,
&case.kv,
layer,
case.scale,
&case.q,
&mut scratch,
&mut got,
)
.unwrap();
for (i, (a, b)) in got.iter().zip(&case.want[layer]).enumerate() {
assert_eq!(
a.to_bits(),
b.to_bits(),
"{} shards {shards} layer {layer} element {i}",
case.label
);
}
}
}
}
}
#[test]
fn attn_scratch_holds_one_carve_per_reachable_shard() {
let (_fx, model) = load_fixture("fwd-attn-carves");
let arch = model.arch();
let n_kv = arch.n_kv_heads as usize;
assert!(n_kv >= 2, "the fixture must have a kv head axis to split");
let cap = 16usize;
let carve = attention_scratch_len(arch.n_heads as usize, n_kv, arch.head_dim as usize, cap);
for threads in [1usize, 2, n_kv, n_kv + 3, n_kv * 4] {
let config = RuntimeConfig {
cache_bytes: 4 * 1024 * 1024,
threads: Some(threads),
placement: Placement::Off,
};
let st = ForwardState::with_config(&model, cap, config).unwrap();
let shards = st.pool.shards();
assert_eq!(
st.attn_scratch.len(),
carve * n_kv.min(shards),
"threads {threads} (pool gave {shards} shards)"
);
assert!(
st.attn_scratch.len() >= carve,
"threads {threads}: the serial fall-through needs a whole carve"
);
}
}
#[test]
fn logits_are_finite_and_vocab_shaped() {
let (_fx, model) = load_fixture("fwd-finite");
let mut st = state(&model, 8);
assert_eq!(st.context_cap(), 8);
let logits = run(&model, &mut st, &[1, 2, 3]);
assert_eq!(logits.len(), VOCAB);
assert!(logits.iter().all(|v| v.is_finite()), "{logits:?}");
assert!(logits.iter().any(|&v| v != logits[0]));
assert_eq!(st.seq_len().unwrap(), 3);
}
#[test]
fn identical_inputs_are_bitwise_deterministic() {
let (_fx, model) = load_fixture("fwd-determinism");
let mut a = state(&model, 8);
let mut b = state(&model, 8);
let la = run(&model, &mut a, &[5, 0, 7]);
let lb = run(&model, &mut b, &[5, 0, 7]);
let bits_a: Vec<u32> = la.iter().map(|v| v.to_bits()).collect();
let bits_b: Vec<u32> = lb.iter().map(|v| v.to_bits()).collect();
assert_eq!(bits_a, bits_b);
}
#[test]
fn shard_count_and_cache_size_do_not_move_a_bit() {
let (_fx, model) = load_fixture("fwd-shards");
let prompt: [u32; 12] = [5, 0, 7, 3, 11, 2, 9, 1, 14, 6, 8, 4];
let single = RuntimeConfig {
cache_bytes: 4 * 1024 * 1024,
threads: Some(1),
placement: Placement::Off,
};
let mut one = ForwardState::with_config(&model, 16, single).unwrap();
let want = run(&model, &mut one, &prompt);
let layout = model.layout();
let widest = layout.layers.iter().map(|l| l.stride).max().unwrap();
let tightest = widest * layout.layers.len() as u64 * u64::from(model.arch().top_k.max(1));
for threads in [2usize, 4] {
for cache_bytes in [tightest, 4 * 1024 * 1024] {
let config = RuntimeConfig {
cache_bytes,
threads: Some(threads),
placement: Placement::Off,
};
let mut many = ForwardState::with_config(&model, 16, config).unwrap();
let _ = run(&model, &mut many, &prompt);
let mut again = ForwardState::with_config(&model, 16, config).unwrap();
let got = run(&model, &mut again, &prompt);
for (i, (&g, &w)) in got.iter().zip(&want).enumerate() {
assert_eq!(
g.to_bits(),
w.to_bits(),
"threads={threads} cache_bytes={cache_bytes}: logit {i}"
);
}
}
}
}
#[test]
fn zeroed_lm_head_yields_all_zero_logits() {
let fx = build_install("fwd-zero-head");
temper_install(&fx);
let entry = &fx.manifest.common_tensors["output.weight"];
let path = fx.root.join("common.bin");
let mut bytes = std::fs::read(&path).unwrap();
let (start, len) = (entry.offset as usize, entry.len as usize);
bytes[start..start + len].fill(0);
std::fs::write(&path, bytes).unwrap();
let model = Model::load(&fx.root, SKIP).unwrap();
let mut st = state(&model, 4);
let logits = run(&model, &mut st, &[3]);
assert!(logits.iter().all(|&v| v == 0.0), "{logits:?}");
}
#[test]
fn positions_must_be_sequential() {
let (_fx, model) = load_fixture("fwd-positions");
let mut st = state(&model, 4);
assert!(matches!(
forward_token(&model, &mut st, 0, 1, false).unwrap_err(),
ForwardError::PositionMismatch {
position: 1,
expected: 0
}
));
forward_token(&model, &mut st, 0, 0, false).unwrap();
assert!(matches!(
forward_token(&model, &mut st, 0, 0, false).unwrap_err(),
ForwardError::PositionMismatch {
position: 0,
expected: 1
}
));
}
#[test]
fn context_cap_exhaustion_is_typed() {
let (_fx, model) = load_fixture("fwd-cap");
let mut st = state(&model, 2);
forward_token(&model, &mut st, 0, 0, false).unwrap();
forward_token(&model, &mut st, 1, 1, false).unwrap();
assert!(matches!(
forward_token(&model, &mut st, 2, 2, false).unwrap_err(),
ForwardError::Kv(KvError::CapacityExceeded { .. })
));
}
#[test]
fn bad_token_is_typed() {
let (_fx, model) = load_fixture("fwd-token");
let mut st = state(&model, 4);
assert!(matches!(
forward_token(&model, &mut st, VOCAB as u32, 0, true).unwrap_err(),
ForwardError::Model(ModelError::TokenOutOfRange { .. })
));
}
#[test]
fn runtime_dials_are_reported() {
let (_fx, model) = load_fixture("fwd-dials");
let st = state(&model, 4);
assert_eq!(st.shards(), 2);
assert!(st.slots_per_layer() >= 1);
assert!(st.cache_bytes() > 0);
let stats = st.stream_stats();
assert_eq!(stats.hits + stats.misses, 0, "no experts read yet");
assert!(format!("{st:?}").contains("ForwardState"));
}
fn with_expert_scratch<R>(
st: &mut ForwardState,
staged: &mut [f32],
done: &mut [bool],
body: impl FnOnce(&mut ExpertStream, &mut ComputePool, &mut MoeScratch<'_>) -> R,
) -> R {
let ForwardState {
acts_q8k_hidden,
acts_q8k_moe,
gate_up,
pool,
stream,
..
} = st;
let mut scratch = MoeScratch {
ffn: FfnScratch {
acts_hidden: &acts_q8k_hidden[..],
acts_moe: &mut acts_q8k_moe[..],
gate_up: &mut gate_up[..],
},
staged,
done,
};
body(stream, pool, &mut scratch)
}
fn dims_with_top_k(model: &Model, top_k: usize) -> MoeDims {
MoeDims {
hidden: model.arch().hidden as usize,
moe: model.arch().moe_intermediate as usize,
top_k,
}
}
fn warm(stream: &mut ExpertStream, layer: u32, expert: u32) -> u32 {
stream.begin_layer(layer, &[expert]).unwrap();
let slot = stream.misses()[0].1;
stream.await_misses().unwrap();
stream.end_layer(layer);
slot
}
#[test]
fn a_failed_hits_phase_drains_its_in_flight_reads() {
let (_fx, model) = load_fixture("fwd-drain");
assert_eq!(model.arch().top_k, 2, "the fixture routes two experts");
let hidden = model.arch().hidden as usize;
let mut st = state(&model, 4);
let dims = dims_with_top_k(&model, 1);
let mut staged = vec![0.0f32; hidden];
let mut done = vec![false; 1];
with_expert_scratch(&mut st, &mut staged, &mut done, |stream, pool, scratch| {
warm(stream, 0, 0);
stream.begin_layer(0, &[1, 0]).unwrap();
assert_eq!(stream.hits().len(), 1, "expert 0 is resident");
assert_eq!(stream.misses().len(), 1, "expert 1 is cold");
assert_eq!(stream.hits()[0].0, 1, "the hit is the second routed id");
let mut timing = PrefillTiming::default();
let err = stream_experts(
stream,
pool,
0,
dims,
scratch,
&mut PhaseClock::new(&mut timing, false),
&mut GemvClock::disarmed(),
)
.unwrap_err();
assert!(
matches!(
err,
ForwardError::StreamPlanIndex {
layer: 0,
index: 1,
top_k: 1
}
),
"{err}"
);
stream.end_layer(0);
stream.begin_layer(1, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(1);
});
let logits = run(&model, &mut st, &[1, 2]);
assert_eq!(logits.len(), VOCAB);
assert!(logits.iter().all(|v| v.is_finite()), "{logits:?}");
}
#[test]
fn a_plan_that_is_not_a_permutation_is_rejected() {
let (_fx, model) = load_fixture("fwd-plan-index");
let hidden = model.arch().hidden as usize;
let mut st = state(&model, 4);
let dims = dims_with_top_k(&model, 2);
let mut staged = vec![0.0f32; 2 * hidden];
let mut done = vec![false; 2];
with_expert_scratch(&mut st, &mut staged, &mut done, |stream, pool, scratch| {
let slot = warm(stream, 0, 0);
scratch.done.fill(false);
let err = run_plan(
stream,
pool,
0,
dims,
&[(2, slot)],
scratch,
&mut GemvClock::disarmed(),
)
.unwrap_err();
assert!(
matches!(
err,
ForwardError::StreamPlanIndex {
layer: 0,
index: 2,
top_k: 2
}
),
"{err}"
);
assert_eq!(scratch.done, &[false, false]);
scratch.done.fill(false);
let err = run_plan(
stream,
pool,
0,
dims,
&[(0, slot), (0, slot)],
scratch,
&mut GemvClock::disarmed(),
)
.unwrap_err();
assert!(
matches!(
err,
ForwardError::StreamPlanIndex {
layer: 0,
index: 0,
top_k: 2
}
),
"{err}"
);
assert_eq!(scratch.done, &[true, false], "only the first was taken");
});
}
#[test]
fn a_plan_that_covers_fewer_than_top_k_experts_is_rejected() {
let (_fx, model) = load_fixture("fwd-plan-coverage");
let hidden = model.arch().hidden as usize;
let mut st = state(&model, 4);
let dims = dims_with_top_k(&model, 2);
let mut staged = vec![0.0f32; 2 * hidden];
let mut done = vec![false; 2];
with_expert_scratch(&mut st, &mut staged, &mut done, |stream, pool, scratch| {
stream.begin_layer(0, &[1, 1]).unwrap();
assert_eq!(
stream.hits().len() + stream.misses().len(),
1,
"a repeated id resolves once"
);
let mut timing = PrefillTiming::default();
let err = stream_experts(
stream,
pool,
0,
dims,
scratch,
&mut PhaseClock::new(&mut timing, false),
&mut GemvClock::disarmed(),
)
.unwrap_err();
assert!(
matches!(
err,
ForwardError::StreamPlanCoverage {
layer: 0,
covered: 1,
top_k: 2
}
),
"{err}"
);
stream.end_layer(0);
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
stream.end_layer(0);
});
}
fn synth_q4_k(in_dim: usize, out_dim: usize, seed: u8) -> Vec<u8> {
let block_bytes = QuantFormat::Q4_K.block_bytes();
let blocks = out_dim * (in_dim / QuantFormat::Q4_K.block_weights());
let mut w = vec![0u8; blocks * block_bytes];
let d = crate::kernels::quants::f32_to_f16(f32::powi(2.0, -8)).to_le_bytes();
for (b, block) in w.chunks_exact_mut(block_bytes).enumerate() {
block[0..2].copy_from_slice(&d);
block[2..4].copy_from_slice(&d);
for (i, byte) in block[4..].iter_mut().enumerate() {
*byte = (i as u8)
.wrapping_mul(31)
.wrapping_add(b as u8)
.wrapping_add(seed);
}
}
w
}
#[test]
fn a_gapped_fused_fan_out_writes_only_its_own_spans() {
let (in_dim, out_dim) = (256usize, 40usize);
let weights: Vec<Vec<u8>> = (0..3).map(|s| synth_q4_k(in_dim, out_dim, s * 7)).collect();
let x: Vec<f32> = (0..in_dim).map(spread).collect();
let mut acts = vec![BlockQ8K::default(); in_dim / QuantFormat::Q8_K.block_weights()];
quantize_row_q8_k(&x, &mut acts).unwrap();
let parts: Vec<FusedPart<'_>> = weights
.iter()
.map(|w| FusedPart {
format: QuantFormat::Q4_K,
weight: w,
in_dim,
out_dim,
acts: &acts,
})
.collect();
let at = [out_dim, 2 * out_dim, 4 * out_dim];
const FILL: f32 = 1.25;
for shards in [1usize, 2, 3, 5] {
let mut pool = ComputePool::with_config(PoolConfig {
shards: Some(shards),
placement: Placement::Off,
pin_caller: false,
inline_caller: true,
..PoolConfig::default()
});
let mut dst = vec![FILL; 8 * out_dim];
pool_gemv_q8_k_fused_at(&mut pool, &parts, &at, &mut dst, false).unwrap();
for (p, &start) in at.iter().enumerate() {
let mut want = vec![f32::NAN; out_dim];
crate::kernels::gemv_q8_k(
QuantFormat::Q4_K,
&weights[p],
in_dim,
out_dim,
&acts,
&mut want,
)
.unwrap();
assert!(
want.iter().all(|v| v.is_finite() && *v != FILL),
"shards={shards}: part {p} reference is degenerate: {want:?}"
);
for (r, (&g, &w)) in dst[start..start + out_dim].iter().zip(&want).enumerate() {
assert_eq!(
g.to_bits(),
w.to_bits(),
"shards={shards}: part {p} row {r} ({g} vs {w})"
);
}
}
for (i, &v) in dst.iter().enumerate() {
let inside = at.iter().any(|&s| (s..s + out_dim).contains(&i));
assert!(
inside || v == FILL,
"shards={shards}: element {i} outside every span was written ({v})"
);
}
}
let mut pool = ComputePool::with_config(PoolConfig {
shards: Some(2),
placement: Placement::Off,
pin_caller: false,
inline_caller: true,
..PoolConfig::default()
});
let mut dst = vec![0.0f32; 8 * out_dim];
let overlap = [0usize, out_dim - 1, 4 * out_dim];
assert!(matches!(
pool_gemv_q8_k_fused_at(&mut pool, &parts, &overlap, &mut dst, false).unwrap_err(),
KernelError::LengthMismatch {
what: FUSED_SPAN_ORDER,
..
}
));
let descending = [2 * out_dim, 0, 4 * out_dim];
assert!(matches!(
pool_gemv_q8_k_fused_at(&mut pool, &parts, &descending, &mut dst, false).unwrap_err(),
KernelError::LengthMismatch {
what: FUSED_SPAN_ORDER,
..
}
));
let past_end = [0usize, out_dim, 8 * out_dim];
assert!(matches!(
pool_gemv_q8_k_fused_at(&mut pool, &parts, &past_end, &mut dst, false).unwrap_err(),
KernelError::LengthMismatch {
what: FUSED_SPAN_BOUND,
..
}
));
assert!(matches!(
pool_gemv_q8_k_fused_at(&mut pool, &parts, &at[..2], &mut dst, false).unwrap_err(),
KernelError::LengthMismatch {
what: FUSED_SPAN_COUNT,
..
}
));
assert!(matches!(
pool_gemv_q8_k_fused(&mut pool, &parts, &mut dst[..3 * out_dim - 1], false)
.unwrap_err(),
KernelError::LengthMismatch {
what: FUSED_OUT_VS_SPACE,
..
}
));
}
fn expert_ffn_unfused(
stream: &ExpertStream,
pool: &mut ComputePool,
layer: u32,
dims: MoeDims,
slot: u32,
acts_hidden: &[BlockQ8K],
out: &mut [f32],
) {
let view = stream.view(layer, slot).unwrap();
let (g, u, d) = (view.gate(), view.up(), view.down());
let mut gate = vec![0.0f32; dims.moe];
let mut up = vec![0.0f32; dims.moe];
let hidden = dims.hidden;
pool_gemv_q8_k(
pool,
g.format,
g.bytes,
hidden,
dims.moe,
acts_hidden,
&mut gate,
)
.unwrap();
pool_gemv_q8_k(
pool,
u.format,
u.bytes,
hidden,
dims.moe,
acts_hidden,
&mut up,
)
.unwrap();
swiglu_combine(&mut gate, &up).unwrap();
let mut mid = vec![BlockQ8K::default(); dims.moe / QuantFormat::Q8_K.block_weights()];
quantize_row_q8_k(&gate, &mut mid).unwrap();
pool_gemv_q8_k(pool, d.format, d.bytes, dims.moe, hidden, &mid, out).unwrap();
}
fn plant_activations(st: &mut ForwardState, hidden: usize) {
let x: Vec<f32> = (0..hidden).map(spread).collect();
quantize_row_q8_k(&x, &mut st.acts_q8k_hidden).unwrap();
}
#[test]
fn a_fused_expert_phase_matches_the_per_expert_sequence_bitwise() {
let (_fx, model) = load_fixture("fwd-fused-experts");
let hidden = model.arch().hidden as usize;
let mut st = state(&model, 4);
let dims = dims_with_top_k(&model, 2);
plant_activations(&mut st, hidden);
let mut staged = vec![f32::NAN; 2 * hidden];
let mut done = vec![false; 2];
let mut want = vec![f32::NAN; 2 * hidden];
let mut got = vec![f32::NAN; 2 * hidden];
let mut swapped_got = vec![f32::NAN; 2 * hidden];
with_expert_scratch(&mut st, &mut staged, &mut done, |stream, pool, scratch| {
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
let mut plan: Vec<(usize, u32)> = stream.hits().to_vec();
plan.extend_from_slice(stream.misses());
plan.sort_unstable();
assert_eq!(plan.len(), 2, "the fixture routes two distinct experts");
assert_ne!(plan[0].1, plan[1].1, "two experts, two slots");
let acts: Vec<BlockQ8K> = scratch.ffn.acts_hidden.to_vec();
for &(index, slot) in &plan {
expert_ffn_unfused(
stream,
pool,
0,
dims,
slot,
&acts,
&mut want[index * hidden..(index + 1) * hidden],
);
}
assert!(
want.iter().all(|v| v.is_finite()),
"the reference is not finite: {want:?}"
);
scratch.done.fill(false);
run_plan(
stream,
pool,
0,
dims,
&plan,
scratch,
&mut GemvClock::disarmed(),
)
.unwrap();
assert_eq!(scratch.done, &[true, true]);
got.copy_from_slice(scratch.staged);
let swapped: Vec<(usize, u32)> = plan
.iter()
.map(|&(index, slot)| (1 - index, slot))
.collect();
scratch.done.fill(false);
scratch.staged.fill(f32::NAN);
run_plan(
stream,
pool,
0,
dims,
&swapped,
scratch,
&mut GemvClock::disarmed(),
)
.unwrap();
swapped_got.copy_from_slice(scratch.staged);
stream.end_layer(0);
});
for (i, (&g, &w)) in got.iter().zip(&want).enumerate() {
assert_eq!(
g.to_bits(),
w.to_bits(),
"fused expert phase differs at slot {} element {} ({g} vs {w})",
i / hidden,
i % hidden,
);
}
assert!(
want[..hidden]
.iter()
.zip(&want[hidden..])
.any(|(a, b)| a.to_bits() != b.to_bits()),
"the fixture's two experts stage the same vector"
);
for (i, (&g, &w)) in swapped_got[..hidden]
.iter()
.zip(&want[hidden..])
.enumerate()
{
assert_eq!(g.to_bits(), w.to_bits(), "swapped slot 0 element {i}");
}
for (i, (&g, &w)) in swapped_got[hidden..]
.iter()
.zip(&want[..hidden])
.enumerate()
{
assert_eq!(g.to_bits(), w.to_bits(), "swapped slot 1 element {i}");
}
}
#[test]
fn a_partial_plan_stages_by_routed_index_and_not_by_window_position() {
let (_fx, model) = load_fixture("fwd-fused-partial");
let hidden = model.arch().hidden as usize;
let mut st = state(&model, 4);
plant_activations(&mut st, hidden);
let top_k = 4usize;
let dims = dims_with_top_k(&model, top_k);
let mut staged = vec![f32::NAN; top_k * hidden];
let mut done = vec![false; top_k];
let mut want_a = vec![f32::NAN; hidden];
let mut want_b = vec![f32::NAN; hidden];
let mut one = vec![f32::NAN; top_k * hidden];
let mut two = vec![f32::NAN; top_k * hidden];
with_expert_scratch(&mut st, &mut staged, &mut done, |stream, pool, scratch| {
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
let mut resident: Vec<(usize, u32)> = stream.hits().to_vec();
resident.extend_from_slice(stream.misses());
resident.sort_unstable();
assert_eq!(resident.len(), 2, "the fixture routes two distinct experts");
let (slot_a, slot_b) = (resident[0].1, resident[1].1);
assert_ne!(slot_a, slot_b, "two experts, two slots");
let acts: Vec<BlockQ8K> = scratch.ffn.acts_hidden.to_vec();
expert_ffn_unfused(stream, pool, 0, dims, slot_a, &acts, &mut want_a);
expert_ffn_unfused(stream, pool, 0, dims, slot_b, &acts, &mut want_b);
scratch.done.fill(false);
scratch.staged.fill(f32::NAN);
run_plan(
stream,
pool,
0,
dims,
&[(1, slot_a)],
scratch,
&mut GemvClock::disarmed(),
)
.unwrap();
assert_eq!(scratch.done, &[false, true, false, false]);
one.copy_from_slice(scratch.staged);
scratch.done.fill(false);
scratch.staged.fill(f32::NAN);
run_plan(
stream,
pool,
0,
dims,
&[(1, slot_a), (3, slot_b)],
scratch,
&mut GemvClock::disarmed(),
)
.unwrap();
assert_eq!(scratch.done, &[false, true, false, true]);
two.copy_from_slice(scratch.staged);
stream.end_layer(0);
});
assert!(
want_a.iter().chain(&want_b).all(|v| v.is_finite()),
"a reference is not finite"
);
assert!(
want_a
.iter()
.zip(&want_b)
.any(|(a, b)| a.to_bits() != b.to_bits()),
"the fixture's two experts stage the same vector"
);
let slot = |buf: &[f32], i: usize| -> Vec<u32> {
buf[i * hidden..(i + 1) * hidden]
.iter()
.map(|v| v.to_bits())
.collect()
};
let untouched =
|buf: &[f32], i: usize| buf[i * hidden..(i + 1) * hidden].iter().all(|v| v.is_nan());
assert_eq!(
slot(&one, 1),
slot(&want_a, 0),
"a single-part down job staged the wrong slot, or staged it wrong"
);
for i in [0usize, 2, 3] {
assert!(
untouched(&one, i),
"a plan covering routed index 1 alone wrote staging slot {i}"
);
}
assert_eq!(slot(&two, 1), slot(&want_a, 0), "routed index 1");
assert_eq!(slot(&two, 3), slot(&want_b, 0), "routed index 3");
for i in [0usize, 2] {
assert!(
untouched(&two, i),
"a plan covering routed indices 1 and 3 wrote staging slot {i}"
);
}
}
#[test]
fn an_expert_phase_is_two_fan_outs_however_many_experts_it_has() {
let (_fx, model) = load_fixture("fwd-fused-count");
let hidden = model.arch().hidden as usize;
let mut st = state(&model, 4);
let dims = dims_with_top_k(&model, 2);
plant_activations(&mut st, hidden);
let mut staged = vec![0.0f32; 2 * hidden];
let mut done = vec![false; 2];
let mut split = GemvSplit::default();
with_expert_scratch(&mut st, &mut staged, &mut done, |stream, pool, scratch| {
stream.begin_layer(0, &[0, 1]).unwrap();
stream.await_misses().unwrap();
let mut plan: Vec<(usize, u32)> = stream.hits().to_vec();
plan.extend_from_slice(stream.misses());
plan.sort_unstable();
assert_eq!(plan.len(), 2);
let mut gemv = GemvClock::new(&mut split, true);
scratch.done.fill(false);
run_plan(stream, pool, 0, dims, &[], scratch, &mut gemv).unwrap();
scratch.done.fill(false);
run_plan(stream, pool, 0, dims, &plan[..1], scratch, &mut gemv).unwrap();
scratch.done.fill(false);
run_plan(stream, pool, 0, dims, &plan, scratch, &mut gemv).unwrap();
stream.end_layer(0);
});
assert_eq!(
split.experts.scatters, 4,
"an empty plan, a one-expert plan and a two-expert plan must charge \
0 + 2 + 2 fan-outs: {split:?}"
);
assert_eq!(
(
split.projections.scatters,
split.lm_head.scatters,
split.router.scatters
),
(0, 0, 0),
"the expert phase charged another site: {split:?}"
);
}
#[test]
fn a_state_refuses_a_model_it_was_not_built_for() {
let (_fx, wide) = load_fixture("fwd-arch-wide");
let (_fx2, narrow) = load_fixture("fwd-arch-narrow");
let mut st = state(&narrow, 4);
forward_token(&wide, &mut st, 1, 0, false).unwrap();
let fingerprint = ArchFingerprint {
top_k: narrow.arch().top_k - 1,
..st.arch
};
st.arch = fingerprint;
let err = forward_token(&narrow, &mut st, 1, 1, false).unwrap_err();
assert!(
matches!(
err,
ForwardError::ArchMismatch {
what: "top_k",
built: 1,
given: 2
}
),
"{err}"
);
}
#[test]
fn default_runtime_config_is_the_documented_budget() {
let config = RuntimeConfig::default();
assert_eq!(config.cache_bytes, DEFAULT_CACHE_BYTES);
assert_eq!(config.cache_bytes, 2_605_711_360);
assert_eq!(config.threads, None);
assert_eq!(config.placement, Placement::Shared);
}
#[test]
fn an_impossible_configuration_names_every_term_and_the_ceiling() {
let footprint = Footprint {
common: 1_073_051_648,
experts: 11 * 24 * (3_059_712 + 2_654_208),
kv: 16_384 * 96 * 1024,
runtime: crate::io::RUNTIME_ANON_BYTES,
context_cap: 16_384,
};
let err = ForwardError::ConfigTooLarge {
footprint,
limit: 3 << 30,
};
assert_eq!(
err.to_string(),
"this configuration needs about 4,113 MiB (common 1,023 + experts 1,439 + \
KV 1,536 at 16384 tokens + runtime 115) but only 3,072 MiB are available; \
reduce --context or --cache-bytes"
);
}
#[test]
fn an_enormous_context_is_refused_before_the_kv_cache_is_allocated() {
let (_fx, model) = load_fixture("fwd-footprint-refusal");
if resident_ceiling().is_none() {
return;
}
let err = ForwardState::with_config(&model, 1 << 40, RuntimeConfig::testing())
.expect_err("a petabyte of KV cache cannot be accepted");
let ForwardError::ConfigTooLarge { footprint, limit } = err else {
panic!("expected ConfigTooLarge, got {err}");
};
assert_eq!(footprint.context_cap, 1 << 40);
assert_eq!(footprint.kv, 1 << 50);
assert!(footprint.total() > u128::from(limit));
let rendered = ForwardError::ConfigTooLarge { footprint, limit }.to_string();
assert!(rendered.contains("at 1099511627776 tokens"), "{rendered}");
assert!(rendered.contains("reduce --context"), "{rendered}");
}
#[test]
fn the_gate_does_not_refuse_a_configuration_that_fits() {
let (_fx, model) = load_fixture("fwd-footprint-accepts");
for context_cap in [1usize, 8, 128, 4096] {
ForwardState::with_config(&model, context_cap, RuntimeConfig::testing())
.unwrap_or_else(|e| panic!("a {context_cap}-token context must be accepted: {e}"));
}
}
#[test]
fn a_rehearsed_failure_keeps_the_variant_it_always_had() {
let (_fx, model) = load_fixture("fwd-footprint-variant");
let err = ForwardState::with_config(
&model,
8,
RuntimeConfig {
cache_bytes: 1,
..RuntimeConfig::testing()
},
)
.expect_err("a one-byte expert budget buys nothing");
assert!(
matches!(
err,
ForwardError::Io(IoError::Slots(crate::io::SlotError::ZeroSlotsPerLayer))
),
"{err}"
);
let err = ForwardState::with_config(&model, 0, RuntimeConfig::testing())
.expect_err("a zero context is not a cache");
assert!(
matches!(err, ForwardError::Kv(KvError::InvalidDims { .. })),
"{err}"
);
}
#[test]
fn the_default_budget_clears_nineteen_slots_per_layer() {
const PER_SLOT_ALL_LAYERS: u64 = 24 * 3_059_712 + 24 * 2_654_208;
assert_eq!(PER_SLOT_ALL_LAYERS, 137_134_080);
assert_eq!(DEFAULT_CACHE_BYTES / PER_SLOT_ALL_LAYERS, 19);
assert_eq!(2484 * 1024 * 1024 / PER_SLOT_ALL_LAYERS, 18);
assert_eq!(1440 * 1024 * 1024 / PER_SLOT_ALL_LAYERS, 11);
assert_eq!(1438 * 1024 * 1024 / PER_SLOT_ALL_LAYERS, 10);
}
fn gemv_accounted(split: &[(&'static str, Duration, Duration, u64); 4]) -> Duration {
split
.iter()
.map(|&(_, own, wait, _)| own + wait)
.sum::<Duration>()
}
fn gemv_scatters(split: &[(&'static str, Duration, Duration, u64); 4], want: &str) -> u64 {
split
.iter()
.find(|&&(label, ..)| label == want)
.unwrap_or_else(|| panic!("no {want} bucket in {split:?}"))
.3
}
#[test]
fn the_gemv_sub_split_cannot_disagree_with_the_coarse_decode_split() {
let (_fx, model) = load_fixture("fwd-gemv-subsplit");
let mut st = state(&model, 16);
let layers = model.n_layers() as u64;
let top_k = model.arch().top_k as u64;
let tokens = 4u64;
for pos in 0..tokens as usize {
forward_token(&model, &mut st, (pos % VOCAB) as u32, pos, true).unwrap();
}
let split = st.decode_gemv_split();
let coarse = st.decode_timing();
assert_eq!(
coarse.tokens, tokens,
"the two blocks cover the same tokens"
);
assert_eq!(
gemv_scatters(&split, "projections"),
4 * layers * tokens,
"attn_q, attn_k, attn_v, attn_output, each once a layer: {split:?}"
);
let experts = gemv_scatters(&split, "experts");
assert_eq!(
experts % 2,
0,
"an expert phase charges a gate/up job and a down job or neither, \
so the count is even: {split:?}"
);
assert!(
experts >= 2 * layers * tokens && experts <= 4 * layers * tokens,
"every layer runs one or two non-empty expert phases at two fused \
fan-outs each, so {experts} must sit in \
[{}, {}]: {split:?}",
2 * layers * tokens,
4 * layers * tokens,
);
assert!(
3 * top_k > 4,
"the fixture routes {top_k} expert(s), so the pre-fusion per-expert \
form would have charged no more than the fused bound above: that \
bound is no longer evidence that anything is fused"
);
assert_eq!(
gemv_scatters(&split, "lm_head"),
tokens,
"one head fan-out a token: {split:?}"
);
assert_eq!(
gemv_scatters(&split, "router"),
layers * tokens,
"one router fan-out a layer: {split:?}"
);
let accounted = gemv_accounted(&split);
let inside = coarse.projections + coarse.router + coarse.expert_compute;
assert!(
accounted <= inside,
"the sub-split {accounted:?} exceeds the coarse phases {inside:?} it \
lives inside — a fan-out is charged to the wrong phase, or twice: \
{split:?}"
);
assert!(
accounted > Duration::ZERO,
"a decode that ran charged nothing: {split:?}"
);
assert!(
accounted * 2 >= inside,
"the sub-split {accounted:?} is under half the coarse phases \
{inside:?} it tiles: a site has stopped charging: {split:?}"
);
for &(label, own, _, _) in &split {
assert!(
own > Duration::ZERO,
"{label} charged no own time: {split:?}"
);
}
let pooled_wait: Duration = split.iter().map(|&(_, _, wait, _)| wait).sum();
assert!(
pooled_wait > Duration::ZERO,
"every pooled fan-out reported a zero barrier: the caller stamp is \
not reaching the submitting thread, so `own` is the whole call and \
the split says nothing: {split:?}"
);
st.reset();
assert_eq!(
st.decode_gemv_split(),
GemvSplit::default().rows(),
"a reset state reports no decode GEMVs at all"
);
}
#[test]
fn a_disarmed_gemv_clock_does_nothing() {
let sentinel = Instant::now();
CALLER_DONE.set(Some(sentinel));
let mut clock = GemvClock::disarmed();
assert!(
clock.open().is_none(),
"a disarmed clock has no instant to hand back, so it read none"
);
assert_eq!(
CALLER_DONE.get(),
Some(sentinel),
"`open` cleared the stamp, so it did more than return"
);
clock.close(GemvSite::Experts, Some(sentinel));
clock.close(GemvSite::Router, Some(sentinel));
assert_eq!(
CALLER_DONE.get(),
Some(sentinel),
"a disarmed close consumed the stamp"
);
let mut split = GemvSplit::default();
let mut clock = GemvClock::new(&mut split, true);
let opened = clock.open();
assert!(opened.is_some());
assert_eq!(CALLER_DONE.get(), None, "`open` arms a fresh stamp");
CALLER_DONE.set(opened);
clock.close(GemvSite::Router, opened);
assert_eq!(split.router.scatters, 1);
assert_eq!(
split.router.own,
Duration::ZERO,
"a stamp at the open instant charges the whole call to the barrier"
);
CALLER_DONE.set(None);
}
}