use std::time::{Duration, Instant};
use thiserror::Error;
use crate::io::StreamStats;
use crate::model::{
ForwardError, ForwardState, Model, PrefillProgressSink, PrefillRouteSink, StreamPhase,
forward_token, forward_token_traced, prefill_prompt_with_progress,
};
use crate::tokenizer::{RvmpTokenizer, SamplingDefaults, TokenizerError};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TracePhase {
Prefill,
Decode,
}
impl From<TracePhase> for StreamPhase {
fn from(phase: TracePhase) -> Self {
match phase {
TracePhase::Prefill => Self::Prefill,
TracePhase::Decode => Self::Decode,
}
}
}
pub type RouteSink<'a> = &'a mut dyn FnMut(TracePhase, usize, u32, &[(u32, f32)]);
#[derive(Clone, Copy, Debug)]
pub enum GenerateProgress {
PrefillChunk {
positions_done: usize,
positions_total: usize,
},
DecodeToken {
index: usize,
stats: StreamStats,
},
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct GenerateParams {
pub max_new: usize,
pub temperature: f32,
pub top_k: Option<u32>,
pub top_p: f32,
pub seed: u64,
pub greedy: bool,
}
impl GenerateParams {
pub fn from_defaults(defaults: SamplingDefaults) -> Self {
Self {
max_new: 128,
temperature: defaults.temperature,
top_k: defaults.top_k,
top_p: defaults.top_p,
seed: 42,
greedy: false,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StopReason {
StopToken(u32),
MaxNew,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GenerateStats {
pub prompt_tokens: usize,
pub generated: usize,
pub generated_ids: Vec<u32>,
pub stop: StopReason,
pub prefill: Duration,
pub decode: Duration,
}
#[derive(Debug, Error)]
pub enum GenerateError {
#[error(transparent)]
Forward(#[from] ForwardError),
#[error(transparent)]
Tokenizer(#[from] TokenizerError),
#[error("generate: empty prompt")]
EmptyPrompt,
#[error("generate: start_position {position}, but the state holds {expected} positions")]
PositionMismatch {
position: usize,
expected: usize,
},
#[error("generate: forward pass returned no logits")]
MissingLogits,
}
#[derive(Debug)]
struct Xorshift(u64);
impl Xorshift {
fn new(seed: u64) -> Self {
Self(if seed == 0 {
0x9E37_79B9_7F4A_7C15
} else {
seed
})
}
fn next_u64(&mut self) -> u64 {
let mut x = self.0;
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
self.0 = x;
x.wrapping_mul(0x2545_F491_4F6C_DD1D)
}
fn next_f64(&mut self) -> f64 {
(self.next_u64() >> 11) as f64 / (1u64 << 53) as f64
}
}
#[derive(Debug)]
struct Sampler {
params: GenerateParams,
rng: Xorshift,
candidates: Vec<(u32, f32)>,
}
impl Sampler {
fn new(params: &GenerateParams, vocab: usize) -> Self {
Self {
params: *params,
rng: Xorshift::new(params.seed),
candidates: Vec::with_capacity(vocab),
}
}
fn argmax(logits: &[f32]) -> u32 {
let mut best = 0usize;
let mut best_v = f32::NEG_INFINITY;
for (i, &v) in logits.iter().enumerate() {
if v > best_v {
best_v = v;
best = i;
}
}
best as u32
}
fn sample(&mut self, logits: &[f32]) -> u32 {
if self.params.greedy || self.params.temperature <= 0.0 || logits.len() < 2 {
return Self::argmax(logits);
}
let cands = &mut self.candidates;
cands.clear();
cands.extend(logits.iter().enumerate().map(|(i, &l)| (i as u32, l)));
if let Some(k) = self.params.top_k {
let k = k as usize;
if k > 0 && k < cands.len() {
cands.select_nth_unstable_by(k - 1, |a, b| b.1.total_cmp(&a.1));
cands.truncate(k);
}
}
cands.sort_unstable_by(|a, b| b.1.total_cmp(&a.1));
if self.params.top_p < 1.0 {
let max = f64::from(cands[0].1);
let total: f64 = cands.iter().map(|c| (f64::from(c.1) - max).exp()).sum();
let mut cum = 0.0f64;
let mut keep = cands.len();
for (i, c) in cands.iter().enumerate() {
cum += (f64::from(c.1) - max).exp() / total;
if cum >= f64::from(self.params.top_p) {
keep = i + 1;
break;
}
}
cands.truncate(keep.max(1));
}
let inv_t = 1.0 / f64::from(self.params.temperature);
let max = f64::from(cands[0].1) * inv_t;
let total: f64 = cands
.iter()
.map(|c| (f64::from(c.1) * inv_t - max).exp())
.sum();
let mut r = self.rng.next_f64() * total;
for c in cands.iter() {
r -= (f64::from(c.1) * inv_t - max).exp();
if r < 0.0 {
return c.0;
}
}
cands.last().map_or(0, |c| c.0)
}
}
const STREAM_WINDOW_KEEP: usize = 8;
const STREAM_WINDOW_MAX: usize = 64;
#[derive(Debug, Default)]
struct StreamDecoder {
window: Vec<u32>,
emitted: usize,
last: Option<u32>,
#[cfg(test)]
decoded_ids: usize,
}
impl StreamDecoder {
fn new() -> Self {
Self::default()
}
fn decode_range(
&mut self,
tokenizer: &RvmpTokenizer,
start: usize,
end: usize,
) -> Result<String, TokenizerError> {
#[cfg(test)]
{
self.decoded_ids += end - start;
}
tokenizer.decode(&self.window[start..end], false)
}
fn push(&mut self, tokenizer: &RvmpTokenizer, id: u32) -> Result<String, TokenizerError> {
self.window.push(id);
self.last = Some(id);
let len = self.window.len();
let text = self.decode_range(tokenizer, 0, len)?;
if len >= STREAM_WINDOW_MAX {
return self.force_cut(tokenizer, &text);
}
let mut safe = text.len();
while text[..safe].ends_with('\u{FFFD}') {
safe -= '\u{FFFD}'.len_utf8();
}
let mut out = String::new();
if safe > self.emitted {
out = text.get(self.emitted..safe).unwrap_or("").to_owned();
self.emitted = safe;
}
self.trim(tokenizer, &text)?;
Ok(out)
}
fn force_cut(
&mut self,
tokenizer: &RvmpTokenizer,
text: &str,
) -> Result<String, TokenizerError> {
let mut safe = text.len();
if text.ends_with('\u{FFFD}') {
safe -= '\u{FFFD}'.len_utf8();
}
let withheld = text.len() - safe;
let out = text.get(self.emitted..safe).unwrap_or("").to_owned();
let len = self.window.len();
let cut = len.saturating_sub(STREAM_WINDOW_KEEP);
let tail = self.decode_range(tokenizer, cut, len)?;
self.window.drain(..cut);
self.emitted = tail.len().saturating_sub(withheld);
Ok(out)
}
fn trim(&mut self, tokenizer: &RvmpTokenizer, text: &str) -> Result<(), TokenizerError> {
let len = self.window.len();
if len <= STREAM_WINDOW_KEEP {
return Ok(());
}
let mut cut = len - STREAM_WINDOW_KEEP;
while cut > 0 {
let head = self.decode_range(tokenizer, 0, cut)?;
if !head.is_empty()
&& !head.ends_with('\u{FFFD}')
&& head.len() <= self.emitted
&& text.starts_with(&head)
{
self.window.drain(..cut);
self.emitted -= head.len();
return Ok(());
}
cut -= 1;
}
Ok(())
}
fn flush(&mut self, tokenizer: &RvmpTokenizer) -> Result<String, TokenizerError> {
if self.window.is_empty() {
return Ok(String::new());
}
let len = self.window.len();
let text = self.decode_range(tokenizer, 0, len)?;
let out = text.get(self.emitted..).unwrap_or("").to_owned();
self.emitted = text.len();
Ok(out)
}
fn last_id(&self) -> Option<u32> {
self.last
}
}
pub fn generate(
model: &Model,
state: &mut ForwardState,
tokenizer: &RvmpTokenizer,
prompt_ids: &[u32],
params: &GenerateParams,
mut on_token: impl FnMut(u32, &str),
) -> Result<GenerateStats, GenerateError> {
generate_with_stops(
model,
state,
tokenizer,
prompt_ids,
0,
params,
tokenizer.stop_tokens(),
&mut on_token,
None,
None,
)
}
pub fn generate_from(
model: &Model,
state: &mut ForwardState,
tokenizer: &RvmpTokenizer,
prompt_ids: &[u32],
start_position: usize,
params: &GenerateParams,
on_token: impl FnMut(u32, &str),
) -> Result<GenerateStats, GenerateError> {
generate_from_with_progress(
model,
state,
tokenizer,
prompt_ids,
start_position,
params,
None,
on_token,
)
}
#[allow(clippy::too_many_arguments)]
pub fn generate_from_with_progress(
model: &Model,
state: &mut ForwardState,
tokenizer: &RvmpTokenizer,
prompt_ids: &[u32],
start_position: usize,
params: &GenerateParams,
on_progress: Option<&mut dyn FnMut(GenerateProgress)>,
mut on_token: impl FnMut(u32, &str),
) -> Result<GenerateStats, GenerateError> {
generate_with_stops(
model,
state,
tokenizer,
prompt_ids,
start_position,
params,
tokenizer.stop_tokens(),
&mut on_token,
None,
on_progress,
)
}
pub fn generate_traced(
model: &Model,
state: &mut ForwardState,
tokenizer: &RvmpTokenizer,
prompt_ids: &[u32],
params: &GenerateParams,
mut on_token: impl FnMut(u32, &str),
trace: RouteSink<'_>,
) -> Result<GenerateStats, GenerateError> {
generate_with_stops(
model,
state,
tokenizer,
prompt_ids,
0,
params,
tokenizer.stop_tokens(),
&mut on_token,
Some(trace),
None,
)
}
fn traced_step<'s>(
model: &Model,
state: &'s mut ForwardState,
token_id: u32,
position: usize,
want_logits: bool,
phase: TracePhase,
trace: &mut Option<RouteSink<'_>>,
) -> Result<Option<&'s [f32]>, ForwardError> {
state.set_stream_phase(StreamPhase::from(phase));
match trace.as_deref_mut() {
Some(sink) => {
let mut on_route = |layer: u32, topk: &[(u32, f32)]| sink(phase, position, layer, topk);
forward_token_traced(
model,
state,
token_id,
position,
want_logits,
Some(&mut on_route),
)
}
None => forward_token(model, state, token_id, position, want_logits),
}
}
#[allow(clippy::too_many_arguments)]
fn generate_with_stops(
model: &Model,
state: &mut ForwardState,
tokenizer: &RvmpTokenizer,
prompt_ids: &[u32],
start_position: usize,
params: &GenerateParams,
stop_tokens: &[u32],
on_token: &mut dyn FnMut(u32, &str),
mut trace: Option<RouteSink<'_>>,
mut on_progress: Option<&mut dyn FnMut(GenerateProgress)>,
) -> Result<GenerateStats, GenerateError> {
if prompt_ids.is_empty() {
return Err(GenerateError::EmptyPrompt);
}
let expected = state
.seq_len()
.map_err(|_| GenerateError::PositionMismatch {
position: start_position,
expected: usize::MAX,
})?;
if start_position != expected {
return Err(GenerateError::PositionMismatch {
position: start_position,
expected,
});
}
let prefill_start = Instant::now();
state.set_stream_phase(StreamPhase::Prefill);
let mut logits = {
let mut tagged;
let sink: Option<PrefillRouteSink<'_>> = match trace.as_deref_mut() {
Some(inner) => {
tagged = |position: usize, layer: u32, topk: &[(u32, f32)]| {
inner(TracePhase::Prefill, position, layer, topk);
};
Some(&mut tagged)
}
None => None,
};
let mut counted;
let progress: Option<PrefillProgressSink<'_>> = match on_progress.as_deref_mut() {
Some(inner) => {
counted = |positions_done: usize, positions_total: usize| {
inner(GenerateProgress::PrefillChunk {
positions_done,
positions_total,
});
};
Some(&mut counted)
}
None => None,
};
prefill_prompt_with_progress(model, state, prompt_ids, sink, progress)?
};
let prefill = prefill_start.elapsed();
let decode_start = Instant::now();
let mut sampler = Sampler::new(params, logits.len());
let mut stream = StreamDecoder::new();
let mut position = start_position + prompt_ids.len();
let mut generated_ids: Vec<u32> = Vec::with_capacity(params.max_new);
let mut stop = StopReason::MaxNew;
while generated_ids.len() < params.max_new {
let next = sampler.sample(logits);
if stop_tokens.contains(&next) {
stop = StopReason::StopToken(next);
break;
}
let text = stream.push(tokenizer, next)?;
if let Some(progress) = on_progress.as_deref_mut() {
progress(GenerateProgress::DecodeToken {
index: generated_ids.len(),
stats: state.stream_stats_in(StreamPhase::Decode),
});
}
on_token(next, &text);
generated_ids.push(next);
if generated_ids.len() == params.max_new {
break;
}
logits = traced_step(
model,
state,
next,
position,
true,
TracePhase::Decode,
&mut trace,
)?
.ok_or(GenerateError::MissingLogits)?;
position += 1;
}
let tail = stream.flush(tokenizer)?;
if !tail.is_empty()
&& let Some(id) = stream.last_id()
{
on_token(id, &tail);
}
Ok(GenerateStats {
prompt_tokens: prompt_ids.len(),
generated: generated_ids.len(),
generated_ids,
stop,
prefill,
decode: decode_start.elapsed(),
})
}
#[cfg(test)]
mod tests {
use std::path::{Path, PathBuf};
use super::*;
use crate::io::LoadOptions;
use crate::io::testutil::build_install;
use crate::model::{ForwardState, PrefillConfig, PrefillMode, RuntimeConfig, prefill_prompt};
fn small(model: &Model, context_cap: usize) -> ForwardState {
ForwardState::with_config(model, context_cap, RuntimeConfig::testing()).unwrap()
}
fn one_chunk_sweep() -> PrefillConfig {
PrefillConfig {
mode: PrefillMode::Sweep,
chunk: 512,
experts_per_window: 1,
windows_in_flight: 1,
}
}
fn fixtures_dir() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("src/tokenizer/fixtures")
}
fn fixture_tokenizer() -> RvmpTokenizer {
RvmpTokenizer::load(&fixtures_dir()).expect("fixture tokenizer loads")
}
fn greedy_params(max_new: usize) -> GenerateParams {
GenerateParams {
max_new,
temperature: 0.7,
top_k: Some(20),
top_p: 0.8,
seed: 42,
greedy: true,
}
}
fn sampled_params(temperature: f32, top_k: Option<u32>, top_p: f32, seed: u64) -> Sampler {
Sampler::new(
&GenerateParams {
max_new: 1,
temperature,
top_k,
top_p,
seed,
greedy: false,
},
8,
)
}
#[test]
fn greedy_is_argmax_and_first_index_wins_ties() {
let logits = [0.5f32, 3.0, -1.0, 3.0];
let mut sampler = Sampler::new(
&GenerateParams {
greedy: true,
..greedy_params(1)
},
logits.len(),
);
for _ in 0..4 {
assert_eq!(sampler.sample(&logits), 1);
}
let mut cold = sampled_params(0.0, None, 1.0, 7);
assert_eq!(cold.sample(&logits), 1);
}
#[test]
fn top_k_one_is_deterministic_argmax() {
let logits = [0.1f32, 0.2, 5.0, 0.3, 4.9];
let mut sampler = sampled_params(1.0, Some(1), 1.0, 123);
for _ in 0..16 {
assert_eq!(sampler.sample(&logits), 2);
}
}
#[test]
fn top_k_filters_to_the_k_largest() {
let logits = [0.0f32, 0.0, 5.0, 0.0, 4.9];
let mut sampler = sampled_params(1.0, Some(2), 1.0, 99);
let mut seen = [0usize; 5];
for _ in 0..200 {
seen[sampler.sample(&logits) as usize] += 1;
}
assert_eq!(seen[0] + seen[1] + seen[3], 0, "{seen:?}");
assert!(seen[2] > 0 && seen[4] > 0, "{seen:?}");
}
#[test]
fn top_p_keeps_the_smallest_covering_prefix() {
let logits = [6.0f32.ln(), 3.0f32.ln(), 1.0f32.ln()];
let mut sampler = sampled_params(1.0, None, 0.7, 0xF00);
let mut seen = [0usize; 3];
for _ in 0..300 {
seen[sampler.sample(&logits) as usize] += 1;
}
assert_eq!(seen[2], 0, "{seen:?}");
assert!(seen[0] > 0 && seen[1] > 0, "{seen:?}");
}
#[test]
fn seeded_sampling_is_deterministic() {
let logits: Vec<f32> = (0..64).map(|i| ((i * 37) % 11) as f32 * 0.3).collect();
let draw = |seed: u64| -> Vec<u32> {
let mut sampler = sampled_params(0.9, Some(8), 0.95, seed);
(0..32).map(|_| sampler.sample(&logits)).collect()
};
assert_eq!(draw(1234), draw(1234));
assert_ne!(draw(1234), draw(4321), "different seeds should diverge");
}
use crate::tokenizer::{ENDOFTEXT_TOKEN_ID, IM_END_TOKEN_ID, IM_START_TOKEN_ID};
fn mixed_ids(tokenizer: &RvmpTokenizer, reps: usize) -> Vec<u32> {
const UNIT: &str = "hi \u{4F60}\u{597D}\u{4E16}\u{754C} \u{1F30D}\u{1F469}\u{200D}\u{1F4BB}\
<|im_end|>\n\u{13000}\u{1D518} ok ";
tokenizer.encode(&UNIT.repeat(reps)).unwrap()
}
fn drive_stream(tokenizer: &RvmpTokenizer, ids: &[u32]) -> (String, Vec<usize>) {
let mut stream = StreamDecoder::new();
let mut assembled = String::new();
let mut per_push = Vec::with_capacity(ids.len());
let mut counted = 0usize;
for (i, &id) in ids.iter().enumerate() {
let piece = stream.push(tokenizer, id).unwrap();
assert!(
!piece.contains('\u{FFFD}'),
"push {i} leaked a replacement char: {piece:?}"
);
assert!(
stream.window.len() <= STREAM_WINDOW_MAX,
"window grew past its cap at push {i}"
);
assembled.push_str(&piece);
per_push.push(stream.decoded_ids - counted);
counted = stream.decoded_ids;
}
assembled.push_str(&stream.flush(tokenizer).unwrap());
(assembled, per_push)
}
#[test]
fn stream_decoder_matches_the_full_decode() {
let tokenizer = fixture_tokenizer();
let enc = |s: &str| tokenizer.encode(s).unwrap();
let mut raw_specials = vec![IM_START_TOKEN_ID];
raw_specials.extend(enc("user\n"));
raw_specials.extend([IM_END_TOKEN_ID, ENDOFTEXT_TOKEN_ID, 151_668]);
let cases: Vec<(&str, Vec<u32>)> = vec![
("empty", Vec::new()),
("single ascii id", enc("hello")[..1].to_vec()),
("single added-token id", vec![IM_END_TOKEN_ID]),
("ascii", enc("hello world, 1 + 2 = 3. done!")),
(
"cjk",
enc("\u{4F60}\u{597D}\u{4E16}\u{754C}\u{3002}\u{6D4B}\u{8BD5}\u{4E00}\u{4E0B}"),
),
(
"emoji",
enc("ok \u{1F30D}\u{1F680}\u{1F469}\u{200D}\u{1F4BB} fine"),
),
("astral planes", enc("\u{13000}\u{1D518}\u{10348}")),
(
"chatml round trip",
enc("<|im_start|>user\nhi \u{1F600}<|im_end|>\n<|im_start|>assistant\n"),
),
("raw added-token ids", raw_specials),
("long mixed, many slides", mixed_ids(&tokenizer, 24)),
];
for (name, ids) in cases {
let (assembled, _) = drive_stream(&tokenizer, &ids);
assert_eq!(
assembled,
tokenizer.decode(&ids, false).unwrap(),
"case {name}"
);
}
}
#[test]
fn stream_decoder_carries_a_character_across_a_window_slide() {
let tokenizer = fixture_tokenizer();
let split = ["\u{13000}", "\u{1D518}", "\u{10348}", "\u{1F30D}"]
.iter()
.map(|s| tokenizer.encode(s).unwrap())
.find(|ids| ids.len() >= 2)
.expect("some exotic character splits into multiple ids");
assert!(split.len() <= STREAM_WINDOW_KEEP);
let prefix = tokenizer
.encode(&"the quick brown fox jumps over the lazy dog ".repeat(3))
.unwrap();
assert!(
prefix.len() > STREAM_WINDOW_KEEP * 2,
"{} ids",
prefix.len()
);
let mut stream = StreamDecoder::new();
let mut assembled = String::new();
for &id in &prefix {
assembled.push_str(&stream.push(&tokenizer, id).unwrap());
}
assert_eq!(stream.window.len(), STREAM_WINDOW_KEEP);
assert_eq!(assembled, tokenizer.decode(&prefix, false).unwrap());
for (i, &id) in split.iter().enumerate() {
let piece = stream.push(&tokenizer, id).unwrap();
assert!(!piece.contains('\u{FFFD}'), "push {i}: {piece:?}");
assert!(stream.window.ends_with(&split[..=i]), "push {i}");
assembled.push_str(&piece);
}
assembled.push_str(&stream.flush(&tokenizer).unwrap());
let all: Vec<u32> = prefix.iter().chain(split.iter()).copied().collect();
assert_eq!(assembled, tokenizer.decode(&all, false).unwrap());
}
fn undecodable_filler(tokenizer: &RvmpTokenizer) -> u32 {
const FFFD: &str = "\u{FFFD}";
(0u32..256)
.find(|&id| {
let one = tokenizer.decode(&[id], false).unwrap_or_default();
let two = tokenizer.decode(&[id, id], false).unwrap_or_default();
one == FFFD && two == FFFD.repeat(2)
})
.expect("the byte alphabet has a stray continuation byte")
}
fn drive_past_the_cap(tokenizer: &RvmpTokenizer, ids: &[u32]) -> (String, Vec<usize>) {
let mut stream = StreamDecoder::new();
let mut assembled = String::new();
let mut lens = Vec::with_capacity(ids.len());
for (i, &id) in ids.iter().enumerate() {
assembled.push_str(&stream.push(tokenizer, id).unwrap());
assert!(
stream.window.len() <= STREAM_WINDOW_MAX,
"window past the cap at push {i}: {}",
stream.window.len()
);
lens.push(stream.window.len());
}
assembled.push_str(&stream.flush(tokenizer).unwrap());
(assembled, lens)
}
#[test]
fn stream_decoder_cap_matches_the_full_decode() {
let tokenizer = fixture_tokenizer();
let filler = undecodable_filler(&tokenizer);
let split = ["\u{13000}", "\u{1D518}", "\u{10348}", "\u{1F30D}"]
.iter()
.map(|s| tokenizer.encode(s).unwrap())
.find(|ids| ids.len() >= 2)
.expect("some exotic character splits into multiple ids");
let mut half_char = vec![filler; STREAM_WINDOW_MAX - 1];
half_char.extend(split.iter().copied());
const ROTATION: u32 = 72_496;
let once = tokenizer.decode(&[ROTATION], false).unwrap();
let twice = tokenizer.decode(&[ROTATION, ROTATION], false).unwrap();
assert_ne!(
twice,
once.repeat(2),
"id {ROTATION} no longer straddles a character boundary"
);
let rotated = vec![ROTATION; STREAM_WINDOW_MAX + STREAM_WINDOW_KEEP];
for (name, ids) in [
("split character at the cap", half_char),
("rotated token, no clean cut anywhere", rotated),
] {
let (assembled, lens) = drive_past_the_cap(&tokenizer, &ids);
assert_eq!(
assembled,
tokenizer.decode(&ids, false).unwrap(),
"case {name}"
);
assert_eq!(
lens[STREAM_WINDOW_MAX - 2],
STREAM_WINDOW_MAX - 1,
"case {name}: window did not reach the cap"
);
assert_eq!(
lens[STREAM_WINDOW_MAX - 1],
STREAM_WINDOW_KEEP,
"case {name}: forced cut did not land on the keep bound"
);
}
}
#[test]
fn stream_decoder_flush_owes_text_at_most_once() {
let tokenizer = fixture_tokenizer();
let mut stream = StreamDecoder::new();
for &id in &tokenizer.encode("all done.").unwrap() {
stream.push(&tokenizer, id).unwrap();
}
assert_eq!(stream.flush(&tokenizer).unwrap(), "");
assert_eq!(stream.flush(&tokenizer).unwrap(), "");
let split = tokenizer.encode("\u{13000}").unwrap();
assert!(split.len() >= 2);
let head = &split[..split.len() - 1];
let mut stream = StreamDecoder::new();
for &id in head {
assert_eq!(stream.push(&tokenizer, id).unwrap(), "");
}
let tail = stream.flush(&tokenizer).unwrap();
assert!(tail.contains('\u{FFFD}'));
assert_eq!(stream.last_id(), head.last().copied());
assert_eq!(stream.flush(&tokenizer).unwrap(), "");
}
#[test]
fn stream_decoder_handles_empty_and_single_id_sequences() {
let tokenizer = fixture_tokenizer();
let mut empty = StreamDecoder::new();
assert_eq!(empty.flush(&tokenizer).unwrap(), "");
assert_eq!(empty.last_id(), None);
for id in [
tokenizer.encode("hi").unwrap()[0],
IM_START_TOKEN_ID,
IM_END_TOKEN_ID,
] {
let mut stream = StreamDecoder::new();
let piece = stream.push(&tokenizer, id).unwrap();
assert_eq!(piece, tokenizer.decode(&[id], false).unwrap(), "id {id}");
assert_eq!(stream.flush(&tokenizer).unwrap(), "");
assert_eq!(stream.last_id(), Some(id));
}
}
#[test]
fn stream_decoder_decode_work_per_push_does_not_grow_with_length() {
let tokenizer = fixture_tokenizer();
let short = mixed_ids(&tokenizer, 3);
let long = mixed_ids(&tokenizer, 24);
assert!(long.len() > short.len() * 4, "{} ids", long.len());
let (short_text, short_push) = drive_stream(&tokenizer, &short);
let (long_text, long_push) = drive_stream(&tokenizer, &long);
assert_eq!(short_text, tokenizer.decode(&short, false).unwrap());
assert_eq!(long_text, tokenizer.decode(&long, false).unwrap());
let short_worst = short_push.iter().copied().max().unwrap();
let long_worst = long_push.iter().copied().max().unwrap();
assert!(
long_worst <= short_worst && long_worst <= STREAM_WINDOW_KEEP * 3,
"per-push decode grew with length: {short_worst} -> {long_worst}"
);
let n = long.len();
let total: usize = long_push.iter().sum();
assert!(total <= n * STREAM_WINDOW_KEEP * 3, "total {total}, n {n}");
assert!(total * 4 < n * (n + 1) / 2, "total {total}, n {n}");
}
#[test]
fn stream_decoder_withholds_split_utf8() {
let tokenizer = fixture_tokenizer();
let ids = ["\u{13000}", "\u{1D518}", "\u{1F30D}", "\u{10348}"]
.iter()
.map(|s| tokenizer.encode(s).unwrap())
.find(|ids| ids.len() >= 2)
.expect("some exotic character splits into multiple tokens");
let full = tokenizer.decode(&ids, false).unwrap();
let mut stream = StreamDecoder::new();
let mut assembled = String::new();
for (i, &id) in ids.iter().enumerate() {
let piece = stream.push(&tokenizer, id).unwrap();
assert!(
!piece.contains('\u{FFFD}'),
"piece {i} leaked a replacement char: {piece:?}"
);
assembled.push_str(&piece);
}
assembled.push_str(&stream.flush(&tokenizer).unwrap());
assert_eq!(assembled, full);
assert_eq!(stream.flush(&tokenizer).unwrap(), "");
}
#[test]
fn stream_decoder_flushes_incomplete_tail_at_stop() {
let tokenizer = fixture_tokenizer();
let ids = tokenizer.encode("\u{13000}").unwrap();
assert!(ids.len() >= 2);
let mut stream = StreamDecoder::new();
let text = stream.push(&tokenizer, ids[0]).unwrap();
assert_eq!(text, "");
let tail = stream.flush(&tokenizer).unwrap();
assert!(tail.contains('\u{FFFD}'));
}
#[test]
fn stream_decoder_ascii_is_immediate() {
let tokenizer = fixture_tokenizer();
let ids = tokenizer.encode("hello world").unwrap();
let mut stream = StreamDecoder::new();
let mut assembled = String::new();
for &id in &ids {
assembled.push_str(&stream.push(&tokenizer, id).unwrap());
}
assert_eq!(assembled, "hello world");
assert_eq!(stream.flush(&tokenizer).unwrap(), "");
}
struct Harness {
_fx: crate::io::testutil::Fixture,
model: Model,
tokenizer: RvmpTokenizer,
}
fn harness(tag: &str) -> Harness {
let fx = build_install(tag);
crate::model::testsupport::temper_install(&fx);
let model = Model::load(
&fx.root,
LoadOptions {
skip_hashes: true,
..LoadOptions::default()
},
)
.unwrap();
Harness {
_fx: fx,
model,
tokenizer: fixture_tokenizer(),
}
}
fn run_greedy(
h: &Harness,
prompt: &[u32],
max_new: usize,
stops: &[u32],
) -> (Vec<u32>, String, GenerateStats) {
let mut state = small(&h.model, 16);
let mut events: Vec<(u32, String)> = Vec::new();
let mut on_token = |id: u32, piece: &str| {
events.push((id, piece.to_owned()));
};
let stats = generate_with_stops(
&h.model,
&mut state,
&h.tokenizer,
prompt,
0,
&greedy_params(max_new),
stops,
&mut on_token,
None,
None,
)
.unwrap();
assert!(events.len() == stats.generated || events.len() == stats.generated + 1);
assert_eq!(stats.generated_ids.len(), stats.generated);
let ids: Vec<u32> = events.iter().take(stats.generated).map(|e| e.0).collect();
assert_eq!(
ids, stats.generated_ids,
"the stream and the stats disagree"
);
let text: String = events.iter().map(|e| e.1.as_str()).collect();
(ids, text, stats)
}
#[test]
fn greedy_generation_is_deterministic_and_counts_match() {
let h = harness("gen-greedy");
let (ids_a, text_a, stats) = run_greedy(&h, &[1, 2, 3], 4, &[]);
assert_eq!(stats.prompt_tokens, 3);
assert_eq!(stats.generated, 4);
assert_eq!(stats.stop, StopReason::MaxNew);
assert_eq!(ids_a.len(), 4);
assert!(ids_a.iter().all(|&id| (id as usize) < 32));
assert_eq!(text_a, h.tokenizer.decode(&ids_a, false).unwrap());
let (ids_b, text_b, _) = run_greedy(&h, &[1, 2, 3], 4, &[]);
assert_eq!(ids_a, ids_b);
assert_eq!(text_a, text_b);
}
#[test]
fn stop_token_halts_generation_and_is_not_streamed() {
let h = harness("gen-stop");
let (ids, _, _) = run_greedy(&h, &[1, 2, 3], 4, &[]);
assert!(ids.len() >= 2);
let j = (1..ids.len())
.find(|&j| !ids[..j].contains(&ids[j]))
.unwrap_or(0);
let stop = ids[j];
let (stopped_ids, text, stats) = run_greedy(&h, &[1, 2, 3], 4, &[stop]);
assert_eq!(stopped_ids, ids[..j].to_vec());
assert_eq!(stats.generated, j);
assert_eq!(stats.stop, StopReason::StopToken(stop));
assert_eq!(text, h.tokenizer.decode(&ids[..j], false).unwrap());
}
#[test]
fn max_new_zero_generates_nothing() {
let h = harness("gen-zero");
let (ids, text, stats) = run_greedy(&h, &[1], 0, &[]);
assert!(ids.is_empty());
assert!(text.is_empty());
assert_eq!(stats.generated, 0);
assert_eq!(stats.stop, StopReason::MaxNew);
assert_eq!(stats.prompt_tokens, 1);
}
#[test]
fn tracing_records_every_layer_and_changes_nothing() {
let h = harness("gen-trace");
let prompt = [1u32, 2, 3];
let (want_ids, want_text, want_stats) = run_greedy(&h, &prompt, 4, &[]);
let arch = h.model.arch();
let (n_layers, top_k) = (arch.n_layers, arch.top_k as usize);
let mut records: Vec<(TracePhase, usize, u32, Vec<u32>)> = Vec::new();
let mut sink = |phase: TracePhase, pos: usize, layer: u32, topk: &[(u32, f32)]| {
records.push((phase, pos, layer, topk.iter().map(|&(e, _)| e).collect()));
};
let mut state = small(&h.model, 16);
state.set_prefill_config(one_chunk_sweep()).unwrap();
let mut ids = Vec::new();
let mut text = String::new();
let stats = generate_traced(
&h.model,
&mut state,
&h.tokenizer,
&prompt,
&greedy_params(4),
|id, piece| {
ids.push(id);
text.push_str(piece);
},
&mut sink,
)
.unwrap();
assert_eq!(stats.stop, StopReason::MaxNew);
assert_eq!(stats.generated, want_stats.generated);
assert_eq!(ids[..want_ids.len()], want_ids[..]);
assert_eq!(text, want_text);
let tokens = prompt.len() + stats.generated - 1;
assert_eq!(records.len(), tokens * n_layers as usize);
for (phase, pos, layer, experts) in &records {
let want_phase = if *pos < prompt.len() {
TracePhase::Prefill
} else {
TracePhase::Decode
};
assert_eq!(*phase, want_phase, "position {pos}");
assert!(*layer < n_layers);
assert_eq!(experts.len(), top_k);
assert!(experts.iter().all(|&e| e < arch.n_experts), "{experts:?}");
let mut unique = experts.clone();
unique.sort_unstable();
unique.dedup();
assert_eq!(unique.len(), top_k, "{experts:?}");
}
let mut pairs: Vec<(usize, u32)> = records.iter().map(|r| (r.1, r.2)).collect();
pairs.sort_unstable();
let want: Vec<(usize, u32)> = (0..tokens)
.flat_map(|pos| (0..n_layers).map(move |layer| (pos, layer)))
.collect();
assert_eq!(pairs, want);
let decode: Vec<_> = records
.iter()
.filter(|r| r.0 == TracePhase::Decode)
.collect();
for (i, record) in decode.iter().enumerate() {
assert_eq!(
record.2 as usize,
i % n_layers as usize,
"decode record {i}"
);
assert_eq!(record.1, prompt.len() + i / n_layers as usize);
}
let prefill: Vec<_> = records
.iter()
.filter(|r| r.0 == TracePhase::Prefill)
.collect();
for (i, record) in prefill.iter().enumerate() {
assert_eq!(
record.2 as usize,
i / prompt.len(),
"prefill record {i} is not layer-major"
);
assert_eq!(record.1, i % prompt.len());
}
}
#[test]
fn streaming_counters_split_prefill_from_decode() {
let h = harness("gen-phase-split");
let prompt = [1u32, 2, 3];
let max_new = 4;
let arch = h.model.arch();
let per_pass = u64::from(arch.n_layers) * u64::from(arch.top_k);
let run = |config: PrefillConfig| {
let mut state = small(&h.model, 16);
state.set_prefill_config(config).unwrap();
let stats = generate_with_stops(
&h.model,
&mut state,
&h.tokenizer,
&prompt,
0,
&greedy_params(max_new),
&[],
&mut |_, _| {},
None,
None,
)
.unwrap();
assert_eq!(stats.generated, max_new);
state
};
let state = run(PrefillConfig {
mode: PrefillMode::TokenMajor,
..one_chunk_sweep()
});
let prefill = state.stream_stats_in(StreamPhase::Prefill);
let decode = state.stream_stats_in(StreamPhase::Decode);
let total = state.stream_stats();
assert_eq!(prefill.accesses(), prompt.len() as u64 * per_pass);
assert_eq!(decode.accesses(), (max_new as u64 - 1) * per_pass);
assert_eq!(prefill.accesses() + decode.accesses(), total.accesses());
assert_eq!(prefill.misses + decode.misses, total.misses);
assert_eq!(prefill.hits + decode.hits, total.hits);
assert_eq!(prefill.bytes_read + decode.bytes_read, total.bytes_read);
assert_ne!(total.accesses(), decode.accesses());
let state = run(one_chunk_sweep());
let prefill = state.stream_stats_in(StreamPhase::Prefill);
let decode = state.stream_stats_in(StreamPhase::Decode);
let total = state.stream_stats();
assert_eq!(prefill.accesses(), 0, "the sweep bypasses the slot cache");
assert_eq!(prefill.hits, 0);
assert_eq!(prefill.misses, 0);
assert_eq!(prefill.bytes_read, 0);
assert!(prefill.sweep_windows_read > 0, "no window was ever read");
assert!(prefill.sweep_bytes_read > 0, "no expert bytes were swept");
assert_eq!(decode.accesses(), (max_new as u64 - 1) * per_pass);
assert_eq!(total.accesses(), decode.accesses());
assert_eq!(decode.sweep_windows_read, 0, "decode never sweeps");
}
#[test]
fn a_second_turn_continues_an_existing_cache() {
let h = harness("gen-continue");
let params = greedy_params(3);
let mut history: Vec<u32> = vec![1, 2, 3];
let mut state = small(&h.model, 24);
state.set_prefill_config(one_chunk_sweep()).unwrap();
let turn1 = generate_with_stops(
&h.model,
&mut state,
&h.tokenizer,
&history.clone(),
0,
¶ms,
&[],
&mut |_, _| {},
None,
None,
)
.unwrap();
assert_eq!(turn1.generated_ids.len(), 3);
history.extend(&turn1.generated_ids);
let fed = state.seq_len().unwrap();
assert_eq!(fed, history.len() - 1);
history.extend([7u32, 8]);
let segment = history[fed..].to_vec();
let turn2 = generate_with_stops(
&h.model,
&mut state,
&h.tokenizer,
&segment,
fed,
¶ms,
&[],
&mut |_, _| {},
None,
None,
)
.unwrap();
assert_eq!(turn2.prompt_tokens, segment.len());
let mut fresh = small(&h.model, 24);
fresh.set_prefill_config(one_chunk_sweep()).unwrap();
let want = generate_with_stops(
&h.model,
&mut fresh,
&h.tokenizer,
&history,
0,
¶ms,
&[],
&mut |_, _| {},
None,
None,
)
.unwrap();
assert_eq!(
turn2.generated_ids, want.generated_ids,
"a continued cache diverged from a rebuilt one"
);
assert_eq!(state.seq_len().unwrap(), fresh.seq_len().unwrap());
}
#[test]
fn a_truncated_cache_is_bit_identical_to_one_that_never_grew() {
let h = harness("gen-truncate");
let prefix = [1u32, 2, 3, 4];
let divergent = [5u32, 6, 7];
let next = 8u32;
for mode in [PrefillMode::Sweep, PrefillMode::TokenMajor] {
let config = PrefillConfig {
mode,
..one_chunk_sweep()
};
let mut whole: Vec<u32> = prefix.to_vec();
whole.extend(divergent);
let mut rewound = small(&h.model, 24);
rewound.set_prefill_config(config).unwrap();
let long_logits = prefill_prompt(&h.model, &mut rewound, &whole, None)
.unwrap()
.to_vec();
assert_eq!(rewound.seq_len().unwrap(), whole.len());
let mut short = small(&h.model, 24);
short.set_prefill_config(config).unwrap();
let short_logits = prefill_prompt(&h.model, &mut short, &prefix, None)
.unwrap()
.to_vec();
assert_ne!(
long_logits, short_logits,
"{mode:?}: the divergent suffix changed nothing, so this test \
would pass with a truncate that did nothing"
);
rewound.truncate(prefix.len()).unwrap();
assert_eq!(rewound.seq_len().unwrap(), prefix.len());
assert_eq!(rewound.seq_len().unwrap(), short.seq_len().unwrap());
let got = forward_token(&h.model, &mut rewound, next, prefix.len(), true)
.unwrap()
.expect("logits were requested");
let got = got.to_vec();
let want = forward_token(&h.model, &mut short, next, prefix.len(), true)
.unwrap()
.expect("logits were requested");
let want = want.to_vec();
assert_eq!(got.len(), want.len());
for (i, (&g, &w)) in got.iter().zip(&want).enumerate() {
assert_eq!(
g.to_bits(),
w.to_bits(),
"{mode:?} logit {i}: off a truncated cache {g:e} vs off one \
that never grew {w:e}"
);
}
let params = greedy_params(3);
let segment = [11u32, 12];
let at = prefix.len() + 1;
let turn = |state: &mut ForwardState| {
generate_with_stops(
&h.model,
state,
&h.tokenizer,
&segment,
at,
¶ms,
&[],
&mut |_, _| {},
None,
None,
)
.unwrap()
};
let continued = turn(&mut rewound);
let reference = turn(&mut short);
assert_eq!(
continued.generated_ids, reference.generated_ids,
"{mode:?}: a rewound cache diverged from one that never grew"
);
assert_eq!(rewound.seq_len().unwrap(), short.seq_len().unwrap());
}
}
#[test]
fn truncate_refuses_what_it_cannot_honour() {
let h = harness("gen-truncate-refuse");
let mut state = small(&h.model, 16);
state.set_prefill_config(one_chunk_sweep()).unwrap();
prefill_prompt(&h.model, &mut state, &[1u32, 2, 3], None).unwrap();
let err = state.truncate(4).unwrap_err();
assert!(
matches!(
err,
ForwardError::Kv(crate::kv::KvError::TruncateBeyondLength {
requested: 4,
len: 3
})
),
"unexpected error: {err}"
);
assert_eq!(state.seq_len().unwrap(), 3, "a refused truncate rewound");
state.truncate(2).unwrap();
assert_eq!(state.seq_len().unwrap(), 2);
forward_token(&h.model, &mut state, 5, 2, true).unwrap();
assert_eq!(state.seq_len().unwrap(), 3);
}
#[test]
fn a_wrong_start_position_is_typed() {
let h = harness("gen-start-position");
let mut state = small(&h.model, 16);
state.set_prefill_config(one_chunk_sweep()).unwrap();
let err = generate_from(
&h.model,
&mut state,
&h.tokenizer,
&[1, 2],
1,
&greedy_params(1),
|_, _| {},
)
.unwrap_err();
assert!(
matches!(
err,
GenerateError::PositionMismatch {
position: 1,
expected: 0
}
),
"unexpected error: {err}"
);
assert_eq!(state.seq_len().unwrap(), 0, "nothing was consumed");
generate_from(
&h.model,
&mut state,
&h.tokenizer,
&[1, 2],
0,
&greedy_params(2),
|_, _| {},
)
.unwrap();
let held = state.seq_len().unwrap();
assert!(held > 0);
let err = generate(
&h.model,
&mut state,
&h.tokenizer,
&[3],
&greedy_params(1),
|_, _| {},
)
.unwrap_err();
assert!(
matches!(
err,
GenerateError::PositionMismatch {
position: 0,
expected,
} if expected == held
),
"unexpected error: {err}"
);
}
#[test]
fn reset_starts_a_fresh_sequence() {
let h = harness("gen-reset");
let prompt = [4u32, 5, 6];
let params = greedy_params(3);
let mut reused = small(&h.model, 16);
reused.set_prefill_config(one_chunk_sweep()).unwrap();
let first = generate_with_stops(
&h.model,
&mut reused,
&h.tokenizer,
&prompt,
0,
¶ms,
&[],
&mut |_, _| {},
None,
None,
)
.unwrap();
assert!(reused.seq_len().unwrap() > 0);
reused.reset();
assert_eq!(reused.seq_len().unwrap(), 0);
for layer in 0..h.model.n_layers() as usize {
assert_eq!(reused.kv_len(layer).unwrap(), 0);
}
let second = generate_with_stops(
&h.model,
&mut reused,
&h.tokenizer,
&prompt,
0,
¶ms,
&[],
&mut |_, _| {},
None,
None,
)
.unwrap();
assert_eq!(second.generated_ids, first.generated_ids);
}
fn run_with_progress(
h: &Harness,
config: PrefillConfig,
prompt: &[u32],
max_new: usize,
) -> (Vec<GenerateProgress>, Vec<(u32, String)>, GenerateStats) {
let mut state = small(&h.model, 24);
state.set_prefill_config(config).unwrap();
let mut progress: Vec<GenerateProgress> = Vec::new();
let mut tokens: Vec<(u32, String)> = Vec::new();
let stats = {
let mut on_progress = |event: GenerateProgress| progress.push(event);
generate_from_with_progress(
&h.model,
&mut state,
&h.tokenizer,
prompt,
0,
&greedy_params(max_new),
Some(&mut on_progress),
|id, piece| tokens.push((id, piece.to_owned())),
)
.unwrap()
};
(progress, tokens, stats)
}
fn prefill_events(events: &[GenerateProgress]) -> Vec<(usize, usize)> {
events
.iter()
.filter_map(|event| match *event {
GenerateProgress::PrefillChunk {
positions_done,
positions_total,
} => Some((positions_done, positions_total)),
GenerateProgress::DecodeToken { .. } => None,
})
.collect()
}
fn decode_events(events: &[GenerateProgress]) -> Vec<usize> {
events
.iter()
.filter_map(|event| match *event {
GenerateProgress::DecodeToken { index, .. } => Some(index),
GenerateProgress::PrefillChunk { .. } => None,
})
.collect()
}
fn run_plain(
h: &Harness,
config: PrefillConfig,
prompt: &[u32],
max_new: usize,
) -> (Vec<(u32, String)>, GenerateStats) {
let mut state = small(&h.model, 24);
state.set_prefill_config(config).unwrap();
let mut tokens: Vec<(u32, String)> = Vec::new();
let stats = generate_from(
&h.model,
&mut state,
&h.tokenizer,
prompt,
0,
&greedy_params(max_new),
|id, piece| tokens.push((id, piece.to_owned())),
)
.unwrap();
(tokens, stats)
}
#[test]
fn prefill_progress_counts_every_chunk_to_the_total() {
let h = harness("gen-progress-prefill");
let prompt = [1u32, 2, 3, 4, 5, 6];
let n = prompt.len();
for (chunk, want) in [
(1usize, vec![1usize, 2, 3, 4, 5, 6]),
(2, vec![2, 4, 6]),
(3, vec![3, 6]),
(512, vec![6]),
] {
let config = PrefillConfig {
chunk,
..one_chunk_sweep()
};
let (events, _, stats) = run_with_progress(&h, config, &prompt, 2);
let prefill = prefill_events(&events);
assert_eq!(
prefill.iter().map(|e| e.0).collect::<Vec<_>>(),
want,
"chunk {chunk}"
);
assert!(
prefill.iter().all(|e| e.1 == n),
"chunk {chunk}: total is not the prompt length: {prefill:?}"
);
assert_eq!(stats.prompt_tokens, n);
}
let (events, _, _) = run_with_progress(
&h,
PrefillConfig {
mode: PrefillMode::TokenMajor,
..one_chunk_sweep()
},
&prompt,
2,
);
assert_eq!(
prefill_events(&events),
(1..=n).map(|done| (done, n)).collect::<Vec<_>>()
);
for config in [
PrefillConfig {
chunk: 1,
..one_chunk_sweep()
},
PrefillConfig {
chunk: 4,
..one_chunk_sweep()
},
one_chunk_sweep(),
PrefillConfig {
mode: PrefillMode::TokenMajor,
..one_chunk_sweep()
},
] {
let (events, _, _) = run_with_progress(&h, config, &prompt, 2);
let prefill = prefill_events(&events);
assert!(!prefill.is_empty(), "{config:?} reported nothing");
let mut previous = 0usize;
for &(done, total) in &prefill {
assert!(done > previous, "{config:?} went backwards: {prefill:?}");
assert!(done <= total, "{config:?} overshot: {prefill:?}");
previous = done;
}
assert_eq!(prefill.last().unwrap(), &(n, n), "{config:?}");
}
}
#[test]
fn decode_progress_fires_once_per_token_and_not_for_the_flush() {
let h = harness("gen-progress-decode");
let (events, tokens, stats) = run_with_progress(&h, one_chunk_sweep(), &[1, 2, 3], 5);
assert_eq!(stats.generated, 5);
assert_eq!(
decode_events(&events),
(0..stats.generated).collect::<Vec<_>>()
);
assert_eq!(
tokens.len(),
stats.generated,
"this fixture's vocabulary is ASCII punctuation; nothing can be withheld"
);
let first_decode = events
.iter()
.position(|e| matches!(e, GenerateProgress::DecodeToken { .. }))
.expect("decode reported");
assert!(
events[..first_decode]
.iter()
.all(|e| matches!(e, GenerateProgress::PrefillChunk { .. })),
"a decode event arrived before prefill finished"
);
let w = byte_harness("gen-progress-flush");
let (events, tokens, stats) = run_with_progress(&w, one_chunk_sweep(), &[3, 15], 3);
assert_eq!(
stats.generated_ids,
vec![134, 134, 134],
"the fixture's greedy output moved; pick a new prompt that ends mid-character"
);
assert_eq!(
tokens.len(),
stats.generated + 1,
"this run was supposed to reach the trailing flush"
);
assert_eq!(
tokens.last().unwrap().0,
*stats.generated_ids.last().unwrap(),
"the flush repeats the last id rather than reporting a new one"
);
assert_eq!(decode_events(&events), vec![0, 1, 2]);
}
#[test]
fn progress_does_not_perturb_generation() {
for tag in ["gen-progress-noop", "gen-progress-noop-wide"] {
let wide = tag.ends_with("-wide");
let h = if wide {
byte_harness(tag)
} else {
harness(tag)
};
let prompt: &[u32] = if wide { &[3, 15] } else { &[1, 2, 3] };
for config in [
one_chunk_sweep(),
PrefillConfig {
chunk: 2,
..one_chunk_sweep()
},
PrefillConfig {
mode: PrefillMode::TokenMajor,
..one_chunk_sweep()
},
] {
let (want_tokens, want) = run_plain(&h, config, prompt, 4);
let (events, tokens, stats) = run_with_progress(&h, config, prompt, 4);
assert_eq!(
tokens, want_tokens,
"{tag} {config:?}: the token stream moved"
);
assert_eq!(stats.generated_ids, want.generated_ids, "{tag} {config:?}");
assert_eq!(stats.generated, want.generated, "{tag} {config:?}");
assert_eq!(stats.prompt_tokens, want.prompt_tokens, "{tag} {config:?}");
assert_eq!(stats.stop, want.stop, "{tag} {config:?}");
assert!(!prefill_events(&events).is_empty(), "{tag} {config:?}");
assert_eq!(
decode_events(&events).len(),
want.generated,
"{tag} {config:?}"
);
}
}
}
fn byte_harness(tag: &str) -> Harness {
let fx = crate::io::testutil::build_install_with(
tag,
&crate::io::testutil::Geometry {
vocab: 256,
..crate::io::testutil::NARROW
},
);
crate::model::testsupport::temper_install(&fx);
let model = Model::load(
&fx.root,
LoadOptions {
skip_hashes: true,
..LoadOptions::default()
},
)
.unwrap();
Harness {
_fx: fx,
model,
tokenizer: fixture_tokenizer(),
}
}
#[test]
fn empty_prompt_is_a_typed_error() {
let h = harness("gen-empty");
let mut state = small(&h.model, 16);
let err = generate(
&h.model,
&mut state,
&h.tokenizer,
&[],
&greedy_params(4),
|_, _| {},
)
.unwrap_err();
assert!(matches!(err, GenerateError::EmptyPrompt));
}
#[test]
fn public_generate_uses_the_pinned_stop_tokens() {
let h = harness("gen-public");
let mut state = small(&h.model, 16);
let mut count = 0usize;
let stats = generate(
&h.model,
&mut state,
&h.tokenizer,
&[4, 5],
&greedy_params(3),
|_, _| count += 1,
)
.unwrap();
assert_eq!(stats.stop, StopReason::MaxNew);
assert_eq!(stats.generated, 3);
assert!(count >= 3);
}
}