use crate::gpu::GpuContext;
use crate::model::{CausalLM, KvCache, ModelError};
use crate::profile;
use crate::tokenizer::BpeTokenizer;
use crate::{IndexTensor, TensorError};
use rand::prelude::*;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, VecDeque};
use std::error::Error;
use std::fmt;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct SamplingConfig {
pub max_new_tokens: usize,
pub temperature: f32,
pub top_k: Option<usize>,
pub top_p: f32,
pub repetition_penalty: f32,
#[serde(default)]
pub repetition_penalty_window: Option<usize>,
#[serde(default)]
pub frequency_penalty: f32,
#[serde(default)]
pub presence_penalty: f32,
pub seed: u64,
pub return_logprobs: bool,
pub top_logprobs: usize,
}
impl Default for SamplingConfig {
fn default() -> Self {
Self {
max_new_tokens: 128,
temperature: 0.8,
top_k: Some(40),
top_p: 0.95,
repetition_penalty: 1.1,
repetition_penalty_window: None,
frequency_penalty: 0.0,
presence_penalty: 0.0,
seed: 42,
return_logprobs: false,
top_logprobs: 0,
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct SpeculativeConfig {
pub num_draft_tokens: usize,
}
impl Default for SpeculativeConfig {
fn default() -> Self {
Self {
num_draft_tokens: 4,
}
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct TokenLogProb {
pub token_id: usize,
pub logprob: f32,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct StepLogProbs {
pub token_id: usize,
pub logprob: f32,
pub top: Vec<TokenLogProb>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct GenerationOutput {
pub token_ids: Vec<usize>,
pub logprobs: Option<Vec<StepLogProbs>>,
}
#[derive(Debug)]
pub enum GeneratorError {
InvalidConfig(String),
Model(ModelError),
Tensor(TensorError),
}
impl fmt::Display for GeneratorError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidConfig(message) => write!(formatter, "generation error: {message}"),
Self::Model(error) => error.fmt(formatter),
Self::Tensor(error) => error.fmt(formatter),
}
}
}
impl Error for GeneratorError {}
impl From<ModelError> for GeneratorError {
fn from(error: ModelError) -> Self {
Self::Model(error)
}
}
impl From<TensorError> for GeneratorError {
fn from(error: TensorError) -> Self {
Self::Tensor(error)
}
}
pub struct Generator<'a> {
model: &'a CausalLM,
tokenizer: &'a BpeTokenizer,
gpu: Option<&'a GpuContext>,
}
impl<'a> Generator<'a> {
pub fn new(model: &'a CausalLM, tokenizer: &'a BpeTokenizer) -> Self {
Self {
model,
tokenizer,
gpu: None,
}
}
pub fn with_gpu(model: &'a CausalLM, tokenizer: &'a BpeTokenizer, gpu: &'a GpuContext) -> Self {
Self {
model,
tokenizer,
gpu: Some(gpu),
}
}
pub fn generate_ids(
&self,
prompt: &str,
config: &SamplingConfig,
) -> Result<Vec<usize>, GeneratorError> {
Ok(self.generate(prompt, config)?.token_ids)
}
pub fn generate(
&self,
prompt: &str,
config: &SamplingConfig,
) -> Result<GenerationOutput, GeneratorError> {
self.validate_tokenizer()?;
let mut context = vec![self.tokenizer.special_tokens.bos];
context.extend(self.tokenizer.encode(prompt));
let mut output = generate_tokens(self.model, &context, config, self.gpu)?;
if output.token_ids.last() == Some(&self.model.config.eos_token_id) {
output.token_ids.pop();
if let Some(logprobs) = output.logprobs.as_mut() {
logprobs.pop();
}
}
Ok(output)
}
pub fn generate_stream<F>(
&self,
prompt: &str,
config: &SamplingConfig,
mut emit: F,
) -> Result<(), GeneratorError>
where
F: FnMut(usize, &[u8]),
{
self.generate_stream_with_logprobs(prompt, config, |token_id, bytes, _logprobs| {
emit(token_id, bytes);
})
}
pub fn generate_stream_with_logprobs<F>(
&self,
prompt: &str,
config: &SamplingConfig,
mut emit: F,
) -> Result<(), GeneratorError>
where
F: FnMut(usize, &[u8], Option<&StepLogProbs>),
{
self.validate_tokenizer()?;
let mut context = vec![self.tokenizer.special_tokens.bos];
context.extend(self.tokenizer.encode(prompt));
generate_token_stream(self.model, &context, config, self.gpu, |next_token, logprobs| {
if next_token != self.model.config.eos_token_id {
let bytes = self.tokenizer.token_bytes(next_token).unwrap_or(b"<UNK>");
emit(next_token, bytes, logprobs.as_ref());
}
})
}
pub fn generate_batch(
&self,
prompts: &[&str],
config: &SamplingConfig,
max_batch_size: usize,
) -> Result<Vec<GenerationOutput>, GeneratorError> {
let mut batcher = match self.gpu {
Some(gpu) => {
ContinuousBatcher::with_gpu(self.model, self.tokenizer, gpu, max_batch_size)
}
None => ContinuousBatcher::new(self.model, self.tokenizer, max_batch_size),
};
let configs = vec![config.clone(); prompts.len()];
batcher.run(prompts, &configs)
}
fn validate_tokenizer(&self) -> Result<(), GeneratorError> {
validate_tokenizer_pair(self.tokenizer, self.model)
}
pub fn with_draft(
&self,
draft: &'a CausalLM,
speculative: SpeculativeConfig,
) -> Result<SpeculativeGenerator<'a>, GeneratorError> {
SpeculativeGenerator::new(
self.model,
draft,
self.tokenizer,
speculative,
self.gpu,
)
}
}
pub struct SpeculativeGenerator<'a> {
target: &'a CausalLM,
draft: &'a CausalLM,
tokenizer: &'a BpeTokenizer,
gpu: Option<&'a GpuContext>,
speculative: SpeculativeConfig,
}
impl<'a> SpeculativeGenerator<'a> {
pub fn new(
target: &'a CausalLM,
draft: &'a CausalLM,
tokenizer: &'a BpeTokenizer,
speculative: SpeculativeConfig,
gpu: Option<&'a GpuContext>,
) -> Result<Self, GeneratorError> {
validate_tokenizer_pair(tokenizer, target)?;
validate_speculative_pair(target, draft, &speculative)?;
Ok(Self {
target,
draft,
tokenizer,
gpu,
speculative,
})
}
pub fn with_gpu(
target: &'a CausalLM,
draft: &'a CausalLM,
tokenizer: &'a BpeTokenizer,
gpu: &'a GpuContext,
speculative: SpeculativeConfig,
) -> Result<Self, GeneratorError> {
Self::new(target, draft, tokenizer, speculative, Some(gpu))
}
pub fn generate_ids(
&self,
prompt: &str,
config: &SamplingConfig,
) -> Result<Vec<usize>, GeneratorError> {
Ok(self.generate(prompt, config)?.token_ids)
}
pub fn generate(
&self,
prompt: &str,
config: &SamplingConfig,
) -> Result<GenerationOutput, GeneratorError> {
let mut context = vec![self.tokenizer.special_tokens.bos];
context.extend(self.tokenizer.encode(prompt));
let mut output =
generate_speculative_tokens(self.target, self.draft, &context, config, &self.speculative, self.gpu)?;
if output.token_ids.last() == Some(&self.target.config.eos_token_id) {
output.token_ids.pop();
if let Some(logprobs) = output.logprobs.as_mut() {
logprobs.pop();
}
}
Ok(output)
}
pub fn generate_stream<F>(
&self,
prompt: &str,
config: &SamplingConfig,
mut emit: F,
) -> Result<(), GeneratorError>
where
F: FnMut(usize, &[u8]),
{
self.generate_stream_with_logprobs(prompt, config, |token_id, bytes, _logprobs| {
emit(token_id, bytes);
})
}
pub fn generate_stream_with_logprobs<F>(
&self,
prompt: &str,
config: &SamplingConfig,
mut emit: F,
) -> Result<(), GeneratorError>
where
F: FnMut(usize, &[u8], Option<&StepLogProbs>),
{
let mut context = vec![self.tokenizer.special_tokens.bos];
context.extend(self.tokenizer.encode(prompt));
generate_speculative_token_stream(
self.target,
self.draft,
&context,
config,
&self.speculative,
self.gpu,
|next_token, logprobs| {
if next_token != self.target.config.eos_token_id {
let bytes = self.tokenizer.token_bytes(next_token).unwrap_or(b"<UNK>");
emit(next_token, bytes, logprobs.as_ref());
}
},
)
}
}
fn validate_tokenizer_pair(
tokenizer: &BpeTokenizer,
model: &CausalLM,
) -> Result<(), GeneratorError> {
if tokenizer.vocab_size() != model.config.vocab_size {
return Err(GeneratorError::InvalidConfig(
"tokenizer and model vocabulary sizes differ".into(),
));
}
if tokenizer.special_tokens.eos != model.config.eos_token_id {
return Err(GeneratorError::InvalidConfig(
"tokenizer and model EOS token IDs differ".into(),
));
}
Ok(())
}
fn validate_speculative_pair(
target: &CausalLM,
draft: &CausalLM,
speculative: &SpeculativeConfig,
) -> Result<(), GeneratorError> {
if speculative.num_draft_tokens == 0 {
return Err(GeneratorError::InvalidConfig(
"num_draft_tokens must be positive".into(),
));
}
if target.config.vocab_size != draft.config.vocab_size {
return Err(GeneratorError::InvalidConfig(
"target and draft vocabulary sizes differ".into(),
));
}
if target.config.eos_token_id != draft.config.eos_token_id {
return Err(GeneratorError::InvalidConfig(
"target and draft EOS token IDs differ".into(),
));
}
Ok(())
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct RequestId(u64);
impl RequestId {
pub fn get(self) -> u64 {
self.0
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct DecodeEvent {
pub request_id: RequestId,
pub token_id: usize,
pub finished: bool,
pub logprobs: Option<StepLogProbs>,
}
struct PendingRequest {
id: RequestId,
prompt_tokens: Vec<usize>,
config: SamplingConfig,
}
struct ActiveRequest {
id: RequestId,
history: Vec<usize>,
cache: KvCache,
config: SamplingConfig,
generated: Vec<usize>,
logprobs: Option<Vec<StepLogProbs>>,
next_logits: Vec<f32>,
rng: StdRng,
scratch: SamplingScratch,
}
pub struct ContinuousBatcher<'a> {
model: &'a CausalLM,
tokenizer: &'a BpeTokenizer,
gpu: Option<&'a GpuContext>,
max_batch_size: usize,
next_id: u64,
waiting: VecDeque<PendingRequest>,
active: Vec<ActiveRequest>,
finished: HashMap<RequestId, GenerationOutput>,
submission_order: Vec<RequestId>,
}
impl<'a> ContinuousBatcher<'a> {
pub fn new(model: &'a CausalLM, tokenizer: &'a BpeTokenizer, max_batch_size: usize) -> Self {
Self {
model,
tokenizer,
gpu: None,
max_batch_size: max_batch_size.max(1),
next_id: 1,
waiting: VecDeque::new(),
active: Vec::new(),
finished: HashMap::new(),
submission_order: Vec::new(),
}
}
pub fn with_gpu(
model: &'a CausalLM,
tokenizer: &'a BpeTokenizer,
gpu: &'a GpuContext,
max_batch_size: usize,
) -> Self {
Self {
model,
tokenizer,
gpu: Some(gpu),
max_batch_size: max_batch_size.max(1),
next_id: 1,
waiting: VecDeque::new(),
active: Vec::new(),
finished: HashMap::new(),
submission_order: Vec::new(),
}
}
pub fn max_batch_size(&self) -> usize {
self.max_batch_size
}
pub fn active_len(&self) -> usize {
self.active.len()
}
pub fn waiting_len(&self) -> usize {
self.waiting.len()
}
pub fn is_idle(&self) -> bool {
self.active.is_empty() && self.waiting.is_empty()
}
pub fn submit(
&mut self,
prompt: &str,
config: &SamplingConfig,
) -> Result<RequestId, GeneratorError> {
validate_tokenizer_pair(self.tokenizer, self.model)?;
let mut prompt_tokens = vec![self.tokenizer.special_tokens.bos];
prompt_tokens.extend(self.tokenizer.encode(prompt));
self.submit_token_ids(&prompt_tokens, config)
}
pub fn submit_token_ids(
&mut self,
prompt_tokens: &[usize],
config: &SamplingConfig,
) -> Result<RequestId, GeneratorError> {
validate_sampling_config(config)?;
validate_prompt_tokens(self.model, prompt_tokens)?;
let id = RequestId(self.next_id);
self.next_id += 1;
self.submission_order.push(id);
self.waiting.push_back(PendingRequest {
id,
prompt_tokens: prompt_tokens.to_vec(),
config: config.clone(),
});
self.admit_waiting()?;
Ok(id)
}
pub fn step(&mut self) -> Result<Vec<DecodeEvent>, GeneratorError> {
let _no_grad = crate::no_grad();
self.admit_waiting()?;
if self.active.is_empty() {
return Ok(Vec::new());
}
let eos = self.model.config.eos_token_id;
let mut events = Vec::with_capacity(self.active.len());
for request in &mut self.active {
let next_token = sample_token(
&request.next_logits,
&request.history,
&request.config,
&mut request.rng,
&mut request.scratch,
);
let step_logprobs = request.config.return_logprobs.then(|| {
compute_step_logprobs(
&request.next_logits,
next_token,
request.config.top_logprobs,
&mut request.scratch,
)
});
request.history.push(next_token);
request.generated.push(next_token);
if let (Some(logprobs), Some(step)) = (request.logprobs.as_mut(), step_logprobs.clone())
{
logprobs.push(step);
}
let finished = request_is_finished(request, eos);
events.push(DecodeEvent {
request_id: request.id,
token_id: next_token,
finished,
logprobs: step_logprobs,
});
}
self.retire_finished(eos);
if !self.active.is_empty() {
let continuing: Vec<usize> = (0..self.active.len()).collect();
let groups = group_indices_by_position(&self.active, &continuing);
for indices in groups {
self.forward_decode_group(&indices)?;
}
}
self.admit_waiting()?;
Ok(events)
}
pub fn run_token_ids(
&mut self,
prompts: &[&[usize]],
configs: &[SamplingConfig],
) -> Result<Vec<GenerationOutput>, GeneratorError> {
if prompts.len() != configs.len() {
return Err(GeneratorError::InvalidConfig(
"prompt and sampling config counts differ".into(),
));
}
let mut ids = Vec::with_capacity(prompts.len());
for (prompt, config) in prompts.iter().zip(configs) {
ids.push(self.submit_token_ids(prompt, config)?);
}
while !self.is_idle() {
self.step()?;
}
let finished = self.take_finished();
let mut by_id: HashMap<RequestId, GenerationOutput> = finished.into_iter().collect();
ids.into_iter()
.map(|id| {
by_id.remove(&id).ok_or_else(|| {
GeneratorError::InvalidConfig(format!(
"missing continuous-batch output for request {}",
id.get()
))
})
})
.collect()
}
pub fn take_finished(&mut self) -> Vec<(RequestId, GenerationOutput)> {
let mut completed = Vec::new();
self.submission_order.retain(|id| {
if let Some(output) = self.finished.remove(id) {
completed.push((*id, output));
false
} else {
true
}
});
completed
}
pub fn run(
&mut self,
prompts: &[&str],
configs: &[SamplingConfig],
) -> Result<Vec<GenerationOutput>, GeneratorError> {
if prompts.len() != configs.len() {
return Err(GeneratorError::InvalidConfig(
"prompt and sampling config counts differ".into(),
));
}
let mut ids = Vec::with_capacity(prompts.len());
for (prompt, config) in prompts.iter().zip(configs) {
ids.push(self.submit(prompt, config)?);
}
while !self.is_idle() {
self.step()?;
}
let finished = self.take_finished();
let mut by_id: HashMap<RequestId, GenerationOutput> = finished.into_iter().collect();
ids.into_iter()
.map(|id| {
by_id.remove(&id).ok_or_else(|| {
GeneratorError::InvalidConfig(format!(
"missing continuous-batch output for request {}",
id.get()
))
})
})
.collect()
}
fn admit_waiting(&mut self) -> Result<(), GeneratorError> {
while self.active.len() < self.max_batch_size {
let Some(pending) = self.waiting.pop_front() else {
break;
};
if pending.config.max_new_tokens == 0 {
self.finished.insert(
pending.id,
GenerationOutput {
token_ids: Vec::new(),
logprobs: pending.config.return_logprobs.then(Vec::new),
},
);
continue;
}
let active = self.prefill_request(pending)?;
self.active.push(active);
}
Ok(())
}
fn prefill_request(&self, pending: PendingRequest) -> Result<ActiveRequest, GeneratorError> {
let history = pending.prompt_tokens;
let start = history
.len()
.saturating_sub(self.model.config.max_sequence_length);
let visible = &history[start..];
let input = IndexTensor::try_new(visible.to_vec(), vec![1, visible.len()])?;
let mut cache = KvCache::new(self.model.layers.len());
let logits = if let Some(gpu) = self.gpu {
self.model.forward_cached_gpu(&input, &mut cache, gpu)?
} else {
self.model.forward_cached(&input, &mut cache)?
};
let values = logits.to_cpu().map_err(ModelError::from)?.data();
let vocab = self.model.config.vocab_size;
let row_start = (logits.shape()[1] - 1) * vocab;
let next_logits = values[row_start..row_start + vocab].to_vec();
Ok(ActiveRequest {
id: pending.id,
history,
cache,
scratch: SamplingScratch::with_vocab(vocab),
rng: StdRng::seed_from_u64(pending.config.seed),
logprobs: pending.config.return_logprobs.then(Vec::new),
generated: Vec::new(),
next_logits,
config: pending.config,
})
}
fn forward_decode_group(&mut self, indices: &[usize]) -> Result<(), GeneratorError> {
if indices.is_empty() {
return Ok(());
}
let tokens: Vec<usize> = indices
.iter()
.map(|&index| *self.active[index].history.last().expect("history non-empty"))
.collect();
let cache_refs: Vec<&KvCache> = indices
.iter()
.map(|&index| &self.active[index].cache)
.collect();
let mut packed_cache = KvCache::concat_batch(&cache_refs)?;
let input = IndexTensor::try_new(tokens, vec![indices.len(), 1])?;
let logits = if let Some(gpu) = self.gpu {
self.model
.forward_cached_gpu(&input, &mut packed_cache, gpu)?
} else {
self.model.forward_cached(&input, &mut packed_cache)?
};
let split_caches = packed_cache.split_batch()?;
if split_caches.len() != indices.len() {
return Err(GeneratorError::InvalidConfig(
"continuous batch cache split size mismatch".into(),
));
}
for (&request_index, cache) in indices.iter().zip(split_caches) {
self.active[request_index].cache = cache;
}
let values = logits.to_cpu().map_err(ModelError::from)?.data();
let vocab = self.model.config.vocab_size;
for (offset, &request_index) in indices.iter().enumerate() {
let row_start = offset * vocab;
self.active[request_index].next_logits =
values[row_start..row_start + vocab].to_vec();
}
Ok(())
}
fn retire_finished(&mut self, eos: usize) {
self.active.retain(|request| {
if request_is_finished(request, eos) {
let mut token_ids = request.generated.clone();
let mut logprobs = request.logprobs.clone();
if token_ids.last() == Some(&eos) {
token_ids.pop();
if let Some(logprobs) = logprobs.as_mut() {
logprobs.pop();
}
}
self.finished.insert(
request.id,
GenerationOutput {
token_ids,
logprobs,
},
);
false
} else {
true
}
});
}
}
fn group_indices_by_position(active: &[ActiveRequest], indices: &[usize]) -> Vec<Vec<usize>> {
let mut order = Vec::new();
let mut groups: HashMap<usize, Vec<usize>> = HashMap::new();
for &index in indices {
let position = active[index].cache.position();
groups
.entry(position)
.or_insert_with(|| {
order.push(position);
Vec::new()
})
.push(index);
}
order
.into_iter()
.filter_map(|position| groups.remove(&position))
.collect()
}
fn request_is_finished(request: &ActiveRequest, eos: usize) -> bool {
request.generated.len() >= request.config.max_new_tokens
|| request.generated.last() == Some(&eos)
}
fn validate_prompt_tokens(model: &CausalLM, prompt: &[usize]) -> Result<(), GeneratorError> {
if prompt.is_empty() {
return Err(GeneratorError::InvalidConfig(
"generation prompt cannot be empty".into(),
));
}
if !prompt.iter().all(|token| *token < model.config.vocab_size) {
return Err(GeneratorError::InvalidConfig(
"generation prompt contains an invalid token".into(),
));
}
Ok(())
}
pub(crate) fn generate_tokens(
model: &CausalLM,
prompt: &[usize],
config: &SamplingConfig,
gpu: Option<&GpuContext>,
) -> Result<GenerationOutput, GeneratorError> {
let mut token_ids = Vec::new();
let mut logprobs = config.return_logprobs.then(Vec::new);
generate_token_stream(model, prompt, config, gpu, |token_id, step| {
token_ids.push(token_id);
if let (Some(logprobs), Some(step)) = (logprobs.as_mut(), step) {
logprobs.push(step);
}
})?;
Ok(GenerationOutput {
token_ids,
logprobs,
})
}
pub(crate) fn generate_token_stream<F>(
model: &CausalLM,
prompt: &[usize],
config: &SamplingConfig,
gpu: Option<&GpuContext>,
mut emit: F,
) -> Result<(), GeneratorError>
where
F: FnMut(usize, Option<StepLogProbs>),
{
let _no_grad = crate::no_grad();
validate_sampling_config(config)?;
if prompt.is_empty() {
return Err(GeneratorError::InvalidConfig(
"generation prompt cannot be empty".into(),
));
}
if !prompt.iter().all(|token| *token < model.config.vocab_size) {
return Err(GeneratorError::InvalidConfig(
"generation prompt contains an invalid token".into(),
));
}
if config.max_new_tokens == 0 {
return Ok(());
}
let mut history = prompt.to_vec();
let start = history
.len()
.saturating_sub(model.config.max_sequence_length);
let visible = &history[start..];
let input = IndexTensor::try_new(visible.to_vec(), vec![1, visible.len()])?;
let mut cache = KvCache::new(model.layers.len());
let mut logits = if let Some(gpu) = gpu {
model.forward_cached_gpu(&input, &mut cache, gpu)?
} else {
model.forward_cached(&input, &mut cache)?
};
let mut rng = StdRng::seed_from_u64(config.seed);
let mut scratch = SamplingScratch::with_vocab(model.config.vocab_size);
for token_index in 0..config.max_new_tokens {
let values = logits.to_cpu().map_err(ModelError::from)?.data();
let row_start = (logits.shape()[1] - 1) * model.config.vocab_size;
let row = &values[row_start..row_start + model.config.vocab_size];
let next_token = sample_token(row, &history, config, &mut rng, &mut scratch);
let step_logprobs = config
.return_logprobs
.then(|| compute_step_logprobs(row, next_token, config.top_logprobs, &mut scratch));
history.push(next_token);
emit(next_token, step_logprobs);
if next_token == model.config.eos_token_id {
break;
}
if token_index + 1 < config.max_new_tokens {
let input = IndexTensor::try_new(vec![next_token], vec![1, 1])?;
logits = if let Some(gpu) = gpu {
model.forward_cached_gpu(&input, &mut cache, gpu)?
} else {
model.forward_cached(&input, &mut cache)?
};
}
}
Ok(())
}
pub(crate) fn generate_speculative_tokens(
target: &CausalLM,
draft: &CausalLM,
prompt: &[usize],
config: &SamplingConfig,
speculative: &SpeculativeConfig,
gpu: Option<&GpuContext>,
) -> Result<GenerationOutput, GeneratorError> {
let mut token_ids = Vec::new();
let mut logprobs = config.return_logprobs.then(Vec::new);
generate_speculative_token_stream(
target,
draft,
prompt,
config,
speculative,
gpu,
|token_id, step| {
token_ids.push(token_id);
if let (Some(logprobs), Some(step)) = (logprobs.as_mut(), step) {
logprobs.push(step);
}
},
)?;
Ok(GenerationOutput {
token_ids,
logprobs,
})
}
pub(crate) fn generate_speculative_token_stream<F>(
target: &CausalLM,
draft: &CausalLM,
prompt: &[usize],
config: &SamplingConfig,
speculative: &SpeculativeConfig,
gpu: Option<&GpuContext>,
mut emit: F,
) -> Result<(), GeneratorError>
where
F: FnMut(usize, Option<StepLogProbs>),
{
let _no_grad = crate::no_grad();
validate_sampling_config(config)?;
validate_speculative_pair(target, draft, speculative)?;
if prompt.is_empty() {
return Err(GeneratorError::InvalidConfig(
"generation prompt cannot be empty".into(),
));
}
if !prompt
.iter()
.all(|token| *token < target.config.vocab_size)
{
return Err(GeneratorError::InvalidConfig(
"generation prompt contains an invalid token".into(),
));
}
if config.max_new_tokens == 0 {
return Ok(());
}
let max_context = target
.config
.max_sequence_length
.min(draft.config.max_sequence_length);
let mut history = prompt.to_vec();
let start = history.len().saturating_sub(max_context);
let visible = &history[start..];
let input = IndexTensor::try_new(visible.to_vec(), vec![1, visible.len()])?;
let mut target_cache = KvCache::new(target.layers.len());
let mut draft_cache = KvCache::new(draft.layers.len());
let mut target_logits = forward_cached_row(target, &input, &mut target_cache, gpu)?;
let mut draft_logits = forward_cached_row(draft, &input, &mut draft_cache, gpu)?;
let mut rng = StdRng::seed_from_u64(config.seed);
let mut scratch = SamplingScratch::with_vocab(target.config.vocab_size);
let mut draft_scratch = SamplingScratch::with_vocab(target.config.vocab_size);
let mut residual_scratch = SamplingScratch::with_vocab(target.config.vocab_size);
let mut emitted = 0usize;
let eos = target.config.eos_token_id;
while emitted < config.max_new_tokens {
let prefix_len = target_cache.sequence_length();
let room = max_context.saturating_sub(prefix_len);
if room == 0 {
break;
}
let gamma = speculative
.num_draft_tokens
.min(config.max_new_tokens - emitted)
.min(room);
let mut draft_tokens = Vec::with_capacity(gamma);
let mut draft_logit_rows = Vec::with_capacity(gamma);
let mut draft_history = history.clone();
let mut next_draft_logits = draft_logits.clone();
for _ in 0..gamma {
draft_logit_rows.push(next_draft_logits.clone());
let token = sample_token(
&next_draft_logits,
&draft_history,
config,
&mut rng,
&mut draft_scratch,
);
draft_tokens.push(token);
draft_history.push(token);
next_draft_logits = forward_one(draft, token, &mut draft_cache, gpu)?;
if token == eos {
break;
}
}
let draft_len = draft_tokens.len();
if draft_len == 0 {
break;
}
let verify_logits =
forward_cached_rows(target, &draft_tokens, &mut target_cache, gpu)?;
let mut accepted = 0usize;
let mut rejected = false;
for index in 0..draft_len {
let target_row = if index == 0 {
target_logits.as_slice()
} else {
verify_logits[index - 1].as_slice()
};
let draft_row = draft_logit_rows[index].as_slice();
let token = draft_tokens[index];
if accept_draft_token(
target_row,
draft_row,
&history,
token,
config,
&mut rng,
&mut scratch,
&mut draft_scratch,
) {
let step_logprobs = config.return_logprobs.then(|| {
compute_step_logprobs(target_row, token, config.top_logprobs, &mut scratch)
});
history.push(token);
emit(token, step_logprobs);
emitted += 1;
accepted += 1;
if token == eos || emitted >= config.max_new_tokens {
break;
}
} else {
let correction = sample_residual_token(
target_row,
draft_row,
&history,
config,
&mut rng,
&mut scratch,
&mut draft_scratch,
&mut residual_scratch,
);
target_cache.truncate(prefix_len + accepted)?;
draft_cache.truncate(prefix_len + accepted)?;
let step_logprobs = config.return_logprobs.then(|| {
compute_step_logprobs(
target_row,
correction,
config.top_logprobs,
&mut scratch,
)
});
history.push(correction);
emit(correction, step_logprobs);
emitted += 1;
rejected = true;
if correction != eos && emitted < config.max_new_tokens {
target_logits = forward_one(target, correction, &mut target_cache, gpu)?;
draft_logits = forward_one(draft, correction, &mut draft_cache, gpu)?;
}
break;
}
}
if history.last() == Some(&eos) || emitted >= config.max_new_tokens {
break;
}
if rejected {
continue;
}
target_cache.truncate(prefix_len + accepted)?;
draft_cache.truncate(prefix_len + accepted)?;
let bonus_row = &verify_logits[accepted - 1];
let bonus = sample_token(bonus_row, &history, config, &mut rng, &mut scratch);
let step_logprobs = config.return_logprobs.then(|| {
compute_step_logprobs(bonus_row, bonus, config.top_logprobs, &mut scratch)
});
history.push(bonus);
emit(bonus, step_logprobs);
emitted += 1;
if bonus == eos || emitted >= config.max_new_tokens {
break;
}
target_logits = forward_one(target, bonus, &mut target_cache, gpu)?;
draft_logits = forward_one(draft, bonus, &mut draft_cache, gpu)?;
}
Ok(())
}
fn forward_cached_row(
model: &CausalLM,
input: &IndexTensor,
cache: &mut KvCache,
gpu: Option<&GpuContext>,
) -> Result<Vec<f32>, GeneratorError> {
let logits = if let Some(gpu) = gpu {
model.forward_cached_gpu(input, cache, gpu)?
} else {
model.forward_cached(input, cache)?
};
let values = logits.to_cpu().map_err(ModelError::from)?.data();
let vocab = model.config.vocab_size;
let row_start = (logits.shape()[1] - 1) * vocab;
Ok(values[row_start..row_start + vocab].to_vec())
}
fn forward_cached_rows(
model: &CausalLM,
tokens: &[usize],
cache: &mut KvCache,
gpu: Option<&GpuContext>,
) -> Result<Vec<Vec<f32>>, GeneratorError> {
let input = IndexTensor::try_new(tokens.to_vec(), vec![1, tokens.len()])?;
let logits = if let Some(gpu) = gpu {
model.forward_cached_gpu(&input, cache, gpu)?
} else {
model.forward_cached(&input, cache)?
};
let values = logits.to_cpu().map_err(ModelError::from)?.data();
let vocab = model.config.vocab_size;
let mut rows = Vec::with_capacity(tokens.len());
for index in 0..tokens.len() {
let start = index * vocab;
rows.push(values[start..start + vocab].to_vec());
}
Ok(rows)
}
fn forward_one(
model: &CausalLM,
token: usize,
cache: &mut KvCache,
gpu: Option<&GpuContext>,
) -> Result<Vec<f32>, GeneratorError> {
let input = IndexTensor::try_new(vec![token], vec![1, 1])?;
forward_cached_row(model, &input, cache, gpu)
}
fn accept_draft_token(
target_logits: &[f32],
draft_logits: &[f32],
history: &[usize],
token: usize,
config: &SamplingConfig,
rng: &mut StdRng,
target_scratch: &mut SamplingScratch,
draft_scratch: &mut SamplingScratch,
) -> bool {
if config.temperature == 0.0 {
let target_token = sample_token(target_logits, history, config, rng, target_scratch);
return target_token == token;
}
let p = token_probability(target_logits, history, config, token, target_scratch);
let q = token_probability(draft_logits, history, config, token, draft_scratch);
if q <= 0.0 {
return p > 0.0;
}
rng.random::<f32>() < (p / q).min(1.0)
}
fn token_probability(
logits: &[f32],
history: &[usize],
config: &SamplingConfig,
token: usize,
scratch: &mut SamplingScratch,
) -> f32 {
prepare_candidates(logits, history, config, scratch);
if config.temperature == 0.0 {
let chosen = scratch
.indices
.iter()
.copied()
.max_by(|&left, &right| scratch.logits[left].total_cmp(&scratch.logits[right]))
.unwrap();
return if chosen == token { 1.0 } else { 0.0 };
}
if !scratch.indices.iter().any(|&index| index == token) {
return 0.0;
}
let maximum = scratch
.indices
.iter()
.map(|&index| scratch.logits[index])
.fold(f32::NEG_INFINITY, f32::max);
let mut total = 0.0;
let mut chosen = 0.0;
for &index in &scratch.indices {
let weight = (scratch.logits[index] - maximum).exp();
total += weight;
if index == token {
chosen = weight;
}
}
if total <= 0.0 {
0.0
} else {
chosen / total
}
}
fn sample_residual_token(
target_logits: &[f32],
draft_logits: &[f32],
history: &[usize],
config: &SamplingConfig,
rng: &mut StdRng,
target_scratch: &mut SamplingScratch,
draft_scratch: &mut SamplingScratch,
residual_scratch: &mut SamplingScratch,
) -> usize {
if config.temperature == 0.0 {
return sample_token(target_logits, history, config, rng, target_scratch);
}
prepare_candidates(target_logits, history, config, target_scratch);
prepare_candidates(draft_logits, history, config, draft_scratch);
let vocab = target_logits.len();
residual_scratch.logits.clear();
residual_scratch.logits.resize(vocab, 0.0);
residual_scratch.indices.clear();
let mut p_mass = vec![0.0f32; vocab];
let mut q_mass = vec![0.0f32; vocab];
fill_candidate_probs(target_scratch, &mut p_mass);
fill_candidate_probs(draft_scratch, &mut q_mass);
let mut total = 0.0f32;
for token in 0..vocab {
let residual = (p_mass[token] - q_mass[token]).max(0.0);
if residual > 0.0 {
residual_scratch.indices.push(token);
residual_scratch.logits[token] = residual;
total += residual;
}
}
if residual_scratch.indices.is_empty() || total <= 0.0 {
return sample_token(target_logits, history, config, rng, target_scratch);
}
let threshold = rng.random::<f32>() * total;
let mut cumulative = 0.0;
for &token in &residual_scratch.indices {
cumulative += residual_scratch.logits[token];
if cumulative >= threshold {
return token;
}
}
*residual_scratch.indices.last().unwrap()
}
fn fill_candidate_probs(scratch: &SamplingScratch, mass: &mut [f32]) {
if scratch.indices.is_empty() {
return;
}
let maximum = scratch
.indices
.iter()
.map(|&index| scratch.logits[index])
.fold(f32::NEG_INFINITY, f32::max);
let mut total = 0.0f32;
let mut weights = Vec::with_capacity(scratch.indices.len());
for &index in &scratch.indices {
let weight = (scratch.logits[index] - maximum).exp();
weights.push(weight);
total += weight;
}
if total <= 0.0 {
return;
}
for (offset, &index) in scratch.indices.iter().enumerate() {
mass[index] = weights[offset] / total;
}
}
struct SamplingScratch {
logits: Vec<f32>,
indices: Vec<usize>,
weights: Vec<f32>,
seen: Vec<bool>,
counts: Vec<u32>,
}
impl SamplingScratch {
fn with_vocab(vocab_size: usize) -> Self {
Self {
logits: Vec::with_capacity(vocab_size),
indices: Vec::with_capacity(vocab_size),
weights: Vec::with_capacity(vocab_size),
seen: vec![false; vocab_size],
counts: vec![0; vocab_size],
}
}
fn resize_seen(&mut self, vocab_size: usize) {
if self.seen.len() != vocab_size {
self.seen.clear();
self.seen.resize(vocab_size, false);
} else {
self.seen.fill(false);
}
}
fn resize_counts(&mut self, vocab_size: usize) {
if self.counts.len() != vocab_size {
self.counts.clear();
self.counts.resize(vocab_size, 0);
} else {
self.counts.fill(0);
}
}
}
pub fn filter_logits(logits: &[f32], history: &[usize], config: &SamplingConfig) -> Vec<f32> {
let mut scratch = SamplingScratch::with_vocab(logits.len());
prepare_candidates(logits, history, config, &mut scratch);
let mut filtered = scratch.logits;
let keep = scratch.indices;
let mut retained = vec![false; filtered.len()];
for index in keep {
retained[index] = true;
}
for (index, value) in filtered.iter_mut().enumerate() {
if !retained[index] {
*value = f32::NEG_INFINITY;
}
}
filtered
}
fn prepare_candidates(
logits: &[f32],
history: &[usize],
config: &SamplingConfig,
scratch: &mut SamplingScratch,
) {
scratch.logits.clear();
scratch.logits.extend_from_slice(logits);
if config.repetition_penalty != 1.0 {
let penalty_history = match config.repetition_penalty_window {
Some(window) => {
let start = history.len().saturating_sub(window);
&history[start..]
}
None => history,
};
scratch.resize_seen(logits.len());
for token in penalty_history {
if *token < scratch.seen.len() && !scratch.seen[*token] {
scratch.seen[*token] = true;
if scratch.logits[*token] >= 0.0 {
scratch.logits[*token] /= config.repetition_penalty;
} else {
scratch.logits[*token] *= config.repetition_penalty;
}
}
}
}
if config.frequency_penalty != 0.0 || config.presence_penalty != 0.0 {
scratch.resize_counts(logits.len());
for &token in history {
if token < scratch.counts.len() {
scratch.counts[token] += 1;
}
}
for (token, &count) in scratch.counts.iter().enumerate() {
if count == 0 {
continue;
}
scratch.logits[token] -= config.presence_penalty;
scratch.logits[token] -= config.frequency_penalty * count as f32;
}
}
if config.temperature > 0.0 {
for value in &mut scratch.logits {
*value /= config.temperature;
}
}
select_top_k_indices(&scratch.logits, config.top_k, &mut scratch.indices);
apply_top_p(
&scratch.logits,
config.top_p,
&mut scratch.indices,
&mut scratch.weights,
);
}
fn select_top_k_indices(logits: &[f32], top_k: Option<usize>, indices: &mut Vec<usize>) {
indices.clear();
indices.extend(0..logits.len());
let Some(top_k) = top_k else {
return;
};
if top_k == 0 || top_k >= indices.len() {
return;
}
indices.select_nth_unstable_by(top_k - 1, |&left, &right| {
logits[right].total_cmp(&logits[left])
});
indices.truncate(top_k);
}
fn apply_top_p(logits: &[f32], top_p: f32, indices: &mut Vec<usize>, weights: &mut Vec<f32>) {
if top_p >= 1.0 || indices.is_empty() {
return;
}
indices.sort_unstable_by(|&left, &right| logits[right].total_cmp(&logits[left]));
let maximum = indices
.iter()
.map(|&index| logits[index])
.fold(f32::NEG_INFINITY, f32::max);
weights.clear();
weights.reserve(indices.len());
let mut total = 0.0;
for &index in indices.iter() {
let weight = (logits[index] - maximum).exp();
weights.push(weight);
total += weight;
}
if total <= 0.0 || !total.is_finite() {
return;
}
let mut cumulative = 0.0;
let mut cutoff = indices.len();
for (position, weight) in weights.iter().enumerate() {
cumulative += *weight / total;
if cumulative >= top_p {
cutoff = position + 1;
break;
}
}
indices.truncate(cutoff);
}
fn sample_token(
logits: &[f32],
history: &[usize],
config: &SamplingConfig,
rng: &mut StdRng,
scratch: &mut SamplingScratch,
) -> usize {
let _span = profile::span("sampling");
prepare_candidates(logits, history, config, scratch);
if config.temperature == 0.0 {
return scratch
.indices
.iter()
.copied()
.max_by(|&left, &right| scratch.logits[left].total_cmp(&scratch.logits[right]))
.unwrap();
}
let maximum = scratch
.indices
.iter()
.map(|&index| scratch.logits[index])
.fold(f32::NEG_INFINITY, f32::max);
scratch.weights.clear();
scratch.weights.reserve(scratch.indices.len());
let mut total = 0.0;
for &index in &scratch.indices {
let weight = (scratch.logits[index] - maximum).exp();
scratch.weights.push(weight);
total += weight;
}
let threshold = rng.random::<f32>() * total;
let mut cumulative = 0.0;
for (offset, weight) in scratch.weights.iter().enumerate() {
cumulative += weight;
if cumulative >= threshold {
return scratch.indices[offset];
}
}
*scratch.indices.last().unwrap()
}
fn compute_step_logprobs(
logits: &[f32],
chosen: usize,
top_n: usize,
scratch: &mut SamplingScratch,
) -> StepLogProbs {
let maximum = logits
.iter()
.copied()
.fold(f32::NEG_INFINITY, f32::max);
let mut sum_exp = 0.0f32;
for &value in logits {
sum_exp += (value - maximum).exp();
}
let log_z = if sum_exp > 0.0 && sum_exp.is_finite() {
maximum + sum_exp.ln()
} else {
maximum
};
let logprob = logits[chosen] - log_z;
let top = if top_n == 0 {
Vec::new()
} else {
select_top_k_indices(logits, Some(top_n.min(logits.len())), &mut scratch.indices);
scratch
.indices
.sort_unstable_by(|&left, &right| logits[right].total_cmp(&logits[left]));
scratch
.indices
.iter()
.map(|&token_id| TokenLogProb {
token_id,
logprob: logits[token_id] - log_z,
})
.collect()
};
StepLogProbs {
token_id: chosen,
logprob,
top,
}
}
pub fn validate_sampling_config(config: &SamplingConfig) -> Result<(), GeneratorError> {
if !config.temperature.is_finite() || config.temperature < 0.0 {
return Err(GeneratorError::InvalidConfig(
"temperature must be finite and non-negative".into(),
));
}
if !(0.0 < config.top_p && config.top_p <= 1.0) {
return Err(GeneratorError::InvalidConfig(
"top-p must be in (0, 1]".into(),
));
}
if config.repetition_penalty < 1.0 || !config.repetition_penalty.is_finite() {
return Err(GeneratorError::InvalidConfig(
"repetition penalty must be finite and at least one".into(),
));
}
if !config.frequency_penalty.is_finite() {
return Err(GeneratorError::InvalidConfig(
"frequency penalty must be finite".into(),
));
}
if !config.presence_penalty.is_finite() {
return Err(GeneratorError::InvalidConfig(
"presence penalty must be finite".into(),
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::{
compute_step_logprobs, filter_logits, select_top_k_indices, ContinuousBatcher,
SamplingConfig, SamplingScratch, SpeculativeConfig,
};
use crate::model::{CausalLM, CausalLmConfig, KvCache};
use crate::IndexTensor;
#[test]
fn top_k_keeps_only_k_logits() {
let config = SamplingConfig {
top_k: Some(2),
top_p: 1.0,
repetition_penalty: 1.0,
..Default::default()
};
let filtered = filter_logits(&[1.0, 4.0, 2.0, 3.0], &[], &config);
assert_eq!(filtered.iter().filter(|value| value.is_finite()).count(), 2);
assert!(filtered[1].is_finite() && filtered[3].is_finite());
}
#[test]
fn top_p_keeps_smallest_probability_prefix() {
let config = SamplingConfig {
top_k: None,
top_p: 0.8,
repetition_penalty: 1.0,
..Default::default()
};
let filtered = filter_logits(&[4.0, 3.0, 0.0], &[], &config);
assert!(filtered[0].is_finite());
assert!(filtered[1].is_finite());
assert!(!filtered[2].is_finite());
}
#[test]
fn top_k_selection_keeps_highest_values_without_full_sort() {
let logits: Vec<f32> = (0..128).map(|value| value as f32).collect();
let mut indices = Vec::new();
select_top_k_indices(&logits, Some(5), &mut indices);
assert_eq!(indices.len(), 5);
let mut selected = indices;
selected.sort_unstable();
assert_eq!(selected, vec![123, 124, 125, 126, 127]);
}
#[test]
fn step_logprobs_match_log_softmax_and_top_order() {
let logits = [1.0f32, 3.0, 0.0, 2.0];
let mut scratch = SamplingScratch::with_vocab(logits.len());
let step = compute_step_logprobs(&logits, 1, 2, &mut scratch);
let maximum = 3.0f32;
let log_z = maximum
+ logits
.iter()
.map(|value| (value - maximum).exp())
.sum::<f32>()
.ln();
assert_eq!(step.token_id, 1);
assert!((step.logprob - (3.0 - log_z)).abs() < 1e-6);
assert_eq!(step.top.len(), 2);
assert_eq!(step.top[0].token_id, 1);
assert_eq!(step.top[1].token_id, 3);
assert!((step.top[0].logprob - step.logprob).abs() < 1e-6);
let total_prob: f32 = step.top.iter().map(|item| item.logprob.exp()).sum();
assert!(total_prob < 1.0 + 1e-5);
}
#[test]
fn step_logprobs_without_top_alternatives() {
let logits = [0.0f32, 1.0];
let mut scratch = SamplingScratch::with_vocab(logits.len());
let step = compute_step_logprobs(&logits, 0, 0, &mut scratch);
assert_eq!(step.token_id, 0);
assert!(step.top.is_empty());
assert!(step.logprob < 0.0);
}
#[test]
fn generate_tokens_returns_chosen_and_top_logprobs() {
use crate::model::{CausalLM, CausalLmConfig};
let model = CausalLM::new(CausalLmConfig {
vocab_size: 16,
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 8,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: 15,
seed: 11,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
let config = SamplingConfig {
max_new_tokens: 3,
temperature: 0.0,
top_k: None,
top_p: 1.0,
repetition_penalty: 1.0,
return_logprobs: true,
top_logprobs: 3,
..Default::default()
};
let output = super::generate_tokens(&model, &[1, 2, 3], &config, None).unwrap();
assert_eq!(output.token_ids.len(), 3);
let logprobs = output.logprobs.expect("logprobs enabled");
assert_eq!(logprobs.len(), 3);
for (token_id, step) in output.token_ids.iter().zip(&logprobs) {
assert_eq!(step.token_id, *token_id);
assert_eq!(step.top.len(), 3);
assert!(step.top.windows(2).all(|pair| pair[0].logprob >= pair[1].logprob));
assert!(step.top.iter().any(|item| item.token_id == *token_id));
}
}
#[test]
fn presence_penalty_lowers_seen_tokens_once() {
let config = SamplingConfig {
temperature: 1.0,
top_k: None,
top_p: 1.0,
repetition_penalty: 1.0,
presence_penalty: 2.0,
frequency_penalty: 0.0,
..Default::default()
};
let filtered = filter_logits(&[5.0, 5.0, 5.0], &[0, 0, 1], &config);
assert!((filtered[0] - 3.0).abs() < 1e-6);
assert!((filtered[1] - 3.0).abs() < 1e-6);
assert!((filtered[2] - 5.0).abs() < 1e-6);
}
#[test]
fn frequency_penalty_scales_with_count() {
let config = SamplingConfig {
temperature: 1.0,
top_k: None,
top_p: 1.0,
repetition_penalty: 1.0,
presence_penalty: 0.0,
frequency_penalty: 0.5,
..Default::default()
};
let filtered = filter_logits(&[5.0, 5.0, 5.0], &[0, 0, 1], &config);
assert!((filtered[0] - 4.0).abs() < 1e-6);
assert!((filtered[1] - 4.5).abs() < 1e-6);
assert!((filtered[2] - 5.0).abs() < 1e-6);
}
#[test]
fn repetition_penalty_window_limits_history() {
let config = SamplingConfig {
top_k: None,
top_p: 1.0,
temperature: 0.0,
repetition_penalty: 2.0,
repetition_penalty_window: Some(1),
..Default::default()
};
let filtered = filter_logits(&[4.0, 4.0], &[0, 1], &config);
assert!((filtered[0] - 4.0).abs() < 1e-6);
assert!((filtered[1] - 2.0).abs() < 1e-6);
}
fn tiny_model(seed: u64) -> CausalLM {
CausalLM::new(CausalLmConfig {
vocab_size: 16,
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 16,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: 15,
seed,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap()
}
#[test]
fn kv_cache_concat_split_round_trip() {
let model = tiny_model(3);
let mut left = KvCache::new(model.layers.len());
let mut right = KvCache::new(model.layers.len());
let _ = model
.forward_cached(&IndexTensor::new(vec![1, 2, 3], vec![1, 3]), &mut left)
.unwrap();
let _ = model
.forward_cached(&IndexTensor::new(vec![4, 5, 6], vec![1, 3]), &mut right)
.unwrap();
let packed = KvCache::concat_batch(&[&left, &right]).unwrap();
assert_eq!(packed.batch_size().unwrap(), 2);
assert_eq!(packed.position(), 3);
let split = packed.split_batch().unwrap();
assert_eq!(split.len(), 2);
for (original, restored) in [&left, &right].iter().zip(&split) {
assert_eq!(original.position(), restored.position());
for (left_layer, right_layer) in original.layers.iter().zip(&restored.layers) {
let left_layer = left_layer.as_ref().unwrap();
let right_layer = right_layer.as_ref().unwrap();
assert_eq!(left_layer.key.data(), right_layer.key.data());
assert_eq!(left_layer.value.data(), right_layer.value.data());
}
}
}
#[test]
fn continuous_batch_matches_independent_generate() {
let model = tiny_model(11);
let tokenizer = crate::tokenizer::BpeTokenizer::new();
let prompts: [&[usize]; 3] = [&[1, 2, 3], &[4, 5, 6], &[7, 8]];
let configs = [
SamplingConfig {
max_new_tokens: 4,
temperature: 0.0,
top_k: None,
top_p: 1.0,
repetition_penalty: 1.0,
seed: 1,
..Default::default()
},
SamplingConfig {
max_new_tokens: 3,
temperature: 0.0,
top_k: None,
top_p: 1.0,
repetition_penalty: 1.0,
seed: 2,
..Default::default()
},
SamplingConfig {
max_new_tokens: 5,
temperature: 0.0,
top_k: None,
top_p: 1.0,
repetition_penalty: 1.0,
seed: 3,
..Default::default()
},
];
let expected: Vec<Vec<usize>> = prompts
.iter()
.zip(&configs)
.map(|(prompt, config)| {
super::generate_tokens(&model, prompt, config, None)
.unwrap()
.token_ids
})
.collect();
let mut batcher = ContinuousBatcher::new(&model, &tokenizer, 2);
let actual = batcher.run_token_ids(&prompts, &configs).unwrap();
assert_eq!(
actual
.iter()
.map(|output| output.token_ids.clone())
.collect::<Vec<_>>(),
expected
);
}
#[test]
fn continuous_batch_drops_finished_and_admits_waiting() {
let model = tiny_model(19);
let tokenizer = crate::tokenizer::BpeTokenizer::new();
let mut batcher = ContinuousBatcher::new(&model, &tokenizer, 2);
let short = SamplingConfig {
max_new_tokens: 1,
temperature: 0.0,
top_k: None,
top_p: 1.0,
repetition_penalty: 1.0,
seed: 1,
..Default::default()
};
let longer = SamplingConfig {
max_new_tokens: 3,
temperature: 0.0,
top_k: None,
top_p: 1.0,
repetition_penalty: 1.0,
seed: 2,
..Default::default()
};
batcher.submit_token_ids(&[1, 2], &short).unwrap();
batcher.submit_token_ids(&[3, 4], &longer).unwrap();
batcher.submit_token_ids(&[5, 6], &longer).unwrap();
assert_eq!(batcher.active_len(), 2);
assert_eq!(batcher.waiting_len(), 1);
let events = batcher.step().unwrap();
assert_eq!(events.len(), 2);
assert!(events.iter().any(|event| event.finished));
assert_eq!(batcher.active_len(), 2);
assert_eq!(batcher.waiting_len(), 0);
while !batcher.is_idle() {
batcher.step().unwrap();
}
let finished = batcher.take_finished();
assert_eq!(finished.len(), 3);
}
#[test]
fn batched_decode_logits_match_single_request() {
let model = tiny_model(23);
let prompt_a = IndexTensor::new(vec![1, 2, 3], vec![1, 3]);
let prompt_b = IndexTensor::new(vec![4, 5, 6], vec![1, 3]);
let mut cache_a = KvCache::new(model.layers.len());
let mut cache_b = KvCache::new(model.layers.len());
let logits_a = model.forward_cached(&prompt_a, &mut cache_a).unwrap();
let logits_b = model.forward_cached(&prompt_b, &mut cache_b).unwrap();
let mut packed = KvCache::concat_batch(&[&cache_a, &cache_b]).unwrap();
let decode = IndexTensor::new(vec![7, 8], vec![2, 1]);
let batched = model.forward_cached(&decode, &mut packed).unwrap().data();
let single_a = model
.forward_cached(&IndexTensor::new(vec![7], vec![1, 1]), &mut cache_a)
.unwrap()
.data();
let single_b = model
.forward_cached(&IndexTensor::new(vec![8], vec![1, 1]), &mut cache_b)
.unwrap()
.data();
let vocab = model.config.vocab_size;
assert_eq!(logits_a.shape()[2], vocab);
assert_eq!(logits_b.shape()[2], vocab);
for index in 0..vocab {
assert!((batched[index] - single_a[index]).abs() < 1e-5);
assert!((batched[vocab + index] - single_b[index]).abs() < 1e-5);
}
}
#[test]
fn kv_cache_truncate_keeps_prefix() {
let model = tiny_model(29);
let mut cache = KvCache::new(model.layers.len());
let _ = model
.forward_cached(&IndexTensor::new(vec![1, 2, 3, 4], vec![1, 4]), &mut cache)
.unwrap();
assert_eq!(cache.sequence_length(), 4);
cache.truncate(2).unwrap();
assert_eq!(cache.sequence_length(), 2);
assert_eq!(cache.position(), 2);
let mut reference = KvCache::new(model.layers.len());
let _ = model
.forward_cached(&IndexTensor::new(vec![1, 2], vec![1, 2]), &mut reference)
.unwrap();
for (left, right) in cache.layers.iter().zip(&reference.layers) {
let left = left.as_ref().unwrap();
let right = right.as_ref().unwrap();
assert_eq!(left.key.data(), right.key.data());
assert_eq!(left.value.data(), right.value.data());
}
}
fn smaller_draft(seed: u64) -> CausalLM {
CausalLM::new(CausalLmConfig {
vocab_size: 16,
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 8,
max_sequence_length: 16,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: 15,
seed,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap()
}
#[test]
fn speculative_greedy_matches_target_when_draft_is_identical() {
let model = tiny_model(31);
let config = SamplingConfig {
max_new_tokens: 6,
temperature: 0.0,
top_k: None,
top_p: 1.0,
repetition_penalty: 1.0,
seed: 7,
..Default::default()
};
let prompt = [1usize, 2, 3];
let baseline = super::generate_tokens(&model, &prompt, &config, None)
.unwrap()
.token_ids;
let speculative = SpeculativeConfig {
num_draft_tokens: 3,
};
let drafted = super::generate_speculative_tokens(
&model,
&model,
&prompt,
&config,
&speculative,
None,
)
.unwrap()
.token_ids;
assert_eq!(drafted, baseline);
}
#[test]
fn speculative_with_smaller_draft_emits_requested_length() {
let target = tiny_model(37);
let draft = smaller_draft(41);
let config = SamplingConfig {
max_new_tokens: 5,
temperature: 0.0,
top_k: None,
top_p: 1.0,
repetition_penalty: 1.0,
seed: 9,
..Default::default()
};
let output = super::generate_speculative_tokens(
&target,
&draft,
&[1, 2, 3],
&config,
&SpeculativeConfig {
num_draft_tokens: 4,
},
None,
)
.unwrap();
assert_eq!(output.token_ids.len(), 5);
assert!(output
.token_ids
.iter()
.all(|token| *token < target.config.vocab_size));
}
}