use crate::error::EngineError;
use crate::model::ModelEngine;
use crate::stream::{GenerationEvent, SpeculativeStream};
use spec_stream_cache::PagedKvCacheManager;
use spec_stream_core::{SpeculativeMetrics, SpeculativeTree, TokenId};
use std::sync::Arc;
use tokio::sync::mpsc;
#[derive(Debug, Clone)]
pub struct TreeSpeculativeConfig {
pub branching_factor: usize,
pub depth: usize,
pub max_new_tokens: usize,
pub eos_token_id: Option<TokenId>,
}
impl Default for TreeSpeculativeConfig {
fn default() -> Self {
Self {
branching_factor: 2,
depth: 2,
max_new_tokens: 64,
eos_token_id: Some(2),
}
}
}
pub struct TreeSpeculativePipeline<D: ModelEngine, T: ModelEngine> {
draft_model: D,
target_model: T,
cache_manager: Arc<PagedKvCacheManager>,
config: TreeSpeculativeConfig,
}
impl<D: ModelEngine + 'static, T: ModelEngine + 'static> TreeSpeculativePipeline<D, T> {
pub fn new(
draft_model: D,
target_model: T,
cache_manager: Arc<PagedKvCacheManager>,
config: TreeSpeculativeConfig,
) -> Self {
Self {
draft_model,
target_model,
cache_manager,
config,
}
}
pub fn generate_stream(
mut self,
prompt: Vec<TokenId>,
sequence_id: u64,
) -> Result<SpeculativeStream, EngineError> {
let (tx, rx) = mpsc::channel(32);
self.cache_manager.create_sequence(sequence_id);
self.cache_manager
.append_tokens(sequence_id, &prompt)
.map_err(EngineError::Cache)?;
tokio::spawn(async move {
let mut current_prefix = prompt;
let mut total_generated = 0;
let mut metrics = SpeculativeMetrics::new();
while total_generated < self.config.max_new_tokens {
let last_token = match current_prefix.last() {
Some(&t) => t,
None => break,
};
if let Some(eos) = self.config.eos_token_id {
if last_token == eos {
let _ = tx
.send(GenerationEvent::Finished {
total_tokens: total_generated,
reason: "EOS token reached".into(),
})
.await;
break;
}
}
let mut tree = SpeculativeTree::new(last_token);
let mut current_level_nodes = vec![0usize];
for _d in 0..self.config.depth {
let mut next_level_nodes = Vec::new();
for &parent_node_idx in ¤t_level_nodes {
let parent_token = tree.nodes[parent_node_idx].token_id;
let logits = match self.draft_model.forward_token(parent_token).await {
Ok(l) => l,
Err(e) => {
tracing::error!("Draft model tree forward error: {:?}", e);
break;
}
};
let probs = logits.softmax().unwrap_or_default();
let mut indexed: Vec<(usize, f32)> =
probs.into_iter().enumerate().collect();
indexed.sort_by(|a, b| {
b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)
});
for &(candidate_id, prob) in
indexed.iter().take(self.config.branching_factor)
{
if let Ok(child_idx) =
tree.add_child(parent_node_idx, candidate_id as TokenId, prob)
{
next_level_nodes.push(child_idx);
}
}
}
if next_level_nodes.is_empty() {
break;
}
current_level_nodes = next_level_nodes;
}
if tree.nodes.len() <= 1 {
break;
}
let tree_tokens = tree.token_ids();
let _tree_mask = tree.compute_tree_attention_mask();
let target_logits = match self.target_model.forward_batch(&tree_tokens).await {
Ok(l) => l,
Err(e) => {
tracing::error!("Target model tree forward error: {:?}", e);
break;
}
};
let tree_result = match tree.verify_greedy(&target_logits) {
Ok(r) => r,
Err(e) => {
tracing::error!("Tree verification error: {:?}", e);
break;
}
};
let valid_tokens_to_keep = current_prefix.len() + tree_result.accepted_tokens.len();
let _ = self
.cache_manager
.rollback_sequence(sequence_id, valid_tokens_to_keep);
let _ = self
.cache_manager
.append_tokens(sequence_id, &[tree_result.emitted_target_token]);
let draft_nodes_count = tree.nodes.len() - 1;
metrics.record_cycle(
draft_nodes_count,
tree_result.accepted_tokens.len(),
tree_result.total_tokens_produced,
);
for &tok in &tree_result.accepted_tokens {
current_prefix.push(tok);
total_generated += 1;
if tx
.send(GenerationEvent::Token {
token_id: tok,
is_draft: true,
})
.await
.is_err()
{
return;
}
}
current_prefix.push(tree_result.emitted_target_token);
total_generated += 1;
if tx
.send(GenerationEvent::Token {
token_id: tree_result.emitted_target_token,
is_draft: false,
})
.await
.is_err()
{
return;
}
let _ = tx
.send(GenerationEvent::Burst {
accepted: tree_result.accepted_tokens.len(),
total_produced: tree_result.total_tokens_produced,
})
.await;
let _ = tx.send(GenerationEvent::Metrics(metrics.clone())).await;
if total_generated >= self.config.max_new_tokens {
let _ = tx
.send(GenerationEvent::Finished {
total_tokens: total_generated,
reason: "Max new tokens reached".into(),
})
.await;
break;
}
}
let _ = self.cache_manager.remove_sequence(sequence_id);
});
Ok(SpeculativeStream::new(rx))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::model::SyntheticModel;
use tokio_stream::StreamExt;
#[tokio::test]
async fn test_tree_speculative_pipeline() {
let vocab_size = 50;
let draft = SyntheticModel::new("draft-tree", vocab_size, 0.8, 42);
let target = SyntheticModel::new("target-tree", vocab_size, 0.95, 1337);
let cache = Arc::new(PagedKvCacheManager::with_defaults(64));
let config = TreeSpeculativeConfig {
branching_factor: 2,
depth: 2,
max_new_tokens: 10,
eos_token_id: None,
};
let pipeline = TreeSpeculativePipeline::new(draft, target, cache, config);
let mut stream = pipeline.generate_stream(vec![5, 6], 200).unwrap();
let mut token_count = 0;
while let Some(event) = stream.next().await {
if let GenerationEvent::Token { .. } = event {
token_count += 1;
}
}
assert!(token_count >= 10);
}
}