Skip to main content

spec_stream_engine/
model.rs

1use crate::error::EngineError;
2use rand::{rngs::StdRng, Rng, SeedableRng};
3use spec_stream_core::{Logits, TokenId};
4use std::future::Future;
5
6/// Abstract inference engine interface for draft and target models.
7/// Compatible with Candle, Burn, ONNX Runtime, or custom model runtimes.
8pub trait ModelEngine: Send + Sync {
9    /// Computes logits for a sequence of tokens in a single parallel forward pass.
10    fn forward_batch(
11        &mut self,
12        tokens: &[TokenId],
13    ) -> impl Future<Output = Result<Vec<Logits>, EngineError>> + Send;
14
15    /// Computes logits for a single token step.
16    fn forward_token(
17        &mut self,
18        token: TokenId,
19    ) -> impl Future<Output = Result<Logits, EngineError>> + Send;
20}
21
22/// A deterministic synthetic model for benchmarking, testing, and pipeline demonstrations.
23/// Simulates model predictions with a controllable agreement/acceptance probability.
24#[derive(Debug, Clone)]
25pub struct SyntheticModel {
26    pub name: String,
27    pub vocab_size: usize,
28    pub agreement_rate: f32,
29    pub step_counter: usize,
30    rng: StdRng,
31}
32
33impl SyntheticModel {
34    pub fn new(name: impl Into<String>, vocab_size: usize, agreement_rate: f32, seed: u64) -> Self {
35        Self {
36            name: name.into(),
37            vocab_size,
38            agreement_rate,
39            step_counter: 0,
40            rng: StdRng::seed_from_u64(seed),
41        }
42    }
43
44    fn generate_logits_for(&mut self, expected_token: TokenId) -> Logits {
45        let mut vals = vec![0.0f32; self.vocab_size];
46        let will_agree = self.rng.gen::<f32>() <= self.agreement_rate;
47
48        let target = if will_agree {
49            expected_token as usize % self.vocab_size
50        } else {
51            self.rng.gen_range(0..self.vocab_size)
52        };
53
54        // Give the target token a prominent peak
55        vals[target] = 8.0;
56
57        // Add minor Gaussian-like noise to others
58        for (i, v) in vals.iter_mut().enumerate() {
59            if i != target {
60                *v = self.rng.gen_range(0.0..1.5);
61            }
62        }
63
64        Logits::new(vals)
65    }
66}
67
68impl ModelEngine for SyntheticModel {
69    async fn forward_batch(&mut self, tokens: &[TokenId]) -> Result<Vec<Logits>, EngineError> {
70        self.step_counter += 1;
71        let mut results = Vec::with_capacity(tokens.len() + 1);
72
73        for &t in tokens {
74            // Predict candidate following token t
75            results.push(self.generate_logits_for(t + 1));
76        }
77
78        // The bonus prediction at the end
79        let last_token = tokens.last().copied().unwrap_or(1);
80        results.push(self.generate_logits_for(last_token + 2));
81
82        Ok(results)
83    }
84
85    async fn forward_token(&mut self, token: TokenId) -> Result<Logits, EngineError> {
86        self.step_counter += 1;
87        Ok(self.generate_logits_for(token + 1))
88    }
89}