Skip to main content

spec_stream_engine/
pipeline.rs

1use crate::error::EngineError;
2use crate::model::ModelEngine;
3use crate::stream::{GenerationEvent, SpeculativeStream};
4use spec_stream_cache::PagedKvCacheManager;
5use spec_stream_core::{
6    DraftProposal, SpeculativeMetrics, SpeculativeVerifier, TokenId, VerificationStrategy,
7};
8use std::sync::Arc;
9use tokio::sync::mpsc;
10
11/// Configuration parameters for speculative decoding.
12#[derive(Debug, Clone)]
13pub struct SpeculativeConfig {
14    /// Number of speculative candidate tokens the draft model proposes per cycle (typically 3..6).
15    pub k_draft_tokens: usize,
16    /// Maximum new tokens to generate before stopping.
17    pub max_new_tokens: usize,
18    /// Optional End-of-Sequence token ID to terminate generation.
19    pub eos_token_id: Option<TokenId>,
20    /// Verification strategy: Greedy or Stochastic with temperature.
21    pub strategy: VerificationStrategy,
22    /// Whether to dynamically adjust K draft tokens based on empirical acceptance rate.
23    pub adaptive_k: bool,
24    /// Minimum allowed K when adaptive_k is enabled (default 1).
25    pub min_k: usize,
26    /// Maximum allowed K when adaptive_k is enabled (default 8).
27    pub max_k: usize,
28}
29
30impl Default for SpeculativeConfig {
31    fn default() -> Self {
32        Self {
33            k_draft_tokens: 4,
34            max_new_tokens: 64,
35            eos_token_id: Some(2),
36            strategy: VerificationStrategy::Greedy,
37            adaptive_k: true,
38            min_k: 1,
39            max_k: 8,
40        }
41    }
42}
43
44/// Orchestrator for speculative decoding execution and streaming.
45pub struct SpeculativePipeline<D: ModelEngine, T: ModelEngine> {
46    draft_model: D,
47    target_model: T,
48    verifier: SpeculativeVerifier,
49    cache_manager: Arc<PagedKvCacheManager>,
50    config: SpeculativeConfig,
51}
52
53impl<D: ModelEngine + 'static, T: ModelEngine + 'static> SpeculativePipeline<D, T> {
54    pub fn new(
55        draft_model: D,
56        target_model: T,
57        cache_manager: Arc<PagedKvCacheManager>,
58        config: SpeculativeConfig,
59    ) -> Self {
60        let verifier = SpeculativeVerifier::new(config.strategy);
61        Self {
62            draft_model,
63            target_model,
64            verifier,
65            cache_manager,
66            config,
67        }
68    }
69
70    /// Starts speculative generation from a prompt and returns an asynchronous stream of events.
71    pub fn generate_stream(
72        mut self,
73        prompt: Vec<TokenId>,
74        sequence_id: u64,
75    ) -> Result<SpeculativeStream, EngineError> {
76        let (tx, rx) = mpsc::channel(32);
77
78        self.cache_manager.create_sequence(sequence_id);
79        self.cache_manager
80            .append_tokens(sequence_id, &prompt)
81            .map_err(EngineError::Cache)?;
82
83        tokio::spawn(async move {
84            let mut current_prefix = prompt;
85            let mut total_generated = 0;
86            let mut metrics = SpeculativeMetrics::new();
87            let mut current_k = self.config.k_draft_tokens;
88            let mut ema_acceptance: Option<f32> = None;
89
90            while total_generated < self.config.max_new_tokens {
91                let last_token = match current_prefix.last() {
92                    Some(&t) => t,
93                    None => break,
94                };
95
96                // Check EOS
97                if let Some(eos) = self.config.eos_token_id {
98                    if last_token == eos {
99                        let _ = tx
100                            .send(GenerationEvent::Finished {
101                                total_tokens: total_generated,
102                                reason: "EOS token reached".into(),
103                            })
104                            .await;
105                        break;
106                    }
107                }
108
109                // 1. DRAFT PHASE: Generate K speculative tokens sequentially using draft model
110                let mut proposal = DraftProposal::new();
111                let mut draft_token = last_token;
112
113                for _ in 0..current_k {
114                    let logits = match self.draft_model.forward_token(draft_token).await {
115                        Ok(l) => l,
116                        Err(e) => {
117                            tracing::error!("Draft model error: {:?}", e);
118                            break;
119                        }
120                    };
121
122                    let next_draft_id = match logits.argmax() {
123                        Ok(id) => id,
124                        Err(_) => break,
125                    };
126
127                    let prob = logits
128                        .softmax()
129                        .ok()
130                        .and_then(|p| p.get(next_draft_id as usize).copied())
131                        .unwrap_or(0.9);
132
133                    proposal.push(next_draft_id, prob);
134                    draft_token = next_draft_id;
135                }
136
137                if proposal.is_empty() {
138                    break;
139                }
140
141                // Append proposed draft tokens to KV-cache
142                let proposed_ids = proposal.token_ids();
143                let _ = self.cache_manager.append_tokens(sequence_id, &proposed_ids);
144
145                // 2. TARGET PHASE: Target model evaluates all proposed positions in ONE parallel forward pass
146                let mut target_inputs = Vec::with_capacity(proposed_ids.len() + 1);
147                target_inputs.push(last_token);
148                target_inputs.extend_from_slice(&proposed_ids);
149
150                let target_logits = match self.target_model.forward_batch(&target_inputs).await {
151                    Ok(l) => l,
152                    Err(e) => {
153                        tracing::error!("Target model error: {:?}", e);
154                        break;
155                    }
156                };
157
158                // 3. VERIFICATION PHASE: Accept valid draft tokens, reject mismatches
159                let verif_result = match self.verifier.verify(&proposal, &target_logits) {
160                    Ok(r) => r,
161                    Err(e) => {
162                        tracing::error!("Verification error: {:?}", e);
163                        break;
164                    }
165                };
166
167                // 4. KV-CACHE COMMIT & ROLLBACK
168                // Keep only valid tokens in the cache table; roll back unaccepted proposals
169                let valid_tokens_to_keep = current_prefix.len() + verif_result.accepted_count;
170                let _ = self
171                    .cache_manager
172                    .rollback_sequence(sequence_id, valid_tokens_to_keep);
173
174                // Commit the target model's emitted token (correction or bonus) to cache
175                let _ = self
176                    .cache_manager
177                    .append_tokens(sequence_id, &[verif_result.emitted_target_token]);
178
179                // 5. UPDATE METRICS & EMIT STREAM EVENTS
180                metrics.record_cycle(
181                    proposal.len(),
182                    verif_result.accepted_count,
183                    verif_result.total_tokens_produced,
184                );
185
186                // Emit accepted draft tokens
187                for &tok in &verif_result.accepted_tokens {
188                    current_prefix.push(tok);
189                    total_generated += 1;
190                    if tx
191                        .send(GenerationEvent::Token {
192                            token_id: tok,
193                            is_draft: true,
194                        })
195                        .await
196                        .is_err()
197                    {
198                        return;
199                    }
200                }
201
202                // Emit the target-emitted token (correction or bonus)
203                current_prefix.push(verif_result.emitted_target_token);
204                total_generated += 1;
205                if tx
206                    .send(GenerationEvent::Token {
207                        token_id: verif_result.emitted_target_token,
208                        is_draft: false,
209                    })
210                    .await
211                    .is_err()
212                {
213                    return;
214                }
215
216                // Emit burst event and updated metrics
217                let _ = tx
218                    .send(GenerationEvent::Burst {
219                        accepted: verif_result.accepted_count,
220                        total_produced: verif_result.total_tokens_produced,
221                    })
222                    .await;
223
224                // 6. ADAPTIVE K CALCULATION (if enabled)
225                let current_acc_rate = if proposal.is_empty() {
226                    0.0
227                } else {
228                    verif_result.accepted_count as f32 / proposal.len() as f32
229                };
230
231                let ema = match ema_acceptance {
232                    Some(prev) => 0.5 * current_acc_rate + 0.5 * prev,
233                    None => current_acc_rate,
234                };
235                ema_acceptance = Some(ema);
236
237                if self.config.adaptive_k {
238                    let old_k = current_k;
239                    if ema >= 0.75 && current_k < self.config.max_k {
240                        current_k += 1;
241                    } else if ema <= 0.35 && current_k > self.config.min_k {
242                        current_k -= 1;
243                    }
244
245                    if current_k != old_k {
246                        let _ = tx
247                            .send(GenerationEvent::AdaptiveKChanged {
248                                old_k,
249                                new_k: current_k,
250                                acceptance_rate: ema,
251                            })
252                            .await;
253                    }
254                }
255
256                let _ = tx.send(GenerationEvent::Metrics(metrics.clone())).await;
257
258                if total_generated >= self.config.max_new_tokens {
259                    let _ = tx
260                        .send(GenerationEvent::Finished {
261                            total_tokens: total_generated,
262                            reason: "Max new tokens reached".into(),
263                        })
264                        .await;
265                    break;
266                }
267            }
268
269            // Cleanup sequence cache resources
270            let _ = self.cache_manager.remove_sequence(sequence_id);
271        });
272
273        Ok(SpeculativeStream::new(rx))
274    }
275}