spec_stream_engine/
pipeline.rs1use crate::error::EngineError;
2use crate::model::ModelEngine;
3use crate::stream::{GenerationEvent, SpeculativeStream};
4use spec_stream_cache::PagedKvCacheManager;
5use spec_stream_core::{
6 DraftProposal, SpeculativeMetrics, SpeculativeVerifier, TokenId, VerificationStrategy,
7};
8use std::sync::Arc;
9use tokio::sync::mpsc;
10
11#[derive(Debug, Clone)]
13pub struct SpeculativeConfig {
14 pub k_draft_tokens: usize,
16 pub max_new_tokens: usize,
18 pub eos_token_id: Option<TokenId>,
20 pub strategy: VerificationStrategy,
22 pub adaptive_k: bool,
24 pub min_k: usize,
26 pub max_k: usize,
28}
29
30impl Default for SpeculativeConfig {
31 fn default() -> Self {
32 Self {
33 k_draft_tokens: 4,
34 max_new_tokens: 64,
35 eos_token_id: Some(2),
36 strategy: VerificationStrategy::Greedy,
37 adaptive_k: true,
38 min_k: 1,
39 max_k: 8,
40 }
41 }
42}
43
44pub struct SpeculativePipeline<D: ModelEngine, T: ModelEngine> {
46 draft_model: D,
47 target_model: T,
48 verifier: SpeculativeVerifier,
49 cache_manager: Arc<PagedKvCacheManager>,
50 config: SpeculativeConfig,
51}
52
53impl<D: ModelEngine + 'static, T: ModelEngine + 'static> SpeculativePipeline<D, T> {
54 pub fn new(
55 draft_model: D,
56 target_model: T,
57 cache_manager: Arc<PagedKvCacheManager>,
58 config: SpeculativeConfig,
59 ) -> Self {
60 let verifier = SpeculativeVerifier::new(config.strategy);
61 Self {
62 draft_model,
63 target_model,
64 verifier,
65 cache_manager,
66 config,
67 }
68 }
69
70 pub fn generate_stream(
72 mut self,
73 prompt: Vec<TokenId>,
74 sequence_id: u64,
75 ) -> Result<SpeculativeStream, EngineError> {
76 let (tx, rx) = mpsc::channel(32);
77
78 self.cache_manager.create_sequence(sequence_id);
79 self.cache_manager
80 .append_tokens(sequence_id, &prompt)
81 .map_err(EngineError::Cache)?;
82
83 tokio::spawn(async move {
84 let mut current_prefix = prompt;
85 let mut total_generated = 0;
86 let mut metrics = SpeculativeMetrics::new();
87 let mut current_k = self.config.k_draft_tokens;
88 let mut ema_acceptance: Option<f32> = None;
89
90 while total_generated < self.config.max_new_tokens {
91 let last_token = match current_prefix.last() {
92 Some(&t) => t,
93 None => break,
94 };
95
96 if let Some(eos) = self.config.eos_token_id {
98 if last_token == eos {
99 let _ = tx
100 .send(GenerationEvent::Finished {
101 total_tokens: total_generated,
102 reason: "EOS token reached".into(),
103 })
104 .await;
105 break;
106 }
107 }
108
109 let mut proposal = DraftProposal::new();
111 let mut draft_token = last_token;
112
113 for _ in 0..current_k {
114 let logits = match self.draft_model.forward_token(draft_token).await {
115 Ok(l) => l,
116 Err(e) => {
117 tracing::error!("Draft model error: {:?}", e);
118 break;
119 }
120 };
121
122 let next_draft_id = match logits.argmax() {
123 Ok(id) => id,
124 Err(_) => break,
125 };
126
127 let prob = logits
128 .softmax()
129 .ok()
130 .and_then(|p| p.get(next_draft_id as usize).copied())
131 .unwrap_or(0.9);
132
133 proposal.push(next_draft_id, prob);
134 draft_token = next_draft_id;
135 }
136
137 if proposal.is_empty() {
138 break;
139 }
140
141 let proposed_ids = proposal.token_ids();
143 let _ = self.cache_manager.append_tokens(sequence_id, &proposed_ids);
144
145 let mut target_inputs = Vec::with_capacity(proposed_ids.len() + 1);
147 target_inputs.push(last_token);
148 target_inputs.extend_from_slice(&proposed_ids);
149
150 let target_logits = match self.target_model.forward_batch(&target_inputs).await {
151 Ok(l) => l,
152 Err(e) => {
153 tracing::error!("Target model error: {:?}", e);
154 break;
155 }
156 };
157
158 let verif_result = match self.verifier.verify(&proposal, &target_logits) {
160 Ok(r) => r,
161 Err(e) => {
162 tracing::error!("Verification error: {:?}", e);
163 break;
164 }
165 };
166
167 let valid_tokens_to_keep = current_prefix.len() + verif_result.accepted_count;
170 let _ = self
171 .cache_manager
172 .rollback_sequence(sequence_id, valid_tokens_to_keep);
173
174 let _ = self
176 .cache_manager
177 .append_tokens(sequence_id, &[verif_result.emitted_target_token]);
178
179 metrics.record_cycle(
181 proposal.len(),
182 verif_result.accepted_count,
183 verif_result.total_tokens_produced,
184 );
185
186 for &tok in &verif_result.accepted_tokens {
188 current_prefix.push(tok);
189 total_generated += 1;
190 if tx
191 .send(GenerationEvent::Token {
192 token_id: tok,
193 is_draft: true,
194 })
195 .await
196 .is_err()
197 {
198 return;
199 }
200 }
201
202 current_prefix.push(verif_result.emitted_target_token);
204 total_generated += 1;
205 if tx
206 .send(GenerationEvent::Token {
207 token_id: verif_result.emitted_target_token,
208 is_draft: false,
209 })
210 .await
211 .is_err()
212 {
213 return;
214 }
215
216 let _ = tx
218 .send(GenerationEvent::Burst {
219 accepted: verif_result.accepted_count,
220 total_produced: verif_result.total_tokens_produced,
221 })
222 .await;
223
224 let current_acc_rate = if proposal.is_empty() {
226 0.0
227 } else {
228 verif_result.accepted_count as f32 / proposal.len() as f32
229 };
230
231 let ema = match ema_acceptance {
232 Some(prev) => 0.5 * current_acc_rate + 0.5 * prev,
233 None => current_acc_rate,
234 };
235 ema_acceptance = Some(ema);
236
237 if self.config.adaptive_k {
238 let old_k = current_k;
239 if ema >= 0.75 && current_k < self.config.max_k {
240 current_k += 1;
241 } else if ema <= 0.35 && current_k > self.config.min_k {
242 current_k -= 1;
243 }
244
245 if current_k != old_k {
246 let _ = tx
247 .send(GenerationEvent::AdaptiveKChanged {
248 old_k,
249 new_k: current_k,
250 acceptance_rate: ema,
251 })
252 .await;
253 }
254 }
255
256 let _ = tx.send(GenerationEvent::Metrics(metrics.clone())).await;
257
258 if total_generated >= self.config.max_new_tokens {
259 let _ = tx
260 .send(GenerationEvent::Finished {
261 total_tokens: total_generated,
262 reason: "Max new tokens reached".into(),
263 })
264 .await;
265 break;
266 }
267 }
268
269 let _ = self.cache_manager.remove_sequence(sequence_id);
271 });
272
273 Ok(SpeculativeStream::new(rx))
274 }
275}