spec_stream_engine/
chunked_prefill.rs1use crate::error::EngineError;
10use crate::model::ModelEngine;
11use spec_stream_cache::PagedKvCacheManager;
12use spec_stream_core::TokenId;
13use std::collections::VecDeque;
14use std::sync::Arc;
15
16#[derive(Debug, Clone)]
18pub struct ChunkedPrefillConfig {
19 pub chunk_size: usize,
21 pub max_concurrent_prefills: usize,
23}
24
25impl Default for ChunkedPrefillConfig {
26 fn default() -> Self {
27 Self {
28 chunk_size: 128,
29 max_concurrent_prefills: 4,
30 }
31 }
32}
33
34#[derive(Debug, Clone)]
36pub struct PrefillTask {
37 pub sequence_id: u64,
38 pub prompt: Vec<TokenId>,
39 pub processed_tokens: usize,
40}
41
42impl PrefillTask {
43 pub fn new(sequence_id: u64, prompt: Vec<TokenId>) -> Self {
44 Self {
45 sequence_id,
46 prompt,
47 processed_tokens: 0,
48 }
49 }
50
51 #[inline]
53 pub fn is_complete(&self) -> bool {
54 self.processed_tokens >= self.prompt.len()
55 }
56
57 #[inline]
59 pub fn remaining_tokens(&self) -> usize {
60 self.prompt.len().saturating_sub(self.processed_tokens)
61 }
62
63 #[inline]
65 pub fn progress(&self) -> f32 {
66 if self.prompt.is_empty() {
67 1.0
68 } else {
69 self.processed_tokens as f32 / self.prompt.len() as f32
70 }
71 }
72}
73
74pub struct ChunkedPrefillScheduler {
76 config: ChunkedPrefillConfig,
77 cache_manager: Arc<PagedKvCacheManager>,
78 pending_tasks: VecDeque<PrefillTask>,
79 completed_tasks: Vec<u64>,
80}
81
82impl ChunkedPrefillScheduler {
83 pub fn new(cache_manager: Arc<PagedKvCacheManager>, config: ChunkedPrefillConfig) -> Self {
84 Self {
85 config,
86 cache_manager,
87 pending_tasks: VecDeque::new(),
88 completed_tasks: Vec::new(),
89 }
90 }
91
92 pub fn submit_prompt(&mut self, sequence_id: u64, prompt: Vec<TokenId>) {
94 self.cache_manager.create_sequence(sequence_id);
95 self.pending_tasks
96 .push_back(PrefillTask::new(sequence_id, prompt));
97 }
98
99 pub fn pending_count(&self) -> usize {
101 self.pending_tasks.len()
102 }
103
104 pub async fn step_prefill<M: ModelEngine>(
107 &mut self,
108 model: &mut M,
109 ) -> Result<Option<(u64, usize, bool)>, EngineError> {
110 let mut task = match self.pending_tasks.pop_front() {
111 Some(t) => t,
112 None => return Ok(None),
113 };
114
115 if task.is_complete() {
116 self.completed_tasks.push(task.sequence_id);
117 return Ok(Some((task.sequence_id, 0, true)));
118 }
119
120 let start = task.processed_tokens;
121 let end = (start + self.config.chunk_size).min(task.prompt.len());
122 let chunk = &task.prompt[start..end];
123 let chunk_len = chunk.len();
124
125 self.cache_manager.append_tokens(task.sequence_id, chunk)?;
127
128 let _logits = model.forward_batch(chunk).await?;
130
131 task.processed_tokens += chunk_len;
132 let is_now_complete = task.is_complete();
133
134 if is_now_complete {
135 self.completed_tasks.push(task.sequence_id);
136 } else {
137 self.pending_tasks.push_back(task.clone());
139 }
140
141 Ok(Some((task.sequence_id, chunk_len, is_now_complete)))
142 }
143
144 pub fn take_completed_sequences(&mut self) -> Vec<u64> {
146 std::mem::take(&mut self.completed_tasks)
147 }
148}
149
150#[cfg(test)]
151mod tests {
152 use super::*;
153 use crate::model::SyntheticModel;
154
155 #[tokio::test]
156 async fn test_chunked_prefill_step_execution() {
157 let cache = Arc::new(PagedKvCacheManager::with_defaults(64));
158 let config = ChunkedPrefillConfig {
159 chunk_size: 10,
160 max_concurrent_prefills: 2,
161 };
162
163 let mut scheduler = ChunkedPrefillScheduler::new(cache.clone(), config);
164 let mut model = SyntheticModel::new("test-model", 100, 0.9, 42);
165
166 let prompt: Vec<TokenId> = (1..=25).collect();
168 scheduler.submit_prompt(501, prompt);
169
170 assert_eq!(scheduler.pending_count(), 1);
171
172 let step1 = scheduler.step_prefill(&mut model).await.unwrap().unwrap();
174 assert_eq!(step1, (501, 10, false));
175
176 let step2 = scheduler.step_prefill(&mut model).await.unwrap().unwrap();
178 assert_eq!(step2, (501, 10, false));
179
180 let step3 = scheduler.step_prefill(&mut model).await.unwrap().unwrap();
182 assert_eq!(step3, (501, 5, true));
183
184 let completed = scheduler.take_completed_sequences();
185 assert_eq!(completed, vec![501]);
186 assert_eq!(scheduler.pending_count(), 0);
187 }
188}