Skip to main content

spec_stream_engine/
chunked_prefill.rs

1//! # Chunked Prefill Scheduler
2//!
3//! Provides chunked prefill scheduling to prevent head-of-line blocking and memory spikes
4//! during long context ingestion.
5//!
6//! Large prompts (e.g. 2k–8k tokens) are segmented into configurable chunks (e.g. 128–512 tokens),
7//! interleaving prompt KV-cache prefilling with active speculative decoding bursts.
8
9use crate::error::EngineError;
10use crate::model::ModelEngine;
11use spec_stream_cache::PagedKvCacheManager;
12use spec_stream_core::TokenId;
13use std::collections::VecDeque;
14use std::sync::Arc;
15
16/// Configuration for chunked prefill processing.
17#[derive(Debug, Clone)]
18pub struct ChunkedPrefillConfig {
19    /// Maximum number of prompt tokens processed in a single prefill step (default: 128).
20    pub chunk_size: usize,
21    /// Maximum number of concurrent prompts in prefill stage (default: 4).
22    pub max_concurrent_prefills: usize,
23}
24
25impl Default for ChunkedPrefillConfig {
26    fn default() -> Self {
27        Self {
28            chunk_size: 128,
29            max_concurrent_prefills: 4,
30        }
31    }
32}
33
34/// An active prompt ingestion task being processed in chunks.
35#[derive(Debug, Clone)]
36pub struct PrefillTask {
37    pub sequence_id: u64,
38    pub prompt: Vec<TokenId>,
39    pub processed_tokens: usize,
40}
41
42impl PrefillTask {
43    pub fn new(sequence_id: u64, prompt: Vec<TokenId>) -> Self {
44        Self {
45            sequence_id,
46            prompt,
47            processed_tokens: 0,
48        }
49    }
50
51    /// Whether all prompt tokens have been completely ingested into the KV-cache.
52    #[inline]
53    pub fn is_complete(&self) -> bool {
54        self.processed_tokens >= self.prompt.len()
55    }
56
57    /// Number of remaining tokens left to prefill.
58    #[inline]
59    pub fn remaining_tokens(&self) -> usize {
60        self.prompt.len().saturating_sub(self.processed_tokens)
61    }
62
63    /// Progress ratio between 0.0 and 1.0.
64    #[inline]
65    pub fn progress(&self) -> f32 {
66        if self.prompt.is_empty() {
67            1.0
68        } else {
69            self.processed_tokens as f32 / self.prompt.len() as f32
70        }
71    }
72}
73
74/// Chunked prefill orchestrator.
75pub struct ChunkedPrefillScheduler {
76    config: ChunkedPrefillConfig,
77    cache_manager: Arc<PagedKvCacheManager>,
78    pending_tasks: VecDeque<PrefillTask>,
79    completed_tasks: Vec<u64>,
80}
81
82impl ChunkedPrefillScheduler {
83    pub fn new(cache_manager: Arc<PagedKvCacheManager>, config: ChunkedPrefillConfig) -> Self {
84        Self {
85            config,
86            cache_manager,
87            pending_tasks: VecDeque::new(),
88            completed_tasks: Vec::new(),
89        }
90    }
91
92    /// Submits a new prompt sequence for chunked prefill.
93    pub fn submit_prompt(&mut self, sequence_id: u64, prompt: Vec<TokenId>) {
94        self.cache_manager.create_sequence(sequence_id);
95        self.pending_tasks
96            .push_back(PrefillTask::new(sequence_id, prompt));
97    }
98
99    /// Number of sequences currently undergoing chunked prefill.
100    pub fn pending_count(&self) -> usize {
101        self.pending_tasks.len()
102    }
103
104    /// Executes one chunked prefill step for the front task using the target model.
105    /// Returns the number of tokens prefilled in this step, and whether the sequence completed.
106    pub async fn step_prefill<M: ModelEngine>(
107        &mut self,
108        model: &mut M,
109    ) -> Result<Option<(u64, usize, bool)>, EngineError> {
110        let mut task = match self.pending_tasks.pop_front() {
111            Some(t) => t,
112            None => return Ok(None),
113        };
114
115        if task.is_complete() {
116            self.completed_tasks.push(task.sequence_id);
117            return Ok(Some((task.sequence_id, 0, true)));
118        }
119
120        let start = task.processed_tokens;
121        let end = (start + self.config.chunk_size).min(task.prompt.len());
122        let chunk = &task.prompt[start..end];
123        let chunk_len = chunk.len();
124
125        // Append this chunk to the Paged KV-Cache
126        self.cache_manager.append_tokens(task.sequence_id, chunk)?;
127
128        // Execute batch forward pass for this prefill chunk
129        let _logits = model.forward_batch(chunk).await?;
130
131        task.processed_tokens += chunk_len;
132        let is_now_complete = task.is_complete();
133
134        if is_now_complete {
135            self.completed_tasks.push(task.sequence_id);
136        } else {
137            // Re-enqueue task for subsequent chunk processing
138            self.pending_tasks.push_back(task.clone());
139        }
140
141        Ok(Some((task.sequence_id, chunk_len, is_now_complete)))
142    }
143
144    /// Drains IDs of sequences that finished their prefill stage and are ready for speculative decoding.
145    pub fn take_completed_sequences(&mut self) -> Vec<u64> {
146        std::mem::take(&mut self.completed_tasks)
147    }
148}
149
150#[cfg(test)]
151mod tests {
152    use super::*;
153    use crate::model::SyntheticModel;
154
155    #[tokio::test]
156    async fn test_chunked_prefill_step_execution() {
157        let cache = Arc::new(PagedKvCacheManager::with_defaults(64));
158        let config = ChunkedPrefillConfig {
159            chunk_size: 10,
160            max_concurrent_prefills: 2,
161        };
162
163        let mut scheduler = ChunkedPrefillScheduler::new(cache.clone(), config);
164        let mut model = SyntheticModel::new("test-model", 100, 0.9, 42);
165
166        // Prompt of 25 tokens -> chunks of 10, 10, 5
167        let prompt: Vec<TokenId> = (1..=25).collect();
168        scheduler.submit_prompt(501, prompt);
169
170        assert_eq!(scheduler.pending_count(), 1);
171
172        // Step 1: 10 tokens
173        let step1 = scheduler.step_prefill(&mut model).await.unwrap().unwrap();
174        assert_eq!(step1, (501, 10, false));
175
176        // Step 2: 10 tokens
177        let step2 = scheduler.step_prefill(&mut model).await.unwrap().unwrap();
178        assert_eq!(step2, (501, 10, false));
179
180        // Step 3: 5 tokens -> complete!
181        let step3 = scheduler.step_prefill(&mut model).await.unwrap().unwrap();
182        assert_eq!(step3, (501, 5, true));
183
184        let completed = scheduler.take_completed_sequences();
185        assert_eq!(completed, vec![501]);
186        assert_eq!(scheduler.pending_count(), 0);
187    }
188}