Skip to main content

spec_stream_engine/
lib.rs

1//! # spec-stream-engine
2//!
3//! Speculative decoding execution engine and streaming orchestrator.
4//! Combines draft models, target models, paged KV-cache rollbacks, and verification
5//! into an asynchronous token generation pipeline.
6
7pub mod chunked_prefill;
8pub mod error;
9pub mod model;
10pub mod pipeline;
11pub mod prompt_lookup;
12pub mod quantization;
13pub mod scheduler;
14pub mod stream;
15pub mod tree_pipeline;
16pub mod weights;
17
18pub use chunked_prefill::{ChunkedPrefillConfig, ChunkedPrefillScheduler, PrefillTask};
19pub use error::EngineError;
20pub use model::{ModelEngine, SyntheticModel};
21pub use pipeline::{SpeculativeConfig, SpeculativePipeline};
22pub use prompt_lookup::{PromptLookupConfig, PromptLookupEngine};
23pub use quantization::{
24    dequantize_q4_0, dequantize_q8_0, f16_to_f32, f32_to_f16, QuantType, QuantizedLinearModel,
25};
26pub use scheduler::{ContinuousBatchScheduler, QueuedRequest, SchedulerStats};
27pub use stream::{GenerationEvent, SpeculativeStream};
28pub use tree_pipeline::{TreeSpeculativeConfig, TreeSpeculativePipeline};
29pub use weights::{
30    GgufMetadata, GgufTensorInfo, GgufValue, LinearProjectionModel, SafeTensorEntry,
31    SafeTensorsMetadata,
32};
33
34#[cfg(test)]
35mod tests {
36    use super::*;
37    use spec_stream_cache::PagedKvCacheManager;
38    use spec_stream_core::VerificationStrategy;
39    use std::sync::Arc;
40    use tokio_stream::StreamExt;
41
42    #[tokio::test]
43    async fn test_speculative_pipeline_execution() {
44        let vocab_size = 50;
45        // Draft model agrees with target ~70% of the time
46        let draft = SyntheticModel::new("draft-small", vocab_size, 0.7, 42);
47        let target = SyntheticModel::new("target-large", vocab_size, 0.95, 1337);
48
49        let cache = Arc::new(PagedKvCacheManager::with_defaults(64));
50        let config = SpeculativeConfig {
51            k_draft_tokens: 3,
52            max_new_tokens: 12,
53            eos_token_id: None,
54            strategy: VerificationStrategy::Greedy,
55            ..Default::default()
56        };
57
58        let pipeline = SpeculativePipeline::new(draft, target, cache, config);
59        let mut stream = pipeline.generate_stream(vec![1, 2], 100).unwrap();
60
61        let mut token_count = 0;
62        let mut finished = false;
63
64        while let Some(event) = stream.next().await {
65            match event {
66                GenerationEvent::Token { .. } => {
67                    token_count += 1;
68                }
69                GenerationEvent::Finished { .. } => {
70                    finished = true;
71                }
72                _ => {}
73            }
74        }
75
76        assert!(token_count >= 12);
77        assert!(finished);
78    }
79}