spec_stream_engine/
lib.rs1pub mod chunked_prefill;
8pub mod error;
9pub mod model;
10pub mod pipeline;
11pub mod prompt_lookup;
12pub mod quantization;
13pub mod scheduler;
14pub mod stream;
15pub mod tree_pipeline;
16pub mod weights;
17
18pub use chunked_prefill::{ChunkedPrefillConfig, ChunkedPrefillScheduler, PrefillTask};
19pub use error::EngineError;
20pub use model::{ModelEngine, SyntheticModel};
21pub use pipeline::{SpeculativeConfig, SpeculativePipeline};
22pub use prompt_lookup::{PromptLookupConfig, PromptLookupEngine};
23pub use quantization::{
24 dequantize_q4_0, dequantize_q8_0, f16_to_f32, f32_to_f16, QuantType, QuantizedLinearModel,
25};
26pub use scheduler::{ContinuousBatchScheduler, QueuedRequest, SchedulerStats};
27pub use stream::{GenerationEvent, SpeculativeStream};
28pub use tree_pipeline::{TreeSpeculativeConfig, TreeSpeculativePipeline};
29pub use weights::{
30 GgufMetadata, GgufTensorInfo, GgufValue, LinearProjectionModel, SafeTensorEntry,
31 SafeTensorsMetadata,
32};
33
34#[cfg(test)]
35mod tests {
36 use super::*;
37 use spec_stream_cache::PagedKvCacheManager;
38 use spec_stream_core::VerificationStrategy;
39 use std::sync::Arc;
40 use tokio_stream::StreamExt;
41
42 #[tokio::test]
43 async fn test_speculative_pipeline_execution() {
44 let vocab_size = 50;
45 let draft = SyntheticModel::new("draft-small", vocab_size, 0.7, 42);
47 let target = SyntheticModel::new("target-large", vocab_size, 0.95, 1337);
48
49 let cache = Arc::new(PagedKvCacheManager::with_defaults(64));
50 let config = SpeculativeConfig {
51 k_draft_tokens: 3,
52 max_new_tokens: 12,
53 eos_token_id: None,
54 strategy: VerificationStrategy::Greedy,
55 ..Default::default()
56 };
57
58 let pipeline = SpeculativePipeline::new(draft, target, cache, config);
59 let mut stream = pipeline.generate_stream(vec![1, 2], 100).unwrap();
60
61 let mut token_count = 0;
62 let mut finished = false;
63
64 while let Some(event) = stream.next().await {
65 match event {
66 GenerationEvent::Token { .. } => {
67 token_count += 1;
68 }
69 GenerationEvent::Finished { .. } => {
70 finished = true;
71 }
72 _ => {}
73 }
74 }
75
76 assert!(token_count >= 12);
77 assert!(finished);
78 }
79}