Skip to main content

spec_stream/
lib.rs

1//! # spec-stream
2//!
3//! **spec-stream** is an ultra-fast, modular, model-agnostic speculative decoding and
4//! paged KV-cache engine in pure Rust.
5//!
6//! Accelerates Large Language Model (LLM) inference by 2x–3x through speculative token burst
7//! verification and $O(1)$ rollback paged memory allocation.
8//!
9//! ## Key Capabilities
10//! - **Mathematical Exactness**: Implements Leviathan et al. (2023) stochastic rejection sampling,
11//!   guaranteeing output probability distribution identical to the target model.
12//! - **Paged KV-Cache with O(1) Rollback**: Physical-to-virtual block tables prevent memory fragmentation
13//!   and allow instant rewind of rejected speculative proposals.
14//! - **Model Agnostic**: Plug in any inference backend: Candle, Burn, ONNX Runtime, or native C/CUDA FFI.
15//! - **Async Token Streaming**: First-class support for Tokio streams and Server-Sent Events (SSE).
16
17pub use spec_stream_core as core;
18
19#[cfg(feature = "cache")]
20pub use spec_stream_cache as cache;
21
22#[cfg(feature = "engine")]
23pub use spec_stream_engine as engine;
24
25/// Convenient prelude re-exporting core traits and common structs.
26pub mod prelude {
27    pub use spec_stream_core::{
28        ByteLevelTokenizer, DraftProposal, Logits, SamplingParams, SpeculativeMetrics,
29        SpeculativeTree, SpeculativeVerifier, TokenCandidate, TokenId, Tokenizer,
30        TreeVerificationResult, VerificationResult, VerificationStrategy,
31    };
32
33    #[cfg(feature = "cache")]
34    pub use spec_stream_cache::{
35        BlockAllocator, BlockTable, EvictionPolicy, LruEvictionPolicy, PagedKvCacheManager,
36        PhysicalBlock, PhysicalBlockId, PrefixCacheSnapshot, PrefixSnapshotEntry,
37    };
38
39    #[cfg(feature = "engine")]
40    pub use spec_stream_engine::{
41        ChunkedPrefillConfig, ChunkedPrefillScheduler, ContinuousBatchScheduler, GenerationEvent,
42        GgufMetadata, LinearProjectionModel, ModelEngine, PromptLookupConfig, PromptLookupEngine,
43        QuantType, QuantizedLinearModel, QueuedRequest, SafeTensorsMetadata, SchedulerStats,
44        SpeculativeConfig, SpeculativePipeline, SpeculativeStream, SyntheticModel,
45        TreeSpeculativeConfig, TreeSpeculativePipeline,
46    };
47}