Expand description
§spec-stream-engine
Speculative decoding execution engine and streaming orchestrator. Combines draft models, target models, paged KV-cache rollbacks, and verification into an asynchronous token generation pipeline.
Re-exports§
pub use chunked_prefill::ChunkedPrefillConfig;pub use chunked_prefill::ChunkedPrefillScheduler;pub use chunked_prefill::PrefillTask;pub use error::EngineError;pub use model::ModelEngine;pub use model::SyntheticModel;pub use pipeline::SpeculativeConfig;pub use pipeline::SpeculativePipeline;pub use prompt_lookup::PromptLookupConfig;pub use prompt_lookup::PromptLookupEngine;pub use quantization::dequantize_q4_0;pub use quantization::dequantize_q8_0;pub use quantization::f16_to_f32;pub use quantization::f32_to_f16;pub use quantization::QuantType;pub use quantization::QuantizedLinearModel;pub use scheduler::ContinuousBatchScheduler;pub use scheduler::QueuedRequest;pub use scheduler::SchedulerStats;pub use stream::GenerationEvent;pub use stream::SpeculativeStream;pub use tree_pipeline::TreeSpeculativeConfig;pub use tree_pipeline::TreeSpeculativePipeline;pub use weights::GgufMetadata;pub use weights::GgufTensorInfo;pub use weights::GgufValue;pub use weights::LinearProjectionModel;pub use weights::SafeTensorEntry;pub use weights::SafeTensorsMetadata;
Modules§
- chunked_
prefill - Chunked Prefill Scheduler
- error
- model
- pipeline
- prompt_
lookup - Prompt Lookup Speculative Decoding Engine
- quantization
- GGUF Quantization Engine
- scheduler
- stream
- tree_
pipeline - weights
- Model Weights & Metadata Parsers