Skip to main content

Crate spec_stream_engine

Crate spec_stream_engine 

Source
Expand description

§spec-stream-engine

Speculative decoding execution engine and streaming orchestrator. Combines draft models, target models, paged KV-cache rollbacks, and verification into an asynchronous token generation pipeline.

Re-exports§

pub use chunked_prefill::ChunkedPrefillConfig;
pub use chunked_prefill::ChunkedPrefillScheduler;
pub use chunked_prefill::PrefillTask;
pub use error::EngineError;
pub use model::ModelEngine;
pub use model::SyntheticModel;
pub use pipeline::SpeculativeConfig;
pub use pipeline::SpeculativePipeline;
pub use prompt_lookup::PromptLookupConfig;
pub use prompt_lookup::PromptLookupEngine;
pub use quantization::dequantize_q4_0;
pub use quantization::dequantize_q8_0;
pub use quantization::f16_to_f32;
pub use quantization::f32_to_f16;
pub use quantization::QuantType;
pub use quantization::QuantizedLinearModel;
pub use scheduler::ContinuousBatchScheduler;
pub use scheduler::QueuedRequest;
pub use scheduler::SchedulerStats;
pub use stream::GenerationEvent;
pub use stream::SpeculativeStream;
pub use tree_pipeline::TreeSpeculativeConfig;
pub use tree_pipeline::TreeSpeculativePipeline;
pub use weights::GgufMetadata;
pub use weights::GgufTensorInfo;
pub use weights::GgufValue;
pub use weights::LinearProjectionModel;
pub use weights::SafeTensorEntry;
pub use weights::SafeTensorsMetadata;

Modules§

chunked_prefill
Chunked Prefill Scheduler
error
model
pipeline
prompt_lookup
Prompt Lookup Speculative Decoding Engine
quantization
GGUF Quantization Engine
scheduler
stream
tree_pipeline
weights
Model Weights & Metadata Parsers