abyo_speculate/lib.rs
1//! # abyo-speculate
2//!
3//! Pure Rust Speculative Decoding library for local LLMs, optimized for batch size 1.
4//!
5//! See the [crate README](https://github.com/abyo-software/abyo-speculate) and
6//! the project plan for design context.
7//!
8//! ## Quick example
9//!
10//! ```no_run
11//! use abyo_speculate::{SpeculateEngine, Method};
12//!
13//! # fn main() -> anyhow::Result<()> {
14//! let mut engine = SpeculateEngine::builder()
15//! .target_model("llama-3.1-8b-instruct")
16//! .method(Method::Vanilla)
17//! .draft_model("tinyllama-1.1b")
18//! .build()?;
19//!
20//! // engine.with_target(...).with_draft(...) attach loaded models;
21//! // see model::qwen2::Qwen2Decoder for a concrete loader.
22//! let _tokens = engine.generate_tokens(&[1u32, 2, 3], 64)?;
23//! # Ok(())
24//! # }
25//! ```
26
27#![cfg_attr(docsrs, feature(doc_cfg))]
28#![warn(missing_docs)]
29#![warn(rust_2018_idioms)]
30
31pub mod cache;
32pub mod device;
33pub mod engine;
34pub mod error;
35pub mod methods;
36pub mod model;
37pub mod presets;
38pub mod sampling;
39pub mod tree;
40
41pub use engine::{GenerationOptions, SpeculateEngine, SpeculateEngineBuilder};
42pub use error::{Error, Result};
43pub use methods::Method;