Skip to main content

llama_cpp_4/
prelude.rs

1//! Convenience re-exports for typical inference workflows.
2//!
3//! # Quick start
4//!
5//! ```
6//! use llama_cpp_4::prelude::*;
7//! ```
8//!
9//! Core types are also available at the crate root (`llama_cpp_4::LlamaModel`, …)
10//! if you prefer explicit paths over a glob import.
11//!
12//! # What's included
13//!
14//! The rule is that anything a prelude-exported API takes, returns or fails
15//! with is exported too — including every error type — so `use
16//! llama_cpp_4::prelude::*` is enough to write a complete call, `match` arm
17//! included. Lower-level corners stay out: the `ggml` module (feature `ggml`)
18//! for raw graph building, and [`crate::quantize`] for the imatrix and
19//! quantization-preview types.
20//!
21//! | Category | Re-exported |
22//! |---|---|
23//! | Backend & model | [`LlamaBackend`], [`LlamaModel`], [`LlamaModelParams`], [`LlamaBackendDevice`], [`LlamaBackendDeviceType`], [`LlamaLoraAdapter`] |
24//! | Context | [`LlamaContext`], [`LlamaContextParams`], [`LlamaContextType`], [`LlamaAttentionType`], [`LlamaFlashAttnType`], [`LlamaPoolingType`], [`RopeScalingType`], [`ParamOverrideValue`] |
25//! | Batching | [`LlamaBatch`] for token batches; [`LlamaBatchExt`] with [`BatchEntry`], [`EmbeddingRows`], [`LlamaProcessType`], [`LlamaDecisionOrder`] and [`MROPE_SECTIONS`] for per-entry embeddings and M-RoPE |
26//! | Tokens | [`LlamaToken`], [`LlamaTokenData`], [`LlamaTokenDataArray`], [`AddBos`], [`Special`], [`StreamDetokenizer`] |
27//! | Sampling | [`LlamaSampler`], [`LlamaSamplerParams`], [`CommonSampler`], [`CommonSamplerParams`], [`CommonSamplerType`], [`GrammarSource`], [`ReasoningBudget`], [`ReasoningBudgetState`] |
28//! | Chat & tools | [`LlamaChatMessage`], [`ChatTemplates`], [`ChatApplyParams`], [`ChatParams`], [`GrammarTrigger`], [`ReasoningFormat`], [`ToolChoice`], [`json_schema_to_grammar`] |
29//! | Speculative decoding | [`MtpSession`], [`MtpSessionConfig`], [`Eagle3Session`], [`Eagle3SessionConfig`], [`DFlashSession`], [`NgramCache`], [`NgramMap`], [`ngram_simple_draft`], [`ngram_cache_draft`], [`SpeculativeType`], [`speculative_types_from_gguf`] |
30//! | Memory & fit | [`fit_params`], [`FitParams`], [`FitParamsResult`], [`get_device_memory_data`], [`DeviceMemoryReport`], [`DeviceMemoryEntry`], [`DeviceMemoryHyperParams`], [`MemoryBreakdownEntry`] |
31//! | Tensor capture | [`TensorCapture`], [`CapturedTensor`], [`TensorTransactions`], [`TensorTransaction`], [`TensorSelector`], [`TensorAccess`], … |
32//! | Quantization | [`QuantizeParams`], [`TensorTypeOverride`], [`GgmlType`], [`LlamaFtype`], [`model_quantize`], [`attn_rot_disabled`], [`set_attn_rot_disabled`] |
33//! | Errors | [`Result`], [`LLamaCppError`], [`ShimError`] (the type behind the shim-backed aliases) and one per area: [`DecodeError`], [`EncodeError`], [`ProcessError`], [`EmbeddingsError`], [`BatchAddError`], [`BatchExtError`], [`ChatError`], [`CommonSamplerError`], [`NgramError`], [`RuntimeError`], [`MtpSessionError`], [`Eagle3SessionError`], [`SpeculativeStateError`], [`DetokenizeError`], [`FitParamsError`], [`DeviceMemoryError`], [`FtypeNameError`], [`QuantPreviewError`], [`SamplerInitError`], [`LlamaModelLoadError`], [`LlamaContextLoadError`], [`LlamaLoraAdapterInitError`], [`LlamaLoraAdapterSetError`], [`LlamaLoraAdapterRemoveError`], [`ApplyChatTemplateError`], [`NewLlamaChatMessageError`], [`ParamsCloneError`] |
34//! | Utilities | [`ggml_time_us`], [`llama_time_us`], [`print_system_info`], [`supports_gpu_offload`], [`max_devices`] |
35//!
36//! With the `mtmd` feature: [`MtmdContext`], [`MtmdBitmap`], [`MtmdContextParams`],
37//! [`MtmdInputChunks`], [`MtmdInputText`], [`MtmdProgressCallback`].\
38//! With the `rpc` feature: `RpcBackend`, `RpcError`, `add_rpc_server`, `serve`.
39//!
40//! # Text generation
41//!
42//! ```no_run
43//! use llama_cpp_4::prelude::*;
44//! use std::num::NonZeroU32;
45//!
46//! fn main() {
47//!     let backend = LlamaBackend::init().unwrap();
48//!     let model = LlamaModel::load_from_file(
49//!         &backend,
50//!         "model.gguf",
51//!         &LlamaModelParams::default(),
52//!     )
53//!     .unwrap();
54//!     let mut ctx = model
55//!         .new_context(
56//!             &backend,
57//!             LlamaContextParams::default().with_n_ctx(NonZeroU32::new(2048)),
58//!         )
59//!         .unwrap();
60//!
61//!     let tokens = model.str_to_token("The answer is", AddBos::Always).unwrap();
62//!     let n_prompt = tokens.len();
63//!     let mut batch = LlamaBatch::new(2048, 1);
64//!     for (i, &tok) in tokens.iter().enumerate() {
65//!         batch
66//!             .add(tok, i as i32, &[0], i == n_prompt - 1)
67//!             .unwrap();
68//!     }
69//!     ctx.decode(&mut batch).unwrap();
70//!
71//!     let sampler = LlamaSampler::chain_simple([
72//!         LlamaSampler::temp(0.8),
73//!         LlamaSampler::dist(0),
74//!     ]);
75//!     let token = sampler.sample(&ctx, 0);
76//!     let _text = model.token_to_bytes(token, Special::Plaintext).unwrap();
77//! }
78//! ```
79//!
80//! # Chat template
81//!
82//! ```no_run
83//! use llama_cpp_4::prelude::*;
84//!
85//! fn main() {
86//!     let backend = LlamaBackend::init().unwrap();
87//!     let model = LlamaModel::load_from_file(
88//!         &backend,
89//!         "model.gguf",
90//!         &LlamaModelParams::default(),
91//!     )
92//!     .unwrap();
93//!
94//!     let messages = vec![
95//!         LlamaChatMessage::new("system".into(), "You are helpful.".into()).unwrap(),
96//!         LlamaChatMessage::new("user".into(), "What is 2+2?".into()).unwrap(),
97//!     ];
98//!     let prompt = model.apply_chat_template(None, &messages, true).unwrap();
99//!     let _tokens = model.str_to_token(&prompt, AddBos::Always).unwrap();
100//! }
101//! ```
102//!
103//! # Embeddings
104//!
105//! ```no_run
106//! use llama_cpp_4::prelude::*;
107//! use std::num::NonZeroU32;
108//!
109//! fn main() {
110//!     let backend = LlamaBackend::init().unwrap();
111//!     let model = LlamaModel::load_from_file(
112//!         &backend,
113//!         "model.gguf",
114//!         &LlamaModelParams::default(),
115//!     )
116//!     .unwrap();
117//!     let mut ctx = model
118//!         .new_context(
119//!             &backend,
120//!             LlamaContextParams::default()
121//!                 .with_embeddings(true)
122//!                 .with_n_ctx(NonZeroU32::new(512)),
123//!         )
124//!         .unwrap();
125//!
126//!     let tokens = model.str_to_token("Hello", AddBos::Always).unwrap();
127//!     let mut batch = LlamaBatch::new(512, 1);
128//!     for (i, &tok) in tokens.iter().enumerate() {
129//!         batch
130//!             .add(tok, i as i32, &[0], i == tokens.len() - 1)
131//!             .unwrap();
132//!     }
133//!     ctx.decode(&mut batch).unwrap();
134//!     let _vec = ctx.embeddings_seq_ith(0).unwrap();
135//! }
136//! ```
137//!
138//! # Memory estimation (before loading fully)
139//!
140//! ```no_run
141//! use llama_cpp_4::prelude::*;
142//! use std::path::Path;
143//!
144//! fn main() {
145//!     let report = get_device_memory_data(
146//!         Path::new("model.gguf"),
147//!         &LlamaModelParams::default(),
148//!         &LlamaContextParams::default(),
149//!         llama_cpp_sys_4::GGML_LOG_LEVEL_ERROR,
150//!     )
151//!     .unwrap();
152//!     for entry in &report.entries {
153//!         println!("projected used: {} bytes", entry.used());
154//!     }
155//! }
156//! ```
157
158// ── Core inference ────────────────────────────────────────────────────────────
159
160pub use crate::context::params::{
161    LlamaAttentionType, LlamaContextParams, LlamaContextType, LlamaFlashAttnType, LlamaPoolingType,
162    ParamsCloneError, RopeScalingType,
163};
164pub use crate::context::{
165    CapturedTensor, CapturedTensorData, LlamaContext, MemoryBreakdownEntry, TensorAccess,
166    TensorBatchRow, TensorCallbackFailure, TensorCapture, TensorDataMut, TensorElementType,
167    TensorFiniteValidation, TensorRowMapping, TensorSelector, TensorShape, TensorTransaction,
168    TensorTransactionError, TensorTransactionHandler, TensorTransactions, TensorWriteback,
169    TransactionalTensorCapture,
170};
171pub use crate::llama_backend::LlamaBackend;
172pub use crate::llama_batch::{BatchAddError, LlamaBatch};
173pub use crate::llama_batch_ext::{
174    BatchEntry, BatchExtError, EmbeddingRows, LlamaBatchExt, LlamaDecisionOrder, LlamaProcessType,
175    MROPE_SECTIONS,
176};
177pub use crate::model::params::kv_overrides::ParamOverrideValue;
178pub use crate::model::params::LlamaModelParams;
179pub use crate::model::{
180    AddBos, LlamaBackendDevice, LlamaBackendDeviceType, LlamaChatMessage, LlamaLoraAdapter,
181    LlamaModel, Special,
182};
183pub use crate::sampling::{LlamaSampler, LlamaSamplerParams, SamplerInitError};
184pub use crate::speculative::{SpeculativeStateError, MAX_SPECULATIVE_STATE_BYTES};
185pub use crate::token::data::LlamaTokenData;
186pub use crate::token::data_array::LlamaTokenDataArray;
187pub use crate::token::detokenizer::{DetokenizeError, StreamDetokenizer};
188pub use crate::token::LlamaToken;
189
190// ── Errors & results ────────────────────────────────────────────────────────
191//
192// Every error type a prelude-exported API can return is itself re-exported, so
193// a caller never has to reach past the prelude just to write a `match` arm.
194
195pub use crate::common_sampler::CommonSamplerError;
196pub use crate::eagle::Eagle3SessionError;
197pub use crate::mtp::MtpSessionError;
198pub use crate::ngram::NgramError;
199pub use crate::quantize::{FtypeNameError, QuantPreviewError};
200pub use crate::runtime::RuntimeError;
201pub use crate::ShimError;
202pub use crate::{
203    ApplyChatTemplateError, DecodeError, EmbeddingsError, EncodeError, LLamaCppError,
204    LlamaContextLoadError, LlamaLoraAdapterInitError, LlamaLoraAdapterRemoveError,
205    LlamaLoraAdapterSetError, LlamaModelLoadError, NewLlamaChatMessageError, ProcessError, Result,
206};
207
208// ── Memory / fit helpers ────────────────────────────────────────────────────
209
210pub use crate::fit::{
211    fit_params, get_device_memory_data, DeviceMemoryEntry, DeviceMemoryError,
212    DeviceMemoryHyperParams, DeviceMemoryReport, FitParams, FitParamsError, FitParamsResult,
213};
214
215// ── Chat templates, tool calling, structured output ─────────────────────────
216
217pub use crate::chat::{
218    json_schema_to_grammar, ChatApplyParams, ChatError, ChatParams, ChatTemplates, GrammarTrigger,
219    ReasoningFormat, ToolChoice,
220};
221
222// ── Sampler chains ──────────────────────────────────────────────────────────
223
224pub use crate::common_sampler::{
225    CommonSampler, CommonSamplerParams, CommonSamplerType, GrammarSource, ReasoningBudget,
226    ReasoningBudgetState,
227};
228
229// ── Speculative decoding ────────────────────────────────────────────────────
230
231pub use crate::eagle::{DFlashSession, Eagle3Session, Eagle3SessionConfig};
232pub use crate::mtp::{MtpSession, MtpSessionConfig};
233pub use crate::ngram::{ngram_cache_draft, ngram_simple_draft, NgramCache, NgramMap};
234pub use crate::runtime::{speculative_types_from_gguf, SpeculativeType};
235
236// ── Multimodal (feature `mtmd`) ─────────────────────────────────────────────
237
238#[cfg(feature = "mtmd")]
239pub use crate::mtmd::{
240    MtmdBitmap, MtmdContext, MtmdContextParams, MtmdInputChunks, MtmdInputText,
241    MtmdProgressCallback,
242};
243
244// ── Remote backend (feature `rpc`) ──────────────────────────────────────────
245
246#[cfg(feature = "rpc")]
247pub use crate::rpc::{add_rpc_server, serve, RpcBackend, RpcError};
248
249// ── Quantization ────────────────────────────────────────────────────────────
250
251pub use crate::quantize::{
252    attn_rot_disabled, set_attn_rot_disabled, GgmlType, LlamaFtype, QuantizeParams,
253    TensorTypeOverride,
254};
255
256// ── Utilities ───────────────────────────────────────────────────────────────
257
258pub use crate::{
259    ggml_time_us, llama_time_us, max_devices, model_quantize, print_system_info,
260    supports_gpu_offload,
261};