llama_cpp_2/context/
params.rs

1//! A safe wrapper around `llama_context_params`.
2mod get_set;
3
4/// A rusty wrapper around `rope_scaling_type`.
5#[repr(i8)]
6#[derive(Copy, Clone, Debug, PartialEq, Eq)]
7pub enum RopeScalingType {
8    /// The scaling type is unspecified
9    Unspecified = -1,
10    /// No scaling
11    None = 0,
12    /// Linear scaling
13    Linear = 1,
14    /// Yarn scaling
15    Yarn = 2,
16}
17
18/// Create a `RopeScalingType` from a `c_int` - returns `RopeScalingType::ScalingUnspecified` if
19/// the value is not recognized.
20impl From<i32> for RopeScalingType {
21    fn from(value: i32) -> Self {
22        match value {
23            0 => Self::None,
24            1 => Self::Linear,
25            2 => Self::Yarn,
26            _ => Self::Unspecified,
27        }
28    }
29}
30
31/// Create a `c_int` from a `RopeScalingType`.
32impl From<RopeScalingType> for i32 {
33    fn from(value: RopeScalingType) -> Self {
34        match value {
35            RopeScalingType::None => 0,
36            RopeScalingType::Linear => 1,
37            RopeScalingType::Yarn => 2,
38            RopeScalingType::Unspecified => -1,
39        }
40    }
41}
42
43/// A rusty wrapper around `LLAMA_POOLING_TYPE`.
44#[repr(i8)]
45#[derive(Copy, Clone, Debug, PartialEq, Eq)]
46pub enum LlamaPoolingType {
47    /// The pooling type is unspecified
48    Unspecified = -1,
49    /// No pooling
50    None = 0,
51    /// Mean pooling
52    Mean = 1,
53    /// CLS pooling
54    Cls = 2,
55    /// Last pooling
56    Last = 3,
57    /// Rank pooling
58    Rank = 4,
59}
60
61/// Create a `LlamaPoolingType` from a `c_int` - returns `LlamaPoolingType::Unspecified` if
62/// the value is not recognized.
63impl From<i32> for LlamaPoolingType {
64    fn from(value: i32) -> Self {
65        match value {
66            0 => Self::None,
67            1 => Self::Mean,
68            2 => Self::Cls,
69            3 => Self::Last,
70            4 => Self::Rank,
71            _ => Self::Unspecified,
72        }
73    }
74}
75
76/// Create a `c_int` from a `LlamaPoolingType`.
77impl From<LlamaPoolingType> for i32 {
78    fn from(value: LlamaPoolingType) -> Self {
79        match value {
80            LlamaPoolingType::None => 0,
81            LlamaPoolingType::Mean => 1,
82            LlamaPoolingType::Cls => 2,
83            LlamaPoolingType::Last => 3,
84            LlamaPoolingType::Rank => 4,
85            LlamaPoolingType::Unspecified => -1,
86        }
87    }
88}
89
90/// A rusty wrapper around `LLAMA_ATTENTION_TYPE`.
91#[repr(i8)]
92#[derive(Copy, Clone, Debug, PartialEq, Eq)]
93pub enum LlamaAttentionType {
94    /// The attention type is unspecified
95    Unspecified = -1,
96    /// Causal attention
97    Causal = 0,
98    /// Non-causal attention
99    NonCausal = 1,
100}
101
102/// Create a `LlamaAttentionType` from a `c_int` - returns `LlamaAttentionType::Unspecified` if
103/// the value is not recognized.
104impl From<i32> for LlamaAttentionType {
105    fn from(value: i32) -> Self {
106        match value {
107            0 => Self::Causal,
108            1 => Self::NonCausal,
109            _ => Self::Unspecified,
110        }
111    }
112}
113
114/// Create a `c_int` from a `LlamaAttentionType`.
115impl From<LlamaAttentionType> for i32 {
116    fn from(value: LlamaAttentionType) -> Self {
117        match value {
118            LlamaAttentionType::Causal => 0,
119            LlamaAttentionType::NonCausal => 1,
120            LlamaAttentionType::Unspecified => -1,
121        }
122    }
123}
124
125/// A rusty wrapper around `llama_context_type`.
126#[derive(Copy, Clone, Debug, PartialEq, Eq)]
127pub enum LlamaContextType {
128    /// Default decoder context.
129    Default,
130    /// Multi-token-prediction draft context.
131    Mtp,
132    /// Unknown context type from a newer llama.cpp.
133    Unknown(llama_cpp_sys_2::llama_context_type),
134}
135
136impl From<llama_cpp_sys_2::llama_context_type> for LlamaContextType {
137    fn from(value: llama_cpp_sys_2::llama_context_type) -> Self {
138        match value {
139            x if x == llama_cpp_sys_2::LLAMA_CONTEXT_TYPE_DEFAULT => Self::Default,
140            x if x == llama_cpp_sys_2::LLAMA_CONTEXT_TYPE_MTP => Self::Mtp,
141            x => Self::Unknown(x),
142        }
143    }
144}
145
146impl From<LlamaContextType> for llama_cpp_sys_2::llama_context_type {
147    fn from(value: LlamaContextType) -> Self {
148        match value {
149            LlamaContextType::Default => llama_cpp_sys_2::LLAMA_CONTEXT_TYPE_DEFAULT,
150            LlamaContextType::Mtp => llama_cpp_sys_2::LLAMA_CONTEXT_TYPE_MTP,
151            LlamaContextType::Unknown(raw) => raw,
152        }
153    }
154}
155
156/// A rusty wrapper around `ggml_type` for KV cache types.
157#[allow(non_camel_case_types, missing_docs)]
158#[derive(Copy, Clone, Debug, PartialEq, Eq)]
159pub enum KvCacheType {
160    /// Represents an unknown or not-yet-mapped `ggml_type` and carries the raw value.
161    /// When passed through FFI, the raw value is used as-is (if llama.cpp supports it,
162    /// the runtime will operate with that type).
163    /// This variant preserves API compatibility when new `ggml_type` values are
164    /// introduced in the future.
165    Unknown(llama_cpp_sys_2::ggml_type),
166    F32,
167    F16,
168    Q4_0,
169    Q4_1,
170    Q5_0,
171    Q5_1,
172    Q8_0,
173    Q8_1,
174    Q2_K,
175    Q3_K,
176    Q4_K,
177    Q5_K,
178    Q6_K,
179    Q8_K,
180    IQ2_XXS,
181    IQ2_XS,
182    IQ3_XXS,
183    IQ1_S,
184    IQ4_NL,
185    IQ3_S,
186    IQ2_S,
187    IQ4_XS,
188    I8,
189    I16,
190    I32,
191    I64,
192    F64,
193    IQ1_M,
194    BF16,
195    TQ1_0,
196    TQ2_0,
197    MXFP4,
198}
199
200impl From<KvCacheType> for llama_cpp_sys_2::ggml_type {
201    fn from(value: KvCacheType) -> Self {
202        match value {
203            KvCacheType::Unknown(raw) => raw,
204            KvCacheType::F32 => llama_cpp_sys_2::GGML_TYPE_F32,
205            KvCacheType::F16 => llama_cpp_sys_2::GGML_TYPE_F16,
206            KvCacheType::Q4_0 => llama_cpp_sys_2::GGML_TYPE_Q4_0,
207            KvCacheType::Q4_1 => llama_cpp_sys_2::GGML_TYPE_Q4_1,
208            KvCacheType::Q5_0 => llama_cpp_sys_2::GGML_TYPE_Q5_0,
209            KvCacheType::Q5_1 => llama_cpp_sys_2::GGML_TYPE_Q5_1,
210            KvCacheType::Q8_0 => llama_cpp_sys_2::GGML_TYPE_Q8_0,
211            KvCacheType::Q8_1 => llama_cpp_sys_2::GGML_TYPE_Q8_1,
212            KvCacheType::Q2_K => llama_cpp_sys_2::GGML_TYPE_Q2_K,
213            KvCacheType::Q3_K => llama_cpp_sys_2::GGML_TYPE_Q3_K,
214            KvCacheType::Q4_K => llama_cpp_sys_2::GGML_TYPE_Q4_K,
215            KvCacheType::Q5_K => llama_cpp_sys_2::GGML_TYPE_Q5_K,
216            KvCacheType::Q6_K => llama_cpp_sys_2::GGML_TYPE_Q6_K,
217            KvCacheType::Q8_K => llama_cpp_sys_2::GGML_TYPE_Q8_K,
218            KvCacheType::IQ2_XXS => llama_cpp_sys_2::GGML_TYPE_IQ2_XXS,
219            KvCacheType::IQ2_XS => llama_cpp_sys_2::GGML_TYPE_IQ2_XS,
220            KvCacheType::IQ3_XXS => llama_cpp_sys_2::GGML_TYPE_IQ3_XXS,
221            KvCacheType::IQ1_S => llama_cpp_sys_2::GGML_TYPE_IQ1_S,
222            KvCacheType::IQ4_NL => llama_cpp_sys_2::GGML_TYPE_IQ4_NL,
223            KvCacheType::IQ3_S => llama_cpp_sys_2::GGML_TYPE_IQ3_S,
224            KvCacheType::IQ2_S => llama_cpp_sys_2::GGML_TYPE_IQ2_S,
225            KvCacheType::IQ4_XS => llama_cpp_sys_2::GGML_TYPE_IQ4_XS,
226            KvCacheType::I8 => llama_cpp_sys_2::GGML_TYPE_I8,
227            KvCacheType::I16 => llama_cpp_sys_2::GGML_TYPE_I16,
228            KvCacheType::I32 => llama_cpp_sys_2::GGML_TYPE_I32,
229            KvCacheType::I64 => llama_cpp_sys_2::GGML_TYPE_I64,
230            KvCacheType::F64 => llama_cpp_sys_2::GGML_TYPE_F64,
231            KvCacheType::IQ1_M => llama_cpp_sys_2::GGML_TYPE_IQ1_M,
232            KvCacheType::BF16 => llama_cpp_sys_2::GGML_TYPE_BF16,
233            KvCacheType::TQ1_0 => llama_cpp_sys_2::GGML_TYPE_TQ1_0,
234            KvCacheType::TQ2_0 => llama_cpp_sys_2::GGML_TYPE_TQ2_0,
235            KvCacheType::MXFP4 => llama_cpp_sys_2::GGML_TYPE_MXFP4,
236        }
237    }
238}
239
240impl From<llama_cpp_sys_2::ggml_type> for KvCacheType {
241    fn from(value: llama_cpp_sys_2::ggml_type) -> Self {
242        match value {
243            x if x == llama_cpp_sys_2::GGML_TYPE_F32 => KvCacheType::F32,
244            x if x == llama_cpp_sys_2::GGML_TYPE_F16 => KvCacheType::F16,
245            x if x == llama_cpp_sys_2::GGML_TYPE_Q4_0 => KvCacheType::Q4_0,
246            x if x == llama_cpp_sys_2::GGML_TYPE_Q4_1 => KvCacheType::Q4_1,
247            x if x == llama_cpp_sys_2::GGML_TYPE_Q5_0 => KvCacheType::Q5_0,
248            x if x == llama_cpp_sys_2::GGML_TYPE_Q5_1 => KvCacheType::Q5_1,
249            x if x == llama_cpp_sys_2::GGML_TYPE_Q8_0 => KvCacheType::Q8_0,
250            x if x == llama_cpp_sys_2::GGML_TYPE_Q8_1 => KvCacheType::Q8_1,
251            x if x == llama_cpp_sys_2::GGML_TYPE_Q2_K => KvCacheType::Q2_K,
252            x if x == llama_cpp_sys_2::GGML_TYPE_Q3_K => KvCacheType::Q3_K,
253            x if x == llama_cpp_sys_2::GGML_TYPE_Q4_K => KvCacheType::Q4_K,
254            x if x == llama_cpp_sys_2::GGML_TYPE_Q5_K => KvCacheType::Q5_K,
255            x if x == llama_cpp_sys_2::GGML_TYPE_Q6_K => KvCacheType::Q6_K,
256            x if x == llama_cpp_sys_2::GGML_TYPE_Q8_K => KvCacheType::Q8_K,
257            x if x == llama_cpp_sys_2::GGML_TYPE_IQ2_XXS => KvCacheType::IQ2_XXS,
258            x if x == llama_cpp_sys_2::GGML_TYPE_IQ2_XS => KvCacheType::IQ2_XS,
259            x if x == llama_cpp_sys_2::GGML_TYPE_IQ3_XXS => KvCacheType::IQ3_XXS,
260            x if x == llama_cpp_sys_2::GGML_TYPE_IQ1_S => KvCacheType::IQ1_S,
261            x if x == llama_cpp_sys_2::GGML_TYPE_IQ4_NL => KvCacheType::IQ4_NL,
262            x if x == llama_cpp_sys_2::GGML_TYPE_IQ3_S => KvCacheType::IQ3_S,
263            x if x == llama_cpp_sys_2::GGML_TYPE_IQ2_S => KvCacheType::IQ2_S,
264            x if x == llama_cpp_sys_2::GGML_TYPE_IQ4_XS => KvCacheType::IQ4_XS,
265            x if x == llama_cpp_sys_2::GGML_TYPE_I8 => KvCacheType::I8,
266            x if x == llama_cpp_sys_2::GGML_TYPE_I16 => KvCacheType::I16,
267            x if x == llama_cpp_sys_2::GGML_TYPE_I32 => KvCacheType::I32,
268            x if x == llama_cpp_sys_2::GGML_TYPE_I64 => KvCacheType::I64,
269            x if x == llama_cpp_sys_2::GGML_TYPE_F64 => KvCacheType::F64,
270            x if x == llama_cpp_sys_2::GGML_TYPE_IQ1_M => KvCacheType::IQ1_M,
271            x if x == llama_cpp_sys_2::GGML_TYPE_BF16 => KvCacheType::BF16,
272            x if x == llama_cpp_sys_2::GGML_TYPE_TQ1_0 => KvCacheType::TQ1_0,
273            x if x == llama_cpp_sys_2::GGML_TYPE_TQ2_0 => KvCacheType::TQ2_0,
274            x if x == llama_cpp_sys_2::GGML_TYPE_MXFP4 => KvCacheType::MXFP4,
275            _ => KvCacheType::Unknown(value),
276        }
277    }
278}
279
280/// A safe wrapper around `llama_context_params`.
281///
282/// Generally this should be created with [`Default::default()`] and then modified with `with_*` methods.
283///
284/// # Examples
285///
286/// ```rust
287/// # use std::num::NonZeroU32;
288/// # use llama_cpp_2::context::params::LlamaContextParams;
289///
290/// let ctx_params = LlamaContextParams::default()
291///     .with_n_ctx(NonZeroU32::new(2048));
292///
293/// assert_eq!(ctx_params.n_ctx(), NonZeroU32::new(2048));
294/// ```
295#[derive(Debug, Clone)]
296#[allow(
297    missing_docs,
298    clippy::struct_excessive_bools,
299    clippy::module_name_repetitions
300)]
301pub struct LlamaContextParams {
302    pub(crate) context_params: llama_cpp_sys_2::llama_context_params,
303}
304
305/// SAFETY: we do not currently allow setting or reading the pointers that cause this to not be automatically send or sync.
306unsafe impl Send for LlamaContextParams {}
307unsafe impl Sync for LlamaContextParams {}
308
309/// Default parameters for `LlamaContext`. (as defined in llama.cpp by `llama_context_default_params`)
310/// ```
311/// # use std::num::NonZeroU32;
312/// # use llama_cpp_2::context::params::{LlamaContextParams, RopeScalingType};
313/// let params = LlamaContextParams::default();
314/// assert_eq!(params.n_ctx(), NonZeroU32::new(512), "n_ctx should be 512");
315/// assert_eq!(params.rope_scaling_type(), RopeScalingType::Unspecified);
316/// ```
317impl Default for LlamaContextParams {
318    fn default() -> Self {
319        let context_params = unsafe { llama_cpp_sys_2::llama_context_default_params() };
320        Self { context_params }
321    }
322}
llama_cpp_2/context/params.rs

llama_cpp_2/context/
params.rs
