llama_cpp_2/
model.rs

1//! A safe wrapper around `llama_model`.
2use std::ffi::{c_char, CStr, CString};
3use std::os::raw::c_int;
4use std::path::Path;
5use std::ptr::NonNull;
6use std::str::Utf8Error;
7
8use crate::context::params::LlamaContextParams;
9use crate::context::LlamaContext;
10use crate::llama_backend::LlamaBackend;
11use crate::model::params::LlamaModelParams;
12use crate::sampling::LlamaSampler;
13use crate::token::LlamaToken;
14use crate::vocab::LlamaVocab;
15use crate::{
16    ApplyChatTemplateError, ChatTemplateError, LlamaContextLoadError, LlamaLoraAdapterInitError,
17    LlamaModelLoadError, MetaValError, NewLlamaChatMessageError,
18};
19
20pub mod params;
21// For backwards compat.
22pub use crate::vocab::{LlamaTokenTypeFromIntError, VocabType};
23
24/// A safe wrapper around `llama_model`.
25#[derive(Debug)]
26#[repr(transparent)]
27#[allow(clippy::module_name_repetitions)]
28pub struct LlamaModel {
29    pub(crate) model: NonNull<llama_cpp_sys_2::llama_model>,
30}
31
32/// A safe wrapper around `llama_lora_adapter`.
33#[derive(Debug)]
34#[repr(transparent)]
35#[allow(clippy::module_name_repetitions)]
36pub struct LlamaLoraAdapter {
37    pub(crate) lora_adapter: NonNull<llama_cpp_sys_2::llama_adapter_lora>,
38}
39
40/// A performance-friendly wrapper around [`LlamaModel::chat_template`] which is then
41/// fed into [`LlamaModel::apply_chat_template`] to convert a list of messages into an LLM
42/// prompt. Internally the template is stored as a `CString` to avoid round-trip conversions
43/// within the FFI.
44#[derive(Eq, PartialEq, Clone, PartialOrd, Ord, Hash)]
45pub struct LlamaChatTemplate(CString);
46
47impl LlamaChatTemplate {
48    /// Create a new template from a string. This can either be the name of a llama.cpp [chat template](https://github.com/ggerganov/llama.cpp/blob/8a8c4ceb6050bd9392609114ca56ae6d26f5b8f5/src/llama-chat.cpp#L27-L61)
49    /// like "chatml" or "llama3" or an actual Jinja template for llama.cpp to interpret.
50    pub fn new(template: &str) -> Result<Self, std::ffi::NulError> {
51        Ok(Self(CString::new(template)?))
52    }
53
54    /// Accesses the template as a c string reference.
55    pub fn as_c_str(&self) -> &CStr {
56        &self.0
57    }
58
59    /// Attempts to convert the `CString` into a Rust str reference.
60    pub fn to_str(&self) -> Result<&str, Utf8Error> {
61        self.0.to_str()
62    }
63
64    /// Convenience method to create an owned String.
65    pub fn to_string(&self) -> Result<String, Utf8Error> {
66        self.to_str().map(str::to_string)
67    }
68}
69
70impl std::fmt::Debug for LlamaChatTemplate {
71    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
72        self.0.fmt(f)
73    }
74}
75
76/// A Safe wrapper around `llama_chat_message`
77#[derive(Debug, Eq, PartialEq, Clone)]
78pub struct LlamaChatMessage {
79    role: CString,
80    content: CString,
81}
82
83impl LlamaChatMessage {
84    /// Create a new `LlamaChatMessage`
85    ///
86    /// # Errors
87    /// If either of ``role`` or ``content`` contain null bytes.
88    pub fn new(role: String, content: String) -> Result<Self, NewLlamaChatMessageError> {
89        Ok(Self {
90            role: CString::new(role)?,
91            content: CString::new(content)?,
92        })
93    }
94}
95
96/// The Rope type that's used within the model.
97#[derive(Debug, Clone, Copy, PartialEq, Eq)]
98pub enum RopeType {
99    Norm,
100    NeoX,
101    MRope,
102    Vision,
103}
104
105unsafe impl Send for LlamaModel {}
106
107unsafe impl Sync for LlamaModel {}
108
109#[allow(deprecated)]
110impl LlamaModel {
111    /// Get the model's vocabulary.
112    #[must_use]
113    pub fn vocab(&self) -> LlamaVocab<'_> {
114        let ptr = unsafe { llama_cpp_sys_2::llama_model_get_vocab(self.model.as_ptr()) };
115        LlamaVocab::new(ptr).expect("model must have vocabulary")
116    }
117
118    /// get the number of tokens the model was trained on
119    ///
120    /// # Panics
121    ///
122    /// If the number of tokens the model was trained on does not fit into an `u32`. This should be impossible on most
123    /// platforms due to llama.cpp returning a `c_int` (i32 on most platforms) which is almost certainly positive.
124    #[must_use]
125    pub fn n_ctx_train(&self) -> u32 {
126        let n_ctx_train = unsafe { llama_cpp_sys_2::llama_n_ctx_train(self.model.as_ptr()) };
127        u32::try_from(n_ctx_train).expect("n_ctx_train fits into an u32")
128    }
129
130    /// Get all tokens in the model.
131    pub fn tokens(&self, decode_special: bool) -> impl Iterator<Item = (LlamaToken, Vec<u8>)> + '_ {
132        let vocab = self.vocab();
133        vocab.tokens().map(move |llama_token| {
134            let bytes = vocab.token_to_piece(llama_token, decode_special, None);
135            (llama_token, bytes)
136        })
137    }
138
139    /// Get the decoder start token.
140    #[must_use]
141    pub fn decode_start_token(&self) -> LlamaToken {
142        let token =
143            unsafe { llama_cpp_sys_2::llama_model_decoder_start_token(self.model.as_ptr()) };
144        LlamaToken(token)
145    }
146
147    /// The number of tokens the model was trained on.
148    ///
149    /// This returns a `c_int` for maximum compatibility. Most of the time it can be cast to an i32
150    /// without issue.
151    #[must_use]
152    pub fn n_vocab(&self) -> i32 {
153        self.vocab().n_tokens()
154    }
155
156    /// This returns a `c_int` for maximum compatibility. Most of the time it can be cast to an i32
157    /// without issue.
158    #[must_use]
159    pub fn n_embd(&self) -> c_int {
160        unsafe { llama_cpp_sys_2::llama_n_embd(self.model.as_ptr()) }
161    }
162
163    /// The model's *output* embedding width (`n_embd_out`). This is the width
164    /// llama.cpp actually extracts embeddings at — `n_embd` and `n_embd_out`
165    /// diverge when `{arch}.embedding_length_out` is present (deepstack models
166    /// like qwen3vl). Returns a `c_int` for maximum compatibility.
167    #[must_use]
168    pub fn n_embd_out(&self) -> c_int {
169        unsafe { llama_cpp_sys_2::llama_model_n_embd_out(self.model.as_ptr()) }
170    }
171
172    /// The model's classification output width (`n_cls_out`, default 1) — the
173    /// width of a RANK-pooled embeddings read (llama.h:1029).
174    #[must_use]
175    pub fn n_cls_out(&self) -> u32 {
176        unsafe { llama_cpp_sys_2::llama_model_n_cls_out(self.model.as_ptr()) }
177    }
178
179    /// Returns the total size of all the tensors in the model in bytes.
180    pub fn size(&self) -> u64 {
181        unsafe { llama_cpp_sys_2::llama_model_size(self.model.as_ptr()) }
182    }
183
184    /// Returns the number of parameters in the model.
185    pub fn n_params(&self) -> u64 {
186        unsafe { llama_cpp_sys_2::llama_model_n_params(self.model.as_ptr()) }
187    }
188
189    /// Returns whether the model is a recurrent network (Mamba, RWKV, etc)
190    pub fn is_recurrent(&self) -> bool {
191        unsafe { llama_cpp_sys_2::llama_model_is_recurrent(self.model.as_ptr()) }
192    }
193
194    /// Returns whether the model is a hybrid network (Jamba, Granite, Qwen3xx, etc)
195    ///
196    /// Hybrid models have both attention layers and recurrent/SSM layers.
197    /// They require special handling for state checkpointing.
198    pub fn is_hybrid(&self) -> bool {
199        unsafe { llama_cpp_sys_2::llama_model_is_hybrid(self.model.as_ptr()) }
200    }
201
202    /// Returns the number of layers within the model.
203    pub fn n_layer(&self) -> u32 {
204        // It's never possible for this to panic because while the API interface is defined as an int32_t,
205        // the field it's accessing is a uint32_t.
206        u32::try_from(unsafe { llama_cpp_sys_2::llama_model_n_layer(self.model.as_ptr()) }).unwrap()
207    }
208
209    /// Returns the number of attention heads within the model.
210    pub fn n_head(&self) -> u32 {
211        // It's never possible for this to panic because while the API interface is defined as an int32_t,
212        // the field it's accessing is a uint32_t.
213        u32::try_from(unsafe { llama_cpp_sys_2::llama_model_n_head(self.model.as_ptr()) }).unwrap()
214    }
215
216    /// Returns the number of KV attention heads.
217    pub fn n_head_kv(&self) -> u32 {
218        // It's never possible for this to panic because while the API interface is defined as an int32_t,
219        // the field it's accessing is a uint32_t.
220        u32::try_from(unsafe { llama_cpp_sys_2::llama_model_n_head_kv(self.model.as_ptr()) })
221            .unwrap()
222    }
223
224    /// Get metadata value as a string by key name
225    pub fn meta_val_str(&self, key: &str) -> Result<String, MetaValError> {
226        let key_cstring = CString::new(key)?;
227        let key_ptr = key_cstring.as_ptr();
228
229        extract_meta_string(
230            |buf_ptr, buf_len| unsafe {
231                llama_cpp_sys_2::llama_model_meta_val_str(
232                    self.model.as_ptr(),
233                    key_ptr,
234                    buf_ptr,
235                    buf_len,
236                )
237            },
238            256,
239        )
240    }
241
242    /// Get the number of metadata key/value pairs
243    pub fn meta_count(&self) -> i32 {
244        unsafe { llama_cpp_sys_2::llama_model_meta_count(self.model.as_ptr()) }
245    }
246
247    /// Get metadata key name by index
248    pub fn meta_key_by_index(&self, index: i32) -> Result<String, MetaValError> {
249        extract_meta_string(
250            |buf_ptr, buf_len| unsafe {
251                llama_cpp_sys_2::llama_model_meta_key_by_index(
252                    self.model.as_ptr(),
253                    index,
254                    buf_ptr,
255                    buf_len,
256                )
257            },
258            256,
259        )
260    }
261
262    /// Get metadata value as a string by index
263    pub fn meta_val_str_by_index(&self, index: i32) -> Result<String, MetaValError> {
264        extract_meta_string(
265            |buf_ptr, buf_len| unsafe {
266                llama_cpp_sys_2::llama_model_meta_val_str_by_index(
267                    self.model.as_ptr(),
268                    index,
269                    buf_ptr,
270                    buf_len,
271                )
272            },
273            256,
274        )
275    }
276
277    /// Returns the rope type of the model.
278    pub fn rope_type(&self) -> Option<RopeType> {
279        match unsafe { llama_cpp_sys_2::llama_model_rope_type(self.model.as_ptr()) } {
280            llama_cpp_sys_2::LLAMA_ROPE_TYPE_NONE => None,
281            llama_cpp_sys_2::LLAMA_ROPE_TYPE_NORM => Some(RopeType::Norm),
282            llama_cpp_sys_2::LLAMA_ROPE_TYPE_NEOX => Some(RopeType::NeoX),
283            llama_cpp_sys_2::LLAMA_ROPE_TYPE_MROPE => Some(RopeType::MRope),
284            llama_cpp_sys_2::LLAMA_ROPE_TYPE_VISION => Some(RopeType::Vision),
285            rope_type => {
286                tracing::error!(rope_type = rope_type, "Unexpected rope type from llama.cpp");
287                None
288            }
289        }
290    }
291
292    /// Get chat template from model by name. If the name parameter is None, the default chat template will be returned.
293    ///
294    /// You supply this into [`Self::apply_chat_template`] to get back a string with the appropriate template
295    /// substitution applied to convert a list of messages into a prompt the LLM can use to complete
296    /// the chat.
297    ///
298    /// You could also use an external jinja parser, like [minijinja](https://github.com/mitsuhiko/minijinja),
299    /// to parse jinja templates not supported by the llama.cpp template engine.
300    ///
301    /// # Errors
302    ///
303    /// * If the model has no chat template by that name
304    /// * If the chat template is not a valid [`CString`].
305    pub fn chat_template(
306        &self,
307        name: Option<&str>,
308    ) -> Result<LlamaChatTemplate, ChatTemplateError> {
309        let name_cstr = name.map(CString::new);
310        let name_ptr = match name_cstr {
311            Some(Ok(name)) => name.as_ptr(),
312            _ => std::ptr::null(),
313        };
314        let result =
315            unsafe { llama_cpp_sys_2::llama_model_chat_template(self.model.as_ptr(), name_ptr) };
316
317        // Convert result to Rust String if not null
318        if result.is_null() {
319            Err(ChatTemplateError::MissingTemplate)
320        } else {
321            let chat_template_cstr = unsafe { CStr::from_ptr(result) };
322            let chat_template = CString::new(chat_template_cstr.to_bytes())?;
323            Ok(LlamaChatTemplate(chat_template))
324        }
325    }
326
327    /// Loads a model from a file.
328    ///
329    /// # Errors
330    ///
331    /// See [`LlamaModelLoadError`] for more information.
332    #[tracing::instrument(skip_all, fields(params))]
333    pub fn load_from_file(
334        _: &LlamaBackend,
335        path: impl AsRef<Path>,
336        params: &LlamaModelParams,
337    ) -> Result<Self, LlamaModelLoadError> {
338        let path = path.as_ref();
339        debug_assert!(Path::new(path).exists(), "{path:?} does not exist");
340        let path = path
341            .to_str()
342            .ok_or(LlamaModelLoadError::PathToStrError(path.to_path_buf()))?;
343
344        let cstr = CString::new(path)?;
345        let llama_model =
346            unsafe { llama_cpp_sys_2::llama_load_model_from_file(cstr.as_ptr(), params.params) };
347
348        let model = NonNull::new(llama_model).ok_or(LlamaModelLoadError::NullResult)?;
349
350        tracing::debug!(?path, "Loaded model");
351        Ok(LlamaModel { model })
352    }
353
354    /// Initializes a lora adapter from a file.
355    ///
356    /// # Errors
357    ///
358    /// See [`LlamaLoraAdapterInitError`] for more information.
359    pub fn lora_adapter_init(
360        &self,
361        path: impl AsRef<Path>,
362    ) -> Result<LlamaLoraAdapter, LlamaLoraAdapterInitError> {
363        let path = path.as_ref();
364        debug_assert!(Path::new(path).exists(), "{path:?} does not exist");
365
366        let path = path
367            .to_str()
368            .ok_or(LlamaLoraAdapterInitError::PathToStrError(
369                path.to_path_buf(),
370            ))?;
371
372        let cstr = CString::new(path)?;
373        let adapter =
374            unsafe { llama_cpp_sys_2::llama_adapter_lora_init(self.model.as_ptr(), cstr.as_ptr()) };
375
376        let adapter = NonNull::new(adapter).ok_or(LlamaLoraAdapterInitError::NullResult)?;
377
378        tracing::debug!(?path, "Initialized lora adapter");
379        Ok(LlamaLoraAdapter {
380            lora_adapter: adapter,
381        })
382    }
383
384    /// Create a new context from this model.
385    ///
386    /// # Errors
387    ///
388    /// There is many ways this can fail. See [`LlamaContextLoadError`] for more information.
389    // we intentionally do not derive Copy on `LlamaContextParams` to allow llama.cpp to change the type to be non-trivially copyable.
390    #[allow(clippy::needless_pass_by_value)]
391    pub fn new_context<'a>(
392        &'a self,
393        _: &LlamaBackend,
394        params: LlamaContextParams,
395    ) -> Result<LlamaContext<'a>, LlamaContextLoadError> {
396        let context_params = params.context_params;
397        let context = unsafe {
398            llama_cpp_sys_2::llama_new_context_with_model(self.model.as_ptr(), context_params)
399        };
400        let context = NonNull::new(context).ok_or(LlamaContextLoadError::NullReturn)?;
401
402        Ok(LlamaContext::new(self, context, params.embeddings()))
403    }
404
405    /// Create a new context bound to another context via llama.cpp's `ctx_other` field.
406    ///
407    /// This is required for MTP speculative decoding when the target model's
408    /// architecture uses `LLM_ARCH_GEMMA4_ASSISTANT`, which asserts that the draft
409    /// context references the target context so KV state can be shared.
410    ///
411    /// # Errors
412    ///
413    /// See [`LlamaContextLoadError`].
414    #[allow(clippy::needless_pass_by_value)]
415    pub fn new_context_with_ctx_other<'a>(
416        &'a self,
417        _: &LlamaBackend,
418        params: LlamaContextParams,
419        ctx_other: &LlamaContext<'_>,
420    ) -> Result<LlamaContext<'a>, LlamaContextLoadError> {
421        let mut context_params = params.context_params;
422        context_params.ctx_other = ctx_other.context.as_ptr();
423        let context = unsafe {
424            llama_cpp_sys_2::llama_new_context_with_model(self.model.as_ptr(), context_params)
425        };
426        let context = NonNull::new(context).ok_or(LlamaContextLoadError::NullReturn)?;
427
428        Ok(LlamaContext::new(self, context, params.embeddings()))
429    }
430
431    /// Creates a new context with backend samplers attached for specific sequences.
432    ///
433    /// Ownership of the samplers is transferred to the context, ensuring they remain
434    /// alive for the context's lifetime. Only samplers that support backend execution
435    /// (greedy, dist, temp, top_k, top_p, min_p, logit_bias) will run on the backend.
436    ///
437    /// # Arguments
438    ///
439    /// * `params` - Context parameters
440    /// * `samplers` - Iterator of `(seq_id, sampler)` pairs where sampler must be a chain
441    ///
442    /// # Example
443    ///
444    /// ```rust,ignore
445    /// let sampler = LlamaSampler::chain([
446    ///     LlamaSampler::min_p(0.01, 64),
447    ///     LlamaSampler::temp(0.1),
448    ///     LlamaSampler::dist(42),
449    /// ], false);
450    ///
451    /// let ctx = model.new_context_with_samplers(
452    ///     &backend,
453    ///     ctx_params,
454    ///     [(0, sampler)],
455    /// )?;
456    /// ```
457    #[allow(clippy::needless_pass_by_value)]
458    pub fn new_context_with_samplers<'a>(
459        &'a self,
460        _: &LlamaBackend,
461        params: LlamaContextParams,
462        samplers: impl IntoIterator<Item = (i32, LlamaSampler)>,
463    ) -> Result<LlamaContext<'a>, LlamaContextLoadError> {
464        let samplers: Vec<_> = samplers.into_iter().collect();
465        let mut context_params = params.context_params;
466
467        let mut sampler_configs: Vec<llama_cpp_sys_2::llama_sampler_seq_config> = samplers
468            .iter()
469            .map(
470                |(seq_id, sampler)| llama_cpp_sys_2::llama_sampler_seq_config {
471                    seq_id: *seq_id,
472                    sampler: sampler.sampler,
473                },
474            )
475            .collect();
476
477        if !sampler_configs.is_empty() {
478            context_params.samplers = sampler_configs.as_mut_ptr();
479            context_params.n_samplers = sampler_configs.len();
480        }
481
482        let context = unsafe {
483            llama_cpp_sys_2::llama_new_context_with_model(self.model.as_ptr(), context_params)
484        };
485        let context = NonNull::new(context).ok_or(LlamaContextLoadError::NullReturn)?;
486
487        Ok(LlamaContext::with_samplers(
488            self,
489            context,
490            params.embeddings(),
491            samplers,
492        ))
493    }
494
495    /// Apply the models chat template to some messages.
496    /// See <https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template>
497    ///
498    /// Unlike the llama.cpp `apply_chat_template` which just randomly uses the ChatML template when given
499    /// a null pointer for the template, this requires an explicit template to be specified. If you want to
500    /// use "chatml", then just do `LlamaChatTemplate::new("chatml")` or any other model name or template
501    /// string.
502    ///
503    /// Use [`Self::chat_template`] to retrieve the template baked into the model (this is the preferred
504    /// mechanism as using the wrong chat template can result in really unexpected responses from the LLM).
505    ///
506    /// You probably want to set `add_ass` to true so that the generated template string ends with a the
507    /// opening tag of the assistant. If you fail to leave a hanging chat tag, the model will likely generate
508    /// one into the output and the output may also have unexpected output aside from that.
509    ///
510    /// # Errors
511    /// There are many ways this can fail. See [`ApplyChatTemplateError`] for more information.
512    #[tracing::instrument(skip_all)]
513    pub fn apply_chat_template(
514        &self,
515        tmpl: &LlamaChatTemplate,
516        chat: &[LlamaChatMessage],
517        add_ass: bool,
518    ) -> Result<String, ApplyChatTemplateError> {
519        // Buffer is twice the length of messages per their recommendation
520        let message_length = chat.iter().fold(0, |acc, c| {
521            acc + c.role.to_bytes().len() + c.content.to_bytes().len()
522        });
523        let mut buff: Vec<u8> = vec![0; message_length * 2];
524
525        // Build our llama_cpp_sys_2 chat messages
526        let chat: Vec<llama_cpp_sys_2::llama_chat_message> = chat
527            .iter()
528            .map(|c| llama_cpp_sys_2::llama_chat_message {
529                role: c.role.as_ptr(),
530                content: c.content.as_ptr(),
531            })
532            .collect();
533
534        let tmpl_ptr = tmpl.0.as_ptr();
535
536        let res = unsafe {
537            llama_cpp_sys_2::llama_chat_apply_template(
538                tmpl_ptr,
539                chat.as_ptr(),
540                chat.len(),
541                add_ass,
542                buff.as_mut_ptr().cast::<c_char>(),
543                buff.len().try_into().expect("Buffer size exceeds i32::MAX"),
544            )
545        };
546
547        if res < 0 {
548            return Err(ApplyChatTemplateError::FfiError(res));
549        }
550
551        if res > buff.len().try_into().expect("Buffer size exceeds i32::MAX") {
552            buff.resize(res.try_into().expect("res is negative"), 0);
553
554            let res = unsafe {
555                llama_cpp_sys_2::llama_chat_apply_template(
556                    tmpl_ptr,
557                    chat.as_ptr(),
558                    chat.len(),
559                    add_ass,
560                    buff.as_mut_ptr().cast::<c_char>(),
561                    buff.len().try_into().expect("Buffer size exceeds i32::MAX"),
562                )
563            };
564            if res < 0 {
565                return Err(ApplyChatTemplateError::FfiError(res));
566            }
567            assert_eq!(Ok(res), buff.len().try_into());
568        }
569        buff.truncate(res.try_into().expect("res is negative"));
570        Ok(String::from_utf8(buff)?)
571    }
572}
573
574/// Generic helper function for extracting string values from the C API
575/// This are specifically useful for the the metadata functions, where we pass in a buffer
576/// to be populated by a string, not yet knowing if the buffer is large enough.
577/// If the buffer was not large enough, we get the correct length back, which can be used to
578/// construct a buffer of appropriate size.
579fn extract_meta_string<F>(c_function: F, capacity: usize) -> Result<String, MetaValError>
580where
581    F: Fn(*mut c_char, usize) -> i32,
582{
583    let mut buffer = vec![0u8; capacity];
584
585    // call the foreign function
586    let result = c_function(buffer.as_mut_ptr().cast::<c_char>(), buffer.len());
587    if result < 0 {
588        return Err(MetaValError::NegativeReturn(result));
589    }
590
591    // check if the response fit in our buffer
592    let returned_len = result as usize;
593    if returned_len >= capacity {
594        // buffer wasn't large enough, try again with the correct capacity.
595        return extract_meta_string(c_function, returned_len + 1);
596    }
597
598    // verify null termination
599    debug_assert_eq!(
600        buffer.get(returned_len),
601        Some(&0),
602        "should end with null byte"
603    );
604
605    // resize, convert, and return
606    buffer.truncate(returned_len);
607    Ok(String::from_utf8(buffer)?)
608}
609
610impl Drop for LlamaModel {
611    fn drop(&mut self) {
612        unsafe { llama_cpp_sys_2::llama_free_model(self.model.as_ptr()) }
613    }
614}
llama_cpp_2/model.rs

llama_cpp_2/
model.rs
