llama_cpp_2/
model.rs

1//! A safe wrapper around `llama_model`.
2use std::ffi::{c_char, CStr, CString};
3use std::num::NonZeroU16;
4use std::os::raw::c_int;
5use std::path::Path;
6use std::ptr::{self, NonNull};
7use std::slice;
8use std::str::Utf8Error;
9
10use crate::context::params::LlamaContextParams;
11use crate::context::LlamaContext;
12use crate::llama_backend::LlamaBackend;
13use crate::model::params::LlamaModelParams;
14use crate::sampling::LlamaSampler;
15use crate::token::LlamaToken;
16use crate::token_type::{LlamaTokenAttr, LlamaTokenAttrs};
17use crate::{
18    ApplyChatTemplateError, ChatTemplateError, LlamaContextLoadError, LlamaLoraAdapterInitError,
19    LlamaModelLoadError, MetaValError, NewLlamaChatMessageError, StringToTokenError,
20    TokenToStringError,
21};
22
23pub mod params;
24
25/// A safe wrapper around `llama_model`.
26#[derive(Debug)]
27#[repr(transparent)]
28#[allow(clippy::module_name_repetitions)]
29pub struct LlamaModel {
30    pub(crate) model: NonNull<llama_cpp_sys_2::llama_model>,
31}
32
33/// A safe wrapper around `llama_lora_adapter`.
34#[derive(Debug)]
35#[repr(transparent)]
36#[allow(clippy::module_name_repetitions)]
37pub struct LlamaLoraAdapter {
38    pub(crate) lora_adapter: NonNull<llama_cpp_sys_2::llama_adapter_lora>,
39}
40
41/// A performance-friendly wrapper around [`LlamaModel::chat_template`] which is then
42/// fed into [`LlamaModel::apply_chat_template`] to convert a list of messages into an LLM
43/// prompt. Internally the template is stored as a `CString` to avoid round-trip conversions
44/// within the FFI.
45#[derive(Eq, PartialEq, Clone, PartialOrd, Ord, Hash)]
46pub struct LlamaChatTemplate(CString);
47
48impl LlamaChatTemplate {
49    /// Create a new template from a string. This can either be the name of a llama.cpp [chat template](https://github.com/ggerganov/llama.cpp/blob/8a8c4ceb6050bd9392609114ca56ae6d26f5b8f5/src/llama-chat.cpp#L27-L61)
50    /// like "chatml" or "llama3" or an actual Jinja template for llama.cpp to interpret.
51    pub fn new(template: &str) -> Result<Self, std::ffi::NulError> {
52        Ok(Self(CString::new(template)?))
53    }
54
55    /// Accesses the template as a c string reference.
56    pub fn as_c_str(&self) -> &CStr {
57        &self.0
58    }
59
60    /// Attempts to convert the `CString` into a Rust str reference.
61    pub fn to_str(&self) -> Result<&str, Utf8Error> {
62        self.0.to_str()
63    }
64
65    /// Convenience method to create an owned String.
66    pub fn to_string(&self) -> Result<String, Utf8Error> {
67        self.to_str().map(str::to_string)
68    }
69}
70
71impl std::fmt::Debug for LlamaChatTemplate {
72    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
73        self.0.fmt(f)
74    }
75}
76
77/// A Safe wrapper around `llama_chat_message`
78#[derive(Debug, Eq, PartialEq, Clone)]
79pub struct LlamaChatMessage {
80    role: CString,
81    content: CString,
82}
83
84impl LlamaChatMessage {
85    /// Create a new `LlamaChatMessage`
86    ///
87    /// # Errors
88    /// If either of ``role`` or ``content`` contain null bytes.
89    pub fn new(role: String, content: String) -> Result<Self, NewLlamaChatMessageError> {
90        Ok(Self {
91            role: CString::new(role)?,
92            content: CString::new(content)?,
93        })
94    }
95}
96
97/// The Rope type that's used within the model.
98#[derive(Debug, Clone, Copy, PartialEq, Eq)]
99pub enum RopeType {
100    Norm,
101    NeoX,
102    MRope,
103    Vision,
104}
105
106/// How to determine if we should prepend a bos token to tokens
107#[derive(Debug, Clone, Copy, PartialEq, Eq)]
108pub enum AddBos {
109    /// Add the beginning of stream token to the start of the string.
110    Always,
111    /// Do not add the beginning of stream token to the start of the string.
112    Never,
113}
114
115/// How to determine if we should tokenize special tokens
116#[deprecated(
117    since = "0.1.0",
118    note = "This enum is a mixture of options for llama cpp providing less flexibility it only used with deprecated methods and will be removed in the future."
119)]
120#[derive(Debug, Clone, Copy, PartialEq, Eq)]
121pub enum Special {
122    /// Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. Does not insert a leading space.
123    Tokenize,
124    /// Treat special and/or control tokens as plaintext.
125    Plaintext,
126}
127
128unsafe impl Send for LlamaModel {}
129
130unsafe impl Sync for LlamaModel {}
131
132impl LlamaModel {
133    pub(crate) fn vocab_ptr(&self) -> *const llama_cpp_sys_2::llama_vocab {
134        unsafe { llama_cpp_sys_2::llama_model_get_vocab(self.model.as_ptr()) }
135    }
136
137    /// get the number of tokens the model was trained on
138    ///
139    /// # Panics
140    ///
141    /// If the number of tokens the model was trained on does not fit into an `u32`. This should be impossible on most
142    /// platforms due to llama.cpp returning a `c_int` (i32 on most platforms) which is almost certainly positive.
143    #[must_use]
144    pub fn n_ctx_train(&self) -> u32 {
145        let n_ctx_train = unsafe { llama_cpp_sys_2::llama_n_ctx_train(self.model.as_ptr()) };
146        u32::try_from(n_ctx_train).expect("n_ctx_train fits into an u32")
147    }
148
149    /// Get all tokens in the model.
150    pub fn tokens(
151        &self,
152        decode_special: bool,
153    ) -> impl Iterator<Item = (LlamaToken, Result<String, TokenToStringError>)> + '_ {
154        (0..self.n_vocab())
155            .map(LlamaToken::new)
156            .map(move |llama_token| {
157                let mut decoder = encoding_rs::UTF_8.new_decoder();
158                (
159                    llama_token,
160                    self.token_to_piece(llama_token, &mut decoder, decode_special, None),
161                )
162            })
163    }
164
165    /// Get the beginning of stream token.
166    #[must_use]
167    pub fn token_bos(&self) -> LlamaToken {
168        let token = unsafe { llama_cpp_sys_2::llama_token_bos(self.vocab_ptr()) };
169        LlamaToken(token)
170    }
171
172    /// Get the end of stream token.
173    #[must_use]
174    pub fn token_eos(&self) -> LlamaToken {
175        let token = unsafe { llama_cpp_sys_2::llama_token_eos(self.vocab_ptr()) };
176        LlamaToken(token)
177    }
178
179    /// Get the newline token.
180    #[must_use]
181    pub fn token_nl(&self) -> LlamaToken {
182        let token = unsafe { llama_cpp_sys_2::llama_token_nl(self.vocab_ptr()) };
183        LlamaToken(token)
184    }
185
186    /// Check if a token represents the end of generation (end of turn, end of sequence, etc.)
187    #[must_use]
188    pub fn is_eog_token(&self, token: LlamaToken) -> bool {
189        unsafe { llama_cpp_sys_2::llama_token_is_eog(self.vocab_ptr(), token.0) }
190    }
191
192    /// Get the decoder start token.
193    #[must_use]
194    pub fn decode_start_token(&self) -> LlamaToken {
195        let token =
196            unsafe { llama_cpp_sys_2::llama_model_decoder_start_token(self.model.as_ptr()) };
197        LlamaToken(token)
198    }
199
200    /// Get the separator token (SEP).
201    #[must_use]
202    pub fn token_sep(&self) -> LlamaToken {
203        let token = unsafe { llama_cpp_sys_2::llama_vocab_sep(self.vocab_ptr()) };
204        LlamaToken(token)
205    }
206
207    /// Convert single token to a string.
208    ///
209    /// # Errors
210    ///
211    /// See [`TokenToStringError`] for more information.
212    #[deprecated(since = "0.1.0", note = "Use `token_to_piece` instead")]
213    pub fn token_to_str(
214        &self,
215        token: LlamaToken,
216        special: Special,
217    ) -> Result<String, TokenToStringError> {
218        // TODO lsptrip None is acutally not quite the origignal behavior of this function,
219        let mut decoder = encoding_rs::UTF_8.new_decoder();
220        self.token_to_piece(
221            token,
222            &mut decoder,
223            matches!(special, Special::Tokenize),
224            None,
225        )
226    }
227
228    /// Convert single token to bytes.
229    ///
230    /// # Errors
231    /// See [`TokenToStringError`] for more information.
232    ///
233    /// # Panics
234    /// If a [`TokenToStringError::InsufficientBufferSpace`] error returned by
235    /// [`Self::token_to_bytes_with_size`] contains a positive nonzero value. This should never
236    /// happen.
237    #[deprecated(since = "0.1.0", note = "Use `token_to_piece_bytes` instead")]
238    pub fn token_to_bytes(
239        &self,
240        token: LlamaToken,
241        special: Special,
242    ) -> Result<Vec<u8>, TokenToStringError> {
243        // TODO lsptrip None is acutally not quite the origignal behavior of this function,
244        match self.token_to_piece_bytes(token, 8, matches!(special, Special::Tokenize), None) {
245            Err(TokenToStringError::InsufficientBufferSpace(i)) => self.token_to_piece_bytes(
246                token,
247                (-i).try_into().expect("Error buffer size is positive"),
248                matches!(special, Special::Tokenize),
249                None,
250            ),
251            x => x,
252        }
253    }
254
255    /// Convert a vector of tokens to a single string.
256    ///
257    /// # Errors
258    ///
259    /// See [`TokenToStringError`] for more information.
260    #[deprecated(
261        since = "0.1.0",
262        note = "Use `token_to_piece` for each token individually instead"
263    )]
264    pub fn tokens_to_str(
265        &self,
266        tokens: &[LlamaToken],
267        special: Special,
268    ) -> Result<String, TokenToStringError> {
269        let mut builder: Vec<u8> = Vec::with_capacity(tokens.len() * 4);
270        for piece in tokens
271            .iter()
272            .copied()
273            .map(|t| self.token_to_piece_bytes(t, 8, matches!(special, Special::Tokenize), None))
274        {
275            builder.extend_from_slice(&piece?);
276        }
277        Ok(String::from_utf8(builder)?)
278    }
279
280    /// Convert a string to a Vector of tokens.
281    ///
282    /// # Errors
283    ///
284    /// - if [`str`] contains a null byte.
285    ///
286    /// # Panics
287    ///
288    /// - if there is more than [`usize::MAX`] [`LlamaToken`]s in [`str`].
289    ///
290    ///
291    /// ```no_run
292    /// use llama_cpp_2::model::LlamaModel;
293    ///
294    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
295    /// use std::path::Path;
296    /// use llama_cpp_2::model::AddBos;
297    /// let backend = llama_cpp_2::llama_backend::LlamaBackend::init()?;
298    /// let model = LlamaModel::load_from_file(&backend, Path::new("path/to/model"), &Default::default())?;
299    /// let tokens = model.str_to_token("Hello, World!", AddBos::Always)?;
300    /// # Ok(())
301    /// # }
302    pub fn str_to_token(
303        &self,
304        str: &str,
305        add_bos: AddBos,
306    ) -> Result<Vec<LlamaToken>, StringToTokenError> {
307        let add_bos = match add_bos {
308            AddBos::Always => true,
309            AddBos::Never => false,
310        };
311
312        let tokens_estimation = std::cmp::max(8, (str.len() / 2) + usize::from(add_bos));
313        let mut buffer: Vec<LlamaToken> = Vec::with_capacity(tokens_estimation);
314
315        let c_string = CString::new(str)?;
316        let buffer_capacity =
317            c_int::try_from(buffer.capacity()).expect("buffer capacity should fit into a c_int");
318
319        let size = unsafe {
320            llama_cpp_sys_2::llama_tokenize(
321                self.vocab_ptr(),
322                c_string.as_ptr(),
323                c_int::try_from(c_string.as_bytes().len())?,
324                buffer.as_mut_ptr().cast::<llama_cpp_sys_2::llama_token>(),
325                buffer_capacity,
326                add_bos,
327                true,
328            )
329        };
330
331        // if we fail the first time we can resize the vector to the correct size and try again. This should never fail.
332        // as a result - size is guaranteed to be positive here.
333        let size = if size.is_negative() {
334            buffer.reserve_exact(usize::try_from(-size).expect("usize's are larger "));
335            unsafe {
336                llama_cpp_sys_2::llama_tokenize(
337                    self.vocab_ptr(),
338                    c_string.as_ptr(),
339                    c_int::try_from(c_string.as_bytes().len())?,
340                    buffer.as_mut_ptr().cast::<llama_cpp_sys_2::llama_token>(),
341                    -size,
342                    add_bos,
343                    true,
344                )
345            }
346        } else {
347            size
348        };
349
350        let size = usize::try_from(size).expect("size is positive and usize ");
351
352        // Safety: `size` < `capacity` and llama-cpp has initialized elements up to `size`
353        unsafe { buffer.set_len(size) }
354        Ok(buffer)
355    }
356
357    /// Get the type of a token.
358    ///
359    /// # Panics
360    ///
361    /// If the token type is not known to this library.
362    #[must_use]
363    pub fn token_attr(&self, LlamaToken(id): LlamaToken) -> LlamaTokenAttrs {
364        let token_type = unsafe { llama_cpp_sys_2::llama_token_get_attr(self.vocab_ptr(), id) };
365        LlamaTokenAttrs::try_from(token_type).expect("token type is valid")
366    }
367
368    /// Convert a token to a string using the underlying llama.cpp `llama_token_to_piece` function.
369    ///
370    /// This is the new default function for token decoding and provides direct access to
371    /// the llama.cpp token decoding functionality without any special logic or filtering.
372    ///
373    /// Decoding raw string requires using an decoder, tokens from language models may not always map
374    /// to full characters depending on the encoding so stateful decoding is required, otherwise partial strings may be lost!
375    /// Invalid characters are mapped to REPLACEMENT CHARACTER making the method safe to use even if the model inherently produces
376    /// garbage.
377    ///
378    /// # Errors
379    ///
380    /// - if the token type is unknown
381    ///
382    /// # Panics
383    ///
384    /// - if the returned size from llama-cpp does not fit into a [`usize`]. (this should never happen)
385    pub fn token_to_piece(
386        &self,
387        token: LlamaToken,
388        decoder: &mut encoding_rs::Decoder,
389        special: bool,
390        lstrip: Option<NonZeroU16>,
391    ) -> Result<String, TokenToStringError> {
392        let bytes = match self.token_to_piece_bytes(token, 8, special, lstrip) {
393            // when there is insufficient space `token_to_piece` will return a negative number with the size that would have been returned
394            // https://github.com/abetlen/llama-cpp-python/blob/c37132bac860fcc333255c36313f89c4f49d4c8d/llama_cpp/llama_cpp.py#L3461
395            Err(TokenToStringError::InsufficientBufferSpace(i)) => self.token_to_piece_bytes(
396                token,
397                (-i).try_into().expect("Error buffer size is positive"),
398                special,
399                lstrip,
400            ),
401            x => x,
402        }?;
403        Ok(decode_piece(decoder, &bytes))
404    }
405
406    /// Raw token decoding to bytes, use if you want to handle the decoding model output yourself
407    ///
408    /// Convert a token to bytes using the underlying llama.cpp `llama_token_to_piece` function. This is mostly
409    /// a thin wrapper around `llama_token_to_piece` function, that handles rust <-> c type conversions while
410    /// letting the caller handle errors. For a safer inteface returing rust strings directly use `token_to_piece` instead!
411    ///
412    /// # Errors
413    ///
414    /// - if the token type is unknown
415    /// - the resultant token is larger than `buffer_size`.
416    ///
417    /// # Panics
418    ///
419    /// - if `buffer_size` does not fit into a [`c_int`].
420    /// - if the returned size from llama-cpp does not fit into a [`usize`]. (this should never happen)
421    pub fn token_to_piece_bytes(
422        &self,
423        token: LlamaToken,
424        buffer_size: usize,
425        special: bool,
426        lstrip: Option<NonZeroU16>,
427    ) -> Result<Vec<u8>, TokenToStringError> {
428        let string = CString::new(vec![b'*'; buffer_size]).expect("no null");
429        let len = string.as_bytes().len();
430        let len = c_int::try_from(len).expect("length fits into c_int");
431        let buf = string.into_raw();
432        let lstrip = lstrip.map_or(0, |it| i32::from(it.get()));
433        let size = unsafe {
434            llama_cpp_sys_2::llama_token_to_piece(
435                self.vocab_ptr(),
436                token.0,
437                buf,
438                len,
439                lstrip,
440                special,
441            )
442        };
443
444        match size {
445            0 => Err(TokenToStringError::UnknownTokenType),
446            i if i.is_negative() => Err(TokenToStringError::InsufficientBufferSpace(i)),
447            size => {
448                let string = unsafe { CString::from_raw(buf) };
449                let mut bytes = string.into_bytes();
450                let len = usize::try_from(size).expect("size is positive and fits into usize");
451                bytes.truncate(len);
452                Ok(bytes)
453            }
454        }
455    }
456
457    /// Convert a token to a string with a specified buffer size.
458    ///
459    /// Generally you should use [`LlamaModel::token_to_str`] as it is able to decode tokens with
460    /// any length.
461    ///
462    /// # Errors
463    ///
464    /// - if the token type is unknown
465    /// - the resultant token is larger than `buffer_size`.
466    /// - the string returend by llama-cpp is not valid utf8.
467    ///
468    /// # Panics
469    ///
470    /// - if `buffer_size` does not fit into a [`c_int`].
471    /// - if the returned size from llama-cpp does not fit into a [`usize`]. (this should never happen)
472    #[deprecated(since = "0.1.0", note = "Use `token_to_piece` instead")]
473    pub fn token_to_str_with_size(
474        &self,
475        token: LlamaToken,
476        buffer_size: usize,
477        special: Special,
478    ) -> Result<String, TokenToStringError> {
479        let bytes = self.token_to_piece_bytes(
480            token,
481            buffer_size,
482            matches!(special, Special::Tokenize),
483            None,
484        )?;
485        Ok(String::from_utf8(bytes)?)
486    }
487
488    /// Convert a token to bytes with a specified buffer size.
489    ///
490    /// Generally you should use [`LlamaModel::token_to_bytes`] as it is able to handle tokens of
491    /// any length.
492    ///
493    /// # Errors
494    ///
495    /// - if the token type is unknown
496    /// - the resultant token is larger than `buffer_size`.
497    ///
498    /// # Panics
499    ///
500    /// - if `buffer_size` does not fit into a [`c_int`].
501    /// - if the returned size from llama-cpp does not fit into a [`usize`]. (this should never happen)
502    #[deprecated(since = "0.1.0", note = "Use `token_to_piece_bytes` instead")]
503    pub fn token_to_bytes_with_size(
504        &self,
505        token: LlamaToken,
506        buffer_size: usize,
507        special: Special,
508        lstrip: Option<NonZeroU16>,
509    ) -> Result<Vec<u8>, TokenToStringError> {
510        if token == self.token_nl() {
511            return Ok(b"\n".to_vec());
512        }
513
514        // unsure what to do with this in the face of the 'special' arg + attr changes
515        let attrs = self.token_attr(token);
516        if attrs.is_empty()
517            || attrs
518                .intersects(LlamaTokenAttr::Unknown | LlamaTokenAttr::Byte | LlamaTokenAttr::Unused)
519            || attrs.contains(LlamaTokenAttr::Control)
520                && (token == self.token_bos() || token == self.token_eos())
521        {
522            return Ok(Vec::new());
523        }
524
525        let special = match special {
526            Special::Tokenize => true,
527            Special::Plaintext => false,
528        };
529
530        let string = CString::new(vec![b'*'; buffer_size]).expect("no null");
531        let len = string.as_bytes().len();
532        let len = c_int::try_from(len).expect("length fits into c_int");
533        let buf = string.into_raw();
534        let lstrip = lstrip.map_or(0, |it| i32::from(it.get()));
535        let size = unsafe {
536            llama_cpp_sys_2::llama_token_to_piece(
537                self.vocab_ptr(),
538                token.0,
539                buf,
540                len,
541                lstrip,
542                special,
543            )
544        };
545
546        match size {
547            0 => Err(TokenToStringError::UnknownTokenType),
548            i if i.is_negative() => Err(TokenToStringError::InsufficientBufferSpace(i)),
549            size => {
550                let string = unsafe { CString::from_raw(buf) };
551                let mut bytes = string.into_bytes();
552                let len = usize::try_from(size).expect("size is positive and fits into usize");
553                bytes.truncate(len);
554                Ok(bytes)
555            }
556        }
557    }
558    /// The number of tokens the model was trained on.
559    ///
560    /// This returns a `c_int` for maximum compatibility. Most of the time it can be cast to an i32
561    /// without issue.
562    #[must_use]
563    pub fn n_vocab(&self) -> i32 {
564        unsafe { llama_cpp_sys_2::llama_n_vocab(self.vocab_ptr()) }
565    }
566
567    /// The type of vocab the model was trained on.
568    ///
569    /// # Panics
570    ///
571    /// If llama-cpp emits a vocab type that is not known to this library.
572    #[must_use]
573    pub fn vocab_type(&self) -> VocabType {
574        // llama_cpp_sys_2::llama_model_get_vocab
575        let vocab_type = unsafe { llama_cpp_sys_2::llama_vocab_type(self.vocab_ptr()) };
576        VocabType::try_from(vocab_type).expect("invalid vocab type")
577    }
578
579    /// This returns a `c_int` for maximum compatibility. Most of the time it can be cast to an i32
580    /// without issue.
581    #[must_use]
582    pub fn n_embd(&self) -> c_int {
583        unsafe { llama_cpp_sys_2::llama_n_embd(self.model.as_ptr()) }
584    }
585
586    /// The model's *output* embedding width (`n_embd_out`). This is the width
587    /// llama.cpp actually extracts embeddings at — `n_embd` and `n_embd_out`
588    /// diverge when `{arch}.embedding_length_out` is present (deepstack models
589    /// like qwen3vl). Returns a `c_int` for maximum compatibility.
590    #[must_use]
591    pub fn n_embd_out(&self) -> c_int {
592        unsafe { llama_cpp_sys_2::llama_model_n_embd_out(self.model.as_ptr()) }
593    }
594
595    /// The model's classification output width (`n_cls_out`, default 1) — the
596    /// width of a RANK-pooled embeddings read (llama.h:1029).
597    #[must_use]
598    pub fn n_cls_out(&self) -> u32 {
599        unsafe { llama_cpp_sys_2::llama_model_n_cls_out(self.model.as_ptr()) }
600    }
601
602    /// Returns the total size of all the tensors in the model in bytes.
603    pub fn size(&self) -> u64 {
604        unsafe { llama_cpp_sys_2::llama_model_size(self.model.as_ptr()) }
605    }
606
607    /// Returns the number of parameters in the model.
608    pub fn n_params(&self) -> u64 {
609        unsafe { llama_cpp_sys_2::llama_model_n_params(self.model.as_ptr()) }
610    }
611
612    /// Returns whether the model is a recurrent network (Mamba, RWKV, etc)
613    pub fn is_recurrent(&self) -> bool {
614        unsafe { llama_cpp_sys_2::llama_model_is_recurrent(self.model.as_ptr()) }
615    }
616
617    /// Returns whether the model is a hybrid network (Jamba, Granite, Qwen3xx, etc)
618    ///
619    /// Hybrid models have both attention layers and recurrent/SSM layers.
620    /// They require special handling for state checkpointing.
621    pub fn is_hybrid(&self) -> bool {
622        unsafe { llama_cpp_sys_2::llama_model_is_hybrid(self.model.as_ptr()) }
623    }
624
625    /// Returns the number of layers within the model.
626    pub fn n_layer(&self) -> u32 {
627        // It's never possible for this to panic because while the API interface is defined as an int32_t,
628        // the field it's accessing is a uint32_t.
629        u32::try_from(unsafe { llama_cpp_sys_2::llama_model_n_layer(self.model.as_ptr()) }).unwrap()
630    }
631
632    /// Returns the number of attention heads within the model.
633    pub fn n_head(&self) -> u32 {
634        // It's never possible for this to panic because while the API interface is defined as an int32_t,
635        // the field it's accessing is a uint32_t.
636        u32::try_from(unsafe { llama_cpp_sys_2::llama_model_n_head(self.model.as_ptr()) }).unwrap()
637    }
638
639    /// Returns the number of KV attention heads.
640    pub fn n_head_kv(&self) -> u32 {
641        // It's never possible for this to panic because while the API interface is defined as an int32_t,
642        // the field it's accessing is a uint32_t.
643        u32::try_from(unsafe { llama_cpp_sys_2::llama_model_n_head_kv(self.model.as_ptr()) })
644            .unwrap()
645    }
646
647    /// Get metadata value as a string by key name
648    pub fn meta_val_str(&self, key: &str) -> Result<String, MetaValError> {
649        let key_cstring = CString::new(key)?;
650        let key_ptr = key_cstring.as_ptr();
651
652        extract_meta_string(
653            |buf_ptr, buf_len| unsafe {
654                llama_cpp_sys_2::llama_model_meta_val_str(
655                    self.model.as_ptr(),
656                    key_ptr,
657                    buf_ptr,
658                    buf_len,
659                )
660            },
661            256,
662        )
663    }
664
665    /// Get the number of metadata key/value pairs
666    pub fn meta_count(&self) -> i32 {
667        unsafe { llama_cpp_sys_2::llama_model_meta_count(self.model.as_ptr()) }
668    }
669
670    /// Get metadata key name by index
671    pub fn meta_key_by_index(&self, index: i32) -> Result<String, MetaValError> {
672        extract_meta_string(
673            |buf_ptr, buf_len| unsafe {
674                llama_cpp_sys_2::llama_model_meta_key_by_index(
675                    self.model.as_ptr(),
676                    index,
677                    buf_ptr,
678                    buf_len,
679                )
680            },
681            256,
682        )
683    }
684
685    /// Get metadata value as a string by index
686    pub fn meta_val_str_by_index(&self, index: i32) -> Result<String, MetaValError> {
687        extract_meta_string(
688            |buf_ptr, buf_len| unsafe {
689                llama_cpp_sys_2::llama_model_meta_val_str_by_index(
690                    self.model.as_ptr(),
691                    index,
692                    buf_ptr,
693                    buf_len,
694                )
695            },
696            256,
697        )
698    }
699
700    /// Returns the rope type of the model.
701    pub fn rope_type(&self) -> Option<RopeType> {
702        match unsafe { llama_cpp_sys_2::llama_model_rope_type(self.model.as_ptr()) } {
703            llama_cpp_sys_2::LLAMA_ROPE_TYPE_NONE => None,
704            llama_cpp_sys_2::LLAMA_ROPE_TYPE_NORM => Some(RopeType::Norm),
705            llama_cpp_sys_2::LLAMA_ROPE_TYPE_NEOX => Some(RopeType::NeoX),
706            llama_cpp_sys_2::LLAMA_ROPE_TYPE_MROPE => Some(RopeType::MRope),
707            llama_cpp_sys_2::LLAMA_ROPE_TYPE_VISION => Some(RopeType::Vision),
708            rope_type => {
709                tracing::error!(rope_type = rope_type, "Unexpected rope type from llama.cpp");
710                None
711            }
712        }
713    }
714
715    /// Get chat template from model by name. If the name parameter is None, the default chat template will be returned.
716    ///
717    /// You supply this into [`Self::apply_chat_template`] to get back a string with the appropriate template
718    /// substitution applied to convert a list of messages into a prompt the LLM can use to complete
719    /// the chat.
720    ///
721    /// You could also use an external jinja parser, like [minijinja](https://github.com/mitsuhiko/minijinja),
722    /// to parse jinja templates not supported by the llama.cpp template engine.
723    ///
724    /// # Errors
725    ///
726    /// * If the model has no chat template by that name
727    /// * If the chat template is not a valid [`CString`].
728    pub fn chat_template(
729        &self,
730        name: Option<&str>,
731    ) -> Result<LlamaChatTemplate, ChatTemplateError> {
732        let name_cstr = name.map(CString::new);
733        let name_ptr = match name_cstr {
734            Some(Ok(name)) => name.as_ptr(),
735            _ => std::ptr::null(),
736        };
737        let result =
738            unsafe { llama_cpp_sys_2::llama_model_chat_template(self.model.as_ptr(), name_ptr) };
739
740        // Convert result to Rust String if not null
741        if result.is_null() {
742            Err(ChatTemplateError::MissingTemplate)
743        } else {
744            let chat_template_cstr = unsafe { CStr::from_ptr(result) };
745            let chat_template = CString::new(chat_template_cstr.to_bytes())?;
746            Ok(LlamaChatTemplate(chat_template))
747        }
748    }
749
750    /// Loads a model from a file.
751    ///
752    /// # Errors
753    ///
754    /// See [`LlamaModelLoadError`] for more information.
755    #[tracing::instrument(skip_all, fields(params))]
756    pub fn load_from_file(
757        _: &LlamaBackend,
758        path: impl AsRef<Path>,
759        params: &LlamaModelParams,
760    ) -> Result<Self, LlamaModelLoadError> {
761        let path = path.as_ref();
762        debug_assert!(Path::new(path).exists(), "{path:?} does not exist");
763        let path = path
764            .to_str()
765            .ok_or(LlamaModelLoadError::PathToStrError(path.to_path_buf()))?;
766
767        let cstr = CString::new(path)?;
768        let llama_model =
769            unsafe { llama_cpp_sys_2::llama_load_model_from_file(cstr.as_ptr(), params.params) };
770
771        let model = NonNull::new(llama_model).ok_or(LlamaModelLoadError::NullResult)?;
772
773        tracing::debug!(?path, "Loaded model");
774        Ok(LlamaModel { model })
775    }
776
777    /// Initializes a lora adapter from a file.
778    ///
779    /// # Errors
780    ///
781    /// See [`LlamaLoraAdapterInitError`] for more information.
782    pub fn lora_adapter_init(
783        &self,
784        path: impl AsRef<Path>,
785    ) -> Result<LlamaLoraAdapter, LlamaLoraAdapterInitError> {
786        let path = path.as_ref();
787        debug_assert!(Path::new(path).exists(), "{path:?} does not exist");
788
789        let path = path
790            .to_str()
791            .ok_or(LlamaLoraAdapterInitError::PathToStrError(
792                path.to_path_buf(),
793            ))?;
794
795        let cstr = CString::new(path)?;
796        let adapter =
797            unsafe { llama_cpp_sys_2::llama_adapter_lora_init(self.model.as_ptr(), cstr.as_ptr()) };
798
799        let adapter = NonNull::new(adapter).ok_or(LlamaLoraAdapterInitError::NullResult)?;
800
801        tracing::debug!(?path, "Initialized lora adapter");
802        Ok(LlamaLoraAdapter {
803            lora_adapter: adapter,
804        })
805    }
806
807    /// Create a new context from this model.
808    ///
809    /// # Errors
810    ///
811    /// There is many ways this can fail. See [`LlamaContextLoadError`] for more information.
812    // we intentionally do not derive Copy on `LlamaContextParams` to allow llama.cpp to change the type to be non-trivially copyable.
813    #[allow(clippy::needless_pass_by_value)]
814    pub fn new_context<'a>(
815        &'a self,
816        _: &LlamaBackend,
817        params: LlamaContextParams,
818    ) -> Result<LlamaContext<'a>, LlamaContextLoadError> {
819        let context_params = params.context_params;
820        let context = unsafe {
821            llama_cpp_sys_2::llama_new_context_with_model(self.model.as_ptr(), context_params)
822        };
823        let context = NonNull::new(context).ok_or(LlamaContextLoadError::NullReturn)?;
824
825        Ok(LlamaContext::new(self, context, params.embeddings()))
826    }
827
828    /// Create a new context bound to another context via llama.cpp's `ctx_other` field.
829    ///
830    /// This is required for MTP speculative decoding when the target model's
831    /// architecture uses `LLM_ARCH_GEMMA4_ASSISTANT`, which asserts that the draft
832    /// context references the target context so KV state can be shared.
833    ///
834    /// # Errors
835    ///
836    /// See [`LlamaContextLoadError`].
837    #[allow(clippy::needless_pass_by_value)]
838    pub fn new_context_with_ctx_other<'a>(
839        &'a self,
840        _: &LlamaBackend,
841        params: LlamaContextParams,
842        ctx_other: &LlamaContext<'_>,
843    ) -> Result<LlamaContext<'a>, LlamaContextLoadError> {
844        let mut context_params = params.context_params;
845        context_params.ctx_other = ctx_other.context.as_ptr();
846        let context = unsafe {
847            llama_cpp_sys_2::llama_new_context_with_model(self.model.as_ptr(), context_params)
848        };
849        let context = NonNull::new(context).ok_or(LlamaContextLoadError::NullReturn)?;
850
851        Ok(LlamaContext::new(self, context, params.embeddings()))
852    }
853
854    /// Creates a new context with backend samplers attached for specific sequences.
855    ///
856    /// Ownership of the samplers is transferred to the context, ensuring they remain
857    /// alive for the context's lifetime. Only samplers that support backend execution
858    /// (greedy, dist, temp, top_k, top_p, min_p, logit_bias) will run on the backend.
859    ///
860    /// # Arguments
861    ///
862    /// * `params` - Context parameters
863    /// * `samplers` - Iterator of `(seq_id, sampler)` pairs where sampler must be a chain
864    ///
865    /// # Example
866    ///
867    /// ```rust,ignore
868    /// let sampler = LlamaSampler::chain([
869    ///     LlamaSampler::min_p(0.01, 64),
870    ///     LlamaSampler::temp(0.1),
871    ///     LlamaSampler::dist(42),
872    /// ], false);
873    ///
874    /// let ctx = model.new_context_with_samplers(
875    ///     &backend,
876    ///     ctx_params,
877    ///     [(0, sampler)],
878    /// )?;
879    /// ```
880    #[allow(clippy::needless_pass_by_value)]
881    pub fn new_context_with_samplers<'a>(
882        &'a self,
883        _: &LlamaBackend,
884        params: LlamaContextParams,
885        samplers: impl IntoIterator<Item = (i32, LlamaSampler)>,
886    ) -> Result<LlamaContext<'a>, LlamaContextLoadError> {
887        let samplers: Vec<_> = samplers.into_iter().collect();
888        let mut context_params = params.context_params;
889
890        let mut sampler_configs: Vec<llama_cpp_sys_2::llama_sampler_seq_config> = samplers
891            .iter()
892            .map(
893                |(seq_id, sampler)| llama_cpp_sys_2::llama_sampler_seq_config {
894                    seq_id: *seq_id,
895                    sampler: sampler.sampler,
896                },
897            )
898            .collect();
899
900        if !sampler_configs.is_empty() {
901            context_params.samplers = sampler_configs.as_mut_ptr();
902            context_params.n_samplers = sampler_configs.len();
903        }
904
905        let context = unsafe {
906            llama_cpp_sys_2::llama_new_context_with_model(self.model.as_ptr(), context_params)
907        };
908        let context = NonNull::new(context).ok_or(LlamaContextLoadError::NullReturn)?;
909
910        Ok(LlamaContext::with_samplers(
911            self,
912            context,
913            params.embeddings(),
914            samplers,
915        ))
916    }
917
918    /// Apply the models chat template to some messages.
919    /// See <https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template>
920    ///
921    /// Unlike the llama.cpp `apply_chat_template` which just randomly uses the ChatML template when given
922    /// a null pointer for the template, this requires an explicit template to be specified. If you want to
923    /// use "chatml", then just do `LlamaChatTemplate::new("chatml")` or any other model name or template
924    /// string.
925    ///
926    /// Use [`Self::chat_template`] to retrieve the template baked into the model (this is the preferred
927    /// mechanism as using the wrong chat template can result in really unexpected responses from the LLM).
928    ///
929    /// You probably want to set `add_ass` to true so that the generated template string ends with a the
930    /// opening tag of the assistant. If you fail to leave a hanging chat tag, the model will likely generate
931    /// one into the output and the output may also have unexpected output aside from that.
932    ///
933    /// # Errors
934    /// There are many ways this can fail. See [`ApplyChatTemplateError`] for more information.
935    #[tracing::instrument(skip_all)]
936    pub fn apply_chat_template(
937        &self,
938        tmpl: &LlamaChatTemplate,
939        chat: &[LlamaChatMessage],
940        add_ass: bool,
941    ) -> Result<String, ApplyChatTemplateError> {
942        // Buffer is twice the length of messages per their recommendation
943        let message_length = chat.iter().fold(0, |acc, c| {
944            acc + c.role.to_bytes().len() + c.content.to_bytes().len()
945        });
946        let mut buff: Vec<u8> = vec![0; message_length * 2];
947
948        // Build our llama_cpp_sys_2 chat messages
949        let chat: Vec<llama_cpp_sys_2::llama_chat_message> = chat
950            .iter()
951            .map(|c| llama_cpp_sys_2::llama_chat_message {
952                role: c.role.as_ptr(),
953                content: c.content.as_ptr(),
954            })
955            .collect();
956
957        let tmpl_ptr = tmpl.0.as_ptr();
958
959        let res = unsafe {
960            llama_cpp_sys_2::llama_chat_apply_template(
961                tmpl_ptr,
962                chat.as_ptr(),
963                chat.len(),
964                add_ass,
965                buff.as_mut_ptr().cast::<c_char>(),
966                buff.len().try_into().expect("Buffer size exceeds i32::MAX"),
967            )
968        };
969
970        if res < 0 {
971            return Err(ApplyChatTemplateError::FfiError(res));
972        }
973
974        if res > buff.len().try_into().expect("Buffer size exceeds i32::MAX") {
975            buff.resize(res.try_into().expect("res is negative"), 0);
976
977            let res = unsafe {
978                llama_cpp_sys_2::llama_chat_apply_template(
979                    tmpl_ptr,
980                    chat.as_ptr(),
981                    chat.len(),
982                    add_ass,
983                    buff.as_mut_ptr().cast::<c_char>(),
984                    buff.len().try_into().expect("Buffer size exceeds i32::MAX"),
985                )
986            };
987            if res < 0 {
988                return Err(ApplyChatTemplateError::FfiError(res));
989            }
990            assert_eq!(Ok(res), buff.len().try_into());
991        }
992        buff.truncate(res.try_into().expect("res is negative"));
993        Ok(String::from_utf8(buff)?)
994    }
995}
996
997/// Generic helper function for extracting string values from the C API
998/// This are specifically useful for the the metadata functions, where we pass in a buffer
999/// to be populated by a string, not yet knowing if the buffer is large enough.
1000/// If the buffer was not large enough, we get the correct length back, which can be used to
1001/// construct a buffer of appropriate size.
1002fn extract_meta_string<F>(c_function: F, capacity: usize) -> Result<String, MetaValError>
1003where
1004    F: Fn(*mut c_char, usize) -> i32,
1005{
1006    let mut buffer = vec![0u8; capacity];
1007
1008    // call the foreign function
1009    let result = c_function(buffer.as_mut_ptr().cast::<c_char>(), buffer.len());
1010    if result < 0 {
1011        return Err(MetaValError::NegativeReturn(result));
1012    }
1013
1014    // check if the response fit in our buffer
1015    let returned_len = result as usize;
1016    if returned_len >= capacity {
1017        // buffer wasn't large enough, try again with the correct capacity.
1018        return extract_meta_string(c_function, returned_len + 1);
1019    }
1020
1021    // verify null termination
1022    debug_assert_eq!(
1023        buffer.get(returned_len),
1024        Some(&0),
1025        "should end with null byte"
1026    );
1027
1028    // resize, convert, and return
1029    buffer.truncate(returned_len);
1030    Ok(String::from_utf8(buffer)?)
1031}
1032
1033impl Drop for LlamaModel {
1034    fn drop(&mut self) {
1035        unsafe { llama_cpp_sys_2::llama_free_model(self.model.as_ptr()) }
1036    }
1037}
1038
1039fn decode_piece(decoder: &mut encoding_rs::Decoder, bytes: &[u8]) -> String {
1040    // `decode_to_string` never grows its destination. The decoder's bound also accounts
1041    // for an incomplete UTF-8 sequence retained from the previous token.
1042    let mut output = String::with_capacity(
1043        decoder
1044            .max_utf8_buffer_length(bytes.len())
1045            .expect("token output is too large to decode"),
1046    );
1047    let (result, read, _) = decoder.decode_to_string(bytes, &mut output, false);
1048    assert!(
1049        matches!(result, encoding_rs::CoderResult::InputEmpty) && read == bytes.len(),
1050        "UTF-8 decoder capacity bound must consume the complete token"
1051    );
1052    output
1053}
1054
1055/// a rusty equivalent of `llama_vocab_type`
1056#[repr(u32)]
1057#[derive(Debug, Eq, Copy, Clone, PartialEq)]
1058pub enum VocabType {
1059    /// Byte Pair Encoding
1060    BPE = llama_cpp_sys_2::LLAMA_VOCAB_TYPE_BPE as _,
1061    /// Sentence Piece Tokenizer
1062    SPM = llama_cpp_sys_2::LLAMA_VOCAB_TYPE_SPM as _,
1063}
1064
1065/// There was an error converting a `llama_vocab_type` to a `VocabType`.
1066#[derive(thiserror::Error, Debug, Eq, PartialEq)]
1067pub enum LlamaTokenTypeFromIntError {
1068    /// The value is not a valid `llama_token_type`. Contains the int value that was invalid.
1069    #[error("Unknown Value {0}")]
1070    UnknownValue(llama_cpp_sys_2::llama_vocab_type),
1071}
1072
1073impl TryFrom<llama_cpp_sys_2::llama_vocab_type> for VocabType {
1074    type Error = LlamaTokenTypeFromIntError;
1075
1076    fn try_from(value: llama_cpp_sys_2::llama_vocab_type) -> Result<Self, Self::Error> {
1077        match value {
1078            llama_cpp_sys_2::LLAMA_VOCAB_TYPE_BPE => Ok(VocabType::BPE),
1079            llama_cpp_sys_2::LLAMA_VOCAB_TYPE_SPM => Ok(VocabType::SPM),
1080            unknown => Err(LlamaTokenTypeFromIntError::UnknownValue(unknown)),
1081        }
1082    }
1083}
1084
1085#[cfg(test)]
1086mod tests {
1087    use super::decode_piece;
1088
1089    #[test]
1090    fn token_decoder_preserves_utf8_split_across_pieces() {
1091        let mut decoder = encoding_rs::UTF_8.new_decoder();
1092
1093        assert_eq!(decode_piece(&mut decoder, &[0xE5, 0x9B]), "");
1094        assert_eq!(decode_piece(&mut decoder, &[0xB2]), "\u{56F2}");
1095    }
1096}
llama_cpp_2/model.rs

llama_cpp_2/
model.rs
