llama_cpp_2/
model.rs

1//! A safe wrapper around `llama_model`.
2use std::ffi::{c_char, CStr, CString};
3use std::os::raw::c_int;
4use std::path::Path;
5use std::str::Utf8Error;
6
7use crate::context::params::LlamaContextParams;
8use crate::context::LlamaContext;
9use crate::llama_backend::LlamaBackend;
10use crate::model::params::LlamaModelParams;
11use crate::ptr::Ptr;
12use crate::sampling::LlamaSampler;
13use crate::token::LlamaToken;
14use crate::vocab::LlamaVocab;
15use crate::{
16    ApplyChatTemplateError, ChatTemplateError, LlamaContextLoadError, LlamaLoraAdapterInitError,
17    LlamaModelLoadError, MetaValError, NewLlamaChatMessageError,
18};
19
20pub mod params;
21// For backwards compat.
22pub use crate::vocab::{LlamaTokenTypeFromIntError, VocabType};
23
24/// A safe wrapper around `llama_model`.
25#[derive(Debug)]
26#[repr(transparent)]
27#[allow(clippy::module_name_repetitions)]
28pub struct LlamaModel {
29    pub(crate) model: Ptr<llama_cpp_sys_2::llama_model>,
30}
31
32// SAFETY: The model is immutable except for methods where it's passed as `&mut`.
33//
34// FIXME(madsmtm): Apart from `llama_new_context_with_model`, which mutates.
35unsafe impl Send for LlamaModel {}
36// SAFETY: Same as above.
37unsafe impl Sync for LlamaModel {}
38
39/// A safe wrapper around `llama_lora_adapter`.
40#[derive(Debug)]
41#[repr(transparent)]
42#[allow(clippy::module_name_repetitions)]
43pub struct LlamaLoraAdapter {
44    pub(crate) lora_adapter: Ptr<llama_cpp_sys_2::llama_adapter_lora>,
45}
46
47// SAFETY: The lora is immutable.
48unsafe impl Send for LlamaLoraAdapter {}
49// SAFETY: Same as above.
50unsafe impl Sync for LlamaLoraAdapter {}
51
52/// A performance-friendly wrapper around [`LlamaModel::chat_template`] which is then
53/// fed into [`LlamaModel::apply_chat_template`] to convert a list of messages into an LLM
54/// prompt. Internally the template is stored as a `CString` to avoid round-trip conversions
55/// within the FFI.
56#[derive(Eq, PartialEq, Clone, PartialOrd, Ord, Hash)]
57pub struct LlamaChatTemplate(CString);
58
59impl LlamaChatTemplate {
60    /// Create a new template from a string. This can either be the name of a llama.cpp [chat template](https://github.com/ggerganov/llama.cpp/blob/8a8c4ceb6050bd9392609114ca56ae6d26f5b8f5/src/llama-chat.cpp#L27-L61)
61    /// like "chatml" or "llama3" or an actual Jinja template for llama.cpp to interpret.
62    pub fn new(template: &str) -> Result<Self, std::ffi::NulError> {
63        Ok(Self(CString::new(template)?))
64    }
65
66    /// Accesses the template as a c string reference.
67    pub fn as_c_str(&self) -> &CStr {
68        &self.0
69    }
70
71    /// Attempts to convert the `CString` into a Rust str reference.
72    pub fn to_str(&self) -> Result<&str, Utf8Error> {
73        self.0.to_str()
74    }
75
76    /// Convenience method to create an owned String.
77    pub fn to_string(&self) -> Result<String, Utf8Error> {
78        self.to_str().map(str::to_string)
79    }
80}
81
82impl std::fmt::Debug for LlamaChatTemplate {
83    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
84        self.0.fmt(f)
85    }
86}
87
88/// A Safe wrapper around `llama_chat_message`
89#[derive(Debug, Eq, PartialEq, Clone)]
90pub struct LlamaChatMessage {
91    role: CString,
92    content: CString,
93}
94
95impl LlamaChatMessage {
96    /// Create a new `LlamaChatMessage`
97    ///
98    /// # Errors
99    /// If either of ``role`` or ``content`` contain null bytes.
100    pub fn new(role: String, content: String) -> Result<Self, NewLlamaChatMessageError> {
101        Ok(Self {
102            role: CString::new(role)?,
103            content: CString::new(content)?,
104        })
105    }
106}
107
108/// The RoPE type that's used within the model.
109#[derive(Debug, Clone, Copy, PartialEq, Eq)]
110pub enum RopeType {
111    /// Normal RoPE.
112    Norm,
113    /// Neox-style dimension interleave.
114    NeoX,
115    /// multi-section rotary embedding (MRoPE).
116    MRope,
117    /// Interleaved MRoPE.
118    IMRope,
119    /// Vision RoPE.
120    Vision,
121}
122
123impl LlamaModel {
124    /// Get the model's vocabulary.
125    #[must_use]
126    pub fn vocab(&self) -> LlamaVocab<'_> {
127        let ptr = unsafe { llama_cpp_sys_2::llama_model_get_vocab(self.model.as_ptr()) };
128        LlamaVocab::new(ptr).expect("model must have vocabulary")
129    }
130
131    /// get the number of tokens the model was trained on
132    ///
133    /// # Panics
134    ///
135    /// If the number of tokens the model was trained on does not fit into an `u32`. This should be impossible on most
136    /// platforms due to llama.cpp returning a `c_int` (i32 on most platforms) which is almost certainly positive.
137    #[must_use]
138    pub fn n_ctx_train(&self) -> u32 {
139        let n_ctx_train = unsafe { llama_cpp_sys_2::llama_n_ctx_train(self.model.as_ptr()) };
140        u32::try_from(n_ctx_train).expect("n_ctx_train fits into an u32")
141    }
142
143    /// Get all tokens in the model.
144    pub fn tokens(&self, decode_special: bool) -> impl Iterator<Item = (LlamaToken, Vec<u8>)> + '_ {
145        let vocab = self.vocab();
146        vocab.tokens().map(move |llama_token| {
147            let bytes = vocab.token_to_piece(llama_token, decode_special, None);
148            (llama_token, bytes)
149        })
150    }
151
152    /// Get the decoder start token.
153    #[must_use]
154    pub fn decode_start_token(&self) -> LlamaToken {
155        let token =
156            unsafe { llama_cpp_sys_2::llama_model_decoder_start_token(self.model.as_ptr()) };
157        LlamaToken(token)
158    }
159
160    /// The number of tokens the model was trained on.
161    ///
162    /// This returns a `c_int` for maximum compatibility. Most of the time it can be cast to an i32
163    /// without issue.
164    #[must_use]
165    pub fn n_vocab(&self) -> i32 {
166        self.vocab().n_tokens()
167    }
168
169    /// This returns a `c_int` for maximum compatibility. Most of the time it can be cast to an i32
170    /// without issue.
171    #[must_use]
172    pub fn n_embd(&self) -> c_int {
173        unsafe { llama_cpp_sys_2::llama_n_embd(self.model.as_ptr()) }
174    }
175
176    /// The model's *output* embedding width (`n_embd_out`). This is the width
177    /// llama.cpp actually extracts embeddings at — `n_embd` and `n_embd_out`
178    /// diverge when `{arch}.embedding_length_out` is present (deepstack models
179    /// like qwen3vl). Returns a `c_int` for maximum compatibility.
180    #[must_use]
181    pub fn n_embd_out(&self) -> c_int {
182        unsafe { llama_cpp_sys_2::llama_model_n_embd_out(self.model.as_ptr()) }
183    }
184
185    /// The model's classification output width (`n_cls_out`, default 1) — the
186    /// width of a RANK-pooled embeddings read (llama.h:1029).
187    #[must_use]
188    pub fn n_cls_out(&self) -> u32 {
189        unsafe { llama_cpp_sys_2::llama_model_n_cls_out(self.model.as_ptr()) }
190    }
191
192    /// Returns the total size of all the tensors in the model in bytes.
193    pub fn size(&self) -> u64 {
194        unsafe { llama_cpp_sys_2::llama_model_size(self.model.as_ptr()) }
195    }
196
197    /// Returns the number of parameters in the model.
198    pub fn n_params(&self) -> u64 {
199        unsafe { llama_cpp_sys_2::llama_model_n_params(self.model.as_ptr()) }
200    }
201
202    /// Returns whether the model is a recurrent network (Mamba, RWKV, etc)
203    pub fn is_recurrent(&self) -> bool {
204        unsafe { llama_cpp_sys_2::llama_model_is_recurrent(self.model.as_ptr()) }
205    }
206
207    /// Returns whether the model is a hybrid network (Jamba, Granite, Qwen3xx, etc)
208    ///
209    /// Hybrid models have both attention layers and recurrent/SSM layers.
210    /// They require special handling for state checkpointing.
211    pub fn is_hybrid(&self) -> bool {
212        unsafe { llama_cpp_sys_2::llama_model_is_hybrid(self.model.as_ptr()) }
213    }
214
215    /// Returns the number of layers within the model.
216    pub fn n_layer(&self) -> u32 {
217        // It's never possible for this to panic because while the API interface is defined as an int32_t,
218        // the field it's accessing is a uint32_t.
219        u32::try_from(unsafe { llama_cpp_sys_2::llama_model_n_layer(self.model.as_ptr()) }).unwrap()
220    }
221
222    /// Returns the number of attention heads within the model.
223    pub fn n_head(&self) -> u32 {
224        // It's never possible for this to panic because while the API interface is defined as an int32_t,
225        // the field it's accessing is a uint32_t.
226        u32::try_from(unsafe { llama_cpp_sys_2::llama_model_n_head(self.model.as_ptr()) }).unwrap()
227    }
228
229    /// Returns the number of KV attention heads.
230    pub fn n_head_kv(&self) -> u32 {
231        // It's never possible for this to panic because while the API interface is defined as an int32_t,
232        // the field it's accessing is a uint32_t.
233        u32::try_from(unsafe { llama_cpp_sys_2::llama_model_n_head_kv(self.model.as_ptr()) })
234            .unwrap()
235    }
236
237    /// Get metadata value as a string by key name
238    pub fn meta_val_str(&self, key: &str) -> Result<String, MetaValError> {
239        let key_cstring = CString::new(key)?;
240        let key_ptr = key_cstring.as_ptr();
241
242        extract_meta_string(
243            |buf_ptr, buf_len| unsafe {
244                llama_cpp_sys_2::llama_model_meta_val_str(
245                    self.model.as_ptr(),
246                    key_ptr,
247                    buf_ptr,
248                    buf_len,
249                )
250            },
251            256,
252        )
253    }
254
255    /// Get the number of metadata key/value pairs
256    pub fn meta_count(&self) -> i32 {
257        unsafe { llama_cpp_sys_2::llama_model_meta_count(self.model.as_ptr()) }
258    }
259
260    /// Get metadata key name by index
261    pub fn meta_key_by_index(&self, index: i32) -> Result<String, MetaValError> {
262        extract_meta_string(
263            |buf_ptr, buf_len| unsafe {
264                llama_cpp_sys_2::llama_model_meta_key_by_index(
265                    self.model.as_ptr(),
266                    index,
267                    buf_ptr,
268                    buf_len,
269                )
270            },
271            256,
272        )
273    }
274
275    /// Get metadata value as a string by index
276    pub fn meta_val_str_by_index(&self, index: i32) -> Result<String, MetaValError> {
277        extract_meta_string(
278            |buf_ptr, buf_len| unsafe {
279                llama_cpp_sys_2::llama_model_meta_val_str_by_index(
280                    self.model.as_ptr(),
281                    index,
282                    buf_ptr,
283                    buf_len,
284                )
285            },
286            256,
287        )
288    }
289
290    /// Returns the rope type of the model.
291    pub fn rope_type(&self) -> Option<RopeType> {
292        match unsafe { llama_cpp_sys_2::llama_model_rope_type(self.model.as_ptr()) } {
293            llama_cpp_sys_2::LLAMA_ROPE_TYPE_NONE => None,
294            llama_cpp_sys_2::LLAMA_ROPE_TYPE_NORM => Some(RopeType::Norm),
295            llama_cpp_sys_2::LLAMA_ROPE_TYPE_NEOX => Some(RopeType::NeoX),
296            llama_cpp_sys_2::LLAMA_ROPE_TYPE_MROPE => Some(RopeType::MRope),
297            llama_cpp_sys_2::LLAMA_ROPE_TYPE_IMROPE => Some(RopeType::IMRope),
298            llama_cpp_sys_2::LLAMA_ROPE_TYPE_VISION => Some(RopeType::Vision),
299            rope_type => {
300                tracing::error!(rope_type = rope_type, "Unexpected rope type from llama.cpp");
301                None
302            }
303        }
304    }
305
306    /// Get chat template from model by name. If the name parameter is None, the default chat template will be returned.
307    ///
308    /// You supply this into [`Self::apply_chat_template`] to get back a string with the appropriate template
309    /// substitution applied to convert a list of messages into a prompt the LLM can use to complete
310    /// the chat.
311    ///
312    /// You could also use an external jinja parser, like [minijinja](https://github.com/mitsuhiko/minijinja),
313    /// to parse jinja templates not supported by the llama.cpp template engine.
314    ///
315    /// # Errors
316    ///
317    /// * If the model has no chat template by that name
318    /// * If the chat template is not a valid [`CString`].
319    pub fn chat_template(
320        &self,
321        name: Option<&str>,
322    ) -> Result<LlamaChatTemplate, ChatTemplateError> {
323        let name_cstr = name.map(CString::new);
324        let name_ptr = match name_cstr {
325            Some(Ok(name)) => name.as_ptr(),
326            _ => std::ptr::null(),
327        };
328        let result =
329            unsafe { llama_cpp_sys_2::llama_model_chat_template(self.model.as_ptr(), name_ptr) };
330
331        // Convert result to Rust String if not null
332        if result.is_null() {
333            Err(ChatTemplateError::MissingTemplate)
334        } else {
335            let chat_template_cstr = unsafe { CStr::from_ptr(result) };
336            let chat_template = CString::new(chat_template_cstr.to_bytes())?;
337            Ok(LlamaChatTemplate(chat_template))
338        }
339    }
340
341    /// Loads a model from a file.
342    ///
343    /// # Errors
344    ///
345    /// See [`LlamaModelLoadError`] for more information.
346    #[tracing::instrument(skip_all, fields(params))]
347    pub fn load_from_file(
348        _: &LlamaBackend,
349        path: impl AsRef<Path>,
350        params: &LlamaModelParams,
351    ) -> Result<Self, LlamaModelLoadError> {
352        let path = path.as_ref();
353        debug_assert!(Path::new(path).exists(), "{path:?} does not exist");
354        let path = path
355            .to_str()
356            .ok_or(LlamaModelLoadError::PathToStrError(path.to_path_buf()))?;
357
358        let cstr = CString::new(path)?;
359        let llama_model =
360            unsafe { llama_cpp_sys_2::llama_load_model_from_file(cstr.as_ptr(), params.params) };
361
362        let model = Ptr::new(llama_model).ok_or(LlamaModelLoadError::NullResult)?;
363
364        tracing::debug!(?path, "Loaded model");
365        Ok(LlamaModel { model })
366    }
367
368    /// Initializes a lora adapter from a file.
369    ///
370    /// # Errors
371    ///
372    /// See [`LlamaLoraAdapterInitError`] for more information.
373    pub fn lora_adapter_init(
374        &mut self,
375        path: impl AsRef<Path>,
376    ) -> Result<LlamaLoraAdapter, LlamaLoraAdapterInitError> {
377        let path = path.as_ref();
378        debug_assert!(Path::new(path).exists(), "{path:?} does not exist");
379
380        let path = path
381            .to_str()
382            .ok_or(LlamaLoraAdapterInitError::PathToStrError(
383                path.to_path_buf(),
384            ))?;
385
386        let cstr = CString::new(path)?;
387        let adapter = unsafe {
388            llama_cpp_sys_2::llama_adapter_lora_init(self.model.as_mut_ptr(), cstr.as_ptr())
389        };
390
391        let adapter = Ptr::new(adapter).ok_or(LlamaLoraAdapterInitError::NullResult)?;
392
393        tracing::debug!(?path, "Initialized lora adapter");
394        Ok(LlamaLoraAdapter {
395            lora_adapter: adapter,
396        })
397    }
398
399    /// Create a new context from this model.
400    ///
401    /// # Errors
402    ///
403    /// There is many ways this can fail. See [`LlamaContextLoadError`] for more information.
404    // we intentionally do not derive Copy on `LlamaContextParams` to allow llama.cpp to change the type to be non-trivially copyable.
405    #[allow(clippy::needless_pass_by_value)]
406    pub fn new_context<'a>(
407        &'a self,
408        _: &LlamaBackend,
409        params: LlamaContextParams,
410    ) -> Result<LlamaContext<'a>, LlamaContextLoadError> {
411        let context_params = params.context_params;
412        // Constructing multiple contexts from a model in parallel is actually
413        // unsound, since `llama_new_context_with_model` mutates the model's
414        // `n_ctx_train` in some cases:
415        // <https://github.com/ggml-org/llama.cpp/blob/f45576aa86c07d60d346b469651a098a15cc4f81/src/llama-context.cpp#L3751>
416        // FIXME(madsmtm): Make this sound!
417        let context = unsafe {
418            llama_cpp_sys_2::llama_new_context_with_model(
419                self.model.as_mut_ptr_unsound(),
420                context_params,
421            )
422        };
423        let context = Ptr::new(context).ok_or(LlamaContextLoadError::NullReturn)?;
424
425        Ok(LlamaContext::new(context))
426    }
427
428    /// Create a new context bound to another context via llama.cpp's `ctx_other` field.
429    ///
430    /// This is required for MTP speculative decoding when the target model's
431    /// architecture uses `LLM_ARCH_GEMMA4_ASSISTANT`, which asserts that the draft
432    /// context references the target context so KV state can be shared.
433    ///
434    /// # Errors
435    ///
436    /// See [`LlamaContextLoadError`].
437    #[allow(clippy::needless_pass_by_value)]
438    pub fn new_context_with_ctx_other<'a>(
439        &'a self,
440        _: &LlamaBackend,
441        params: LlamaContextParams,
442        ctx_other: &LlamaContext<'_>,
443    ) -> Result<LlamaContext<'a>, LlamaContextLoadError> {
444        let mut context_params = params.context_params;
445        // FIXME(madsmtm): Use `.as_ptr()` after:
446        // https://github.com/ggml-org/llama.cpp/pull/28316
447        context_params.ctx_other = unsafe { ctx_other.context.as_mut_ptr_unsound() };
448        // Unsoundness: See `LlamaMode::new_context`.
449        let context = unsafe {
450            llama_cpp_sys_2::llama_new_context_with_model(
451                self.model.as_mut_ptr_unsound(),
452                context_params,
453            )
454        };
455        let context = Ptr::new(context).ok_or(LlamaContextLoadError::NullReturn)?;
456
457        Ok(LlamaContext::new(context))
458    }
459
460    /// Creates a new context with backend samplers attached for specific sequences.
461    ///
462    /// Ownership of the samplers is transferred to the context, ensuring they remain
463    /// alive for the context's lifetime. Only samplers that support backend execution
464    /// (greedy, dist, temp, top_k, top_p, min_p, logit_bias) will run on the backend.
465    ///
466    /// # Arguments
467    ///
468    /// * `params` - Context parameters
469    /// * `samplers` - Iterator of `(seq_id, sampler)` pairs where sampler must be a chain
470    ///
471    /// # Example
472    ///
473    /// ```rust,ignore
474    /// let sampler = LlamaSampler::chain([
475    ///     LlamaSampler::min_p(0.01, 64),
476    ///     LlamaSampler::temp(0.1),
477    ///     LlamaSampler::dist(42),
478    /// ], false);
479    ///
480    /// let ctx = model.new_context_with_samplers(
481    ///     &backend,
482    ///     ctx_params,
483    ///     [(0, sampler)],
484    /// )?;
485    /// ```
486    #[allow(clippy::needless_pass_by_value)]
487    pub fn new_context_with_samplers<'a>(
488        &'a self,
489        _: &LlamaBackend,
490        params: LlamaContextParams,
491        samplers: impl IntoIterator<Item = (i32, LlamaSampler)>,
492    ) -> Result<LlamaContext<'a>, LlamaContextLoadError> {
493        let mut samplers: Vec<_> = samplers.into_iter().collect();
494        let mut context_params = params.context_params;
495
496        let mut sampler_configs: Vec<llama_cpp_sys_2::llama_sampler_seq_config> = samplers
497            .iter_mut()
498            .map(
499                |(seq_id, sampler)| llama_cpp_sys_2::llama_sampler_seq_config {
500                    seq_id: *seq_id,
501                    sampler: sampler.sampler.as_mut_ptr(),
502                },
503            )
504            .collect();
505
506        if !sampler_configs.is_empty() {
507            context_params.samplers = sampler_configs.as_mut_ptr();
508            context_params.n_samplers = sampler_configs.len();
509        }
510
511        // Unsoundness: See `LlamaMode::new_context`.
512        let context = unsafe {
513            llama_cpp_sys_2::llama_new_context_with_model(
514                self.model.as_mut_ptr_unsound(),
515                context_params,
516            )
517        };
518        let context = Ptr::new(context).ok_or(LlamaContextLoadError::NullReturn)?;
519
520        Ok(LlamaContext::with_samplers(context, samplers))
521    }
522
523    /// Apply the models chat template to some messages.
524    /// See <https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template>
525    ///
526    /// Unlike the llama.cpp `apply_chat_template` which just randomly uses the ChatML template when given
527    /// a null pointer for the template, this requires an explicit template to be specified. If you want to
528    /// use "chatml", then just do `LlamaChatTemplate::new("chatml")` or any other model name or template
529    /// string.
530    ///
531    /// Use [`Self::chat_template`] to retrieve the template baked into the model (this is the preferred
532    /// mechanism as using the wrong chat template can result in really unexpected responses from the LLM).
533    ///
534    /// You probably want to set `add_ass` to true so that the generated template string ends with a the
535    /// opening tag of the assistant. If you fail to leave a hanging chat tag, the model will likely generate
536    /// one into the output and the output may also have unexpected output aside from that.
537    ///
538    /// # Errors
539    /// There are many ways this can fail. See [`ApplyChatTemplateError`] for more information.
540    #[tracing::instrument(skip_all)]
541    pub fn apply_chat_template(
542        &self,
543        tmpl: &LlamaChatTemplate,
544        chat: &[LlamaChatMessage],
545        add_ass: bool,
546    ) -> Result<String, ApplyChatTemplateError> {
547        // Buffer is twice the length of messages per their recommendation
548        let message_length = chat.iter().fold(0, |acc, c| {
549            acc + c.role.to_bytes().len() + c.content.to_bytes().len()
550        });
551        let mut buff: Vec<u8> = vec![0; message_length * 2];
552
553        // Build our llama_cpp_sys_2 chat messages
554        let chat: Vec<llama_cpp_sys_2::llama_chat_message> = chat
555            .iter()
556            .map(|c| llama_cpp_sys_2::llama_chat_message {
557                role: c.role.as_ptr(),
558                content: c.content.as_ptr(),
559            })
560            .collect();
561
562        let tmpl_ptr = tmpl.0.as_ptr();
563
564        let res = unsafe {
565            llama_cpp_sys_2::llama_chat_apply_template(
566                tmpl_ptr,
567                chat.as_ptr(),
568                chat.len(),
569                add_ass,
570                buff.as_mut_ptr().cast::<c_char>(),
571                buff.len().try_into().expect("Buffer size exceeds i32::MAX"),
572            )
573        };
574
575        if res < 0 {
576            return Err(ApplyChatTemplateError::FfiError(res));
577        }
578
579        if res > buff.len().try_into().expect("Buffer size exceeds i32::MAX") {
580            buff.resize(res.try_into().expect("res is negative"), 0);
581
582            let res = unsafe {
583                llama_cpp_sys_2::llama_chat_apply_template(
584                    tmpl_ptr,
585                    chat.as_ptr(),
586                    chat.len(),
587                    add_ass,
588                    buff.as_mut_ptr().cast::<c_char>(),
589                    buff.len().try_into().expect("Buffer size exceeds i32::MAX"),
590                )
591            };
592            if res < 0 {
593                return Err(ApplyChatTemplateError::FfiError(res));
594            }
595            assert_eq!(Ok(res), buff.len().try_into());
596        }
597        buff.truncate(res.try_into().expect("res is negative"));
598        Ok(String::from_utf8(buff)?)
599    }
600}
601
602/// Generic helper function for extracting string values from the C API
603/// This are specifically useful for the the metadata functions, where we pass in a buffer
604/// to be populated by a string, not yet knowing if the buffer is large enough.
605/// If the buffer was not large enough, we get the correct length back, which can be used to
606/// construct a buffer of appropriate size.
607fn extract_meta_string<F>(c_function: F, capacity: usize) -> Result<String, MetaValError>
608where
609    F: Fn(*mut c_char, usize) -> i32,
610{
611    let mut buffer = vec![0u8; capacity];
612
613    // call the foreign function
614    let result = c_function(buffer.as_mut_ptr().cast::<c_char>(), buffer.len());
615    if result < 0 {
616        return Err(MetaValError::NegativeReturn(result));
617    }
618
619    // check if the response fit in our buffer
620    let returned_len = result as usize;
621    if returned_len >= capacity {
622        // buffer wasn't large enough, try again with the correct capacity.
623        return extract_meta_string(c_function, returned_len + 1);
624    }
625
626    // verify null termination
627    debug_assert_eq!(
628        buffer.get(returned_len),
629        Some(&0),
630        "should end with null byte"
631    );
632
633    // resize, convert, and return
634    buffer.truncate(returned_len);
635    Ok(String::from_utf8(buffer)?)
636}
637
638impl Drop for LlamaModel {
639    fn drop(&mut self) {
640        unsafe { llama_cpp_sys_2::llama_free_model(self.model.as_mut_ptr()) }
641    }
642}
llama_cpp_2/model.rs

llama_cpp_2/
model.rs
