llama_cpp_bindings/model/params/
kv_overrides.rs

1//! Key-value overrides for a model.
2
3use crate::model::params::LlamaModelParams;
4use crate::model::params::param_override_value::ParamOverrideValue;
5use std::ffi::{CStr, CString};
6use std::fmt::Debug;
7
8/// A struct implementing [`IntoIterator`] over the key-value overrides for a model.
9#[derive(Debug)]
10pub struct KvOverrides<'model_params> {
11    model_params: &'model_params LlamaModelParams,
12}
13
14impl KvOverrides<'_> {
15    /// Creates a new `KvOverrides` view over the given model parameters.
16    #[must_use]
17    pub fn new(model_params: &LlamaModelParams) -> KvOverrides<'_> {
18        KvOverrides { model_params }
19    }
20}
21
22impl<'model_params> IntoIterator for KvOverrides<'model_params> {
23    // I'm fairly certain this could be written returning by reference, but I'm not sure how to do it safely. I do not
24    // expect this to be a performance bottleneck so the copy should be fine. (let me know if it's not fine!)
25    type Item = (CString, ParamOverrideValue);
26    type IntoIter = KvOverrideValueIterator<'model_params>;
27
28    fn into_iter(self) -> Self::IntoIter {
29        KvOverrideValueIterator {
30            model_params: self.model_params,
31            current: 0,
32        }
33    }
34}
35
36/// An iterator over the key-value overrides for a model.
37#[derive(Debug)]
38pub struct KvOverrideValueIterator<'model_params> {
39    model_params: &'model_params LlamaModelParams,
40    current: usize,
41}
42
43impl Iterator for KvOverrideValueIterator<'_> {
44    type Item = (CString, ParamOverrideValue);
45
46    fn next(&mut self) -> Option<Self::Item> {
47        let overrides = self.model_params.params.kv_overrides;
48        if overrides.is_null() {
49            return None;
50        }
51
52        // SAFETY: llama.cpp seems to guarantee that the last element contains an empty key or is valid. We've checked
53        // the prev one in the last iteration, the next one should be valid or 0 (and thus safe to deref)
54        let current = unsafe { *overrides.add(self.current) };
55
56        if current.key[0] == 0 {
57            return None;
58        }
59
60        let value = ParamOverrideValue::from(&current);
61
62        let key = unsafe { CStr::from_ptr(current.key.as_ptr()).to_owned() };
63
64        self.current += 1;
65
66        Some((key, value))
67    }
68}
69
70#[cfg(test)]
71mod tests {
72    #[test]
73    fn kv_overrides_empty_by_default() {
74        let params = crate::model::params::LlamaModelParams::default();
75        let overrides = params.kv_overrides();
76        let count = overrides.into_iter().count();
77
78        assert_eq!(count, 0);
79    }
80}
llama_cpp_bindings/model/params/kv_overrides.rs

llama_cpp_bindings/model/params/
kv_overrides.rs
