Struct LlamaModelParams 

pub struct LlamaModelParams { /* private fields */ }

A safe wrapper around llama_model_params.

Implementations

impl LlamaModelParams

pub fn kv_overrides(&self) -> KvOverrides<'_>

See KvOverrides

Examples

let params = Box::pin(LlamaModelParams::default());
let kv_overrides = params.kv_overrides();
let count = kv_overrides.into_iter().count();
assert_eq!(count, 0);

pub fn append_kv_override( self: Pin<&mut Self>, key: &CStr, value: ParamOverrideValue, )

Appends a key-value override to the model parameters. It must be pinned as this creates a self-referential struct.

Examples

use std::pin::pin;
let mut params = pin!(LlamaModelParams::default());
let key = CString::new("key").expect("CString::new failed");
params.as_mut().append_kv_override(&key, ParamOverrideValue::Int(50));

let kv_overrides = params.kv_overrides().into_iter().collect::<Vec<_>>();
assert_eq!(kv_overrides.len(), 1);

let (k, v) = &kv_overrides[0];
assert_eq!(v, &ParamOverrideValue::Int(50));

assert_eq!(k.to_bytes(), b"key", "expected key to be 'key', was {:?}", k);

impl LlamaModelParams

pub fn n_gpu_layers(&self) -> i32

Get the number of layers to offload to the GPU.

pub fn main_gpu(&self) -> i32

The GPU that is used for scratch and small tensors

pub fn vocab_only(&self) -> bool

only load the vocabulary, no weights

pub fn load_mode(&self) -> LlamaLoadMode

Returns the exact model-file loading strategy.

pub fn load_mtp(&self) -> bool

Whether the model’s MTP (multi-token prediction) layers will be loaded.

MTP layers drive multi-token-prediction speculative decoding for models that ship them (e.g. DeepSeek V4). Once loaded, the speculative state is captured and restored through crate::speculative. Defaults to false because most models carry no MTP weights.

pub fn use_mmap(&self) -> bool

use mmap if possible

LlamaLoadMode::Auto counts as “possible”: llama.cpp memory-maps under Auto unless one of the backend devices lacks mmap support, which is only known once the model is loaded.

pub fn use_mlock(&self) -> bool

force system to keep model in RAM

pub fn with_n_gpu_layers(self, n_gpu_layers: u32) -> Self

sets the number of gpu layers to offload to the GPU.

let params = LlamaModelParams::default();
let params = params.with_n_gpu_layers(1);
assert_eq!(params.n_gpu_layers(), 1);

pub fn with_main_gpu(self, main_gpu: i32) -> Self

sets the main GPU

pub fn with_vocab_only(self, vocab_only: bool) -> Self

sets vocab_only

pub fn with_load_mode(self, load_mode: LlamaLoadMode) -> Self

Sets the exact model-file loading strategy.

pub fn with_load_mtp(self, load_mtp: bool) -> Self

Sets whether to load the model’s MTP (multi-token prediction) layers.

Enable this for models that ship MTP weights (e.g. DeepSeek V4) when you intend to use MTP-based speculative decoding, then drive the speculative state via crate::speculative. For models without MTP layers the flag has no effect. Corresponds to llama_model_params.load_mtp, added upstream in llama.cpp PR #25784 (DeepSeek V4 MTP + DSpark).

let params = LlamaModelParams::default().with_load_mtp(true);
assert!(params.load_mtp());

pub fn with_use_mlock(self, use_mlock: bool) -> Self

sets use_mlock

Trait Implementations

impl Debug for LlamaModelParams

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for LlamaModelParams

Default parameters for LlamaModel. (as defined in llama.cpp by llama_model_default_params)

let params = LlamaModelParams::default();
assert_eq!(params.n_gpu_layers(), -1, "n_gpu_layers should be -1 (all layers)");
assert_eq!(params.main_gpu(), 0, "main_gpu should be 0");
assert_eq!(params.vocab_only(), false, "vocab_only should be false");
assert_eq!(params.use_mmap(), true, "use_mmap should be true");
assert_eq!(params.use_mlock(), false, "use_mlock should be false");

fn default() -> Self

Returns the “default value” for a type.