Struct llama_cpp_2::model::params::LlamaModelParams

pub struct LlamaModelParams { /* private fields */ }

A safe wrapper around llama_model_params.

Implementations

impl LlamaModelParams

pub fn n_gpu_layers(&self) -> i32

Get the number of layers to offload to the GPU.

pub fn main_gpu(&self) -> i32

The GPU that is used for scratch and small tensors

pub fn vocab_only(&self) -> bool

only load the vocabulary, no weights

pub fn use_mmap(&self) -> bool

use mmap if possible

pub fn use_mlock(&self) -> bool

force system to keep model in RAM

pub fn with_n_gpu_layers(self, n_gpu_layers: u32) -> Self

sets the number of gpu layers to offload to the GPU.

let params = LlamaModelParams::default();
let params = params.with_n_gpu_layers(1);
assert_eq!(params.n_gpu_layers(), 1);

pub fn with_vocab_only(self, vocab_only: bool) -> Self

sets vocab_only

Trait Implementations

impl Debug for LlamaModelParams

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for LlamaModelParams

Default parameters for LlamaModel. (as defined in llama.cpp by llama_model_default_params)

let params = LlamaModelParams::default();
assert_eq!(params.n_gpu_layers(), 0, "n_gpu_layers should be 0");
assert_eq!(params.main_gpu(), 0, "main_gpu should be 0");
assert_eq!(params.vocab_only(), false, "vocab_only should be false");
assert_eq!(params.use_mmap(), true, "use_mmap should be true");
assert_eq!(params.use_mlock(), false, "use_mlock should be false");

fn default() -> Self

Returns the “default value” for a type.