Struct LlamaModel 

pub struct LlamaModel {
    pub model: NonNull<llama_model>,
    /* private fields */
}

Fields

model: NonNull<llama_model>

Implementations

impl LlamaModel

pub fn vocab_ptr(&self) -> *const llama_vocab

pub fn n_ctx_train(&self) -> Result<u32, TryFromIntError>

Errors

Returns an error if the value returned by llama.cpp does not fit into a u32.

pub fn tokens( &self, decode_special: bool, ) -> impl Iterator<Item = (LlamaToken, Result<String, TokenToStringError>)> + '_

pub fn token_bos(&self) -> LlamaToken

pub fn token_eos(&self) -> LlamaToken

pub fn token_nl(&self) -> LlamaToken

pub fn is_eog_token(&self, token: &SampledToken) -> bool

pub fn decode_start_token(&self) -> LlamaToken

pub fn token_sep(&self) -> LlamaToken

pub fn str_to_token( &self, str: &str, add_bos: AddBos, ) -> Result<Vec<LlamaToken>, StringToTokenError>

Errors

• if str contains a null byte
• if an integer conversion fails during tokenization

use llama_cpp_bindings::model::LlamaModel;

pub fn token_attr( &self, LlamaToken: LlamaToken, ) -> Result<LlamaTokenAttrs, LlamaTokenAttrsFromIntError>

Errors

Returns an error if the token type is not known to this library.

pub fn token_to_piece( &self, token: &SampledToken, decoder: &mut Decoder, special: bool, lstrip: Option<NonZeroU16>, ) -> Result<String, TokenToStringError>

Errors

• if the token type is unknown

• if the returned size from llama.cpp does not fit into a usize

pub fn token_to_piece_bytes( &self, token: LlamaToken, buffer_size: usize, special: bool, lstrip: Option<NonZeroU16>, ) -> Result<Vec<u8>, TokenToStringError>

Errors

• if the token type is unknown
• the resultant token is larger than buffer_size.
• if an integer conversion fails

pub fn n_vocab(&self) -> i32

pub fn vocab_type(&self) -> Result<VocabType, VocabTypeFromIntError>

Errors

Returns an error if llama.cpp emits a vocab type that is not known to this library.

pub fn n_embd(&self) -> c_int

pub fn size(&self) -> u64

pub fn n_params(&self) -> u64

pub fn is_recurrent(&self) -> bool

pub fn n_layer(&self) -> Result<u32, TryFromIntError>

Errors

Returns an error if the layer count returned by llama.cpp does not fit into a u32.

pub fn n_head(&self) -> Result<u32, TryFromIntError>

Errors

Returns an error if the head count returned by llama.cpp does not fit into a u32.

pub fn n_head_kv(&self) -> Result<u32, TryFromIntError>

Errors

Returns an error if the KV head count returned by llama.cpp does not fit into a u32.

pub fn is_hybrid(&self) -> bool

pub fn meta_val_str(&self, key: &str) -> Result<String, MetaValError>

Errors

Returns an error if the key is not found or the value is not valid UTF-8.

pub fn meta_count(&self) -> i32

pub fn meta_key_by_index(&self, index: i32) -> Result<String, MetaValError>

Errors

Returns an error if the index is out of range or the key is not valid UTF-8.

pub fn meta_val_str_by_index(&self, index: i32) -> Result<String, MetaValError>

Errors

Returns an error if the index is out of range or the value is not valid UTF-8.

pub fn rope_type(&self) -> Option<RopeType>

pub fn chat_template( &self, name: Option<&str>, ) -> Result<LlamaChatTemplate, ChatTemplateError>

Errors

• If the model has no chat template by that name

Panics

Panics if the C-returned chat template string contains interior null bytes (should never happen with valid model data).

pub fn load_from_file( _: &LlamaBackend, path: impl AsRef<Path>, params: &LlamaModelParams, ) -> Result<Self, LlamaModelLoadError>

Errors

See LlamaModelLoadError for more information.

Panics

Panics if a valid UTF-8 path somehow contains interior null bytes (should never happen).

pub fn lora_adapter_init( &self, path: impl AsRef<Path>, ) -> Result<LlamaLoraAdapter, LlamaLoraAdapterInitError>

Errors

See LlamaLoraAdapterInitError for more information.

pub fn apply_chat_template( &self, tmpl: &LlamaChatTemplate, chat: &[LlamaChatMessage], add_ass: bool, ) -> Result<String, ApplyChatTemplateError>

Errors

There are many ways this can fail. See ApplyChatTemplateError for more information.

pub fn sampled_token_classifier(&self) -> SampledTokenClassifier<'_>

pub fn streaming_markers( &self, ) -> Result<StreamingMarkers, MarkerDetectionError>

Errors

Returns MarkerDetectionError when any underlying FFI call fails.

pub fn reasoning_markers( &self, ) -> Result<Option<ReasoningMarkers>, MarkerDetectionError>

Errors

Returns MarkerDetectionError when the underlying FFI call fails.

pub fn tool_call_markers(&self) -> Option<ToolCallMarkers>

pub fn parse_chat_message( &self, tools_json: &str, input: &str, is_partial: bool, ) -> Result<ChatMessageParseOutcome, ParseChatMessageError>

Errors

Returns ParseChatMessageError when tools_json is not valid JSON, the FFI returns a non-OK status other than ParseException, or accessor strings are not valid UTF-8.

pub fn diagnose_tool_call_synthetic_renders( &self, ) -> Result<(String, String), MarkerDetectionError>

Errors

Returns MarkerDetectionError when the C++ analyzer throws or the FFI returns a non-OK status.

impl LlamaModel

pub fn approximate_tok_env(&self) -> Arc<ApproximateTokEnv>

Trait Implementations

impl Debug for LlamaModel

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Drop for LlamaModel

fn drop(&mut self)

Executes the destructor for this type.

fn pin_drop(self: Pin<&mut Self>)

🔬This is a nightly-only experimental API. (pin_ergonomics)

Execute the destructor for this type, but different to Drop::drop, it requires self to be pinned.

impl Send for LlamaModel

impl Sync for LlamaModel