Struct LlamaContext 

pub struct LlamaContext<'model> {
    pub context: NonNull<llama_context>,
    pub model: &'model LlamaModel,
    /* private fields */
}

Fields

context: NonNull<llama_context>

model: &'model LlamaModel

Implementations

impl LlamaContext<'_>

pub fn copy_cache(&mut self, src: i32, dest: i32, size: i32)

pub fn copy_kv_cache_seq( &mut self, src: i32, dest: i32, p0: Option<u32>, p1: Option<u32>, ) -> Result<(), KvCacheConversionError>

Errors

If either position exceeds i32::MAX.

pub fn clear_kv_cache_seq( &mut self, src: Option<u32>, p0: Option<u32>, p1: Option<u32>, ) -> Result<bool, KvCacheConversionError>

Errors

If the sequence id or either position exceeds i32::MAX.

pub fn clear_kv_cache(&mut self)

pub fn kv_cache_seq_keep(&mut self, seq_id: i32)

pub fn kv_cache_seq_add( &mut self, seq_id: i32, p0: Option<u32>, p1: Option<u32>, delta: i32, ) -> Result<(), KvCacheSeqAddError>

Errors

If either position exceeds i32::MAX, or the underlying memory operation reports a failure.

pub fn kv_cache_seq_div( &mut self, seq_id: i32, p0: Option<u32>, p1: Option<u32>, d: NonZeroU8, ) -> Result<(), KvCacheSeqDivError>

Errors

If either position exceeds i32::MAX, or the underlying memory operation reports a failure.

pub fn kv_cache_seq_pos_max(&self, seq_id: i32) -> i32

impl LlamaContext<'_>

pub fn state_save_file( &self, path_session: impl AsRef<Path>, tokens: &[LlamaToken], ) -> Result<(), SaveSessionError>

Errors

Fails if the path is not a valid utf8 or llama.cpp fails to save the state file.

pub fn state_load_file( &mut self, path_session: impl AsRef<Path>, max_tokens: usize, ) -> Result<Vec<LlamaToken>, LoadSessionError>

Errors

Fails if the path is not a valid utf8 or llama.cpp fails to load the state file.

pub fn state_seq_save_file( &self, filepath: impl AsRef<Path>, seq_id: i32, tokens: &[LlamaToken], ) -> Result<usize, SaveSeqStateError>

Errors

Fails if the path is not a valid utf8 or llama.cpp fails to save the sequence state file.

pub fn state_seq_load_file( &mut self, filepath: impl AsRef<Path>, dest_seq_id: i32, max_tokens: usize, ) -> Result<(Vec<LlamaToken>, usize), LoadSeqStateError>

Errors

Fails if the path is not a valid utf8 or llama.cpp fails to load the sequence state file.

pub fn get_state_size(&self) -> usize

pub unsafe fn copy_state_data(&self, dest: &mut [u8]) -> usize

Safety

The dest buffer must be large enough to hold the complete state data.

pub unsafe fn set_state_data(&mut self, src: &[u8]) -> usize

Safety

The src buffer must contain data previously obtained from copy_state_data on a compatible context (same model and parameters). Passing arbitrary or corrupted bytes will lead to undefined behavior.

pub fn state_seq_get_size_ext( &self, seq_id: i32, flags: &LlamaStateSeqFlags, ) -> usize

pub unsafe fn state_seq_get_data_ext( &self, dest: &mut [u8], seq_id: i32, flags: &LlamaStateSeqFlags, ) -> usize

Safety

The dest buffer must be large enough to hold the complete state data.

pub unsafe fn state_seq_set_data_ext( &mut self, src: &[u8], dest_seq_id: i32, flags: &LlamaStateSeqFlags, ) -> usize

Safety

The src buffer must contain data previously obtained from state_seq_get_data_ext on a compatible context.

impl<'model> LlamaContext<'model>

pub const fn new( llama_model: &'model LlamaModel, llama_context: NonNull<llama_context>, embeddings_enabled: bool, ) -> Self

pub fn from_model( model: &'model LlamaModel, _backend: &LlamaBackend, params: LlamaContextParams, ) -> Result<Self, LlamaContextLoadError>

Errors

Returns LlamaContextLoadError when llama.cpp fails to allocate the context.

pub fn n_batch(&self) -> u32

pub fn n_ubatch(&self) -> u32

pub fn n_ctx(&self) -> u32

pub fn set_abort_flag(&mut self, flag: Arc<AtomicBool>)

pub fn clear_abort_callback(&mut self)

pub fn synchronize(&self)

pub fn detach_threadpool(&self)

pub fn mark_logits_initialized(&mut self, token_index: i32)

pub fn decode(&mut self, batch: &mut LlamaBatch<'_>) -> Result<(), DecodeError>

Errors

• DecodeError if the decoding failed.

pub fn encode(&mut self, batch: &mut LlamaBatch<'_>) -> Result<(), EncodeError>

Errors

• EncodeError if the encoding failed.

pub fn embeddings_seq_ith( &self, sequence_index: i32, ) -> Result<&[f32], EmbeddingsError>

Errors

• When the current context was constructed without enabling embeddings.
• If the current model had a pooling type of llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_NONE
• If the given sequence index exceeds the max sequence id.

pub fn embeddings_ith( &self, token_index: i32, ) -> Result<&[f32], EmbeddingsError>

Errors

• When the current context was constructed without enabling embeddings.
• When the given token didn’t have logits enabled when it was passed.
• If the given token index exceeds the max token id.

pub fn candidates( &self, ) -> Result<impl Iterator<Item = LlamaTokenData> + '_, LogitsError>

Errors

Returns LogitsError if logits are null or n_vocab overflows.

pub fn token_data_array(&self) -> Result<LlamaTokenDataArray, LogitsError>

Errors

Returns LogitsError if logits are null or n_vocab overflows.

pub fn get_logits(&self) -> Result<&[f32], LogitsError>

Errors

Returns LogitsError if the logits pointer is null or n_vocab overflows.

pub fn candidates_ith( &self, token_index: i32, ) -> Result<impl Iterator<Item = LlamaTokenData> + '_, LogitsError>

Errors

Returns LogitsError if the token is not initialized or out of range.

pub fn token_data_array_ith( &self, token_index: i32, ) -> Result<LlamaTokenDataArray, LogitsError>

Errors

Returns LogitsError if the token is not initialized or out of range.

pub fn get_logits_ith(&self, token_index: i32) -> Result<&[f32], LogitsError>

Errors

Returns LogitsError if the token is not initialized, out of range, or n_vocab overflows.

pub fn reset_timings(&mut self)

pub fn timings(&mut self) -> LlamaTimings

pub fn lora_adapter_set( &self, adapter: &mut LlamaLoraAdapter, scale: f32, ) -> Result<(), LlamaLoraAdapterSetError>

Errors

See LlamaLoraAdapterSetError for more information.

pub fn lora_adapter_remove( &self, _adapter: &mut LlamaLoraAdapter, ) -> Result<(), LlamaLoraAdapterRemoveError>

Errors

See LlamaLoraAdapterRemoveError for more information.

Trait Implementations

impl Debug for LlamaContext<'_>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Drop for LlamaContext<'_>

fn drop(&mut self)

Executes the destructor for this type.

fn pin_drop(self: Pin<&mut Self>)

🔬This is a nightly-only experimental API. (pin_ergonomics)

Execute the destructor for this type, but different to Drop::drop, it requires self to be pinned.