Struct llama_cpp_2::context::LlamaContext

pub struct LlamaContext<'a> {
    pub model: &'a LlamaModel,
    /* private fields */
}

Safe wrapper around llama_context.

Fields

model: &'a LlamaModel

a reference to the contexts model.

Implementations

impl LlamaContext<'_>

pub fn copy_cache(&mut self, src: i32, dest: i32, size: i32)

Copy the cache from one sequence to another.

Parameters

• src - The sequence id to copy the cache from.
• dest - The sequence id to copy the cache to.
• size - The size of the cache to copy.

pub fn clear_kv_cache_seq(&mut self, src: i32, p0: Option<u16>, p1: Option<u16>)

Clear the kv cache for the given sequence.

Parameters

• src - The sequence id to clear the cache for.
• p0 - The start position of the cache to clear. If None, the entire cache is cleared up to [p1].
• p1 - The end position of the cache to clear. If None, the entire cache is cleared from [p0].

impl LlamaContext<'_>

pub fn sample(&mut self, sampler: Sampler<'_>) -> LlamaToken

Sample a token.

Panics

• sampler contains no tokens

pub fn grammar_accept_token( &mut self, grammar: &mut LlamaGrammar, token: LlamaToken )

Accept a token into the grammar.

pub fn sample_grammar( &mut self, llama_token_data_array: &mut LlamaTokenDataArray, llama_grammar: &LlamaGrammar )

Perform grammar sampling.

pub fn sample_temp( &self, token_data: &mut LlamaTokenDataArray, temperature: f32 )

Modify [token_data] in place using temperature sampling.

Panics

• [temperature] is not between 0.0 and 1.0

pub fn sample_token_greedy(&self, token_data: LlamaTokenDataArray) -> LlamaToken

Sample a token greedily.

Panics

• [token_data] is empty

pub fn sample_token_softmax(&self, token_data: &mut LlamaTokenDataArray)

Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.

impl<'model> LlamaContext<'model>

pub fn n_ctx(&self) -> u32

Gets the size of the context.

pub fn decode(&mut self, batch: &mut LlamaBatch) -> Result<(), DecodeError>

Decodes the batch.

Errors

• DecodeError if the decoding failed.

Panics

• the returned [c_int] from llama-cpp does not fit into a i32 (this should never happen on most systems)

pub fn candidates_ith( &self, i: i32 ) -> impl Iterator<Item = LlamaTokenData> + '_

Get the logits for the ith token in the context.

Panics

• logit i is not initialized.

pub fn get_logits_ith(&self, i: i32) -> &[f32]

Get the logits for the ith token in the context.

Panics

• i is greater than n_ctx
• n_vocab does not fit into a usize
• logit i is not initialized.

pub fn reset_timings(&mut self)

Reset the timings for the context.

pub fn timings(&mut self) -> LlamaTimings

Returns the timings for the context.

Trait Implementations

impl Debug for LlamaContext<'_>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Drop for LlamaContext<'_>

fn drop(&mut self)

Executes the destructor for this type.