1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
//! Safe wrapper around `llama_context`.
use std::fmt::{Debug, Formatter};
use std::num::NonZeroI32;
use std::ptr::NonNull;
use std::slice;
use crate::llama_batch::LlamaBatch;
use crate::model::LlamaModel;
use crate::timing::LlamaTimings;
use crate::token::data::LlamaTokenData;
use crate::token::LlamaToken;
use crate::{DecodeError, EmbeddingsError};
pub mod kv_cache;
pub mod params;
pub mod sample;
pub mod session;
/// Safe wrapper around `llama_context`.
#[allow(clippy::module_name_repetitions)]
pub struct LlamaContext<'a> {
pub(crate) context: NonNull<llama_cpp_sys_2::llama_context>,
/// a reference to the contexts model.
pub model: &'a LlamaModel,
initialized_logits: Vec<i32>,
embeddings_enabled: bool,
}
impl Debug for LlamaContext<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LlamaContext")
.field("context", &self.context)
.finish()
}
}
impl<'model> LlamaContext<'model> {
pub(crate) fn new(
llama_model: &'model LlamaModel,
llama_context: NonNull<llama_cpp_sys_2::llama_context>,
embeddings_enabled: bool,
) -> Self {
Self {
context: llama_context,
model: llama_model,
initialized_logits: Vec::new(),
embeddings_enabled,
}
}
/// Gets the max number of tokens in a batch.
#[must_use]
pub fn n_batch(&self) -> u32 {
unsafe { llama_cpp_sys_2::llama_n_batch(self.context.as_ptr()) }
}
/// Gets the size of the context.
#[must_use]
pub fn n_ctx(&self) -> u32 {
unsafe { llama_cpp_sys_2::llama_n_ctx(self.context.as_ptr()) }
}
/// Decodes the batch.
///
/// # Errors
///
/// - `DecodeError` if the decoding failed.
///
/// # Panics
///
/// - the returned [`std::ffi::c_int`] from llama-cpp does not fit into a i32 (this should never happen on most systems)
pub fn decode(&mut self, batch: &mut LlamaBatch) -> Result<(), DecodeError> {
let result =
unsafe { llama_cpp_sys_2::llama_decode(self.context.as_ptr(), batch.llama_batch) };
match NonZeroI32::new(result) {
None => {
self.initialized_logits = batch.initialized_logits.clone();
Ok(())
}
Some(error) => Err(DecodeError::from(error)),
}
}
/// Get the embeddings for the `i`th sequence in the current context.
///
/// # Returns
///
/// A slice containing the embeddings for the last decoded batch.
/// The size corresponds to the `n_embd` parameter of the context's model.
///
/// # Errors
///
/// - When the current context was constructed without enabling embeddings.
/// - If the current model had a pooling type of [`llama_cpp_sys_2::LLAMA_POOLING_TYPE_NONE`]
/// - If the given sequence index exceeds the max sequence id.
///
/// # Panics
///
/// * `n_embd` does not fit into a usize
pub fn embeddings_seq_ith(&self, i: i32) -> Result<&[f32], EmbeddingsError> {
if !self.embeddings_enabled {
return Err(EmbeddingsError::NotEnabled);
}
let n_embd =
usize::try_from(self.model.n_embd()).expect("n_embd does not fit into a usize");
unsafe {
let embedding = llama_cpp_sys_2::llama_get_embeddings_seq(self.context.as_ptr(), i);
// Technically also possible whenever `i >= max(batch.n_seq)`, but can't check that here.
if embedding.is_null() {
Err(EmbeddingsError::NonePoolType)
} else {
Ok(slice::from_raw_parts(embedding, n_embd))
}
}
}
/// Get the embeddings for the `i`th token in the current context.
///
/// # Returns
///
/// A slice containing the embeddings for the last decoded batch of the given token.
/// The size corresponds to the `n_embd` parameter of the context's model.
///
/// # Errors
///
/// - When the current context was constructed without enabling embeddings.
/// - When the given token didn't have logits enabled when it was passed.
/// - If the given token index exceeds the max token id.
///
/// # Panics
///
/// * `n_embd` does not fit into a usize
pub fn embeddings_ith(&self, i: i32) -> Result<&[f32], EmbeddingsError> {
if !self.embeddings_enabled {
return Err(EmbeddingsError::NotEnabled);
}
let n_embd =
usize::try_from(self.model.n_embd()).expect("n_embd does not fit into a usize");
unsafe {
let embedding = llama_cpp_sys_2::llama_get_embeddings_ith(self.context.as_ptr(), i);
// Technically also possible whenever `i >= batch.n_tokens`, but no good way of checking `n_tokens` here.
if embedding.is_null() {
Err(EmbeddingsError::LogitsNotEnabled)
} else {
Ok(slice::from_raw_parts(embedding, n_embd))
}
}
}
/// Get the logits for the ith token in the context.
///
/// # Panics
///
/// - logit `i` is not initialized.
pub fn candidates_ith(&self, i: i32) -> impl Iterator<Item = LlamaTokenData> + '_ {
(0_i32..).zip(self.get_logits_ith(i)).map(|(i, logit)| {
let token = LlamaToken::new(i);
LlamaTokenData::new(token, *logit, 0_f32)
})
}
/// Get the logits for the ith token in the context.
///
/// # Panics
///
/// - `i` is greater than `n_ctx`
/// - `n_vocab` does not fit into a usize
/// - logit `i` is not initialized.
#[must_use]
pub fn get_logits_ith(&self, i: i32) -> &[f32] {
assert!(
self.initialized_logits.contains(&i),
"logit {i} is not initialized. only {:?} is",
self.initialized_logits
);
assert!(
self.n_ctx() > u32::try_from(i).expect("i does not fit into a u32"),
"n_ctx ({}) must be greater than i ({})",
self.n_ctx(),
i
);
let data = unsafe { llama_cpp_sys_2::llama_get_logits_ith(self.context.as_ptr(), i) };
let len = usize::try_from(self.model.n_vocab()).expect("n_vocab does not fit into a usize");
unsafe { slice::from_raw_parts(data, len) }
}
/// Reset the timings for the context.
pub fn reset_timings(&mut self) {
unsafe { llama_cpp_sys_2::llama_reset_timings(self.context.as_ptr()) }
}
/// Returns the timings for the context.
pub fn timings(&mut self) -> LlamaTimings {
let timings = unsafe { llama_cpp_sys_2::llama_get_timings(self.context.as_ptr()) };
LlamaTimings { timings }
}
}
impl Drop for LlamaContext<'_> {
fn drop(&mut self) {
unsafe { llama_cpp_sys_2::llama_free(self.context.as_ptr()) }
}
}