1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
//! Safe wrapper around `llama_batch`.
use crate::token::LlamaToken;
use llama_cpp_sys_2::{llama_batch, llama_batch_free, llama_batch_init, llama_pos, llama_seq_id};
/// A safe wrapper around `llama_batch`.
#[derive(Debug)]
pub struct LlamaBatch {
/// The number of tokens the batch was allocated with. they are safe to write to - but not necessarily read from as they are not necessarily initilized
allocated: usize,
/// The logits that are initilized. Used by [`LlamaContext`] to ensure that only initilized logits are accessed.
pub(crate) initialized_logits: Vec<i32>,
/// The llama_cpp batch. always initilize by `llama_cpp_sys_2::llama_batch_init(allocated, <unknown>, <unknown>)`
pub(crate) llama_batch: llama_batch,
}
impl LlamaBatch {
/// Clear the batch. This does not free the memory associated with the batch, but it does reset
/// the number of tokens to 0.
pub fn clear(&mut self) {
self.llama_batch.n_tokens = 0;
self.initialized_logits.clear();
}
/// Set the last token in the batch to [value]. If [value] is true, the token will be initilized
/// after a decode and can be read from. If [value] is false, the token will not be initilized (this is the default).
///
/// # Panics
///
/// Panics if there are no tokens in the batch.
#[deprecated(
note = "not compatible with multiple sequences. prefer setting logits while adding tokens"
)]
pub fn set_last_logit(&mut self, value: bool) {
let last_index = self.llama_batch.n_tokens - 1;
let last_index_usize =
usize::try_from(last_index).expect("cannot fit n_tokens - 1 into a usize");
if value {
self.initialized_logits.push(last_index);
} else {
self.initialized_logits.retain(|&x| x != last_index);
}
let value = i8::from(value);
unsafe {
let last: *mut i8 = self.llama_batch.logits.add(last_index_usize);
*last = value;
}
}
/// add a token to the batch for sequences [`seq_ids`] at position [pos]. If [logits] is true, the
/// token will be initilized and can be read from after the next decode.
///
/// # Panics
///
/// - [`self.llama_batch.n_tokens`] does not fit into a usize
/// - [`seq_ids.len()`] does not fit into a [`llama_seq_id`]
pub fn add(
&mut self,
LlamaToken(id): LlamaToken,
pos: llama_pos,
seq_ids: &[i32],
logits: bool,
) {
assert!(self.allocated > (usize::try_from(self.n_tokens() + 1).expect("self.n_tokens does not fit into a usize")), "there are only {} tokens allocated for the batch, but {} tokens in the batch when you tried to add one", self.allocated, self.n_tokens());
unsafe {
// batch.token [batch.n_tokens] = id;
let offset = self.llama_batch.n_tokens;
let offset_usize = usize::try_from(offset).expect("cannot fit n_tokens into a usize");
*self.llama_batch.token.add(offset_usize) = id;
// batch.pos [batch.n_tokens] = pos,
*self.llama_batch.pos.add(offset_usize) = pos;
// batch.n_seq_id[batch.n_tokens] = seq_ids.size();
*self.llama_batch.n_seq_id.add(offset_usize) = llama_seq_id::try_from(seq_ids.len())
.expect("cannot fit seq_ids.len() into a llama_seq_id");
// for (size_t i = 0; i < seq_ids.size(); ++i) {
// batch.seq_id[batch.n_tokens][i] = seq_ids[i];
// }
for (i, seq_id) in seq_ids.iter().enumerate() {
let tmp = *self.llama_batch.seq_id.add(offset_usize);
*tmp.add(i) = *seq_id;
}
// batch.logits [batch.n_tokens] = logits;
*self.llama_batch.logits.add(offset_usize) = i8::from(logits);
if logits {
self.initialized_logits.push(offset);
} else {
self.initialized_logits.retain(|l| l != &offset);
}
// batch.n_tokens++;
self.llama_batch.n_tokens += 1;
}
}
/// Create a new `LlamaBatch` that cab contain up to `n_tokens` tokens.
///
/// # Panics
///
/// Panics if `n_tokens` is greater than `i32::MAX`.
#[must_use]
pub fn new(n_tokens: usize, embd: i32, n_seq_max: i32) -> Self {
let n_tokens_i32 = i32::try_from(n_tokens).expect("cannot fit n_tokens into a i32");
let batch = unsafe { llama_batch_init(n_tokens_i32, embd, n_seq_max) };
LlamaBatch {
allocated: n_tokens,
initialized_logits: vec![],
llama_batch: batch,
}
}
/// add a prompt to the batch at sequence id 0
#[deprecated(note = "not compatible with multiple sequences. use `add_prompt_seq` instead")]
pub fn add_prompt(&mut self, prompt: &[LlamaToken]) {
self.add_prompt_seq(prompt, &[0]);
}
/// add a prompt to the batch at the given sequence ids. This must be the initial prompt as it
/// will be added to the batch starting at position 0.
pub fn add_prompt_seq(&mut self, prompt: &[LlamaToken], seq_ids: &[i32]) {
for (i, &token) in (0_i32..).zip(prompt) {
self.add(token, i, seq_ids, false);
}
}
/// Returns the number of tokens in the batch.
#[must_use]
pub fn n_tokens(&self) -> i32 {
self.llama_batch.n_tokens
}
}
impl Drop for LlamaBatch {
/// Drops the `LlamaBatch`.
///
/// ```
/// # use llama_cpp::llama_batch::LlamaBatch;
/// # use std::error::Error;
/// # fn main() -> Result<(), Box<dyn Error>> {
/// let batch = LlamaBatch::new_from_prompt(&[]);
/// // frees the memory associated with the batch. (allocated by llama.cpp)
/// drop(batch);
/// # Ok(())
/// # }
fn drop(&mut self) {
unsafe {
llama_batch_free(self.llama_batch);
}
}
}