ik_llama_cpp_2/
token.rs

1//! Token newtype.
2
3pub mod data;
4pub mod data_array;
5
6/// A single vocabulary token id (newtype over ik_llama.cpp's `llama_token` = `i32`).
7///
8/// `#[repr(transparent)]` guarantees identical layout to `llama_token`, so
9/// `*const llama_token` ↔ `*const LlamaToken` casts (e.g. in mtmd `text_tokens`)
10/// are sound.
11#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
12#[repr(transparent)]
13pub struct LlamaToken(pub ik_llama_cpp_sys::llama_token);
14
15impl LlamaToken {
16    /// Construct from a raw token id.
17    #[must_use]
18    pub fn new(id: ik_llama_cpp_sys::llama_token) -> Self {
19        Self(id)
20    }
21
22    /// The raw token id.
23    #[must_use]
24    pub fn raw(self) -> ik_llama_cpp_sys::llama_token {
25        self.0
26    }
27}
ik_llama_cpp_2/token.rs

ik_llama_cpp_2/
token.rs
