llama_cpp_bindings/
llama_token_attr.rs

1use enumflags2::bitflags;
2
3/// A rust flavored equivalent of `llama_token_type`.
4#[derive(Eq, PartialEq, Debug, Clone, Copy)]
5#[bitflags]
6#[repr(u32)]
7pub enum LlamaTokenAttr {
8    /// Unknown token attribute.
9    Unknown = llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_UNKNOWN as _,
10    /// Unused token attribute.
11    Unused = llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_UNUSED as _,
12    /// Normal text token.
13    Normal = llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_NORMAL as _,
14    /// Control token (e.g. BOS, EOS).
15    Control = llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_CONTROL as _,
16    /// User-defined token.
17    UserDefined = llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_USER_DEFINED as _,
18    /// Byte-level fallback token.
19    Byte = llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_BYTE as _,
20    /// Token with normalized text.
21    Normalized = llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_NORMALIZED as _,
22    /// Token with left-stripped whitespace.
23    LStrip = llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_LSTRIP as _,
24    /// Token with right-stripped whitespace.
25    RStrip = llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_RSTRIP as _,
26    /// Token representing a single word.
27    SingleWord = llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_SINGLE_WORD as _,
28}
llama_cpp_bindings/llama_token_attr.rs

llama_cpp_bindings/
llama_token_attr.rs
