Struct TokTrie

pub struct TokTrie { /* private fields */ }

Implementations

impl TokTrie

pub const SPECIAL_TOKEN_MARKER: u8 = 255u8

pub const MAX_DBG_TOKENS: usize = 200usize

pub fn from(info: &TokRxInfo, words: &[Vec<u8>]) -> Self

pub fn filter(&self, filter: &SimpleVob) -> Self

pub fn with_eos_token(&self, eos_token: TokenId) -> Self

pub fn with_info(&self, info: TokRxInfo) -> Self

pub fn build_chat_mode_trie(&self) -> Self

pub fn info(&self) -> &TokRxInfo

pub fn eos_token(&self) -> TokenId

pub fn vocab_size(&self) -> usize

pub fn alloc_token_set(&self) -> SimpleVob

pub fn singleton_token_set(&self, tok: TokenId) -> SimpleVob

pub fn token_set_dbg(&self, ts: &SimpleVob) -> String

pub fn alloc_logits(&self) -> Vec<f32>

pub fn test_trace_tokens(&self, toks: &[u32]) -> String

pub fn tokens_dbg(&self, toks: &[u32]) -> String

pub fn token_dbg(&self, idx: u32) -> String

pub fn token_str(&self, idx: u32) -> String

pub fn token_len(&self, idx: u32) -> usize

pub fn token(&self, idx: u32) -> &[u8]

pub fn decode(&self, tokens: &[TokenId]) -> Vec<u8>

pub fn decode_ext(&self, tokens: &[TokenId], include_special: bool) -> Vec<u8>

pub fn decode_as_special(&self, tok: TokenId) -> Vec<u8>

pub fn decode_raw(&self, tokens: &[TokenId]) -> Vec<u8>

pub fn decode_str(&self, tokens: &[TokenId]) -> String

pub fn decode_raw_to_decode(&self, bytes: &[u8]) -> Vec<u8>

pub fn is_special_token(&self, tok: TokenId) -> bool

pub fn get_special_token(&self, name: &str) -> Option<TokenId>

pub fn get_special_tokens(&self) -> Vec<TokenId>

pub fn greedy_tokenize(&self, bytes: &[u8]) -> Vec<TokenId>

pub fn tokenize_with_greedy_fallback( &self, bytes: &[u8], str_tokenize: impl Fn(&str) -> Vec<TokenId>, ) -> Vec<TokenId>

pub fn has_extensions(&self, bytes: &[u8]) -> bool

pub fn token_id(&self, bytes: &[u8]) -> Option<TokenId>

pub fn prefix_token_id(&self, bytes: &[u8]) -> (TokenId, usize)

pub fn max_token_len(&self) -> usize

pub fn root(&self) -> &TrieNode

pub fn check_against(&self, tokens: &[Vec<u8>])

pub fn child_at_byte<'a>( &'a self, n: &'a TrieNode, byte: u8, ) -> Option<&'a TrieNode>

pub fn all_subtokens(&self, bytes: &[u8]) -> Vec<TokenId>

pub fn node_children(&self, n: &TrieNode) -> NodeChildren<'_>

pub fn child_at_bytes<'a>( &'a self, n: &'a TrieNode, bytes: &[u8], ) -> Option<&'a TrieNode>

pub fn token_id_at_bytes(&self, bytes: &[u8]) -> Option<TokenId>

pub fn chop_tokens( &self, r: &mut impl Recognizer, tokens: &[TokenId], ) -> (usize, usize)

Return how many tokens and bytes need to chopped off tokens, so that we do not limit all possible future tokenizations matching the recognizer.

pub fn has_valid_extensions( &self, r: &mut impl Recognizer, start: &[u8], ) -> bool

Check if add_bias() would have returned any tokens.

pub fn all_prefixes(&self, bytes: &[u8]) -> Vec<TokenId>

pub fn add_bias( &self, r: &mut impl Recognizer, toks: &mut SimpleVob, start: &[u8], )

pub fn all_tokens(&self) -> Vec<Vec<u8>>

pub fn sorted_tokens(&self) -> Vec<(u32, Vec<u8>)>

pub fn trie_stats(&self) -> String

Trait Implementations

impl Clone for TokTrie

fn clone(&self) -> TokTrie

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.