Struct CachedTokenizer 

pub struct CachedTokenizer { /* private fields */ }

A caching wrapper around any tokenizer

Implementations

impl CachedTokenizer

pub fn new(inner: Arc<dyn Tokenizer>, config: CacheConfig) -> Self

Create a new cached tokenizer

pub fn cache_stats(&self) -> Option<CacheStats>

Get L0 cache statistics

pub fn l1_cache_stats(&self) -> Option<L1CacheStats>

Get L1 cache statistics

pub fn clear_cache(&self)

Clear the cache

pub fn fingerprint(&self) -> &TokenizerFingerprint

Get the fingerprint of the underlying tokenizer

pub fn inner(&self) -> &Arc<dyn Tokenizer>

Get a reference to the inner (wrapped) tokenizer

Trait Implementations

impl Decoder for CachedTokenizer

fn decode( &self, token_ids: &[TokenIdType], skip_special_tokens: bool, ) -> Result<String>

fn decode_step( &self, token_id: TokenIdType, ids: &mut Vec<TokenIdType>, prefix: &mut String, prefix_index: &mut usize, skip_special_tokens: bool, ) -> Result<Option<String>>

Incremental decode step — called once per generated token.

impl Encoder for CachedTokenizer

fn encode(&self, input: &str, add_special_tokens: bool) -> Result<Encoding>

fn encode_batch( &self, inputs: &[&str], add_special_tokens: bool, ) -> Result<Vec<Encoding>>

impl Tokenizer for CachedTokenizer

fn vocab_size(&self) -> usize

fn get_special_tokens(&self) -> &SpecialTokens

fn token_to_id(&self, token: &str) -> Option<TokenIdType>

fn id_to_token(&self, id: TokenIdType) -> Option<String>

fn as_any(&self) -> &dyn Any

Enable downcasting to concrete types

fn apply_chat_template( &self, messages: &[Value], params: ChatTemplateParams<'_>, ) -> Result<String>

Apply chat template to messages. Default returns an error for tokenizers without template support.

fn chat_template_content_format(&self) -> ChatTemplateContentFormat

Get the content format expected by the chat template.

fn thinking_toggle(&self) -> ThinkingToggle

Get the thinking toggle support for this template.

fn thinking_key_name(&self) -> Option<ThinkingKeyName>

The variable name the template uses for the thinking toggle.

fn think_in_prefill(&self) -> bool

Whether the template injects <think> in the generation prompt.

fn set_chat_template(&mut self, _template: String) -> Result<()>

Set or override the chat template.