Struct CachedTokenizer 

pub struct CachedTokenizer { /* private fields */ }

A caching wrapper around any tokenizer

Implementations

impl CachedTokenizer

pub fn new(inner: Arc<dyn Tokenizer>, config: CacheConfig) -> Self

Create a new cached tokenizer

pub fn cache_stats(&self) -> Option<CacheStats>

Get L0 cache statistics

pub fn l1_cache_stats(&self) -> Option<L1CacheStats>

Get L1 cache statistics

pub fn clear_cache(&self)

Clear the cache

pub fn fingerprint(&self) -> &TokenizerFingerprint

Get the fingerprint of the underlying tokenizer

pub fn inner(&self) -> &Arc<dyn Tokenizer>

Get a reference to the inner (wrapped) tokenizer

Trait Implementations

impl Decoder for CachedTokenizer

fn decode( &self, token_ids: &[TokenIdType], skip_special_tokens: bool, ) -> Result<String>

impl Encoder for CachedTokenizer

fn encode(&self, input: &str, add_special_tokens: bool) -> Result<Encoding>

fn encode_batch( &self, inputs: &[&str], add_special_tokens: bool, ) -> Result<Vec<Encoding>>

impl Tokenizer for CachedTokenizer

fn vocab_size(&self) -> usize

fn get_special_tokens(&self) -> &SpecialTokens

fn token_to_id(&self, token: &str) -> Option<TokenIdType>

fn id_to_token(&self, id: TokenIdType) -> Option<String>

fn as_any(&self) -> &dyn Any

Enable downcasting to concrete types

fn apply_chat_template( &self, messages: &[Value], params: ChatTemplateParams<'_>, ) -> Result<String>

Apply chat template to messages. Default returns an error for tokenizers without template support.

fn chat_template_content_format(&self) -> ChatTemplateContentFormat

Get the content format expected by the chat template.

fn set_chat_template(&mut self, _template: String) -> Result<()>

Set or override the chat template.