List of all items

Structs

• Tokenizer
• cache::CacheConfig
• cache::CacheStats
• cache::CachedTokenizer
• cache::L0Cache
• cache::L1Cache
• cache::L1CacheStats
• cache::TokenizerFingerprint
• chat_template::ChatTemplateParams
• chat_template::ChatTemplateProcessor
• chat_template::ChatTemplateState
• huggingface::HuggingFaceTokenizer
• mock::MockTokenizer
• registry::TokenizerEntry
• registry::TokenizerRegistry
• sequence::Sequence
• stop::StopSequenceConfig
• stop::StopSequenceDecoder
• stop::StopSequenceDecoderBuilder
• stream::DecodeStream
• tiktoken::TiktokenTokenizer
• traits::SpecialTokens

Enums

• chat_template::ChatTemplateContentFormat
• chat_template::ThinkingKeyName
• chat_template::ThinkingToggle
• factory::TokenizerType
• registry::LoadError
• registry::LoadOutcome
• stop::SequenceDecoderOutput
• tiktoken::TiktokenModel
• traits::Encoding

Traits

• traits::Decoder
• traits::Encoder
• traits::Tokenizer

Functions

• chat_template::detect_chat_template_content_format
• chat_template::detect_thinking_toggle
• chat_template::load_chat_template_from_config
• chat_template::load_chat_template_from_file
• factory::create_tokenizer
• factory::create_tokenizer_async
• factory::create_tokenizer_async_with_chat_template
• factory::create_tokenizer_from_file
• factory::create_tokenizer_with_chat_template
• factory::create_tokenizer_with_chat_template_blocking
• factory::discover_chat_template_in_dir
• factory::get_tokenizer_info
• hub::download_tokenizer_from_hf
• hub::from_hf
• tiktoken::has_tiktoken_file
• tiktoken::is_tiktoken_file

Type Aliases

• traits::TokenIdType