Re-exports§
pub use cache::CacheConfig;pub use cache::CacheStats;pub use cache::CachedTokenizer;pub use cache::L0Cache;pub use cache::L1Cache;pub use cache::TokenizerFingerprint;pub use chat_template::ChatTemplateState;pub use factory::create_tokenizer;pub use factory::create_tokenizer_from_file;pub use factory::create_tokenizer_with_chat_template;pub use factory::TokenizerType;pub use huggingface::HuggingFaceTokenizer;pub use mock::MockTokenizer;pub use registry::LoadError;pub use registry::LoadOutcome;pub use registry::TokenizerRegistry;pub use sequence::Sequence;pub use stop::SequenceDecoderOutput;pub use stop::StopSequenceConfig;pub use stop::StopSequenceDecoder;pub use stream::DecodeStream;pub use tiktoken::TiktokenModel;pub use tiktoken::TiktokenTokenizer;pub use traits::Decoder;pub use traits::Encoder;pub use traits::Encoding;pub use traits::SpecialTokens;pub use traits::TokenIdType;pub use traits::Tokenizer as TokenizerTrait;
Modules§
- cache
- Tokenizer Caching Layer
- chat_
template - Chat template support for tokenizers using Jinja2 templates
- factory
- hub
- huggingface
- mock
- Mock tokenizer implementation for testing
- registry
- Tokenizer Registry for dynamic tokenizer loading
- sequence
- stop
- stream
- tiktoken
- traits
Structs§
- Tokenizer
- Main tokenizer wrapper that provides a unified interface for different tokenizer implementations