tokenizers 0.22.2

Provides an implementation of today's most used tokenizers, with a focus on performances and versatility.
Documentation 
1
2
3
4
5

mod pre_tokenizer;
mod scripts;

// Re-export the PreTokenizer
pub use pre_tokenizer::UnicodeScripts;