tokenizers/pre_tokenizers/unicode_scripts/
mod.rs

1
2
3
4
5
mod pre_tokenizer;
mod scripts;

// Re-export the PreTokenizer
pub use pre_tokenizer::UnicodeScripts;