Expand description
Axonml Text - Natural Language Processing Utilities
§File
crates/axonml-text/src/lib.rs
§Author
Andrew Jewell Sr - AutomataNexus
§Updated
March 8, 2026
§Disclaimer
Use at own risk. This software is provided “as is”, without warranty of any kind, express or implied. The author and AutomataNexus shall not be held liable for any damages arising from the use of this software.
Re-exports§
pub use vocab::BOS_TOKEN;pub use vocab::EOS_TOKEN;pub use vocab::MASK_TOKEN;pub use vocab::PAD_TOKEN;pub use vocab::UNK_TOKEN;pub use vocab::Vocab;pub use tokenizer::BasicBPETokenizer;pub use tokenizer::CharTokenizer;pub use tokenizer::NGramTokenizer;pub use tokenizer::Tokenizer;pub use tokenizer::UnigramTokenizer;pub use tokenizer::WhitespaceTokenizer;pub use tokenizer::WordPunctTokenizer;pub use datasets::LanguageModelDataset;pub use datasets::SyntheticSentimentDataset;pub use datasets::SyntheticSeq2SeqDataset;pub use datasets::TextDataset;