ferrum-tokenizer 0.4.0

Tokenization wrapper for Ferrum inference engine
Documentation 
1
2
3

pub mod huggingface;

pub use huggingface::{HuggingFaceTokenizer, HuggingFaceTokenizerFactory, IncrementalState};