ferrum-tokenizer 0.7.0

Tokenization wrapper for Ferrum inference engine
Documentation 
1
2
3

pub mod huggingface;

pub use huggingface::{HuggingFaceTokenizer, HuggingFaceTokenizerFactory, IncrementalState};