bm_25/
tokenizer.rs

1/// A tokenizer splits text into a sequence of tokens. Implement this trait to use this crate with
2/// your own tokenizer.
3pub trait Tokenizer {
4    /// Tokenizes the input text.
5    fn tokenize<'a>(&'a self, input_text: &'a str) -> impl Iterator<Item = String> + 'a;
6}