1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#[cfg(test)]
mod tests;
use std::fmt::Debug;
mod wordpiece;
pub use wordpiece::WordpieceTokenizer;
mod bpe;
pub use bpe::BPETokenizer;
mod whitespace;
pub use whitespace::WhitespaceTokenizer;
mod alphabet;
pub use alphabet::AlphabetTokenizer;
pub trait Tokenizer: Debug + Send + Sync {
fn load() -> Self;
fn tokenize(&self, string: &str) -> Vec<String>;
fn batch_tokenize(&self, strings: Vec<String>) -> Vec<Vec<String>>;
fn untokenize(&self, tokens: Vec<String>) -> String;
fn batch_untokenize(&self, tokens: Vec<Vec<String>>) -> Vec<String>;
}