Function mako::tokenization::batch_tokenize_wordpiece [−][src]
pub fn batch_tokenize_wordpiece(
strings: Vec<String>,
tokenizer: Option<&Tokenizer>
) -> Vec<Vec<String>>
Expand description
Tokenizes strings using WordPiece tokenization