Function mako::tokenization::batch_tokenize_wordpiece[][src]

pub fn batch_tokenize_wordpiece(
    strings: Vec<String>,
    tokenizer: Option<&Tokenizer>
) -> Vec<Vec<String>>
Expand description

Tokenizes strings using WordPiece tokenization