Function mako::tokenization::tokenize_wordpiece

pub fn tokenize_wordpiece(
    string: String, 
    tokenizer: Option<&Tokenizer>
) -> Vec<String>

Tokenizes string using BPE tokenization