Function mako::tokenization::tokenize_bpe[−][src]

pub fn tokenize_bpe(
    string: String, 
    tokenizer: Option<&Tokenizer>
) -> Vec<String>
Expand description

Tokenizes string using BPE tokenization