[−][src]Function rust_tokenizers::preprocessing::tokenizer::tokenization_utils::ctrl_bpe
pub fn ctrl_bpe(
token: &str,
bpe_ranks: &BpePairVocab
) -> (Vec<String>, Vec<usize>)
pub fn ctrl_bpe(
token: &str,
bpe_ranks: &BpePairVocab
) -> (Vec<String>, Vec<usize>)