[−][src]Function rust_tokenizers::preprocessing::tokenizer::tokenization_utils::group_common_pairs
pub fn group_common_pairs(
tokens: Vec<String>,
bpe_ranks: &BpePairVocab
) -> (Vec<String>, bool)