[][src]Function rust_tokenizers::preprocessing::tokenizer::tokenization_utils::group_common_pairs

pub fn group_common_pairs(
    tokens: Vec<String>,
    bpe_ranks: &BpePairVocab
) -> (Vec<String>, bool)