Function rust_tokenizers::preprocessing::tokenizer::tokenization_utils::split_on_bpe_pairs

pub fn split_on_bpe_pairs<'a, F>(
    token: TokenRef<'a>, 
    bpe_function: F, 
    bpe_ranks: &BpePairVocab, 
    cache: &RefCell<HashMap<String, (Vec<String>, Vec<usize>)>>, 
    as_bytes: bool
) -> Vec<Token> where
    F: Fn(&str, &BpePairVocab) -> (Vec<String>, Vec<usize>),