[][src]Function rust_tokenizers::preprocessing::tokenizer::tokenization_utils::split_on_regex_with_lookahead

pub fn split_on_regex_with_lookahead<'a>(
    token: TokenRef<'a>,
    pattern_lookahead: &Regex,
    pattern_tokenization: &Regex
) -> Vec<TokenRef<'a>>