Function rust_tokenizers::preprocessing::tokenizer::tokenization_utils::tokenize_cjk_chars

pub fn tokenize_cjk_chars(token: TokenRef) -> Vec<TokenRef>

Tokenizes CJK characters, each character will be a token