Function rust_tokenizers::preprocessing::tokenizer::tokenization_utils::tokenize_cjk_chars

pub fn tokenize_cjk_chars(token: TokenRef<'_>) -> Vec<TokenRef<'_>>

Tokenizes CJK characters, each character will be a token