Function wana_kana::tokenize::tokenize

pub fn tokenize(input: &str) -> Vec<String> 

Available on crate feature tokenize only.

Tokenizes the text. Splits input into array of strings separated by opinionated TokenType.

Example

use wana_kana::tokenize::*;
let empty: Vec<String> = vec![];
assert_eq!(tokenize(""), empty);
assert_eq!(tokenize("ふふフフ"), vec!["ふふ", "フフ"]);
assert_eq!(tokenize("感じ"), vec!["感", "じ"]);
assert_eq!(tokenize("私は悲しい"), vec!["私", "は", "悲", "しい"] );