1 2 3 4 5 6 7 8
use unicode_segmentation::UnicodeSegmentation; pub fn tokenize(content: &str) -> Vec<String> { content.unicode_words() .map(|w| w.trim_matches(|c: char| !c.is_alphanumeric()).to_lowercase()) .filter(|w| !w.is_empty()) .collect() }