1use unicode_segmentation::UnicodeSegmentation;
13
14#[inline]
16pub fn is_cjk(c: char) -> bool {
17 matches!(c as u32,
18 0x1100..=0x11FF | 0x3040..=0x30FF | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xA960..=0xA97F | 0xAC00..=0xD7AF | 0xF900..=0xFAFF | 0xFF66..=0xFF9D | 0x20000..=0x2FA1F
27 )
28}
29
30#[inline]
32fn is_meaningful(w: &str) -> bool {
33 w.chars().any(|c| c.is_alphanumeric())
34}
35
36pub fn tokenize(text: &str) -> Vec<String> {
42 let mut out = Vec::with_capacity(text.len() / 5);
43 for word in text.unicode_words() {
44 if !is_meaningful(word) {
45 continue;
46 }
47 let cjk_chars: Vec<char> = word.chars().filter(|c| is_cjk(*c)).collect();
48 if cjk_chars.len() * 2 >= word.chars().count() && !cjk_chars.is_empty() {
49 for c in &cjk_chars {
50 out.push(c.to_lowercase().collect());
51 }
52 for pair in cjk_chars.windows(2) {
53 out.push(pair.iter().collect());
54 }
55 } else {
56 out.push(word.to_lowercase());
57 }
58 }
59 out
60}
61
62pub fn estimate_tokens(text: &str) -> usize {
69 let mut cjk = 0usize;
70 let mut other_bytes = 0usize;
71 for c in text.chars() {
72 if is_cjk(c) {
73 cjk += 1;
74 } else {
75 other_bytes += c.len_utf8();
76 }
77 }
78 let latin = other_bytes.div_ceil(4);
79 let cjk_tokens = (cjk as f64 / 1.4).ceil() as usize;
80 (latin + cjk_tokens).max(usize::from(!text.trim().is_empty()))
81}
82
83pub fn normalize_ws(s: &str) -> String {
85 let mut out = String::with_capacity(s.len());
86 let mut pending_space = false;
87 for c in s.chars() {
88 if c.is_whitespace() {
89 pending_space = !out.is_empty();
90 } else {
91 if pending_space {
92 out.push(' ');
93 pending_space = false;
94 }
95 out.push(c);
96 }
97 }
98 out
99}
100
101pub fn visible_len(s: &str) -> usize {
110 s.chars().count()
111}
112
113#[cfg(test)]
114mod visible_len_tests {
115 use super::*;
116 use unicode_segmentation::UnicodeSegmentation;
117
118 #[test]
119 fn agrees_with_grapheme_counting_on_ordinary_text() {
120 for s in ["hello world", "한국어 텍스트입니다", "日本語のテキスト", "Ελληνικά", ""]
123 {
124 assert_eq!(visible_len(s), s.graphemes(true).count(), "{s:?}");
125 }
126 }
127}
128
129#[cfg(test)]
130mod tests {
131 use super::*;
132
133 #[test]
134 fn latin_tokens_are_lowercased_words() {
135 assert_eq!(tokenize("Hello, World! 42"), ["hello", "world", "42"]);
136 }
137
138 #[test]
139 fn korean_yields_unigrams_and_bigrams() {
140 let t = tokenize("한국어");
141 assert!(t.contains(&"한".to_string()));
142 assert!(t.contains(&"한국".to_string()));
143 assert!(t.contains(&"국어".to_string()));
144 }
145
146 #[test]
147 fn cjk_costs_more_tokens_per_char_than_latin() {
148 assert!(estimate_tokens("데이터 수집") > estimate_tokens("data"));
149 assert_eq!(estimate_tokens(""), 0);
150 }
151
152 #[test]
153 fn whitespace_collapses() {
154 assert_eq!(normalize_ws(" a \n\t b "), "a b");
155 }
156}