1pub mod cjk;
9
10use regex::Regex;
11use std::collections::HashSet;
12use std::sync::LazyLock;
13
14use crate::config::CjkTokenizerMode;
15
16pub static RE_HAS_CJK: LazyLock<Regex> = LazyLock::new(|| {
22 Regex::new(r"[\u4e00-\u9fff\u3400-\u4dbf\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af\uf900-\ufaff]")
23 .unwrap()
24});
25
26static RE_SPLIT_EN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\W+").unwrap());
28
29static RE_FTS5_SPECIAL: LazyLock<Regex> = LazyLock::new(|| {
32 Regex::new(r"[^\w\u4e00-\u9fff\u3400-\u4dbf\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]").unwrap()
33});
34
35const FTS5_OPERATORS: &[&str] = &["AND", "OR", "NOT", "NEAR"];
37
38static EN_STOPWORDS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
44 [
45 "a", "an", "the", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had",
46 "do", "does", "did", "will", "would", "could", "should", "may", "might", "shall", "can",
47 "of", "in", "to", "for", "with", "on", "at", "by", "from", "as", "into", "through",
48 "during", "before", "after", "above", "below", "between", "and", "but", "or", "not", "no",
49 "so", "if", "than", "too", "very", "just", "about", "also", "then", "this", "that",
50 "these", "those", "it", "its", "what", "which", "who", "whom", "how", "when", "where",
51 "why", "all", "each", "every", "both", "few", "more", "most", "other", "some", "such",
52 "only", "own", "same", "we", "they", "he", "she", "us", "our", "their", "your", "my", "i",
53 "me", "you", "nor", "need", "dare", "him", "his", "her", "them", "any", "again", "under",
54 "over",
55 ]
56 .into_iter()
57 .collect()
58});
59
60static ZH_STOPWORDS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
62 [
63 "的",
64 "了",
65 "在",
66 "是",
67 "我",
68 "有",
69 "和",
70 "就",
71 "不",
72 "人",
73 "都",
74 "一",
75 "一个",
76 "上",
77 "也",
78 "很",
79 "到",
80 "说",
81 "要",
82 "去",
83 "你",
84 "会",
85 "着",
86 "没有",
87 "看",
88 "好",
89 "自己",
90 "这",
91 "他",
92 "她",
93 "它",
94 "吗",
95 "吧",
96 "呢",
97 "啊",
98 "呀",
99 "哦",
100 "嗯",
101 "嘛",
102 "哈",
103 "怎样",
104 "怎么",
105 "什么",
106 "哪",
107 "哪个",
108 "哪些",
109 "为什么",
110 "如何",
111 "可以",
112 "能",
113 "把",
114 "被",
115 "让",
116 "给",
117 "对",
118 "从",
119 "向",
120 "跟",
121 "还",
122 "又",
123 "再",
124 "已",
125 "已经",
126 "正在",
127 "将",
128 "将要",
129 ]
130 .into_iter()
131 .collect()
132});
133
134pub fn has_cjk(text: &str) -> bool {
140 RE_HAS_CJK.is_match(text)
141}
142
143pub fn tokenize_for_fts(text: &str, cjk_mode: CjkTokenizerMode) -> String {
150 if text.is_empty() || text.trim().is_empty() {
151 return String::new();
152 }
153 if RE_HAS_CJK.is_match(text) {
154 let tokens = tokenize(text, false, cjk_mode);
155 tokens.join(" ")
156 } else {
157 text.to_string()
159 }
160}
161
162pub fn tokenize_fts_expression(expr: &str, cjk_mode: CjkTokenizerMode) -> String {
169 let parts: Vec<&str> = expr.split_whitespace().collect();
170 let mut result = Vec::new();
171
172 for part in parts {
173 if FTS5_OPERATORS.contains(&part.to_uppercase().as_str()) {
174 result.push(part.to_uppercase());
175 } else {
176 let cleaned = RE_FTS5_SPECIAL.replace_all(part, "");
178 if cleaned.is_empty() {
179 continue;
180 }
181 let tokenized = tokenize_for_fts(&cleaned, cjk_mode);
182 let trimmed = tokenized.trim().to_string();
183 if !trimmed.is_empty() {
184 result.push(trimmed);
185 }
186 }
187 }
188
189 result.join(" ")
190}
191
192pub fn tokenize(text: &str, remove_stopwords: bool, cjk_mode: CjkTokenizerMode) -> Vec<String> {
201 if text.is_empty() {
202 return Vec::new();
203 }
204
205 let is_cjk = RE_HAS_CJK.is_match(text);
206
207 let raw_tokens: Vec<String> = if is_cjk {
208 match cjk_mode {
209 CjkTokenizerMode::Jieba | CjkTokenizerMode::Auto => cjk::jieba_cut(text),
210 CjkTokenizerMode::Bigram => cjk::bigrams(text),
211 CjkTokenizerMode::Char => cjk::chars(text),
212 }
213 } else {
214 RE_SPLIT_EN
215 .split(text)
216 .filter(|s| !s.is_empty())
217 .map(|s| s.to_lowercase())
218 .collect()
219 };
220
221 let filtered: Vec<String> = raw_tokens
223 .into_iter()
224 .filter(|t| {
225 let trimmed = t.trim();
226 if trimmed.is_empty() {
227 return false;
228 }
229 if trimmed.chars().count() == 1 {
230 let ch = trimmed.chars().next().unwrap();
232 is_cjk_char(ch)
233 } else {
234 true
235 }
236 })
237 .map(|t| t.trim().to_string())
238 .collect();
239
240 if !remove_stopwords {
241 return filtered;
242 }
243
244 filtered
246 .into_iter()
247 .filter(|t| {
248 let lower = t.to_lowercase();
249 !EN_STOPWORDS.contains(lower.as_str()) && !ZH_STOPWORDS.contains(t.as_str())
250 })
251 .collect()
252}
253
254fn is_cjk_char(ch: char) -> bool {
256 matches!(ch,
257 '\u{4e00}'..='\u{9fff}' | '\u{3400}'..='\u{4dbf}' | '\u{3040}'..='\u{309f}' | '\u{30a0}'..='\u{30ff}' | '\u{ac00}'..='\u{d7af}' | '\u{f900}'..='\u{faff}' )
264}
265
266#[cfg(test)]
271mod tests {
272 use super::*;
273
274 #[test]
277 fn test_has_cjk_chinese() {
278 assert!(has_cjk("hello你好"));
279 assert!(has_cjk("机器学习"));
280 }
281
282 #[test]
283 fn test_has_cjk_japanese() {
284 assert!(has_cjk("こんにちは"));
285 assert!(has_cjk("カタカナ"));
286 }
287
288 #[test]
289 fn test_has_cjk_korean() {
290 assert!(has_cjk("한글"));
291 }
292
293 #[test]
294 fn test_has_cjk_english_only() {
295 assert!(!has_cjk("hello world"));
296 assert!(!has_cjk("rust programming 123"));
297 }
298
299 #[test]
300 fn test_has_cjk_empty() {
301 assert!(!has_cjk(""));
302 }
303
304 #[test]
307 fn test_tokenize_for_fts_english() {
308 let result = tokenize_for_fts("hello world test", CjkTokenizerMode::Auto);
309 assert_eq!(result, "hello world test");
311 }
312
313 #[test]
314 fn test_tokenize_for_fts_chinese() {
315 let result = tokenize_for_fts("机器学习是人工智能", CjkTokenizerMode::Auto);
316 assert!(!result.is_empty());
318 assert!(result.contains("机器"));
319 assert!(result.contains("学习"));
320 }
321
322 #[test]
323 fn test_tokenize_for_fts_empty() {
324 assert_eq!(tokenize_for_fts("", CjkTokenizerMode::Auto), "");
325 assert_eq!(tokenize_for_fts(" ", CjkTokenizerMode::Auto), "");
326 }
327
328 #[test]
329 fn test_tokenize_for_fts_bigram_mode() {
330 let result = tokenize_for_fts("机器学习", CjkTokenizerMode::Bigram);
331 assert!(result.contains("机器"));
332 assert!(result.contains("器学"));
333 assert!(result.contains("学习"));
334 }
335
336 #[test]
337 fn test_tokenize_for_fts_char_mode() {
338 let result = tokenize_for_fts("机器学习", CjkTokenizerMode::Char);
339 assert!(result.contains("机"));
340 assert!(result.contains("器"));
341 assert!(result.contains("学"));
342 assert!(result.contains("习"));
343 }
344
345 #[test]
348 fn test_fts_expression_preserves_operators() {
349 let result = tokenize_fts_expression("machine AND learning", CjkTokenizerMode::Auto);
350 assert!(result.contains("AND"));
351 assert!(result.contains("machine"));
352 assert!(result.contains("learning"));
353 }
354
355 #[test]
356 fn test_fts_expression_or_operator() {
357 let result = tokenize_fts_expression("rust OR python", CjkTokenizerMode::Auto);
358 assert!(result.contains("OR"));
359 }
360
361 #[test]
362 fn test_fts_expression_not_operator() {
363 let result = tokenize_fts_expression("search NOT vector", CjkTokenizerMode::Auto);
364 assert!(result.contains("NOT"));
365 }
366
367 #[test]
368 fn test_fts_expression_near_operator() {
369 let result = tokenize_fts_expression("tree NEAR search", CjkTokenizerMode::Auto);
370 assert!(result.contains("NEAR"));
371 }
372
373 #[test]
374 fn test_fts_expression_case_insensitive_operators() {
375 let result = tokenize_fts_expression("rust and python", CjkTokenizerMode::Auto);
376 assert!(result.contains("AND"));
377 }
378
379 #[test]
380 fn test_fts_expression_chinese() {
381 let result = tokenize_fts_expression("机器学习 AND 深度学习", CjkTokenizerMode::Auto);
382 assert!(result.contains("AND"));
383 assert!(result.contains("机器"));
384 }
385
386 #[test]
387 fn test_fts_expression_strips_special_chars() {
388 let result = tokenize_fts_expression("hello! AND world?", CjkTokenizerMode::Auto);
389 assert!(result.contains("AND"));
390 assert!(!result.contains("!"));
392 assert!(!result.contains("?"));
393 }
394
395 #[test]
398 fn test_tokenize_english_no_stopwords() {
399 let tokens = tokenize("hello world test", false, CjkTokenizerMode::Auto);
400 assert_eq!(tokens, vec!["hello", "world", "test"]);
401 }
402
403 #[test]
404 fn test_tokenize_english_with_stopword_removal() {
405 let tokens = tokenize(
406 "the quick brown fox is a test",
407 true,
408 CjkTokenizerMode::Auto,
409 );
410 assert!(tokens.contains(&"quick".to_string()));
411 assert!(tokens.contains(&"brown".to_string()));
412 assert!(tokens.contains(&"fox".to_string()));
413 assert!(tokens.contains(&"test".to_string()));
414 assert!(!tokens.contains(&"the".to_string()));
416 assert!(!tokens.contains(&"is".to_string()));
417 assert!(!tokens.contains(&"a".to_string()));
418 }
419
420 #[test]
421 fn test_tokenize_chinese_no_stopwords() {
422 let tokens = tokenize("机器学习是人工智能的分支", false, CjkTokenizerMode::Auto);
423 assert!(!tokens.is_empty());
424 let joined = tokens.join(" ");
425 assert!(joined.contains("机器"));
426 assert!(joined.contains("学习"));
427 }
428
429 #[test]
430 fn test_tokenize_chinese_with_stopword_removal() {
431 let tokens = tokenize("机器学习是人工智能的分支", true, CjkTokenizerMode::Auto);
432 assert!(!tokens.contains(&"是".to_string()));
434 assert!(!tokens.contains(&"的".to_string()));
435 }
436
437 #[test]
438 fn test_tokenize_mixed_text() {
439 let tokens = tokenize("Python机器学习", false, CjkTokenizerMode::Auto);
440 assert!(!tokens.is_empty());
441 let joined = tokens.join(" ");
443 assert!(
444 joined.contains("机器") || joined.contains("机"),
445 "expected CJK tokens in: {joined}"
446 );
447 }
448
449 #[test]
450 fn test_tokenize_empty() {
451 let tokens = tokenize("", false, CjkTokenizerMode::Auto);
452 assert!(tokens.is_empty());
453 }
454
455 #[test]
456 fn test_tokenize_filters_single_english_chars() {
457 let tokens = tokenize("a b c hello", false, CjkTokenizerMode::Auto);
459 assert!(!tokens.contains(&"b".to_string()));
460 assert!(!tokens.contains(&"c".to_string()));
461 assert!(tokens.contains(&"hello".to_string()));
462 }
463
464 #[test]
465 fn test_tokenize_keeps_single_cjk_chars() {
466 let tokens = tokenize("我", false, CjkTokenizerMode::Auto);
467 assert!(tokens.contains(&"我".to_string()));
468 }
469
470 #[test]
473 fn test_english_stopwords_coverage() {
474 assert!(EN_STOPWORDS.contains("the"));
475 assert!(EN_STOPWORDS.contains("is"));
476 assert!(EN_STOPWORDS.contains("and"));
477 assert!(EN_STOPWORDS.contains("or"));
478 assert!(EN_STOPWORDS.contains("not"));
479 assert!(EN_STOPWORDS.contains("i"));
480 assert!(EN_STOPWORDS.contains("you"));
481 assert!(EN_STOPWORDS.contains("we"));
482 }
483
484 #[test]
485 fn test_chinese_stopwords_coverage() {
486 assert!(ZH_STOPWORDS.contains("的"));
487 assert!(ZH_STOPWORDS.contains("了"));
488 assert!(ZH_STOPWORDS.contains("是"));
489 assert!(ZH_STOPWORDS.contains("在"));
490 assert!(ZH_STOPWORDS.contains("什么"));
491 assert!(ZH_STOPWORDS.contains("为什么"));
492 }
493
494 #[test]
495 fn test_non_stopword_not_present() {
496 assert!(!EN_STOPWORDS.contains("machine"));
497 assert!(!EN_STOPWORDS.contains("learning"));
498 assert!(!ZH_STOPWORDS.contains("机器"));
499 assert!(!ZH_STOPWORDS.contains("学习"));
500 }
501}