Skip to main content

treesearch/tokenizer/
mod.rs

1//! Tokenizer module for TreeSearch FTS5 and search indexing.
2//!
3//! Supports CJK (Chinese, Japanese, Korean) and English tokenization.
4//! CJK tokenization mode is configurable via `CjkTokenizerMode` in config.
5//!
6//! Ported from Python `treesearch/tokenizer.py` and `treesearch/fts.py`.
7
8pub mod cjk;
9
10use regex::Regex;
11use std::collections::HashSet;
12use std::sync::LazyLock;
13
14use crate::config::CjkTokenizerMode;
15
16// ---------------------------------------------------------------------------
17// Global regex patterns
18// ---------------------------------------------------------------------------
19
20/// Regex to detect CJK characters (Chinese, Japanese, Korean).
21pub static RE_HAS_CJK: LazyLock<Regex> = LazyLock::new(|| {
22    Regex::new(r"[\u4e00-\u9fff\u3400-\u4dbf\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af\uf900-\ufaff]")
23        .unwrap()
24});
25
26/// Regex to split English text on non-word boundaries.
27static RE_SPLIT_EN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\W+").unwrap());
28
29/// Regex to strip FTS5 special characters.
30/// Keeps only word characters (letters, digits, underscore) and CJK ranges.
31static RE_FTS5_SPECIAL: LazyLock<Regex> = LazyLock::new(|| {
32    Regex::new(r"[^\w\u4e00-\u9fff\u3400-\u4dbf\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]").unwrap()
33});
34
35/// FTS5 operators that must NOT be tokenized.
36const FTS5_OPERATORS: &[&str] = &["AND", "OR", "NOT", "NEAR"];
37
38// ---------------------------------------------------------------------------
39// Stop words (ported from Python treesearch/heuristics.py)
40// ---------------------------------------------------------------------------
41
42/// English stop words.
43static EN_STOPWORDS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
44    [
45        "a", "an", "the", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had",
46        "do", "does", "did", "will", "would", "could", "should", "may", "might", "shall", "can",
47        "of", "in", "to", "for", "with", "on", "at", "by", "from", "as", "into", "through",
48        "during", "before", "after", "above", "below", "between", "and", "but", "or", "not", "no",
49        "so", "if", "than", "too", "very", "just", "about", "also", "then", "this", "that",
50        "these", "those", "it", "its", "what", "which", "who", "whom", "how", "when", "where",
51        "why", "all", "each", "every", "both", "few", "more", "most", "other", "some", "such",
52        "only", "own", "same", "we", "they", "he", "she", "us", "our", "their", "your", "my", "i",
53        "me", "you", "nor", "need", "dare", "him", "his", "her", "them", "any", "again", "under",
54        "over",
55    ]
56    .into_iter()
57    .collect()
58});
59
60/// Chinese stop words.
61static ZH_STOPWORDS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
62    [
63        "的",
64        "了",
65        "在",
66        "是",
67        "我",
68        "有",
69        "和",
70        "就",
71        "不",
72        "人",
73        "都",
74        "一",
75        "一个",
76        "上",
77        "也",
78        "很",
79        "到",
80        "说",
81        "要",
82        "去",
83        "你",
84        "会",
85        "着",
86        "没有",
87        "看",
88        "好",
89        "自己",
90        "这",
91        "他",
92        "她",
93        "它",
94        "吗",
95        "吧",
96        "呢",
97        "啊",
98        "呀",
99        "哦",
100        "嗯",
101        "嘛",
102        "哈",
103        "怎样",
104        "怎么",
105        "什么",
106        "哪",
107        "哪个",
108        "哪些",
109        "为什么",
110        "如何",
111        "可以",
112        "能",
113        "把",
114        "被",
115        "让",
116        "给",
117        "对",
118        "从",
119        "向",
120        "跟",
121        "还",
122        "又",
123        "再",
124        "已",
125        "已经",
126        "正在",
127        "将",
128        "将要",
129    ]
130    .into_iter()
131    .collect()
132});
133
134// ---------------------------------------------------------------------------
135// Public API
136// ---------------------------------------------------------------------------
137
138/// Check whether a string contains any CJK character.
139pub fn has_cjk(text: &str) -> bool {
140    RE_HAS_CJK.is_match(text)
141}
142
143/// Tokenize text for FTS5 indexing. Returns space-separated tokens.
144///
145/// Only uses jieba for CJK text; English text is returned as-is
146/// (FTS5 unicode61 handles it natively).
147///
148/// Equivalent to Python's `_tokenize_for_fts()` in `fts.py`.
149pub fn tokenize_for_fts(text: &str, cjk_mode: CjkTokenizerMode) -> String {
150    if text.is_empty() || text.trim().is_empty() {
151        return String::new();
152    }
153    if RE_HAS_CJK.is_match(text) {
154        let tokens = tokenize(text, false, cjk_mode);
155        tokens.join(" ")
156    } else {
157        // English / non-CJK: return as-is, FTS5 unicode61 handles tokenization
158        text.to_string()
159    }
160}
161
162/// Tokenize a query string, preserving FTS5 operators (AND, OR, NOT, NEAR).
163///
164/// Non-operator parts are tokenized via `tokenize_for_fts`. FTS5 special
165/// characters are stripped from each token.
166///
167/// Equivalent to Python's `_tokenize_fts_expression()` in `fts.py`.
168pub fn tokenize_fts_expression(expr: &str, cjk_mode: CjkTokenizerMode) -> String {
169    let parts: Vec<&str> = expr.split_whitespace().collect();
170    let mut result = Vec::new();
171
172    for part in parts {
173        if FTS5_OPERATORS.contains(&part.to_uppercase().as_str()) {
174            result.push(part.to_uppercase());
175        } else {
176            // Strip FTS5 special characters
177            let cleaned = RE_FTS5_SPECIAL.replace_all(part, "");
178            if cleaned.is_empty() {
179                continue;
180            }
181            let tokenized = tokenize_for_fts(&cleaned, cjk_mode);
182            let trimmed = tokenized.trim().to_string();
183            if !trimmed.is_empty() {
184                result.push(trimmed);
185            }
186        }
187    }
188
189    result.join(" ")
190}
191
192/// General tokenization with optional stopword removal.
193///
194/// CJK tokenization mode is determined by `cjk_mode`:
195///   - `Auto` / `Jieba`: jieba word segmentation
196///   - `Bigram`: CJK character 2-grams
197///   - `Char`: single-character splitting
198///
199/// Equivalent to Python's `tokenize()` in `tokenizer.py`.
200pub fn tokenize(text: &str, remove_stopwords: bool, cjk_mode: CjkTokenizerMode) -> Vec<String> {
201    if text.is_empty() {
202        return Vec::new();
203    }
204
205    let is_cjk = RE_HAS_CJK.is_match(text);
206
207    let raw_tokens: Vec<String> = if is_cjk {
208        match cjk_mode {
209            CjkTokenizerMode::Jieba | CjkTokenizerMode::Auto => cjk::jieba_cut(text),
210            CjkTokenizerMode::Bigram => cjk::bigrams(text),
211            CjkTokenizerMode::Char => cjk::chars(text),
212        }
213    } else {
214        RE_SPLIT_EN
215            .split(text)
216            .filter(|s| !s.is_empty())
217            .map(|s| s.to_lowercase())
218            .collect()
219    };
220
221    // Filter: keep CJK single chars, require len>1 for English tokens
222    let filtered: Vec<String> = raw_tokens
223        .into_iter()
224        .filter(|t| {
225            let trimmed = t.trim();
226            if trimmed.is_empty() {
227                return false;
228            }
229            if trimmed.chars().count() == 1 {
230                // Single char: only keep if CJK
231                let ch = trimmed.chars().next().unwrap();
232                is_cjk_char(ch)
233            } else {
234                true
235            }
236        })
237        .map(|t| t.trim().to_string())
238        .collect();
239
240    if !remove_stopwords {
241        return filtered;
242    }
243
244    // Remove stop words
245    filtered
246        .into_iter()
247        .filter(|t| {
248            let lower = t.to_lowercase();
249            !EN_STOPWORDS.contains(lower.as_str()) && !ZH_STOPWORDS.contains(t.as_str())
250        })
251        .collect()
252}
253
254/// Check if a character is in CJK ranges (inline, no regex).
255fn is_cjk_char(ch: char) -> bool {
256    matches!(ch,
257        '\u{4e00}'..='\u{9fff}'   // CJK Unified Ideographs
258        | '\u{3400}'..='\u{4dbf}' // CJK Unified Ideographs Extension A
259        | '\u{3040}'..='\u{309f}' // Hiragana
260        | '\u{30a0}'..='\u{30ff}' // Katakana
261        | '\u{ac00}'..='\u{d7af}' // Hangul Syllables
262        | '\u{f900}'..='\u{faff}' // CJK Compatibility Ideographs
263    )
264}
265
266// ---------------------------------------------------------------------------
267// Tests
268// ---------------------------------------------------------------------------
269
270#[cfg(test)]
271mod tests {
272    use super::*;
273
274    // --- has_cjk ---
275
276    #[test]
277    fn test_has_cjk_chinese() {
278        assert!(has_cjk("hello你好"));
279        assert!(has_cjk("机器学习"));
280    }
281
282    #[test]
283    fn test_has_cjk_japanese() {
284        assert!(has_cjk("こんにちは"));
285        assert!(has_cjk("カタカナ"));
286    }
287
288    #[test]
289    fn test_has_cjk_korean() {
290        assert!(has_cjk("한글"));
291    }
292
293    #[test]
294    fn test_has_cjk_english_only() {
295        assert!(!has_cjk("hello world"));
296        assert!(!has_cjk("rust programming 123"));
297    }
298
299    #[test]
300    fn test_has_cjk_empty() {
301        assert!(!has_cjk(""));
302    }
303
304    // --- tokenize_for_fts ---
305
306    #[test]
307    fn test_tokenize_for_fts_english() {
308        let result = tokenize_for_fts("hello world test", CjkTokenizerMode::Auto);
309        // English text returned as-is for FTS5 unicode61
310        assert_eq!(result, "hello world test");
311    }
312
313    #[test]
314    fn test_tokenize_for_fts_chinese() {
315        let result = tokenize_for_fts("机器学习是人工智能", CjkTokenizerMode::Auto);
316        // Should be space-separated jieba tokens
317        assert!(!result.is_empty());
318        assert!(result.contains("机器"));
319        assert!(result.contains("学习"));
320    }
321
322    #[test]
323    fn test_tokenize_for_fts_empty() {
324        assert_eq!(tokenize_for_fts("", CjkTokenizerMode::Auto), "");
325        assert_eq!(tokenize_for_fts("   ", CjkTokenizerMode::Auto), "");
326    }
327
328    #[test]
329    fn test_tokenize_for_fts_bigram_mode() {
330        let result = tokenize_for_fts("机器学习", CjkTokenizerMode::Bigram);
331        assert!(result.contains("机器"));
332        assert!(result.contains("器学"));
333        assert!(result.contains("学习"));
334    }
335
336    #[test]
337    fn test_tokenize_for_fts_char_mode() {
338        let result = tokenize_for_fts("机器学习", CjkTokenizerMode::Char);
339        assert!(result.contains("机"));
340        assert!(result.contains("器"));
341        assert!(result.contains("学"));
342        assert!(result.contains("习"));
343    }
344
345    // --- tokenize_fts_expression ---
346
347    #[test]
348    fn test_fts_expression_preserves_operators() {
349        let result = tokenize_fts_expression("machine AND learning", CjkTokenizerMode::Auto);
350        assert!(result.contains("AND"));
351        assert!(result.contains("machine"));
352        assert!(result.contains("learning"));
353    }
354
355    #[test]
356    fn test_fts_expression_or_operator() {
357        let result = tokenize_fts_expression("rust OR python", CjkTokenizerMode::Auto);
358        assert!(result.contains("OR"));
359    }
360
361    #[test]
362    fn test_fts_expression_not_operator() {
363        let result = tokenize_fts_expression("search NOT vector", CjkTokenizerMode::Auto);
364        assert!(result.contains("NOT"));
365    }
366
367    #[test]
368    fn test_fts_expression_near_operator() {
369        let result = tokenize_fts_expression("tree NEAR search", CjkTokenizerMode::Auto);
370        assert!(result.contains("NEAR"));
371    }
372
373    #[test]
374    fn test_fts_expression_case_insensitive_operators() {
375        let result = tokenize_fts_expression("rust and python", CjkTokenizerMode::Auto);
376        assert!(result.contains("AND"));
377    }
378
379    #[test]
380    fn test_fts_expression_chinese() {
381        let result = tokenize_fts_expression("机器学习 AND 深度学习", CjkTokenizerMode::Auto);
382        assert!(result.contains("AND"));
383        assert!(result.contains("机器"));
384    }
385
386    #[test]
387    fn test_fts_expression_strips_special_chars() {
388        let result = tokenize_fts_expression("hello! AND world?", CjkTokenizerMode::Auto);
389        assert!(result.contains("AND"));
390        // Special chars should be stripped
391        assert!(!result.contains("!"));
392        assert!(!result.contains("?"));
393    }
394
395    // --- tokenize (general) ---
396
397    #[test]
398    fn test_tokenize_english_no_stopwords() {
399        let tokens = tokenize("hello world test", false, CjkTokenizerMode::Auto);
400        assert_eq!(tokens, vec!["hello", "world", "test"]);
401    }
402
403    #[test]
404    fn test_tokenize_english_with_stopword_removal() {
405        let tokens = tokenize(
406            "the quick brown fox is a test",
407            true,
408            CjkTokenizerMode::Auto,
409        );
410        assert!(tokens.contains(&"quick".to_string()));
411        assert!(tokens.contains(&"brown".to_string()));
412        assert!(tokens.contains(&"fox".to_string()));
413        assert!(tokens.contains(&"test".to_string()));
414        // Stop words removed
415        assert!(!tokens.contains(&"the".to_string()));
416        assert!(!tokens.contains(&"is".to_string()));
417        assert!(!tokens.contains(&"a".to_string()));
418    }
419
420    #[test]
421    fn test_tokenize_chinese_no_stopwords() {
422        let tokens = tokenize("机器学习是人工智能的分支", false, CjkTokenizerMode::Auto);
423        assert!(!tokens.is_empty());
424        let joined = tokens.join(" ");
425        assert!(joined.contains("机器"));
426        assert!(joined.contains("学习"));
427    }
428
429    #[test]
430    fn test_tokenize_chinese_with_stopword_removal() {
431        let tokens = tokenize("机器学习是人工智能的分支", true, CjkTokenizerMode::Auto);
432        // "是" and "的" are Chinese stop words
433        assert!(!tokens.contains(&"是".to_string()));
434        assert!(!tokens.contains(&"的".to_string()));
435    }
436
437    #[test]
438    fn test_tokenize_mixed_text() {
439        let tokens = tokenize("Python机器学习", false, CjkTokenizerMode::Auto);
440        assert!(!tokens.is_empty());
441        // Should contain both English and CJK tokens
442        let joined = tokens.join(" ");
443        assert!(
444            joined.contains("机器") || joined.contains("机"),
445            "expected CJK tokens in: {joined}"
446        );
447    }
448
449    #[test]
450    fn test_tokenize_empty() {
451        let tokens = tokenize("", false, CjkTokenizerMode::Auto);
452        assert!(tokens.is_empty());
453    }
454
455    #[test]
456    fn test_tokenize_filters_single_english_chars() {
457        // Single English characters (non-CJK) should be filtered out
458        let tokens = tokenize("a b c hello", false, CjkTokenizerMode::Auto);
459        assert!(!tokens.contains(&"b".to_string()));
460        assert!(!tokens.contains(&"c".to_string()));
461        assert!(tokens.contains(&"hello".to_string()));
462    }
463
464    #[test]
465    fn test_tokenize_keeps_single_cjk_chars() {
466        let tokens = tokenize("我", false, CjkTokenizerMode::Auto);
467        assert!(tokens.contains(&"我".to_string()));
468    }
469
470    // --- stopwords ---
471
472    #[test]
473    fn test_english_stopwords_coverage() {
474        assert!(EN_STOPWORDS.contains("the"));
475        assert!(EN_STOPWORDS.contains("is"));
476        assert!(EN_STOPWORDS.contains("and"));
477        assert!(EN_STOPWORDS.contains("or"));
478        assert!(EN_STOPWORDS.contains("not"));
479        assert!(EN_STOPWORDS.contains("i"));
480        assert!(EN_STOPWORDS.contains("you"));
481        assert!(EN_STOPWORDS.contains("we"));
482    }
483
484    #[test]
485    fn test_chinese_stopwords_coverage() {
486        assert!(ZH_STOPWORDS.contains("的"));
487        assert!(ZH_STOPWORDS.contains("了"));
488        assert!(ZH_STOPWORDS.contains("是"));
489        assert!(ZH_STOPWORDS.contains("在"));
490        assert!(ZH_STOPWORDS.contains("什么"));
491        assert!(ZH_STOPWORDS.contains("为什么"));
492    }
493
494    #[test]
495    fn test_non_stopword_not_present() {
496        assert!(!EN_STOPWORDS.contains("machine"));
497        assert!(!EN_STOPWORDS.contains("learning"));
498        assert!(!ZH_STOPWORDS.contains("机器"));
499        assert!(!ZH_STOPWORDS.contains("学习"));
500    }
501}