Skip to main content

rustai_core/
text.rs

1//! Shared tokenisation.
2//!
3//! One tokeniser feeds BM25, the density scorer and the redundancy filter, so
4//! that a "token" means the same thing everywhere — including in the budget the
5//! caller sets for their local model's context window.
6//!
7//! Latin-script text is split on word boundaries and lowercased. CJK runs are
8//! additionally emitted as character bigrams, the standard trick that lets a
9//! bag-of-words ranker work on Korean, Japanese and Chinese without shipping a
10//! morphological analyser.
11
12use unicode_segmentation::UnicodeSegmentation;
13
14/// True for Hangul, Hiragana/Katakana and CJK ideographs.
15#[inline]
16pub fn is_cjk(c: char) -> bool {
17    matches!(c as u32,
18        0x1100..=0x11FF   // Hangul Jamo
19        | 0x3040..=0x30FF // Kana
20        | 0x3400..=0x4DBF // CJK ext A
21        | 0x4E00..=0x9FFF // CJK unified
22        | 0xA960..=0xA97F // Hangul Jamo ext A
23        | 0xAC00..=0xD7AF // Hangul syllables
24        | 0xF900..=0xFAFF // CJK compat
25        | 0xFF66..=0xFF9D // halfwidth kana
26        | 0x20000..=0x2FA1F
27    )
28}
29
30/// A token is kept if it carries any letter or digit.
31#[inline]
32fn is_meaningful(w: &str) -> bool {
33    w.chars().any(|c| c.is_alphanumeric())
34}
35
36/// Split text into lowercase tokens.
37///
38/// CJK runs contribute both single characters and adjacent bigrams; a bigram is
39/// what actually discriminates in those scripts, while the unigrams keep recall
40/// for single-character terms.
41pub fn tokenize(text: &str) -> Vec<String> {
42    let mut out = Vec::with_capacity(text.len() / 5);
43    for word in text.unicode_words() {
44        if !is_meaningful(word) {
45            continue;
46        }
47        let cjk_chars: Vec<char> = word.chars().filter(|c| is_cjk(*c)).collect();
48        if cjk_chars.len() * 2 >= word.chars().count() && !cjk_chars.is_empty() {
49            for c in &cjk_chars {
50                out.push(c.to_lowercase().collect());
51            }
52            for pair in cjk_chars.windows(2) {
53                out.push(pair.iter().collect());
54            }
55        } else {
56            out.push(word.to_lowercase());
57        }
58    }
59    out
60}
61
62/// Approximate token count for an LLM context budget.
63///
64/// Calibrated against byte-pair vocabularies: roughly 4 bytes per token for
65/// Latin script, and closer to 1.4 characters per token for CJK, which BPE
66/// vocabularies split much more finely. This is deliberately a slight
67/// over-estimate so a budget is never blown.
68pub fn estimate_tokens(text: &str) -> usize {
69    let mut cjk = 0usize;
70    let mut other_bytes = 0usize;
71    for c in text.chars() {
72        if is_cjk(c) {
73            cjk += 1;
74        } else {
75            other_bytes += c.len_utf8();
76        }
77    }
78    let latin = other_bytes.div_ceil(4);
79    let cjk_tokens = (cjk as f64 / 1.4).ceil() as usize;
80    (latin + cjk_tokens).max(usize::from(!text.trim().is_empty()))
81}
82
83/// Collapse runs of whitespace and trim, without allocating when unnecessary.
84pub fn normalize_ws(s: &str) -> String {
85    let mut out = String::with_capacity(s.len());
86    let mut pending_space = false;
87    for c in s.chars() {
88        if c.is_whitespace() {
89            pending_space = !out.is_empty();
90        } else {
91            if pending_space {
92                out.push(' ');
93                pending_space = false;
94            }
95            out.push(c);
96        }
97    }
98    out
99}
100
101/// Visible length in characters — a fairer "length" than `len()` for CJK.
102///
103/// Deliberately `chars()`, not grapheme clusters. Every use of this is a
104/// heuristic threshold or a ratio, none of which can tell the difference, and
105/// Unicode segmentation over every text node in a document measured out at
106/// roughly 78% of total extraction time — a 4.4x slowdown for an exactness
107/// nothing here consumes. Combining marks and emoji sequences count slightly
108/// high, which only ever errs toward keeping content.
109pub fn visible_len(s: &str) -> usize {
110    s.chars().count()
111}
112
113#[cfg(test)]
114mod visible_len_tests {
115    use super::*;
116    use unicode_segmentation::UnicodeSegmentation;
117
118    #[test]
119    fn agrees_with_grapheme_counting_on_ordinary_text() {
120        // The two differ only on combining marks and emoji sequences, which no
121        // threshold in this crate is sensitive to.
122        for s in ["hello world", "한국어 텍스트입니다", "日本語のテキスト", "Ελληνικά", ""]
123        {
124            assert_eq!(visible_len(s), s.graphemes(true).count(), "{s:?}");
125        }
126    }
127}
128
129#[cfg(test)]
130mod tests {
131    use super::*;
132
133    #[test]
134    fn latin_tokens_are_lowercased_words() {
135        assert_eq!(tokenize("Hello,  World! 42"), ["hello", "world", "42"]);
136    }
137
138    #[test]
139    fn korean_yields_unigrams_and_bigrams() {
140        let t = tokenize("한국어");
141        assert!(t.contains(&"한".to_string()));
142        assert!(t.contains(&"한국".to_string()));
143        assert!(t.contains(&"국어".to_string()));
144    }
145
146    #[test]
147    fn cjk_costs_more_tokens_per_char_than_latin() {
148        assert!(estimate_tokens("데이터 수집") > estimate_tokens("data"));
149        assert_eq!(estimate_tokens(""), 0);
150    }
151
152    #[test]
153    fn whitespace_collapses() {
154        assert_eq!(normalize_ws("  a \n\t b  "), "a b");
155    }
156}