khmer_tokenizer_core/
kcc.rs

1//! Khmer Character Cluster (KCC) splitting.
2//!
3//! Written Khmer has no spaces between words and uses an abugida script: a base
4//! consonant can carry stacked subscript consonants (each introduced by COENG,
5//! `U+17D2`) plus dependent vowels and signs. The smallest meaningful unit for
6//! word segmentation is therefore the *orthographic cluster*, not the Unicode
7//! scalar value.
8//!
9//! Splitting into clusters first is what keeps the segmenter from ever cutting a
10//! base character away from its subscripts or vowels — the core correctness bug
11//! in naive char-by-char tokenizers.
12
13/// `U+17D2` KHMER SIGN COENG — joins a following consonant as a subscript.
14pub(crate) const COENG: char = '\u{17D2}';
15
16/// `U+17CC` KHMER SIGN ROBAT — the one combining mark that legitimately
17/// precedes a `COENG`+consonant subscript pair (see `crate::normalize`).
18pub(crate) const ROBAT: char = '\u{17CC}';
19
20/// A Khmer *base*: consonants (`U+1780..=U+17A2`) and independent vowels
21/// (`U+17A3..=U+17B3`). A cluster always begins with one of these.
22pub(crate) fn is_khmer_base(c: char) -> bool {
23    matches!(c as u32, 0x1780..=0x17B3)
24}
25
26/// Dependent vowels, signs and diacritics that attach to a base, plus the
27/// zero-width joiners that may appear inside a cluster.
28pub(crate) fn is_khmer_combining(c: char) -> bool {
29    let o = c as u32;
30    matches!(o, 0x17B4..=0x17D1)   // dependent vowels & most signs
31        || o == 0x17D3             // bathamasat
32        || o == 0x17DD             // atthacan
33        || matches!(o, 0x200C..=0x200D) // ZWNJ / ZWJ
34}
35
36/// True if `c` falls anywhere in the Khmer Unicode block (`U+1780..=U+17FF`).
37pub fn is_khmer(c: char) -> bool {
38    matches!(c as u32, 0x1780..=0x17FF)
39}
40
41/// Split `text` into Khmer Character Clusters.
42///
43/// Each Khmer cluster is a base character followed by any number of
44/// COENG+consonant subscripts and combining marks. Non-Khmer scalars (spaces,
45/// Latin letters, digits, punctuation) are returned as individual single-char
46/// clusters, so downstream code can group or separate them as needed.
47///
48/// # Example
49/// ```
50/// use khmer_tokenizer_core::split_kcc;
51/// assert_eq!(split_kcc("ខ្មែរ"), vec!["ខ្មែ", "រ"]);
52/// assert_eq!(split_kcc("ស្ត្រី"), vec!["ស្ត្រី"]); // stacked subscripts stay whole
53/// ```
54pub fn split_kcc(text: &str) -> Vec<String> {
55    let chars: Vec<char> = text.chars().collect();
56    let n = chars.len();
57    let mut clusters: Vec<String> = Vec::new();
58    let mut i = 0;
59
60    while i < n {
61        let c = chars[i];
62        if is_khmer_base(c) {
63            let start = i;
64            i += 1;
65            while i < n {
66                let d = chars[i];
67                if d == COENG && i + 1 < n {
68                    // COENG plus the consonant it subscripts.
69                    i += 2;
70                } else if is_khmer_combining(d) {
71                    i += 1;
72                } else {
73                    break;
74                }
75            }
76            clusters.push(chars[start..i].iter().collect());
77        } else {
78            // Non-base scalar (space, Latin, digit, punctuation, stray mark).
79            clusters.push(c.to_string());
80            i += 1;
81        }
82    }
83
84    clusters
85}
86
87#[cfg(test)]
88mod tests {
89    use super::*;
90
91    #[test]
92    fn keeps_base_with_subscripts_and_vowels() {
93        assert_eq!(split_kcc("ខ្មែរ"), vec!["ខ្មែ", "រ"]);
94        assert_eq!(split_kcc("ស្ត្រី"), vec!["ស្ត្រី"]);
95    }
96
97    #[test]
98    fn separates_non_khmer_scalars() {
99        assert_eq!(split_kcc("ab 1"), vec!["a", "b", " ", "1"]);
100    }
101
102    #[test]
103    fn classifies_khmer_block() {
104        assert!(is_khmer('ខ'));
105        assert!(!is_khmer('a'));
106        assert!(!is_khmer('1'));
107    }
108}
khmer_tokenizer_core/kcc.rs

khmer_tokenizer_core/
kcc.rs
