khmer_tokenizer_core/
lib.rs

1//! # khmer-tokenizer-core
2//!
3//! A fast, dependency-free Khmer word segmenter.
4//!
5//! Written Khmer does not put spaces between words, so segmentation is the first
6//! step of nearly every Khmer NLP pipeline. This crate does it in two passes:
7//!
8//! 1. **Cluster pass** — [`split_kcc`] groups the text into Khmer Character
9//!    Clusters (a base character plus its subscripts and vowels). This keeps the
10//!    segmenter from ever splitting *inside* an orthographic syllable.
11//! 2. **Boundary pass** — [`KhmerTokenizer`] walks a cluster-keyed trie to
12//!    place word boundaries, using one of a few [`Strategy`] algorithms
13//!    (default: greedy longest-match, falling back to a single cluster when
14//!    nothing matches).
15//!
16//! The engine is `std`-only (no external dependencies) and deterministic.
17//!
18//! ## Quick start
19//! ```
20//! use khmer_tokenizer_core::KhmerTokenizer;
21//!
22//! let tk = KhmerTokenizer::with_default_dict();
23//! let tokens = tk.segment("សួស្តីអ្នកទាំងអស់គ្នា");
24//! assert_eq!(tokens, vec!["សួស្តី", "អ្នក", "ទាំងអស់គ្នា"]);
25//! ```
26
27mod hmm;
28mod kcc;
29mod normalize;
30mod strategy;
31mod trie;
32
33pub use hmm::HmmModel;
34pub use kcc::{is_khmer, split_kcc};
35pub use normalize::normalize;
36pub use strategy::Strategy;
37pub use trie::KhmerTokenizer;
38
39/// The embedded default dictionary (59,526 words; see `ATTRIBUTION.md`): one
40/// word per line; blank lines and lines starting with `#` are ignored.
41/// Replace or extend it for your own use case — see the dictionary notes in
42/// the project README.
43pub const DEFAULT_DICT: &str = include_str!("dict.txt");
44
45impl KhmerTokenizer {
46    /// Build a tokenizer pre-loaded with the embedded default dictionary
47    /// ([`DEFAULT_DICT`]).
48    pub fn with_default_dict() -> Self {
49        Self::from_dict_str(DEFAULT_DICT)
50    }
51
52    /// Build a tokenizer from a newline-separated word list. Blank lines and
53    /// lines beginning with `#` are skipped, so dictionary files may carry
54    /// comments.
55    pub fn from_dict_str(dict: &str) -> Self {
56        let words = dict
57            .lines()
58            .map(str::trim)
59            .filter(|l| !l.is_empty() && !l.starts_with('#'));
60        Self::from_words(words)
61    }
62}
63
64#[cfg(test)]
65mod tests {
66    use super::*;
67
68    fn tk() -> KhmerTokenizer {
69        KhmerTokenizer::from_words([
70            "សួស្តី",
71            "អ្នក",
72            "ទាំងអស់គ្នា",
73            "កម្ពុជា",
74            "ភាសា",
75            "ខ្មែរ",
76            "ខ្ញុំ",
77            "ស្រឡាញ់",
78        ])
79    }
80
81    #[test]
82    fn segments_known_words() {
83        assert_eq!(
84            tk().segment("សួស្តីអ្នកទាំងអស់គ្នា"),
85            vec!["សួស្តី", "អ្នក", "ទាំងអស់គ្នា"]
86        );
87        assert_eq!(
88            tk().segment("ខ្ញុំស្រឡាញ់កម្ពុជា"),
89            vec!["ខ្ញុំ", "ស្រឡាញ់", "កម្ពុជា"]
90        );
91        assert_eq!(tk().segment("ភាសាខ្មែរ"), vec!["ភាសា", "ខ្មែរ"]);
92    }
93
94    #[test]
95    fn handles_mixed_scripts() {
96        assert_eq!(
97            tk().segment("ខ្ញុំស្រឡាញ់ Rust 2026 កម្ពុជា"),
98            vec!["ខ្ញុំ", "ស្រឡាញ់", "Rust", "2026", "កម្ពុជា"]
99        );
100    }
101
102    #[test]
103    fn oov_falls_back_to_clusters() {
104        // ឆ្នាំ and ថ្មី are absent from this dictionary -> single clusters.
105        assert_eq!(tk().segment("ឆ្នាំថ្មី"), vec!["ឆ្នាំ", "ថ្មី"]);
106    }
107
108    #[test]
109    fn default_dict_loads_and_segments() {
110        let tk = KhmerTokenizer::with_default_dict();
111        assert!(!tk.is_empty());
112        assert_eq!(
113            tk.segment("សួស្តីអ្នកទាំងអស់គ្នា"),
114            vec!["សួស្តី", "អ្នក", "ទាំងអស់គ្នា"]
115        );
116    }
117
118    #[test]
119    fn dict_str_skips_comments_and_blanks() {
120        let tk = KhmerTokenizer::from_dict_str("# comment\n\nខ្មែរ\n  ភាសា  \n");
121        assert_eq!(tk.len(), 2);
122    }
123}
khmer_tokenizer_core/lib.rs

khmer_tokenizer_core/
lib.rs
