kham-core 0.4.0

Pure Rust Thai word segmentation engine — no_std compatible
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152

use kham_core::fts::FtsTokenizer;
use kham_core::romanizer::RomanizationMap;
use kham_core::stopwords::StopwordSet;

#[test]
fn builtin_map_is_non_empty() {
    let map = RomanizationMap::builtin();
    assert!(map.len() > 100, "expected at least 100 built-in entries");
}

#[test]
fn common_words_roundtrip() {
    let map = RomanizationMap::builtin();
    let cases = [
        ("กิน", "kin"),
        ("ข้าว", "khao"),
        ("น้ำ", "nam"),
        ("ปลา", "pla"),
        ("ไฟ", "fai"),
        ("ดี", "di"),
        ("มา", "ma"),
        ("ทำ", "tham"),
    ];
    for (thai, rtgs) in cases {
        assert_eq!(map.romanize(thai), Some(rtgs), "failed for {thai}");
    }
}

#[test]
fn unknown_word_is_none() {
    let map = RomanizationMap::builtin();
    assert_eq!(map.romanize("เปปซี่"), None);
    assert_eq!(map.romanize(""), None);
}

#[test]
fn romanize_or_raw_returns_word_when_missing() {
    let map = RomanizationMap::builtin();
    let oov = "เปปซี่";
    assert_eq!(map.romanize_or_raw(oov), oov);
}

#[test]
fn romanize_tokens_mixed_known_and_unknown() {
    let map = RomanizationMap::builtin();
    let tokens = vec!["กิน", "ข้าว", "เปปซี่"];
    let out = map.romanize_tokens(&tokens);
    assert_eq!(out[0], "kin");
    assert_eq!(out[1], "khao");
    assert_eq!(out[2], "เปปซี่");
}

#[test]
fn romanize_tokens_length_matches_input() {
    let map = RomanizationMap::builtin();
    let tokens: Vec<&str> = vec!["กิน", "ข้าว", "ปลา", "น้ำ", "ไฟ"];
    let out = map.romanize_tokens(&tokens);
    assert_eq!(out.len(), tokens.len());
}

#[test]
fn from_tsv_override_last_wins() {
    let map = RomanizationMap::from_tsv("กิน\tkin\nกิน\tgin\n");
    assert_eq!(map.romanize("กิน"), Some("gin"));
}

#[test]
fn from_tsv_trims_whitespace() {
    let map = RomanizationMap::from_tsv("กิน\t kin \n");
    assert_eq!(map.romanize("กิน"), Some("kin"));
}

// ── FtsTokenizer integration ──────────────────────────────────────────────────

#[test]
fn fts_romanization_appended_to_synonyms() {
    let fts = FtsTokenizer::builder()
        .romanization(RomanizationMap::builtin())
        .stopwords(StopwordSet::from_text(""))
        .build();
    let tokens = fts.segment_for_fts("กิน");
    let t = tokens.iter().find(|t| t.text == "กิน");
    assert!(t.is_some(), "expected 'กิน' token");
    let t = t.unwrap();
    assert!(
        t.synonyms.contains(&String::from("kin")),
        "expected 'kin' in synonyms, got {:?}",
        t.synonyms
    );
}

#[test]
fn fts_without_romanization_no_rtgs_in_synonyms() {
    let fts = FtsTokenizer::builder()
        .stopwords(StopwordSet::from_text(""))
        .build();
    let tokens = fts.segment_for_fts("กิน");
    let t = tokens.iter().find(|t| t.text == "กิน").unwrap();
    assert!(
        !t.synonyms.contains(&String::from("kin")),
        "romanization should not appear when not configured"
    );
}

#[test]
fn fts_romanization_coexists_with_synonyms() {
    let fts = FtsTokenizer::builder()
        .romanization(RomanizationMap::from_tsv("กิน\tkin\n"))
        .synonyms(kham_core::synonym::SynonymMap::from_tsv("กิน\teat\n"))
        .stopwords(StopwordSet::from_text(""))
        .build();
    let tokens = fts.segment_for_fts("กิน");
    let t = tokens.iter().find(|t| t.text == "กิน").unwrap();
    assert!(t.synonyms.contains(&String::from("eat")), "synonym missing");
    assert!(t.synonyms.contains(&String::from("kin")), "rtgs missing");
}

#[test]
fn fts_oov_token_gets_no_rtgs_synonym() {
    let fts = FtsTokenizer::builder()
        .romanization(RomanizationMap::from_tsv("กิน\tkin\n"))
        .stopwords(StopwordSet::from_text(""))
        .build();
    let tokens = fts.segment_for_fts("ข้าว");
    let t = tokens.iter().find(|t| t.text == "ข้าว");
    if let Some(t) = t {
        // "ข้าว" not in our custom map → no RTGS synonym added
        assert!(
            t.synonyms.is_empty(),
            "OOV word should have no synonyms, got {:?}",
            t.synonyms
        );
    }
}

#[test]
fn fts_named_entity_gets_rtgs_synonym() {
    // "ไทย" is PLACE in the NE gazetteer — TokenKind::Named
    let fts = FtsTokenizer::builder()
        .romanization(RomanizationMap::builtin())
        .stopwords(StopwordSet::from_text(""))
        .build();
    let tokens = fts.segment_for_fts("ไทย");
    let t = tokens.iter().find(|t| t.text == "ไทย");
    if let Some(t) = t {
        assert!(
            t.synonyms.contains(&String::from("thai")),
            "Named token 'ไทย' should have RTGS synonym 'thai', got {:?}",
            t.synonyms
        );
    }
}