charabia 0.9.9

A simple library to detect the language, tokenize the text and normalize the tokens
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144

use std::collections::HashMap;

use serde::Deserialize;
use std::sync::LazyLock;

#[derive(Debug, PartialEq, Eq)]
pub enum KVariantClass {
    Wrong,
    SementicVariant,
    Simplified,
    Old,
    Equal,
}

#[derive(Debug, PartialEq, Eq)]
pub struct KVariant {
    pub source_ideograph: char,
    pub classification: KVariantClass,
    pub destination_ideograph: char,
}

#[derive(Deserialize)]
pub struct TsvRow {
    lhs: String,
    relation: String,
    rhs: String,
}

pub static KVARIANTS: LazyLock<HashMap<char, KVariant>> = LazyLock::new(|| {
    // The tab separated format is like:
    //
    //   㨲 (U+3A32)	wrong!	㩍 (U+3A4D)
    //   铿 (U+94FF)	simp	鏗 (U+93D7)
    //   㓻 (U+34FB)	sem	    剛 (U+525B)
    //   ...
    //
    let tsv = include_str!("../../../dictionaries/txt/chinese/kVariants.tsv");
    let mut reader =
        csv::ReaderBuilder::new().delimiter(b'\t').has_headers(false).from_reader(tsv.as_bytes());

    let mut map: HashMap<char, KVariant> = HashMap::new();
    for result in reader.deserialize() {
        let line: TsvRow = result.unwrap();
        let rhs = line.rhs.chars().next().unwrap(); // Extract "㨲" from "㨲 (U+3A32)"
        let lhs = line.lhs.chars().next().unwrap(); // Extract "㩍" from "㩍 (U+3A4D)"

        if let Some(classification) = match line.relation.as_str() {
            "wrong!" => Some(KVariantClass::Wrong),
            "sem" => Some(KVariantClass::SementicVariant),
            "simp" => Some(KVariantClass::Simplified),
            "old" => Some(KVariantClass::Old),
            "=" => Some(KVariantClass::Equal),
            unexpected_classification => {
                debug_assert!(
                    false,
                    "Unexpected classification {unexpected_classification:?} encountered. Consider handling or ignore explicaitly."
                );
                None
            }
        } {
            debug_assert!(
                !map.contains_key(&lhs),
                "Unexpected one source ideograph mapping to multiple destination ideographs.
                 If this happens in the future when we update kVariants.tsv, we would need to handle it
                 by, for example, deciding priorities for different classification types. "
            );

            map.insert(
                lhs,
                KVariant { source_ideograph: lhs, classification, destination_ideograph: rhs },
            );
        }
    }

    map
});

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_kvariants() {
        assert_eq!(
            KVARIANTS.get(&'澚'),
            Some(&KVariant {
                source_ideograph: '澚',
                classification: KVariantClass::Wrong,
                destination_ideograph: '澳'
            }),
        );
        assert_eq!(
            KVARIANTS.get(&'䀾'),
            Some(&KVariant {
                source_ideograph: '䀾',
                classification: KVariantClass::SementicVariant,
                destination_ideograph: '䁈',
            }),
        );
        assert_eq!(
            KVARIANTS.get(&'亚'),
            Some(&KVariant {
                source_ideograph: '亚',
                classification: KVariantClass::Simplified,
                destination_ideograph: '亞',
            }),
        );
        assert_eq!(
            KVARIANTS.get(&'㮺'),
            Some(&KVariant {
                source_ideograph: '㮺',
                classification: KVariantClass::Old,
                destination_ideograph: '本',
            }),
        );
        assert_eq!(
            KVARIANTS.get(&'刄'),
            Some(&KVariant {
                source_ideograph: '刄',
                classification: KVariantClass::Equal,
                destination_ideograph: '刃',
            }),
        );
        assert_eq!(KVARIANTS.get(&'刃'), None);
    }

    #[test]
    fn test_no_loop() {
        for value in KVARIANTS.values() {
            match KVARIANTS.get(&value.destination_ideograph) {
                // e.g. when value is "栄", reverse lookup would yield nothing.
                None => (),

                // e.g. when value is "椉", reverse lookup would yield "椉 old 乘".
                Some(reverse_lookup_value) => {
                    assert_ne!(
                        value.destination_ideograph,
                        reverse_lookup_value.destination_ideograph
                    );
                }
            }
        }
    }
}