lindera_dictionary/dictionary/
character_definition.rs

1use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
2use serde::{Deserialize, Serialize};
3
4use crate::LinderaResult;
5use crate::error::LinderaErrorKind;
6
7#[derive(Serialize, Deserialize, Debug, Copy, Clone, Archive, RkyvSerialize, RkyvDeserialize)]
8
9pub struct CategoryData {
10    pub invoke: bool,
11    pub group: bool,
12    pub length: u32,
13}
14
15#[derive(
16    Serialize,
17    Deserialize,
18    Clone,
19    Debug,
20    Hash,
21    Copy,
22    PartialOrd,
23    Ord,
24    Eq,
25    PartialEq,
26    Archive,
27    RkyvSerialize,
28    RkyvDeserialize,
29)]
30
31pub struct CategoryId(pub usize);
32
33#[derive(Serialize, Deserialize, Clone, Archive, RkyvSerialize, RkyvDeserialize)]
34
35pub struct LookupTable<T: Copy + Clone> {
36    boundaries: Vec<u32>,
37    values: Vec<Vec<T>>,
38}
39
40impl<T: Copy + Clone> LookupTable<T> {
41    pub fn from_fn(mut boundaries: Vec<u32>, funct: &dyn Fn(u32, &mut Vec<T>)) -> LookupTable<T> {
42        if !boundaries.contains(&0) {
43            boundaries.push(0);
44        }
45        boundaries.sort_unstable();
46        let mut values = Vec::new();
47        for &boundary in &boundaries {
48            let mut output = Vec::default();
49            funct(boundary, &mut output);
50            values.push(output);
51        }
52        LookupTable { boundaries, values }
53    }
54
55    pub fn eval(&self, target: u32) -> &[T] {
56        let idx = self
57            .boundaries
58            .binary_search(&target)
59            .unwrap_or_else(|val| val - 1);
60        &self.values[idx][..]
61    }
62}
63
64#[derive(Clone, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)]
65
66pub struct CharacterDefinition {
67    pub category_definitions: Vec<CategoryData>,
68    pub category_names: Vec<String>,
69    pub mapping: LookupTable<CategoryId>,
70}
71
72impl CharacterDefinition {
73    pub fn categories(&self) -> &[String] {
74        &self.category_names[..]
75    }
76
77    pub fn load(char_def_data: &[u8]) -> LinderaResult<CharacterDefinition> {
78        let mut aligned = rkyv::util::AlignedVec::<16>::new();
79        aligned.extend_from_slice(char_def_data);
80        rkyv::from_bytes::<CharacterDefinition, rkyv::rancor::Error>(&aligned).map_err(|err| {
81            LinderaErrorKind::Deserialize.with_error(anyhow::anyhow!(err.to_string()))
82        })
83    }
84
85    pub fn lookup_definition(&self, category_id: CategoryId) -> &CategoryData {
86        &self.category_definitions[category_id.0]
87    }
88
89    pub fn category_name(&self, category_id: CategoryId) -> &str {
90        &self.category_names[category_id.0]
91    }
92
93    pub fn category_id_by_name(&self, name: &str) -> Option<CategoryId> {
94        self.category_names
95            .iter()
96            .position(|n| n == name)
97            .map(CategoryId)
98    }
99
100    pub fn lookup_categories(&self, c: char) -> &[CategoryId] {
101        self.mapping.eval(c as u32)
102    }
103}
104
105#[cfg(test)]
106mod tests {
107    use crate::dictionary::character_definition::LookupTable;
108
109    #[test]
110    fn test_lookup_table() {
111        let funct = |c: u32, output: &mut Vec<u32>| {
112            if c >= 10u32 {
113                output.push(1u32);
114            } else {
115                output.push(0u32);
116            }
117        };
118        let lookup_table = LookupTable::from_fn(vec![0u32, 10u32], &funct);
119        for i in 0..100 {
120            let mut v = Vec::default();
121            funct(i, &mut v);
122            assert_eq!(lookup_table.eval(i), &v[..]);
123        }
124    }
125}
lindera_dictionary/dictionary/character_definition.rs

lindera_dictionary/dictionary/
character_definition.rs
