lindera_core/dictionary/
character_definition.rs

1use std::collections::{BTreeSet, HashMap};
2
3use byteorder::{ByteOrder, LittleEndian};
4use encoding_rs::UTF_16LE;
5use serde::{Deserialize, Serialize};
6
7use crate::error::LinderaErrorKind;
8use crate::LinderaResult;
9
10const DEFAULT_CATEGORY_NAME: &str = "DEFAULT";
11
12fn ucs2_to_unicode(ucs2_codepoint: u16) -> LinderaResult<u32> {
13    let mut buf = [0u8; 2];
14    LittleEndian::write_u16(&mut buf[..], ucs2_codepoint);
15
16    let s = UTF_16LE.decode(&buf[..]).0.into_owned();
17    let chrs: Vec<char> = s.chars().collect();
18
19    match chrs.len() {
20        1 => Ok(chrs[0] as u32),
21        _ => Err(LinderaErrorKind::Parse.with_error(anyhow::anyhow!("unusual char length"))),
22    }
23}
24
25fn parse_hex_codepoint(s: &str) -> LinderaResult<u32> {
26    let removed_0x = s.trim_start_matches("0x");
27    let ucs2_codepoint = u16::from_str_radix(removed_0x, 16)
28        .map_err(|err| LinderaErrorKind::Parse.with_error(anyhow::anyhow!(err)))?;
29
30    ucs2_to_unicode(ucs2_codepoint)
31}
32
33#[derive(Serialize, Deserialize, Debug, Copy, Clone)]
34pub struct CategoryData {
35    pub invoke: bool,
36    pub group: bool,
37    pub length: u32,
38}
39
40#[derive(Serialize, Deserialize, Clone, Debug, Hash, Copy, PartialOrd, Ord, Eq, PartialEq)]
41pub struct CategoryId(pub usize);
42
43#[derive(Clone, Serialize, Deserialize)]
44pub struct CharacterDefinitions {
45    pub category_definitions: Vec<CategoryData>,
46    pub category_names: Vec<String>,
47    pub mapping: LookupTable<CategoryId>,
48}
49
50#[derive(Serialize, Deserialize, Clone)]
51pub struct LookupTable<T: Copy + Clone> {
52    boundaries: Vec<u32>,
53    values: Vec<Vec<T>>,
54}
55
56impl<T: Copy + Clone> LookupTable<T> {
57    pub fn from_fn(mut boundaries: Vec<u32>, funct: &dyn Fn(u32, &mut Vec<T>)) -> LookupTable<T> {
58        if !boundaries.contains(&0) {
59            boundaries.push(0);
60        }
61        boundaries.sort_unstable();
62        let mut values = Vec::new();
63        for &boundary in &boundaries {
64            let mut output = Vec::default();
65            funct(boundary, &mut output);
66            values.push(output);
67        }
68        LookupTable { boundaries, values }
69    }
70
71    pub fn eval(&self, target: u32) -> &[T] {
72        let idx = self
73            .boundaries
74            .binary_search(&target)
75            .unwrap_or_else(|val| val - 1);
76        &self.values[idx][..]
77    }
78}
79
80impl CharacterDefinitions {
81    pub fn categories(&self) -> &[String] {
82        &self.category_names[..]
83    }
84
85    pub fn load(char_def_data: &[u8]) -> LinderaResult<CharacterDefinitions> {
86        bincode::deserialize(char_def_data)
87            .map_err(|err| LinderaErrorKind::Deserialize.with_error(anyhow::anyhow!(err)))
88    }
89
90    pub fn lookup_definition(&self, category_id: CategoryId) -> &CategoryData {
91        &self.category_definitions[category_id.0]
92    }
93
94    pub fn category_name(&self, category_id: CategoryId) -> &str {
95        &self.category_names[category_id.0]
96    }
97
98    pub fn lookup_categories(&self, c: char) -> &[CategoryId] {
99        self.mapping.eval(c as u32)
100    }
101}
102
103#[derive(Default)]
104pub struct CharacterDefinitionsBuilder {
105    category_definition: Vec<CategoryData>,
106    category_index: HashMap<String, CategoryId>,
107    char_ranges: Vec<(u32, u32, Vec<CategoryId>)>,
108}
109
110impl CharacterDefinitionsBuilder {
111    pub fn category_id(&mut self, category_name: &str) -> CategoryId {
112        let num_categories = self.category_index.len();
113        *self
114            .category_index
115            .entry(category_name.to_string())
116            .or_insert(CategoryId(num_categories))
117    }
118
119    fn lookup_categories(&self, c: u32, categories_buffer: &mut Vec<CategoryId>) {
120        categories_buffer.clear();
121        for (start, stop, category_ids) in &self.char_ranges {
122            if *start <= c && *stop >= c {
123                for cat in category_ids {
124                    if !categories_buffer.contains(cat) {
125                        categories_buffer.push(*cat);
126                    }
127                }
128            }
129        }
130        if categories_buffer.is_empty() {
131            if let Some(default_category) = self.category_index.get(DEFAULT_CATEGORY_NAME) {
132                categories_buffer.push(*default_category);
133            }
134        }
135    }
136
137    fn build_lookup_table(&self) -> LookupTable<CategoryId> {
138        let boundaries_set: BTreeSet<u32> = self
139            .char_ranges
140            .iter()
141            .flat_map(|(low, high, _)| vec![*low, *high + 1u32])
142            .collect();
143        let boundaries: Vec<u32> = boundaries_set.into_iter().collect();
144        LookupTable::from_fn(boundaries, &|c, buff| self.lookup_categories(c, buff))
145    }
146
147    pub fn parse(&mut self, content: &str) -> LinderaResult<()> {
148        for line in content.lines() {
149            let line_str = line
150                .split('#')
151                .next()
152                .ok_or_else(|| {
153                    LinderaErrorKind::Parse.with_error(anyhow::anyhow!("failed to parse line"))
154                })?
155                .trim();
156            if line_str.is_empty() {
157                continue;
158            }
159            if line_str.starts_with("0x") {
160                self.parse_range(line_str)?;
161            } else {
162                self.parse_category(line_str)?;
163            }
164        }
165        Ok(())
166    }
167
168    fn parse_range(&mut self, line: &str) -> LinderaResult<()> {
169        let fields: Vec<&str> = line.split_whitespace().collect();
170        let range_bounds: Vec<&str> = fields[0].split("..").collect();
171        let lower_bound: u32;
172        let higher_bound: u32;
173        match range_bounds.len() {
174            1 => {
175                lower_bound = parse_hex_codepoint(range_bounds[0])?;
176                higher_bound = lower_bound;
177            }
178            2 => {
179                lower_bound = parse_hex_codepoint(range_bounds[0])?;
180                // the right bound is included in the file.
181                higher_bound = parse_hex_codepoint(range_bounds[1])?;
182            }
183            _ => {
184                return Err(
185                    LinderaErrorKind::Content.with_error(anyhow::anyhow!("Invalid line: {}", line))
186                );
187            }
188        }
189        let category_ids: Vec<CategoryId> = fields[1..]
190            .iter()
191            .map(|category| self.category_id(category))
192            .collect();
193
194        self.char_ranges
195            .push((lower_bound, higher_bound, category_ids));
196
197        Ok(())
198    }
199
200    fn parse_category(&mut self, line: &str) -> LinderaResult<()> {
201        let fields = line.split_ascii_whitespace().collect::<Vec<&str>>();
202        if fields.len() != 4 {
203            return Err(LinderaErrorKind::Content.with_error(anyhow::anyhow!(
204                "Expected 4 fields. Got {} in {}",
205                fields.len(),
206                line
207            )));
208        }
209        let invoke = fields[1]
210            .parse::<u32>()
211            .map_err(|err| LinderaErrorKind::Parse.with_error(anyhow::anyhow!(err)))?
212            == 1;
213        let group = fields[2]
214            .parse::<u32>()
215            .map_err(|err| LinderaErrorKind::Parse.with_error(anyhow::anyhow!(err)))?
216            == 1;
217        let length = fields[3]
218            .parse::<u32>()
219            .map_err(|err| LinderaErrorKind::Parse.with_error(anyhow::anyhow!(err)))?;
220        let category_data = CategoryData {
221            invoke,
222            group,
223            length,
224        };
225        // force a category_id allocation
226        self.category_id(fields[0]);
227        self.category_definition.push(category_data);
228
229        Ok(())
230    }
231
232    pub fn build(self) -> CharacterDefinitions {
233        let mut category_names: Vec<String> = (0..self.category_index.len())
234            .map(|_| String::new())
235            .collect();
236        for (category_name, category_id) in &self.category_index {
237            category_names[category_id.0] = category_name.clone();
238        }
239        let mapping = self.build_lookup_table();
240        CharacterDefinitions {
241            category_definitions: self.category_definition,
242            category_names,
243            mapping,
244        }
245    }
246}
247
248#[cfg(test)]
249mod tests {
250    use crate::dictionary::character_definition::LookupTable;
251
252    #[test]
253    fn test_lookup_table() {
254        let funct = |c: u32, output: &mut Vec<u32>| {
255            if c >= 10u32 {
256                output.push(1u32);
257            } else {
258                output.push(0u32);
259            }
260        };
261        let lookup_table = LookupTable::from_fn(vec![0u32, 10u32], &funct);
262        for i in 0..100 {
263            let mut v = Vec::default();
264            funct(i, &mut v);
265            assert_eq!(lookup_table.eval(i), &v[..]);
266        }
267    }
268}
lindera_core/dictionary/character_definition.rs

lindera_core/dictionary/
character_definition.rs
