lindera_core/dictionary/
unknown_dictionary.rs

1use std::str::FromStr;
2
3use log::warn;
4use serde::{Deserialize, Serialize};
5
6use crate::dictionary::character_definition::CategoryId;
7use crate::dictionary::word_entry::{WordEntry, WordId};
8use crate::error::LinderaErrorKind;
9use crate::LinderaResult;
10
11#[derive(Serialize, Deserialize, Clone)]
12pub struct UnknownDictionary {
13    pub category_references: Vec<Vec<u32>>,
14    pub costs: Vec<WordEntry>,
15}
16
17impl UnknownDictionary {
18    pub fn load(unknown_data: &[u8]) -> LinderaResult<UnknownDictionary> {
19        bincode::deserialize(unknown_data)
20            .map_err(|err| LinderaErrorKind::Deserialize.with_error(anyhow::anyhow!(err)))
21    }
22
23    pub fn word_entry(&self, word_id: u32) -> WordEntry {
24        self.costs[word_id as usize]
25    }
26
27    pub fn lookup_word_ids(&self, category_id: CategoryId) -> &[u32] {
28        &self.category_references[category_id.0][..]
29    }
30}
31
32#[derive(Debug)]
33pub struct UnknownDictionaryEntry {
34    pub surface: String,
35    pub left_id: u32,
36    pub right_id: u32,
37    pub word_cost: i32,
38}
39
40fn parse_dictionary_entry(
41    fields: &[&str],
42    expected_fields_len: usize,
43) -> LinderaResult<UnknownDictionaryEntry> {
44    if fields.len() != expected_fields_len {
45        return Err(LinderaErrorKind::Content.with_error(anyhow::anyhow!(
46            "Invalid number of fields. Expect {}, got {}",
47            expected_fields_len,
48            fields.len()
49        )));
50    }
51    let surface = fields[0];
52    let left_id = u32::from_str(fields[1])
53        .map_err(|err| LinderaErrorKind::Parse.with_error(anyhow::anyhow!(err)))?;
54    let right_id = u32::from_str(fields[2])
55        .map_err(|err| LinderaErrorKind::Parse.with_error(anyhow::anyhow!(err)))?;
56    let word_cost = i32::from_str(fields[3])
57        .map_err(|err| LinderaErrorKind::Parse.with_error(anyhow::anyhow!(err)))?;
58
59    Ok(UnknownDictionaryEntry {
60        surface: surface.to_string(),
61        left_id,
62        right_id,
63        word_cost,
64    })
65}
66
67fn get_entry_id_matching_surface(
68    entries: &[UnknownDictionaryEntry],
69    target_surface: &str,
70) -> Vec<u32> {
71    entries
72        .iter()
73        .enumerate()
74        .filter_map(|(entry_id, entry)| {
75            if entry.surface == *target_surface {
76                Some(entry_id as u32)
77            } else {
78                None
79            }
80        })
81        .collect()
82}
83
84fn make_category_references(
85    categories: &[String],
86    entries: &[UnknownDictionaryEntry],
87) -> Vec<Vec<u32>> {
88    categories
89        .iter()
90        .map(|category| get_entry_id_matching_surface(entries, category))
91        .collect()
92}
93
94fn make_costs_array(entries: &[UnknownDictionaryEntry]) -> Vec<WordEntry> {
95    entries
96        .iter()
97        .map(|e| {
98            // Do not perform strict checks on left context id and right context id in unk.def.
99            // Just output a warning.
100            if e.left_id != e.right_id {
101                warn!("left id and right id are not same: {:?}", e);
102            }
103            WordEntry {
104                word_id: WordId(u32::MAX, true),
105                left_id: e.left_id as u16,
106                right_id: e.right_id as u16,
107                word_cost: e.word_cost as i16,
108            }
109        })
110        .collect()
111}
112
113pub fn parse_unk(
114    categories: &[String],
115    file_content: &str,
116    expected_fields_len: usize,
117) -> LinderaResult<UnknownDictionary> {
118    let mut unknown_dict_entries = Vec::new();
119    for line in file_content.lines() {
120        let fields: Vec<&str> = line.split(',').collect::<Vec<&str>>();
121        let entry = parse_dictionary_entry(&fields[..], expected_fields_len)?;
122        unknown_dict_entries.push(entry);
123    }
124
125    let category_references = make_category_references(categories, &unknown_dict_entries[..]);
126    let costs = make_costs_array(&unknown_dict_entries[..]);
127    Ok(UnknownDictionary {
128        category_references,
129        costs,
130    })
131}
132
133#[cfg(test)]
134mod tests {}
lindera_core/dictionary/unknown_dictionary.rs

lindera_core/dictionary/
unknown_dictionary.rs
