lindera_core/
dictionary.rs

1pub mod character_definition;
2pub mod connection;
3pub mod prefix_dict;
4pub mod unknown_dictionary;
5pub mod viterbi;
6pub mod word_entry;
7
8use std::borrow::Cow;
9use std::str;
10
11use byteorder::{ByteOrder, LittleEndian};
12use once_cell::sync::Lazy;
13use serde::{Deserialize, Serialize};
14
15use crate::dictionary::character_definition::CharacterDefinitions;
16use crate::dictionary::connection::ConnectionCostMatrix;
17use crate::dictionary::prefix_dict::PrefixDict;
18use crate::dictionary::unknown_dictionary::UnknownDictionary;
19use crate::error::LinderaErrorKind;
20use crate::LinderaResult;
21
22pub static UNK: Lazy<Vec<&str>> = Lazy::new(|| vec!["UNK"]);
23
24#[derive(Clone, Serialize, Deserialize)]
25pub struct Dictionary {
26    pub dict: PrefixDict<Vec<u8>>,
27    pub cost_matrix: ConnectionCostMatrix,
28    pub char_definitions: CharacterDefinitions,
29    pub unknown_dictionary: UnknownDictionary,
30    pub words_idx_data: Cow<'static, [u8]>,
31    pub words_data: Cow<'static, [u8]>,
32}
33
34impl Dictionary {
35    pub fn word_details(&self, word_id: usize) -> Vec<&str> {
36        if 4 * word_id >= self.words_idx_data.len() {
37            return vec![];
38        }
39
40        let idx: usize =
41            match LittleEndian::read_u32(&self.words_idx_data[4 * word_id..][..4]).try_into() {
42                Ok(value) => value,
43                Err(_) => return UNK.to_vec(), // return empty vector if conversion fails
44            };
45        let data = &self.words_data[idx..];
46        let joined_details_len: usize = match LittleEndian::read_u32(data).try_into() {
47            Ok(value) => value,
48            Err(_) => return UNK.to_vec(), // return empty vector if conversion fails
49        };
50        let joined_details_bytes: &[u8] = &self.words_data[idx + 4..idx + 4 + joined_details_len];
51
52        let mut details = Vec::new();
53        for bytes in joined_details_bytes.split(|&b| b == 0) {
54            let detail = match str::from_utf8(bytes) {
55                Ok(s) => s,
56                Err(_) => return UNK.to_vec(), // return empty vector if conversion fails
57            };
58            details.push(detail);
59        }
60        details
61    }
62}
63
64#[derive(Clone, Serialize, Deserialize)]
65pub struct UserDictionary {
66    pub dict: PrefixDict<Vec<u8>>,
67    pub words_idx_data: Vec<u8>,
68    pub words_data: Vec<u8>,
69}
70
71impl UserDictionary {
72    pub fn load(user_dict_data: &[u8]) -> LinderaResult<UserDictionary> {
73        bincode::deserialize(user_dict_data)
74            .map_err(|err| LinderaErrorKind::Deserialize.with_error(anyhow::anyhow!(err)))
75    }
76
77    pub fn word_details(&self, word_id: usize) -> Vec<&str> {
78        if 4 * word_id >= self.words_idx_data.len() {
79            return UNK.to_vec(); // return empty vector if conversion fails
80        }
81        let idx = LittleEndian::read_u32(&self.words_idx_data[4 * word_id..][..4]);
82        let data = &self.words_data[idx as usize..];
83        match bincode::deserialize(data) {
84            Ok(details) => details,
85            Err(_) => UNK.to_vec(), // return empty vector if conversion fails
86        }
87    }
88}
lindera_core/dictionary.rs

lindera_core/
dictionary.rs
