lindera_core/dictionary_builder/
ipadic.rs

1use std::fs;
2use std::path::Path;
3
4use csv::StringRecord;
5
6use crate::decompress::Algorithm;
7use crate::dictionary::character_definition::CharacterDefinitions;
8use crate::dictionary::UserDictionary;
9use crate::dictionary_builder::DictionaryBuilder;
10use crate::dictionary_builder::{
11    build_user_dictionary, CharDefBuilderOptions, CostMatrixBuilderOptions, DictBuilderOptions,
12    UnkBuilderOptions, UserDictBuilderOptions,
13};
14use crate::error::LinderaErrorKind;
15use crate::LinderaResult;
16
17const SIMPLE_USERDIC_FIELDS_NUM: usize = 3;
18const SIMPLE_WORD_COST: i16 = -10000;
19const SIMPLE_CONTEXT_ID: u16 = 0;
20const DETAILED_USERDIC_FIELDS_NUM: usize = 13;
21const COMPRESS_ALGORITHM: Algorithm = Algorithm::Deflate;
22const UNK_FIELDS_NUM: usize = 11;
23const ENCODING: &str = "EUC-JP";
24
25pub struct IpadicBuilder {}
26
27impl IpadicBuilder {
28    pub fn new() -> Self {
29        IpadicBuilder {}
30    }
31}
32
33impl Default for IpadicBuilder {
34    fn default() -> Self {
35        Self::new()
36    }
37}
38
39impl DictionaryBuilder for IpadicBuilder {
40    fn build_dictionary(&self, input_dir: &Path, output_dir: &Path) -> LinderaResult<()> {
41        fs::create_dir_all(output_dir)
42            .map_err(|err| LinderaErrorKind::Io.with_error(anyhow::anyhow!(err)))?;
43
44        let chardef = self.build_chardef(input_dir, output_dir)?;
45        self.build_unk(input_dir, &chardef, output_dir)?;
46        self.build_dict(input_dir, output_dir)?;
47        self.build_cost_matrix(input_dir, output_dir)?;
48
49        Ok(())
50    }
51
52    fn build_user_dictionary(&self, input_file: &Path, output_file: &Path) -> LinderaResult<()> {
53        let user_dict = self.build_user_dict(input_file)?;
54        build_user_dictionary(user_dict, output_file)
55    }
56
57    fn build_chardef(
58        &self,
59        input_dir: &Path,
60        output_dir: &Path,
61    ) -> LinderaResult<CharacterDefinitions> {
62        CharDefBuilderOptions::default()
63            .encoding(ENCODING)
64            .compress_algorithm(COMPRESS_ALGORITHM)
65            .builder()
66            .unwrap()
67            .build(input_dir, output_dir)
68    }
69
70    fn build_unk(
71        &self,
72        input_dir: &Path,
73        chardef: &CharacterDefinitions,
74        output_dir: &Path,
75    ) -> LinderaResult<()> {
76        UnkBuilderOptions::default()
77            .encoding(ENCODING)
78            .compress_algorithm(COMPRESS_ALGORITHM)
79            .unk_fields_num(UNK_FIELDS_NUM)
80            .builder()
81            .unwrap()
82            .build(input_dir, chardef, output_dir)
83    }
84
85    fn build_dict(&self, input_dir: &Path, output_dir: &Path) -> LinderaResult<()> {
86        DictBuilderOptions::default()
87            .flexible_csv(false)
88            .encoding(ENCODING)
89            .compress_algorithm(COMPRESS_ALGORITHM)
90            .normalize_details(true)
91            .builder()
92            .unwrap()
93            .build(input_dir, output_dir)
94    }
95
96    fn build_cost_matrix(&self, input_dir: &Path, output_dir: &Path) -> LinderaResult<()> {
97        CostMatrixBuilderOptions::default()
98            .encoding(ENCODING)
99            .compress_algorithm(COMPRESS_ALGORITHM)
100            .builder()
101            .unwrap()
102            .build(input_dir, output_dir)
103    }
104
105    fn build_user_dict(&self, input_file: &Path) -> LinderaResult<UserDictionary> {
106        UserDictBuilderOptions::default()
107            .simple_userdic_fields_num(SIMPLE_USERDIC_FIELDS_NUM)
108            .detailed_userdic_fields_num(DETAILED_USERDIC_FIELDS_NUM)
109            .simple_word_cost(SIMPLE_WORD_COST)
110            .simple_context_id(SIMPLE_CONTEXT_ID)
111            .flexible_csv(true)
112            .simple_userdic_details_handler(Some(Box::new(|row: &StringRecord| {
113                Ok(vec![
114                    row[1].to_string(), // POS
115                    "*".to_string(),    // POS subcategory 1
116                    "*".to_string(),    // POS subcategory 2
117                    "*".to_string(),    // POS subcategory 3
118                    "*".to_string(),    // Conjugation type
119                    "*".to_string(),    // Conjugation form
120                    row[0].to_string(), // Base form
121                    row[2].to_string(), // Reading
122                    "*".to_string(),    // Pronunciation
123                ])
124            })))
125            .builder()
126            .unwrap()
127            .build(input_file)
128    }
129}
lindera_core/dictionary_builder/ipadic.rs

lindera_core/dictionary_builder/
ipadic.rs
