jpreprocess_dictionary/tokenizer/
default.rs

1use jpreprocess_core::{
2    token::{Token, Tokenizer},
3    word_entry::WordEntry,
4    JPreprocessResult,
5};
6use lindera_dictionary::dictionary::prefix_dictionary::PrefixDictionary;
7
8use super::{
9    identify_dictionary::DictionaryIdent,
10    jpreprocess::{JPreprocessToken, JPreprocessTokenizer},
11};
12
13pub struct DefaultTokenizer {
14    lindera_tokenizer: lindera::tokenizer::Tokenizer,
15    system: TokenizerType,
16    user: Option<TokenizerType>,
17}
18
19enum TokenizerType {
20    JPreprocessTokenizer,
21    LinderaTokenizer,
22}
23
24impl DefaultTokenizer {
25    pub fn new(tokenizer: lindera::tokenizer::Tokenizer) -> Self {
26        fn identify_tokenizer(prefix_dictionary: &PrefixDictionary) -> TokenizerType {
27            let ident = DictionaryIdent::from_idx_data(
28                &prefix_dictionary.words_idx_data,
29                &prefix_dictionary.words_data,
30            );
31            match ident {
32                DictionaryIdent::JPreprocess => TokenizerType::JPreprocessTokenizer,
33                DictionaryIdent::Lindera => TokenizerType::LinderaTokenizer,
34            }
35        }
36
37        Self {
38            system: identify_tokenizer(&tokenizer.segmenter.dictionary.prefix_dictionary),
39            user: tokenizer
40                .segmenter
41                .user_dictionary
42                .as_ref()
43                .map(|d| identify_tokenizer(&d.dict)),
44            lindera_tokenizer: tokenizer,
45        }
46    }
47}
48
49impl Tokenizer for DefaultTokenizer {
50    fn tokenize<'a>(&'a self, text: &'a str) -> JPreprocessResult<Vec<impl 'a + Token>> {
51        let tokens = self.lindera_tokenizer.tokenize(text)?;
52
53        tokens
54            .into_iter()
55            .map(|token| {
56                if token.word_id.is_unknown() {
57                    Ok(DefaultToken::from_token(token))
58                } else if token.word_id.is_system() {
59                    match self.system {
60                        TokenizerType::JPreprocessTokenizer => {
61                            Ok(DefaultToken::from_token(JPreprocessToken::new(
62                                token.surface,
63                                JPreprocessTokenizer::get_word_from_prefixdict(
64                                    &token.dictionary.prefix_dictionary,
65                                    token.word_id,
66                                )?,
67                            )))
68                        }
69                        TokenizerType::LinderaTokenizer => Ok(DefaultToken::from_token(token)),
70                    }
71                } else {
72                    match self.user {
73                        Some(TokenizerType::JPreprocessTokenizer) => {
74                            Ok(DefaultToken::from_token(JPreprocessToken::new(
75                                token.surface,
76                                JPreprocessTokenizer::get_word_from_prefixdict(
77                                    &token.user_dictionary.as_ref().unwrap().dict,
78                                    token.word_id,
79                                )?,
80                            )))
81                        }
82                        Some(TokenizerType::LinderaTokenizer) => {
83                            Ok(DefaultToken::from_token(token))
84                        }
85                        None => Ok(DefaultToken::from_token(token)),
86                    }
87                }
88            })
89            .collect()
90    }
91}
92
93struct DefaultToken<'a> {
94    inner: Box<dyn 'a + Token>,
95}
96
97impl<'a> DefaultToken<'a> {
98    fn from_token(inner: impl 'a + Token) -> Self {
99        DefaultToken {
100            inner: Box::new(inner),
101        }
102    }
103}
104
105impl Token for DefaultToken<'_> {
106    fn fetch(&mut self) -> JPreprocessResult<(&str, WordEntry)> {
107        self.inner.fetch()
108    }
109}
jpreprocess_dictionary/tokenizer/default.rs

jpreprocess_dictionary/tokenizer/
default.rs
