jpreprocess-dictionary 0.15.0

Japanese text preprocessor for Text-to-Speech application (OpenJTalk rewrite in rust language).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

use jpreprocess_core::{
    token::{Token, Tokenizer},
    word_entry::WordEntry,
    JPreprocessResult,
};
use lindera_dictionary::dictionary::prefix_dictionary::PrefixDictionary;

use super::{
    identify_dictionary::DictionaryIdent,
    jpreprocess::{JPreprocessToken, JPreprocessTokenizer},
};

pub struct DefaultTokenizer {
    lindera_tokenizer: lindera::tokenizer::Tokenizer,
    system: TokenizerType,
    user: Option<TokenizerType>,
}

enum TokenizerType {
    JPreprocessTokenizer,
    LinderaTokenizer,
}

impl DefaultTokenizer {
    pub fn new(tokenizer: lindera::tokenizer::Tokenizer) -> Self {
        fn identify_tokenizer(prefix_dictionary: &PrefixDictionary) -> TokenizerType {
            let ident = DictionaryIdent::from_idx_data(
                &prefix_dictionary.words_idx_data,
                &prefix_dictionary.words_data,
            );
            match ident {
                DictionaryIdent::JPreprocess => TokenizerType::JPreprocessTokenizer,
                DictionaryIdent::Lindera => TokenizerType::LinderaTokenizer,
            }
        }

        Self {
            system: identify_tokenizer(&tokenizer.segmenter.dictionary.prefix_dictionary),
            user: tokenizer
                .segmenter
                .user_dictionary
                .as_ref()
                .map(|d| identify_tokenizer(&d.dict)),
            lindera_tokenizer: tokenizer,
        }
    }
}

impl Tokenizer for DefaultTokenizer {
    fn tokenize<'a>(&'a self, text: &'a str) -> JPreprocessResult<Vec<impl 'a + Token>> {
        let tokens = self.lindera_tokenizer.tokenize(text)?;

        tokens
            .into_iter()
            .map(|token| {
                if token.word_id.is_unknown() {
                    Ok(DefaultToken::from_token(token))
                } else if token.word_id.is_system() {
                    match self.system {
                        TokenizerType::JPreprocessTokenizer => {
                            Ok(DefaultToken::from_token(JPreprocessToken::new(
                                token.surface,
                                JPreprocessTokenizer::get_word_from_prefixdict(
                                    &token.dictionary.prefix_dictionary,
                                    token.word_id,
                                )?,
                            )))
                        }
                        TokenizerType::LinderaTokenizer => Ok(DefaultToken::from_token(token)),
                    }
                } else {
                    match self.user {
                        Some(TokenizerType::JPreprocessTokenizer) => {
                            Ok(DefaultToken::from_token(JPreprocessToken::new(
                                token.surface,
                                JPreprocessTokenizer::get_word_from_prefixdict(
                                    &token.user_dictionary.as_ref().unwrap().dict,
                                    token.word_id,
                                )?,
                            )))
                        }
                        Some(TokenizerType::LinderaTokenizer) => {
                            Ok(DefaultToken::from_token(token))
                        }
                        None => Ok(DefaultToken::from_token(token)),
                    }
                }
            })
            .collect()
    }
}

struct DefaultToken<'a> {
    inner: Box<dyn 'a + Token>,
}

impl<'a> DefaultToken<'a> {
    fn from_token(inner: impl 'a + Token) -> Self {
        DefaultToken {
            inner: Box::new(inner),
        }
    }
}

impl Token for DefaultToken<'_> {
    fn fetch(&mut self) -> JPreprocessResult<(&str, WordEntry)> {
        self.inner.fetch()
    }
}