jpreprocess-dictionary 0.15.0

Japanese text preprocessor for Text-to-Speech application (OpenJTalk rewrite in rust language).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

use std::borrow::Cow;

use jpreprocess_core::{
    error::DictionaryError,
    token::{Token, Tokenizer},
    word_entry::WordEntry,
    JPreprocessResult,
};

use crate::{
    dictionary::word_encoding::JPreprocessDictionaryWordEncoding, word_data::get_word_data,
};

pub struct JPreprocessTokenizer {
    tokenizer: lindera::tokenizer::Tokenizer,
}

impl JPreprocessTokenizer {
    pub fn new(tokenizer: lindera::tokenizer::Tokenizer) -> Self {
        Self { tokenizer }
    }

    fn get_word(
        &self,
        word_id: lindera_dictionary::viterbi::WordId,
    ) -> Result<WordEntry, DictionaryError> {
        if word_id.is_unknown() {
            Ok(WordEntry::default())
        } else if word_id.is_system() {
            Self::get_word_from_prefixdict(
                &self.tokenizer.segmenter.dictionary.prefix_dictionary,
                word_id,
            )
        } else {
            let user = &self.tokenizer.segmenter.user_dictionary;
            user.as_ref()
                .map_or(Err(DictionaryError::UserDictionaryNotProvided), |user| {
                    Self::get_word_from_prefixdict(&user.dict, word_id)
                })
        }
    }

    /// PANIC: It must be ensured that the prefix_dict is the correct dictionary for the word_id.
    pub(super) fn get_word_from_prefixdict(
        prefix_dict: &lindera_dictionary::dictionary::prefix_dictionary::PrefixDictionary,
        word_id: lindera_dictionary::viterbi::WordId,
    ) -> Result<WordEntry, DictionaryError> {
        if word_id.is_unknown() {
            Ok(WordEntry::default())
        } else {
            let data = get_word_data(
                &prefix_dict.words_idx_data,
                &prefix_dict.words_data,
                Some(word_id.id as usize),
            )
            .ok_or(DictionaryError::IdNotFound(word_id.id))?;
            Ok(JPreprocessDictionaryWordEncoding::deserialize(data)?)
        }
    }
}

impl Tokenizer for JPreprocessTokenizer {
    fn tokenize<'a>(&'a self, text: &'a str) -> JPreprocessResult<Vec<impl 'a + Token>> {
        let words = self.tokenizer.tokenize(text).unwrap();
        words
            .into_iter()
            .map(|token| {
                Ok(JPreprocessToken::new(
                    token.surface,
                    self.get_word(token.word_id)?,
                ))
            })
            .collect::<Result<_, _>>()
    }
}

pub struct JPreprocessToken<'a> {
    text: Cow<'a, str>,
    entry: WordEntry,
}

impl<'a> JPreprocessToken<'a> {
    pub(crate) fn new(text: Cow<'a, str>, entry: WordEntry) -> Self {
        Self { text, entry }
    }
}

impl Token for JPreprocessToken<'_> {
    fn fetch(&mut self) -> Result<(&str, WordEntry), jpreprocess_core::JPreprocessError> {
        Ok((&self.text, self.entry.clone()))
    }
}