jpreprocess-core 0.15.0

Japanese text preprocessor for Text-to-Speech application (OpenJTalk rewrite in rust language).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

use crate::{word_entry::WordEntry, JPreprocessResult};

pub trait Tokenizer {
    fn tokenize<'a>(&'a self, text: &'a str) -> JPreprocessResult<Vec<impl 'a + Token>>;
}

pub trait Token {
    fn fetch(&mut self) -> JPreprocessResult<(&str, WordEntry)>;
}

impl Token for (String, WordEntry) {
    fn fetch(&mut self) -> JPreprocessResult<(&str, WordEntry)> {
        let (string, entry) = self;
        Ok((string.as_str(), entry.to_owned()))
    }
}

#[cfg(feature = "lindera")]
impl Tokenizer for lindera::tokenizer::Tokenizer {
    fn tokenize<'a>(&'a self, text: &'a str) -> JPreprocessResult<Vec<impl 'a + Token>> {
        Ok(self.tokenize(text)?)
    }
}

#[cfg(feature = "lindera")]
impl Token for lindera::token::Token<'_> {
    fn fetch(&mut self) -> JPreprocessResult<(&str, WordEntry)> {
        use lindera_dictionary::dictionary::UNK;

        let mut details = self.details();
        let entry = if details == *UNK {
            WordEntry::default()
        } else {
            details.resize(12, "");
            WordEntry::load(&details)?
        };

        Ok((&self.surface, entry))
    }
}

/// Vibrato support is experimental and may be removed or changed in the future. Use with caution.
#[cfg(feature = "vibrato")]
impl Tokenizer for vibrato::tokenizer::Tokenizer {
    fn tokenize<'a>(&'a self, text: &'a str) -> JPreprocessResult<Vec<impl 'a + Token>> {
        let mut worker = self.new_worker();
        worker.reset_sentence(text);
        worker.tokenize();

        worker
            .token_iter()
            .map(|token| {
                let features = token.feature();
                let mut details = features.split(',').collect::<Vec<_>>();
                details.resize(12, "*");
                let entry = WordEntry::load(&details)?;

                Ok((token.surface().to_string(), entry))
            })
            .collect()
    }
}