Skip to main content

haqumei_jpreprocess/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3mod normalize_open_jtalk;
4pub use normalize_open_jtalk::normalize_text_for_open_jtalk;
5pub use normalize_open_jtalk::normalize_text_for_open_jtalk_with_mapping;
6
7#[doc(hidden)]
8pub const VERSION: &str = env!("CARGO_PKG_VERSION");
9
10mod normalize_text;
11pub use normalize_text::normalize_text_for_naist_jdic;
12
13pub use haqumei_jpreprocess_core::error;
14use haqumei_jpreprocess_core::{token::Tokenizer, *};
15pub use haqumei_jpreprocess_njd::NJD;
16
17pub struct JPreprocess<T: Tokenizer> {
18    tokenizer: T,
19}
20
21impl<T: Tokenizer> JPreprocess<T> {
22    /// Creates JPreprocess from provided tokenizer.
23    pub fn from_tokenizer(tokenizer: T) -> Self {
24        Self { tokenizer }
25    }
26
27    /// テキストを形態素解析して NJD の単語列を返します。
28    pub fn text_to_njd(&self, text: &str) -> JPreprocessResult<NJD> {
29        let normalized_input_text = normalize_text_for_naist_jdic(text);
30        let tokens = self.tokenizer.tokenize(normalized_input_text.as_str())?;
31
32        NJD::from_tokens(tokens)
33    }
34
35    /// テキストを形態素解析し、前処理済みの NJD 特徴量を返します。
36    pub fn run_frontend(&self, text: &str) -> JPreprocessResult<Vec<String>> {
37        let mut njd = Self::text_to_njd(self, text)?;
38        njd.preprocess();
39        Ok(njd.into())
40    }
41
42    /// Generate jpcommon features from NJD features(returned by [`run_frontend`]).
43    ///
44    /// [`run_frontend`]: #method.run_frontend
45    pub fn make_label(&self, njd_features: Vec<String>) -> Vec<haqumei_jlabel::Label> {
46        let njd = NJD::from_strings(njd_features);
47        haqumei_jpreprocess_jpcommon::njdnodes_to_features(&njd.nodes)
48    }
49
50    /// Generate jpcommon features from a text.
51    ///
52    /// This is not guaranteed to be same as calling [`run_frontend`] and [`make_label`].
53    ///
54    /// [`run_frontend`]: #method.run_frontend
55    /// [`make_label`]: #method.make_label
56    pub fn extract_fullcontext(&self, text: &str) -> JPreprocessResult<Vec<haqumei_jlabel::Label>> {
57        let mut njd = Self::text_to_njd(self, text)?;
58        njd.preprocess();
59        Ok(haqumei_jpreprocess_jpcommon::njdnodes_to_features(
60            &njd.nodes,
61        ))
62    }
63}
64
65#[cfg(feature = "tokenizer")]
66mod dictionary;
67#[cfg(feature = "tokenizer")]
68pub use dictionary::*;