haqumei_jpreprocess/
lib.rs1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3mod normalize_open_jtalk;
4pub use normalize_open_jtalk::normalize_text_for_open_jtalk;
5pub use normalize_open_jtalk::normalize_text_for_open_jtalk_with_mapping;
6
7#[doc(hidden)]
8pub const VERSION: &str = env!("CARGO_PKG_VERSION");
9
10mod normalize_text;
11pub use normalize_text::normalize_text_for_naist_jdic;
12
13pub use haqumei_jpreprocess_core::error;
14use haqumei_jpreprocess_core::{token::Tokenizer, *};
15pub use haqumei_jpreprocess_njd::NJD;
16
17pub struct JPreprocess<T: Tokenizer> {
18 tokenizer: T,
19}
20
21impl<T: Tokenizer> JPreprocess<T> {
22 pub fn from_tokenizer(tokenizer: T) -> Self {
24 Self { tokenizer }
25 }
26
27 pub fn text_to_njd(&self, text: &str) -> JPreprocessResult<NJD> {
29 let normalized_input_text = normalize_text_for_naist_jdic(text);
30 let tokens = self.tokenizer.tokenize(normalized_input_text.as_str())?;
31
32 NJD::from_tokens(tokens)
33 }
34
35 pub fn run_frontend(&self, text: &str) -> JPreprocessResult<Vec<String>> {
37 let mut njd = Self::text_to_njd(self, text)?;
38 njd.preprocess();
39 Ok(njd.into())
40 }
41
42 pub fn make_label(&self, njd_features: Vec<String>) -> Vec<haqumei_jlabel::Label> {
46 let njd = NJD::from_strings(njd_features);
47 haqumei_jpreprocess_jpcommon::njdnodes_to_features(&njd.nodes)
48 }
49
50 pub fn extract_fullcontext(&self, text: &str) -> JPreprocessResult<Vec<haqumei_jlabel::Label>> {
57 let mut njd = Self::text_to_njd(self, text)?;
58 njd.preprocess();
59 Ok(haqumei_jpreprocess_jpcommon::njdnodes_to_features(
60 &njd.nodes,
61 ))
62 }
63}
64
65#[cfg(feature = "tokenizer")]
66mod dictionary;
67#[cfg(feature = "tokenizer")]
68pub use dictionary::*;