haqumei_jpreprocess_jpcommon/
lib.rs1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3mod feature;
4mod label;
5mod word_attr;
6
7pub use feature::*;
8use haqumei_jlabel::Label;
9pub use label::*;
10
11use haqumei_jpreprocess_njd::NJDNode;
12
13pub fn njdnodes_to_features(njd_nodes: &[NJDNode]) -> Vec<Label> {
15 let utterance = Utterance::from(njd_nodes);
16 utterance_to_features(&utterance)
17}
18
19#[derive(Clone, Debug, PartialEq, Eq)]
21pub struct LabelWithSource {
22 pub label: Label,
23 pub source_index: Option<usize>,
26}
27
28pub fn njdnodes_to_features_with_sources(njd_nodes: &[NJDNode]) -> Vec<LabelWithSource> {
30 let utterance = Utterance::from(njd_nodes);
31 utterance_to_features_with_sources(&utterance)
32}
33
34#[derive(Clone, Debug, PartialEq, Eq)]
36pub struct PhonemeWithSource {
37 pub phoneme: String,
38 pub source_index: Option<usize>,
41}
42
43pub fn njdnodes_to_phonemes_with_sources(njd_nodes: &[NJDNode]) -> Vec<PhonemeWithSource> {
46 utterance_to_phonemes_with_sources(&Utterance::from(njd_nodes))
47}
48
49#[cfg(test)]
50mod tests {
51 use haqumei_jpreprocess_core::{pronunciation::Pronunciation, word_details::WordDetails};
52
53 use super::*;
54
55 fn node(pron: &str, accent: usize, chain: bool) -> NJDNode {
56 NJDNode::from_details(
57 pron.to_owned(),
58 WordDetails {
59 pron: Pronunciation::parse(pron, accent).unwrap(),
60 chain_flag: Some(chain),
61 ..WordDetails::default()
62 },
63 )
64 }
65
66 #[test]
67 fn sources_follow_long_vowels_and_pause_boundaries() {
68 let nodes = vec![
69 node("キ’", 1, false),
70 node("ー", 0, false),
71 node("、", 0, false),
72 node("ー", 0, false),
73 node("ア", 0, false),
74 node("?", 0, false),
75 node("!", 0, false),
76 node("イ", 1, false),
77 ];
78 let features = njdnodes_to_features_with_sources(&nodes);
79 let phonemes = njdnodes_to_phonemes_with_sources(&nodes);
80 assert_eq!(
81 phonemes,
82 features
83 .iter()
84 .map(|feature| PhonemeWithSource {
85 phoneme: feature.label.phoneme.c.clone().unwrap(),
86 source_index: feature.source_index,
87 })
88 .collect::<Vec<_>>()
89 );
90 let actual: Vec<_> = features
91 .iter()
92 .map(|feature| {
93 (
94 feature.label.phoneme.c.as_deref().unwrap(),
95 feature.source_index,
96 )
97 })
98 .collect();
99 assert_eq!(
100 actual,
101 [
102 ("sil", None),
103 ("k", Some(0)),
104 ("I", Some(0)),
105 ("I", Some(0)),
106 ("pau", None),
107 ("a", Some(4)),
108 ("pau", None),
109 ("i", Some(7)),
110 ("sil", None),
111 ]
112 );
113 let phrase = features[5].label.accent_phrase_curr.as_ref().unwrap();
114 assert!(phrase.is_interrogative);
115 assert!(phrase.is_exclamatory);
116 assert_eq!(phrase.mora_count, 1);
117 let pause = features[6].label.accent_phrase_prev.as_ref().unwrap();
118 assert!(pause.is_interrogative);
119 assert!(pause.is_exclamatory);
120 assert_eq!(features[0].label.utterance.breath_group_count, 3);
121 assert_eq!(features[0].label.utterance.mora_count, 4);
122 }
123
124 #[test]
125 fn leading_long_vowels_keep_their_source_inside_the_previous_phrase() {
126 let nodes = [node("キ", 1, false), node("ーーア", 1, false)];
127 let features = njdnodes_to_features_with_sources(&nodes);
128 let sources: Vec<_> = features
129 .iter()
130 .map(|feature| feature.source_index)
131 .collect();
132 assert_eq!(
133 sources,
134 [None, Some(0), Some(0), Some(1), Some(1), Some(1), None]
135 );
136 assert_eq!(
137 njdnodes_to_phonemes_with_sources(&nodes),
138 features
139 .iter()
140 .map(|f| PhonemeWithSource {
141 phoneme: f.label.phoneme.c.clone().unwrap(),
142 source_index: f.source_index,
143 })
144 .collect::<Vec<_>>()
145 );
146 assert_eq!(
147 features[1]
148 .label
149 .accent_phrase_curr
150 .as_ref()
151 .unwrap()
152 .mora_count,
153 3
154 );
155 assert_eq!(
156 features[5]
157 .label
158 .accent_phrase_curr
159 .as_ref()
160 .unwrap()
161 .mora_count,
162 1
163 );
164 }
165
166 #[test]
167 fn a_long_vowel_can_be_unvoiced_without_changing_the_preceding_mora() {
168 let features = njdnodes_to_features(&[node("ツォー’", 1, false)]);
169 let phonemes: Vec<_> = features
170 .iter()
171 .map(|feature| feature.phoneme.c.as_deref().unwrap())
172 .collect();
173 assert_eq!(phonemes, ["sil", "ts", "o", "O", "sil"]);
174 }
175
176 #[test]
177 fn leading_long_vowels_do_not_take_phonemes_across_a_pause() {
178 let nodes = [
179 node("キ", 1, false),
180 node("、", 0, false),
181 node("ーーア", 1, false),
182 ];
183 let phonemes = njdnodes_to_phonemes_with_sources(&nodes);
184 let actual: Vec<_> = phonemes
185 .iter()
186 .map(|p| (p.phoneme.as_str(), p.source_index))
187 .collect();
188 assert_eq!(
189 actual,
190 [
191 ("sil", None),
192 ("k", Some(0)),
193 ("i", Some(0)),
194 ("pau", None),
195 ("a", Some(2)),
196 ("sil", None)
197 ]
198 );
199 }
200
201 #[test]
202 fn unvoicing_after_a_long_geminate_keeps_the_consonant() {
203 let features = njdnodes_to_features(&[node("ッー’", 1, false)]);
204 let phonemes: Vec<_> = features
205 .iter()
206 .map(|feature| feature.phoneme.c.as_deref().unwrap())
207 .collect();
208 assert_eq!(phonemes, ["sil", "cl", "cl", "sil"]);
209 }
210
211 #[test]
212 fn explicit_chaining_keeps_a_pause_inside_the_accent_phrase() {
213 let nodes = [
214 node("ア", 1, false),
215 node("、", 0, false),
216 node("イ", 0, true),
217 ];
218 let features = njdnodes_to_features_with_sources(&nodes);
219 let phonemes = njdnodes_to_phonemes_with_sources(&nodes);
220 let actual: Vec<_> = phonemes
221 .iter()
222 .map(|feature| (feature.phoneme.as_str(), feature.source_index))
223 .collect();
224 assert_eq!(
225 actual,
226 [
227 ("sil", None),
228 ("a", Some(0)),
229 ("pau", None),
230 ("i", Some(2)),
231 ("sil", None)
232 ]
233 );
234 assert_eq!(features[1].label.utterance.breath_group_count, 1);
235 assert_eq!(features[1].label.utterance.accent_phrase_count, 1);
236 assert_eq!(features[3].label.mora.as_ref().unwrap().position_forward, 2);
237 assert_eq!(
238 features[2]
239 .label
240 .accent_phrase_prev
241 .as_ref()
242 .unwrap()
243 .mora_count,
244 2
245 );
246 assert_eq!(
247 features[2]
248 .label
249 .accent_phrase_next
250 .as_ref()
251 .unwrap()
252 .mora_count,
253 2
254 );
255 }
256
257 #[test]
258 fn empty_and_punctuation_only_inputs_have_no_labels() {
259 assert!(njdnodes_to_features(&[]).is_empty());
260 assert!(
261 njdnodes_to_features(&[
262 node("ー", 0, false),
263 node("?", 0, false),
264 node("!", 0, false),
265 node("、", 0, false),
266 node("*", 0, false),
267 ])
268 .is_empty()
269 );
270 }
271}