1use std::collections::{HashMap, HashSet};
16
17use crate::generated::enums::WrittenUnit;
18use crate::shaper::Shaper;
19use crate::tables::{Fvs, NormalizeData, Position, UnitEntry};
20use crate::unicode::is_mongolian_word_char;
21use crate::Error;
22
23const MAX_KEY_LEN: usize = 3;
25
26#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
28struct UnitKey {
29 len: u8,
30 units: [WrittenUnit; MAX_KEY_LEN],
31}
32
33impl UnitKey {
34 fn new(units: &[WrittenUnit]) -> UnitKey {
35 debug_assert!(!units.is_empty() && units.len() <= MAX_KEY_LEN);
36 let mut padded = [units[0]; MAX_KEY_LEN];
39 padded[..units.len()].copy_from_slice(units);
40 UnitKey {
41 len: units.len() as u8,
42 units: padded,
43 }
44 }
45}
46
47type Encoding = (u32, Option<Fvs>);
49
50pub(crate) struct NormalizeTable {
52 pub canonical_version: &'static str,
53 max_len: usize,
54 table: HashMap<(Position, UnitKey), Encoding>,
55 feminine: HashMap<(Position, UnitKey), Encoding>,
56 velar_fem_units: HashSet<WrittenUnit>,
57 masculine_cps: HashSet<u32>,
58 pub known_units: HashSet<WrittenUnit>,
61 pub sorted_vocabulary: Vec<&'static str>,
64 pub positioned_units: HashSet<(WrittenUnit, Position)>,
66}
67
68fn sorted_vocabulary(units: &HashSet<WrittenUnit>) -> Vec<&'static str> {
70 let mut names: Vec<&'static str> = units.iter().map(|unit| unit.as_str()).collect();
71 names.sort_by(|a, b| b.len().cmp(&a.len()).then_with(|| a.cmp(b)));
72 names
73}
74
75fn index_entries(entries: &'static [UnitEntry]) -> HashMap<(Position, UnitKey), Encoding> {
76 entries
77 .iter()
78 .map(|entry| {
79 (
80 (entry.position, UnitKey::new(entry.units)),
81 (entry.cp, entry.fvs),
82 )
83 })
84 .collect()
85}
86
87impl NormalizeTable {
88 pub fn new(data: &'static NormalizeData) -> NormalizeTable {
89 let mut known_units: HashSet<WrittenUnit> = data
90 .unit_table
91 .iter()
92 .flat_map(|entry| entry.units.iter().copied())
93 .collect();
94 known_units.extend([WrittenUnit::Mvs, WrittenUnit::Nirugu, WrittenUnit::Zwj]);
95 let sorted_vocabulary = sorted_vocabulary(&known_units);
96 NormalizeTable {
97 canonical_version: data.canonical_version,
98 max_len: data.unit_enc_max_len,
99 table: index_entries(data.unit_table),
100 feminine: index_entries(data.velar_fem),
101 velar_fem_units: data.velar_fem_units.iter().copied().collect(),
102 masculine_cps: data.masc_to_fem.iter().map(|(masc, _)| *masc).collect(),
103 known_units,
104 sorted_vocabulary,
105 positioned_units: data.positioned_units.iter().copied().collect(),
106 }
107 }
108
109 #[cfg(test)]
112 pub fn empty(canonical_version: &'static str) -> NormalizeTable {
113 let known_units: HashSet<WrittenUnit> =
114 [WrittenUnit::Mvs, WrittenUnit::Nirugu, WrittenUnit::Zwj]
115 .into_iter()
116 .collect();
117 NormalizeTable {
118 canonical_version,
119 max_len: 1,
120 table: HashMap::new(),
121 feminine: HashMap::new(),
122 velar_fem_units: HashSet::new(),
123 masculine_cps: HashSet::new(),
124 known_units: known_units.clone(),
125 sorted_vocabulary: sorted_vocabulary(&known_units),
126 positioned_units: HashSet::new(),
127 }
128 }
129
130 fn get(&self, position: Position, units: &[WrittenUnit]) -> Option<Encoding> {
131 self.table.get(&(position, UnitKey::new(units))).copied()
132 }
133
134 fn get_feminine(&self, position: Position, units: &[WrittenUnit]) -> Option<Encoding> {
135 self.feminine.get(&(position, UnitKey::new(units))).copied()
136 }
137}
138
139pub(crate) fn structural_char(unit: WrittenUnit) -> Option<char> {
141 match unit {
142 WrittenUnit::Mvs => Some('\u{180E}'),
143 WrittenUnit::Nirugu => Some('\u{180A}'),
144 WrittenUnit::Zwj => Some('\u{200D}'),
145 _ => None,
146 }
147}
148
149pub(crate) fn is_joiner(unit: WrittenUnit) -> bool {
151 matches!(unit, WrittenUnit::Nirugu | WrittenUnit::Zwj)
152}
153
154fn structural_text(units: &[WrittenUnit]) -> String {
155 units
156 .iter()
157 .map(|unit| structural_char(*unit).expect("structural token"))
158 .collect()
159}
160
161enum Part {
162 Structural(WrittenUnit),
163 Chain(Vec<WrittenUnit>),
164}
165
166fn split_parts(shape: &[WrittenUnit]) -> Vec<Part> {
168 let mut parts = Vec::new();
169 let mut chain = Vec::new();
170 for &unit in shape {
171 if unit.is_structural() {
172 if !chain.is_empty() {
173 parts.push(Part::Chain(std::mem::take(&mut chain)));
174 }
175 parts.push(Part::Structural(unit));
176 } else {
177 chain.push(unit);
178 }
179 }
180 if !chain.is_empty() {
181 parts.push(Part::Chain(chain));
182 }
183 parts
184}
185
186pub(crate) fn slot_position(start: usize, length: usize, unit_count: usize) -> Position {
188 if start == 0 && start + length == unit_count {
189 Position::Isol
190 } else if start == 0 {
191 Position::Init
192 } else if start + length == unit_count {
193 Position::Fina
194 } else {
195 Position::Medi
196 }
197}
198
199fn letter_position(letter_index: usize, total: usize) -> Position {
201 if total == 1 {
202 Position::Isol
203 } else if letter_index == 0 {
204 Position::Init
205 } else if letter_index == total - 1 {
206 Position::Fina
207 } else {
208 Position::Medi
209 }
210}
211
212fn unit_partition(
216 table: &NormalizeTable,
217 chain: &[WrittenUnit],
218 joined_left: bool,
219 joined_right: bool,
220) -> Option<String> {
221 let unit_count = chain.len();
222 let pad_left = usize::from(joined_left);
223 let pad_right = usize::from(joined_right);
224 let padded_count = unit_count + pad_left + pad_right;
225 let mut letters: Vec<Encoding> = Vec::new();
226 let mut unit_at: Vec<Option<WrittenUnit>> = Vec::new();
227 let mut index = 0;
228 while index < unit_count {
229 let span = table.max_len.min(unit_count - index);
230 let mut hit: Option<(Encoding, usize)> = None;
231 let position = slot_position(index + pad_left, 1, padded_count);
233 if let Some(encoding) = table.get(position, &chain[index..index + 1]) {
234 hit = Some((encoding, 1));
235 }
236 if hit.is_none() {
238 for length in (2..=span).rev() {
239 let position = slot_position(index + pad_left, length, padded_count);
240 if let Some(encoding) = table.get(position, &chain[index..index + length]) {
241 hit = Some((encoding, length));
242 break;
243 }
244 }
245 }
246 let (encoding, length) = hit?;
247 letters.push(encoding);
248 unit_at.push((length == 1).then_some(chain[index]));
249 index += length;
250 }
251 apply_velar_fem(table, &mut letters, &unit_at, pad_left, pad_right);
252 let mut text = String::new();
253 for (cp, fvs) in letters {
254 text.push(char::from_u32(cp).expect("table code points are scalar values"));
255 if let Some(fvs) = fvs {
256 text.push(fvs.as_char());
257 }
258 }
259 Some(text)
260}
261
262fn apply_velar_fem(
266 table: &NormalizeTable,
267 letters: &mut [Encoding],
268 unit_at: &[Option<WrittenUnit>],
269 pad_left: usize,
270 pad_right: usize,
271) {
272 let total = letters.len();
276 let padded_total = total + pad_left + pad_right;
277 for (letter_index, unit) in unit_at.iter().enumerate() {
278 let Some(unit) = *unit else {
279 continue;
280 };
281 if !table.velar_fem_units.contains(&unit) {
282 continue;
283 }
284 let position = letter_position(letter_index + pad_left, padded_total);
291 if !matches!(position, Position::Init | Position::Medi) {
292 continue;
293 }
294 let target_index = letter_index + 1;
295 if target_index >= total {
296 continue;
297 }
298 let Some(target_unit) = unit_at[target_index] else {
299 continue; };
301 let (cp, _) = letters[target_index];
302 if !table.masculine_cps.contains(&cp) {
303 continue; }
305 let target_position = letter_position(target_index + pad_left, padded_total);
306 let Some(feminine) = table.get_feminine(target_position, &[target_unit]) else {
307 continue; };
309 letters[target_index] = feminine;
310 }
311}
312
313impl Shaper {
314 pub(crate) fn table(&self) -> Result<&NormalizeTable, Error> {
316 self.normalize.as_ref().ok_or(Error::NormalizeUnsupported {
317 locale: self.locale(),
318 })
319 }
320
321 pub fn canonical_version(&self) -> Option<&'static str> {
324 self.normalize.as_ref().map(|table| table.canonical_version)
325 }
326
327 pub(crate) fn canonical_for_shape(&self, shape: &[WrittenUnit]) -> Result<String, Error> {
339 let parts = split_parts(shape);
340 let mut suffix_text = String::new();
344 let mut suffix_target: Vec<WrittenUnit> = Vec::new();
345 for index in (0..parts.len()).rev() {
346 match &parts[index] {
347 Part::Structural(unit) => {
348 let text = structural_char(*unit)
349 .expect("structural token")
350 .to_string();
351 suffix_text.insert_str(0, &text);
352 suffix_target.insert(0, *unit);
353 }
354 Part::Chain(body) => {
355 let table = self.table()?;
356 let mut prefix_tokens: Vec<WrittenUnit> = Vec::new();
360 let mut scan = index;
361 while scan > 0 {
362 scan -= 1;
363 match &parts[scan] {
364 Part::Structural(unit) => prefix_tokens.insert(0, *unit),
365 Part::Chain(_) => break,
366 }
367 }
368 let mut chain_canonical: Option<String> = None;
369 if prefix_tokens.last() == Some(&WrittenUnit::Mvs) {
372 let candidate = if body.as_slice() == [WrittenUnit::Aa] {
373 String::from('\u{1820}')
374 } else {
375 self.encode_chain_canonical(table, body, &[], "", &[])?
376 };
377 if !candidate.is_empty() {
378 let prefix_text = structural_text(&prefix_tokens);
379 let mut want = prefix_tokens.clone();
380 want.extend_from_slice(body);
381 want.extend_from_slice(&suffix_target);
382 if self.shape(&format!("{prefix_text}{candidate}{suffix_text}"))?
383 == want
384 {
385 chain_canonical = Some(candidate);
386 }
387 }
388 }
389 let chain_canonical = match chain_canonical {
390 Some(text) => text,
391 None => self.encode_chain_canonical(
392 table,
393 body,
394 &prefix_tokens,
395 &suffix_text,
396 &suffix_target,
397 )?,
398 };
399 suffix_text.insert_str(0, &chain_canonical);
400 let mut target = body.clone();
401 target.extend_from_slice(&suffix_target);
402 suffix_target = target;
403 }
404 }
405 }
406 Ok(suffix_text)
407 }
408
409 fn encode_chain_canonical(
412 &self,
413 table: &NormalizeTable,
414 chain: &[WrittenUnit],
415 prefix_tokens: &[WrittenUnit],
416 suffix_text: &str,
417 suffix_target: &[WrittenUnit],
418 ) -> Result<String, Error> {
419 Ok(self
420 .unit_encode_chain(table, chain, prefix_tokens, suffix_text, suffix_target)?
421 .unwrap_or_default())
422 }
423
424 fn unit_encode_chain(
427 &self,
428 table: &NormalizeTable,
429 chain: &[WrittenUnit],
430 prefix_tokens: &[WrittenUnit],
431 suffix_text: &str,
432 suffix_target: &[WrittenUnit],
433 ) -> Result<Option<String>, Error> {
434 let joined_left = prefix_tokens.last().is_some_and(|unit| is_joiner(*unit));
435 let joined_right = suffix_target.first().is_some_and(|unit| is_joiner(*unit));
436 let Some(text) = unit_partition(table, chain, joined_left, joined_right) else {
437 return Ok(None);
438 };
439 let prefix_text = structural_text(prefix_tokens);
440 let mut verify_target = prefix_tokens.to_vec();
444 verify_target.extend_from_slice(chain);
445 verify_target.extend_from_slice(suffix_target);
446 if self.shape(&format!("{prefix_text}{text}{suffix_text}"))? == verify_target {
447 Ok(Some(text))
448 } else {
449 Ok(None)
450 }
451 }
452
453 fn normalize_impl(&self, text: &str, strict: bool) -> Result<String, Error> {
454 if text.is_empty() {
455 return Ok(String::new());
456 }
457 let target = self.shape(text)?;
458 if target.is_empty() {
459 return Ok(String::new());
462 }
463 let canonical = self.canonical_for_shape(&target)?;
464 if canonical.is_empty() || self.shape(&canonical)? != target {
465 if strict {
466 return Err(Error::NormalizationFallback {
467 text: text.to_owned(),
468 written_units: target,
469 });
470 }
471 return Ok(text.to_owned());
472 }
473 Ok(canonical)
474 }
475
476 pub fn normalize(&self, text: &str) -> Result<String, Error> {
484 self.normalize_impl(text, true)
485 }
486
487 pub fn normalize_allow_fallback(&self, text: &str) -> Result<String, Error> {
490 self.normalize_impl(text, false)
491 }
492
493 fn normalize_text_impl(&self, text: &str, strict: bool) -> Result<String, Error> {
494 if text.is_empty() {
495 return Ok(String::new());
496 }
497 let mut out = String::with_capacity(text.len());
498 let mut run = String::new();
499 let mut run_is_mongolian: Option<bool> = None;
500 for ch in text.chars() {
501 let is_mongolian = is_mongolian_word_char(ch);
502 match run_is_mongolian {
503 Some(current) if current != is_mongolian => {
504 self.flush_run(&mut out, &run, current, strict)?;
505 run.clear();
506 run_is_mongolian = Some(is_mongolian);
507 }
508 Some(_) => {}
509 None => run_is_mongolian = Some(is_mongolian),
510 }
511 run.push(ch);
512 }
513 if let Some(current) = run_is_mongolian {
514 self.flush_run(&mut out, &run, current, strict)?;
515 }
516 Ok(out)
517 }
518
519 fn flush_run(
520 &self,
521 out: &mut String,
522 run: &str,
523 is_mongolian: bool,
524 strict: bool,
525 ) -> Result<(), Error> {
526 if is_mongolian {
527 out.push_str(&self.normalize_impl(run, strict)?);
528 } else {
529 out.push_str(run);
530 }
531 Ok(())
532 }
533
534 pub fn normalize_text(&self, text: &str) -> Result<String, Error> {
537 self.normalize_text_impl(text, true)
538 }
539
540 pub fn normalize_text_allow_fallback(&self, text: &str) -> Result<String, Error> {
542 self.normalize_text_impl(text, false)
543 }
544}
545
546#[cfg(test)]
547mod tests {
548 use super::*;
549 use crate::Locale;
550
551 const SAIN: &str = "\u{1830}\u{1820}\u{1822}\u{1828}";
552
553 #[test]
554 fn strict_mode_raises_when_canonicalization_falls_back() {
555 let shaper = Shaper::with_empty_normalize_table(Locale::Mng);
556 let error = shaper.normalize(SAIN).unwrap_err();
557 assert_eq!(
558 error,
559 Error::NormalizationFallback {
560 text: SAIN.to_owned(),
561 written_units: vec![
562 WrittenUnit::S,
563 WrittenUnit::A,
564 WrittenUnit::I,
565 WrittenUnit::I,
566 WrittenUnit::A
567 ],
568 }
569 );
570 assert_eq!(
571 error.to_string(),
572 "normalization fallback: no canonical encoding for written units S+A+I+I+A"
573 );
574 }
575
576 #[test]
577 fn allow_fallback_preserves_input_when_canonicalization_falls_back() {
578 let shaper = Shaper::with_empty_normalize_table(Locale::Mng);
579 assert_eq!(shaper.normalize_allow_fallback(SAIN).unwrap(), SAIN);
580 }
581
582 #[test]
583 fn strict_mode_reports_a_fallback_inside_mixed_text() {
584 let shaper = Shaper::with_empty_normalize_table(Locale::Mng);
585 let text = format!("Hello {SAIN} world");
586 assert!(matches!(
587 shaper.normalize_text(&text),
588 Err(Error::NormalizationFallback { .. })
589 ));
590 }
591
592 #[test]
593 fn allow_fallback_preserves_a_fallback_inside_mixed_text() {
594 let shaper = Shaper::with_empty_normalize_table(Locale::Mng);
595 let text = format!("Hello {SAIN} world");
596 assert_eq!(shaper.normalize_text_allow_fallback(&text).unwrap(), text);
597 }
598
599 #[test]
600 fn locales_without_a_table_reject_normalization_of_letters() {
601 let shaper = Shaper::new(Locale::Tod);
602 assert_eq!(shaper.canonical_version(), None);
603 assert_eq!(shaper.normalize("").unwrap(), "");
604 assert_eq!(shaper.normalize("\u{180B}").unwrap(), ""); assert_eq!(shaper.normalize("\u{180A}").unwrap(), "\u{180A}"); assert_eq!(
607 shaper.normalize("\u{1820}"),
608 Err(Error::NormalizeUnsupported {
609 locale: Locale::Tod
610 })
611 );
612 assert_eq!(
613 shaper.normalize_written_units(&[WrittenUnit::Mvs]),
614 Err(Error::NormalizeUnsupported {
615 locale: Locale::Tod
616 })
617 );
618 assert_eq!(
619 Shaper::new(Locale::Mng).canonical_version(),
620 Some("mng-canonical/1")
621 );
622 }
623
624 #[test]
625 fn positions_of_partition_slots_and_letters() {
626 assert_eq!(slot_position(0, 1, 1), Position::Isol);
627 assert_eq!(slot_position(0, 2, 2), Position::Isol);
628 assert_eq!(slot_position(0, 1, 3), Position::Init);
629 assert_eq!(slot_position(1, 1, 3), Position::Medi);
630 assert_eq!(slot_position(1, 2, 3), Position::Fina);
631 assert_eq!(letter_position(0, 1), Position::Isol);
632 assert_eq!(letter_position(0, 2), Position::Init);
633 assert_eq!(letter_position(1, 2), Position::Fina);
634 assert_eq!(letter_position(1, 3), Position::Medi);
635 assert_eq!(
636 UnitKey::new(&[WrittenUnit::A]),
637 UnitKey::new(&[WrittenUnit::A])
638 );
639 assert_ne!(
640 UnitKey::new(&[WrittenUnit::A]),
641 UnitKey::new(&[WrittenUnit::A, WrittenUnit::A])
642 );
643 }
644}