use std::collections::{HashMap, HashSet};
use crate::generated::enums::WrittenUnit;
use crate::shaper::Shaper;
use crate::tables::{Fvs, NormalizeData, Position, UnitEntry};
use crate::unicode::is_mongolian_word_char;
use crate::Error;
const MAX_KEY_LEN: usize = 3;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
struct UnitKey {
len: u8,
units: [WrittenUnit; MAX_KEY_LEN],
}
impl UnitKey {
fn new(units: &[WrittenUnit]) -> UnitKey {
debug_assert!(!units.is_empty() && units.len() <= MAX_KEY_LEN);
let mut padded = [units[0]; MAX_KEY_LEN];
padded[..units.len()].copy_from_slice(units);
UnitKey {
len: units.len() as u8,
units: padded,
}
}
}
type Encoding = (u32, Option<Fvs>);
pub(crate) struct NormalizeTable {
pub canonical_version: &'static str,
max_len: usize,
table: HashMap<(Position, UnitKey), Encoding>,
feminine: HashMap<(Position, UnitKey), Encoding>,
velar_fem_units: HashSet<WrittenUnit>,
masculine_cps: HashSet<u32>,
pub known_units: HashSet<WrittenUnit>,
pub sorted_vocabulary: Vec<&'static str>,
pub positioned_units: HashSet<(WrittenUnit, Position)>,
}
fn sorted_vocabulary(units: &HashSet<WrittenUnit>) -> Vec<&'static str> {
let mut names: Vec<&'static str> = units.iter().map(|unit| unit.as_str()).collect();
names.sort_by(|a, b| b.len().cmp(&a.len()).then_with(|| a.cmp(b)));
names
}
fn index_entries(entries: &'static [UnitEntry]) -> HashMap<(Position, UnitKey), Encoding> {
entries
.iter()
.map(|entry| {
(
(entry.position, UnitKey::new(entry.units)),
(entry.cp, entry.fvs),
)
})
.collect()
}
impl NormalizeTable {
pub fn new(data: &'static NormalizeData) -> NormalizeTable {
let mut known_units: HashSet<WrittenUnit> = data
.unit_table
.iter()
.flat_map(|entry| entry.units.iter().copied())
.collect();
known_units.extend([WrittenUnit::Mvs, WrittenUnit::Nirugu, WrittenUnit::Zwj]);
let sorted_vocabulary = sorted_vocabulary(&known_units);
NormalizeTable {
canonical_version: data.canonical_version,
max_len: data.unit_enc_max_len,
table: index_entries(data.unit_table),
feminine: index_entries(data.velar_fem),
velar_fem_units: data.velar_fem_units.iter().copied().collect(),
masculine_cps: data.masc_to_fem.iter().map(|(masc, _)| *masc).collect(),
known_units,
sorted_vocabulary,
positioned_units: data.positioned_units.iter().copied().collect(),
}
}
#[cfg(test)]
pub fn empty(canonical_version: &'static str) -> NormalizeTable {
let known_units: HashSet<WrittenUnit> =
[WrittenUnit::Mvs, WrittenUnit::Nirugu, WrittenUnit::Zwj]
.into_iter()
.collect();
NormalizeTable {
canonical_version,
max_len: 1,
table: HashMap::new(),
feminine: HashMap::new(),
velar_fem_units: HashSet::new(),
masculine_cps: HashSet::new(),
known_units: known_units.clone(),
sorted_vocabulary: sorted_vocabulary(&known_units),
positioned_units: HashSet::new(),
}
}
fn get(&self, position: Position, units: &[WrittenUnit]) -> Option<Encoding> {
self.table.get(&(position, UnitKey::new(units))).copied()
}
fn get_feminine(&self, position: Position, units: &[WrittenUnit]) -> Option<Encoding> {
self.feminine.get(&(position, UnitKey::new(units))).copied()
}
}
pub(crate) fn structural_char(unit: WrittenUnit) -> Option<char> {
match unit {
WrittenUnit::Mvs => Some('\u{180E}'),
WrittenUnit::Nirugu => Some('\u{180A}'),
WrittenUnit::Zwj => Some('\u{200D}'),
_ => None,
}
}
pub(crate) fn is_joiner(unit: WrittenUnit) -> bool {
matches!(unit, WrittenUnit::Nirugu | WrittenUnit::Zwj)
}
fn structural_text(units: &[WrittenUnit]) -> String {
units
.iter()
.map(|unit| structural_char(*unit).expect("structural token"))
.collect()
}
enum Part {
Structural(WrittenUnit),
Chain(Vec<WrittenUnit>),
}
fn split_parts(shape: &[WrittenUnit]) -> Vec<Part> {
let mut parts = Vec::new();
let mut chain = Vec::new();
for &unit in shape {
if unit.is_structural() {
if !chain.is_empty() {
parts.push(Part::Chain(std::mem::take(&mut chain)));
}
parts.push(Part::Structural(unit));
} else {
chain.push(unit);
}
}
if !chain.is_empty() {
parts.push(Part::Chain(chain));
}
parts
}
pub(crate) fn slot_position(start: usize, length: usize, unit_count: usize) -> Position {
if start == 0 && start + length == unit_count {
Position::Isol
} else if start == 0 {
Position::Init
} else if start + length == unit_count {
Position::Fina
} else {
Position::Medi
}
}
fn letter_position(letter_index: usize, total: usize) -> Position {
if total == 1 {
Position::Isol
} else if letter_index == 0 {
Position::Init
} else if letter_index == total - 1 {
Position::Fina
} else {
Position::Medi
}
}
fn unit_partition(
table: &NormalizeTable,
chain: &[WrittenUnit],
joined_left: bool,
joined_right: bool,
) -> Option<String> {
let unit_count = chain.len();
let pad_left = usize::from(joined_left);
let pad_right = usize::from(joined_right);
let padded_count = unit_count + pad_left + pad_right;
let mut letters: Vec<Encoding> = Vec::new();
let mut unit_at: Vec<Option<WrittenUnit>> = Vec::new();
let mut index = 0;
while index < unit_count {
let span = table.max_len.min(unit_count - index);
let mut hit: Option<(Encoding, usize)> = None;
let position = slot_position(index + pad_left, 1, padded_count);
if let Some(encoding) = table.get(position, &chain[index..index + 1]) {
hit = Some((encoding, 1));
}
if hit.is_none() {
for length in (2..=span).rev() {
let position = slot_position(index + pad_left, length, padded_count);
if let Some(encoding) = table.get(position, &chain[index..index + length]) {
hit = Some((encoding, length));
break;
}
}
}
let (encoding, length) = hit?;
letters.push(encoding);
unit_at.push((length == 1).then_some(chain[index]));
index += length;
}
apply_velar_fem(table, &mut letters, &unit_at, pad_left, pad_right);
let mut text = String::new();
for (cp, fvs) in letters {
text.push(char::from_u32(cp).expect("table code points are scalar values"));
if let Some(fvs) = fvs {
text.push(fvs.as_char());
}
}
Some(text)
}
fn apply_velar_fem(
table: &NormalizeTable,
letters: &mut [Encoding],
unit_at: &[Option<WrittenUnit>],
pad_left: usize,
pad_right: usize,
) {
let total = letters.len();
let padded_total = total + pad_left + pad_right;
for (letter_index, unit) in unit_at.iter().enumerate() {
let Some(unit) = *unit else {
continue;
};
if !table.velar_fem_units.contains(&unit) {
continue;
}
let position = letter_position(letter_index + pad_left, padded_total);
if !matches!(position, Position::Init | Position::Medi) {
continue;
}
let target_index = letter_index + 1;
if target_index >= total {
continue;
}
let Some(target_unit) = unit_at[target_index] else {
continue; };
let (cp, _) = letters[target_index];
if !table.masculine_cps.contains(&cp) {
continue; }
let target_position = letter_position(target_index + pad_left, padded_total);
let Some(feminine) = table.get_feminine(target_position, &[target_unit]) else {
continue; };
letters[target_index] = feminine;
}
}
impl Shaper {
pub(crate) fn table(&self) -> Result<&NormalizeTable, Error> {
self.normalize.as_ref().ok_or(Error::NormalizeUnsupported {
locale: self.locale(),
})
}
pub fn canonical_version(&self) -> Option<&'static str> {
self.normalize.as_ref().map(|table| table.canonical_version)
}
pub(crate) fn canonical_for_shape(&self, shape: &[WrittenUnit]) -> Result<String, Error> {
let parts = split_parts(shape);
let mut suffix_text = String::new();
let mut suffix_target: Vec<WrittenUnit> = Vec::new();
for index in (0..parts.len()).rev() {
match &parts[index] {
Part::Structural(unit) => {
let text = structural_char(*unit)
.expect("structural token")
.to_string();
suffix_text.insert_str(0, &text);
suffix_target.insert(0, *unit);
}
Part::Chain(body) => {
let table = self.table()?;
let mut prefix_tokens: Vec<WrittenUnit> = Vec::new();
let mut scan = index;
while scan > 0 {
scan -= 1;
match &parts[scan] {
Part::Structural(unit) => prefix_tokens.insert(0, *unit),
Part::Chain(_) => break,
}
}
let mut chain_canonical: Option<String> = None;
if prefix_tokens.last() == Some(&WrittenUnit::Mvs) {
let candidate = if body.as_slice() == [WrittenUnit::Aa] {
String::from('\u{1820}')
} else {
self.encode_chain_canonical(table, body, &[], "", &[])?
};
if !candidate.is_empty() {
let prefix_text = structural_text(&prefix_tokens);
let mut want = prefix_tokens.clone();
want.extend_from_slice(body);
want.extend_from_slice(&suffix_target);
if self.shape(&format!("{prefix_text}{candidate}{suffix_text}"))?
== want
{
chain_canonical = Some(candidate);
}
}
}
let chain_canonical = match chain_canonical {
Some(text) => text,
None => self.encode_chain_canonical(
table,
body,
&prefix_tokens,
&suffix_text,
&suffix_target,
)?,
};
suffix_text.insert_str(0, &chain_canonical);
let mut target = body.clone();
target.extend_from_slice(&suffix_target);
suffix_target = target;
}
}
}
Ok(suffix_text)
}
fn encode_chain_canonical(
&self,
table: &NormalizeTable,
chain: &[WrittenUnit],
prefix_tokens: &[WrittenUnit],
suffix_text: &str,
suffix_target: &[WrittenUnit],
) -> Result<String, Error> {
Ok(self
.unit_encode_chain(table, chain, prefix_tokens, suffix_text, suffix_target)?
.unwrap_or_default())
}
fn unit_encode_chain(
&self,
table: &NormalizeTable,
chain: &[WrittenUnit],
prefix_tokens: &[WrittenUnit],
suffix_text: &str,
suffix_target: &[WrittenUnit],
) -> Result<Option<String>, Error> {
let joined_left = prefix_tokens.last().is_some_and(|unit| is_joiner(*unit));
let joined_right = suffix_target.first().is_some_and(|unit| is_joiner(*unit));
let Some(text) = unit_partition(table, chain, joined_left, joined_right) else {
return Ok(None);
};
let prefix_text = structural_text(prefix_tokens);
let mut verify_target = prefix_tokens.to_vec();
verify_target.extend_from_slice(chain);
verify_target.extend_from_slice(suffix_target);
if self.shape(&format!("{prefix_text}{text}{suffix_text}"))? == verify_target {
Ok(Some(text))
} else {
Ok(None)
}
}
fn normalize_impl(&self, text: &str, strict: bool) -> Result<String, Error> {
if text.is_empty() {
return Ok(String::new());
}
let target = self.shape(text)?;
if target.is_empty() {
return Ok(String::new());
}
let canonical = self.canonical_for_shape(&target)?;
if canonical.is_empty() || self.shape(&canonical)? != target {
if strict {
return Err(Error::NormalizationFallback {
text: text.to_owned(),
written_units: target,
});
}
return Ok(text.to_owned());
}
Ok(canonical)
}
pub fn normalize(&self, text: &str) -> Result<String, Error> {
self.normalize_impl(text, true)
}
pub fn normalize_allow_fallback(&self, text: &str) -> Result<String, Error> {
self.normalize_impl(text, false)
}
fn normalize_text_impl(&self, text: &str, strict: bool) -> Result<String, Error> {
if text.is_empty() {
return Ok(String::new());
}
let mut out = String::with_capacity(text.len());
let mut run = String::new();
let mut run_is_mongolian: Option<bool> = None;
for ch in text.chars() {
let is_mongolian = is_mongolian_word_char(ch);
match run_is_mongolian {
Some(current) if current != is_mongolian => {
self.flush_run(&mut out, &run, current, strict)?;
run.clear();
run_is_mongolian = Some(is_mongolian);
}
Some(_) => {}
None => run_is_mongolian = Some(is_mongolian),
}
run.push(ch);
}
if let Some(current) = run_is_mongolian {
self.flush_run(&mut out, &run, current, strict)?;
}
Ok(out)
}
fn flush_run(
&self,
out: &mut String,
run: &str,
is_mongolian: bool,
strict: bool,
) -> Result<(), Error> {
if is_mongolian {
out.push_str(&self.normalize_impl(run, strict)?);
} else {
out.push_str(run);
}
Ok(())
}
pub fn normalize_text(&self, text: &str) -> Result<String, Error> {
self.normalize_text_impl(text, true)
}
pub fn normalize_text_allow_fallback(&self, text: &str) -> Result<String, Error> {
self.normalize_text_impl(text, false)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::Locale;
const SAIN: &str = "\u{1830}\u{1820}\u{1822}\u{1828}";
#[test]
fn strict_mode_raises_when_canonicalization_falls_back() {
let shaper = Shaper::with_empty_normalize_table(Locale::Mng);
let error = shaper.normalize(SAIN).unwrap_err();
assert_eq!(
error,
Error::NormalizationFallback {
text: SAIN.to_owned(),
written_units: vec![
WrittenUnit::S,
WrittenUnit::A,
WrittenUnit::I,
WrittenUnit::I,
WrittenUnit::A
],
}
);
assert_eq!(
error.to_string(),
"normalization fallback: no canonical encoding for written units S+A+I+I+A"
);
}
#[test]
fn allow_fallback_preserves_input_when_canonicalization_falls_back() {
let shaper = Shaper::with_empty_normalize_table(Locale::Mng);
assert_eq!(shaper.normalize_allow_fallback(SAIN).unwrap(), SAIN);
}
#[test]
fn strict_mode_reports_a_fallback_inside_mixed_text() {
let shaper = Shaper::with_empty_normalize_table(Locale::Mng);
let text = format!("Hello {SAIN} world");
assert!(matches!(
shaper.normalize_text(&text),
Err(Error::NormalizationFallback { .. })
));
}
#[test]
fn allow_fallback_preserves_a_fallback_inside_mixed_text() {
let shaper = Shaper::with_empty_normalize_table(Locale::Mng);
let text = format!("Hello {SAIN} world");
assert_eq!(shaper.normalize_text_allow_fallback(&text).unwrap(), text);
}
#[test]
fn locales_without_a_table_reject_normalization_of_letters() {
let shaper = Shaper::new(Locale::Tod);
assert_eq!(shaper.canonical_version(), None);
assert_eq!(shaper.normalize("").unwrap(), "");
assert_eq!(shaper.normalize("\u{180B}").unwrap(), ""); assert_eq!(shaper.normalize("\u{180A}").unwrap(), "\u{180A}"); assert_eq!(
shaper.normalize("\u{1820}"),
Err(Error::NormalizeUnsupported {
locale: Locale::Tod
})
);
assert_eq!(
shaper.normalize_written_units(&[WrittenUnit::Mvs]),
Err(Error::NormalizeUnsupported {
locale: Locale::Tod
})
);
assert_eq!(
Shaper::new(Locale::Mng).canonical_version(),
Some("mng-canonical/1")
);
}
#[test]
fn positions_of_partition_slots_and_letters() {
assert_eq!(slot_position(0, 1, 1), Position::Isol);
assert_eq!(slot_position(0, 2, 2), Position::Isol);
assert_eq!(slot_position(0, 1, 3), Position::Init);
assert_eq!(slot_position(1, 1, 3), Position::Medi);
assert_eq!(slot_position(1, 2, 3), Position::Fina);
assert_eq!(letter_position(0, 1), Position::Isol);
assert_eq!(letter_position(0, 2), Position::Init);
assert_eq!(letter_position(1, 2), Position::Fina);
assert_eq!(letter_position(1, 3), Position::Medi);
assert_eq!(
UnitKey::new(&[WrittenUnit::A]),
UnitKey::new(&[WrittenUnit::A])
);
assert_ne!(
UnitKey::new(&[WrittenUnit::A]),
UnitKey::new(&[WrittenUnit::A, WrittenUnit::A])
);
}
}