Skip to main content

mongol_norm/
normalize.rs

1//! Canonical normalization — a port of the normalize core of `mongol_norm/shaper.py`
2//! (`_canonical_for_shape`, `_unit_encode_chain`, `_unit_partition`, `_apply_velar_fem`,
3//! `_slot_position`, `_letter_position`, `normalize`, `normalize_text`).
4//!
5//! Within the table's domain `normalize` is a pure function of shape: each written unit is
6//! encoded by a context-independent, FVS-pinned `(letter, fvs)` from the per-`(position, unit)`
7//! table; every chain is verified by reshaping it in full context; a chain right after MVS takes
8//! its standalone canonical so a suffix's spelling never depends on the MVS. There is no search
9//! fallback — an uncovered shape is reported (strict) or echoed back unchanged.
10//!
11//! The remedy for a genuine table gap is to widen the table, not the runtime: the table is
12//! generated offline by `scripts/gen_normalize_table.py` (Python), so extending coverage means
13//! regenerating it there and rerunning `scripts/gen_rust_tables.py`.
14
15use std::collections::{HashMap, HashSet};
16
17use crate::generated::enums::WrittenUnit;
18use crate::shaper::Shaper;
19use crate::tables::{Fvs, NormalizeData, Position, UnitEntry};
20use crate::unicode::is_mongolian_word_char;
21use crate::Error;
22
23/// Longest written-unit tuple a table key holds (the generator asserts `unit_enc_max_len <= 3`).
24const MAX_KEY_LEN: usize = 3;
25
26/// A fixed-capacity `(written units)` key — lookups never allocate.
27#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
28struct UnitKey {
29    len: u8,
30    units: [WrittenUnit; MAX_KEY_LEN],
31}
32
33impl UnitKey {
34    fn new(units: &[WrittenUnit]) -> UnitKey {
35        debug_assert!(!units.is_empty() && units.len() <= MAX_KEY_LEN);
36        // The padding value is arbitrary but deterministic per slice, and `len` disambiguates
37        // shorter keys from longer ones, so `Hash` and `Eq` stay consistent.
38        let mut padded = [units[0]; MAX_KEY_LEN];
39        padded[..units.len()].copy_from_slice(units);
40        UnitKey {
41            len: units.len() as u8,
42            units: padded,
43        }
44    }
45}
46
47/// `(letter code point, FVS)`.
48type Encoding = (u32, Option<Fvs>);
49
50/// The runtime form of `MNG.normalize.json`.
51pub(crate) struct NormalizeTable {
52    pub canonical_version: &'static str,
53    max_len: usize,
54    table: HashMap<(Position, UnitKey), Encoding>,
55    feminine: HashMap<(Position, UnitKey), Encoding>,
56    velar_fem_units: HashSet<WrittenUnit>,
57    masculine_cps: HashSet<u32>,
58    /// Every unit that occurs in a table key, plus the three structural tokens — the vocabulary
59    /// of `normalize_written_units` and `parse_written_units`.
60    pub known_units: HashSet<WrittenUnit>,
61    /// `known_units` as names, in Python's `(-len, name)` order — the compact-segmentation
62    /// vocabulary, built once here so `parse_written_units` never sorts per call.
63    pub sorted_vocabulary: Vec<&'static str>,
64    /// The authoritative HUD `(unit, position)` inventory.
65    pub positioned_units: HashSet<(WrittenUnit, Position)>,
66}
67
68/// The unit names of `units` in Python's `sorted(known, key=lambda u: (-len(u), u))` order.
69fn sorted_vocabulary(units: &HashSet<WrittenUnit>) -> Vec<&'static str> {
70    let mut names: Vec<&'static str> = units.iter().map(|unit| unit.as_str()).collect();
71    names.sort_by(|a, b| b.len().cmp(&a.len()).then_with(|| a.cmp(b)));
72    names
73}
74
75fn index_entries(entries: &'static [UnitEntry]) -> HashMap<(Position, UnitKey), Encoding> {
76    entries
77        .iter()
78        .map(|entry| {
79            (
80                (entry.position, UnitKey::new(entry.units)),
81                (entry.cp, entry.fvs),
82            )
83        })
84        .collect()
85}
86
87impl NormalizeTable {
88    pub fn new(data: &'static NormalizeData) -> NormalizeTable {
89        let mut known_units: HashSet<WrittenUnit> = data
90            .unit_table
91            .iter()
92            .flat_map(|entry| entry.units.iter().copied())
93            .collect();
94        known_units.extend([WrittenUnit::Mvs, WrittenUnit::Nirugu, WrittenUnit::Zwj]);
95        let sorted_vocabulary = sorted_vocabulary(&known_units);
96        NormalizeTable {
97            canonical_version: data.canonical_version,
98            max_len: data.unit_enc_max_len,
99            table: index_entries(data.unit_table),
100            feminine: index_entries(data.velar_fem),
101            velar_fem_units: data.velar_fem_units.iter().copied().collect(),
102            masculine_cps: data.masc_to_fem.iter().map(|(masc, _)| *masc).collect(),
103            known_units,
104            sorted_vocabulary,
105            positioned_units: data.positioned_units.iter().copied().collect(),
106        }
107    }
108
109    /// Python's monkeypatched empty table (`_unit_enc = {}`, `_unit_enc_max_len = 1`): no
110    /// encodings at all, so every chain falls back.
111    #[cfg(test)]
112    pub fn empty(canonical_version: &'static str) -> NormalizeTable {
113        let known_units: HashSet<WrittenUnit> =
114            [WrittenUnit::Mvs, WrittenUnit::Nirugu, WrittenUnit::Zwj]
115                .into_iter()
116                .collect();
117        NormalizeTable {
118            canonical_version,
119            max_len: 1,
120            table: HashMap::new(),
121            feminine: HashMap::new(),
122            velar_fem_units: HashSet::new(),
123            masculine_cps: HashSet::new(),
124            known_units: known_units.clone(),
125            sorted_vocabulary: sorted_vocabulary(&known_units),
126            positioned_units: HashSet::new(),
127        }
128    }
129
130    fn get(&self, position: Position, units: &[WrittenUnit]) -> Option<Encoding> {
131        self.table.get(&(position, UnitKey::new(units))).copied()
132    }
133
134    fn get_feminine(&self, position: Position, units: &[WrittenUnit]) -> Option<Encoding> {
135        self.feminine.get(&(position, UnitKey::new(units))).copied()
136    }
137}
138
139/// The character a structural shape token encodes to, verbatim (Python `_STRUCTURAL_CHARS`).
140pub(crate) fn structural_char(unit: WrittenUnit) -> Option<char> {
141    match unit {
142        WrittenUnit::Mvs => Some('\u{180E}'),
143        WrittenUnit::Nirugu => Some('\u{180A}'),
144        WrittenUnit::Zwj => Some('\u{200D}'),
145        _ => None,
146    }
147}
148
149/// Joiners force cursive connection on the adjacent letter (Python `_JOINER_TOKENS`).
150pub(crate) fn is_joiner(unit: WrittenUnit) -> bool {
151    matches!(unit, WrittenUnit::Nirugu | WrittenUnit::Zwj)
152}
153
154fn structural_text(units: &[WrittenUnit]) -> String {
155    units
156        .iter()
157        .map(|unit| structural_char(*unit).expect("structural token"))
158        .collect()
159}
160
161enum Part {
162    Structural(WrittenUnit),
163    Chain(Vec<WrittenUnit>),
164}
165
166/// Split a shape at its structural tokens (which are copied through verbatim).
167fn split_parts(shape: &[WrittenUnit]) -> Vec<Part> {
168    let mut parts = Vec::new();
169    let mut chain = Vec::new();
170    for &unit in shape {
171        if unit.is_structural() {
172            if !chain.is_empty() {
173                parts.push(Part::Chain(std::mem::take(&mut chain)));
174            }
175            parts.push(Part::Structural(unit));
176        } else {
177            chain.push(unit);
178        }
179    }
180    if !chain.is_empty() {
181        parts.push(Part::Chain(chain));
182    }
183    parts
184}
185
186/// Position of a letter spanning units `[start, start + length)` in a chain of `unit_count` units.
187pub(crate) fn slot_position(start: usize, length: usize, unit_count: usize) -> Position {
188    if start == 0 && start + length == unit_count {
189        Position::Isol
190    } else if start == 0 {
191        Position::Init
192    } else if start + length == unit_count {
193        Position::Fina
194    } else {
195        Position::Medi
196    }
197}
198
199/// Position of the `letter_index`-th letter out of `total` letters.
200fn letter_position(letter_index: usize, total: usize) -> Position {
201    if total == 1 {
202        Position::Isol
203    } else if letter_index == 0 {
204        Position::Init
205    } else if letter_index == total - 1 {
206        Position::Fina
207    } else {
208        Position::Medi
209    }
210}
211
212/// Python `_unit_partition`: single deterministic local partition + encode pass. At each index
213/// take the single unit if the table has it, else the longest multi-unit entry. Joiners on either
214/// side shift the positions as if one extra unit padded that side.
215fn unit_partition(
216    table: &NormalizeTable,
217    chain: &[WrittenUnit],
218    joined_left: bool,
219    joined_right: bool,
220) -> Option<String> {
221    let unit_count = chain.len();
222    let pad_left = usize::from(joined_left);
223    let pad_right = usize::from(joined_right);
224    let padded_count = unit_count + pad_left + pad_right;
225    let mut letters: Vec<Encoding> = Vec::new();
226    let mut unit_at: Vec<Option<WrittenUnit>> = Vec::new();
227    let mut index = 0;
228    while index < unit_count {
229        let span = table.max_len.min(unit_count - index);
230        let mut hit: Option<(Encoding, usize)> = None;
231        // 1) single unit (preferred — clean output)
232        let position = slot_position(index + pad_left, 1, padded_count);
233        if let Some(encoding) = table.get(position, &chain[index..index + 1]) {
234            hit = Some((encoding, 1));
235        }
236        // 2) else the longest available multi-unit entry (last resort)
237        if hit.is_none() {
238            for length in (2..=span).rev() {
239                let position = slot_position(index + pad_left, length, padded_count);
240                if let Some(encoding) = table.get(position, &chain[index..index + length]) {
241                    hit = Some((encoding, length));
242                    break;
243                }
244            }
245        }
246        let (encoding, length) = hit?;
247        letters.push(encoding);
248        unit_at.push((length == 1).then_some(chain[index]));
249        index += length;
250    }
251    apply_velar_fem(table, &mut letters, &unit_at, pad_left, pad_right);
252    let mut text = String::new();
253    for (cp, fvs) in letters {
254        text.push(char::from_u32(cp).expect("table code points are scalar values"));
255        if let Some(fvs) = fvs {
256            text.push(fvs.as_char());
257        }
258    }
259    Some(text)
260}
261
262/// Python `_apply_velar_fem`: switch the vowel forward-coupled to each init/medi `G`/`Gx` velar to
263/// its feminine letter (only masculine a/o/u flip; backward coupling is deliberately skipped for
264/// prefix-stability).
265fn apply_velar_fem(
266    table: &NormalizeTable,
267    letters: &mut [Encoding],
268    unit_at: &[Option<WrittenUnit>],
269    pad_left: usize,
270    pad_right: usize,
271) {
272    // `letters` and `unit_at` are pushed in lockstep by `unit_partition`, so they have equal
273    // length; iterating `unit_at` (a separate slice, so no borrow conflict with `letters`)
274    // yields exactly the indices of `letters`.
275    let total = letters.len();
276    let padded_total = total + pad_left + pad_right;
277    for (letter_index, unit) in unit_at.iter().enumerate() {
278        let Some(unit) = *unit else {
279            continue;
280        };
281        if !table.velar_fem_units.contains(&unit) {
282            continue;
283        }
284        // FORWARD coupling only (init/medi velar → following vowel). Backward coupling (fina
285        // velar → preceding vowel) is deliberately skipped: a fina velar becomes medi when a
286        // suffix is appended, flipping its coupling direction, which would make the shared-prefix
287        // vowel diverge between word B and word A. The FVS-pinned velar renders `G` regardless,
288        // so a masculine preceding vowel still round-trips — that one's prettiness is traded for
289        // prefix-stability. — see shaper.py::_apply_velar_fem
290        let position = letter_position(letter_index + pad_left, padded_total);
291        if !matches!(position, Position::Init | Position::Medi) {
292            continue;
293        }
294        let target_index = letter_index + 1;
295        if target_index >= total {
296            continue;
297        }
298        let Some(target_unit) = unit_at[target_index] else {
299            continue; // multi-unit coupled letter — leave it
300        };
301        let (cp, _) = letters[target_index];
302        if !table.masculine_cps.contains(&cp) {
303            continue; // only flip a currently-masculine vowel
304        }
305        let target_position = letter_position(target_index + pad_left, padded_total);
306        let Some(feminine) = table.get_feminine(target_position, &[target_unit]) else {
307            continue; // no round-trip-safe feminine form → leave masculine
308        };
309        letters[target_index] = feminine;
310    }
311}
312
313impl Shaper {
314    /// The normalize table, or [`Error::NormalizeUnsupported`] for locales without one.
315    pub(crate) fn table(&self) -> Result<&NormalizeTable, Error> {
316        self.normalize.as_ref().ok_or(Error::NormalizeUnsupported {
317            locale: self.locale(),
318        })
319    }
320
321    /// Version of the canonical Unicode selection policy (`"mng-canonical/1"` for MNG; `None`
322    /// for locales without a normalize table). Persist it next to stored normalized keys.
323    pub fn canonical_version(&self) -> Option<&'static str> {
324        self.normalize.as_ref().map(|table| table.canonical_version)
325    }
326
327    /// Python `_canonical_for_shape`: encode a full shape (chains right-to-left so each chain is
328    /// verified with the already-encoded suffix; structural tokens copied verbatim).
329    ///
330    /// Right-to-left with the encoded suffix in hand is necessary because rules interact across
331    /// MVS: a masculine vowel after an MVS can propagate backward through the MVS and mark a g/h
332    /// in the previous chain, changing its rendering between `G` and `H`. Per-chain verification
333    /// with only the adjacent MVS is insufficient. — see shaper.py::_canonical_for_shape
334    ///
335    /// The table is fetched only when a chain has to be encoded (Python calls `_build_unit_enc`
336    /// lazily), so a structural-only shape — e.g. a lone nirugu — is copied through even for
337    /// locales without a normalize table.
338    pub(crate) fn canonical_for_shape(&self, shape: &[WrittenUnit]) -> Result<String, Error> {
339        let parts = split_parts(shape);
340        // `suffix_text` accumulates the whole result: each part is prepended as it is encoded, so
341        // after the last (leftmost) part it *is* the canonical text. (Python keeps a parallel
342        // `encoded` list and joins it at the end — this deviates from that structure on purpose.)
343        let mut suffix_text = String::new();
344        let mut suffix_target: Vec<WrittenUnit> = Vec::new();
345        for index in (0..parts.len()).rev() {
346            match &parts[index] {
347                Part::Structural(unit) => {
348                    let text = structural_char(*unit)
349                        .expect("structural token")
350                        .to_string();
351                    suffix_text.insert_str(0, &text);
352                    suffix_target.insert(0, *unit);
353                }
354                Part::Chain(body) => {
355                    let table = self.table()?;
356                    // Context = the full run of structural tokens right before this chain, not
357                    // just the adjacent one: an MVS behind a nirugu still matters, because
358                    // chachlag looks through nirugu. — see shaper.py::_canonical_for_shape
359                    let mut prefix_tokens: Vec<WrittenUnit> = Vec::new();
360                    let mut scan = index;
361                    while scan > 0 {
362                        scan -= 1;
363                        match &parts[scan] {
364                            Part::Structural(unit) => prefix_tokens.insert(0, *unit),
365                            Part::Chain(_) => break,
366                        }
367                    }
368                    let mut chain_canonical: Option<String> = None;
369                    // A chain directly after MVS is a suffix particle: encode it STANDALONE (drop
370                    // the MVS, normalize, re-attach). Exception: chachlag `Aa` is bare `a`.
371                    if prefix_tokens.last() == Some(&WrittenUnit::Mvs) {
372                        let candidate = if body.as_slice() == [WrittenUnit::Aa] {
373                            String::from('\u{1820}')
374                        } else {
375                            self.encode_chain_canonical(table, body, &[], "", &[])?
376                        };
377                        if !candidate.is_empty() {
378                            let prefix_text = structural_text(&prefix_tokens);
379                            let mut want = prefix_tokens.clone();
380                            want.extend_from_slice(body);
381                            want.extend_from_slice(&suffix_target);
382                            if self.shape(&format!("{prefix_text}{candidate}{suffix_text}"))?
383                                == want
384                            {
385                                chain_canonical = Some(candidate);
386                            }
387                        }
388                    }
389                    let chain_canonical = match chain_canonical {
390                        Some(text) => text,
391                        None => self.encode_chain_canonical(
392                            table,
393                            body,
394                            &prefix_tokens,
395                            &suffix_text,
396                            &suffix_target,
397                        )?,
398                    };
399                    suffix_text.insert_str(0, &chain_canonical);
400                    let mut target = body.clone();
401                    target.extend_from_slice(&suffix_target);
402                    suffix_target = target;
403                }
404            }
405        }
406        Ok(suffix_text)
407    }
408
409    /// Python `_encode_chain_canonical` / `_compute_chain_canonical`: the table encoding of one
410    /// chain in its structural context, or `""` on a genuine table gap.
411    fn encode_chain_canonical(
412        &self,
413        table: &NormalizeTable,
414        chain: &[WrittenUnit],
415        prefix_tokens: &[WrittenUnit],
416        suffix_text: &str,
417        suffix_target: &[WrittenUnit],
418    ) -> Result<String, Error> {
419        Ok(self
420            .unit_encode_chain(table, chain, prefix_tokens, suffix_text, suffix_target)?
421            .unwrap_or_default())
422    }
423
424    /// Python `_unit_encode_chain`: partition + encode, then verify in FULL context (the
425    /// structural prefix run and the already-encoded following chains).
426    fn unit_encode_chain(
427        &self,
428        table: &NormalizeTable,
429        chain: &[WrittenUnit],
430        prefix_tokens: &[WrittenUnit],
431        suffix_text: &str,
432        suffix_target: &[WrittenUnit],
433    ) -> Result<Option<String>, Error> {
434        let joined_left = prefix_tokens.last().is_some_and(|unit| is_joiner(*unit));
435        let joined_right = suffix_target.first().is_some_and(|unit| is_joiner(*unit));
436        let Some(text) = unit_partition(table, chain, joined_left, joined_right) else {
437            return Ok(None);
438        };
439        let prefix_text = structural_text(prefix_tokens);
440        // `verify_target` MUST include `suffix_target`: without it the non-last chains of a
441        // multi-chain word never verify, and every one of them falls back.
442        // — see shaper.py::_unit_encode_chain
443        let mut verify_target = prefix_tokens.to_vec();
444        verify_target.extend_from_slice(chain);
445        verify_target.extend_from_slice(suffix_target);
446        if self.shape(&format!("{prefix_text}{text}{suffix_text}"))? == verify_target {
447            Ok(Some(text))
448        } else {
449            Ok(None)
450        }
451    }
452
453    fn normalize_impl(&self, text: &str, strict: bool) -> Result<String, Error> {
454        if text.is_empty() {
455            return Ok(String::new());
456        }
457        let target = self.shape(text)?;
458        if target.is_empty() {
459            // Only FVS marks, no letter — canonical is the empty string. (Joiners are *not*
460            // dropped here: a lone nirugu/ZWJ shapes to a structural token and round-trips.)
461            return Ok(String::new());
462        }
463        let canonical = self.canonical_for_shape(&target)?;
464        if canonical.is_empty() || self.shape(&canonical)? != target {
465            if strict {
466                return Err(Error::NormalizationFallback {
467                    text: text.to_owned(),
468                    written_units: target,
469                });
470            }
471            return Ok(text.to_owned());
472        }
473        Ok(canonical)
474    }
475
476    /// Canonical, FVS-pinned encoding of one Mongolian word: within the normalize table's domain,
477    /// `shape(x) == shape(y)` ⟹ `normalize(x) == normalize(y)`, and
478    /// `shape(normalize(x)) == shape(x)`.
479    ///
480    /// Strict (the Python default): an uncovered shape is [`Error::NormalizationFallback`].
481    /// Errors with [`Error::NonMongolianChar`] on mixed-script input — see
482    /// [`Shaper::normalize_text`].
483    pub fn normalize(&self, text: &str) -> Result<String, Error> {
484        self.normalize_impl(text, true)
485    }
486
487    /// Like [`Shaper::normalize`], but an uncovered shape returns the input unchanged
488    /// (Python `strict=False`).
489    pub fn normalize_allow_fallback(&self, text: &str) -> Result<String, Error> {
490        self.normalize_impl(text, false)
491    }
492
493    fn normalize_text_impl(&self, text: &str, strict: bool) -> Result<String, Error> {
494        if text.is_empty() {
495            return Ok(String::new());
496        }
497        let mut out = String::with_capacity(text.len());
498        let mut run = String::new();
499        let mut run_is_mongolian: Option<bool> = None;
500        for ch in text.chars() {
501            let is_mongolian = is_mongolian_word_char(ch);
502            match run_is_mongolian {
503                Some(current) if current != is_mongolian => {
504                    self.flush_run(&mut out, &run, current, strict)?;
505                    run.clear();
506                    run_is_mongolian = Some(is_mongolian);
507                }
508                Some(_) => {}
509                None => run_is_mongolian = Some(is_mongolian),
510            }
511            run.push(ch);
512        }
513        if let Some(current) = run_is_mongolian {
514            self.flush_run(&mut out, &run, current, strict)?;
515        }
516        Ok(out)
517    }
518
519    fn flush_run(
520        &self,
521        out: &mut String,
522        run: &str,
523        is_mongolian: bool,
524        strict: bool,
525    ) -> Result<(), Error> {
526        if is_mongolian {
527            out.push_str(&self.normalize_impl(run, strict)?);
528        } else {
529            out.push_str(run);
530        }
531        Ok(())
532    }
533
534    /// Normalize free-form text: every Mongolian word run is normalized independently, everything
535    /// else (spaces, punctuation, Latin, …) is copied verbatim. Strict like [`Shaper::normalize`].
536    pub fn normalize_text(&self, text: &str) -> Result<String, Error> {
537        self.normalize_text_impl(text, true)
538    }
539
540    /// Like [`Shaper::normalize_text`], but an uncovered word is preserved unchanged.
541    pub fn normalize_text_allow_fallback(&self, text: &str) -> Result<String, Error> {
542        self.normalize_text_impl(text, false)
543    }
544}
545
546#[cfg(test)]
547mod tests {
548    use super::*;
549    use crate::Locale;
550
551    const SAIN: &str = "\u{1830}\u{1820}\u{1822}\u{1828}";
552
553    #[test]
554    fn strict_mode_raises_when_canonicalization_falls_back() {
555        let shaper = Shaper::with_empty_normalize_table(Locale::Mng);
556        let error = shaper.normalize(SAIN).unwrap_err();
557        assert_eq!(
558            error,
559            Error::NormalizationFallback {
560                text: SAIN.to_owned(),
561                written_units: vec![
562                    WrittenUnit::S,
563                    WrittenUnit::A,
564                    WrittenUnit::I,
565                    WrittenUnit::I,
566                    WrittenUnit::A
567                ],
568            }
569        );
570        assert_eq!(
571            error.to_string(),
572            "normalization fallback: no canonical encoding for written units S+A+I+I+A"
573        );
574    }
575
576    #[test]
577    fn allow_fallback_preserves_input_when_canonicalization_falls_back() {
578        let shaper = Shaper::with_empty_normalize_table(Locale::Mng);
579        assert_eq!(shaper.normalize_allow_fallback(SAIN).unwrap(), SAIN);
580    }
581
582    #[test]
583    fn strict_mode_reports_a_fallback_inside_mixed_text() {
584        let shaper = Shaper::with_empty_normalize_table(Locale::Mng);
585        let text = format!("Hello {SAIN} world");
586        assert!(matches!(
587            shaper.normalize_text(&text),
588            Err(Error::NormalizationFallback { .. })
589        ));
590    }
591
592    #[test]
593    fn allow_fallback_preserves_a_fallback_inside_mixed_text() {
594        let shaper = Shaper::with_empty_normalize_table(Locale::Mng);
595        let text = format!("Hello {SAIN} world");
596        assert_eq!(shaper.normalize_text_allow_fallback(&text).unwrap(), text);
597    }
598
599    #[test]
600    fn locales_without_a_table_reject_normalization_of_letters() {
601        let shaper = Shaper::new(Locale::Tod);
602        assert_eq!(shaper.canonical_version(), None);
603        assert_eq!(shaper.normalize("").unwrap(), "");
604        assert_eq!(shaper.normalize("\u{180B}").unwrap(), ""); // FVS only: empty shape short-circuits
605        assert_eq!(shaper.normalize("\u{180A}").unwrap(), "\u{180A}"); // structural-only shape needs no table (Python parity)
606        assert_eq!(
607            shaper.normalize("\u{1820}"),
608            Err(Error::NormalizeUnsupported {
609                locale: Locale::Tod
610            })
611        );
612        assert_eq!(
613            shaper.normalize_written_units(&[WrittenUnit::Mvs]),
614            Err(Error::NormalizeUnsupported {
615                locale: Locale::Tod
616            })
617        );
618        assert_eq!(
619            Shaper::new(Locale::Mng).canonical_version(),
620            Some("mng-canonical/1")
621        );
622    }
623
624    #[test]
625    fn positions_of_partition_slots_and_letters() {
626        assert_eq!(slot_position(0, 1, 1), Position::Isol);
627        assert_eq!(slot_position(0, 2, 2), Position::Isol);
628        assert_eq!(slot_position(0, 1, 3), Position::Init);
629        assert_eq!(slot_position(1, 1, 3), Position::Medi);
630        assert_eq!(slot_position(1, 2, 3), Position::Fina);
631        assert_eq!(letter_position(0, 1), Position::Isol);
632        assert_eq!(letter_position(0, 2), Position::Init);
633        assert_eq!(letter_position(1, 2), Position::Fina);
634        assert_eq!(letter_position(1, 3), Position::Medi);
635        assert_eq!(
636            UnitKey::new(&[WrittenUnit::A]),
637            UnitKey::new(&[WrittenUnit::A])
638        );
639        assert_ne!(
640            UnitKey::new(&[WrittenUnit::A]),
641            UnitKey::new(&[WrittenUnit::A, WrittenUnit::A])
642        );
643    }
644}