Skip to main content

talk/string/
normalize.rs

1//! Fold the many ways IPA gets written down into the one this library reads.
2//!
3//! Mirrors `code/string/normalize.ts` in the TypeScript port.
4
5use std::collections::HashMap;
6use std::sync::OnceLock;
7
8use unicode_normalization::UnicodeNormalization;
9
10/// Characters that mean the same thing as a character the parser knows, and
11/// are replaced one for one.
12///
13/// WITHDRAWN SYMBOLS. `ʆ` and `ʓ` carried the palatal hook, which the IPA
14/// withdrew in 1989 in favour of a following superscript j.
15///
16/// SANCTIONED ABBREVIATIONS. `ɚ` and `ɝ` are the IPA's own shorthand for a
17/// rhotic schwa.
18///
19/// LOOKALIKES. A transcription typed on a normal keyboard reaches for ASCII
20/// `g`, `:` and `'` where the IPA wants `ɡ`, `ː` and `ʼ`.
21fn replace() -> &'static HashMap<char, &'static str> {
22  static CELL: OnceLock<HashMap<char, &'static str>> = OnceLock::new();
23
24  CELL.get_or_init(|| {
25    [
26      // Withdrawn 1989, palatal hook.
27      ('ʆ', "ʃʲ"),
28      ('ʓ', "ʒʲ"),
29      // IPA-sanctioned abbreviations for rhotic vowels.
30      ('ɚ', "ə˞"),
31      ('ɝ', "ɜ˞"),
32      // ASCII lookalikes.
33      ('g', "ɡ"),
34      (':', "ː"),
35      ('\'', "ʼ"),
36      ('?', "ʔ"),
37      (',', "\u{0329}"),
38      // Sinological letters for the alveolo-palatal series, written as the
39      // palatal plus the advancing mark. `ȵ` is nearer `ɲ̟` than `ɲ`, and
40      // the advancement now has a modifier to land on, so it is kept.
41      ('ȵ', "ɲ\u{031f}"),
42      ('ȶ', "c\u{031f}"),
43      ('ȡ', "ɟ\u{031f}"),
44      ('ȴ', "ʎ\u{031f}"),
45      // GREEK LETTERS THAT ARE NOT IPA.
46      //
47      // Three Greek letters ARE official IPA and are deliberately absent:
48      // β the voiced bilabial fricative, θ the voiceless dental, and χ the
49      // voiceless uvular. Together they account for 166,790 uses across
50      // 171 languages, so folding them would rewrite correct
51      // transcription.
52      ('γ', "ɣ"), // U+03B3 -> U+0263, 26 uses
53      ('δ', "ð"), // U+03B4 -> U+00F0, 77 uses
54      ('ϕ', "ɸ"), // U+03D5 -> U+0278
55      ('ε', "ɛ"), // U+03B5 -> U+025B
56      ('ι', "ɪ"), // U+03B9 -> U+026A
57      ('υ', "ʊ"), // U+03C5 -> U+028A
58      ('α', "ɑ"), // U+03B1 -> U+0251
59      // φ is the ambiguous one and IS folded: every observed use is the
60      // bilabial fricative. λ is deliberately NOT folded, because
61      // Americanist notation uses it for `tɬ` and other sources for `ʎ`.
62      ('φ', "ɸ"),
63      // CYRILLIC LOOKALIKES. `ӕ` renders identically to the Latin `æ` IPA
64      // wants. 83 uses, all in one language.
65      ('ӕ', "æ"), // U+04D5 -> U+00E6
66      ('Ӕ', "Æ"),
67      // TYPOGRAPHIC QUOTES. A source that ran through a word processor has
68      // the curly apostrophe where IPA wants the ejective mark. 570 uses.
69      ('\u{2019}', "ʼ"),
70      ('\u{2018}', "ʼ"),
71      ('`', "ʼ"),
72      ('´', "ʼ"),
73      ('"', "ˈ"), // ASCII double quote -> primary stress
74      // THE TIE, ABOVE OR BELOW. U+035C is the same tie as U+0361,
75      // written underneath when a descender leaves no room above. One
76      // thing, two spellings, which is what this table is for. The tie
77      // itself is kept: `t͡ʃ` is one affricate and `tʃ` may be two
78      // segments meeting.
79      ('\u{035c}', "\u{0361}"),
80      // THE RETROFLEX CLICK, IN THE NOTATION THAT PREDATES ITS LETTER.
81      // Khoisanist work wrote it `‼` because no letter existed; Unicode
82      // encoded U+1DF0A in 2021 and the catalog spells it that way. phoible
83      // still writes the old form, in !Xun alone, so its 12 spellings
84      // matched nothing. One sound, two spellings.
85      ('\u{203c}', "\u{1df0a}"),
86      // SYMBOLS THE IPA WITHDREW, AND THE SPELLINGS THAT REPLACED THEM.
87      //
88      // Each was current notation once and every one is EXACTLY equal to what
89      // it maps to, so folding gives nothing up. Until now all eighteen parsed
90      // as `unknown`: the parser did not reject them, it simply had no phone to
91      // offer, so a reader saw a gap where a sound was written.
92      //
93      // The affricate ligatures expand to the two letters and the tie. That is
94      // only correct because the tie now SURVIVES normalizing: expanding `ʤ` to
95      // a bare `dʒ` would turn one segment into two, which is the distinction
96      // the tie exists to make.
97      //
98      // The withdrawn voiceless implosives take the voiced letter plus the
99      // voiceless ring, which is what the IPA recommended when it retired them.
100      ('\u{02a3}', "\u{0064}\u{0361}\u{007a}"), // ligature, withdrawn 1989
101      ('\u{02a4}', "\u{0064}\u{0361}\u{0292}"), // ligature, withdrawn 1989
102      ('\u{02a5}', "\u{0064}\u{0361}\u{0291}"), // ligature, withdrawn 1989
103      ('\u{02a6}', "\u{0074}\u{0361}\u{0073}"), // ligature, withdrawn 1989
104      ('\u{02a7}', "\u{0074}\u{0361}\u{0283}"), // ligature, withdrawn 1989. 4 uses
105      ('\u{02a8}', "\u{0074}\u{0361}\u{0255}"), // ligature, withdrawn 1989
106      ('\u{0287}', "\u{01c0}"), // old dental click, withdrawn 1989
107      ('\u{0297}', "\u{01c3}"), // old alveolar click, withdrawn 1989
108      ('\u{0296}', "\u{01c1}"), // old lateral click, withdrawn 1989
109      ('\u{0269}', "\u{026a}"), // old iota, withdrawn 1989. 76 uses in 5 languages
110      ('\u{0277}', "\u{028a}"), // old closed omega, withdrawn 1989. 74 uses in 7 languages
111      ('\u{029a}', "\u{025e}"), // closed open-e, a variant spelling
112      ('\u{027c}', "\u{0072}\u{031d}"), // r with long leg, withdrawn 1989
113      ('\u{01a5}', "\u{0253}\u{0325}"), // hooktop p, withdrawn 1993
114      ('\u{01ad}', "\u{0257}\u{0325}"), // hooktop t, withdrawn 1993
115      ('\u{0188}', "\u{0284}\u{0325}"), // hooktop c, withdrawn 1993
116      ('\u{0199}', "\u{0260}\u{0325}"), // hooktop k, withdrawn 1993
117      ('\u{02a0}', "\u{029b}\u{0325}"), // hooktop q, withdrawn 1993
118    ]
119    .into_iter()
120    .collect()
121  })
122}
123
124/// Chao tone letters, extra-low to extra-high.
125const TONE_DIGIT: [(char, char); 5] =
126  [('1', '˩'), ('2', '˨'), ('3', '˧'), ('4', '˦'), ('5', '˥')];
127
128/// Superscript digits, which several sources use for tone instead of Chao
129/// letters. `ma³³` and `ma˧˧` are the same word.
130const SUPERSCRIPT_DIGIT: [(char, char); 5] =
131  [('¹', '1'), ('²', '2'), ('³', '3'), ('⁴', '4'), ('⁵', '5')];
132
133fn tone_for(digit: char) -> Option<char> {
134  TONE_DIGIT
135    .iter()
136    .find(|(from, _)| *from == digit)
137    .map(|(_, to)| *to)
138}
139
140/// The ring above is the same feature as the ring below, voiceless. IPA uses
141/// the one above when a descender leaves no room underneath.
142const RING_ABOVE: char = '\u{030a}';
143const RING_BELOW: char = '\u{0325}';
144
145/// NOTHING IS DROPPED ANY MORE.
146///
147/// This held `\u{0361}`, `\u{035c}`, `.` and `\u{203f}`, and deleting them was a
148/// category error. Normalizing folds the several ways of writing ONE thing
149/// into one of them. It does not decide a thing is unimportant. The syllable
150/// boundary, the link between two words spoken as one, and the tie binding
151/// two letters into a single segment each say something the source meant,
152/// and `t͡ʃ` is one affricate where `tʃ` may be a stop meeting a fricative.
153///
154/// They pass through now and the parser reports them as symbols. The one
155/// genuine fold among the four is in `replace`: the tie below is the tie
156/// above, chosen by whether a descender leaves room.
157
158/// Fine phonetic detail used to be stripped here so the segment underneath
159/// still resolved. It no longer is, and there is no option to bring it back.
160///
161/// Every mark that was dropped now has an entry in `modifiers.json`, so `b̤`
162/// reads as `b` plus breathy and a caller sees the feature instead of losing
163/// it. Breathy voice is contrastive across South Asia and creaky voice across
164/// Mesoamerica, so the old default merged phonemes that contrast.
165///
166/// A caller wanting `b̤` to MATCH `b` wants the tone encoding, which is coarse
167/// by design and gives both the same code.
168
169/// Characters carrying nothing recoverable, removed before anything else.
170///
171/// PRIVATE USE AREA. A font assigns these whatever glyph it likes and no two
172/// agree, so a source using one meant a symbol its own font drew and nothing
173/// downstream can know which. 98 such characters were observed in five
174/// languages.
175fn is_private_use(character: char) -> bool {
176  matches!(
177    character,
178    '\u{e000}'..='\u{f8ff}'
179      | '\u{f0000}'..='\u{ffffd}'
180      | '\u{100000}'..='\u{10fffd}'
181  )
182}
183
184/// Strip the delimiters WRAPPING a whole string, which say what kind of
185/// transcription it is rather than form part of it. 6,903 uses across four
186/// languages.
187///
188/// Only when they wrap. A slash INSIDE a string separates two readings, and
189/// cutting it would join them into a word that is neither.
190fn unwrap_delimiters(text: &str) -> &str {
191  let trimmed = text.trim();
192  let mut characters = trimmed.chars();
193
194  match (characters.next(), characters.next_back()) {
195    (Some('/'), Some('/')) | (Some('['), Some(']'))
196      if trimmed.chars().count() > 2 =>
197    {
198      characters.as_str()
199    }
200    // UNTRIMMED WHEN NOTHING WRAPPED. The trim above is how the delimiters
201    // are FOUND, not something to keep: returning it stripped a lone space
202    // to nothing, and a space is a real symbol that separates two words.
203    // The reference build trims only to test for the delimiters and hands
204    // back the string it was given.
205    _ => text,
206  }
207}
208
209/// The canonical order for the marks that follow a base, DERIVED FROM PHOIBLE
210/// rather than chosen.
211///
212/// Combining marks sharing a Unicode combining class keep their input order
213/// under NFD, so `n̪̥` and `n̥̪` stay two strings for one sound, and the parser
214/// matches greedily so the two read as different segments. Sorting fixes it.
215///
216/// WHICH ORDER HAS NO STANDARD. Unicode declines to decide: dental U+032A and
217/// voiceless U+0325 are both combining class 220, and NFD only reorders across
218/// different classes. The IPA Handbook says what each diacritic means and never
219/// what sequence to write two of them in.
220///
221/// So it is recovered from the corpus. `pnpm derive:mark-order` in mesh walks
222/// phoible's 3,142 distinct phoneme spellings, takes one vote per stacked pair
223/// and topologically sorts them. phoible proved completely self-consistent:
224/// 135 pairs decided, none written both ways, no cycles.
225///
226/// THE EVIDENCE IS UNEVEN. `ː` rests on 367 stack observations and `ʰ` on 135,
227/// so those positions are solid. U+031A was stacked ONCE and its place is the
228/// tie-break rather than a fact. Re-run the derivation instead of editing here.
229const MARK_ORDER: &[char] = &[
230  '\u{0327}', // cedilla
231  '\u{032a}', // dental
232  '\u{033c}', // linguolabial
233  '\u{033a}', // apical
234  '\u{033b}', // laminal
235  '\u{031f}', // advanced
236  '\u{0320}', // retracted
237  '\u{031d}', // raised
238  '\u{031e}', // lowered
239  '\u{033d}', // mid-centralized
240  '\u{0318}', // advanced tongue root
241  '\u{0319}', // retracted tongue root
242  '\u{1da3}', // labial-palatalized
243  '\u{0339}', // more rounded
244  '\u{031c}', // less rounded
245  '\u{032c}', // voiced
246  '\u{0324}', // breathy
247  '\u{02b1}', // murmured
248  '\u{0330}', // creaky
249  '\u{1d31}', // sphincteric
250  '\u{0348}', // fortis
251  '\u{0349}', // lenis
252  '\u{0353}', // frictionalized
253  '\u{0325}', // voiceless
254  '\u{0347}', // non-sibilant
255  '\u{207f}', // nasal release
256  '\u{02e1}', // lateral release
257  '\u{031a}', // no audible release
258  '\u{0329}', // syllabic
259  '\u{02b7}', // labialized
260  '\u{02b2}', // palatalized
261  '\u{02c0}', // glottalized
262  '\u{032f}', // non-syllabic
263  '\u{02d1}', // half-long
264  '\u{0306}', // extra-short
265  '\u{0303}', // nasalized
266  '\u{0308}', // centralized
267  '\u{02e0}', // velarized
268  '\u{02de}', // rhotic
269  '\u{02e4}', // pharyngealized
270  '\u{02e5}', // tone extra-high
271  '\u{02e6}', // tone high
272  '\u{02e7}', // tone mid
273  '\u{02e8}', // tone low
274  '\u{02e9}', // tone extra-low
275  '\u{2193}', // downstep
276  '\u{02b0}', // aspirated
277  '\u{02bc}', // ejective
278  '\u{02d0}', // long
279];
280
281/// The tone marks, which all share ONE rank.
282///
283/// A CONTOUR IS A SEQUENCE, NOT A SET. `˩˩˦` is a rise and `˦˩˩` is a fall,
284/// and the two are spelled with the same three letters in a different order.
285/// Ranking the five Chao letters individually meant every run of them was
286/// sorted into descending pitch, so every rising tone became a falling one
287/// and the original was not recoverable.
288///
289/// MEASURED, in a corpus normalized by this function: 599,336 of
290/// Vietnamese's 599,339 tone-letter runs came out descending, and all 23,292
291/// of Thai's. Vietnamese `mả` is a dipping-rise and was stored `maː˦˨˩`.
292/// Neither language has a tone that only falls.
293///
294/// REMOVING THEM FROM `MARK_ORDER` DOES NOT FIX IT. `rank_of` would fall
295/// through to the codepoint, and U+02E5 through U+02E9 run high pitch to
296/// low, so the sort reaches the same descending order by another route.
297///
298/// An equal rank is the fix, because `sort_by_key` is stable: a run of tone
299/// marks keeps the order it arrived in, while still sorting as a group
300/// against every other kind of mark.
301///
302/// THE COMBINING TONE ACCENTS ARE HERE FOR THE SAME REASON. Acute, grave,
303/// circumflex, caron, macron and the doubled pair are how many sources write
304/// tone, two of them stacked make a contour, and none appears in
305/// `MARK_ORDER`, so they were ordered by codepoint against each other.
306const TONE: &[char] = &[
307  '˥',
308  '˦',
309  '˧',
310  '˨',
311  '˩',
312  '\u{2193}', // downstep, which lowers what follows and so is positional too
313  '\u{0300}', // grave, low
314  '\u{0301}', // acute, high
315  '\u{0302}', // circumflex, falling
316  '\u{0304}', // macron, mid
317  '\u{030b}', // double acute, extra high
318  '\u{030c}', // caron, rising
319  '\u{030f}', // double grave, extra low
320  '\u{0311}', // inverted breve, peaking
321];
322
323fn rank_of(character: char) -> usize {
324  if TONE.contains(&character) {
325    // Where the tone group sorts: exactly where the Chao letters already
326    // did, which is last among the suprasegmentals.
327    return MARK_ORDER
328      .iter()
329      .position(|mark| *mark == '˥')
330      .expect("MARK_ORDER holds the extra-high tone letter");
331  }
332
333  MARK_ORDER
334    .iter()
335    .position(|mark| *mark == character)
336    .unwrap_or(MARK_ORDER.len() + character as usize)
337}
338
339/// Whether a character attaches to a preceding base rather than standing
340/// alone.
341fn is_affix(character: char) -> bool {
342  use unicode_normalization::char::canonical_combining_class;
343
344  canonical_combining_class(character) != 0
345    // The modifier-letter blocks IPA draws from. Rust has no general
346    // category table, so the ranges are named: spacing modifiers, the two
347    // phonetic-extension blocks (the second holds `ᶣ` at U+1DA3), and the
348    // two strays IPA uses.
349    || matches!(character,
350      '\u{02b0}'..='\u{02ff}'
351        | '\u{1d2c}'..='\u{1d6a}'
352        | '\u{1d78}'
353        | '\u{1d9b}'..='\u{1dbf}'
354        | '\u{2071}'
355        | '\u{207f}')
356}
357
358fn is_base(character: char) -> bool {
359  // A LETTER IS A BASE EVEN WHEN IT IS ALSO AN AFFIX. The reference build
360  // asks only whether the general category starts with `L`, and the modifier
361  // letters are `Lm`, so `ˈ` and `ʰ` are bases there as well as affixes. The
362  // `&& !is_affix` here excluded exactly those, so a run opened by one was
363  // never sorted: `ˈ˷̩` kept its input order while the reference moved the
364  // syllabic mark ahead of the phone, and the mark then read as trailing
365  // rather than leading.
366  //
367  // Ordering the two tests this way is safe because `order_marks` checks
368  // the affix branch FIRST, so a modifier letter with nothing before it
369  // still falls through and opens a run rather than joining one.
370  character.is_alphabetic()
371}
372
373/// Marks whose place in a run says WHEN, not merely what.
374///
375/// Sorting a run is right when the marks describe one moment from several
376/// angles. `n̪̥` and `n̥̪` are one sound written two ways, dental and
377/// voiceless, both true of the whole segment, so an order has to be picked
378/// and either will do.
379///
380/// It is wrong the moment a run describes a sequence. These establish points
381/// in time, and every mark between two of them is timed against them: the
382/// Chao pitch targets in the order the voice reaches them, downstep which
383/// lowers everything after it, and the length marks.
384///
385/// MEASURED. Vietnamese writes its ngã tone `˦ˀ˥`, a rise interrupted by
386/// glottalisation partway up, and sorting produced `ˀ˦˥` in 2,838 rows.
387/// Navajo writes `óː` and sorting pushed the acute past the length mark to
388/// `oː́`, where it attaches to the `ː`, in 3,707 rows across three languages.
389/// The two look unrelated and are the same defect.
390const ANCHOR: &[char] =
391  &['˥', '˦', '˧', '˨', '˩', '\u{2193}', 'ː', 'ˑ', '\u{0306}'];
392
393/// One run, sorted between its anchors and never across them.
394///
395/// The anchors keep their positions and each stretch between two of them is
396/// sorted alone, so `˦ˀ˥` keeps the glottal where the voice makes it and `óː`
397/// keeps the accent on the vowel, while `n̥̪` still canonicalises because
398/// neither of its marks is an anchor.
399fn ordered(run: &[char]) -> Vec<char> {
400  let mut out: Vec<char> = Vec::with_capacity(run.len());
401  let mut piece: Vec<char> = Vec::new();
402
403  for character in run {
404    if ANCHOR.contains(character) {
405      piece.sort_by_key(|mark| rank_of(*mark));
406      out.append(&mut piece);
407      out.push(*character);
408      continue;
409    }
410
411    piece.push(*character);
412  }
413
414  piece.sort_by_key(|mark| rank_of(*mark));
415  out.append(&mut piece);
416
417  out
418}
419
420/// Put the affixes following each base into canonical order.
421///
422/// Only a run that FOLLOWS a base is sorted. A leading modifier is a
423/// different sound rather than a reordering: `ʰk` is pre-aspirated and
424/// `kʰ` post-aspirated, so moving one to the other would change what was
425/// said.
426fn order_marks(text: &str) -> String {
427  let mut out = String::with_capacity(text.len());
428  let mut run: Vec<char> = Vec::new();
429  let mut after_base = false;
430
431  for character in text.chars() {
432    if is_affix(character) && (after_base || !run.is_empty()) {
433      run.push(character);
434      continue;
435    }
436
437    if !run.is_empty() {
438      if after_base {
439        run = ordered(&run);
440      }
441      out.extend(run.drain(..));
442    }
443
444    out.push(character);
445    after_base = is_base(character);
446  }
447
448  if !run.is_empty() {
449    if after_base {
450      run = ordered(&run);
451    }
452    out.extend(run.drain(..));
453  }
454
455  out
456}
457
458/// `bare_digit_tone` reads bare digits as tone, off by default because a bare
459/// digit is ambiguous.
460/// caller can audit what this encoding drops.
461#[derive(Debug, Clone, Copy, Default)]
462pub struct NormalizeIpaOptions {
463  pub bare_digit_tone: bool,
464}
465
466/// Normalize an IPA string into the form the parser reads.
467///
468/// Idempotent, and returns NFD because every combining mark in the tries is
469/// stored decomposed.
470pub fn normalize_ipa_with(text: &str, options: NormalizeIpaOptions) -> String {
471  // Before anything else: drop the private-use characters no reader can
472  // resolve, and unwrap the delimiters that say what kind of transcription
473  // this is. Both are about the string rather than in it.
474  let stripped: String =
475    text.chars().filter(|one| !is_private_use(*one)).collect();
476  let bare = unwrap_delimiters(&stripped);
477
478  // Two passes, because a replacement can itself contain something the second
479  // pass acts on: `ȵ` expands to `ɲ̟`, and the advancing mark then has to
480  // decompose and sort with the rest of its run.
481  let mut source = String::with_capacity(bare.len());
482
483  for character in bare.chars() {
484    if let Some((_, digit)) =
485      SUPERSCRIPT_DIGIT.iter().find(|(from, _)| *from == character)
486    {
487      if let Some(tone) = tone_for(*digit) {
488        source.push(tone);
489        continue;
490      }
491    }
492
493    match replace().get(&character) {
494      Some(replacement) => source.push_str(replacement),
495      None => source.push(character),
496    }
497  }
498
499  let mut out = String::with_capacity(source.len());
500
501  for character in source.nfd() {
502    if character == RING_ABOVE {
503      out.push(RING_BELOW);
504      continue;
505    }
506
507    if options.bare_digit_tone {
508      if let Some(tone) = tone_for(character) {
509        out.push(tone);
510        continue;
511      }
512    }
513
514    out.push(character);
515  }
516
517  order_marks(&out)
518}
519
520/// Normalize with the default options.
521pub fn normalize_ipa(text: &str) -> String {
522  normalize_ipa_with(text, NormalizeIpaOptions::default())
523}