talk/string/normalize.rs
1//! Fold the many ways IPA gets written down into the one this library reads.
2//!
3//! Mirrors `code/string/normalize.ts` in the TypeScript port.
4
5use std::collections::HashMap;
6use std::sync::OnceLock;
7
8use unicode_normalization::UnicodeNormalization;
9
10/// Characters that mean the same thing as a character the parser knows, and
11/// are replaced one for one.
12///
13/// WITHDRAWN SYMBOLS. `ʆ` and `ʓ` carried the palatal hook, which the IPA
14/// withdrew in 1989 in favour of a following superscript j.
15///
16/// SANCTIONED ABBREVIATIONS. `ɚ` and `ɝ` are the IPA's own shorthand for a
17/// rhotic schwa.
18///
19/// LOOKALIKES. A transcription typed on a normal keyboard reaches for ASCII
20/// `g`, `:` and `'` where the IPA wants `ɡ`, `ː` and `ʼ`.
21fn replace() -> &'static HashMap<char, &'static str> {
22 static CELL: OnceLock<HashMap<char, &'static str>> = OnceLock::new();
23
24 CELL.get_or_init(|| {
25 [
26 // Withdrawn 1989, palatal hook.
27 ('ʆ', "ʃʲ"),
28 ('ʓ', "ʒʲ"),
29 // IPA-sanctioned abbreviations for rhotic vowels.
30 ('ɚ', "ə˞"),
31 ('ɝ', "ɜ˞"),
32 // ASCII lookalikes.
33 ('g', "ɡ"),
34 (':', "ː"),
35 ('\'', "ʼ"),
36 ('?', "ʔ"),
37 (',', "\u{0329}"),
38 // Sinological letters for the alveolo-palatal series, written as the
39 // palatal plus the advancing mark. `ȵ` is nearer `ɲ̟` than `ɲ`, and
40 // the advancement now has a modifier to land on, so it is kept.
41 ('ȵ', "ɲ\u{031f}"),
42 ('ȶ', "c\u{031f}"),
43 ('ȡ', "ɟ\u{031f}"),
44 ('ȴ', "ʎ\u{031f}"),
45 // GREEK LETTERS THAT ARE NOT IPA.
46 //
47 // Three Greek letters ARE official IPA and are deliberately absent:
48 // β the voiced bilabial fricative, θ the voiceless dental, and χ the
49 // voiceless uvular. Together they account for 166,790 uses across
50 // 171 languages, so folding them would rewrite correct
51 // transcription.
52 ('γ', "ɣ"), // U+03B3 -> U+0263, 26 uses
53 ('δ', "ð"), // U+03B4 -> U+00F0, 77 uses
54 ('ϕ', "ɸ"), // U+03D5 -> U+0278
55 ('ε', "ɛ"), // U+03B5 -> U+025B
56 ('ι', "ɪ"), // U+03B9 -> U+026A
57 ('υ', "ʊ"), // U+03C5 -> U+028A
58 ('α', "ɑ"), // U+03B1 -> U+0251
59 // φ is the ambiguous one and IS folded: every observed use is the
60 // bilabial fricative. λ is deliberately NOT folded, because
61 // Americanist notation uses it for `tɬ` and other sources for `ʎ`.
62 ('φ', "ɸ"),
63 // CYRILLIC LOOKALIKES. `ӕ` renders identically to the Latin `æ` IPA
64 // wants. 83 uses, all in one language.
65 ('ӕ', "æ"), // U+04D5 -> U+00E6
66 ('Ӕ', "Æ"),
67 // TYPOGRAPHIC QUOTES. A source that ran through a word processor has
68 // the curly apostrophe where IPA wants the ejective mark. 570 uses.
69 ('\u{2019}', "ʼ"),
70 ('\u{2018}', "ʼ"),
71 ('`', "ʼ"),
72 ('´', "ʼ"),
73 ('"', "ˈ"), // ASCII double quote -> primary stress
74 // THE TIE, ABOVE OR BELOW. U+035C is the same tie as U+0361,
75 // written underneath when a descender leaves no room above. One
76 // thing, two spellings, which is what this table is for. The tie
77 // itself is kept: `t͡ʃ` is one affricate and `tʃ` may be two
78 // segments meeting.
79 ('\u{035c}', "\u{0361}"),
80 // THE RETROFLEX CLICK, IN THE NOTATION THAT PREDATES ITS LETTER.
81 // Khoisanist work wrote it `‼` because no letter existed; Unicode
82 // encoded U+1DF0A in 2021 and the catalog spells it that way. phoible
83 // still writes the old form, in !Xun alone, so its 12 spellings
84 // matched nothing. One sound, two spellings.
85 ('\u{203c}', "\u{1df0a}"),
86 // SYMBOLS THE IPA WITHDREW, AND THE SPELLINGS THAT REPLACED THEM.
87 //
88 // Each was current notation once and every one is EXACTLY equal to what
89 // it maps to, so folding gives nothing up. Until now all eighteen parsed
90 // as `unknown`: the parser did not reject them, it simply had no phone to
91 // offer, so a reader saw a gap where a sound was written.
92 //
93 // The affricate ligatures expand to the two letters and the tie. That is
94 // only correct because the tie now SURVIVES normalizing: expanding `ʤ` to
95 // a bare `dʒ` would turn one segment into two, which is the distinction
96 // the tie exists to make.
97 //
98 // The withdrawn voiceless implosives take the voiced letter plus the
99 // voiceless ring, which is what the IPA recommended when it retired them.
100 ('\u{02a3}', "\u{0064}\u{0361}\u{007a}"), // ligature, withdrawn 1989
101 ('\u{02a4}', "\u{0064}\u{0361}\u{0292}"), // ligature, withdrawn 1989
102 ('\u{02a5}', "\u{0064}\u{0361}\u{0291}"), // ligature, withdrawn 1989
103 ('\u{02a6}', "\u{0074}\u{0361}\u{0073}"), // ligature, withdrawn 1989
104 ('\u{02a7}', "\u{0074}\u{0361}\u{0283}"), // ligature, withdrawn 1989. 4 uses
105 ('\u{02a8}', "\u{0074}\u{0361}\u{0255}"), // ligature, withdrawn 1989
106 ('\u{0287}', "\u{01c0}"), // old dental click, withdrawn 1989
107 ('\u{0297}', "\u{01c3}"), // old alveolar click, withdrawn 1989
108 ('\u{0296}', "\u{01c1}"), // old lateral click, withdrawn 1989
109 ('\u{0269}', "\u{026a}"), // old iota, withdrawn 1989. 76 uses in 5 languages
110 ('\u{0277}', "\u{028a}"), // old closed omega, withdrawn 1989. 74 uses in 7 languages
111 ('\u{029a}', "\u{025e}"), // closed open-e, a variant spelling
112 ('\u{027c}', "\u{0072}\u{031d}"), // r with long leg, withdrawn 1989
113 ('\u{01a5}', "\u{0253}\u{0325}"), // hooktop p, withdrawn 1993
114 ('\u{01ad}', "\u{0257}\u{0325}"), // hooktop t, withdrawn 1993
115 ('\u{0188}', "\u{0284}\u{0325}"), // hooktop c, withdrawn 1993
116 ('\u{0199}', "\u{0260}\u{0325}"), // hooktop k, withdrawn 1993
117 ('\u{02a0}', "\u{029b}\u{0325}"), // hooktop q, withdrawn 1993
118 ]
119 .into_iter()
120 .collect()
121 })
122}
123
124/// Chao tone letters, extra-low to extra-high.
125const TONE_DIGIT: [(char, char); 5] =
126 [('1', '˩'), ('2', '˨'), ('3', '˧'), ('4', '˦'), ('5', '˥')];
127
128/// Superscript digits, which several sources use for tone instead of Chao
129/// letters. `ma³³` and `ma˧˧` are the same word.
130const SUPERSCRIPT_DIGIT: [(char, char); 5] =
131 [('¹', '1'), ('²', '2'), ('³', '3'), ('⁴', '4'), ('⁵', '5')];
132
133fn tone_for(digit: char) -> Option<char> {
134 TONE_DIGIT
135 .iter()
136 .find(|(from, _)| *from == digit)
137 .map(|(_, to)| *to)
138}
139
140/// The ring above is the same feature as the ring below, voiceless. IPA uses
141/// the one above when a descender leaves no room underneath.
142const RING_ABOVE: char = '\u{030a}';
143const RING_BELOW: char = '\u{0325}';
144
145/// NOTHING IS DROPPED ANY MORE.
146///
147/// This held `\u{0361}`, `\u{035c}`, `.` and `\u{203f}`, and deleting them was a
148/// category error. Normalizing folds the several ways of writing ONE thing
149/// into one of them. It does not decide a thing is unimportant. The syllable
150/// boundary, the link between two words spoken as one, and the tie binding
151/// two letters into a single segment each say something the source meant,
152/// and `t͡ʃ` is one affricate where `tʃ` may be a stop meeting a fricative.
153///
154/// They pass through now and the parser reports them as symbols. The one
155/// genuine fold among the four is in `replace`: the tie below is the tie
156/// above, chosen by whether a descender leaves room.
157
158/// Fine phonetic detail used to be stripped here so the segment underneath
159/// still resolved. It no longer is, and there is no option to bring it back.
160///
161/// Every mark that was dropped now has an entry in `modifiers.json`, so `b̤`
162/// reads as `b` plus breathy and a caller sees the feature instead of losing
163/// it. Breathy voice is contrastive across South Asia and creaky voice across
164/// Mesoamerica, so the old default merged phonemes that contrast.
165///
166/// A caller wanting `b̤` to MATCH `b` wants the tone encoding, which is coarse
167/// by design and gives both the same code.
168
169/// Characters carrying nothing recoverable, removed before anything else.
170///
171/// PRIVATE USE AREA. A font assigns these whatever glyph it likes and no two
172/// agree, so a source using one meant a symbol its own font drew and nothing
173/// downstream can know which. 98 such characters were observed in five
174/// languages.
175fn is_private_use(character: char) -> bool {
176 matches!(
177 character,
178 '\u{e000}'..='\u{f8ff}'
179 | '\u{f0000}'..='\u{ffffd}'
180 | '\u{100000}'..='\u{10fffd}'
181 )
182}
183
184/// Strip the delimiters WRAPPING a whole string, which say what kind of
185/// transcription it is rather than form part of it. 6,903 uses across four
186/// languages.
187///
188/// Only when they wrap. A slash INSIDE a string separates two readings, and
189/// cutting it would join them into a word that is neither.
190fn unwrap_delimiters(text: &str) -> &str {
191 let trimmed = text.trim();
192 let mut characters = trimmed.chars();
193
194 match (characters.next(), characters.next_back()) {
195 (Some('/'), Some('/')) | (Some('['), Some(']'))
196 if trimmed.chars().count() > 2 =>
197 {
198 characters.as_str()
199 }
200 // UNTRIMMED WHEN NOTHING WRAPPED. The trim above is how the delimiters
201 // are FOUND, not something to keep: returning it stripped a lone space
202 // to nothing, and a space is a real symbol that separates two words.
203 // The reference build trims only to test for the delimiters and hands
204 // back the string it was given.
205 _ => text,
206 }
207}
208
209/// The canonical order for the marks that follow a base, DERIVED FROM PHOIBLE
210/// rather than chosen.
211///
212/// Combining marks sharing a Unicode combining class keep their input order
213/// under NFD, so `n̪̥` and `n̥̪` stay two strings for one sound, and the parser
214/// matches greedily so the two read as different segments. Sorting fixes it.
215///
216/// WHICH ORDER HAS NO STANDARD. Unicode declines to decide: dental U+032A and
217/// voiceless U+0325 are both combining class 220, and NFD only reorders across
218/// different classes. The IPA Handbook says what each diacritic means and never
219/// what sequence to write two of them in.
220///
221/// So it is recovered from the corpus. `pnpm derive:mark-order` in mesh walks
222/// phoible's 3,142 distinct phoneme spellings, takes one vote per stacked pair
223/// and topologically sorts them. phoible proved completely self-consistent:
224/// 135 pairs decided, none written both ways, no cycles.
225///
226/// THE EVIDENCE IS UNEVEN. `ː` rests on 367 stack observations and `ʰ` on 135,
227/// so those positions are solid. U+031A was stacked ONCE and its place is the
228/// tie-break rather than a fact. Re-run the derivation instead of editing here.
229const MARK_ORDER: &[char] = &[
230 '\u{0327}', // cedilla
231 '\u{032a}', // dental
232 '\u{033c}', // linguolabial
233 '\u{033a}', // apical
234 '\u{033b}', // laminal
235 '\u{031f}', // advanced
236 '\u{0320}', // retracted
237 '\u{031d}', // raised
238 '\u{031e}', // lowered
239 '\u{033d}', // mid-centralized
240 '\u{0318}', // advanced tongue root
241 '\u{0319}', // retracted tongue root
242 '\u{1da3}', // labial-palatalized
243 '\u{0339}', // more rounded
244 '\u{031c}', // less rounded
245 '\u{032c}', // voiced
246 '\u{0324}', // breathy
247 '\u{02b1}', // murmured
248 '\u{0330}', // creaky
249 '\u{1d31}', // sphincteric
250 '\u{0348}', // fortis
251 '\u{0349}', // lenis
252 '\u{0353}', // frictionalized
253 '\u{0325}', // voiceless
254 '\u{0347}', // non-sibilant
255 '\u{207f}', // nasal release
256 '\u{02e1}', // lateral release
257 '\u{031a}', // no audible release
258 '\u{0329}', // syllabic
259 '\u{02b7}', // labialized
260 '\u{02b2}', // palatalized
261 '\u{02c0}', // glottalized
262 '\u{032f}', // non-syllabic
263 '\u{02d1}', // half-long
264 '\u{0306}', // extra-short
265 '\u{0303}', // nasalized
266 '\u{0308}', // centralized
267 '\u{02e0}', // velarized
268 '\u{02de}', // rhotic
269 '\u{02e4}', // pharyngealized
270 '\u{02e5}', // tone extra-high
271 '\u{02e6}', // tone high
272 '\u{02e7}', // tone mid
273 '\u{02e8}', // tone low
274 '\u{02e9}', // tone extra-low
275 '\u{2193}', // downstep
276 '\u{02b0}', // aspirated
277 '\u{02bc}', // ejective
278 '\u{02d0}', // long
279];
280
281/// The tone marks, which all share ONE rank.
282///
283/// A CONTOUR IS A SEQUENCE, NOT A SET. `˩˩˦` is a rise and `˦˩˩` is a fall,
284/// and the two are spelled with the same three letters in a different order.
285/// Ranking the five Chao letters individually meant every run of them was
286/// sorted into descending pitch, so every rising tone became a falling one
287/// and the original was not recoverable.
288///
289/// MEASURED, in a corpus normalized by this function: 599,336 of
290/// Vietnamese's 599,339 tone-letter runs came out descending, and all 23,292
291/// of Thai's. Vietnamese `mả` is a dipping-rise and was stored `maː˦˨˩`.
292/// Neither language has a tone that only falls.
293///
294/// REMOVING THEM FROM `MARK_ORDER` DOES NOT FIX IT. `rank_of` would fall
295/// through to the codepoint, and U+02E5 through U+02E9 run high pitch to
296/// low, so the sort reaches the same descending order by another route.
297///
298/// An equal rank is the fix, because `sort_by_key` is stable: a run of tone
299/// marks keeps the order it arrived in, while still sorting as a group
300/// against every other kind of mark.
301///
302/// THE COMBINING TONE ACCENTS ARE HERE FOR THE SAME REASON. Acute, grave,
303/// circumflex, caron, macron and the doubled pair are how many sources write
304/// tone, two of them stacked make a contour, and none appears in
305/// `MARK_ORDER`, so they were ordered by codepoint against each other.
306const TONE: &[char] = &[
307 '˥',
308 '˦',
309 '˧',
310 '˨',
311 '˩',
312 '\u{2193}', // downstep, which lowers what follows and so is positional too
313 '\u{0300}', // grave, low
314 '\u{0301}', // acute, high
315 '\u{0302}', // circumflex, falling
316 '\u{0304}', // macron, mid
317 '\u{030b}', // double acute, extra high
318 '\u{030c}', // caron, rising
319 '\u{030f}', // double grave, extra low
320 '\u{0311}', // inverted breve, peaking
321];
322
323fn rank_of(character: char) -> usize {
324 if TONE.contains(&character) {
325 // Where the tone group sorts: exactly where the Chao letters already
326 // did, which is last among the suprasegmentals.
327 return MARK_ORDER
328 .iter()
329 .position(|mark| *mark == '˥')
330 .expect("MARK_ORDER holds the extra-high tone letter");
331 }
332
333 MARK_ORDER
334 .iter()
335 .position(|mark| *mark == character)
336 .unwrap_or(MARK_ORDER.len() + character as usize)
337}
338
339/// Whether a character attaches to a preceding base rather than standing
340/// alone.
341fn is_affix(character: char) -> bool {
342 use unicode_normalization::char::canonical_combining_class;
343
344 canonical_combining_class(character) != 0
345 // The modifier-letter blocks IPA draws from. Rust has no general
346 // category table, so the ranges are named: spacing modifiers, the two
347 // phonetic-extension blocks (the second holds `ᶣ` at U+1DA3), and the
348 // two strays IPA uses.
349 || matches!(character,
350 '\u{02b0}'..='\u{02ff}'
351 | '\u{1d2c}'..='\u{1d6a}'
352 | '\u{1d78}'
353 | '\u{1d9b}'..='\u{1dbf}'
354 | '\u{2071}'
355 | '\u{207f}')
356}
357
358fn is_base(character: char) -> bool {
359 // A LETTER IS A BASE EVEN WHEN IT IS ALSO AN AFFIX. The reference build
360 // asks only whether the general category starts with `L`, and the modifier
361 // letters are `Lm`, so `ˈ` and `ʰ` are bases there as well as affixes. The
362 // `&& !is_affix` here excluded exactly those, so a run opened by one was
363 // never sorted: `ˈ˷̩` kept its input order while the reference moved the
364 // syllabic mark ahead of the phone, and the mark then read as trailing
365 // rather than leading.
366 //
367 // Ordering the two tests this way is safe because `order_marks` checks
368 // the affix branch FIRST, so a modifier letter with nothing before it
369 // still falls through and opens a run rather than joining one.
370 character.is_alphabetic()
371}
372
373/// Marks whose place in a run says WHEN, not merely what.
374///
375/// Sorting a run is right when the marks describe one moment from several
376/// angles. `n̪̥` and `n̥̪` are one sound written two ways, dental and
377/// voiceless, both true of the whole segment, so an order has to be picked
378/// and either will do.
379///
380/// It is wrong the moment a run describes a sequence. These establish points
381/// in time, and every mark between two of them is timed against them: the
382/// Chao pitch targets in the order the voice reaches them, downstep which
383/// lowers everything after it, and the length marks.
384///
385/// MEASURED. Vietnamese writes its ngã tone `˦ˀ˥`, a rise interrupted by
386/// glottalisation partway up, and sorting produced `ˀ˦˥` in 2,838 rows.
387/// Navajo writes `óː` and sorting pushed the acute past the length mark to
388/// `oː́`, where it attaches to the `ː`, in 3,707 rows across three languages.
389/// The two look unrelated and are the same defect.
390const ANCHOR: &[char] =
391 &['˥', '˦', '˧', '˨', '˩', '\u{2193}', 'ː', 'ˑ', '\u{0306}'];
392
393/// One run, sorted between its anchors and never across them.
394///
395/// The anchors keep their positions and each stretch between two of them is
396/// sorted alone, so `˦ˀ˥` keeps the glottal where the voice makes it and `óː`
397/// keeps the accent on the vowel, while `n̥̪` still canonicalises because
398/// neither of its marks is an anchor.
399fn ordered(run: &[char]) -> Vec<char> {
400 let mut out: Vec<char> = Vec::with_capacity(run.len());
401 let mut piece: Vec<char> = Vec::new();
402
403 for character in run {
404 if ANCHOR.contains(character) {
405 piece.sort_by_key(|mark| rank_of(*mark));
406 out.append(&mut piece);
407 out.push(*character);
408 continue;
409 }
410
411 piece.push(*character);
412 }
413
414 piece.sort_by_key(|mark| rank_of(*mark));
415 out.append(&mut piece);
416
417 out
418}
419
420/// Put the affixes following each base into canonical order.
421///
422/// Only a run that FOLLOWS a base is sorted. A leading modifier is a
423/// different sound rather than a reordering: `ʰk` is pre-aspirated and
424/// `kʰ` post-aspirated, so moving one to the other would change what was
425/// said.
426fn order_marks(text: &str) -> String {
427 let mut out = String::with_capacity(text.len());
428 let mut run: Vec<char> = Vec::new();
429 let mut after_base = false;
430
431 for character in text.chars() {
432 if is_affix(character) && (after_base || !run.is_empty()) {
433 run.push(character);
434 continue;
435 }
436
437 if !run.is_empty() {
438 if after_base {
439 run = ordered(&run);
440 }
441 out.extend(run.drain(..));
442 }
443
444 out.push(character);
445 after_base = is_base(character);
446 }
447
448 if !run.is_empty() {
449 if after_base {
450 run = ordered(&run);
451 }
452 out.extend(run.drain(..));
453 }
454
455 out
456}
457
458/// `bare_digit_tone` reads bare digits as tone, off by default because a bare
459/// digit is ambiguous.
460/// caller can audit what this encoding drops.
461#[derive(Debug, Clone, Copy, Default)]
462pub struct NormalizeIpaOptions {
463 pub bare_digit_tone: bool,
464}
465
466/// Normalize an IPA string into the form the parser reads.
467///
468/// Idempotent, and returns NFD because every combining mark in the tries is
469/// stored decomposed.
470pub fn normalize_ipa_with(text: &str, options: NormalizeIpaOptions) -> String {
471 // Before anything else: drop the private-use characters no reader can
472 // resolve, and unwrap the delimiters that say what kind of transcription
473 // this is. Both are about the string rather than in it.
474 let stripped: String =
475 text.chars().filter(|one| !is_private_use(*one)).collect();
476 let bare = unwrap_delimiters(&stripped);
477
478 // Two passes, because a replacement can itself contain something the second
479 // pass acts on: `ȵ` expands to `ɲ̟`, and the advancing mark then has to
480 // decompose and sort with the rest of its run.
481 let mut source = String::with_capacity(bare.len());
482
483 for character in bare.chars() {
484 if let Some((_, digit)) =
485 SUPERSCRIPT_DIGIT.iter().find(|(from, _)| *from == character)
486 {
487 if let Some(tone) = tone_for(*digit) {
488 source.push(tone);
489 continue;
490 }
491 }
492
493 match replace().get(&character) {
494 Some(replacement) => source.push_str(replacement),
495 None => source.push(character),
496 }
497 }
498
499 let mut out = String::with_capacity(source.len());
500
501 for character in source.nfd() {
502 if character == RING_ABOVE {
503 out.push(RING_BELOW);
504 continue;
505 }
506
507 if options.bare_digit_tone {
508 if let Some(tone) = tone_for(character) {
509 out.push(tone);
510 continue;
511 }
512 }
513
514 out.push(character);
515 }
516
517 order_marks(&out)
518}
519
520/// Normalize with the default options.
521pub fn normalize_ipa(text: &str) -> String {
522 normalize_ipa_with(text, NormalizeIpaOptions::default())
523}