Skip to main content

nameparser/
unicode.rs

1// SPDX-License-Identifier: Apache-2.0
2//! Java `org.gbif.nameparser.util.UnicodeUtils` — ported subset: `normalizeQuotes`, plus
3//! the homoglyph-replacement table (`HOMOGLYHPS`/`containsHomoglyphs`/`replaceHomoglyphs`)
4//! used by `pipeline::stripandstash`'s step 13. `decompose`/`foldToAscii`/
5//! `replaceSpecialCases`/`removeNonAscii`/`replaceNonAscii`/`unescapeUnicodeChars`/
6//! `decodeUtf8Garbage`/`containsDiacritics`/`findDiacritics` are NOT ported: no call site
7//! anywhere in this crate's (now complete) pipeline reaches them, and the full-corpus
8//! Rust-vs-Java parity cross-validation (see `BENCHMARKS.md`/`cross-validation.md`, 0
9//! mismatches) confirms their absence isn't a gap.
10
11/// Unicode apostrophe / single-quote variants that normalise to the ASCII apostrophe `'`.
12/// Verbatim transcription of Java `UnicodeUtils.SINGLE_QUOTES`
13/// (`"`´ʹʻʼʽˊˋ‘’‘’‚‛′‵❛❜'"`).
14/// Every codepoint is a 4-hex-digit Java `\u` escape, i.e. a single UTF-16 code unit in
15/// the Basic Multilingual Plane outside the surrogate range — so iterating Rust `char`s
16/// (Unicode scalar values) reproduces Java's `char`-at-a-time (UTF-16 code unit) loop
17/// exactly for this fixed table.
18const SINGLE_QUOTES: &[char] = &[
19    '\u{0060}', '\u{00B4}', '\u{02B9}', '\u{02BB}', '\u{02BC}', '\u{02BD}', '\u{02CA}', '\u{02CB}',
20    '\u{0091}', '\u{0092}', '\u{2018}', '\u{2019}', '\u{201A}', '\u{201B}', '\u{2032}', '\u{2035}',
21    '\u{275B}', '\u{275C}', '\u{FF07}',
22];
23
24/// Unicode double-quote / double-prime variants that normalise to the ASCII double quote
25/// `"`. Verbatim transcription of Java `UnicodeUtils.DOUBLE_QUOTES`
26/// (`"“”“”„‟″‶❝❞〝〞〟""`).
27const DOUBLE_QUOTES: &[char] = &[
28    '\u{0093}', '\u{0094}', '\u{201C}', '\u{201D}', '\u{201E}', '\u{201F}', '\u{2033}', '\u{2036}',
29    '\u{275D}', '\u{275E}', '\u{301D}', '\u{301E}', '\u{301F}', '\u{FF02}',
30];
31
32// ---------------------------------------------------------------------------------------
33// Homoglyph table: Java `UnicodeUtils.HOMOGLYHPS`/`containsHomoglyphs`/`replaceHomoglyphs`
34// (`UnicodeUtils.java:58-139` for the static-init loader, `:194-215` and `:239-276` for the
35// public methods). Used by `pipeline::stripandstash`'s step 13 (`replaceHomoglyphs`) to
36// fold Latin look-alike characters from other scripts (e.g. Cyrillic 'а' U+0430) — and a
37// handful of archaic Latin typography, notably U+017F LATIN SMALL LETTER LONG S "ſ" — down
38// to their canonical ASCII/Latin letter.
39// ---------------------------------------------------------------------------------------
40
41use std::borrow::Cow;
42use std::collections::HashMap;
43use std::sync::LazyLock;
44use unicode_normalization::UnicodeNormalization;
45
46/// The homoglyph table, embedded at compile time (copied verbatim from
47/// `name-parser-api/src/main/resources/unicode/homoglyphs.txt`, itself sourced from
48/// <https://raw.githubusercontent.com/codebox/homoglyph/master/raw_data/chars.txt> per the
49/// resource file's own header comment — `diff`-verified byte-for-byte identical to the
50/// Java classpath resource, but for 18 distinct letters removed in #65: the caron and the comma
51/// below of Czech `ě`, pinyin `ǎ`/`ǐ`/`ǒ`/`ǔ`, `ǧ`, `ǵ`, Romanian `ț`, Hungarian `Ő` and `ȧ`
52/// rewrote real letters to their breve, cedilla or diaeresis twins, "homoglyphs replaced"). Each line lists a "canonical" character (the line's first
53/// char) followed by every codepoint the codebox project considers a visual look-alike of
54/// it. 1861 lines total (`\n`-delimited; the file's own last line has no trailing
55/// terminator, the same off-by-one `wc -l` undercounts documented on
56/// `pipeline::blacklisted_epithets`'s `BLACKLIST_TXT`) — but see [`HOMOGLYPHS`]'s own doc
57/// comment: Java's loader, and this port, only ever read a small prefix of it.
58const HOMOGLYPHS_TXT: &str = include_str!("../resources/homoglyphs.txt");
59
60/// U+00D7 MULTIPLICATION SIGN, the botanical hybrid marker ("Abies × Picea"). Java
61/// `NameFormatter.HYBRID_MARKER` (`NameFormatter.java:14`: `public static final char
62/// HYBRID_MARKER = '×';`). Excluded from the table [`HOMOGLYPHS`] builds below — even
63/// though it appears, unremoved, as one of the look-alikes on `homoglyphs.txt`'s `x` row
64/// (line 88) — so a genuine hybrid marker is never folded into a plain "x", which would
65/// silently destroy hybrid-name semantics. Java's own `UnicodeUtilsTest.containsHomoglyphs`
66/// asserts exactly this: `"Abies × Picea"` is NOT flagged as containing a homoglyph.
67const HYBRID_MARKER: char = '\u{00D7}';
68
69/// Java `ignoredCanonicals` (`UnicodeUtils.java:67`): `new CharArraySet(new char[]{' ',
70/// '\'', '-', '﹘'})`. A `homoglyphs.txt` row whose canonical (first) character is one of
71/// these four is skipped ENTIRELY — no homoglyph entries at all are taken from that row.
72/// U+0020 SPACE, U+0027 APOSTROPHE, U+002D HYPHEN-MINUS, U+FE58 SMALL EM DASH.
73const IGNORED_CANONICALS: [char; 4] = [' ', '\'', '-', '\u{FE58}'];
74
75/// Java's local `ignore` set (`UnicodeUtils.java:93-103`, rebuilt from the literal
76/// `"‘’“”"` on every loop iteration but with identical fixed content
77/// each time — a code-style quirk, not per-row-varying data). Curly single/double-quote
78/// variants are never captured as a homoglyph VALUE on ANY row — quote normalisation is
79/// [`normalize_quotes`]'s job, not this table's.
80const QUOTE_IGNORE: [char; 4] = ['\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}'];
81
82/// Java `DIACRITICS` (`UnicodeUtils.java:33-56`, static-init block): standalone
83/// combining/spacing diacritic marks, excluded from ever being captured as a homoglyph
84/// VALUE (a bare combining accent isn't meaningfully "the same as" some base letter, even
85/// where the auto-generated codebox table lists it as a look-alike on some canonical's
86/// row). Verbatim transcription of the Java string literal
87/// `"´˝\` ̏ˆˇ˘ ̑¸¨· ̡ ̢ ̉ ̛ˉ˛ ˚˳῾᾿"` with its 7 embedded literal spaces filtered out
88/// (`.filter(cp -> cp != 32)`, exactly like the Java) — 21 codepoints, extracted
89/// programmatically from the Java source (not hand-transcribed from the hard-to-read
90/// rendered glyphs, several of which are zero-width combining marks that visually vanish
91/// or merge with a neighbour when displayed).
92const DIACRITICS: [char; 21] = [
93    '\u{00B4}', '\u{02DD}', '\u{0060}', '\u{030F}', '\u{02C6}', '\u{02C7}', '\u{02D8}', '\u{0311}',
94    '\u{00B8}', '\u{00A8}', '\u{00B7}', '\u{0321}', '\u{0322}', '\u{0309}', '\u{031B}', '\u{02C9}',
95    '\u{02DB}', '\u{02DA}', '\u{02F3}', '\u{1FFE}', '\u{1FBF}',
96];
97
98/// Java `UnicodeUtils.HOMOGLYHPS` (`UnicodeUtils.java:59-139`, static-init block): a
99/// codepoint -> canonical-char map, built once from [`HOMOGLYPHS_TXT`]. Faithful port of
100/// the loader's exact algorithm, verified against a compiled run of the real Java
101/// `LineReader` over the real resource file (see the loop body comments below for the
102/// specific findings that verification surfaced):
103///
104/// 1. `'ſ'` (U+017F LATIN SMALL LETTER LONG S) -> `'s'` is inserted FIRST, before any table
105///    row loads, so it wins over row `f`'s own listing of `'ſ'` as one of ITS look-alikes
106///    (`homoglyphs.txt` line 72, canonical `f`) — the loop below never overwrites an
107///    existing key. Java `UnicodeUtils.java:71`: `homoglyphs.put(toCodePoint('ſ'), 's');`.
108/// 2. Each remaining `homoglyphs.txt` line is a candidate row, in file order. Java's
109///    `LineReader` (`skipBlank=true, skipComments=true`, the defaults its 1-arg
110///    `InputStream` constructor chains to) skips blank and `#`-comment lines before they
111///    ever reach this loop. A plain `line.is_empty()` check stands in for `LineReader`'s
112///    fuller `StringUtils.isBlank` (which additionally treats any all-whitespace line as
113///    blank): the ONE line in this file that's all-whitespace-but-not-empty (line 9, a run
114///    of Unicode space variants: U+0020 U+0020 U+1680 U+2000-U+200A U+2028 U+2029 U+202F
115///    U+205F) is NOT actually blank under Java's exact `Character.isWhitespace` (which
116///    excludes U+2007 FIGURE SPACE and U+202F NARROW NO-BREAK SPACE, both present on that
117///    line) — so Java's `LineReader` does NOT skip it either, and it reaches
118///    `UnicodeUtils`'s own loop body same as any other row. It's discarded there instead,
119///    by step 3's `IGNORED_CANONICALS` check (its canonical, the first char, is U+0020
120///    SPACE) — so the outcome is identical either way; `line.is_empty()` here is not a
121///    faithfulness gap.
122/// 3. The row's canonical is its first char (`.chars().next()` on a non-empty `str` is
123///    exactly Java's `line.charAt(0)` here — the canonical is never a surrogate pair, and a
124///    Rust `char` is already a full Unicode scalar value). If the canonical is one of
125///    [`IGNORED_CANONICALS`], the entire row is skipped.
126/// 4. Otherwise, every remaining char on the row becomes a candidate homoglyph VALUE,
127///    filtered exactly as `UnicodeUtils.java:104-111` filters it: codepoint > 128 (skip
128///    ASCII/Latin-1-control), not [`HYBRID_MARKER`], not in [`DIACRITICS`], not already a
129///    map key (first row wins — relevant beyond just the `'ſ'` pre-seed whenever the same
130///    look-alike appears on more than one row), not in [`QUOTE_IGNORE`].
131/// 5. After a (non-ignored-canonical) row is fully processed, Java breaks the loop
132///    (`UnicodeUtils.java:128`: `if (lr.getRow() > 175 || 'ɸ' == canonical) break;`,
133///    comment "skip all rare chars"). `lr.getRow()` is the row's own 1-indexed PHYSICAL
134///    line number in the source file (confirmed by compiling and running the real
135///    `LineReader` against the real resource file) — i.e. exactly this loop's `row`
136///    variable below, a straight `enumerate()` index, not a count of rows actually
137///    accepted. The `> 175` numeric fallback is dead code against the actual resource file
138///    — canonical `'ɸ'` (U+0278, physical line 135) always fires first — but ported
139///    verbatim anyway, matching this port's established precedent of preserving
140///    apparently-dead Java guards faithfully (see e.g. `stash_trailing_strain_code`'s
141///    `DIGITS_ONLY` guard in `pipeline::stripandstash`). Net effect: only `homoglyphs.txt`
142///    lines 1-135 are ever consulted — lines 1-8 are the file's own comment header (all
143///    `#`-prefixed), line 9 is the ignored space-canonical row, 2 more rows in range (14,
144///    canonical `'`; 20, canonical `-`) are also [`IGNORED_CANONICALS`], and the remaining
145///    124 rows (lines 10-135, minus those 2) each contribute at least one map entry — lines
146///    136-1845 (the file's long CJK "duplicate codepoint" tail, e.g. `"𦰶𦰶"`) are never
147///    read into the map at all — by Java's own design, not a deferral this port introduces.
148static HOMOGLYPHS: LazyLock<HashMap<char, char>> = LazyLock::new(|| {
149    let mut map = HashMap::new();
150    map.insert('\u{017F}', 's'); // 'ſ' -> 's', pre-seeded (see step 1 above)
151
152    for (idx, line) in HOMOGLYPHS_TXT.lines().enumerate() {
153        let row = idx + 1; // 1-indexed physical line number == Java `lr.getRow()`
154        if line.is_empty() || line.starts_with('#') {
155            continue;
156        }
157        let canonical = line.chars().next().expect("checked non-empty above");
158        if IGNORED_CANONICALS.contains(&canonical) {
159            continue;
160        }
161        // Rust-only: a digit row listing ASCII letters too ("0Oo…", "1Il…") folds a look-alike
162        // letter to the ASCII letter of its case, not to the digit — a Cyrillic "о" in
163        // "оleiformis" is an "o", and Java made it "0leiformis".
164        let letter = |upper: bool| {
165            line.chars()
166                .skip(1)
167                .find(|c| c.is_ascii_alphabetic() && c.is_ascii_uppercase() == upper)
168        };
169        let (upper_letter, lower_letter) = if canonical.is_ascii_digit() {
170            (letter(true), letter(false))
171        } else {
172            (None, None)
173        };
174        for cp in line.chars().skip(1) {
175            if (cp as u32) > 128
176                && cp != HYBRID_MARKER
177                && !DIACRITICS.contains(&cp)
178                && !map.contains_key(&cp)
179                && !QUOTE_IGNORE.contains(&cp)
180            {
181                let folded = match (cp.is_uppercase(), cp.is_lowercase()) {
182                    (true, _) => upper_letter,
183                    (_, true) => lower_letter,
184                    _ => None,
185                };
186                map.insert(cp, folded.unwrap_or(canonical));
187            }
188        }
189        if row > 175 || canonical == 'ɸ' {
190            break;
191        }
192    }
193    map
194});
195
196/// Java `UnicodeUtils.containsHomoglyphs(CharSequence)` (`UnicodeUtils.java:194-196`,
197/// delegating to `findHomoglyph` at `:203-215`). Returns true if `s` contains at least one
198/// character that is a known homoglyph of a Latin letter. Java's version short-circuits via
199/// a codepoint lower/upper-bound check before the map lookup (an `Int2CharMap`-specific
200/// micro-optimisation); a `HashMap<char, _>` lookup here is already O(1) average without
201/// needing that bound check reproduced.
202pub fn contains_homoglyphs(s: &str) -> bool {
203    s.chars().any(|c| HOMOGLYPHS.contains_key(&c))
204}
205
206/// Java `UnicodeUtils.replaceHomoglyphs(CharSequence, boolean)` / its 3-arg sibling
207/// (`UnicodeUtils.java:239-276`), specialised to the only shape any ported call site ever
208/// needs: `inclHyphens=false` and no `keep` set. `pipeline::stripandstash`'s step 13 (the
209/// sole caller) always passes `inclHyphens=false` — Unicode hyphen variants are already
210/// normalised to ASCII `-` by the immediately-preceding step 12 (`normaliseHyphens`), per
211/// that Java method's own comment ("Hyphen homoglyphs are intentionally excluded — those
212/// are normalised above already") — so `HYPHEN_HOMOGLYHPS` (`UnicodeUtils.java:141-167`)
213/// and the `keep`-list parameter are both unreachable dead weight for this port's actual
214/// call graph and are not ported. Replaces every character present as a key in
215/// [`HOMOGLYPHS`] with its mapped canonical; every other character passes through
216/// unchanged.
217pub fn replace_homoglyphs(s: &str) -> String {
218    s.chars()
219        .map(|c| *HOMOGLYPHS.get(&c).unwrap_or(&c))
220        .collect()
221}
222
223/// [`replace_homoglyphs`], except that a Turkish dotless ı (U+0131) stays in a capitalised word
224/// other than the first: the surnames `Altıner`, `Yıldırım`. In the genus or an epithet it is a
225/// look-alike of `i` all the same. Rust-only: Java folded it everywhere (`Altiner`).
226pub fn replace_homoglyphs_in_name(s: &str) -> String {
227    let mut out = String::with_capacity(s.len());
228    let mut at_word_start = true;
229    let mut surname = false;
230    for (i, c) in s.char_indices() {
231        if c.is_whitespace() {
232            at_word_start = true;
233            surname = false;
234            out.push(c);
235            continue;
236        }
237        if at_word_start && !matches!(c, '(' | '[' | '"' | '\'') {
238            surname = i > 0 && c.is_uppercase();
239            at_word_start = false;
240        }
241        if c == '\u{0131}' && surname {
242            out.push(c);
243        } else {
244            out.push(*HOMOGLYPHS.get(&c).unwrap_or(&c));
245        }
246    }
247    out
248}
249
250/// Faithful port of Java `String.trim()`: strips only leading/trailing chars whose
251/// codepoint is <= U+0020 (NOT the full Unicode White_Space set that Rust's str::trim uses).
252/// Use this everywhere the Java source calls `.trim()`.
253///
254/// Java's `String.trim()` only removes characters with codepoint values 0–32 (space through
255/// control chars), while Rust's `str::trim()` removes the entire Unicode `White_Space` category.
256/// This means non-breaking space (U+00A0) and other Unicode spaces are trimmed by Rust but NOT
257/// by Java, causing empty/length guard logic to diverge.
258pub fn java_trim(s: &str) -> &str {
259    s.trim_matches(|c: char| (c as u32) <= 0x20)
260}
261
262/// Folds every Unicode space separator (category `Zs`: NBSP U+00A0, U+1680, U+2000–U+200A, the
263/// narrow NBSP U+202F, U+205F, U+3000) to an ASCII space. The parser's own whitespace handling is
264/// ASCII-only (Java parity: `String.trim()`, `Character.isWhitespace`, and the `(?-u:\s)` regexes
265/// all skip NBSP), so an NBSP left in place glues words together and hides keywords from every
266/// stage — `Assimineidae\u{a0}incertae\u{a0}sedis` read as a monomial with the author
267/// `incertae sedis`. Runs once on the raw input, before [`java_trim`], so the guards and every
268/// later stage see plain spaces. Borrows when there is nothing to fold (the common case).
269pub fn normalize_spaces(x: &str) -> Cow<'_, str> {
270    if x.chars().any(is_unicode_space) {
271        Cow::Owned(
272            x.chars()
273                .map(|c| if is_unicode_space(c) { ' ' } else { c })
274                .collect(),
275        )
276    } else {
277        Cow::Borrowed(x)
278    }
279}
280
281/// The input clean-up every parse starts with: [`normalize_spaces`], invisible format characters
282/// removed — zero-width space and joiners U+200B–U+200D, the word joiner U+2060 and the byte order
283/// mark U+FEFF (the first cell of a UTF-8 file with a BOM) — and the fullwidth ASCII forms
284/// U+FF01–U+FF5E folded to ASCII, as East Asian keyboards type them (`(Tubangui,1928)`). The
285/// homoglyph table cannot do the latter: it maps fullwidth `l` and `I` to the digit `1`. The soft
286/// hyphen U+00AD only marks where a word may break: it is dropped (`novae\u{ad}zelandiae`,
287/// `Ramí\u{ad}rez`), except between a lower-case and a capital letter, where it stands for a real
288/// hyphen (`Miranda\u{ad}Ribeiro`). Mojibake is repaired first ([`repair_mojibake`]): its
289/// continuation bytes include the soft hyphen and the no-break space (`Ã\u{ad}` is `í`, `Ã\u{a0}`
290/// is `à`). Decomposed letters are composed (NFC): a combining accent after its letter
291/// (`Za\u{301}gors\u{30c}ek`) otherwise splits the word, and Java read `Za gors ek`. Borrows when
292/// there is nothing to change.
293pub fn normalize_input(x: &str) -> Cow<'_, str> {
294    let x = repair_mojibake(x);
295    let x: Cow<'_, str> = if x
296        .chars()
297        .any(unicode_normalization::char::is_combining_mark)
298    {
299        Cow::Owned(x.nfc().collect())
300    } else {
301        x
302    };
303    if !x
304        .chars()
305        .any(|c| is_unicode_space(c) || is_invisible(c) || is_fullwidth(c) || c == SOFT_HYPHEN)
306    {
307        return x;
308    }
309    let chars: Vec<char> = x.chars().collect();
310    let mut out = String::with_capacity(x.len());
311    for (i, &c) in chars.iter().enumerate() {
312        if is_invisible(c) {
313            continue;
314        }
315        if c == SOFT_HYPHEN {
316            let prev = i.checked_sub(1).map(|k| chars[k]);
317            let next = chars.get(i + 1).copied();
318            if prev.is_some_and(char::is_lowercase) && next.is_some_and(char::is_uppercase) {
319                out.push('-');
320            }
321            continue;
322        }
323        out.push(if is_unicode_space(c) {
324            ' '
325        } else if is_fullwidth(c) {
326            char::from_u32(c as u32 - 0xFEE0).unwrap_or(c)
327        } else {
328            c
329        });
330    }
331    Cow::Owned(out)
332}
333
334/// UTF-8 text decoded as Windows-1252 (or Latin-1), the commonest mojibake in author names:
335/// `Gürke` for `Gürke`, `André` for `André`, `Komárek` for `Komárek`, `Å¡` for `š` (#188). A
336/// UTF-8 lead byte (`Â`–`ô`) followed by the continuation bytes its sequence needs is encoded back
337/// to those bytes and decoded as UTF-8, where that gives a Latin letter. Broken beyond that — a byte lost
338/// or turned into a space (`Å korniÄ k.`) — it is left alone. Borrows when there is nothing to
339/// repair.
340pub fn repair_mojibake(x: &str) -> Cow<'_, str> {
341    if !x.chars().any(|c| ('\u{C2}'..='\u{F4}').contains(&c)) {
342        return Cow::Borrowed(x);
343    }
344    let chars: Vec<char> = x.chars().collect();
345    let mut out = String::with_capacity(x.len());
346    let mut changed = false;
347    let mut i = 0;
348    while i < chars.len() {
349        if let Some((c, len)) = mojibake_at(&chars[i..]) {
350            out.push(c);
351            i += len;
352            changed = true;
353        } else {
354            out.push(chars[i]);
355            i += 1;
356        }
357    }
358    if changed {
359        Cow::Owned(out)
360    } else {
361        Cow::Borrowed(x)
362    }
363}
364
365/// True if [`repair_mojibake`] would change `x`.
366pub fn has_mojibake(x: &str) -> bool {
367    matches!(repair_mojibake(x), Cow::Owned(_))
368}
369
370/// The letter a mojibake sequence at the start of `chars` stands for, and how many chars it takes.
371fn mojibake_at(chars: &[char]) -> Option<(char, usize)> {
372    let lead = cp1252_byte(*chars.first()?)?;
373    let n = match lead {
374        0xC2..=0xDF => 1,
375        0xE0..=0xEF => 2,
376        0xF0..=0xF4 => 3,
377        _ => return None,
378    };
379    let mut bytes = vec![lead];
380    for &c in chars.get(1..=n)? {
381        let b = cp1252_byte(c).filter(|b| (0x80..=0xBF).contains(b))?;
382        bytes.push(b);
383    }
384    let c = std::str::from_utf8(&bytes).ok()?.chars().next()?;
385    // a Latin letter; anything else is a chance match, `ß’` in `‘Gartenspaß’` decodes to N'Ko `ߒ`
386    let latin = matches!(c, '\u{C0}'..='\u{24F}' | '\u{1E00}'..='\u{1EFF}') && c.is_alphabetic();
387    latin.then_some((c, n + 1))
388}
389
390/// The Windows-1252 byte of `c`: Latin-1 for U+0000–U+00FF (the C1 controls included, as
391/// Latin-1 decoders leave them), the Windows-1252 punctuation and letters for 0x80–0x9F.
392fn cp1252_byte(c: char) -> Option<u8> {
393    let cp = c as u32;
394    if cp <= 0xFF {
395        return Some(cp as u8);
396    }
397    Some(match c {
398        '€' => 0x80,
399        '‚' => 0x82,
400        'ƒ' => 0x83,
401        '„' => 0x84,
402        '…' => 0x85,
403        '†' => 0x86,
404        '‡' => 0x87,
405        'ˆ' => 0x88,
406        '‰' => 0x89,
407        'Š' => 0x8A,
408        '‹' => 0x8B,
409        'Œ' => 0x8C,
410        'Ž' => 0x8E,
411        '‘' => 0x91,
412        '’' => 0x92,
413        '“' => 0x93,
414        '”' => 0x94,
415        '•' => 0x95,
416        '–' => 0x96,
417        '—' => 0x97,
418        '˜' => 0x98,
419        '™' => 0x99,
420        'š' => 0x9A,
421        '›' => 0x9B,
422        'œ' => 0x9C,
423        'ž' => 0x9E,
424        'Ÿ' => 0x9F,
425        _ => return None,
426    })
427}
428
429/// U+00AD, see [`normalize_input`].
430const SOFT_HYPHEN: char = '\u{AD}';
431
432/// A fullwidth ASCII form, U+FF01–U+FF5E: `!` … `~`.
433pub fn is_fullwidth(c: char) -> bool {
434    matches!(c, '\u{FF01}'..='\u{FF5E}')
435}
436
437fn is_invisible(c: char) -> bool {
438    matches!(c, '\u{200B}'..='\u{200D}' | '\u{2060}' | '\u{FEFF}')
439}
440
441fn is_unicode_space(c: char) -> bool {
442    matches!(
443        c,
444        '\u{00A0}' | '\u{1680}' | '\u{2000}'..='\u{200A}' | '\u{202F}' | '\u{205F}' | '\u{3000}'
445    )
446}
447
448/// Normalises the many unicode apostrophe / single-quote variants to the ASCII apostrophe
449/// (') and the unicode double-quote variants to the ASCII double quote ("). Author names
450/// and quoted/provisional names routinely arrive with curly, prime, modifier-letter,
451/// low-9, fullwidth or Windows-1252 quotes; collapsing them to ASCII keeps tokenisation
452/// and the parsed output consistent regardless of the input's quote style.
453///
454/// Faithful port of Java `UnicodeUtils.normalizeQuotes(String)`. Java's `null`-in/`null`-out
455/// case is not ported: this takes and returns non-nullable Rust types, so callers threading
456/// an optional authorship string (e.g. `Option<&str>`) map over it instead
457/// (`authorship.map(normalize_quotes)`), which reproduces the same null-forwarding at the
458/// `Option` level.
459pub fn normalize_quotes(x: &str) -> String {
460    x.chars()
461        .map(|c| {
462            if SINGLE_QUOTES.contains(&c) {
463                '\''
464            } else if DOUBLE_QUOTES.contains(&c) {
465                '"'
466            } else {
467                c
468            }
469        })
470        .collect()
471}
472
473#[cfg(test)]
474mod tests {
475    use super::*;
476
477    // ---- Homoglyph table ----
478
479    /// `homoglyphs.txt` lines 10-135 (126 physical lines) are the candidate data rows once
480    /// the 8-line `#`-comment header and line 9's ignored space-canonical row are behind us
481    /// (see [`HOMOGLYPHS`]'s own doc comment); 2 more of those 126 (line 14, canonical `'`;
482    /// line 20, canonical `-`) are also [`IGNORED_CANONICALS`], leaving 124 rows that each
483    /// contribute at least one homoglyph codepoint (independently verified against the
484    /// source file — no row contributes zero) — plus the `'ſ'` pre-seed, for an exact total
485    /// of 1733 map entries (1751 before the 18 distinct letters of #65 were removed). Asserted
486    /// exactly (not just "a lot"), as a direct regression guard against a bad resource copy or
487    /// a loader loop that silently stops early/never reaches `'ɸ'`/double-counts/under-filters.
488    #[test]
489    fn distinct_letters_are_no_homoglyphs() {
490        // #65: Czech, Hungarian, Romanian and pinyin letters are no look-alikes
491        for c in [
492            'ě', 'Ě', 'ǎ', 'ǐ', 'ǒ', 'ǔ', 'ǧ', 'ǵ', 'ț', 'Ț', 'Ő', 'ȧ', 'Ȧ',
493        ] {
494            assert!(!HOMOGLYPHS.contains_key(&c), "{c} must not be replaced");
495        }
496        // a Cyrillic look-alike of Ö still is
497        assert_eq!(HOMOGLYPHS.get(&'Ӧ'), Some(&'Ö'));
498    }
499
500    #[test]
501    fn resource_loads_exactly_1733_entries() {
502        assert_eq!(
503            HOMOGLYPHS.len(),
504            1733,
505            "loaded homoglyph count drifted — see this test's own doc comment"
506        );
507    }
508
509    /// Java `UnicodeUtilsTest.replaceHomoglyphs` (`UnicodeUtilsTest.java:117`): long-s
510    /// folds to plain "s" (NOT "f", even though `ſ` sits on `homoglyphs.txt`'s `f`-canonical
511    /// row too — the manual pre-seed wins), while `æ` — a genuine Latin ligature, not a
512    /// look-alike — is untouched; this table is not `foldToAscii`. Same shape as the
513    /// corpus's `specificEpithet`-affecting rows ("Musca domeſtica", "Amphisbæna
514    /// fuliginoſa") — see the sibling test below and `pipeline::stripandstash`'s golden
515    /// harness `ALLOWLIST` (now empty, see that file's own history).
516    #[test]
517    fn long_s_folds_to_s_but_ae_ligature_is_untouched() {
518        assert_eq!(
519            replace_homoglyphs("Coccinella 2-puſtulata Linnæus, 1758"),
520            "Coccinella 2-pustulata Linnæus, 1758"
521        );
522    }
523
524    /// The two golden-corpus names this port's `replace_homoglyphs` stub used to leave
525    /// untouched (`crates/nameparser/tests/parse_golden.rs`, `specificEpithet`/`warnings`
526    /// mismatches, now 0). `genus="Amphisbæna"` keeps its `æ` — only the specific epithet's
527    /// `ſ` is a homoglyph here.
528    #[test]
529    fn corpus_long_s_names_fold_to_s() {
530        assert_eq!(replace_homoglyphs("Musca domeſtica"), "Musca domestica");
531        assert_eq!(
532            replace_homoglyphs("Amphisbæna fuliginoſa"),
533            "Amphisbæna fuliginosa"
534        );
535    }
536
537    /// Java `UnicodeUtilsTest.containsHomoglyphs` (`UnicodeUtilsTest.java:77`, comment
538    /// "hybrid marker is fine in out domain!"): U+00D7 '×' is excluded from the table even
539    /// though it sits on the `x`-canonical row, so a genuine hybrid-formula name survives
540    /// `replace_homoglyphs` untouched.
541    #[test]
542    fn hybrid_marker_is_not_a_homoglyph() {
543        assert!(!contains_homoglyphs("Abies × Picea"));
544        assert_eq!(replace_homoglyphs("Abies × Picea"), "Abies × Picea");
545    }
546
547    /// Java `UnicodeUtilsTest.replaceHomoglyphs` (`UnicodeUtilsTest.java:119-120`, the
548    /// `composed` string): digraphs/ligatures round-trip unchanged — this table only
549    /// touches genuine cross-script look-alikes, never legitimate distinct Latin letters.
550    #[test]
551    fn ligatures_and_composed_letters_are_untouched() {
552        let composed = "æÆœŒIJijLjljȸȹßstſtfffiflffiffl";
553        assert!(!contains_homoglyphs(composed));
554        assert_eq!(replace_homoglyphs(composed), composed);
555    }
556
557    /// A Cyrillic 'а' (U+0430, `homoglyphs.txt` line 67's `a`-canonical row) look-alike
558    /// mid-word folds to plain "a", flagging as a homoglyph.
559    #[test]
560    fn cyrillic_a_lookalike_folds_to_latin_a() {
561        assert!(contains_homoglyphs("Aus \u{0430}bus"));
562        assert_eq!(replace_homoglyphs("Aus \u{0430}bus"), "Aus abus");
563    }
564
565    /// A plain-ASCII scientific name contains no homoglyphs and round-trips byte-identical.
566    #[test]
567    fn plain_ascii_name_has_no_homoglyphs() {
568        assert!(!contains_homoglyphs("Abies alba Mill."));
569        assert_eq!(replace_homoglyphs("Abies alba Mill."), "Abies alba Mill.");
570    }
571
572    /// `¡` (U+00A1) is a Win-1252 transcription artefact handled by the SEPARATE
573    /// `repairWin1252Artefacts` step (`pipeline::stripandstash`'s `repair_win1252_artefacts`
574    /// step 14) — it is not in the homoglyph table itself. Java
575    /// `UnicodeUtilsTest.replaceHomoglyphs` (`UnicodeUtilsTest.java:115`, comment "should
576    /// this be a homoglyph?") asserts the same non-membership.
577    #[test]
578    fn win1252_artefacts_are_not_homoglyphs() {
579        assert!(!contains_homoglyphs("¡i"));
580        assert_eq!(replace_homoglyphs("¡i"), "¡i");
581    }
582
583    #[test]
584    fn folds_curly_single_quotes_to_ascii_apostrophe() {
585        assert_eq!(normalize_quotes("d\u{2019}Urville"), "d'Urville");
586        assert_eq!(normalize_quotes("\u{2018}Aus\u{2019}"), "'Aus'");
587    }
588
589    #[test]
590    fn repair_mojibake_decodes_utf8_read_as_windows_1252() {
591        assert_eq!(repair_mojibake("Gürke"), "Gürke");
592        assert_eq!(repair_mojibake("(Fée) Alston"), "(Fée) Alston");
593        assert_eq!(repair_mojibake("Å¡korniÄ\u{8d}ka"), "škornička");
594        assert_eq!(repair_mojibake("RamÃ\u{ad}rez"), "Ramírez");
595        assert_eq!(repair_mojibake("Ã\u{a0} la"), "à la");
596        // nothing to repair, or broken beyond it: borrowed as is
597        for s in [
598            "Schätti",
599            "Ålesund",
600            "Å korniÄ k.",
601            "Va^Zut",
602            "Müller–Schmidt",
603            "‘Gartenspaß’",
604        ] {
605            assert!(matches!(repair_mojibake(s), Cow::Borrowed(_)), "{s}");
606        }
607    }
608
609    #[test]
610    fn normalize_input_drops_invisibles_and_folds_spaces_and_fullwidth_forms() {
611        assert_eq!(
612            normalize_input("\u{feff}Abies\u{a0}alba Mi\u{200d}ll.\u{200b}"),
613            "Abies alba Mill."
614        );
615        assert_eq!(normalize_input("(Tubangui,1928)"), "(Tubangui,1928)");
616        assert_eq!(normalize_input("Mill."), "Mill.");
617        assert_eq!(normalize_input("Za\u{301}gors\u{30c}ek"), "Zágoršek");
618        assert!(matches!(normalize_input("Abies alba L."), Cow::Borrowed(_)));
619        // the soft hyphen in the mojibake for `í` is repaired with it, stands for a hyphen before a
620        // capital and is dropped elsewhere
621        assert_eq!(normalize_input("Ort\u{c3}\u{ad}z"), "Ortíz");
622        assert_eq!(normalize_input("Miranda\u{ad}Ribeiro"), "Miranda-Ribeiro");
623        assert_eq!(normalize_input("novae\u{ad}zelandiae"), "novaezelandiae");
624        assert_eq!(normalize_input("D.\u{ad}Hawksw."), "D.Hawksw.");
625    }
626
627    #[test]
628    fn normalize_spaces_folds_unicode_space_separators_only() {
629        assert_eq!(
630            normalize_spaces("Abies\u{a0}alba\u{202f}L.\u{3000}x"),
631            "Abies alba L. x"
632        );
633        assert!(matches!(
634            normalize_spaces("Abies alba\tL."),
635            Cow::Borrowed(_)
636        ));
637    }
638
639    #[test]
640    fn folds_curly_double_quotes_to_ascii_double_quote() {
641        assert_eq!(normalize_quotes("\u{201C}Aus bus\u{201D}"), "\"Aus bus\"");
642    }
643
644    #[test]
645    fn folds_backtick_acute_and_fullwidth_apostrophe_variants() {
646        assert_eq!(normalize_quotes("\u{0060}"), "'");
647        assert_eq!(normalize_quotes("\u{00B4}"), "'");
648        assert_eq!(normalize_quotes("\u{FF07}"), "'");
649    }
650
651    #[test]
652    fn folds_low9_and_prime_double_quote_variants() {
653        assert_eq!(normalize_quotes("\u{201E}Aus\u{201F}"), "\"Aus\"");
654        assert_eq!(normalize_quotes("\u{2033}"), "\"");
655    }
656
657    #[test]
658    fn leaves_plain_ascii_and_already_normalised_quotes_untouched() {
659        assert_eq!(normalize_quotes("O'Brien"), "O'Brien");
660        assert_eq!(normalize_quotes("\"Aus bus\" Smith"), "\"Aus bus\" Smith");
661        assert_eq!(normalize_quotes("Abies alba Mill."), "Abies alba Mill.");
662    }
663
664    #[test]
665    fn java_trim_strips_only_up_to_u0020() {
666        assert_eq!(java_trim("  x\t"), "x"); // ASCII space + tab (<=0x20) stripped
667        assert_eq!(java_trim("\n x \r"), "x"); // newline/CR stripped
668        assert_eq!(java_trim("\u{00A0}x\u{00A0}"), "\u{00A0}x\u{00A0}"); // NBSP (>0x20) NOT stripped
669        assert_eq!(java_trim("\u{00A0}"), "\u{00A0}"); // NBSP-only stays non-empty, matching Java
670    }
671}