Skip to main content

nameparser/
unicode.rs

1// SPDX-License-Identifier: Apache-2.0
2//! Java `org.gbif.nameparser.util.UnicodeUtils` — ported subset: `normalizeQuotes`, plus
3//! the homoglyph-replacement table (`HOMOGLYHPS`/`containsHomoglyphs`/`replaceHomoglyphs`)
4//! used by `pipeline::stripandstash`'s step 13. `decompose`/`foldToAscii`/
5//! `replaceSpecialCases`/`removeNonAscii`/`replaceNonAscii`/`unescapeUnicodeChars`/
6//! `decodeUtf8Garbage`/`containsDiacritics`/`findDiacritics` are NOT ported: no call site
7//! anywhere in this crate's (now complete) pipeline reaches them, and the full-corpus
8//! Rust-vs-Java parity cross-validation (see `BENCHMARKS.md`/`cross-validation.md`, 0
9//! mismatches) confirms their absence isn't a gap.
10
11/// Unicode apostrophe / single-quote variants that normalise to the ASCII apostrophe `'`.
12/// Verbatim transcription of Java `UnicodeUtils.SINGLE_QUOTES`
13/// (`"`´ʹʻʼʽˊˋ‘’‘’‚‛′‵❛❜'"`).
14/// Every codepoint is a 4-hex-digit Java `\u` escape, i.e. a single UTF-16 code unit in
15/// the Basic Multilingual Plane outside the surrogate range — so iterating Rust `char`s
16/// (Unicode scalar values) reproduces Java's `char`-at-a-time (UTF-16 code unit) loop
17/// exactly for this fixed table.
18const SINGLE_QUOTES: &[char] = &[
19    '\u{0060}', '\u{00B4}', '\u{02B9}', '\u{02BB}', '\u{02BC}', '\u{02BD}', '\u{02CA}', '\u{02CB}',
20    '\u{0091}', '\u{0092}', '\u{2018}', '\u{2019}', '\u{201A}', '\u{201B}', '\u{2032}', '\u{2035}',
21    '\u{275B}', '\u{275C}', '\u{FF07}',
22];
23
24/// Unicode double-quote / double-prime variants that normalise to the ASCII double quote
25/// `"`. Verbatim transcription of Java `UnicodeUtils.DOUBLE_QUOTES`
26/// (`"“”“”„‟″‶❝❞〝〞〟""`).
27const DOUBLE_QUOTES: &[char] = &[
28    '\u{0093}', '\u{0094}', '\u{201C}', '\u{201D}', '\u{201E}', '\u{201F}', '\u{2033}', '\u{2036}',
29    '\u{275D}', '\u{275E}', '\u{301D}', '\u{301E}', '\u{301F}', '\u{FF02}',
30];
31
32// ---------------------------------------------------------------------------------------
33// Homoglyph table: Java `UnicodeUtils.HOMOGLYHPS`/`containsHomoglyphs`/`replaceHomoglyphs`
34// (`UnicodeUtils.java:58-139` for the static-init loader, `:194-215` and `:239-276` for the
35// public methods). Used by `pipeline::stripandstash`'s step 13 (`replaceHomoglyphs`) to
36// fold Latin look-alike characters from other scripts (e.g. Cyrillic 'а' U+0430) — and a
37// handful of archaic Latin typography, notably U+017F LATIN SMALL LETTER LONG S "ſ" — down
38// to their canonical ASCII/Latin letter.
39// ---------------------------------------------------------------------------------------
40
41use std::collections::HashMap;
42use std::sync::LazyLock;
43
44/// The homoglyph table, embedded at compile time (copied verbatim from
45/// `name-parser-api/src/main/resources/unicode/homoglyphs.txt`, itself sourced from
46/// <https://raw.githubusercontent.com/codebox/homoglyph/master/raw_data/chars.txt> per the
47/// resource file's own header comment — `diff`-verified byte-for-byte identical to the
48/// Java classpath resource). Each line lists a "canonical" character (the line's first
49/// char) followed by every codepoint the codebox project considers a visual look-alike of
50/// it. 1861 lines total (`\n`-delimited; the file's own last line has no trailing
51/// terminator, the same off-by-one `wc -l` undercounts documented on
52/// `pipeline::blacklisted_epithets`'s `BLACKLIST_TXT`) — but see [`HOMOGLYPHS`]'s own doc
53/// comment: Java's loader, and this port, only ever read a small prefix of it.
54const HOMOGLYPHS_TXT: &str = include_str!("../resources/homoglyphs.txt");
55
56/// U+00D7 MULTIPLICATION SIGN, the botanical hybrid marker ("Abies × Picea"). Java
57/// `NameFormatter.HYBRID_MARKER` (`NameFormatter.java:14`: `public static final char
58/// HYBRID_MARKER = '×';`). Excluded from the table [`HOMOGLYPHS`] builds below — even
59/// though it appears, unremoved, as one of the look-alikes on `homoglyphs.txt`'s `x` row
60/// (line 88) — so a genuine hybrid marker is never folded into a plain "x", which would
61/// silently destroy hybrid-name semantics. Java's own `UnicodeUtilsTest.containsHomoglyphs`
62/// asserts exactly this: `"Abies × Picea"` is NOT flagged as containing a homoglyph.
63const HYBRID_MARKER: char = '\u{00D7}';
64
65/// Java `ignoredCanonicals` (`UnicodeUtils.java:67`): `new CharArraySet(new char[]{' ',
66/// '\'', '-', '﹘'})`. A `homoglyphs.txt` row whose canonical (first) character is one of
67/// these four is skipped ENTIRELY — no homoglyph entries at all are taken from that row.
68/// U+0020 SPACE, U+0027 APOSTROPHE, U+002D HYPHEN-MINUS, U+FE58 SMALL EM DASH.
69const IGNORED_CANONICALS: [char; 4] = [' ', '\'', '-', '\u{FE58}'];
70
71/// Java's local `ignore` set (`UnicodeUtils.java:93-103`, rebuilt from the literal
72/// `"‘’“”"` on every loop iteration but with identical fixed content
73/// each time — a code-style quirk, not per-row-varying data). Curly single/double-quote
74/// variants are never captured as a homoglyph VALUE on ANY row — quote normalisation is
75/// [`normalize_quotes`]'s job, not this table's.
76const QUOTE_IGNORE: [char; 4] = ['\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}'];
77
78/// Java `DIACRITICS` (`UnicodeUtils.java:33-56`, static-init block): standalone
79/// combining/spacing diacritic marks, excluded from ever being captured as a homoglyph
80/// VALUE (a bare combining accent isn't meaningfully "the same as" some base letter, even
81/// where the auto-generated codebox table lists it as a look-alike on some canonical's
82/// row). Verbatim transcription of the Java string literal
83/// `"´˝\` ̏ˆˇ˘ ̑¸¨· ̡ ̢ ̉ ̛ˉ˛ ˚˳῾᾿"` with its 7 embedded literal spaces filtered out
84/// (`.filter(cp -> cp != 32)`, exactly like the Java) — 21 codepoints, extracted
85/// programmatically from the Java source (not hand-transcribed from the hard-to-read
86/// rendered glyphs, several of which are zero-width combining marks that visually vanish
87/// or merge with a neighbour when displayed).
88const DIACRITICS: [char; 21] = [
89    '\u{00B4}', '\u{02DD}', '\u{0060}', '\u{030F}', '\u{02C6}', '\u{02C7}', '\u{02D8}', '\u{0311}',
90    '\u{00B8}', '\u{00A8}', '\u{00B7}', '\u{0321}', '\u{0322}', '\u{0309}', '\u{031B}', '\u{02C9}',
91    '\u{02DB}', '\u{02DA}', '\u{02F3}', '\u{1FFE}', '\u{1FBF}',
92];
93
94/// Java `UnicodeUtils.HOMOGLYHPS` (`UnicodeUtils.java:59-139`, static-init block): a
95/// codepoint -> canonical-char map, built once from [`HOMOGLYPHS_TXT`]. Faithful port of
96/// the loader's exact algorithm, verified against a compiled run of the real Java
97/// `LineReader` over the real resource file (see the loop body comments below for the
98/// specific findings that verification surfaced):
99///
100/// 1. `'ſ'` (U+017F LATIN SMALL LETTER LONG S) -> `'s'` is inserted FIRST, before any table
101///    row loads, so it wins over row `f`'s own listing of `'ſ'` as one of ITS look-alikes
102///    (`homoglyphs.txt` line 72, canonical `f`) — the loop below never overwrites an
103///    existing key. Java `UnicodeUtils.java:71`: `homoglyphs.put(toCodePoint('ſ'), 's');`.
104/// 2. Each remaining `homoglyphs.txt` line is a candidate row, in file order. Java's
105///    `LineReader` (`skipBlank=true, skipComments=true`, the defaults its 1-arg
106///    `InputStream` constructor chains to) skips blank and `#`-comment lines before they
107///    ever reach this loop. A plain `line.is_empty()` check stands in for `LineReader`'s
108///    fuller `StringUtils.isBlank` (which additionally treats any all-whitespace line as
109///    blank): the ONE line in this file that's all-whitespace-but-not-empty (line 9, a run
110///    of Unicode space variants: U+0020 U+0020 U+1680 U+2000-U+200A U+2028 U+2029 U+202F
111///    U+205F) is NOT actually blank under Java's exact `Character.isWhitespace` (which
112///    excludes U+2007 FIGURE SPACE and U+202F NARROW NO-BREAK SPACE, both present on that
113///    line) — so Java's `LineReader` does NOT skip it either, and it reaches
114///    `UnicodeUtils`'s own loop body same as any other row. It's discarded there instead,
115///    by step 3's `IGNORED_CANONICALS` check (its canonical, the first char, is U+0020
116///    SPACE) — so the outcome is identical either way; `line.is_empty()` here is not a
117///    faithfulness gap.
118/// 3. The row's canonical is its first char (`.chars().next()` on a non-empty `str` is
119///    exactly Java's `line.charAt(0)` here — the canonical is never a surrogate pair, and a
120///    Rust `char` is already a full Unicode scalar value). If the canonical is one of
121///    [`IGNORED_CANONICALS`], the entire row is skipped.
122/// 4. Otherwise, every remaining char on the row becomes a candidate homoglyph VALUE,
123///    filtered exactly as `UnicodeUtils.java:104-111` filters it: codepoint > 128 (skip
124///    ASCII/Latin-1-control), not [`HYBRID_MARKER`], not in [`DIACRITICS`], not already a
125///    map key (first row wins — relevant beyond just the `'ſ'` pre-seed whenever the same
126///    look-alike appears on more than one row), not in [`QUOTE_IGNORE`].
127/// 5. After a (non-ignored-canonical) row is fully processed, Java breaks the loop
128///    (`UnicodeUtils.java:128`: `if (lr.getRow() > 175 || 'ɸ' == canonical) break;`,
129///    comment "skip all rare chars"). `lr.getRow()` is the row's own 1-indexed PHYSICAL
130///    line number in the source file (confirmed by compiling and running the real
131///    `LineReader` against the real resource file) — i.e. exactly this loop's `row`
132///    variable below, a straight `enumerate()` index, not a count of rows actually
133///    accepted. The `> 175` numeric fallback is dead code against the actual resource file
134///    — canonical `'ɸ'` (U+0278, physical line 151) always fires first — but ported
135///    verbatim anyway, matching this port's established precedent of preserving
136///    apparently-dead Java guards faithfully (see e.g. `stash_trailing_strain_code`'s
137///    `DIGITS_ONLY` guard in `pipeline::stripandstash`). Net effect: only `homoglyphs.txt`
138///    lines 1-151 are ever consulted — lines 1-8 are the file's own comment header (all
139///    `#`-prefixed), line 9 is the ignored space-canonical row, 2 more rows in range (14,
140///    canonical `'`; 20, canonical `-`) are also [`IGNORED_CANONICALS`], and the remaining
141///    140 rows (lines 10-151, minus those 2) each contribute at least one map entry — lines
142///    152-1861 (the file's long CJK "duplicate codepoint" tail, e.g. `"𦰶𦰶"`) are never
143///    read into the map at all — by Java's own design, not a deferral this port introduces.
144static HOMOGLYPHS: LazyLock<HashMap<char, char>> = LazyLock::new(|| {
145    let mut map = HashMap::new();
146    map.insert('\u{017F}', 's'); // 'ſ' -> 's', pre-seeded (see step 1 above)
147
148    for (idx, line) in HOMOGLYPHS_TXT.lines().enumerate() {
149        let row = idx + 1; // 1-indexed physical line number == Java `lr.getRow()`
150        if line.is_empty() || line.starts_with('#') {
151            continue;
152        }
153        let canonical = line.chars().next().expect("checked non-empty above");
154        if IGNORED_CANONICALS.contains(&canonical) {
155            continue;
156        }
157        for cp in line.chars().skip(1) {
158            if (cp as u32) > 128
159                && cp != HYBRID_MARKER
160                && !DIACRITICS.contains(&cp)
161                && !map.contains_key(&cp)
162                && !QUOTE_IGNORE.contains(&cp)
163            {
164                map.insert(cp, canonical);
165            }
166        }
167        if row > 175 || canonical == 'ɸ' {
168            break;
169        }
170    }
171    map
172});
173
174/// Java `UnicodeUtils.containsHomoglyphs(CharSequence)` (`UnicodeUtils.java:194-196`,
175/// delegating to `findHomoglyph` at `:203-215`). Returns true if `s` contains at least one
176/// character that is a known homoglyph of a Latin letter. Java's version short-circuits via
177/// a codepoint lower/upper-bound check before the map lookup (an `Int2CharMap`-specific
178/// micro-optimisation); a `HashMap<char, _>` lookup here is already O(1) average without
179/// needing that bound check reproduced.
180pub fn contains_homoglyphs(s: &str) -> bool {
181    s.chars().any(|c| HOMOGLYPHS.contains_key(&c))
182}
183
184/// Java `UnicodeUtils.replaceHomoglyphs(CharSequence, boolean)` / its 3-arg sibling
185/// (`UnicodeUtils.java:239-276`), specialised to the only shape any ported call site ever
186/// needs: `inclHyphens=false` and no `keep` set. `pipeline::stripandstash`'s step 13 (the
187/// sole caller) always passes `inclHyphens=false` — Unicode hyphen variants are already
188/// normalised to ASCII `-` by the immediately-preceding step 12 (`normaliseHyphens`), per
189/// that Java method's own comment ("Hyphen homoglyphs are intentionally excluded — those
190/// are normalised above already") — so `HYPHEN_HOMOGLYHPS` (`UnicodeUtils.java:141-167`)
191/// and the `keep`-list parameter are both unreachable dead weight for this port's actual
192/// call graph and are not ported. Replaces every character present as a key in
193/// [`HOMOGLYPHS`] with its mapped canonical; every other character passes through
194/// unchanged.
195pub fn replace_homoglyphs(s: &str) -> String {
196    s.chars()
197        .map(|c| *HOMOGLYPHS.get(&c).unwrap_or(&c))
198        .collect()
199}
200
201/// Faithful port of Java `String.trim()`: strips only leading/trailing chars whose
202/// codepoint is <= U+0020 (NOT the full Unicode White_Space set that Rust's str::trim uses).
203/// Use this everywhere the Java source calls `.trim()`.
204///
205/// Java's `String.trim()` only removes characters with codepoint values 0–32 (space through
206/// control chars), while Rust's `str::trim()` removes the entire Unicode `White_Space` category.
207/// This means non-breaking space (U+00A0) and other Unicode spaces are trimmed by Rust but NOT
208/// by Java, causing empty/length guard logic to diverge.
209pub fn java_trim(s: &str) -> &str {
210    s.trim_matches(|c: char| (c as u32) <= 0x20)
211}
212
213/// Normalises the many unicode apostrophe / single-quote variants to the ASCII apostrophe
214/// (') and the unicode double-quote variants to the ASCII double quote ("). Author names
215/// and quoted/provisional names routinely arrive with curly, prime, modifier-letter,
216/// low-9, fullwidth or Windows-1252 quotes; collapsing them to ASCII keeps tokenisation
217/// and the parsed output consistent regardless of the input's quote style.
218///
219/// Faithful port of Java `UnicodeUtils.normalizeQuotes(String)`. Java's `null`-in/`null`-out
220/// case is not ported: this takes and returns non-nullable Rust types, so callers threading
221/// an optional authorship string (e.g. `Option<&str>`) map over it instead
222/// (`authorship.map(normalize_quotes)`), which reproduces the same null-forwarding at the
223/// `Option` level.
224pub fn normalize_quotes(x: &str) -> String {
225    x.chars()
226        .map(|c| {
227            if SINGLE_QUOTES.contains(&c) {
228                '\''
229            } else if DOUBLE_QUOTES.contains(&c) {
230                '"'
231            } else {
232                c
233            }
234        })
235        .collect()
236}
237
238#[cfg(test)]
239mod tests {
240    use super::*;
241
242    // ---- Homoglyph table ----
243
244    /// `homoglyphs.txt` lines 10-151 (142 physical lines) are the candidate data rows once
245    /// the 8-line `#`-comment header and line 9's ignored space-canonical row are behind us
246    /// (see [`HOMOGLYPHS`]'s own doc comment); 2 more of those 142 (line 14, canonical `'`;
247    /// line 20, canonical `-`) are also [`IGNORED_CANONICALS`], leaving 140 rows that each
248    /// contribute at least one homoglyph codepoint (independently verified against the
249    /// source file — no row contributes zero) — plus the `'ſ'` pre-seed, for an exact total
250    /// of 1751 map entries. Asserted exactly (not just "a lot"), as a direct regression
251    /// guard against a bad resource copy or a loader loop that silently stops early/never
252    /// reaches `'ɸ'`/double-counts/under-filters.
253    #[test]
254    fn resource_loads_exactly_1751_entries() {
255        assert_eq!(
256            HOMOGLYPHS.len(),
257            1751,
258            "loaded homoglyph count drifted — see this test's own doc comment"
259        );
260    }
261
262    /// Java `UnicodeUtilsTest.replaceHomoglyphs` (`UnicodeUtilsTest.java:117`): long-s
263    /// folds to plain "s" (NOT "f", even though `ſ` sits on `homoglyphs.txt`'s `f`-canonical
264    /// row too — the manual pre-seed wins), while `æ` — a genuine Latin ligature, not a
265    /// look-alike — is untouched; this table is not `foldToAscii`. Same shape as the
266    /// corpus's `specificEpithet`-affecting rows ("Musca domeſtica", "Amphisbæna
267    /// fuliginoſa") — see the sibling test below and `pipeline::stripandstash`'s golden
268    /// harness `ALLOWLIST` (now empty, see that file's own history).
269    #[test]
270    fn long_s_folds_to_s_but_ae_ligature_is_untouched() {
271        assert_eq!(
272            replace_homoglyphs("Coccinella 2-puſtulata Linnæus, 1758"),
273            "Coccinella 2-pustulata Linnæus, 1758"
274        );
275    }
276
277    /// The two golden-corpus names this port's `replace_homoglyphs` stub used to leave
278    /// untouched (`crates/nameparser/tests/parse_golden.rs`, `specificEpithet`/`warnings`
279    /// mismatches, now 0). `genus="Amphisbæna"` keeps its `æ` — only the specific epithet's
280    /// `ſ` is a homoglyph here.
281    #[test]
282    fn corpus_long_s_names_fold_to_s() {
283        assert_eq!(replace_homoglyphs("Musca domeſtica"), "Musca domestica");
284        assert_eq!(
285            replace_homoglyphs("Amphisbæna fuliginoſa"),
286            "Amphisbæna fuliginosa"
287        );
288    }
289
290    /// Java `UnicodeUtilsTest.containsHomoglyphs` (`UnicodeUtilsTest.java:77`, comment
291    /// "hybrid marker is fine in out domain!"): U+00D7 '×' is excluded from the table even
292    /// though it sits on the `x`-canonical row, so a genuine hybrid-formula name survives
293    /// `replace_homoglyphs` untouched.
294    #[test]
295    fn hybrid_marker_is_not_a_homoglyph() {
296        assert!(!contains_homoglyphs("Abies × Picea"));
297        assert_eq!(replace_homoglyphs("Abies × Picea"), "Abies × Picea");
298    }
299
300    /// Java `UnicodeUtilsTest.replaceHomoglyphs` (`UnicodeUtilsTest.java:119-120`, the
301    /// `composed` string): digraphs/ligatures round-trip unchanged — this table only
302    /// touches genuine cross-script look-alikes, never legitimate distinct Latin letters.
303    #[test]
304    fn ligatures_and_composed_letters_are_untouched() {
305        let composed = "æÆœŒIJijLjljȸȹßstſtfffiflffiffl";
306        assert!(!contains_homoglyphs(composed));
307        assert_eq!(replace_homoglyphs(composed), composed);
308    }
309
310    /// A Cyrillic 'а' (U+0430, `homoglyphs.txt` line 67's `a`-canonical row) look-alike
311    /// mid-word folds to plain "a", flagging as a homoglyph.
312    #[test]
313    fn cyrillic_a_lookalike_folds_to_latin_a() {
314        assert!(contains_homoglyphs("Aus \u{0430}bus"));
315        assert_eq!(replace_homoglyphs("Aus \u{0430}bus"), "Aus abus");
316    }
317
318    /// A plain-ASCII scientific name contains no homoglyphs and round-trips byte-identical.
319    #[test]
320    fn plain_ascii_name_has_no_homoglyphs() {
321        assert!(!contains_homoglyphs("Abies alba Mill."));
322        assert_eq!(replace_homoglyphs("Abies alba Mill."), "Abies alba Mill.");
323    }
324
325    /// `¡` (U+00A1) is a Win-1252 transcription artefact handled by the SEPARATE
326    /// `repairWin1252Artefacts` step (`pipeline::stripandstash`'s `repair_win1252_artefacts`
327    /// step 14) — it is not in the homoglyph table itself. Java
328    /// `UnicodeUtilsTest.replaceHomoglyphs` (`UnicodeUtilsTest.java:115`, comment "should
329    /// this be a homoglyph?") asserts the same non-membership.
330    #[test]
331    fn win1252_artefacts_are_not_homoglyphs() {
332        assert!(!contains_homoglyphs("¡i"));
333        assert_eq!(replace_homoglyphs("¡i"), "¡i");
334    }
335
336    #[test]
337    fn folds_curly_single_quotes_to_ascii_apostrophe() {
338        assert_eq!(normalize_quotes("d\u{2019}Urville"), "d'Urville");
339        assert_eq!(normalize_quotes("\u{2018}Aus\u{2019}"), "'Aus'");
340    }
341
342    #[test]
343    fn folds_curly_double_quotes_to_ascii_double_quote() {
344        assert_eq!(normalize_quotes("\u{201C}Aus bus\u{201D}"), "\"Aus bus\"");
345    }
346
347    #[test]
348    fn folds_backtick_acute_and_fullwidth_apostrophe_variants() {
349        assert_eq!(normalize_quotes("\u{0060}"), "'");
350        assert_eq!(normalize_quotes("\u{00B4}"), "'");
351        assert_eq!(normalize_quotes("\u{FF07}"), "'");
352    }
353
354    #[test]
355    fn folds_low9_and_prime_double_quote_variants() {
356        assert_eq!(normalize_quotes("\u{201E}Aus\u{201F}"), "\"Aus\"");
357        assert_eq!(normalize_quotes("\u{2033}"), "\"");
358    }
359
360    #[test]
361    fn leaves_plain_ascii_and_already_normalised_quotes_untouched() {
362        assert_eq!(normalize_quotes("O'Brien"), "O'Brien");
363        assert_eq!(normalize_quotes("\"Aus bus\" Smith"), "\"Aus bus\" Smith");
364        assert_eq!(normalize_quotes("Abies alba Mill."), "Abies alba Mill.");
365    }
366
367    #[test]
368    fn java_trim_strips_only_up_to_u0020() {
369        assert_eq!(java_trim("  x\t"), "x"); // ASCII space + tab (<=0x20) stripped
370        assert_eq!(java_trim("\n x \r"), "x"); // newline/CR stripped
371        assert_eq!(java_trim("\u{00A0}x\u{00A0}"), "\u{00A0}x\u{00A0}"); // NBSP (>0x20) NOT stripped
372        assert_eq!(java_trim("\u{00A0}"), "\u{00A0}"); // NBSP-only stays non-empty, matching Java
373    }
374}