Skip to main content

nameparser/
token.rs

1// SPDX-License-Identifier: Apache-2.0
2
3use std::sync::LazyLock;
4
5use regex::Regex;
6
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub enum TokenKind {
9    Word,
10    Number,
11    HybridMark,
12    OpenParen,
13    CloseParen,
14    OpenBracket,
15    CloseBracket,
16    OpenBrace,
17    CloseBrace,
18    Comma,
19    Semicolon,
20    Colon,
21    Dot,
22    Ampersand,
23    Dagger,
24    Other,
25}
26
27#[derive(Debug, Clone, PartialEq, Eq)]
28pub struct Token {
29    pub kind: TokenKind,
30    /// The verbatim matched text.
31    pub text: String,
32    /// Byte offset into the input (UTF-8). NB: Java uses UTF-16 code-unit offsets.
33    pub start: usize,
34    pub end: usize,
35}
36
37/// java.lang.Character.isWhitespace — Unicode space separators EXCEPT the non-breaking
38/// ones (U+00A0, U+2007, U+202F), plus the ASCII/control whitespace incl. the file/group/
39/// record/unit separators. Rust's `char::is_whitespace` differs: it INCLUDES U+00A0 and
40/// EXCLUDES U+001C–U+001F. This helper matches Java. (Global constraint: Unicode semantics.)
41/// `pub(crate)`: also used by `pipeline::stripandstash`'s `first_word` (Java
42/// `StripAndStash.firstWord`, Phase 1 Slice 2 Task 3), not just this module's own tokenizer.
43pub(crate) fn is_whitespace_java(c: char) -> bool {
44    match c {
45        '\u{00A0}' | '\u{2007}' | '\u{202F}' => false,
46        '\u{001C}' | '\u{001D}' | '\u{001E}' | '\u{001F}' => true,
47        _ => c.is_whitespace(),
48    }
49}
50
51/// Java `org.gbif.nameparser.token.AuthorParticles.PARTICLES` — lower-case particles that
52/// combine with a following capitalised author surname ("de Vriese", "Van Heurck", "von
53/// der Linde"). Full 50-entry set transcribed verbatim from `AuthorParticles.java` (and
54/// verified programmatically against it — an exact set match, no entries dropped or
55/// added). The first call site (`StripAndStash::apply_missing_genus_placeholder`, Phase 1
56/// Slice 2 Task 3) only needed the boolean `is_particle` — `NameTokens`/`AuthorshipSplit`/
57/// `AuthorshipParser` now call it too (mirroring Java's own `isCapitalisedParticle`, which
58/// needs the same table), so porting the full set once here rather than a hand-trimmed
59/// subset paid off.
60const PARTICLES: &[&str] = &[
61    "a", "ab", "af", "ap", "auf", "d", "da", "dal", "dalla", "dalle", "dallo", "das", "de",
62    "degli", "dei", "del", "della", "delle", "delli", "dello", "der", "des", "di", "do", "dos",
63    "du", "el", "in", "la", "las", "le", "les", "lo", "los", "of", "ofver", "te", "ten", "ter",
64    "und", "v", "van", "vd", "ven", "vom", "von", "y", "zu", "zum", "zur",
65];
66
67/// Java `AuthorParticles.isParticle(String)`: `PARTICLES.contains(word.toLowerCase())`.
68/// Java's `.toLowerCase()` uses the JVM default locale; Rust's `.to_lowercase()` is the
69/// full-Unicode locale-independent mapping — every entry in `PARTICLES` is pure ASCII, so
70/// the two only-differ-for-Turkish-dotless-I-etc. divergences that exist in principle
71/// cannot change whether any of these ASCII words match (same reasoning already applied to
72/// `Preflight::run`'s `s.to_lowercase()` call).
73pub(crate) fn is_particle(word: &str) -> bool {
74    let word = word.to_lowercase();
75    // a hyphenated run of particles too: "van-der Land", "de-la Cruz"
76    PARTICLES.contains(&word.as_str())
77        || (word.contains('-') && word.split('-').all(|p| PARTICLES.contains(&p)))
78}
79
80#[cfg(test)]
81mod author_particles_tests {
82    use super::is_particle;
83
84    #[test]
85    fn recognises_lowercase_and_mixed_case_particles() {
86        assert!(is_particle("van"));
87        assert!(is_particle("Van"));
88        assert!(is_particle("VAN"));
89        assert!(is_particle("de"));
90        assert!(is_particle("van-der"));
91        assert!(!is_particle("da-silvae"));
92    }
93
94    #[test]
95    fn rejects_non_particle_words() {
96        assert!(!is_particle("Smith"));
97        assert!(!is_particle("denheyeri"));
98        assert!(!is_particle(""));
99    }
100}
101
102/// java.lang.Character.isLetter — Unicode general category L*. Rust's `is_alphabetic` is the
103/// (slightly broader) Alphabetic property. Divergences are surfaced by the Task 3 golden diff.
104fn is_letter(c: char) -> bool {
105    c.is_alphabetic()
106}
107
108/// java.lang.Character.isDigit — Unicode Nd. Spike approximation: ASCII digits only.
109/// Non-ASCII decimal digits (rare in names) will show up as Task 3 mismatches for triage.
110fn is_digit(c: char) -> bool {
111    c.is_ascii_digit()
112}
113
114/// A numeral-prefixed LATIN EPITHET — `11-punctata`, `7-maculatus`, `4maculatus`, `2-pustulata`.
115/// One or two digits with no leading zero, an optional hyphen, then a run of at least three
116/// lowercase letters to the end.
117///
118/// The single authority for that shape, because three places must agree on it or they undo each
119/// other: [`tokenize`]'s two leading-numeral rules (which produce these WORD tokens),
120/// `StripAndStash::stash_trailing_strain_code` (which must NOT stash one as a strain code) and
121/// `NameTokens`' numbered-infraspecific phrase capture (which must NOT stash one as a phrase —
122/// `Benthogone rosea var. 4-lineata R. Perrier, 1896` has a real epithet there, and the 6.4M COL
123/// corpus found it). Real strain codes never take this shape: they keep going in digits
124/// (`18-41`), carry an uppercase segment (`47-T17`, `1-CPM-2005`) or have at most one trailing
125/// letter (`5a`).
126pub(crate) fn is_numeral_epithet(s: &str) -> bool {
127    static NUMERAL_EPITHET: LazyLock<Regex> =
128        LazyLock::new(|| Regex::new(r"^(?-u:[1-9]\d?)-?\p{Ll}{3,}$").unwrap());
129    NUMERAL_EPITHET.is_match(s)
130}
131
132/// A lowercase letter, for the unhyphenated leading-numeral epithet rule in [`tokenize`].
133/// `is_letter` + `is_lowercase` rather than `is_ascii_lowercase`, so a Latinised epithet
134/// with a non-ASCII letter is glued on the same terms as an ASCII one.
135fn is_lowercase_letter(c: char) -> bool {
136    is_letter(c) && c.is_lowercase()
137}
138
139fn is_all_upper(s: &str) -> bool {
140    for c in s.chars() {
141        if is_letter(c) && !c.is_uppercase() {
142            return false;
143        }
144    }
145    true
146}
147
148/// Single-pass tokenizer. Faithful port of the Java `Tokenizer.tokenize`.
149/// Whitespace is consumed and not emitted. Letter runs may include internal hyphens,
150/// apostrophes, underscores and stray '!' when flanked by letters/digits.
151pub fn tokenize(input: &str) -> Vec<Token> {
152    let cs: Vec<(usize, char)> = input.char_indices().collect();
153    let n = cs.len();
154    let byte_at = |k: usize| -> usize {
155        if k < n {
156            cs[k].0
157        } else {
158            input.len()
159        }
160    };
161    let mut out: Vec<Token> = Vec::with_capacity((n / 4).max(4));
162    let mut k = 0usize;
163
164    while k < n {
165        let c = cs[k].1;
166
167        if is_whitespace_java(c) {
168            k += 1;
169            continue;
170        }
171
172        if is_letter(c) {
173            let word_start = k;
174            k += 1;
175            while k < n {
176                let c2 = cs[k].1;
177                if is_letter(c2) || is_digit(c2) {
178                    k += 1;
179                    continue;
180                }
181                // internal hyphen / apostrophe / underscore / stray '!' glued between letters/digits
182                if matches!(
183                    c2,
184                    '-' | '\''
185                        | '\u{2019}'
186                        | '_'
187                        | '!'
188                        | '\u{2010}'
189                        | '\u{2011}'
190                        | '\u{2012}'
191                        | '\u{2013}'
192                        | '\u{2014}'
193                ) && k + 1 < n
194                {
195                    let next = cs[k + 1].1;
196                    if is_letter(next) || is_digit(next) {
197                        k += 1;
198                        continue;
199                    }
200                }
201                break;
202            }
203            // A lower-case "st."/"s." (sancti) hyphenated to the rest of the epithet is one word:
204            // "Macromitrium st.-johnii", "Cyathea st.-helenae" — not a French "s.-esp." (sous-espèce).
205            let abbreviated = &cs[word_start..k];
206            if matches!(abbreviated, [(_, 's')] | [(_, 's'), (_, 't')])
207                && k + 5 < n
208                && cs[k].1 == '.'
209                && cs[k + 1].1 == '-'
210                && cs[k + 2..k + 6].iter().all(|&(_, c)| c.is_lowercase())
211            {
212                k += 2;
213                while k < n
214                    && (is_letter(cs[k].1)
215                        || (cs[k].1 == '-' && k + 1 < n && is_letter(cs[k + 1].1)))
216                {
217                    k += 1;
218                }
219            }
220            let start_b = byte_at(word_start);
221            let end_b = byte_at(k);
222            let word = &input[start_b..end_b];
223            let first = word.chars().next().unwrap();
224            let char_count = word.chars().count();
225
226            if char_count == 1 && (first == 'x' || first == 'X') {
227                let left_ok = word_start == 0 || is_whitespace_java(cs[word_start - 1].1);
228                let right_ok = k == n || is_whitespace_java(cs[k].1);
229                let kind = if left_ok && right_ok {
230                    TokenKind::HybridMark
231                } else {
232                    TokenKind::Word
233                };
234                out.push(Token {
235                    kind,
236                    text: word.to_string(),
237                    start: start_b,
238                    end: end_b,
239                });
240                continue;
241            } else if char_count >= 2 && (first == 'x' || first == 'X') {
242                let second = word.chars().nth(1).unwrap();
243                if second.is_uppercase() && !is_all_upper(word) {
244                    let first_len = first.len_utf8();
245                    out.push(Token {
246                        kind: TokenKind::HybridMark,
247                        text: word[..first_len].to_string(),
248                        start: start_b,
249                        end: start_b + first_len,
250                    });
251                    out.push(Token {
252                        kind: TokenKind::Word,
253                        text: word[first_len..].to_string(),
254                        start: start_b + first_len,
255                        end: end_b,
256                    });
257                    continue;
258                }
259            }
260            out.push(Token {
261                kind: TokenKind::Word,
262                text: word.to_string(),
263                start: start_b,
264                end: end_b,
265            });
266            continue;
267        }
268
269        if is_digit(c) {
270            let num_start = k;
271            k += 1;
272            while k < n && is_digit(cs[k].1) {
273                k += 1;
274            }
275            // "11-punctata": a number glued to hyphen + letter is a leading-numeral epithet word
276            if k + 1 < n && cs[k].1 == '-' && is_letter(cs[k + 1].1) {
277                k += 1; // consume hyphen
278                while k < n {
279                    let c2 = cs[k].1;
280                    if is_letter(c2) || is_digit(c2) {
281                        k += 1;
282                        continue;
283                    }
284                    if c2 == '-' && k + 1 < n {
285                        let next = cs[k + 1].1;
286                        if is_letter(next) || is_digit(next) {
287                            k += 1;
288                            continue;
289                        }
290                    }
291                    break;
292                }
293                let s = byte_at(num_start);
294                let e = byte_at(k);
295                out.push(Token {
296                    kind: TokenKind::Word,
297                    text: input[s..e].to_string(),
298                    start: s,
299                    end: e,
300                });
301                continue;
302            }
303            // gbif/name-parser-rust#16: the UNHYPHENATED spelling of the same leading-numeral
304            // epithet — "4maculatus" for "quadrimaculatus", "11punctata". Java (and this port
305            // until now) only glued the hyphenated form, so `Camponotus sericeus 4maculatus`
306            // tokenised as Number(4) + Word(maculatus) and the authorship parser dropped the 4
307            // and promoted "maculatus" to an author, leaving a canonical string identical to the
308            // real `Camponotus sericeus`.
309            //
310            // Deliberately much narrower than the hyphenated rule above, because without the
311            // hyphen the shape collides with things that must STAY split: at most two digits (a
312            // numeral epithet is `2`..`24`, never a year — so "1976var" keeps its Number), no
313            // leading zero (a numeral epithet never has one, but the OCR of a capital O does —
314            // "0ersted" for Ørsted, "0lsson" for Olsson, both in the golden corpus, where gluing
315            // would swallow the AUTHOR into the name), then at least three LOWERCASE letters (so
316            // the year disambiguator "1935h" and the strain codes "5a" / "16S" / "2016Iso3" are
317            // all untouched), and the word must end there (so "12abc4" stays a Number too).
318            let digits = k - num_start;
319            let leading_zero = cs[num_start].1 == '0';
320            if digits <= 2 && !leading_zero && k < n && is_lowercase_letter(cs[k].1) {
321                let mut k2 = k;
322                while k2 < n && is_lowercase_letter(cs[k2].1) {
323                    k2 += 1;
324                }
325                let word_ends = k2 == n || !(is_letter(cs[k2].1) || is_digit(cs[k2].1));
326                if k2 - k >= 3 && word_ends {
327                    let s = byte_at(num_start);
328                    let e = byte_at(k2);
329                    out.push(Token {
330                        kind: TokenKind::Word,
331                        text: input[s..e].to_string(),
332                        start: s,
333                        end: e,
334                    });
335                    k = k2;
336                    continue;
337                }
338            }
339            let s = byte_at(num_start);
340            let e = byte_at(k);
341            out.push(Token {
342                kind: TokenKind::Number,
343                text: input[s..e].to_string(),
344                start: s,
345                end: e,
346            });
347            continue;
348        }
349
350        let kind = match c {
351            '(' => TokenKind::OpenParen,
352            ')' => TokenKind::CloseParen,
353            '[' => TokenKind::OpenBracket,
354            ']' => TokenKind::CloseBracket,
355            '{' => TokenKind::OpenBrace,
356            '}' => TokenKind::CloseBrace,
357            ',' => TokenKind::Comma,
358            ';' => TokenKind::Semicolon,
359            ':' => TokenKind::Colon,
360            '.' => TokenKind::Dot,
361            '&' => TokenKind::Ampersand,
362            '\u{00D7}' => TokenKind::HybridMark, // ×
363            '+' => TokenKind::HybridMark,
364            '\u{2020}' => TokenKind::Dagger, // †
365            _ => TokenKind::Other,
366        };
367        let s = byte_at(k);
368        let e = byte_at(k + 1);
369        out.push(Token {
370            kind,
371            text: input[s..e].to_string(),
372            start: s,
373            end: e,
374        });
375        k += 1;
376    }
377    out
378}
379
380#[cfg(test)]
381mod tests {
382    use super::*;
383
384    fn kinds(input: &str) -> Vec<(TokenKind, String)> {
385        tokenize(input)
386            .into_iter()
387            .map(|t| (t.kind, t.text))
388            .collect()
389    }
390
391    #[test]
392    fn simple_binomial() {
393        use TokenKind::*;
394        assert_eq!(
395            kinds("Aus bus"),
396            vec![(Word, "Aus".into()), (Word, "bus".into())]
397        );
398    }
399
400    #[test]
401    fn author_year() {
402        use TokenKind::*;
403        assert_eq!(
404            kinds("Aus bus L., 1758"),
405            vec![
406                (Word, "Aus".into()),
407                (Word, "bus".into()),
408                (Word, "L".into()),
409                (Dot, ".".into()),
410                (Comma, ",".into()),
411                (Number, "1758".into()),
412            ]
413        );
414    }
415
416    #[test]
417    fn internal_hyphen_and_apostrophe_stay_in_word() {
418        assert_eq!(kinds("Hartmann-Schroder").len(), 1);
419        assert_eq!(kinds("d'urvilleana").len(), 1);
420        assert_eq!(kinds("pu!chra").len(), 1); // stray '!' between letters
421    }
422
423    #[test]
424    fn leading_numeral_epithet_is_one_word() {
425        use TokenKind::*;
426        assert_eq!(kinds("11-punctata"), vec![(Word, "11-punctata".into())]);
427        assert_eq!(kinds("2-pustulata"), vec![(Word, "2-pustulata".into())]);
428    }
429
430    #[test]
431    fn unhyphenated_leading_numeral_epithet_is_one_word() {
432        // gbif/name-parser-rust#16
433        use TokenKind::*;
434        assert_eq!(kinds("4maculatus"), vec![(Word, "4maculatus".into())]);
435        assert_eq!(kinds("11punctata"), vec![(Word, "11punctata".into())]);
436        assert_eq!(kinds("13punctatus"), vec![(Word, "13punctatus".into())]);
437    }
438
439    #[test]
440    fn the_unhyphenated_glue_leaves_every_near_miss_split() {
441        use TokenKind::*;
442        for (input, number, rest) in [
443            // >2 digits: a year with a word glued on, not a numeral epithet
444            ("1976var", "1976", "var"),
445            // a leading zero is the OCR of a capital O ("Ørsted", "Olsson"), and gluing it
446            // would swallow the author into the name
447            ("0ersted", "0", "ersted"),
448            ("0lsson", "0", "lsson"),
449        ] {
450            assert_eq!(
451                kinds(input),
452                vec![(Number, number.into()), (Word, rest.into())],
453                "for {input:?}"
454            );
455        }
456        // <3 letters: the year disambiguator "1935h" and short strain codes stay split
457        assert_eq!(
458            kinds("1935h"),
459            vec![(Number, "1935".into()), (Word, "h".into())]
460        );
461        assert_eq!(kinds("5a"), vec![(Number, "5".into()), (Word, "a".into())]);
462        // uppercase letters are not an epithet
463        assert_eq!(
464            kinds("16S"),
465            vec![(Number, "16".into()), (Word, "S".into())]
466        );
467        // the word must END at the letter run
468        assert_eq!(
469            kinds("12abc4"),
470            vec![(Number, "12".into()), (Word, "abc4".into())]
471        );
472    }
473
474    #[test]
475    fn bare_number_is_number() {
476        use TokenKind::*;
477        assert_eq!(kinds("1758"), vec![(Number, "1758".into())]);
478    }
479
480    #[test]
481    fn hybrid_marks() {
482        use TokenKind::*;
483        // Unicode ×
484        assert_eq!(
485            kinds("Aus \u{00D7}bus"),
486            vec![
487                (Word, "Aus".into()),
488                (HybridMark, "\u{00D7}".into()),
489                (Word, "bus".into())
490            ]
491        );
492        // bare ASCII x between spaces
493        assert_eq!(
494            kinds("Aus x bus"),
495            vec![
496                (Word, "Aus".into()),
497                (HybridMark, "x".into()),
498                (Word, "bus".into())
499            ]
500        );
501        // xFoo — leading x glued to a capitalised word splits off
502        assert_eq!(
503            kinds("xBus"),
504            vec![(HybridMark, "x".into()), (Word, "Bus".into())]
505        );
506    }
507
508    #[test]
509    fn punctuation_and_dagger() {
510        use TokenKind::*;
511        assert_eq!(
512            kinds("\u{2020}Aus (Bus)"),
513            vec![
514                (Dagger, "\u{2020}".into()),
515                (Word, "Aus".into()),
516                (OpenParen, "(".into()),
517                (Word, "Bus".into()),
518                (CloseParen, ")".into()),
519            ]
520        );
521    }
522}