Skip to main content

nameparser/
token.rs

1// SPDX-License-Identifier: Apache-2.0
2
3use std::sync::LazyLock;
4
5use regex::Regex;
6
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub enum TokenKind {
9    Word,
10    Number,
11    HybridMark,
12    OpenParen,
13    CloseParen,
14    OpenBracket,
15    CloseBracket,
16    OpenBrace,
17    CloseBrace,
18    Comma,
19    Semicolon,
20    Colon,
21    Dot,
22    Ampersand,
23    Dagger,
24    Other,
25}
26
27#[derive(Debug, Clone, PartialEq, Eq)]
28pub struct Token {
29    pub kind: TokenKind,
30    /// The verbatim matched text.
31    pub text: String,
32    /// Byte offset into the input (UTF-8). NB: Java uses UTF-16 code-unit offsets.
33    pub start: usize,
34    pub end: usize,
35}
36
37/// java.lang.Character.isWhitespace — Unicode space separators EXCEPT the non-breaking
38/// ones (U+00A0, U+2007, U+202F), plus the ASCII/control whitespace incl. the file/group/
39/// record/unit separators. Rust's `char::is_whitespace` differs: it INCLUDES U+00A0 and
40/// EXCLUDES U+001C–U+001F. This helper matches Java. (Global constraint: Unicode semantics.)
41/// `pub(crate)`: also used by `pipeline::stripandstash`'s `first_word` (Java
42/// `StripAndStash.firstWord`, Phase 1 Slice 2 Task 3), not just this module's own tokenizer.
43pub(crate) fn is_whitespace_java(c: char) -> bool {
44    match c {
45        '\u{00A0}' | '\u{2007}' | '\u{202F}' => false,
46        '\u{001C}' | '\u{001D}' | '\u{001E}' | '\u{001F}' => true,
47        _ => c.is_whitespace(),
48    }
49}
50
51/// Java `org.gbif.nameparser.token.AuthorParticles.PARTICLES` — lower-case particles that
52/// combine with a following capitalised author surname ("de Vriese", "Van Heurck", "von
53/// der Linde"). Full 50-entry set transcribed verbatim from `AuthorParticles.java` (and
54/// verified programmatically against it — an exact set match, no entries dropped or
55/// added). The first call site (`StripAndStash::apply_missing_genus_placeholder`, Phase 1
56/// Slice 2 Task 3) only needed the boolean `is_particle` — `NameTokens`/`AuthorshipSplit`/
57/// `AuthorshipParser` now call it too (mirroring Java's own `isCapitalisedParticle`, which
58/// needs the same table), so porting the full set once here rather than a hand-trimmed
59/// subset paid off.
60const PARTICLES: &[&str] = &[
61    "a", "ab", "af", "ap", "auf", "d", "da", "dal", "dalla", "dalle", "dallo", "das", "de",
62    "degli", "dei", "del", "della", "delle", "delli", "dello", "der", "des", "di", "do", "dos",
63    "du", "el", "in", "la", "las", "le", "les", "lo", "los", "of", "ofver", "te", "ten", "ter",
64    "und", "v", "van", "vd", "ven", "vom", "von", "y", "zu", "zum", "zur",
65];
66
67/// Java `AuthorParticles.isParticle(String)`: `PARTICLES.contains(word.toLowerCase())`.
68/// Java's `.toLowerCase()` uses the JVM default locale; Rust's `.to_lowercase()` is the
69/// full-Unicode locale-independent mapping — every entry in `PARTICLES` is pure ASCII, so
70/// the two only-differ-for-Turkish-dotless-I-etc. divergences that exist in principle
71/// cannot change whether any of these ASCII words match (same reasoning already applied to
72/// `Preflight::run`'s `s.to_lowercase()` call).
73pub(crate) fn is_particle(word: &str) -> bool {
74    PARTICLES.contains(&word.to_lowercase().as_str())
75}
76
77#[cfg(test)]
78mod author_particles_tests {
79    use super::is_particle;
80
81    #[test]
82    fn recognises_lowercase_and_mixed_case_particles() {
83        assert!(is_particle("van"));
84        assert!(is_particle("Van"));
85        assert!(is_particle("VAN"));
86        assert!(is_particle("de"));
87    }
88
89    #[test]
90    fn rejects_non_particle_words() {
91        assert!(!is_particle("Smith"));
92        assert!(!is_particle("denheyeri"));
93        assert!(!is_particle(""));
94    }
95}
96
97/// java.lang.Character.isLetter — Unicode general category L*. Rust's `is_alphabetic` is the
98/// (slightly broader) Alphabetic property. Divergences are surfaced by the Task 3 golden diff.
99fn is_letter(c: char) -> bool {
100    c.is_alphabetic()
101}
102
103/// java.lang.Character.isDigit — Unicode Nd. Spike approximation: ASCII digits only.
104/// Non-ASCII decimal digits (rare in names) will show up as Task 3 mismatches for triage.
105fn is_digit(c: char) -> bool {
106    c.is_ascii_digit()
107}
108
109/// A numeral-prefixed LATIN EPITHET — `11-punctata`, `7-maculatus`, `4maculatus`, `2-pustulata`.
110/// One or two digits with no leading zero, an optional hyphen, then a run of at least three
111/// lowercase letters to the end.
112///
113/// The single authority for that shape, because three places must agree on it or they undo each
114/// other: [`tokenize`]'s two leading-numeral rules (which produce these WORD tokens),
115/// `StripAndStash::stash_trailing_strain_code` (which must NOT stash one as a strain code) and
116/// `NameTokens`' numbered-infraspecific phrase capture (which must NOT stash one as a phrase —
117/// `Benthogone rosea var. 4-lineata R. Perrier, 1896` has a real epithet there, and the 6.4M COL
118/// corpus found it). Real strain codes never take this shape: they keep going in digits
119/// (`18-41`), carry an uppercase segment (`47-T17`, `1-CPM-2005`) or have at most one trailing
120/// letter (`5a`).
121pub(crate) fn is_numeral_epithet(s: &str) -> bool {
122    static NUMERAL_EPITHET: LazyLock<Regex> =
123        LazyLock::new(|| Regex::new(r"^(?-u:[1-9]\d?)-?\p{Ll}{3,}$").unwrap());
124    NUMERAL_EPITHET.is_match(s)
125}
126
127/// A lowercase letter, for the unhyphenated leading-numeral epithet rule in [`tokenize`].
128/// `is_letter` + `is_lowercase` rather than `is_ascii_lowercase`, so a Latinised epithet
129/// with a non-ASCII letter is glued on the same terms as an ASCII one.
130fn is_lowercase_letter(c: char) -> bool {
131    is_letter(c) && c.is_lowercase()
132}
133
134fn is_all_upper(s: &str) -> bool {
135    for c in s.chars() {
136        if is_letter(c) && !c.is_uppercase() {
137            return false;
138        }
139    }
140    true
141}
142
143/// Single-pass tokenizer. Faithful port of the Java `Tokenizer.tokenize`.
144/// Whitespace is consumed and not emitted. Letter runs may include internal hyphens,
145/// apostrophes, underscores and stray '!' when flanked by letters/digits.
146pub fn tokenize(input: &str) -> Vec<Token> {
147    let cs: Vec<(usize, char)> = input.char_indices().collect();
148    let n = cs.len();
149    let byte_at = |k: usize| -> usize {
150        if k < n {
151            cs[k].0
152        } else {
153            input.len()
154        }
155    };
156    let mut out: Vec<Token> = Vec::with_capacity((n / 4).max(4));
157    let mut k = 0usize;
158
159    while k < n {
160        let c = cs[k].1;
161
162        if is_whitespace_java(c) {
163            k += 1;
164            continue;
165        }
166
167        if is_letter(c) {
168            let word_start = k;
169            k += 1;
170            while k < n {
171                let c2 = cs[k].1;
172                if is_letter(c2) || is_digit(c2) {
173                    k += 1;
174                    continue;
175                }
176                // internal hyphen / apostrophe / underscore / stray '!' glued between letters/digits
177                if matches!(
178                    c2,
179                    '-' | '\''
180                        | '\u{2019}'
181                        | '_'
182                        | '!'
183                        | '\u{2010}'
184                        | '\u{2011}'
185                        | '\u{2012}'
186                        | '\u{2013}'
187                        | '\u{2014}'
188                ) && k + 1 < n
189                {
190                    let next = cs[k + 1].1;
191                    if is_letter(next) || is_digit(next) {
192                        k += 1;
193                        continue;
194                    }
195                }
196                break;
197            }
198            let start_b = byte_at(word_start);
199            let end_b = byte_at(k);
200            let word = &input[start_b..end_b];
201            let first = word.chars().next().unwrap();
202            let char_count = word.chars().count();
203
204            if char_count == 1 && (first == 'x' || first == 'X') {
205                let left_ok = word_start == 0 || is_whitespace_java(cs[word_start - 1].1);
206                let right_ok = k == n || is_whitespace_java(cs[k].1);
207                let kind = if left_ok && right_ok {
208                    TokenKind::HybridMark
209                } else {
210                    TokenKind::Word
211                };
212                out.push(Token {
213                    kind,
214                    text: word.to_string(),
215                    start: start_b,
216                    end: end_b,
217                });
218                continue;
219            } else if char_count >= 2 && (first == 'x' || first == 'X') {
220                let second = word.chars().nth(1).unwrap();
221                if second.is_uppercase() && !is_all_upper(word) {
222                    let first_len = first.len_utf8();
223                    out.push(Token {
224                        kind: TokenKind::HybridMark,
225                        text: word[..first_len].to_string(),
226                        start: start_b,
227                        end: start_b + first_len,
228                    });
229                    out.push(Token {
230                        kind: TokenKind::Word,
231                        text: word[first_len..].to_string(),
232                        start: start_b + first_len,
233                        end: end_b,
234                    });
235                    continue;
236                }
237            }
238            out.push(Token {
239                kind: TokenKind::Word,
240                text: word.to_string(),
241                start: start_b,
242                end: end_b,
243            });
244            continue;
245        }
246
247        if is_digit(c) {
248            let num_start = k;
249            k += 1;
250            while k < n && is_digit(cs[k].1) {
251                k += 1;
252            }
253            // "11-punctata": a number glued to hyphen + letter is a leading-numeral epithet word
254            if k + 1 < n && cs[k].1 == '-' && is_letter(cs[k + 1].1) {
255                k += 1; // consume hyphen
256                while k < n {
257                    let c2 = cs[k].1;
258                    if is_letter(c2) || is_digit(c2) {
259                        k += 1;
260                        continue;
261                    }
262                    if c2 == '-' && k + 1 < n {
263                        let next = cs[k + 1].1;
264                        if is_letter(next) || is_digit(next) {
265                            k += 1;
266                            continue;
267                        }
268                    }
269                    break;
270                }
271                let s = byte_at(num_start);
272                let e = byte_at(k);
273                out.push(Token {
274                    kind: TokenKind::Word,
275                    text: input[s..e].to_string(),
276                    start: s,
277                    end: e,
278                });
279                continue;
280            }
281            // gbif/name-parser-rust#16: the UNHYPHENATED spelling of the same leading-numeral
282            // epithet — "4maculatus" for "quadrimaculatus", "11punctata". Java (and this port
283            // until now) only glued the hyphenated form, so `Camponotus sericeus 4maculatus`
284            // tokenised as Number(4) + Word(maculatus) and the authorship parser dropped the 4
285            // and promoted "maculatus" to an author, leaving a canonical string identical to the
286            // real `Camponotus sericeus`.
287            //
288            // Deliberately much narrower than the hyphenated rule above, because without the
289            // hyphen the shape collides with things that must STAY split: at most two digits (a
290            // numeral epithet is `2`..`24`, never a year — so "1976var" keeps its Number), no
291            // leading zero (a numeral epithet never has one, but the OCR of a capital O does —
292            // "0ersted" for Ørsted, "0lsson" for Olsson, both in the golden corpus, where gluing
293            // would swallow the AUTHOR into the name), then at least three LOWERCASE letters (so
294            // the year disambiguator "1935h" and the strain codes "5a" / "16S" / "2016Iso3" are
295            // all untouched), and the word must end there (so "12abc4" stays a Number too).
296            let digits = k - num_start;
297            let leading_zero = cs[num_start].1 == '0';
298            if digits <= 2 && !leading_zero && k < n && is_lowercase_letter(cs[k].1) {
299                let mut k2 = k;
300                while k2 < n && is_lowercase_letter(cs[k2].1) {
301                    k2 += 1;
302                }
303                let word_ends = k2 == n || !(is_letter(cs[k2].1) || is_digit(cs[k2].1));
304                if k2 - k >= 3 && word_ends {
305                    let s = byte_at(num_start);
306                    let e = byte_at(k2);
307                    out.push(Token {
308                        kind: TokenKind::Word,
309                        text: input[s..e].to_string(),
310                        start: s,
311                        end: e,
312                    });
313                    k = k2;
314                    continue;
315                }
316            }
317            let s = byte_at(num_start);
318            let e = byte_at(k);
319            out.push(Token {
320                kind: TokenKind::Number,
321                text: input[s..e].to_string(),
322                start: s,
323                end: e,
324            });
325            continue;
326        }
327
328        let kind = match c {
329            '(' => TokenKind::OpenParen,
330            ')' => TokenKind::CloseParen,
331            '[' => TokenKind::OpenBracket,
332            ']' => TokenKind::CloseBracket,
333            '{' => TokenKind::OpenBrace,
334            '}' => TokenKind::CloseBrace,
335            ',' => TokenKind::Comma,
336            ';' => TokenKind::Semicolon,
337            ':' => TokenKind::Colon,
338            '.' => TokenKind::Dot,
339            '&' => TokenKind::Ampersand,
340            '\u{00D7}' => TokenKind::HybridMark, // ×
341            '+' => TokenKind::HybridMark,
342            '\u{2020}' => TokenKind::Dagger, // †
343            _ => TokenKind::Other,
344        };
345        let s = byte_at(k);
346        let e = byte_at(k + 1);
347        out.push(Token {
348            kind,
349            text: input[s..e].to_string(),
350            start: s,
351            end: e,
352        });
353        k += 1;
354    }
355    out
356}
357
358#[cfg(test)]
359mod tests {
360    use super::*;
361
362    fn kinds(input: &str) -> Vec<(TokenKind, String)> {
363        tokenize(input)
364            .into_iter()
365            .map(|t| (t.kind, t.text))
366            .collect()
367    }
368
369    #[test]
370    fn simple_binomial() {
371        use TokenKind::*;
372        assert_eq!(
373            kinds("Aus bus"),
374            vec![(Word, "Aus".into()), (Word, "bus".into())]
375        );
376    }
377
378    #[test]
379    fn author_year() {
380        use TokenKind::*;
381        assert_eq!(
382            kinds("Aus bus L., 1758"),
383            vec![
384                (Word, "Aus".into()),
385                (Word, "bus".into()),
386                (Word, "L".into()),
387                (Dot, ".".into()),
388                (Comma, ",".into()),
389                (Number, "1758".into()),
390            ]
391        );
392    }
393
394    #[test]
395    fn internal_hyphen_and_apostrophe_stay_in_word() {
396        assert_eq!(kinds("Hartmann-Schroder").len(), 1);
397        assert_eq!(kinds("d'urvilleana").len(), 1);
398        assert_eq!(kinds("pu!chra").len(), 1); // stray '!' between letters
399    }
400
401    #[test]
402    fn leading_numeral_epithet_is_one_word() {
403        use TokenKind::*;
404        assert_eq!(kinds("11-punctata"), vec![(Word, "11-punctata".into())]);
405        assert_eq!(kinds("2-pustulata"), vec![(Word, "2-pustulata".into())]);
406    }
407
408    #[test]
409    fn unhyphenated_leading_numeral_epithet_is_one_word() {
410        // gbif/name-parser-rust#16
411        use TokenKind::*;
412        assert_eq!(kinds("4maculatus"), vec![(Word, "4maculatus".into())]);
413        assert_eq!(kinds("11punctata"), vec![(Word, "11punctata".into())]);
414        assert_eq!(kinds("13punctatus"), vec![(Word, "13punctatus".into())]);
415    }
416
417    #[test]
418    fn the_unhyphenated_glue_leaves_every_near_miss_split() {
419        use TokenKind::*;
420        for (input, number, rest) in [
421            // >2 digits: a year with a word glued on, not a numeral epithet
422            ("1976var", "1976", "var"),
423            // a leading zero is the OCR of a capital O ("Ørsted", "Olsson"), and gluing it
424            // would swallow the author into the name
425            ("0ersted", "0", "ersted"),
426            ("0lsson", "0", "lsson"),
427        ] {
428            assert_eq!(
429                kinds(input),
430                vec![(Number, number.into()), (Word, rest.into())],
431                "for {input:?}"
432            );
433        }
434        // <3 letters: the year disambiguator "1935h" and short strain codes stay split
435        assert_eq!(
436            kinds("1935h"),
437            vec![(Number, "1935".into()), (Word, "h".into())]
438        );
439        assert_eq!(kinds("5a"), vec![(Number, "5".into()), (Word, "a".into())]);
440        // uppercase letters are not an epithet
441        assert_eq!(
442            kinds("16S"),
443            vec![(Number, "16".into()), (Word, "S".into())]
444        );
445        // the word must END at the letter run
446        assert_eq!(
447            kinds("12abc4"),
448            vec![(Number, "12".into()), (Word, "abc4".into())]
449        );
450    }
451
452    #[test]
453    fn bare_number_is_number() {
454        use TokenKind::*;
455        assert_eq!(kinds("1758"), vec![(Number, "1758".into())]);
456    }
457
458    #[test]
459    fn hybrid_marks() {
460        use TokenKind::*;
461        // Unicode ×
462        assert_eq!(
463            kinds("Aus \u{00D7}bus"),
464            vec![
465                (Word, "Aus".into()),
466                (HybridMark, "\u{00D7}".into()),
467                (Word, "bus".into())
468            ]
469        );
470        // bare ASCII x between spaces
471        assert_eq!(
472            kinds("Aus x bus"),
473            vec![
474                (Word, "Aus".into()),
475                (HybridMark, "x".into()),
476                (Word, "bus".into())
477            ]
478        );
479        // xFoo — leading x glued to a capitalised word splits off
480        assert_eq!(
481            kinds("xBus"),
482            vec![(HybridMark, "x".into()), (Word, "Bus".into())]
483        );
484    }
485
486    #[test]
487    fn punctuation_and_dagger() {
488        use TokenKind::*;
489        assert_eq!(
490            kinds("\u{2020}Aus (Bus)"),
491            vec![
492                (Dagger, "\u{2020}".into()),
493                (Word, "Aus".into()),
494                (OpenParen, "(".into()),
495                (Word, "Bus".into()),
496                (CloseParen, ")".into()),
497            ]
498        );
499    }
500}