Skip to main content

nameparser/
token.rs

1// SPDX-License-Identifier: Apache-2.0
2
3#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4pub enum TokenKind {
5    Word,
6    Number,
7    HybridMark,
8    OpenParen,
9    CloseParen,
10    OpenBracket,
11    CloseBracket,
12    OpenBrace,
13    CloseBrace,
14    Comma,
15    Semicolon,
16    Colon,
17    Dot,
18    Ampersand,
19    Dagger,
20    Other,
21}
22
23#[derive(Debug, Clone, PartialEq, Eq)]
24pub struct Token {
25    pub kind: TokenKind,
26    /// The verbatim matched text.
27    pub text: String,
28    /// Byte offset into the input (UTF-8). NB: Java uses UTF-16 code-unit offsets.
29    pub start: usize,
30    pub end: usize,
31}
32
33/// java.lang.Character.isWhitespace — Unicode space separators EXCEPT the non-breaking
34/// ones (U+00A0, U+2007, U+202F), plus the ASCII/control whitespace incl. the file/group/
35/// record/unit separators. Rust's `char::is_whitespace` differs: it INCLUDES U+00A0 and
36/// EXCLUDES U+001C–U+001F. This helper matches Java. (Global constraint: Unicode semantics.)
37/// `pub(crate)`: also used by `pipeline::stripandstash`'s `first_word` (Java
38/// `StripAndStash.firstWord`, Phase 1 Slice 2 Task 3), not just this module's own tokenizer.
39pub(crate) fn is_whitespace_java(c: char) -> bool {
40    match c {
41        '\u{00A0}' | '\u{2007}' | '\u{202F}' => false,
42        '\u{001C}' | '\u{001D}' | '\u{001E}' | '\u{001F}' => true,
43        _ => c.is_whitespace(),
44    }
45}
46
47/// Java `org.gbif.nameparser.token.AuthorParticles.PARTICLES` — lower-case particles that
48/// combine with a following capitalised author surname ("de Vriese", "Van Heurck", "von
49/// der Linde"). Full 50-entry set transcribed verbatim from `AuthorParticles.java` (and
50/// verified programmatically against it — an exact set match, no entries dropped or
51/// added). The first call site (`StripAndStash::apply_missing_genus_placeholder`, Phase 1
52/// Slice 2 Task 3) only needed the boolean `is_particle` — `NameTokens`/`AuthorshipSplit`/
53/// `AuthorshipParser` now call it too (mirroring Java's own `isCapitalisedParticle`, which
54/// needs the same table), so porting the full set once here rather than a hand-trimmed
55/// subset paid off.
56const PARTICLES: &[&str] = &[
57    "a", "ab", "af", "ap", "auf", "d", "da", "dal", "dalla", "dalle", "dallo", "das", "de",
58    "degli", "dei", "del", "della", "delle", "delli", "dello", "der", "des", "di", "do", "dos",
59    "du", "el", "in", "la", "las", "le", "les", "lo", "los", "of", "ofver", "te", "ten", "ter",
60    "und", "v", "van", "vd", "ven", "vom", "von", "y", "zu", "zum", "zur",
61];
62
63/// Java `AuthorParticles.isParticle(String)`: `PARTICLES.contains(word.toLowerCase())`.
64/// Java's `.toLowerCase()` uses the JVM default locale; Rust's `.to_lowercase()` is the
65/// full-Unicode locale-independent mapping — every entry in `PARTICLES` is pure ASCII, so
66/// the two only-differ-for-Turkish-dotless-I-etc. divergences that exist in principle
67/// cannot change whether any of these ASCII words match (same reasoning already applied to
68/// `Preflight::run`'s `s.to_lowercase()` call).
69pub(crate) fn is_particle(word: &str) -> bool {
70    PARTICLES.contains(&word.to_lowercase().as_str())
71}
72
73#[cfg(test)]
74mod author_particles_tests {
75    use super::is_particle;
76
77    #[test]
78    fn recognises_lowercase_and_mixed_case_particles() {
79        assert!(is_particle("van"));
80        assert!(is_particle("Van"));
81        assert!(is_particle("VAN"));
82        assert!(is_particle("de"));
83    }
84
85    #[test]
86    fn rejects_non_particle_words() {
87        assert!(!is_particle("Smith"));
88        assert!(!is_particle("denheyeri"));
89        assert!(!is_particle(""));
90    }
91}
92
93/// java.lang.Character.isLetter — Unicode general category L*. Rust's `is_alphabetic` is the
94/// (slightly broader) Alphabetic property. Divergences are surfaced by the Task 3 golden diff.
95fn is_letter(c: char) -> bool {
96    c.is_alphabetic()
97}
98
99/// java.lang.Character.isDigit — Unicode Nd. Spike approximation: ASCII digits only.
100/// Non-ASCII decimal digits (rare in names) will show up as Task 3 mismatches for triage.
101fn is_digit(c: char) -> bool {
102    c.is_ascii_digit()
103}
104
105fn is_all_upper(s: &str) -> bool {
106    for c in s.chars() {
107        if is_letter(c) && !c.is_uppercase() {
108            return false;
109        }
110    }
111    true
112}
113
114/// Single-pass tokenizer. Faithful port of the Java `Tokenizer.tokenize`.
115/// Whitespace is consumed and not emitted. Letter runs may include internal hyphens,
116/// apostrophes, underscores and stray '!' when flanked by letters/digits.
117pub fn tokenize(input: &str) -> Vec<Token> {
118    let cs: Vec<(usize, char)> = input.char_indices().collect();
119    let n = cs.len();
120    let byte_at = |k: usize| -> usize {
121        if k < n {
122            cs[k].0
123        } else {
124            input.len()
125        }
126    };
127    let mut out: Vec<Token> = Vec::with_capacity((n / 4).max(4));
128    let mut k = 0usize;
129
130    while k < n {
131        let c = cs[k].1;
132
133        if is_whitespace_java(c) {
134            k += 1;
135            continue;
136        }
137
138        if is_letter(c) {
139            let word_start = k;
140            k += 1;
141            while k < n {
142                let c2 = cs[k].1;
143                if is_letter(c2) || is_digit(c2) {
144                    k += 1;
145                    continue;
146                }
147                // internal hyphen / apostrophe / underscore / stray '!' glued between letters/digits
148                if matches!(
149                    c2,
150                    '-' | '\''
151                        | '\u{2019}'
152                        | '_'
153                        | '!'
154                        | '\u{2010}'
155                        | '\u{2011}'
156                        | '\u{2012}'
157                        | '\u{2013}'
158                        | '\u{2014}'
159                ) && k + 1 < n
160                {
161                    let next = cs[k + 1].1;
162                    if is_letter(next) || is_digit(next) {
163                        k += 1;
164                        continue;
165                    }
166                }
167                break;
168            }
169            let start_b = byte_at(word_start);
170            let end_b = byte_at(k);
171            let word = &input[start_b..end_b];
172            let first = word.chars().next().unwrap();
173            let char_count = word.chars().count();
174
175            if char_count == 1 && (first == 'x' || first == 'X') {
176                let left_ok = word_start == 0 || is_whitespace_java(cs[word_start - 1].1);
177                let right_ok = k == n || is_whitespace_java(cs[k].1);
178                let kind = if left_ok && right_ok {
179                    TokenKind::HybridMark
180                } else {
181                    TokenKind::Word
182                };
183                out.push(Token {
184                    kind,
185                    text: word.to_string(),
186                    start: start_b,
187                    end: end_b,
188                });
189                continue;
190            } else if char_count >= 2 && (first == 'x' || first == 'X') {
191                let second = word.chars().nth(1).unwrap();
192                if second.is_uppercase() && !is_all_upper(word) {
193                    let first_len = first.len_utf8();
194                    out.push(Token {
195                        kind: TokenKind::HybridMark,
196                        text: word[..first_len].to_string(),
197                        start: start_b,
198                        end: start_b + first_len,
199                    });
200                    out.push(Token {
201                        kind: TokenKind::Word,
202                        text: word[first_len..].to_string(),
203                        start: start_b + first_len,
204                        end: end_b,
205                    });
206                    continue;
207                }
208            }
209            out.push(Token {
210                kind: TokenKind::Word,
211                text: word.to_string(),
212                start: start_b,
213                end: end_b,
214            });
215            continue;
216        }
217
218        if is_digit(c) {
219            let num_start = k;
220            k += 1;
221            while k < n && is_digit(cs[k].1) {
222                k += 1;
223            }
224            // "11-punctata": a number glued to hyphen + letter is a leading-numeral epithet word
225            if k + 1 < n && cs[k].1 == '-' && is_letter(cs[k + 1].1) {
226                k += 1; // consume hyphen
227                while k < n {
228                    let c2 = cs[k].1;
229                    if is_letter(c2) || is_digit(c2) {
230                        k += 1;
231                        continue;
232                    }
233                    if c2 == '-' && k + 1 < n {
234                        let next = cs[k + 1].1;
235                        if is_letter(next) || is_digit(next) {
236                            k += 1;
237                            continue;
238                        }
239                    }
240                    break;
241                }
242                let s = byte_at(num_start);
243                let e = byte_at(k);
244                out.push(Token {
245                    kind: TokenKind::Word,
246                    text: input[s..e].to_string(),
247                    start: s,
248                    end: e,
249                });
250                continue;
251            }
252            let s = byte_at(num_start);
253            let e = byte_at(k);
254            out.push(Token {
255                kind: TokenKind::Number,
256                text: input[s..e].to_string(),
257                start: s,
258                end: e,
259            });
260            continue;
261        }
262
263        let kind = match c {
264            '(' => TokenKind::OpenParen,
265            ')' => TokenKind::CloseParen,
266            '[' => TokenKind::OpenBracket,
267            ']' => TokenKind::CloseBracket,
268            '{' => TokenKind::OpenBrace,
269            '}' => TokenKind::CloseBrace,
270            ',' => TokenKind::Comma,
271            ';' => TokenKind::Semicolon,
272            ':' => TokenKind::Colon,
273            '.' => TokenKind::Dot,
274            '&' => TokenKind::Ampersand,
275            '\u{00D7}' => TokenKind::HybridMark, // ×
276            '+' => TokenKind::HybridMark,
277            '\u{2020}' => TokenKind::Dagger, // †
278            _ => TokenKind::Other,
279        };
280        let s = byte_at(k);
281        let e = byte_at(k + 1);
282        out.push(Token {
283            kind,
284            text: input[s..e].to_string(),
285            start: s,
286            end: e,
287        });
288        k += 1;
289    }
290    out
291}
292
293#[cfg(test)]
294mod tests {
295    use super::*;
296
297    fn kinds(input: &str) -> Vec<(TokenKind, String)> {
298        tokenize(input)
299            .into_iter()
300            .map(|t| (t.kind, t.text))
301            .collect()
302    }
303
304    #[test]
305    fn simple_binomial() {
306        use TokenKind::*;
307        assert_eq!(
308            kinds("Aus bus"),
309            vec![(Word, "Aus".into()), (Word, "bus".into())]
310        );
311    }
312
313    #[test]
314    fn author_year() {
315        use TokenKind::*;
316        assert_eq!(
317            kinds("Aus bus L., 1758"),
318            vec![
319                (Word, "Aus".into()),
320                (Word, "bus".into()),
321                (Word, "L".into()),
322                (Dot, ".".into()),
323                (Comma, ",".into()),
324                (Number, "1758".into()),
325            ]
326        );
327    }
328
329    #[test]
330    fn internal_hyphen_and_apostrophe_stay_in_word() {
331        assert_eq!(kinds("Hartmann-Schroder").len(), 1);
332        assert_eq!(kinds("d'urvilleana").len(), 1);
333        assert_eq!(kinds("pu!chra").len(), 1); // stray '!' between letters
334    }
335
336    #[test]
337    fn leading_numeral_epithet_is_one_word() {
338        use TokenKind::*;
339        assert_eq!(kinds("11-punctata"), vec![(Word, "11-punctata".into())]);
340        assert_eq!(kinds("2-pustulata"), vec![(Word, "2-pustulata".into())]);
341    }
342
343    #[test]
344    fn bare_number_is_number() {
345        use TokenKind::*;
346        assert_eq!(kinds("1758"), vec![(Number, "1758".into())]);
347    }
348
349    #[test]
350    fn hybrid_marks() {
351        use TokenKind::*;
352        // Unicode ×
353        assert_eq!(
354            kinds("Aus \u{00D7}bus"),
355            vec![
356                (Word, "Aus".into()),
357                (HybridMark, "\u{00D7}".into()),
358                (Word, "bus".into())
359            ]
360        );
361        // bare ASCII x between spaces
362        assert_eq!(
363            kinds("Aus x bus"),
364            vec![
365                (Word, "Aus".into()),
366                (HybridMark, "x".into()),
367                (Word, "bus".into())
368            ]
369        );
370        // xFoo — leading x glued to a capitalised word splits off
371        assert_eq!(
372            kinds("xBus"),
373            vec![(HybridMark, "x".into()), (Word, "Bus".into())]
374        );
375    }
376
377    #[test]
378    fn punctuation_and_dagger() {
379        use TokenKind::*;
380        assert_eq!(
381            kinds("\u{2020}Aus (Bus)"),
382            vec![
383                (Dagger, "\u{2020}".into()),
384                (Word, "Aus".into()),
385                (OpenParen, "(".into()),
386                (Word, "Bus".into()),
387                (CloseParen, ")".into()),
388            ]
389        );
390    }
391}