nameparser/token.rs
1// SPDX-License-Identifier: Apache-2.0
2
3use std::sync::LazyLock;
4
5use regex::Regex;
6
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub enum TokenKind {
9 Word,
10 Number,
11 HybridMark,
12 OpenParen,
13 CloseParen,
14 OpenBracket,
15 CloseBracket,
16 OpenBrace,
17 CloseBrace,
18 Comma,
19 Semicolon,
20 Colon,
21 Dot,
22 Ampersand,
23 Dagger,
24 Other,
25}
26
27#[derive(Debug, Clone, PartialEq, Eq)]
28pub struct Token {
29 pub kind: TokenKind,
30 /// The verbatim matched text.
31 pub text: String,
32 /// Byte offset into the input (UTF-8). NB: Java uses UTF-16 code-unit offsets.
33 pub start: usize,
34 pub end: usize,
35}
36
37/// java.lang.Character.isWhitespace — Unicode space separators EXCEPT the non-breaking
38/// ones (U+00A0, U+2007, U+202F), plus the ASCII/control whitespace incl. the file/group/
39/// record/unit separators. Rust's `char::is_whitespace` differs: it INCLUDES U+00A0 and
40/// EXCLUDES U+001C–U+001F. This helper matches Java. (Global constraint: Unicode semantics.)
41/// `pub(crate)`: also used by `pipeline::stripandstash`'s `first_word` (Java
42/// `StripAndStash.firstWord`, Phase 1 Slice 2 Task 3), not just this module's own tokenizer.
43pub(crate) fn is_whitespace_java(c: char) -> bool {
44 match c {
45 '\u{00A0}' | '\u{2007}' | '\u{202F}' => false,
46 '\u{001C}' | '\u{001D}' | '\u{001E}' | '\u{001F}' => true,
47 _ => c.is_whitespace(),
48 }
49}
50
51/// Java `org.gbif.nameparser.token.AuthorParticles.PARTICLES` — lower-case particles that
52/// combine with a following capitalised author surname ("de Vriese", "Van Heurck", "von
53/// der Linde"). Full 50-entry set transcribed verbatim from `AuthorParticles.java` (and
54/// verified programmatically against it — an exact set match, no entries dropped or
55/// added). The first call site (`StripAndStash::apply_missing_genus_placeholder`, Phase 1
56/// Slice 2 Task 3) only needed the boolean `is_particle` — `NameTokens`/`AuthorshipSplit`/
57/// `AuthorshipParser` now call it too (mirroring Java's own `isCapitalisedParticle`, which
58/// needs the same table), so porting the full set once here rather than a hand-trimmed
59/// subset paid off.
60const PARTICLES: &[&str] = &[
61 "a", "ab", "af", "ap", "auf", "d", "da", "dal", "dalla", "dalle", "dallo", "das", "de",
62 "degli", "dei", "del", "della", "delle", "delli", "dello", "der", "des", "di", "do", "dos",
63 "du", "el", "in", "la", "las", "le", "les", "lo", "los", "of", "ofver", "te", "ten", "ter",
64 "und", "v", "van", "vd", "ven", "vom", "von", "y", "zu", "zum", "zur",
65];
66
67/// Java `AuthorParticles.isParticle(String)`: `PARTICLES.contains(word.toLowerCase())`.
68/// Java's `.toLowerCase()` uses the JVM default locale; Rust's `.to_lowercase()` is the
69/// full-Unicode locale-independent mapping — every entry in `PARTICLES` is pure ASCII, so
70/// the two only-differ-for-Turkish-dotless-I-etc. divergences that exist in principle
71/// cannot change whether any of these ASCII words match (same reasoning already applied to
72/// `Preflight::run`'s `s.to_lowercase()` call).
73pub(crate) fn is_particle(word: &str) -> bool {
74 let word = word.to_lowercase();
75 // a hyphenated run of particles too: "van-der Land", "de-la Cruz"
76 PARTICLES.contains(&word.as_str())
77 || (word.contains('-') && word.split('-').all(|p| PARTICLES.contains(&p)))
78}
79
80#[cfg(test)]
81mod author_particles_tests {
82 use super::is_particle;
83
84 #[test]
85 fn recognises_lowercase_and_mixed_case_particles() {
86 assert!(is_particle("van"));
87 assert!(is_particle("Van"));
88 assert!(is_particle("VAN"));
89 assert!(is_particle("de"));
90 assert!(is_particle("van-der"));
91 assert!(!is_particle("da-silvae"));
92 }
93
94 #[test]
95 fn rejects_non_particle_words() {
96 assert!(!is_particle("Smith"));
97 assert!(!is_particle("denheyeri"));
98 assert!(!is_particle(""));
99 }
100}
101
102/// java.lang.Character.isLetter — Unicode general category L*. Rust's `is_alphabetic` is the
103/// (slightly broader) Alphabetic property. Divergences are surfaced by the Task 3 golden diff.
104fn is_letter(c: char) -> bool {
105 c.is_alphabetic()
106}
107
108/// java.lang.Character.isDigit — Unicode Nd. Spike approximation: ASCII digits only.
109/// Non-ASCII decimal digits (rare in names) will show up as Task 3 mismatches for triage.
110fn is_digit(c: char) -> bool {
111 c.is_ascii_digit()
112}
113
114/// A numeral-prefixed LATIN EPITHET — `11-punctata`, `7-maculatus`, `4maculatus`, `2-pustulata`.
115/// One or two digits with no leading zero, an optional hyphen, then a run of at least three
116/// lowercase letters to the end.
117///
118/// The single authority for that shape, because three places must agree on it or they undo each
119/// other: [`tokenize`]'s two leading-numeral rules (which produce these WORD tokens),
120/// `StripAndStash::stash_trailing_strain_code` (which must NOT stash one as a strain code) and
121/// `NameTokens`' numbered-infraspecific phrase capture (which must NOT stash one as a phrase —
122/// `Benthogone rosea var. 4-lineata R. Perrier, 1896` has a real epithet there, and the 6.4M COL
123/// corpus found it). Real strain codes never take this shape: they keep going in digits
124/// (`18-41`), carry an uppercase segment (`47-T17`, `1-CPM-2005`) or have at most one trailing
125/// letter (`5a`).
126pub(crate) fn is_numeral_epithet(s: &str) -> bool {
127 static NUMERAL_EPITHET: LazyLock<Regex> =
128 LazyLock::new(|| Regex::new(r"^(?-u:[1-9]\d?)-?\p{Ll}{3,}$").unwrap());
129 NUMERAL_EPITHET.is_match(s)
130}
131
132/// A lowercase letter, for the unhyphenated leading-numeral epithet rule in [`tokenize`].
133/// `is_letter` + `is_lowercase` rather than `is_ascii_lowercase`, so a Latinised epithet
134/// with a non-ASCII letter is glued on the same terms as an ASCII one.
135fn is_lowercase_letter(c: char) -> bool {
136 is_letter(c) && c.is_lowercase()
137}
138
139fn is_all_upper(s: &str) -> bool {
140 for c in s.chars() {
141 if is_letter(c) && !c.is_uppercase() {
142 return false;
143 }
144 }
145 true
146}
147
148/// Single-pass tokenizer. Faithful port of the Java `Tokenizer.tokenize`.
149/// Whitespace is consumed and not emitted. Letter runs may include internal hyphens,
150/// apostrophes, underscores and stray '!' when flanked by letters/digits.
151pub fn tokenize(input: &str) -> Vec<Token> {
152 let cs: Vec<(usize, char)> = input.char_indices().collect();
153 let n = cs.len();
154 let byte_at = |k: usize| -> usize {
155 if k < n {
156 cs[k].0
157 } else {
158 input.len()
159 }
160 };
161 let mut out: Vec<Token> = Vec::with_capacity((n / 4).max(4));
162 let mut k = 0usize;
163
164 while k < n {
165 let c = cs[k].1;
166
167 if is_whitespace_java(c) {
168 k += 1;
169 continue;
170 }
171
172 if is_letter(c) {
173 let word_start = k;
174 k += 1;
175 while k < n {
176 let c2 = cs[k].1;
177 if is_letter(c2) || is_digit(c2) {
178 k += 1;
179 continue;
180 }
181 // internal hyphen / apostrophe / underscore / stray '!' glued between letters/digits
182 if matches!(
183 c2,
184 '-' | '\''
185 | '\u{2019}'
186 | '_'
187 | '!'
188 | '\u{2010}'
189 | '\u{2011}'
190 | '\u{2012}'
191 | '\u{2013}'
192 | '\u{2014}'
193 ) && k + 1 < n
194 {
195 let next = cs[k + 1].1;
196 if is_letter(next) || is_digit(next) {
197 k += 1;
198 continue;
199 }
200 }
201 break;
202 }
203 // A lower-case "st."/"s." (sancti) hyphenated to the rest of the epithet is one word:
204 // "Macromitrium st.-johnii", "Cyathea st.-helenae" — not a French "s.-esp." (sous-espèce).
205 let abbreviated = &cs[word_start..k];
206 if matches!(abbreviated, [(_, 's')] | [(_, 's'), (_, 't')])
207 && k + 5 < n
208 && cs[k].1 == '.'
209 && cs[k + 1].1 == '-'
210 && cs[k + 2..k + 6].iter().all(|&(_, c)| c.is_lowercase())
211 {
212 k += 2;
213 while k < n
214 && (is_letter(cs[k].1)
215 || (cs[k].1 == '-' && k + 1 < n && is_letter(cs[k + 1].1)))
216 {
217 k += 1;
218 }
219 }
220 let start_b = byte_at(word_start);
221 let end_b = byte_at(k);
222 let word = &input[start_b..end_b];
223 let first = word.chars().next().unwrap();
224 let char_count = word.chars().count();
225
226 if char_count == 1 && (first == 'x' || first == 'X') {
227 let left_ok = word_start == 0 || is_whitespace_java(cs[word_start - 1].1);
228 let right_ok = k == n || is_whitespace_java(cs[k].1);
229 let kind = if left_ok && right_ok {
230 TokenKind::HybridMark
231 } else {
232 TokenKind::Word
233 };
234 out.push(Token {
235 kind,
236 text: word.to_string(),
237 start: start_b,
238 end: end_b,
239 });
240 continue;
241 } else if char_count >= 2 && (first == 'x' || first == 'X') {
242 let second = word.chars().nth(1).unwrap();
243 if second.is_uppercase() && !is_all_upper(word) {
244 let first_len = first.len_utf8();
245 out.push(Token {
246 kind: TokenKind::HybridMark,
247 text: word[..first_len].to_string(),
248 start: start_b,
249 end: start_b + first_len,
250 });
251 out.push(Token {
252 kind: TokenKind::Word,
253 text: word[first_len..].to_string(),
254 start: start_b + first_len,
255 end: end_b,
256 });
257 continue;
258 }
259 }
260 out.push(Token {
261 kind: TokenKind::Word,
262 text: word.to_string(),
263 start: start_b,
264 end: end_b,
265 });
266 continue;
267 }
268
269 if is_digit(c) {
270 let num_start = k;
271 k += 1;
272 while k < n && is_digit(cs[k].1) {
273 k += 1;
274 }
275 // "11-punctata": a number glued to hyphen + letter is a leading-numeral epithet word
276 if k + 1 < n && cs[k].1 == '-' && is_letter(cs[k + 1].1) {
277 k += 1; // consume hyphen
278 while k < n {
279 let c2 = cs[k].1;
280 if is_letter(c2) || is_digit(c2) {
281 k += 1;
282 continue;
283 }
284 if c2 == '-' && k + 1 < n {
285 let next = cs[k + 1].1;
286 if is_letter(next) || is_digit(next) {
287 k += 1;
288 continue;
289 }
290 }
291 break;
292 }
293 let s = byte_at(num_start);
294 let e = byte_at(k);
295 out.push(Token {
296 kind: TokenKind::Word,
297 text: input[s..e].to_string(),
298 start: s,
299 end: e,
300 });
301 continue;
302 }
303 // gbif/name-parser-rust#16: the UNHYPHENATED spelling of the same leading-numeral
304 // epithet — "4maculatus" for "quadrimaculatus", "11punctata". Java (and this port
305 // until now) only glued the hyphenated form, so `Camponotus sericeus 4maculatus`
306 // tokenised as Number(4) + Word(maculatus) and the authorship parser dropped the 4
307 // and promoted "maculatus" to an author, leaving a canonical string identical to the
308 // real `Camponotus sericeus`.
309 //
310 // Deliberately much narrower than the hyphenated rule above, because without the
311 // hyphen the shape collides with things that must STAY split: at most two digits (a
312 // numeral epithet is `2`..`24`, never a year — so "1976var" keeps its Number), no
313 // leading zero (a numeral epithet never has one, but the OCR of a capital O does —
314 // "0ersted" for Ørsted, "0lsson" for Olsson, both in the golden corpus, where gluing
315 // would swallow the AUTHOR into the name), then at least three LOWERCASE letters (so
316 // the year disambiguator "1935h" and the strain codes "5a" / "16S" / "2016Iso3" are
317 // all untouched), and the word must end there (so "12abc4" stays a Number too).
318 let digits = k - num_start;
319 let leading_zero = cs[num_start].1 == '0';
320 if digits <= 2 && !leading_zero && k < n && is_lowercase_letter(cs[k].1) {
321 let mut k2 = k;
322 while k2 < n && is_lowercase_letter(cs[k2].1) {
323 k2 += 1;
324 }
325 let word_ends = k2 == n || !(is_letter(cs[k2].1) || is_digit(cs[k2].1));
326 if k2 - k >= 3 && word_ends {
327 let s = byte_at(num_start);
328 let e = byte_at(k2);
329 out.push(Token {
330 kind: TokenKind::Word,
331 text: input[s..e].to_string(),
332 start: s,
333 end: e,
334 });
335 k = k2;
336 continue;
337 }
338 }
339 let s = byte_at(num_start);
340 let e = byte_at(k);
341 out.push(Token {
342 kind: TokenKind::Number,
343 text: input[s..e].to_string(),
344 start: s,
345 end: e,
346 });
347 continue;
348 }
349
350 let kind = match c {
351 '(' => TokenKind::OpenParen,
352 ')' => TokenKind::CloseParen,
353 '[' => TokenKind::OpenBracket,
354 ']' => TokenKind::CloseBracket,
355 '{' => TokenKind::OpenBrace,
356 '}' => TokenKind::CloseBrace,
357 ',' => TokenKind::Comma,
358 ';' => TokenKind::Semicolon,
359 ':' => TokenKind::Colon,
360 '.' => TokenKind::Dot,
361 '&' => TokenKind::Ampersand,
362 '\u{00D7}' => TokenKind::HybridMark, // ×
363 '+' => TokenKind::HybridMark,
364 '\u{2020}' => TokenKind::Dagger, // †
365 _ => TokenKind::Other,
366 };
367 let s = byte_at(k);
368 let e = byte_at(k + 1);
369 out.push(Token {
370 kind,
371 text: input[s..e].to_string(),
372 start: s,
373 end: e,
374 });
375 k += 1;
376 }
377 out
378}
379
380#[cfg(test)]
381mod tests {
382 use super::*;
383
384 fn kinds(input: &str) -> Vec<(TokenKind, String)> {
385 tokenize(input)
386 .into_iter()
387 .map(|t| (t.kind, t.text))
388 .collect()
389 }
390
391 #[test]
392 fn simple_binomial() {
393 use TokenKind::*;
394 assert_eq!(
395 kinds("Aus bus"),
396 vec![(Word, "Aus".into()), (Word, "bus".into())]
397 );
398 }
399
400 #[test]
401 fn author_year() {
402 use TokenKind::*;
403 assert_eq!(
404 kinds("Aus bus L., 1758"),
405 vec![
406 (Word, "Aus".into()),
407 (Word, "bus".into()),
408 (Word, "L".into()),
409 (Dot, ".".into()),
410 (Comma, ",".into()),
411 (Number, "1758".into()),
412 ]
413 );
414 }
415
416 #[test]
417 fn internal_hyphen_and_apostrophe_stay_in_word() {
418 assert_eq!(kinds("Hartmann-Schroder").len(), 1);
419 assert_eq!(kinds("d'urvilleana").len(), 1);
420 assert_eq!(kinds("pu!chra").len(), 1); // stray '!' between letters
421 }
422
423 #[test]
424 fn leading_numeral_epithet_is_one_word() {
425 use TokenKind::*;
426 assert_eq!(kinds("11-punctata"), vec![(Word, "11-punctata".into())]);
427 assert_eq!(kinds("2-pustulata"), vec![(Word, "2-pustulata".into())]);
428 }
429
430 #[test]
431 fn unhyphenated_leading_numeral_epithet_is_one_word() {
432 // gbif/name-parser-rust#16
433 use TokenKind::*;
434 assert_eq!(kinds("4maculatus"), vec![(Word, "4maculatus".into())]);
435 assert_eq!(kinds("11punctata"), vec![(Word, "11punctata".into())]);
436 assert_eq!(kinds("13punctatus"), vec![(Word, "13punctatus".into())]);
437 }
438
439 #[test]
440 fn the_unhyphenated_glue_leaves_every_near_miss_split() {
441 use TokenKind::*;
442 for (input, number, rest) in [
443 // >2 digits: a year with a word glued on, not a numeral epithet
444 ("1976var", "1976", "var"),
445 // a leading zero is the OCR of a capital O ("Ørsted", "Olsson"), and gluing it
446 // would swallow the author into the name
447 ("0ersted", "0", "ersted"),
448 ("0lsson", "0", "lsson"),
449 ] {
450 assert_eq!(
451 kinds(input),
452 vec![(Number, number.into()), (Word, rest.into())],
453 "for {input:?}"
454 );
455 }
456 // <3 letters: the year disambiguator "1935h" and short strain codes stay split
457 assert_eq!(
458 kinds("1935h"),
459 vec![(Number, "1935".into()), (Word, "h".into())]
460 );
461 assert_eq!(kinds("5a"), vec![(Number, "5".into()), (Word, "a".into())]);
462 // uppercase letters are not an epithet
463 assert_eq!(
464 kinds("16S"),
465 vec![(Number, "16".into()), (Word, "S".into())]
466 );
467 // the word must END at the letter run
468 assert_eq!(
469 kinds("12abc4"),
470 vec![(Number, "12".into()), (Word, "abc4".into())]
471 );
472 }
473
474 #[test]
475 fn bare_number_is_number() {
476 use TokenKind::*;
477 assert_eq!(kinds("1758"), vec![(Number, "1758".into())]);
478 }
479
480 #[test]
481 fn hybrid_marks() {
482 use TokenKind::*;
483 // Unicode ×
484 assert_eq!(
485 kinds("Aus \u{00D7}bus"),
486 vec![
487 (Word, "Aus".into()),
488 (HybridMark, "\u{00D7}".into()),
489 (Word, "bus".into())
490 ]
491 );
492 // bare ASCII x between spaces
493 assert_eq!(
494 kinds("Aus x bus"),
495 vec![
496 (Word, "Aus".into()),
497 (HybridMark, "x".into()),
498 (Word, "bus".into())
499 ]
500 );
501 // xFoo — leading x glued to a capitalised word splits off
502 assert_eq!(
503 kinds("xBus"),
504 vec![(HybridMark, "x".into()), (Word, "Bus".into())]
505 );
506 }
507
508 #[test]
509 fn punctuation_and_dagger() {
510 use TokenKind::*;
511 assert_eq!(
512 kinds("\u{2020}Aus (Bus)"),
513 vec![
514 (Dagger, "\u{2020}".into()),
515 (Word, "Aus".into()),
516 (OpenParen, "(".into()),
517 (Word, "Bus".into()),
518 (CloseParen, ")".into()),
519 ]
520 );
521 }
522}