nameparser/token.rs
1// SPDX-License-Identifier: Apache-2.0
2
3use std::sync::LazyLock;
4
5use regex::Regex;
6
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub enum TokenKind {
9 Word,
10 Number,
11 HybridMark,
12 OpenParen,
13 CloseParen,
14 OpenBracket,
15 CloseBracket,
16 OpenBrace,
17 CloseBrace,
18 Comma,
19 Semicolon,
20 Colon,
21 Dot,
22 Ampersand,
23 Dagger,
24 Other,
25}
26
27#[derive(Debug, Clone, PartialEq, Eq)]
28pub struct Token {
29 pub kind: TokenKind,
30 /// The verbatim matched text.
31 pub text: String,
32 /// Byte offset into the input (UTF-8). NB: Java uses UTF-16 code-unit offsets.
33 pub start: usize,
34 pub end: usize,
35}
36
37/// java.lang.Character.isWhitespace — Unicode space separators EXCEPT the non-breaking
38/// ones (U+00A0, U+2007, U+202F), plus the ASCII/control whitespace incl. the file/group/
39/// record/unit separators. Rust's `char::is_whitespace` differs: it INCLUDES U+00A0 and
40/// EXCLUDES U+001C–U+001F. This helper matches Java. (Global constraint: Unicode semantics.)
41/// `pub(crate)`: also used by `pipeline::stripandstash`'s `first_word` (Java
42/// `StripAndStash.firstWord`, Phase 1 Slice 2 Task 3), not just this module's own tokenizer.
43pub(crate) fn is_whitespace_java(c: char) -> bool {
44 match c {
45 '\u{00A0}' | '\u{2007}' | '\u{202F}' => false,
46 '\u{001C}' | '\u{001D}' | '\u{001E}' | '\u{001F}' => true,
47 _ => c.is_whitespace(),
48 }
49}
50
51/// Java `org.gbif.nameparser.token.AuthorParticles.PARTICLES` — lower-case particles that
52/// combine with a following capitalised author surname ("de Vriese", "Van Heurck", "von
53/// der Linde"). Full 50-entry set transcribed verbatim from `AuthorParticles.java` (and
54/// verified programmatically against it — an exact set match, no entries dropped or
55/// added). The first call site (`StripAndStash::apply_missing_genus_placeholder`, Phase 1
56/// Slice 2 Task 3) only needed the boolean `is_particle` — `NameTokens`/`AuthorshipSplit`/
57/// `AuthorshipParser` now call it too (mirroring Java's own `isCapitalisedParticle`, which
58/// needs the same table), so porting the full set once here rather than a hand-trimmed
59/// subset paid off.
60const PARTICLES: &[&str] = &[
61 "a", "ab", "af", "ap", "auf", "d", "da", "dal", "dalla", "dalle", "dallo", "das", "de",
62 "degli", "dei", "del", "della", "delle", "delli", "dello", "der", "des", "di", "do", "dos",
63 "du", "el", "in", "la", "las", "le", "les", "lo", "los", "of", "ofver", "te", "ten", "ter",
64 "und", "v", "van", "vd", "ven", "vom", "von", "y", "zu", "zum", "zur",
65];
66
67/// Java `AuthorParticles.isParticle(String)`: `PARTICLES.contains(word.toLowerCase())`.
68/// Java's `.toLowerCase()` uses the JVM default locale; Rust's `.to_lowercase()` is the
69/// full-Unicode locale-independent mapping — every entry in `PARTICLES` is pure ASCII, so
70/// the two only-differ-for-Turkish-dotless-I-etc. divergences that exist in principle
71/// cannot change whether any of these ASCII words match (same reasoning already applied to
72/// `Preflight::run`'s `s.to_lowercase()` call).
73pub(crate) fn is_particle(word: &str) -> bool {
74 PARTICLES.contains(&word.to_lowercase().as_str())
75}
76
77#[cfg(test)]
78mod author_particles_tests {
79 use super::is_particle;
80
81 #[test]
82 fn recognises_lowercase_and_mixed_case_particles() {
83 assert!(is_particle("van"));
84 assert!(is_particle("Van"));
85 assert!(is_particle("VAN"));
86 assert!(is_particle("de"));
87 }
88
89 #[test]
90 fn rejects_non_particle_words() {
91 assert!(!is_particle("Smith"));
92 assert!(!is_particle("denheyeri"));
93 assert!(!is_particle(""));
94 }
95}
96
97/// java.lang.Character.isLetter — Unicode general category L*. Rust's `is_alphabetic` is the
98/// (slightly broader) Alphabetic property. Divergences are surfaced by the Task 3 golden diff.
99fn is_letter(c: char) -> bool {
100 c.is_alphabetic()
101}
102
103/// java.lang.Character.isDigit — Unicode Nd. Spike approximation: ASCII digits only.
104/// Non-ASCII decimal digits (rare in names) will show up as Task 3 mismatches for triage.
105fn is_digit(c: char) -> bool {
106 c.is_ascii_digit()
107}
108
109/// A numeral-prefixed LATIN EPITHET — `11-punctata`, `7-maculatus`, `4maculatus`, `2-pustulata`.
110/// One or two digits with no leading zero, an optional hyphen, then a run of at least three
111/// lowercase letters to the end.
112///
113/// The single authority for that shape, because three places must agree on it or they undo each
114/// other: [`tokenize`]'s two leading-numeral rules (which produce these WORD tokens),
115/// `StripAndStash::stash_trailing_strain_code` (which must NOT stash one as a strain code) and
116/// `NameTokens`' numbered-infraspecific phrase capture (which must NOT stash one as a phrase —
117/// `Benthogone rosea var. 4-lineata R. Perrier, 1896` has a real epithet there, and the 6.4M COL
118/// corpus found it). Real strain codes never take this shape: they keep going in digits
119/// (`18-41`), carry an uppercase segment (`47-T17`, `1-CPM-2005`) or have at most one trailing
120/// letter (`5a`).
121pub(crate) fn is_numeral_epithet(s: &str) -> bool {
122 static NUMERAL_EPITHET: LazyLock<Regex> =
123 LazyLock::new(|| Regex::new(r"^(?-u:[1-9]\d?)-?\p{Ll}{3,}$").unwrap());
124 NUMERAL_EPITHET.is_match(s)
125}
126
127/// A lowercase letter, for the unhyphenated leading-numeral epithet rule in [`tokenize`].
128/// `is_letter` + `is_lowercase` rather than `is_ascii_lowercase`, so a Latinised epithet
129/// with a non-ASCII letter is glued on the same terms as an ASCII one.
130fn is_lowercase_letter(c: char) -> bool {
131 is_letter(c) && c.is_lowercase()
132}
133
134fn is_all_upper(s: &str) -> bool {
135 for c in s.chars() {
136 if is_letter(c) && !c.is_uppercase() {
137 return false;
138 }
139 }
140 true
141}
142
143/// Single-pass tokenizer. Faithful port of the Java `Tokenizer.tokenize`.
144/// Whitespace is consumed and not emitted. Letter runs may include internal hyphens,
145/// apostrophes, underscores and stray '!' when flanked by letters/digits.
146pub fn tokenize(input: &str) -> Vec<Token> {
147 let cs: Vec<(usize, char)> = input.char_indices().collect();
148 let n = cs.len();
149 let byte_at = |k: usize| -> usize {
150 if k < n {
151 cs[k].0
152 } else {
153 input.len()
154 }
155 };
156 let mut out: Vec<Token> = Vec::with_capacity((n / 4).max(4));
157 let mut k = 0usize;
158
159 while k < n {
160 let c = cs[k].1;
161
162 if is_whitespace_java(c) {
163 k += 1;
164 continue;
165 }
166
167 if is_letter(c) {
168 let word_start = k;
169 k += 1;
170 while k < n {
171 let c2 = cs[k].1;
172 if is_letter(c2) || is_digit(c2) {
173 k += 1;
174 continue;
175 }
176 // internal hyphen / apostrophe / underscore / stray '!' glued between letters/digits
177 if matches!(
178 c2,
179 '-' | '\''
180 | '\u{2019}'
181 | '_'
182 | '!'
183 | '\u{2010}'
184 | '\u{2011}'
185 | '\u{2012}'
186 | '\u{2013}'
187 | '\u{2014}'
188 ) && k + 1 < n
189 {
190 let next = cs[k + 1].1;
191 if is_letter(next) || is_digit(next) {
192 k += 1;
193 continue;
194 }
195 }
196 break;
197 }
198 let start_b = byte_at(word_start);
199 let end_b = byte_at(k);
200 let word = &input[start_b..end_b];
201 let first = word.chars().next().unwrap();
202 let char_count = word.chars().count();
203
204 if char_count == 1 && (first == 'x' || first == 'X') {
205 let left_ok = word_start == 0 || is_whitespace_java(cs[word_start - 1].1);
206 let right_ok = k == n || is_whitespace_java(cs[k].1);
207 let kind = if left_ok && right_ok {
208 TokenKind::HybridMark
209 } else {
210 TokenKind::Word
211 };
212 out.push(Token {
213 kind,
214 text: word.to_string(),
215 start: start_b,
216 end: end_b,
217 });
218 continue;
219 } else if char_count >= 2 && (first == 'x' || first == 'X') {
220 let second = word.chars().nth(1).unwrap();
221 if second.is_uppercase() && !is_all_upper(word) {
222 let first_len = first.len_utf8();
223 out.push(Token {
224 kind: TokenKind::HybridMark,
225 text: word[..first_len].to_string(),
226 start: start_b,
227 end: start_b + first_len,
228 });
229 out.push(Token {
230 kind: TokenKind::Word,
231 text: word[first_len..].to_string(),
232 start: start_b + first_len,
233 end: end_b,
234 });
235 continue;
236 }
237 }
238 out.push(Token {
239 kind: TokenKind::Word,
240 text: word.to_string(),
241 start: start_b,
242 end: end_b,
243 });
244 continue;
245 }
246
247 if is_digit(c) {
248 let num_start = k;
249 k += 1;
250 while k < n && is_digit(cs[k].1) {
251 k += 1;
252 }
253 // "11-punctata": a number glued to hyphen + letter is a leading-numeral epithet word
254 if k + 1 < n && cs[k].1 == '-' && is_letter(cs[k + 1].1) {
255 k += 1; // consume hyphen
256 while k < n {
257 let c2 = cs[k].1;
258 if is_letter(c2) || is_digit(c2) {
259 k += 1;
260 continue;
261 }
262 if c2 == '-' && k + 1 < n {
263 let next = cs[k + 1].1;
264 if is_letter(next) || is_digit(next) {
265 k += 1;
266 continue;
267 }
268 }
269 break;
270 }
271 let s = byte_at(num_start);
272 let e = byte_at(k);
273 out.push(Token {
274 kind: TokenKind::Word,
275 text: input[s..e].to_string(),
276 start: s,
277 end: e,
278 });
279 continue;
280 }
281 // gbif/name-parser-rust#16: the UNHYPHENATED spelling of the same leading-numeral
282 // epithet — "4maculatus" for "quadrimaculatus", "11punctata". Java (and this port
283 // until now) only glued the hyphenated form, so `Camponotus sericeus 4maculatus`
284 // tokenised as Number(4) + Word(maculatus) and the authorship parser dropped the 4
285 // and promoted "maculatus" to an author, leaving a canonical string identical to the
286 // real `Camponotus sericeus`.
287 //
288 // Deliberately much narrower than the hyphenated rule above, because without the
289 // hyphen the shape collides with things that must STAY split: at most two digits (a
290 // numeral epithet is `2`..`24`, never a year — so "1976var" keeps its Number), no
291 // leading zero (a numeral epithet never has one, but the OCR of a capital O does —
292 // "0ersted" for Ørsted, "0lsson" for Olsson, both in the golden corpus, where gluing
293 // would swallow the AUTHOR into the name), then at least three LOWERCASE letters (so
294 // the year disambiguator "1935h" and the strain codes "5a" / "16S" / "2016Iso3" are
295 // all untouched), and the word must end there (so "12abc4" stays a Number too).
296 let digits = k - num_start;
297 let leading_zero = cs[num_start].1 == '0';
298 if digits <= 2 && !leading_zero && k < n && is_lowercase_letter(cs[k].1) {
299 let mut k2 = k;
300 while k2 < n && is_lowercase_letter(cs[k2].1) {
301 k2 += 1;
302 }
303 let word_ends = k2 == n || !(is_letter(cs[k2].1) || is_digit(cs[k2].1));
304 if k2 - k >= 3 && word_ends {
305 let s = byte_at(num_start);
306 let e = byte_at(k2);
307 out.push(Token {
308 kind: TokenKind::Word,
309 text: input[s..e].to_string(),
310 start: s,
311 end: e,
312 });
313 k = k2;
314 continue;
315 }
316 }
317 let s = byte_at(num_start);
318 let e = byte_at(k);
319 out.push(Token {
320 kind: TokenKind::Number,
321 text: input[s..e].to_string(),
322 start: s,
323 end: e,
324 });
325 continue;
326 }
327
328 let kind = match c {
329 '(' => TokenKind::OpenParen,
330 ')' => TokenKind::CloseParen,
331 '[' => TokenKind::OpenBracket,
332 ']' => TokenKind::CloseBracket,
333 '{' => TokenKind::OpenBrace,
334 '}' => TokenKind::CloseBrace,
335 ',' => TokenKind::Comma,
336 ';' => TokenKind::Semicolon,
337 ':' => TokenKind::Colon,
338 '.' => TokenKind::Dot,
339 '&' => TokenKind::Ampersand,
340 '\u{00D7}' => TokenKind::HybridMark, // ×
341 '+' => TokenKind::HybridMark,
342 '\u{2020}' => TokenKind::Dagger, // †
343 _ => TokenKind::Other,
344 };
345 let s = byte_at(k);
346 let e = byte_at(k + 1);
347 out.push(Token {
348 kind,
349 text: input[s..e].to_string(),
350 start: s,
351 end: e,
352 });
353 k += 1;
354 }
355 out
356}
357
358#[cfg(test)]
359mod tests {
360 use super::*;
361
362 fn kinds(input: &str) -> Vec<(TokenKind, String)> {
363 tokenize(input)
364 .into_iter()
365 .map(|t| (t.kind, t.text))
366 .collect()
367 }
368
369 #[test]
370 fn simple_binomial() {
371 use TokenKind::*;
372 assert_eq!(
373 kinds("Aus bus"),
374 vec![(Word, "Aus".into()), (Word, "bus".into())]
375 );
376 }
377
378 #[test]
379 fn author_year() {
380 use TokenKind::*;
381 assert_eq!(
382 kinds("Aus bus L., 1758"),
383 vec![
384 (Word, "Aus".into()),
385 (Word, "bus".into()),
386 (Word, "L".into()),
387 (Dot, ".".into()),
388 (Comma, ",".into()),
389 (Number, "1758".into()),
390 ]
391 );
392 }
393
394 #[test]
395 fn internal_hyphen_and_apostrophe_stay_in_word() {
396 assert_eq!(kinds("Hartmann-Schroder").len(), 1);
397 assert_eq!(kinds("d'urvilleana").len(), 1);
398 assert_eq!(kinds("pu!chra").len(), 1); // stray '!' between letters
399 }
400
401 #[test]
402 fn leading_numeral_epithet_is_one_word() {
403 use TokenKind::*;
404 assert_eq!(kinds("11-punctata"), vec![(Word, "11-punctata".into())]);
405 assert_eq!(kinds("2-pustulata"), vec![(Word, "2-pustulata".into())]);
406 }
407
408 #[test]
409 fn unhyphenated_leading_numeral_epithet_is_one_word() {
410 // gbif/name-parser-rust#16
411 use TokenKind::*;
412 assert_eq!(kinds("4maculatus"), vec![(Word, "4maculatus".into())]);
413 assert_eq!(kinds("11punctata"), vec![(Word, "11punctata".into())]);
414 assert_eq!(kinds("13punctatus"), vec![(Word, "13punctatus".into())]);
415 }
416
417 #[test]
418 fn the_unhyphenated_glue_leaves_every_near_miss_split() {
419 use TokenKind::*;
420 for (input, number, rest) in [
421 // >2 digits: a year with a word glued on, not a numeral epithet
422 ("1976var", "1976", "var"),
423 // a leading zero is the OCR of a capital O ("Ørsted", "Olsson"), and gluing it
424 // would swallow the author into the name
425 ("0ersted", "0", "ersted"),
426 ("0lsson", "0", "lsson"),
427 ] {
428 assert_eq!(
429 kinds(input),
430 vec![(Number, number.into()), (Word, rest.into())],
431 "for {input:?}"
432 );
433 }
434 // <3 letters: the year disambiguator "1935h" and short strain codes stay split
435 assert_eq!(
436 kinds("1935h"),
437 vec![(Number, "1935".into()), (Word, "h".into())]
438 );
439 assert_eq!(kinds("5a"), vec![(Number, "5".into()), (Word, "a".into())]);
440 // uppercase letters are not an epithet
441 assert_eq!(
442 kinds("16S"),
443 vec![(Number, "16".into()), (Word, "S".into())]
444 );
445 // the word must END at the letter run
446 assert_eq!(
447 kinds("12abc4"),
448 vec![(Number, "12".into()), (Word, "abc4".into())]
449 );
450 }
451
452 #[test]
453 fn bare_number_is_number() {
454 use TokenKind::*;
455 assert_eq!(kinds("1758"), vec![(Number, "1758".into())]);
456 }
457
458 #[test]
459 fn hybrid_marks() {
460 use TokenKind::*;
461 // Unicode ×
462 assert_eq!(
463 kinds("Aus \u{00D7}bus"),
464 vec![
465 (Word, "Aus".into()),
466 (HybridMark, "\u{00D7}".into()),
467 (Word, "bus".into())
468 ]
469 );
470 // bare ASCII x between spaces
471 assert_eq!(
472 kinds("Aus x bus"),
473 vec![
474 (Word, "Aus".into()),
475 (HybridMark, "x".into()),
476 (Word, "bus".into())
477 ]
478 );
479 // xFoo — leading x glued to a capitalised word splits off
480 assert_eq!(
481 kinds("xBus"),
482 vec![(HybridMark, "x".into()), (Word, "Bus".into())]
483 );
484 }
485
486 #[test]
487 fn punctuation_and_dagger() {
488 use TokenKind::*;
489 assert_eq!(
490 kinds("\u{2020}Aus (Bus)"),
491 vec![
492 (Dagger, "\u{2020}".into()),
493 (Word, "Aus".into()),
494 (OpenParen, "(".into()),
495 (Word, "Bus".into()),
496 (CloseParen, ")".into()),
497 ]
498 );
499 }
500}