1#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4pub enum TokenKind {
5 Word,
6 Number,
7 HybridMark,
8 OpenParen,
9 CloseParen,
10 OpenBracket,
11 CloseBracket,
12 OpenBrace,
13 CloseBrace,
14 Comma,
15 Semicolon,
16 Colon,
17 Dot,
18 Ampersand,
19 Dagger,
20 Other,
21}
22
23#[derive(Debug, Clone, PartialEq, Eq)]
24pub struct Token {
25 pub kind: TokenKind,
26 pub text: String,
28 pub start: usize,
30 pub end: usize,
31}
32
33pub(crate) fn is_whitespace_java(c: char) -> bool {
40 match c {
41 '\u{00A0}' | '\u{2007}' | '\u{202F}' => false,
42 '\u{001C}' | '\u{001D}' | '\u{001E}' | '\u{001F}' => true,
43 _ => c.is_whitespace(),
44 }
45}
46
47const PARTICLES: &[&str] = &[
57 "a", "ab", "af", "ap", "auf", "d", "da", "dal", "dalla", "dalle", "dallo", "das", "de",
58 "degli", "dei", "del", "della", "delle", "delli", "dello", "der", "des", "di", "do", "dos",
59 "du", "el", "in", "la", "las", "le", "les", "lo", "los", "of", "ofver", "te", "ten", "ter",
60 "und", "v", "van", "vd", "ven", "vom", "von", "y", "zu", "zum", "zur",
61];
62
63pub(crate) fn is_particle(word: &str) -> bool {
70 PARTICLES.contains(&word.to_lowercase().as_str())
71}
72
73#[cfg(test)]
74mod author_particles_tests {
75 use super::is_particle;
76
77 #[test]
78 fn recognises_lowercase_and_mixed_case_particles() {
79 assert!(is_particle("van"));
80 assert!(is_particle("Van"));
81 assert!(is_particle("VAN"));
82 assert!(is_particle("de"));
83 }
84
85 #[test]
86 fn rejects_non_particle_words() {
87 assert!(!is_particle("Smith"));
88 assert!(!is_particle("denheyeri"));
89 assert!(!is_particle(""));
90 }
91}
92
93fn is_letter(c: char) -> bool {
96 c.is_alphabetic()
97}
98
99fn is_digit(c: char) -> bool {
102 c.is_ascii_digit()
103}
104
105fn is_all_upper(s: &str) -> bool {
106 for c in s.chars() {
107 if is_letter(c) && !c.is_uppercase() {
108 return false;
109 }
110 }
111 true
112}
113
114pub fn tokenize(input: &str) -> Vec<Token> {
118 let cs: Vec<(usize, char)> = input.char_indices().collect();
119 let n = cs.len();
120 let byte_at = |k: usize| -> usize {
121 if k < n {
122 cs[k].0
123 } else {
124 input.len()
125 }
126 };
127 let mut out: Vec<Token> = Vec::with_capacity((n / 4).max(4));
128 let mut k = 0usize;
129
130 while k < n {
131 let c = cs[k].1;
132
133 if is_whitespace_java(c) {
134 k += 1;
135 continue;
136 }
137
138 if is_letter(c) {
139 let word_start = k;
140 k += 1;
141 while k < n {
142 let c2 = cs[k].1;
143 if is_letter(c2) || is_digit(c2) {
144 k += 1;
145 continue;
146 }
147 if matches!(
149 c2,
150 '-' | '\''
151 | '\u{2019}'
152 | '_'
153 | '!'
154 | '\u{2010}'
155 | '\u{2011}'
156 | '\u{2012}'
157 | '\u{2013}'
158 | '\u{2014}'
159 ) && k + 1 < n
160 {
161 let next = cs[k + 1].1;
162 if is_letter(next) || is_digit(next) {
163 k += 1;
164 continue;
165 }
166 }
167 break;
168 }
169 let start_b = byte_at(word_start);
170 let end_b = byte_at(k);
171 let word = &input[start_b..end_b];
172 let first = word.chars().next().unwrap();
173 let char_count = word.chars().count();
174
175 if char_count == 1 && (first == 'x' || first == 'X') {
176 let left_ok = word_start == 0 || is_whitespace_java(cs[word_start - 1].1);
177 let right_ok = k == n || is_whitespace_java(cs[k].1);
178 let kind = if left_ok && right_ok {
179 TokenKind::HybridMark
180 } else {
181 TokenKind::Word
182 };
183 out.push(Token {
184 kind,
185 text: word.to_string(),
186 start: start_b,
187 end: end_b,
188 });
189 continue;
190 } else if char_count >= 2 && (first == 'x' || first == 'X') {
191 let second = word.chars().nth(1).unwrap();
192 if second.is_uppercase() && !is_all_upper(word) {
193 let first_len = first.len_utf8();
194 out.push(Token {
195 kind: TokenKind::HybridMark,
196 text: word[..first_len].to_string(),
197 start: start_b,
198 end: start_b + first_len,
199 });
200 out.push(Token {
201 kind: TokenKind::Word,
202 text: word[first_len..].to_string(),
203 start: start_b + first_len,
204 end: end_b,
205 });
206 continue;
207 }
208 }
209 out.push(Token {
210 kind: TokenKind::Word,
211 text: word.to_string(),
212 start: start_b,
213 end: end_b,
214 });
215 continue;
216 }
217
218 if is_digit(c) {
219 let num_start = k;
220 k += 1;
221 while k < n && is_digit(cs[k].1) {
222 k += 1;
223 }
224 if k + 1 < n && cs[k].1 == '-' && is_letter(cs[k + 1].1) {
226 k += 1; while k < n {
228 let c2 = cs[k].1;
229 if is_letter(c2) || is_digit(c2) {
230 k += 1;
231 continue;
232 }
233 if c2 == '-' && k + 1 < n {
234 let next = cs[k + 1].1;
235 if is_letter(next) || is_digit(next) {
236 k += 1;
237 continue;
238 }
239 }
240 break;
241 }
242 let s = byte_at(num_start);
243 let e = byte_at(k);
244 out.push(Token {
245 kind: TokenKind::Word,
246 text: input[s..e].to_string(),
247 start: s,
248 end: e,
249 });
250 continue;
251 }
252 let s = byte_at(num_start);
253 let e = byte_at(k);
254 out.push(Token {
255 kind: TokenKind::Number,
256 text: input[s..e].to_string(),
257 start: s,
258 end: e,
259 });
260 continue;
261 }
262
263 let kind = match c {
264 '(' => TokenKind::OpenParen,
265 ')' => TokenKind::CloseParen,
266 '[' => TokenKind::OpenBracket,
267 ']' => TokenKind::CloseBracket,
268 '{' => TokenKind::OpenBrace,
269 '}' => TokenKind::CloseBrace,
270 ',' => TokenKind::Comma,
271 ';' => TokenKind::Semicolon,
272 ':' => TokenKind::Colon,
273 '.' => TokenKind::Dot,
274 '&' => TokenKind::Ampersand,
275 '\u{00D7}' => TokenKind::HybridMark, '+' => TokenKind::HybridMark,
277 '\u{2020}' => TokenKind::Dagger, _ => TokenKind::Other,
279 };
280 let s = byte_at(k);
281 let e = byte_at(k + 1);
282 out.push(Token {
283 kind,
284 text: input[s..e].to_string(),
285 start: s,
286 end: e,
287 });
288 k += 1;
289 }
290 out
291}
292
293#[cfg(test)]
294mod tests {
295 use super::*;
296
297 fn kinds(input: &str) -> Vec<(TokenKind, String)> {
298 tokenize(input)
299 .into_iter()
300 .map(|t| (t.kind, t.text))
301 .collect()
302 }
303
304 #[test]
305 fn simple_binomial() {
306 use TokenKind::*;
307 assert_eq!(
308 kinds("Aus bus"),
309 vec![(Word, "Aus".into()), (Word, "bus".into())]
310 );
311 }
312
313 #[test]
314 fn author_year() {
315 use TokenKind::*;
316 assert_eq!(
317 kinds("Aus bus L., 1758"),
318 vec![
319 (Word, "Aus".into()),
320 (Word, "bus".into()),
321 (Word, "L".into()),
322 (Dot, ".".into()),
323 (Comma, ",".into()),
324 (Number, "1758".into()),
325 ]
326 );
327 }
328
329 #[test]
330 fn internal_hyphen_and_apostrophe_stay_in_word() {
331 assert_eq!(kinds("Hartmann-Schroder").len(), 1);
332 assert_eq!(kinds("d'urvilleana").len(), 1);
333 assert_eq!(kinds("pu!chra").len(), 1); }
335
336 #[test]
337 fn leading_numeral_epithet_is_one_word() {
338 use TokenKind::*;
339 assert_eq!(kinds("11-punctata"), vec![(Word, "11-punctata".into())]);
340 assert_eq!(kinds("2-pustulata"), vec![(Word, "2-pustulata".into())]);
341 }
342
343 #[test]
344 fn bare_number_is_number() {
345 use TokenKind::*;
346 assert_eq!(kinds("1758"), vec![(Number, "1758".into())]);
347 }
348
349 #[test]
350 fn hybrid_marks() {
351 use TokenKind::*;
352 assert_eq!(
354 kinds("Aus \u{00D7}bus"),
355 vec![
356 (Word, "Aus".into()),
357 (HybridMark, "\u{00D7}".into()),
358 (Word, "bus".into())
359 ]
360 );
361 assert_eq!(
363 kinds("Aus x bus"),
364 vec![
365 (Word, "Aus".into()),
366 (HybridMark, "x".into()),
367 (Word, "bus".into())
368 ]
369 );
370 assert_eq!(
372 kinds("xBus"),
373 vec![(HybridMark, "x".into()), (Word, "Bus".into())]
374 );
375 }
376
377 #[test]
378 fn punctuation_and_dagger() {
379 use TokenKind::*;
380 assert_eq!(
381 kinds("\u{2020}Aus (Bus)"),
382 vec![
383 (Dagger, "\u{2020}".into()),
384 (Word, "Aus".into()),
385 (OpenParen, "(".into()),
386 (Word, "Bus".into()),
387 (CloseParen, ")".into()),
388 ]
389 );
390 }
391}