1use std::ops::Range;
22
23use crate::Alternative;
24use crate::fold::Folded;
25
26#[derive(Debug, Clone, PartialEq, Eq)]
28#[non_exhaustive]
29pub struct Reference {
30 pub text: String,
32 pub span: Range<usize>,
35 pub alternatives: Vec<Alternative>,
38}
39
40pub fn split_references(text: &str) -> Vec<Reference> {
44 let folded = Folded::new(text);
45 let list = List::new(&folded.text);
46 let groups = list.groups();
47 references(text, &folded, &list, groups)
48}
49
50#[derive(Debug, Clone, PartialEq)]
53#[non_exhaustive]
54pub struct IndentedLine {
55 pub text: String,
56 pub indent: f64,
57}
58
59impl IndentedLine {
60 pub fn new(text: impl Into<String>, indent: f64) -> Self {
61 IndentedLine {
62 text: text.into(),
63 indent,
64 }
65 }
66}
67
68#[derive(Debug, Clone, PartialEq, Eq)]
70#[non_exhaustive]
71pub struct LineReference {
72 pub text: String,
74 pub lines: Range<usize>,
76 pub alternatives: Vec<Alternative>,
77}
78
79pub fn split_lines(lines: &[IndentedLine]) -> Vec<LineReference> {
84 let text = lines
85 .iter()
86 .map(|l| l.text.replace(['\r', '\n'], " "))
87 .collect::<Vec<_>>()
88 .join("\n");
89 let folded = Folded::new(&text);
90 let list = List::new(&folded.text);
91 let groups = match starts_by_indent(lines) {
92 Some(starts) if list.numbered().is_none() => {
93 let mut groups: Vec<Vec<Range<usize>>> = Vec::new();
94 for (i, line) in list.lines.iter().enumerate() {
95 let Some(line) = line else { continue };
96 match groups.last_mut() {
97 Some(group) if !starts[i] => group.push(line.clone()),
98 _ => groups.push(vec![line.clone()]),
99 }
100 }
101 groups
102 }
103 _ => list.groups(),
104 };
105 let line_starts: Vec<usize> = std::iter::once(0)
106 .chain(text.match_indices('\n').map(|(i, _)| i + 1))
107 .collect();
108 let line_of = |byte: usize| line_starts.partition_point(|&s| s <= byte) - 1;
109 references(&text, &folded, &list, groups)
110 .into_iter()
111 .map(|r| LineReference {
112 lines: line_of(r.span.start)..line_of(r.span.end.saturating_sub(1)) + 1,
113 text: r.text,
114 alternatives: r.alternatives,
115 })
116 .collect()
117}
118
119fn starts_by_indent(lines: &[IndentedLine]) -> Option<Vec<bool>> {
123 let content: Vec<&IndentedLine> = lines.iter().filter(|l| !l.text.trim().is_empty()).collect();
124 let margin = content
125 .iter()
126 .map(|l| l.indent)
127 .fold(f64::INFINITY, f64::min);
128 let next = content
129 .iter()
130 .map(|l| l.indent)
131 .filter(|&i| i > margin + 1.0)
132 .fold(f64::INFINITY, f64::min);
133 if !next.is_finite() {
134 return None;
135 }
136 let at_margin = |l: &IndentedLine| l.indent - margin < (next - margin) / 2.0;
137 let first = content.iter().find(|l| !is_heading(l.text.trim()))?;
138 let starting = at_margin(first);
139 Some(
140 lines
141 .iter()
142 .map(|l| !l.text.trim().is_empty() && at_margin(l) == starting)
143 .collect(),
144 )
145}
146
147fn references(
149 text: &str,
150 folded: &Folded,
151 list: &List,
152 groups: Vec<Vec<Range<usize>>>,
153) -> Vec<Reference> {
154 groups
155 .into_iter()
156 .map(|g| {
157 let span = folded.original(&(g[0].start..g[g.len() - 1].end));
158 let written = &text[span.clone()];
159 let alternatives = crate::id::scan(written)
160 .iter()
161 .filter_map(|m| Some(Alternative::new(m.id.clone(), m.alternative(written)?)))
162 .collect();
163 Reference {
164 text: list.join(&g),
165 span,
166 alternatives,
167 }
168 })
169 .filter(|r| !is_heading(&r.text))
170 .collect()
171}
172
173struct List<'a> {
174 text: &'a str,
175 lines: Vec<Option<Range<usize>>>,
177}
178
179impl<'a> List<'a> {
180 fn new(text: &'a str) -> Self {
181 let mut lines = Vec::new();
182 let mut at = 0;
183 for raw in text.split_inclusive('\n') {
184 let trimmed = raw.trim();
185 lines.push((!trimmed.is_empty()).then(|| {
186 let start = at + (raw.len() - raw.trim_start().len());
187 start..start + trimmed.len()
188 }));
189 at += raw.len();
190 }
191 List { text, lines }
192 }
193
194 fn groups(&self) -> Vec<Vec<Range<usize>>> {
197 self.numbered().unwrap_or_else(|| {
198 let paragraphs: Vec<Vec<Range<usize>>> = self
199 .lines
200 .split(Option::is_none)
201 .filter(|p| !p.is_empty())
202 .map(|p| p.iter().flatten().cloned().collect())
203 .collect();
204 let alone = paragraphs.len() == 1;
205 paragraphs
206 .into_iter()
207 .flat_map(|p| self.unnumbered(p, alone))
208 .collect()
209 })
210 }
211
212 fn line(&self, r: &Range<usize>) -> &'a str {
213 &self.text[r.clone()]
214 }
215
216 fn numbered(&self) -> Option<Vec<Vec<Range<usize>>>> {
219 let mut groups: Vec<Vec<Range<usize>>> = Vec::new();
220 let mut last: Option<Marker> = None;
221 for line in self.lines.iter().flatten() {
222 let starts = marker(self.line(line)).filter(|(m, _)| match (last, m) {
223 (None, _) => true,
224 (Some(Marker::Number(n)), Marker::Number(m)) => *m == n + 1,
225 (Some(Marker::Bullet), Marker::Bullet) => true,
226 _ => false,
227 });
228 match (starts, groups.last_mut()) {
229 (Some((m, used)), _) => {
230 last = Some(m);
231 let first = line.start + used..line.end;
233 groups.push(Vec::new());
234 if !first.is_empty() {
235 groups.last_mut().expect("just pushed").push(first);
236 }
237 }
238 (None, Some(group)) => group.push(line.clone()),
239 (None, None) => {}
240 }
241 }
242 let first_marked = self
245 .lines
246 .iter()
247 .flatten()
248 .next()
249 .is_some_and(|l| marker(self.line(l)).is_some());
250 groups.retain(|g| !g.is_empty());
251 (groups.len() >= 2 || groups.len() == 1 && first_marked).then_some(groups)
252 }
253
254 fn unnumbered(&self, lines: Vec<Range<usize>>, alone: bool) -> Vec<Vec<Range<usize>>> {
258 let one_per_line = alone
259 && lines.iter().all(|l| dated(self.line(l)))
260 && lines.windows(2).all(|w| {
261 let (line, next) = (self.line(&w[0]), self.line(&w[1]));
262 ends_reference(line) && !continues(next) && !link_continues(line, next)
263 });
264 if one_per_line {
265 return lines.into_iter().map(|l| vec![l]).collect();
266 }
267 let mut groups: Vec<Vec<Range<usize>>> = Vec::new();
268 for line in lines {
269 let text = self.line(&line);
270 match groups.last_mut() {
271 Some(group) if !(ends_sentence(&self.join(group)) && starts_with_author(text)) => {
272 group.push(line)
273 }
274 _ => groups.push(vec![line]),
275 }
276 }
277 groups
278 }
279
280 fn join(&self, group: &[Range<usize>]) -> String {
283 let mut out = String::new();
284 for line in group {
285 let next = self.line(line);
286 let first = next.chars().next().unwrap_or(' ');
287 if out.is_empty() {
288 } else if out.ends_with('-') && first.is_lowercase() && hyphenated_word(&out) {
290 out.pop();
291 } else if !link_continues(&out, next) {
292 out.push(' ');
293 }
294 out.push_str(next);
295 }
296 out
297 }
298}
299
300#[derive(Debug, Clone, Copy, PartialEq, Eq)]
301enum Marker {
302 Number(u32),
303 Bullet,
304}
305
306fn marker(line: &str) -> Option<(Marker, usize)> {
308 let spaced = |rest: &str, used: usize, needs_space: bool| {
309 let gap = rest.len() - rest.trim_start().len();
310 (gap > 0 || !needs_space).then_some(used + gap)
311 };
312 if let Some(rest) = line.strip_prefix(['•', '–', '-', '*', '·']) {
313 return Some((Marker::Bullet, spaced(rest, line.len() - rest.len(), true)?));
314 }
315 let (open, close): (usize, &[char]) = match line.as_bytes().first()? {
316 b'[' => (1, &[']']),
317 b'(' => (1, &[')']),
318 _ => (0, &['.', ')']),
319 };
320 let inner = &line[open..];
321 let digits = inner.len() - inner.trim_start_matches(|c: char| c.is_ascii_digit()).len();
322 if !(1..=4).contains(&digits) {
323 return None;
324 }
325 let n: u32 = inner[..digits].parse().ok()?;
326 let rest = inner[digits..].strip_prefix(close)?;
327 Some((
329 Marker::Number(n),
330 spaced(rest, line.len() - rest.len(), open == 0)?,
331 ))
332}
333
334fn ends_reference(line: &str) -> bool {
337 ends_sentence(line) || line.ends_with(|c: char| c.is_ascii_digit() || c == ')' || c == ']')
338}
339
340fn ends_sentence(line: &str) -> bool {
343 line.ends_with(['.', '?', '!']) || in_link(line) && !line.ends_with(['/', '-'])
344}
345
346fn dated(line: &str) -> bool {
348 let year = line
349 .split(|c: char| !c.is_ascii_digit())
350 .any(|n| n.len() == 4 && ("1600".."2100").contains(&n));
351 year || line.contains("doi") || line.contains("10.") || line.contains("http")
352}
353
354fn continues(line: &str) -> bool {
357 line.starts_with(|c: char| c.is_lowercase() || ",;:)&".contains(c))
358}
359
360fn starts_with_author(line: &str) -> bool {
363 let surname_len = line
364 .char_indices()
365 .take_while(|(_, c)| c.is_alphabetic() || "'’-".contains(*c))
366 .last()
367 .map_or(0, |(i, c)| i + c.len_utf8());
368 let surname = &line[..surname_len];
369 if surname.chars().count() < 2 || !surname.starts_with(char::is_uppercase) {
370 return false;
371 }
372 let rest = &line[surname_len..];
373 if rest.starts_with(',') {
374 return true;
375 }
376 let Some(rest) = rest.strip_prefix(' ') else {
378 return false;
379 };
380 let initials: String = rest
381 .chars()
382 .take_while(|c| c.is_uppercase() || *c == '.')
383 .collect();
384 let capitals = initials.chars().filter(|c| c.is_uppercase()).count();
385 let after = rest[initials.len()..].chars().next();
386 (1..=3).contains(&capitals) && after.is_none_or(|c| ", .;".contains(c))
387}
388
389fn hyphenated_word(text: &str) -> bool {
391 let word = text.rsplit(char::is_whitespace).next().unwrap_or_default();
392 !in_link(text) && word.trim_end_matches('-').chars().all(char::is_alphabetic)
393}
394
395pub(crate) fn link_continues(text: &str, next: &str) -> bool {
403 let token = next.split_whitespace().next().unwrap_or_default();
404 let Some(first) = token.chars().next() else {
405 return false;
406 };
407 if !in_link(text) || !first.is_alphanumeric() {
408 return false;
409 }
410 let lower_or_digit = first.is_lowercase() || first.is_ascii_digit();
411 if text.ends_with(['/', '-', '_']) {
412 true
413 } else if text.ends_with('.') {
414 lower_or_digit
415 } else {
416 let inner = token.trim_end_matches(|c: char| !c.is_alphanumeric());
418 lower_or_digit && inner.contains(|c: char| "./-_".contains(c))
419 }
420}
421
422fn in_link(text: &str) -> bool {
424 let word = text.rsplit(char::is_whitespace).next().unwrap_or_default();
425 let word = word.to_ascii_lowercase();
426 word.starts_with("http") || word.starts_with("doi:") || word.starts_with("10.")
427}
428
429fn is_heading(text: &str) -> bool {
430 let t = text.trim_end_matches(':').trim().to_lowercase();
431 matches!(
432 t.as_str(),
433 "references"
434 | "reference list"
435 | "bibliography"
436 | "works cited"
437 | "literature cited"
438 | "sources"
439 | "citations"
440 )
441}
442
443#[cfg(test)]
444mod tests {
445 use super::*;
446
447 fn texts(list: &str) -> Vec<String> {
448 split_references(list).into_iter().map(|r| r.text).collect()
449 }
450
451 #[test]
452 fn numbered_lists_split_at_each_number_in_sequence() {
453 let list = "References\n\
454 1. LeCun Y, Bengio Y, Hinton G. Deep learning. Nature.\n\
455 2015;521:436-44.\n\
456 2. Vaswani A, et al. Attention is all you need. NeurIPS 2017.\n";
457 assert_eq!(
458 texts(list),
459 [
460 "LeCun Y, Bengio Y, Hinton G. Deep learning. Nature. 2015;521:436-44.",
461 "Vaswani A, et al. Attention is all you need. NeurIPS 2017.",
462 ]
463 );
464 }
465
466 #[test]
467 fn bracketed_and_parenthesised_numbers() {
468 let list = "[1] Smith J. One. 2020.\n[2]Jones K. Two. 2021.\n";
469 assert_eq!(texts(list), ["Smith J. One. 2020.", "Jones K. Two. 2021."]);
470 let list = "(1) Smith J. One. 2020.\n(2) Jones K. Two. 2021.\n";
471 assert_eq!(texts(list), ["Smith J. One. 2020.", "Jones K. Two. 2021."]);
472 }
473
474 #[test]
475 fn numbers_out_of_sequence_continue_the_reference() {
476 let list = "1. Smith J. A study of things. Journal\n\
477 2015. 12(3):1-9.\n\
478 2. Jones K. Another. 2021.\n";
479 assert_eq!(
480 texts(list),
481 [
482 "Smith J. A study of things. Journal 2015. 12(3):1-9.",
483 "Jones K. Another. 2021.",
484 ]
485 );
486 }
487
488 #[test]
489 fn bullets_mark_references() {
490 let list = "• Smith J. One. 2020.\n• Jones K. Two,\n continued. 2021.\n";
491 assert_eq!(
492 texts(list),
493 ["Smith J. One. 2020.", "Jones K. Two, continued. 2021."]
494 );
495 }
496
497 #[test]
498 fn blank_lines_separate_wrapped_references() {
499 let list = "Vaswani A, Shazeer N. Attention is all you need.\n\
500 Advances in Neural Information Processing Systems. 2017.\n\
501 \n\
502 LeCun Y, Bengio Y, Hinton G. Deep learning.\n\
503 Nature. 2015;521:436-44.\n";
504 assert_eq!(
505 texts(list),
506 [
507 "Vaswani A, Shazeer N. Attention is all you need. Advances in Neural Information Processing Systems. 2017.",
508 "LeCun Y, Bengio Y, Hinton G. Deep learning. Nature. 2015;521:436-44.",
509 ]
510 );
511 }
512
513 #[test]
514 fn one_per_line() {
515 let list = "Smith J. One. 2020.\n\
516 World Health Organization. Two. 2021.\n\
517 Jones K. Three. https://example.org/three\n";
518 assert_eq!(texts(list).len(), 3);
519 }
520
521 #[test]
522 fn wrapped_lines_without_blank_lines_split_at_authors() {
523 let list = "Smith, J. (2020). A study of\n\
524 things. Journal of Studies, 12, 1-9.\n\
525 Jones, K. (2021). Another study. Nature,\n\
526 521, 436.\n";
527 assert_eq!(
528 texts(list),
529 [
530 "Smith, J. (2020). A study of things. Journal of Studies, 12, 1-9.",
531 "Jones, K. (2021). Another study. Nature, 521, 436.",
532 ]
533 );
534 }
535
536 #[test]
537 fn hyphenation_and_broken_links_are_mended() {
538 let list = "1. Smith J. Neuro-\nlogical find-\nings. doi:10.1038/\nnature14539\n\
539 2. Jones K. https://example.org/a-\nb/c\n";
540 assert_eq!(
541 texts(list),
542 [
543 "Smith J. Neurological findings. doi:10.1038/nature14539",
544 "Jones K. https://example.org/a-b/c",
545 ]
546 );
547 }
548
549 #[test]
550 fn links_broken_by_pdf_extraction_are_rejoined_as_printed() {
551 let list = "[1] Smith J. Cells. Cell. 2020. https://doi.org/10.1016/j.cell.\n\
552 2020.01.001\n\
553 [2] Jones K. Folds. Nature. 2020. doi:10.1038/s41586-\n\
554 020-2649-2\n\
555 [3] Brown L. Stems. https://doi.org/10.1016/j.ce\n\
556 ll.2021.02.003\n";
557 assert_eq!(
558 texts(list),
559 [
560 "Smith J. Cells. Cell. 2020. https://doi.org/10.1016/j.cell.2020.01.001",
561 "Jones K. Folds. Nature. 2020. doi:10.1038/s41586-020-2649-2",
562 "Brown L. Stems. https://doi.org/10.1016/j.cell.2021.02.003",
563 ]
564 );
565 let list = "Smith J. Cells. Cell. 2020. doi:10.1016/j.cell.\n2020.01.001\n\
567 Jones K. Folds. Nature. 2020.\n";
568 assert_eq!(texts(list).len(), 2);
569 assert_eq!(
571 texts("1. Smith J. doi:10.1038/x.\nNature. 2020.\n2. Jones K. 2021.")[0],
572 "Smith J. doi:10.1038/x. Nature. 2020."
573 );
574 }
575
576 #[test]
577 fn a_single_numbered_reference_loses_its_number() {
578 assert_eq!(
579 texts("1. LeCun Y. Deep learning. Nature. 2015."),
580 ["LeCun Y. Deep learning. Nature. 2015."]
581 );
582 assert_eq!(
584 texts("Smith J. A book.\n2. ed. Berlin: Springer; 2010."),
585 ["Smith J. A book. 2. ed. Berlin: Springer; 2010."]
586 );
587 }
588
589 #[test]
590 fn markers_alone_on_their_line_number_the_next() {
591 assert_eq!(
592 texts("[1]\nSmith J. One. 2020.\n[2]\nJones K. Two. 2021.\n[3] "),
593 ["Smith J. One. 2020.", "Jones K. Two. 2021."]
594 );
595 }
596
597 #[test]
598 fn pdf_text_is_cleaned_and_spans_point_into_it() {
599 let list = "1. Smith J. Neuro\u{AD}\nlogical findings. doi:10.1038/s41586\u{2010}020-2649-2\n\
600 2. Jones K. Two. 2021.";
601 let refs = split_references(list);
602 assert_eq!(
603 refs[0].text,
604 "Smith J. Neurological findings. doi:10.1038/s41586-020-2649-2"
605 );
606 assert!(list[refs[0].span.clone()].starts_with("Smith J. Neuro\u{AD}"));
607 assert!(list[refs[0].span.clone()].ends_with("\u{2010}020-2649-2"));
608 }
609
610 #[test]
611 fn dois_broken_at_a_hyphen_carry_the_reading_without_it() {
612 let refs = split_references("1. Smith J. doi:10.1038/nat-\nure14539\n2. Jones K. 2021.");
613 assert_eq!(refs[0].text, "Smith J. doi:10.1038/nat-ure14539");
614 let alt = &refs[0].alternatives[0];
615 assert_eq!(alt.claimed.to_string(), "doi:10.1038/nat-ure14539");
616 assert_eq!(alt.alternative.to_string(), "doi:10.1038/nature14539");
617 assert!(refs[1].alternatives.is_empty());
618 }
619
620 #[test]
621 fn hanging_indents_mark_starts_that_text_cannot() {
622 let lines = [
625 IndentedLine::new("References", 72.4),
626 IndentedLine::new("World Health Organization. Global report.", 72.0),
627 IndentedLine::new("Geneva: WHO; 2020.", 90.2),
628 IndentedLine::new("Smith J. A study. Nature. 2021.", 71.8),
629 ];
630 let refs = split_lines(&lines);
631 assert_eq!(
632 refs.iter().map(|r| r.text.as_str()).collect::<Vec<_>>(),
633 [
634 "World Health Organization. Global report. Geneva: WHO; 2020.",
635 "Smith J. A study. Nature. 2021.",
636 ]
637 );
638 assert_eq!(refs[0].lines, 1..3);
639 assert_eq!(refs[1].lines, 3..4);
640 }
641
642 #[test]
643 fn first_line_indents_and_flat_lists() {
644 let lines = [
645 IndentedLine::new("Smith J. A study of", 90.0),
646 IndentedLine::new("Things. Nature. 2021.", 72.0),
647 IndentedLine::new("Jones K. Another. 2022.", 90.0),
648 ];
649 assert_eq!(split_lines(&lines).len(), 2);
650 let flat = [
652 IndentedLine::new("1. Smith J. One.", 72.0),
653 IndentedLine::new("2020.", 72.0),
654 IndentedLine::new("2. Jones K. Two. 2021.", 72.0),
655 ];
656 let refs = split_lines(&flat);
657 assert_eq!(refs[0].text, "Smith J. One. 2020.");
658 assert_eq!(refs[1].lines, 2..3);
659 }
660
661 #[test]
662 fn spans_cover_the_reference_without_its_marker() {
663 let list = "1. Smith J. One.\r\n 2020.\r\n2. Jones K. Two. 2021.";
664 let refs = split_references(list);
665 assert_eq!(&list[refs[0].span.clone()], "Smith J. One.\r\n 2020.");
666 assert_eq!(&list[refs[1].span.clone()], "Jones K. Two. 2021.");
667 }
668
669 #[test]
670 fn a_single_reference_is_one() {
671 assert_eq!(
672 texts("Smith J. A study.\nJournal 2020;1:2."),
673 ["Smith J. A study. Journal 2020;1:2."]
674 );
675 assert!(texts(" \n\n").is_empty());
676 }
677}