Skip to main content

aspero_core/
references.rs

1//! Splitting a pasted reference list into one citation each, offline.
2//!
3//! Lists arrive numbered (`1.`, `[1]`, `(1)`), bulleted, separated by blank
4//! lines, or one per line, with lines wrapped by PDFs and word processors.
5//! The heuristics, in order:
6//!
7//! 1. **Numbered or bulleted**: two or more lines start with a marker
8//!    (numbers in sequence, so `2015. Title` inside a reference doesn't
9//!    count). Each marker starts a reference; other lines continue it.
10//! 2. **Paragraphs**: blank lines separate references; within one, a line
11//!    starts another reference only if it looks like an author list after
12//!    a line that ended a sentence.
13//! 3. **Lines**: one reference per line when every line has a year or an
14//!    identifier, ends as a reference ends, and doesn't start in lower
15//!    case; otherwise wrapped, split as within a paragraph.
16//!
17//! Wrapped lines are joined with a space; a word hyphenated across lines
18//! is mended, and DOIs and URLs broken after `/` or `-` are rejoined.
19//! Headings such as "References" are dropped.
20
21use std::ops::Range;
22
23use crate::Alternative;
24use crate::fold::Folded;
25
26/// One reference found in a list.
27#[derive(Debug, Clone, PartialEq, Eq)]
28#[non_exhaustive]
29pub struct Reference {
30    /// On one line, without its number or bullet.
31    pub text: String,
32    /// Byte range in the list: from after the marker to the end of its
33    /// last line.
34    pub span: Range<usize>,
35    /// DOIs broken across lines at a hyphen, read without it: to try if
36    /// the DOI as printed doesn't exist.
37    pub alternatives: Vec<Alternative>,
38}
39
40/// The references in `text`, in order. Text from PDFs is cleaned first
41/// (ligatures, invisible characters, look-alike dashes and quotes); spans
42/// point into `text` as given.
43pub fn split_references(text: &str) -> Vec<Reference> {
44    let folded = Folded::new(text);
45    let list = List::new(&folded.text);
46    let groups = list.groups();
47    references(text, &folded, &list, groups)
48}
49
50/// A line of a reference list as laid out, with its distance from the
51/// left edge (any unit, such as PDF points).
52#[derive(Debug, Clone, PartialEq)]
53#[non_exhaustive]
54pub struct IndentedLine {
55    pub text: String,
56    pub indent: f64,
57}
58
59impl IndentedLine {
60    pub fn new(text: impl Into<String>, indent: f64) -> Self {
61        IndentedLine {
62            text: text.into(),
63            indent,
64        }
65    }
66}
67
68/// One reference found in laid-out lines.
69#[derive(Debug, Clone, PartialEq, Eq)]
70#[non_exhaustive]
71pub struct LineReference {
72    /// On one line, without its number or bullet.
73    pub text: String,
74    /// Indices of the lines it spans.
75    pub lines: Range<usize>,
76    pub alternatives: Vec<Alternative>,
77}
78
79/// The references in laid-out lines (as from a PDF), using the layout
80/// plain text loses: with a hanging indent, references start at the
81/// margin; with a first-line indent, indented. Numbered lists, and lines
82/// all at one indent, are split by the text rules of [`split_references`].
83pub fn split_lines(lines: &[IndentedLine]) -> Vec<LineReference> {
84    let text = lines
85        .iter()
86        .map(|l| l.text.replace(['\r', '\n'], " "))
87        .collect::<Vec<_>>()
88        .join("\n");
89    let folded = Folded::new(&text);
90    let list = List::new(&folded.text);
91    let groups = match starts_by_indent(lines) {
92        Some(starts) if list.numbered().is_none() => {
93            let mut groups: Vec<Vec<Range<usize>>> = Vec::new();
94            for (i, line) in list.lines.iter().enumerate() {
95                let Some(line) = line else { continue };
96                match groups.last_mut() {
97                    Some(group) if !starts[i] => group.push(line.clone()),
98                    _ => groups.push(vec![line.clone()]),
99                }
100            }
101            groups
102        }
103        _ => list.groups(),
104    };
105    let line_starts: Vec<usize> = std::iter::once(0)
106        .chain(text.match_indices('\n').map(|(i, _)| i + 1))
107        .collect();
108    let line_of = |byte: usize| line_starts.partition_point(|&s| s <= byte) - 1;
109    references(&text, &folded, &list, groups)
110        .into_iter()
111        .map(|r| LineReference {
112            lines: line_of(r.span.start)..line_of(r.span.end.saturating_sub(1)) + 1,
113            text: r.text,
114            alternatives: r.alternatives,
115        })
116        .collect()
117}
118
119/// Which lines start a reference, by indent: `None` if the lines don't
120/// differ enough to tell. Indents within half the gap to the next level
121/// count as the same, so jitter in a PDF's positions doesn't matter.
122fn starts_by_indent(lines: &[IndentedLine]) -> Option<Vec<bool>> {
123    let content: Vec<&IndentedLine> = lines.iter().filter(|l| !l.text.trim().is_empty()).collect();
124    let margin = content
125        .iter()
126        .map(|l| l.indent)
127        .fold(f64::INFINITY, f64::min);
128    let next = content
129        .iter()
130        .map(|l| l.indent)
131        .filter(|&i| i > margin + 1.0)
132        .fold(f64::INFINITY, f64::min);
133    if !next.is_finite() {
134        return None;
135    }
136    let at_margin = |l: &IndentedLine| l.indent - margin < (next - margin) / 2.0;
137    let first = content.iter().find(|l| !is_heading(l.text.trim()))?;
138    let starting = at_margin(first);
139    Some(
140        lines
141            .iter()
142            .map(|l| !l.text.trim().is_empty() && at_margin(l) == starting)
143            .collect(),
144    )
145}
146
147/// Groups of lines (ranges in `folded`) as references in `text`.
148fn references(
149    text: &str,
150    folded: &Folded,
151    list: &List,
152    groups: Vec<Vec<Range<usize>>>,
153) -> Vec<Reference> {
154    groups
155        .into_iter()
156        .map(|g| {
157            let span = folded.original(&(g[0].start..g[g.len() - 1].end));
158            let written = &text[span.clone()];
159            let alternatives = crate::id::scan(written)
160                .iter()
161                .filter_map(|m| Some(Alternative::new(m.id.clone(), m.alternative(written)?)))
162                .collect();
163            Reference {
164                text: list.join(&g),
165                span,
166                alternatives,
167            }
168        })
169        .filter(|r| !is_heading(&r.text))
170        .collect()
171}
172
173struct List<'a> {
174    text: &'a str,
175    /// Every line trimmed, as a byte range; `None` for blank ones.
176    lines: Vec<Option<Range<usize>>>,
177}
178
179impl<'a> List<'a> {
180    fn new(text: &'a str) -> Self {
181        let mut lines = Vec::new();
182        let mut at = 0;
183        for raw in text.split_inclusive('\n') {
184            let trimmed = raw.trim();
185            lines.push((!trimmed.is_empty()).then(|| {
186                let start = at + (raw.len() - raw.trim_start().len());
187                start..start + trimmed.len()
188            }));
189            at += raw.len();
190        }
191        List { text, lines }
192    }
193
194    /// Groups by the text rules: numbers or bullets, else paragraphs, else
195    /// lines.
196    fn groups(&self) -> Vec<Vec<Range<usize>>> {
197        self.numbered().unwrap_or_else(|| {
198            let paragraphs: Vec<Vec<Range<usize>>> = self
199                .lines
200                .split(Option::is_none)
201                .filter(|p| !p.is_empty())
202                .map(|p| p.iter().flatten().cloned().collect())
203                .collect();
204            let alone = paragraphs.len() == 1;
205            paragraphs
206                .into_iter()
207                .flat_map(|p| self.unnumbered(p, alone))
208                .collect()
209        })
210    }
211
212    fn line(&self, r: &Range<usize>) -> &'a str {
213        &self.text[r.clone()]
214    }
215
216    /// Groups by number or bullet, if two or more lines start with one.
217    /// Lines before the first marker are dropped (a heading, usually).
218    fn numbered(&self) -> Option<Vec<Vec<Range<usize>>>> {
219        let mut groups: Vec<Vec<Range<usize>>> = Vec::new();
220        let mut last: Option<Marker> = None;
221        for line in self.lines.iter().flatten() {
222            let starts = marker(self.line(line)).filter(|(m, _)| match (last, m) {
223                (None, _) => true,
224                (Some(Marker::Number(n)), Marker::Number(m)) => *m == n + 1,
225                (Some(Marker::Bullet), Marker::Bullet) => true,
226                _ => false,
227            });
228            match (starts, groups.last_mut()) {
229                (Some((m, used)), _) => {
230                    last = Some(m);
231                    // A marker alone on its line: the text follows.
232                    let first = line.start + used..line.end;
233                    groups.push(Vec::new());
234                    if !first.is_empty() {
235                        groups.last_mut().expect("just pushed").push(first);
236                    }
237                }
238                (None, Some(group)) => group.push(line.clone()),
239                (None, None) => {}
240            }
241        }
242        // One marker counts only on the first line ("1. Smith J. …"): later,
243        // it may be a wrapped line ("2. ed. Berlin").
244        let first_marked = self
245            .lines
246            .iter()
247            .flatten()
248            .next()
249            .is_some_and(|l| marker(self.line(l)).is_some());
250        groups.retain(|g| !g.is_empty());
251        (groups.len() >= 2 || groups.len() == 1 && first_marked).then_some(groups)
252    }
253
254    /// Groups lines without markers: each its own reference if `alone`
255    /// (the whole list) and they read as one per line; otherwise split
256    /// where an author list follows a finished sentence.
257    fn unnumbered(&self, lines: Vec<Range<usize>>, alone: bool) -> Vec<Vec<Range<usize>>> {
258        let one_per_line = alone
259            && lines.iter().all(|l| dated(self.line(l)))
260            && lines.windows(2).all(|w| {
261                let (line, next) = (self.line(&w[0]), self.line(&w[1]));
262                ends_reference(line) && !continues(next) && !link_continues(line, next)
263            });
264        if one_per_line {
265            return lines.into_iter().map(|l| vec![l]).collect();
266        }
267        let mut groups: Vec<Vec<Range<usize>>> = Vec::new();
268        for line in lines {
269            let text = self.line(&line);
270            match groups.last_mut() {
271                Some(group) if !(ends_sentence(&self.join(group)) && starts_with_author(text)) => {
272                    group.push(line)
273                }
274                _ => groups.push(vec![line]),
275            }
276        }
277        groups
278    }
279
280    /// A group's lines as one: joined with a space, or with none where a
281    /// word, DOI or URL was broken across lines.
282    fn join(&self, group: &[Range<usize>]) -> String {
283        let mut out = String::new();
284        for line in group {
285            let next = self.line(line);
286            let first = next.chars().next().unwrap_or(' ');
287            if out.is_empty() {
288                // Nothing to join to.
289            } else if out.ends_with('-') && first.is_lowercase() && hyphenated_word(&out) {
290                out.pop();
291            } else if !link_continues(&out, next) {
292                out.push(' ');
293            }
294            out.push_str(next);
295        }
296        out
297    }
298}
299
300#[derive(Debug, Clone, Copy, PartialEq, Eq)]
301enum Marker {
302    Number(u32),
303    Bullet,
304}
305
306/// The marker a line starts with, and its length with the space after.
307fn marker(line: &str) -> Option<(Marker, usize)> {
308    let spaced = |rest: &str, used: usize, needs_space: bool| {
309        let gap = rest.len() - rest.trim_start().len();
310        (gap > 0 || !needs_space).then_some(used + gap)
311    };
312    if let Some(rest) = line.strip_prefix(['•', '–', '-', '*', '·']) {
313        return Some((Marker::Bullet, spaced(rest, line.len() - rest.len(), true)?));
314    }
315    let (open, close): (usize, &[char]) = match line.as_bytes().first()? {
316        b'[' => (1, &[']']),
317        b'(' => (1, &[')']),
318        _ => (0, &['.', ')']),
319    };
320    let inner = &line[open..];
321    let digits = inner.len() - inner.trim_start_matches(|c: char| c.is_ascii_digit()).len();
322    if !(1..=4).contains(&digits) {
323        return None;
324    }
325    let n: u32 = inner[..digits].parse().ok()?;
326    let rest = inner[digits..].strip_prefix(close)?;
327    // `[1]Smith` is unambiguous; `1.` needs a space, so `1.5` isn't one.
328    Some((
329        Marker::Number(n),
330        spaced(rest, line.len() - rest.len(), open == 0)?,
331    ))
332}
333
334/// Ends as a reference does: a full stop, page numbers, a closing bracket,
335/// or an identifier.
336fn ends_reference(line: &str) -> bool {
337    ends_sentence(line) || line.ends_with(|c: char| c.is_ascii_digit() || c == ')' || c == ']')
338}
339
340/// Ends with a full stop (or `?`, `!`), or with a whole DOI or URL, as
341/// references often do without one.
342fn ends_sentence(line: &str) -> bool {
343    line.ends_with(['.', '?', '!']) || in_link(line) && !line.ends_with(['/', '-'])
344}
345
346/// Has a year (1600 to 2099) or an identifier, as a whole reference does.
347fn dated(line: &str) -> bool {
348    let year = line
349        .split(|c: char| !c.is_ascii_digit())
350        .any(|n| n.len() == 4 && ("1600".."2100").contains(&n));
351    year || line.contains("doi") || line.contains("10.") || line.contains("http")
352}
353
354/// Starts as a wrapped line's continuation does: in lower case, or with
355/// punctuation.
356fn continues(line: &str) -> bool {
357    line.starts_with(|c: char| c.is_lowercase() || ",;:)&".contains(c))
358}
359
360/// Starts with a surname and initials or a comma, as author lists do:
361/// "Smith J,", "Smith JA.", "Smith, J.", "SMITH, J.", "O'Neil, K.".
362fn starts_with_author(line: &str) -> bool {
363    let surname_len = line
364        .char_indices()
365        .take_while(|(_, c)| c.is_alphabetic() || "'’-".contains(*c))
366        .last()
367        .map_or(0, |(i, c)| i + c.len_utf8());
368    let surname = &line[..surname_len];
369    if surname.chars().count() < 2 || !surname.starts_with(char::is_uppercase) {
370        return false;
371    }
372    let rest = &line[surname_len..];
373    if rest.starts_with(',') {
374        return true;
375    }
376    // Initials: one to three capitals, maybe dotted, then a break.
377    let Some(rest) = rest.strip_prefix(' ') else {
378        return false;
379    };
380    let initials: String = rest
381        .chars()
382        .take_while(|c| c.is_uppercase() || *c == '.')
383        .collect();
384    let capitals = initials.chars().filter(|c| c.is_uppercase()).count();
385    let after = rest[initials.len()..].chars().next();
386    (1..=3).contains(&capitals) && after.is_none_or(|c| ", .;".contains(c))
387}
388
389/// A word broken with a hyphen at the end of `text`: letters before it.
390fn hyphenated_word(text: &str) -> bool {
391    let word = text.rsplit(char::is_whitespace).next().unwrap_or_default();
392    !in_link(text) && word.trim_end_matches('-').chars().all(char::is_alphabetic)
393}
394
395/// `next` continues a DOI or URL that `text` ends with, broken where PDFs
396/// break them: after `/`, `-` or `_` (`10.1038/s41586-` + `020-2649-2`),
397/// after a dot before a digit or lower case (`j.cell.` + `2020.01.001`),
398/// or mid-word before more of the link (`j.ce` + `ll.2020.01.001`, not
399/// `2015` or `2015.`).
400/// Hyphens are kept as printed: DOIs have real ones, and typesetters
401/// rarely add their own inside a link.
402pub(crate) fn link_continues(text: &str, next: &str) -> bool {
403    let token = next.split_whitespace().next().unwrap_or_default();
404    let Some(first) = token.chars().next() else {
405        return false;
406    };
407    if !in_link(text) || !first.is_alphanumeric() {
408        return false;
409    }
410    let lower_or_digit = first.is_lowercase() || first.is_ascii_digit();
411    if text.ends_with(['/', '-', '_']) {
412        true
413    } else if text.ends_with('.') {
414        lower_or_digit
415    } else {
416        // Mid-word: only before more of a link, never a word or a year.
417        let inner = token.trim_end_matches(|c: char| !c.is_alphanumeric());
418        lower_or_digit && inner.contains(|c: char| "./-_".contains(c))
419    }
420}
421
422/// `text` ends inside a DOI or URL.
423fn in_link(text: &str) -> bool {
424    let word = text.rsplit(char::is_whitespace).next().unwrap_or_default();
425    let word = word.to_ascii_lowercase();
426    word.starts_with("http") || word.starts_with("doi:") || word.starts_with("10.")
427}
428
429fn is_heading(text: &str) -> bool {
430    let t = text.trim_end_matches(':').trim().to_lowercase();
431    matches!(
432        t.as_str(),
433        "references"
434            | "reference list"
435            | "bibliography"
436            | "works cited"
437            | "literature cited"
438            | "sources"
439            | "citations"
440    )
441}
442
443#[cfg(test)]
444mod tests {
445    use super::*;
446
447    fn texts(list: &str) -> Vec<String> {
448        split_references(list).into_iter().map(|r| r.text).collect()
449    }
450
451    #[test]
452    fn numbered_lists_split_at_each_number_in_sequence() {
453        let list = "References\n\
454            1. LeCun Y, Bengio Y, Hinton G. Deep learning. Nature.\n\
455            2015;521:436-44.\n\
456            2. Vaswani A, et al. Attention is all you need. NeurIPS 2017.\n";
457        assert_eq!(
458            texts(list),
459            [
460                "LeCun Y, Bengio Y, Hinton G. Deep learning. Nature. 2015;521:436-44.",
461                "Vaswani A, et al. Attention is all you need. NeurIPS 2017.",
462            ]
463        );
464    }
465
466    #[test]
467    fn bracketed_and_parenthesised_numbers() {
468        let list = "[1] Smith J. One. 2020.\n[2]Jones K. Two. 2021.\n";
469        assert_eq!(texts(list), ["Smith J. One. 2020.", "Jones K. Two. 2021."]);
470        let list = "(1) Smith J. One. 2020.\n(2) Jones K. Two. 2021.\n";
471        assert_eq!(texts(list), ["Smith J. One. 2020.", "Jones K. Two. 2021."]);
472    }
473
474    #[test]
475    fn numbers_out_of_sequence_continue_the_reference() {
476        let list = "1. Smith J. A study of things. Journal\n\
477            2015. 12(3):1-9.\n\
478            2. Jones K. Another. 2021.\n";
479        assert_eq!(
480            texts(list),
481            [
482                "Smith J. A study of things. Journal 2015. 12(3):1-9.",
483                "Jones K. Another. 2021.",
484            ]
485        );
486    }
487
488    #[test]
489    fn bullets_mark_references() {
490        let list = "• Smith J. One. 2020.\n• Jones K. Two,\n  continued. 2021.\n";
491        assert_eq!(
492            texts(list),
493            ["Smith J. One. 2020.", "Jones K. Two, continued. 2021."]
494        );
495    }
496
497    #[test]
498    fn blank_lines_separate_wrapped_references() {
499        let list = "Vaswani A, Shazeer N. Attention is all you need.\n\
500            Advances in Neural Information Processing Systems. 2017.\n\
501            \n\
502            LeCun Y, Bengio Y, Hinton G. Deep learning.\n\
503            Nature. 2015;521:436-44.\n";
504        assert_eq!(
505            texts(list),
506            [
507                "Vaswani A, Shazeer N. Attention is all you need. Advances in Neural Information Processing Systems. 2017.",
508                "LeCun Y, Bengio Y, Hinton G. Deep learning. Nature. 2015;521:436-44.",
509            ]
510        );
511    }
512
513    #[test]
514    fn one_per_line() {
515        let list = "Smith J. One. 2020.\n\
516            World Health Organization. Two. 2021.\n\
517            Jones K. Three. https://example.org/three\n";
518        assert_eq!(texts(list).len(), 3);
519    }
520
521    #[test]
522    fn wrapped_lines_without_blank_lines_split_at_authors() {
523        let list = "Smith, J. (2020). A study of\n\
524            things. Journal of Studies, 12, 1-9.\n\
525            Jones, K. (2021). Another study. Nature,\n\
526            521, 436.\n";
527        assert_eq!(
528            texts(list),
529            [
530                "Smith, J. (2020). A study of things. Journal of Studies, 12, 1-9.",
531                "Jones, K. (2021). Another study. Nature, 521, 436.",
532            ]
533        );
534    }
535
536    #[test]
537    fn hyphenation_and_broken_links_are_mended() {
538        let list = "1. Smith J. Neuro-\nlogical find-\nings. doi:10.1038/\nnature14539\n\
539            2. Jones K. https://example.org/a-\nb/c\n";
540        assert_eq!(
541            texts(list),
542            [
543                "Smith J. Neurological findings. doi:10.1038/nature14539",
544                "Jones K. https://example.org/a-b/c",
545            ]
546        );
547    }
548
549    #[test]
550    fn links_broken_by_pdf_extraction_are_rejoined_as_printed() {
551        let list = "[1] Smith J. Cells. Cell. 2020. https://doi.org/10.1016/j.cell.\n\
552            2020.01.001\n\
553            [2] Jones K. Folds. Nature. 2020. doi:10.1038/s41586-\n\
554            020-2649-2\n\
555            [3] Brown L. Stems. https://doi.org/10.1016/j.ce\n\
556            ll.2021.02.003\n";
557        assert_eq!(
558            texts(list),
559            [
560                "Smith J. Cells. Cell. 2020. https://doi.org/10.1016/j.cell.2020.01.001",
561                "Jones K. Folds. Nature. 2020. doi:10.1038/s41586-020-2649-2",
562                "Brown L. Stems. https://doi.org/10.1016/j.cell.2021.02.003",
563            ]
564        );
565        // Unnumbered, the continuation isn't a reference of its own.
566        let list = "Smith J. Cells. Cell. 2020. doi:10.1016/j.cell.\n2020.01.001\n\
567            Jones K. Folds. Nature. 2020.\n";
568        assert_eq!(texts(list).len(), 2);
569        // A link that ends the line, then a new sentence, stays apart.
570        assert_eq!(
571            texts("1. Smith J. doi:10.1038/x.\nNature. 2020.\n2. Jones K. 2021.")[0],
572            "Smith J. doi:10.1038/x. Nature. 2020."
573        );
574    }
575
576    #[test]
577    fn a_single_numbered_reference_loses_its_number() {
578        assert_eq!(
579            texts("1. LeCun Y. Deep learning. Nature. 2015."),
580            ["LeCun Y. Deep learning. Nature. 2015."]
581        );
582        // A number further down is part of the reference.
583        assert_eq!(
584            texts("Smith J. A book.\n2. ed. Berlin: Springer; 2010."),
585            ["Smith J. A book. 2. ed. Berlin: Springer; 2010."]
586        );
587    }
588
589    #[test]
590    fn markers_alone_on_their_line_number_the_next() {
591        assert_eq!(
592            texts("[1]\nSmith J. One. 2020.\n[2]\nJones K. Two. 2021.\n[3] "),
593            ["Smith J. One. 2020.", "Jones K. Two. 2021."]
594        );
595    }
596
597    #[test]
598    fn pdf_text_is_cleaned_and_spans_point_into_it() {
599        let list = "1. Smith J. Neuro\u{AD}\nlogical findings. doi:10.1038/s41586\u{2010}020-2649-2\n\
600            2. Jones K. Two. 2021.";
601        let refs = split_references(list);
602        assert_eq!(
603            refs[0].text,
604            "Smith J. Neurological findings. doi:10.1038/s41586-020-2649-2"
605        );
606        assert!(list[refs[0].span.clone()].starts_with("Smith J. Neuro\u{AD}"));
607        assert!(list[refs[0].span.clone()].ends_with("\u{2010}020-2649-2"));
608    }
609
610    #[test]
611    fn dois_broken_at_a_hyphen_carry_the_reading_without_it() {
612        let refs = split_references("1. Smith J. doi:10.1038/nat-\nure14539\n2. Jones K. 2021.");
613        assert_eq!(refs[0].text, "Smith J. doi:10.1038/nat-ure14539");
614        let alt = &refs[0].alternatives[0];
615        assert_eq!(alt.claimed.to_string(), "doi:10.1038/nat-ure14539");
616        assert_eq!(alt.alternative.to_string(), "doi:10.1038/nature14539");
617        assert!(refs[1].alternatives.is_empty());
618    }
619
620    #[test]
621    fn hanging_indents_mark_starts_that_text_cannot() {
622        // Each line ends a sentence and the next starts with a capital,
623        // so the text alone reads three references.
624        let lines = [
625            IndentedLine::new("References", 72.4),
626            IndentedLine::new("World Health Organization. Global report.", 72.0),
627            IndentedLine::new("Geneva: WHO; 2020.", 90.2),
628            IndentedLine::new("Smith J. A study. Nature. 2021.", 71.8),
629        ];
630        let refs = split_lines(&lines);
631        assert_eq!(
632            refs.iter().map(|r| r.text.as_str()).collect::<Vec<_>>(),
633            [
634                "World Health Organization. Global report. Geneva: WHO; 2020.",
635                "Smith J. A study. Nature. 2021.",
636            ]
637        );
638        assert_eq!(refs[0].lines, 1..3);
639        assert_eq!(refs[1].lines, 3..4);
640    }
641
642    #[test]
643    fn first_line_indents_and_flat_lists() {
644        let lines = [
645            IndentedLine::new("Smith J. A study of", 90.0),
646            IndentedLine::new("Things. Nature. 2021.", 72.0),
647            IndentedLine::new("Jones K. Another. 2022.", 90.0),
648        ];
649        assert_eq!(split_lines(&lines).len(), 2);
650        // One indent throughout: the text rules decide.
651        let flat = [
652            IndentedLine::new("1. Smith J. One.", 72.0),
653            IndentedLine::new("2020.", 72.0),
654            IndentedLine::new("2. Jones K. Two. 2021.", 72.0),
655        ];
656        let refs = split_lines(&flat);
657        assert_eq!(refs[0].text, "Smith J. One. 2020.");
658        assert_eq!(refs[1].lines, 2..3);
659    }
660
661    #[test]
662    fn spans_cover_the_reference_without_its_marker() {
663        let list = "1. Smith J. One.\r\n   2020.\r\n2. Jones K. Two. 2021.";
664        let refs = split_references(list);
665        assert_eq!(&list[refs[0].span.clone()], "Smith J. One.\r\n   2020.");
666        assert_eq!(&list[refs[1].span.clone()], "Jones K. Two. 2021.");
667    }
668
669    #[test]
670    fn a_single_reference_is_one() {
671        assert_eq!(
672            texts("Smith J. A study.\nJournal 2020;1:2."),
673            ["Smith J. A study. Journal 2020;1:2."]
674        );
675        assert!(texts("  \n\n").is_empty());
676    }
677}