aspero_engine/
identify.rs

1//! Which work a document is, from its first page: not the works it cites
2//! or links to.
3//!
4//! A first page prints the paper's own DOI, but often others too: the
5//! journal's, data or code, a related article, the first references. So
6//! no identifier is taken on trust. Each one on the page (those in the
7//! header and footer first, at most [`MAX_IDENTIFIERS`]) is looked up, and
8//! its work must be confirmed: by the title the caller gives (the largest
9//! type at the top of the page, or the file's title metadata unless junk),
10//! or by its own title and first author appearing on the page, as a
11//! paper's do and the works it cites usually don't. With none confirmed,
12//! the title and the top of the page are verified as a citation.
13
14use aspero_core::id::{Mention, scan};
15use aspero_core::matching::{self, Agreement};
16use aspero_core::{Claim, FieldSet, SourceRecord, WorkId};
17
18use crate::{Engine, Verdict, Verification};
19
20/// Identifiers looked up per page.
21pub const MAX_IDENTIFIERS: usize = 6;
22/// The share of a page's lines at its top and at its bottom that count as
23/// header and footer.
24const EDGE: f64 = 0.2;
25/// Characters from the top of the page verified as a citation when no
26/// identifier is confirmed: title, authors and affiliations.
27const TOP_CHARS: usize = 800;
28
29/// What decided which work a document is, from surest to least sure.
30#[derive(Debug, Clone, Copy, PartialEq, Eq)]
31#[non_exhaustive]
32pub enum Decision {
33    /// An identifier on the page, its work's title matching the title
34    /// given.
35    IdentifierAndTitle,
36    /// An identifier on the page, its work's title and first author on the
37    /// page too.
38    IdentifierAndPage,
39    /// The page's only identifier, with no title to check it against.
40    OnlyIdentifier,
41    /// No identifier confirmed: a search for the title and the top of the
42    /// page.
43    TitleSearch,
44}
45
46impl Decision {
47    pub fn as_str(self) -> &'static str {
48        match self {
49            Decision::IdentifierAndTitle => "identifier_and_title",
50            Decision::IdentifierAndPage => "identifier_and_page",
51            Decision::OnlyIdentifier => "only_identifier",
52            Decision::TitleSearch => "title_search",
53        }
54    }
55}
56
57/// The work a document is, and how that was decided.
58#[derive(Debug, Clone)]
59#[non_exhaustive]
60pub struct Identified {
61    pub decided_by: Decision,
62    /// The claim verified: the identifier chosen, or the title and top of
63    /// the page.
64    pub claim: Claim,
65    /// The work as a verified (or, by title search, plausible) citation.
66    pub verification: Verification,
67}
68
69impl Identified {
70    /// What was verified, on one line: the identifier, or the start of the
71    /// page.
72    pub fn input(&self) -> String {
73        if !self.claim.ids.is_empty() {
74            let ids: Vec<String> = self.claim.ids.iter().map(ToString::to_string).collect();
75            return ids.join(" ");
76        }
77        let text = self.claim.reference().unwrap_or_default();
78        let line = text.split_whitespace().collect::<Vec<_>>().join(" ");
79        match line.char_indices().nth(100) {
80            Some((at, _)) => format!("{}…", &line[..at]),
81            None => line,
82        }
83    }
84}
85
86impl Engine {
87    /// The work a document is, from its first page's text and, optionally,
88    /// its title: `title_hint` (the largest type at the top of the page)
89    /// and `title` (the file's metadata, ignored when junk such as
90    /// "Microsoft Word - draft3.docx"). `None` when nothing is sure enough.
91    pub async fn identify(
92        &self,
93        first_page: &str,
94        title: Option<&str>,
95        title_hint: Option<&str>,
96    ) -> Option<Identified> {
97        // Metadata titles of a word or two are more often placeholders than
98        // titles; the largest type on the page is the title, however short.
99        let titles: Vec<&str> = [(title_hint, 1), (title, 3)]
100            .into_iter()
101            .filter_map(|(t, min_words)| t.map(str::trim).filter(|t| !junk_title(t, min_words)))
102            .collect();
103        let ids = candidates(first_page);
104        let found = self.lookup(&ids, FieldSet::DEFAULT).await;
105        let records: Vec<(&WorkId, &SourceRecord)> = ids
106            .iter()
107            .zip(&found.results)
108            .filter_map(|(id, r)| Some((id, r.record.as_ref()?)))
109            .collect();
110
111        let by_title = records
112            .iter()
113            .filter_map(|&(id, r)| {
114                let best = titles.iter().map(|t| title_score(t, r)).fold(0.0, f64::max);
115                (best >= TITLE_MATCH).then_some((best, id))
116            })
117            // On a tie, the first: the page's header and footer come before
118            // its body (a preprint's DOI and the published one both match).
119            .rev()
120            .max_by(|a, b| a.0.total_cmp(&b.0))
121            .map(|(_, id)| (id, Decision::IdentifierAndTitle));
122        // A short letter's references may start on its first page, titles
123        // and authors and all: the paper's own title is the one at the top.
124        let on_page = || {
125            let page = Claim::from_text(first_page);
126            let norm = matching::normalise(first_page);
127            records
128                .iter()
129                .filter(|(_, r)| {
130                    let a = matching::assess(&page, r);
131                    a.title.agreement == Agreement::Match && a.authors.agreement == Agreement::Match
132                })
133                .min_by_key(|(_, r)| {
134                    let title = r.work.title.as_deref().map(matching::normalise);
135                    title.and_then(|t| norm.find(&t)).unwrap_or(usize::MAX)
136                })
137                .map(|&(id, _)| (id, Decision::IdentifierAndPage))
138        };
139        let only = || match (&ids[..], titles.is_empty(), &records[..]) {
140            ([_], true, [(id, _)]) => Some((*id, Decision::OnlyIdentifier)),
141            _ => None,
142        };
143        if let Some((id, decided_by)) = by_title.or_else(on_page).or_else(only) {
144            let mut claim = Claim::default();
145            claim.ids = vec![id.clone()];
146            let verification = self.verify(&claim, 5).await;
147            return Some(Identified {
148                decided_by,
149                claim,
150                verification,
151            });
152        }
153
154        // No identifier confirmed: the top of the page as a citation, with
155        // the identifiers already ruled out left aside. A title given is
156        // there already when it's the largest type; one that isn't (a
157        // banner, "PLOS ONE") mustn't be taken for the title, so it only
158        // joins the text.
159        let mut top: String = first_page.chars().take(TOP_CHARS).collect();
160        let norm = matching::normalise(&top);
161        for t in &titles {
162            if !norm.contains(&matching::normalise(t)) {
163                top = format!("{t}. {top}");
164            }
165        }
166        let mut claim = Claim::default();
167        claim.text = Some(top);
168        let verification = self.verify(&claim, 5).await;
169        matches!(verification.verdict, Verdict::Verified | Verdict::Plausible).then_some(
170            Identified {
171                decided_by: Decision::TitleSearch,
172                claim,
173                verification,
174            },
175        )
176    }
177}
178
179/// Title similarity that counts as the same title.
180const TITLE_MATCH: f64 = 0.9;
181
182/// How well a record's title matches a title given for the document.
183fn title_score(title: &str, record: &SourceRecord) -> f64 {
184    record
185        .work
186        .title
187        .as_deref()
188        .map_or(0.0, |actual| matching::title_similarity(title, actual))
189}
190
191/// The page's identifiers to look up: distinct works, those in its header
192/// and footer before those in its body, at most [`MAX_IDENTIFIERS`].
193fn candidates(page: &str) -> Vec<WorkId> {
194    let lines = page.lines().count().max(1);
195    let edge = |m: &Mention| {
196        let at = (m.line - 1) as f64 / lines as f64;
197        at < EDGE || at >= 1.0 - EDGE
198    };
199    let mentions = scan(page);
200    let (mut ordered, body): (Vec<&Mention>, Vec<&Mention>) =
201        mentions.iter().partition(|m| edge(m));
202    ordered.extend(body);
203    let mut ids: Vec<WorkId> = Vec::new();
204    for m in ordered {
205        if !ids.iter().any(|id| matching::same_work_id(id, &m.id)) {
206            ids.push(m.id.clone());
207        }
208    }
209    ids.truncate(MAX_IDENTIFIERS);
210    ids
211}
212
213/// A title that isn't one: empty, a placeholder, a word processor's file
214/// name ("Microsoft Word - draft3.docx"), a file name, or under
215/// `min_words` words.
216fn junk_title(title: &str, min_words: usize) -> bool {
217    let lower = title.trim().to_lowercase();
218    let placeholder = [
219        "untitled",
220        "title",
221        "document",
222        "manuscript",
223        "article",
224        "pdf",
225    ]
226    .iter()
227    .any(|p| lower == *p || lower.starts_with(&format!("{p} ")) && lower.len() < 16);
228    let tool = [
229        "microsoft word",
230        "microsoft powerpoint",
231        "untitled document",
232        "doi:",
233        "http",
234    ]
235    .iter()
236    .any(|p| lower.starts_with(p));
237    let file = [
238        ".doc", ".docx", ".pdf", ".tex", ".dvi", ".indd", ".rtf", ".odt",
239    ]
240    .iter()
241    .any(|ext| lower.ends_with(ext))
242        || (!lower.contains(' ') && lower.contains(['_', '-', '.']));
243    // A margin stamp ("arXiv:1706.03762v7 [cs.CL] 2 Aug 2023") or a link.
244    let identifier = !scan(title).is_empty();
245    let words = lower
246        .split_whitespace()
247        .filter(|w| w.chars().any(char::is_alphabetic))
248        .count();
249    placeholder || tool || file || identifier || words < min_words.max(1)
250}
251
252#[cfg(test)]
253mod tests {
254    use super::*;
255
256    #[test]
257    fn metadata_titles_that_are_not_titles() {
258        for junk in [
259            "",
260            "untitled",
261            "Microsoft Word - draft3.docx",
262            "paper_final_v2",
263            "manuscript.pdf",
264            "Title",
265            "arXiv:1706.03762v7 [cs.CL] 2 Aug 2023",
266        ] {
267            assert!(junk_title(junk, 1), "{junk:?}");
268        }
269        assert!(junk_title("Deep learning", 3));
270        assert!(!junk_title("Deep learning", 1));
271        assert!(!junk_title(
272            "Chest wall thickness and decompression failure",
273            3
274        ));
275    }
276
277    #[test]
278    fn header_and_footer_identifiers_come_first() {
279        let mut page =
280            String::from("Journal of Things 12 (2020) 1-9 https://doi.org/10.1234/jt.2020.1\n");
281        for i in 0..20 {
282            page.push_str(&format!("Body line {i} citing doi:10.5555/ref.{i}\n"));
283        }
284        page.push_str("© 2020 Elsevier. doi:10.1234/jt.2020.1\n");
285        let ids = candidates(&page);
286        assert_eq!(ids.len(), MAX_IDENTIFIERS);
287        assert_eq!(ids[0].to_string(), "doi:10.1234/jt.2020.1");
288        // The repeat in the footer is the same work, counted once.
289        assert_eq!(
290            ids.iter()
291                .filter(|id| id.to_string() == "doi:10.1234/jt.2020.1")
292                .count(),
293            1
294        );
295    }
296}
aspero_engine/identify.rs

aspero_engine/
identify.rs
