aspero_engine/
identify.rs

1//! Which work a document is, from its first page: not the works it cites
2//! or links to.
3//!
4//! A first page prints the paper's own DOI, but often others too: the
5//! journal's, data or code, a related article, the first references. So
6//! no identifier is taken on trust. Each one on the page (those in the
7//! header and footer first, at most [`MAX_IDENTIFIERS`]) is looked up, and
8//! its work must be confirmed: by the title the caller gives (the largest
9//! type at the top of the page, or the file's title metadata unless junk),
10//! or by its own title and first author appearing on the page, as a
11//! paper's do and the works it cites usually don't. With none confirmed,
12//! the title and the top of the page are verified as a citation.
13
14use aspero_core::id::{Mention, scan};
15use aspero_core::matching::{self, Agreement};
16use aspero_core::{Claim, FieldSet, SourceRecord, WorkId};
17
18use crate::{Engine, Verdict, Verification};
19
20/// Identifiers looked up per page.
21pub const MAX_IDENTIFIERS: usize = 6;
22/// The share of a page's lines at its top and at its bottom that count as
23/// header and footer.
24const EDGE: f64 = 0.2;
25/// Characters from the top of the page verified as a citation when no
26/// identifier is confirmed: title, authors and affiliations.
27const TOP_CHARS: usize = 800;
28
29/// What decided which work a document is, from surest to least sure.
30#[derive(Debug, Clone, Copy, PartialEq, Eq)]
31#[non_exhaustive]
32pub enum Decision {
33    /// An identifier on the page, its work's title matching the title
34    /// given.
35    IdentifierAndTitle,
36    /// An identifier on the page, its work's title and first author on the
37    /// page too.
38    IdentifierAndPage,
39    /// The page's only identifier, with no title to check it against.
40    OnlyIdentifier,
41    /// No identifier confirmed: a search for the title and the top of the
42    /// page.
43    TitleSearch,
44}
45
46impl Decision {
47    pub fn as_str(self) -> &'static str {
48        match self {
49            Decision::IdentifierAndTitle => "identifier_and_title",
50            Decision::IdentifierAndPage => "identifier_and_page",
51            Decision::OnlyIdentifier => "only_identifier",
52            Decision::TitleSearch => "title_search",
53        }
54    }
55}
56
57/// The work a document is, and how that was decided.
58#[derive(Debug, Clone)]
59#[non_exhaustive]
60pub struct Identified {
61    pub decided_by: Decision,
62    /// The claim verified: the identifier chosen, or the title and top of
63    /// the page.
64    pub claim: Claim,
65    /// The work as a verified (or, by title search, plausible) citation.
66    pub verification: Verification,
67}
68
69impl Identified {
70    /// What was verified, on one line: the identifier, or the start of the
71    /// page.
72    pub fn input(&self) -> String {
73        if !self.claim.ids.is_empty() {
74            let ids: Vec<String> = self.claim.ids.iter().map(ToString::to_string).collect();
75            return ids.join(" ");
76        }
77        let text = self.claim.reference().unwrap_or_default();
78        let line = text.split_whitespace().collect::<Vec<_>>().join(" ");
79        match line.char_indices().nth(100) {
80            Some((at, _)) => format!("{}…", &line[..at]),
81            None => line,
82        }
83    }
84}
85
86impl Engine {
87    /// The work a document is, from its first page's text and, optionally,
88    /// its title: `title_hint` (the largest type at the top of the page)
89    /// and `title` (the file's metadata, ignored when junk such as
90    /// "Microsoft Word - draft3.docx"). `None` when nothing is sure enough.
91    pub async fn identify(
92        &self,
93        first_page: &str,
94        title: Option<&str>,
95        title_hint: Option<&str>,
96    ) -> Option<Identified> {
97        // Metadata titles of a word or two are more often placeholders than
98        // titles; the largest type on the page is the title, however short.
99        let titles: Vec<&str> = [(title_hint, 1), (title, 3)]
100            .into_iter()
101            .filter_map(|(t, min_words)| t.map(str::trim).filter(|t| !junk_title(t, min_words)))
102            .collect();
103        let ids = candidates(first_page);
104        let found = self.lookup(&ids, FieldSet::DEFAULT).await;
105        let records: Vec<(&WorkId, &SourceRecord)> = ids
106            .iter()
107            .zip(&found.results)
108            .filter_map(|(id, r)| Some((id, r.record.as_ref()?)))
109            .collect();
110
111        let by_title = records
112            .iter()
113            .filter_map(|&(id, r)| {
114                let best = titles.iter().map(|t| title_score(t, r)).fold(0.0, f64::max);
115                (best >= TITLE_MATCH).then_some((best, id))
116            })
117            .max_by(|a, b| a.0.total_cmp(&b.0))
118            .map(|(_, id)| (id, Decision::IdentifierAndTitle));
119        // A short letter's references may start on its first page, titles
120        // and authors and all: the paper's own title is the one at the top.
121        let on_page = || {
122            let page = Claim::from_text(first_page);
123            let norm = matching::normalise(first_page);
124            records
125                .iter()
126                .filter(|(_, r)| {
127                    let a = matching::assess(&page, r);
128                    a.title.agreement == Agreement::Match && a.authors.agreement == Agreement::Match
129                })
130                .min_by_key(|(_, r)| {
131                    let title = r.work.title.as_deref().map(matching::normalise);
132                    title.and_then(|t| norm.find(&t)).unwrap_or(usize::MAX)
133                })
134                .map(|&(id, _)| (id, Decision::IdentifierAndPage))
135        };
136        let only = || match (&ids[..], titles.is_empty(), &records[..]) {
137            ([_], true, [(id, _)]) => Some((*id, Decision::OnlyIdentifier)),
138            _ => None,
139        };
140        if let Some((id, decided_by)) = by_title.or_else(on_page).or_else(only) {
141            let mut claim = Claim::default();
142            claim.ids = vec![id.clone()];
143            let verification = self.verify(&claim, 5).await;
144            return Some(Identified {
145                decided_by,
146                claim,
147                verification,
148            });
149        }
150
151        // No identifier confirmed: the top of the page as a citation, with
152        // the identifiers already ruled out left aside. A title given is
153        // there already when it's the largest type; one that isn't (a
154        // banner, "PLOS ONE") mustn't be taken for the title, so it only
155        // joins the text.
156        let mut top: String = first_page.chars().take(TOP_CHARS).collect();
157        let norm = matching::normalise(&top);
158        for t in &titles {
159            if !norm.contains(&matching::normalise(t)) {
160                top = format!("{t}. {top}");
161            }
162        }
163        let mut claim = Claim::default();
164        claim.text = Some(top);
165        let verification = self.verify(&claim, 5).await;
166        matches!(verification.verdict, Verdict::Verified | Verdict::Plausible).then_some(
167            Identified {
168                decided_by: Decision::TitleSearch,
169                claim,
170                verification,
171            },
172        )
173    }
174}
175
176/// Title similarity that counts as the same title.
177const TITLE_MATCH: f64 = 0.9;
178
179/// How well a record's title matches a title given for the document.
180fn title_score(title: &str, record: &SourceRecord) -> f64 {
181    record
182        .work
183        .title
184        .as_deref()
185        .map_or(0.0, |actual| matching::title_similarity(title, actual))
186}
187
188/// The page's identifiers to look up: distinct works, those in its header
189/// and footer before those in its body, at most [`MAX_IDENTIFIERS`].
190fn candidates(page: &str) -> Vec<WorkId> {
191    let lines = page.lines().count().max(1);
192    let edge = |m: &Mention| {
193        let at = (m.line - 1) as f64 / lines as f64;
194        at < EDGE || at >= 1.0 - EDGE
195    };
196    let mentions = scan(page);
197    let (mut ordered, body): (Vec<&Mention>, Vec<&Mention>) =
198        mentions.iter().partition(|m| edge(m));
199    ordered.extend(body);
200    let mut ids: Vec<WorkId> = Vec::new();
201    for m in ordered {
202        if !ids.iter().any(|id| matching::same_work_id(id, &m.id)) {
203            ids.push(m.id.clone());
204        }
205    }
206    ids.truncate(MAX_IDENTIFIERS);
207    ids
208}
209
210/// A title that isn't one: empty, a placeholder, a word processor's file
211/// name ("Microsoft Word - draft3.docx"), a file name, or under
212/// `min_words` words.
213fn junk_title(title: &str, min_words: usize) -> bool {
214    let lower = title.trim().to_lowercase();
215    let placeholder = [
216        "untitled",
217        "title",
218        "document",
219        "manuscript",
220        "article",
221        "pdf",
222    ]
223    .iter()
224    .any(|p| lower == *p || lower.starts_with(&format!("{p} ")) && lower.len() < 16);
225    let tool = [
226        "microsoft word",
227        "microsoft powerpoint",
228        "untitled document",
229        "doi:",
230        "http",
231    ]
232    .iter()
233    .any(|p| lower.starts_with(p));
234    let file = [
235        ".doc", ".docx", ".pdf", ".tex", ".dvi", ".indd", ".rtf", ".odt",
236    ]
237    .iter()
238    .any(|ext| lower.ends_with(ext))
239        || (!lower.contains(' ') && lower.contains(['_', '-', '.']));
240    // A margin stamp ("arXiv:1706.03762v7 [cs.CL] 2 Aug 2023") or a link.
241    let identifier = !scan(title).is_empty();
242    let words = lower
243        .split_whitespace()
244        .filter(|w| w.chars().any(char::is_alphabetic))
245        .count();
246    placeholder || tool || file || identifier || words < min_words.max(1)
247}
248
249#[cfg(test)]
250mod tests {
251    use super::*;
252
253    #[test]
254    fn metadata_titles_that_are_not_titles() {
255        for junk in [
256            "",
257            "untitled",
258            "Microsoft Word - draft3.docx",
259            "paper_final_v2",
260            "manuscript.pdf",
261            "Title",
262            "arXiv:1706.03762v7 [cs.CL] 2 Aug 2023",
263        ] {
264            assert!(junk_title(junk, 1), "{junk:?}");
265        }
266        assert!(junk_title("Deep learning", 3));
267        assert!(!junk_title("Deep learning", 1));
268        assert!(!junk_title(
269            "Chest wall thickness and decompression failure",
270            3
271        ));
272    }
273
274    #[test]
275    fn header_and_footer_identifiers_come_first() {
276        let mut page =
277            String::from("Journal of Things 12 (2020) 1-9 https://doi.org/10.1234/jt.2020.1\n");
278        for i in 0..20 {
279            page.push_str(&format!("Body line {i} citing doi:10.5555/ref.{i}\n"));
280        }
281        page.push_str("© 2020 Elsevier. doi:10.1234/jt.2020.1\n");
282        let ids = candidates(&page);
283        assert_eq!(ids.len(), MAX_IDENTIFIERS);
284        assert_eq!(ids[0].to_string(), "doi:10.1234/jt.2020.1");
285        // The repeat in the footer is the same work, counted once.
286        assert_eq!(
287            ids.iter()
288                .filter(|id| id.to_string() == "doi:10.1234/jt.2020.1")
289                .count(),
290            1
291        );
292    }
293}
aspero_engine/identify.rs

aspero_engine/
identify.rs
