aspero_core/matching/
mod.rs

1//! Does a record match what a citation claims? Pure scoring, no I/O.
2//!
3//! A [`Claim`] is what someone (often an agent) asserted: a free-text
4//! reference, structured fields, identifiers, or a mix. [`assess`] compares
5//! one candidate record against it, field by field, and combines the fields
6//! into a confidence in `0.0..=1.0`.
7//!
8//! Lessons from real data shape the rules (see DESIGN §9):
9//! - **Lookalikes exist**: re-deposits with the real title, authors and
10//!   abstract under a different year and publisher. A wrong year is a strong
11//!   penalty, never a detail.
12//! - **One field is not proof**: a title alone ("Deep learning") matches
13//!   many works, so confidence is capped unless at least two fields agree.
14//! - Free-text references are not parsed into fields. Instead the record's
15//!   fields are looked for *in* the text: its title as a phrase, its authors'
16//!   family names, its year. This is robust to citation styles.
17
18mod authors;
19mod derivative;
20mod text;
21
22use serde::{Deserialize, Serialize};
23
24use crate::id::WorkId;
25use crate::publication::{self, first_page, journal_in, same_journal};
26use crate::record::SourceRecord;
27
28use authors::{author_overlap, family_name};
29use derivative::claim_mentions_derivative;
30pub use derivative::is_derivative;
31use text::{Normalised, content_words};
32pub use text::{normalise, search_terms, shared_title, title_similarity, unmarked};
33
34/// Field weights in the confidence (before penalties).
35const W_TITLE: f64 = 0.5;
36const W_AUTHORS: f64 = 0.25;
37const W_YEAR: f64 = 0.25;
38
39/// Multiplier when the year is off by two or more.
40const YEAR_MISMATCH_PENALTY: f64 = 0.5;
41/// Multiplier when the title clearly differs.
42const TITLE_MISMATCH_PENALTY: f64 = 0.5;
43/// Multiplier when the authors clearly differ.
44const AUTHORS_MISMATCH_PENALTY: f64 = 0.5;
45/// Confidence ceiling when fewer than two fields could be compared.
46const SINGLE_FIELD_CAP: f64 = 0.7;
47/// Confidence ceiling without corroborating authors or identifiers.
48const UNCORROBORATED_CAP: f64 = 0.8;
49/// Author score that corroborates a match: the first author present
50/// (0.75 for structured claims, 0.9 in free text).
51const CORROBORATING_AUTHORS: f64 = 0.75;
52/// Confidence ceiling, below plausible, without evidence of which work is
53/// cited: only a year or journal agreeing, or a short title found inside
54/// free text.
55const NO_EVIDENCE_CAP: f64 = 0.5;
56/// Content words below which a title found inside free text is short: one
57/// or two words turn up in any text ("Aspero", "Deep learning").
58const SHORT_TITLE_WORDS: usize = 3;
59/// Multiplier for replies, corrections, errata and commentaries.
60const DERIVATIVE_PENALTY: f64 = 0.5;
61/// Confidence ceiling when the claimed volume or first page differs.
62const LOCATION_MISMATCH_CAP: f64 = 0.8;
63/// Multiplier when a structured claim names a different journal.
64const JOURNAL_MISMATCH_PENALTY: f64 = 0.9;
65
66/// Verdict thresholds on confidence.
67pub const VERIFIED: f64 = 0.85;
68pub const PLAUSIBLE: f64 = 0.6;
69
70/// Authors compared from the start of a byline.
71const AUTHORS_COMPARED: usize = 3;
72
73/// What a citation asserts. Any part may be missing.
74#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
75#[non_exhaustive]
76pub struct Claim {
77    /// A free-text reference, in any style.
78    #[serde(default, skip_serializing_if = "Option::is_none")]
79    pub text: Option<String>,
80    /// Identifiers asserted for the work.
81    #[serde(default, skip_serializing_if = "Vec::is_empty")]
82    pub ids: Vec<WorkId>,
83    #[serde(default, skip_serializing_if = "Option::is_none")]
84    pub title: Option<String>,
85    /// Names as written: "Ashish Vaswani", "Vaswani, A.", "A. Vaswani".
86    #[serde(default, skip_serializing_if = "Vec::is_empty")]
87    pub authors: Vec<String>,
88    #[serde(default, skip_serializing_if = "Option::is_none")]
89    pub year: Option<i16>,
90    /// The journal (or other container), full or abbreviated.
91    #[serde(default, skip_serializing_if = "Option::is_none")]
92    pub container: Option<String>,
93    #[serde(default, skip_serializing_if = "Option::is_none")]
94    pub volume: Option<String>,
95    /// The first page (or a range, of which the first page counts).
96    #[serde(default, skip_serializing_if = "Option::is_none")]
97    pub page: Option<String>,
98    /// Other readings of claimed identifiers (a DOI without the hyphen at
99    /// a line break), to try only if the claimed one doesn't exist.
100    #[serde(default, skip_serializing_if = "Vec::is_empty")]
101    pub alternatives: Vec<Alternative>,
102}
103
104/// Another reading of a claimed identifier.
105#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
106#[non_exhaustive]
107pub struct Alternative {
108    pub claimed: WorkId,
109    pub alternative: WorkId,
110}
111
112impl Alternative {
113    pub fn new(claimed: WorkId, alternative: WorkId) -> Self {
114        Alternative {
115            claimed,
116            alternative,
117        }
118    }
119}
120
121impl Claim {
122    /// A claim from a free-text reference, with any DOIs and arXiv IDs it
123    /// mentions extracted as asserted identifiers.
124    pub fn from_text(text: impl Into<String>) -> Self {
125        let text = text.into();
126        Claim {
127            ids: find_ids(&text),
128            text: Some(text),
129            ..Claim::default()
130        }
131    }
132
133    /// A claim from whatever a user typed: if the whole input is an
134    /// identifier, a claim of just that identifier (an existence check);
135    /// otherwise a free-text reference (see [`Claim::from_text`]).
136    pub fn from_input(input: &str) -> Self {
137        let input = input.trim();
138        let whole = || match crate::id::scan(input).as_slice() {
139            // As a copy spaced it out: `https:// doi. org/ 10. 1183/ …`.
140            [m] if m.span == (0..input.len()) => Some(m.id.clone()),
141            _ => None,
142        };
143        match input.parse::<WorkId>().ok().or_else(whole) {
144            Some(id) => Claim {
145                ids: vec![id],
146                ..Claim::default()
147            },
148            None => Claim::from_text(input),
149        }
150    }
151
152    /// Nothing to search with: no identifiers, and no text, title or
153    /// authors that aren't blank.
154    pub fn is_empty(&self) -> bool {
155        let blank = |s: &Option<String>| s.as_deref().is_none_or(|s| s.trim().is_empty());
156        blank(&self.text)
157            && self.ids.is_empty()
158            && blank(&self.title)
159            && self.authors.iter().all(|a| a.trim().is_empty())
160    }
161
162    /// A reference string for citation matchers: the text, or one composed
163    /// from the fields.
164    pub fn reference(&self) -> Option<String> {
165        if let Some(text) = &self.text {
166            return Some(text.clone());
167        }
168        // Vancouver-like: "Authors. Title. Journal. 2017;376:2545".
169        let location = match (&self.volume, &self.page) {
170            (Some(v), Some(p)) => Some(format!("{v}:{p}")),
171            (Some(v), None) => Some(v.clone()),
172            (None, p) => p.clone(),
173        };
174        let when = [self.year.map(|y| y.to_string()), location]
175            .into_iter()
176            .flatten()
177            .collect::<Vec<_>>()
178            .join(";");
179        let parts: Vec<String> = [
180            (!self.authors.is_empty()).then(|| self.authors.join(", ")),
181            self.title.clone(),
182            self.container.clone(),
183            (!when.is_empty()).then_some(when),
184        ]
185        .into_iter()
186        .flatten()
187        .collect();
188        (!parts.is_empty()).then(|| parts.join(". "))
189    }
190}
191
192#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
193#[serde(rename_all = "snake_case")]
194pub enum Agreement {
195    Match,
196    Partial,
197    Mismatch,
198    /// The claim or the record doesn't say.
199    Unknown,
200}
201
202/// One field's comparison: a score in `0.0..=1.0` when comparable.
203#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
204pub struct FieldCheck {
205    pub agreement: Agreement,
206    #[serde(skip_serializing_if = "Option::is_none")]
207    pub score: Option<f64>,
208}
209
210impl FieldCheck {
211    const UNKNOWN: FieldCheck = FieldCheck {
212        agreement: Agreement::Unknown,
213        score: None,
214    };
215
216    fn exact(agrees: bool) -> Self {
217        FieldCheck::scored(if agrees { 1.0 } else { 0.0 })
218    }
219
220    fn scored(score: f64) -> Self {
221        let agreement = match score {
222            s if s >= 0.9 => Agreement::Match,
223            s if s >= 0.5 => Agreement::Partial,
224            _ => Agreement::Mismatch,
225        };
226        FieldCheck {
227            agreement,
228            score: Some(score),
229        }
230    }
231}
232
233#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
234#[non_exhaustive]
235pub struct Assessment {
236    pub confidence: f64,
237    pub title: FieldCheck,
238    pub authors: FieldCheck,
239    pub year: FieldCheck,
240    /// The journal: `Match` when the claim names it (or an abbreviation),
241    /// `Mismatch` when a structured claim names another.
242    pub container: FieldCheck,
243    pub volume: FieldCheck,
244    /// The first page.
245    pub page: FieldCheck,
246    /// Whether the claim's identifiers are the record's: `Match` if any
247    /// claimed ID is the record's, `Mismatch` if the record has a different
248    /// ID of the same kind, else `Unknown`.
249    pub ids: Agreement,
250}
251
252/// A title that is a URL.
253fn is_url(title: &str) -> bool {
254    let t = title.trim();
255    let starts = |p: &str| t.get(..p.len()).is_some_and(|h| h.eq_ignore_ascii_case(p));
256    starts("http://") || starts("https://") || starts("www.") || t.contains("://")
257}
258
259/// Scores how well `record` matches `claim`.
260pub fn assess(claim: &Claim, record: &SourceRecord) -> Assessment {
261    let work = &record.work;
262    let text = claim.text.as_deref().map(Normalised::new);
263    // A placeholder record's "title" may be a URL (a registration agency's
264    // notice page): no title at all.
265    let work_title = work.title.clone().filter(|t| !is_url(t));
266
267    let title = match (&claim.title, &work_title, &text) {
268        (Some(claimed), Some(actual), _) => FieldCheck::scored(title_similarity(claimed, actual)),
269        (None, Some(actual), Some(text)) => FieldCheck::scored(text.title_coverage(actual)),
270        _ => FieldCheck::UNKNOWN,
271    };
272
273    let families: Vec<String> = work.authors.iter().filter_map(family_name).collect();
274    let authors = if families.is_empty() {
275        FieldCheck::UNKNOWN
276    } else if !claim.authors.is_empty() {
277        FieldCheck::scored(author_overlap(&claim.authors, &families))
278    } else if let Some(text) = &text {
279        FieldCheck::scored(text.author_coverage(&families))
280    } else {
281        FieldCheck::UNKNOWN
282    };
283
284    let actual_years: Vec<i16> = work
285        .issued
286        .map(|d| d.year())
287        .into_iter()
288        .chain(work.other_years.iter().copied())
289        .collect();
290    let claimed_years: Vec<i16> = match (claim.year, &text) {
291        (Some(y), _) => vec![y],
292        (None, Some(text)) => text.years(),
293        _ => Vec::new(),
294    };
295    let year = match actual_years.is_empty() || claimed_years.is_empty() {
296        false => {
297            // Nearest pairing: any edition's year may be the one cited.
298            let delta = claimed_years
299                .iter()
300                .flat_map(|c| actual_years.iter().map(move |a| (c - a).unsigned_abs()))
301                .min()
302                .unwrap_or(u16::MAX);
303            // Halves per year off: off by one is common (online-first vs
304            // print, preprint vs published) and counts as partial; two or
305            // more is a mismatch, but a nearer year still ranks higher.
306            FieldCheck::scored(0.5f64.powi(i32::from(delta.min(30))))
307        }
308        true => FieldCheck::UNKNOWN,
309    };
310
311    let container = match (&claim.container, &work.container_title, &text) {
312        (Some(claimed), Some(actual), _) => FieldCheck::exact(same_journal(claimed, actual)),
313        (None, Some(actual), Some(text)) if journal_in(&text.words, actual) => {
314            FieldCheck::exact(true)
315        }
316        _ => FieldCheck::UNKNOWN,
317    };
318    let located = match &claim.text {
319        Some(text) if claim.volume.is_none() && claim.page.is_none() => publication::locator(text),
320        _ => publication::Locator {
321            volume: claim.volume.clone(),
322            page: claim.page.as_deref().map(|p| first_page(p).to_owned()),
323        },
324    };
325    let same = |claimed: Option<&str>, actual: Option<&str>| match (claimed, actual) {
326        (Some(c), Some(a)) if !c.is_empty() && !a.is_empty() => {
327            FieldCheck::exact(c.trim().eq_ignore_ascii_case(a.trim()))
328        }
329        _ => FieldCheck::UNKNOWN,
330    };
331    let volume = same(located.volume.as_deref(), work.volume.as_deref());
332    let page = same(
333        located.page.as_deref(),
334        work.page.as_deref().map(first_page),
335    );
336
337    // Citing the exact place: volume and first page.
338    let located = volume.agreement == Agreement::Match && page.agreement == Agreement::Match;
339    // Short Vancouver references often leave the title out ("Daum RS et
340    // al. N Engl J Med 2017;376:2545"); with the journal and volume
341    // agreeing, a title missing from the text is not a different one (the
342    // page then decides between verified and a wrong page).
343    let placed = container.agreement == Agreement::Match && volume.agreement == Agreement::Match;
344    let title = match placed {
345        true if claim.title.is_none() && title.agreement == Agreement::Mismatch => {
346            FieldCheck::UNKNOWN
347        }
348        _ => title,
349    };
350
351    let ids = id_agreement(&claim.ids, &record.ids);
352    // A claim of identifiers alone is fully confirmed by a matching ID.
353    let id_only = [title, authors, year].iter().all(|c| c.score.is_none());
354    let mut confidence = if id_only && ids == Agreement::Match {
355        1.0
356    } else {
357        combine(title, authors, year)
358    };
359    // Title and year alone can't verify: a short or generic title ("Health
360    // technology assessment") turns up inside many references, and letters
361    // and corrections often carry the original's title without authors.
362    // Corroboration means the first author matches, or a claimed ID does.
363    // So does citing the exact place.
364    let corroborated = authors.score.is_some_and(|s| s >= CORROBORATING_AUTHORS)
365        || ids == Agreement::Match
366        || located;
367    if !id_only && !corroborated {
368        confidence = confidence.min(UNCORROBORATED_CAP);
369    }
370    // A title of a word or two inside free text ("Aspero" in "aspero 0.6.5
371    // is in") is no evidence by itself, nor with a year: it takes the
372    // authors, the journal, the place or an identifier.
373    let short_title_in_text = claim.title.is_none()
374        && text.is_some()
375        && work_title
376            .as_deref()
377            .is_some_and(|t| content_words(t).len() < SHORT_TITLE_WORDS);
378    if short_title_in_text && !corroborated && container.agreement != Agreement::Match {
379        confidence = confidence.min(NO_EVIDENCE_CAP);
380    }
381    // A year, or a year and journal, identify nothing: some work matches
382    // them by chance. It takes the title, authors, an identifier or the
383    // place.
384    let identified =
385        matches!(title.agreement, Agreement::Match | Agreement::Partial) || corroborated;
386    if !id_only && !identified {
387        confidence = confidence.min(NO_EVIDENCE_CAP);
388    }
389    // A wrong volume or page is an error in the citation, and tells an
390    // original from the letters and corrections in the same journal.
391    if [volume, page]
392        .iter()
393        .any(|c| c.agreement == Agreement::Mismatch)
394    {
395        confidence = confidence.min(LOCATION_MISMATCH_CAP);
396    }
397    // Journal names compare by rule, which misses some pairs ("Br Med J" ↔
398    // "BMJ"), so a different journal only lowers the ranking.
399    if container.agreement == Agreement::Mismatch {
400        confidence *= JOURNAL_MISMATCH_PENALTY;
401    }
402    // A reply, correction or commentary is not the work it responds to,
403    // unless the citation says it is citing one.
404    if let Some(actual) = &work_title
405        && is_derivative(actual)
406        && !claim_mentions_derivative(claim)
407    {
408        confidence *= DERIVATIVE_PENALTY;
409    }
410
411    Assessment {
412        confidence,
413        title,
414        authors,
415        year,
416        container,
417        volume,
418        page,
419        ids,
420    }
421}
422
423fn combine(title: FieldCheck, authors: FieldCheck, year: FieldCheck) -> f64 {
424    let parts = [(title, W_TITLE), (authors, W_AUTHORS), (year, W_YEAR)];
425    let known: Vec<(f64, f64)> = parts
426        .iter()
427        .filter_map(|(check, w)| check.score.map(|s| (s, *w)))
428        .collect();
429    let weight: f64 = known.iter().map(|(_, w)| w).sum();
430    if weight == 0.0 {
431        return 0.0;
432    }
433    let mut confidence = known.iter().map(|(s, w)| s * w).sum::<f64>() / weight;
434    if year.agreement == Agreement::Mismatch {
435        confidence *= YEAR_MISMATCH_PENALTY;
436    }
437    if title.agreement == Agreement::Mismatch {
438        confidence *= TITLE_MISMATCH_PENALTY;
439    }
440    // Different authors usually mean a different work (a similarly titled
441    // book or survey), even when title and year agree.
442    if authors.agreement == Agreement::Mismatch {
443        confidence *= AUTHORS_MISMATCH_PENALTY;
444    }
445    if known.len() < 2 {
446        confidence = confidence.min(SINGLE_FIELD_CAP);
447    }
448    confidence
449}
450
451/// Whether claimed identifiers are among a work's identifiers (see
452/// [`Assessment::ids`]).
453pub fn id_agreement(claimed: &[WorkId], actual: &[WorkId]) -> Agreement {
454    if claimed.is_empty() {
455        return Agreement::Unknown;
456    }
457    if claimed
458        .iter()
459        .any(|c| actual.iter().any(|a| same_work_id(c, a)))
460    {
461        return Agreement::Match;
462    }
463    let shares_kind = claimed
464        .iter()
465        .any(|c| actual.iter().any(|a| a.kind() == c.kind()));
466    if shares_kind {
467        Agreement::Mismatch
468    } else {
469        Agreement::Unknown
470    }
471}
472
473/// Identifier equality at the level of works: arXiv versions are the same work.
474pub fn same_work_id(a: &WorkId, b: &WorkId) -> bool {
475    match (a, b) {
476        (WorkId::Arxiv(x), WorkId::Arxiv(y)) => x.unversioned() == y.unversioned(),
477        _ => a == b,
478    }
479}
480
481/// The distinct identifiers mentioned in free text (see [`crate::id::scan`]).
482pub fn find_ids(text: &str) -> Vec<WorkId> {
483    let mut found: Vec<WorkId> = Vec::new();
484    for mention in crate::id::scan(text) {
485        if !found.iter().any(|f| same_work_id(f, &mention.id)) {
486            found.push(mention.id);
487        }
488    }
489    found
490}
491
492#[cfg(test)]
493mod tests;
aspero_core/matching/mod.rs

aspero_core/matching/
mod.rs
