aspero_europepmc/
lib.rs

1//! Europe PMC provider: biomedical and life-science literature.
2//!
3//! Covers PubMed, PubMed Central and preprints; free and keyless. Adds
4//! what the other providers lack: PMCID lookup, broad abstract coverage
5//! for biomedical work, and "Retracted Publication" flags.
6//!
7//! - Lookups batch PMIDs, PMCIDs and DOIs into one OR query
8//!   (`(EXT_ID:… AND SRC:MED) OR PMCID:… OR DOI:"…"`); results come back
9//!   unordered and are matched by identifier.
10//! - Search is relevance-ordered and pages by `cursorMark` *without*
11//!   losing that order (checked 2026-11-10, unlike Crossref and DataCite).
12
13use std::sync::Arc;
14use std::time::Duration;
15
16use aspero_core::csl::strip_markup;
17use aspero_core::http::escape_lucene;
18use aspero_core::{
19    Capabilities, Cursor, Doi, Error, FieldSet, Filter, Hit, HttpClient, HttpRequest, HttpResponse,
20    IdKind, Lookup, Page, PartialDate, Person, Pmcid, Pmid, Provider, ProviderId, Raw,
21    ResiliencePolicy, Response, Result, SearchMode, SearchQuery, SourceRecord, Update, UpdateKind,
22    Work, WorkId, WorkType, tidy_abstract,
23};
24use async_trait::async_trait;
25use jiff::Timestamp;
26use serde::Deserialize;
27
28pub const ID: ProviderId = ProviderId::from_static("europepmc");
29
30pub const DEFAULT_BASE_URL: &str = "https://www.ebi.ac.uk/europepmc/webservices/rest";
31
32/// Identifiers per batched lookup.
33const MAX_BATCH: usize = 50;
34
35/// Europe PMC's page-size cap.
36const MAX_PAGE: u32 = 1000;
37
38#[derive(Debug, Clone)]
39#[non_exhaustive]
40pub struct EuropePmcConfig {
41    pub base_url: String,
42}
43
44impl Default for EuropePmcConfig {
45    fn default() -> Self {
46        EuropePmcConfig {
47            base_url: DEFAULT_BASE_URL.to_owned(),
48        }
49    }
50}
51
52pub struct EuropePmc {
53    config: EuropePmcConfig,
54    http: Arc<dyn HttpClient>,
55    capabilities: Capabilities,
56}
57
58impl EuropePmc {
59    pub fn new(config: EuropePmcConfig, http: Arc<dyn HttpClient>) -> Self {
60        let mut capabilities = Capabilities::default();
61        capabilities.lookup_kinds = vec![IdKind::Pmid, IdKind::Pmcid, IdKind::Doi];
62        capabilities.max_batch = MAX_BATCH;
63        capabilities.fields = FieldSet::ALL;
64        capabilities.search = true;
65        capabilities.search_filters = vec![Filter::Years, Filter::OpenAccess];
66        EuropePmc {
67            config,
68            http,
69            capabilities,
70        }
71    }
72
73    pub fn default_policy() -> ResiliencePolicy {
74        let mut policy = ResiliencePolicy::default();
75        policy.min_interval = Duration::from_millis(200);
76        policy
77    }
78
79    async fn search_raw(&self, params: &[(&str, String)]) -> Result<(WireResults, Timestamp)> {
80        let base = format!("{}/search", self.config.base_url.trim_end_matches('/'));
81        let url = url::Url::parse_with_params(&base, params)
82            .map_err(|e| Error::BadRequest(format!("invalid Europe PMC base URL: {e}")))?;
83        let resp = self
84            .http
85            .send(HttpRequest::get(String::from(url)).header("accept", "application/json"))
86            .await?;
87        check(&resp)?;
88        let results = serde_json::from_slice(&resp.body).map_err(Error::decode)?;
89        Ok((results, resp.received_at))
90    }
91}
92
93fn check(resp: &HttpResponse) -> Result<()> {
94    if resp.is_success() {
95        return Ok(());
96    }
97    Err(match resp.status {
98        400 => Error::BadRequest("rejected by Europe PMC".into()),
99        404 => Error::NotFound,
100        429 => Error::RateLimited {
101            retry_after: resp
102                .header("retry-after")
103                .and_then(|v| v.trim().parse().ok())
104                .map(Duration::from_secs),
105        },
106        status => Error::Server {
107            status,
108            message: None,
109        },
110    })
111}
112
113#[derive(Debug, Default, Deserialize)]
114#[serde(default, rename_all = "camelCase")]
115struct WireResults {
116    hit_count: Option<u64>,
117    next_cursor_mark: Option<String>,
118    result_list: ResultList,
119}
120
121#[derive(Debug, Default, Deserialize)]
122#[serde(default)]
123struct ResultList {
124    result: Vec<serde_json::Value>,
125}
126
127#[derive(Debug, Default, Deserialize)]
128#[serde(default, rename_all = "camelCase")]
129struct WireRecord {
130    id: Option<String>,
131    source: Option<String>,
132    pmid: Option<String>,
133    pmcid: Option<String>,
134    doi: Option<String>,
135    title: Option<String>,
136    author_list: AuthorList,
137    pub_year: Option<String>,
138    first_publication_date: Option<String>,
139    journal_info: JournalInfo,
140    page_info: Option<String>,
141    abstract_text: Option<String>,
142    pub_type_list: PubTypes,
143    cited_by_count: Option<u64>,
144}
145
146#[derive(Debug, Default, Deserialize)]
147#[serde(default)]
148struct AuthorList {
149    author: Vec<Author>,
150}
151
152#[derive(Debug, Default, Deserialize)]
153#[serde(default, rename_all = "camelCase")]
154struct Author {
155    full_name: Option<String>,
156    first_name: Option<String>,
157    last_name: Option<String>,
158    collective_name: Option<String>,
159    author_id: Option<AuthorId>,
160}
161
162#[derive(Debug, Default, Deserialize)]
163#[serde(default, rename_all = "camelCase")]
164struct AuthorId {
165    #[serde(rename = "type")]
166    id_type: Option<String>,
167    value: Option<String>,
168}
169
170#[derive(Debug, Default, Deserialize)]
171#[serde(default)]
172#[serde(rename_all = "camelCase")]
173struct JournalInfo {
174    year_of_publication: Option<i16>,
175    volume: Option<String>,
176    issue: Option<String>,
177    journal: Journal,
178}
179
180#[derive(Debug, Default, Deserialize)]
181#[serde(default)]
182struct Journal {
183    title: Option<String>,
184}
185
186#[derive(Debug, Default, Deserialize)]
187#[serde(default, rename_all = "camelCase")]
188struct PubTypes {
189    pub_type: Vec<String>,
190}
191
192impl Author {
193    fn person(self) -> Option<Person> {
194        let mut person = Person::default();
195        match (self.last_name, self.collective_name, self.full_name) {
196            (Some(last), _, _) => {
197                person.family = Some(last);
198                person.given = self.first_name;
199            }
200            (None, Some(name), _) | (None, None, Some(name)) => person.literal = Some(name),
201            (None, None, None) => return None,
202        }
203        person.orcid = self
204            .author_id
205            .filter(|id| id.id_type.as_deref() == Some("ORCID"))
206            .and_then(|id| id.value);
207        Some(person)
208    }
209}
210
211fn to_record(value: &serde_json::Value, at: Timestamp) -> serde_json::Result<SourceRecord> {
212    let r = WireRecord::deserialize(value)?;
213    let mut ids: Vec<WorkId> = Vec::new();
214    ids.extend(
215        r.pmid
216            .as_deref()
217            .and_then(|p| p.parse::<Pmid>().ok())
218            .map(WorkId::Pmid),
219    );
220    ids.extend(
221        r.pmcid
222            .as_deref()
223            .and_then(|p| p.parse::<Pmcid>().ok())
224            .map(WorkId::Pmcid),
225    );
226    ids.extend(
227        r.doi
228            .as_deref()
229            .and_then(|d| d.parse::<Doi>().ok())
230            .map(WorkId::Doi),
231    );
232
233    let types = &r.pub_type_list.pub_type;
234    let has = |t: &str| types.iter().any(|p| p.eq_ignore_ascii_case(t));
235    let mut work = Work::default();
236    work.title = r.title.map(|t| t.trim_end_matches('.').to_owned());
237    work.authors = r
238        .author_list
239        .author
240        .into_iter()
241        .filter_map(Author::person)
242        .collect();
243    work.issued = r
244        .first_publication_date
245        .as_deref()
246        .and_then(|d| d.parse::<PartialDate>().ok())
247        .or_else(|| {
248            r.pub_year
249                .as_deref()?
250                .parse()
251                .ok()
252                .map(PartialDate::from_year)
253        });
254    work.work_type = if r.source.as_deref() == Some("PPR") || has("Preprint") {
255        Some(WorkType::Preprint)
256    } else if has("Journal Article") || has("research-article") || has("Review") {
257        Some(WorkType::JournalArticle)
258    } else {
259        None
260    };
261    // First publication (often online) vs the journal issue's year.
262    let issued_year = work.issued.map(|d| d.year());
263    let mut other: Vec<i16> = r
264        .journal_info
265        .year_of_publication
266        .into_iter()
267        .chain(r.pub_year.as_deref().and_then(|y| y.parse().ok()))
268        .filter(|y| Some(*y) != issued_year)
269        .collect();
270    other.sort_unstable();
271    other.dedup();
272    work.other_years = other;
273    work.container_title = r.journal_info.journal.title;
274    work.volume = r.journal_info.volume;
275    work.issue = r.journal_info.issue;
276    work.page = r.page_info;
277    work.abstract_text = r
278        .abstract_text
279        .as_deref()
280        .map(|a| tidy_abstract(&strip_markup(a)))
281        .filter(|a| !a.is_empty());
282    work.url = match (r.source.as_deref(), r.id.as_deref()) {
283        (Some(source), Some(id)) => Some(format!("https://europepmc.org/article/{source}/{id}")),
284        _ => None,
285    };
286    work.cited_by_count = r.cited_by_count;
287    if has("Retracted Publication") {
288        let mut retraction = Update::new(UpdateKind::Retraction);
289        retraction.source = Some("europepmc".into());
290        work.updates.push(retraction);
291    }
292    Ok(SourceRecord::new(
293        ID,
294        ids,
295        work,
296        FieldSet::ALL,
297        Raw::json(value),
298        at,
299    ))
300}
301
302/// The query clause that finds `id`.
303fn clause(id: &WorkId) -> Result<String> {
304    Ok(match id {
305        WorkId::Pmid(p) => format!("(EXT_ID:{p} AND SRC:MED)"),
306        WorkId::Pmcid(p) => format!("PMCID:{p}"),
307        WorkId::Doi(d) => format!("DOI:\"{}\"", d.as_str().replace('"', "\\\"")),
308        _ => {
309            return Err(Error::Unsupported(
310                "Europe PMC lookup of this identifier kind",
311            ));
312        }
313    })
314}
315
316#[async_trait]
317impl Provider for EuropePmc {
318    fn id(&self) -> ProviderId {
319        ID
320    }
321
322    fn capabilities(&self) -> &Capabilities {
323        &self.capabilities
324    }
325
326    async fn lookup(&self, ids: &[WorkId], _fields: FieldSet) -> Result<Response<Vec<Lookup>>> {
327        let clauses = ids.iter().map(clause).collect::<Result<Vec<_>>>()?;
328        let mut found = Vec::new();
329        for chunk in clauses.chunks(MAX_BATCH) {
330            let params = [
331                ("query", chunk.join(" OR ")),
332                ("resultType", "core".to_owned()),
333                ("format", "json".to_owned()),
334                ("pageSize", chunk.len().to_string()),
335            ];
336            let (results, at) = self.search_raw(&params).await?;
337            for v in &results.result_list.result {
338                found.push(to_record(v, at).map_err(Error::decode)?);
339            }
340        }
341        let lookups = ids
342            .iter()
343            .map(|id| Lookup {
344                id: id.clone(),
345                record: found.iter().find(|r| r.ids.contains(id)).cloned(),
346            })
347            .collect();
348        Ok(Response::new(lookups))
349    }
350
351    async fn search(&self, query: &SearchQuery) -> Result<Response<Page>> {
352        if query.mode != SearchMode::Keywords {
353            return Err(Error::Unsupported("Europe PMC has no citation matcher"));
354        }
355        let mut clauses = Vec::new();
356        if !query.text.trim().is_empty() {
357            clauses.push(format!("({})", escape_lucene(query.text.trim())));
358        }
359        for (key, value) in &query.extra {
360            match key.as_str() {
361                // Raw Europe PMC query syntax, ANDed with the rest.
362                "query" => clauses.push(format!("({value})")),
363                other => {
364                    return Err(Error::BadRequest(format!(
365                        "unknown Europe PMC parameter {other:?} (supported: query)"
366                    )));
367                }
368            }
369        }
370        if let Some(years) = query.years {
371            let end = |y: Option<i16>| y.map_or("*".to_owned(), |y| y.to_string());
372            clauses.push(format!(
373                "PUB_YEAR:[{} TO {}]",
374                end(years.from),
375                end(years.to)
376            ));
377        }
378        if let Some(oa) = query.open_access {
379            clauses.push(format!("OPEN_ACCESS:{}", if oa { "y" } else { "n" }));
380        }
381        let cursor = query.page.cursor.as_ref().map_or("*", |c| c.0.as_str());
382        let params = [
383            ("query", clauses.join(" AND ")),
384            ("resultType", "core".to_owned()),
385            ("format", "json".to_owned()),
386            ("pageSize", query.page.limit.clamp(1, MAX_PAGE).to_string()),
387            ("cursorMark", cursor.to_owned()),
388        ];
389        let (results, at) = self.search_raw(&params).await?;
390        let hits: Vec<Hit> = results
391            .result_list
392            .result
393            .iter()
394            .map(|v| to_record(v, at).map(|r| Hit::new(r, None)))
395            .collect::<serde_json::Result<_>>()
396            .map_err(Error::decode)?;
397        // The last page repeats its cursor; an empty page is also the end.
398        let next = results
399            .next_cursor_mark
400            .filter(|next| !hits.is_empty() && next != cursor)
401            .map(Cursor);
402        Ok(Response::new(Page::new(hits, results.hit_count, next)))
403    }
404}
aspero_europepmc/lib.rs

aspero_europepmc/
lib.rs
