aspero_datacite/
lib.rs

1//! DataCite provider.
2//!
3//! DataCite registers DOIs for arXiv, Zenodo, Figshare, Dryad and most
4//! research data and software. Its REST API (JSON:API) gives lookups,
5//! batched lookups, and keyword search with year and type filters; records
6//! carry abstracts (as "Abstract" descriptions) and stated arXiv IDs. Free;
7//! about 3,000 requests per 5 minutes.
8//!
9//! Observed 2026-11-10: **relevance ranking only works with page-number
10//! paging** (`sort=relevance&page[number]=N`); with cursor paging results
11//! come back by creation date regardless. So the provider's [`Cursor`] is
12//! the next page number. Year ranges use the query syntax
13//! (`publicationYear:[2017 TO *]`); `published=` only takes year lists.
14
15mod wire;
16
17use std::sync::Arc;
18use std::time::Duration;
19
20use aspero_core::http::{escape_lucene, path_segment};
21use aspero_core::{
22    Capabilities, Cursor, Doi, Error, Field, FieldSet, Filter, Hit, HttpClient, HttpRequest,
23    HttpResponse, IdKind, Lookup, Page, Provider, ProviderId, ResiliencePolicy, Response, Result,
24    SearchMode, SearchQuery, SourceRecord, WorkId,
25};
26use async_trait::async_trait;
27
28use crate::wire::WireList;
29
30pub const ID: ProviderId = ProviderId::from_static("datacite");
31
32pub const DEFAULT_BASE_URL: &str = "https://api.datacite.org";
33
34pub(crate) const ARXIV_DOI_PREFIX: &str = "10.48550";
35
36/// DOIs per batched `query=doi:(…)` lookup.
37const MAX_BATCH: usize = 50;
38
39/// DataCite's page-size cap.
40const MAX_PAGE: u32 = 1000;
41
42/// Extra parameters passed through to DataCite's search filters.
43const PASSTHROUGH: &[&str] = &["client-id", "prefix", "affiliation-id", "has-citations"];
44
45#[derive(Debug, Clone)]
46#[non_exhaustive]
47pub struct DataciteConfig {
48    pub base_url: String,
49}
50
51impl Default for DataciteConfig {
52    fn default() -> Self {
53        DataciteConfig {
54            base_url: DEFAULT_BASE_URL.to_owned(),
55        }
56    }
57}
58
59pub struct Datacite {
60    config: DataciteConfig,
61    http: Arc<dyn HttpClient>,
62    capabilities: Capabilities,
63}
64
65impl Datacite {
66    pub fn new(config: DataciteConfig, http: Arc<dyn HttpClient>) -> Self {
67        let mut capabilities = Capabilities::default();
68        capabilities.lookup_kinds = vec![IdKind::Doi, IdKind::Arxiv];
69        capabilities.max_batch = MAX_BATCH;
70        // Records say nothing about free copies.
71        capabilities.fields = FieldSet::ALL.difference(FieldSet::of(&[Field::OpenAccess]));
72        capabilities.search = true;
73        capabilities.search_filters = vec![Filter::Years, Filter::WorkType];
74        Datacite {
75            config,
76            http,
77            capabilities,
78        }
79    }
80
81    /// About 10 req/s allowed; stay well under.
82    pub fn default_policy() -> ResiliencePolicy {
83        let mut policy = ResiliencePolicy::default();
84        policy.min_interval = Duration::from_millis(200);
85        policy
86    }
87
88    fn url(&self, rest: &str, params: &[(&str, String)]) -> Result<String> {
89        let base = format!("{}/dois{rest}", self.config.base_url.trim_end_matches('/'));
90        let url = if params.is_empty() {
91            url::Url::parse(&base)
92        } else {
93            url::Url::parse_with_params(&base, params)
94        };
95        url.map(String::from)
96            .map_err(|e| Error::BadRequest(format!("invalid DataCite base URL: {e}")))
97    }
98
99    async fn get(&self, url: String) -> Result<HttpResponse> {
100        let req = HttpRequest::get(url).header("accept", "application/vnd.api+json");
101        let resp = self.http.send(req).await?;
102        check(&resp)?;
103        Ok(resp)
104    }
105
106    async fn lookup_one(&self, doi: &Doi) -> Result<Option<SourceRecord>> {
107        let resp = match self
108            .get(self.url(&format!("/{}", path_segment(doi.as_str())), &[])?)
109            .await
110        {
111            Ok(resp) => resp,
112            Err(Error::NotFound) => return Ok(None),
113            Err(e) => return Err(e),
114        };
115        let body: serde_json::Value = serde_json::from_slice(&resp.body).map_err(Error::decode)?;
116        let data = body.get("data").ok_or_else(|| Error::decode(MissingData))?;
117        wire::to_record(data, resp.received_at)
118            .map(Some)
119            .map_err(Error::decode)
120    }
121
122    async fn list(&self, params: &[(&str, String)]) -> Result<(Vec<Hit>, wire::WireMeta)> {
123        let resp = self.get(self.url("", params)?).await?;
124        let list: WireList = serde_json::from_slice(&resp.body).map_err(Error::decode)?;
125        let hits = list
126            .data
127            .iter()
128            .map(|v| wire::to_record(v, resp.received_at).map(|r| Hit::new(r, None)))
129            .collect::<serde_json::Result<_>>()
130            .map_err(Error::decode)?;
131        Ok((hits, list.meta))
132    }
133}
134
135#[derive(Debug, thiserror::Error)]
136#[error("DataCite response has no `data`")]
137struct MissingData;
138
139fn to_doi(id: &WorkId) -> Result<Doi> {
140    match id {
141        WorkId::Doi(doi) => Ok(doi.clone()),
142        WorkId::Arxiv(arxiv) => Ok(format!("{ARXIV_DOI_PREFIX}/arxiv.{}", arxiv.id()).parse()?),
143        _ => Err(Error::Unsupported("DataCite lookup of non-DOI identifiers")),
144    }
145}
146
147fn check(resp: &HttpResponse) -> Result<()> {
148    if resp.is_success() {
149        return Ok(());
150    }
151    let message = message(resp);
152    Err(match resp.status {
153        400 | 422 => Error::BadRequest(message.unwrap_or_else(|| "rejected by DataCite".into())),
154        401 | 403 => Error::AuthRejected { message },
155        404 => Error::NotFound,
156        429 => Error::RateLimited {
157            retry_after: resp
158                .header("retry-after")
159                .and_then(|v| v.trim().parse().ok())
160                .map(Duration::from_secs),
161        },
162        status => Error::Server { status, message },
163    })
164}
165
166/// The `title` of a JSON:API `errors` value, which DataCite sends as an
167/// object or an array.
168fn message(resp: &HttpResponse) -> Option<String> {
169    let json: serde_json::Value = serde_json::from_slice(&resp.body).ok()?;
170    let errors = json.get("errors")?;
171    let first = errors.as_array().and_then(|a| a.first()).unwrap_or(errors);
172    first.get("title")?.as_str().map(str::to_owned)
173}
174
175#[async_trait]
176impl Provider for Datacite {
177    fn id(&self) -> ProviderId {
178        ID
179    }
180
181    fn capabilities(&self) -> &Capabilities {
182        &self.capabilities
183    }
184
185    async fn lookup(&self, ids: &[WorkId], _fields: FieldSet) -> Result<Response<Vec<Lookup>>> {
186        let dois = ids.iter().map(to_doi).collect::<Result<Vec<_>>>()?;
187        if let [doi] = dois.as_slice() {
188            let record = self.lookup_one(doi).await?;
189            return Ok(Response::new(vec![Lookup {
190                id: ids[0].clone(),
191                record,
192            }]));
193        }
194        let mut found = Vec::new();
195        for chunk in dois.chunks(MAX_BATCH) {
196            let terms: Vec<String> = chunk
197                .iter()
198                .map(|d| format!("\"{}\"", d.as_str().replace('"', "\\\"")))
199                .collect();
200            let params = [
201                ("query", format!("doi:({})", terms.join(" OR "))),
202                ("page[size]", chunk.len().to_string()),
203            ];
204            let (hits, _) = self.list(&params).await?;
205            found.extend(hits.into_iter().map(|h| h.record));
206        }
207        let lookups = ids
208            .iter()
209            .zip(&dois)
210            .map(|(id, doi)| Lookup {
211                id: id.clone(),
212                record: found
213                    .iter()
214                    .find(|r| {
215                        r.ids
216                            .iter()
217                            .any(|i| matches!(i, WorkId::Doi(d) if d == doi))
218                    })
219                    .cloned(),
220            })
221            .collect();
222        Ok(Response::new(lookups))
223    }
224
225    async fn search(&self, query: &SearchQuery) -> Result<Response<Page>> {
226        if query.mode != SearchMode::Keywords {
227            return Err(Error::Unsupported("DataCite has no citation matcher"));
228        }
229        let mut clauses = Vec::new();
230        if !query.text.trim().is_empty() {
231            clauses.push(format!("({})", escape_lucene(query.text.trim())));
232        }
233        if let Some(years) = query.years {
234            let end = |y: Option<i16>| y.map_or("*".to_owned(), |y| y.to_string());
235            clauses.push(format!(
236                "publicationYear:[{} TO {}]",
237                end(years.from),
238                end(years.to)
239            ));
240        }
241        let mut params: Vec<(&str, String)> = Vec::new();
242        for (key, value) in &query.extra {
243            match key.as_str() {
244                // Raw query syntax, ANDed with the rest.
245                "query" => clauses.push(format!("({value})")),
246                k if PASSTHROUGH.contains(&k) => params.push((k, value.clone())),
247                other => {
248                    return Err(Error::BadRequest(format!(
249                        "unknown DataCite parameter {other:?} (supported: query, {})",
250                        PASSTHROUGH.join(", ")
251                    )));
252                }
253            }
254        }
255        if !clauses.is_empty() {
256            params.insert(0, ("query", clauses.join(" AND ")));
257        }
258        if let Some(t) = &query.work_type {
259            let id = wire::resource_type_id(t).ok_or_else(|| {
260                Error::BadRequest(format!(
261                    "DataCite has no resource type matching {:?}",
262                    t.as_csl()
263                ))
264            })?;
265            params.push(("resource-type-id", id.to_owned()));
266        }
267        let page_number: u32 = match &query.page.cursor {
268            None => 1,
269            Some(c) => {
270                c.0.parse()
271                    .map_err(|_| Error::BadRequest(format!("invalid DataCite cursor {:?}", c.0)))?
272            }
273        };
274        params.push(("sort", "relevance".into()));
275        params.push(("page[number]", page_number.to_string()));
276        params.push((
277            "page[size]",
278            query.page.limit.clamp(1, MAX_PAGE).to_string(),
279        ));
280
281        let (hits, meta) = self.list(&params).await?;
282        let more = meta.total_pages.is_some_and(|pages| page_number < pages) && !hits.is_empty();
283        let next = more.then(|| Cursor((page_number + 1).to_string()));
284        Ok(Response::new(Page::new(hits, meta.total, next)))
285    }
286}
aspero_datacite/lib.rs

aspero_datacite/
lib.rs
