aspero_crossref/
lib.rs

1//! Crossref provider.
2//!
3//! Crossref is the registration agency for most journal and conference DOIs,
4//! so it is authoritative for their bibliographic fields. Its
5//! `query.bibliographic` matcher, built for reference strings, backs
6//! [`SearchMode::Citation`].
7//!
8//! It only knows Crossref DOIs: arXiv's DOIs are registered with DataCite,
9//! so preprints such as arXiv:1706.03762 are absent here.
10//!
11//! Free to use. Requests carry a `mailto`, which puts them in the "polite"
12//! pool (observed 2026-11: 10 req/s for single works, 3 req/s for lists, 3
13//! concurrent).
14
15mod http;
16mod wire;
17
18use std::sync::Arc;
19use std::time::Duration;
20
21use aspero_core::http::path_segment;
22use aspero_core::{
23    Capabilities, Cursor, Doi, Error, FieldSet, Filter, Hit, HttpClient, HttpRequest, HttpResponse,
24    IdKind, Lookup, Page, Provider, ProviderId, ResiliencePolicy, Response, Result, SearchMode,
25    SearchQuery, SourceRecord, WorkId,
26};
27use async_trait::async_trait;
28
29use crate::wire::{Envelope, WireList};
30
31pub const ID: ProviderId = ProviderId::from_static("crossref");
32
33pub const DEFAULT_BASE_URL: &str = "https://api.crossref.org";
34
35/// DOIs per batched `filter=doi:…,doi:…` call; keeps URLs a sane length.
36const MAX_BATCH: usize = 50;
37
38/// Crossref's cap on `rows`.
39const MAX_ROWS: u32 = 1000;
40
41#[derive(Debug, Clone)]
42#[non_exhaustive]
43pub struct CrossrefConfig {
44    /// Contact address for the polite pool. Without it requests share the
45    /// slower public pool.
46    pub mailto: Option<String>,
47    pub base_url: String,
48}
49
50impl Default for CrossrefConfig {
51    fn default() -> Self {
52        CrossrefConfig {
53            mailto: None,
54            base_url: DEFAULT_BASE_URL.to_owned(),
55        }
56    }
57}
58
59pub struct Crossref {
60    config: CrossrefConfig,
61    http: Arc<dyn HttpClient>,
62    capabilities: Capabilities,
63}
64
65impl Crossref {
66    pub fn new(config: CrossrefConfig, http: Arc<dyn HttpClient>) -> Self {
67        let mut capabilities = Capabilities::default();
68        capabilities.lookup_kinds = vec![IdKind::Doi];
69        capabilities.max_batch = MAX_BATCH;
70        capabilities.fields = FieldSet::ALL;
71        capabilities.search = true;
72        capabilities.citation_search = true;
73        capabilities.search_filters = vec![Filter::Years, Filter::WorkType];
74        Crossref {
75            config,
76            http,
77            capabilities,
78        }
79    }
80
81    /// Paced for the polite pool's list limit (3 req/s), the stricter of
82    /// its two.
83    pub fn default_policy() -> ResiliencePolicy {
84        let mut policy = ResiliencePolicy::default();
85        policy.min_interval = Duration::from_millis(350);
86        policy
87    }
88
89    /// `/works<rest>` with `params` and the `mailto` appended.
90    fn url(&self, rest: &str, params: &[(&str, String)]) -> Result<String> {
91        let base = format!("{}/works{rest}", self.config.base_url.trim_end_matches('/'));
92        let mut url = url::Url::parse(&base)
93            .map_err(|e| Error::BadRequest(format!("invalid Crossref base URL: {e}")))?;
94        {
95            let mut query = url.query_pairs_mut();
96            for (k, v) in params {
97                query.append_pair(k, v);
98            }
99            if let Some(mailto) = &self.config.mailto {
100                query.append_pair("mailto", mailto);
101            }
102        }
103        // An empty query would leave a dangling `?`.
104        if url.query() == Some("") {
105            url.set_query(None);
106        }
107        Ok(url.into())
108    }
109
110    async fn get(&self, url: String) -> Result<HttpResponse> {
111        let req = HttpRequest::get(url).header("accept", "application/json");
112        let resp = self.http.send(req).await?;
113        http::check(&resp)?;
114        Ok(resp)
115    }
116
117    async fn lookup_one(&self, doi: &Doi) -> Result<Option<SourceRecord>> {
118        let url = self.url(&format!("/{}", path_segment(doi.as_str())), &[])?;
119        let resp = match self.get(url).await {
120            Ok(resp) => resp,
121            Err(Error::NotFound) => return Ok(None),
122            Err(e) => return Err(e),
123        };
124        let envelope: Envelope<serde_json::Value> =
125            serde_json::from_slice(&resp.body).map_err(Error::decode)?;
126        wire::to_record(&envelope.message, resp.received_at)
127            .map(Some)
128            .map_err(Error::decode)
129    }
130
131    async fn list(&self, params: &[(&str, String)]) -> Result<(Vec<Hit>, WireList)> {
132        let resp = self.get(self.url("", params)?).await?;
133        let envelope: Envelope<WireList> =
134            serde_json::from_slice(&resp.body).map_err(Error::decode)?;
135        let list = envelope.message;
136        let hits = list
137            .items
138            .iter()
139            .map(|v| {
140                let score = v.get("score").and_then(serde_json::Value::as_f64);
141                wire::to_record(v, resp.received_at).map(|r| Hit::new(r, score))
142            })
143            .collect::<serde_json::Result<_>>()
144            .map_err(Error::decode)?;
145        Ok((hits, list))
146    }
147}
148
149#[async_trait]
150impl Provider for Crossref {
151    fn id(&self) -> ProviderId {
152        ID
153    }
154
155    fn capabilities(&self) -> &Capabilities {
156        &self.capabilities
157    }
158
159    async fn lookup(&self, ids: &[WorkId], _fields: FieldSet) -> Result<Response<Vec<Lookup>>> {
160        let dois = ids
161            .iter()
162            .map(|id| match id {
163                WorkId::Doi(doi) => Ok(doi),
164                _ => Err(Error::Unsupported("Crossref lookup of non-DOI identifiers")),
165            })
166            .collect::<Result<Vec<_>>>()?;
167
168        if let [doi] = dois.as_slice() {
169            let record = self.lookup_one(doi).await?;
170            return Ok(Response::new(vec![Lookup {
171                id: ids[0].clone(),
172                record,
173            }]));
174        }
175
176        // `,` separates filters, so such DOIs can't be batched.
177        let (batchable, singles): (Vec<&Doi>, Vec<&Doi>) =
178            dois.iter().partition(|d| !d.as_str().contains(','));
179        let mut found = Vec::new();
180        for chunk in batchable.chunks(MAX_BATCH) {
181            let filter = chunk
182                .iter()
183                .map(|d| format!("doi:{}", d.as_str()))
184                .collect::<Vec<_>>()
185                .join(",");
186            let params = [("filter", filter), ("rows", chunk.len().to_string())];
187            let (hits, _) = self.list(&params).await?;
188            found.extend(hits.into_iter().map(|h| h.record));
189        }
190        for doi in singles {
191            found.extend(self.lookup_one(doi).await?);
192        }
193
194        let lookups = ids
195            .iter()
196            .map(|id| Lookup {
197                id: id.clone(),
198                record: found.iter().find(|r| r.ids.contains(id)).cloned(),
199            })
200            .collect();
201        Ok(Response::new(lookups))
202    }
203
204    async fn search(&self, query: &SearchQuery) -> Result<Response<Page>> {
205        let mut filters = Vec::new();
206        if let Some(years) = query.years {
207            if let Some(from) = years.from {
208                filters.push(format!("from-pub-date:{from:04}"));
209            }
210            if let Some(to) = years.to {
211                filters.push(format!("until-pub-date:{to:04}"));
212            }
213        }
214        if let Some(t) = &query.work_type {
215            let crossref = wire::crossref_type(t).ok_or_else(|| {
216                Error::BadRequest(format!(
217                    "Crossref has no work type matching {:?}",
218                    t.as_csl()
219                ))
220            })?;
221            filters.push(format!("type:{crossref}"));
222        }
223
224        let mut params: Vec<(&str, String)> = Vec::new();
225        for (key, value) in &query.extra {
226            match key.as_str() {
227                "filter" => filters.push(value.clone()),
228                "sort" | "order" => params.push((key.as_str(), value.clone())),
229                k if k.starts_with("query.") => params.push((k, value.clone())),
230                other => {
231                    return Err(Error::BadRequest(format!(
232                        "unknown Crossref parameter {other:?} (supported: filter, sort, order, query.*)"
233                    )));
234                }
235            }
236        }
237        let text = query.text.trim();
238        if !text.is_empty() {
239            let field = match query.mode {
240                SearchMode::Citation => "query.bibliographic",
241                _ => "query",
242            };
243            params.push((field, text.to_owned()));
244        }
245        if !filters.is_empty() {
246            params.push(("filter", filters.join(",")));
247        }
248        // Cursor paging silently drops relevance order (observed 2026-11-10:
249        // the right match at 92.1 vanished from the top 3), so ask for it
250        // explicitly whenever there is text to rank by.
251        let has_query = params.iter().any(|(k, _)| k.starts_with("query"));
252        if has_query && !params.iter().any(|(k, _)| *k == "sort") {
253            params.push(("sort", "score".into()));
254            params.push(("order", "desc".into()));
255        }
256        params.push(("rows", query.page.limit.clamp(1, MAX_ROWS).to_string()));
257        let cursor = query.page.cursor.as_ref().map_or("*", |c| c.0.as_str());
258        params.push(("cursor", cursor.to_owned()));
259
260        let (hits, list) = self.list(&params).await?;
261        // Crossref keeps returning a cursor after the last page; an empty
262        // page is the real end.
263        let next = list.next_cursor.filter(|_| !hits.is_empty()).map(Cursor);
264        Ok(Response::new(Page::new(hits, list.total_results, next)))
265    }
266}
aspero_crossref/lib.rs

aspero_crossref/
lib.rs
