aspero_crossref/
lib.rs

1//! Crossref provider.
2//!
3//! Crossref is the registration agency for most journal and conference DOIs,
4//! so it is authoritative for their bibliographic fields. Its
5//! `query.bibliographic` matcher, built for reference strings, backs
6//! [`SearchMode::Citation`].
7//!
8//! It only knows Crossref DOIs: arXiv's DOIs are registered with DataCite,
9//! so preprints such as arXiv:1706.03762 are absent here.
10//!
11//! Free to use. Requests carry a `mailto`, which puts them in the "polite"
12//! pool (observed 2026-11: 10 req/s for single works, 3 req/s for lists, 3
13//! concurrent).
14
15mod http;
16mod wire;
17
18use std::sync::Arc;
19use std::time::Duration;
20
21use aspero_core::http::path_segment;
22use aspero_core::{
23    Capabilities, Cursor, Doi, Error, FieldSet, Filter, Hit, HttpClient, HttpRequest, HttpResponse,
24    IdKind, Lookup, Page, Provider, ProviderId, ResiliencePolicy, Response, Result, SearchMode,
25    SearchQuery, SourceRecord, WorkId,
26};
27use async_trait::async_trait;
28
29use crate::wire::{Envelope, WireList};
30
31pub const ID: ProviderId = ProviderId::from_static("crossref");
32
33pub const DEFAULT_BASE_URL: &str = "https://api.crossref.org";
34
35/// DOIs per batched `filter=doi:…,doi:…` call; keeps URLs a sane length.
36const MAX_BATCH: usize = 50;
37
38/// Crossref's cap on `rows`.
39const MAX_ROWS: u32 = 1000;
40
41#[derive(Debug, Clone)]
42#[non_exhaustive]
43pub struct CrossrefConfig {
44    /// Contact address for the polite pool. Without it requests share the
45    /// slower public pool.
46    pub mailto: Option<String>,
47    pub base_url: String,
48}
49
50impl Default for CrossrefConfig {
51    fn default() -> Self {
52        CrossrefConfig {
53            mailto: None,
54            base_url: DEFAULT_BASE_URL.to_owned(),
55        }
56    }
57}
58
59pub struct Crossref {
60    config: CrossrefConfig,
61    http: Arc<dyn HttpClient>,
62    capabilities: Capabilities,
63}
64
65impl Crossref {
66    pub fn new(config: CrossrefConfig, http: Arc<dyn HttpClient>) -> Self {
67        let mut capabilities = Capabilities::default();
68        capabilities.lookup_kinds = vec![IdKind::Doi];
69        capabilities.max_batch = MAX_BATCH;
70        capabilities.fields = FieldSet::ALL;
71        capabilities.search = true;
72        capabilities.citation_search = true;
73        capabilities.notices = true;
74        capabilities.search_filters = vec![Filter::Years, Filter::WorkType];
75        Crossref {
76            config,
77            http,
78            capabilities,
79        }
80    }
81
82    /// Paced for the polite pool's list limit (3 req/s), the stricter of
83    /// its two.
84    pub fn default_policy() -> ResiliencePolicy {
85        let mut policy = ResiliencePolicy::default();
86        policy.min_interval = Duration::from_millis(350);
87        policy
88    }
89
90    /// `/works<rest>` with `params` and the `mailto` appended.
91    fn url(&self, rest: &str, params: &[(&str, String)]) -> Result<String> {
92        let base = format!("{}/works{rest}", self.config.base_url.trim_end_matches('/'));
93        let mut url = url::Url::parse(&base)
94            .map_err(|e| Error::BadRequest(format!("invalid Crossref base URL: {e}")))?;
95        {
96            let mut query = url.query_pairs_mut();
97            for (k, v) in params {
98                query.append_pair(k, v);
99            }
100            if let Some(mailto) = &self.config.mailto {
101                query.append_pair("mailto", mailto);
102            }
103        }
104        // An empty query would leave a dangling `?`.
105        if url.query() == Some("") {
106            url.set_query(None);
107        }
108        Ok(url.into())
109    }
110
111    async fn get(&self, url: String) -> Result<HttpResponse> {
112        let req = HttpRequest::get(url).header("accept", "application/json");
113        let resp = self.http.send(req).await?;
114        http::check(&resp)?;
115        Ok(resp)
116    }
117
118    async fn lookup_one(&self, doi: &Doi) -> Result<Option<SourceRecord>> {
119        let url = self.url(&format!("/{}", path_segment(doi.as_str())), &[])?;
120        let resp = match self.get(url).await {
121            Ok(resp) => resp,
122            Err(Error::NotFound) => return Ok(None),
123            Err(e) => return Err(e),
124        };
125        let envelope: Envelope<serde_json::Value> =
126            serde_json::from_slice(&resp.body).map_err(Error::decode)?;
127        wire::to_record(&envelope.message, resp.received_at)
128            .map(Some)
129            .map_err(Error::decode)
130    }
131
132    async fn list(&self, params: &[(&str, String)]) -> Result<(Vec<Hit>, WireList)> {
133        let resp = self.get(self.url("", params)?).await?;
134        let envelope: Envelope<WireList> =
135            serde_json::from_slice(&resp.body).map_err(Error::decode)?;
136        let list = envelope.message;
137        let hits = list
138            .items
139            .iter()
140            .map(|v| {
141                let score = v.get("score").and_then(serde_json::Value::as_f64);
142                wire::to_record(v, resp.received_at).map(|r| Hit::new(r, score))
143            })
144            .collect::<serde_json::Result<_>>()
145            .map_err(Error::decode)?;
146        Ok((hits, list))
147    }
148}
149
150#[async_trait]
151impl Provider for Crossref {
152    fn id(&self) -> ProviderId {
153        ID
154    }
155
156    fn capabilities(&self) -> &Capabilities {
157        &self.capabilities
158    }
159
160    async fn lookup(&self, ids: &[WorkId], _fields: FieldSet) -> Result<Response<Vec<Lookup>>> {
161        let dois = ids
162            .iter()
163            .map(|id| match id {
164                WorkId::Doi(doi) => Ok(doi),
165                _ => Err(Error::Unsupported("Crossref lookup of non-DOI identifiers")),
166            })
167            .collect::<Result<Vec<_>>>()?;
168
169        if let [doi] = dois.as_slice() {
170            let record = self.lookup_one(doi).await?;
171            return Ok(Response::new(vec![Lookup {
172                id: ids[0].clone(),
173                record,
174            }]));
175        }
176
177        // `,` separates filters, so such DOIs can't be batched.
178        let (batchable, singles): (Vec<&Doi>, Vec<&Doi>) =
179            dois.iter().partition(|d| !d.as_str().contains(','));
180        let mut found = Vec::new();
181        for chunk in batchable.chunks(MAX_BATCH) {
182            let filter = chunk
183                .iter()
184                .map(|d| format!("doi:{}", d.as_str()))
185                .collect::<Vec<_>>()
186                .join(",");
187            let params = [("filter", filter), ("rows", chunk.len().to_string())];
188            let (hits, _) = self.list(&params).await?;
189            found.extend(hits.into_iter().map(|h| h.record));
190        }
191        for doi in singles {
192            found.extend(self.lookup_one(doi).await?);
193        }
194
195        let lookups = ids
196            .iter()
197            .map(|id| Lookup {
198                id: id.clone(),
199                record: found.iter().find(|r| r.ids.contains(id)).cloned(),
200            })
201            .collect();
202        Ok(Response::new(lookups))
203    }
204
205    async fn search(&self, query: &SearchQuery) -> Result<Response<Page>> {
206        let mut filters = Vec::new();
207        if let Some(years) = query.years {
208            if let Some(from) = years.from {
209                filters.push(format!("from-pub-date:{from:04}"));
210            }
211            if let Some(to) = years.to {
212                filters.push(format!("until-pub-date:{to:04}"));
213            }
214        }
215        if let Some(t) = &query.work_type {
216            let crossref = wire::crossref_type(t).ok_or_else(|| {
217                Error::BadRequest(format!(
218                    "Crossref has no work type matching {:?}",
219                    t.as_csl()
220                ))
221            })?;
222            filters.push(format!("type:{crossref}"));
223        }
224
225        let mut params: Vec<(&str, String)> = Vec::new();
226        for (key, value) in &query.extra {
227            match key.as_str() {
228                "filter" => filters.push(value.clone()),
229                "sort" | "order" => params.push((key.as_str(), value.clone())),
230                k if k.starts_with("query.") => params.push((k, value.clone())),
231                other => {
232                    return Err(Error::BadRequest(format!(
233                        "unknown Crossref parameter {other:?} (supported: filter, sort, order, query.*)"
234                    )));
235                }
236            }
237        }
238        let text = query.text.trim();
239        if !text.is_empty() {
240            let field = match query.mode {
241                SearchMode::Citation => "query.bibliographic",
242                _ => "query",
243            };
244            params.push((field, text.to_owned()));
245        }
246        if !filters.is_empty() {
247            params.push(("filter", filters.join(",")));
248        }
249        // Cursor paging silently drops relevance order (observed 2026-11-10:
250        // the right match at 92.1 vanished from the top 3), so ask for it
251        // explicitly whenever there is text to rank by.
252        let has_query = params.iter().any(|(k, _)| k.starts_with("query"));
253        if has_query && !params.iter().any(|(k, _)| *k == "sort") {
254            params.push(("sort", "score".into()));
255            params.push(("order", "desc".into()));
256        }
257        params.push(("rows", query.page.limit.clamp(1, MAX_ROWS).to_string()));
258        let cursor = query.page.cursor.as_ref().map_or("*", |c| c.0.as_str());
259        params.push(("cursor", cursor.to_owned()));
260
261        let (hits, list) = self.list(&params).await?;
262        // Crossref keeps returning a cursor after the last page; an empty
263        // page is the real end.
264        let next = list.next_cursor.filter(|_| !hits.is_empty()).map(Cursor);
265        Ok(Response::new(Page::new(hits, list.total_results, next)))
266    }
267}
aspero_crossref/lib.rs

aspero_crossref/
lib.rs
