aspero_crossref/
lib.rs

1//! Crossref provider.
2//!
3//! Crossref is the registration agency for most journal and conference DOIs,
4//! so it is authoritative for their bibliographic fields. Its
5//! `query.bibliographic` matcher, built for reference strings, backs
6//! [`SearchMode::Citation`].
7//!
8//! It only knows Crossref DOIs: arXiv's DOIs are registered with DataCite,
9//! so preprints such as arXiv:1706.03762 are absent here.
10//!
11//! Free to use. Requests carry a `mailto`, which puts them in the "polite"
12//! pool (observed 2026-11: 10 req/s for single works, 3 req/s for lists, 3
13//! concurrent).
14
15mod http;
16mod wire;
17
18use std::sync::Arc;
19use std::time::Duration;
20
21use aspero_core::http::path_segment;
22use aspero_core::{
23    Capabilities, Cursor, Doi, Error, Field, FieldSet, Filter, Hit, HttpClient, HttpRequest,
24    HttpResponse, IdKind, Lookup, Page, Provider, ProviderId, ResiliencePolicy, Response, Result,
25    SearchMode, SearchQuery, SourceRecord, WorkId,
26};
27use async_trait::async_trait;
28
29use crate::wire::{Envelope, WireList};
30
31pub const ID: ProviderId = ProviderId::from_static("crossref");
32
33pub const DEFAULT_BASE_URL: &str = "https://api.crossref.org";
34
35/// DOIs per batched `filter=doi:…,doi:…` call; keeps URLs a sane length.
36const MAX_BATCH: usize = 50;
37
38/// Crossref's cap on `rows`.
39const MAX_ROWS: u32 = 1000;
40
41#[derive(Debug, Clone)]
42#[non_exhaustive]
43pub struct CrossrefConfig {
44    /// Contact address for the polite pool. Without it requests share the
45    /// slower public pool.
46    pub mailto: Option<String>,
47    pub base_url: String,
48}
49
50impl Default for CrossrefConfig {
51    fn default() -> Self {
52        CrossrefConfig {
53            mailto: None,
54            base_url: DEFAULT_BASE_URL.to_owned(),
55        }
56    }
57}
58
59pub struct Crossref {
60    config: CrossrefConfig,
61    http: Arc<dyn HttpClient>,
62    capabilities: Capabilities,
63}
64
65impl Crossref {
66    pub fn new(config: CrossrefConfig, http: Arc<dyn HttpClient>) -> Self {
67        let mut capabilities = Capabilities::default();
68        capabilities.lookup_kinds = vec![IdKind::Doi];
69        capabilities.max_batch = MAX_BATCH;
70        // Records say nothing about free copies.
71        capabilities.fields = FieldSet::ALL.difference(FieldSet::of(&[Field::OpenAccess]));
72        capabilities.search = true;
73        capabilities.citation_search = true;
74        capabilities.notices = true;
75        capabilities.search_filters = vec![Filter::Years, Filter::WorkType];
76        Crossref {
77            config,
78            http,
79            capabilities,
80        }
81    }
82
83    /// Paced for the polite pool's list limit (3 req/s), the stricter of
84    /// its two.
85    pub fn default_policy() -> ResiliencePolicy {
86        let mut policy = ResiliencePolicy::default();
87        policy.min_interval = Duration::from_millis(350);
88        policy
89    }
90
91    /// `/works<rest>` with `params` and the `mailto` appended.
92    fn url(&self, rest: &str, params: &[(&str, String)]) -> Result<String> {
93        let base = format!("{}/works{rest}", self.config.base_url.trim_end_matches('/'));
94        let mut url = url::Url::parse(&base)
95            .map_err(|e| Error::BadRequest(format!("invalid Crossref base URL: {e}")))?;
96        {
97            let mut query = url.query_pairs_mut();
98            for (k, v) in params {
99                query.append_pair(k, v);
100            }
101            if let Some(mailto) = &self.config.mailto {
102                query.append_pair("mailto", mailto);
103            }
104        }
105        // An empty query would leave a dangling `?`.
106        if url.query() == Some("") {
107            url.set_query(None);
108        }
109        Ok(url.into())
110    }
111
112    async fn get(&self, url: String) -> Result<HttpResponse> {
113        let req = HttpRequest::get(url).header("accept", "application/json");
114        let resp = self.http.send(req).await?;
115        http::check(&resp)?;
116        Ok(resp)
117    }
118
119    async fn lookup_one(&self, doi: &Doi) -> Result<Option<SourceRecord>> {
120        let url = self.url(&format!("/{}", path_segment(doi.as_str())), &[])?;
121        let resp = match self.get(url).await {
122            Ok(resp) => resp,
123            Err(Error::NotFound) => return Ok(None),
124            Err(e) => return Err(e),
125        };
126        let envelope: Envelope<serde_json::Value> =
127            serde_json::from_slice(&resp.body).map_err(Error::decode)?;
128        wire::to_record(&envelope.message, resp.received_at)
129            .map(Some)
130            .map_err(Error::decode)
131    }
132
133    async fn list(&self, params: &[(&str, String)]) -> Result<(Vec<Hit>, WireList)> {
134        let resp = self.get(self.url("", params)?).await?;
135        let envelope: Envelope<WireList> =
136            serde_json::from_slice(&resp.body).map_err(Error::decode)?;
137        let list = envelope.message;
138        let hits = list
139            .items
140            .iter()
141            .map(|v| {
142                let score = v.get("score").and_then(serde_json::Value::as_f64);
143                wire::to_record(v, resp.received_at).map(|r| Hit::new(r, score))
144            })
145            .collect::<serde_json::Result<_>>()
146            .map_err(Error::decode)?;
147        Ok((hits, list))
148    }
149}
150
151#[async_trait]
152impl Provider for Crossref {
153    fn id(&self) -> ProviderId {
154        ID
155    }
156
157    fn capabilities(&self) -> &Capabilities {
158        &self.capabilities
159    }
160
161    async fn lookup(&self, ids: &[WorkId], _fields: FieldSet) -> Result<Response<Vec<Lookup>>> {
162        let dois = ids
163            .iter()
164            .map(|id| match id {
165                WorkId::Doi(doi) => Ok(doi),
166                _ => Err(Error::Unsupported("Crossref lookup of non-DOI identifiers")),
167            })
168            .collect::<Result<Vec<_>>>()?;
169
170        if let [doi] = dois.as_slice() {
171            let record = self.lookup_one(doi).await?;
172            return Ok(Response::new(vec![Lookup {
173                id: ids[0].clone(),
174                record,
175            }]));
176        }
177
178        // `,` separates filters, so such DOIs can't be batched.
179        let (batchable, singles): (Vec<&Doi>, Vec<&Doi>) =
180            dois.iter().partition(|d| !d.as_str().contains(','));
181        let mut found = Vec::new();
182        for chunk in batchable.chunks(MAX_BATCH) {
183            let filter = chunk
184                .iter()
185                .map(|d| format!("doi:{}", d.as_str()))
186                .collect::<Vec<_>>()
187                .join(",");
188            let params = [("filter", filter), ("rows", chunk.len().to_string())];
189            let (hits, _) = self.list(&params).await?;
190            found.extend(hits.into_iter().map(|h| h.record));
191        }
192        for doi in singles {
193            found.extend(self.lookup_one(doi).await?);
194        }
195
196        let lookups = ids
197            .iter()
198            .map(|id| Lookup {
199                id: id.clone(),
200                record: found.iter().find(|r| r.ids.contains(id)).cloned(),
201            })
202            .collect();
203        Ok(Response::new(lookups))
204    }
205
206    async fn search(&self, query: &SearchQuery) -> Result<Response<Page>> {
207        let mut filters = Vec::new();
208        if let Some(years) = query.years {
209            if let Some(from) = years.from {
210                filters.push(format!("from-pub-date:{from:04}"));
211            }
212            if let Some(to) = years.to {
213                filters.push(format!("until-pub-date:{to:04}"));
214            }
215        }
216        if let Some(t) = &query.work_type {
217            let crossref = wire::crossref_type(t).ok_or_else(|| {
218                Error::BadRequest(format!(
219                    "Crossref has no work type matching {:?}",
220                    t.as_csl()
221                ))
222            })?;
223            filters.push(format!("type:{crossref}"));
224        }
225
226        let mut params: Vec<(&str, String)> = Vec::new();
227        for (key, value) in &query.extra {
228            match key.as_str() {
229                "filter" => filters.push(value.clone()),
230                "sort" | "order" => params.push((key.as_str(), value.clone())),
231                k if k.starts_with("query.") => params.push((k, value.clone())),
232                other => {
233                    return Err(Error::BadRequest(format!(
234                        "unknown Crossref parameter {other:?} (supported: filter, sort, order, query.*)"
235                    )));
236                }
237            }
238        }
239        let text = query.text.trim();
240        if !text.is_empty() {
241            let field = match query.mode {
242                SearchMode::Citation => "query.bibliographic",
243                _ => "query",
244            };
245            params.push((field, text.to_owned()));
246        }
247        if !filters.is_empty() {
248            params.push(("filter", filters.join(",")));
249        }
250        // Cursor paging silently drops relevance order (observed 2026-11-10:
251        // the right match at 92.1 vanished from the top 3), so ask for it
252        // explicitly whenever there is text to rank by.
253        let has_query = params.iter().any(|(k, _)| k.starts_with("query"));
254        if has_query && !params.iter().any(|(k, _)| *k == "sort") {
255            params.push(("sort", "score".into()));
256            params.push(("order", "desc".into()));
257        }
258        params.push(("rows", query.page.limit.clamp(1, MAX_ROWS).to_string()));
259        let cursor = query.page.cursor.as_ref().map_or("*", |c| c.0.as_str());
260        params.push(("cursor", cursor.to_owned()));
261
262        let (hits, list) = self.list(&params).await?;
263        // Crossref keeps returning a cursor after the last page; an empty
264        // page is the real end.
265        let next = list.next_cursor.filter(|_| !hits.is_empty()).map(Cursor);
266        Ok(Response::new(Page::new(hits, list.total_results, next)))
267    }
268}
aspero_crossref/lib.rs

aspero_crossref/
lib.rs
