aspero_crossref/
lib.rs

1//! Crossref provider.
2//!
3//! Crossref is the registration agency for most journal and conference DOIs,
4//! so it is authoritative for their bibliographic fields. Its
5//! `query.bibliographic` matcher, built for reference strings, backs
6//! [`SearchMode::Citation`].
7//!
8//! It only knows Crossref DOIs: arXiv's DOIs are registered with DataCite,
9//! so preprints such as arXiv:1706.03762 are absent here.
10//!
11//! Free to use. Requests carry a `mailto`, which puts them in the "polite"
12//! pool (observed 2026-11: 10 req/s for single works, 3 req/s for lists, 3
13//! concurrent).
14
15mod http;
16mod wire;
17
18use std::sync::Arc;
19use std::time::Duration;
20
21use aspero_core::http::path_segment;
22use aspero_core::{
23    Capabilities, Cursor, Doi, Error, Field, FieldSet, Filter, Hit, HttpClient, HttpRequest,
24    HttpResponse, IdKind, Lookup, Page, Provider, ProviderId, ResiliencePolicy, Response, Result,
25    SearchMode, SearchQuery, SourceRecord, WorkId,
26};
27use async_trait::async_trait;
28
29use crate::wire::{Envelope, WireList};
30
31pub const ID: ProviderId = ProviderId::from_static("crossref");
32
33pub const DEFAULT_BASE_URL: &str = "https://api.crossref.org";
34
35/// DOIs per batched `filter=doi:…,doi:…` call; keeps URLs a sane length.
36const MAX_BATCH: usize = 50;
37
38/// Crossref's cap on `rows`.
39const MAX_ROWS: u32 = 1000;
40
41#[derive(Debug, Clone)]
42#[non_exhaustive]
43pub struct CrossrefConfig {
44    /// Contact address for the polite pool. Without it requests share the
45    /// slower public pool.
46    pub mailto: Option<String>,
47    pub base_url: String,
48}
49
50impl Default for CrossrefConfig {
51    fn default() -> Self {
52        CrossrefConfig {
53            mailto: None,
54            base_url: DEFAULT_BASE_URL.to_owned(),
55        }
56    }
57}
58
59pub struct Crossref {
60    config: CrossrefConfig,
61    http: Arc<dyn HttpClient>,
62    capabilities: Capabilities,
63}
64
65impl Crossref {
66    pub fn new(config: CrossrefConfig, http: Arc<dyn HttpClient>) -> Self {
67        let mut capabilities = Capabilities::default();
68        capabilities.lookup_kinds = vec![IdKind::Doi];
69        capabilities.max_batch = MAX_BATCH;
70        // Records say nothing about free copies.
71        capabilities.fields = FieldSet::ALL.difference(FieldSet::of(&[Field::OpenAccess]));
72        capabilities.search = true;
73        capabilities.citation_search = true;
74        capabilities.notices = true;
75        capabilities.search_filters = vec![Filter::Years, Filter::WorkType];
76        Crossref {
77            config,
78            http,
79            capabilities,
80        }
81    }
82
83    /// Paced for the polite pool's list limit (3 req/s), the stricter of
84    /// its two.
85    pub fn default_policy() -> ResiliencePolicy {
86        let mut policy = ResiliencePolicy::default();
87        policy.min_interval = Duration::from_millis(350);
88        policy
89    }
90
91    /// `/works<rest>` with `params` and the `mailto` appended.
92    fn url(&self, rest: &str, params: &[(&str, String)]) -> Result<String> {
93        let base = format!("{}/works{rest}", self.config.base_url.trim_end_matches('/'));
94        let mut url = url::Url::parse(&base)
95            .map_err(|e| Error::BadRequest(format!("invalid Crossref base URL: {e}")))?;
96        {
97            let mut query = url.query_pairs_mut();
98            for (k, v) in params {
99                query.append_pair(k, v);
100            }
101            if let Some(mailto) = &self.config.mailto {
102                query.append_pair("mailto", mailto);
103            }
104        }
105        // An empty query would leave a dangling `?`.
106        if url.query() == Some("") {
107            url.set_query(None);
108        }
109        Ok(url.into())
110    }
111
112    async fn get(&self, url: String) -> Result<HttpResponse> {
113        let req = HttpRequest::get(url).header("accept", "application/json");
114        let resp = self.http.send(req).await?;
115        http::check(&resp)?;
116        Ok(resp)
117    }
118
119    async fn lookup_one(&self, doi: &Doi) -> Result<Option<SourceRecord>> {
120        let url = self.url(&format!("/{}", path_segment(doi.as_str())), &[])?;
121        let resp = match self.get(url).await {
122            Ok(resp) => resp,
123            Err(Error::NotFound) => return Ok(None),
124            Err(e) => return Err(e),
125        };
126        let envelope: Envelope<serde_json::Value> =
127            serde_json::from_slice(&resp.body).map_err(Error::decode)?;
128        wire::to_record(&envelope.message, resp.received_at)
129            .map(Some)
130            .map_err(Error::decode)
131    }
132
133    async fn list(&self, params: &[(&str, String)]) -> Result<(Vec<Hit>, WireList)> {
134        let resp = self.get(self.url("", params)?).await?;
135        let envelope: Envelope<WireList> =
136            serde_json::from_slice(&resp.body).map_err(Error::decode)?;
137        let list = envelope.message;
138        let hits = list
139            .items
140            .iter()
141            .filter_map(|v| {
142                let score = v.get("score").and_then(serde_json::Value::as_f64);
143                or_skip(wire::to_record(v, resp.received_at)).map(|r| Hit::new(r, score))
144            })
145            .collect();
146        Ok((hits, list))
147    }
148}
149
150/// The record, or `None` with a warning when it doesn't decode: one bad
151/// record shouldn't cost the rest of the page.
152fn or_skip(record: serde_json::Result<SourceRecord>) -> Option<SourceRecord> {
153    record
154        .inspect_err(|e| {
155            tracing::warn!(
156                target: "aspero::decode",
157                provider = %ID, error = %e, "skipped a record that doesn't decode"
158            );
159        })
160        .ok()
161}
162
163#[async_trait]
164impl Provider for Crossref {
165    fn id(&self) -> ProviderId {
166        ID
167    }
168
169    fn capabilities(&self) -> &Capabilities {
170        &self.capabilities
171    }
172
173    async fn lookup(&self, ids: &[WorkId], _fields: FieldSet) -> Result<Response<Vec<Lookup>>> {
174        let dois = ids
175            .iter()
176            .map(|id| match id {
177                WorkId::Doi(doi) => Ok(doi),
178                _ => Err(Error::Unsupported("Crossref lookup of non-DOI identifiers")),
179            })
180            .collect::<Result<Vec<_>>>()?;
181
182        if let [doi] = dois.as_slice() {
183            let record = self.lookup_one(doi).await?;
184            return Ok(Response::new(vec![Lookup {
185                id: ids[0].clone(),
186                record,
187            }]));
188        }
189
190        // `,` separates filters, so such DOIs can't be batched.
191        let (batchable, singles): (Vec<&Doi>, Vec<&Doi>) =
192            dois.iter().partition(|d| !d.as_str().contains(','));
193        let mut found = Vec::new();
194        for chunk in batchable.chunks(MAX_BATCH) {
195            let filter = chunk
196                .iter()
197                .map(|d| format!("doi:{}", d.as_str()))
198                .collect::<Vec<_>>()
199                .join(",");
200            let params = [("filter", filter), ("rows", chunk.len().to_string())];
201            let (hits, _) = self.list(&params).await?;
202            found.extend(hits.into_iter().map(|h| h.record));
203        }
204        for doi in singles {
205            found.extend(self.lookup_one(doi).await?);
206        }
207
208        let lookups = ids
209            .iter()
210            .map(|id| Lookup {
211                id: id.clone(),
212                record: found.iter().find(|r| r.ids.contains(id)).cloned(),
213            })
214            .collect();
215        Ok(Response::new(lookups))
216    }
217
218    async fn search(&self, query: &SearchQuery) -> Result<Response<Page>> {
219        let mut filters = Vec::new();
220        if let Some(years) = query.years {
221            if let Some(from) = years.from {
222                filters.push(format!("from-pub-date:{from:04}"));
223            }
224            if let Some(to) = years.to {
225                filters.push(format!("until-pub-date:{to:04}"));
226            }
227        }
228        if let Some(t) = &query.work_type {
229            let crossref = wire::crossref_type(t).ok_or_else(|| {
230                Error::BadRequest(format!(
231                    "Crossref has no work type matching {:?}",
232                    t.as_csl()
233                ))
234            })?;
235            filters.push(format!("type:{crossref}"));
236        }
237
238        let mut params: Vec<(&str, String)> = Vec::new();
239        for (key, value) in &query.extra {
240            match key.as_str() {
241                "filter" => filters.push(value.clone()),
242                "sort" | "order" => params.push((key.as_str(), value.clone())),
243                k if k.starts_with("query.") => params.push((k, value.clone())),
244                other => {
245                    return Err(Error::BadRequest(format!(
246                        "unknown Crossref parameter {other:?} (supported: filter, sort, order, query.*)"
247                    )));
248                }
249            }
250        }
251        let text = query.text.trim();
252        if !text.is_empty() {
253            let field = match query.mode {
254                SearchMode::Citation => "query.bibliographic",
255                _ => "query",
256            };
257            params.push((field, text.to_owned()));
258        }
259        if !filters.is_empty() {
260            params.push(("filter", filters.join(",")));
261        }
262        // Cursor paging silently drops relevance order (observed 2026-11-10:
263        // the right match at 92.1 vanished from the top 3), so ask for it
264        // explicitly whenever there is text to rank by.
265        let has_query = params.iter().any(|(k, _)| k.starts_with("query"));
266        if has_query && !params.iter().any(|(k, _)| *k == "sort") {
267            params.push(("sort", "score".into()));
268            params.push(("order", "desc".into()));
269        }
270        params.push(("rows", query.page.limit.clamp(1, MAX_ROWS).to_string()));
271        let cursor = query.page.cursor.as_ref().map_or("*", |c| c.0.as_str());
272        params.push(("cursor", cursor.to_owned()));
273
274        let (hits, list) = self.list(&params).await?;
275        // Crossref keeps returning a cursor after the last page; an empty
276        // page is the real end.
277        let next = list.next_cursor.filter(|_| !hits.is_empty()).map(Cursor);
278        Ok(Response::new(Page::new(hits, list.total_results, next)))
279    }
280}
aspero_crossref/lib.rs

aspero_crossref/
lib.rs
