Skip to main content

opencitations_client/
lib.rs

1//! An async client for the [OpenCitations](https://opencitations.net) REST apis.
2//!
3//! OpenCitations answers the question crossref's public api cannot: **which
4//! works cite this one**. Crossref holds the reference list a publisher
5//! deposited -- what a work points *at* -- and exposes the number of incoming
6//! citations but never the list. OpenCitations inverts those deposited
7//! references into a citation index and serves both directions openly, with no
8//! account and no credentials.
9//!
10//! Two apis sit behind one client here, because a citation is only useful
11//! alongside what it connects:
12//!
13//! * the **[Index](https://api.opencitations.net/index/v2)** — the citation
14//!   edges. [`OpenCitations::citations`] for the works citing something,
15//!   [`OpenCitations::references`] for the works it cites, and the counts of
16//!   both.
17//! * **[Meta](https://api.opencitations.net/meta/v1)** — the bibliographic
18//!   record behind an identifier, via [`OpenCitations::metadata`].
19//!
20//! ## Everything is addressed by a prefixed identifier
21//!
22//! A bare DOI means nothing to OpenCitations; `doi:` in front of it does. See
23//! [`Identifier`], which parses the `scheme:value` form and refuses a bare one
24//! rather than guessing — a guess that picks the wrong scheme comes back as an
25//! empty list, which reads exactly like a work nobody has cited.
26//!
27//! ```no_run
28//! use opencitations_client::{Identifier, OpenCitations};
29//!
30//! # async fn run() -> Result<(), opencitations_client::Error> {
31//! let client = OpenCitations::builder().build()?;
32//! let paper = Identifier::doi("10.1016/j.marpol.2021.104855");
33//!
34//! for citation in client.citations(&paper).await? {
35//!     println!("{:?} cited it in {:?}", citation.citing_doi(), citation.creation);
36//! }
37//! # Ok(())
38//! # }
39//! ```
40//!
41//! ## Nothing found is not an error
42//!
43//! Both apis answer an identifier they hold nothing about with `200` and an
44//! empty list, so a work nobody has cited, a work OpenCitations has never
45//! indexed, and a typo in a DOI are indistinguishable from one another. An
46//! empty [`Vec`] is the answer to all three; only a malformed identifier
47//! fails, as [`Error::NoSuchOperation`].
48//!
49//! ## Rate
50//!
51//! OpenCitations documents 180 requests a minute per address and reports
52//! nothing about the budget on a response, so the client paces itself blindly
53//! at that rate and retries a `429`. Cloning is cheap and shares the limiter;
54//! build one client and clone it rather than building several, or they will not
55//! queue against one another. An access token from
56//! [opencitations.net/accesstoken](https://opencitations.net/accesstoken) is
57//! optional and raises the limit — see [`OpenCitationsBuilder::token`].
58
59#![deny(missing_docs)]
60
61mod date;
62mod error;
63mod id;
64mod index;
65mod limit;
66mod meta;
67
68pub use crate::date::{DateParseError, PartialDate, Timespan, TimespanParseError};
69pub use crate::error::{Error, Result};
70pub use crate::id::{Identifier, Oci};
71pub use crate::index::{Citation, Comparison, Field, Filter, Order, Refine, Sort};
72pub use crate::limit::RateLimit;
73pub use crate::meta::{Entity, Metadata};
74
75use crate::index::Count;
76use crate::limit::{Limiter, retry_after};
77use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode};
78use reqwest::{Client, StatusCode};
79use serde::de::DeserializeOwned;
80use std::sync::Arc;
81use std::time::Duration;
82
83/// The characters that would end the path if they were left in an identifier.
84///
85/// `:` and `/` are deliberately *not* encoded: every identifier carries a `:`,
86/// most DOIs carry a `/`, and OpenCitations reads both forms — so leaving them
87/// alone keeps the url readable in a log or an error message.
88const IN_PATH: &AsciiSet = &CONTROLS
89    .add(b' ')
90    .add(b'"')
91    .add(b'<')
92    .add(b'>')
93    .add(b'?')
94    .add(b'#')
95    .add(b'%')
96    .add(b'`')
97    .add(b'{')
98    .add(b'}')
99    .add(b'\\')
100    .add(b'^')
101    .add(b'|');
102
103/// An async client for the OpenCitations Index and Meta apis.
104///
105/// Cloning is cheap and shares the connection pool and the rate limiter, so
106/// concurrent callers should clone one client rather than build several.
107#[derive(Debug, Clone)]
108pub struct OpenCitations {
109    base_url: String,
110    client: Client,
111    limiter: Arc<Limiter>,
112    max_retries: u32,
113}
114
115impl OpenCitations {
116    const BASE_URL: &'static str = "https://api.opencitations.net";
117    /// the Index api, the half that holds the citation edges
118    const INDEX: &'static str = "index/v2";
119    /// the Meta api, the half that holds the bibliographic records
120    const META: &'static str = "meta/v1";
121
122    /// How many identifiers [`Self::metadata`] puts in one request.
123    ///
124    /// Meta answers a `500` at fifteen identifiers and succeeds at fourteen,
125    /// whichever ids they are — a cap it does not document and does not
126    /// mention in the error. Ten leaves room for that edge to move without
127    /// this crate finding out from a user.
128    pub const META_BATCH: usize = 10;
129
130    /// Constructs a new [`OpenCitationsBuilder`].
131    pub fn builder() -> OpenCitationsBuilder {
132        OpenCitationsBuilder::new()
133    }
134
135    /// The url every route is appended to, `https://api.opencitations.net`
136    /// unless [`OpenCitationsBuilder::base_url`] said otherwise.
137    pub fn base_url(&self) -> &str {
138        &self.base_url
139    }
140
141    /// The rate this client is pacing itself at.
142    ///
143    /// OpenCitations reports no budget of its own, so this is whatever the
144    /// builder was given and never changes.
145    pub fn rate_limit(&self) -> RateLimit {
146        self.limiter.rate()
147    }
148
149    /// The works that cite `id` — the direction crossref cannot answer.
150    ///
151    /// Empty when nothing cites it *and* when OpenCitations has never indexed
152    /// it; the two are not distinguishable. Unpaged, so a heavily cited work
153    /// arrives in one large response — narrow it with
154    /// [`OpenCitations::citations_where`] rather than here.
155    pub async fn citations(&self, id: &Identifier) -> Result<Vec<Citation>> {
156        self.citations_where(id, &Refine::new()).await
157    }
158
159    /// The works that cite `id`, narrowed and ordered by the api.
160    ///
161    /// ```no_run
162    /// use opencitations_client::{Field, Filter, Identifier, OpenCitations, Refine, Sort};
163    ///
164    /// # async fn run() -> Result<(), opencitations_client::Error> {
165    /// # let client = OpenCitations::builder().build()?;
166    /// let recent = Refine::new()
167    ///     .filter(Filter::greater_than(Field::Creation, "2023"))
168    ///     .sort(Sort::descending(Field::Creation));
169    ///
170    /// let citations = client
171    ///     .citations_where(&Identifier::doi("10.1016/j.marpol.2021.104855"), &recent)
172    ///     .await?;
173    /// # Ok(())
174    /// # }
175    /// ```
176    pub async fn citations_where(&self, id: &Identifier, refine: &Refine) -> Result<Vec<Citation>> {
177        self.get(
178            &format!("{}/citations/{}", Self::INDEX, encode(id)),
179            &refine.params(),
180            "citations",
181        )
182        .await
183    }
184
185    /// The works `id` cites, as OpenCitations indexed them.
186    ///
187    /// This is the same direction as crossref's `reference[]` and a useful
188    /// cross-check on it: OpenCitations resolves references to identifiers,
189    /// where a crossref reference may be an unstructured string.
190    pub async fn references(&self, id: &Identifier) -> Result<Vec<Citation>> {
191        self.references_where(id, &Refine::new()).await
192    }
193
194    /// The works `id` cites, narrowed and ordered by the api.
195    pub async fn references_where(
196        &self,
197        id: &Identifier,
198        refine: &Refine,
199    ) -> Result<Vec<Citation>> {
200        self.get(
201            &format!("{}/references/{}", Self::INDEX, encode(id)),
202            &refine.params(),
203            "references",
204        )
205        .await
206    }
207
208    /// One citation, by its own identifier.
209    ///
210    /// [`None`] when OpenCitations holds no such citation.
211    pub async fn citation(&self, oci: &Oci) -> Result<Option<Citation>> {
212        let found: Vec<Citation> = self
213            .get(
214                &format!("{}/citation/{oci}", Self::INDEX),
215                &[],
216                "the citation",
217            )
218            .await?;
219        Ok(found.into_iter().next())
220    }
221
222    /// How many works cite `id`.
223    ///
224    /// Cheaper than counting [`OpenCitations::citations`], which fetches every
225    /// edge to do it.
226    pub async fn citation_count(&self, id: &Identifier) -> Result<u64> {
227        self.count(&format!("{}/citation-count/{}", Self::INDEX, encode(id)))
228            .await
229    }
230
231    /// How many works `id` cites.
232    pub async fn reference_count(&self, id: &Identifier) -> Result<u64> {
233        self.count(&format!("{}/reference-count/{}", Self::INDEX, encode(id)))
234            .await
235    }
236
237    /// How many citations everything published in a venue has received.
238    ///
239    /// Addressed by ISSN — `Identifier::issn("0308-597X")`.
240    pub async fn venue_citation_count(&self, issn: &Identifier) -> Result<u64> {
241        self.count(&format!(
242            "{}/venue-citation-count/{}",
243            Self::INDEX,
244            encode(issn)
245        ))
246        .await
247    }
248
249    /// The bibliographic records behind any number of identifiers.
250    ///
251    /// Ids are batched rather than asked about one at a time, which is worth
252    /// doing: a batch is one slot of the rate limit rather than one each. More
253    /// than [`Self::META_BATCH`] of them is split across several requests,
254    /// paced like any other.
255    ///
256    /// Records come back in whatever order Meta answers in, and an id it holds
257    /// nothing about is simply absent — so match the results up by
258    /// [`Metadata::id`] rather than by position. An empty slice makes no
259    /// request.
260    ///
261    /// ```no_run
262    /// # use opencitations_client::{Identifier, OpenCitations};
263    /// # async fn run() -> Result<(), opencitations_client::Error> {
264    /// # let client = OpenCitations::builder().build()?;
265    /// let records = client
266    ///     .metadata(&[
267    ///         Identifier::doi("10.1016/j.marpol.2021.104855"),
268    ///         Identifier::doi("10.1038/nature12373"),
269    ///     ])
270    ///     .await?;
271    /// # Ok(())
272    /// # }
273    /// ```
274    pub async fn metadata(&self, ids: &[Identifier]) -> Result<Vec<Metadata>> {
275        let mut records = Vec::with_capacity(ids.len());
276
277        for batch in ids.chunks(Self::META_BATCH) {
278            // Meta joins several ids with a double underscore
279            let joined = batch.iter().map(encode).collect::<Vec<_>>().join("__");
280
281            let found: Vec<Metadata> = self
282                .get(
283                    &format!("{}/metadata/{joined}", Self::META),
284                    &[],
285                    "the metadata",
286                )
287                .await?;
288            records.extend(found);
289        }
290
291        Ok(records)
292    }
293
294    /// Reads one of the count routes, which answer with a list holding one
295    /// object holding the number as a string.
296    async fn count(&self, route: &str) -> Result<u64> {
297        let counts: Vec<Count> = self.get(route, &[], "the count").await?;
298        let Some(count) = counts.first() else {
299            // an entity it holds nothing about, which is a count of zero
300            return Ok(0);
301        };
302        count.count.trim().parse().map_err(|_| Error::Api {
303            status: 200,
304            body: format!("`{}` is not a count", count.count),
305        })
306    }
307
308    /// Sends a request, retrying a `429`, and reads the body as `T`.
309    async fn get<T: DeserializeOwned>(
310        &self,
311        route: &str,
312        params: &[(&'static str, String)],
313        what: &'static str,
314    ) -> Result<T> {
315        let url = format!("{}/{route}", self.base_url);
316
317        for attempt in 1.. {
318            self.limiter.acquire().await;
319            tracing::debug!(%url, attempt, "requesting");
320
321            let response = self.client.get(&url).query(params).send().await?;
322            let status = response.status();
323
324            if status == StatusCode::TOO_MANY_REQUESTS {
325                if attempt > self.max_retries {
326                    return Err(Error::RateLimited { attempts: attempt });
327                }
328                // back off for everything sharing the limiter, not just this
329                // request -- the rate we were pacing at was wrong
330                let delay = retry_after(response.headers())
331                    .unwrap_or_else(|| Duration::from_secs(u64::from(attempt)));
332                self.limiter.back_off(delay);
333                continue;
334            }
335
336            let body = response.text().await?;
337            if !status.is_success() {
338                let body = body.trim().to_string();
339                // an identifier it could not even parse answers `404` with a
340                // line of plain text rather than json
341                return Err(if status == StatusCode::NOT_FOUND {
342                    Error::NoSuchOperation { message: body }
343                } else {
344                    Error::Api {
345                        status: status.as_u16(),
346                        body,
347                    }
348                });
349            }
350
351            return serde_json::from_str(&body).map_err(|source| Error::Malformed { what, source });
352        }
353
354        unreachable!("the retry loop returns or continues")
355    }
356}
357
358/// Percent-encodes an identifier for the path it goes into.
359fn encode(id: &Identifier) -> String {
360    utf8_percent_encode(&id.to_string(), IN_PATH).to_string()
361}
362
363/// Builds an [`OpenCitations`] client.
364///
365/// Every option has a working default, so the shortest form is
366/// `OpenCitations::builder().build()?`.
367#[derive(Debug, Default)]
368pub struct OpenCitationsBuilder {
369    user_agent: Option<String>,
370    token: Option<String>,
371    base_url: Option<String>,
372    rate_limit: Option<RateLimit>,
373    max_retries: Option<u32>,
374}
375
376impl OpenCitationsBuilder {
377    /// How many times a `429` is retried, when the builder was not told
378    /// otherwise.
379    const DEFAULT_MAX_RETRIES: u32 = 3;
380
381    /// Constructs a new `OpenCitationsBuilder`.
382    pub fn new() -> Self {
383        Self::default()
384    }
385
386    /// An access token from [opencitations.net/accesstoken](https://opencitations.net/accesstoken),
387    /// sent as the `authorization` header.
388    ///
389    /// Optional — every route works without one — but it raises the rate limit
390    /// and is how OpenCitations tells its users apart. Getting one is free and
391    /// wants only an email address.
392    pub fn token<'a>(mut self, token: impl Into<Option<&'a str>>) -> Self {
393        self.token = token.into().map(str::to_owned);
394        self
395    }
396
397    /// Identify the caller in the `User-Agent` header.
398    ///
399    /// Not required by OpenCitations, unlike crossref's polite pool, but it is
400    /// the courtesy that lets them tell a misbehaving script from a busy one.
401    /// Defaults to this crate's name and version.
402    pub fn user_agent<'a>(mut self, user_agent: impl Into<Option<&'a str>>) -> Self {
403        self.user_agent = user_agent.into().map(str::to_owned);
404        self
405    }
406
407    /// Send requests somewhere other than `https://api.opencitations.net`.
408    ///
409    /// Routes are appended verbatim, so the url must not end in a `/`. Mainly
410    /// useful for pointing the client at a mock or a proxy in tests.
411    pub fn base_url<'a>(mut self, base_url: impl Into<Option<&'a str>>) -> Self {
412        self.base_url = base_url.into().map(str::to_owned);
413        self
414    }
415
416    /// The rate to pace requests at. Defaults to [`RateLimit::DOCUMENTED`].
417    ///
418    /// The documented limit is per ip address rather than per client, so lower
419    /// this when something else on the same machine is querying too.
420    pub fn rate_limit(mut self, rate_limit: impl Into<Option<RateLimit>>) -> Self {
421        self.rate_limit = rate_limit.into();
422        self
423    }
424
425    /// How many times a `429` is retried before giving up with
426    /// [`Error::RateLimited`]. Defaults to 3; `0` disables retrying.
427    pub fn max_retries(mut self, max_retries: impl Into<Option<u32>>) -> Self {
428        self.max_retries = max_retries.into();
429        self
430    }
431
432    /// Returns an [`OpenCitations`] using this configuration.
433    ///
434    /// # Errors
435    ///
436    /// Fails if a header cannot be built from what it was given, or if the TLS
437    /// backend cannot be initialized.
438    pub fn build(self) -> Result<OpenCitations> {
439        use reqwest::header;
440
441        let mut headers = header::HeaderMap::new();
442        let agent = self.user_agent.unwrap_or_else(|| {
443            format!(
444                "{}/{} ({})",
445                env!("CARGO_PKG_NAME"),
446                env!("CARGO_PKG_VERSION"),
447                env!("CARGO_PKG_REPOSITORY")
448            )
449        });
450        headers.insert(
451            header::USER_AGENT,
452            header::HeaderValue::from_str(&agent).map_err(|_| Error::Config {
453                msg: format!("failed to create User-Agent header for `{agent}`"),
454            })?,
455        );
456        if let Some(token) = &self.token {
457            headers.insert(
458                header::AUTHORIZATION,
459                header::HeaderValue::from_str(token).map_err(|_| Error::Config {
460                    msg: "failed to create authorization header for the access token".to_string(),
461                })?,
462            );
463        }
464
465        let client = Client::builder()
466            .default_headers(headers)
467            .build()
468            .map_err(|_| Error::Config {
469                msg: "failed to initialize TLS backend".to_string(),
470            })?;
471
472        Ok(OpenCitations {
473            base_url: self
474                .base_url
475                .unwrap_or_else(|| OpenCitations::BASE_URL.to_string()),
476            client,
477            limiter: Arc::new(Limiter::new(self.rate_limit.unwrap_or_default())),
478            max_retries: self.max_retries.unwrap_or(Self::DEFAULT_MAX_RETRIES),
479        })
480    }
481}
482
483#[cfg(test)]
484mod tests {
485    use super::*;
486
487    #[test]
488    fn a_default_client_talks_to_opencitations_at_the_documented_rate() {
489        let client = OpenCitations::builder().build().unwrap();
490        assert_eq!(client.base_url(), "https://api.opencitations.net");
491        assert_eq!(client.rate_limit(), RateLimit::DOCUMENTED);
492    }
493
494    /// The `:` and the `/` every identifier is built from stay readable; the
495    /// characters that would end the path do not.
496    #[test]
497    fn an_identifier_is_encoded_only_where_it_has_to_be() {
498        assert_eq!(
499            encode(&Identifier::doi("10.1016/j.marpol.2021.104855")),
500            "doi:10.1016/j.marpol.2021.104855"
501        );
502        assert_eq!(
503            encode(&Identifier::doi("10.1000/a b?c#d")),
504            "doi:10.1000/a%20b%3Fc%23d"
505        );
506    }
507
508    #[tokio::test]
509    async fn asking_for_no_metadata_makes_no_request() {
510        // the base url is a black hole; reaching it would hang or fail
511        let client = OpenCitations::builder()
512            .base_url("http://127.0.0.1:1")
513            .build()
514            .unwrap();
515
516        assert!(client.metadata(&[]).await.unwrap().is_empty());
517    }
518}