opencitations-client 0.1.0

An async Rust client for the OpenCitations Index and Meta REST APIs
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
//! An async client for the [OpenCitations](https://opencitations.net) REST apis.
//!
//! OpenCitations answers the question crossref's public api cannot: **which
//! works cite this one**. Crossref holds the reference list a publisher
//! deposited -- what a work points *at* -- and exposes the number of incoming
//! citations but never the list. OpenCitations inverts those deposited
//! references into a citation index and serves both directions openly, with no
//! account and no credentials.
//!
//! Two apis sit behind one client here, because a citation is only useful
//! alongside what it connects:
//!
//! * the **[Index](https://api.opencitations.net/index/v2)** — the citation
//!   edges. [`OpenCitations::citations`] for the works citing something,
//!   [`OpenCitations::references`] for the works it cites, and the counts of
//!   both.
//! * **[Meta](https://api.opencitations.net/meta/v1)** — the bibliographic
//!   record behind an identifier, via [`OpenCitations::metadata`].
//!
//! ## Everything is addressed by a prefixed identifier
//!
//! A bare DOI means nothing to OpenCitations; `doi:` in front of it does. See
//! [`Identifier`], which parses the `scheme:value` form and refuses a bare one
//! rather than guessing — a guess that picks the wrong scheme comes back as an
//! empty list, which reads exactly like a work nobody has cited.
//!
//! ```no_run
//! use opencitations_client::{Identifier, OpenCitations};
//!
//! # async fn run() -> Result<(), opencitations_client::Error> {
//! let client = OpenCitations::builder().build()?;
//! let paper = Identifier::doi("10.1016/j.marpol.2021.104855");
//!
//! for citation in client.citations(&paper).await? {
//!     println!("{:?} cited it in {:?}", citation.citing_doi(), citation.creation);
//! }
//! # Ok(())
//! # }
//! ```
//!
//! ## Nothing found is not an error
//!
//! Both apis answer an identifier they hold nothing about with `200` and an
//! empty list, so a work nobody has cited, a work OpenCitations has never
//! indexed, and a typo in a DOI are indistinguishable from one another. An
//! empty [`Vec`] is the answer to all three; only a malformed identifier
//! fails, as [`Error::NoSuchOperation`].
//!
//! ## Rate
//!
//! OpenCitations documents 180 requests a minute per address and reports
//! nothing about the budget on a response, so the client paces itself blindly
//! at that rate and retries a `429`. Cloning is cheap and shares the limiter;
//! build one client and clone it rather than building several, or they will not
//! queue against one another. An access token from
//! [opencitations.net/accesstoken](https://opencitations.net/accesstoken) is
//! optional and raises the limit — see [`OpenCitationsBuilder::token`].

#![deny(missing_docs)]

mod date;
mod error;
mod id;
mod index;
mod limit;
mod meta;

pub use crate::date::{DateParseError, PartialDate, Timespan, TimespanParseError};
pub use crate::error::{Error, Result};
pub use crate::id::{Identifier, Oci};
pub use crate::index::{Citation, Comparison, Field, Filter, Order, Refine, Sort};
pub use crate::limit::RateLimit;
pub use crate::meta::{Entity, Metadata};

use crate::index::Count;
use crate::limit::{Limiter, retry_after};
use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode};
use reqwest::{Client, StatusCode};
use serde::de::DeserializeOwned;
use std::sync::Arc;
use std::time::Duration;

/// The characters that would end the path if they were left in an identifier.
///
/// `:` and `/` are deliberately *not* encoded: every identifier carries a `:`,
/// most DOIs carry a `/`, and OpenCitations reads both forms — so leaving them
/// alone keeps the url readable in a log or an error message.
const IN_PATH: &AsciiSet = &CONTROLS
    .add(b' ')
    .add(b'"')
    .add(b'<')
    .add(b'>')
    .add(b'?')
    .add(b'#')
    .add(b'%')
    .add(b'`')
    .add(b'{')
    .add(b'}')
    .add(b'\\')
    .add(b'^')
    .add(b'|');

/// An async client for the OpenCitations Index and Meta apis.
///
/// Cloning is cheap and shares the connection pool and the rate limiter, so
/// concurrent callers should clone one client rather than build several.
#[derive(Debug, Clone)]
pub struct OpenCitations {
    base_url: String,
    client: Client,
    limiter: Arc<Limiter>,
    max_retries: u32,
}

impl OpenCitations {
    const BASE_URL: &'static str = "https://api.opencitations.net";
    /// the Index api, the half that holds the citation edges
    const INDEX: &'static str = "index/v2";
    /// the Meta api, the half that holds the bibliographic records
    const META: &'static str = "meta/v1";

    /// How many identifiers [`Self::metadata`] puts in one request.
    ///
    /// Meta answers a `500` at fifteen identifiers and succeeds at fourteen,
    /// whichever ids they are — a cap it does not document and does not
    /// mention in the error. Ten leaves room for that edge to move without
    /// this crate finding out from a user.
    pub const META_BATCH: usize = 10;

    /// Constructs a new [`OpenCitationsBuilder`].
    pub fn builder() -> OpenCitationsBuilder {
        OpenCitationsBuilder::new()
    }

    /// The url every route is appended to, `https://api.opencitations.net`
    /// unless [`OpenCitationsBuilder::base_url`] said otherwise.
    pub fn base_url(&self) -> &str {
        &self.base_url
    }

    /// The rate this client is pacing itself at.
    ///
    /// OpenCitations reports no budget of its own, so this is whatever the
    /// builder was given and never changes.
    pub fn rate_limit(&self) -> RateLimit {
        self.limiter.rate()
    }

    /// The works that cite `id` — the direction crossref cannot answer.
    ///
    /// Empty when nothing cites it *and* when OpenCitations has never indexed
    /// it; the two are not distinguishable. Unpaged, so a heavily cited work
    /// arrives in one large response — narrow it with
    /// [`OpenCitations::citations_where`] rather than here.
    pub async fn citations(&self, id: &Identifier) -> Result<Vec<Citation>> {
        self.citations_where(id, &Refine::new()).await
    }

    /// The works that cite `id`, narrowed and ordered by the api.
    ///
    /// ```no_run
    /// use opencitations_client::{Field, Filter, Identifier, OpenCitations, Refine, Sort};
    ///
    /// # async fn run() -> Result<(), opencitations_client::Error> {
    /// # let client = OpenCitations::builder().build()?;
    /// let recent = Refine::new()
    ///     .filter(Filter::greater_than(Field::Creation, "2023"))
    ///     .sort(Sort::descending(Field::Creation));
    ///
    /// let citations = client
    ///     .citations_where(&Identifier::doi("10.1016/j.marpol.2021.104855"), &recent)
    ///     .await?;
    /// # Ok(())
    /// # }
    /// ```
    pub async fn citations_where(&self, id: &Identifier, refine: &Refine) -> Result<Vec<Citation>> {
        self.get(
            &format!("{}/citations/{}", Self::INDEX, encode(id)),
            &refine.params(),
            "citations",
        )
        .await
    }

    /// The works `id` cites, as OpenCitations indexed them.
    ///
    /// This is the same direction as crossref's `reference[]` and a useful
    /// cross-check on it: OpenCitations resolves references to identifiers,
    /// where a crossref reference may be an unstructured string.
    pub async fn references(&self, id: &Identifier) -> Result<Vec<Citation>> {
        self.references_where(id, &Refine::new()).await
    }

    /// The works `id` cites, narrowed and ordered by the api.
    pub async fn references_where(
        &self,
        id: &Identifier,
        refine: &Refine,
    ) -> Result<Vec<Citation>> {
        self.get(
            &format!("{}/references/{}", Self::INDEX, encode(id)),
            &refine.params(),
            "references",
        )
        .await
    }

    /// One citation, by its own identifier.
    ///
    /// [`None`] when OpenCitations holds no such citation.
    pub async fn citation(&self, oci: &Oci) -> Result<Option<Citation>> {
        let found: Vec<Citation> = self
            .get(
                &format!("{}/citation/{oci}", Self::INDEX),
                &[],
                "the citation",
            )
            .await?;
        Ok(found.into_iter().next())
    }

    /// How many works cite `id`.
    ///
    /// Cheaper than counting [`OpenCitations::citations`], which fetches every
    /// edge to do it.
    pub async fn citation_count(&self, id: &Identifier) -> Result<u64> {
        self.count(&format!("{}/citation-count/{}", Self::INDEX, encode(id)))
            .await
    }

    /// How many works `id` cites.
    pub async fn reference_count(&self, id: &Identifier) -> Result<u64> {
        self.count(&format!("{}/reference-count/{}", Self::INDEX, encode(id)))
            .await
    }

    /// How many citations everything published in a venue has received.
    ///
    /// Addressed by ISSN — `Identifier::issn("0308-597X")`.
    pub async fn venue_citation_count(&self, issn: &Identifier) -> Result<u64> {
        self.count(&format!(
            "{}/venue-citation-count/{}",
            Self::INDEX,
            encode(issn)
        ))
        .await
    }

    /// The bibliographic records behind any number of identifiers.
    ///
    /// Ids are batched rather than asked about one at a time, which is worth
    /// doing: a batch is one slot of the rate limit rather than one each. More
    /// than [`Self::META_BATCH`] of them is split across several requests,
    /// paced like any other.
    ///
    /// Records come back in whatever order Meta answers in, and an id it holds
    /// nothing about is simply absent — so match the results up by
    /// [`Metadata::id`] rather than by position. An empty slice makes no
    /// request.
    ///
    /// ```no_run
    /// # use opencitations_client::{Identifier, OpenCitations};
    /// # async fn run() -> Result<(), opencitations_client::Error> {
    /// # let client = OpenCitations::builder().build()?;
    /// let records = client
    ///     .metadata(&[
    ///         Identifier::doi("10.1016/j.marpol.2021.104855"),
    ///         Identifier::doi("10.1038/nature12373"),
    ///     ])
    ///     .await?;
    /// # Ok(())
    /// # }
    /// ```
    pub async fn metadata(&self, ids: &[Identifier]) -> Result<Vec<Metadata>> {
        let mut records = Vec::with_capacity(ids.len());

        for batch in ids.chunks(Self::META_BATCH) {
            // Meta joins several ids with a double underscore
            let joined = batch.iter().map(encode).collect::<Vec<_>>().join("__");

            let found: Vec<Metadata> = self
                .get(
                    &format!("{}/metadata/{joined}", Self::META),
                    &[],
                    "the metadata",
                )
                .await?;
            records.extend(found);
        }

        Ok(records)
    }

    /// Reads one of the count routes, which answer with a list holding one
    /// object holding the number as a string.
    async fn count(&self, route: &str) -> Result<u64> {
        let counts: Vec<Count> = self.get(route, &[], "the count").await?;
        let Some(count) = counts.first() else {
            // an entity it holds nothing about, which is a count of zero
            return Ok(0);
        };
        count.count.trim().parse().map_err(|_| Error::Api {
            status: 200,
            body: format!("`{}` is not a count", count.count),
        })
    }

    /// Sends a request, retrying a `429`, and reads the body as `T`.
    async fn get<T: DeserializeOwned>(
        &self,
        route: &str,
        params: &[(&'static str, String)],
        what: &'static str,
    ) -> Result<T> {
        let url = format!("{}/{route}", self.base_url);

        for attempt in 1.. {
            self.limiter.acquire().await;
            tracing::debug!(%url, attempt, "requesting");

            let response = self.client.get(&url).query(params).send().await?;
            let status = response.status();

            if status == StatusCode::TOO_MANY_REQUESTS {
                if attempt > self.max_retries {
                    return Err(Error::RateLimited { attempts: attempt });
                }
                // back off for everything sharing the limiter, not just this
                // request -- the rate we were pacing at was wrong
                let delay = retry_after(response.headers())
                    .unwrap_or_else(|| Duration::from_secs(u64::from(attempt)));
                self.limiter.back_off(delay);
                continue;
            }

            let body = response.text().await?;
            if !status.is_success() {
                let body = body.trim().to_string();
                // an identifier it could not even parse answers `404` with a
                // line of plain text rather than json
                return Err(if status == StatusCode::NOT_FOUND {
                    Error::NoSuchOperation { message: body }
                } else {
                    Error::Api {
                        status: status.as_u16(),
                        body,
                    }
                });
            }

            return serde_json::from_str(&body).map_err(|source| Error::Malformed { what, source });
        }

        unreachable!("the retry loop returns or continues")
    }
}

/// Percent-encodes an identifier for the path it goes into.
fn encode(id: &Identifier) -> String {
    utf8_percent_encode(&id.to_string(), IN_PATH).to_string()
}

/// Builds an [`OpenCitations`] client.
///
/// Every option has a working default, so the shortest form is
/// `OpenCitations::builder().build()?`.
#[derive(Debug, Default)]
pub struct OpenCitationsBuilder {
    user_agent: Option<String>,
    token: Option<String>,
    base_url: Option<String>,
    rate_limit: Option<RateLimit>,
    max_retries: Option<u32>,
}

impl OpenCitationsBuilder {
    /// How many times a `429` is retried, when the builder was not told
    /// otherwise.
    const DEFAULT_MAX_RETRIES: u32 = 3;

    /// Constructs a new `OpenCitationsBuilder`.
    pub fn new() -> Self {
        Self::default()
    }

    /// An access token from [opencitations.net/accesstoken](https://opencitations.net/accesstoken),
    /// sent as the `authorization` header.
    ///
    /// Optional — every route works without one — but it raises the rate limit
    /// and is how OpenCitations tells its users apart. Getting one is free and
    /// wants only an email address.
    pub fn token<'a>(mut self, token: impl Into<Option<&'a str>>) -> Self {
        self.token = token.into().map(str::to_owned);
        self
    }

    /// Identify the caller in the `User-Agent` header.
    ///
    /// Not required by OpenCitations, unlike crossref's polite pool, but it is
    /// the courtesy that lets them tell a misbehaving script from a busy one.
    /// Defaults to this crate's name and version.
    pub fn user_agent<'a>(mut self, user_agent: impl Into<Option<&'a str>>) -> Self {
        self.user_agent = user_agent.into().map(str::to_owned);
        self
    }

    /// Send requests somewhere other than `https://api.opencitations.net`.
    ///
    /// Routes are appended verbatim, so the url must not end in a `/`. Mainly
    /// useful for pointing the client at a mock or a proxy in tests.
    pub fn base_url<'a>(mut self, base_url: impl Into<Option<&'a str>>) -> Self {
        self.base_url = base_url.into().map(str::to_owned);
        self
    }

    /// The rate to pace requests at. Defaults to [`RateLimit::DOCUMENTED`].
    ///
    /// The documented limit is per ip address rather than per client, so lower
    /// this when something else on the same machine is querying too.
    pub fn rate_limit(mut self, rate_limit: impl Into<Option<RateLimit>>) -> Self {
        self.rate_limit = rate_limit.into();
        self
    }

    /// How many times a `429` is retried before giving up with
    /// [`Error::RateLimited`]. Defaults to 3; `0` disables retrying.
    pub fn max_retries(mut self, max_retries: impl Into<Option<u32>>) -> Self {
        self.max_retries = max_retries.into();
        self
    }

    /// Returns an [`OpenCitations`] using this configuration.
    ///
    /// # Errors
    ///
    /// Fails if a header cannot be built from what it was given, or if the TLS
    /// backend cannot be initialized.
    pub fn build(self) -> Result<OpenCitations> {
        use reqwest::header;

        let mut headers = header::HeaderMap::new();
        let agent = self.user_agent.unwrap_or_else(|| {
            format!(
                "{}/{} ({})",
                env!("CARGO_PKG_NAME"),
                env!("CARGO_PKG_VERSION"),
                env!("CARGO_PKG_REPOSITORY")
            )
        });
        headers.insert(
            header::USER_AGENT,
            header::HeaderValue::from_str(&agent).map_err(|_| Error::Config {
                msg: format!("failed to create User-Agent header for `{agent}`"),
            })?,
        );
        if let Some(token) = &self.token {
            headers.insert(
                header::AUTHORIZATION,
                header::HeaderValue::from_str(token).map_err(|_| Error::Config {
                    msg: "failed to create authorization header for the access token".to_string(),
                })?,
            );
        }

        let client = Client::builder()
            .default_headers(headers)
            .build()
            .map_err(|_| Error::Config {
                msg: "failed to initialize TLS backend".to_string(),
            })?;

        Ok(OpenCitations {
            base_url: self
                .base_url
                .unwrap_or_else(|| OpenCitations::BASE_URL.to_string()),
            client,
            limiter: Arc::new(Limiter::new(self.rate_limit.unwrap_or_default())),
            max_retries: self.max_retries.unwrap_or(Self::DEFAULT_MAX_RETRIES),
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn a_default_client_talks_to_opencitations_at_the_documented_rate() {
        let client = OpenCitations::builder().build().unwrap();
        assert_eq!(client.base_url(), "https://api.opencitations.net");
        assert_eq!(client.rate_limit(), RateLimit::DOCUMENTED);
    }

    /// The `:` and the `/` every identifier is built from stay readable; the
    /// characters that would end the path do not.
    #[test]
    fn an_identifier_is_encoded_only_where_it_has_to_be() {
        assert_eq!(
            encode(&Identifier::doi("10.1016/j.marpol.2021.104855")),
            "doi:10.1016/j.marpol.2021.104855"
        );
        assert_eq!(
            encode(&Identifier::doi("10.1000/a b?c#d")),
            "doi:10.1000/a%20b%3Fc%23d"
        );
    }

    #[tokio::test]
    async fn asking_for_no_metadata_makes_no_request() {
        // the base url is a black hole; reaching it would hang or fail
        let client = OpenCitations::builder()
            .base_url("http://127.0.0.1:1")
            .build()
            .unwrap();

        assert!(client.metadata(&[]).await.unwrap().is_empty());
    }
}