Skip to main content

arxiv_client/
lib.rs

1//! An async client for the [arXiv API](https://info.arxiv.org/help/api/index.html).
2//!
3//! arXiv exposes one route, `/api/query`, which answers with an
4//! [Atom feed](https://info.arxiv.org/help/api/user-manual.html#31-calling-the-api).
5//! A request asks for a [`Search`] expression, a list of [`ArticleId`]s, or
6//! both; everything else is paging and sorting.
7//!
8//! ## Usage
9//!
10//! ### Create an `Arxiv` client
11//!
12//! ```no_run
13//! # use arxiv_client::Arxiv;
14//! # fn run() -> Result<(), arxiv_client::Error> {
15//! let client = Arxiv::builder().build()?;
16//! # Ok(())
17//! # }
18//! ```
19//!
20//! arXiv has no api key and no polite pool, but it does ask callers to
21//! [identify themselves and to pace their requests](https://info.arxiv.org/help/api/tou.html).
22//! [`ArxivBuilder::contact`] does the first, and the client does the second on
23//! its own -- requests are serialized and spaced three seconds apart unless
24//! [`ArxivBuilder::interval`] says otherwise.
25//!
26//! ```no_run
27//! # use arxiv_client::Arxiv;
28//! # fn run() -> Result<(), arxiv_client::Error> {
29//! let client = Arxiv::builder().contact("you@example.com").build()?;
30//! # Ok(())
31//! # }
32//! ```
33//!
34//! ### Search
35//!
36//! ```no_run
37//! # use arxiv_client::{Arxiv, Search, SortBy};
38//! # async fn run() -> Result<(), arxiv_client::Error> {
39//! # let client = Arxiv::builder().build()?;
40//! let feed = client
41//!     .search(
42//!         Search::category("cs.LG")
43//!             .and(Search::title("attention"))
44//!             .into_query()
45//!             .max_results(20)
46//!             .sort_by(SortBy::SubmittedDate),
47//!     )
48//!     .await?;
49//!
50//! for entry in &feed.entries {
51//!     println!("{} {}", entry.id, entry.title);
52//! }
53//! # Ok(())
54//! # }
55//! ```
56//!
57//! Free form text is read as a search of every field, which is the common case:
58//!
59//! ```no_run
60//! # use arxiv_client::Arxiv;
61//! # async fn run() -> Result<(), arxiv_client::Error> {
62//! # let client = Arxiv::builder().build()?;
63//! let feed = client.search("quantum error correction").await?;
64//! # Ok(())
65//! # }
66//! ```
67//!
68//! ### Get one article
69//!
70//! ```no_run
71//! # use arxiv_client::Arxiv;
72//! # async fn run() -> Result<(), arxiv_client::Error> {
73//! # let client = Arxiv::builder().build()?;
74//! let entry = client.entry("1706.03762").await?;
75//!
76//! println!("{}", entry.title);
77//! println!("{}", entry.to_bibtex());
78//! # Ok(())
79//! # }
80//! ```
81//!
82//! ### Page through a whole result set
83//!
84//! [`Arxiv::paginate`] returns an iterator over pages, threading `start`
85//! forward and pacing the requests. arXiv serves at most
86//! [`Query::MAX_RESULTS`] per request and stops answering past
87//! [`Query::MAX_REACHABLE`], so a search that has to reach further than that
88//! has to be split into narrower ones.
89//!
90//! ```no_run
91//! use arxiv_client::{Arxiv, AsyncIterator, Entry, Search};
92//! # async fn run() -> Result<(), arxiv_client::Error> {
93//! let client = Arxiv::builder().contact("you@example.com").build()?;
94//!
95//! let mut pages = client.paginate(Search::category("cs.LG").into_query());
96//! let mut all: Vec<Entry> = Vec::new();
97//! while let Some(page) = pages.next().await {
98//!     all.extend(page?.entries);
99//! }
100//! # Ok(())
101//! # }
102//! ```
103//!
104//! Which can be flattened to the entries themselves:
105//!
106//! ```no_run
107//! use arxiv_client::{Arxiv, AsyncIterator, Search};
108//! # async fn run() -> Result<(), arxiv_client::Error> {
109//! # let client = Arxiv::builder().build()?;
110//! let mut entries = client
111//!     .paginate(Search::category("cs.LG").into_query())
112//!     .into_entry_iter();
113//!
114//! while let Some(entry) = entries.next().await {
115//!     println!("{}", entry?.id);
116//! }
117//! # Ok(())
118//! # }
119//! ```
120
121#![warn(missing_docs)]
122
123mod error;
124/// client side rate limiting
125pub mod limit;
126/// provides the types to construct a request
127pub mod query;
128/// provides the response types of the arXiv api
129pub mod response;
130
131#[doc(inline)]
132pub use self::error::{Error, Result};
133
134#[doc(inline)]
135pub use self::limit::POLITE_INTERVAL;
136
137#[doc(inline)]
138pub use self::query::{ArticleId, DateField, Field, Op, Query, Search, SortBy, SortOrder};
139
140#[doc(inline)]
141pub use self::response::{Author, Category, Entry, Feed, Link, unwrap_lines};
142
143/// The async iterator trait implemented by [`PageIterator`] and
144/// [`EntryIterator`].
145///
146/// Re-exported so callers can page through results without depending on
147/// `async-iterator` directly.
148pub use async_iterator::Iterator as AsyncIterator;
149
150use crate::limit::{Limiter, retry_after};
151use reqwest::{Client, StatusCode};
152use std::str::FromStr;
153use std::sync::Arc;
154use std::time::Duration;
155
156/// A client for the arXiv api.
157///
158/// Cloning is cheap and shares the connection pool and the rate limiter, so
159/// concurrent callers should clone one client rather than build several --
160/// arXiv asks for one request at a time, and only a shared limiter can promise
161/// that.
162#[derive(Debug, Clone)]
163pub struct Arxiv {
164    /// use another base url than `export.arxiv.org`
165    base_url: String,
166    /// the reqwest client that handles the requests
167    client: Client,
168    /// paces requests, shared between clones
169    limiter: Arc<Limiter>,
170    /// how many times a `429`/`503` is retried before giving up
171    max_retries: u32,
172}
173
174impl Arxiv {
175    /// The host arXiv asks api traffic to go to.
176    const BASE_URL: &'static str = "https://export.arxiv.org";
177
178    /// The page size [`Arxiv::paginate`] uses when a query names none.
179    ///
180    /// Well under [`Query::MAX_RESULTS`]: arXiv answers a large slice slowly
181    /// enough that the request sometimes times out before it arrives.
182    pub const PAGE_SIZE: usize = 100;
183
184    /// Constructs a new [`ArxivBuilder`].
185    pub fn builder() -> ArxivBuilder {
186        ArxivBuilder::new()
187    }
188
189    /// The url every route is appended to, `https://export.arxiv.org` unless
190    /// [`ArxivBuilder::base_url`] said otherwise.
191    pub fn base_url(&self) -> &str {
192        &self.base_url
193    }
194
195    /// The gap this client leaves between two requests, shared with every clone
196    /// of it.
197    ///
198    /// [`POLITE_INTERVAL`] unless [`ArxivBuilder::interval`] said otherwise.
199    pub fn interval(&self) -> Duration {
200        self.limiter.interval()
201    }
202
203    /// One page of the articles matching `query`.
204    ///
205    /// # Errors
206    ///
207    /// [`Error::Api`] if arXiv answered with its error feed -- see
208    /// [`Query::route`] for the requests that are refused before they get that
209    /// far -- and [`Error::Xml`] if the body was not the feed arXiv documents.
210    pub async fn search(&self, query: impl Into<Query>) -> Result<Feed> {
211        let query = query.into();
212        let url = query.to_url(&self.base_url)?;
213        Feed::from_atom(&self.get(&url).await?)
214    }
215
216    /// The article with this identifier.
217    ///
218    /// Accepts every shape an identifier is written in -- see [`ArticleId`].
219    ///
220    /// # Errors
221    ///
222    /// [`Error::InvalidArticleId`] if `id` is not one, and [`Error::NotFound`]
223    /// if arXiv has no such article.
224    pub async fn entry(&self, id: &str) -> Result<Entry> {
225        let id = ArticleId::from_str(id)?;
226        let feed = self.search(Query::from(id.clone())).await?;
227
228        feed.entries
229            .into_iter()
230            .next()
231            .ok_or_else(|| Error::NotFound { id: id.to_string() })
232    }
233
234    /// The articles with these identifiers, in the order arXiv returns them.
235    ///
236    /// arXiv drops identifiers it does not know rather than reporting them, so
237    /// a shorter list than was asked for is how a bad identifier surfaces.
238    pub async fn entries<I>(&self, ids: I) -> Result<Vec<Entry>>
239    where
240        I: IntoIterator,
241        I::Item: AsRef<str>,
242    {
243        Ok(self.search(Query::ids(ids)?).await?.entries)
244    }
245
246    /// Pages through everything matching `query`, one [`Feed`] at a time.
247    ///
248    /// `query.max_results` becomes the page size, defaulting to
249    /// [`Arxiv::PAGE_SIZE`], and `query.start` the offset to begin at. Requests
250    /// are paced by the client's [interval](Arxiv::interval), so a long crawl
251    /// takes as long as arXiv asks it to.
252    pub fn paginate(&self, query: impl Into<Query>) -> PageIterator<'_> {
253        PageIterator::new(query.into(), self)
254    }
255
256    /// The pdf arXiv holds for this article.
257    ///
258    /// Buffered whole, so this is for one paper at a time rather than for
259    /// mirroring an archive.
260    ///
261    /// # Errors
262    ///
263    /// [`Error::NotFound`] if arXiv holds no pdf for the article -- which
264    /// happens for the few submitted in formats it cannot render.
265    pub async fn pdf(&self, id: &ArticleId) -> Result<Vec<u8>> {
266        let url = format!("{}/pdf/{}", self.base_url, id);
267        let response = self.send(&url).await?;
268
269        if response.status() == StatusCode::NOT_FOUND {
270            return Err(Error::NotFound { id: id.to_string() });
271        }
272        Ok(response.error_for_status()?.bytes().await?.to_vec())
273    }
274
275    /// Sends a `GET` and reads the body as text.
276    async fn get(&self, url: &str) -> Result<String> {
277        let response = self.send(url).await?.error_for_status()?;
278        Ok(response.text().await?)
279    }
280
281    /// Sends a `GET`, pacing it against the interval and retrying a refusal.
282    ///
283    /// Waits for a slot from the shared [`Limiter`] first. A `429` or `503`
284    /// means arXiv is refusing the load it is being offered, so the whole
285    /// client backs off -- for as long as `retry-after` asks, or exponentially
286    /// from the interval -- and the request is sent again up to
287    /// [`ArxivBuilder::max_retries`] times.
288    async fn send(&self, url: &str) -> Result<reqwest::Response> {
289        let mut attempt = 0u32;
290        loop {
291            self.limiter.acquire().await;
292            tracing::debug!(%url, attempt, "arxiv request");
293
294            let response = self.client.get(url).send().await?;
295            let status = response.status();
296
297            if status != StatusCode::TOO_MANY_REQUESTS && status != StatusCode::SERVICE_UNAVAILABLE
298            {
299                return Ok(response);
300            }
301
302            let interval = self.limiter.interval();
303            let delay = retry_after(response.headers())
304                .unwrap_or_else(|| interval * 2u32.pow(attempt.min(5)));
305            self.limiter.back_off(delay);
306            attempt += 1;
307
308            if attempt > self.max_retries {
309                tracing::warn!(%url, attempt, "arxiv refused the request, retries exhausted");
310                return Err(Error::RateLimited {
311                    attempts: attempt,
312                    interval,
313                });
314            }
315            tracing::debug!(%url, attempt, ?delay, "arxiv refused the request, retrying");
316        }
317    }
318}
319
320/// An `ArxivBuilder` can be used to create an [`Arxiv`] with additional config.
321///
322/// ```no_run
323/// # use arxiv_client::Arxiv;
324/// # fn run() -> Result<(), arxiv_client::Error> {
325/// let client = Arxiv::builder()
326///     .contact("you@example.com")
327///     .max_retries(5)
328///     .build()?;
329/// # Ok(())
330/// # }
331/// ```
332///
333/// Every setter accepts `Option`, so options that are themselves optional -- a
334/// CLI flag, a config field -- can be passed straight through:
335///
336/// ```no_run
337/// # use arxiv_client::Arxiv;
338/// # fn run(email: Option<String>) -> Result<(), arxiv_client::Error> {
339/// let client = Arxiv::builder().contact(email.as_deref()).build()?;
340/// # Ok(())
341/// # }
342/// ```
343#[derive(Debug, Default)]
344pub struct ArxivBuilder {
345    /// identifies the caller to arXiv, as its terms of use ask
346    user_agent: Option<String>,
347    /// use a different base url than [`Arxiv::BASE_URL`]
348    base_url: Option<String>,
349    /// the gap to leave between two requests
350    interval: Option<Duration>,
351    /// how many times a `429`/`503` is retried
352    max_retries: Option<u32>,
353}
354
355impl ArxivBuilder {
356    /// How many times a refusal is retried before [`Error::RateLimited`] is
357    /// returned, when the builder was not told otherwise.
358    const DEFAULT_MAX_RETRIES: u32 = 3;
359
360    /// Constructs a new `ArxivBuilder`.
361    pub fn new() -> ArxivBuilder {
362        ArxivBuilder::default()
363    }
364
365    /// Identify yourself by email, as arXiv's
366    /// [terms of use](https://info.arxiv.org/help/api/tou.html) ask.
367    ///
368    /// Sends a `User-Agent` of the shape
369    /// `arxiv-client/0.1.0 (https://github.com/…; mailto:you@example.com)`, so
370    /// arXiv can get in touch before it blocks a script that misbehaves. There
371    /// is no faster pool to be let into -- this is etiquette rather than
372    /// currency.
373    ///
374    /// Use [`ArxivBuilder::user_agent`] instead to send your own application's
375    /// name; include `mailto:<email>` in it either way.
376    pub fn contact<'a>(mut self, email: impl Into<Option<&'a str>>) -> Self {
377        if let Some(email) = email.into() {
378            self.user_agent = Some(format!(
379                "{}/{} ({}; mailto:{})",
380                env!("CARGO_PKG_NAME"),
381                env!("CARGO_PKG_VERSION"),
382                env!("CARGO_PKG_REPOSITORY"),
383                email
384            ));
385        }
386        self
387    }
388
389    /// Set the user agent directly.
390    pub fn user_agent<'a>(mut self, user_agent: impl Into<Option<&'a str>>) -> Self {
391        self.user_agent = user_agent.into().map(str::to_owned);
392        self
393    }
394
395    /// Send requests somewhere other than `https://export.arxiv.org`.
396    ///
397    /// Routes are appended verbatim, so the url must not end in a `/`. Mainly
398    /// useful for pointing the client at a mock or a proxy in tests.
399    pub fn base_url<'a>(mut self, base_url: impl Into<Option<&'a str>>) -> Self {
400        self.base_url = base_url.into().map(str::to_owned);
401        self
402    }
403
404    /// The gap to leave between two requests.
405    ///
406    /// Defaults to [`POLITE_INTERVAL`], the three seconds
407    /// [arXiv's user manual asks for](https://info.arxiv.org/help/api/user-manual.html).
408    /// Shortening it is between you and arXiv; lengthening it is worth doing
409    /// for a sustained crawl, which arXiv starts answering `503` to well inside
410    /// that rate.
411    pub fn interval(mut self, interval: impl Into<Option<Duration>>) -> Self {
412        self.interval = interval.into();
413        self
414    }
415
416    /// How many times a `429`/`503` is retried before giving up with
417    /// [`Error::RateLimited`]. Defaults to 3; `0` disables retrying.
418    pub fn max_retries(mut self, max_retries: impl Into<Option<u32>>) -> Self {
419        self.max_retries = max_retries.into();
420        self
421    }
422
423    /// Returns an [`Arxiv`] that uses this builder's configuration.
424    ///
425    /// # Errors
426    ///
427    /// Fails if the user agent cannot be sent as a header, or if the TLS
428    /// backend cannot be initialized -- see [`reqwest::ClientBuilder::build`].
429    pub fn build(self) -> Result<Arxiv> {
430        use reqwest::header;
431        let mut headers = header::HeaderMap::new();
432        if let Some(agent) = &self.user_agent {
433            headers.insert(
434                header::USER_AGENT,
435                header::HeaderValue::from_str(agent).map_err(|_| Error::Config {
436                    msg: format!("failed to create User Agent header for `{agent}`"),
437                })?,
438            );
439        }
440        let client = reqwest::Client::builder()
441            .default_headers(headers)
442            .build()
443            .map_err(|_| Error::Config {
444                msg: "failed to initialize TLS backend".to_string(),
445            })?;
446
447        Ok(Arxiv {
448            base_url: self.base_url.unwrap_or_else(|| Arxiv::BASE_URL.to_string()),
449            client,
450            limiter: Arc::new(Limiter::new(
451                self.interval.unwrap_or(limit::POLITE_INTERVAL),
452            )),
453            max_retries: self.max_retries.unwrap_or(Self::DEFAULT_MAX_RETRIES),
454        })
455    }
456}
457
458/// Pages through a query's whole result set, one [`Feed`] at a time.
459///
460/// Created by [`Arxiv::paginate`].
461pub struct PageIterator<'a> {
462    /// the query, with `start` moved forward after each page
463    query: Query,
464    /// performs each request
465    client: &'a Arxiv,
466    /// whether the iteration is over
467    finished: bool,
468}
469
470impl<'a> PageIterator<'a> {
471    /// Creates an iterator that pages `query` through `client`.
472    pub fn new(mut query: Query, client: &'a Arxiv) -> Self {
473        query.start.get_or_insert(0);
474        query.max_results.get_or_insert(Arxiv::PAGE_SIZE);
475        Self {
476            query,
477            client,
478            finished: false,
479        }
480    }
481
482    /// Flatten the paged results into an iterator over individual [`Entry`]
483    /// items.
484    ///
485    /// Pages are fetched lazily, one request at a time, as the buffer drains.
486    pub fn into_entry_iter(self) -> EntryIterator<'a> {
487        EntryIterator {
488            pages: self,
489            buffer: Vec::new().into_iter(),
490        }
491    }
492
493    /// Ends the iteration and reports `error`.
494    fn fail(&mut self, error: Error) -> Option<Result<Feed>> {
495        self.finished = true;
496        Some(Err(error))
497    }
498}
499
500impl async_iterator::Iterator for PageIterator<'_> {
501    type Item = Result<Feed>;
502
503    async fn next(&mut self) -> Option<Self::Item> {
504        if self.finished {
505            return None;
506        }
507        let start = self.query.start.unwrap_or(0);
508
509        // arXiv intermittently serves an empty page in the middle of a result
510        // set. Ending here would look exactly like having reached the end, so
511        // the page is asked for again before that is believed.
512        let mut attempts = 0;
513        let feed = loop {
514            attempts += 1;
515            let feed = match self.client.search(self.query.clone()).await {
516                Ok(feed) => feed,
517                Err(error) => return self.fail(error),
518            };
519
520            if !feed.entries.is_empty() {
521                break feed;
522            }
523            if start >= feed.total_results {
524                // the end, and the reason the last page had nothing in it
525                self.finished = true;
526                return None;
527            }
528            if attempts > self.client.max_retries {
529                return self.fail(Error::EmptyPage {
530                    start,
531                    total_results: feed.total_results,
532                    attempts,
533                });
534            }
535            tracing::debug!(start, attempts, "arxiv served an empty page, retrying");
536        };
537
538        let next_start = start + feed.entries.len();
539        self.query.start = Some(next_start);
540        if next_start >= feed.total_results.min(Query::MAX_REACHABLE) {
541            self.finished = true;
542        }
543        Some(Ok(feed))
544    }
545}
546
547/// Yields individual [`Entry`] items across the paged [`Feed`]s.
548///
549/// Created by [`PageIterator::into_entry_iter`].
550pub struct EntryIterator<'a> {
551    pages: PageIterator<'a>,
552    buffer: std::vec::IntoIter<Entry>,
553}
554
555impl async_iterator::Iterator for EntryIterator<'_> {
556    type Item = Result<Entry>;
557
558    async fn next(&mut self) -> Option<Self::Item> {
559        loop {
560            if let Some(entry) = self.buffer.next() {
561                return Some(Ok(entry));
562            }
563            // `PageIterator` stops after an error and at the end of the
564            // results, so this terminates
565            match self.pages.next().await? {
566                Ok(page) => self.buffer = page.entries.into_iter(),
567                Err(error) => return Some(Err(error)),
568            }
569        }
570    }
571}