arxiv_client/lib.rs
1//! An async client for the [arXiv API](https://info.arxiv.org/help/api/index.html).
2//!
3//! arXiv exposes one route, `/api/query`, which answers with an
4//! [Atom feed](https://info.arxiv.org/help/api/user-manual.html#31-calling-the-api).
5//! A request asks for a [`Search`] expression, a list of [`ArticleId`]s, or
6//! both; everything else is paging and sorting.
7//!
8//! ## Usage
9//!
10//! ### Create an `Arxiv` client
11//!
12//! ```no_run
13//! # use arxiv_client::Arxiv;
14//! # fn run() -> Result<(), arxiv_client::Error> {
15//! let client = Arxiv::builder().build()?;
16//! # Ok(())
17//! # }
18//! ```
19//!
20//! arXiv has no api key and no polite pool, but it does ask callers to
21//! [identify themselves and to pace their requests](https://info.arxiv.org/help/api/tou.html).
22//! [`ArxivBuilder::contact`] does the first, and the client does the second on
23//! its own -- requests are serialized and spaced three seconds apart unless
24//! [`ArxivBuilder::interval`] says otherwise.
25//!
26//! ```no_run
27//! # use arxiv_client::Arxiv;
28//! # fn run() -> Result<(), arxiv_client::Error> {
29//! let client = Arxiv::builder().contact("you@example.com").build()?;
30//! # Ok(())
31//! # }
32//! ```
33//!
34//! ### Search
35//!
36//! ```no_run
37//! # use arxiv_client::{Arxiv, Search, SortBy};
38//! # async fn run() -> Result<(), arxiv_client::Error> {
39//! # let client = Arxiv::builder().build()?;
40//! let feed = client
41//! .search(
42//! Search::category("cs.LG")
43//! .and(Search::title("attention"))
44//! .into_query()
45//! .max_results(20)
46//! .sort_by(SortBy::SubmittedDate),
47//! )
48//! .await?;
49//!
50//! for entry in &feed.entries {
51//! println!("{} {}", entry.id, entry.title);
52//! }
53//! # Ok(())
54//! # }
55//! ```
56//!
57//! Free form text is read as a search of every field, which is the common case:
58//!
59//! ```no_run
60//! # use arxiv_client::Arxiv;
61//! # async fn run() -> Result<(), arxiv_client::Error> {
62//! # let client = Arxiv::builder().build()?;
63//! let feed = client.search("quantum error correction").await?;
64//! # Ok(())
65//! # }
66//! ```
67//!
68//! ### Get one article
69//!
70//! ```no_run
71//! # use arxiv_client::Arxiv;
72//! # async fn run() -> Result<(), arxiv_client::Error> {
73//! # let client = Arxiv::builder().build()?;
74//! let entry = client.entry("1706.03762").await?;
75//!
76//! println!("{}", entry.title);
77//! println!("{}", entry.to_bibtex());
78//! # Ok(())
79//! # }
80//! ```
81//!
82//! ### Page through a whole result set
83//!
84//! [`Arxiv::paginate`] returns an iterator over pages, threading `start`
85//! forward and pacing the requests. arXiv serves at most
86//! [`Query::MAX_RESULTS`] per request and stops answering past
87//! [`Query::MAX_REACHABLE`], so a search that has to reach further than that
88//! has to be split into narrower ones.
89//!
90//! ```no_run
91//! use arxiv_client::{Arxiv, AsyncIterator, Entry, Search};
92//! # async fn run() -> Result<(), arxiv_client::Error> {
93//! let client = Arxiv::builder().contact("you@example.com").build()?;
94//!
95//! let mut pages = client.paginate(Search::category("cs.LG").into_query());
96//! let mut all: Vec<Entry> = Vec::new();
97//! while let Some(page) = pages.next().await {
98//! all.extend(page?.entries);
99//! }
100//! # Ok(())
101//! # }
102//! ```
103//!
104//! Which can be flattened to the entries themselves:
105//!
106//! ```no_run
107//! use arxiv_client::{Arxiv, AsyncIterator, Search};
108//! # async fn run() -> Result<(), arxiv_client::Error> {
109//! # let client = Arxiv::builder().build()?;
110//! let mut entries = client
111//! .paginate(Search::category("cs.LG").into_query())
112//! .into_entry_iter();
113//!
114//! while let Some(entry) = entries.next().await {
115//! println!("{}", entry?.id);
116//! }
117//! # Ok(())
118//! # }
119//! ```
120
121#![warn(missing_docs)]
122
123mod error;
124/// client side rate limiting
125pub mod limit;
126/// provides the types to construct a request
127pub mod query;
128/// provides the response types of the arXiv api
129pub mod response;
130
131#[doc(inline)]
132pub use self::error::{Error, Result};
133
134#[doc(inline)]
135pub use self::limit::POLITE_INTERVAL;
136
137#[doc(inline)]
138pub use self::query::{ArticleId, DateField, Field, Op, Query, Search, SortBy, SortOrder};
139
140#[doc(inline)]
141pub use self::response::{Author, Category, Entry, Feed, Link, unwrap_lines};
142
143/// The async iterator trait implemented by [`PageIterator`] and
144/// [`EntryIterator`].
145///
146/// Re-exported so callers can page through results without depending on
147/// `async-iterator` directly.
148pub use async_iterator::Iterator as AsyncIterator;
149
150use crate::limit::{Limiter, retry_after};
151use reqwest::{Client, StatusCode};
152use std::str::FromStr;
153use std::sync::Arc;
154use std::time::Duration;
155
156/// A client for the arXiv api.
157///
158/// Cloning is cheap and shares the connection pool and the rate limiter, so
159/// concurrent callers should clone one client rather than build several --
160/// arXiv asks for one request at a time, and only a shared limiter can promise
161/// that.
162#[derive(Debug, Clone)]
163pub struct Arxiv {
164 /// use another base url than `export.arxiv.org`
165 base_url: String,
166 /// the reqwest client that handles the requests
167 client: Client,
168 /// paces requests, shared between clones
169 limiter: Arc<Limiter>,
170 /// how many times a `429`/`503` is retried before giving up
171 max_retries: u32,
172}
173
174impl Arxiv {
175 /// The host arXiv asks api traffic to go to.
176 const BASE_URL: &'static str = "https://export.arxiv.org";
177
178 /// The page size [`Arxiv::paginate`] uses when a query names none.
179 ///
180 /// Well under [`Query::MAX_RESULTS`]: arXiv answers a large slice slowly
181 /// enough that the request sometimes times out before it arrives.
182 pub const PAGE_SIZE: usize = 100;
183
184 /// Constructs a new [`ArxivBuilder`].
185 pub fn builder() -> ArxivBuilder {
186 ArxivBuilder::new()
187 }
188
189 /// The url every route is appended to, `https://export.arxiv.org` unless
190 /// [`ArxivBuilder::base_url`] said otherwise.
191 pub fn base_url(&self) -> &str {
192 &self.base_url
193 }
194
195 /// The gap this client leaves between two requests, shared with every clone
196 /// of it.
197 ///
198 /// [`POLITE_INTERVAL`] unless [`ArxivBuilder::interval`] said otherwise.
199 pub fn interval(&self) -> Duration {
200 self.limiter.interval()
201 }
202
203 /// One page of the articles matching `query`.
204 ///
205 /// # Errors
206 ///
207 /// [`Error::Api`] if arXiv answered with its error feed -- see
208 /// [`Query::route`] for the requests that are refused before they get that
209 /// far -- and [`Error::Xml`] if the body was not the feed arXiv documents.
210 pub async fn search(&self, query: impl Into<Query>) -> Result<Feed> {
211 let query = query.into();
212 let url = query.to_url(&self.base_url)?;
213 Feed::from_atom(&self.get(&url).await?)
214 }
215
216 /// The article with this identifier.
217 ///
218 /// Accepts every shape an identifier is written in -- see [`ArticleId`].
219 ///
220 /// # Errors
221 ///
222 /// [`Error::InvalidArticleId`] if `id` is not one, and [`Error::NotFound`]
223 /// if arXiv has no such article.
224 pub async fn entry(&self, id: &str) -> Result<Entry> {
225 let id = ArticleId::from_str(id)?;
226 let feed = self.search(Query::from(id.clone())).await?;
227
228 feed.entries
229 .into_iter()
230 .next()
231 .ok_or_else(|| Error::NotFound { id: id.to_string() })
232 }
233
234 /// The articles with these identifiers, in the order arXiv returns them.
235 ///
236 /// arXiv drops identifiers it does not know rather than reporting them, so
237 /// a shorter list than was asked for is how a bad identifier surfaces.
238 pub async fn entries<I>(&self, ids: I) -> Result<Vec<Entry>>
239 where
240 I: IntoIterator,
241 I::Item: AsRef<str>,
242 {
243 Ok(self.search(Query::ids(ids)?).await?.entries)
244 }
245
246 /// Pages through everything matching `query`, one [`Feed`] at a time.
247 ///
248 /// `query.max_results` becomes the page size, defaulting to
249 /// [`Arxiv::PAGE_SIZE`], and `query.start` the offset to begin at. Requests
250 /// are paced by the client's [interval](Arxiv::interval), so a long crawl
251 /// takes as long as arXiv asks it to.
252 pub fn paginate(&self, query: impl Into<Query>) -> PageIterator<'_> {
253 PageIterator::new(query.into(), self)
254 }
255
256 /// The pdf arXiv holds for this article.
257 ///
258 /// Buffered whole, so this is for one paper at a time rather than for
259 /// mirroring an archive.
260 ///
261 /// # Errors
262 ///
263 /// [`Error::NotFound`] if arXiv holds no pdf for the article -- which
264 /// happens for the few submitted in formats it cannot render.
265 pub async fn pdf(&self, id: &ArticleId) -> Result<Vec<u8>> {
266 let url = format!("{}/pdf/{}", self.base_url, id);
267 let response = self.send(&url).await?;
268
269 if response.status() == StatusCode::NOT_FOUND {
270 return Err(Error::NotFound { id: id.to_string() });
271 }
272 Ok(response.error_for_status()?.bytes().await?.to_vec())
273 }
274
275 /// Sends a `GET` and reads the body as text.
276 async fn get(&self, url: &str) -> Result<String> {
277 let response = self.send(url).await?.error_for_status()?;
278 Ok(response.text().await?)
279 }
280
281 /// Sends a `GET`, pacing it against the interval and retrying a refusal.
282 ///
283 /// Waits for a slot from the shared [`Limiter`] first. A `429` or `503`
284 /// means arXiv is refusing the load it is being offered, so the whole
285 /// client backs off -- for as long as `retry-after` asks, or exponentially
286 /// from the interval -- and the request is sent again up to
287 /// [`ArxivBuilder::max_retries`] times.
288 async fn send(&self, url: &str) -> Result<reqwest::Response> {
289 let mut attempt = 0u32;
290 loop {
291 self.limiter.acquire().await;
292 tracing::debug!(%url, attempt, "arxiv request");
293
294 let response = self.client.get(url).send().await?;
295 let status = response.status();
296
297 if status != StatusCode::TOO_MANY_REQUESTS && status != StatusCode::SERVICE_UNAVAILABLE
298 {
299 return Ok(response);
300 }
301
302 let interval = self.limiter.interval();
303 let delay = retry_after(response.headers())
304 .unwrap_or_else(|| interval * 2u32.pow(attempt.min(5)));
305 self.limiter.back_off(delay);
306 attempt += 1;
307
308 if attempt > self.max_retries {
309 tracing::warn!(%url, attempt, "arxiv refused the request, retries exhausted");
310 return Err(Error::RateLimited {
311 attempts: attempt,
312 interval,
313 });
314 }
315 tracing::debug!(%url, attempt, ?delay, "arxiv refused the request, retrying");
316 }
317 }
318}
319
320/// An `ArxivBuilder` can be used to create an [`Arxiv`] with additional config.
321///
322/// ```no_run
323/// # use arxiv_client::Arxiv;
324/// # fn run() -> Result<(), arxiv_client::Error> {
325/// let client = Arxiv::builder()
326/// .contact("you@example.com")
327/// .max_retries(5)
328/// .build()?;
329/// # Ok(())
330/// # }
331/// ```
332///
333/// Every setter accepts `Option`, so options that are themselves optional -- a
334/// CLI flag, a config field -- can be passed straight through:
335///
336/// ```no_run
337/// # use arxiv_client::Arxiv;
338/// # fn run(email: Option<String>) -> Result<(), arxiv_client::Error> {
339/// let client = Arxiv::builder().contact(email.as_deref()).build()?;
340/// # Ok(())
341/// # }
342/// ```
343#[derive(Debug, Default)]
344pub struct ArxivBuilder {
345 /// identifies the caller to arXiv, as its terms of use ask
346 user_agent: Option<String>,
347 /// use a different base url than [`Arxiv::BASE_URL`]
348 base_url: Option<String>,
349 /// the gap to leave between two requests
350 interval: Option<Duration>,
351 /// how many times a `429`/`503` is retried
352 max_retries: Option<u32>,
353}
354
355impl ArxivBuilder {
356 /// How many times a refusal is retried before [`Error::RateLimited`] is
357 /// returned, when the builder was not told otherwise.
358 const DEFAULT_MAX_RETRIES: u32 = 3;
359
360 /// Constructs a new `ArxivBuilder`.
361 pub fn new() -> ArxivBuilder {
362 ArxivBuilder::default()
363 }
364
365 /// Identify yourself by email, as arXiv's
366 /// [terms of use](https://info.arxiv.org/help/api/tou.html) ask.
367 ///
368 /// Sends a `User-Agent` of the shape
369 /// `arxiv-client/0.1.0 (https://github.com/…; mailto:you@example.com)`, so
370 /// arXiv can get in touch before it blocks a script that misbehaves. There
371 /// is no faster pool to be let into -- this is etiquette rather than
372 /// currency.
373 ///
374 /// Use [`ArxivBuilder::user_agent`] instead to send your own application's
375 /// name; include `mailto:<email>` in it either way.
376 pub fn contact<'a>(mut self, email: impl Into<Option<&'a str>>) -> Self {
377 if let Some(email) = email.into() {
378 self.user_agent = Some(format!(
379 "{}/{} ({}; mailto:{})",
380 env!("CARGO_PKG_NAME"),
381 env!("CARGO_PKG_VERSION"),
382 env!("CARGO_PKG_REPOSITORY"),
383 email
384 ));
385 }
386 self
387 }
388
389 /// Set the user agent directly.
390 pub fn user_agent<'a>(mut self, user_agent: impl Into<Option<&'a str>>) -> Self {
391 self.user_agent = user_agent.into().map(str::to_owned);
392 self
393 }
394
395 /// Send requests somewhere other than `https://export.arxiv.org`.
396 ///
397 /// Routes are appended verbatim, so the url must not end in a `/`. Mainly
398 /// useful for pointing the client at a mock or a proxy in tests.
399 pub fn base_url<'a>(mut self, base_url: impl Into<Option<&'a str>>) -> Self {
400 self.base_url = base_url.into().map(str::to_owned);
401 self
402 }
403
404 /// The gap to leave between two requests.
405 ///
406 /// Defaults to [`POLITE_INTERVAL`], the three seconds
407 /// [arXiv's user manual asks for](https://info.arxiv.org/help/api/user-manual.html).
408 /// Shortening it is between you and arXiv; lengthening it is worth doing
409 /// for a sustained crawl, which arXiv starts answering `503` to well inside
410 /// that rate.
411 pub fn interval(mut self, interval: impl Into<Option<Duration>>) -> Self {
412 self.interval = interval.into();
413 self
414 }
415
416 /// How many times a `429`/`503` is retried before giving up with
417 /// [`Error::RateLimited`]. Defaults to 3; `0` disables retrying.
418 pub fn max_retries(mut self, max_retries: impl Into<Option<u32>>) -> Self {
419 self.max_retries = max_retries.into();
420 self
421 }
422
423 /// Returns an [`Arxiv`] that uses this builder's configuration.
424 ///
425 /// # Errors
426 ///
427 /// Fails if the user agent cannot be sent as a header, or if the TLS
428 /// backend cannot be initialized -- see [`reqwest::ClientBuilder::build`].
429 pub fn build(self) -> Result<Arxiv> {
430 use reqwest::header;
431 let mut headers = header::HeaderMap::new();
432 if let Some(agent) = &self.user_agent {
433 headers.insert(
434 header::USER_AGENT,
435 header::HeaderValue::from_str(agent).map_err(|_| Error::Config {
436 msg: format!("failed to create User Agent header for `{agent}`"),
437 })?,
438 );
439 }
440 let client = reqwest::Client::builder()
441 .default_headers(headers)
442 .build()
443 .map_err(|_| Error::Config {
444 msg: "failed to initialize TLS backend".to_string(),
445 })?;
446
447 Ok(Arxiv {
448 base_url: self.base_url.unwrap_or_else(|| Arxiv::BASE_URL.to_string()),
449 client,
450 limiter: Arc::new(Limiter::new(
451 self.interval.unwrap_or(limit::POLITE_INTERVAL),
452 )),
453 max_retries: self.max_retries.unwrap_or(Self::DEFAULT_MAX_RETRIES),
454 })
455 }
456}
457
458/// Pages through a query's whole result set, one [`Feed`] at a time.
459///
460/// Created by [`Arxiv::paginate`].
461pub struct PageIterator<'a> {
462 /// the query, with `start` moved forward after each page
463 query: Query,
464 /// performs each request
465 client: &'a Arxiv,
466 /// whether the iteration is over
467 finished: bool,
468}
469
470impl<'a> PageIterator<'a> {
471 /// Creates an iterator that pages `query` through `client`.
472 pub fn new(mut query: Query, client: &'a Arxiv) -> Self {
473 query.start.get_or_insert(0);
474 query.max_results.get_or_insert(Arxiv::PAGE_SIZE);
475 Self {
476 query,
477 client,
478 finished: false,
479 }
480 }
481
482 /// Flatten the paged results into an iterator over individual [`Entry`]
483 /// items.
484 ///
485 /// Pages are fetched lazily, one request at a time, as the buffer drains.
486 pub fn into_entry_iter(self) -> EntryIterator<'a> {
487 EntryIterator {
488 pages: self,
489 buffer: Vec::new().into_iter(),
490 }
491 }
492
493 /// Ends the iteration and reports `error`.
494 fn fail(&mut self, error: Error) -> Option<Result<Feed>> {
495 self.finished = true;
496 Some(Err(error))
497 }
498}
499
500impl async_iterator::Iterator for PageIterator<'_> {
501 type Item = Result<Feed>;
502
503 async fn next(&mut self) -> Option<Self::Item> {
504 if self.finished {
505 return None;
506 }
507 let start = self.query.start.unwrap_or(0);
508
509 // arXiv intermittently serves an empty page in the middle of a result
510 // set. Ending here would look exactly like having reached the end, so
511 // the page is asked for again before that is believed.
512 let mut attempts = 0;
513 let feed = loop {
514 attempts += 1;
515 let feed = match self.client.search(self.query.clone()).await {
516 Ok(feed) => feed,
517 Err(error) => return self.fail(error),
518 };
519
520 if !feed.entries.is_empty() {
521 break feed;
522 }
523 if start >= feed.total_results {
524 // the end, and the reason the last page had nothing in it
525 self.finished = true;
526 return None;
527 }
528 if attempts > self.client.max_retries {
529 return self.fail(Error::EmptyPage {
530 start,
531 total_results: feed.total_results,
532 attempts,
533 });
534 }
535 tracing::debug!(start, attempts, "arxiv served an empty page, retrying");
536 };
537
538 let next_start = start + feed.entries.len();
539 self.query.start = Some(next_start);
540 if next_start >= feed.total_results.min(Query::MAX_REACHABLE) {
541 self.finished = true;
542 }
543 Some(Ok(feed))
544 }
545}
546
547/// Yields individual [`Entry`] items across the paged [`Feed`]s.
548///
549/// Created by [`PageIterator::into_entry_iter`].
550pub struct EntryIterator<'a> {
551 pages: PageIterator<'a>,
552 buffer: std::vec::IntoIter<Entry>,
553}
554
555impl async_iterator::Iterator for EntryIterator<'_> {
556 type Item = Result<Entry>;
557
558 async fn next(&mut self) -> Option<Self::Item> {
559 loop {
560 if let Some(entry) = self.buffer.next() {
561 return Some(Ok(entry));
562 }
563 // `PageIterator` stops after an error and at the end of the
564 // results, so this terminates
565 match self.pages.next().await? {
566 Ok(page) => self.buffer = page.entries.into_iter(),
567 Err(error) => return Some(Err(error)),
568 }
569 }
570 }
571}