Skip to main content

arxiv_client/query/
mod.rs

1//! The one request arXiv's api takes, and the pieces it is built from.
2//!
3//! arXiv has a single route, `/api/query`, which is asked for either a
4//! [`Search`] expression, a list of [`ArticleId`]s, or both -- with the
5//! identifiers narrowed by the search when both are given. Everything else is
6//! paging and sorting.
7
8use crate::error::{Error, Result};
9use std::fmt;
10use std::str::FromStr;
11
12/// percent-encoding of the arXiv query string
13pub(crate) mod encode;
14/// arXiv article identifiers
15pub mod id;
16/// the expression language arXiv reads out of `search_query`
17pub mod search;
18
19pub use self::id::ArticleId;
20pub use self::search::{DateField, Field, Op, Search};
21
22/// What the results are ordered by.
23#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
24pub enum SortBy {
25    /// how well the article matches the search
26    Relevance,
27    /// when the most recent version was submitted
28    LastUpdatedDate,
29    /// when the first version was submitted
30    SubmittedDate,
31}
32
33impl SortBy {
34    /// Every ordering arXiv offers, in declaration order.
35    pub const ALL: &'static [SortBy] = &[
36        SortBy::Relevance,
37        SortBy::LastUpdatedDate,
38        SortBy::SubmittedDate,
39    ];
40
41    /// the value arXiv knows this by
42    pub fn as_str(&self) -> &'static str {
43        match self {
44            SortBy::Relevance => "relevance",
45            SortBy::LastUpdatedDate => "lastUpdatedDate",
46            SortBy::SubmittedDate => "submittedDate",
47        }
48    }
49}
50
51impl fmt::Display for SortBy {
52    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
53        f.write_str(self.as_str())
54    }
55}
56
57impl FromStr for SortBy {
58    type Err = Error;
59
60    /// Reads either the spelling arXiv uses or the kebab-case one a command
61    /// line is more likely to be given.
62    fn from_str(value: &str) -> Result<Self> {
63        match value.to_ascii_lowercase().replace('-', "").as_str() {
64            "relevance" => Ok(SortBy::Relevance),
65            "lastupdateddate" | "lastupdated" => Ok(SortBy::LastUpdatedDate),
66            "submitteddate" | "submitted" => Ok(SortBy::SubmittedDate),
67            _ => Err(Error::Config {
68                msg: format!(
69                    "`{value}` is not something arxiv sorts by. Try one of: relevance, last-updated-date, submitted-date"
70                ),
71            }),
72        }
73    }
74}
75
76/// Which end of the ordering comes first.
77#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
78pub enum SortOrder {
79    /// smallest, oldest or least relevant first
80    Ascending,
81    /// largest, newest or most relevant first
82    Descending,
83}
84
85impl SortOrder {
86    /// Every order arXiv offers, in declaration order.
87    pub const ALL: &'static [SortOrder] = &[SortOrder::Ascending, SortOrder::Descending];
88
89    /// the value arXiv knows this by
90    pub fn as_str(&self) -> &'static str {
91        match self {
92            SortOrder::Ascending => "ascending",
93            SortOrder::Descending => "descending",
94        }
95    }
96}
97
98impl fmt::Display for SortOrder {
99    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
100        f.write_str(self.as_str())
101    }
102}
103
104impl FromStr for SortOrder {
105    type Err = Error;
106
107    fn from_str(value: &str) -> Result<Self> {
108        match value.to_ascii_lowercase().as_str() {
109            "ascending" | "asc" => Ok(SortOrder::Ascending),
110            "descending" | "desc" => Ok(SortOrder::Descending),
111            _ => Err(Error::Config {
112                msg: format!("`{value}` is not an order. Try one of: ascending, descending"),
113            }),
114        }
115    }
116}
117
118/// One request to arXiv's `/api/query` route.
119///
120/// ```
121/// # use arxiv_client::{Query, Search, SortBy};
122/// let query = Query::new(Search::category("cs.LG").and(Search::title("transformer")))
123///     .max_results(50)
124///     .sort_by(SortBy::SubmittedDate);
125/// ```
126///
127/// A [`Search`], a `&str` (read as a search of every field) and an
128/// [`ArticleId`] all convert into a `Query`, so the common cases need no
129/// builder at all.
130#[derive(Debug, Clone, Default, PartialEq, Eq)]
131pub struct Query {
132    /// the expression to match, sent as `search_query`
133    pub search: Option<Search>,
134    /// the articles to fetch, sent as `id_list`
135    ///
136    /// With a `search` beside it these are narrowed to the ones that also
137    /// match; on their own they are returned as they are.
138    pub id_list: Vec<ArticleId>,
139    /// the offset into the result set to start at, sent as `start`
140    pub start: Option<usize>,
141    /// how many results to return, sent as `max_results`
142    pub max_results: Option<usize>,
143    /// what to order the results by, sent as `sortBy`
144    pub sort_by: Option<SortBy>,
145    /// which end of that ordering comes first, sent as `sortOrder`
146    pub sort_order: Option<SortOrder>,
147}
148
149impl Query {
150    /// The most results
151    /// [arXiv's user manual](https://info.arxiv.org/help/api/user-manual.html)
152    /// says it serves in one request.
153    ///
154    /// [`Query::route`] refuses more, and [`Arxiv::paginate`](crate::Arxiv::paginate)
155    /// is how to reach the rest. The api does not police the number itself --
156    /// asked for 2100 it currently answers with 2100 -- but that is undocumented
157    /// behaviour of a request arXiv already describes as slow, and it gives out
158    /// entirely, with a `500`, not far above.
159    pub const MAX_RESULTS: usize = 2_000;
160
161    /// The deepest arXiv lets a query page.
162    ///
163    /// A `start` at or near this is answered with a `500`, whatever
164    /// `totalResults` said, so a search that has to reach further has to be
165    /// split into narrower ones -- by date range, or by category.
166    /// [`Arxiv::paginate`](crate::Arxiv::paginate) stops here rather than
167    /// walking into it.
168    pub const MAX_REACHABLE: usize = 30_000;
169
170    /// A query for everything matching `search`.
171    pub fn new(search: Search) -> Self {
172        Query {
173            search: Some(search),
174            ..Query::default()
175        }
176    }
177
178    /// A query for the articles with these identifiers.
179    ///
180    /// ```
181    /// # use arxiv_client::Query;
182    /// let query = Query::ids(["2101.00001", "hep-ex/0307015"])?;
183    /// # Ok::<(), arxiv_client::Error>(())
184    /// ```
185    pub fn ids<I>(ids: I) -> Result<Self>
186    where
187        I: IntoIterator,
188        I::Item: AsRef<str>,
189    {
190        let id_list = ids
191            .into_iter()
192            .map(|id| ArticleId::from_str(id.as_ref()))
193            .collect::<Result<Vec<_>>>()?;
194        Ok(Query {
195            id_list,
196            ..Query::default()
197        })
198    }
199
200    /// Narrow this query to articles matching `search` as well.
201    pub fn search(mut self, search: impl Into<Option<Search>>) -> Self {
202        self.search = search.into();
203        self
204    }
205
206    /// Add an article to the ones this query asks for.
207    pub fn id(mut self, id: ArticleId) -> Self {
208        self.id_list.push(id);
209        self
210    }
211
212    /// Start at the `start`th result rather than the first.
213    pub fn start(mut self, start: impl Into<Option<usize>>) -> Self {
214        self.start = start.into();
215        self
216    }
217
218    /// Return at most `max_results` results, at most [`Query::MAX_RESULTS`] of
219    /// them. Defaults to the 10 arXiv returns when not asked.
220    pub fn max_results(mut self, max_results: impl Into<Option<usize>>) -> Self {
221        self.max_results = max_results.into();
222        self
223    }
224
225    /// Order the results by `sort_by`. Defaults to arXiv's own relevance order.
226    pub fn sort_by(mut self, sort_by: impl Into<Option<SortBy>>) -> Self {
227        self.sort_by = sort_by.into();
228        self
229    }
230
231    /// Return the results ascending or descending.
232    pub fn sort_order(mut self, sort_order: impl Into<Option<SortOrder>>) -> Self {
233        self.sort_order = sort_order.into();
234        self
235    }
236
237    /// The route this query renders to, `/api/query?…`.
238    ///
239    /// # Errors
240    ///
241    /// Refuses a query arXiv would answer with an error feed rather than
242    /// results: one that names neither a search nor an identifier, one whose
243    /// `max_results` is over [`Query::MAX_RESULTS`], and one carrying a term
244    /// that cannot survive the query syntax -- see
245    /// [`Error::UnsendableTerm`].
246    pub fn route(&self) -> Result<String> {
247        if self.search.is_none() && self.id_list.is_empty() {
248            return Err(Error::EmptyQuery);
249        }
250        if let Some(requested) = self.max_results.filter(|n| *n > Self::MAX_RESULTS) {
251            return Err(Error::MaxResultsTooLarge {
252                requested,
253                max: Self::MAX_RESULTS,
254            });
255        }
256
257        let mut params: Vec<(&str, String)> = Vec::new();
258        if let Some(search) = &self.search {
259            search.check_sendable()?;
260            params.push(("search_query", search.to_string()));
261        }
262        if !self.id_list.is_empty() {
263            let ids: Vec<&str> = self.id_list.iter().map(ArticleId::as_str).collect();
264            params.push(("id_list", ids.join(",")));
265        }
266        if let Some(start) = self.start {
267            params.push(("start", start.to_string()));
268        }
269        if let Some(max_results) = self.max_results {
270            params.push(("max_results", max_results.to_string()));
271        }
272        if let Some(sort_by) = self.sort_by {
273            params.push(("sortBy", sort_by.as_str().to_string()));
274        }
275        if let Some(sort_order) = self.sort_order {
276            params.push(("sortOrder", sort_order.as_str().to_string()));
277        }
278
279        Ok(format!("/api/query{}", encode::query_string(&params)))
280    }
281
282    /// The full url this query is sent to, `base_url` with the route appended.
283    pub fn to_url(&self, base_url: &str) -> Result<String> {
284        Ok(format!("{base_url}{}", self.route()?))
285    }
286}
287
288impl Search {
289    /// This search as a [`Query`], ready for paging and sorting to be added.
290    ///
291    /// ```
292    /// # use arxiv_client::{Search, SortBy};
293    /// let query = Search::category("cs.LG").into_query().sort_by(SortBy::SubmittedDate);
294    /// ```
295    pub fn into_query(self) -> Query {
296        Query::new(self)
297    }
298}
299
300impl From<Search> for Query {
301    fn from(search: Search) -> Self {
302        Query::new(search)
303    }
304}
305
306impl From<ArticleId> for Query {
307    fn from(id: ArticleId) -> Self {
308        Query::default().id(id)
309    }
310}
311
312/// Reads free form text as a search of every field, so
313/// `client.search("electron")` means what it looks like.
314impl From<&str> for Query {
315    fn from(terms: &str) -> Self {
316        Query::new(Search::all(terms))
317    }
318}
319
320impl From<String> for Query {
321    fn from(terms: String) -> Self {
322        Query::new(Search::all(terms))
323    }
324}
325
326#[cfg(test)]
327mod tests {
328    use super::*;
329
330    #[test]
331    fn a_search_renders_into_the_route_arxiv_documents() {
332        let query = Query::new(Search::all("electron"));
333
334        assert_eq!(
335            "/api/query?search_query=all%3Aelectron",
336            query.route().unwrap()
337        );
338    }
339
340    #[test]
341    fn free_form_text_becomes_a_search_of_every_field() {
342        let query: Query = "quantum computing".into();
343
344        assert_eq!(
345            "/api/query?search_query=all%3A%22quantum+computing%22",
346            query.route().unwrap()
347        );
348    }
349
350    #[test]
351    fn identifiers_are_sent_as_a_comma_separated_id_list() {
352        let query = Query::ids(["2101.00001v2", "hep-ex/0307015"]).unwrap();
353
354        assert_eq!(
355            "/api/query?id_list=2101.00001v2%2Chep-ex%2F0307015",
356            query.route().unwrap()
357        );
358    }
359
360    #[test]
361    fn a_search_and_an_id_list_are_sent_together() {
362        let query = Query::ids(["2101.00001"])
363            .unwrap()
364            .search(Search::category("cs.LG"));
365        let route = query.route().unwrap();
366
367        assert!(route.contains("search_query=cat%3Acs.LG"), "{route}");
368        assert!(route.contains("id_list=2101.00001"), "{route}");
369    }
370
371    #[test]
372    fn paging_and_sorting_render_under_the_names_arxiv_reads() {
373        let route = Query::new(Search::title("electron"))
374            .start(30)
375            .max_results(10)
376            .sort_by(SortBy::SubmittedDate)
377            .sort_order(SortOrder::Descending)
378            .route()
379            .unwrap();
380
381        assert!(route.contains("start=30"), "{route}");
382        assert!(route.contains("max_results=10"), "{route}");
383        assert!(route.contains("sortBy=submittedDate"), "{route}");
384        assert!(route.contains("sortOrder=descending"), "{route}");
385    }
386
387    #[test]
388    fn a_query_that_asks_for_nothing_is_refused() {
389        assert!(matches!(Query::default().route(), Err(Error::EmptyQuery)));
390    }
391
392    #[test]
393    fn asking_for_more_than_one_slice_is_refused_rather_than_truncated() {
394        let query = Query::new(Search::all("electron")).max_results(5_000);
395
396        assert!(matches!(
397            query.route(),
398            Err(Error::MaxResultsTooLarge {
399                requested: 5_000,
400                max: 2_000
401            })
402        ));
403        assert!(
404            Query::new(Search::all("electron"))
405                .max_results(Query::MAX_RESULTS)
406                .route()
407                .is_ok()
408        );
409    }
410
411    #[test]
412    fn a_term_arxiv_could_not_read_back_never_leaves() {
413        let query = Query::new(Search::title(r#"the "war" on drugs"#));
414
415        assert!(matches!(query.route(), Err(Error::UnsendableTerm { .. })));
416    }
417
418    #[test]
419    fn a_malformed_identifier_is_caught_where_it_is_offered() {
420        let err = Query::ids(["2101.00001", "not-an-id"]).unwrap_err();
421
422        assert!(matches!(err, Error::InvalidArticleId { .. }), "{err}");
423    }
424
425    #[test]
426    fn sort_by_reads_both_spellings_and_round_trips() {
427        for sort_by in SortBy::ALL {
428            assert_eq!(*sort_by, SortBy::from_str(sort_by.as_str()).unwrap());
429        }
430        assert_eq!(
431            SortBy::LastUpdatedDate,
432            SortBy::from_str("last-updated-date").unwrap()
433        );
434        assert_eq!(
435            SortBy::SubmittedDate,
436            SortBy::from_str("submitted").unwrap()
437        );
438        assert!(SortBy::from_str("date").is_err());
439    }
440
441    #[test]
442    fn sort_order_reads_both_spellings_and_round_trips() {
443        for order in SortOrder::ALL {
444            assert_eq!(*order, SortOrder::from_str(order.as_str()).unwrap());
445        }
446        assert_eq!(SortOrder::Descending, SortOrder::from_str("desc").unwrap());
447        assert!(SortOrder::from_str("down").is_err());
448    }
449
450    #[test]
451    fn the_url_is_the_base_with_the_route_appended() {
452        let url = Query::new(Search::all("electron"))
453            .to_url("https://export.arxiv.org")
454            .unwrap();
455
456        assert_eq!(
457            "https://export.arxiv.org/api/query?search_query=all%3Aelectron",
458            url
459        );
460    }
461}