fastpaper/sources/
crossref.rs

1use super::Paper;
2
3/// Beyond this Crossref requires cursor paging instead of `offset`.
4const MAX_OFFSET: u32 = 10_000;
5
6/// Build the /works search URL.
7///
8/// `mailto` opts into Crossref's polite pool; without it the request goes to
9/// the public pool, which still works. Date filters go into a single
10/// comma-separated `filter` parameter, which Crossref ANDs together.
11fn build_search_url(
12    base_url: &str,
13    q: &super::SearchQuery,
14    email: Option<&str>,
15) -> Result<String, String> {
16    if q.offset > MAX_OFFSET {
17        return Err(format!(
18            "crossref supports an offset up to {} (asked for {}); deeper paging needs a cursor",
19            MAX_OFFSET, q.offset
20        ));
21    }
22
23    let mut url = format!(
24        "{}/works?query={}&rows={}&offset={}",
25        base_url,
26        super::encode_query(&q.query),
27        q.limit,
28        q.offset
29    );
30
31    if let Some(ref author) = q.author {
32        url.push_str(&format!("&query.author={}", super::encode_query(author)));
33    }
34
35    let mut filters = Vec::new();
36    match q.year {
37        Some(year) => {
38            filters.push(format!("from-pub-date:{}-01-01", year));
39            filters.push(format!("until-pub-date:{}-12-31", year));
40        }
41        None => {
42            if let Some(ref after) = q.after {
43                filters.push(format!("from-pub-date:{}", super::validate_ymd(after)?));
44            }
45            if let Some(ref before) = q.before {
46                filters.push(format!("until-pub-date:{}", super::validate_ymd(before)?));
47            }
48        }
49    }
50    if !filters.is_empty() {
51        url.push_str(&format!("&filter={}", filters.join(",")));
52    }
53
54    if let Some(sort) = q.sort {
55        let by = match sort {
56            super::SortField::Relevance => "score",
57            super::SortField::Date => "published",
58            super::SortField::Citations => "is-referenced-by-count",
59        };
60        let order = match q.order {
61            super::SortOrder::Asc => "asc",
62            super::SortOrder::Desc => "desc",
63        };
64        url.push_str(&format!("&sort={}&order={}", by, order));
65    }
66
67    if let Some(email) = email {
68        url.push_str(&format!("&mailto={}", email));
69    }
70    Ok(url)
71}
72
73fn build_work_url(base_url: &str, doi: &str, email: Option<&str>) -> String {
74    let mut url = format!("{}/works/{}", base_url, doi);
75    if let Some(email) = email {
76        url.push_str(&format!("?mailto={}", email));
77    }
78    url
79}
80
81/// Search CrossRef API.
82pub fn search(base_url: &str, q: &super::SearchQuery) -> Result<Vec<Paper>, String> {
83    let email = super::contact_email();
84    let url = build_search_url(base_url, q, email.as_deref())?;
85    let body = http_get(&url)?;
86    parse_search_response(&body)
87}
88
89/// Get a single paper by DOI.
90pub fn get_by_doi(base_url: &str, doi: &str) -> Result<Option<Paper>, String> {
91    let email = super::contact_email();
92    let url = build_work_url(base_url, doi, email.as_deref());
93    match ureq::get(&url).call() {
94        Ok(resp) => {
95            let body = resp
96                .into_body()
97                .read_to_string()
98                .map_err(|e| format!("Failed to read response: {}", e))?;
99            let root: serde_json::Value =
100                serde_json::from_str(&body).map_err(|e| format!("JSON parse error: {}", e))?;
101            let item = &root["message"];
102            if item.is_null() {
103                return Ok(None);
104            }
105            Ok(Some(parse_crossref_item(item)))
106        }
107        Err(ureq::Error::StatusCode(404)) => Ok(None),
108        Err(ureq::Error::StatusCode(429)) => Err("rate limited (429)".to_string()),
109        Err(ureq::Error::StatusCode(code)) if code >= 500 => Err(format!("Server error: {}", code)),
110        Err(e) => Err(format!("HTTP error: {}", e)),
111    }
112}
113
114fn http_get(url: &str) -> Result<String, String> {
115    let mut last_err = String::new();
116    for attempt in 0..3 {
117        if attempt > 0 {
118            std::thread::sleep(std::time::Duration::from_millis(100 * (1 << attempt)));
119        }
120        match ureq::get(url).call() {
121            Ok(resp) => {
122                return resp
123                    .into_body()
124                    .read_to_string()
125                    .map_err(|e| format!("Failed to read response: {}", e));
126            }
127            Err(ureq::Error::StatusCode(429)) => {
128                last_err = "rate limited (429)".to_string();
129                continue;
130            }
131            Err(ureq::Error::StatusCode(code)) if code >= 500 => {
132                return Err(format!("Server error: {}", code));
133            }
134            Err(e) => {
135                return Err(format!("HTTP error: {}", e));
136            }
137        }
138    }
139    Err(last_err)
140}
141
142/// Parse CrossRef JSON search response into a list of Papers.
143pub fn parse_search_response(json: &str) -> Result<Vec<Paper>, String> {
144    let root: serde_json::Value =
145        serde_json::from_str(json).map_err(|e| format!("JSON parse error: {}", e))?;
146
147    let items = root["message"]["items"]
148        .as_array()
149        .ok_or("missing 'message.items' array")?;
150
151    Ok(items.iter().map(parse_crossref_item).collect())
152}
153
154fn parse_crossref_item(item: &serde_json::Value) -> Paper {
155    let doi = item["DOI"].as_str().unwrap_or("").to_string();
156    let title = item["title"]
157        .as_array()
158        .and_then(|arr| arr.first())
159        .and_then(|v| v.as_str())
160        .unwrap_or("")
161        .to_string();
162
163    let authors: Vec<String> = item["author"]
164        .as_array()
165        .map(|arr| {
166            arr.iter()
167                .map(|a| {
168                    let given = a["given"].as_str().unwrap_or("");
169                    let family = a["family"].as_str().unwrap_or("");
170                    format!("{} {}", given, family).trim().to_string()
171                })
172                .collect()
173        })
174        .unwrap_or_default();
175
176    let year = extract_year(item, "published")
177        .or_else(|| extract_year(item, "issued"))
178        .or_else(|| extract_year(item, "created"));
179
180    let url = item["URL"]
181        .as_str()
182        .map(|s| s.to_string())
183        .unwrap_or_else(|| format!("https://doi.org/{}", doi));
184
185    let venue = item["container-title"]
186        .as_array()
187        .and_then(|arr| arr.first())
188        .and_then(|v| v.as_str())
189        .map(|s| s.to_string());
190
191    let citations = item["is-referenced-by-count"].as_u64().map(|n| n as u32);
192
193    let abstract_text = item["abstract"].as_str().map(|s| s.to_string());
194
195    Paper {
196        id: doi.clone(),
197        title,
198        authors,
199        abstract_text,
200        year,
201        doi: Some(doi),
202        url: Some(url),
203        pdf_url: None,
204        venue,
205        citations,
206        fields: vec![],
207        open_access: None,
208        source: "crossref".to_string(),
209    }
210}
211
212fn extract_year(item: &serde_json::Value, field: &str) -> Option<u16> {
213    item[field]["date-parts"]
214        .as_array()
215        .and_then(|arr| arr.first())
216        .and_then(|parts| parts.as_array())
217        .and_then(|parts| parts.first())
218        .and_then(|y| y.as_u64())
219        .map(|y| y as u16)
220}
221
222#[cfg(test)]
223mod tests {
224    use super::*;
225    use serial_test::serial;
226
227    const FIXTURE: &str = include_str!("../../tests/fixtures/crossref_search.json");
228
229    #[test]
230    fn parse_returns_ok() {
231        let result = parse_search_response(FIXTURE);
232        assert!(result.is_ok());
233    }
234
235    #[test]
236    fn parse_titles_from_array() {
237        let papers = parse_search_response(FIXTURE).unwrap();
238        for p in &papers {
239            assert!(!p.title.is_empty(), "paper {} has empty title", p.id);
240        }
241    }
242
243    #[test]
244    fn parse_authors_given_family() {
245        let papers = parse_search_response(FIXTURE).unwrap();
246        // At least one paper should have authors
247        let with_authors: Vec<_> = papers.iter().filter(|p| !p.authors.is_empty()).collect();
248        assert!(!with_authors.is_empty(), "no papers with authors");
249        for p in &with_authors {
250            for a in &p.authors {
251                assert!(!a.is_empty(), "empty author name in paper {}", p.id);
252            }
253        }
254    }
255
256    #[test]
257    fn parse_doi() {
258        let papers = parse_search_response(FIXTURE).unwrap();
259        for p in &papers {
260            assert!(p.doi.is_some(), "paper missing DOI");
261            assert!(!p.doi.as_ref().unwrap().is_empty());
262        }
263    }
264
265    #[test]
266    fn parse_citations() {
267        let papers = parse_search_response(FIXTURE).unwrap();
268        for p in &papers {
269            assert!(p.citations.is_some(), "paper {} missing citations", p.id);
270        }
271    }
272
273    #[test]
274    fn parse_year_from_date_parts() {
275        let papers = parse_search_response(FIXTURE).unwrap();
276        let with_year: Vec<_> = papers.iter().filter(|p| p.year.is_some()).collect();
277        assert!(!with_year.is_empty(), "no papers with year");
278        for p in &with_year {
279            assert!(p.year.unwrap() > 2000);
280        }
281    }
282
283    #[test]
284    fn parse_source_is_crossref() {
285        let papers = parse_search_response(FIXTURE).unwrap();
286        for p in &papers {
287            assert_eq!(p.source, "crossref");
288        }
289    }
290
291    #[test]
292    fn search_request_path_is_works() {
293        let mut server = mockito::Server::new();
294        let mock = server
295            .mock("GET", mockito::Matcher::Regex(r"/works\?".to_string()))
296            .with_status(200)
297            .with_body(FIXTURE)
298            .create();
299        let _ = search(
300            &server.url(),
301            &crate::sources::SearchQuery::simple("attention", 3),
302        );
303        mock.assert();
304    }
305
306    // Single-item response for get_by_doi tests
307    fn single_item_response() -> String {
308        // Wrap the first item from fixture as a single-item response
309        let root: serde_json::Value = serde_json::from_str(FIXTURE).unwrap();
310        let first_item = &root["message"]["items"][0];
311        serde_json::json!({
312            "status": "ok",
313            "message": first_item,
314        })
315        .to_string()
316    }
317
318    #[test]
319    fn get_by_doi_request_path() {
320        let mut server = mockito::Server::new();
321        let mock = server
322            .mock(
323                "GET",
324                mockito::Matcher::Regex(r"/works/10\.1038/nature12373".to_string()),
325            )
326            .with_status(200)
327            .with_body(single_item_response())
328            .create();
329        let _ = get_by_doi(&server.url(), "10.1038/nature12373");
330        mock.assert();
331    }
332
333    #[test]
334    fn get_by_doi_returns_paper() {
335        let mut server = mockito::Server::new();
336        let mock = server
337            .mock("GET", mockito::Matcher::Any)
338            .with_status(200)
339            .with_body(single_item_response())
340            .create();
341        let result = get_by_doi(&server.url(), "10.1038/nature12373").unwrap();
342        assert!(result.is_some());
343        let paper = result.unwrap();
344        assert_eq!(paper.source, "crossref");
345        mock.assert();
346    }
347
348    #[test]
349    fn get_by_doi_returns_none_on_404() {
350        let mut server = mockito::Server::new();
351        let mock = server
352            .mock("GET", mockito::Matcher::Any)
353            .with_status(404)
354            .create();
355        let result = get_by_doi(&server.url(), "10.9999/nonexistent").unwrap();
356        assert!(result.is_none());
357        mock.assert();
358    }
359
360    // Without FASTPAPER_EMAIL, fall back to Crossref's public pool rather than
361    // identifying every user's traffic as the maintainer. Asserted on the built
362    // URL rather than through mockito: later-created mocks take precedence
363    // there, so an `expect(0)` probe behind a catch-all passes vacuously.
364    #[test]
365    fn build_search_url_omits_mailto_without_email() {
366        let url = build_search_url(
367            "https://api.crossref.org",
368            &crate::sources::SearchQuery::simple("attention", 3),
369            None,
370        )
371        .unwrap();
372        assert!(
373            !url.contains("mailto="),
374            "url should carry no contact address: {}",
375            url
376        );
377    }
378
379    #[test]
380    fn build_search_url_includes_mailto_with_email() {
381        let url = build_search_url(
382            "https://api.crossref.org",
383            &crate::sources::SearchQuery::simple("attention", 3),
384            Some("a@b.com"),
385        )
386        .unwrap();
387        assert!(url.contains("mailto=a@b.com"), "got: {}", url);
388    }
389
390    #[test]
391    fn build_work_url_omits_mailto_without_email() {
392        let url = build_work_url("https://api.crossref.org", "10.1038/nature12373", None);
393        assert!(!url.contains("mailto="), "got: {}", url);
394    }
395
396    #[test]
397    #[serial]
398    fn search_sends_env_email_when_set() {
399        unsafe { std::env::set_var("FASTPAPER_EMAIL", "custom@example.com") };
400        let mut server = mockito::Server::new();
401        let mock = server
402            .mock("GET", mockito::Matcher::Regex("mailto=custom".to_string()))
403            .with_status(200)
404            .with_body(FIXTURE)
405            .create();
406        let _ = search(
407            &server.url(),
408            &crate::sources::SearchQuery::simple("test", 3),
409        );
410        unsafe { std::env::remove_var("FASTPAPER_EMAIL") };
411        mock.assert();
412    }
413}
414
415#[cfg(test)]
416mod query_tests {
417    use super::*;
418    use crate::sources::{SearchQuery, SortField, SortOrder};
419
420    fn url(q: &SearchQuery) -> String {
421        build_search_url("https://api.crossref.org", q, None).unwrap()
422    }
423
424    #[test]
425    fn plain_query_uses_query_and_rows() {
426        let u = url(&SearchQuery::simple("attention", 10));
427        assert!(u.contains("query=attention"), "got: {}", u);
428        assert!(u.contains("rows=10"), "got: {}", u);
429    }
430
431    #[test]
432    fn offset_is_passed_through() {
433        let mut q = SearchQuery::simple("attention", 10);
434        q.offset = 30;
435        assert!(url(&q).contains("offset=30"));
436    }
437
438    // Crossref caps offset at 10000 and wants a cursor beyond that.
439    #[test]
440    fn offset_beyond_the_api_limit_is_rejected() {
441        let mut q = SearchQuery::simple("attention", 10);
442        q.offset = 10_001;
443        let err = build_search_url("https://api.crossref.org", &q, None).unwrap_err();
444        assert!(err.contains("10000"), "got: {}", err);
445    }
446
447    #[test]
448    fn author_uses_the_dedicated_field_query() {
449        let mut q = SearchQuery::simple("attention", 10);
450        q.author = Some("Vaswani".into());
451        assert!(url(&q).contains("query.author=Vaswani"), "got: {}", url(&q));
452    }
453
454    #[test]
455    fn year_becomes_a_publication_date_filter() {
456        let mut q = SearchQuery::simple("attention", 10);
457        q.year = Some(2024);
458        let u = url(&q);
459        assert!(u.contains("from-pub-date:2024-01-01"), "got: {}", u);
460        assert!(u.contains("until-pub-date:2024-12-31"), "got: {}", u);
461    }
462
463    #[test]
464    fn after_and_before_become_date_filters() {
465        let mut q = SearchQuery::simple("attention", 10);
466        q.after = Some("2023-06-01".into());
467        q.before = Some("2023-12-31".into());
468        let u = url(&q);
469        assert!(u.contains("from-pub-date:2023-06-01"), "got: {}", u);
470        assert!(u.contains("until-pub-date:2023-12-31"), "got: {}", u);
471    }
472
473    #[test]
474    fn multiple_filters_are_comma_separated_in_one_parameter() {
475        let mut q = SearchQuery::simple("attention", 10);
476        q.after = Some("2023-06-01".into());
477        q.before = Some("2023-12-31".into());
478        let u = url(&q);
479        assert_eq!(
480            u.matches("filter=").count(),
481            1,
482            "one filter parameter: {}",
483            u
484        );
485        assert!(u.contains(','), "filters are comma separated: {}", u);
486    }
487
488    #[test]
489    fn malformed_date_is_rejected() {
490        let mut q = SearchQuery::simple("attention", 10);
491        q.before = Some("2023/12/31".into());
492        let err = build_search_url("https://api.crossref.org", &q, None).unwrap_err();
493        assert!(err.contains("YYYY-MM-DD"), "got: {}", err);
494    }
495
496    #[test]
497    fn sort_by_citations_uses_the_reference_count() {
498        let mut q = SearchQuery::simple("attention", 10);
499        q.sort = Some(SortField::Citations);
500        let u = url(&q);
501        assert!(u.contains("sort=is-referenced-by-count"), "got: {}", u);
502        assert!(u.contains("order=desc"), "got: {}", u);
503    }
504
505    #[test]
506    fn sort_by_date_uses_published() {
507        let mut q = SearchQuery::simple("attention", 10);
508        q.sort = Some(SortField::Date);
509        assert!(url(&q).contains("sort=published"));
510    }
511
512    #[test]
513    fn sort_by_relevance_uses_score() {
514        let mut q = SearchQuery::simple("attention", 10);
515        q.sort = Some(SortField::Relevance);
516        assert!(url(&q).contains("sort=score"));
517    }
518
519    #[test]
520    fn ascending_order_is_honoured() {
521        let mut q = SearchQuery::simple("attention", 10);
522        q.sort = Some(SortField::Date);
523        q.order = SortOrder::Asc;
524        assert!(url(&q).contains("order=asc"));
525    }
526
527    #[test]
528    fn mailto_is_still_appended_when_supplied() {
529        let q = SearchQuery::simple("attention", 10);
530        let u = build_search_url("https://api.crossref.org", &q, Some("a@b.com")).unwrap();
531        assert!(u.contains("mailto=a@b.com"), "got: {}", u);
532    }
533}
fastpaper/sources/crossref.rs

fastpaper/sources/
crossref.rs
