fastpaper/sources/
core.rs

1use super::Paper;
2
3/// Search CORE API.
4/// Build the /v3/search/works URL.
5///
6/// CORE takes field-scoped terms inside `q` and pages with limit/offset.
7///
8/// The author field is `authors.name`, and the value must be quoted: bare
9/// `authors.name:Doudna` matches nothing, and the shorter `authors:"Doudna"`
10/// silently *widens* the result set (60460 -> 874397 for the same query)
11/// instead of narrowing it.
12///
13/// The trailing slash matters too -- `/v3/search/works` answers 301 to
14/// `/v3/search/works/`.
15fn build_search_url(base_url: &str, q: &super::SearchQuery) -> Result<String, String> {
16    if q.limit > MAX_LIMIT {
17        return Err(format!(
18            "core returns at most {} results per request (asked for {}); \
19             larger limits time out server-side",
20            MAX_LIMIT, q.limit
21        ));
22    }
23    let mut terms = vec![super::encode_query(&q.query)];
24    if let Some(ref author) = q.author {
25        terms.push(format!(
26            "authors.name:%22{}%22",
27            super::encode_query(author)
28        ));
29    }
30    if let Some(year) = q.year {
31        terms.push(format!("yearPublished:{}", year));
32    }
33    Ok(format!(
34        "{}/v3/search/works/?q={}&limit={}&offset={}",
35        base_url,
36        terms.join("+AND+"),
37        q.limit,
38        q.offset
39    ))
40}
41
42/// Beyond this CORE answers 504; 100 is the largest verified working value.
43const MAX_LIMIT: u32 = 100;
44
45/// Fetch a single work by CORE id, DOI or other supported identifier.
46///
47/// `GET /v3/works/{identifier}` returns one work object rather than the
48/// `{results: [...]}` envelope the search endpoint uses.
49pub fn get_by_id(base_url: &str, identifier: &str) -> Result<Option<Paper>, String> {
50    let url = format!("{}/v3/works/{}", base_url, super::encode_query(identifier));
51    let api_key = std::env::var("CORE_API_KEY").ok();
52    match http_get_core(&url, api_key.as_deref()) {
53        Ok(body) => {
54            // parse_search_response expects the list envelope.
55            let wrapped = format!(r#"{{"results":[{}]}}"#, body);
56            Ok(parse_search_response(&wrapped)?.into_iter().next())
57        }
58        Err(e) if e.contains("404") => Ok(None),
59        Err(e) => Err(e),
60    }
61}
62
63pub fn search(base_url: &str, q: &super::SearchQuery) -> Result<Vec<Paper>, String> {
64    let url = build_search_url(base_url, q)?;
65    let api_key = std::env::var("CORE_API_KEY").ok();
66
67    // Try with key first, then without on 403
68    let result = http_get_core(&url, api_key.as_deref());
69    match result {
70        Err(ref e) if e.contains("403") && api_key.is_some() => {
71            // Retry without key
72            let body = http_get_core(&url, None)?;
73            parse_search_response(&body)
74        }
75        Err(e) => Err(e),
76        Ok(body) => parse_search_response(&body),
77    }
78}
79
80fn http_get_core(url: &str, api_key: Option<&str>) -> Result<String, String> {
81    let mut last_err = String::new();
82    for attempt in 0..3 {
83        if attempt > 0 {
84            std::thread::sleep(std::time::Duration::from_millis(100 * (1 << attempt)));
85        }
86        let mut req = ureq::get(url);
87        if let Some(key) = api_key {
88            req = req.header("Authorization", &format!("Bearer {}", key));
89        }
90        match req.call() {
91            Ok(resp) => {
92                return resp
93                    .into_body()
94                    .read_to_string()
95                    .map_err(|e| format!("Failed to read response: {}", e));
96            }
97            Err(ureq::Error::StatusCode(429)) => {
98                last_err = "rate limited (429)".to_string();
99                continue;
100            }
101            Err(ureq::Error::StatusCode(403)) => {
102                return Err("403 Forbidden".to_string());
103            }
104            Err(ureq::Error::StatusCode(code)) if code >= 500 => {
105                return Err(format!("Server error: {}", code));
106            }
107            Err(e) => {
108                return Err(format!("HTTP error: {}", e));
109            }
110        }
111    }
112    Err(last_err)
113}
114
115/// Parse CORE JSON search response into a list of Papers.
116pub fn parse_search_response(json: &str) -> Result<Vec<Paper>, String> {
117    let root: serde_json::Value =
118        serde_json::from_str(json).map_err(|e| format!("JSON parse error: {}", e))?;
119
120    let results = root["results"]
121        .as_array()
122        .ok_or("missing 'results' array")?;
123
124    let mut papers = Vec::new();
125    for item in results {
126        let title = item["title"].as_str().unwrap_or("").to_string();
127        if title.is_empty() {
128            continue;
129        }
130
131        let id = item["id"]
132            .as_u64()
133            .map(|n| n.to_string())
134            .or_else(|| item["id"].as_str().map(|s| s.to_string()))
135            .unwrap_or_default();
136
137        let authors: Vec<String> = item["authors"]
138            .as_array()
139            .map(|arr| {
140                arr.iter()
141                    .filter_map(|a| a["name"].as_str().map(|s| s.to_string()))
142                    .collect()
143            })
144            .unwrap_or_default();
145
146        let doi = item["doi"].as_str().map(|s| s.to_string());
147        let abstract_text = item["abstract"].as_str().map(|s| s.to_string());
148
149        let download_url = item["downloadUrl"]
150            .as_str()
151            .filter(|s| !s.is_empty())
152            .map(|s| s.to_string());
153
154        let citations = item["citationCount"].as_u64().map(|n| n as u32);
155
156        let year = item["publishedDate"]
157            .as_str()
158            .and_then(|s| s.get(..4))
159            .and_then(|y| y.parse::<u16>().ok())
160            .or_else(|| item["yearPublished"].as_u64().map(|y| y as u16));
161
162        papers.push(Paper {
163            id,
164            title,
165            authors,
166            abstract_text,
167            year,
168            doi,
169            url: item["links"]
170                .as_array()
171                .and_then(|arr| arr.first())
172                .and_then(|l| l["url"].as_str())
173                .map(|s| s.to_string()),
174            pdf_url: download_url,
175            venue: None,
176            citations,
177            fields: item["fieldOfStudy"]
178                .as_str()
179                .map(|s| vec![s.to_string()])
180                .unwrap_or_default(),
181            open_access: Some(true),
182            source: "core".to_string(),
183        });
184    }
185
186    Ok(papers)
187}
188
189#[cfg(test)]
190mod tests {
191    use super::*;
192    use serial_test::serial;
193
194    const FIXTURE: &str = include_str!("../../tests/fixtures/core_search.json");
195
196    #[test]
197    fn parse_returns_ok() {
198        let result = parse_search_response(FIXTURE);
199        assert!(result.is_ok());
200    }
201
202    #[test]
203    fn parse_papers_not_empty() {
204        let papers = parse_search_response(FIXTURE).unwrap();
205        assert!(!papers.is_empty());
206    }
207
208    #[test]
209    fn parse_source_is_core() {
210        let papers = parse_search_response(FIXTURE).unwrap();
211        for p in &papers {
212            assert_eq!(p.source, "core");
213        }
214    }
215
216    #[test]
217    fn parse_title_not_empty() {
218        let papers = parse_search_response(FIXTURE).unwrap();
219        for p in &papers {
220            assert!(!p.title.is_empty(), "paper {} has empty title", p.id);
221        }
222    }
223
224    #[test]
225    fn parse_authors() {
226        let papers = parse_search_response(FIXTURE).unwrap();
227        let with_authors: Vec<_> = papers.iter().filter(|p| !p.authors.is_empty()).collect();
228        assert!(!with_authors.is_empty(), "no papers with authors");
229    }
230
231    #[test]
232    fn parse_doi() {
233        let papers = parse_search_response(FIXTURE).unwrap();
234        let with_doi: Vec<_> = papers.iter().filter(|p| p.doi.is_some()).collect();
235        assert!(!with_doi.is_empty(), "no papers with DOI");
236        for p in &with_doi {
237            assert!(p.doi.as_ref().unwrap().starts_with("10."));
238        }
239    }
240
241    #[test]
242    fn parse_abstract() {
243        let papers = parse_search_response(FIXTURE).unwrap();
244        let with_abstract: Vec<_> = papers
245            .iter()
246            .filter(|p| p.abstract_text.is_some())
247            .collect();
248        assert!(!with_abstract.is_empty(), "no papers with abstract");
249    }
250
251    #[test]
252    fn parse_pdf_url() {
253        let papers = parse_search_response(FIXTURE).unwrap();
254        let with_pdf: Vec<_> = papers.iter().filter(|p| p.pdf_url.is_some()).collect();
255        assert!(!with_pdf.is_empty(), "no papers with pdf_url");
256    }
257
258    #[test]
259    fn parse_citations() {
260        let papers = parse_search_response(FIXTURE).unwrap();
261        for p in &papers {
262            assert!(p.citations.is_some(), "paper {} missing citations", p.id);
263        }
264    }
265
266    #[test]
267    fn search_returns_papers() {
268        let mut server = mockito::Server::new();
269        let mock = server
270            .mock("GET", mockito::Matcher::Any)
271            .with_status(200)
272            .with_body(FIXTURE)
273            .create();
274        let papers = search(
275            &server.url(),
276            &crate::sources::SearchQuery::simple("test", 3),
277        )
278        .unwrap();
279        assert!(!papers.is_empty());
280        mock.assert();
281    }
282
283    #[test]
284    fn search_request_path() {
285        let mut server = mockito::Server::new();
286        let mock = server
287            .mock(
288                "GET",
289                mockito::Matcher::Regex("/v3/search/works".to_string()),
290            )
291            .with_status(200)
292            .with_body(FIXTURE)
293            .create();
294        let _ = search(
295            &server.url(),
296            &crate::sources::SearchQuery::simple("test", 3),
297        );
298        mock.assert();
299    }
300
301    #[test]
302    fn search_request_contains_limit() {
303        let mut server = mockito::Server::new();
304        let mock = server
305            .mock("GET", mockito::Matcher::Regex("limit=3".to_string()))
306            .with_status(200)
307            .with_body(FIXTURE)
308            .create();
309        let _ = search(
310            &server.url(),
311            &crate::sources::SearchQuery::simple("test", 3),
312        );
313        mock.assert();
314    }
315
316    #[test]
317    #[serial]
318    fn search_with_api_key_sends_bearer() {
319        unsafe { std::env::set_var("CORE_API_KEY", "core-test-key") };
320        let mut server = mockito::Server::new();
321        let mock = server
322            .mock("GET", mockito::Matcher::Any)
323            .match_header("Authorization", "Bearer core-test-key")
324            .with_status(200)
325            .with_body(FIXTURE)
326            .create();
327        let result = search(
328            &server.url(),
329            &crate::sources::SearchQuery::simple("test", 3),
330        );
331        unsafe { std::env::remove_var("CORE_API_KEY") };
332        assert!(result.is_ok());
333        mock.assert();
334    }
335
336    #[test]
337    #[serial]
338    fn search_works_without_api_key() {
339        unsafe { std::env::remove_var("CORE_API_KEY") };
340        let mut server = mockito::Server::new();
341        server
342            .mock("GET", mockito::Matcher::Any)
343            .with_status(200)
344            .with_body(FIXTURE)
345            .create();
346        let result = search(
347            &server.url(),
348            &crate::sources::SearchQuery::simple("test", 3),
349        );
350        assert!(result.is_ok());
351    }
352
353    #[test]
354    #[serial]
355    fn search_403_with_key_retries_without() {
356        unsafe { std::env::set_var("CORE_API_KEY", "bad-key") };
357        let mut server = mockito::Server::new();
358        // First request with key → 403
359        server
360            .mock("GET", mockito::Matcher::Any)
361            .match_header("Authorization", "Bearer bad-key")
362            .with_status(403)
363            .create();
364        // Second request without key → 200
365        server
366            .mock("GET", mockito::Matcher::Any)
367            .match_header("Authorization", mockito::Matcher::Missing)
368            .with_status(200)
369            .with_body(FIXTURE)
370            .create();
371        let result = search(
372            &server.url(),
373            &crate::sources::SearchQuery::simple("test", 3),
374        );
375        unsafe { std::env::remove_var("CORE_API_KEY") };
376        assert!(result.is_ok());
377    }
378}
379
380#[cfg(test)]
381mod get_tests {
382    use super::*;
383
384    // /v3/works/{id} answers with a bare work object, not the search envelope.
385    const WORK: &str = r#"{"id":12345678,"title":"A CORE work",
386"authors":[{"name":"Jane Doe"}],"yearPublished":2021,
387"doi":"10.1234/abcd","abstract":"Some abstract.",
388"downloadUrl":"https://core.ac.uk/download/12345678.pdf"}"#;
389
390    #[test]
391    fn get_by_id_parses_a_single_work() {
392        let mut server = mockito::Server::new();
393        server
394            .mock("GET", mockito::Matcher::Regex("/v3/works/".to_string()))
395            .with_status(200)
396            .with_body(WORK)
397            .create();
398        let paper = get_by_id(&server.url(), "12345678").unwrap().unwrap();
399        assert_eq!(paper.title, "A CORE work");
400        assert_eq!(paper.year, Some(2021));
401        assert_eq!(paper.source, "core");
402    }
403
404    #[test]
405    fn get_by_id_encodes_a_doi_identifier() {
406        let mut server = mockito::Server::new();
407        let mock = server
408            .mock("GET", mockito::Matcher::Regex("/v3/works/10".to_string()))
409            .with_status(200)
410            .with_body(WORK)
411            .create();
412        let _ = get_by_id(&server.url(), "10.1234/abcd");
413        mock.assert();
414    }
415
416    #[test]
417    fn get_by_id_returns_none_on_404() {
418        let mut server = mockito::Server::new();
419        server
420            .mock("GET", mockito::Matcher::Any)
421            .with_status(404)
422            .create();
423        assert!(get_by_id(&server.url(), "nope").unwrap().is_none());
424    }
425}
426
427#[cfg(test)]
428mod url_tests {
429    use super::*;
430    use crate::sources::SearchQuery;
431
432    fn url(q: &SearchQuery) -> String {
433        build_search_url("https://api.core.ac.uk", q).unwrap()
434    }
435
436    // /v3/search/works answers 301 to the trailing-slash form.
437    #[test]
438    fn search_path_carries_the_trailing_slash() {
439        assert!(url(&SearchQuery::simple("crispr", 10)).contains("/v3/search/works/?"));
440    }
441
442    // `authors:"X"` widens the result set instead of narrowing it; the field is
443    // authors.name, and the value has to be quoted or it matches nothing.
444    #[test]
445    fn author_uses_the_quoted_authors_name_field() {
446        let mut q = SearchQuery::simple("crispr", 10);
447        q.author = Some("Doudna".into());
448        let u = url(&q);
449        assert!(u.contains("authors.name:%22Doudna%22"), "got: {}", u);
450        assert!(
451            !u.contains("authors:%22"),
452            "wrong field widens results: {}",
453            u
454        );
455    }
456
457    #[test]
458    fn year_uses_year_published() {
459        let mut q = SearchQuery::simple("crispr", 10);
460        q.year = Some(2020);
461        assert!(url(&q).contains("yearPublished:2020"), "got: {}", url(&q));
462    }
463
464    #[test]
465    fn offset_is_passed_through() {
466        let mut q = SearchQuery::simple("crispr", 10);
467        q.offset = 30;
468        assert!(url(&q).contains("offset=30"));
469    }
470
471    // 200 already times out server-side with a 504.
472    #[test]
473    fn limit_above_the_cap_is_rejected() {
474        let err =
475            build_search_url("https://api.core.ac.uk", &SearchQuery::simple("x", 200)).unwrap_err();
476        assert!(err.contains("100"), "got: {}", err);
477    }
478
479    #[test]
480    fn limit_at_the_cap_is_allowed() {
481        assert!(build_search_url("https://api.core.ac.uk", &SearchQuery::simple("x", 100)).is_ok());
482    }
483}
fastpaper/sources/core.rs

fastpaper/sources/
core.rs
