fastpaper/sources/
hal.rs

1use super::Paper;
2
3const FIELDS: &str =
4    "halId_s,title_s,authFullName_s,abstract_s,doiId_s,publicationDateY_i,fileMain_s,uri_s";
5
6/// Build the Solr search URL.
7///
8/// HAL is a Solr index: `q` holds the query and author term, structured
9/// restrictions go into repeated `fq` parameters, and `start`/`rows` page.
10fn build_search_url(base_url: &str, q: &super::SearchQuery) -> Result<String, String> {
11    let mut terms = vec![super::encode_query(&q.query)];
12    if let Some(ref author) = q.author {
13        terms.push(format!(
14            "authFullName_s:%22{}%22",
15            super::encode_query(author)
16        ));
17    }
18
19    let mut url = format!(
20        "{}/search/?q={}&rows={}&start={}&wt=json&fl={}",
21        base_url,
22        terms.join("+AND+"),
23        q.limit,
24        q.offset,
25        FIELDS
26    );
27
28    if let Some(year) = q.year {
29        url.push_str(&format!("&fq=publicationDateY_i:{}", year));
30    }
31    if q.open_access {
32        url.push_str("&fq=openAccess_bool:true");
33    }
34    // domain_s values are hierarchical codes like "0.sdv" / "1.sdv.bbm", so a
35    // bare "sdv" only matches when wrapped in wildcards.
36    if let Some(ref field) = q.field {
37        url.push_str(&format!("&fq=domain_s:*{}*", super::encode_query(field)));
38    }
39
40    if let Some(sort) = q.sort {
41        match sort {
42            // Solr's default ordering is by relevance score.
43            super::SortField::Relevance => {}
44            super::SortField::Date => {
45                let order = match q.order {
46                    super::SortOrder::Asc => "asc",
47                    super::SortOrder::Desc => "desc",
48                };
49                url.push_str(&format!("&sort=publicationDateY_i+{}", order));
50            }
51            super::SortField::Citations => {
52                return Err(
53                    "hal cannot sort by citations: it indexes no citation counts.\n\
54                     Try semantic or openalex."
55                        .to_string(),
56                );
57            }
58        }
59    }
60
61    Ok(url)
62}
63
64/// Fetch a single document by HAL id or DOI.
65///
66/// HAL has no by-id path; a field-scoped Solr query is the documented way.
67pub fn get_by_id(base_url: &str, identifier: &str) -> Result<Option<Paper>, String> {
68    let field = if identifier.starts_with("10.") {
69        "doiId_s"
70    } else {
71        "halId_s"
72    };
73    let url = format!(
74        "{}/search/?q={}:%22{}%22&rows=1&wt=json&fl={}",
75        base_url,
76        field,
77        super::encode_query(identifier),
78        FIELDS
79    );
80    let body = ureq::get(&url)
81        .call()
82        .map_err(|e| format!("HTTP error: {}", e))?
83        .into_body()
84        .read_to_string()
85        .map_err(|e| format!("Failed to read response: {}", e))?;
86    Ok(parse_search_response(&body)?.into_iter().next())
87}
88
89/// Search HAL API.
90pub fn search(base_url: &str, q: &super::SearchQuery) -> Result<Vec<Paper>, String> {
91    let url = build_search_url(base_url, q)?;
92
93    let mut last_err = String::new();
94    for attempt in 0..3 {
95        if attempt > 0 {
96            std::thread::sleep(std::time::Duration::from_millis(100 * (1 << attempt)));
97        }
98        match ureq::get(&url).call() {
99            Ok(resp) => {
100                let body = resp
101                    .into_body()
102                    .read_to_string()
103                    .map_err(|e| format!("Failed to read response: {}", e))?;
104                return parse_search_response(&body);
105            }
106            Err(ureq::Error::StatusCode(429)) => {
107                last_err = "rate limited (429)".to_string();
108                continue;
109            }
110            Err(ureq::Error::StatusCode(code)) if code >= 500 => {
111                return Err(format!("Server error: {}", code));
112            }
113            Err(e) => {
114                return Err(format!("HTTP error: {}", e));
115            }
116        }
117    }
118    Err(last_err)
119}
120
121/// Parse HAL JSON search response into a list of Papers.
122pub fn parse_search_response(json: &str) -> Result<Vec<Paper>, String> {
123    let root: serde_json::Value =
124        serde_json::from_str(json).map_err(|e| format!("JSON parse error: {}", e))?;
125
126    let docs = root["response"]["docs"]
127        .as_array()
128        .ok_or("missing 'response.docs' array")?;
129
130    let mut papers = Vec::new();
131    for item in docs {
132        // title_s can be array or string
133        let title = if let Some(arr) = item["title_s"].as_array() {
134            arr.first()
135                .and_then(|v| v.as_str())
136                .unwrap_or("")
137                .to_string()
138        } else {
139            item["title_s"].as_str().unwrap_or("").to_string()
140        };
141        if title.is_empty() {
142            continue;
143        }
144
145        let id = item["halId_s"].as_str().unwrap_or("").to_string();
146
147        let authors: Vec<String> = item["authFullName_s"]
148            .as_array()
149            .map(|arr| {
150                arr.iter()
151                    .filter_map(|a| a.as_str().map(|s| s.to_string()))
152                    .collect()
153            })
154            .unwrap_or_default();
155
156        let doi = item["doiId_s"].as_str().map(|s| s.to_string());
157
158        // abstract_s can be array or string
159        let abstract_text = if let Some(arr) = item["abstract_s"].as_array() {
160            arr.first().and_then(|v| v.as_str()).map(|s| s.to_string())
161        } else {
162            item["abstract_s"].as_str().map(|s| s.to_string())
163        };
164
165        let year = item["publicationDateY_i"].as_u64().map(|y| y as u16);
166
167        let pdf_url = item["fileMain_s"].as_str().map(|s| s.to_string());
168
169        papers.push(Paper {
170            id,
171            title,
172            authors,
173            abstract_text,
174            year,
175            doi,
176            url: item["uri_s"].as_str().map(|s| s.to_string()),
177            pdf_url,
178            venue: None,
179            citations: None,
180            fields: vec![],
181            open_access: Some(true),
182            source: "hal".to_string(),
183        });
184    }
185
186    Ok(papers)
187}
188
189#[cfg(test)]
190mod tests {
191    use super::*;
192
193    const FIXTURE: &str = include_str!("../../tests/fixtures/hal_search.json");
194
195    #[test]
196    fn parse_returns_ok() {
197        let result = parse_search_response(FIXTURE);
198        assert!(result.is_ok());
199    }
200
201    #[test]
202    fn parse_papers_not_empty() {
203        let papers = parse_search_response(FIXTURE).unwrap();
204        assert!(!papers.is_empty());
205    }
206
207    #[test]
208    fn parse_source_is_hal() {
209        let papers = parse_search_response(FIXTURE).unwrap();
210        for p in &papers {
211            assert_eq!(p.source, "hal");
212        }
213    }
214
215    #[test]
216    fn parse_id_is_hal_format() {
217        let papers = parse_search_response(FIXTURE).unwrap();
218        for p in &papers {
219            assert!(
220                p.id.starts_with("hal-"),
221                "id should start with hal-: {}",
222                p.id
223            );
224        }
225    }
226
227    #[test]
228    fn parse_title() {
229        let papers = parse_search_response(FIXTURE).unwrap();
230        for p in &papers {
231            assert!(!p.title.is_empty(), "paper {} has empty title", p.id);
232        }
233    }
234
235    #[test]
236    fn parse_authors() {
237        let papers = parse_search_response(FIXTURE).unwrap();
238        let with_authors: Vec<_> = papers.iter().filter(|p| !p.authors.is_empty()).collect();
239        assert!(!with_authors.is_empty(), "no papers with authors");
240    }
241
242    #[test]
243    fn parse_doi() {
244        let papers = parse_search_response(FIXTURE).unwrap();
245        let with_doi: Vec<_> = papers.iter().filter(|p| p.doi.is_some()).collect();
246        assert!(!with_doi.is_empty(), "no papers with DOI");
247        for p in &with_doi {
248            assert!(p.doi.as_ref().unwrap().starts_with("10."));
249        }
250    }
251
252    #[test]
253    fn parse_abstract() {
254        let papers = parse_search_response(FIXTURE).unwrap();
255        let with_abstract: Vec<_> = papers
256            .iter()
257            .filter(|p| p.abstract_text.is_some())
258            .collect();
259        assert!(!with_abstract.is_empty(), "no papers with abstract");
260    }
261
262    #[test]
263    fn parse_year() {
264        let papers = parse_search_response(FIXTURE).unwrap();
265        let with_year: Vec<_> = papers.iter().filter(|p| p.year.is_some()).collect();
266        assert!(!with_year.is_empty(), "no papers with year");
267        for p in &with_year {
268            assert!(p.year.unwrap() >= 2000);
269        }
270    }
271
272    #[test]
273    fn parse_pdf_url() {
274        let papers = parse_search_response(FIXTURE).unwrap();
275        let with_pdf: Vec<_> = papers.iter().filter(|p| p.pdf_url.is_some()).collect();
276        assert!(!with_pdf.is_empty(), "no papers with pdf_url");
277    }
278
279    #[test]
280    fn parse_open_access_always_true() {
281        let papers = parse_search_response(FIXTURE).unwrap();
282        for p in &papers {
283            assert_eq!(p.open_access, Some(true), "HAL papers are always OA");
284        }
285    }
286
287    #[test]
288    fn search_returns_papers() {
289        let mut server = mockito::Server::new();
290        let mock = server
291            .mock("GET", mockito::Matcher::Any)
292            .with_status(200)
293            .with_body(FIXTURE)
294            .create();
295        let papers = search(
296            &server.url(),
297            &crate::sources::SearchQuery::simple("test", 3),
298        )
299        .unwrap();
300        assert!(!papers.is_empty());
301        mock.assert();
302    }
303
304    #[test]
305    fn search_request_path() {
306        let mut server = mockito::Server::new();
307        let mock = server
308            .mock("GET", mockito::Matcher::Regex("/search/".to_string()))
309            .with_status(200)
310            .with_body(FIXTURE)
311            .create();
312        let _ = search(
313            &server.url(),
314            &crate::sources::SearchQuery::simple("test", 3),
315        );
316        mock.assert();
317    }
318
319    #[test]
320    fn search_request_contains_wt_json() {
321        let mut server = mockito::Server::new();
322        let mock = server
323            .mock("GET", mockito::Matcher::Regex("wt=json".to_string()))
324            .with_status(200)
325            .with_body(FIXTURE)
326            .create();
327        let _ = search(
328            &server.url(),
329            &crate::sources::SearchQuery::simple("test", 3),
330        );
331        mock.assert();
332    }
333
334    #[test]
335    fn search_request_contains_rows() {
336        let mut server = mockito::Server::new();
337        let mock = server
338            .mock("GET", mockito::Matcher::Regex("rows=3".to_string()))
339            .with_status(200)
340            .with_body(FIXTURE)
341            .create();
342        let _ = search(
343            &server.url(),
344            &crate::sources::SearchQuery::simple("test", 3),
345        );
346        mock.assert();
347    }
348
349    #[test]
350    fn search_request_contains_fl() {
351        let mut server = mockito::Server::new();
352        let mock = server
353            .mock("GET", mockito::Matcher::Regex("fl=".to_string()))
354            .with_status(200)
355            .with_body(FIXTURE)
356            .create();
357        let _ = search(
358            &server.url(),
359            &crate::sources::SearchQuery::simple("test", 3),
360        );
361        mock.assert();
362    }
363}
364
365#[cfg(test)]
366mod query_tests {
367    use super::*;
368    use crate::sources::{SearchQuery, SortField, SortOrder};
369
370    fn url(q: &SearchQuery) -> String {
371        build_search_url("https://api.archives-ouvertes.fr", q).unwrap()
372    }
373
374    #[test]
375    fn author_becomes_a_solr_term_on_the_query() {
376        let mut q = SearchQuery::simple("crispr", 10);
377        q.author = Some("Aubry".into());
378        assert!(
379            url(&q).contains("authFullName_s:%22Aubry%22"),
380            "got: {}",
381            url(&q)
382        );
383    }
384
385    #[test]
386    fn year_becomes_a_filter_query() {
387        let mut q = SearchQuery::simple("crispr", 10);
388        q.year = Some(2024);
389        assert!(
390            url(&q).contains("fq=publicationDateY_i:2024"),
391            "got: {}",
392            url(&q)
393        );
394    }
395
396    #[test]
397    fn open_access_becomes_a_filter_query() {
398        let mut q = SearchQuery::simple("crispr", 10);
399        q.open_access = true;
400        assert!(
401            url(&q).contains("fq=openAccess_bool:true"),
402            "got: {}",
403            url(&q)
404        );
405    }
406
407    // domain_s holds hierarchical codes ("0.sdv", "1.sdv.bbm"), so a plain
408    // "sdv" only matches wrapped in wildcards.
409    #[test]
410    fn field_is_wrapped_in_wildcards() {
411        let mut q = SearchQuery::simple("crispr", 10);
412        q.field = Some("sdv".into());
413        assert!(url(&q).contains("fq=domain_s:*sdv*"), "got: {}", url(&q));
414    }
415
416    #[test]
417    fn offset_becomes_start() {
418        let mut q = SearchQuery::simple("crispr", 10);
419        q.offset = 30;
420        assert!(url(&q).contains("start=30"));
421    }
422
423    #[test]
424    fn sort_by_date_uses_the_publication_year_field() {
425        let mut q = SearchQuery::simple("crispr", 10);
426        q.sort = Some(SortField::Date);
427        q.order = SortOrder::Asc;
428        assert!(
429            url(&q).contains("sort=publicationDateY_i+asc"),
430            "got: {}",
431            url(&q)
432        );
433    }
434
435    #[test]
436    fn sort_by_citations_is_rejected() {
437        let mut q = SearchQuery::simple("crispr", 10);
438        q.sort = Some(SortField::Citations);
439        assert!(build_search_url("https://api.archives-ouvertes.fr", &q).is_err());
440    }
441}
442
443#[cfg(test)]
444mod get_tests {
445    use super::*;
446
447    const FIXTURE: &str = include_str!("../../tests/fixtures/hal_search.json");
448
449    #[test]
450    fn get_by_id_queries_hal_id_for_a_hal_identifier() {
451        let mut server = mockito::Server::new();
452        let m = server
453            .mock("GET", mockito::Matcher::Regex("halId_s".to_string()))
454            .with_status(200)
455            .with_body(FIXTURE)
456            .create();
457        let _ = get_by_id(&server.url(), "hal-00000001");
458        m.assert();
459    }
460
461    // HAL has no by-id path, so a DOI has to target the DOI field instead.
462    #[test]
463    fn get_by_id_queries_doi_id_for_a_doi() {
464        let mut server = mockito::Server::new();
465        let m = server
466            .mock("GET", mockito::Matcher::Regex("doiId_s".to_string()))
467            .with_status(200)
468            .with_body(FIXTURE)
469            .create();
470        let _ = get_by_id(&server.url(), "10.1038/nature12373");
471        m.assert();
472    }
473
474    #[test]
475    fn get_by_id_returns_the_first_document() {
476        let mut server = mockito::Server::new();
477        server
478            .mock("GET", mockito::Matcher::Any)
479            .with_status(200)
480            .with_body(FIXTURE)
481            .create();
482        assert!(get_by_id(&server.url(), "hal-00000001").unwrap().is_some());
483    }
484}
fastpaper/sources/hal.rs

fastpaper/sources/
hal.rs
