1use quick_xml::Reader;
2use quick_xml::events::Event;
3
4use super::Paper;
5
6const ESEARCH_URL: &str = "/entrez/eutils/esearch.fcgi";
7const EFETCH_URL: &str = "/entrez/eutils/efetch.fcgi";
8
9fn build_esearch_url_q(
15 base_url: &str,
16 q: &super::SearchQuery,
17 api_key: Option<&str>,
18 email: Option<&str>,
19) -> Result<String, String> {
20 let mut term = super::encode_query(&q.query);
21 if let Some(ref author) = q.author {
22 term.push_str(&format!("+AND+{}%5Bau%5D", super::encode_query(author)));
23 }
24 if let Some(year) = q.year {
25 term.push_str(&format!("+AND+{}%5Bdp%5D", year));
26 }
27
28 let mut url = format!(
29 "{}{}?db=pubmed&term={}&retmax={}&retstart={}&retmode=json&tool=fastpaper",
30 base_url, ESEARCH_URL, term, q.limit, q.offset
31 );
32
33 if q.year.is_none() && (q.after.is_some() || q.before.is_some()) {
35 url.push_str("&datetype=pdat");
36 let min = match q.after {
37 Some(ref d) => super::validate_ymd(d)?.replace('-', "/"),
38 None => "1800/01/01".to_string(),
39 };
40 let max = match q.before {
41 Some(ref d) => super::validate_ymd(d)?.replace('-', "/"),
42 None => "3000/12/31".to_string(),
43 };
44 url.push_str(&format!("&mindate={}&maxdate={}", min, max));
45 }
46
47 if let Some(sort) = q.sort {
48 let by = match sort {
49 super::SortField::Relevance => "relevance",
50 super::SortField::Date => "pub_date",
51 super::SortField::Citations => {
52 return Err(
53 "pubmed cannot sort by citations: E-utilities offers no citation ordering.\n\
54 Try semantic or openalex."
55 .to_string(),
56 );
57 }
58 };
59 url.push_str(&format!("&sort={}", by));
60 }
61
62 if let Some(email) = email {
63 url.push_str(&format!("&email={}", email));
64 }
65 if let Some(key) = api_key {
66 url.push_str(&format!("&api_key={}", key));
67 }
68 Ok(url)
69}
70
71fn build_efetch_url(
72 base_url: &str,
73 ids: &str,
74 api_key: Option<&str>,
75 email: Option<&str>,
76) -> String {
77 let mut url = format!(
78 "{}{}?db=pubmed&id={}&retmode=xml&tool=fastpaper",
79 base_url, EFETCH_URL, ids
80 );
81 if let Some(email) = email {
82 url.push_str(&format!("&email={}", email));
83 }
84 if let Some(key) = api_key {
85 url.push_str(&format!("&api_key={}", key));
86 }
87 url
88}
89
90pub fn search(base_url: &str, q: &super::SearchQuery) -> Result<Vec<Paper>, String> {
92 let api_key = std::env::var("NCBI_API_KEY").ok();
93 let email = super::contact_email();
94
95 let esearch_url = build_esearch_url_q(base_url, q, api_key.as_deref(), email.as_deref())?;
97
98 let esearch_body = http_get(&esearch_url)?;
99 let esearch_json: serde_json::Value =
100 serde_json::from_str(&esearch_body).map_err(|e| format!("JSON parse error: {}", e))?;
101
102 let ids: Vec<&str> = esearch_json["esearchresult"]["idlist"]
103 .as_array()
104 .map(|arr| arr.iter().filter_map(|v| v.as_str()).collect())
105 .unwrap_or_default();
106
107 if ids.is_empty() {
108 return Ok(vec![]);
109 }
110
111 let efetch_url = build_efetch_url(
113 base_url,
114 &ids.join(","),
115 api_key.as_deref(),
116 email.as_deref(),
117 );
118
119 let efetch_body = http_get(&efetch_url)?;
120 parse_efetch_response(&efetch_body)
121}
122
123fn http_get(url: &str) -> Result<String, String> {
124 let mut last_err = String::new();
125 for attempt in 0..3 {
126 if attempt > 0 {
127 std::thread::sleep(std::time::Duration::from_millis(100 * (1 << attempt)));
128 }
129 match ureq::get(url).call() {
130 Ok(resp) => {
131 return resp
132 .into_body()
133 .read_to_string()
134 .map_err(|e| format!("Failed to read response: {}", e));
135 }
136 Err(ureq::Error::StatusCode(429)) => {
137 last_err = "rate limited (429)".to_string();
138 continue;
139 }
140 Err(ureq::Error::StatusCode(code)) if code >= 500 => {
141 return Err(format!("Server error: {}", code));
142 }
143 Err(e) => {
144 return Err(format!("HTTP error: {}", e));
145 }
146 }
147 }
148 Err(last_err)
149}
150
151pub fn get_by_pmid(base_url: &str, pmid: &str) -> Result<Option<Paper>, String> {
153 let api_key = std::env::var("NCBI_API_KEY").ok();
154 let email = super::contact_email();
155 let url = build_efetch_url(base_url, pmid, api_key.as_deref(), email.as_deref());
156 let body = http_get(&url)?;
157 let papers = parse_efetch_response(&body)?;
158 Ok(papers.into_iter().next())
159}
160
161pub fn parse_efetch_response(xml: &str) -> Result<Vec<Paper>, String> {
163 let mut reader = Reader::from_str(xml);
164 let mut buf = Vec::new();
165 let mut papers = Vec::new();
166
167 let mut in_article = false;
169 let mut in_author = false;
170 let mut tag_stack: Vec<String> = Vec::new();
171 let mut pmid = String::new();
172 let mut title = String::new();
173 let mut authors: Vec<String> = Vec::new();
174 let mut last_name = String::new();
175 let mut initials = String::new();
176 let mut abstract_text = String::new();
177 let mut year = String::new();
178 let mut doi = String::new();
179
180 loop {
181 match reader.read_event_into(&mut buf) {
182 Ok(Event::Start(ref e)) => {
183 let name = e.name();
184 let local = local_name(name.as_ref());
185 match local {
186 "PubmedArticle" => {
187 in_article = true;
188 pmid.clear();
189 title.clear();
190 authors.clear();
191 abstract_text.clear();
192 year.clear();
193 doi.clear();
194 }
195 "Author" if in_article => {
196 in_author = true;
197 last_name.clear();
198 initials.clear();
199 }
200 "ELocationID" if in_article => {
201 let mut is_doi = false;
203 for attr in e.attributes().flatten() {
204 if attr.key.as_ref() == b"EIdType" && attr.value.as_ref() == b"doi" {
205 is_doi = true;
206 }
207 }
208 if is_doi {
209 tag_stack.push("doi".to_string());
210 } else {
211 tag_stack.push(local.to_string());
212 }
213 buf.clear();
214 continue;
215 }
216 _ => {}
217 }
218 tag_stack.push(local.to_string());
219 }
220 Ok(Event::Text(ref e)) => {
221 if !in_article {
222 buf.clear();
223 continue;
224 }
225 let text = e.decode().unwrap_or_default().to_string();
226 let current = tag_stack.last().map(|s| s.as_str()).unwrap_or("");
227 match current {
228 "PMID" if pmid.is_empty() => pmid.push_str(text.trim()),
229 "ArticleTitle" => title.push_str(&text),
230 "LastName" if in_author => last_name.push_str(text.trim()),
231 "Initials" if in_author => initials.push_str(text.trim()),
232 "AbstractText" => {
233 if !abstract_text.is_empty() {
234 abstract_text.push(' ');
235 }
236 abstract_text.push_str(text.trim());
237 }
238 "Year"
243 if year.is_empty()
244 && tag_stack
245 .iter()
246 .rev()
247 .nth(1)
248 .map(|parent| parent == "PubDate")
249 .unwrap_or(false) =>
250 {
251 year.push_str(text.trim())
252 }
253 "doi" => doi.push_str(text.trim()),
254 _ => {}
255 }
256 }
257 Ok(Event::End(ref e)) => {
258 let name = e.name();
259 let local = local_name(name.as_ref());
260 match local {
261 "PubmedArticle" => {
262 if !pmid.is_empty() && !title.is_empty() {
263 papers.push(Paper {
264 id: pmid.clone(),
265 title: title.trim().to_string(),
266 authors: authors.clone(),
267 abstract_text: if abstract_text.is_empty() {
268 None
269 } else {
270 Some(abstract_text.clone())
271 },
272 year: year.parse::<u16>().ok(),
273 doi: if doi.is_empty() {
274 None
275 } else {
276 Some(doi.clone())
277 },
278 url: Some(format!("https://pubmed.ncbi.nlm.nih.gov/{}/", pmid)),
279 pdf_url: None,
280 venue: None,
281 citations: None,
282 fields: vec![],
283 open_access: None,
284 source: "pubmed".to_string(),
285 });
286 }
287 in_article = false;
288 }
289 "Author" if in_author => {
290 let name_str = format!("{} {}", last_name, initials).trim().to_string();
291 if !name_str.is_empty() {
292 authors.push(name_str);
293 }
294 in_author = false;
295 }
296 _ => {}
297 }
298 tag_stack.pop();
299 }
300 Ok(Event::Eof) => break,
301 Err(e) => return Err(format!("XML parse error: {}", e)),
302 _ => {}
303 }
304 buf.clear();
305 }
306
307 Ok(papers)
308}
309
310fn local_name(name: &[u8]) -> &str {
311 let s = std::str::from_utf8(name).unwrap_or("");
312 s.rsplit(':').next().unwrap_or(s)
313}
314
315#[cfg(test)]
316mod tests {
317 use super::*;
318 use serial_test::serial;
319
320 const FIXTURE: &str = include_str!("../../tests/fixtures/pubmed_efetch.xml");
321
322 #[test]
323 fn parse_returns_ok() {
324 let result = parse_efetch_response(FIXTURE);
325 assert!(result.is_ok());
326 }
327
328 #[test]
329 fn parse_papers_not_empty() {
330 let papers = parse_efetch_response(FIXTURE).unwrap();
331 assert!(!papers.is_empty());
332 }
333
334 #[test]
335 fn parse_source_is_pubmed() {
336 let papers = parse_efetch_response(FIXTURE).unwrap();
337 for p in &papers {
338 assert_eq!(p.source, "pubmed");
339 }
340 }
341
342 #[test]
343 fn parse_title_from_article_title() {
344 let papers = parse_efetch_response(FIXTURE).unwrap();
345 for p in &papers {
346 assert!(!p.title.is_empty(), "paper {} has empty title", p.id);
347 }
348 }
349
350 #[test]
351 fn parse_authors_lastname_initials() {
352 let papers = parse_efetch_response(FIXTURE).unwrap();
353 let with_authors: Vec<_> = papers.iter().filter(|p| !p.authors.is_empty()).collect();
354 assert!(!with_authors.is_empty(), "no papers with authors");
355 for p in &with_authors {
356 for a in &p.authors {
357 assert!(!a.is_empty());
358 assert!(a.len() > 1, "author name too short: {}", a);
360 }
361 }
362 }
363
364 #[test]
365 fn parse_abstract_from_abstract_text() {
366 let papers = parse_efetch_response(FIXTURE).unwrap();
367 let with_abstract: Vec<_> = papers
368 .iter()
369 .filter(|p| p.abstract_text.is_some())
370 .collect();
371 assert!(!with_abstract.is_empty(), "no papers with abstract");
372 for p in &with_abstract {
373 assert!(p.abstract_text.as_ref().unwrap().len() > 10);
374 }
375 }
376
377 #[test]
378 fn parse_year_from_pubdate() {
379 let papers = parse_efetch_response(FIXTURE).unwrap();
380 for p in &papers {
381 assert!(p.year.is_some(), "paper {} missing year", p.id);
382 assert!(p.year.unwrap() >= 2000);
383 }
384 }
385
386 #[test]
387 fn parse_doi_from_elocationid() {
388 let papers = parse_efetch_response(FIXTURE).unwrap();
389 let with_doi: Vec<_> = papers.iter().filter(|p| p.doi.is_some()).collect();
390 assert!(!with_doi.is_empty(), "no papers with DOI");
391 for p in &with_doi {
392 let doi = p.doi.as_ref().unwrap();
393 assert!(doi.starts_with("10."), "DOI should start with 10.: {}", doi);
394 }
395 }
396
397 #[test]
398 fn parse_id_is_pmid() {
399 let papers = parse_efetch_response(FIXTURE).unwrap();
400 for p in &papers {
401 assert!(
402 p.id.chars().all(|c| c.is_ascii_digit()),
403 "PMID should be numeric: {}",
404 p.id
405 );
406 }
407 }
408
409 #[test]
410 fn parse_pdf_url_is_none() {
411 let papers = parse_efetch_response(FIXTURE).unwrap();
412 for p in &papers {
413 assert!(p.pdf_url.is_none(), "pubmed should not have pdf_url");
414 }
415 }
416
417 const ESEARCH_FIXTURE: &str = include_str!("../../tests/fixtures/pubmed_esearch.json");
418
419 #[test]
420 fn search_calls_esearch_then_efetch() {
421 let mut server = mockito::Server::new();
422 let esearch_mock = server
423 .mock("GET", mockito::Matcher::Regex("esearch".to_string()))
424 .with_status(200)
425 .with_body(ESEARCH_FIXTURE)
426 .create();
427 let efetch_mock = server
428 .mock("GET", mockito::Matcher::Regex("efetch".to_string()))
429 .with_status(200)
430 .with_body(FIXTURE)
431 .create();
432 let papers = search(
433 &server.url(),
434 &crate::sources::SearchQuery::simple("test", 3),
435 )
436 .unwrap();
437 assert!(!papers.is_empty());
438 esearch_mock.assert();
439 efetch_mock.assert();
440 }
441
442 #[test]
443 fn search_esearch_contains_db_pubmed() {
444 let mut server = mockito::Server::new();
445 let mock = server
446 .mock("GET", mockito::Matcher::Regex("db=pubmed".to_string()))
447 .with_status(200)
448 .with_body(ESEARCH_FIXTURE)
449 .expect_at_least(1)
450 .create();
451 server
453 .mock("GET", mockito::Matcher::Any)
454 .with_status(200)
455 .with_body(FIXTURE)
456 .create();
457 let _ = search(
458 &server.url(),
459 &crate::sources::SearchQuery::simple("test", 3),
460 );
461 mock.assert();
462 }
463
464 #[test]
465 fn search_esearch_contains_retmax() {
466 let mut server = mockito::Server::new();
467 let esearch_mock = server
468 .mock("GET", mockito::Matcher::Regex("retmax=3".to_string()))
469 .with_status(200)
470 .with_body(ESEARCH_FIXTURE)
471 .create();
472 server
473 .mock("GET", mockito::Matcher::Any)
474 .with_status(200)
475 .with_body(FIXTURE)
476 .create();
477 let _ = search(
478 &server.url(),
479 &crate::sources::SearchQuery::simple("test", 3),
480 );
481 esearch_mock.assert();
482 }
483
484 #[test]
485 #[serial]
486 fn search_with_api_key() {
487 unsafe { std::env::set_var("NCBI_API_KEY", "test-ncbi-key") };
488 let mut server = mockito::Server::new();
489 server
491 .mock(
492 "GET",
493 mockito::Matcher::Regex("esearch.*api_key=test-ncbi-key".to_string()),
494 )
495 .with_status(200)
496 .with_body(ESEARCH_FIXTURE)
497 .create();
498 server
499 .mock(
500 "GET",
501 mockito::Matcher::Regex("efetch.*api_key=test-ncbi-key".to_string()),
502 )
503 .with_status(200)
504 .with_body(FIXTURE)
505 .create();
506 let result = search(
507 &server.url(),
508 &crate::sources::SearchQuery::simple("test", 3),
509 );
510 unsafe { std::env::remove_var("NCBI_API_KEY") };
511 assert!(
512 result.is_ok(),
513 "search should succeed with api key: {:?}",
514 result.err()
515 );
516 }
517
518 #[test]
519 #[serial]
520 fn search_works_without_api_key() {
521 unsafe { std::env::remove_var("NCBI_API_KEY") };
522 let mut server = mockito::Server::new();
523 server
524 .mock("GET", mockito::Matcher::Regex("esearch".to_string()))
525 .with_status(200)
526 .with_body(ESEARCH_FIXTURE)
527 .create();
528 server
529 .mock("GET", mockito::Matcher::Regex("efetch".to_string()))
530 .with_status(200)
531 .with_body(FIXTURE)
532 .create();
533 let result = search(
534 &server.url(),
535 &crate::sources::SearchQuery::simple("test", 3),
536 );
537 assert!(result.is_ok());
538 }
539
540 #[test]
545 fn build_esearch_url_omits_email_when_none() {
546 let url = build_esearch_url_q(
547 "https://eutils.ncbi.nlm.nih.gov",
548 &crate::sources::SearchQuery::simple("test", 3),
549 None,
550 None,
551 )
552 .unwrap();
553 assert!(!url.contains("email="), "got: {}", url);
554 assert!(url.contains("tool=fastpaper"), "tool should stay: {}", url);
555 }
556
557 #[test]
558 fn build_esearch_url_includes_email_when_some() {
559 let url = build_esearch_url_q(
560 "https://eutils.ncbi.nlm.nih.gov",
561 &crate::sources::SearchQuery::simple("test", 3),
562 None,
563 Some("a@b.com"),
564 )
565 .unwrap();
566 assert!(url.contains("email=a@b.com"), "got: {}", url);
567 }
568
569 #[test]
570 fn build_efetch_url_omits_email_when_none() {
571 let url = build_efetch_url("https://eutils.ncbi.nlm.nih.gov", "123,456", None, None);
572 assert!(!url.contains("email="), "got: {}", url);
573 }
574
575 #[test]
576 #[serial]
577 fn search_sends_env_email_when_set() {
578 unsafe { std::env::set_var("FASTPAPER_EMAIL", "custom@example.com") };
579 let mut server = mockito::Server::new();
580 let mock = server
581 .mock(
582 "GET",
583 mockito::Matcher::Regex("esearch.*email=custom".to_string()),
584 )
585 .with_status(200)
586 .with_body(ESEARCH_FIXTURE)
587 .create();
588 server
589 .mock("GET", mockito::Matcher::Regex("efetch".to_string()))
590 .with_status(200)
591 .with_body(FIXTURE)
592 .create();
593 let _ = search(
594 &server.url(),
595 &crate::sources::SearchQuery::simple("test", 3),
596 );
597 unsafe { std::env::remove_var("FASTPAPER_EMAIL") };
598 mock.assert();
599 }
600}
601
602#[cfg(test)]
603mod query_tests {
604 use super::*;
605 use crate::sources::{SearchQuery, SortField};
606
607 fn url(q: &SearchQuery) -> String {
608 build_esearch_url_q("https://eutils.ncbi.nlm.nih.gov", q, None, None).unwrap()
609 }
610
611 #[test]
613 fn author_becomes_an_au_field_tag() {
614 let mut q = SearchQuery::simple("crispr", 10);
615 q.author = Some("Doudna".into());
616 let u = url(&q);
617 assert!(u.contains("Doudna%5Bau%5D"), "got: {}", u);
618 assert!(u.contains("+AND+"), "got: {}", u);
619 }
620
621 #[test]
622 fn year_becomes_a_dp_field_tag() {
623 let mut q = SearchQuery::simple("crispr", 10);
624 q.year = Some(2017);
625 assert!(url(&q).contains("2017%5Bdp%5D"), "got: {}", url(&q));
626 }
627
628 #[test]
629 fn offset_becomes_retstart() {
630 let mut q = SearchQuery::simple("crispr", 10);
631 q.offset = 20;
632 assert!(url(&q).contains("retstart=20"));
633 }
634
635 #[test]
636 fn dates_use_datetype_mindate_maxdate() {
637 let mut q = SearchQuery::simple("crispr", 10);
638 q.after = Some("2024-01-01".into());
639 q.before = Some("2024-03-31".into());
640 let u = url(&q);
641 assert!(u.contains("datetype=pdat"), "got: {}", u);
642 assert!(u.contains("mindate=2024/01/01"), "got: {}", u);
643 assert!(u.contains("maxdate=2024/03/31"), "got: {}", u);
644 }
645
646 #[test]
647 fn year_takes_precedence_over_a_date_range() {
648 let mut q = SearchQuery::simple("crispr", 10);
649 q.year = Some(2017);
650 q.after = Some("2024-01-01".into());
651 let u = url(&q);
652 assert!(u.contains("2017%5Bdp%5D"), "got: {}", u);
653 assert!(
654 !u.contains("mindate"),
655 "should not also send a range: {}",
656 u
657 );
658 }
659
660 #[test]
661 fn malformed_date_is_rejected() {
662 let mut q = SearchQuery::simple("crispr", 10);
663 q.before = Some("March 2024".into());
664 let err =
665 build_esearch_url_q("https://eutils.ncbi.nlm.nih.gov", &q, None, None).unwrap_err();
666 assert!(err.contains("YYYY-MM-DD"), "got: {}", err);
667 }
668
669 #[test]
670 fn sort_by_date_uses_pub_date() {
671 let mut q = SearchQuery::simple("crispr", 10);
672 q.sort = Some(SortField::Date);
673 assert!(url(&q).contains("sort=pub_date"));
674 }
675
676 #[test]
677 fn sort_by_citations_is_rejected_with_alternatives() {
678 let mut q = SearchQuery::simple("crispr", 10);
679 q.sort = Some(SortField::Citations);
680 let err =
681 build_esearch_url_q("https://eutils.ncbi.nlm.nih.gov", &q, None, None).unwrap_err();
682 assert!(err.contains("semantic"), "got: {}", err);
683 }
684
685 #[test]
686 fn a_filterless_query_is_unchanged() {
687 let u = url(&SearchQuery::simple("crispr", 10));
688 assert!(u.contains("term=crispr&"), "got: {}", u);
689 assert!(!u.contains("AND"), "no filter terms expected: {}", u);
690 }
691}
692
693#[cfg(test)]
694mod year_tests {
695 use super::*;
696
697 const MIXED_DATES: &str = r#"<?xml version="1.0"?>
702<PubmedArticleSet><PubmedArticle><MedlineCitation>
703<PMID Version="1">35349206</PMID>
704<DateCompleted><Year>2022</Year><Month>04</Month><Day>04</Day></DateCompleted>
705<DateRevised><Year>2022</Year><Month>04</Month><Day>05</Day></DateRevised>
706<Article PubModel="Print-Electronic"><Journal><JournalIssue CitedMedium="Internet">
707<PubDate><Year>2021</Year><Month>Mar</Month></PubDate>
708</JournalIssue><Title>Reviews in medical virology</Title></Journal>
709<ArticleTitle>SARS-CoV-2: Mechanism of infection.</ArticleTitle>
710</Article></MedlineCitation></PubmedArticle></PubmedArticleSet>"#;
711
712 #[test]
713 fn year_comes_from_pubdate_not_the_nlm_processing_date() {
714 let papers = parse_efetch_response(MIXED_DATES).unwrap();
715 assert_eq!(papers.len(), 1);
716 assert_eq!(
717 papers[0].year,
718 Some(2021),
719 "should report the PubDate year, not DateCompleted"
720 );
721 }
722}