fastpaper/sources/
pmc.rs

1use quick_xml::Reader;
2use quick_xml::events::Event;
3
4use super::Paper;
5
6const ESEARCH_URL: &str = "/entrez/eutils/esearch.fcgi";
7const EFETCH_URL: &str = "/entrez/eutils/efetch.fcgi";
8
9/// Build the esearch URL for a full query.
10///
11/// Like PubMed, PMC carries author, year and the open access restriction as
12/// Entrez tags inside `term`; date ranges use datetype/mindate/maxdate.
13fn build_esearch_url_q(
14    base_url: &str,
15    q: &super::SearchQuery,
16    api_key: Option<&str>,
17    email: Option<&str>,
18) -> Result<String, String> {
19    let mut term = super::encode_query(&q.query);
20    if let Some(ref author) = q.author {
21        term.push_str(&format!("+AND+{}%5Bau%5D", super::encode_query(author)));
22    }
23    if let Some(year) = q.year {
24        term.push_str(&format!("+AND+{}%5Bdp%5D", year));
25    }
26    if q.open_access {
27        term.push_str("+AND+open+access%5Bfilter%5D");
28    }
29
30    let mut url = format!(
31        "{}{}?db=pmc&term={}&retmax={}&retstart={}&retmode=json&tool=fastpaper",
32        base_url, ESEARCH_URL, term, q.limit, q.offset
33    );
34
35    if q.year.is_none() && (q.after.is_some() || q.before.is_some()) {
36        url.push_str("&datetype=pdat");
37        let min = match q.after {
38            Some(ref d) => super::validate_ymd(d)?.replace('-', "/"),
39            None => "1800/01/01".to_string(),
40        };
41        let max = match q.before {
42            Some(ref d) => super::validate_ymd(d)?.replace('-', "/"),
43            None => "3000/12/31".to_string(),
44        };
45        url.push_str(&format!("&mindate={}&maxdate={}", min, max));
46    }
47
48    if let Some(sort) = q.sort {
49        let by = match sort {
50            super::SortField::Relevance => "relevance",
51            super::SortField::Date => "pub_date",
52            super::SortField::Citations => {
53                return Err(
54                    "pmc cannot sort by citations: E-utilities offers no citation ordering.\n\
55                     Try semantic or openalex."
56                        .to_string(),
57                );
58            }
59        };
60        url.push_str(&format!("&sort={}", by));
61    }
62
63    if let Some(email) = email {
64        url.push_str(&format!("&email={}", email));
65    }
66    if let Some(key) = api_key {
67        url.push_str(&format!("&api_key={}", key));
68    }
69    Ok(url)
70}
71
72fn build_efetch_url(
73    base_url: &str,
74    ids: &str,
75    api_key: Option<&str>,
76    email: Option<&str>,
77) -> String {
78    let mut url = format!(
79        "{}{}?db=pmc&id={}&rettype=xml&tool=fastpaper",
80        base_url, EFETCH_URL, ids
81    );
82    if let Some(email) = email {
83        url.push_str(&format!("&email={}", email));
84    }
85    if let Some(key) = api_key {
86        url.push_str(&format!("&api_key={}", key));
87    }
88    url
89}
90
91/// Search PMC: esearch for IDs, then efetch for details.
92pub fn search(base_url: &str, q: &super::SearchQuery) -> Result<Vec<Paper>, String> {
93    let api_key = std::env::var("NCBI_API_KEY").ok();
94    let email = super::contact_email();
95
96    // Step 1: esearch
97    let esearch_url = build_esearch_url_q(base_url, q, api_key.as_deref(), email.as_deref())?;
98
99    let esearch_body = http_get(&esearch_url)?;
100    let esearch_json: serde_json::Value =
101        serde_json::from_str(&esearch_body).map_err(|e| format!("JSON parse error: {}", e))?;
102
103    let ids: Vec<&str> = esearch_json["esearchresult"]["idlist"]
104        .as_array()
105        .map(|arr| arr.iter().filter_map(|v| v.as_str()).collect())
106        .unwrap_or_default();
107
108    if ids.is_empty() {
109        return Ok(vec![]);
110    }
111
112    // Step 2: efetch
113    let efetch_url = build_efetch_url(
114        base_url,
115        &ids.join(","),
116        api_key.as_deref(),
117        email.as_deref(),
118    );
119
120    let efetch_body = http_get(&efetch_url)?;
121    parse_efetch_response(&efetch_body)
122}
123
124fn http_get(url: &str) -> Result<String, String> {
125    let mut last_err = String::new();
126    for attempt in 0..3 {
127        if attempt > 0 {
128            std::thread::sleep(std::time::Duration::from_millis(100 * (1 << attempt)));
129        }
130        match ureq::get(url).call() {
131            Ok(resp) => {
132                return resp
133                    .into_body()
134                    .read_to_string()
135                    .map_err(|e| format!("Failed to read response: {}", e));
136            }
137            Err(ureq::Error::StatusCode(429)) => {
138                last_err = "rate limited (429)".to_string();
139                continue;
140            }
141            Err(ureq::Error::StatusCode(code)) if code >= 500 => {
142                return Err(format!("Server error: {}", code));
143            }
144            Err(e) => {
145                return Err(format!("HTTP error: {}", e));
146            }
147        }
148    }
149    Err(last_err)
150}
151
152/// Fetch a single paper by PMC ID.
153pub fn get_by_pmc_id(base_url: &str, pmc_id: &str) -> Result<Option<Paper>, String> {
154    // Strip "PMC" prefix if present to get numeric ID
155    let numeric_id = pmc_id.strip_prefix("PMC").unwrap_or(pmc_id);
156    let api_key = std::env::var("NCBI_API_KEY").ok();
157    let email = super::contact_email();
158    let url = build_efetch_url(base_url, numeric_id, api_key.as_deref(), email.as_deref());
159    let body = http_get(&url)?;
160    let papers = parse_efetch_response(&body)?;
161    Ok(papers.into_iter().next())
162}
163
164/// Parse PMC efetch XML response into a list of Papers.
165pub fn parse_efetch_response(xml: &str) -> Result<Vec<Paper>, String> {
166    let mut reader = Reader::from_str(xml);
167    let mut buf = Vec::new();
168    let mut papers = Vec::new();
169
170    let mut in_article = false;
171    let mut in_front = false;
172    let mut in_contrib_author = false;
173    let mut tag_stack: Vec<String> = Vec::new();
174
175    let mut pmcid = String::new();
176    let mut doi = String::new();
177    let mut title = String::new();
178    let mut authors: Vec<String> = Vec::new();
179    let mut surname = String::new();
180    let mut given_names = String::new();
181    let mut abstract_text = String::new();
182    let mut year = String::new();
183    let mut journal = String::new();
184    let mut in_abstract = false;
185
186    loop {
187        match reader.read_event_into(&mut buf) {
188            Ok(Event::Start(ref e)) => {
189                let name = e.name();
190                let local = local_name(name.as_ref());
191                match local {
192                    "article" => {
193                        in_article = true;
194                        pmcid.clear();
195                        doi.clear();
196                        title.clear();
197                        authors.clear();
198                        abstract_text.clear();
199                        year.clear();
200                        journal.clear();
201                    }
202                    "front" if in_article => in_front = true,
203                    "article-id" if in_front => {
204                        let mut id_type = String::new();
205                        for attr in e.attributes().flatten() {
206                            if attr.key.as_ref() == b"pub-id-type" {
207                                id_type = String::from_utf8_lossy(&attr.value).to_string();
208                            }
209                        }
210                        tag_stack.push(format!("article-id:{}", id_type));
211                        buf.clear();
212                        continue;
213                    }
214                    "contrib" if in_front => {
215                        for attr in e.attributes().flatten() {
216                            if attr.key.as_ref() == b"contrib-type"
217                                && attr.value.as_ref() == b"author"
218                            {
219                                in_contrib_author = true;
220                                surname.clear();
221                                given_names.clear();
222                            }
223                        }
224                    }
225                    "abstract" if in_front => in_abstract = true,
226                    _ => {}
227                }
228                tag_stack.push(local.to_string());
229            }
230            Ok(Event::Text(ref e)) => {
231                if !in_article {
232                    buf.clear();
233                    continue;
234                }
235                let text = e.decode().unwrap_or_default().to_string();
236                let current = tag_stack.last().map(|s| s.as_str()).unwrap_or("");
237                if current == "article-id:pmcid" && pmcid.is_empty() {
238                    pmcid.push_str(text.trim());
239                } else if current == "article-id:doi" && doi.is_empty() {
240                    doi.push_str(text.trim());
241                } else if current == "article-title" && in_front {
242                    title.push_str(&text);
243                } else if current == "surname" && in_contrib_author {
244                    surname.push_str(text.trim());
245                } else if current == "given-names" && in_contrib_author {
246                    given_names.push_str(text.trim());
247                } else if in_abstract && !text.trim().is_empty() {
248                    if !abstract_text.is_empty() {
249                        abstract_text.push(' ');
250                    }
251                    abstract_text.push_str(text.trim());
252                } else if current == "year" && in_front && year.is_empty() {
253                    year.push_str(text.trim());
254                } else if current == "journal-title" && in_front && journal.is_empty() {
255                    journal.push_str(text.trim());
256                }
257            }
258            Ok(Event::End(ref e)) => {
259                let name = e.name();
260                let local = local_name(name.as_ref());
261                match local {
262                    "article" => {
263                        if !pmcid.is_empty() && !title.is_empty() {
264                            papers.push(Paper {
265                                id: pmcid.clone(),
266                                title: title.trim().to_string(),
267                                authors: authors.clone(),
268                                abstract_text: if abstract_text.is_empty() {
269                                    None
270                                } else {
271                                    Some(abstract_text.clone())
272                                },
273                                year: year.parse::<u16>().ok(),
274                                doi: if doi.is_empty() {
275                                    None
276                                } else {
277                                    Some(doi.clone())
278                                },
279                                url: Some(format!(
280                                    "https://www.ncbi.nlm.nih.gov/pmc/articles/{}/",
281                                    pmcid
282                                )),
283                                pdf_url: Some(format!(
284                                    "https://www.ncbi.nlm.nih.gov/pmc/articles/{}/pdf/",
285                                    pmcid
286                                )),
287                                venue: if journal.is_empty() {
288                                    None
289                                } else {
290                                    Some(journal.clone())
291                                },
292                                citations: None,
293                                fields: vec![],
294                                open_access: Some(true),
295                                source: "pmc".to_string(),
296                            });
297                        }
298                        in_article = false;
299                        in_front = false;
300                    }
301                    "front" => in_front = false,
302                    "contrib" if in_contrib_author => {
303                        let name_str = format!("{} {}", given_names, surname).trim().to_string();
304                        if !name_str.is_empty() {
305                            authors.push(name_str);
306                        }
307                        in_contrib_author = false;
308                    }
309                    "abstract" => in_abstract = false,
310                    _ => {}
311                }
312                tag_stack.pop();
313            }
314            Ok(Event::Eof) => break,
315            Err(e) => return Err(format!("XML parse error: {}", e)),
316            _ => {}
317        }
318        buf.clear();
319    }
320
321    Ok(papers)
322}
323
324fn local_name(name: &[u8]) -> &str {
325    let s = std::str::from_utf8(name).unwrap_or("");
326    s.rsplit(':').next().unwrap_or(s)
327}
328
329#[cfg(test)]
330mod tests {
331    use super::*;
332    use serial_test::serial;
333
334    const FIXTURE: &str = include_str!("../../tests/fixtures/pmc_efetch.xml");
335
336    #[test]
337    fn parse_returns_ok() {
338        let result = parse_efetch_response(FIXTURE);
339        assert!(result.is_ok());
340    }
341
342    #[test]
343    fn parse_papers_not_empty() {
344        let papers = parse_efetch_response(FIXTURE).unwrap();
345        assert!(!papers.is_empty());
346    }
347
348    #[test]
349    fn parse_source_is_pmc() {
350        let papers = parse_efetch_response(FIXTURE).unwrap();
351        for p in &papers {
352            assert_eq!(p.source, "pmc");
353        }
354    }
355
356    #[test]
357    fn parse_id_is_pmc_format() {
358        let papers = parse_efetch_response(FIXTURE).unwrap();
359        for p in &papers {
360            assert!(
361                p.id.starts_with("PMC"),
362                "id should start with PMC: {}",
363                p.id
364            );
365        }
366    }
367
368    #[test]
369    fn parse_title_not_empty() {
370        let papers = parse_efetch_response(FIXTURE).unwrap();
371        for p in &papers {
372            assert!(!p.title.is_empty(), "paper {} has empty title", p.id);
373        }
374    }
375
376    #[test]
377    fn parse_authors_surname_given() {
378        let papers = parse_efetch_response(FIXTURE).unwrap();
379        let with_authors: Vec<_> = papers.iter().filter(|p| !p.authors.is_empty()).collect();
380        assert!(!with_authors.is_empty(), "no papers with authors");
381    }
382
383    #[test]
384    fn parse_doi() {
385        let papers = parse_efetch_response(FIXTURE).unwrap();
386        let with_doi: Vec<_> = papers.iter().filter(|p| p.doi.is_some()).collect();
387        assert!(!with_doi.is_empty(), "no papers with DOI");
388        for p in &with_doi {
389            assert!(p.doi.as_ref().unwrap().starts_with("10."));
390        }
391    }
392
393    #[test]
394    fn parse_year() {
395        let papers = parse_efetch_response(FIXTURE).unwrap();
396        for p in &papers {
397            assert!(p.year.is_some(), "paper {} missing year", p.id);
398            assert!(p.year.unwrap() >= 2000);
399        }
400    }
401
402    #[test]
403    fn parse_venue() {
404        let papers = parse_efetch_response(FIXTURE).unwrap();
405        let with_venue: Vec<_> = papers.iter().filter(|p| p.venue.is_some()).collect();
406        assert!(!with_venue.is_empty(), "no papers with venue");
407    }
408
409    #[test]
410    fn parse_abstract() {
411        let papers = parse_efetch_response(FIXTURE).unwrap();
412        let with_abstract: Vec<_> = papers
413            .iter()
414            .filter(|p| p.abstract_text.is_some())
415            .collect();
416        assert!(!with_abstract.is_empty(), "no papers with abstract");
417        for p in &with_abstract {
418            assert!(p.abstract_text.as_ref().unwrap().len() > 10);
419        }
420    }
421
422    const ESEARCH_FIXTURE: &str = include_str!("../../tests/fixtures/pmc_esearch.json");
423
424    #[test]
425    fn search_calls_esearch_then_efetch() {
426        let mut server = mockito::Server::new();
427        let esearch_mock = server
428            .mock("GET", mockito::Matcher::Regex("esearch".to_string()))
429            .with_status(200)
430            .with_body(ESEARCH_FIXTURE)
431            .create();
432        let efetch_mock = server
433            .mock("GET", mockito::Matcher::Regex("efetch".to_string()))
434            .with_status(200)
435            .with_body(FIXTURE)
436            .create();
437        let papers = search(
438            &server.url(),
439            &crate::sources::SearchQuery::simple("test", 3),
440        )
441        .unwrap();
442        assert!(!papers.is_empty());
443        esearch_mock.assert();
444        efetch_mock.assert();
445    }
446
447    #[test]
448    fn search_esearch_contains_db_pmc() {
449        let mut server = mockito::Server::new();
450        let mock = server
451            .mock(
452                "GET",
453                mockito::Matcher::Regex("esearch.*db=pmc".to_string()),
454            )
455            .with_status(200)
456            .with_body(ESEARCH_FIXTURE)
457            .create();
458        server
459            .mock("GET", mockito::Matcher::Any)
460            .with_status(200)
461            .with_body(FIXTURE)
462            .create();
463        let _ = search(
464            &server.url(),
465            &crate::sources::SearchQuery::simple("test", 3),
466        );
467        mock.assert();
468    }
469
470    #[test]
471    fn search_efetch_contains_rettype_xml() {
472        let mut server = mockito::Server::new();
473        server
474            .mock("GET", mockito::Matcher::Regex("esearch".to_string()))
475            .with_status(200)
476            .with_body(ESEARCH_FIXTURE)
477            .create();
478        let mock = server
479            .mock(
480                "GET",
481                mockito::Matcher::Regex("efetch.*rettype=xml".to_string()),
482            )
483            .with_status(200)
484            .with_body(FIXTURE)
485            .create();
486        let _ = search(
487            &server.url(),
488            &crate::sources::SearchQuery::simple("test", 3),
489        );
490        mock.assert();
491    }
492
493    #[test]
494    #[serial]
495    fn search_with_api_key() {
496        unsafe { std::env::set_var("NCBI_API_KEY", "pmc-test-key") };
497        let mut server = mockito::Server::new();
498        server
499            .mock(
500                "GET",
501                mockito::Matcher::Regex("esearch.*api_key=pmc-test-key".to_string()),
502            )
503            .with_status(200)
504            .with_body(ESEARCH_FIXTURE)
505            .create();
506        server
507            .mock(
508                "GET",
509                mockito::Matcher::Regex("efetch.*api_key=pmc-test-key".to_string()),
510            )
511            .with_status(200)
512            .with_body(FIXTURE)
513            .create();
514        let result = search(
515            &server.url(),
516            &crate::sources::SearchQuery::simple("test", 3),
517        );
518        unsafe { std::env::remove_var("NCBI_API_KEY") };
519        assert!(result.is_ok());
520    }
521
522    #[test]
523    fn search_empty_ids_skips_efetch() {
524        let empty_esearch = r#"{"esearchresult":{"idlist":[]}}"#;
525        let mut server = mockito::Server::new();
526        server
527            .mock("GET", mockito::Matcher::Regex("esearch".to_string()))
528            .with_status(200)
529            .with_body(empty_esearch)
530            .create();
531        // efetch should NOT be called
532        let efetch_mock = server
533            .mock("GET", mockito::Matcher::Regex("efetch".to_string()))
534            .with_status(200)
535            .with_body(FIXTURE)
536            .expect(0)
537            .create();
538        let papers = search(
539            &server.url(),
540            &crate::sources::SearchQuery::simple("nonexistent", 3),
541        )
542        .unwrap();
543        assert!(papers.is_empty());
544        efetch_mock.assert();
545    }
546
547    // See the matching test in pubmed.rs: never send the maintainer's address
548    // as the caller identity.
549    #[test]
550    fn build_esearch_url_omits_email_when_none() {
551        let url = build_esearch_url_q(
552            "https://eutils.ncbi.nlm.nih.gov",
553            &crate::sources::SearchQuery::simple("test", 3),
554            None,
555            None,
556        )
557        .unwrap();
558        assert!(!url.contains("email="), "got: {}", url);
559        assert!(url.contains("db=pmc"), "got: {}", url);
560    }
561
562    #[test]
563    fn build_efetch_url_omits_email_when_none() {
564        let url = build_efetch_url("https://eutils.ncbi.nlm.nih.gov", "7318926", None, None);
565        assert!(!url.contains("email="), "got: {}", url);
566    }
567
568    #[test]
569    fn build_efetch_url_includes_email_when_some() {
570        let url = build_efetch_url(
571            "https://eutils.ncbi.nlm.nih.gov",
572            "7318926",
573            None,
574            Some("a@b.com"),
575        );
576        assert!(url.contains("email=a@b.com"), "got: {}", url);
577    }
578
579    #[test]
580    #[serial]
581    fn get_by_pmc_id_sends_env_email_when_set() {
582        unsafe { std::env::set_var("FASTPAPER_EMAIL", "custom@example.com") };
583        let mut server = mockito::Server::new();
584        let mock = server
585            .mock("GET", mockito::Matcher::Regex("email=custom".to_string()))
586            .with_status(200)
587            .with_body(FIXTURE)
588            .create();
589        let _ = get_by_pmc_id(&server.url(), "PMC7318926");
590        unsafe { std::env::remove_var("FASTPAPER_EMAIL") };
591        mock.assert();
592    }
593}
594
595#[cfg(test)]
596mod query_tests {
597    use super::*;
598    use crate::sources::SearchQuery;
599
600    fn url(q: &SearchQuery) -> String {
601        build_esearch_url_q("https://eutils.ncbi.nlm.nih.gov", q, None, None).unwrap()
602    }
603
604    #[test]
605    fn author_becomes_an_au_field_tag() {
606        let mut q = SearchQuery::simple("crispr", 10);
607        q.author = Some("Doudna".into());
608        assert!(url(&q).contains("Doudna%5Bau%5D"), "got: {}", url(&q));
609    }
610
611    // Unlike PubMed, PMC can restrict to its open access subset.
612    #[test]
613    fn open_access_uses_the_entrez_filter() {
614        let mut q = SearchQuery::simple("crispr", 10);
615        q.open_access = true;
616        assert!(
617            url(&q).contains("open+access%5Bfilter%5D"),
618            "got: {}",
619            url(&q)
620        );
621    }
622
623    #[test]
624    fn offset_becomes_retstart() {
625        let mut q = SearchQuery::simple("crispr", 10);
626        q.offset = 20;
627        assert!(url(&q).contains("retstart=20"));
628    }
629
630    #[test]
631    fn dates_use_datetype_mindate_maxdate() {
632        let mut q = SearchQuery::simple("crispr", 10);
633        q.after = Some("2024-01-01".into());
634        let u = url(&q);
635        assert!(u.contains("datetype=pdat"), "got: {}", u);
636        assert!(u.contains("mindate=2024/01/01"), "got: {}", u);
637    }
638
639    #[test]
640    fn query_still_targets_the_pmc_database() {
641        assert!(url(&SearchQuery::simple("crispr", 10)).contains("db=pmc"));
642    }
643}
fastpaper/sources/pmc.rs

fastpaper/sources/
pmc.rs
