fastpaper/sources/
pmc.rs

1use quick_xml::events::Event;
2use quick_xml::Reader;
3
4use super::Paper;
5
6const ESEARCH_URL: &str = "/entrez/eutils/esearch.fcgi";
7const EFETCH_URL: &str = "/entrez/eutils/efetch.fcgi";
8
9/// Search PMC: esearch for IDs, then efetch for details.
10pub fn search(base_url: &str, query: &str, max_results: u32) -> Result<Vec<Paper>, String> {
11    let api_key = std::env::var("NCBI_API_KEY").ok();
12
13    // Step 1: esearch
14    let encoded = super::encode_query(query);
15    let mut esearch_url = format!(
16        "{}{}?db=pmc&term={}&retmax={}&retmode=json&tool=fastpaper&email=yee.zhang@gmail.com",
17        base_url, ESEARCH_URL, encoded, max_results
18    );
19    if let Some(ref key) = api_key {
20        esearch_url.push_str(&format!("&api_key={}", key));
21    }
22
23    let esearch_body = http_get(&esearch_url)?;
24    let esearch_json: serde_json::Value =
25        serde_json::from_str(&esearch_body).map_err(|e| format!("JSON parse error: {}", e))?;
26
27    let ids: Vec<&str> = esearch_json["esearchresult"]["idlist"]
28        .as_array()
29        .map(|arr| arr.iter().filter_map(|v| v.as_str()).collect())
30        .unwrap_or_default();
31
32    if ids.is_empty() {
33        return Ok(vec![]);
34    }
35
36    // Step 2: efetch
37    let mut efetch_url = format!(
38        "{}{}?db=pmc&id={}&rettype=xml&tool=fastpaper&email=yee.zhang@gmail.com",
39        base_url,
40        EFETCH_URL,
41        ids.join(",")
42    );
43    if let Some(ref key) = api_key {
44        efetch_url.push_str(&format!("&api_key={}", key));
45    }
46
47    let efetch_body = http_get(&efetch_url)?;
48    parse_efetch_response(&efetch_body)
49}
50
51fn http_get(url: &str) -> Result<String, String> {
52    let mut last_err = String::new();
53    for attempt in 0..3 {
54        if attempt > 0 {
55            std::thread::sleep(std::time::Duration::from_millis(100 * (1 << attempt)));
56        }
57        match ureq::get(url).call() {
58            Ok(resp) => {
59                return resp
60                    .into_body()
61                    .read_to_string()
62                    .map_err(|e| format!("Failed to read response: {}", e));
63            }
64            Err(ureq::Error::StatusCode(429)) => {
65                last_err = "rate limited (429)".to_string();
66                continue;
67            }
68            Err(ureq::Error::StatusCode(code)) if code >= 500 => {
69                return Err(format!("Server error: {}", code));
70            }
71            Err(e) => {
72                return Err(format!("HTTP error: {}", e));
73            }
74        }
75    }
76    Err(last_err)
77}
78
79/// Fetch a single paper by PMC ID.
80pub fn get_by_pmc_id(base_url: &str, pmc_id: &str) -> Result<Option<Paper>, String> {
81    // Strip "PMC" prefix if present to get numeric ID
82    let numeric_id = pmc_id.strip_prefix("PMC").unwrap_or(pmc_id);
83    let url = format!(
84        "{}/entrez/eutils/efetch.fcgi?db=pmc&id={}&rettype=xml&tool=fastpaper&email=yee.zhang@gmail.com",
85        base_url, numeric_id
86    );
87    let body = http_get(&url)?;
88    let papers = parse_efetch_response(&body)?;
89    Ok(papers.into_iter().next())
90}
91
92/// Parse PMC efetch XML response into a list of Papers.
93pub fn parse_efetch_response(xml: &str) -> Result<Vec<Paper>, String> {
94    let mut reader = Reader::from_str(xml);
95    let mut buf = Vec::new();
96    let mut papers = Vec::new();
97
98    let mut in_article = false;
99    let mut in_front = false;
100    let mut in_contrib_author = false;
101    let mut tag_stack: Vec<String> = Vec::new();
102
103    let mut pmcid = String::new();
104    let mut doi = String::new();
105    let mut title = String::new();
106    let mut authors: Vec<String> = Vec::new();
107    let mut surname = String::new();
108    let mut given_names = String::new();
109    let mut abstract_text = String::new();
110    let mut year = String::new();
111    let mut journal = String::new();
112    let mut in_abstract = false;
113
114    loop {
115        match reader.read_event_into(&mut buf) {
116            Ok(Event::Start(ref e)) => {
117                let name = e.name();
118                let local = local_name(name.as_ref());
119                match local {
120                    "article" => {
121                        in_article = true;
122                        pmcid.clear();
123                        doi.clear();
124                        title.clear();
125                        authors.clear();
126                        abstract_text.clear();
127                        year.clear();
128                        journal.clear();
129                    }
130                    "front" if in_article => in_front = true,
131                    "article-id" if in_front => {
132                        let mut id_type = String::new();
133                        for attr in e.attributes().flatten() {
134                            if attr.key.as_ref() == b"pub-id-type" {
135                                id_type = String::from_utf8_lossy(&attr.value).to_string();
136                            }
137                        }
138                        tag_stack.push(format!("article-id:{}", id_type));
139                        buf.clear();
140                        continue;
141                    }
142                    "contrib" if in_front => {
143                        for attr in e.attributes().flatten() {
144                            if attr.key.as_ref() == b"contrib-type"
145                                && attr.value.as_ref() == b"author"
146                            {
147                                in_contrib_author = true;
148                                surname.clear();
149                                given_names.clear();
150                            }
151                        }
152                    }
153                    "abstract" if in_front => in_abstract = true,
154                    _ => {}
155                }
156                tag_stack.push(local.to_string());
157            }
158            Ok(Event::Text(ref e)) => {
159                if !in_article {
160                    buf.clear();
161                    continue;
162                }
163                let text = e.decode().unwrap_or_default().to_string();
164                let current = tag_stack.last().map(|s| s.as_str()).unwrap_or("");
165                if current == "article-id:pmcid" && pmcid.is_empty() {
166                    pmcid.push_str(text.trim());
167                } else if current == "article-id:doi" && doi.is_empty() {
168                    doi.push_str(text.trim());
169                } else if current == "article-title" && in_front {
170                    title.push_str(&text);
171                } else if current == "surname" && in_contrib_author {
172                    surname.push_str(text.trim());
173                } else if current == "given-names" && in_contrib_author {
174                    given_names.push_str(text.trim());
175                } else if in_abstract && !text.trim().is_empty() {
176                    if !abstract_text.is_empty() {
177                        abstract_text.push(' ');
178                    }
179                    abstract_text.push_str(text.trim());
180                } else if current == "year" && in_front && year.is_empty() {
181                    year.push_str(text.trim());
182                } else if current == "journal-title" && in_front && journal.is_empty() {
183                    journal.push_str(text.trim());
184                }
185            }
186            Ok(Event::End(ref e)) => {
187                let name = e.name();
188                let local = local_name(name.as_ref());
189                match local {
190                    "article" => {
191                        if !pmcid.is_empty() && !title.is_empty() {
192                            papers.push(Paper {
193                                id: pmcid.clone(),
194                                title: title.trim().to_string(),
195                                authors: authors.clone(),
196                                abstract_text: if abstract_text.is_empty() {
197                                    None
198                                } else {
199                                    Some(abstract_text.clone())
200                                },
201                                year: year.parse::<u16>().ok(),
202                                doi: if doi.is_empty() { None } else { Some(doi.clone()) },
203                                url: Some(format!(
204                                    "https://www.ncbi.nlm.nih.gov/pmc/articles/{}/",
205                                    pmcid
206                                )),
207                                pdf_url: Some(format!(
208                                    "https://www.ncbi.nlm.nih.gov/pmc/articles/{}/pdf/",
209                                    pmcid
210                                )),
211                                venue: if journal.is_empty() {
212                                    None
213                                } else {
214                                    Some(journal.clone())
215                                },
216                                citations: None,
217                                fields: vec![],
218                                open_access: Some(true),
219                                source: "pmc".to_string(),
220                            });
221                        }
222                        in_article = false;
223                        in_front = false;
224                    }
225                    "front" => in_front = false,
226                    "contrib" if in_contrib_author => {
227                        let name_str =
228                            format!("{} {}", given_names, surname).trim().to_string();
229                        if !name_str.is_empty() {
230                            authors.push(name_str);
231                        }
232                        in_contrib_author = false;
233                    }
234                    "abstract" => in_abstract = false,
235                    _ => {}
236                }
237                tag_stack.pop();
238            }
239            Ok(Event::Eof) => break,
240            Err(e) => return Err(format!("XML parse error: {}", e)),
241            _ => {}
242        }
243        buf.clear();
244    }
245
246    Ok(papers)
247}
248
249fn local_name(name: &[u8]) -> &str {
250    let s = std::str::from_utf8(name).unwrap_or("");
251    s.rsplit(':').next().unwrap_or(s)
252}
253
254#[cfg(test)]
255mod tests {
256    use super::*;
257    use serial_test::serial;
258
259    const FIXTURE: &str = include_str!("../../tests/fixtures/pmc_efetch.xml");
260
261    #[test]
262    fn parse_returns_ok() {
263        let result = parse_efetch_response(FIXTURE);
264        assert!(result.is_ok());
265    }
266
267    #[test]
268    fn parse_papers_not_empty() {
269        let papers = parse_efetch_response(FIXTURE).unwrap();
270        assert!(!papers.is_empty());
271    }
272
273    #[test]
274    fn parse_source_is_pmc() {
275        let papers = parse_efetch_response(FIXTURE).unwrap();
276        for p in &papers {
277            assert_eq!(p.source, "pmc");
278        }
279    }
280
281    #[test]
282    fn parse_id_is_pmc_format() {
283        let papers = parse_efetch_response(FIXTURE).unwrap();
284        for p in &papers {
285            assert!(p.id.starts_with("PMC"), "id should start with PMC: {}", p.id);
286        }
287    }
288
289    #[test]
290    fn parse_title_not_empty() {
291        let papers = parse_efetch_response(FIXTURE).unwrap();
292        for p in &papers {
293            assert!(!p.title.is_empty(), "paper {} has empty title", p.id);
294        }
295    }
296
297    #[test]
298    fn parse_authors_surname_given() {
299        let papers = parse_efetch_response(FIXTURE).unwrap();
300        let with_authors: Vec<_> = papers.iter().filter(|p| !p.authors.is_empty()).collect();
301        assert!(!with_authors.is_empty(), "no papers with authors");
302    }
303
304    #[test]
305    fn parse_doi() {
306        let papers = parse_efetch_response(FIXTURE).unwrap();
307        let with_doi: Vec<_> = papers.iter().filter(|p| p.doi.is_some()).collect();
308        assert!(!with_doi.is_empty(), "no papers with DOI");
309        for p in &with_doi {
310            assert!(p.doi.as_ref().unwrap().starts_with("10."));
311        }
312    }
313
314    #[test]
315    fn parse_year() {
316        let papers = parse_efetch_response(FIXTURE).unwrap();
317        for p in &papers {
318            assert!(p.year.is_some(), "paper {} missing year", p.id);
319            assert!(p.year.unwrap() >= 2000);
320        }
321    }
322
323    #[test]
324    fn parse_venue() {
325        let papers = parse_efetch_response(FIXTURE).unwrap();
326        let with_venue: Vec<_> = papers.iter().filter(|p| p.venue.is_some()).collect();
327        assert!(!with_venue.is_empty(), "no papers with venue");
328    }
329
330    #[test]
331    fn parse_abstract() {
332        let papers = parse_efetch_response(FIXTURE).unwrap();
333        let with_abstract: Vec<_> = papers.iter().filter(|p| p.abstract_text.is_some()).collect();
334        assert!(!with_abstract.is_empty(), "no papers with abstract");
335        for p in &with_abstract {
336            assert!(p.abstract_text.as_ref().unwrap().len() > 10);
337        }
338    }
339
340    const ESEARCH_FIXTURE: &str = include_str!("../../tests/fixtures/pmc_esearch.json");
341
342    #[test]
343    fn search_calls_esearch_then_efetch() {
344        let mut server = mockito::Server::new();
345        let esearch_mock = server
346            .mock("GET", mockito::Matcher::Regex("esearch".to_string()))
347            .with_status(200)
348            .with_body(ESEARCH_FIXTURE)
349            .create();
350        let efetch_mock = server
351            .mock("GET", mockito::Matcher::Regex("efetch".to_string()))
352            .with_status(200)
353            .with_body(FIXTURE)
354            .create();
355        let papers = search(&server.url(), "test", 3).unwrap();
356        assert!(!papers.is_empty());
357        esearch_mock.assert();
358        efetch_mock.assert();
359    }
360
361    #[test]
362    fn search_esearch_contains_db_pmc() {
363        let mut server = mockito::Server::new();
364        let mock = server
365            .mock("GET", mockito::Matcher::Regex("esearch.*db=pmc".to_string()))
366            .with_status(200)
367            .with_body(ESEARCH_FIXTURE)
368            .create();
369        server
370            .mock("GET", mockito::Matcher::Any)
371            .with_status(200)
372            .with_body(FIXTURE)
373            .create();
374        let _ = search(&server.url(), "test", 3);
375        mock.assert();
376    }
377
378    #[test]
379    fn search_efetch_contains_rettype_xml() {
380        let mut server = mockito::Server::new();
381        server
382            .mock("GET", mockito::Matcher::Regex("esearch".to_string()))
383            .with_status(200)
384            .with_body(ESEARCH_FIXTURE)
385            .create();
386        let mock = server
387            .mock("GET", mockito::Matcher::Regex("efetch.*rettype=xml".to_string()))
388            .with_status(200)
389            .with_body(FIXTURE)
390            .create();
391        let _ = search(&server.url(), "test", 3);
392        mock.assert();
393    }
394
395    #[test]
396    #[serial]
397    fn search_with_api_key() {
398        unsafe { std::env::set_var("NCBI_API_KEY", "pmc-test-key") };
399        let mut server = mockito::Server::new();
400        server
401            .mock("GET", mockito::Matcher::Regex("esearch.*api_key=pmc-test-key".to_string()))
402            .with_status(200)
403            .with_body(ESEARCH_FIXTURE)
404            .create();
405        server
406            .mock("GET", mockito::Matcher::Regex("efetch.*api_key=pmc-test-key".to_string()))
407            .with_status(200)
408            .with_body(FIXTURE)
409            .create();
410        let result = search(&server.url(), "test", 3);
411        unsafe { std::env::remove_var("NCBI_API_KEY") };
412        assert!(result.is_ok());
413    }
414
415    #[test]
416    fn search_empty_ids_skips_efetch() {
417        let empty_esearch = r#"{"esearchresult":{"idlist":[]}}"#;
418        let mut server = mockito::Server::new();
419        server
420            .mock("GET", mockito::Matcher::Regex("esearch".to_string()))
421            .with_status(200)
422            .with_body(empty_esearch)
423            .create();
424        // efetch should NOT be called
425        let efetch_mock = server
426            .mock("GET", mockito::Matcher::Regex("efetch".to_string()))
427            .with_status(200)
428            .with_body(FIXTURE)
429            .expect(0)
430            .create();
431        let papers = search(&server.url(), "nonexistent", 3).unwrap();
432        assert!(papers.is_empty());
433        efetch_mock.assert();
434    }
435}
fastpaper/sources/pmc.rs

fastpaper/sources/
pmc.rs
