1use clap::ValueEnum;
11
12use crate::download;
13use crate::sources::{self, Capabilities, Direction, Paper, SearchCaps, SearchQuery};
14
15#[derive(ValueEnum, Clone, Copy, Debug, PartialEq, Eq)]
16pub enum Source {
17 Arxiv,
18 Biorxiv,
19 Medrxiv,
20 Pubmed,
21 Pmc,
22 Europepmc,
23 Scholar,
24 Xueshu,
25 Semantic,
26 Crossref,
27 Openalex,
28 Dblp,
29 Core,
30 Openaire,
31 Doaj,
32 Unpaywall,
33 Zenodo,
34 Hal,
35}
36
37pub const ALL: &[Source] = &[
39 Source::Arxiv,
40 Source::Biorxiv,
41 Source::Medrxiv,
42 Source::Pubmed,
43 Source::Pmc,
44 Source::Europepmc,
45 Source::Scholar,
46 Source::Xueshu,
47 Source::Semantic,
48 Source::Crossref,
49 Source::Openalex,
50 Source::Dblp,
51 Source::Core,
52 Source::Openaire,
53 Source::Doaj,
54 Source::Unpaywall,
55 Source::Zenodo,
56 Source::Hal,
57];
58
59pub struct SourceEntry {
60 pub name: &'static str,
61 pub caps: Capabilities,
62 pub env_var: &'static str,
64 pub default_base: &'static str,
65 pub pdf_env_var: Option<&'static str>,
67 pub pdf_default_base: Option<&'static str>,
68 pub search: Option<fn(&str, &SearchQuery) -> Result<Vec<Paper>, String>>,
69 pub get: Option<fn(&str, &str) -> Result<Option<Paper>, String>>,
70 pub pdf: Option<fn(&str, &str) -> Result<Vec<u8>, String>>,
71 pub cite: Option<fn(&str, &str, Direction, u32) -> Result<Vec<Paper>, String>>,
72}
73
74impl Source {
75 pub fn name(&self) -> &'static str {
76 self.entry().name
77 }
78
79 pub fn caps(&self) -> Capabilities {
80 self.entry().caps
81 }
82
83 pub fn base_url(&self) -> String {
85 let e = self.entry();
86 std::env::var(e.env_var).unwrap_or_else(|_| e.default_base.to_string())
87 }
88
89 pub fn pdf_base_url(&self) -> String {
96 let e = self.entry();
97 if let Some(var) = e.pdf_env_var {
98 if let Ok(value) = std::env::var(var) {
99 return value;
100 }
101 }
102 if let Ok(value) = std::env::var(e.env_var) {
103 return value;
104 }
105 e.pdf_default_base.unwrap_or(e.default_base).to_string()
106 }
107
108 pub fn from_name(token: &str) -> Option<Source> {
111 ALL.iter().copied().find(|s| s.name() == token)
112 }
113
114 pub fn entry(&self) -> &'static SourceEntry {
115 match self {
116 Source::Arxiv => &ARXIV,
117 Source::Biorxiv => &BIORXIV,
118 Source::Medrxiv => &MEDRXIV,
119 Source::Pubmed => &PUBMED,
120 Source::Pmc => &PMC,
121 Source::Europepmc => &EUROPEPMC,
122 Source::Scholar => &SCHOLAR,
123 Source::Xueshu => &XUESHU,
124 Source::Semantic => &SEMANTIC,
125 Source::Crossref => &CROSSREF,
126 Source::Openalex => &OPENALEX,
127 Source::Dblp => &DBLP,
128 Source::Core => &CORE,
129 Source::Openaire => &OPENAIRE,
130 Source::Doaj => &DOAJ,
131 Source::Unpaywall => &UNPAYWALL,
132 Source::Zenodo => &ZENODO,
133 Source::Hal => &HAL,
134 }
135 }
136}
137
138fn g_unpaywall(base: &str, id: &str) -> Result<Option<Paper>, String> {
142 sources::unpaywall::lookup_doi(base, id).map(Some)
143}
144
145static ARXIV: SourceEntry = SourceEntry {
148 name: "arxiv",
149 caps: Capabilities {
150 search: Some(SearchCaps {
151 offset: true,
152 sort: true,
153 year: true,
154 date_range: true,
155 author: true,
156 field: true,
157 open_access: true,
160 patents: false,
161 }),
162 get: true,
163 download: true,
164 cite: false,
165 max_limit: Some(2000),
166 notes: "--sort citations is unavailable: arXiv publishes no citation counts",
167 },
168 env_var: "FASTPAPER_ARXIV_URL",
169 default_base: "https://export.arxiv.org",
170 pdf_env_var: Some("FASTPAPER_ARXIV_PDF_URL"),
171 pdf_default_base: Some("https://arxiv.org"),
172 search: Some(sources::arxiv::search),
173 get: Some(sources::arxiv::get_by_id),
174 pdf: Some(download::pdf_bytes_arxiv),
175 cite: None,
176};
177
178static BIORXIV: SourceEntry = SourceEntry {
179 name: "biorxiv",
180 caps: Capabilities {
181 search: Some(SearchCaps {
182 offset: true,
183 sort: false,
184 year: true,
185 date_range: true,
186 author: false,
187 field: false,
188 open_access: true,
190 patents: false,
191 }),
192 get: true,
193 download: true,
194 cite: false,
195 max_limit: None,
196 notes: "no keyword search API: browses a date window and matches the \
197 keyword locally, so --after/--before/--year decide what is searched",
198 },
199 env_var: "FASTPAPER_BIORXIV_URL",
200 default_base: "https://api.biorxiv.org",
201 pdf_env_var: Some("FASTPAPER_BIORXIV_DL_URL"),
202 pdf_default_base: Some("https://www.biorxiv.org"),
203 search: Some(sources::biorxiv::search),
204 get: Some(sources::biorxiv::get_by_id),
205 pdf: Some(download::pdf_bytes_biorxiv),
206 cite: None,
207};
208
209static MEDRXIV: SourceEntry = SourceEntry {
210 name: "medrxiv",
211 caps: Capabilities {
212 search: Some(SearchCaps {
213 offset: true,
214 sort: false,
215 year: true,
216 date_range: true,
217 author: false,
218 field: false,
219 open_access: true,
221 patents: false,
222 }),
223 get: true,
224 download: false,
225 cite: false,
226 max_limit: None,
227 notes: "no keyword search API: browses a date window and matches the \
228 keyword locally, so --after/--before/--year decide what is \
229 searched; PDF downloads are blocked by medRxiv (HTTP 403)",
230 },
231 env_var: "FASTPAPER_MEDRXIV_URL",
232 default_base: "https://api.biorxiv.org",
233 pdf_env_var: Some("FASTPAPER_MEDRXIV_DL_URL"),
234 pdf_default_base: Some("https://www.medrxiv.org"),
235 search: Some(sources::medrxiv::search),
236 get: Some(sources::medrxiv::get_by_id),
237 pdf: None,
238 cite: None,
239};
240
241static PUBMED: SourceEntry = SourceEntry {
242 name: "pubmed",
243 caps: Capabilities {
244 search: Some(SearchCaps {
245 offset: true,
246 sort: true,
247 year: true,
248 date_range: true,
249 author: true,
250 field: false,
253 open_access: false,
254 patents: false,
255 }),
256 get: true,
257 download: false,
258 cite: false,
259 max_limit: Some(10000),
260 notes: "metadata only, for full text try pmc; --sort citations unavailable",
261 },
262 env_var: "FASTPAPER_PUBMED_URL",
263 default_base: "https://eutils.ncbi.nlm.nih.gov",
264 pdf_env_var: None,
265 pdf_default_base: None,
266 search: Some(sources::pubmed::search),
267 get: Some(sources::pubmed::get_by_pmid),
268 pdf: None,
269 cite: None,
270};
271
272static PMC: SourceEntry = SourceEntry {
273 name: "pmc",
274 caps: Capabilities {
275 search: Some(SearchCaps {
276 offset: true,
277 sort: true,
278 year: true,
279 date_range: true,
280 author: true,
281 field: false,
282 open_access: true,
283 patents: false,
284 }),
285 get: true,
286 download: true,
287 cite: false,
288 max_limit: Some(10000),
289 notes: "--sort citations unavailable; PDFs come from the OA subset only",
290 },
291 env_var: "FASTPAPER_PMC_URL",
292 default_base: "https://eutils.ncbi.nlm.nih.gov",
293 pdf_env_var: Some("FASTPAPER_PMC_DL_URL"),
294 pdf_default_base: Some("https://pmc-oa-opendata.s3.amazonaws.com"),
298 search: Some(sources::pmc::search),
299 get: Some(sources::pmc::get_by_pmc_id),
300 pdf: Some(download::pdf_bytes_pmc),
301 cite: None,
302};
303
304static EUROPEPMC: SourceEntry = SourceEntry {
305 name: "europepmc",
306 caps: Capabilities {
307 search: Some(SearchCaps {
308 offset: true,
309 sort: true,
310 year: true,
311 date_range: true,
312 author: true,
313 field: false,
314 open_access: true,
315 patents: true,
316 }),
317 get: true,
318 download: true,
319 cite: false,
320 max_limit: Some(1000),
321 notes: "orders newest/most-cited first only, --order asc unavailable; \
322 --offset must be a multiple of -n; PDFs come from the open \
323 access subset",
324 },
325 env_var: "FASTPAPER_EUROPEPMC_URL",
326 default_base: "https://www.ebi.ac.uk",
327 pdf_env_var: None,
328 pdf_default_base: None,
329 search: Some(sources::europepmc::search),
330 get: Some(sources::europepmc::get_by_id),
331 pdf: Some(download::pdf_bytes_europepmc),
332 cite: None,
333};
334
335static SCHOLAR: SourceEntry = SourceEntry {
336 name: "scholar",
337 caps: Capabilities {
338 search: Some(SearchCaps {
339 offset: true,
340 ..SearchCaps::BASIC
341 }),
342 get: false,
343 download: false,
344 cite: false,
345 max_limit: None,
346 notes: "HTML scraping, no official API; brittle and rate-limited",
347 },
348 env_var: "FASTPAPER_SCHOLAR_URL",
349 default_base: "https://scholar.google.com",
350 pdf_env_var: None,
351 pdf_default_base: None,
352 search: Some(sources::scholar::search),
353 get: None,
354 pdf: None,
355 cite: None,
356};
357
358static XUESHU: SourceEntry = SourceEntry {
359 name: "xueshu",
360 caps: Capabilities {
361 search: Some(SearchCaps {
362 offset: true,
363 patents: true,
364 ..SearchCaps::BASIC
365 }),
366 get: false,
367 download: false,
368 cite: false,
369 max_limit: None,
370 notes: "unofficial endpoint; subject to bot detection; --patents filters \
371 locally, so it pages further to fill -n",
372 },
373 env_var: "FASTPAPER_XUESHU_URL",
374 default_base: "https://xueshu.baidu.com",
375 pdf_env_var: None,
376 pdf_default_base: None,
377 search: Some(sources::xueshu::search),
378 get: None,
379 pdf: None,
380 cite: None,
381};
382
383static SEMANTIC: SourceEntry = SourceEntry {
384 name: "semantic",
385 caps: Capabilities {
386 search: Some(SearchCaps {
387 offset: true,
388 sort: true,
389 year: true,
390 date_range: true,
391 author: false,
393 field: true,
394 open_access: true,
395 patents: false,
396 }),
397 get: true,
398 download: true,
399 cite: true,
400 max_limit: Some(100),
401 notes: "--sort switches to the bulk endpoint, which pages by token, so it \
402 cannot be combined with --offset; set SEMANTIC_SCHOLAR_API_KEY to \
403 avoid heavy throttling",
404 },
405 env_var: "FASTPAPER_SEMANTIC_URL",
406 default_base: "https://api.semanticscholar.org",
407 pdf_env_var: None,
408 pdf_default_base: None,
409 search: Some(sources::semantic::search),
410 get: Some(sources::semantic::get_by_id),
411 pdf: Some(download::pdf_bytes_semantic),
412 cite: Some(sources::semantic::cite),
413};
414
415static CROSSREF: SourceEntry = SourceEntry {
416 name: "crossref",
417 caps: Capabilities {
418 search: Some(SearchCaps {
419 offset: true,
420 sort: true,
421 year: true,
422 date_range: true,
423 author: true,
424 field: false,
427 open_access: false,
428 patents: false,
429 }),
430 get: true,
431 download: false,
432 cite: false,
433 max_limit: Some(1000),
434 notes: "metadata only, no PDF links; --offset caps at 10000",
435 },
436 env_var: "FASTPAPER_CROSSREF_URL",
437 default_base: "https://api.crossref.org",
438 pdf_env_var: None,
439 pdf_default_base: None,
440 search: Some(sources::crossref::search),
441 get: Some(sources::crossref::get_by_doi),
442 pdf: None,
443 cite: None,
444};
445
446static OPENALEX: SourceEntry = SourceEntry {
447 name: "openalex",
448 caps: Capabilities {
449 search: Some(SearchCaps {
450 offset: true,
451 sort: true,
452 year: true,
453 date_range: true,
454 author: true,
455 field: true,
456 open_access: true,
457 patents: false,
458 }),
459 get: true,
460 download: false,
461 cite: true,
462 max_limit: Some(100),
463 notes: "usage-metered since 2026-02, set OPENALEX_API_KEY for the larger free tier; \
464 --field takes a concept ID (e.g. C154945302), not a name; \
465 --offset must be a multiple of -n",
466 },
467 env_var: "FASTPAPER_OPENALEX_URL",
468 default_base: "https://api.openalex.org",
469 pdf_env_var: None,
470 pdf_default_base: None,
471 search: Some(sources::openalex::search),
472 get: Some(sources::openalex::get_by_id),
473 pdf: None,
474 cite: Some(sources::openalex::cite),
475};
476
477static DBLP: SourceEntry = SourceEntry {
478 name: "dblp",
479 caps: Capabilities {
480 search: Some(SearchCaps {
481 offset: true,
482 ..SearchCaps::BASIC
483 }),
484 get: false,
485 download: false,
486 cite: false,
487 max_limit: Some(1000),
488 notes: "computer science only, metadata only; the API takes a query and \
489 paging, nothing else",
490 },
491 env_var: "FASTPAPER_DBLP_URL",
492 default_base: "https://dblp.uni-trier.de",
494 pdf_env_var: None,
495 pdf_default_base: None,
496 search: Some(sources::dblp::search),
497 get: None,
498 pdf: None,
499 cite: None,
500};
501
502static CORE: SourceEntry = SourceEntry {
503 name: "core",
504 caps: Capabilities {
505 search: Some(SearchCaps {
506 offset: true,
507 year: true,
508 author: true,
509 ..SearchCaps::BASIC
510 }),
511 get: true,
512 download: true,
513 cite: false,
514 max_limit: Some(100),
515 notes: "CORE_API_KEY is effectively required -- anonymous requests are \
516 throttled to 429; --author matches authors.name",
517 },
518 env_var: "FASTPAPER_CORE_URL",
519 default_base: "https://api.core.ac.uk",
520 pdf_env_var: None,
521 pdf_default_base: None,
522 search: Some(sources::core::search),
523 get: Some(sources::core::get_by_id),
524 pdf: Some(download::pdf_bytes_core),
525 cite: None,
526};
527
528static OPENAIRE: SourceEntry = SourceEntry {
529 name: "openaire",
530 caps: Capabilities {
531 search: Some(SearchCaps {
532 offset: true,
533 sort: true,
534 year: true,
535 date_range: true,
536 author: true,
537 field: true,
538 open_access: true,
539 patents: false,
540 }),
541 get: true,
542 download: false,
543 cite: false,
544 max_limit: None,
545 notes: "--field takes an OpenAIRE field-of-science value; an unrecognised \
546 one is answered with the allowed list",
547 },
548 env_var: "FASTPAPER_OPENAIRE_URL",
549 default_base: "https://api.openaire.eu",
550 pdf_env_var: None,
551 pdf_default_base: None,
552 search: Some(sources::openaire::search),
553 get: Some(sources::openaire::get_by_id),
554 pdf: None,
555 cite: None,
556};
557
558static DOAJ: SourceEntry = SourceEntry {
559 name: "doaj",
560 caps: Capabilities {
561 search: Some(SearchCaps {
562 offset: true,
563 sort: false,
566 year: true,
567 date_range: false,
570 author: true,
571 field: false,
572 open_access: true,
575 patents: false,
576 }),
577 get: true,
578 download: false,
579 cite: false,
580 max_limit: Some(100),
581 notes: "year granularity only, use --year rather than --after/--before; \
582 no PDFs -- its fulltext links point at publisher landing pages, \
583 not files",
584 },
585 env_var: "FASTPAPER_DOAJ_URL",
586 default_base: "https://doaj.org",
587 pdf_env_var: None,
588 pdf_default_base: None,
589 search: Some(sources::doaj::search),
590 get: Some(sources::doaj::get_by_id),
591 pdf: None,
592 cite: None,
593};
594
595static UNPAYWALL: SourceEntry = SourceEntry {
596 name: "unpaywall",
597 caps: Capabilities {
598 search: None,
600 get: true,
601 download: false,
602 cite: false,
603 max_limit: None,
604 notes: "DOI lookup only; requires a real address in FASTPAPER_EMAIL",
605 },
606 env_var: "FASTPAPER_UNPAYWALL_URL",
607 default_base: "https://api.unpaywall.org",
608 pdf_env_var: None,
609 pdf_default_base: None,
610 search: None,
611 get: Some(g_unpaywall),
612 pdf: None,
613 cite: None,
614};
615
616static ZENODO: SourceEntry = SourceEntry {
617 name: "zenodo",
618 caps: Capabilities {
619 search: Some(SearchCaps {
620 offset: true,
621 sort: true,
622 year: true,
623 date_range: true,
624 author: true,
625 field: false,
626 open_access: true,
627 patents: false,
628 }),
629 get: true,
630 download: true,
631 cite: false,
632 max_limit: Some(25),
633 notes: "anonymous callers get 25 results per request; --sort citations \
634 unavailable; --offset must be a multiple of -n",
635 },
636 env_var: "FASTPAPER_ZENODO_URL",
637 default_base: "https://zenodo.org",
638 pdf_env_var: None,
639 pdf_default_base: None,
640 search: Some(sources::zenodo::search),
641 get: Some(sources::zenodo::get_by_id),
642 pdf: Some(download::pdf_bytes_zenodo),
643 cite: None,
644};
645
646static HAL: SourceEntry = SourceEntry {
647 name: "hal",
648 caps: Capabilities {
649 search: Some(SearchCaps {
650 offset: true,
651 sort: true,
652 year: true,
653 date_range: false,
656 author: true,
657 field: true,
658 open_access: true,
659 patents: false,
660 }),
661 get: true,
662 download: true,
663 cite: false,
664 max_limit: Some(10000),
665 notes: "year granularity only, use --year rather than --after/--before; \
666 --field takes a HAL domain code such as sdv or info",
667 },
668 env_var: "FASTPAPER_HAL_URL",
669 default_base: "https://api.archives-ouvertes.fr",
670 pdf_env_var: None,
671 pdf_default_base: None,
672 search: Some(sources::hal::search),
673 get: Some(sources::hal::get_by_id),
674 pdf: Some(download::pdf_bytes_hal),
675 cite: None,
676};
677
678#[cfg(test)]
679mod tests {
680 use super::*;
681
682 #[test]
683 fn every_source_has_a_matching_entry() {
684 for s in ALL {
685 assert_eq!(
686 s.name(),
687 s.entry().name,
688 "entry for {:?} is wired to the wrong record",
689 s
690 );
691 }
692 }
693
694 #[test]
695 fn source_names_are_unique() {
696 let mut names: Vec<_> = ALL.iter().map(|s| s.name()).collect();
697 names.sort_unstable();
698 let before = names.len();
699 names.dedup();
700 assert_eq!(before, names.len(), "duplicate source name in ALL");
701 }
702
703 #[test]
704 fn from_name_round_trips() {
705 for s in ALL {
706 assert_eq!(Source::from_name(s.name()), Some(*s));
707 }
708 }
709
710 #[test]
711 fn from_name_rejects_unknown() {
712 assert_eq!(Source::from_name("nope"), None);
713 assert_eq!(Source::from_name("10.1038/nature12373"), None);
714 }
715
716 #[test]
717 fn download_capable_sources_have_a_pdf_fn() {
718 for s in ALL {
719 let e = s.entry();
720 assert_eq!(
721 e.caps.download,
722 e.pdf.is_some(),
723 "{} declares download={} but pdf fn present={}",
724 e.name,
725 e.caps.download,
726 e.pdf.is_some()
727 );
728 }
729 }
730
731 #[test]
732 fn get_capable_sources_have_a_get_fn() {
733 for s in ALL {
734 let e = s.entry();
735 assert_eq!(
736 e.caps.get,
737 e.get.is_some(),
738 "{} declares get={} but get fn present={}",
739 e.name,
740 e.caps.get,
741 e.get.is_some()
742 );
743 }
744 }
745
746 #[test]
751 fn only_europepmc_and_xueshu_take_patents() {
752 for s in ALL {
753 let declared = s
754 .caps()
755 .search
756 .is_some_and(|caps| caps.supports("--patents"));
757 let expected = matches!(s, Source::Europepmc | Source::Xueshu);
758 assert_eq!(
759 declared,
760 expected,
761 "{} declares --patents support = {}",
762 s.name(),
763 declared
764 );
765 }
766 }
767
768 #[test]
769 fn search_capable_sources_have_a_search_fn() {
770 for s in ALL {
771 let e = s.entry();
772 assert_eq!(
773 e.caps.search.is_some(),
774 e.search.is_some(),
775 "{} declares search={} but search fn present={}",
776 e.name,
777 e.caps.search.is_some(),
778 e.search.is_some()
779 );
780 }
781 }
782
783 #[test]
784 #[serial_test::serial]
785 fn base_url_prefers_env_override() {
786 unsafe { std::env::set_var("FASTPAPER_ARXIV_URL", "http://localhost:1234") };
787 assert_eq!(Source::Arxiv.base_url(), "http://localhost:1234");
788 unsafe { std::env::remove_var("FASTPAPER_ARXIV_URL") };
789 assert_eq!(Source::Arxiv.base_url(), "https://export.arxiv.org");
790 }
791
792 #[test]
793 fn pdf_base_url_falls_back_to_base_url() {
794 assert_eq!(Source::Semantic.pdf_base_url(), Source::Semantic.base_url());
796 }
797
798 #[test]
799 #[serial_test::serial]
800 fn pdf_base_url_defaults_to_the_file_host_not_the_api_host() {
801 unsafe { std::env::remove_var("FASTPAPER_ARXIV_URL") };
802 unsafe { std::env::remove_var("FASTPAPER_ARXIV_PDF_URL") };
803 assert_eq!(Source::Arxiv.base_url(), "https://export.arxiv.org");
804 assert_eq!(Source::Arxiv.pdf_base_url(), "https://arxiv.org");
805 }
806
807 #[test]
810 #[serial_test::serial]
811 fn general_override_still_redirects_pdf_downloads() {
812 unsafe { std::env::remove_var("FASTPAPER_ARXIV_PDF_URL") };
813 unsafe { std::env::set_var("FASTPAPER_ARXIV_URL", "http://localhost:9999") };
814 assert_eq!(Source::Arxiv.pdf_base_url(), "http://localhost:9999");
815 unsafe { std::env::remove_var("FASTPAPER_ARXIV_URL") };
816 }
817
818 #[test]
819 #[serial_test::serial]
820 fn pdf_override_wins_over_the_general_one() {
821 unsafe { std::env::set_var("FASTPAPER_ARXIV_URL", "http://api.test") };
822 unsafe { std::env::set_var("FASTPAPER_ARXIV_PDF_URL", "http://files.test") };
823 assert_eq!(Source::Arxiv.pdf_base_url(), "http://files.test");
824 unsafe { std::env::remove_var("FASTPAPER_ARXIV_URL") };
825 unsafe { std::env::remove_var("FASTPAPER_ARXIV_PDF_URL") };
826 }
827}