paper_gap/
openalex.rs

1use crate::{Result, arxiv, paper::Paper};
2use chrono::NaiveDate;
3use serde::Deserialize;
4use std::{collections::HashMap, time::Duration};
5
6const BASE_URL: &str = "https://api.openalex.org/works";
7
8#[derive(Debug, Clone)]
9pub struct SearchQuery {
10    pub query: Option<String>,
11    pub from: Option<NaiveDate>,
12    pub to: Option<NaiveDate>,
13    pub limit: usize,
14}
15
16pub fn search_url(query: &SearchQuery, email: Option<&str>) -> String {
17    let mut url = format!(
18        "{BASE_URL}?per-page={}&sort=publication_date:desc",
19        query.limit
20    );
21    if let Some(search) = query
22        .query
23        .as_deref()
24        .filter(|query| !query.trim().is_empty())
25    {
26        url.push_str("&search=");
27        url.push_str(&urlencoding::encode(search));
28    }
29    let mut filters = Vec::new();
30    if let Some(from) = query.from {
31        filters.push(format!("from_publication_date:{from}"));
32    }
33    if let Some(to) = query.to {
34        filters.push(format!("to_publication_date:{to}"));
35    }
36    if !filters.is_empty() {
37        url.push_str("&filter=");
38        url.push_str(&urlencoding::encode(&filters.join(",")));
39    }
40    if let Some(email) = email.filter(|email| !email.trim().is_empty()) {
41        url.push_str("&mailto=");
42        url.push_str(&urlencoding::encode(email));
43    }
44    url
45}
46
47pub async fn search(query: &SearchQuery) -> Result<Vec<Paper>> {
48    let client = reqwest::Client::builder()
49        .user_agent("paper-gap/0.2")
50        .timeout(Duration::from_secs(20))
51        .build()?;
52    let email = std::env::var("OPENALEX_EMAIL").ok();
53    let body = client
54        .get(search_url(query, email.as_deref()))
55        .send()
56        .await?
57        .error_for_status()?
58        .text()
59        .await?;
60    parse(&body)
61}
62
63pub fn parse(json: &str) -> Result<Vec<Paper>> {
64    let response: Response = serde_json::from_str(json)?;
65    Ok(response.results.into_iter().map(normalize).collect())
66}
67
68#[derive(Deserialize)]
69struct Response {
70    results: Vec<Work>,
71}
72
73#[derive(Deserialize)]
74struct Work {
75    id: String,
76    title: String,
77    doi: Option<String>,
78    publication_date: Option<String>,
79    updated_date: Option<String>,
80    abstract_inverted_index: Option<HashMap<String, Vec<usize>>>,
81    authorships: Vec<Authorship>,
82    ids: HashMap<String, String>,
83    primary_location: Option<Location>,
84    #[serde(default)]
85    topics: Vec<Topic>,
86}
87
88#[derive(Deserialize)]
89struct Authorship {
90    author: Author,
91}
92
93#[derive(Deserialize)]
94struct Author {
95    display_name: String,
96}
97
98#[derive(Deserialize)]
99struct Location {
100    landing_page_url: Option<String>,
101}
102
103#[derive(Deserialize)]
104struct Topic {
105    display_name: String,
106}
107
108fn normalize(work: Work) -> Paper {
109    let abstract_text = reconstruct_abstract(work.abstract_inverted_index.as_ref());
110    let arxiv_id = work
111        .ids
112        .get("arxiv")
113        .and_then(|url| url.rsplit('/').next())
114        .map(str::to_string);
115    let doi = work
116        .doi
117        .as_deref()
118        .map(|doi| doi.trim_start_matches("https://doi.org/").to_string());
119    let url = work
120        .primary_location
121        .and_then(|location| location.landing_page_url)
122        .unwrap_or_else(|| work.id.clone());
123    let extracted_methods = arxiv::extract_acronyms(&format!("{} {}", work.title, abstract_text));
124
125    Paper {
126        id: work.id,
127        title: work.title,
128        abstract_text,
129        authors: work
130            .authorships
131            .into_iter()
132            .map(|entry| entry.author.display_name)
133            .collect(),
134        published_date: work.publication_date,
135        updated_date: work.updated_date,
136        doi,
137        arxiv_id,
138        url,
139        categories: work
140            .topics
141            .into_iter()
142            .map(|topic| topic.display_name)
143            .collect(),
144        extracted_methods,
145        source_provider: "openalex".into(),
146    }
147}
148
149fn reconstruct_abstract(index: Option<&HashMap<String, Vec<usize>>>) -> String {
150    let Some(index) = index else {
151        return String::new();
152    };
153    let Some(max) = index.values().flatten().max().copied() else {
154        return String::new();
155    };
156    let mut words = vec![String::new(); max + 1];
157    for (word, positions) in index {
158        for &position in positions {
159            words[position] = word.clone();
160        }
161    }
162    words
163        .into_iter()
164        .filter(|word| !word.is_empty())
165        .collect::<Vec<_>>()
166        .join(" ")
167}
paper_gap/openalex.rs

paper_gap/
openalex.rs
