paper_gap/
openalex.rs

1use crate::{Result, arxiv, paper::Paper};
2use chrono::NaiveDate;
3use serde::Deserialize;
4use std::{collections::HashMap, time::Duration};
5
6const BASE_URL: &str = "https://api.openalex.org/works";
7
8#[derive(Debug, Clone)]
9pub struct SearchQuery {
10    pub query: String,
11    pub from: Option<NaiveDate>,
12    pub to: Option<NaiveDate>,
13    pub limit: usize,
14}
15
16pub fn search_url(query: &SearchQuery, email: Option<&str>) -> String {
17    let mut url = format!(
18        "{BASE_URL}?search={}&per-page={}&sort=publication_date:desc",
19        urlencoding::encode(&query.query),
20        query.limit
21    );
22    let mut filters = Vec::new();
23    if let Some(from) = query.from {
24        filters.push(format!("from_publication_date:{from}"));
25    }
26    if let Some(to) = query.to {
27        filters.push(format!("to_publication_date:{to}"));
28    }
29    if !filters.is_empty() {
30        url.push_str("&filter=");
31        url.push_str(&urlencoding::encode(&filters.join(",")));
32    }
33    if let Some(email) = email.filter(|email| !email.trim().is_empty()) {
34        url.push_str("&mailto=");
35        url.push_str(&urlencoding::encode(email));
36    }
37    url
38}
39
40pub async fn search(query: &SearchQuery) -> Result<Vec<Paper>> {
41    let client = reqwest::Client::builder()
42        .user_agent("paper-gap/0.2")
43        .timeout(Duration::from_secs(20))
44        .build()?;
45    let email = std::env::var("OPENALEX_EMAIL").ok();
46    let body = client
47        .get(search_url(query, email.as_deref()))
48        .send()
49        .await?
50        .error_for_status()?
51        .text()
52        .await?;
53    parse(&body)
54}
55
56pub fn parse(json: &str) -> Result<Vec<Paper>> {
57    let response: Response = serde_json::from_str(json)?;
58    Ok(response.results.into_iter().map(normalize).collect())
59}
60
61#[derive(Deserialize)]
62struct Response {
63    results: Vec<Work>,
64}
65
66#[derive(Deserialize)]
67struct Work {
68    id: String,
69    title: String,
70    doi: Option<String>,
71    publication_date: Option<String>,
72    updated_date: Option<String>,
73    abstract_inverted_index: Option<HashMap<String, Vec<usize>>>,
74    authorships: Vec<Authorship>,
75    ids: HashMap<String, String>,
76    primary_location: Option<Location>,
77    #[serde(default)]
78    topics: Vec<Topic>,
79}
80
81#[derive(Deserialize)]
82struct Authorship {
83    author: Author,
84}
85
86#[derive(Deserialize)]
87struct Author {
88    display_name: String,
89}
90
91#[derive(Deserialize)]
92struct Location {
93    landing_page_url: Option<String>,
94}
95
96#[derive(Deserialize)]
97struct Topic {
98    display_name: String,
99}
100
101fn normalize(work: Work) -> Paper {
102    let abstract_text = reconstruct_abstract(work.abstract_inverted_index.as_ref());
103    let arxiv_id = work
104        .ids
105        .get("arxiv")
106        .and_then(|url| url.rsplit('/').next())
107        .map(str::to_string);
108    let doi = work
109        .doi
110        .as_deref()
111        .map(|doi| doi.trim_start_matches("https://doi.org/").to_string());
112    let url = work
113        .primary_location
114        .and_then(|location| location.landing_page_url)
115        .unwrap_or_else(|| work.id.clone());
116    let extracted_methods = arxiv::extract_acronyms(&format!("{} {}", work.title, abstract_text));
117
118    Paper {
119        id: work.id,
120        title: work.title,
121        abstract_text,
122        authors: work
123            .authorships
124            .into_iter()
125            .map(|entry| entry.author.display_name)
126            .collect(),
127        published_date: work.publication_date,
128        updated_date: work.updated_date,
129        doi,
130        arxiv_id,
131        url,
132        categories: work
133            .topics
134            .into_iter()
135            .map(|topic| topic.display_name)
136            .collect(),
137        extracted_methods,
138        source_provider: "openalex".into(),
139    }
140}
141
142fn reconstruct_abstract(index: Option<&HashMap<String, Vec<usize>>>) -> String {
143    let Some(index) = index else {
144        return String::new();
145    };
146    let Some(max) = index.values().flatten().max().copied() else {
147        return String::new();
148    };
149    let mut words = vec![String::new(); max + 1];
150    for (word, positions) in index {
151        for &position in positions {
152            words[position] = word.clone();
153        }
154    }
155    words
156        .into_iter()
157        .filter(|word| !word.is_empty())
158        .collect::<Vec<_>>()
159        .join(" ")
160}
paper_gap/openalex.rs

paper_gap/
openalex.rs
