paper_gap/
openalex.rs

1use crate::{Result, arxiv, paper::Paper, provider};
2use chrono::NaiveDate;
3use serde::Deserialize;
4use std::{collections::HashMap, time::Duration};
5
6const BASE_URL: &str = "https://api.openalex.org/works";
7
8#[derive(Debug, Clone)]
9pub struct SearchQuery {
10    pub query: Option<String>,
11    pub from: Option<NaiveDate>,
12    pub to: Option<NaiveDate>,
13    pub limit: usize,
14}
15
16pub fn search_url(query: &SearchQuery, email: Option<&str>) -> String {
17    let mut url = format!(
18        "{BASE_URL}?per-page={}&sort=publication_date:desc",
19        query.limit
20    );
21    if let Some(search) = query
22        .query
23        .as_deref()
24        .filter(|query| !query.trim().is_empty())
25    {
26        url.push_str("&search=");
27        url.push_str(&urlencoding::encode(search));
28    }
29    let mut filters = Vec::new();
30    if let Some(from) = query.from {
31        filters.push(format!("from_publication_date:{from}"));
32    }
33    if let Some(to) = query.to {
34        filters.push(format!("to_publication_date:{to}"));
35    }
36    if !filters.is_empty() {
37        url.push_str("&filter=");
38        url.push_str(&urlencoding::encode(&filters.join(",")));
39    }
40    if let Some(email) = email.filter(|email| !email.trim().is_empty()) {
41        url.push_str("&mailto=");
42        url.push_str(&urlencoding::encode(email));
43    }
44    url
45}
46
47pub async fn search(query: &SearchQuery) -> Result<Vec<Paper>> {
48    let client = reqwest::Client::builder()
49        .user_agent("paper-gap/0.2")
50        .timeout(Duration::from_secs(20))
51        .build()?;
52    let email = std::env::var("OPENALEX_EMAIL").ok();
53    let body = client.get(search_url(query, email.as_deref()));
54    let body = provider::send_with_retry(body)
55        .await?
56        .error_for_status()?
57        .text()
58        .await?;
59    parse(&body)
60}
61
62pub fn parse(json: &str) -> Result<Vec<Paper>> {
63    let response: Response = serde_json::from_str(json)?;
64    Ok(response.results.into_iter().map(normalize).collect())
65}
66
67#[derive(Deserialize)]
68struct Response {
69    results: Vec<Work>,
70}
71
72#[derive(Deserialize)]
73struct Work {
74    id: String,
75    title: String,
76    doi: Option<String>,
77    publication_date: Option<String>,
78    updated_date: Option<String>,
79    abstract_inverted_index: Option<HashMap<String, Vec<usize>>>,
80    authorships: Vec<Authorship>,
81    ids: HashMap<String, String>,
82    primary_location: Option<Location>,
83    #[serde(default)]
84    topics: Vec<Topic>,
85}
86
87#[derive(Deserialize)]
88struct Authorship {
89    author: Author,
90}
91
92#[derive(Deserialize)]
93struct Author {
94    display_name: String,
95}
96
97#[derive(Deserialize)]
98struct Location {
99    landing_page_url: Option<String>,
100}
101
102#[derive(Deserialize)]
103struct Topic {
104    display_name: String,
105}
106
107fn normalize(work: Work) -> Paper {
108    let abstract_text = reconstruct_abstract(work.abstract_inverted_index.as_ref());
109    let arxiv_id = work
110        .ids
111        .get("arxiv")
112        .and_then(|url| url.rsplit('/').next())
113        .map(str::to_string);
114    let doi = work
115        .doi
116        .as_deref()
117        .map(|doi| doi.trim_start_matches("https://doi.org/").to_string());
118    let url = work
119        .primary_location
120        .and_then(|location| location.landing_page_url)
121        .unwrap_or_else(|| work.id.clone());
122    let extracted_methods = arxiv::extract_acronyms(&format!("{} {}", work.title, abstract_text));
123
124    Paper {
125        id: work.id,
126        title: work.title,
127        abstract_text,
128        authors: work
129            .authorships
130            .into_iter()
131            .map(|entry| entry.author.display_name)
132            .collect(),
133        published_date: work.publication_date,
134        updated_date: work.updated_date,
135        doi,
136        arxiv_id,
137        url,
138        categories: work
139            .topics
140            .into_iter()
141            .map(|topic| topic.display_name)
142            .collect(),
143        extracted_methods,
144        source_provider: "openalex".into(),
145    }
146}
147
148fn reconstruct_abstract(index: Option<&HashMap<String, Vec<usize>>>) -> String {
149    let Some(index) = index else {
150        return String::new();
151    };
152    let Some(max) = index.values().flatten().max().copied() else {
153        return String::new();
154    };
155    let mut words = vec![String::new(); max + 1];
156    for (word, positions) in index {
157        for &position in positions {
158            words[position] = word.clone();
159        }
160    }
161    words
162        .into_iter()
163        .filter(|word| !word.is_empty())
164        .collect::<Vec<_>>()
165        .join(" ")
166}
paper_gap/openalex.rs

paper_gap/
openalex.rs
