paper_gap/
repos.rs

1use crate::Result;
2use crate::arxiv::Paper;
3use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT};
4use serde::{Deserialize, Serialize};
5use std::time::Duration;
6
7#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
8pub struct Repository {
9    pub forge: String,
10    pub url: String,
11    pub owner: String,
12    pub name: String,
13    pub description: Option<String>,
14    pub primary_language: Option<String>,
15    pub stars: Option<u64>,
16    pub forks: Option<u64>,
17    pub license: Option<String>,
18    pub archived: Option<bool>,
19    pub default_branch: Option<String>,
20    pub last_commit_date: Option<String>,
21    pub source_provider: String,
22}
23
24#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
25pub struct RepositoryRef {
26    pub forge: String,
27    pub owner: String,
28    pub name: String,
29    pub url: String,
30}
31
32#[derive(Debug, Clone)]
33pub struct RepoSearchQuery {
34    pub terms: Vec<String>,
35    pub limit: usize,
36}
37
38pub fn search_terms(paper: &Paper) -> Vec<String> {
39    let mut terms = vec![paper.title.clone()];
40    if let Some(arxiv_id) = paper.arxiv_id.as_deref() {
41        terms.push(trim_arxiv_version(arxiv_id).to_string());
42    }
43    if let Some(doi) = paper.doi.as_deref() {
44        terms.push(doi.to_string());
45    }
46    terms.extend(paper.extracted_methods.iter().cloned());
47    dedup_strings(terms)
48}
49
50pub async fn search_for_paper(paper: &Paper, limit: usize) -> Result<Vec<Repository>> {
51    let query = RepoSearchQuery {
52        // ponytail: cap live forge searches; search_terms still exposes the full list for reports/tests.
53        terms: search_terms(paper).into_iter().take(3).collect(),
54        limit,
55    };
56    let mut repos = Vec::new();
57    repos.extend(github_search(&query).await.unwrap_or_default());
58    repos.extend(gitlab_search(&query).await.unwrap_or_default());
59    repos.extend(
60        forgejo_search("codeberg", "https://codeberg.org/api/v1", &query)
61            .await
62            .unwrap_or_default(),
63    );
64    Ok(dedup_repositories(repos))
65}
66
67pub fn github_search_url(term: &str, limit: usize) -> String {
68    format!(
69        "https://api.github.com/search/repositories?q={}&per_page={}",
70        urlencoding::encode(term),
71        limit
72    )
73}
74
75pub fn gitlab_search_url(term: &str, limit: usize) -> String {
76    format!(
77        "https://gitlab.com/api/v4/projects?search={}&per_page={}",
78        urlencoding::encode(term),
79        limit
80    )
81}
82
83pub fn forgejo_search_url(base_url: &str, term: &str, limit: usize) -> String {
84    format!(
85        "{}/repos/search?q={}&limit={}",
86        base_url.trim_end_matches('/'),
87        urlencoding::encode(term),
88        limit
89    )
90}
91
92pub fn github_inspect_url(owner: &str, name: &str) -> String {
93    format!("https://api.github.com/repos/{}/{}", owner, name)
94}
95
96pub fn gitlab_inspect_url(owner: &str, name: &str) -> String {
97    format!(
98        "https://gitlab.com/api/v4/projects/{}",
99        urlencoding::encode(&format!("{owner}/{name}"))
100    )
101}
102
103pub fn forgejo_inspect_url(base_url: &str, owner: &str, name: &str) -> String {
104    format!(
105        "{}/repos/{}/{}",
106        base_url.trim_end_matches('/'),
107        owner,
108        name
109    )
110}
111
112pub async fn github_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
113    let client = client(Some("GITHUB_TOKEN"));
114    let mut repos = Vec::new();
115    for term in &query.terms {
116        let response = client
117            .get(github_search_url(term, query.limit))
118            .send()
119            .await?;
120        if response.status() == reqwest::StatusCode::FORBIDDEN
121            || response.status() == reqwest::StatusCode::UNAUTHORIZED
122        {
123            continue;
124        }
125        repos.extend(parse_github_search(
126            &response.error_for_status()?.text().await?,
127        )?);
128    }
129    Ok(dedup_repositories(repos))
130}
131
132pub async fn gitlab_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
133    let client = client(Some("GITLAB_TOKEN"));
134    let mut repos = Vec::new();
135    for term in &query.terms {
136        let response = client
137            .get(gitlab_search_url(term, query.limit))
138            .send()
139            .await?;
140        if response.status() == reqwest::StatusCode::FORBIDDEN
141            || response.status() == reqwest::StatusCode::UNAUTHORIZED
142        {
143            continue;
144        }
145        repos.extend(parse_gitlab_search(
146            &response.error_for_status()?.text().await?,
147        )?);
148    }
149    Ok(dedup_repositories(repos))
150}
151
152pub async fn forgejo_search(
153    provider: &str,
154    base_url: &str,
155    query: &RepoSearchQuery,
156) -> Result<Vec<Repository>> {
157    let client = client(None);
158    let mut repos = Vec::new();
159    for term in &query.terms {
160        let response = client
161            .get(forgejo_search_url(base_url, term, query.limit))
162            .send()
163            .await?;
164        if response.status() == reqwest::StatusCode::FORBIDDEN
165            || response.status() == reqwest::StatusCode::UNAUTHORIZED
166        {
167            continue;
168        }
169        repos.extend(parse_forgejo_search(
170            provider,
171            &response.error_for_status()?.text().await?,
172        )?);
173    }
174    Ok(dedup_repositories(repos))
175}
176
177pub async fn inspect(repo: &RepositoryRef) -> Result<Option<Repository>> {
178    let client = client(match repo.forge.as_str() {
179        "github" => Some("GITHUB_TOKEN"),
180        "gitlab" => Some("GITLAB_TOKEN"),
181        _ => None,
182    });
183    let url = match repo.forge.as_str() {
184        "github" => github_inspect_url(&repo.owner, &repo.name),
185        "gitlab" => gitlab_inspect_url(&repo.owner, &repo.name),
186        "codeberg" => forgejo_inspect_url("https://codeberg.org/api/v1", &repo.owner, &repo.name),
187        _ => return Ok(None),
188    };
189    let response = client.get(url).send().await?;
190    if response.status() == reqwest::StatusCode::NOT_FOUND {
191        return Ok(None);
192    }
193    let text = response.error_for_status()?.text().await?;
194    Ok(Some(match repo.forge.as_str() {
195        "github" => parse_github_repo(&text)?,
196        "gitlab" => parse_gitlab_repo(&text)?,
197        _ => parse_forgejo_repo(&repo.forge, &text)?,
198    }))
199}
200
201pub fn parse_github_search(json: &str) -> Result<Vec<Repository>> {
202    #[derive(Deserialize)]
203    struct Search {
204        items: Vec<GithubRepo>,
205    }
206    Ok(serde_json::from_str::<Search>(json)?
207        .items
208        .into_iter()
209        .map(Into::into)
210        .collect())
211}
212
213pub fn parse_github_repo(json: &str) -> Result<Repository> {
214    Ok(serde_json::from_str::<GithubRepo>(json)?.into())
215}
216
217pub fn parse_gitlab_search(json: &str) -> Result<Vec<Repository>> {
218    Ok(serde_json::from_str::<Vec<GitlabRepo>>(json)?
219        .into_iter()
220        .map(Into::into)
221        .collect())
222}
223
224pub fn parse_gitlab_repo(json: &str) -> Result<Repository> {
225    Ok(serde_json::from_str::<GitlabRepo>(json)?.into())
226}
227
228pub fn parse_forgejo_search(provider: &str, json: &str) -> Result<Vec<Repository>> {
229    #[derive(Deserialize)]
230    struct Search {
231        data: Vec<ForgejoRepo>,
232    }
233    Ok(serde_json::from_str::<Search>(json)?
234        .data
235        .into_iter()
236        .map(|r| r.into_repo(provider))
237        .collect())
238}
239
240pub fn parse_forgejo_repo(provider: &str, json: &str) -> Result<Repository> {
241    Ok(serde_json::from_str::<ForgejoRepo>(json)?.into_repo(provider))
242}
243
244fn client(token_env: Option<&str>) -> reqwest::Client {
245    let mut headers = HeaderMap::new();
246    headers.insert(USER_AGENT, HeaderValue::from_static("paper-gap"));
247    if let Some(token_env) = token_env.and_then(|name| std::env::var(name).ok()) {
248        if let Ok(value) = HeaderValue::from_str(&format!("Bearer {token_env}")) {
249            headers.insert(AUTHORIZATION, value);
250        }
251    }
252    reqwest::Client::builder()
253        .default_headers(headers)
254        .timeout(Duration::from_secs(8))
255        .build()
256        .expect("valid reqwest client")
257}
258
259#[derive(Deserialize)]
260struct GithubRepo {
261    html_url: String,
262    name: String,
263    owner: GithubOwner,
264    description: Option<String>,
265    language: Option<String>,
266    stargazers_count: Option<u64>,
267    forks_count: Option<u64>,
268    license: Option<GithubLicense>,
269    archived: Option<bool>,
270    default_branch: Option<String>,
271    pushed_at: Option<String>,
272}
273#[derive(Deserialize)]
274struct GithubOwner {
275    login: String,
276}
277#[derive(Deserialize)]
278struct GithubLicense {
279    spdx_id: Option<String>,
280    name: Option<String>,
281}
282
283impl From<GithubRepo> for Repository {
284    fn from(repo: GithubRepo) -> Self {
285        Repository {
286            forge: "github".into(),
287            url: repo.html_url,
288            owner: repo.owner.login,
289            name: repo.name,
290            description: repo.description,
291            primary_language: repo.language,
292            stars: repo.stargazers_count,
293            forks: repo.forks_count,
294            license: repo.license.and_then(|l| l.spdx_id.or(l.name)),
295            archived: repo.archived,
296            default_branch: repo.default_branch,
297            last_commit_date: repo.pushed_at,
298            source_provider: "github".into(),
299        }
300    }
301}
302
303#[derive(Deserialize)]
304struct GitlabRepo {
305    web_url: String,
306    path: String,
307    path_with_namespace: String,
308    description: Option<String>,
309    star_count: Option<u64>,
310    forks_count: Option<u64>,
311    archived: Option<bool>,
312    default_branch: Option<String>,
313    last_activity_at: Option<String>,
314}
315
316impl From<GitlabRepo> for Repository {
317    fn from(repo: GitlabRepo) -> Self {
318        let owner = repo
319            .path_with_namespace
320            .rsplit_once('/')
321            .map_or("", |(owner, _)| owner)
322            .to_string();
323        Repository {
324            forge: "gitlab".into(),
325            url: repo.web_url,
326            owner,
327            name: repo.path,
328            description: repo.description,
329            primary_language: None,
330            stars: repo.star_count,
331            forks: repo.forks_count,
332            license: None,
333            archived: repo.archived,
334            default_branch: repo.default_branch,
335            last_commit_date: repo.last_activity_at,
336            source_provider: "gitlab".into(),
337        }
338    }
339}
340
341#[derive(Deserialize)]
342struct ForgejoRepo {
343    html_url: String,
344    name: String,
345    owner: ForgejoOwner,
346    description: Option<String>,
347    language: Option<String>,
348    stars_count: Option<u64>,
349    forks_count: Option<u64>,
350    archived: Option<bool>,
351    default_branch: Option<String>,
352    updated_at: Option<String>,
353}
354#[derive(Deserialize)]
355struct ForgejoOwner {
356    login: String,
357}
358
359impl ForgejoRepo {
360    fn into_repo(self, provider: &str) -> Repository {
361        Repository {
362            forge: provider.into(),
363            url: self.html_url,
364            owner: self.owner.login,
365            name: self.name,
366            description: self.description,
367            primary_language: self.language,
368            stars: self.stars_count,
369            forks: self.forks_count,
370            license: None,
371            archived: self.archived,
372            default_branch: self.default_branch,
373            last_commit_date: self.updated_at,
374            source_provider: provider.into(),
375        }
376    }
377}
378
379fn dedup_repositories(repos: Vec<Repository>) -> Vec<Repository> {
380    let mut out = Vec::new();
381    for repo in repos {
382        if !out
383            .iter()
384            .any(|existing: &Repository| existing.url == repo.url)
385        {
386            out.push(repo);
387        }
388    }
389    out
390}
391
392fn dedup_strings(values: Vec<String>) -> Vec<String> {
393    let mut out = Vec::new();
394    for value in values.into_iter().filter(|s| !s.trim().is_empty()) {
395        if !out.iter().any(|existing| existing == &value) {
396            out.push(value);
397        }
398    }
399    out
400}
401
402fn trim_arxiv_version(arxiv_id: &str) -> &str {
403    arxiv_id.rsplit_once('v').map_or(arxiv_id, |(id, version)| {
404        if version.chars().all(|c| c.is_ascii_digit()) {
405            id
406        } else {
407            arxiv_id
408        }
409    })
410}
paper_gap/repos.rs

paper_gap/
repos.rs
