paper_gap/
repos.rs

1use crate::Result;
2use crate::arxiv::Paper;
3use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT};
4use serde::{Deserialize, Serialize};
5use std::time::Duration;
6
7#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
8pub struct Repository {
9    pub forge: String,
10    pub url: String,
11    pub owner: String,
12    pub name: String,
13    pub description: Option<String>,
14    pub primary_language: Option<String>,
15    pub stars: Option<u64>,
16    pub forks: Option<u64>,
17    pub license: Option<String>,
18    pub archived: Option<bool>,
19    pub default_branch: Option<String>,
20    pub last_commit_date: Option<String>,
21    pub source_provider: String,
22}
23
24#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
25pub struct RepositoryRef {
26    pub forge: String,
27    pub owner: String,
28    pub name: String,
29    pub url: String,
30}
31
32#[derive(Debug, Clone)]
33pub struct RepoSearchQuery {
34    pub terms: Vec<String>,
35    pub limit: usize,
36}
37
38pub fn search_terms(paper: &Paper) -> Vec<String> {
39    let mut terms = vec![format!("\"{}\"", paper.title)];
40    if let Some(arxiv_id) = paper.arxiv_id.as_deref() {
41        terms.push(trim_arxiv_version(arxiv_id).to_string());
42    }
43    if let Some(doi) = paper.doi.as_deref() {
44        terms.push(doi.to_string());
45    }
46    terms.extend(
47        paper
48            .extracted_methods
49            .iter()
50            .filter(|term| is_specific_method_term(term))
51            .cloned(),
52    );
53    dedup_strings(terms)
54}
55
56pub async fn search_for_paper(paper: &Paper, limit: usize) -> Result<Vec<Repository>> {
57    let query = RepoSearchQuery {
58        // ponytail: cap live forge searches; search_terms still exposes the full list for reports/tests.
59        terms: search_terms(paper).into_iter().take(3).collect(),
60        limit,
61    };
62    let mut repos = Vec::new();
63    repos.extend(github_search(&query).await.unwrap_or_default());
64    repos.extend(gitlab_search(&query).await.unwrap_or_default());
65    repos.extend(
66        forgejo_search("codeberg", "https://codeberg.org/api/v1", &query)
67            .await
68            .unwrap_or_default(),
69    );
70    Ok(filter_repositories(paper, dedup_repositories(repos)))
71}
72
73pub fn filter_repositories(paper: &Paper, repositories: Vec<Repository>) -> Vec<Repository> {
74    repositories
75        .into_iter()
76        .filter(|repository| repository_matches_paper(paper, repository))
77        .collect()
78}
79
80fn repository_matches_paper(paper: &Paper, repository: &Repository) -> bool {
81    let haystack = normalize(&format!(
82        "{} {} {} {} {}",
83        repository.owner,
84        repository.name,
85        repository.url,
86        repository.description.as_deref().unwrap_or_default(),
87        repository.primary_language.as_deref().unwrap_or_default()
88    ));
89
90    if paper
91        .arxiv_id
92        .as_deref()
93        .map(trim_arxiv_version)
94        .is_some_and(|id| haystack.contains(&normalize(id)))
95    {
96        return true;
97    }
98    if paper
99        .doi
100        .as_deref()
101        .is_some_and(|doi| haystack.contains(&normalize(doi)))
102    {
103        return true;
104    }
105
106    let terms = meaningful_title_terms(&paper.title);
107    let matches = terms
108        .iter()
109        .filter(|term| haystack.contains(term.as_str()))
110        .count();
111    matches >= 2 || (terms.len() == 1 && matches == 1)
112}
113
114fn meaningful_title_terms(title: &str) -> Vec<String> {
115    let stop = [
116        "with", "from", "into", "using", "based", "towards", "toward", "paper", "study", "method",
117        "methods", "system", "systems", "program", "programs",
118    ];
119    title
120        .split(|c: char| !c.is_alphanumeric())
121        .map(str::to_ascii_lowercase)
122        .filter(|term| term.len() >= 4 && !stop.contains(&term.as_str()))
123        .collect()
124}
125
126fn is_specific_method_term(term: &str) -> bool {
127    let broad = [
128        "ai", "ml", "rl", "api", "cpu", "gpu", "fft", "pde", "ode", "chc", "nlp", "llm", "cnn",
129        "rnn", "gcn", "sql", "http", "json", "xml",
130    ];
131    let lower = term.to_ascii_lowercase();
132    term.len() > 3 && !broad.contains(&lower.as_str())
133}
134
135fn normalize(text: &str) -> String {
136    text.chars()
137        .filter(|c| c.is_ascii_alphanumeric())
138        .flat_map(char::to_lowercase)
139        .collect()
140}
141
142pub fn github_search_url(term: &str, limit: usize) -> String {
143    format!(
144        "https://api.github.com/search/repositories?q={}&per_page={}",
145        urlencoding::encode(term),
146        limit
147    )
148}
149
150pub fn gitlab_search_url(term: &str, limit: usize) -> String {
151    format!(
152        "https://gitlab.com/api/v4/projects?search={}&per_page={}",
153        urlencoding::encode(term),
154        limit
155    )
156}
157
158pub fn forgejo_search_url(base_url: &str, term: &str, limit: usize) -> String {
159    format!(
160        "{}/repos/search?q={}&limit={}",
161        base_url.trim_end_matches('/'),
162        urlencoding::encode(term),
163        limit
164    )
165}
166
167pub fn github_inspect_url(owner: &str, name: &str) -> String {
168    format!("https://api.github.com/repos/{}/{}", owner, name)
169}
170
171pub fn gitlab_inspect_url(owner: &str, name: &str) -> String {
172    format!(
173        "https://gitlab.com/api/v4/projects/{}",
174        urlencoding::encode(&format!("{owner}/{name}"))
175    )
176}
177
178pub fn forgejo_inspect_url(base_url: &str, owner: &str, name: &str) -> String {
179    format!(
180        "{}/repos/{}/{}",
181        base_url.trim_end_matches('/'),
182        owner,
183        name
184    )
185}
186
187pub async fn github_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
188    let client = client(Some("GITHUB_TOKEN"));
189    let mut repos = Vec::new();
190    for term in &query.terms {
191        let response = client
192            .get(github_search_url(term, query.limit))
193            .send()
194            .await?;
195        if response.status() == reqwest::StatusCode::FORBIDDEN
196            || response.status() == reqwest::StatusCode::UNAUTHORIZED
197        {
198            continue;
199        }
200        repos.extend(parse_github_search(
201            &response.error_for_status()?.text().await?,
202        )?);
203    }
204    Ok(dedup_repositories(repos))
205}
206
207pub async fn gitlab_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
208    let client = client(Some("GITLAB_TOKEN"));
209    let mut repos = Vec::new();
210    for term in &query.terms {
211        let response = client
212            .get(gitlab_search_url(term, query.limit))
213            .send()
214            .await?;
215        if response.status() == reqwest::StatusCode::FORBIDDEN
216            || response.status() == reqwest::StatusCode::UNAUTHORIZED
217        {
218            continue;
219        }
220        repos.extend(parse_gitlab_search(
221            &response.error_for_status()?.text().await?,
222        )?);
223    }
224    Ok(dedup_repositories(repos))
225}
226
227pub async fn forgejo_search(
228    provider: &str,
229    base_url: &str,
230    query: &RepoSearchQuery,
231) -> Result<Vec<Repository>> {
232    let client = client(None);
233    let mut repos = Vec::new();
234    for term in &query.terms {
235        let response = client
236            .get(forgejo_search_url(base_url, term, query.limit))
237            .send()
238            .await?;
239        if response.status() == reqwest::StatusCode::FORBIDDEN
240            || response.status() == reqwest::StatusCode::UNAUTHORIZED
241        {
242            continue;
243        }
244        repos.extend(parse_forgejo_search(
245            provider,
246            &response.error_for_status()?.text().await?,
247        )?);
248    }
249    Ok(dedup_repositories(repos))
250}
251
252pub async fn inspect(repo: &RepositoryRef) -> Result<Option<Repository>> {
253    let client = client(match repo.forge.as_str() {
254        "github" => Some("GITHUB_TOKEN"),
255        "gitlab" => Some("GITLAB_TOKEN"),
256        _ => None,
257    });
258    let url = match repo.forge.as_str() {
259        "github" => github_inspect_url(&repo.owner, &repo.name),
260        "gitlab" => gitlab_inspect_url(&repo.owner, &repo.name),
261        "codeberg" => forgejo_inspect_url("https://codeberg.org/api/v1", &repo.owner, &repo.name),
262        _ => return Ok(None),
263    };
264    let response = client.get(url).send().await?;
265    if response.status() == reqwest::StatusCode::NOT_FOUND {
266        return Ok(None);
267    }
268    let text = response.error_for_status()?.text().await?;
269    Ok(Some(match repo.forge.as_str() {
270        "github" => parse_github_repo(&text)?,
271        "gitlab" => parse_gitlab_repo(&text)?,
272        _ => parse_forgejo_repo(&repo.forge, &text)?,
273    }))
274}
275
276pub fn parse_github_search(json: &str) -> Result<Vec<Repository>> {
277    #[derive(Deserialize)]
278    struct Search {
279        items: Vec<GithubRepo>,
280    }
281    Ok(serde_json::from_str::<Search>(json)?
282        .items
283        .into_iter()
284        .map(Into::into)
285        .collect())
286}
287
288pub fn parse_github_repo(json: &str) -> Result<Repository> {
289    Ok(serde_json::from_str::<GithubRepo>(json)?.into())
290}
291
292pub fn parse_gitlab_search(json: &str) -> Result<Vec<Repository>> {
293    Ok(serde_json::from_str::<Vec<GitlabRepo>>(json)?
294        .into_iter()
295        .map(Into::into)
296        .collect())
297}
298
299pub fn parse_gitlab_repo(json: &str) -> Result<Repository> {
300    Ok(serde_json::from_str::<GitlabRepo>(json)?.into())
301}
302
303pub fn parse_forgejo_search(provider: &str, json: &str) -> Result<Vec<Repository>> {
304    #[derive(Deserialize)]
305    struct Search {
306        data: Vec<ForgejoRepo>,
307    }
308    Ok(serde_json::from_str::<Search>(json)?
309        .data
310        .into_iter()
311        .map(|r| r.into_repo(provider))
312        .collect())
313}
314
315pub fn parse_forgejo_repo(provider: &str, json: &str) -> Result<Repository> {
316    Ok(serde_json::from_str::<ForgejoRepo>(json)?.into_repo(provider))
317}
318
319fn client(token_env: Option<&str>) -> reqwest::Client {
320    let mut headers = HeaderMap::new();
321    headers.insert(USER_AGENT, HeaderValue::from_static("paper-gap"));
322    if let Some(token_env) = token_env.and_then(|name| std::env::var(name).ok()) {
323        if let Ok(value) = HeaderValue::from_str(&format!("Bearer {token_env}")) {
324            headers.insert(AUTHORIZATION, value);
325        }
326    }
327    reqwest::Client::builder()
328        .default_headers(headers)
329        .timeout(Duration::from_secs(8))
330        .build()
331        .expect("valid reqwest client")
332}
333
334#[derive(Deserialize)]
335struct GithubRepo {
336    html_url: String,
337    name: String,
338    owner: GithubOwner,
339    description: Option<String>,
340    language: Option<String>,
341    stargazers_count: Option<u64>,
342    forks_count: Option<u64>,
343    license: Option<GithubLicense>,
344    archived: Option<bool>,
345    default_branch: Option<String>,
346    pushed_at: Option<String>,
347}
348#[derive(Deserialize)]
349struct GithubOwner {
350    login: String,
351}
352#[derive(Deserialize)]
353struct GithubLicense {
354    spdx_id: Option<String>,
355    name: Option<String>,
356}
357
358impl From<GithubRepo> for Repository {
359    fn from(repo: GithubRepo) -> Self {
360        Repository {
361            forge: "github".into(),
362            url: repo.html_url,
363            owner: repo.owner.login,
364            name: repo.name,
365            description: repo.description,
366            primary_language: repo.language,
367            stars: repo.stargazers_count,
368            forks: repo.forks_count,
369            license: repo.license.and_then(|l| l.spdx_id.or(l.name)),
370            archived: repo.archived,
371            default_branch: repo.default_branch,
372            last_commit_date: repo.pushed_at,
373            source_provider: "github".into(),
374        }
375    }
376}
377
378#[derive(Deserialize)]
379struct GitlabRepo {
380    web_url: String,
381    path: String,
382    path_with_namespace: String,
383    description: Option<String>,
384    star_count: Option<u64>,
385    forks_count: Option<u64>,
386    archived: Option<bool>,
387    default_branch: Option<String>,
388    last_activity_at: Option<String>,
389}
390
391impl From<GitlabRepo> for Repository {
392    fn from(repo: GitlabRepo) -> Self {
393        let owner = repo
394            .path_with_namespace
395            .rsplit_once('/')
396            .map_or("", |(owner, _)| owner)
397            .to_string();
398        Repository {
399            forge: "gitlab".into(),
400            url: repo.web_url,
401            owner,
402            name: repo.path,
403            description: repo.description,
404            primary_language: None,
405            stars: repo.star_count,
406            forks: repo.forks_count,
407            license: None,
408            archived: repo.archived,
409            default_branch: repo.default_branch,
410            last_commit_date: repo.last_activity_at,
411            source_provider: "gitlab".into(),
412        }
413    }
414}
415
416#[derive(Deserialize)]
417struct ForgejoRepo {
418    html_url: String,
419    name: String,
420    owner: ForgejoOwner,
421    description: Option<String>,
422    language: Option<String>,
423    stars_count: Option<u64>,
424    forks_count: Option<u64>,
425    archived: Option<bool>,
426    default_branch: Option<String>,
427    updated_at: Option<String>,
428}
429#[derive(Deserialize)]
430struct ForgejoOwner {
431    login: String,
432}
433
434impl ForgejoRepo {
435    fn into_repo(self, provider: &str) -> Repository {
436        Repository {
437            forge: provider.into(),
438            url: self.html_url,
439            owner: self.owner.login,
440            name: self.name,
441            description: self.description,
442            primary_language: self.language,
443            stars: self.stars_count,
444            forks: self.forks_count,
445            license: None,
446            archived: self.archived,
447            default_branch: self.default_branch,
448            last_commit_date: self.updated_at,
449            source_provider: provider.into(),
450        }
451    }
452}
453
454fn dedup_repositories(repos: Vec<Repository>) -> Vec<Repository> {
455    let mut out = Vec::new();
456    for repo in repos {
457        if !out
458            .iter()
459            .any(|existing: &Repository| existing.url == repo.url)
460        {
461            out.push(repo);
462        }
463    }
464    out
465}
466
467fn dedup_strings(values: Vec<String>) -> Vec<String> {
468    let mut out = Vec::new();
469    for value in values.into_iter().filter(|s| !s.trim().is_empty()) {
470        if !out.iter().any(|existing| existing == &value) {
471            out.push(value);
472        }
473    }
474    out
475}
476
477fn trim_arxiv_version(arxiv_id: &str) -> &str {
478    arxiv_id.rsplit_once('v').map_or(arxiv_id, |(id, version)| {
479        if version.chars().all(|c| c.is_ascii_digit()) {
480            id
481        } else {
482            arxiv_id
483        }
484    })
485}
paper_gap/repos.rs

paper_gap/
repos.rs
