paper_gap/
repos.rs

1use crate::Result;
2use crate::arxiv::Paper;
3use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT};
4use serde::{Deserialize, Serialize};
5use std::time::Duration;
6
7#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
8pub struct Repository {
9    pub forge: String,
10    pub url: String,
11    pub owner: String,
12    pub name: String,
13    pub description: Option<String>,
14    pub primary_language: Option<String>,
15    pub stars: Option<u64>,
16    pub forks: Option<u64>,
17    pub license: Option<String>,
18    pub archived: Option<bool>,
19    pub default_branch: Option<String>,
20    pub last_commit_date: Option<String>,
21    pub source_provider: String,
22}
23
24#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
25pub struct RepositoryRef {
26    pub forge: String,
27    pub owner: String,
28    pub name: String,
29    pub url: String,
30}
31
32#[derive(Debug, Clone)]
33pub struct RepoSearchQuery {
34    pub terms: Vec<String>,
35    pub limit: usize,
36}
37
38pub fn search_terms(paper: &Paper) -> Vec<String> {
39    let mut terms = vec![format!("\"{}\"", paper.title)];
40    if let Some(arxiv_id) = paper.arxiv_id.as_deref() {
41        terms.push(trim_arxiv_version(arxiv_id).to_string());
42    }
43    if let Some(doi) = paper.doi.as_deref() {
44        terms.push(doi.to_string());
45    }
46    terms.extend(
47        paper
48            .extracted_methods
49            .iter()
50            .filter(|term| is_specific_method_term(term))
51            .cloned(),
52    );
53    dedup_strings(terms)
54}
55
56pub async fn search_for_paper(paper: &Paper, limit: usize) -> Result<Vec<Repository>> {
57    let query = RepoSearchQuery {
58        // ponytail: cap live forge searches; search_terms still exposes the full list for reports/tests.
59        terms: search_terms(paper).into_iter().take(3).collect(),
60        limit,
61    };
62    let mut repos = Vec::new();
63    repos.extend(github_search(&query).await.unwrap_or_default());
64    repos.extend(gitlab_search(&query).await.unwrap_or_default());
65    repos.extend(
66        forgejo_search("codeberg", "https://codeberg.org/api/v1", &query)
67            .await
68            .unwrap_or_default(),
69    );
70    Ok(filter_repositories(paper, dedup_repositories(repos)))
71}
72
73pub fn filter_repositories(paper: &Paper, repositories: Vec<Repository>) -> Vec<Repository> {
74    repositories
75        .into_iter()
76        .filter(|repository| repository_matches_paper(paper, repository))
77        .collect()
78}
79
80fn repository_matches_paper(paper: &Paper, repository: &Repository) -> bool {
81    let haystack = normalize(&format!(
82        "{} {} {} {} {}",
83        repository.owner,
84        repository.name,
85        repository.url,
86        repository.description.as_deref().unwrap_or_default(),
87        repository.primary_language.as_deref().unwrap_or_default()
88    ));
89
90    if paper
91        .arxiv_id
92        .as_deref()
93        .map(trim_arxiv_version)
94        .is_some_and(|id| haystack.contains(&normalize(id)))
95    {
96        return true;
97    }
98    if paper
99        .doi
100        .as_deref()
101        .is_some_and(|doi| haystack.contains(&normalize(doi)))
102    {
103        return true;
104    }
105
106    let terms = meaningful_title_terms(&paper.title);
107    let matches = terms
108        .iter()
109        .filter(|term| haystack.contains(term.as_str()))
110        .count();
111    matches >= 2 || (terms.len() == 1 && matches == 1)
112}
113
114fn meaningful_title_terms(title: &str) -> Vec<String> {
115    let stop = [
116        "with", "from", "into", "using", "based", "towards", "toward", "paper", "study", "method",
117        "methods", "system", "systems", "program", "programs",
118    ];
119    title
120        .split(|c: char| !c.is_alphanumeric())
121        .map(str::to_ascii_lowercase)
122        .filter(|term| term.len() >= 4 && !stop.contains(&term.as_str()))
123        .collect()
124}
125
126fn is_specific_method_term(term: &str) -> bool {
127    let broad = [
128        "ai", "ml", "rl", "api", "cpu", "gpu", "fft", "pde", "ode", "chc", "nlp", "llm", "cnn",
129        "rnn", "gcn", "sql", "http", "json", "xml",
130    ];
131    let lower = term.to_ascii_lowercase();
132    term.len() > 3 && !broad.contains(&lower.as_str())
133}
134
135fn normalize(text: &str) -> String {
136    text.chars()
137        .filter(|c| c.is_ascii_alphanumeric())
138        .flat_map(char::to_lowercase)
139        .collect()
140}
141
142pub fn github_search_url(term: &str, limit: usize) -> String {
143    format!(
144        "https://api.github.com/search/repositories?q={}&per_page={}",
145        urlencoding::encode(term),
146        limit
147    )
148}
149
150pub fn gitlab_search_url(term: &str, limit: usize) -> String {
151    format!(
152        "https://gitlab.com/api/v4/projects?search={}&per_page={}",
153        urlencoding::encode(term),
154        limit
155    )
156}
157
158pub fn forgejo_search_url(base_url: &str, term: &str, limit: usize) -> String {
159    format!(
160        "{}/repos/search?q={}&limit={}",
161        base_url.trim_end_matches('/'),
162        urlencoding::encode(term),
163        limit
164    )
165}
166
167#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
168pub struct RepositoryFile {
169    pub path: String,
170}
171
172pub fn github_tree_url(owner: &str, name: &str, branch: &str) -> String {
173    format!(
174        "https://api.github.com/repos/{}/{}/git/trees/{}?recursive=1",
175        owner,
176        name,
177        urlencoding::encode(branch)
178    )
179}
180
181pub fn gitlab_tree_url(owner: &str, name: &str, branch: &str) -> String {
182    format!(
183        "https://gitlab.com/api/v4/projects/{}/repository/tree?recursive=true&per_page=100&ref={}",
184        urlencoding::encode(&format!("{owner}/{name}")),
185        urlencoding::encode(branch)
186    )
187}
188
189pub fn forgejo_tree_url(base_url: &str, owner: &str, name: &str, branch: &str) -> String {
190    format!(
191        "{}/repos/{}/{}/git/trees/{}?recursive=true",
192        base_url.trim_end_matches('/'),
193        owner,
194        name,
195        urlencoding::encode(branch)
196    )
197}
198
199pub async fn file_paths(repo: &Repository) -> Result<Vec<String>> {
200    let client = client(match repo.forge.as_str() {
201        "github" => Some("GITHUB_TOKEN"),
202        "gitlab" => Some("GITLAB_TOKEN"),
203        _ => None,
204    });
205    let branch = repo.default_branch.as_deref().unwrap_or("main");
206    let url = match repo.forge.as_str() {
207        "github" => github_tree_url(&repo.owner, &repo.name, branch),
208        "gitlab" => gitlab_tree_url(&repo.owner, &repo.name, branch),
209        "codeberg" => forgejo_tree_url(
210            "https://codeberg.org/api/v1",
211            &repo.owner,
212            &repo.name,
213            branch,
214        ),
215        _ => return Ok(Vec::new()),
216    };
217    let response = client.get(url).send().await?;
218    if response.status() == reqwest::StatusCode::NOT_FOUND {
219        return Ok(Vec::new());
220    }
221    let text = response.error_for_status()?.text().await?;
222    match repo.forge.as_str() {
223        "github" => parse_github_tree(&text),
224        "gitlab" => parse_gitlab_tree(&text),
225        _ => parse_forgejo_tree(&text),
226    }
227}
228
229pub fn parse_github_tree(json: &str) -> Result<Vec<String>> {
230    #[derive(Deserialize)]
231    struct Tree {
232        tree: Vec<TreeItem>,
233    }
234    #[derive(Deserialize)]
235    struct TreeItem {
236        path: String,
237    }
238    Ok(serde_json::from_str::<Tree>(json)?
239        .tree
240        .into_iter()
241        .map(|item| item.path)
242        .collect())
243}
244
245pub fn parse_gitlab_tree(json: &str) -> Result<Vec<String>> {
246    #[derive(Deserialize)]
247    struct TreeItem {
248        path: String,
249    }
250    Ok(serde_json::from_str::<Vec<TreeItem>>(json)?
251        .into_iter()
252        .map(|item| item.path)
253        .collect())
254}
255
256pub fn parse_forgejo_tree(json: &str) -> Result<Vec<String>> {
257    parse_github_tree(json)
258}
259
260pub fn github_inspect_url(owner: &str, name: &str) -> String {
261    format!("https://api.github.com/repos/{}/{}", owner, name)
262}
263
264pub fn gitlab_inspect_url(owner: &str, name: &str) -> String {
265    format!(
266        "https://gitlab.com/api/v4/projects/{}",
267        urlencoding::encode(&format!("{owner}/{name}"))
268    )
269}
270
271pub fn forgejo_inspect_url(base_url: &str, owner: &str, name: &str) -> String {
272    format!(
273        "{}/repos/{}/{}",
274        base_url.trim_end_matches('/'),
275        owner,
276        name
277    )
278}
279
280pub async fn github_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
281    let client = client(Some("GITHUB_TOKEN"));
282    let mut repos = Vec::new();
283    for term in &query.terms {
284        let response = client
285            .get(github_search_url(term, query.limit))
286            .send()
287            .await?;
288        if response.status() == reqwest::StatusCode::FORBIDDEN
289            || response.status() == reqwest::StatusCode::UNAUTHORIZED
290        {
291            continue;
292        }
293        repos.extend(parse_github_search(
294            &response.error_for_status()?.text().await?,
295        )?);
296    }
297    Ok(dedup_repositories(repos))
298}
299
300pub async fn gitlab_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
301    let client = client(Some("GITLAB_TOKEN"));
302    let mut repos = Vec::new();
303    for term in &query.terms {
304        let response = client
305            .get(gitlab_search_url(term, query.limit))
306            .send()
307            .await?;
308        if response.status() == reqwest::StatusCode::FORBIDDEN
309            || response.status() == reqwest::StatusCode::UNAUTHORIZED
310        {
311            continue;
312        }
313        repos.extend(parse_gitlab_search(
314            &response.error_for_status()?.text().await?,
315        )?);
316    }
317    Ok(dedup_repositories(repos))
318}
319
320pub async fn forgejo_search(
321    provider: &str,
322    base_url: &str,
323    query: &RepoSearchQuery,
324) -> Result<Vec<Repository>> {
325    let client = client(None);
326    let mut repos = Vec::new();
327    for term in &query.terms {
328        let response = client
329            .get(forgejo_search_url(base_url, term, query.limit))
330            .send()
331            .await?;
332        if response.status() == reqwest::StatusCode::FORBIDDEN
333            || response.status() == reqwest::StatusCode::UNAUTHORIZED
334        {
335            continue;
336        }
337        repos.extend(parse_forgejo_search(
338            provider,
339            &response.error_for_status()?.text().await?,
340        )?);
341    }
342    Ok(dedup_repositories(repos))
343}
344
345pub async fn inspect(repo: &RepositoryRef) -> Result<Option<Repository>> {
346    let client = client(match repo.forge.as_str() {
347        "github" => Some("GITHUB_TOKEN"),
348        "gitlab" => Some("GITLAB_TOKEN"),
349        _ => None,
350    });
351    let url = match repo.forge.as_str() {
352        "github" => github_inspect_url(&repo.owner, &repo.name),
353        "gitlab" => gitlab_inspect_url(&repo.owner, &repo.name),
354        "codeberg" => forgejo_inspect_url("https://codeberg.org/api/v1", &repo.owner, &repo.name),
355        _ => return Ok(None),
356    };
357    let response = client.get(url).send().await?;
358    if response.status() == reqwest::StatusCode::NOT_FOUND {
359        return Ok(None);
360    }
361    let text = response.error_for_status()?.text().await?;
362    Ok(Some(match repo.forge.as_str() {
363        "github" => parse_github_repo(&text)?,
364        "gitlab" => parse_gitlab_repo(&text)?,
365        _ => parse_forgejo_repo(&repo.forge, &text)?,
366    }))
367}
368
369pub fn parse_github_search(json: &str) -> Result<Vec<Repository>> {
370    #[derive(Deserialize)]
371    struct Search {
372        items: Vec<GithubRepo>,
373    }
374    Ok(serde_json::from_str::<Search>(json)?
375        .items
376        .into_iter()
377        .map(Into::into)
378        .collect())
379}
380
381pub fn parse_github_repo(json: &str) -> Result<Repository> {
382    Ok(serde_json::from_str::<GithubRepo>(json)?.into())
383}
384
385pub fn parse_gitlab_search(json: &str) -> Result<Vec<Repository>> {
386    Ok(serde_json::from_str::<Vec<GitlabRepo>>(json)?
387        .into_iter()
388        .map(Into::into)
389        .collect())
390}
391
392pub fn parse_gitlab_repo(json: &str) -> Result<Repository> {
393    Ok(serde_json::from_str::<GitlabRepo>(json)?.into())
394}
395
396pub fn parse_forgejo_search(provider: &str, json: &str) -> Result<Vec<Repository>> {
397    #[derive(Deserialize)]
398    struct Search {
399        data: Vec<ForgejoRepo>,
400    }
401    Ok(serde_json::from_str::<Search>(json)?
402        .data
403        .into_iter()
404        .map(|r| r.into_repo(provider))
405        .collect())
406}
407
408pub fn parse_forgejo_repo(provider: &str, json: &str) -> Result<Repository> {
409    Ok(serde_json::from_str::<ForgejoRepo>(json)?.into_repo(provider))
410}
411
412fn client(token_env: Option<&str>) -> reqwest::Client {
413    let mut headers = HeaderMap::new();
414    headers.insert(USER_AGENT, HeaderValue::from_static("paper-gap"));
415    if let Some(token_env) = token_env.and_then(|name| std::env::var(name).ok()) {
416        if let Ok(value) = HeaderValue::from_str(&format!("Bearer {token_env}")) {
417            headers.insert(AUTHORIZATION, value);
418        }
419    }
420    reqwest::Client::builder()
421        .default_headers(headers)
422        .timeout(Duration::from_secs(8))
423        .build()
424        .expect("valid reqwest client")
425}
426
427#[derive(Deserialize)]
428struct GithubRepo {
429    html_url: String,
430    name: String,
431    owner: GithubOwner,
432    description: Option<String>,
433    language: Option<String>,
434    stargazers_count: Option<u64>,
435    forks_count: Option<u64>,
436    license: Option<GithubLicense>,
437    archived: Option<bool>,
438    default_branch: Option<String>,
439    pushed_at: Option<String>,
440}
441#[derive(Deserialize)]
442struct GithubOwner {
443    login: String,
444}
445#[derive(Deserialize)]
446struct GithubLicense {
447    spdx_id: Option<String>,
448    name: Option<String>,
449}
450
451impl From<GithubRepo> for Repository {
452    fn from(repo: GithubRepo) -> Self {
453        Repository {
454            forge: "github".into(),
455            url: repo.html_url,
456            owner: repo.owner.login,
457            name: repo.name,
458            description: repo.description,
459            primary_language: repo.language,
460            stars: repo.stargazers_count,
461            forks: repo.forks_count,
462            license: repo.license.and_then(|l| l.spdx_id.or(l.name)),
463            archived: repo.archived,
464            default_branch: repo.default_branch,
465            last_commit_date: repo.pushed_at,
466            source_provider: "github".into(),
467        }
468    }
469}
470
471#[derive(Deserialize)]
472struct GitlabRepo {
473    web_url: String,
474    path: String,
475    path_with_namespace: String,
476    description: Option<String>,
477    star_count: Option<u64>,
478    forks_count: Option<u64>,
479    archived: Option<bool>,
480    default_branch: Option<String>,
481    last_activity_at: Option<String>,
482}
483
484impl From<GitlabRepo> for Repository {
485    fn from(repo: GitlabRepo) -> Self {
486        let owner = repo
487            .path_with_namespace
488            .rsplit_once('/')
489            .map_or("", |(owner, _)| owner)
490            .to_string();
491        Repository {
492            forge: "gitlab".into(),
493            url: repo.web_url,
494            owner,
495            name: repo.path,
496            description: repo.description,
497            primary_language: None,
498            stars: repo.star_count,
499            forks: repo.forks_count,
500            license: None,
501            archived: repo.archived,
502            default_branch: repo.default_branch,
503            last_commit_date: repo.last_activity_at,
504            source_provider: "gitlab".into(),
505        }
506    }
507}
508
509#[derive(Deserialize)]
510struct ForgejoRepo {
511    html_url: String,
512    name: String,
513    owner: ForgejoOwner,
514    description: Option<String>,
515    language: Option<String>,
516    stars_count: Option<u64>,
517    forks_count: Option<u64>,
518    archived: Option<bool>,
519    default_branch: Option<String>,
520    updated_at: Option<String>,
521}
522#[derive(Deserialize)]
523struct ForgejoOwner {
524    login: String,
525}
526
527impl ForgejoRepo {
528    fn into_repo(self, provider: &str) -> Repository {
529        Repository {
530            forge: provider.into(),
531            url: self.html_url,
532            owner: self.owner.login,
533            name: self.name,
534            description: self.description,
535            primary_language: self.language,
536            stars: self.stars_count,
537            forks: self.forks_count,
538            license: None,
539            archived: self.archived,
540            default_branch: self.default_branch,
541            last_commit_date: self.updated_at,
542            source_provider: provider.into(),
543        }
544    }
545}
546
547fn dedup_repositories(repos: Vec<Repository>) -> Vec<Repository> {
548    let mut out = Vec::new();
549    for repo in repos {
550        if !out
551            .iter()
552            .any(|existing: &Repository| existing.url == repo.url)
553        {
554            out.push(repo);
555        }
556    }
557    out
558}
559
560fn dedup_strings(values: Vec<String>) -> Vec<String> {
561    let mut out = Vec::new();
562    for value in values.into_iter().filter(|s| !s.trim().is_empty()) {
563        if !out.iter().any(|existing| existing == &value) {
564            out.push(value);
565        }
566    }
567    out
568}
569
570fn trim_arxiv_version(arxiv_id: &str) -> &str {
571    arxiv_id.rsplit_once('v').map_or(arxiv_id, |(id, version)| {
572        if version.chars().all(|c| c.is_ascii_digit()) {
573            id
574        } else {
575            arxiv_id
576        }
577    })
578}
paper_gap/repos.rs

paper_gap/
repos.rs
