1use crate::arxiv::Paper;
2use crate::{Result, provider::ProviderOutcome};
3use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT};
4use serde::{Deserialize, Serialize};
5use std::time::Duration;
6
7#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
8pub struct Repository {
9 pub forge: String,
10 pub url: String,
11 pub owner: String,
12 pub name: String,
13 pub description: Option<String>,
14 pub primary_language: Option<String>,
15 pub stars: Option<u64>,
16 pub forks: Option<u64>,
17 pub license: Option<String>,
18 pub archived: Option<bool>,
19 pub default_branch: Option<String>,
20 pub last_commit_date: Option<String>,
21 pub source_provider: String,
22}
23
24#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
25pub struct RepositoryRef {
26 pub forge: String,
27 pub owner: String,
28 pub name: String,
29 pub url: String,
30}
31
32#[derive(Debug, Clone)]
33pub struct RepoSearchQuery {
34 pub terms: Vec<String>,
35 pub limit: usize,
36}
37
38pub fn search_terms(paper: &Paper) -> Vec<String> {
39 let mut terms = vec![format!("\"{}\"", paper.title)];
40 if let Some(arxiv_id) = paper.arxiv_id.as_deref() {
41 terms.push(trim_arxiv_version(arxiv_id).to_string());
42 }
43 if let Some(doi) = paper.doi.as_deref() {
44 terms.push(doi.to_string());
45 }
46 terms.extend(
47 paper
48 .extracted_methods
49 .iter()
50 .filter(|term| is_specific_method_term(term))
51 .cloned(),
52 );
53 dedup_strings(terms)
54}
55
56pub async fn search_for_paper(paper: &Paper, limit: usize) -> Result<Vec<Repository>> {
57 let query = RepoSearchQuery {
58 terms: search_terms(paper).into_iter().take(3).collect(),
60 limit,
61 };
62 let mut repos = Vec::new();
63 repos.extend(github_search(&query).await.unwrap_or_default());
64 repos.extend(gitlab_search(&query).await.unwrap_or_default());
65 repos.extend(
66 forgejo_search("codeberg", "https://codeberg.org/api/v1", &query)
67 .await
68 .unwrap_or_default(),
69 );
70 Ok(filter_repositories(paper, dedup_repositories(repos)))
71}
72
73pub async fn search_for_paper_with_outcome(
74 paper: &Paper,
75 limit: usize,
76) -> (Vec<Repository>, ProviderOutcome) {
77 let query = RepoSearchQuery {
78 terms: search_terms(paper).into_iter().take(3).collect(),
79 limit,
80 };
81 let mut repositories = Vec::new();
82 let mut outcomes = Vec::new();
83
84 for (provider, result) in [
85 ("github", github_search(&query).await),
86 ("gitlab", gitlab_search(&query).await),
87 (
88 "codeberg",
89 forgejo_search("codeberg", "https://codeberg.org/api/v1", &query).await,
90 ),
91 ] {
92 match result {
93 Ok(found) => {
94 repositories.extend(found);
95 outcomes.push(ProviderOutcome::success(provider));
96 }
97 Err(error) => outcomes.push(ProviderOutcome::from_error(provider, &error)),
98 }
99 }
100
101 (
102 filter_repositories(paper, dedup_repositories(repositories)),
103 ProviderOutcome::aggregate("repository-search", &outcomes),
104 )
105}
106
107pub fn filter_repositories(paper: &Paper, repositories: Vec<Repository>) -> Vec<Repository> {
108 repositories
109 .into_iter()
110 .filter(|repository| repository_matches_paper(paper, repository))
111 .collect()
112}
113
114fn repository_matches_paper(paper: &Paper, repository: &Repository) -> bool {
115 let haystack = normalize(&format!(
116 "{} {} {} {} {}",
117 repository.owner,
118 repository.name,
119 repository.url,
120 repository.description.as_deref().unwrap_or_default(),
121 repository.primary_language.as_deref().unwrap_or_default()
122 ));
123
124 if paper
125 .arxiv_id
126 .as_deref()
127 .map(trim_arxiv_version)
128 .is_some_and(|id| haystack.contains(&normalize(id)))
129 {
130 return true;
131 }
132 if paper
133 .doi
134 .as_deref()
135 .is_some_and(|doi| haystack.contains(&normalize(doi)))
136 {
137 return true;
138 }
139
140 let terms = meaningful_title_terms(&paper.title);
141 let matches = terms
142 .iter()
143 .filter(|term| haystack.contains(term.as_str()))
144 .count();
145 matches >= 2 || (terms.len() == 1 && matches == 1)
146}
147
148fn meaningful_title_terms(title: &str) -> Vec<String> {
149 let stop = [
150 "with", "from", "into", "using", "based", "towards", "toward", "paper", "study", "method",
151 "methods", "system", "systems", "program", "programs",
152 ];
153 title
154 .split(|c: char| !c.is_alphanumeric())
155 .map(str::to_ascii_lowercase)
156 .filter(|term| term.len() >= 4 && !stop.contains(&term.as_str()))
157 .collect()
158}
159
160fn is_specific_method_term(term: &str) -> bool {
161 let broad = [
162 "ai", "ml", "rl", "api", "cpu", "gpu", "fft", "pde", "ode", "chc", "nlp", "llm", "cnn",
163 "rnn", "gcn", "sql", "http", "json", "xml",
164 ];
165 let lower = term.to_ascii_lowercase();
166 term.len() > 3 && !broad.contains(&lower.as_str())
167}
168
169fn normalize(text: &str) -> String {
170 text.chars()
171 .filter(|c| c.is_ascii_alphanumeric())
172 .flat_map(char::to_lowercase)
173 .collect()
174}
175
176pub fn github_search_url(term: &str, limit: usize) -> String {
177 format!(
178 "https://api.github.com/search/repositories?q={}&per_page={}",
179 urlencoding::encode(term),
180 limit
181 )
182}
183
184pub fn gitlab_search_url(term: &str, limit: usize) -> String {
185 format!(
186 "https://gitlab.com/api/v4/projects?search={}&per_page={}",
187 urlencoding::encode(term),
188 limit
189 )
190}
191
192pub fn forgejo_search_url(base_url: &str, term: &str, limit: usize) -> String {
193 format!(
194 "{}/repos/search?q={}&limit={}",
195 base_url.trim_end_matches('/'),
196 urlencoding::encode(term),
197 limit
198 )
199}
200
201#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
202pub struct RepositoryFile {
203 pub path: String,
204}
205
206pub fn github_tree_url(owner: &str, name: &str, branch: &str) -> String {
207 format!(
208 "https://api.github.com/repos/{}/{}/git/trees/{}?recursive=1",
209 owner,
210 name,
211 urlencoding::encode(branch)
212 )
213}
214
215pub fn gitlab_tree_url(owner: &str, name: &str, branch: &str) -> String {
216 format!(
217 "https://gitlab.com/api/v4/projects/{}/repository/tree?recursive=true&per_page=100&ref={}",
218 urlencoding::encode(&format!("{owner}/{name}")),
219 urlencoding::encode(branch)
220 )
221}
222
223pub fn forgejo_tree_url(base_url: &str, owner: &str, name: &str, branch: &str) -> String {
224 format!(
225 "{}/repos/{}/{}/git/trees/{}?recursive=true",
226 base_url.trim_end_matches('/'),
227 owner,
228 name,
229 urlencoding::encode(branch)
230 )
231}
232
233pub async fn file_paths(repo: &Repository) -> Result<Vec<String>> {
234 let client = client(match repo.forge.as_str() {
235 "github" => Some("GITHUB_TOKEN"),
236 "gitlab" => Some("GITLAB_TOKEN"),
237 _ => None,
238 });
239 let branch = repo.default_branch.as_deref().unwrap_or("main");
240 let url = match repo.forge.as_str() {
241 "github" => github_tree_url(&repo.owner, &repo.name, branch),
242 "gitlab" => gitlab_tree_url(&repo.owner, &repo.name, branch),
243 "codeberg" => forgejo_tree_url(
244 "https://codeberg.org/api/v1",
245 &repo.owner,
246 &repo.name,
247 branch,
248 ),
249 _ => return Ok(Vec::new()),
250 };
251 let response = client.get(url).send().await?;
252 if response.status() == reqwest::StatusCode::NOT_FOUND {
253 return Ok(Vec::new());
254 }
255 let text = response.error_for_status()?.text().await?;
256 match repo.forge.as_str() {
257 "github" => parse_github_tree(&text),
258 "gitlab" => parse_gitlab_tree(&text),
259 _ => parse_forgejo_tree(&text),
260 }
261}
262
263pub fn parse_github_tree(json: &str) -> Result<Vec<String>> {
264 #[derive(Deserialize)]
265 struct Tree {
266 tree: Vec<TreeItem>,
267 }
268 #[derive(Deserialize)]
269 struct TreeItem {
270 path: String,
271 }
272 Ok(serde_json::from_str::<Tree>(json)?
273 .tree
274 .into_iter()
275 .map(|item| item.path)
276 .collect())
277}
278
279pub fn parse_gitlab_tree(json: &str) -> Result<Vec<String>> {
280 #[derive(Deserialize)]
281 struct TreeItem {
282 path: String,
283 }
284 Ok(serde_json::from_str::<Vec<TreeItem>>(json)?
285 .into_iter()
286 .map(|item| item.path)
287 .collect())
288}
289
290pub fn parse_forgejo_tree(json: &str) -> Result<Vec<String>> {
291 parse_github_tree(json)
292}
293
294pub fn github_inspect_url(owner: &str, name: &str) -> String {
295 format!("https://api.github.com/repos/{}/{}", owner, name)
296}
297
298pub fn gitlab_inspect_url(owner: &str, name: &str) -> String {
299 format!(
300 "https://gitlab.com/api/v4/projects/{}",
301 urlencoding::encode(&format!("{owner}/{name}"))
302 )
303}
304
305pub fn forgejo_inspect_url(base_url: &str, owner: &str, name: &str) -> String {
306 format!(
307 "{}/repos/{}/{}",
308 base_url.trim_end_matches('/'),
309 owner,
310 name
311 )
312}
313
314pub async fn github_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
315 let client = client(Some("GITHUB_TOKEN"));
316 let mut repos = Vec::new();
317 for term in &query.terms {
318 let response = client
319 .get(github_search_url(term, query.limit))
320 .send()
321 .await?;
322 if response.status() == reqwest::StatusCode::FORBIDDEN
323 || response.status() == reqwest::StatusCode::UNAUTHORIZED
324 {
325 continue;
326 }
327 repos.extend(parse_github_search(
328 &response.error_for_status()?.text().await?,
329 )?);
330 }
331 Ok(dedup_repositories(repos))
332}
333
334pub async fn gitlab_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
335 let client = client(Some("GITLAB_TOKEN"));
336 let mut repos = Vec::new();
337 for term in &query.terms {
338 let response = client
339 .get(gitlab_search_url(term, query.limit))
340 .send()
341 .await?;
342 if response.status() == reqwest::StatusCode::FORBIDDEN
343 || response.status() == reqwest::StatusCode::UNAUTHORIZED
344 {
345 continue;
346 }
347 repos.extend(parse_gitlab_search(
348 &response.error_for_status()?.text().await?,
349 )?);
350 }
351 Ok(dedup_repositories(repos))
352}
353
354pub async fn forgejo_search(
355 provider: &str,
356 base_url: &str,
357 query: &RepoSearchQuery,
358) -> Result<Vec<Repository>> {
359 let client = client(None);
360 let mut repos = Vec::new();
361 for term in &query.terms {
362 let response = client
363 .get(forgejo_search_url(base_url, term, query.limit))
364 .send()
365 .await?;
366 if response.status() == reqwest::StatusCode::FORBIDDEN
367 || response.status() == reqwest::StatusCode::UNAUTHORIZED
368 {
369 continue;
370 }
371 repos.extend(parse_forgejo_search(
372 provider,
373 &response.error_for_status()?.text().await?,
374 )?);
375 }
376 Ok(dedup_repositories(repos))
377}
378
379pub async fn inspect(repo: &RepositoryRef) -> Result<Option<Repository>> {
380 let client = client(match repo.forge.as_str() {
381 "github" => Some("GITHUB_TOKEN"),
382 "gitlab" => Some("GITLAB_TOKEN"),
383 _ => None,
384 });
385 let url = match repo.forge.as_str() {
386 "github" => github_inspect_url(&repo.owner, &repo.name),
387 "gitlab" => gitlab_inspect_url(&repo.owner, &repo.name),
388 "codeberg" => forgejo_inspect_url("https://codeberg.org/api/v1", &repo.owner, &repo.name),
389 _ => return Ok(None),
390 };
391 let response = client.get(url).send().await?;
392 if response.status() == reqwest::StatusCode::NOT_FOUND {
393 return Ok(None);
394 }
395 let text = response.error_for_status()?.text().await?;
396 Ok(Some(match repo.forge.as_str() {
397 "github" => parse_github_repo(&text)?,
398 "gitlab" => parse_gitlab_repo(&text)?,
399 _ => parse_forgejo_repo(&repo.forge, &text)?,
400 }))
401}
402
403pub fn parse_github_search(json: &str) -> Result<Vec<Repository>> {
404 #[derive(Deserialize)]
405 struct Search {
406 items: Vec<GithubRepo>,
407 }
408 Ok(serde_json::from_str::<Search>(json)?
409 .items
410 .into_iter()
411 .map(Into::into)
412 .collect())
413}
414
415pub fn parse_github_repo(json: &str) -> Result<Repository> {
416 Ok(serde_json::from_str::<GithubRepo>(json)?.into())
417}
418
419pub fn parse_gitlab_search(json: &str) -> Result<Vec<Repository>> {
420 Ok(serde_json::from_str::<Vec<GitlabRepo>>(json)?
421 .into_iter()
422 .map(Into::into)
423 .collect())
424}
425
426pub fn parse_gitlab_repo(json: &str) -> Result<Repository> {
427 Ok(serde_json::from_str::<GitlabRepo>(json)?.into())
428}
429
430pub fn parse_forgejo_search(provider: &str, json: &str) -> Result<Vec<Repository>> {
431 #[derive(Deserialize)]
432 struct Search {
433 data: Vec<ForgejoRepo>,
434 }
435 Ok(serde_json::from_str::<Search>(json)?
436 .data
437 .into_iter()
438 .map(|r| r.into_repo(provider))
439 .collect())
440}
441
442pub fn parse_forgejo_repo(provider: &str, json: &str) -> Result<Repository> {
443 Ok(serde_json::from_str::<ForgejoRepo>(json)?.into_repo(provider))
444}
445
446fn client(token_env: Option<&str>) -> reqwest::Client {
447 let mut headers = HeaderMap::new();
448 headers.insert(USER_AGENT, HeaderValue::from_static("paper-gap"));
449 if let Some(token_env) = token_env.and_then(|name| std::env::var(name).ok()) {
450 if let Ok(value) = HeaderValue::from_str(&format!("Bearer {token_env}")) {
451 headers.insert(AUTHORIZATION, value);
452 }
453 }
454 reqwest::Client::builder()
455 .default_headers(headers)
456 .timeout(Duration::from_secs(8))
457 .build()
458 .expect("valid reqwest client")
459}
460
461#[derive(Deserialize)]
462struct GithubRepo {
463 html_url: String,
464 name: String,
465 owner: GithubOwner,
466 description: Option<String>,
467 language: Option<String>,
468 stargazers_count: Option<u64>,
469 forks_count: Option<u64>,
470 license: Option<GithubLicense>,
471 archived: Option<bool>,
472 default_branch: Option<String>,
473 pushed_at: Option<String>,
474}
475#[derive(Deserialize)]
476struct GithubOwner {
477 login: String,
478}
479#[derive(Deserialize)]
480struct GithubLicense {
481 spdx_id: Option<String>,
482 name: Option<String>,
483}
484
485impl From<GithubRepo> for Repository {
486 fn from(repo: GithubRepo) -> Self {
487 Repository {
488 forge: "github".into(),
489 url: repo.html_url,
490 owner: repo.owner.login,
491 name: repo.name,
492 description: repo.description,
493 primary_language: repo.language,
494 stars: repo.stargazers_count,
495 forks: repo.forks_count,
496 license: repo.license.and_then(|l| l.spdx_id.or(l.name)),
497 archived: repo.archived,
498 default_branch: repo.default_branch,
499 last_commit_date: repo.pushed_at,
500 source_provider: "github".into(),
501 }
502 }
503}
504
505#[derive(Deserialize)]
506struct GitlabRepo {
507 web_url: String,
508 path: String,
509 path_with_namespace: String,
510 description: Option<String>,
511 star_count: Option<u64>,
512 forks_count: Option<u64>,
513 archived: Option<bool>,
514 default_branch: Option<String>,
515 last_activity_at: Option<String>,
516}
517
518impl From<GitlabRepo> for Repository {
519 fn from(repo: GitlabRepo) -> Self {
520 let owner = repo
521 .path_with_namespace
522 .rsplit_once('/')
523 .map_or("", |(owner, _)| owner)
524 .to_string();
525 Repository {
526 forge: "gitlab".into(),
527 url: repo.web_url,
528 owner,
529 name: repo.path,
530 description: repo.description,
531 primary_language: None,
532 stars: repo.star_count,
533 forks: repo.forks_count,
534 license: None,
535 archived: repo.archived,
536 default_branch: repo.default_branch,
537 last_commit_date: repo.last_activity_at,
538 source_provider: "gitlab".into(),
539 }
540 }
541}
542
543#[derive(Deserialize)]
544struct ForgejoRepo {
545 html_url: String,
546 name: String,
547 owner: ForgejoOwner,
548 description: Option<String>,
549 language: Option<String>,
550 stars_count: Option<u64>,
551 forks_count: Option<u64>,
552 archived: Option<bool>,
553 default_branch: Option<String>,
554 updated_at: Option<String>,
555}
556#[derive(Deserialize)]
557struct ForgejoOwner {
558 login: String,
559}
560
561impl ForgejoRepo {
562 fn into_repo(self, provider: &str) -> Repository {
563 Repository {
564 forge: provider.into(),
565 url: self.html_url,
566 owner: self.owner.login,
567 name: self.name,
568 description: self.description,
569 primary_language: self.language,
570 stars: self.stars_count,
571 forks: self.forks_count,
572 license: None,
573 archived: self.archived,
574 default_branch: self.default_branch,
575 last_commit_date: self.updated_at,
576 source_provider: provider.into(),
577 }
578 }
579}
580
581fn dedup_repositories(repos: Vec<Repository>) -> Vec<Repository> {
582 let mut out = Vec::new();
583 for repo in repos {
584 if !out
585 .iter()
586 .any(|existing: &Repository| existing.url == repo.url)
587 {
588 out.push(repo);
589 }
590 }
591 out
592}
593
594fn dedup_strings(values: Vec<String>) -> Vec<String> {
595 let mut out = Vec::new();
596 for value in values.into_iter().filter(|s| !s.trim().is_empty()) {
597 if !out.iter().any(|existing| existing == &value) {
598 out.push(value);
599 }
600 }
601 out
602}
603
604fn trim_arxiv_version(arxiv_id: &str) -> &str {
605 arxiv_id.rsplit_once('v').map_or(arxiv_id, |(id, version)| {
606 if version.chars().all(|c| c.is_ascii_digit()) {
607 id
608 } else {
609 arxiv_id
610 }
611 })
612}