1use crate::arxiv::Paper;
2use crate::{Result, provider::ProviderOutcome};
3use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT};
4use serde::{Deserialize, Serialize};
5use std::time::Duration;
6
7#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
8pub struct Repository {
9 pub forge: String,
10 pub url: String,
11 pub owner: String,
12 pub name: String,
13 pub description: Option<String>,
14 pub primary_language: Option<String>,
15 pub stars: Option<u64>,
16 pub forks: Option<u64>,
17 pub license: Option<String>,
18 pub archived: Option<bool>,
19 pub default_branch: Option<String>,
20 pub last_commit_date: Option<String>,
21 pub source_provider: String,
22}
23
24#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
25pub struct RepositoryMatch {
26 pub repository: Repository,
27 pub confidence_score: u32,
28 pub evidence: Vec<MatchEvidence>,
29}
30
31#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
32pub struct MatchEvidence {
33 pub kind: String,
34 pub value: String,
35 pub points: u32,
36}
37
38const MIN_MATCH_CONFIDENCE: u32 = 30;
39
40#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
41pub struct RepositoryRef {
42 pub forge: String,
43 pub owner: String,
44 pub name: String,
45 pub url: String,
46}
47
48#[derive(Debug, Clone)]
49pub struct RepoSearchQuery {
50 pub terms: Vec<String>,
51 pub limit: usize,
52}
53
54pub fn search_terms(paper: &Paper) -> Vec<String> {
55 let mut terms = vec![format!("\"{}\"", paper.title)];
56 if let Some(arxiv_id) = paper.arxiv_id.as_deref() {
57 terms.push(trim_arxiv_version(arxiv_id).to_string());
58 }
59 if let Some(doi) = paper.doi.as_deref() {
60 terms.push(doi.to_string());
61 }
62 terms.extend(
63 paper
64 .extracted_methods
65 .iter()
66 .filter(|term| is_specific_method_term(term))
67 .cloned(),
68 );
69 dedup_strings(terms)
70}
71
72pub async fn search_for_paper(paper: &Paper, limit: usize) -> Result<Vec<Repository>> {
73 let query = RepoSearchQuery {
74 terms: search_terms(paper).into_iter().take(3).collect(),
76 limit,
77 };
78 let mut repos = Vec::new();
79 repos.extend(github_search(&query).await.unwrap_or_default());
80 repos.extend(gitlab_search(&query).await.unwrap_or_default());
81 repos.extend(
82 forgejo_search("codeberg", "https://codeberg.org/api/v1", &query)
83 .await
84 .unwrap_or_default(),
85 );
86 Ok(filter_repositories(paper, dedup_repositories(repos)))
87}
88
89pub async fn search_for_paper_with_outcome(
90 paper: &Paper,
91 limit: usize,
92) -> (Vec<Repository>, ProviderOutcome) {
93 let query = RepoSearchQuery {
94 terms: search_terms(paper).into_iter().take(3).collect(),
95 limit,
96 };
97 let mut repositories = Vec::new();
98 let mut outcomes = Vec::new();
99
100 for (provider, result) in [
101 ("github", github_search(&query).await),
102 ("gitlab", gitlab_search(&query).await),
103 (
104 "codeberg",
105 forgejo_search("codeberg", "https://codeberg.org/api/v1", &query).await,
106 ),
107 ] {
108 match result {
109 Ok(found) => {
110 repositories.extend(found);
111 outcomes.push(ProviderOutcome::success(provider));
112 }
113 Err(error) => outcomes.push(ProviderOutcome::from_error(provider, &error)),
114 }
115 }
116
117 (
118 dedup_repositories(repositories),
119 ProviderOutcome::aggregate("repository-search", &outcomes),
120 )
121}
122
123pub fn filter_repositories(paper: &Paper, repositories: Vec<Repository>) -> Vec<Repository> {
124 rank_repositories(paper, repositories, &[])
125 .into_iter()
126 .map(|matched| matched.repository)
127 .collect()
128}
129
130pub fn rank_repositories(
131 paper: &Paper,
132 repositories: Vec<Repository>,
133 official_urls: &[String],
134) -> Vec<RepositoryMatch> {
135 let mut matches: Vec<_> = repositories
136 .into_iter()
137 .filter_map(|repository| {
138 let matched = match_repository(paper, repository, official_urls);
139 (matched.confidence_score >= MIN_MATCH_CONFIDENCE).then_some(matched)
140 })
141 .collect();
142 matches.sort_by_key(|matched| std::cmp::Reverse(matched.confidence_score));
143 matches
144}
145
146fn match_repository(
147 paper: &Paper,
148 repository: Repository,
149 official_urls: &[String],
150) -> RepositoryMatch {
151 let mut evidence = Vec::new();
152 let identity = normalize(&format!(
153 "{} {} {}",
154 repository.owner, repository.name, repository.url
155 ));
156 let description = normalize(repository.description.as_deref().unwrap_or_default());
157 let whole = format!("{identity}{description}");
158
159 if official_urls
160 .iter()
161 .any(|url| normalized_repo_url(url) == normalized_repo_url(&repository.url))
162 {
163 push_evidence(
164 &mut evidence,
165 "official_pwc_link",
166 repository.url.clone(),
167 100,
168 );
169 }
170 if let Some(id) = paper.arxiv_id.as_deref().map(trim_arxiv_version) {
171 if whole.contains(&normalize(id)) {
172 push_evidence(&mut evidence, "exact_arxiv_id", id.into(), 100);
173 }
174 }
175 if let Some(doi) = paper.doi.as_deref() {
176 if whole.contains(&normalize(doi)) {
177 push_evidence(&mut evidence, "exact_doi", doi.into(), 100);
178 }
179 }
180
181 let title = normalize(&paper.title);
182 if !title.is_empty() && whole.contains(&title) {
183 push_evidence(&mut evidence, "exact_title", paper.title.clone(), 60);
184 } else {
185 let title_terms = meaningful_title_terms(&paper.title);
186 let identity_terms: Vec<_> = title_terms
187 .iter()
188 .filter(|term| identity.contains(term.as_str()))
189 .cloned()
190 .collect();
191 let description_terms: Vec<_> = title_terms
192 .iter()
193 .filter(|term| !identity_terms.contains(term) && description.contains(term.as_str()))
194 .cloned()
195 .collect();
196 if !identity_terms.is_empty() {
197 push_evidence(
198 &mut evidence,
199 "title_identity_overlap",
200 identity_terms.join(", "),
201 (identity_terms.len() as u32 * 15).min(45),
202 );
203 }
204 if !description_terms.is_empty() {
205 push_evidence(
206 &mut evidence,
207 "description_overlap",
208 description_terms.join(", "),
209 (description_terms.len() as u32 * 10).min(30),
210 );
211 }
212 }
213
214 let methods: Vec<_> = paper
215 .extracted_methods
216 .iter()
217 .filter(|term| is_specific_method_term(term) && whole.contains(&normalize(term)))
218 .cloned()
219 .collect();
220 if !methods.is_empty() {
221 push_evidence(
222 &mut evidence,
223 "method_overlap",
224 methods.join(", "),
225 (methods.len() as u32 * 10).min(20),
226 );
227 }
228
229 let confidence_score = evidence
230 .iter()
231 .map(|item| item.points)
232 .sum::<u32>()
233 .min(100);
234 RepositoryMatch {
235 repository,
236 confidence_score,
237 evidence,
238 }
239}
240
241fn push_evidence(evidence: &mut Vec<MatchEvidence>, kind: &str, value: String, points: u32) {
242 evidence.push(MatchEvidence {
243 kind: kind.into(),
244 value,
245 points,
246 });
247}
248
249fn normalized_repo_url(url: &str) -> String {
250 url.trim_end_matches('/')
251 .trim_end_matches(".git")
252 .to_ascii_lowercase()
253}
254
255fn meaningful_title_terms(title: &str) -> Vec<String> {
256 let stop = [
257 "with", "from", "into", "using", "based", "towards", "toward", "paper", "study", "method",
258 "methods", "system", "systems", "program", "programs",
259 ];
260 title
261 .split(|c: char| !c.is_alphanumeric())
262 .map(str::to_ascii_lowercase)
263 .filter(|term| term.len() >= 4 && !stop.contains(&term.as_str()))
264 .collect()
265}
266
267fn is_specific_method_term(term: &str) -> bool {
268 let broad = [
269 "ai", "ml", "rl", "api", "cpu", "gpu", "fft", "pde", "ode", "chc", "nlp", "llm", "cnn",
270 "rnn", "gcn", "sql", "http", "json", "xml",
271 ];
272 let lower = term.to_ascii_lowercase();
273 term.len() > 3 && !broad.contains(&lower.as_str())
274}
275
276fn normalize(text: &str) -> String {
277 text.chars()
278 .filter(|c| c.is_ascii_alphanumeric())
279 .flat_map(char::to_lowercase)
280 .collect()
281}
282
283pub fn github_search_url(term: &str, limit: usize) -> String {
284 format!(
285 "https://api.github.com/search/repositories?q={}&per_page={}",
286 urlencoding::encode(term),
287 limit
288 )
289}
290
291pub fn gitlab_search_url(term: &str, limit: usize) -> String {
292 format!(
293 "https://gitlab.com/api/v4/projects?search={}&per_page={}",
294 urlencoding::encode(term),
295 limit
296 )
297}
298
299pub fn forgejo_search_url(base_url: &str, term: &str, limit: usize) -> String {
300 format!(
301 "{}/repos/search?q={}&limit={}",
302 base_url.trim_end_matches('/'),
303 urlencoding::encode(term),
304 limit
305 )
306}
307
308#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
309pub struct RepositoryFile {
310 pub path: String,
311}
312
313pub fn github_tree_url(owner: &str, name: &str, branch: &str) -> String {
314 format!(
315 "https://api.github.com/repos/{}/{}/git/trees/{}?recursive=1",
316 owner,
317 name,
318 urlencoding::encode(branch)
319 )
320}
321
322pub fn gitlab_tree_url(owner: &str, name: &str, branch: &str) -> String {
323 format!(
324 "https://gitlab.com/api/v4/projects/{}/repository/tree?recursive=true&per_page=100&ref={}",
325 urlencoding::encode(&format!("{owner}/{name}")),
326 urlencoding::encode(branch)
327 )
328}
329
330pub fn forgejo_tree_url(base_url: &str, owner: &str, name: &str, branch: &str) -> String {
331 format!(
332 "{}/repos/{}/{}/git/trees/{}?recursive=true",
333 base_url.trim_end_matches('/'),
334 owner,
335 name,
336 urlencoding::encode(branch)
337 )
338}
339
340pub async fn file_paths(repo: &Repository) -> Result<Vec<String>> {
341 let client = client(match repo.forge.as_str() {
342 "github" => Some("GITHUB_TOKEN"),
343 "gitlab" => Some("GITLAB_TOKEN"),
344 _ => None,
345 });
346 let branch = repo.default_branch.as_deref().unwrap_or("main");
347 let url = match repo.forge.as_str() {
348 "github" => github_tree_url(&repo.owner, &repo.name, branch),
349 "gitlab" => gitlab_tree_url(&repo.owner, &repo.name, branch),
350 "codeberg" => forgejo_tree_url(
351 "https://codeberg.org/api/v1",
352 &repo.owner,
353 &repo.name,
354 branch,
355 ),
356 _ => return Ok(Vec::new()),
357 };
358 let response = client.get(url).send().await?;
359 if response.status() == reqwest::StatusCode::NOT_FOUND {
360 return Ok(Vec::new());
361 }
362 let text = response.error_for_status()?.text().await?;
363 match repo.forge.as_str() {
364 "github" => parse_github_tree(&text),
365 "gitlab" => parse_gitlab_tree(&text),
366 _ => parse_forgejo_tree(&text),
367 }
368}
369
370pub fn parse_github_tree(json: &str) -> Result<Vec<String>> {
371 #[derive(Deserialize)]
372 struct Tree {
373 tree: Vec<TreeItem>,
374 }
375 #[derive(Deserialize)]
376 struct TreeItem {
377 path: String,
378 }
379 Ok(serde_json::from_str::<Tree>(json)?
380 .tree
381 .into_iter()
382 .map(|item| item.path)
383 .collect())
384}
385
386pub fn parse_gitlab_tree(json: &str) -> Result<Vec<String>> {
387 #[derive(Deserialize)]
388 struct TreeItem {
389 path: String,
390 }
391 Ok(serde_json::from_str::<Vec<TreeItem>>(json)?
392 .into_iter()
393 .map(|item| item.path)
394 .collect())
395}
396
397pub fn parse_forgejo_tree(json: &str) -> Result<Vec<String>> {
398 parse_github_tree(json)
399}
400
401pub fn github_inspect_url(owner: &str, name: &str) -> String {
402 format!("https://api.github.com/repos/{}/{}", owner, name)
403}
404
405pub fn gitlab_inspect_url(owner: &str, name: &str) -> String {
406 format!(
407 "https://gitlab.com/api/v4/projects/{}",
408 urlencoding::encode(&format!("{owner}/{name}"))
409 )
410}
411
412pub fn forgejo_inspect_url(base_url: &str, owner: &str, name: &str) -> String {
413 format!(
414 "{}/repos/{}/{}",
415 base_url.trim_end_matches('/'),
416 owner,
417 name
418 )
419}
420
421pub async fn github_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
422 let client = client(Some("GITHUB_TOKEN"));
423 let mut repos = Vec::new();
424 for term in &query.terms {
425 let response = client
426 .get(github_search_url(term, query.limit))
427 .send()
428 .await?;
429 if response.status() == reqwest::StatusCode::FORBIDDEN
430 || response.status() == reqwest::StatusCode::UNAUTHORIZED
431 {
432 continue;
433 }
434 repos.extend(parse_github_search(
435 &response.error_for_status()?.text().await?,
436 )?);
437 }
438 Ok(dedup_repositories(repos))
439}
440
441pub async fn gitlab_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
442 let client = client(Some("GITLAB_TOKEN"));
443 let mut repos = Vec::new();
444 for term in &query.terms {
445 let response = client
446 .get(gitlab_search_url(term, query.limit))
447 .send()
448 .await?;
449 if response.status() == reqwest::StatusCode::FORBIDDEN
450 || response.status() == reqwest::StatusCode::UNAUTHORIZED
451 {
452 continue;
453 }
454 repos.extend(parse_gitlab_search(
455 &response.error_for_status()?.text().await?,
456 )?);
457 }
458 Ok(dedup_repositories(repos))
459}
460
461pub async fn forgejo_search(
462 provider: &str,
463 base_url: &str,
464 query: &RepoSearchQuery,
465) -> Result<Vec<Repository>> {
466 let client = client(None);
467 let mut repos = Vec::new();
468 for term in &query.terms {
469 let response = client
470 .get(forgejo_search_url(base_url, term, query.limit))
471 .send()
472 .await?;
473 if response.status() == reqwest::StatusCode::FORBIDDEN
474 || response.status() == reqwest::StatusCode::UNAUTHORIZED
475 {
476 continue;
477 }
478 repos.extend(parse_forgejo_search(
479 provider,
480 &response.error_for_status()?.text().await?,
481 )?);
482 }
483 Ok(dedup_repositories(repos))
484}
485
486pub async fn inspect(repo: &RepositoryRef) -> Result<Option<Repository>> {
487 let client = client(match repo.forge.as_str() {
488 "github" => Some("GITHUB_TOKEN"),
489 "gitlab" => Some("GITLAB_TOKEN"),
490 _ => None,
491 });
492 let url = match repo.forge.as_str() {
493 "github" => github_inspect_url(&repo.owner, &repo.name),
494 "gitlab" => gitlab_inspect_url(&repo.owner, &repo.name),
495 "codeberg" => forgejo_inspect_url("https://codeberg.org/api/v1", &repo.owner, &repo.name),
496 _ => return Ok(None),
497 };
498 let response = client.get(url).send().await?;
499 if response.status() == reqwest::StatusCode::NOT_FOUND {
500 return Ok(None);
501 }
502 let text = response.error_for_status()?.text().await?;
503 Ok(Some(match repo.forge.as_str() {
504 "github" => parse_github_repo(&text)?,
505 "gitlab" => parse_gitlab_repo(&text)?,
506 _ => parse_forgejo_repo(&repo.forge, &text)?,
507 }))
508}
509
510pub fn parse_github_search(json: &str) -> Result<Vec<Repository>> {
511 #[derive(Deserialize)]
512 struct Search {
513 items: Vec<GithubRepo>,
514 }
515 Ok(serde_json::from_str::<Search>(json)?
516 .items
517 .into_iter()
518 .map(Into::into)
519 .collect())
520}
521
522pub fn parse_github_repo(json: &str) -> Result<Repository> {
523 Ok(serde_json::from_str::<GithubRepo>(json)?.into())
524}
525
526pub fn parse_gitlab_search(json: &str) -> Result<Vec<Repository>> {
527 Ok(serde_json::from_str::<Vec<GitlabRepo>>(json)?
528 .into_iter()
529 .map(Into::into)
530 .collect())
531}
532
533pub fn parse_gitlab_repo(json: &str) -> Result<Repository> {
534 Ok(serde_json::from_str::<GitlabRepo>(json)?.into())
535}
536
537pub fn parse_forgejo_search(provider: &str, json: &str) -> Result<Vec<Repository>> {
538 #[derive(Deserialize)]
539 struct Search {
540 data: Vec<ForgejoRepo>,
541 }
542 Ok(serde_json::from_str::<Search>(json)?
543 .data
544 .into_iter()
545 .map(|r| r.into_repo(provider))
546 .collect())
547}
548
549pub fn parse_forgejo_repo(provider: &str, json: &str) -> Result<Repository> {
550 Ok(serde_json::from_str::<ForgejoRepo>(json)?.into_repo(provider))
551}
552
553fn client(token_env: Option<&str>) -> reqwest::Client {
554 let mut headers = HeaderMap::new();
555 headers.insert(USER_AGENT, HeaderValue::from_static("paper-gap"));
556 if let Some(token_env) = token_env.and_then(|name| std::env::var(name).ok()) {
557 if let Ok(value) = HeaderValue::from_str(&format!("Bearer {token_env}")) {
558 headers.insert(AUTHORIZATION, value);
559 }
560 }
561 reqwest::Client::builder()
562 .default_headers(headers)
563 .timeout(Duration::from_secs(8))
564 .build()
565 .expect("valid reqwest client")
566}
567
568#[derive(Deserialize)]
569struct GithubRepo {
570 html_url: String,
571 name: String,
572 owner: GithubOwner,
573 description: Option<String>,
574 language: Option<String>,
575 stargazers_count: Option<u64>,
576 forks_count: Option<u64>,
577 license: Option<GithubLicense>,
578 archived: Option<bool>,
579 default_branch: Option<String>,
580 pushed_at: Option<String>,
581}
582#[derive(Deserialize)]
583struct GithubOwner {
584 login: String,
585}
586#[derive(Deserialize)]
587struct GithubLicense {
588 spdx_id: Option<String>,
589 name: Option<String>,
590}
591
592impl From<GithubRepo> for Repository {
593 fn from(repo: GithubRepo) -> Self {
594 Repository {
595 forge: "github".into(),
596 url: repo.html_url,
597 owner: repo.owner.login,
598 name: repo.name,
599 description: repo.description,
600 primary_language: repo.language,
601 stars: repo.stargazers_count,
602 forks: repo.forks_count,
603 license: repo.license.and_then(|l| l.spdx_id.or(l.name)),
604 archived: repo.archived,
605 default_branch: repo.default_branch,
606 last_commit_date: repo.pushed_at,
607 source_provider: "github".into(),
608 }
609 }
610}
611
612#[derive(Deserialize)]
613struct GitlabRepo {
614 web_url: String,
615 path: String,
616 path_with_namespace: String,
617 description: Option<String>,
618 star_count: Option<u64>,
619 forks_count: Option<u64>,
620 archived: Option<bool>,
621 default_branch: Option<String>,
622 last_activity_at: Option<String>,
623}
624
625impl From<GitlabRepo> for Repository {
626 fn from(repo: GitlabRepo) -> Self {
627 let owner = repo
628 .path_with_namespace
629 .rsplit_once('/')
630 .map_or("", |(owner, _)| owner)
631 .to_string();
632 Repository {
633 forge: "gitlab".into(),
634 url: repo.web_url,
635 owner,
636 name: repo.path,
637 description: repo.description,
638 primary_language: None,
639 stars: repo.star_count,
640 forks: repo.forks_count,
641 license: None,
642 archived: repo.archived,
643 default_branch: repo.default_branch,
644 last_commit_date: repo.last_activity_at,
645 source_provider: "gitlab".into(),
646 }
647 }
648}
649
650#[derive(Deserialize)]
651struct ForgejoRepo {
652 html_url: String,
653 name: String,
654 owner: ForgejoOwner,
655 description: Option<String>,
656 language: Option<String>,
657 stars_count: Option<u64>,
658 forks_count: Option<u64>,
659 archived: Option<bool>,
660 default_branch: Option<String>,
661 updated_at: Option<String>,
662}
663#[derive(Deserialize)]
664struct ForgejoOwner {
665 login: String,
666}
667
668impl ForgejoRepo {
669 fn into_repo(self, provider: &str) -> Repository {
670 Repository {
671 forge: provider.into(),
672 url: self.html_url,
673 owner: self.owner.login,
674 name: self.name,
675 description: self.description,
676 primary_language: self.language,
677 stars: self.stars_count,
678 forks: self.forks_count,
679 license: None,
680 archived: self.archived,
681 default_branch: self.default_branch,
682 last_commit_date: self.updated_at,
683 source_provider: provider.into(),
684 }
685 }
686}
687
688fn dedup_repositories(repos: Vec<Repository>) -> Vec<Repository> {
689 let mut out = Vec::new();
690 for repo in repos {
691 if !out
692 .iter()
693 .any(|existing: &Repository| existing.url == repo.url)
694 {
695 out.push(repo);
696 }
697 }
698 out
699}
700
701fn dedup_strings(values: Vec<String>) -> Vec<String> {
702 let mut out = Vec::new();
703 for value in values.into_iter().filter(|s| !s.trim().is_empty()) {
704 if !out.iter().any(|existing| existing == &value) {
705 out.push(value);
706 }
707 }
708 out
709}
710
711fn trim_arxiv_version(arxiv_id: &str) -> &str {
712 arxiv_id.rsplit_once('v').map_or(arxiv_id, |(id, version)| {
713 if version.chars().all(|c| c.is_ascii_digit()) {
714 id
715 } else {
716 arxiv_id
717 }
718 })
719}