use crate::arxiv::Paper;
use crate::{
Result,
provider::{self, ProviderOutcome},
};
use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT};
use serde::{Deserialize, Serialize};
use std::time::Duration;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Repository {
pub forge: String,
pub url: String,
pub owner: String,
pub name: String,
pub description: Option<String>,
pub primary_language: Option<String>,
pub stars: Option<u64>,
pub forks: Option<u64>,
pub license: Option<String>,
pub archived: Option<bool>,
pub default_branch: Option<String>,
pub last_commit_date: Option<String>,
pub source_provider: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RepositoryMatch {
pub repository: Repository,
pub confidence_score: u32,
pub evidence: Vec<MatchEvidence>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct MatchEvidence {
pub kind: String,
pub value: String,
pub points: u32,
}
const MIN_MATCH_CONFIDENCE: u32 = 30;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RepositoryRef {
pub forge: String,
pub owner: String,
pub name: String,
pub url: String,
}
pub fn repository_from_url(url: &str) -> Option<Repository> {
let normalized = url.trim().trim_end_matches('/').trim_end_matches(".git");
let rest = normalized
.strip_prefix("https://")
.or_else(|| normalized.strip_prefix("http://"))?;
let (host, path) = rest.split_once('/')?;
let forge = match host.to_ascii_lowercase().as_str() {
"github.com" => "github",
"gitlab.com" => "gitlab",
"codeberg.org" => "codeberg",
_ => return None,
};
let parts: Vec<_> = path.split('/').filter(|part| !part.is_empty()).collect();
if parts.len() < 2 || (forge != "gitlab" && parts.len() != 2) {
return None;
}
let name = parts.last()?.to_string();
let owner = parts[..parts.len() - 1].join("/");
Some(Repository {
forge: forge.into(),
url: format!("https://{host}/{path}")
.trim_end_matches('/')
.trim_end_matches(".git")
.to_string(),
owner,
name,
description: None,
primary_language: None,
stars: None,
forks: None,
license: None,
archived: None,
default_branch: None,
last_commit_date: None,
source_provider: "papers-with-code".into(),
})
}
#[derive(Debug, Clone)]
pub struct RepoSearchQuery {
pub terms: Vec<String>,
pub limit: usize,
}
pub fn search_terms(paper: &Paper) -> Vec<String> {
let mut terms = vec![format!("\"{}\"", paper.title)];
if let Some(arxiv_id) = paper.arxiv_id.as_deref() {
terms.push(trim_arxiv_version(arxiv_id).to_string());
}
if let Some(doi) = paper.doi.as_deref() {
terms.push(doi.to_string());
}
terms.extend(
paper
.extracted_methods
.iter()
.filter(|term| is_specific_method_term(term))
.cloned(),
);
dedup_strings(terms)
}
pub async fn search_for_paper(paper: &Paper, limit: usize) -> Result<Vec<Repository>> {
let query = RepoSearchQuery {
terms: search_terms(paper).into_iter().take(3).collect(),
limit,
};
let mut repos = Vec::new();
repos.extend(github_search(&query).await.unwrap_or_default());
repos.extend(gitlab_search(&query).await.unwrap_or_default());
repos.extend(
forgejo_search("codeberg", "https://codeberg.org/api/v1", &query)
.await
.unwrap_or_default(),
);
Ok(filter_repositories(paper, dedup_repositories(repos)))
}
pub async fn search_for_paper_with_outcome(
paper: &Paper,
limit: usize,
) -> (Vec<Repository>, ProviderOutcome) {
search_for_paper_with_providers_and_outcome(paper, limit, &["github", "gitlab", "codeberg"])
.await
}
pub async fn search_for_paper_with_providers_and_outcome(
paper: &Paper,
limit: usize,
providers: &[&str],
) -> (Vec<Repository>, ProviderOutcome) {
let query = RepoSearchQuery {
terms: search_terms(paper).into_iter().take(3).collect(),
limit,
};
let mut repositories = Vec::new();
let mut outcomes = Vec::new();
for provider in providers {
let result = match *provider {
"github" => github_search(&query).await,
"gitlab" => gitlab_search(&query).await,
"codeberg" => forgejo_search("codeberg", "https://codeberg.org/api/v1", &query).await,
_ => continue,
};
match result {
Ok(found) => {
repositories.extend(found);
outcomes.push(ProviderOutcome::success(*provider));
}
Err(error) => outcomes.push(ProviderOutcome::from_error(*provider, &error)),
}
}
(
dedup_repositories(repositories),
ProviderOutcome::aggregate("repository-search", &outcomes),
)
}
pub fn filter_repositories(paper: &Paper, repositories: Vec<Repository>) -> Vec<Repository> {
rank_repositories(paper, repositories, &[])
.into_iter()
.map(|matched| matched.repository)
.collect()
}
pub fn rank_repositories(
paper: &Paper,
repositories: Vec<Repository>,
official_urls: &[String],
) -> Vec<RepositoryMatch> {
let mut matches: Vec<_> = repositories
.into_iter()
.filter_map(|repository| {
let matched = match_repository(paper, repository, official_urls);
(matched.confidence_score >= MIN_MATCH_CONFIDENCE).then_some(matched)
})
.collect();
matches.sort_by_key(|matched| std::cmp::Reverse(matched.confidence_score));
matches
}
pub fn match_repository(
paper: &Paper,
repository: Repository,
official_urls: &[String],
) -> RepositoryMatch {
let mut evidence = Vec::new();
let identity_text = format!(
"{} {} {}",
repository.owner, repository.name, repository.url
);
let description_text = repository.description.as_deref().unwrap_or_default();
let identity = normalize(&identity_text);
let description = normalize(description_text);
let whole = format!("{identity}{description}");
if official_urls
.iter()
.any(|url| normalized_repo_url(url) == normalized_repo_url(&repository.url))
{
push_evidence(
&mut evidence,
"official_pwc_link",
repository.url.clone(),
100,
);
}
if let Some(id) = paper.arxiv_id.as_deref().map(trim_arxiv_version) {
if whole.contains(&normalize(id)) {
push_evidence(&mut evidence, "exact_arxiv_id", id.into(), 100);
}
}
if let Some(doi) = paper.doi.as_deref() {
if whole.contains(&normalize(doi)) {
push_evidence(&mut evidence, "exact_doi", doi.into(), 100);
}
}
let title = normalize(&paper.title);
if !title.is_empty() && whole.contains(&title) {
push_evidence(&mut evidence, "exact_title", paper.title.clone(), 60);
} else {
let title_terms = meaningful_title_terms(&paper.title);
let identity_terms: Vec<_> = title_terms
.iter()
.filter(|term| contains_term(&identity_text, term))
.cloned()
.collect();
let description_terms: Vec<_> = title_terms
.iter()
.filter(|term| !identity_terms.contains(term) && contains_term(description_text, term))
.cloned()
.collect();
if !identity_terms.is_empty() {
push_evidence(
&mut evidence,
"title_identity_overlap",
identity_terms.join(", "),
(identity_terms.len() as u32 * 15).min(45),
);
}
if !description_terms.is_empty() {
push_evidence(
&mut evidence,
"description_overlap",
description_terms.join(", "),
(description_terms.len() as u32 * 10).min(30),
);
}
}
let methods: Vec<_> = paper
.extracted_methods
.iter()
.filter(|term| {
is_specific_method_term(term)
&& (contains_term(&identity_text, term) || contains_term(description_text, term))
})
.cloned()
.collect();
if !methods.is_empty() {
push_evidence(
&mut evidence,
"method_overlap",
methods.join(", "),
(methods.len() as u32 * 10).min(20),
);
}
let confidence_score = evidence
.iter()
.map(|item| item.points)
.sum::<u32>()
.min(100);
RepositoryMatch {
repository,
confidence_score,
evidence,
}
}
fn push_evidence(evidence: &mut Vec<MatchEvidence>, kind: &str, value: String, points: u32) {
evidence.push(MatchEvidence {
kind: kind.into(),
value,
points,
});
}
fn normalized_repo_url(url: &str) -> String {
url.trim_end_matches('/')
.trim_end_matches(".git")
.to_ascii_lowercase()
}
fn meaningful_title_terms(title: &str) -> Vec<String> {
let stop = [
"with", "from", "into", "using", "based", "towards", "toward", "paper", "study", "method",
"methods", "system", "systems", "program", "programs",
];
title
.split(|c: char| !c.is_alphanumeric())
.map(str::to_ascii_lowercase)
.filter(|term| term.len() >= 4 && !stop.contains(&term.as_str()))
.collect()
}
fn is_specific_method_term(term: &str) -> bool {
let broad = [
"ai", "ml", "rl", "api", "cpu", "gpu", "fft", "pde", "ode", "chc", "nlp", "llm", "cnn",
"rnn", "gcn", "sql", "http", "json", "xml",
];
let lower = term.to_ascii_lowercase();
term.len() > 3 && !broad.contains(&lower.as_str())
}
fn contains_term(text: &str, term: &str) -> bool {
let term: Vec<_> = term
.split(|c: char| !c.is_ascii_alphanumeric())
.map(normalize)
.filter(|part| !part.is_empty())
.collect();
let text: Vec<_> = text
.split(|c: char| !c.is_ascii_alphanumeric())
.map(normalize)
.filter(|part| !part.is_empty())
.collect();
!term.is_empty() && text.windows(term.len()).any(|window| window == term)
}
fn normalize(text: &str) -> String {
text.chars()
.filter(|c| c.is_ascii_alphanumeric())
.flat_map(char::to_lowercase)
.collect()
}
pub fn github_search_url(term: &str, limit: usize) -> String {
format!(
"https://api.github.com/search/repositories?q={}&per_page={}",
urlencoding::encode(term),
limit
)
}
pub fn gitlab_search_url(term: &str, limit: usize) -> String {
format!(
"https://gitlab.com/api/v4/projects?search={}&per_page={}",
urlencoding::encode(term),
limit
)
}
pub fn forgejo_search_url(base_url: &str, term: &str, limit: usize) -> String {
format!(
"{}/repos/search?q={}&limit={}",
base_url.trim_end_matches('/'),
urlencoding::encode(term),
limit
)
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RepositoryFile {
pub path: String,
}
pub fn github_tree_url(owner: &str, name: &str, branch: &str) -> String {
format!(
"https://api.github.com/repos/{}/{}/git/trees/{}?recursive=1",
owner,
name,
urlencoding::encode(branch)
)
}
pub fn gitlab_tree_url(owner: &str, name: &str, branch: &str) -> String {
format!(
"https://gitlab.com/api/v4/projects/{}/repository/tree?recursive=true&per_page=100&ref={}",
urlencoding::encode(&format!("{owner}/{name}")),
urlencoding::encode(branch)
)
}
pub fn forgejo_tree_url(base_url: &str, owner: &str, name: &str, branch: &str) -> String {
format!(
"{}/repos/{}/{}/git/trees/{}?recursive=true",
base_url.trim_end_matches('/'),
owner,
name,
urlencoding::encode(branch)
)
}
pub async fn file_paths(repo: &Repository) -> Result<Vec<String>> {
let client = client(match repo.forge.as_str() {
"github" => Some("GITHUB_TOKEN"),
"gitlab" => Some("GITLAB_TOKEN"),
_ => None,
});
let branch = repo.default_branch.as_deref().unwrap_or("main");
let url = match repo.forge.as_str() {
"github" => github_tree_url(&repo.owner, &repo.name, branch),
"gitlab" => gitlab_tree_url(&repo.owner, &repo.name, branch),
"codeberg" => forgejo_tree_url(
"https://codeberg.org/api/v1",
&repo.owner,
&repo.name,
branch,
),
_ => return Ok(Vec::new()),
};
let response = provider::send_with_retry(client.get(url)).await?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(Vec::new());
}
let text = response.error_for_status()?.text().await?;
match repo.forge.as_str() {
"github" => parse_github_tree(&text),
"gitlab" => parse_gitlab_tree(&text),
_ => parse_forgejo_tree(&text),
}
}
pub fn parse_github_tree(json: &str) -> Result<Vec<String>> {
#[derive(Deserialize)]
struct Tree {
tree: Vec<TreeItem>,
}
#[derive(Deserialize)]
struct TreeItem {
path: String,
}
Ok(serde_json::from_str::<Tree>(json)?
.tree
.into_iter()
.map(|item| item.path)
.collect())
}
pub fn parse_gitlab_tree(json: &str) -> Result<Vec<String>> {
#[derive(Deserialize)]
struct TreeItem {
path: String,
}
Ok(serde_json::from_str::<Vec<TreeItem>>(json)?
.into_iter()
.map(|item| item.path)
.collect())
}
pub fn parse_forgejo_tree(json: &str) -> Result<Vec<String>> {
parse_github_tree(json)
}
pub fn github_inspect_url(owner: &str, name: &str) -> String {
format!("https://api.github.com/repos/{}/{}", owner, name)
}
pub fn gitlab_inspect_url(owner: &str, name: &str) -> String {
format!(
"https://gitlab.com/api/v4/projects/{}",
urlencoding::encode(&format!("{owner}/{name}"))
)
}
pub fn forgejo_inspect_url(base_url: &str, owner: &str, name: &str) -> String {
format!(
"{}/repos/{}/{}",
base_url.trim_end_matches('/'),
owner,
name
)
}
pub async fn github_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
let client = client(Some("GITHUB_TOKEN"));
let mut repos = Vec::new();
for term in &query.terms {
let response = client.get(github_search_url(term, query.limit));
let response = provider::send_with_retry(response).await?;
if response.status() == reqwest::StatusCode::FORBIDDEN
|| response.status() == reqwest::StatusCode::UNAUTHORIZED
{
continue;
}
repos.extend(parse_github_search(
&response.error_for_status()?.text().await?,
)?);
}
Ok(dedup_repositories(repos))
}
pub async fn gitlab_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
let client = client(Some("GITLAB_TOKEN"));
let mut repos = Vec::new();
for term in &query.terms {
let response = client.get(gitlab_search_url(term, query.limit));
let response = provider::send_with_retry(response).await?;
if response.status() == reqwest::StatusCode::FORBIDDEN
|| response.status() == reqwest::StatusCode::UNAUTHORIZED
{
continue;
}
repos.extend(parse_gitlab_search(
&response.error_for_status()?.text().await?,
)?);
}
Ok(dedup_repositories(repos))
}
pub async fn forgejo_search(
provider: &str,
base_url: &str,
query: &RepoSearchQuery,
) -> Result<Vec<Repository>> {
let client = client(None);
let mut repos = Vec::new();
for term in &query.terms {
let response = client.get(forgejo_search_url(base_url, term, query.limit));
let response = provider::send_with_retry(response).await?;
if response.status() == reqwest::StatusCode::FORBIDDEN
|| response.status() == reqwest::StatusCode::UNAUTHORIZED
{
continue;
}
repos.extend(parse_forgejo_search(
provider,
&response.error_for_status()?.text().await?,
)?);
}
Ok(dedup_repositories(repos))
}
pub async fn inspect(repo: &RepositoryRef) -> Result<Option<Repository>> {
let client = client(match repo.forge.as_str() {
"github" => Some("GITHUB_TOKEN"),
"gitlab" => Some("GITLAB_TOKEN"),
_ => None,
});
let url = match repo.forge.as_str() {
"github" => github_inspect_url(&repo.owner, &repo.name),
"gitlab" => gitlab_inspect_url(&repo.owner, &repo.name),
"codeberg" => forgejo_inspect_url("https://codeberg.org/api/v1", &repo.owner, &repo.name),
_ => return Ok(None),
};
let response = provider::send_with_retry(client.get(url)).await?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(None);
}
let text = response.error_for_status()?.text().await?;
Ok(Some(match repo.forge.as_str() {
"github" => parse_github_repo(&text)?,
"gitlab" => parse_gitlab_repo(&text)?,
_ => parse_forgejo_repo(&repo.forge, &text)?,
}))
}
pub fn parse_github_search(json: &str) -> Result<Vec<Repository>> {
#[derive(Deserialize)]
struct Search {
items: Vec<GithubRepo>,
}
Ok(serde_json::from_str::<Search>(json)?
.items
.into_iter()
.map(Into::into)
.collect())
}
pub fn parse_github_repo(json: &str) -> Result<Repository> {
Ok(serde_json::from_str::<GithubRepo>(json)?.into())
}
pub fn parse_gitlab_search(json: &str) -> Result<Vec<Repository>> {
Ok(serde_json::from_str::<Vec<GitlabRepo>>(json)?
.into_iter()
.map(Into::into)
.collect())
}
pub fn parse_gitlab_repo(json: &str) -> Result<Repository> {
Ok(serde_json::from_str::<GitlabRepo>(json)?.into())
}
pub fn parse_forgejo_search(provider: &str, json: &str) -> Result<Vec<Repository>> {
#[derive(Deserialize)]
struct Search {
data: Vec<ForgejoRepo>,
}
Ok(serde_json::from_str::<Search>(json)?
.data
.into_iter()
.map(|r| r.into_repo(provider))
.collect())
}
pub fn parse_forgejo_repo(provider: &str, json: &str) -> Result<Repository> {
Ok(serde_json::from_str::<ForgejoRepo>(json)?.into_repo(provider))
}
fn client(token_env: Option<&str>) -> reqwest::Client {
let mut headers = HeaderMap::new();
headers.insert(USER_AGENT, HeaderValue::from_static("paper-gap"));
if let Some(token_env) = token_env.and_then(|name| std::env::var(name).ok()) {
if let Ok(value) = HeaderValue::from_str(&format!("Bearer {token_env}")) {
headers.insert(AUTHORIZATION, value);
}
}
reqwest::Client::builder()
.default_headers(headers)
.timeout(Duration::from_secs(8))
.build()
.expect("valid reqwest client")
}
#[derive(Deserialize)]
struct GithubRepo {
html_url: String,
name: String,
owner: GithubOwner,
description: Option<String>,
language: Option<String>,
stargazers_count: Option<u64>,
forks_count: Option<u64>,
license: Option<GithubLicense>,
archived: Option<bool>,
default_branch: Option<String>,
pushed_at: Option<String>,
}
#[derive(Deserialize)]
struct GithubOwner {
login: String,
}
#[derive(Deserialize)]
struct GithubLicense {
spdx_id: Option<String>,
name: Option<String>,
}
impl From<GithubRepo> for Repository {
fn from(repo: GithubRepo) -> Self {
Repository {
forge: "github".into(),
url: repo.html_url,
owner: repo.owner.login,
name: repo.name,
description: repo.description,
primary_language: repo.language,
stars: repo.stargazers_count,
forks: repo.forks_count,
license: repo.license.and_then(|l| l.spdx_id.or(l.name)),
archived: repo.archived,
default_branch: repo.default_branch,
last_commit_date: repo.pushed_at,
source_provider: "github".into(),
}
}
}
#[derive(Deserialize)]
struct GitlabRepo {
web_url: String,
path: String,
path_with_namespace: String,
description: Option<String>,
star_count: Option<u64>,
forks_count: Option<u64>,
archived: Option<bool>,
default_branch: Option<String>,
last_activity_at: Option<String>,
}
impl From<GitlabRepo> for Repository {
fn from(repo: GitlabRepo) -> Self {
let owner = repo
.path_with_namespace
.rsplit_once('/')
.map_or("", |(owner, _)| owner)
.to_string();
Repository {
forge: "gitlab".into(),
url: repo.web_url,
owner,
name: repo.path,
description: repo.description,
primary_language: None,
stars: repo.star_count,
forks: repo.forks_count,
license: None,
archived: repo.archived,
default_branch: repo.default_branch,
last_commit_date: repo.last_activity_at,
source_provider: "gitlab".into(),
}
}
}
#[derive(Deserialize)]
struct ForgejoRepo {
html_url: String,
name: String,
owner: ForgejoOwner,
description: Option<String>,
language: Option<String>,
stars_count: Option<u64>,
forks_count: Option<u64>,
archived: Option<bool>,
default_branch: Option<String>,
updated_at: Option<String>,
}
#[derive(Deserialize)]
struct ForgejoOwner {
login: String,
}
impl ForgejoRepo {
fn into_repo(self, provider: &str) -> Repository {
Repository {
forge: provider.into(),
url: self.html_url,
owner: self.owner.login,
name: self.name,
description: self.description,
primary_language: self.language,
stars: self.stars_count,
forks: self.forks_count,
license: None,
archived: self.archived,
default_branch: self.default_branch,
last_commit_date: self.updated_at,
source_provider: provider.into(),
}
}
}
fn dedup_repositories(repos: Vec<Repository>) -> Vec<Repository> {
let mut out = Vec::new();
for repo in repos {
if !out
.iter()
.any(|existing: &Repository| existing.url == repo.url)
{
out.push(repo);
}
}
out
}
fn dedup_strings(values: Vec<String>) -> Vec<String> {
let mut out = Vec::new();
for value in values.into_iter().filter(|s| !s.trim().is_empty()) {
if !out.iter().any(|existing| existing == &value) {
out.push(value);
}
}
out
}
fn trim_arxiv_version(arxiv_id: &str) -> &str {
arxiv_id.rsplit_once('v').map_or(arxiv_id, |(id, version)| {
if version.chars().all(|c| c.is_ascii_digit()) {
id
} else {
arxiv_id
}
})
}