use crate::Result;
use crate::arxiv::Paper;
use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT};
use serde::{Deserialize, Serialize};
use std::time::Duration;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Repository {
pub forge: String,
pub url: String,
pub owner: String,
pub name: String,
pub description: Option<String>,
pub primary_language: Option<String>,
pub stars: Option<u64>,
pub forks: Option<u64>,
pub license: Option<String>,
pub archived: Option<bool>,
pub default_branch: Option<String>,
pub last_commit_date: Option<String>,
pub source_provider: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RepositoryRef {
pub forge: String,
pub owner: String,
pub name: String,
pub url: String,
}
#[derive(Debug, Clone)]
pub struct RepoSearchQuery {
pub terms: Vec<String>,
pub limit: usize,
}
pub fn search_terms(paper: &Paper) -> Vec<String> {
let mut terms = vec![format!("\"{}\"", paper.title)];
if let Some(arxiv_id) = paper.arxiv_id.as_deref() {
terms.push(trim_arxiv_version(arxiv_id).to_string());
}
if let Some(doi) = paper.doi.as_deref() {
terms.push(doi.to_string());
}
terms.extend(
paper
.extracted_methods
.iter()
.filter(|term| is_specific_method_term(term))
.cloned(),
);
dedup_strings(terms)
}
pub async fn search_for_paper(paper: &Paper, limit: usize) -> Result<Vec<Repository>> {
let query = RepoSearchQuery {
terms: search_terms(paper).into_iter().take(3).collect(),
limit,
};
let mut repos = Vec::new();
repos.extend(github_search(&query).await.unwrap_or_default());
repos.extend(gitlab_search(&query).await.unwrap_or_default());
repos.extend(
forgejo_search("codeberg", "https://codeberg.org/api/v1", &query)
.await
.unwrap_or_default(),
);
Ok(filter_repositories(paper, dedup_repositories(repos)))
}
pub fn filter_repositories(paper: &Paper, repositories: Vec<Repository>) -> Vec<Repository> {
repositories
.into_iter()
.filter(|repository| repository_matches_paper(paper, repository))
.collect()
}
fn repository_matches_paper(paper: &Paper, repository: &Repository) -> bool {
let haystack = normalize(&format!(
"{} {} {} {} {}",
repository.owner,
repository.name,
repository.url,
repository.description.as_deref().unwrap_or_default(),
repository.primary_language.as_deref().unwrap_or_default()
));
if paper
.arxiv_id
.as_deref()
.map(trim_arxiv_version)
.is_some_and(|id| haystack.contains(&normalize(id)))
{
return true;
}
if paper
.doi
.as_deref()
.is_some_and(|doi| haystack.contains(&normalize(doi)))
{
return true;
}
let terms = meaningful_title_terms(&paper.title);
let matches = terms
.iter()
.filter(|term| haystack.contains(term.as_str()))
.count();
matches >= 2 || (terms.len() == 1 && matches == 1)
}
fn meaningful_title_terms(title: &str) -> Vec<String> {
let stop = [
"with", "from", "into", "using", "based", "towards", "toward", "paper", "study", "method",
"methods", "system", "systems", "program", "programs",
];
title
.split(|c: char| !c.is_alphanumeric())
.map(str::to_ascii_lowercase)
.filter(|term| term.len() >= 4 && !stop.contains(&term.as_str()))
.collect()
}
fn is_specific_method_term(term: &str) -> bool {
let broad = [
"ai", "ml", "rl", "api", "cpu", "gpu", "fft", "pde", "ode", "chc", "nlp", "llm", "cnn",
"rnn", "gcn", "sql", "http", "json", "xml",
];
let lower = term.to_ascii_lowercase();
term.len() > 3 && !broad.contains(&lower.as_str())
}
fn normalize(text: &str) -> String {
text.chars()
.filter(|c| c.is_ascii_alphanumeric())
.flat_map(char::to_lowercase)
.collect()
}
pub fn github_search_url(term: &str, limit: usize) -> String {
format!(
"https://api.github.com/search/repositories?q={}&per_page={}",
urlencoding::encode(term),
limit
)
}
pub fn gitlab_search_url(term: &str, limit: usize) -> String {
format!(
"https://gitlab.com/api/v4/projects?search={}&per_page={}",
urlencoding::encode(term),
limit
)
}
pub fn forgejo_search_url(base_url: &str, term: &str, limit: usize) -> String {
format!(
"{}/repos/search?q={}&limit={}",
base_url.trim_end_matches('/'),
urlencoding::encode(term),
limit
)
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RepositoryFile {
pub path: String,
}
pub fn github_tree_url(owner: &str, name: &str, branch: &str) -> String {
format!(
"https://api.github.com/repos/{}/{}/git/trees/{}?recursive=1",
owner,
name,
urlencoding::encode(branch)
)
}
pub fn gitlab_tree_url(owner: &str, name: &str, branch: &str) -> String {
format!(
"https://gitlab.com/api/v4/projects/{}/repository/tree?recursive=true&per_page=100&ref={}",
urlencoding::encode(&format!("{owner}/{name}")),
urlencoding::encode(branch)
)
}
pub fn forgejo_tree_url(base_url: &str, owner: &str, name: &str, branch: &str) -> String {
format!(
"{}/repos/{}/{}/git/trees/{}?recursive=true",
base_url.trim_end_matches('/'),
owner,
name,
urlencoding::encode(branch)
)
}
pub async fn file_paths(repo: &Repository) -> Result<Vec<String>> {
let client = client(match repo.forge.as_str() {
"github" => Some("GITHUB_TOKEN"),
"gitlab" => Some("GITLAB_TOKEN"),
_ => None,
});
let branch = repo.default_branch.as_deref().unwrap_or("main");
let url = match repo.forge.as_str() {
"github" => github_tree_url(&repo.owner, &repo.name, branch),
"gitlab" => gitlab_tree_url(&repo.owner, &repo.name, branch),
"codeberg" => forgejo_tree_url(
"https://codeberg.org/api/v1",
&repo.owner,
&repo.name,
branch,
),
_ => return Ok(Vec::new()),
};
let response = client.get(url).send().await?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(Vec::new());
}
let text = response.error_for_status()?.text().await?;
match repo.forge.as_str() {
"github" => parse_github_tree(&text),
"gitlab" => parse_gitlab_tree(&text),
_ => parse_forgejo_tree(&text),
}
}
pub fn parse_github_tree(json: &str) -> Result<Vec<String>> {
#[derive(Deserialize)]
struct Tree {
tree: Vec<TreeItem>,
}
#[derive(Deserialize)]
struct TreeItem {
path: String,
}
Ok(serde_json::from_str::<Tree>(json)?
.tree
.into_iter()
.map(|item| item.path)
.collect())
}
pub fn parse_gitlab_tree(json: &str) -> Result<Vec<String>> {
#[derive(Deserialize)]
struct TreeItem {
path: String,
}
Ok(serde_json::from_str::<Vec<TreeItem>>(json)?
.into_iter()
.map(|item| item.path)
.collect())
}
pub fn parse_forgejo_tree(json: &str) -> Result<Vec<String>> {
parse_github_tree(json)
}
pub fn github_inspect_url(owner: &str, name: &str) -> String {
format!("https://api.github.com/repos/{}/{}", owner, name)
}
pub fn gitlab_inspect_url(owner: &str, name: &str) -> String {
format!(
"https://gitlab.com/api/v4/projects/{}",
urlencoding::encode(&format!("{owner}/{name}"))
)
}
pub fn forgejo_inspect_url(base_url: &str, owner: &str, name: &str) -> String {
format!(
"{}/repos/{}/{}",
base_url.trim_end_matches('/'),
owner,
name
)
}
pub async fn github_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
let client = client(Some("GITHUB_TOKEN"));
let mut repos = Vec::new();
for term in &query.terms {
let response = client
.get(github_search_url(term, query.limit))
.send()
.await?;
if response.status() == reqwest::StatusCode::FORBIDDEN
|| response.status() == reqwest::StatusCode::UNAUTHORIZED
{
continue;
}
repos.extend(parse_github_search(
&response.error_for_status()?.text().await?,
)?);
}
Ok(dedup_repositories(repos))
}
pub async fn gitlab_search(query: &RepoSearchQuery) -> Result<Vec<Repository>> {
let client = client(Some("GITLAB_TOKEN"));
let mut repos = Vec::new();
for term in &query.terms {
let response = client
.get(gitlab_search_url(term, query.limit))
.send()
.await?;
if response.status() == reqwest::StatusCode::FORBIDDEN
|| response.status() == reqwest::StatusCode::UNAUTHORIZED
{
continue;
}
repos.extend(parse_gitlab_search(
&response.error_for_status()?.text().await?,
)?);
}
Ok(dedup_repositories(repos))
}
pub async fn forgejo_search(
provider: &str,
base_url: &str,
query: &RepoSearchQuery,
) -> Result<Vec<Repository>> {
let client = client(None);
let mut repos = Vec::new();
for term in &query.terms {
let response = client
.get(forgejo_search_url(base_url, term, query.limit))
.send()
.await?;
if response.status() == reqwest::StatusCode::FORBIDDEN
|| response.status() == reqwest::StatusCode::UNAUTHORIZED
{
continue;
}
repos.extend(parse_forgejo_search(
provider,
&response.error_for_status()?.text().await?,
)?);
}
Ok(dedup_repositories(repos))
}
pub async fn inspect(repo: &RepositoryRef) -> Result<Option<Repository>> {
let client = client(match repo.forge.as_str() {
"github" => Some("GITHUB_TOKEN"),
"gitlab" => Some("GITLAB_TOKEN"),
_ => None,
});
let url = match repo.forge.as_str() {
"github" => github_inspect_url(&repo.owner, &repo.name),
"gitlab" => gitlab_inspect_url(&repo.owner, &repo.name),
"codeberg" => forgejo_inspect_url("https://codeberg.org/api/v1", &repo.owner, &repo.name),
_ => return Ok(None),
};
let response = client.get(url).send().await?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(None);
}
let text = response.error_for_status()?.text().await?;
Ok(Some(match repo.forge.as_str() {
"github" => parse_github_repo(&text)?,
"gitlab" => parse_gitlab_repo(&text)?,
_ => parse_forgejo_repo(&repo.forge, &text)?,
}))
}
pub fn parse_github_search(json: &str) -> Result<Vec<Repository>> {
#[derive(Deserialize)]
struct Search {
items: Vec<GithubRepo>,
}
Ok(serde_json::from_str::<Search>(json)?
.items
.into_iter()
.map(Into::into)
.collect())
}
pub fn parse_github_repo(json: &str) -> Result<Repository> {
Ok(serde_json::from_str::<GithubRepo>(json)?.into())
}
pub fn parse_gitlab_search(json: &str) -> Result<Vec<Repository>> {
Ok(serde_json::from_str::<Vec<GitlabRepo>>(json)?
.into_iter()
.map(Into::into)
.collect())
}
pub fn parse_gitlab_repo(json: &str) -> Result<Repository> {
Ok(serde_json::from_str::<GitlabRepo>(json)?.into())
}
pub fn parse_forgejo_search(provider: &str, json: &str) -> Result<Vec<Repository>> {
#[derive(Deserialize)]
struct Search {
data: Vec<ForgejoRepo>,
}
Ok(serde_json::from_str::<Search>(json)?
.data
.into_iter()
.map(|r| r.into_repo(provider))
.collect())
}
pub fn parse_forgejo_repo(provider: &str, json: &str) -> Result<Repository> {
Ok(serde_json::from_str::<ForgejoRepo>(json)?.into_repo(provider))
}
fn client(token_env: Option<&str>) -> reqwest::Client {
let mut headers = HeaderMap::new();
headers.insert(USER_AGENT, HeaderValue::from_static("paper-gap"));
if let Some(token_env) = token_env.and_then(|name| std::env::var(name).ok()) {
if let Ok(value) = HeaderValue::from_str(&format!("Bearer {token_env}")) {
headers.insert(AUTHORIZATION, value);
}
}
reqwest::Client::builder()
.default_headers(headers)
.timeout(Duration::from_secs(8))
.build()
.expect("valid reqwest client")
}
#[derive(Deserialize)]
struct GithubRepo {
html_url: String,
name: String,
owner: GithubOwner,
description: Option<String>,
language: Option<String>,
stargazers_count: Option<u64>,
forks_count: Option<u64>,
license: Option<GithubLicense>,
archived: Option<bool>,
default_branch: Option<String>,
pushed_at: Option<String>,
}
#[derive(Deserialize)]
struct GithubOwner {
login: String,
}
#[derive(Deserialize)]
struct GithubLicense {
spdx_id: Option<String>,
name: Option<String>,
}
impl From<GithubRepo> for Repository {
fn from(repo: GithubRepo) -> Self {
Repository {
forge: "github".into(),
url: repo.html_url,
owner: repo.owner.login,
name: repo.name,
description: repo.description,
primary_language: repo.language,
stars: repo.stargazers_count,
forks: repo.forks_count,
license: repo.license.and_then(|l| l.spdx_id.or(l.name)),
archived: repo.archived,
default_branch: repo.default_branch,
last_commit_date: repo.pushed_at,
source_provider: "github".into(),
}
}
}
#[derive(Deserialize)]
struct GitlabRepo {
web_url: String,
path: String,
path_with_namespace: String,
description: Option<String>,
star_count: Option<u64>,
forks_count: Option<u64>,
archived: Option<bool>,
default_branch: Option<String>,
last_activity_at: Option<String>,
}
impl From<GitlabRepo> for Repository {
fn from(repo: GitlabRepo) -> Self {
let owner = repo
.path_with_namespace
.rsplit_once('/')
.map_or("", |(owner, _)| owner)
.to_string();
Repository {
forge: "gitlab".into(),
url: repo.web_url,
owner,
name: repo.path,
description: repo.description,
primary_language: None,
stars: repo.star_count,
forks: repo.forks_count,
license: None,
archived: repo.archived,
default_branch: repo.default_branch,
last_commit_date: repo.last_activity_at,
source_provider: "gitlab".into(),
}
}
}
#[derive(Deserialize)]
struct ForgejoRepo {
html_url: String,
name: String,
owner: ForgejoOwner,
description: Option<String>,
language: Option<String>,
stars_count: Option<u64>,
forks_count: Option<u64>,
archived: Option<bool>,
default_branch: Option<String>,
updated_at: Option<String>,
}
#[derive(Deserialize)]
struct ForgejoOwner {
login: String,
}
impl ForgejoRepo {
fn into_repo(self, provider: &str) -> Repository {
Repository {
forge: provider.into(),
url: self.html_url,
owner: self.owner.login,
name: self.name,
description: self.description,
primary_language: self.language,
stars: self.stars_count,
forks: self.forks_count,
license: None,
archived: self.archived,
default_branch: self.default_branch,
last_commit_date: self.updated_at,
source_provider: provider.into(),
}
}
}
fn dedup_repositories(repos: Vec<Repository>) -> Vec<Repository> {
let mut out = Vec::new();
for repo in repos {
if !out
.iter()
.any(|existing: &Repository| existing.url == repo.url)
{
out.push(repo);
}
}
out
}
fn dedup_strings(values: Vec<String>) -> Vec<String> {
let mut out = Vec::new();
for value in values.into_iter().filter(|s| !s.trim().is_empty()) {
if !out.iter().any(|existing| existing == &value) {
out.push(value);
}
}
out
}
fn trim_arxiv_version(arxiv_id: &str) -> &str {
arxiv_id.rsplit_once('v').map_or(arxiv_id, |(id, version)| {
if version.chars().all(|c| c.is_ascii_digit()) {
id
} else {
arxiv_id
}
})
}