use crate::pwc::CodeLink;
use crate::repos::Repository;
use chrono::{DateTime, Duration, Utc};
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RepositoryAnalysis {
#[serde(default = "default_true")]
pub inspection_succeeded: bool,
pub has_readme: bool,
pub has_license: bool,
pub has_tests: bool,
pub has_ci: bool,
pub has_docs: bool,
pub has_examples: bool,
pub has_package_manifest: bool,
pub has_reproducibility_scripts: bool,
pub has_benchmarks: bool,
pub is_notebook_only: bool,
pub is_stale: bool,
pub is_archived: bool,
pub primary_language: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct GapAssessment {
pub implementation_gap_score: u32,
pub components: ScoreComponents,
pub gaps: Vec<Gap>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScoreComponents {
pub no_code_score: u32,
pub prototype_only_score: u32,
pub stale_repo_score: u32,
pub packaging_gap_score: u32,
pub docs_gap_score: u32,
pub tests_gap_score: u32,
pub license_gap_score: u32,
pub reproducibility_gap_score: u32,
pub ecosystem_gap_score: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Gap {
pub gap_type: GapType,
pub points: u32,
pub explanation: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum GapType {
NoPublicImplementationFound,
NoKnownPapersWithCodeEntry,
OnlyNotebookOrPrototypeCode,
NoMaintainedPackage,
NoTests,
NoDocumentation,
NoReproducibilityScripts,
NoBenchmarkReproduction,
StaleRepository,
ArchivedRepository,
UnclearOrRestrictiveLicense,
}
pub fn analyze_repository(repo: &Repository) -> RepositoryAnalysis {
let mut analysis = analyze_repository_with_paths(repo, &[]);
analysis.inspection_succeeded = false;
analysis
}
pub fn analyze_repository_with_paths(repo: &Repository, paths: &[String]) -> RepositoryAnalysis {
let lower_paths: Vec<String> = paths.iter().map(|path| path.to_ascii_lowercase()).collect();
RepositoryAnalysis {
inspection_succeeded: true,
has_readme: has_any(&lower_paths, &["readme"]),
has_license: repo
.license
.as_deref()
.is_some_and(|license| !license.trim().is_empty() && license != "NOASSERTION")
|| has_any(&lower_paths, &["license", "copying"]),
has_tests: has_any(&lower_paths, &["test", "tests", "spec"]),
has_ci: has_any(
&lower_paths,
&[
".github/workflows",
".gitlab-ci",
".woodpecker",
"buildkite",
"circleci",
],
),
has_docs: has_any(&lower_paths, &["docs/", "doc/", "documentation"]),
has_examples: has_any(&lower_paths, &["examples/", "example/"]),
has_package_manifest: has_any(
&lower_paths,
&[
"cargo.toml",
"pyproject.toml",
"package.json",
"setup.py",
"setup.cfg",
"go.mod",
"pom.xml",
"build.gradle",
"cmakelists.txt",
"makefile",
],
),
has_reproducibility_scripts: has_any(
&lower_paths,
&[
"reproduce",
"replicate",
"artifact",
"run_experiment",
"run-experiment",
],
),
has_benchmarks: has_any(&lower_paths, &["bench", "benchmark"]),
is_notebook_only: is_notebook_only(&lower_paths),
is_stale: repo.last_commit_date.as_deref().is_some_and(is_stale_date),
is_archived: repo.archived.unwrap_or(false),
primary_language: repo.primary_language.clone(),
}
}
pub fn assess(code_links: &[CodeLink], repositories: &[Repository]) -> GapAssessment {
let repositories = dedup_repositories(repositories.to_vec());
let analyses: Vec<_> = repositories.iter().map(analyze_repository).collect();
assess_with_evidence(code_links, &repositories, &analyses, true, true)
}
pub fn assess_with_analyses(
code_links: &[CodeLink],
repositories: &[Repository],
analyses: &[RepositoryAnalysis],
) -> GapAssessment {
assess_with_evidence(code_links, repositories, analyses, true, true)
}
pub fn assess_with_evidence(
code_links: &[CodeLink],
repositories: &[Repository],
analyses: &[RepositoryAnalysis],
pwc_succeeded: bool,
repository_search_succeeded: bool,
) -> GapAssessment {
let mut components = ScoreComponents::default();
let mut gaps = Vec::new();
let inspected: Vec<_> = analyses
.iter()
.filter(|analysis| analysis.inspection_succeeded)
.collect();
if pwc_succeeded && code_links.is_empty() {
components.no_code_score += 10;
gaps.push(Gap {
gap_type: GapType::NoKnownPapersWithCodeEntry,
points: 10,
explanation: "Papers with Code returned no known code links.".into(),
});
}
if pwc_succeeded
&& repository_search_succeeded
&& code_links.is_empty()
&& repositories.is_empty()
{
components.no_code_score += 40;
gaps.push(Gap {
gap_type: GapType::NoPublicImplementationFound,
points: 40,
explanation:
"No public implementation was found from Papers with Code or repository search."
.into(),
});
}
if analyses.iter().any(|a| a.is_archived) {
components.stale_repo_score += 15;
gaps.push(Gap {
gap_type: GapType::ArchivedRepository,
points: 15,
explanation: "At least one discovered repository is archived.".into(),
});
}
if !analyses.is_empty() && analyses.iter().all(|a| a.is_stale) {
components.stale_repo_score += 15;
gaps.push(Gap {
gap_type: GapType::StaleRepository,
points: 15,
explanation: "All discovered repositories with commit dates look stale.".into(),
});
}
if !analyses.is_empty() && analyses.iter().all(|a| !a.has_license) {
components.license_gap_score += 10;
gaps.push(Gap {
gap_type: GapType::UnclearOrRestrictiveLicense,
points: 10,
explanation: "No discovered repository has a clear license signal.".into(),
});
}
if !inspected.is_empty() && inspected.iter().all(|a| !a.has_package_manifest) {
components.packaging_gap_score += 10;
gaps.push(Gap {
gap_type: GapType::NoMaintainedPackage,
points: 10,
explanation: "No discovered repository has a known package manifest.".into(),
});
}
if !inspected.is_empty() && inspected.iter().all(|a| !a.has_readme && !a.has_docs) {
components.docs_gap_score += 10;
gaps.push(Gap {
gap_type: GapType::NoDocumentation,
points: 10,
explanation: "No discovered repository has README/docs signals.".into(),
});
}
if !inspected.is_empty() && inspected.iter().all(|a| !a.has_tests) {
components.tests_gap_score += 10;
gaps.push(Gap {
gap_type: GapType::NoTests,
points: 10,
explanation: "No discovered repository has test path signals.".into(),
});
}
if !inspected.is_empty() && inspected.iter().all(|a| !a.has_reproducibility_scripts) {
components.reproducibility_gap_score += 5;
gaps.push(Gap {
gap_type: GapType::NoReproducibilityScripts,
points: 5,
explanation: "No discovered repository has reproducibility script signals.".into(),
});
}
if !inspected.is_empty() && inspected.iter().all(|a| !a.has_benchmarks) {
components.reproducibility_gap_score += 5;
gaps.push(Gap {
gap_type: GapType::NoBenchmarkReproduction,
points: 5,
explanation: "No discovered repository has benchmark signals.".into(),
});
}
if !inspected.is_empty() && inspected.iter().all(|a| a.is_notebook_only) {
components.prototype_only_score += 15;
gaps.push(Gap {
gap_type: GapType::OnlyNotebookOrPrototypeCode,
points: 15,
explanation: "Discovered repositories appear to be notebook-only prototypes.".into(),
});
}
let implementation_gap_score = components.no_code_score
+ components.prototype_only_score
+ components.stale_repo_score
+ components.packaging_gap_score
+ components.docs_gap_score
+ components.tests_gap_score
+ components.license_gap_score
+ components.reproducibility_gap_score
+ components.ecosystem_gap_score;
GapAssessment {
implementation_gap_score,
components,
gaps,
}
}
pub fn dedup_repositories(repositories: Vec<Repository>) -> Vec<Repository> {
let mut out = Vec::new();
for repo in repositories {
let key = normalized_url(&repo.url);
if !out
.iter()
.any(|existing: &Repository| normalized_url(&existing.url) == key)
{
out.push(repo);
}
}
out
}
fn normalized_url(url: &str) -> String {
url.trim_end_matches('/')
.trim_end_matches(".git")
.to_ascii_lowercase()
}
fn is_stale_date(date: &str) -> bool {
DateTime::parse_from_rfc3339(date)
.map(|date| {
Utc::now().signed_duration_since(date.with_timezone(&Utc)) > Duration::days(365)
})
.unwrap_or(false)
}
fn has_any(paths: &[String], needles: &[&str]) -> bool {
paths
.iter()
.any(|path| needles.iter().any(|needle| path.contains(needle)))
}
fn is_notebook_only(paths: &[String]) -> bool {
let code_paths: Vec<_> = paths
.iter()
.filter(|path| {
path.ends_with(".py")
|| path.ends_with(".rs")
|| path.ends_with(".cpp")
|| path.ends_with(".c")
|| path.ends_with(".h")
|| path.ends_with(".jl")
|| path.ends_with(".m")
})
.collect();
!paths.is_empty() && paths.iter().any(|path| path.ends_with(".ipynb")) && code_paths.is_empty()
}
fn default_true() -> bool {
true
}