1use crate::pwc::CodeLink;
2use crate::repos::Repository;
3use chrono::{DateTime, Duration, Utc};
4use serde::{Deserialize, Serialize};
5
6#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
7pub struct RepositoryAnalysis {
8 pub has_readme: bool,
9 pub has_license: bool,
10 pub has_tests: bool,
11 pub has_ci: bool,
12 pub has_docs: bool,
13 pub has_examples: bool,
14 pub has_package_manifest: bool,
15 pub has_reproducibility_scripts: bool,
16 pub has_benchmarks: bool,
17 pub is_notebook_only: bool,
18 pub is_stale: bool,
19 pub is_archived: bool,
20 pub primary_language: Option<String>,
21}
22
23#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
24pub struct GapAssessment {
25 pub implementation_gap_score: u32,
26 pub components: ScoreComponents,
27 pub gaps: Vec<Gap>,
28}
29
30#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
31pub struct ScoreComponents {
32 pub no_code_score: u32,
33 pub prototype_only_score: u32,
34 pub stale_repo_score: u32,
35 pub packaging_gap_score: u32,
36 pub docs_gap_score: u32,
37 pub tests_gap_score: u32,
38 pub license_gap_score: u32,
39 pub reproducibility_gap_score: u32,
40 pub ecosystem_gap_score: u32,
41}
42
43#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
44pub struct Gap {
45 pub gap_type: GapType,
46 pub points: u32,
47 pub explanation: String,
48}
49
50#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
51#[serde(rename_all = "snake_case")]
52pub enum GapType {
53 NoPublicImplementationFound,
54 NoKnownPapersWithCodeEntry,
55 OnlyNotebookOrPrototypeCode,
56 NoMaintainedPackage,
57 NoTests,
58 NoDocumentation,
59 NoReproducibilityScripts,
60 NoBenchmarkReproduction,
61 StaleRepository,
62 ArchivedRepository,
63 UnclearOrRestrictiveLicense,
64}
65
66pub fn analyze_repository(repo: &Repository) -> RepositoryAnalysis {
67 analyze_repository_with_paths(repo, &[])
68}
69
70pub fn analyze_repository_with_paths(repo: &Repository, paths: &[String]) -> RepositoryAnalysis {
71 let lower_paths: Vec<String> = paths.iter().map(|path| path.to_ascii_lowercase()).collect();
72 RepositoryAnalysis {
73 has_readme: has_any(&lower_paths, &["readme"]),
74 has_license: repo
75 .license
76 .as_deref()
77 .is_some_and(|license| !license.trim().is_empty() && license != "NOASSERTION")
78 || has_any(&lower_paths, &["license", "copying"]),
79 has_tests: has_any(&lower_paths, &["test", "tests", "spec"]),
80 has_ci: has_any(
81 &lower_paths,
82 &[
83 ".github/workflows",
84 ".gitlab-ci",
85 ".woodpecker",
86 "buildkite",
87 "circleci",
88 ],
89 ),
90 has_docs: has_any(&lower_paths, &["docs/", "doc/", "documentation"]),
91 has_examples: has_any(&lower_paths, &["examples/", "example/"]),
92 has_package_manifest: has_any(
93 &lower_paths,
94 &[
95 "cargo.toml",
96 "pyproject.toml",
97 "package.json",
98 "setup.py",
99 "setup.cfg",
100 "go.mod",
101 "pom.xml",
102 "build.gradle",
103 "cmakelists.txt",
104 "makefile",
105 ],
106 ),
107 has_reproducibility_scripts: has_any(
108 &lower_paths,
109 &[
110 "reproduce",
111 "replicate",
112 "artifact",
113 "run_experiment",
114 "run-experiment",
115 ],
116 ),
117 has_benchmarks: has_any(&lower_paths, &["bench", "benchmark"]),
118 is_notebook_only: is_notebook_only(&lower_paths),
119 is_stale: repo.last_commit_date.as_deref().is_some_and(is_stale_date),
120 is_archived: repo.archived.unwrap_or(false),
121 primary_language: repo.primary_language.clone(),
122 }
123}
124
125pub fn assess(code_links: &[CodeLink], repositories: &[Repository]) -> GapAssessment {
126 let repositories = dedup_repositories(repositories.to_vec());
127 let analyses: Vec<_> = repositories.iter().map(analyze_repository).collect();
128 assess_with_analyses(code_links, &repositories, &analyses)
129}
130
131pub fn assess_with_analyses(
132 code_links: &[CodeLink],
133 repositories: &[Repository],
134 analyses: &[RepositoryAnalysis],
135) -> GapAssessment {
136 let mut components = ScoreComponents::default();
137 let mut gaps = Vec::new();
138
139 if code_links.is_empty() {
140 components.no_code_score += 10;
141 gaps.push(Gap {
142 gap_type: GapType::NoKnownPapersWithCodeEntry,
143 points: 10,
144 explanation: "Papers with Code returned no known code links.".into(),
145 });
146 }
147 if code_links.is_empty() && repositories.is_empty() {
148 components.no_code_score += 40;
149 gaps.push(Gap {
150 gap_type: GapType::NoPublicImplementationFound,
151 points: 40,
152 explanation:
153 "No public implementation was found from Papers with Code or repository search."
154 .into(),
155 });
156 }
157 if analyses.iter().any(|a| a.is_archived) {
158 components.stale_repo_score += 15;
159 gaps.push(Gap {
160 gap_type: GapType::ArchivedRepository,
161 points: 15,
162 explanation: "At least one discovered repository is archived.".into(),
163 });
164 }
165 if !analyses.is_empty() && analyses.iter().all(|a| a.is_stale) {
166 components.stale_repo_score += 15;
167 gaps.push(Gap {
168 gap_type: GapType::StaleRepository,
169 points: 15,
170 explanation: "All discovered repositories with commit dates look stale.".into(),
171 });
172 }
173 if !analyses.is_empty() && analyses.iter().all(|a| !a.has_license) {
174 components.license_gap_score += 10;
175 gaps.push(Gap {
176 gap_type: GapType::UnclearOrRestrictiveLicense,
177 points: 10,
178 explanation: "No discovered repository has a clear license signal.".into(),
179 });
180 }
181 if !analyses.is_empty() && analyses.iter().all(|a| !a.has_package_manifest) {
182 components.packaging_gap_score += 10;
183 gaps.push(Gap {
184 gap_type: GapType::NoMaintainedPackage,
185 points: 10,
186 explanation: "No discovered repository has a known package manifest.".into(),
187 });
188 }
189 if !analyses.is_empty() && analyses.iter().all(|a| !a.has_readme && !a.has_docs) {
190 components.docs_gap_score += 10;
191 gaps.push(Gap {
192 gap_type: GapType::NoDocumentation,
193 points: 10,
194 explanation: "No discovered repository has README/docs signals.".into(),
195 });
196 }
197 if !analyses.is_empty() && analyses.iter().all(|a| !a.has_tests) {
198 components.tests_gap_score += 10;
199 gaps.push(Gap {
200 gap_type: GapType::NoTests,
201 points: 10,
202 explanation: "No discovered repository has test path signals.".into(),
203 });
204 }
205 if !analyses.is_empty() && analyses.iter().all(|a| !a.has_reproducibility_scripts) {
206 components.reproducibility_gap_score += 5;
207 gaps.push(Gap {
208 gap_type: GapType::NoReproducibilityScripts,
209 points: 5,
210 explanation: "No discovered repository has reproducibility script signals.".into(),
211 });
212 }
213 if !analyses.is_empty() && analyses.iter().all(|a| !a.has_benchmarks) {
214 components.reproducibility_gap_score += 5;
215 gaps.push(Gap {
216 gap_type: GapType::NoBenchmarkReproduction,
217 points: 5,
218 explanation: "No discovered repository has benchmark signals.".into(),
219 });
220 }
221 if !analyses.is_empty() && analyses.iter().all(|a| a.is_notebook_only) {
222 components.prototype_only_score += 15;
223 gaps.push(Gap {
224 gap_type: GapType::OnlyNotebookOrPrototypeCode,
225 points: 15,
226 explanation: "Discovered repositories appear to be notebook-only prototypes.".into(),
227 });
228 }
229
230 let implementation_gap_score = components.no_code_score
231 + components.prototype_only_score
232 + components.stale_repo_score
233 + components.packaging_gap_score
234 + components.docs_gap_score
235 + components.tests_gap_score
236 + components.license_gap_score
237 + components.reproducibility_gap_score
238 + components.ecosystem_gap_score;
239
240 GapAssessment {
241 implementation_gap_score,
242 components,
243 gaps,
244 }
245}
246
247pub fn dedup_repositories(repositories: Vec<Repository>) -> Vec<Repository> {
248 let mut out = Vec::new();
249 for repo in repositories {
250 let key = normalized_url(&repo.url);
251 if !out
252 .iter()
253 .any(|existing: &Repository| normalized_url(&existing.url) == key)
254 {
255 out.push(repo);
256 }
257 }
258 out
259}
260
261fn normalized_url(url: &str) -> String {
262 url.trim_end_matches('/')
263 .trim_end_matches(".git")
264 .to_ascii_lowercase()
265}
266
267fn is_stale_date(date: &str) -> bool {
268 DateTime::parse_from_rfc3339(date)
269 .map(|date| {
270 Utc::now().signed_duration_since(date.with_timezone(&Utc)) > Duration::days(365)
271 })
272 .unwrap_or(false)
273}
274
275fn has_any(paths: &[String], needles: &[&str]) -> bool {
276 paths
277 .iter()
278 .any(|path| needles.iter().any(|needle| path.contains(needle)))
279}
280
281fn is_notebook_only(paths: &[String]) -> bool {
282 let code_paths: Vec<_> = paths
283 .iter()
284 .filter(|path| {
285 path.ends_with(".py")
286 || path.ends_with(".rs")
287 || path.ends_with(".cpp")
288 || path.ends_with(".c")
289 || path.ends_with(".h")
290 || path.ends_with(".jl")
291 || path.ends_with(".m")
292 })
293 .collect();
294 !paths.is_empty() && paths.iter().any(|path| path.ends_with(".ipynb")) && code_paths.is_empty()
295}