1use crate::pwc::CodeLink;
2use crate::repos::Repository;
3use chrono::{DateTime, Duration, Utc};
4use serde::{Deserialize, Serialize};
5
6#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
7pub struct RepositoryAnalysis {
8 #[serde(default = "default_true")]
9 pub inspection_succeeded: bool,
10 pub has_readme: bool,
11 pub has_license: bool,
12 pub has_tests: bool,
13 pub has_ci: bool,
14 pub has_docs: bool,
15 pub has_examples: bool,
16 pub has_package_manifest: bool,
17 pub has_reproducibility_scripts: bool,
18 pub has_benchmarks: bool,
19 pub is_notebook_only: bool,
20 pub is_stale: bool,
21 pub is_archived: bool,
22 pub primary_language: Option<String>,
23}
24
25#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
26pub struct GapAssessment {
27 pub implementation_gap_score: u32,
28 pub components: ScoreComponents,
29 pub gaps: Vec<Gap>,
30}
31
32#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
33pub struct ScoreComponents {
34 pub no_code_score: u32,
35 pub prototype_only_score: u32,
36 pub stale_repo_score: u32,
37 pub packaging_gap_score: u32,
38 pub docs_gap_score: u32,
39 pub tests_gap_score: u32,
40 pub license_gap_score: u32,
41 pub reproducibility_gap_score: u32,
42 pub ecosystem_gap_score: u32,
43}
44
45#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
46pub struct Gap {
47 pub gap_type: GapType,
48 pub points: u32,
49 pub explanation: String,
50}
51
52#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
53#[serde(rename_all = "snake_case")]
54pub enum GapType {
55 NoPublicImplementationFound,
56 NoKnownPapersWithCodeEntry,
57 OnlyNotebookOrPrototypeCode,
58 NoMaintainedPackage,
59 NoTests,
60 NoDocumentation,
61 NoReproducibilityScripts,
62 NoBenchmarkReproduction,
63 StaleRepository,
64 ArchivedRepository,
65 UnclearOrRestrictiveLicense,
66}
67
68pub fn analyze_repository(repo: &Repository) -> RepositoryAnalysis {
69 let mut analysis = analyze_repository_with_paths(repo, &[]);
70 analysis.inspection_succeeded = false;
71 analysis
72}
73
74pub fn analyze_repository_with_paths(repo: &Repository, paths: &[String]) -> RepositoryAnalysis {
75 let lower_paths: Vec<String> = paths.iter().map(|path| path.to_ascii_lowercase()).collect();
76 RepositoryAnalysis {
77 inspection_succeeded: true,
78 has_readme: has_any(&lower_paths, &["readme"]),
79 has_license: repo
80 .license
81 .as_deref()
82 .is_some_and(|license| !license.trim().is_empty() && license != "NOASSERTION")
83 || has_any(&lower_paths, &["license", "copying"]),
84 has_tests: has_any(&lower_paths, &["test", "tests", "spec"]),
85 has_ci: has_any(
86 &lower_paths,
87 &[
88 ".github/workflows",
89 ".gitlab-ci",
90 ".woodpecker",
91 "buildkite",
92 "circleci",
93 ],
94 ),
95 has_docs: has_any(&lower_paths, &["docs/", "doc/", "documentation"]),
96 has_examples: has_any(&lower_paths, &["examples/", "example/"]),
97 has_package_manifest: has_any(
98 &lower_paths,
99 &[
100 "cargo.toml",
101 "pyproject.toml",
102 "package.json",
103 "setup.py",
104 "setup.cfg",
105 "go.mod",
106 "pom.xml",
107 "build.gradle",
108 "cmakelists.txt",
109 "makefile",
110 ],
111 ),
112 has_reproducibility_scripts: has_any(
113 &lower_paths,
114 &[
115 "reproduce",
116 "replicate",
117 "artifact",
118 "run_experiment",
119 "run-experiment",
120 ],
121 ),
122 has_benchmarks: has_any(&lower_paths, &["bench", "benchmark"]),
123 is_notebook_only: is_notebook_only(&lower_paths),
124 is_stale: repo.last_commit_date.as_deref().is_some_and(is_stale_date),
125 is_archived: repo.archived.unwrap_or(false),
126 primary_language: repo.primary_language.clone(),
127 }
128}
129
130pub fn assess(code_links: &[CodeLink], repositories: &[Repository]) -> GapAssessment {
131 let repositories = dedup_repositories(repositories.to_vec());
132 let analyses: Vec<_> = repositories.iter().map(analyze_repository).collect();
133 assess_with_evidence(code_links, &repositories, &analyses, true, true)
134}
135
136pub fn assess_with_analyses(
137 code_links: &[CodeLink],
138 repositories: &[Repository],
139 analyses: &[RepositoryAnalysis],
140) -> GapAssessment {
141 assess_with_evidence(code_links, repositories, analyses, true, true)
142}
143
144pub fn assess_with_evidence(
145 code_links: &[CodeLink],
146 repositories: &[Repository],
147 analyses: &[RepositoryAnalysis],
148 pwc_succeeded: bool,
149 repository_search_succeeded: bool,
150) -> GapAssessment {
151 let mut components = ScoreComponents::default();
152 let mut gaps = Vec::new();
153 let inspected: Vec<_> = analyses
154 .iter()
155 .filter(|analysis| analysis.inspection_succeeded)
156 .collect();
157
158 if pwc_succeeded && code_links.is_empty() {
159 components.no_code_score += 10;
160 gaps.push(Gap {
161 gap_type: GapType::NoKnownPapersWithCodeEntry,
162 points: 10,
163 explanation: "Papers with Code returned no known code links.".into(),
164 });
165 }
166 if pwc_succeeded
167 && repository_search_succeeded
168 && code_links.is_empty()
169 && repositories.is_empty()
170 {
171 components.no_code_score += 40;
172 gaps.push(Gap {
173 gap_type: GapType::NoPublicImplementationFound,
174 points: 40,
175 explanation:
176 "No public implementation was found from Papers with Code or repository search."
177 .into(),
178 });
179 }
180 if analyses.iter().any(|a| a.is_archived) {
181 components.stale_repo_score += 15;
182 gaps.push(Gap {
183 gap_type: GapType::ArchivedRepository,
184 points: 15,
185 explanation: "At least one discovered repository is archived.".into(),
186 });
187 }
188 if !analyses.is_empty() && analyses.iter().all(|a| a.is_stale) {
189 components.stale_repo_score += 15;
190 gaps.push(Gap {
191 gap_type: GapType::StaleRepository,
192 points: 15,
193 explanation: "All discovered repositories with commit dates look stale.".into(),
194 });
195 }
196 if !analyses.is_empty() && analyses.iter().all(|a| !a.has_license) {
197 components.license_gap_score += 10;
198 gaps.push(Gap {
199 gap_type: GapType::UnclearOrRestrictiveLicense,
200 points: 10,
201 explanation: "No discovered repository has a clear license signal.".into(),
202 });
203 }
204 if !inspected.is_empty() && inspected.iter().all(|a| !a.has_package_manifest) {
205 components.packaging_gap_score += 10;
206 gaps.push(Gap {
207 gap_type: GapType::NoMaintainedPackage,
208 points: 10,
209 explanation: "No discovered repository has a known package manifest.".into(),
210 });
211 }
212 if !inspected.is_empty() && inspected.iter().all(|a| !a.has_readme && !a.has_docs) {
213 components.docs_gap_score += 10;
214 gaps.push(Gap {
215 gap_type: GapType::NoDocumentation,
216 points: 10,
217 explanation: "No discovered repository has README/docs signals.".into(),
218 });
219 }
220 if !inspected.is_empty() && inspected.iter().all(|a| !a.has_tests) {
221 components.tests_gap_score += 10;
222 gaps.push(Gap {
223 gap_type: GapType::NoTests,
224 points: 10,
225 explanation: "No discovered repository has test path signals.".into(),
226 });
227 }
228 if !inspected.is_empty() && inspected.iter().all(|a| !a.has_reproducibility_scripts) {
229 components.reproducibility_gap_score += 5;
230 gaps.push(Gap {
231 gap_type: GapType::NoReproducibilityScripts,
232 points: 5,
233 explanation: "No discovered repository has reproducibility script signals.".into(),
234 });
235 }
236 if !inspected.is_empty() && inspected.iter().all(|a| !a.has_benchmarks) {
237 components.reproducibility_gap_score += 5;
238 gaps.push(Gap {
239 gap_type: GapType::NoBenchmarkReproduction,
240 points: 5,
241 explanation: "No discovered repository has benchmark signals.".into(),
242 });
243 }
244 if !inspected.is_empty() && inspected.iter().all(|a| a.is_notebook_only) {
245 components.prototype_only_score += 15;
246 gaps.push(Gap {
247 gap_type: GapType::OnlyNotebookOrPrototypeCode,
248 points: 15,
249 explanation: "Discovered repositories appear to be notebook-only prototypes.".into(),
250 });
251 }
252
253 let implementation_gap_score = components.no_code_score
254 + components.prototype_only_score
255 + components.stale_repo_score
256 + components.packaging_gap_score
257 + components.docs_gap_score
258 + components.tests_gap_score
259 + components.license_gap_score
260 + components.reproducibility_gap_score
261 + components.ecosystem_gap_score;
262
263 GapAssessment {
264 implementation_gap_score,
265 components,
266 gaps,
267 }
268}
269
270pub fn dedup_repositories(repositories: Vec<Repository>) -> Vec<Repository> {
271 let mut out = Vec::new();
272 for repo in repositories {
273 let key = normalized_url(&repo.url);
274 if !out
275 .iter()
276 .any(|existing: &Repository| normalized_url(&existing.url) == key)
277 {
278 out.push(repo);
279 }
280 }
281 out
282}
283
284fn normalized_url(url: &str) -> String {
285 url.trim_end_matches('/')
286 .trim_end_matches(".git")
287 .to_ascii_lowercase()
288}
289
290fn is_stale_date(date: &str) -> bool {
291 DateTime::parse_from_rfc3339(date)
292 .map(|date| {
293 Utc::now().signed_duration_since(date.with_timezone(&Utc)) > Duration::days(365)
294 })
295 .unwrap_or(false)
296}
297
298fn has_any(paths: &[String], needles: &[&str]) -> bool {
299 paths
300 .iter()
301 .any(|path| needles.iter().any(|needle| path.contains(needle)))
302}
303
304fn is_notebook_only(paths: &[String]) -> bool {
305 let code_paths: Vec<_> = paths
306 .iter()
307 .filter(|path| {
308 path.ends_with(".py")
309 || path.ends_with(".rs")
310 || path.ends_with(".cpp")
311 || path.ends_with(".c")
312 || path.ends_with(".h")
313 || path.ends_with(".jl")
314 || path.ends_with(".m")
315 })
316 .collect();
317 !paths.is_empty() && paths.iter().any(|path| path.ends_with(".ipynb")) && code_paths.is_empty()
318}
319
320fn default_true() -> bool {
321 true
322}