1use std::{
2 collections::HashMap,
3 fs,
4 path::{Path, PathBuf},
5 process::Command,
6};
7
8use anyhow::{Context, Result, bail};
9use rayon::prelude::*;
10
11use crate::model::{AnalyzerReport, Atlas, BuildTimings, Metric, Node, NodeKind, Rect};
12
13#[derive(Debug)]
14struct FileStats {
15 path: String,
16 bytes: u64,
17 loc: u64,
18 commits: u64,
19 language: String,
20}
21
22pub fn scan_repository(
23 root: &Path,
24 metric: Metric,
25 include_tests: bool,
26 include_hidden: bool,
27 excluded_paths: &[String],
28) -> Result<Atlas> {
29 let root = root
30 .canonicalize()
31 .with_context(|| format!("cannot open repository {}", root.display()))?;
32 if !root.join(".git").exists() {
33 bail!("{} is not a Git checkout", root.display());
34 }
35
36 let excluded_paths = normalize_excluded_paths(excluded_paths)?;
37 let tracked = tracked_paths(&root, include_tests, include_hidden, &excluded_paths)?;
38 let paths = tracked.paths;
39 let commit_counts = commit_counts(&root)?;
40 let stats: Result<Vec<_>> = paths
41 .par_iter()
42 .map(|path| read_file_stats(&root, path, &commit_counts))
43 .collect();
44 let mut stats = stats?;
45 stats.sort_by(|left, right| left.path.cmp(&right.path));
46
47 let revision = git_output(&root, &["rev-parse", "HEAD"])
48 .unwrap_or_else(|_| "unborn".to_owned())
49 .trim()
50 .to_owned();
51 let dirty = !git_output(&root, &["status", "--porcelain"])
52 .unwrap_or_default()
53 .trim()
54 .is_empty();
55
56 let mut nodes = vec![Node {
57 id: 0,
58 parent: None,
59 children: Vec::new(),
60 kind: NodeKind::Directory,
61 name: root
62 .file_name()
63 .and_then(|name| name.to_str())
64 .unwrap_or("repository")
65 .to_owned(),
66 path: String::new(),
67 depth: 0,
68 bytes: 0,
69 loc: 0,
70 commits: 0,
71 weight: 0.0,
72 language: "directory".to_owned(),
73 rect: Rect::default(),
74 }];
75 let mut path_to_id = HashMap::new();
76 path_to_id.insert(String::new(), 0);
77
78 for file in stats {
79 let file_path = Path::new(&file.path);
80 let components: Vec<_> = file_path.components().collect();
81 let mut parent = 0;
82 let mut accumulated = PathBuf::new();
83
84 for component in components.iter().take(components.len().saturating_sub(1)) {
85 accumulated.push(component.as_os_str());
86 let directory_path = accumulated.to_string_lossy().replace('\\', "/");
87 if let Some(existing) = path_to_id.get(&directory_path) {
88 parent = *existing;
89 continue;
90 }
91
92 let id = nodes.len();
93 nodes.push(Node {
94 id,
95 parent: Some(parent),
96 children: Vec::new(),
97 kind: NodeKind::Directory,
98 name: component.as_os_str().to_string_lossy().to_string(),
99 path: directory_path.clone(),
100 depth: nodes[parent].depth + 1,
101 bytes: 0,
102 loc: 0,
103 commits: 0,
104 weight: 0.0,
105 language: "directory".to_owned(),
106 rect: Rect::default(),
107 });
108 nodes[parent].children.push(id);
109 path_to_id.insert(directory_path, id);
110 parent = id;
111 }
112
113 let id = nodes.len();
114 let name = file_path
115 .file_name()
116 .and_then(|name| name.to_str())
117 .unwrap_or(&file.path)
118 .to_owned();
119 let weight = metric.value(file.bytes, file.loc, file.commits);
120 nodes.push(Node {
121 id,
122 parent: Some(parent),
123 children: Vec::new(),
124 kind: NodeKind::File,
125 name,
126 path: file.path.clone(),
127 depth: nodes[parent].depth + 1,
128 bytes: file.bytes,
129 loc: file.loc,
130 commits: file.commits,
131 weight,
132 language: file.language,
133 rect: Rect::default(),
134 });
135 nodes[parent].children.push(id);
136 path_to_id.insert(file.path, id);
137 }
138
139 roll_up_metrics(0, &mut nodes);
140
141 Ok(Atlas {
142 root_path: root,
143 revision,
144 dirty,
145 excluded_test_files: tracked.excluded_test_files,
146 excluded_hidden_files: tracked.excluded_hidden_files,
147 excluded_custom_files: tracked.excluded_custom_files,
148 excluded_paths,
149 nodes,
150 calls: Vec::new(),
151 path_to_id,
152 report: AnalyzerReport::default(),
153 timings: BuildTimings::default(),
154 })
155}
156
157struct TrackedPaths {
158 paths: Vec<String>,
159 excluded_test_files: usize,
160 excluded_hidden_files: usize,
161 excluded_custom_files: usize,
162}
163
164fn tracked_paths(
165 root: &Path,
166 include_tests: bool,
167 include_hidden: bool,
168 excluded_paths: &[String],
169) -> Result<TrackedPaths> {
170 let output = Command::new("git")
171 .args(["-C"])
172 .arg(root)
173 .args(["ls-files", "-z"])
174 .output()
175 .context("failed to run git ls-files")?;
176 if !output.status.success() {
177 bail!("git ls-files failed");
178 }
179 let mut tracked: Vec<_> = output
180 .stdout
181 .split(|byte| *byte == 0)
182 .filter(|path| !path.is_empty())
183 .map(|path| String::from_utf8_lossy(path).into_owned())
184 .collect();
185 tracked.retain(|path| root.join(path).is_file());
189 let mut excluded_test_files = 0;
190 if !include_tests {
191 tracked.retain(|path| {
192 let include = !is_test_path(path);
193 excluded_test_files += usize::from(!include);
194 include
195 });
196 }
197
198 let mut excluded_hidden_files = 0;
199 if !include_hidden {
200 tracked.retain(|path| {
201 let include = !is_hidden_path(path);
202 excluded_hidden_files += usize::from(!include);
203 include
204 });
205 }
206
207 let mut excluded_custom_files = 0;
208 if !excluded_paths.is_empty() {
209 tracked.retain(|path| {
210 let include = !excluded_paths
211 .iter()
212 .any(|excluded| path == excluded || path.starts_with(&format!("{excluded}/")));
213 excluded_custom_files += usize::from(!include);
214 include
215 });
216 }
217
218 Ok(TrackedPaths {
219 paths: tracked,
220 excluded_test_files,
221 excluded_hidden_files,
222 excluded_custom_files,
223 })
224}
225
226fn normalize_excluded_paths(paths: &[String]) -> Result<Vec<String>> {
227 let mut normalized = Vec::new();
228 for path in paths {
229 let path = path.trim().replace('\\', "/");
230 let path = path.trim_matches('/');
231 if path.is_empty() {
232 bail!("custom exclusion paths cannot be empty");
233 }
234 if Path::new(path).is_absolute()
235 || path
236 .split('/')
237 .any(|component| component == ".." || component.is_empty())
238 {
239 bail!("custom exclusion {path:?} must be a repository-relative path");
240 }
241 if !normalized.iter().any(|existing| existing == path) {
242 normalized.push(path.to_owned());
243 }
244 }
245 normalized.sort();
246 Ok(normalized)
247}
248
249pub fn is_hidden_path(path: &str) -> bool {
252 path.replace('\\', "/")
253 .split('/')
254 .any(|component| component.starts_with('.') && component.len() > 1)
255}
256
257pub fn is_test_path(path: &str) -> bool {
260 let normalized = path.replace('\\', "/").to_ascii_lowercase();
261 let components: Vec<_> = normalized
262 .split('/')
263 .filter(|component| !component.is_empty())
264 .collect();
265 let Some(file_name) = components.last().copied() else {
266 return false;
267 };
268
269 if components
270 .iter()
271 .take(components.len().saturating_sub(1))
272 .any(|component| {
273 matches!(
274 *component,
275 "test" | "tests" | "spec" | "specs" | "__tests__" | "testdata"
276 )
277 })
278 {
279 return true;
280 }
281
282 if matches!(file_name, "test_helper.exs" | "conftest.py")
283 || file_name.starts_with("test.")
284 || file_name.contains(".test.")
285 || file_name.contains(".spec.")
286 {
287 return true;
288 }
289
290 const PREFIX_EXTENSIONS: &[&str] = &[
291 ".py", ".rb", ".exs", ".rs", ".go", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts",
292 ".cts",
293 ];
294 if file_name.starts_with("test_")
295 && PREFIX_EXTENSIONS
296 .iter()
297 .any(|extension| file_name.ends_with(extension))
298 {
299 return true;
300 }
301
302 const SUFFIXES: &[&str] = &[
303 "_test.ex",
304 "_test.exs",
305 "_test.rs",
306 "_tests.rs",
307 "_test.go",
308 "_test.py",
309 "_test.rb",
310 "_spec.exs",
311 "_spec.rb",
312 ];
313 SUFFIXES.iter().any(|suffix| file_name.ends_with(suffix))
314}
315
316fn commit_counts(root: &Path) -> Result<HashMap<String, u64>> {
317 let output = Command::new("git")
318 .args(["-C"])
319 .arg(root)
320 .args(["log", "--format=", "--name-only", "--no-renames", "HEAD"])
321 .output()
322 .context("failed to run git log")?;
323 if !output.status.success() {
324 return Ok(HashMap::new());
325 }
326
327 let mut counts = HashMap::new();
328 for line in String::from_utf8_lossy(&output.stdout).lines() {
329 let path = line.trim();
330 if !path.is_empty() {
331 *counts.entry(path.to_owned()).or_insert(0) += 1;
332 }
333 }
334 Ok(counts)
335}
336
337fn read_file_stats(
338 root: &Path,
339 path: &str,
340 commit_counts: &HashMap<String, u64>,
341) -> Result<FileStats> {
342 let bytes =
343 fs::read(root.join(path)).with_context(|| format!("cannot read tracked file {path}"))?;
344 let binary = bytes.iter().take(8192).any(|byte| *byte == 0);
345 let loc = if binary {
346 0
347 } else {
348 String::from_utf8_lossy(&bytes).lines().count() as u64
349 };
350 Ok(FileStats {
351 path: path.to_owned(),
352 bytes: bytes.len() as u64,
353 loc,
354 commits: commit_counts.get(path).copied().unwrap_or(1),
355 language: language_for_path(path).to_owned(),
356 })
357}
358
359fn language_for_path(path: &str) -> &'static str {
360 match Path::new(path)
361 .extension()
362 .and_then(|extension| extension.to_str())
363 {
364 Some("ex") | Some("exs") => "elixir",
365 Some("rs") => "rust",
366 Some("js") | Some("jsx") | Some("mjs") | Some("cjs") => "javascript",
367 Some("ts") | Some("tsx") | Some("mts") | Some("cts") => "typescript",
368 Some("gleam") => "gleam",
369 Some("py") => "python",
370 Some("go") => "go",
371 Some("c") | Some("h") => "c",
372 Some("cc") | Some("cpp") | Some("hpp") => "cpp",
373 Some("java") => "java",
374 Some("rb") => "ruby",
375 Some("md") => "markdown",
376 Some("toml") | Some("lock") => "config",
377 Some("json") | Some("yml") | Some("yaml") => "data",
378 Some("wit") | Some("wat") | Some("wasm") => "wasm",
379 _ => "other",
380 }
381}
382
383fn roll_up_metrics(id: usize, nodes: &mut [Node]) -> (u64, u64, u64, f64) {
384 if nodes[id].is_file() {
385 return (
386 nodes[id].bytes,
387 nodes[id].loc,
388 nodes[id].commits,
389 nodes[id].weight,
390 );
391 }
392
393 let children = nodes[id].children.clone();
394 let mut totals = (0, 0, 0, 0.0);
395 for child in children {
396 let child_totals = roll_up_metrics(child, nodes);
397 totals.0 += child_totals.0;
398 totals.1 += child_totals.1;
399 totals.2 += child_totals.2;
400 totals.3 += child_totals.3;
401 }
402 nodes[id].bytes = totals.0;
403 nodes[id].loc = totals.1;
404 nodes[id].commits = totals.2;
405 nodes[id].weight = totals.3.max(1.0);
406 totals
407}
408
409fn git_output(root: &Path, args: &[&str]) -> Result<String> {
410 let output = Command::new("git")
411 .args(["-C"])
412 .arg(root)
413 .args(args)
414 .output()
415 .context("failed to invoke git")?;
416 if !output.status.success() {
417 bail!(
418 "git command failed: {}",
419 String::from_utf8_lossy(&output.stderr)
420 );
421 }
422 Ok(String::from_utf8_lossy(&output.stdout).into_owned())
423}
424
425#[cfg(test)]
426mod tests {
427 use super::*;
428
429 #[test]
430 fn recognizes_conventional_test_directories_and_files() {
431 for path in [
432 "test/wasmex_test.exs",
433 "tests/integration.rs",
434 "spec/models/widget_spec.rb",
435 "src/__tests__/widget.test.ts",
436 "pkg/testdata/input.json",
437 "lib/widget_test.exs",
438 "src/widget_test.go",
439 "src/test_widget.py",
440 "src/widget.test.tsx",
441 "src/widget.spec.js",
442 "src/widget.test.mts",
443 "src/widget.spec.cjs",
444 "src/widget_spec.rb",
445 "conftest.py",
446 "test_helper.exs",
447 "config/test.exs",
448 "test.json",
449 ] {
450 assert!(
451 is_test_path(path),
452 "expected {path} to be classified as a test"
453 );
454 }
455 }
456
457 #[test]
458 fn recognizes_analyzed_language_extensions() {
459 for path in ["src/a.js", "src/a.jsx", "src/a.mjs", "src/a.cjs"] {
460 assert_eq!(language_for_path(path), "javascript");
461 }
462 for path in ["src/a.ts", "src/a.tsx", "src/a.mts", "src/a.cts"] {
463 assert_eq!(language_for_path(path), "typescript");
464 }
465 assert_eq!(language_for_path("src/a.gleam"), "gleam");
466 }
467
468 #[test]
469 fn does_not_match_test_words_inside_production_names() {
470 for path in [
471 "src/contest.rs",
472 "lib/testing_tools.ex",
473 "docs/test-strategy.md",
474 "src/specification.rs",
475 "src/testimonial.ts",
476 "lib/fixture_loader.ex",
477 ] {
478 assert!(
479 !is_test_path(path),
480 "expected {path} to remain in the production atlas"
481 );
482 }
483 }
484
485 #[test]
486 fn recognizes_hidden_files_and_directories() {
487 for path in [
488 ".github/workflows/ci.yml",
489 ".formatter.exs",
490 "lib/.generated/module.ex",
491 "assets/.cache/data.json",
492 ] {
493 assert!(
494 is_hidden_path(path),
495 "expected {path} to be classified as hidden"
496 );
497 }
498 for path in [
499 "github/workflows/ci.yml",
500 "formatter.exs",
501 "lib/generated/module.ex",
502 "docs/dot-files.md",
503 ] {
504 assert!(!is_hidden_path(path), "expected {path} to remain visible");
505 }
506 }
507
508 #[test]
509 fn scanner_excludes_tests_and_hidden_paths_with_independent_overrides() {
510 let directory = tempfile::tempdir().unwrap();
511 fs::create_dir_all(directory.path().join("lib")).unwrap();
512 fs::create_dir_all(directory.path().join("lib/.generated")).unwrap();
513 fs::create_dir_all(directory.path().join("test")).unwrap();
514 fs::create_dir_all(directory.path().join(".github/workflows")).unwrap();
515 fs::write(
516 directory.path().join("lib/widget.ex"),
517 "defmodule Widget do\nend\n",
518 )
519 .unwrap();
520 fs::write(
521 directory.path().join("lib/.generated/widget.ex"),
522 "defmodule GeneratedWidget do\nend\n",
523 )
524 .unwrap();
525 fs::write(directory.path().join(".formatter.exs"), "[inputs: []]\n").unwrap();
526 fs::write(
527 directory.path().join(".github/workflows/ci.yml"),
528 "name: CI\n",
529 )
530 .unwrap();
531 fs::write(
532 directory.path().join("lib/widget_test.exs"),
533 "defmodule WidgetTest do\nend\n",
534 )
535 .unwrap();
536 fs::write(
537 directory.path().join("test/widget_test.exs"),
538 "defmodule WidgetTest do\nend\n",
539 )
540 .unwrap();
541
542 let init = Command::new("git")
543 .args(["init", "--quiet"])
544 .arg(directory.path())
545 .status()
546 .unwrap();
547 assert!(init.success());
548 let add = Command::new("git")
549 .args(["-C"])
550 .arg(directory.path())
551 .args(["add", "."])
552 .status()
553 .unwrap();
554 assert!(add.success());
555
556 let atlas = scan_repository(directory.path(), Metric::Loc, false, false, &[]).unwrap();
557 assert_eq!(atlas.files().count(), 1);
558 assert_eq!(atlas.excluded_test_files, 2);
559 assert_eq!(atlas.excluded_hidden_files, 3);
560 assert!(atlas.path_to_id.contains_key("lib/widget.ex"));
561
562 let atlas = scan_repository(directory.path(), Metric::Loc, true, false, &[]).unwrap();
563 assert_eq!(atlas.files().count(), 3);
564 assert_eq!(atlas.excluded_test_files, 0);
565 assert_eq!(atlas.excluded_hidden_files, 3);
566
567 let atlas = scan_repository(directory.path(), Metric::Loc, false, true, &[]).unwrap();
568 assert_eq!(atlas.files().count(), 4);
569 assert_eq!(atlas.excluded_test_files, 2);
570 assert_eq!(atlas.excluded_hidden_files, 0);
571
572 let atlas = scan_repository(directory.path(), Metric::Loc, true, true, &[]).unwrap();
573 assert_eq!(atlas.files().count(), 6);
574 assert_eq!(atlas.excluded_test_files, 0);
575 assert_eq!(atlas.excluded_hidden_files, 0);
576 }
577
578 #[test]
579 fn scanner_excludes_repeatable_repository_relative_paths() {
580 let directory = tempfile::tempdir().unwrap();
581 fs::create_dir_all(directory.path().join("assets/src")).unwrap();
582 fs::create_dir_all(directory.path().join("lib")).unwrap();
583 fs::write(directory.path().join("assets/src/app.ts"), "run();\n").unwrap();
584 fs::write(directory.path().join("assets/logo.svg"), "<svg/>\n").unwrap();
585 fs::write(
586 directory.path().join("lib/app.ex"),
587 "defmodule App do\nend\n",
588 )
589 .unwrap();
590 assert!(
591 Command::new("git")
592 .args(["init", "--quiet"])
593 .arg(directory.path())
594 .status()
595 .unwrap()
596 .success()
597 );
598 assert!(
599 Command::new("git")
600 .args(["-C"])
601 .arg(directory.path())
602 .args(["add", "."])
603 .status()
604 .unwrap()
605 .success()
606 );
607
608 let atlas = scan_repository(
609 directory.path(),
610 Metric::Loc,
611 false,
612 false,
613 &["assets".to_owned()],
614 )
615 .unwrap();
616 assert_eq!(atlas.files().count(), 1);
617 assert_eq!(atlas.excluded_custom_files, 2);
618 assert_eq!(atlas.excluded_paths, vec!["assets"]);
619 assert!(atlas.path_to_id.contains_key("lib/app.ex"));
620 }
621
622 #[test]
623 fn scanner_skips_tracked_files_deleted_from_the_worktree() {
624 let directory = tempfile::tempdir().unwrap();
625 fs::write(directory.path().join("present.rs"), "fn present() {}\n").unwrap();
626 fs::write(directory.path().join("deleted.rs"), "fn deleted() {}\n").unwrap();
627 assert!(
628 Command::new("git")
629 .args(["init", "--quiet"])
630 .arg(directory.path())
631 .status()
632 .unwrap()
633 .success()
634 );
635 assert!(
636 Command::new("git")
637 .args(["-C"])
638 .arg(directory.path())
639 .args(["add", "."])
640 .status()
641 .unwrap()
642 .success()
643 );
644 fs::remove_file(directory.path().join("deleted.rs")).unwrap();
645
646 let atlas = scan_repository(directory.path(), Metric::Loc, false, false, &[]).unwrap();
647
648 assert_eq!(atlas.files().count(), 1);
649 assert!(atlas.path_to_id.contains_key("present.rs"));
650 assert!(!atlas.path_to_id.contains_key("deleted.rs"));
651 assert!(atlas.dirty);
652 }
653}