1use std::{
2 collections::HashMap,
3 fs,
4 path::{Path, PathBuf},
5 process::Command,
6};
7
8use anyhow::{Context, Result, bail};
9use rayon::prelude::*;
10
11use crate::model::{AnalyzerReport, Atlas, BuildTimings, Metric, Node, NodeKind, Rect};
12
13#[derive(Debug)]
14struct FileStats {
15 path: String,
16 bytes: u64,
17 loc: u64,
18 commits: u64,
19 language: String,
20}
21
22pub fn scan_repository(
23 root: &Path,
24 metric: Metric,
25 include_tests: bool,
26 include_hidden: bool,
27 excluded_paths: &[String],
28) -> Result<Atlas> {
29 let root = root
30 .canonicalize()
31 .with_context(|| format!("cannot open repository {}", root.display()))?;
32 if !root.join(".git").exists() {
33 bail!("{} is not a Git checkout", root.display());
34 }
35
36 let excluded_paths = normalize_excluded_paths(excluded_paths)?;
37 let tracked = tracked_paths(&root, include_tests, include_hidden, &excluded_paths)?;
38 let paths = tracked.paths;
39 let commit_counts = if metric == Metric::Commits {
42 commit_counts(&root)?
43 } else {
44 HashMap::new()
45 };
46 let stats: Result<Vec<_>> = paths
47 .par_iter()
48 .map(|path| read_file_stats(&root, path, &commit_counts))
49 .collect();
50 let mut stats = stats?;
51 stats.sort_by(|left, right| left.path.cmp(&right.path));
52
53 let revision = git_output(&root, &["rev-parse", "HEAD"])
54 .unwrap_or_else(|_| "unborn".to_owned())
55 .trim()
56 .to_owned();
57 let dirty = !git_output(&root, &["status", "--porcelain"])
58 .unwrap_or_default()
59 .trim()
60 .is_empty();
61
62 let mut nodes = vec![Node {
63 id: 0,
64 parent: None,
65 children: Vec::new(),
66 kind: NodeKind::Directory,
67 name: root
68 .file_name()
69 .and_then(|name| name.to_str())
70 .unwrap_or("repository")
71 .to_owned(),
72 path: String::new(),
73 depth: 0,
74 bytes: 0,
75 loc: 0,
76 commits: 0,
77 weight: 0.0,
78 language: "directory".to_owned(),
79 rect: Rect::default(),
80 }];
81 let mut path_to_id = HashMap::new();
82 path_to_id.insert(String::new(), 0);
83
84 for file in stats {
85 let file_path = Path::new(&file.path);
86 let components: Vec<_> = file_path.components().collect();
87 let mut parent = 0;
88 let mut accumulated = PathBuf::new();
89
90 for component in components.iter().take(components.len().saturating_sub(1)) {
91 accumulated.push(component.as_os_str());
92 let directory_path = accumulated.to_string_lossy().replace('\\', "/");
93 if let Some(existing) = path_to_id.get(&directory_path) {
94 parent = *existing;
95 continue;
96 }
97
98 let id = nodes.len();
99 nodes.push(Node {
100 id,
101 parent: Some(parent),
102 children: Vec::new(),
103 kind: NodeKind::Directory,
104 name: component.as_os_str().to_string_lossy().to_string(),
105 path: directory_path.clone(),
106 depth: nodes[parent].depth + 1,
107 bytes: 0,
108 loc: 0,
109 commits: 0,
110 weight: 0.0,
111 language: "directory".to_owned(),
112 rect: Rect::default(),
113 });
114 nodes[parent].children.push(id);
115 path_to_id.insert(directory_path, id);
116 parent = id;
117 }
118
119 let id = nodes.len();
120 let name = file_path
121 .file_name()
122 .and_then(|name| name.to_str())
123 .unwrap_or(&file.path)
124 .to_owned();
125 let weight = metric.value(file.bytes, file.loc, file.commits);
126 nodes.push(Node {
127 id,
128 parent: Some(parent),
129 children: Vec::new(),
130 kind: NodeKind::File,
131 name,
132 path: file.path.clone(),
133 depth: nodes[parent].depth + 1,
134 bytes: file.bytes,
135 loc: file.loc,
136 commits: file.commits,
137 weight,
138 language: file.language,
139 rect: Rect::default(),
140 });
141 nodes[parent].children.push(id);
142 path_to_id.insert(file.path, id);
143 }
144
145 roll_up_metrics(0, &mut nodes);
146
147 Ok(Atlas {
148 root_path: root,
149 revision,
150 dirty,
151 excluded_test_files: tracked.excluded_test_files,
152 excluded_hidden_files: tracked.excluded_hidden_files,
153 excluded_custom_files: tracked.excluded_custom_files,
154 excluded_paths,
155 nodes,
156 calls: Vec::new(),
157 path_to_id,
158 report: AnalyzerReport::default(),
159 timings: BuildTimings::default(),
160 })
161}
162
163struct TrackedPaths {
164 paths: Vec<String>,
165 excluded_test_files: usize,
166 excluded_hidden_files: usize,
167 excluded_custom_files: usize,
168}
169
170fn tracked_paths(
171 root: &Path,
172 include_tests: bool,
173 include_hidden: bool,
174 excluded_paths: &[String],
175) -> Result<TrackedPaths> {
176 let output = Command::new("git")
177 .args(["-C"])
178 .arg(root)
179 .args(["ls-files", "-z"])
180 .output()
181 .context("failed to run git ls-files")?;
182 if !output.status.success() {
183 bail!("git ls-files failed");
184 }
185 let mut tracked: Vec<_> = output
186 .stdout
187 .split(|byte| *byte == 0)
188 .filter(|path| !path.is_empty())
189 .map(|path| String::from_utf8_lossy(path).into_owned())
190 .collect();
191 tracked.retain(|path| root.join(path).is_file());
195 let mut excluded_test_files = 0;
196 if !include_tests {
197 tracked.retain(|path| {
198 let include = !is_test_path(path);
199 excluded_test_files += usize::from(!include);
200 include
201 });
202 }
203
204 let mut excluded_hidden_files = 0;
205 if !include_hidden {
206 tracked.retain(|path| {
207 let include = !is_hidden_path(path);
208 excluded_hidden_files += usize::from(!include);
209 include
210 });
211 }
212
213 let mut excluded_custom_files = 0;
214 if !excluded_paths.is_empty() {
215 tracked.retain(|path| {
216 let include = !excluded_paths
217 .iter()
218 .any(|excluded| path == excluded || path.starts_with(&format!("{excluded}/")));
219 excluded_custom_files += usize::from(!include);
220 include
221 });
222 }
223
224 Ok(TrackedPaths {
225 paths: tracked,
226 excluded_test_files,
227 excluded_hidden_files,
228 excluded_custom_files,
229 })
230}
231
232fn normalize_excluded_paths(paths: &[String]) -> Result<Vec<String>> {
233 let mut normalized = Vec::new();
234 for path in paths {
235 let path = path.trim().replace('\\', "/");
236 let path = path.trim_matches('/');
237 if path.is_empty() {
238 bail!("custom exclusion paths cannot be empty");
239 }
240 if Path::new(path).is_absolute()
241 || path
242 .split('/')
243 .any(|component| component == ".." || component.is_empty())
244 {
245 bail!("custom exclusion {path:?} must be a repository-relative path");
246 }
247 if !normalized.iter().any(|existing| existing == path) {
248 normalized.push(path.to_owned());
249 }
250 }
251 normalized.sort();
252 Ok(normalized)
253}
254
255pub fn is_hidden_path(path: &str) -> bool {
258 path.replace('\\', "/")
259 .split('/')
260 .any(|component| component.starts_with('.') && component.len() > 1)
261}
262
263pub fn is_test_path(path: &str) -> bool {
266 let normalized = path.replace('\\', "/").to_ascii_lowercase();
267 let components: Vec<_> = normalized
268 .split('/')
269 .filter(|component| !component.is_empty())
270 .collect();
271 let Some(file_name) = components.last().copied() else {
272 return false;
273 };
274
275 if components
276 .iter()
277 .take(components.len().saturating_sub(1))
278 .any(|component| {
279 matches!(
280 *component,
281 "test" | "tests" | "spec" | "specs" | "__tests__" | "testdata"
282 )
283 })
284 {
285 return true;
286 }
287
288 if matches!(file_name, "test_helper.exs" | "conftest.py")
289 || file_name.starts_with("test.")
290 || file_name.contains(".test.")
291 || file_name.contains(".spec.")
292 {
293 return true;
294 }
295
296 const PREFIX_EXTENSIONS: &[&str] = &[
297 ".py", ".rb", ".exs", ".rs", ".go", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts",
298 ".cts",
299 ];
300 if file_name.starts_with("test_")
301 && PREFIX_EXTENSIONS
302 .iter()
303 .any(|extension| file_name.ends_with(extension))
304 {
305 return true;
306 }
307
308 const SUFFIXES: &[&str] = &[
309 "_test.ex",
310 "_test.exs",
311 "_test.rs",
312 "_tests.rs",
313 "_test.go",
314 "_test.py",
315 "_test.rb",
316 "_spec.exs",
317 "_spec.rb",
318 ];
319 SUFFIXES.iter().any(|suffix| file_name.ends_with(suffix))
320}
321
322fn commit_counts(root: &Path) -> Result<HashMap<String, u64>> {
323 let output = Command::new("git")
324 .args(["-C"])
325 .arg(root)
326 .args(["log", "--format=", "--name-only", "--no-renames", "HEAD"])
327 .output()
328 .context("failed to run git log")?;
329 if !output.status.success() {
330 return Ok(HashMap::new());
331 }
332
333 let mut counts = HashMap::new();
334 for line in String::from_utf8_lossy(&output.stdout).lines() {
335 let path = line.trim();
336 if !path.is_empty() {
337 *counts.entry(path.to_owned()).or_insert(0) += 1;
338 }
339 }
340 Ok(counts)
341}
342
343fn read_file_stats(
344 root: &Path,
345 path: &str,
346 commit_counts: &HashMap<String, u64>,
347) -> Result<FileStats> {
348 let bytes =
349 fs::read(root.join(path)).with_context(|| format!("cannot read tracked file {path}"))?;
350 let binary = bytes.iter().take(8192).any(|byte| *byte == 0);
351 let loc = if binary {
352 0
353 } else {
354 String::from_utf8_lossy(&bytes).lines().count() as u64
355 };
356 Ok(FileStats {
357 path: path.to_owned(),
358 bytes: bytes.len() as u64,
359 loc,
360 commits: commit_counts.get(path).copied().unwrap_or(1),
361 language: language_for_path(path).to_owned(),
362 })
363}
364
365fn language_for_path(path: &str) -> &'static str {
366 match Path::new(path)
367 .extension()
368 .and_then(|extension| extension.to_str())
369 {
370 Some("ex") | Some("exs") | Some("eex") => "elixir",
371 Some("erl") => "erlang",
372 Some("rs") => "rust",
373 Some("js") | Some("jsx") | Some("mjs") | Some("cjs") => "javascript",
374 Some("ts") | Some("tsx") | Some("mts") | Some("cts") => "typescript",
375 Some("gleam") => "gleam",
376 Some("py") => "python",
377 Some("go") => "go",
378 Some("c") | Some("h") => "c",
379 Some("cc") | Some("cpp") | Some("hpp") => "cpp",
380 Some("java") => "java",
381 Some("rb") => "ruby",
382 Some("md") => "markdown",
383 Some("toml") | Some("lock") => "config",
384 Some("json") | Some("yml") | Some("yaml") => "data",
385 Some("wit") | Some("wat") | Some("wasm") => "wasm",
386 _ => "other",
387 }
388}
389
390fn roll_up_metrics(id: usize, nodes: &mut [Node]) -> (u64, u64, u64, f64) {
391 if nodes[id].is_file() {
392 return (
393 nodes[id].bytes,
394 nodes[id].loc,
395 nodes[id].commits,
396 nodes[id].weight,
397 );
398 }
399
400 let children = nodes[id].children.clone();
401 let mut totals = (0, 0, 0, 0.0);
402 for child in children {
403 let child_totals = roll_up_metrics(child, nodes);
404 totals.0 += child_totals.0;
405 totals.1 += child_totals.1;
406 totals.2 += child_totals.2;
407 totals.3 += child_totals.3;
408 }
409 nodes[id].bytes = totals.0;
410 nodes[id].loc = totals.1;
411 nodes[id].commits = totals.2;
412 nodes[id].weight = totals.3.max(1.0);
413 totals
414}
415
416fn git_output(root: &Path, args: &[&str]) -> Result<String> {
417 let output = Command::new("git")
418 .args(["-C"])
419 .arg(root)
420 .args(args)
421 .output()
422 .context("failed to invoke git")?;
423 if !output.status.success() {
424 bail!(
425 "git command failed: {}",
426 String::from_utf8_lossy(&output.stderr)
427 );
428 }
429 Ok(String::from_utf8_lossy(&output.stdout).into_owned())
430}
431
432#[cfg(test)]
433mod tests {
434 use super::*;
435
436 #[test]
437 fn recognizes_conventional_test_directories_and_files() {
438 for path in [
439 "test/wasmex_test.exs",
440 "tests/integration.rs",
441 "spec/models/widget_spec.rb",
442 "src/__tests__/widget.test.ts",
443 "pkg/testdata/input.json",
444 "lib/widget_test.exs",
445 "src/widget_test.go",
446 "src/test_widget.py",
447 "src/widget.test.tsx",
448 "src/widget.spec.js",
449 "src/widget.test.mts",
450 "src/widget.spec.cjs",
451 "src/widget_spec.rb",
452 "conftest.py",
453 "test_helper.exs",
454 "config/test.exs",
455 "test.json",
456 ] {
457 assert!(
458 is_test_path(path),
459 "expected {path} to be classified as a test"
460 );
461 }
462 }
463
464 #[test]
465 fn recognizes_analyzed_language_extensions() {
466 for path in ["src/a.js", "src/a.jsx", "src/a.mjs", "src/a.cjs"] {
467 assert_eq!(language_for_path(path), "javascript");
468 }
469 for path in ["src/a.ts", "src/a.tsx", "src/a.mts", "src/a.cts"] {
470 assert_eq!(language_for_path(path), "typescript");
471 }
472 assert_eq!(language_for_path("lib/templates/a.eex"), "elixir");
473 assert_eq!(language_for_path("lib/compiler/a.erl"), "erlang");
474 assert_eq!(language_for_path("src/a.gleam"), "gleam");
475 }
476
477 #[test]
478 fn does_not_match_test_words_inside_production_names() {
479 for path in [
480 "src/contest.rs",
481 "lib/testing_tools.ex",
482 "docs/test-strategy.md",
483 "src/specification.rs",
484 "src/testimonial.ts",
485 "lib/fixture_loader.ex",
486 ] {
487 assert!(
488 !is_test_path(path),
489 "expected {path} to remain in the production atlas"
490 );
491 }
492 }
493
494 #[test]
495 fn recognizes_hidden_files_and_directories() {
496 for path in [
497 ".github/workflows/ci.yml",
498 ".formatter.exs",
499 "lib/.generated/module.ex",
500 "assets/.cache/data.json",
501 ] {
502 assert!(
503 is_hidden_path(path),
504 "expected {path} to be classified as hidden"
505 );
506 }
507 for path in [
508 "github/workflows/ci.yml",
509 "formatter.exs",
510 "lib/generated/module.ex",
511 "docs/dot-files.md",
512 ] {
513 assert!(!is_hidden_path(path), "expected {path} to remain visible");
514 }
515 }
516
517 #[test]
518 fn scanner_excludes_tests_and_hidden_paths_with_independent_overrides() {
519 let directory = tempfile::tempdir().unwrap();
520 fs::create_dir_all(directory.path().join("lib")).unwrap();
521 fs::create_dir_all(directory.path().join("lib/.generated")).unwrap();
522 fs::create_dir_all(directory.path().join("test")).unwrap();
523 fs::create_dir_all(directory.path().join(".github/workflows")).unwrap();
524 fs::write(
525 directory.path().join("lib/widget.ex"),
526 "defmodule Widget do\nend\n",
527 )
528 .unwrap();
529 fs::write(
530 directory.path().join("lib/.generated/widget.ex"),
531 "defmodule GeneratedWidget do\nend\n",
532 )
533 .unwrap();
534 fs::write(directory.path().join(".formatter.exs"), "[inputs: []]\n").unwrap();
535 fs::write(
536 directory.path().join(".github/workflows/ci.yml"),
537 "name: CI\n",
538 )
539 .unwrap();
540 fs::write(
541 directory.path().join("lib/widget_test.exs"),
542 "defmodule WidgetTest do\nend\n",
543 )
544 .unwrap();
545 fs::write(
546 directory.path().join("test/widget_test.exs"),
547 "defmodule WidgetTest do\nend\n",
548 )
549 .unwrap();
550
551 let init = Command::new("git")
552 .args(["init", "--quiet"])
553 .arg(directory.path())
554 .status()
555 .unwrap();
556 assert!(init.success());
557 let add = Command::new("git")
558 .args(["-C"])
559 .arg(directory.path())
560 .args(["add", "."])
561 .status()
562 .unwrap();
563 assert!(add.success());
564
565 let atlas = scan_repository(directory.path(), Metric::Loc, false, false, &[]).unwrap();
566 assert_eq!(atlas.files().count(), 1);
567 assert_eq!(atlas.excluded_test_files, 2);
568 assert_eq!(atlas.excluded_hidden_files, 3);
569 assert!(atlas.path_to_id.contains_key("lib/widget.ex"));
570
571 let atlas = scan_repository(directory.path(), Metric::Loc, true, false, &[]).unwrap();
572 assert_eq!(atlas.files().count(), 3);
573 assert_eq!(atlas.excluded_test_files, 0);
574 assert_eq!(atlas.excluded_hidden_files, 3);
575
576 let atlas = scan_repository(directory.path(), Metric::Loc, false, true, &[]).unwrap();
577 assert_eq!(atlas.files().count(), 4);
578 assert_eq!(atlas.excluded_test_files, 2);
579 assert_eq!(atlas.excluded_hidden_files, 0);
580
581 let atlas = scan_repository(directory.path(), Metric::Loc, true, true, &[]).unwrap();
582 assert_eq!(atlas.files().count(), 6);
583 assert_eq!(atlas.excluded_test_files, 0);
584 assert_eq!(atlas.excluded_hidden_files, 0);
585 }
586
587 #[test]
588 fn scanner_excludes_repeatable_repository_relative_paths() {
589 let directory = tempfile::tempdir().unwrap();
590 fs::create_dir_all(directory.path().join("assets/src")).unwrap();
591 fs::create_dir_all(directory.path().join("lib")).unwrap();
592 fs::write(directory.path().join("assets/src/app.ts"), "run();\n").unwrap();
593 fs::write(directory.path().join("assets/logo.svg"), "<svg/>\n").unwrap();
594 fs::write(
595 directory.path().join("lib/app.ex"),
596 "defmodule App do\nend\n",
597 )
598 .unwrap();
599 assert!(
600 Command::new("git")
601 .args(["init", "--quiet"])
602 .arg(directory.path())
603 .status()
604 .unwrap()
605 .success()
606 );
607 assert!(
608 Command::new("git")
609 .args(["-C"])
610 .arg(directory.path())
611 .args(["add", "."])
612 .status()
613 .unwrap()
614 .success()
615 );
616
617 let atlas = scan_repository(
618 directory.path(),
619 Metric::Loc,
620 false,
621 false,
622 &["assets".to_owned()],
623 )
624 .unwrap();
625 assert_eq!(atlas.files().count(), 1);
626 assert_eq!(atlas.excluded_custom_files, 2);
627 assert_eq!(atlas.excluded_paths, vec!["assets"]);
628 assert!(atlas.path_to_id.contains_key("lib/app.ex"));
629 }
630
631 #[test]
632 fn scanner_skips_tracked_files_deleted_from_the_worktree() {
633 let directory = tempfile::tempdir().unwrap();
634 fs::write(directory.path().join("present.rs"), "fn present() {}\n").unwrap();
635 fs::write(directory.path().join("deleted.rs"), "fn deleted() {}\n").unwrap();
636 assert!(
637 Command::new("git")
638 .args(["init", "--quiet"])
639 .arg(directory.path())
640 .status()
641 .unwrap()
642 .success()
643 );
644 assert!(
645 Command::new("git")
646 .args(["-C"])
647 .arg(directory.path())
648 .args(["add", "."])
649 .status()
650 .unwrap()
651 .success()
652 );
653 fs::remove_file(directory.path().join("deleted.rs")).unwrap();
654
655 let atlas = scan_repository(directory.path(), Metric::Loc, false, false, &[]).unwrap();
656
657 assert_eq!(atlas.files().count(), 1);
658 assert!(atlas.path_to_id.contains_key("present.rs"));
659 assert!(!atlas.path_to_id.contains_key("deleted.rs"));
660 assert!(atlas.dirty);
661 }
662}