Skip to main content

code_atlas/
git.rs

1use std::{
2    collections::HashMap,
3    fs,
4    path::{Path, PathBuf},
5    process::Command,
6};
7
8use anyhow::{Context, Result, bail};
9use rayon::prelude::*;
10
11use crate::model::{AnalyzerReport, Atlas, BuildTimings, Metric, Node, NodeKind, Rect};
12
13#[derive(Debug)]
14struct FileStats {
15    path: String,
16    bytes: u64,
17    loc: u64,
18    commits: u64,
19    language: String,
20}
21
22pub fn scan_repository(
23    root: &Path,
24    metric: Metric,
25    include_tests: bool,
26    include_hidden: bool,
27    excluded_paths: &[String],
28) -> Result<Atlas> {
29    let root = root
30        .canonicalize()
31        .with_context(|| format!("cannot open repository {}", root.display()))?;
32    if !root.join(".git").exists() {
33        bail!("{} is not a Git checkout", root.display());
34    }
35
36    let excluded_paths = normalize_excluded_paths(excluded_paths)?;
37    let tracked = tracked_paths(&root, include_tests, include_hidden, &excluded_paths)?;
38    let paths = tracked.paths;
39    // Walking the complete repository history dominates scans of large
40    // monorepos. Only pay that cost when commit count is the selected metric.
41    let commit_counts = if metric == Metric::Commits {
42        commit_counts(&root)?
43    } else {
44        HashMap::new()
45    };
46    let stats: Result<Vec<_>> = paths
47        .par_iter()
48        .map(|path| read_file_stats(&root, path, &commit_counts))
49        .collect();
50    let mut stats = stats?;
51    stats.sort_by(|left, right| left.path.cmp(&right.path));
52
53    let revision = git_output(&root, &["rev-parse", "HEAD"])
54        .unwrap_or_else(|_| "unborn".to_owned())
55        .trim()
56        .to_owned();
57    let dirty = !git_output(&root, &["status", "--porcelain"])
58        .unwrap_or_default()
59        .trim()
60        .is_empty();
61
62    let mut nodes = vec![Node {
63        id: 0,
64        parent: None,
65        children: Vec::new(),
66        kind: NodeKind::Directory,
67        name: root
68            .file_name()
69            .and_then(|name| name.to_str())
70            .unwrap_or("repository")
71            .to_owned(),
72        path: String::new(),
73        depth: 0,
74        bytes: 0,
75        loc: 0,
76        commits: 0,
77        weight: 0.0,
78        language: "directory".to_owned(),
79        rect: Rect::default(),
80    }];
81    let mut path_to_id = HashMap::new();
82    path_to_id.insert(String::new(), 0);
83
84    for file in stats {
85        let file_path = Path::new(&file.path);
86        let components: Vec<_> = file_path.components().collect();
87        let mut parent = 0;
88        let mut accumulated = PathBuf::new();
89
90        for component in components.iter().take(components.len().saturating_sub(1)) {
91            accumulated.push(component.as_os_str());
92            let directory_path = accumulated.to_string_lossy().replace('\\', "/");
93            if let Some(existing) = path_to_id.get(&directory_path) {
94                parent = *existing;
95                continue;
96            }
97
98            let id = nodes.len();
99            nodes.push(Node {
100                id,
101                parent: Some(parent),
102                children: Vec::new(),
103                kind: NodeKind::Directory,
104                name: component.as_os_str().to_string_lossy().to_string(),
105                path: directory_path.clone(),
106                depth: nodes[parent].depth + 1,
107                bytes: 0,
108                loc: 0,
109                commits: 0,
110                weight: 0.0,
111                language: "directory".to_owned(),
112                rect: Rect::default(),
113            });
114            nodes[parent].children.push(id);
115            path_to_id.insert(directory_path, id);
116            parent = id;
117        }
118
119        let id = nodes.len();
120        let name = file_path
121            .file_name()
122            .and_then(|name| name.to_str())
123            .unwrap_or(&file.path)
124            .to_owned();
125        let weight = metric.value(file.bytes, file.loc, file.commits);
126        nodes.push(Node {
127            id,
128            parent: Some(parent),
129            children: Vec::new(),
130            kind: NodeKind::File,
131            name,
132            path: file.path.clone(),
133            depth: nodes[parent].depth + 1,
134            bytes: file.bytes,
135            loc: file.loc,
136            commits: file.commits,
137            weight,
138            language: file.language,
139            rect: Rect::default(),
140        });
141        nodes[parent].children.push(id);
142        path_to_id.insert(file.path, id);
143    }
144
145    roll_up_metrics(0, &mut nodes);
146
147    Ok(Atlas {
148        root_path: root,
149        revision,
150        dirty,
151        excluded_test_files: tracked.excluded_test_files,
152        excluded_hidden_files: tracked.excluded_hidden_files,
153        excluded_custom_files: tracked.excluded_custom_files,
154        excluded_paths,
155        nodes,
156        calls: Vec::new(),
157        path_to_id,
158        report: AnalyzerReport::default(),
159        timings: BuildTimings::default(),
160    })
161}
162
163struct TrackedPaths {
164    paths: Vec<String>,
165    excluded_test_files: usize,
166    excluded_hidden_files: usize,
167    excluded_custom_files: usize,
168}
169
170fn tracked_paths(
171    root: &Path,
172    include_tests: bool,
173    include_hidden: bool,
174    excluded_paths: &[String],
175) -> Result<TrackedPaths> {
176    let output = Command::new("git")
177        .args(["-C"])
178        .arg(root)
179        .args(["ls-files", "-z"])
180        .output()
181        .context("failed to run git ls-files")?;
182    if !output.status.success() {
183        bail!("git ls-files failed");
184    }
185    let mut tracked: Vec<_> = output
186        .stdout
187        .split(|byte| *byte == 0)
188        .filter(|path| !path.is_empty())
189        .map(|path| String::from_utf8_lossy(path).into_owned())
190        .collect();
191    // `git ls-files` includes index entries deleted from the working tree. The
192    // atlas represents the checkout as it exists now, so those entries are not
193    // readable parcels and must not abort an otherwise valid dirty-tree scan.
194    tracked.retain(|path| root.join(path).is_file());
195    let mut excluded_test_files = 0;
196    if !include_tests {
197        tracked.retain(|path| {
198            let include = !is_test_path(path);
199            excluded_test_files += usize::from(!include);
200            include
201        });
202    }
203
204    let mut excluded_hidden_files = 0;
205    if !include_hidden {
206        tracked.retain(|path| {
207            let include = !is_hidden_path(path);
208            excluded_hidden_files += usize::from(!include);
209            include
210        });
211    }
212
213    let mut excluded_custom_files = 0;
214    if !excluded_paths.is_empty() {
215        tracked.retain(|path| {
216            let include = !excluded_paths
217                .iter()
218                .any(|excluded| path == excluded || path.starts_with(&format!("{excluded}/")));
219            excluded_custom_files += usize::from(!include);
220            include
221        });
222    }
223
224    Ok(TrackedPaths {
225        paths: tracked,
226        excluded_test_files,
227        excluded_hidden_files,
228        excluded_custom_files,
229    })
230}
231
232fn normalize_excluded_paths(paths: &[String]) -> Result<Vec<String>> {
233    let mut normalized = Vec::new();
234    for path in paths {
235        let path = path.trim().replace('\\', "/");
236        let path = path.trim_matches('/');
237        if path.is_empty() {
238            bail!("custom exclusion paths cannot be empty");
239        }
240        if Path::new(path).is_absolute()
241            || path
242                .split('/')
243                .any(|component| component == ".." || component.is_empty())
244        {
245            bail!("custom exclusion {path:?} must be a repository-relative path");
246        }
247        if !normalized.iter().any(|existing| existing == path) {
248            normalized.push(path.to_owned());
249        }
250    }
251    normalized.sort();
252    Ok(normalized)
253}
254
255/// A hidden path has at least one file or directory component beginning with
256/// a dot, such as `.github/workflows/ci.yml` or `lib/.generated/module.ex`.
257pub fn is_hidden_path(path: &str) -> bool {
258    path.replace('\\', "/")
259        .split('/')
260        .any(|component| component.starts_with('.') && component.len() > 1)
261}
262
263/// Classifies conventional, language-specific test paths without excluding
264/// production files that merely contain words such as `test` or `spec`.
265pub fn is_test_path(path: &str) -> bool {
266    let normalized = path.replace('\\', "/").to_ascii_lowercase();
267    let components: Vec<_> = normalized
268        .split('/')
269        .filter(|component| !component.is_empty())
270        .collect();
271    let Some(file_name) = components.last().copied() else {
272        return false;
273    };
274
275    if components
276        .iter()
277        .take(components.len().saturating_sub(1))
278        .any(|component| {
279            matches!(
280                *component,
281                "test" | "tests" | "spec" | "specs" | "__tests__" | "testdata"
282            )
283        })
284    {
285        return true;
286    }
287
288    if matches!(file_name, "test_helper.exs" | "conftest.py")
289        || file_name.starts_with("test.")
290        || file_name.contains(".test.")
291        || file_name.contains(".spec.")
292    {
293        return true;
294    }
295
296    const PREFIX_EXTENSIONS: &[&str] = &[
297        ".py", ".rb", ".exs", ".rs", ".go", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts",
298        ".cts",
299    ];
300    if file_name.starts_with("test_")
301        && PREFIX_EXTENSIONS
302            .iter()
303            .any(|extension| file_name.ends_with(extension))
304    {
305        return true;
306    }
307
308    const SUFFIXES: &[&str] = &[
309        "_test.ex",
310        "_test.exs",
311        "_test.rs",
312        "_tests.rs",
313        "_test.go",
314        "_test.py",
315        "_test.rb",
316        "_spec.exs",
317        "_spec.rb",
318    ];
319    SUFFIXES.iter().any(|suffix| file_name.ends_with(suffix))
320}
321
322fn commit_counts(root: &Path) -> Result<HashMap<String, u64>> {
323    let output = Command::new("git")
324        .args(["-C"])
325        .arg(root)
326        .args(["log", "--format=", "--name-only", "--no-renames", "HEAD"])
327        .output()
328        .context("failed to run git log")?;
329    if !output.status.success() {
330        return Ok(HashMap::new());
331    }
332
333    let mut counts = HashMap::new();
334    for line in String::from_utf8_lossy(&output.stdout).lines() {
335        let path = line.trim();
336        if !path.is_empty() {
337            *counts.entry(path.to_owned()).or_insert(0) += 1;
338        }
339    }
340    Ok(counts)
341}
342
343fn read_file_stats(
344    root: &Path,
345    path: &str,
346    commit_counts: &HashMap<String, u64>,
347) -> Result<FileStats> {
348    let bytes =
349        fs::read(root.join(path)).with_context(|| format!("cannot read tracked file {path}"))?;
350    let binary = bytes.iter().take(8192).any(|byte| *byte == 0);
351    let loc = if binary {
352        0
353    } else {
354        String::from_utf8_lossy(&bytes).lines().count() as u64
355    };
356    Ok(FileStats {
357        path: path.to_owned(),
358        bytes: bytes.len() as u64,
359        loc,
360        commits: commit_counts.get(path).copied().unwrap_or(1),
361        language: language_for_path(path).to_owned(),
362    })
363}
364
365fn language_for_path(path: &str) -> &'static str {
366    match Path::new(path)
367        .extension()
368        .and_then(|extension| extension.to_str())
369    {
370        Some("ex") | Some("exs") | Some("eex") => "elixir",
371        Some("erl") => "erlang",
372        Some("rs") => "rust",
373        Some("js") | Some("jsx") | Some("mjs") | Some("cjs") => "javascript",
374        Some("ts") | Some("tsx") | Some("mts") | Some("cts") => "typescript",
375        Some("gleam") => "gleam",
376        Some("py") => "python",
377        Some("go") => "go",
378        Some("c") | Some("h") => "c",
379        Some("cc") | Some("cpp") | Some("hpp") => "cpp",
380        Some("java") => "java",
381        Some("rb") => "ruby",
382        Some("md") => "markdown",
383        Some("toml") | Some("lock") => "config",
384        Some("json") | Some("yml") | Some("yaml") => "data",
385        Some("wit") | Some("wat") | Some("wasm") => "wasm",
386        _ => "other",
387    }
388}
389
390fn roll_up_metrics(id: usize, nodes: &mut [Node]) -> (u64, u64, u64, f64) {
391    if nodes[id].is_file() {
392        return (
393            nodes[id].bytes,
394            nodes[id].loc,
395            nodes[id].commits,
396            nodes[id].weight,
397        );
398    }
399
400    let children = nodes[id].children.clone();
401    let mut totals = (0, 0, 0, 0.0);
402    for child in children {
403        let child_totals = roll_up_metrics(child, nodes);
404        totals.0 += child_totals.0;
405        totals.1 += child_totals.1;
406        totals.2 += child_totals.2;
407        totals.3 += child_totals.3;
408    }
409    nodes[id].bytes = totals.0;
410    nodes[id].loc = totals.1;
411    nodes[id].commits = totals.2;
412    nodes[id].weight = totals.3.max(1.0);
413    totals
414}
415
416fn git_output(root: &Path, args: &[&str]) -> Result<String> {
417    let output = Command::new("git")
418        .args(["-C"])
419        .arg(root)
420        .args(args)
421        .output()
422        .context("failed to invoke git")?;
423    if !output.status.success() {
424        bail!(
425            "git command failed: {}",
426            String::from_utf8_lossy(&output.stderr)
427        );
428    }
429    Ok(String::from_utf8_lossy(&output.stdout).into_owned())
430}
431
432#[cfg(test)]
433mod tests {
434    use super::*;
435
436    #[test]
437    fn recognizes_conventional_test_directories_and_files() {
438        for path in [
439            "test/wasmex_test.exs",
440            "tests/integration.rs",
441            "spec/models/widget_spec.rb",
442            "src/__tests__/widget.test.ts",
443            "pkg/testdata/input.json",
444            "lib/widget_test.exs",
445            "src/widget_test.go",
446            "src/test_widget.py",
447            "src/widget.test.tsx",
448            "src/widget.spec.js",
449            "src/widget.test.mts",
450            "src/widget.spec.cjs",
451            "src/widget_spec.rb",
452            "conftest.py",
453            "test_helper.exs",
454            "config/test.exs",
455            "test.json",
456        ] {
457            assert!(
458                is_test_path(path),
459                "expected {path} to be classified as a test"
460            );
461        }
462    }
463
464    #[test]
465    fn recognizes_analyzed_language_extensions() {
466        for path in ["src/a.js", "src/a.jsx", "src/a.mjs", "src/a.cjs"] {
467            assert_eq!(language_for_path(path), "javascript");
468        }
469        for path in ["src/a.ts", "src/a.tsx", "src/a.mts", "src/a.cts"] {
470            assert_eq!(language_for_path(path), "typescript");
471        }
472        assert_eq!(language_for_path("lib/templates/a.eex"), "elixir");
473        assert_eq!(language_for_path("lib/compiler/a.erl"), "erlang");
474        assert_eq!(language_for_path("src/a.gleam"), "gleam");
475    }
476
477    #[test]
478    fn does_not_match_test_words_inside_production_names() {
479        for path in [
480            "src/contest.rs",
481            "lib/testing_tools.ex",
482            "docs/test-strategy.md",
483            "src/specification.rs",
484            "src/testimonial.ts",
485            "lib/fixture_loader.ex",
486        ] {
487            assert!(
488                !is_test_path(path),
489                "expected {path} to remain in the production atlas"
490            );
491        }
492    }
493
494    #[test]
495    fn recognizes_hidden_files_and_directories() {
496        for path in [
497            ".github/workflows/ci.yml",
498            ".formatter.exs",
499            "lib/.generated/module.ex",
500            "assets/.cache/data.json",
501        ] {
502            assert!(
503                is_hidden_path(path),
504                "expected {path} to be classified as hidden"
505            );
506        }
507        for path in [
508            "github/workflows/ci.yml",
509            "formatter.exs",
510            "lib/generated/module.ex",
511            "docs/dot-files.md",
512        ] {
513            assert!(!is_hidden_path(path), "expected {path} to remain visible");
514        }
515    }
516
517    #[test]
518    fn scanner_excludes_tests_and_hidden_paths_with_independent_overrides() {
519        let directory = tempfile::tempdir().unwrap();
520        fs::create_dir_all(directory.path().join("lib")).unwrap();
521        fs::create_dir_all(directory.path().join("lib/.generated")).unwrap();
522        fs::create_dir_all(directory.path().join("test")).unwrap();
523        fs::create_dir_all(directory.path().join(".github/workflows")).unwrap();
524        fs::write(
525            directory.path().join("lib/widget.ex"),
526            "defmodule Widget do\nend\n",
527        )
528        .unwrap();
529        fs::write(
530            directory.path().join("lib/.generated/widget.ex"),
531            "defmodule GeneratedWidget do\nend\n",
532        )
533        .unwrap();
534        fs::write(directory.path().join(".formatter.exs"), "[inputs: []]\n").unwrap();
535        fs::write(
536            directory.path().join(".github/workflows/ci.yml"),
537            "name: CI\n",
538        )
539        .unwrap();
540        fs::write(
541            directory.path().join("lib/widget_test.exs"),
542            "defmodule WidgetTest do\nend\n",
543        )
544        .unwrap();
545        fs::write(
546            directory.path().join("test/widget_test.exs"),
547            "defmodule WidgetTest do\nend\n",
548        )
549        .unwrap();
550
551        let init = Command::new("git")
552            .args(["init", "--quiet"])
553            .arg(directory.path())
554            .status()
555            .unwrap();
556        assert!(init.success());
557        let add = Command::new("git")
558            .args(["-C"])
559            .arg(directory.path())
560            .args(["add", "."])
561            .status()
562            .unwrap();
563        assert!(add.success());
564
565        let atlas = scan_repository(directory.path(), Metric::Loc, false, false, &[]).unwrap();
566        assert_eq!(atlas.files().count(), 1);
567        assert_eq!(atlas.excluded_test_files, 2);
568        assert_eq!(atlas.excluded_hidden_files, 3);
569        assert!(atlas.path_to_id.contains_key("lib/widget.ex"));
570
571        let atlas = scan_repository(directory.path(), Metric::Loc, true, false, &[]).unwrap();
572        assert_eq!(atlas.files().count(), 3);
573        assert_eq!(atlas.excluded_test_files, 0);
574        assert_eq!(atlas.excluded_hidden_files, 3);
575
576        let atlas = scan_repository(directory.path(), Metric::Loc, false, true, &[]).unwrap();
577        assert_eq!(atlas.files().count(), 4);
578        assert_eq!(atlas.excluded_test_files, 2);
579        assert_eq!(atlas.excluded_hidden_files, 0);
580
581        let atlas = scan_repository(directory.path(), Metric::Loc, true, true, &[]).unwrap();
582        assert_eq!(atlas.files().count(), 6);
583        assert_eq!(atlas.excluded_test_files, 0);
584        assert_eq!(atlas.excluded_hidden_files, 0);
585    }
586
587    #[test]
588    fn scanner_excludes_repeatable_repository_relative_paths() {
589        let directory = tempfile::tempdir().unwrap();
590        fs::create_dir_all(directory.path().join("assets/src")).unwrap();
591        fs::create_dir_all(directory.path().join("lib")).unwrap();
592        fs::write(directory.path().join("assets/src/app.ts"), "run();\n").unwrap();
593        fs::write(directory.path().join("assets/logo.svg"), "<svg/>\n").unwrap();
594        fs::write(
595            directory.path().join("lib/app.ex"),
596            "defmodule App do\nend\n",
597        )
598        .unwrap();
599        assert!(
600            Command::new("git")
601                .args(["init", "--quiet"])
602                .arg(directory.path())
603                .status()
604                .unwrap()
605                .success()
606        );
607        assert!(
608            Command::new("git")
609                .args(["-C"])
610                .arg(directory.path())
611                .args(["add", "."])
612                .status()
613                .unwrap()
614                .success()
615        );
616
617        let atlas = scan_repository(
618            directory.path(),
619            Metric::Loc,
620            false,
621            false,
622            &["assets".to_owned()],
623        )
624        .unwrap();
625        assert_eq!(atlas.files().count(), 1);
626        assert_eq!(atlas.excluded_custom_files, 2);
627        assert_eq!(atlas.excluded_paths, vec!["assets"]);
628        assert!(atlas.path_to_id.contains_key("lib/app.ex"));
629    }
630
631    #[test]
632    fn scanner_skips_tracked_files_deleted_from_the_worktree() {
633        let directory = tempfile::tempdir().unwrap();
634        fs::write(directory.path().join("present.rs"), "fn present() {}\n").unwrap();
635        fs::write(directory.path().join("deleted.rs"), "fn deleted() {}\n").unwrap();
636        assert!(
637            Command::new("git")
638                .args(["init", "--quiet"])
639                .arg(directory.path())
640                .status()
641                .unwrap()
642                .success()
643        );
644        assert!(
645            Command::new("git")
646                .args(["-C"])
647                .arg(directory.path())
648                .args(["add", "."])
649                .status()
650                .unwrap()
651                .success()
652        );
653        fs::remove_file(directory.path().join("deleted.rs")).unwrap();
654
655        let atlas = scan_repository(directory.path(), Metric::Loc, false, false, &[]).unwrap();
656
657        assert_eq!(atlas.files().count(), 1);
658        assert!(atlas.path_to_id.contains_key("present.rs"));
659        assert!(!atlas.path_to_id.contains_key("deleted.rs"));
660        assert!(atlas.dirty);
661    }
662}