Skip to main content

code_atlas/
git.rs

1use std::{
2    collections::HashMap,
3    fs,
4    path::{Path, PathBuf},
5    process::Command,
6};
7
8use anyhow::{Context, Result, bail};
9use rayon::prelude::*;
10
11use crate::model::{AnalyzerReport, Atlas, BuildTimings, Metric, Node, NodeKind, Rect};
12
13#[derive(Debug)]
14struct FileStats {
15    path: String,
16    bytes: u64,
17    loc: u64,
18    commits: u64,
19    language: String,
20}
21
22pub fn scan_repository(
23    root: &Path,
24    metric: Metric,
25    include_tests: bool,
26    include_hidden: bool,
27    excluded_paths: &[String],
28) -> Result<Atlas> {
29    let root = root
30        .canonicalize()
31        .with_context(|| format!("cannot open repository {}", root.display()))?;
32    if !root.join(".git").exists() {
33        bail!("{} is not a Git checkout", root.display());
34    }
35
36    let excluded_paths = normalize_excluded_paths(excluded_paths)?;
37    let tracked = tracked_paths(&root, include_tests, include_hidden, &excluded_paths)?;
38    let paths = tracked.paths;
39    let commit_counts = commit_counts(&root)?;
40    let stats: Result<Vec<_>> = paths
41        .par_iter()
42        .map(|path| read_file_stats(&root, path, &commit_counts))
43        .collect();
44    let mut stats = stats?;
45    stats.sort_by(|left, right| left.path.cmp(&right.path));
46
47    let revision = git_output(&root, &["rev-parse", "HEAD"])
48        .unwrap_or_else(|_| "unborn".to_owned())
49        .trim()
50        .to_owned();
51    let dirty = !git_output(&root, &["status", "--porcelain"])
52        .unwrap_or_default()
53        .trim()
54        .is_empty();
55
56    let mut nodes = vec![Node {
57        id: 0,
58        parent: None,
59        children: Vec::new(),
60        kind: NodeKind::Directory,
61        name: root
62            .file_name()
63            .and_then(|name| name.to_str())
64            .unwrap_or("repository")
65            .to_owned(),
66        path: String::new(),
67        depth: 0,
68        bytes: 0,
69        loc: 0,
70        commits: 0,
71        weight: 0.0,
72        language: "directory".to_owned(),
73        rect: Rect::default(),
74    }];
75    let mut path_to_id = HashMap::new();
76    path_to_id.insert(String::new(), 0);
77
78    for file in stats {
79        let file_path = Path::new(&file.path);
80        let components: Vec<_> = file_path.components().collect();
81        let mut parent = 0;
82        let mut accumulated = PathBuf::new();
83
84        for component in components.iter().take(components.len().saturating_sub(1)) {
85            accumulated.push(component.as_os_str());
86            let directory_path = accumulated.to_string_lossy().replace('\\', "/");
87            if let Some(existing) = path_to_id.get(&directory_path) {
88                parent = *existing;
89                continue;
90            }
91
92            let id = nodes.len();
93            nodes.push(Node {
94                id,
95                parent: Some(parent),
96                children: Vec::new(),
97                kind: NodeKind::Directory,
98                name: component.as_os_str().to_string_lossy().to_string(),
99                path: directory_path.clone(),
100                depth: nodes[parent].depth + 1,
101                bytes: 0,
102                loc: 0,
103                commits: 0,
104                weight: 0.0,
105                language: "directory".to_owned(),
106                rect: Rect::default(),
107            });
108            nodes[parent].children.push(id);
109            path_to_id.insert(directory_path, id);
110            parent = id;
111        }
112
113        let id = nodes.len();
114        let name = file_path
115            .file_name()
116            .and_then(|name| name.to_str())
117            .unwrap_or(&file.path)
118            .to_owned();
119        let weight = metric.value(file.bytes, file.loc, file.commits);
120        nodes.push(Node {
121            id,
122            parent: Some(parent),
123            children: Vec::new(),
124            kind: NodeKind::File,
125            name,
126            path: file.path.clone(),
127            depth: nodes[parent].depth + 1,
128            bytes: file.bytes,
129            loc: file.loc,
130            commits: file.commits,
131            weight,
132            language: file.language,
133            rect: Rect::default(),
134        });
135        nodes[parent].children.push(id);
136        path_to_id.insert(file.path, id);
137    }
138
139    roll_up_metrics(0, &mut nodes);
140
141    Ok(Atlas {
142        root_path: root,
143        revision,
144        dirty,
145        excluded_test_files: tracked.excluded_test_files,
146        excluded_hidden_files: tracked.excluded_hidden_files,
147        excluded_custom_files: tracked.excluded_custom_files,
148        excluded_paths,
149        nodes,
150        calls: Vec::new(),
151        path_to_id,
152        report: AnalyzerReport::default(),
153        timings: BuildTimings::default(),
154    })
155}
156
157struct TrackedPaths {
158    paths: Vec<String>,
159    excluded_test_files: usize,
160    excluded_hidden_files: usize,
161    excluded_custom_files: usize,
162}
163
164fn tracked_paths(
165    root: &Path,
166    include_tests: bool,
167    include_hidden: bool,
168    excluded_paths: &[String],
169) -> Result<TrackedPaths> {
170    let output = Command::new("git")
171        .args(["-C"])
172        .arg(root)
173        .args(["ls-files", "-z"])
174        .output()
175        .context("failed to run git ls-files")?;
176    if !output.status.success() {
177        bail!("git ls-files failed");
178    }
179    let mut tracked: Vec<_> = output
180        .stdout
181        .split(|byte| *byte == 0)
182        .filter(|path| !path.is_empty())
183        .map(|path| String::from_utf8_lossy(path).into_owned())
184        .collect();
185    // `git ls-files` includes index entries deleted from the working tree. The
186    // atlas represents the checkout as it exists now, so those entries are not
187    // readable parcels and must not abort an otherwise valid dirty-tree scan.
188    tracked.retain(|path| root.join(path).is_file());
189    let mut excluded_test_files = 0;
190    if !include_tests {
191        tracked.retain(|path| {
192            let include = !is_test_path(path);
193            excluded_test_files += usize::from(!include);
194            include
195        });
196    }
197
198    let mut excluded_hidden_files = 0;
199    if !include_hidden {
200        tracked.retain(|path| {
201            let include = !is_hidden_path(path);
202            excluded_hidden_files += usize::from(!include);
203            include
204        });
205    }
206
207    let mut excluded_custom_files = 0;
208    if !excluded_paths.is_empty() {
209        tracked.retain(|path| {
210            let include = !excluded_paths
211                .iter()
212                .any(|excluded| path == excluded || path.starts_with(&format!("{excluded}/")));
213            excluded_custom_files += usize::from(!include);
214            include
215        });
216    }
217
218    Ok(TrackedPaths {
219        paths: tracked,
220        excluded_test_files,
221        excluded_hidden_files,
222        excluded_custom_files,
223    })
224}
225
226fn normalize_excluded_paths(paths: &[String]) -> Result<Vec<String>> {
227    let mut normalized = Vec::new();
228    for path in paths {
229        let path = path.trim().replace('\\', "/");
230        let path = path.trim_matches('/');
231        if path.is_empty() {
232            bail!("custom exclusion paths cannot be empty");
233        }
234        if Path::new(path).is_absolute()
235            || path
236                .split('/')
237                .any(|component| component == ".." || component.is_empty())
238        {
239            bail!("custom exclusion {path:?} must be a repository-relative path");
240        }
241        if !normalized.iter().any(|existing| existing == path) {
242            normalized.push(path.to_owned());
243        }
244    }
245    normalized.sort();
246    Ok(normalized)
247}
248
249/// A hidden path has at least one file or directory component beginning with
250/// a dot, such as `.github/workflows/ci.yml` or `lib/.generated/module.ex`.
251pub fn is_hidden_path(path: &str) -> bool {
252    path.replace('\\', "/")
253        .split('/')
254        .any(|component| component.starts_with('.') && component.len() > 1)
255}
256
257/// Classifies conventional, language-specific test paths without excluding
258/// production files that merely contain words such as `test` or `spec`.
259pub fn is_test_path(path: &str) -> bool {
260    let normalized = path.replace('\\', "/").to_ascii_lowercase();
261    let components: Vec<_> = normalized
262        .split('/')
263        .filter(|component| !component.is_empty())
264        .collect();
265    let Some(file_name) = components.last().copied() else {
266        return false;
267    };
268
269    if components
270        .iter()
271        .take(components.len().saturating_sub(1))
272        .any(|component| {
273            matches!(
274                *component,
275                "test" | "tests" | "spec" | "specs" | "__tests__" | "testdata"
276            )
277        })
278    {
279        return true;
280    }
281
282    if matches!(file_name, "test_helper.exs" | "conftest.py")
283        || file_name.starts_with("test.")
284        || file_name.contains(".test.")
285        || file_name.contains(".spec.")
286    {
287        return true;
288    }
289
290    const PREFIX_EXTENSIONS: &[&str] = &[
291        ".py", ".rb", ".exs", ".rs", ".go", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts",
292        ".cts",
293    ];
294    if file_name.starts_with("test_")
295        && PREFIX_EXTENSIONS
296            .iter()
297            .any(|extension| file_name.ends_with(extension))
298    {
299        return true;
300    }
301
302    const SUFFIXES: &[&str] = &[
303        "_test.ex",
304        "_test.exs",
305        "_test.rs",
306        "_tests.rs",
307        "_test.go",
308        "_test.py",
309        "_test.rb",
310        "_spec.exs",
311        "_spec.rb",
312    ];
313    SUFFIXES.iter().any(|suffix| file_name.ends_with(suffix))
314}
315
316fn commit_counts(root: &Path) -> Result<HashMap<String, u64>> {
317    let output = Command::new("git")
318        .args(["-C"])
319        .arg(root)
320        .args(["log", "--format=", "--name-only", "--no-renames", "HEAD"])
321        .output()
322        .context("failed to run git log")?;
323    if !output.status.success() {
324        return Ok(HashMap::new());
325    }
326
327    let mut counts = HashMap::new();
328    for line in String::from_utf8_lossy(&output.stdout).lines() {
329        let path = line.trim();
330        if !path.is_empty() {
331            *counts.entry(path.to_owned()).or_insert(0) += 1;
332        }
333    }
334    Ok(counts)
335}
336
337fn read_file_stats(
338    root: &Path,
339    path: &str,
340    commit_counts: &HashMap<String, u64>,
341) -> Result<FileStats> {
342    let bytes =
343        fs::read(root.join(path)).with_context(|| format!("cannot read tracked file {path}"))?;
344    let binary = bytes.iter().take(8192).any(|byte| *byte == 0);
345    let loc = if binary {
346        0
347    } else {
348        String::from_utf8_lossy(&bytes).lines().count() as u64
349    };
350    Ok(FileStats {
351        path: path.to_owned(),
352        bytes: bytes.len() as u64,
353        loc,
354        commits: commit_counts.get(path).copied().unwrap_or(1),
355        language: language_for_path(path).to_owned(),
356    })
357}
358
359fn language_for_path(path: &str) -> &'static str {
360    match Path::new(path)
361        .extension()
362        .and_then(|extension| extension.to_str())
363    {
364        Some("ex") | Some("exs") => "elixir",
365        Some("rs") => "rust",
366        Some("js") | Some("jsx") | Some("mjs") | Some("cjs") => "javascript",
367        Some("ts") | Some("tsx") | Some("mts") | Some("cts") => "typescript",
368        Some("gleam") => "gleam",
369        Some("py") => "python",
370        Some("go") => "go",
371        Some("c") | Some("h") => "c",
372        Some("cc") | Some("cpp") | Some("hpp") => "cpp",
373        Some("java") => "java",
374        Some("rb") => "ruby",
375        Some("md") => "markdown",
376        Some("toml") | Some("lock") => "config",
377        Some("json") | Some("yml") | Some("yaml") => "data",
378        Some("wit") | Some("wat") | Some("wasm") => "wasm",
379        _ => "other",
380    }
381}
382
383fn roll_up_metrics(id: usize, nodes: &mut [Node]) -> (u64, u64, u64, f64) {
384    if nodes[id].is_file() {
385        return (
386            nodes[id].bytes,
387            nodes[id].loc,
388            nodes[id].commits,
389            nodes[id].weight,
390        );
391    }
392
393    let children = nodes[id].children.clone();
394    let mut totals = (0, 0, 0, 0.0);
395    for child in children {
396        let child_totals = roll_up_metrics(child, nodes);
397        totals.0 += child_totals.0;
398        totals.1 += child_totals.1;
399        totals.2 += child_totals.2;
400        totals.3 += child_totals.3;
401    }
402    nodes[id].bytes = totals.0;
403    nodes[id].loc = totals.1;
404    nodes[id].commits = totals.2;
405    nodes[id].weight = totals.3.max(1.0);
406    totals
407}
408
409fn git_output(root: &Path, args: &[&str]) -> Result<String> {
410    let output = Command::new("git")
411        .args(["-C"])
412        .arg(root)
413        .args(args)
414        .output()
415        .context("failed to invoke git")?;
416    if !output.status.success() {
417        bail!(
418            "git command failed: {}",
419            String::from_utf8_lossy(&output.stderr)
420        );
421    }
422    Ok(String::from_utf8_lossy(&output.stdout).into_owned())
423}
424
425#[cfg(test)]
426mod tests {
427    use super::*;
428
429    #[test]
430    fn recognizes_conventional_test_directories_and_files() {
431        for path in [
432            "test/wasmex_test.exs",
433            "tests/integration.rs",
434            "spec/models/widget_spec.rb",
435            "src/__tests__/widget.test.ts",
436            "pkg/testdata/input.json",
437            "lib/widget_test.exs",
438            "src/widget_test.go",
439            "src/test_widget.py",
440            "src/widget.test.tsx",
441            "src/widget.spec.js",
442            "src/widget.test.mts",
443            "src/widget.spec.cjs",
444            "src/widget_spec.rb",
445            "conftest.py",
446            "test_helper.exs",
447            "config/test.exs",
448            "test.json",
449        ] {
450            assert!(
451                is_test_path(path),
452                "expected {path} to be classified as a test"
453            );
454        }
455    }
456
457    #[test]
458    fn recognizes_analyzed_language_extensions() {
459        for path in ["src/a.js", "src/a.jsx", "src/a.mjs", "src/a.cjs"] {
460            assert_eq!(language_for_path(path), "javascript");
461        }
462        for path in ["src/a.ts", "src/a.tsx", "src/a.mts", "src/a.cts"] {
463            assert_eq!(language_for_path(path), "typescript");
464        }
465        assert_eq!(language_for_path("src/a.gleam"), "gleam");
466    }
467
468    #[test]
469    fn does_not_match_test_words_inside_production_names() {
470        for path in [
471            "src/contest.rs",
472            "lib/testing_tools.ex",
473            "docs/test-strategy.md",
474            "src/specification.rs",
475            "src/testimonial.ts",
476            "lib/fixture_loader.ex",
477        ] {
478            assert!(
479                !is_test_path(path),
480                "expected {path} to remain in the production atlas"
481            );
482        }
483    }
484
485    #[test]
486    fn recognizes_hidden_files_and_directories() {
487        for path in [
488            ".github/workflows/ci.yml",
489            ".formatter.exs",
490            "lib/.generated/module.ex",
491            "assets/.cache/data.json",
492        ] {
493            assert!(
494                is_hidden_path(path),
495                "expected {path} to be classified as hidden"
496            );
497        }
498        for path in [
499            "github/workflows/ci.yml",
500            "formatter.exs",
501            "lib/generated/module.ex",
502            "docs/dot-files.md",
503        ] {
504            assert!(!is_hidden_path(path), "expected {path} to remain visible");
505        }
506    }
507
508    #[test]
509    fn scanner_excludes_tests_and_hidden_paths_with_independent_overrides() {
510        let directory = tempfile::tempdir().unwrap();
511        fs::create_dir_all(directory.path().join("lib")).unwrap();
512        fs::create_dir_all(directory.path().join("lib/.generated")).unwrap();
513        fs::create_dir_all(directory.path().join("test")).unwrap();
514        fs::create_dir_all(directory.path().join(".github/workflows")).unwrap();
515        fs::write(
516            directory.path().join("lib/widget.ex"),
517            "defmodule Widget do\nend\n",
518        )
519        .unwrap();
520        fs::write(
521            directory.path().join("lib/.generated/widget.ex"),
522            "defmodule GeneratedWidget do\nend\n",
523        )
524        .unwrap();
525        fs::write(directory.path().join(".formatter.exs"), "[inputs: []]\n").unwrap();
526        fs::write(
527            directory.path().join(".github/workflows/ci.yml"),
528            "name: CI\n",
529        )
530        .unwrap();
531        fs::write(
532            directory.path().join("lib/widget_test.exs"),
533            "defmodule WidgetTest do\nend\n",
534        )
535        .unwrap();
536        fs::write(
537            directory.path().join("test/widget_test.exs"),
538            "defmodule WidgetTest do\nend\n",
539        )
540        .unwrap();
541
542        let init = Command::new("git")
543            .args(["init", "--quiet"])
544            .arg(directory.path())
545            .status()
546            .unwrap();
547        assert!(init.success());
548        let add = Command::new("git")
549            .args(["-C"])
550            .arg(directory.path())
551            .args(["add", "."])
552            .status()
553            .unwrap();
554        assert!(add.success());
555
556        let atlas = scan_repository(directory.path(), Metric::Loc, false, false, &[]).unwrap();
557        assert_eq!(atlas.files().count(), 1);
558        assert_eq!(atlas.excluded_test_files, 2);
559        assert_eq!(atlas.excluded_hidden_files, 3);
560        assert!(atlas.path_to_id.contains_key("lib/widget.ex"));
561
562        let atlas = scan_repository(directory.path(), Metric::Loc, true, false, &[]).unwrap();
563        assert_eq!(atlas.files().count(), 3);
564        assert_eq!(atlas.excluded_test_files, 0);
565        assert_eq!(atlas.excluded_hidden_files, 3);
566
567        let atlas = scan_repository(directory.path(), Metric::Loc, false, true, &[]).unwrap();
568        assert_eq!(atlas.files().count(), 4);
569        assert_eq!(atlas.excluded_test_files, 2);
570        assert_eq!(atlas.excluded_hidden_files, 0);
571
572        let atlas = scan_repository(directory.path(), Metric::Loc, true, true, &[]).unwrap();
573        assert_eq!(atlas.files().count(), 6);
574        assert_eq!(atlas.excluded_test_files, 0);
575        assert_eq!(atlas.excluded_hidden_files, 0);
576    }
577
578    #[test]
579    fn scanner_excludes_repeatable_repository_relative_paths() {
580        let directory = tempfile::tempdir().unwrap();
581        fs::create_dir_all(directory.path().join("assets/src")).unwrap();
582        fs::create_dir_all(directory.path().join("lib")).unwrap();
583        fs::write(directory.path().join("assets/src/app.ts"), "run();\n").unwrap();
584        fs::write(directory.path().join("assets/logo.svg"), "<svg/>\n").unwrap();
585        fs::write(
586            directory.path().join("lib/app.ex"),
587            "defmodule App do\nend\n",
588        )
589        .unwrap();
590        assert!(
591            Command::new("git")
592                .args(["init", "--quiet"])
593                .arg(directory.path())
594                .status()
595                .unwrap()
596                .success()
597        );
598        assert!(
599            Command::new("git")
600                .args(["-C"])
601                .arg(directory.path())
602                .args(["add", "."])
603                .status()
604                .unwrap()
605                .success()
606        );
607
608        let atlas = scan_repository(
609            directory.path(),
610            Metric::Loc,
611            false,
612            false,
613            &["assets".to_owned()],
614        )
615        .unwrap();
616        assert_eq!(atlas.files().count(), 1);
617        assert_eq!(atlas.excluded_custom_files, 2);
618        assert_eq!(atlas.excluded_paths, vec!["assets"]);
619        assert!(atlas.path_to_id.contains_key("lib/app.ex"));
620    }
621
622    #[test]
623    fn scanner_skips_tracked_files_deleted_from_the_worktree() {
624        let directory = tempfile::tempdir().unwrap();
625        fs::write(directory.path().join("present.rs"), "fn present() {}\n").unwrap();
626        fs::write(directory.path().join("deleted.rs"), "fn deleted() {}\n").unwrap();
627        assert!(
628            Command::new("git")
629                .args(["init", "--quiet"])
630                .arg(directory.path())
631                .status()
632                .unwrap()
633                .success()
634        );
635        assert!(
636            Command::new("git")
637                .args(["-C"])
638                .arg(directory.path())
639                .args(["add", "."])
640                .status()
641                .unwrap()
642                .success()
643        );
644        fs::remove_file(directory.path().join("deleted.rs")).unwrap();
645
646        let atlas = scan_repository(directory.path(), Metric::Loc, false, false, &[]).unwrap();
647
648        assert_eq!(atlas.files().count(), 1);
649        assert!(atlas.path_to_id.contains_key("present.rs"));
650        assert!(!atlas.path_to_id.contains_key("deleted.rs"));
651        assert!(atlas.dirty);
652    }
653}