diffler_core/
stats.rs

1//! Lines per language for the working tree, and churn per language for a
2//! review.
3
4use std::collections::HashMap;
5use std::path::{Path, PathBuf};
6
7use crate::classify::{Kind, Rules};
8use crate::git::GitVcs;
9use crate::language;
10use crate::model::FileDiff;
11use crate::review::ReviewError;
12use crate::vcs::Vcs;
13
14/// We skip files past this size so a checked-in data dump never dominates the
15/// breakdown.
16const MAX_FILE_BYTES: u64 = 4 * 1024 * 1024;
17
18#[derive(Debug, Clone, PartialEq, Eq)]
19pub struct LanguageCount {
20    pub name: &'static str,
21    pub color: language::Rgb,
22    pub files: usize,
23    pub lines: usize,
24    pub code: usize,
25    pub comments: usize,
26    pub blanks: usize,
27    pub bytes: u64,
28}
29
30#[derive(Debug, Clone, Default, PartialEq, Eq)]
31pub struct RepoStats {
32    /// Heaviest first.
33    pub languages: Vec<LanguageCount>,
34    pub unknown_files: usize,
35    /// Binary, oversized or unreadable.
36    pub skipped_files: usize,
37    /// Lockfiles included.
38    pub generated_files: usize,
39}
40
41impl RepoStats {
42    #[must_use]
43    pub fn totals(&self) -> LanguageCount {
44        let mut total = LanguageCount {
45            name: "total",
46            color: (0, 0, 0),
47            files: 0,
48            lines: 0,
49            code: 0,
50            comments: 0,
51            blanks: 0,
52            bytes: 0,
53        };
54        for language in &self.languages {
55            total.files += language.files;
56            total.lines += language.lines;
57            total.code += language.code;
58            total.comments += language.comments;
59            total.blanks += language.blanks;
60            total.bytes += language.bytes;
61        }
62        total
63    }
64}
65
66/// Count `paths` by language, ordered by code lines. We leave out what `rules`
67/// calls generated, since a lockfile would outrank the code beside it.
68#[must_use]
69pub fn scan(root: &Path, paths: &[PathBuf], rules: &Rules) -> RepoStats {
70    let mut stats = RepoStats::default();
71    let mut counts: HashMap<&'static str, LanguageCount> = HashMap::new();
72    for path in paths {
73        let relative = path.to_string_lossy();
74        if rules.kind(&relative, None) == Kind::Generated {
75            stats.generated_files += 1;
76            continue;
77        }
78        let full = root.join(path);
79        let Ok(metadata) = std::fs::metadata(&full) else {
80            stats.skipped_files += 1;
81            continue;
82        };
83        if !metadata.is_file() || metadata.len() > MAX_FILE_BYTES {
84            stats.skipped_files += 1;
85            continue;
86        }
87        let Ok(bytes) = std::fs::read(&full) else {
88            stats.skipped_files += 1;
89            continue;
90        };
91        if is_binary(&bytes) {
92            stats.skipped_files += 1;
93            continue;
94        }
95        let text = String::from_utf8_lossy(&bytes);
96        let Some(language) = language::of_path(&relative) else {
97            stats.unknown_files += 1;
98            continue;
99        };
100        let (code, comments, blanks) = language::count_lines(&text, Some(language));
101        let entry = counts.entry(language.name).or_insert(LanguageCount {
102            name: language.name,
103            color: language.color,
104            files: 0,
105            lines: 0,
106            code: 0,
107            comments: 0,
108            blanks: 0,
109            bytes: 0,
110        });
111        entry.files += 1;
112        entry.lines += code + comments + blanks;
113        entry.code += code;
114        entry.comments += comments;
115        entry.blanks += blanks;
116        entry.bytes += metadata.len();
117    }
118    // we break ties by name so two scans of one tree order the same
119    let mut languages: Vec<LanguageCount> = counts.into_values().collect();
120    languages.sort_by(|a, b| b.code.cmp(&a.code).then_with(|| a.name.cmp(b.name)));
121    stats.languages = languages;
122    stats
123}
124
125/// [`scan`] over the index plus `extra` untracked paths, index read included,
126/// on the caller's thread.
127pub fn scan_repo(
128    repo_root: &Path,
129    extra: &[String],
130    rules: &Rules,
131) -> Result<RepoStats, ReviewError> {
132    let vcs = GitVcs::open(repo_root)?;
133    let mut paths = vcs.tracked_files()?;
134    paths.extend(extra.iter().map(PathBuf::from));
135    Ok(scan(repo_root, &paths, rules))
136}
137
138/// git's own rule: a NUL byte in the first 8000 bytes.
139fn is_binary(bytes: &[u8]) -> bool {
140    bytes.iter().take(8000).any(|byte| *byte == 0)
141}
142
143#[derive(Debug, Clone, Copy, PartialEq, Eq)]
144pub struct LanguageChurn {
145    pub name: &'static str,
146    pub color: language::Rgb,
147    pub files: usize,
148    pub added: usize,
149    pub deleted: usize,
150}
151
152impl LanguageChurn {
153    #[must_use]
154    pub fn churn(&self) -> usize {
155        self.added + self.deleted
156    }
157}
158
159/// Churn per language, busiest first, leaving out files in no known language.
160pub fn review_mix<'a>(files: impl IntoIterator<Item = &'a FileDiff>) -> Vec<LanguageChurn> {
161    let mut totals: HashMap<&'static str, LanguageChurn> = HashMap::new();
162    for file in files {
163        let Some(language) = language::of_path(&file.path) else {
164            continue;
165        };
166        let (added, deleted) = file.diffstat();
167        let entry = totals.entry(language.name).or_insert(LanguageChurn {
168            name: language.name,
169            color: language.color,
170            files: 0,
171            added: 0,
172            deleted: 0,
173        });
174        entry.files += 1;
175        entry.added += added;
176        entry.deleted += deleted;
177    }
178    let mut mix: Vec<LanguageChurn> = totals.into_values().collect();
179    mix.sort_by(|a, b| b.churn().cmp(&a.churn()).then_with(|| a.name.cmp(b.name)));
180    mix
181}
182
183#[cfg(test)]
184mod tests {
185    use super::*;
186    use crate::model::{BlobIds, DiffLine, FileStatus, HashCache, Hunk, HunkId, LineKind};
187
188    fn file(path: &str, added: usize, deleted: usize) -> FileDiff {
189        let mut lines = Vec::new();
190        for _ in 0..added {
191            lines.push(DiffLine {
192                kind: LineKind::Added,
193                old_no: None,
194                new_no: Some(1),
195                text: "x".into(),
196                emphasis: Vec::new(),
197                reformat_only: false,
198            });
199        }
200        for _ in 0..deleted {
201            lines.push(DiffLine {
202                kind: LineKind::Deleted,
203                old_no: Some(1),
204                new_no: None,
205                text: "y".into(),
206                emphasis: Vec::new(),
207                reformat_only: false,
208            });
209        }
210        FileDiff {
211            path: path.to_owned(),
212            old_path: None,
213            status: FileStatus::Modified,
214            binary: false,
215            old_text: None,
216            new_text: None,
217            hunks: vec![Hunk {
218                id: HunkId(String::new()),
219                old_start: 1,
220                old_lines: 1,
221                new_start: 1,
222                new_lines: 1,
223                context: String::new(),
224                lines,
225            }],
226            hashes: HashCache::default(),
227            blobs: BlobIds::default(),
228        }
229    }
230
231    #[test]
232    fn the_review_mix_groups_churn_by_language_busiest_first() {
233        let files = vec![
234            file("README.md", 3, 1),
235            file("src/main.rs", 40, 10),
236            file("src/lib.rs", 5, 5),
237            file("data.bin", 99, 0),
238        ];
239        let mix = review_mix(&files);
240        let shape: Vec<(&str, usize, usize, usize)> = mix
241            .iter()
242            .map(|entry| (entry.name, entry.files, entry.added, entry.deleted))
243            .collect();
244        assert_eq!(
245            shape,
246            vec![("Rust", 2, 45, 15), ("Markdown", 1, 3, 1)],
247            "two rust files fold into one row; the binary has no language"
248        );
249    }
250
251    #[test]
252    fn scanning_counts_each_language_and_reports_what_it_left_out() {
253        let dir = tempfile::tempdir().expect("tempdir");
254        let root = dir.path();
255        std::fs::create_dir_all(root.join("src")).expect("mkdir");
256        std::fs::write(
257            root.join("src/main.rs"),
258            "// note\nfn main() {\n\n    let x = 1;\n}\n",
259        )
260        .expect("write");
261        std::fs::write(root.join("run.sh"), "#!/bin/sh\n# comment\necho hi\n").expect("write");
262        std::fs::write(root.join("logo.bin"), [0u8, 1, 2, 3]).expect("write");
263        std::fs::write(root.join("mystery.qqq"), "some text\n").expect("write");
264
265        let paths = [
266            PathBuf::from("src/main.rs"),
267            PathBuf::from("run.sh"),
268            PathBuf::from("logo.bin"),
269            PathBuf::from("mystery.qqq"),
270            PathBuf::from("gone.rs"),
271        ];
272        let stats = scan(root, &paths, &Rules::default());
273
274        let shape: Vec<(&str, usize, usize, usize, usize)> = stats
275            .languages
276            .iter()
277            .map(|l| (l.name, l.files, l.code, l.comments, l.blanks))
278            .collect();
279        assert_eq!(shape, vec![("Rust", 1, 3, 1, 1), ("Shell", 1, 2, 1, 0)]);
280        assert_eq!(stats.unknown_files, 1, "the .qqq file has no language");
281        assert_eq!(stats.skipped_files, 2, "the binary and the missing path");
282        assert_eq!(stats.totals().lines, 8);
283    }
284
285    #[test]
286    fn scan_repo_counts_the_index_and_the_untracked_files_it_is_given() {
287        let dir = tempfile::tempdir().expect("tempdir");
288        let root = dir.path();
289        let repo = crate::test_git::init_repo(root, Some("main"));
290        std::fs::write(root.join("tracked.rs"), "fn main() {}\n").expect("write");
291        let time = git2::Time::new(1_700_000_000, 0);
292        let sig = git2::Signature::new("test", "test@test", &time).expect("sig");
293        crate::test_git::commit_all(&repo, "base", &sig);
294        std::fs::write(root.join("fresh.py"), "print(1)\n").expect("write");
295
296        let counted = scan_repo(root, &[], &Rules::default()).expect("scan");
297        assert_eq!(
298            counted.languages.iter().map(|l| l.name).collect::<Vec<_>>(),
299            vec!["Rust"],
300            "the index alone knows nothing of fresh.py"
301        );
302
303        let both = scan_repo(root, &["fresh.py".to_owned()], &Rules::default()).expect("scan");
304        assert_eq!(
305            both.languages.iter().map(|l| l.name).collect::<Vec<_>>(),
306            vec!["Python", "Rust"],
307            "python outranks rust by a line, and the untracked file counts"
308        );
309    }
310
311    #[test]
312    fn an_empty_scan_totals_zero() {
313        let dir = tempfile::tempdir().expect("tempdir");
314        let stats = scan(dir.path(), &[], &Rules::default());
315        assert!(stats.languages.is_empty());
316        assert_eq!(stats.totals().code, 0);
317    }
318}
diffler_core/stats.rs

diffler_core/
stats.rs
