fallow_engine/
duplicates.rs

1//! Duplication result types exposed through the engine boundary.
2
3use std::path::{Path, PathBuf};
4
5use fallow_config::DuplicatesConfig;
6use fallow_types::discover::DiscoveredFile;
7use rustc_hash::{FxHashMap, FxHashSet};
8
9use crate::results::DuplicationAnalysis;
10
11#[path = "duplication_detector/mod.rs"]
12mod detector;
13
14#[cfg(test)]
15pub(crate) use detector::token_types;
16pub(crate) use detector::types;
17
18/// Detector internals re-exported for the engine's own benches and
19/// integration tests; not part of the supported engine API surface.
20#[doc(hidden)]
21pub use detector::{detect, normalize, tokenize};
22
23/// Engine alias for [`fallow_types::duplicates::CloneGroup`].
24pub type CloneGroup = fallow_types::duplicates::CloneGroup;
25/// Engine alias for [`fallow_types::duplicates::CloneGroupKind`].
26pub type CloneGroupKind = fallow_types::duplicates::CloneGroupKind;
27/// Engine alias for [`fallow_types::duplicates::CloneInstance`].
28pub type CloneInstance = fallow_types::duplicates::CloneInstance;
29/// Engine alias for [`fallow_types::duplicates::DefaultIgnoreSkips`].
30pub type DefaultIgnoreSkips = fallow_types::duplicates::DefaultIgnoreSkips;
31/// Engine alias for [`fallow_types::duplicates::DuplicationReport`].
32pub type DuplicationReport = fallow_types::duplicates::DuplicationReport;
33/// Engine alias for [`fallow_types::duplicates::DuplicationStats`].
34pub type DuplicationStats = fallow_types::duplicates::DuplicationStats;
35/// Engine alias for [`fallow_types::duplicates::RefactoringKind`].
36pub type RefactoringKind = fallow_types::duplicates::RefactoringKind;
37/// Engine alias for [`fallow_types::duplicates::RefactoringSuggestion`].
38pub type RefactoringSuggestion = fallow_types::duplicates::RefactoringSuggestion;
39
40pub use detector::{
41    CloneFingerprintKey, CloneFingerprintSet, FINGERPRINT_PREFIX, clone_fingerprint,
42    dominant_identifier, fingerprint_for_fragment, group_refactoring_suggestion,
43};
44
45/// Refresh clone-family and mirrored-directory fields after clone groups change.
46pub fn refresh_clone_families(report: &mut DuplicationReport, root: &Path) {
47    report.clone_families = detector::families::group_into_families(&report.clone_groups, root);
48    report.mirrored_directories =
49        detector::families::detect_mirrored_directories(&report.clone_families, root);
50}
51
52/// Refresh near-clone metrics after a caller filters group instances.
53#[doc(hidden)]
54pub fn refresh_clone_group_metrics(group: &mut CloneGroup) {
55    detector::refresh_near_group_metrics(group);
56}
57
58/// Recompute duplication statistics after clone groups have been filtered.
59///
60/// Uses per-file line deduplication, matching the detector's stats model, so
61/// overlapping clone instances do not inflate the duplicated line count.
62///
63/// `clone_families` is read from `report.clone_families`, so a scope filter
64/// must call [`refresh_clone_families`] before this. A presentation cap such
65/// as `--top` must not call this at all: it truncates the arrays while `stats`
66/// keeps describing the corpus the run measured.
67#[must_use]
68pub fn recompute_stats(report: &DuplicationReport) -> DuplicationStats {
69    let mut files_with_clones: FxHashSet<&Path> = FxHashSet::default();
70    let mut file_dup_lines: FxHashMap<&Path, FxHashSet<usize>> = FxHashMap::default();
71    let mut duplicated_tokens = 0usize;
72    let mut clone_instances = 0usize;
73
74    for group in &report.clone_groups {
75        for instance in &group.instances {
76            files_with_clones.insert(&instance.file);
77            clone_instances += 1;
78            let lines = file_dup_lines.entry(&instance.file).or_default();
79            for line in instance.start_line..=instance.end_line {
80                lines.insert(line);
81            }
82        }
83        duplicated_tokens += group.token_count * group.instances.len().saturating_sub(1);
84    }
85
86    let duplicated_lines: usize = file_dup_lines.values().map(FxHashSet::len).sum();
87
88    DuplicationStats {
89        total_files: report.stats.total_files,
90        files_with_clones: files_with_clones.len(),
91        total_lines: report.stats.total_lines,
92        duplicated_lines,
93        total_tokens: report.stats.total_tokens,
94        duplicated_tokens: duplicated_tokens.min(report.stats.total_tokens),
95        clone_groups: report.clone_groups.len(),
96        clone_families: report.clone_families.len(),
97        clone_instances,
98        duplication_percentage: if report.stats.total_lines > 0 {
99            (duplicated_lines as f64 / report.stats.total_lines as f64) * 100.0
100        } else {
101            0.0
102        },
103        clone_groups_below_min_occurrences: report.stats.clone_groups_below_min_occurrences,
104        clone_groups_ignored: report.stats.clone_groups_ignored,
105        near_candidates_skipped: report.stats.near_candidates_skipped,
106    }
107}
108
109/// Compare two JS/TS sources by duplicate-token kind sequence.
110///
111/// This keeps CLI audit's non-behavioral change check from depending on the
112/// tokenizer module shape.
113#[must_use]
114pub fn source_token_kinds_equivalent(
115    path: &Path,
116    current: &str,
117    base: &str,
118    cross_language: bool,
119) -> bool {
120    let current_tokens = detector::tokenize::tokenize_file(path, current, cross_language);
121    let base_tokens = detector::tokenize::tokenize_file(path, base, cross_language);
122    current_tokens
123        .tokens
124        .iter()
125        .map(|token| &token.kind)
126        .eq(base_tokens.tokens.iter().map(|token| &token.kind))
127}
128
129/// Run duplication detection on a discovered file set.
130#[must_use]
131pub fn find_duplicates(
132    root: &Path,
133    files: &[DiscoveredFile],
134    config: &DuplicatesConfig,
135) -> DuplicationReport {
136    detector::find_duplicates(root, files, config)
137}
138
139/// Run cached duplication detection inside the engine boundary.
140#[must_use]
141pub(crate) fn find_duplicates_cached(
142    root: &Path,
143    files: &[DiscoveredFile],
144    config: &DuplicatesConfig,
145    cache_dir: &Path,
146) -> DuplicationReport {
147    detector::find_duplicates_cached(root, files, config, cache_dir)
148}
149
150/// Run duplication detection and include metadata about built-in ignored files.
151#[must_use]
152pub fn find_duplicates_with_defaults(
153    root: &Path,
154    files: &[DiscoveredFile],
155    config: &DuplicatesConfig,
156    cache_dir: Option<&Path>,
157) -> DuplicationAnalysis {
158    let (report, default_ignore_skips) = if let Some(cache_dir) = cache_dir {
159        detector::find_duplicates_cached_with_default_ignore_skips(root, files, config, cache_dir)
160    } else {
161        detector::find_duplicates_with_default_ignore_skips(root, files, config)
162    };
163    DuplicationAnalysis {
164        report,
165        default_ignore_skips,
166    }
167}
168
169/// Run focused duplication detection and include metadata about built-in ignored files.
170#[must_use]
171pub fn find_duplicates_touching_files_with_defaults(
172    root: &Path,
173    files: &[DiscoveredFile],
174    config: &DuplicatesConfig,
175    changed_files: &[PathBuf],
176    cache_dir: Option<&Path>,
177) -> DuplicationAnalysis {
178    let changed_files = changed_files.iter().cloned().collect::<FxHashSet<_>>();
179    let (report, default_ignore_skips) = if let Some(cache_dir) = cache_dir {
180        detector::find_duplicates_touching_files_cached_with_default_ignore_skips(
181            root,
182            files,
183            config,
184            &changed_files,
185            cache_dir,
186        )
187    } else {
188        detector::find_duplicates_touching_files_with_default_ignore_skips(
189            root,
190            files,
191            config,
192            &changed_files,
193        )
194    };
195    DuplicationAnalysis {
196        report,
197        default_ignore_skips,
198    }
199}
200
201#[cfg(test)]
202mod tests {
203    use std::path::PathBuf;
204
205    use super::*;
206
207    fn instance(file: &str, start_line: usize, end_line: usize) -> CloneInstance {
208        CloneInstance {
209            file: PathBuf::from(file),
210            start_line,
211            end_line,
212            start_col: 0,
213            end_col: 0,
214            fragment: String::new(),
215        }
216    }
217
218    fn report(clone_groups: Vec<CloneGroup>) -> DuplicationReport {
219        DuplicationReport {
220            clone_groups,
221            clone_families: Vec::new(),
222            mirrored_directories: Vec::new(),
223            stats: DuplicationStats {
224                total_files: 3,
225                total_lines: 100,
226                total_tokens: 1_000,
227                clone_groups_below_min_occurrences: 4,
228                ..DuplicationStats::default()
229            },
230        }
231    }
232
233    #[test]
234    fn recompute_stats_deduplicates_overlapping_lines_per_file() {
235        let report = report(vec![
236            CloneGroup {
237                instances: vec![instance("src/a.ts", 1, 10), instance("src/b.ts", 20, 24)],
238                token_count: 30,
239                line_count: 10,
240                similarity: None,
241            },
242            CloneGroup {
243                instances: vec![instance("src/a.ts", 5, 12), instance("src/c.ts", 40, 44)],
244                token_count: 20,
245                line_count: 8,
246                similarity: None,
247            },
248        ]);
249
250        let stats = recompute_stats(&report);
251
252        assert_eq!(stats.total_files, 3);
253        assert_eq!(stats.files_with_clones, 3);
254        assert_eq!(stats.total_lines, 100);
255        assert_eq!(stats.duplicated_lines, 22);
256        assert_eq!(stats.total_tokens, 1_000);
257        assert_eq!(stats.duplicated_tokens, 50);
258        assert_eq!(stats.clone_groups, 2);
259        assert_eq!(stats.clone_instances, 4);
260        assert!((stats.duplication_percentage - 22.0).abs() < f64::EPSILON);
261        assert_eq!(stats.clone_groups_below_min_occurrences, 4);
262    }
263
264    #[test]
265    fn recompute_stats_handles_zero_total_lines() {
266        let mut report = report(vec![CloneGroup {
267            instances: vec![instance("src/a.ts", 1, 1)],
268            token_count: 5,
269            line_count: 1,
270            similarity: None,
271        }]);
272        report.stats.total_lines = 0;
273
274        let stats = recompute_stats(&report);
275
276        assert_eq!(stats.duplicated_lines, 1);
277        assert!(stats.duplication_percentage.abs() < f64::EPSILON);
278    }
279
280    #[test]
281    fn clone_fingerprint_set_delegates_without_leaking_core_type() {
282        let groups = vec![CloneGroup {
283            instances: vec![
284                CloneInstance {
285                    fragment: "const value = 1;".to_string(),
286                    ..instance("src/a.ts", 1, 1)
287                },
288                CloneInstance {
289                    fragment: "const value = 1;".to_string(),
290                    ..instance("src/b.ts", 2, 2)
291                },
292            ],
293            token_count: 5,
294            line_count: 1,
295            similarity: None,
296        }];
297        let fingerprints = CloneFingerprintSet::from_groups(&groups);
298        let fingerprint = fingerprints.fingerprint_for_group(&groups[0]);
299
300        assert!(fingerprint.starts_with(FINGERPRINT_PREFIX));
301        assert!(fingerprints.find_group(&groups, &fingerprint).is_some());
302    }
303}
fallow_engine/duplicates.rs

fallow_engine/
duplicates.rs
