fallow_engine/
duplicates.rs

1//! Duplication result types exposed through the engine boundary.
2
3use std::path::{Path, PathBuf};
4
5use fallow_config::DuplicatesConfig;
6use fallow_types::discover::DiscoveredFile;
7use rustc_hash::{FxHashMap, FxHashSet};
8
9use crate::results::DuplicationAnalysis;
10
11#[path = "duplication_detector/mod.rs"]
12mod detector;
13
14#[cfg(test)]
15pub(crate) use detector::token_types;
16pub(crate) use detector::types;
17
18/// Detector internals re-exported for the engine's own benches and
19/// integration tests; not part of the supported engine API surface.
20#[doc(hidden)]
21pub use detector::{detect, normalize, tokenize};
22
23pub type CloneGroup = fallow_types::duplicates::CloneGroup;
24pub type CloneInstance = fallow_types::duplicates::CloneInstance;
25pub type DefaultIgnoreSkips = fallow_types::duplicates::DefaultIgnoreSkips;
26pub type DuplicationReport = fallow_types::duplicates::DuplicationReport;
27pub type DuplicationStats = fallow_types::duplicates::DuplicationStats;
28pub type RefactoringKind = fallow_types::duplicates::RefactoringKind;
29pub type RefactoringSuggestion = fallow_types::duplicates::RefactoringSuggestion;
30
31pub use detector::{
32    CloneFingerprintKey, CloneFingerprintSet, FINGERPRINT_PREFIX, clone_fingerprint,
33    dominant_identifier, fingerprint_for_fragment, group_refactoring_suggestion,
34};
35
36/// Refresh clone-family and mirrored-directory fields after clone groups change.
37pub fn refresh_clone_families(report: &mut DuplicationReport, root: &Path) {
38    report.clone_families = detector::families::group_into_families(&report.clone_groups, root);
39    report.mirrored_directories =
40        detector::families::detect_mirrored_directories(&report.clone_families, root);
41}
42
43/// Recompute duplication statistics after clone groups have been filtered.
44///
45/// Uses per-file line deduplication, matching the detector's stats model, so
46/// overlapping clone instances do not inflate the duplicated line count.
47#[must_use]
48pub fn recompute_stats(report: &DuplicationReport) -> DuplicationStats {
49    let mut files_with_clones: FxHashSet<&Path> = FxHashSet::default();
50    let mut file_dup_lines: FxHashMap<&Path, FxHashSet<usize>> = FxHashMap::default();
51    let mut duplicated_tokens = 0usize;
52    let mut clone_instances = 0usize;
53
54    for group in &report.clone_groups {
55        for instance in &group.instances {
56            files_with_clones.insert(&instance.file);
57            clone_instances += 1;
58            let lines = file_dup_lines.entry(&instance.file).or_default();
59            for line in instance.start_line..=instance.end_line {
60                lines.insert(line);
61            }
62        }
63        duplicated_tokens += group.token_count * group.instances.len();
64    }
65
66    let duplicated_lines: usize = file_dup_lines.values().map(FxHashSet::len).sum();
67
68    DuplicationStats {
69        total_files: report.stats.total_files,
70        files_with_clones: files_with_clones.len(),
71        total_lines: report.stats.total_lines,
72        duplicated_lines,
73        total_tokens: report.stats.total_tokens,
74        duplicated_tokens,
75        clone_groups: report.clone_groups.len(),
76        clone_instances,
77        duplication_percentage: if report.stats.total_lines > 0 {
78            (duplicated_lines as f64 / report.stats.total_lines as f64) * 100.0
79        } else {
80            0.0
81        },
82        clone_groups_below_min_occurrences: report.stats.clone_groups_below_min_occurrences,
83    }
84}
85
86/// Compare two JS/TS sources by duplicate-token kind sequence.
87///
88/// This keeps CLI audit's non-behavioral change check from depending on the
89/// tokenizer module shape.
90#[must_use]
91pub fn source_token_kinds_equivalent(
92    path: &Path,
93    current: &str,
94    base: &str,
95    cross_language: bool,
96) -> bool {
97    let current_tokens = detector::tokenize::tokenize_file(path, current, cross_language);
98    let base_tokens = detector::tokenize::tokenize_file(path, base, cross_language);
99    current_tokens
100        .tokens
101        .iter()
102        .map(|token| &token.kind)
103        .eq(base_tokens.tokens.iter().map(|token| &token.kind))
104}
105
106/// Run duplication detection on a discovered file set.
107#[must_use]
108pub fn find_duplicates(
109    root: &Path,
110    files: &[DiscoveredFile],
111    config: &DuplicatesConfig,
112) -> DuplicationReport {
113    detector::find_duplicates(root, files, config)
114}
115
116/// Run cached duplication detection inside the engine boundary.
117#[must_use]
118pub fn find_duplicates_cached(
119    root: &Path,
120    files: &[DiscoveredFile],
121    config: &DuplicatesConfig,
122    cache_dir: &Path,
123) -> DuplicationReport {
124    detector::find_duplicates_cached(root, files, config, cache_dir)
125}
126
127/// Run duplication detection and include metadata about built-in ignored files.
128#[must_use]
129pub fn find_duplicates_with_defaults(
130    root: &Path,
131    files: &[DiscoveredFile],
132    config: &DuplicatesConfig,
133    cache_dir: Option<&Path>,
134) -> DuplicationAnalysis {
135    let (report, default_ignore_skips) = if let Some(cache_dir) = cache_dir {
136        detector::find_duplicates_cached_with_default_ignore_skips(root, files, config, cache_dir)
137    } else {
138        detector::find_duplicates_with_default_ignore_skips(root, files, config)
139    };
140    DuplicationAnalysis {
141        report,
142        default_ignore_skips,
143    }
144}
145
146/// Run focused duplication detection and include metadata about built-in ignored files.
147#[must_use]
148pub fn find_duplicates_touching_files_with_defaults(
149    root: &Path,
150    files: &[DiscoveredFile],
151    config: &DuplicatesConfig,
152    changed_files: &[PathBuf],
153    cache_dir: Option<&Path>,
154) -> DuplicationAnalysis {
155    let changed_files = changed_files.iter().cloned().collect::<FxHashSet<_>>();
156    let (report, default_ignore_skips) = if let Some(cache_dir) = cache_dir {
157        detector::find_duplicates_touching_files_cached_with_default_ignore_skips(
158            root,
159            files,
160            config,
161            &changed_files,
162            cache_dir,
163        )
164    } else {
165        detector::find_duplicates_touching_files_with_default_ignore_skips(
166            root,
167            files,
168            config,
169            &changed_files,
170        )
171    };
172    DuplicationAnalysis {
173        report,
174        default_ignore_skips,
175    }
176}
177
178#[cfg(test)]
179mod tests {
180    use std::path::PathBuf;
181
182    use super::*;
183
184    fn instance(file: &str, start_line: usize, end_line: usize) -> CloneInstance {
185        CloneInstance {
186            file: PathBuf::from(file),
187            start_line,
188            end_line,
189            start_col: 0,
190            end_col: 0,
191            fragment: String::new(),
192        }
193    }
194
195    fn report(clone_groups: Vec<CloneGroup>) -> DuplicationReport {
196        DuplicationReport {
197            clone_groups,
198            clone_families: Vec::new(),
199            mirrored_directories: Vec::new(),
200            stats: DuplicationStats {
201                total_files: 3,
202                total_lines: 100,
203                total_tokens: 1_000,
204                clone_groups_below_min_occurrences: 4,
205                ..DuplicationStats::default()
206            },
207        }
208    }
209
210    #[test]
211    fn recompute_stats_deduplicates_overlapping_lines_per_file() {
212        let report = report(vec![
213            CloneGroup {
214                instances: vec![instance("src/a.ts", 1, 10), instance("src/b.ts", 20, 24)],
215                token_count: 30,
216                line_count: 10,
217            },
218            CloneGroup {
219                instances: vec![instance("src/a.ts", 5, 12), instance("src/c.ts", 40, 44)],
220                token_count: 20,
221                line_count: 8,
222            },
223        ]);
224
225        let stats = recompute_stats(&report);
226
227        assert_eq!(stats.total_files, 3);
228        assert_eq!(stats.files_with_clones, 3);
229        assert_eq!(stats.total_lines, 100);
230        assert_eq!(stats.duplicated_lines, 22);
231        assert_eq!(stats.total_tokens, 1_000);
232        assert_eq!(stats.duplicated_tokens, 100);
233        assert_eq!(stats.clone_groups, 2);
234        assert_eq!(stats.clone_instances, 4);
235        assert!((stats.duplication_percentage - 22.0).abs() < f64::EPSILON);
236        assert_eq!(stats.clone_groups_below_min_occurrences, 4);
237    }
238
239    #[test]
240    fn recompute_stats_handles_zero_total_lines() {
241        let mut report = report(vec![CloneGroup {
242            instances: vec![instance("src/a.ts", 1, 1)],
243            token_count: 5,
244            line_count: 1,
245        }]);
246        report.stats.total_lines = 0;
247
248        let stats = recompute_stats(&report);
249
250        assert_eq!(stats.duplicated_lines, 1);
251        assert!(stats.duplication_percentage.abs() < f64::EPSILON);
252    }
253
254    #[test]
255    fn clone_fingerprint_set_delegates_without_leaking_core_type() {
256        let groups = vec![CloneGroup {
257            instances: vec![
258                CloneInstance {
259                    fragment: "const value = 1;".to_string(),
260                    ..instance("src/a.ts", 1, 1)
261                },
262                CloneInstance {
263                    fragment: "const value = 1;".to_string(),
264                    ..instance("src/b.ts", 2, 2)
265                },
266            ],
267            token_count: 5,
268            line_count: 1,
269        }];
270        let fingerprints = CloneFingerprintSet::from_groups(&groups);
271        let fingerprint = fingerprints.fingerprint_for_group(&groups[0]);
272
273        assert!(fingerprint.starts_with(FINGERPRINT_PREFIX));
274        assert!(fingerprints.find_group(&groups, &fingerprint).is_some());
275    }
276}
fallow_engine/duplicates.rs

fallow_engine/
duplicates.rs
