fallow_engine/
duplicates.rs

1//! Duplication result types exposed through the engine boundary.
2
3use std::path::{Path, PathBuf};
4
5use fallow_config::DuplicatesConfig;
6use fallow_types::discover::DiscoveredFile;
7use rustc_hash::{FxHashMap, FxHashSet};
8
9use crate::results::DuplicationAnalysis;
10
11#[path = "duplication_detector/mod.rs"]
12mod detector;
13
14#[cfg(test)]
15pub(crate) use detector::token_types;
16pub(crate) use detector::types;
17
18/// Detector internals re-exported for the engine's own benches and
19/// integration tests; not part of the supported engine API surface.
20#[doc(hidden)]
21pub use detector::{detect, normalize, tokenize};
22
23/// Engine alias for [`fallow_types::duplicates::CloneGroup`].
24pub type CloneGroup = fallow_types::duplicates::CloneGroup;
25/// Engine alias for [`fallow_types::duplicates::CloneGroupKind`].
26pub type CloneGroupKind = fallow_types::duplicates::CloneGroupKind;
27/// Engine alias for [`fallow_types::duplicates::CloneInstance`].
28pub type CloneInstance = fallow_types::duplicates::CloneInstance;
29/// Engine alias for [`fallow_types::duplicates::DefaultIgnoreSkips`].
30pub type DefaultIgnoreSkips = fallow_types::duplicates::DefaultIgnoreSkips;
31/// Engine alias for [`fallow_types::duplicates::DuplicationReport`].
32pub type DuplicationReport = fallow_types::duplicates::DuplicationReport;
33/// Engine alias for [`fallow_types::duplicates::DuplicationStats`].
34pub type DuplicationStats = fallow_types::duplicates::DuplicationStats;
35/// Engine alias for [`fallow_types::duplicates::RefactoringKind`].
36pub type RefactoringKind = fallow_types::duplicates::RefactoringKind;
37/// Engine alias for [`fallow_types::duplicates::RefactoringSuggestion`].
38pub type RefactoringSuggestion = fallow_types::duplicates::RefactoringSuggestion;
39
40pub use detector::{
41    CloneFingerprintKey, CloneFingerprintSet, FINGERPRINT_PREFIX, clone_fingerprint,
42    dominant_identifier, fingerprint_for_fragment, group_refactoring_suggestion,
43};
44
45/// Refresh clone-family and mirrored-directory fields after clone groups change.
46pub fn refresh_clone_families(report: &mut DuplicationReport, root: &Path) {
47    report.clone_families = detector::families::group_into_families(&report.clone_groups, root);
48    report.mirrored_directories =
49        detector::families::detect_mirrored_directories(&report.clone_families, root);
50}
51
52/// Refresh near-clone metrics after a caller filters group instances.
53#[doc(hidden)]
54pub fn refresh_clone_group_metrics(group: &mut CloneGroup) {
55    detector::refresh_near_group_metrics(group);
56}
57
58/// Recompute duplication statistics after clone groups have been filtered.
59///
60/// Uses per-file line deduplication, matching the detector's stats model, so
61/// overlapping clone instances do not inflate the duplicated line count.
62#[must_use]
63pub fn recompute_stats(report: &DuplicationReport) -> DuplicationStats {
64    let mut files_with_clones: FxHashSet<&Path> = FxHashSet::default();
65    let mut file_dup_lines: FxHashMap<&Path, FxHashSet<usize>> = FxHashMap::default();
66    let mut duplicated_tokens = 0usize;
67    let mut clone_instances = 0usize;
68
69    for group in &report.clone_groups {
70        for instance in &group.instances {
71            files_with_clones.insert(&instance.file);
72            clone_instances += 1;
73            let lines = file_dup_lines.entry(&instance.file).or_default();
74            for line in instance.start_line..=instance.end_line {
75                lines.insert(line);
76            }
77        }
78        duplicated_tokens += group.token_count * group.instances.len().saturating_sub(1);
79    }
80
81    let duplicated_lines: usize = file_dup_lines.values().map(FxHashSet::len).sum();
82
83    DuplicationStats {
84        total_files: report.stats.total_files,
85        files_with_clones: files_with_clones.len(),
86        total_lines: report.stats.total_lines,
87        duplicated_lines,
88        total_tokens: report.stats.total_tokens,
89        duplicated_tokens: duplicated_tokens.min(report.stats.total_tokens),
90        clone_groups: report.clone_groups.len(),
91        clone_instances,
92        duplication_percentage: if report.stats.total_lines > 0 {
93            (duplicated_lines as f64 / report.stats.total_lines as f64) * 100.0
94        } else {
95            0.0
96        },
97        clone_groups_below_min_occurrences: report.stats.clone_groups_below_min_occurrences,
98        clone_groups_ignored: report.stats.clone_groups_ignored,
99        near_candidates_skipped: report.stats.near_candidates_skipped,
100    }
101}
102
103/// Compare two JS/TS sources by duplicate-token kind sequence.
104///
105/// This keeps CLI audit's non-behavioral change check from depending on the
106/// tokenizer module shape.
107#[must_use]
108pub fn source_token_kinds_equivalent(
109    path: &Path,
110    current: &str,
111    base: &str,
112    cross_language: bool,
113) -> bool {
114    let current_tokens = detector::tokenize::tokenize_file(path, current, cross_language);
115    let base_tokens = detector::tokenize::tokenize_file(path, base, cross_language);
116    current_tokens
117        .tokens
118        .iter()
119        .map(|token| &token.kind)
120        .eq(base_tokens.tokens.iter().map(|token| &token.kind))
121}
122
123/// Run duplication detection on a discovered file set.
124#[must_use]
125pub fn find_duplicates(
126    root: &Path,
127    files: &[DiscoveredFile],
128    config: &DuplicatesConfig,
129) -> DuplicationReport {
130    detector::find_duplicates(root, files, config)
131}
132
133/// Run cached duplication detection inside the engine boundary.
134#[must_use]
135pub(crate) fn find_duplicates_cached(
136    root: &Path,
137    files: &[DiscoveredFile],
138    config: &DuplicatesConfig,
139    cache_dir: &Path,
140) -> DuplicationReport {
141    detector::find_duplicates_cached(root, files, config, cache_dir)
142}
143
144/// Run duplication detection and include metadata about built-in ignored files.
145#[must_use]
146pub fn find_duplicates_with_defaults(
147    root: &Path,
148    files: &[DiscoveredFile],
149    config: &DuplicatesConfig,
150    cache_dir: Option<&Path>,
151) -> DuplicationAnalysis {
152    let (report, default_ignore_skips) = if let Some(cache_dir) = cache_dir {
153        detector::find_duplicates_cached_with_default_ignore_skips(root, files, config, cache_dir)
154    } else {
155        detector::find_duplicates_with_default_ignore_skips(root, files, config)
156    };
157    DuplicationAnalysis {
158        report,
159        default_ignore_skips,
160    }
161}
162
163/// Run focused duplication detection and include metadata about built-in ignored files.
164#[must_use]
165pub fn find_duplicates_touching_files_with_defaults(
166    root: &Path,
167    files: &[DiscoveredFile],
168    config: &DuplicatesConfig,
169    changed_files: &[PathBuf],
170    cache_dir: Option<&Path>,
171) -> DuplicationAnalysis {
172    let changed_files = changed_files.iter().cloned().collect::<FxHashSet<_>>();
173    let (report, default_ignore_skips) = if let Some(cache_dir) = cache_dir {
174        detector::find_duplicates_touching_files_cached_with_default_ignore_skips(
175            root,
176            files,
177            config,
178            &changed_files,
179            cache_dir,
180        )
181    } else {
182        detector::find_duplicates_touching_files_with_default_ignore_skips(
183            root,
184            files,
185            config,
186            &changed_files,
187        )
188    };
189    DuplicationAnalysis {
190        report,
191        default_ignore_skips,
192    }
193}
194
195#[cfg(test)]
196mod tests {
197    use std::path::PathBuf;
198
199    use super::*;
200
201    fn instance(file: &str, start_line: usize, end_line: usize) -> CloneInstance {
202        CloneInstance {
203            file: PathBuf::from(file),
204            start_line,
205            end_line,
206            start_col: 0,
207            end_col: 0,
208            fragment: String::new(),
209        }
210    }
211
212    fn report(clone_groups: Vec<CloneGroup>) -> DuplicationReport {
213        DuplicationReport {
214            clone_groups,
215            clone_families: Vec::new(),
216            mirrored_directories: Vec::new(),
217            stats: DuplicationStats {
218                total_files: 3,
219                total_lines: 100,
220                total_tokens: 1_000,
221                clone_groups_below_min_occurrences: 4,
222                ..DuplicationStats::default()
223            },
224        }
225    }
226
227    #[test]
228    fn recompute_stats_deduplicates_overlapping_lines_per_file() {
229        let report = report(vec![
230            CloneGroup {
231                instances: vec![instance("src/a.ts", 1, 10), instance("src/b.ts", 20, 24)],
232                token_count: 30,
233                line_count: 10,
234                similarity: None,
235            },
236            CloneGroup {
237                instances: vec![instance("src/a.ts", 5, 12), instance("src/c.ts", 40, 44)],
238                token_count: 20,
239                line_count: 8,
240                similarity: None,
241            },
242        ]);
243
244        let stats = recompute_stats(&report);
245
246        assert_eq!(stats.total_files, 3);
247        assert_eq!(stats.files_with_clones, 3);
248        assert_eq!(stats.total_lines, 100);
249        assert_eq!(stats.duplicated_lines, 22);
250        assert_eq!(stats.total_tokens, 1_000);
251        assert_eq!(stats.duplicated_tokens, 50);
252        assert_eq!(stats.clone_groups, 2);
253        assert_eq!(stats.clone_instances, 4);
254        assert!((stats.duplication_percentage - 22.0).abs() < f64::EPSILON);
255        assert_eq!(stats.clone_groups_below_min_occurrences, 4);
256    }
257
258    #[test]
259    fn recompute_stats_handles_zero_total_lines() {
260        let mut report = report(vec![CloneGroup {
261            instances: vec![instance("src/a.ts", 1, 1)],
262            token_count: 5,
263            line_count: 1,
264            similarity: None,
265        }]);
266        report.stats.total_lines = 0;
267
268        let stats = recompute_stats(&report);
269
270        assert_eq!(stats.duplicated_lines, 1);
271        assert!(stats.duplication_percentage.abs() < f64::EPSILON);
272    }
273
274    #[test]
275    fn clone_fingerprint_set_delegates_without_leaking_core_type() {
276        let groups = vec![CloneGroup {
277            instances: vec![
278                CloneInstance {
279                    fragment: "const value = 1;".to_string(),
280                    ..instance("src/a.ts", 1, 1)
281                },
282                CloneInstance {
283                    fragment: "const value = 1;".to_string(),
284                    ..instance("src/b.ts", 2, 2)
285                },
286            ],
287            token_count: 5,
288            line_count: 1,
289            similarity: None,
290        }];
291        let fingerprints = CloneFingerprintSet::from_groups(&groups);
292        let fingerprint = fingerprints.fingerprint_for_group(&groups[0]);
293
294        assert!(fingerprint.starts_with(FINGERPRINT_PREFIX));
295        assert!(fingerprints.find_group(&groups, &fingerprint).is_some());
296    }
297}
fallow_engine/duplicates.rs

fallow_engine/
duplicates.rs
