1use std::cmp::{Ordering, Reverse};
4use std::collections::hash_map::Entry;
5use std::path::{Component, Path, PathBuf};
6
7use rustc_hash::FxHashMap;
8use serde::{Deserialize, Serialize};
9
10use crate::serde_path;
11
12#[derive(Debug, Clone, Serialize, Deserialize)]
14#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
15pub struct CloneInstance {
16 #[serde(serialize_with = "serde_path::serialize")]
18 pub file: PathBuf,
19 pub start_line: usize,
21 pub end_line: usize,
23 pub start_col: usize,
25 pub end_col: usize,
27 #[serde(default, skip_serializing_if = "String::is_empty")]
34 pub fragment: String,
35 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
42 pub is_symlink: bool,
43}
44
45#[derive(Debug, Clone, Serialize, Deserialize)]
48#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
49pub struct CloneGroup {
50 pub instances: Vec<CloneInstance>,
52 pub token_count: usize,
54 pub line_count: usize,
56 #[serde(default, skip_serializing_if = "Option::is_none")]
59 #[cfg_attr(feature = "schema", schemars(with = "f64"))]
60 pub similarity: Option<f64>,
61}
62
63#[derive(Debug, Clone, Copy, PartialEq)]
68#[non_exhaustive]
69pub enum CloneGroupKind {
70 Exact,
72 Near {
74 similarity: f64,
76 },
77}
78
79impl CloneGroup {
80 #[must_use]
82 pub fn kind(&self) -> CloneGroupKind {
83 self.similarity
84 .map_or(CloneGroupKind::Exact, |similarity| CloneGroupKind::Near {
85 similarity,
86 })
87 }
88
89 #[must_use]
91 pub fn spread(&self) -> usize {
92 clone_group_spread(&self.instances)
93 }
94
95 pub fn strip_fragments(&mut self) {
97 for instance in &mut self.instances {
98 instance.fragment.clear();
99 }
100 }
101}
102
103const SAME_FILE_SPREAD_STEP: usize = 250;
104const MAX_RANKED_SPREAD: usize = 8;
105const SPREAD_RANK_WEIGHTS: [u64; MAX_RANKED_SPREAD + 1] = [
106 1_000_000_000,
107 1_047_319_732,
108 1_075_000_000,
109 1_094_639_463,
110 1_109_873_014,
111 1_122_319_732,
112 1_132_843_281,
113 1_141_959_195,
114 1_150_000_000,
115];
116
117#[must_use]
123pub fn clone_group_spread(instances: &[CloneInstance]) -> usize {
124 clone_location_spread(instances.iter().map(|instance| {
125 (
126 instance.file.as_path(),
127 instance.start_line,
128 instance.end_line,
129 )
130 }))
131}
132
133#[must_use]
138pub fn clone_location_spread<'a>(
139 locations: impl IntoIterator<Item = (&'a Path, usize, usize)>,
140) -> usize {
141 let mut location_count = 0;
142 let mut file_indices: FxHashMap<&'a Path, usize> = FxHashMap::default();
143 let mut by_file: Vec<FileSpread<'a>> = Vec::new();
144
145 for (file, start_line, end_line) in locations {
146 location_count += 1;
147 let next_index = by_file.len();
148 match file_indices.entry(file) {
149 Entry::Occupied(entry) => by_file[*entry.get()].include(start_line, end_line),
150 Entry::Vacant(entry) => {
151 entry.insert(next_index);
152 by_file.push(FileSpread::new(file, start_line, end_line));
153 }
154 }
155 }
156
157 if location_count < 2 {
158 return 0;
159 }
160
161 let same_file_max = by_file
162 .iter()
163 .filter(|file| file.occurrences >= 2)
164 .map(FileSpread::same_file_spread)
165 .max()
166 .unwrap_or(0);
167
168 same_file_max.max(directory_tree_diameter(&by_file))
169}
170
171struct FileSpread<'a> {
172 parent_components: Vec<Component<'a>>,
173 min_end: usize,
174 max_start: usize,
175 occurrences: usize,
176}
177
178impl<'a> FileSpread<'a> {
179 fn new(file: &'a Path, start_line: usize, end_line: usize) -> Self {
180 Self {
181 parent_components: path_parent_components(file),
182 min_end: end_line,
183 max_start: start_line,
184 occurrences: 1,
185 }
186 }
187
188 fn include(&mut self, start_line: usize, end_line: usize) {
189 self.min_end = self.min_end.min(end_line);
190 self.max_start = self.max_start.max(start_line);
191 self.occurrences += 1;
192 }
193
194 fn same_file_spread(&self) -> usize {
195 self.max_start
196 .saturating_sub(self.min_end)
197 .saturating_sub(1)
198 .div_ceil(SAME_FILE_SPREAD_STEP)
199 }
200}
201
202#[must_use]
204pub fn compare_clone_groups(left: &CloneGroup, right: &CloneGroup) -> Ordering {
205 clone_group_rank_key(left).cmp(&clone_group_rank_key(right))
206}
207
208#[cfg(test)]
209fn instance_pair_spread(left: &CloneInstance, right: &CloneInstance) -> usize {
210 if left.file == right.file {
211 return same_file_spread(left, right);
212 }
213 directory_distance(&left.file, &right.file)
214}
215
216#[cfg(test)]
217fn same_file_spread(left: &CloneInstance, right: &CloneInstance) -> usize {
218 let gap = if left.end_line < right.start_line {
219 right
220 .start_line
221 .saturating_sub(left.end_line)
222 .saturating_sub(1)
223 } else if right.end_line < left.start_line {
224 left.start_line
225 .saturating_sub(right.end_line)
226 .saturating_sub(1)
227 } else {
228 0
229 };
230 gap.div_ceil(SAME_FILE_SPREAD_STEP)
231}
232
233fn path_parent_components(path: &Path) -> Vec<Component<'_>> {
234 path.parent()
235 .unwrap_or_else(|| Path::new(""))
236 .components()
237 .collect()
238}
239
240fn directory_tree_diameter(paths: &[FileSpread<'_>]) -> usize {
241 if paths.len() < 2 {
242 return 0;
243 }
244
245 let endpoint = farthest_path(paths, 0).0;
246 farthest_path(paths, endpoint).1
247}
248
249fn farthest_path(paths: &[FileSpread<'_>], origin: usize) -> (usize, usize) {
250 paths
251 .iter()
252 .enumerate()
253 .map(|(index, path)| {
254 (
255 index,
256 component_distance(&paths[origin].parent_components, &path.parent_components),
257 )
258 })
259 .max_by_key(|&(index, distance)| (distance, index))
260 .unwrap_or((origin, 0))
261}
262
263fn component_distance(left: &[Component<'_>], right: &[Component<'_>]) -> usize {
264 let shared = left
265 .iter()
266 .zip(right)
267 .take_while(|(left, right)| left == right)
268 .count();
269 left.len()
270 .saturating_sub(shared)
271 .saturating_add(right.len().saturating_sub(shared))
272}
273
274#[cfg(test)]
275fn directory_distance(left: &Path, right: &Path) -> usize {
276 component_distance(
277 &path_parent_components(left),
278 &path_parent_components(right),
279 )
280}
281
282type CloneGroupRankKey = (
283 Reverse<u128>,
284 Reverse<usize>,
285 Reverse<usize>,
286 Reverse<usize>,
287 Reverse<usize>,
288 bool,
289 PathBuf,
290 usize,
291);
292
293fn clone_group_rank_key(group: &CloneGroup) -> CloneGroupRankKey {
294 let spread = group.spread();
295 let weight = SPREAD_RANK_WEIGHTS[spread.min(MAX_RANKED_SPREAD)];
296 let token_count = u128::try_from(group.token_count).unwrap_or(u128::MAX);
297 let instance_count = u128::try_from(group.instances.len()).unwrap_or(u128::MAX);
298 let score = token_count
299 .saturating_mul(instance_count)
300 .saturating_mul(u128::from(weight));
301 let first = group.instances.iter().min_by(|left, right| {
302 left.file
303 .cmp(&right.file)
304 .then(left.start_line.cmp(&right.start_line))
305 });
306 (
307 Reverse(score),
308 Reverse(spread),
309 Reverse(group.token_count),
310 Reverse(group.instances.len()),
311 Reverse(group.line_count),
312 first.is_none(),
313 first.map_or_else(PathBuf::new, |instance| instance.file.clone()),
314 first.map_or(0, |instance| instance.start_line),
315 )
316}
317
318fn sort_clone_groups(groups: &mut [CloneGroup]) {
319 groups.sort_by_cached_key(clone_group_rank_key);
320}
321
322#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
324#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
325pub enum RefactoringKind {
326 ExtractFunction,
328 ExtractModule,
330}
331
332#[derive(Debug, Clone, Serialize, Deserialize)]
334#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
335pub struct RefactoringSuggestion {
336 pub kind: RefactoringKind,
338 pub description: String,
340 pub estimated_savings: usize,
342}
343
344#[derive(Debug, Clone, Serialize, Deserialize)]
350#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
351pub struct CloneFamily {
352 #[serde(serialize_with = "serde_path::serialize_vec")]
354 pub files: Vec<PathBuf>,
355 pub groups: Vec<CloneGroup>,
357 pub total_duplicated_lines: usize,
359 pub total_duplicated_tokens: usize,
361 pub suggestions: Vec<RefactoringSuggestion>,
363}
364
365#[derive(Debug, Clone, Serialize, Deserialize)]
368#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
369pub struct MirroredDirectory {
370 pub dir_a: String,
372 pub dir_b: String,
374 pub shared_files: Vec<String>,
376 pub total_lines: usize,
378}
379
380#[derive(Debug, Clone, Default)]
382pub struct DefaultIgnoreSkipCount {
383 pub pattern: &'static str,
385 pub count: usize,
387}
388
389#[derive(Debug, Clone, Default)]
391pub struct DefaultIgnoreSkips {
392 pub total: usize,
394 pub by_pattern: Vec<DefaultIgnoreSkipCount>,
396}
397
398#[derive(Debug, Clone, Default, Serialize, Deserialize)]
400#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
401pub struct DuplicationReport {
402 pub clone_groups: Vec<CloneGroup>,
405 pub clone_families: Vec<CloneFamily>,
408 #[serde(default, skip_serializing_if = "Vec::is_empty")]
410 pub mirrored_directories: Vec<MirroredDirectory>,
411 pub stats: DuplicationStats,
413}
414
415impl DuplicationReport {
416 pub fn sort(&mut self) {
421 for group in &mut self.clone_groups {
422 group
423 .instances
424 .sort_by(|a, b| a.file.cmp(&b.file).then(a.start_line.cmp(&b.start_line)));
425 }
426 sort_clone_groups(&mut self.clone_groups);
427
428 for family in &mut self.clone_families {
429 for group in &mut family.groups {
430 group
431 .instances
432 .sort_by(|a, b| a.file.cmp(&b.file).then(a.start_line.cmp(&b.start_line)));
433 }
434 sort_clone_groups(&mut family.groups);
435 }
436 self.clone_families.sort_by(|a, b| a.files.cmp(&b.files));
437 }
438
439 #[must_use]
441 pub fn clone_groups_shown(&self) -> usize {
442 self.clone_groups.len()
443 }
444
445 #[must_use]
451 pub fn clone_groups_omitted(&self) -> usize {
452 self.stats
453 .clone_groups
454 .saturating_sub(self.clone_groups.len())
455 }
456
457 #[must_use]
463 pub fn clone_groups_total(&self) -> usize {
464 self.clone_groups_shown() + self.clone_groups_omitted()
465 }
466
467 #[must_use]
469 pub fn clone_families_shown(&self) -> usize {
470 self.clone_families.len()
471 }
472
473 #[must_use]
481 pub fn clone_families_omitted(&self) -> usize {
482 self.stats
483 .clone_families
484 .saturating_sub(self.clone_families.len())
485 }
486
487 #[must_use]
489 pub fn clone_families_total(&self) -> usize {
490 self.clone_families_shown() + self.clone_families_omitted()
491 }
492
493 pub fn strip_fragments(&mut self) {
500 for group in &mut self.clone_groups {
501 group.strip_fragments();
502 }
503 for family in &mut self.clone_families {
504 for group in &mut family.groups {
505 group.strip_fragments();
506 }
507 }
508 }
509}
510
511#[derive(Debug, Clone, Default, Serialize, Deserialize)]
513#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
514pub struct DuplicationStats {
515 pub total_files: usize,
517 pub files_with_clones: usize,
519 pub total_lines: usize,
521 pub duplicated_lines: usize,
523 pub total_tokens: usize,
525 pub duplicated_tokens: usize,
527 pub clone_groups: usize,
531 pub clone_families: usize,
536 pub clone_instances: usize,
539 pub duplication_percentage: f64,
542 #[serde(default, skip_serializing_if = "is_zero_usize")]
546 pub clone_groups_below_min_occurrences: usize,
547 #[serde(default, skip_serializing_if = "is_zero_usize")]
549 pub clone_groups_ignored: usize,
550 #[serde(default, skip_serializing_if = "is_zero_usize")]
552 pub near_candidates_skipped: usize,
553}
554
555#[expect(
556 clippy::trivially_copy_pass_by_ref,
557 reason = "serde skip_serializing_if requires &T signature"
558)]
559const fn is_zero_usize(value: &usize) -> bool {
560 *value == 0
561}
562
563#[cfg(test)]
564mod tests {
565 use super::*;
566 use proptest::prelude::*;
567
568 fn pairwise_clone_group_spread(instances: &[CloneInstance]) -> usize {
569 let mut spread = 0;
570 for (index, left) in instances.iter().enumerate() {
571 for right in &instances[index + 1..] {
572 spread = spread.max(instance_pair_spread(left, right));
573 }
574 }
575 spread
576 }
577
578 fn instance(file: impl Into<PathBuf>, start_line: usize, end_line: usize) -> CloneInstance {
579 CloneInstance {
580 is_symlink: false,
581 file: file.into(),
582 start_line,
583 end_line,
584 start_col: 0,
585 end_col: 0,
586 fragment: String::new(),
587 }
588 }
589
590 fn group(instances: Vec<CloneInstance>, token_count: usize, line_count: usize) -> CloneGroup {
591 CloneGroup {
592 instances,
593 token_count,
594 line_count,
595 similarity: None,
596 }
597 }
598
599 #[test]
600 fn spread_counts_non_shared_parent_components() {
601 let clone = group(
602 vec![
603 instance("/repo/packages/a/src/a.ts", 1, 10),
604 instance("/repo/packages/b/src/b.ts", 1, 10),
605 ],
606 100,
607 10,
608 );
609 assert_eq!(clone.spread(), 4);
610 }
611
612 #[test]
613 fn spread_is_zero_for_different_files_in_the_same_directory() {
614 let clone = group(
615 vec![
616 instance("/repo/src/a.ts", 1, 10),
617 instance("/repo/src/b.ts", 1, 10),
618 ],
619 100,
620 10,
621 );
622 assert_eq!(clone.spread(), 0);
623 }
624
625 #[test]
626 fn adjacent_same_file_instances_have_zero_spread() {
627 let clone = group(
628 vec![
629 instance("/repo/src/a.ts", 1, 10),
630 instance("/repo/src/a.ts", 11, 20),
631 ],
632 100,
633 10,
634 );
635 assert_eq!(clone.spread(), 0);
636 }
637
638 #[test]
639 fn same_file_spread_uses_ceiling_rounded_intervening_lines() {
640 let clone = group(
641 vec![
642 instance("/repo/src/a.ts", 1, 10),
643 instance("/repo/src/a.ts", 260, 269),
644 instance("/repo/src/a.ts", 261, 270),
645 instance("/repo/src/a.ts", 262, 271),
646 ],
647 100,
648 10,
649 );
650 assert_eq!(clone_group_spread(&clone.instances[..2]), 1);
651 assert_eq!(clone_group_spread(&clone.instances[..3]), 1);
652 assert_eq!(clone.spread(), 2);
653 }
654
655 #[test]
656 fn overlapping_same_file_instances_have_zero_spread() {
657 let clone = group(
658 vec![
659 instance("/repo/src/a.ts", 1, 20),
660 instance("/repo/src/a.ts", 10, 30),
661 ],
662 100,
663 20,
664 );
665 assert_eq!(clone.spread(), 0);
666 }
667
668 #[test]
669 fn directory_diameter_handles_ties_and_mixed_roots() {
670 let instances = vec![
671 instance("src/a.ts", 1, 10),
672 instance("packages/a/b.ts", 1, 10),
673 instance("packages/c/d.ts", 1, 10),
674 instance("/repo/src/e.ts", 1, 10),
675 ];
676
677 assert_eq!(
678 clone_group_spread(&instances),
679 pairwise_clone_group_spread(&instances)
680 );
681 }
682
683 #[test]
684 fn spread_matches_reference_for_repeated_mixed_and_nested_files() {
685 let instances = vec![
686 instance("src/a.ts", 900, 920),
687 instance("packages/a/src/nested/b.ts", 40, 60),
688 instance("src/a.ts", 1, 20),
689 instance("/repo/apps/web/c.ts", 300, 325),
690 instance("packages/b/test/d.ts", 70, 90),
691 instance("/repo/apps/web/c.ts", 1, 25),
692 ];
693
694 assert_eq!(
695 clone_group_spread(&instances),
696 pairwise_clone_group_spread(&instances)
697 );
698 }
699
700 #[cfg(unix)]
701 #[test]
702 fn spread_matches_reference_for_non_utf8_paths() {
703 use std::ffi::OsString;
704 use std::os::unix::ffi::OsStringExt;
705
706 let first = PathBuf::from(OsString::from_vec(b"/repo/packages/\x80/src/a.ts".to_vec()));
707 let second = PathBuf::from(OsString::from_vec(b"/repo/packages/\x81/src/b.ts".to_vec()));
708 let instances = vec![
709 instance(first.clone(), 1, 20),
710 instance(second, 100, 120),
711 instance(first, 800, 820),
712 ];
713
714 assert_eq!(
715 clone_group_spread(&instances),
716 pairwise_clone_group_spread(&instances)
717 );
718 }
719
720 #[cfg(windows)]
721 #[test]
722 fn directory_diameter_handles_windows_prefixes() {
723 let instances = vec![
724 instance(r"C:\repo\src\a.ts", 1, 10),
725 instance(r"C:\repo\packages\b.ts", 1, 10),
726 instance(r"D:\other\c.ts", 1, 10),
727 ];
728
729 assert_eq!(
730 clone_group_spread(&instances),
731 pairwise_clone_group_spread(&instances)
732 );
733 }
734
735 #[test]
736 fn clone_group_kind_does_not_change_serialized_contract() {
737 let mut clone = group(vec![instance("src/a.ts", 1, 10)], 20, 10);
738 assert_eq!(clone.kind(), CloneGroupKind::Exact);
739 assert!(serde_json::to_value(&clone).unwrap()["similarity"].is_null());
740
741 clone.similarity = Some(0.85);
742 assert_eq!(clone.kind(), CloneGroupKind::Near { similarity: 0.85 });
743 assert_eq!(serde_json::to_value(&clone).unwrap()["similarity"], 0.85);
744 }
745
746 mod proptests {
747 use super::*;
748
749 proptest! {
750 #[test]
751 fn optimized_spread_matches_pairwise_reference(
752 entries in prop::collection::vec((0_u8..8, 1_usize..4_000, 1_usize..500), 0..80)
753 ) {
754 let paths = [
755 "src/a.ts",
756 "src/b.ts",
757 "packages/a/src/c.ts",
758 "packages/b/src/d.ts",
759 "packages/b/test/e.ts",
760 "/repo/apps/web/f.ts",
761 "/repo/crates/core/g.ts",
762 "h.ts",
763 ];
764 let instances = entries
765 .into_iter()
766 .map(|(path, start, len)| instance(paths[usize::from(path)], start, start + len))
767 .collect::<Vec<_>>();
768
769 prop_assert_eq!(
770 clone_group_spread(&instances),
771 pairwise_clone_group_spread(&instances)
772 );
773 }
774 }
775 }
776
777 #[test]
778 fn ranking_uses_spread_without_overriding_a_larger_base_score() {
779 let distant = group(
780 vec![
781 instance("/repo/a/b/c/d/e/a.ts", 1, 10),
782 instance("/repo/f/g/h/i/j/b.ts", 1, 10),
783 ],
784 100,
785 10,
786 );
787 let slightly_larger_local = group(
788 vec![
789 instance("/repo/src/a.ts", 1, 10),
790 instance("/repo/src/b.ts", 1, 10),
791 ],
792 116,
793 10,
794 );
795 assert_eq!(distant.spread(), 10);
796 assert_eq!(
797 compare_clone_groups(&distant, &slightly_larger_local),
798 Ordering::Greater
799 );
800
801 let slightly_smaller_local = group(
802 vec![
803 instance("/repo/src/c.ts", 1, 10),
804 instance("/repo/src/d.ts", 1, 10),
805 ],
806 114,
807 10,
808 );
809 assert_eq!(
810 compare_clone_groups(&distant, &slightly_smaller_local),
811 Ordering::Less
812 );
813 }
814
815 #[test]
816 fn report_sort_uses_canonical_location_as_final_tiebreaker() {
817 let later = group(
818 vec![
819 instance("/repo/src/z.ts", 1, 10),
820 instance("/repo/src/y.ts", 1, 10),
821 ],
822 100,
823 10,
824 );
825 let earlier = group(
826 vec![
827 instance("/repo/src/a.ts", 20, 29),
828 instance("/repo/src/b.ts", 20, 29),
829 ],
830 100,
831 10,
832 );
833 let mut report = DuplicationReport {
834 clone_groups: vec![later, earlier],
835 ..DuplicationReport::default()
836 };
837 report.sort();
838 assert_eq!(
839 report.clone_groups[0].instances[0].file,
840 Path::new("/repo/src/a.ts")
841 );
842 }
843
844 #[test]
845 fn exact_clone_similarity_is_omitted() {
846 let clone = group(Vec::new(), 100, 10);
847 let value = serde_json::to_value(clone).unwrap();
848 assert!(value.get("similarity").is_none());
849 }
850
851 #[test]
852 fn near_clone_similarity_is_serialized() {
853 let mut clone = group(Vec::new(), 100, 10);
854 clone.similarity = Some(0.85);
855 let value = serde_json::to_value(clone).unwrap();
856 assert_eq!(value["similarity"], 0.85);
857 }
858
859 #[test]
860 fn optional_duplication_stats_are_omitted_at_zero() {
861 let empty = serde_json::to_value(DuplicationStats::default()).unwrap();
862 assert!(empty.get("clone_groups_ignored").is_none());
863 assert!(empty.get("near_candidates_skipped").is_none());
864
865 let populated = serde_json::to_value(DuplicationStats {
866 clone_groups_ignored: 2,
867 near_candidates_skipped: 3,
868 ..DuplicationStats::default()
869 })
870 .unwrap();
871 assert_eq!(populated["clone_groups_ignored"], 2);
872 assert_eq!(populated["near_candidates_skipped"], 3);
873 }
874}