tokmd_analysis_types/
duplication.rs

1//! Duplicate and near-duplicate receipt DTOs.
2//!
3//! These contract types remain re-exported from the crate root to preserve
4//! existing `tokmd_analysis_types::...` names.
5
6use serde::{Deserialize, Serialize};
7
8// ----------------------------
9// Near-duplicate detection
10// ----------------------------
11
12/// Scope for near-duplicate comparison partitioning.
13#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
14#[serde(rename_all = "kebab-case")]
15pub enum NearDupScope {
16    /// Compare files within the same module.
17    #[default]
18    Module,
19    /// Compare files within the same language.
20    Lang,
21    /// Compare all files globally.
22    Global,
23}
24
25/// Parameters for near-duplicate detection.
26#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct NearDupParams {
28    pub scope: NearDupScope,
29    pub threshold: f64,
30    pub max_files: usize,
31    /// Maximum pairs to emit (truncation guardrail).
32    #[serde(default, skip_serializing_if = "Option::is_none")]
33    pub max_pairs: Option<usize>,
34    /// Effective per-file byte limit used for eligibility filtering.
35    #[serde(default, skip_serializing_if = "Option::is_none")]
36    pub max_file_bytes: Option<u64>,
37    /// How files were selected for analysis.
38    #[serde(default, skip_serializing_if = "Option::is_none")]
39    pub selection_method: Option<String>,
40    /// Algorithm constants used for fingerprinting.
41    #[serde(default, skip_serializing_if = "Option::is_none")]
42    pub algorithm: Option<NearDupAlgorithm>,
43    /// Glob patterns used to exclude files from near-dup analysis.
44    #[serde(default, skip_serializing_if = "Vec::is_empty")]
45    pub exclude_patterns: Vec<String>,
46}
47
48/// Algorithm constants for near-duplicate fingerprinting.
49#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
50pub struct NearDupAlgorithm {
51    /// Number of tokens per k-gram shingle.
52    pub k_gram_size: usize,
53    /// Winnowing window size.
54    pub window_size: usize,
55    /// Skip fingerprints appearing in more than this many files.
56    pub max_postings: usize,
57}
58
59/// Report of near-duplicate file pairs.
60#[derive(Debug, Clone, Serialize, Deserialize)]
61pub struct NearDuplicateReport {
62    pub params: NearDupParams,
63    pub pairs: Vec<NearDupPairRow>,
64    pub files_analyzed: usize,
65    pub files_skipped: usize,
66    /// Number of files eligible before the max_files cap.
67    #[serde(default, skip_serializing_if = "Option::is_none")]
68    pub eligible_files: Option<usize>,
69    /// Connected-component clusters derived from pairs.
70    #[serde(default, skip_serializing_if = "Option::is_none")]
71    pub clusters: Option<Vec<NearDupCluster>>,
72    /// Whether the pairs list was truncated by `max_pairs`.
73    /// Clusters are built from the complete pair set before truncation.
74    #[serde(default)]
75    pub truncated: bool,
76    /// Number of files excluded by glob patterns.
77    #[serde(default, skip_serializing_if = "Option::is_none")]
78    pub excluded_by_pattern: Option<usize>,
79    /// Runtime performance statistics.
80    #[serde(default, skip_serializing_if = "Option::is_none")]
81    pub stats: Option<NearDupStats>,
82}
83
84/// A connected component of near-duplicate files.
85#[derive(Debug, Clone, Serialize, Deserialize)]
86pub struct NearDupCluster {
87    /// Files in this cluster, sorted alphabetically.
88    pub files: Vec<String>,
89    /// Maximum pairwise similarity in the cluster.
90    pub max_similarity: f64,
91    /// Most-connected file (tie-break alphabetical).
92    pub representative: String,
93    /// Number of pairs within this cluster.
94    pub pair_count: usize,
95}
96
97/// Runtime statistics for near-duplicate detection.
98#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
99pub struct NearDupStats {
100    /// Time spent computing fingerprints (milliseconds).
101    pub fingerprinting_ms: u64,
102    /// Time spent computing pair similarities (milliseconds).
103    pub pairing_ms: u64,
104    /// Total bytes of source files processed.
105    pub bytes_processed: u64,
106}
107
108/// A pair of near-duplicate files with similarity score.
109#[derive(Debug, Clone, Serialize, Deserialize)]
110pub struct NearDupPairRow {
111    pub left: String,
112    pub right: String,
113    pub similarity: f64,
114    pub shared_fingerprints: usize,
115    pub left_fingerprints: usize,
116    pub right_fingerprints: usize,
117}
118
119// -------------------
120// Duplication metrics
121// -------------------
122
123#[derive(Debug, Clone, Serialize, Deserialize)]
124pub struct DuplicateReport {
125    pub groups: Vec<DuplicateGroup>,
126    pub wasted_bytes: u64,
127    pub strategy: String,
128    /// Duplication density summary overall and by module.
129    #[serde(skip_serializing_if = "Option::is_none")]
130    pub density: Option<DuplicationDensityReport>,
131    /// Near-duplicate file pairs detected by fingerprint similarity.
132    #[serde(default, skip_serializing_if = "Option::is_none")]
133    pub near: Option<NearDuplicateReport>,
134}
135
136#[derive(Debug, Clone, Serialize, Deserialize)]
137pub struct DuplicateGroup {
138    pub hash: String,
139    pub bytes: u64,
140    pub files: Vec<String>,
141}
142
143#[derive(Debug, Clone, Serialize, Deserialize)]
144pub struct DuplicationDensityReport {
145    pub duplicate_groups: usize,
146    pub duplicate_files: usize,
147    pub duplicated_bytes: u64,
148    pub wasted_bytes: u64,
149    pub wasted_pct_of_codebase: f64,
150    pub by_module: Vec<ModuleDuplicationDensityRow>,
151}
152
153#[derive(Debug, Clone, Serialize, Deserialize)]
154pub struct ModuleDuplicationDensityRow {
155    pub module: String,
156    pub duplicate_files: usize,
157    pub wasted_files: usize,
158    pub duplicated_bytes: u64,
159    pub wasted_bytes: u64,
160    pub module_bytes: u64,
161    pub density: f64,
162}
tokmd_analysis_types/duplication.rs

tokmd_analysis_types/
duplication.rs
