#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum CloneType {
Exact,
Renamed,
NearMiss,
}
impl CloneType {
#[must_use]
pub fn label(self) -> &'static str {
match self {
CloneType::Exact => "exact",
CloneType::Renamed => "renamed",
CloneType::NearMiss => "near-miss",
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DuplicateBlock {
pub hash: String,
pub locations: Vec<DuplicateLocation>,
pub lines: usize,
pub tokens: usize,
pub similarity: f32,
pub clone_type: CloneType,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DuplicateLocation {
pub file: String,
pub start_line: usize,
pub end_line: usize,
pub content_preview: String,
}
#[derive(Debug, Serialize)]
pub struct DuplicateReport {
pub total_duplicates: usize,
pub duplicate_lines: usize,
pub total_lines: usize,
pub duplication_percentage: f32,
pub duplicate_blocks: Vec<DuplicateBlock>,
pub file_statistics: BTreeMap<String, FileStats>,
}
#[derive(Debug, Serialize)]
pub struct FileStats {
pub duplicate_lines: usize,
pub total_lines: usize,
pub duplication_percentage: f32,
}
#[allow(clippy::too_many_arguments)]
#[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")]
pub async fn handle_analyze_duplicates(
project_path: PathBuf,
detection_type: crate::cli::DuplicateType,
threshold: f32,
min_lines: usize,
max_tokens: usize,
format: crate::cli::DuplicateOutputFormat,
perf: bool,
include: Option<String>,
exclude: Option<String>,
output: Option<PathBuf>,
top_files: usize,
) -> Result<()> {
crate::cli::ensure_analysis_path_exists(&project_path)?;
{
use crate::cli::colors as c;
crate::status_eprintln!("{}", c::dim("Analyzing code similarity..."));
}
let start_time = std::time::Instant::now();
let report = run_duplicate_detection(
&project_path,
detection_type,
threshold,
min_lines,
max_tokens,
&include,
&exclude,
)
.await?;
print_duplicate_summary(&report);
if perf {
use crate::cli::colors as c;
let duration = start_time.elapsed();
eprintln!("\n{}Performance Metrics:{}", c::BOLD, c::RESET);
eprintln!(
" {}Analysis time:{} {}{:.2}ms{}",
c::BOLD,
c::RESET,
c::BOLD_WHITE,
duration.as_millis(),
c::RESET
);
eprintln!(
" {}Files processed:{} {}{}{}",
c::BOLD,
c::RESET,
c::BOLD_WHITE,
report.file_statistics.len(),
c::RESET
);
eprintln!(
" {}Blocks analyzed:{} {}{}{}",
c::BOLD,
c::RESET,
c::BOLD_WHITE,
report.duplicate_blocks.len(),
c::RESET
);
}
{
use crate::cli::colors as c;
crate::status_eprintln!("\n{}", c::pass("Analysis Complete"));
}
write_duplicate_output(&report, format, output, top_files).await
}
#[cfg_attr(coverage_nightly, coverage(off))]
#[cfg(test)]
mod top_files_does_not_change_the_measurement_tests {
use super::*;
#[tokio::test]
async fn duplication_metrics_are_independent_of_top_files() {
use tempfile::TempDir;
let project = TempDir::new().unwrap();
let out = TempDir::new().unwrap();
let family_a: String = (0..14)
.map(|line| format!(" let a{} = {line} * 3;\n", line % 3))
.collect();
let family_b: String = (0..14)
.map(|line| format!(" let b{} = {line} - 7;\n", line % 4))
.collect();
for name in ["a0.rs", "a1.rs", "a2.rs"] {
std::fs::write(project.path().join(name), &family_a).unwrap();
}
for name in ["b0.rs", "b1.rs"] {
std::fs::write(project.path().join(name), &family_b).unwrap();
}
std::fs::write(
project.path().join("c0.rs"),
"fn unique() {\n let only = 1;\n}\n",
)
.unwrap();
let mut baseline: Option<(u64, u64, String)> = None;
for top_files in [1usize, 2, 3, 10, 0] {
let report_path = out.path().join(format!("dup-{top_files}.json"));
handle_analyze_duplicates(
project.path().to_path_buf(),
crate::cli::DuplicateType::Exact,
0.8,
5,
100,
crate::cli::DuplicateOutputFormat::Json,
false,
None,
None,
Some(report_path.clone()),
top_files,
)
.await
.expect("analysis must succeed");
let json: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(&report_path).unwrap()).unwrap();
let measured = (
json["total_duplicates"].as_u64().unwrap(),
json["duplicate_lines"].as_u64().unwrap(),
format!("{:.4}", json["duplication_percentage"].as_f64().unwrap()),
);
match &baseline {
None => baseline = Some(measured),
Some(expected) => assert_eq!(
*expected, measured,
"--top-files {top_files} changed the measured duplication"
),
}
}
}
}
async fn run_duplicate_detection(
project_path: &Path,
detection_type: crate::cli::DuplicateType,
threshold: f32,
min_lines: usize,
max_tokens: usize,
include: &Option<String>,
exclude: &Option<String>,
) -> Result<DuplicateReport> {
detect_duplicates(
project_path,
detection_type,
threshold,
min_lines,
max_tokens,
include,
exclude,
)
.await
}
fn print_duplicate_summary(report: &DuplicateReport) {
use crate::cli::colors as c;
crate::status_eprintln!(
"{} Found {} duplicate blocks",
c::pass(""),
c::number(&report.total_duplicates.to_string())
);
crate::status_eprintln!(
" {}Duplication:{} {} ({} / {} lines)",
c::BOLD,
c::RESET,
c::pct(report.duplication_percentage as f64, 5.0, 15.0),
c::number(&report.duplicate_lines.to_string()),
c::number(&report.total_lines.to_string()),
);
}
async fn write_duplicate_output(
report: &DuplicateReport,
format: crate::cli::DuplicateOutputFormat,
output: Option<PathBuf>,
top_files: usize,
) -> Result<()> {
let content = match format {
crate::cli::DuplicateOutputFormat::Human => {
format_text_output(report, top_files, TextDetail::Human)?
}
crate::cli::DuplicateOutputFormat::Summary => {
format_text_output(report, top_files, TextDetail::Summary)?
}
crate::cli::DuplicateOutputFormat::Detailed => {
format_text_output(report, top_files, TextDetail::Detailed)?
}
other => format_output(report, other)?,
};
if let Some(output_path) = output {
tokio::fs::write(&output_path, &content).await?;
crate::status_eprintln!("📄 Report written to: {}", output_path.display());
} else {
println!("{content}");
}
Ok(())
}
async fn detect_duplicates(
project_path: &Path,
detection_type: crate::cli::DuplicateType,
threshold: f32,
min_lines: usize,
max_tokens: usize,
include: &Option<String>,
exclude: &Option<String>,
) -> Result<DuplicateReport> {
let (all_blocks, sources, total_lines, mut file_stats) = collect_code_blocks(
project_path,
detection_type.clone(),
min_lines,
max_tokens,
include,
exclude,
)
.await?;
warn_threshold_has_no_effect(threshold, &detection_type);
let mut duplicate_blocks = find_duplicate_blocks(all_blocks);
duplicate_blocks.extend(find_structural_similarities(
&sources,
detection_type,
threshold,
min_lines,
));
sort_duplicate_blocks(&mut duplicate_blocks);
let duplicate_lines = calculate_duplicate_statistics(&duplicate_blocks, &mut file_stats);
let duplication_percentage = calculate_duplication_percentage(duplicate_lines, total_lines);
Ok(build_duplicate_report(
duplicate_blocks,
duplicate_lines,
total_lines,
duplication_percentage,
file_stats,
))
}
async fn collect_code_blocks(
project_path: &Path,
detection_type: crate::cli::DuplicateType,
min_lines: usize,
max_tokens: usize,
include: &Option<String>,
exclude: &Option<String>,
) -> Result<(
Vec<(String, String, usize, usize, String)>,
Vec<(PathBuf, String)>,
usize,
BTreeMap<String, FileStats>,
)> {
use crate::services::file_discovery::ProjectFileDiscovery;
let mut all_blocks = Vec::new();
let mut sources = Vec::new();
let mut total_lines = 0usize;
let mut file_stats = BTreeMap::new();
let discovered_files = ProjectFileDiscovery::new(project_path.to_path_buf())
.discover_files()
.unwrap_or_default();
for path in discovered_files {
let path = path.as_path();
if should_analyze_file(path, include, exclude) {
if let Some((blocks, lines_count, content)) =
process_source_file(path, detection_type.clone(), min_lines, max_tokens).await
{
all_blocks.extend(blocks);
total_lines += lines_count;
sources.push((path.to_path_buf(), content));
file_stats.insert(
path.to_string_lossy().to_string(),
FileStats {
duplicate_lines: 0,
total_lines: lines_count,
duplication_percentage: 0.0,
},
);
}
}
}
Ok((all_blocks, sources, total_lines, file_stats))
}
fn should_analyze_file(path: &Path, include: &Option<String>, exclude: &Option<String>) -> bool {
path.is_file() && is_source_file(path) && should_process_file(path, include, exclude)
}
async fn process_source_file(
path: &Path,
detection_type: crate::cli::DuplicateType,
min_lines: usize,
max_tokens: usize,
) -> Option<(Vec<(String, String, usize, usize, String)>, usize, String)> {
if let Ok(content) = tokio::fs::read_to_string(path).await {
let lines: Vec<&str> = content.lines().collect();
let blocks = extract_blocks(&lines, path, min_lines, max_tokens, detection_type);
let line_count = lines.len();
Some((blocks, line_count, content))
} else {
None
}
}
fn near_miss_enabled(detection_type: &crate::cli::DuplicateType) -> bool {
matches!(
detection_type,
crate::cli::DuplicateType::Gapped
| crate::cli::DuplicateType::Fuzzy
| crate::cli::DuplicateType::All
)
}
fn engine_language(path: &Path) -> Option<crate::services::duplicate_detector::Language> {
use crate::services::duplicate_detector::Language;
match path.extension().and_then(|e| e.to_str()) {
Some("rs") => Some(Language::Rust),
Some("ts") => Some(Language::TypeScript),
Some("js") => Some(Language::JavaScript),
Some("py") => Some(Language::Python),
Some("c") => Some(Language::C),
Some("cpp" | "cc" | "cxx") => Some(Language::Cpp),
Some("kt" | "kts") => Some(Language::Kotlin),
_ => None,
}
}
fn find_structural_similarities(
sources: &[(PathBuf, String)],
detection_type: crate::cli::DuplicateType,
threshold: f32,
min_lines: usize,
) -> Vec<DuplicateBlock> {
use crate::services::duplicate_detector::{DuplicateDetectionConfig, DuplicateDetectionEngine};
if !near_miss_enabled(&detection_type) {
return Vec::new();
}
let files: Vec<_> = sources
.iter()
.filter_map(|(path, content)| {
engine_language(path).map(|lang| (path.clone(), content.clone(), lang))
})
.collect();
if files.is_empty() {
return Vec::new();
}
let engine = DuplicateDetectionEngine::new(DuplicateDetectionConfig {
similarity_threshold: f64::from(threshold),
min_group_size: 2,
..DuplicateDetectionConfig::default()
});
let Ok(report) = engine.detect_duplicates(&files) else {
return Vec::new();
};
let contents: BTreeMap<&Path, &str> = sources
.iter()
.map(|(path, content)| (path.as_path(), content.as_str()))
.collect();
let mut blocks: Vec<DuplicateBlock> = report
.groups
.iter()
.filter(|group| group.average_similarity < 1.0)
.filter_map(|group| near_miss_block(group, &contents, min_lines))
.collect();
sort_duplicate_blocks(&mut blocks);
blocks
}
fn near_miss_block(
group: &crate::services::duplicate_detector::CloneGroup,
contents: &BTreeMap<&Path, &str>,
min_lines: usize,
) -> Option<DuplicateBlock> {
let mut sites: Vec<(String, usize, usize, String)> = group
.fragments
.iter()
.filter_map(|f| {
let content = fragment_text(contents, &f.file, f.start_line, f.end_line, min_lines)?;
Some((
f.file.to_string_lossy().to_string(),
f.start_line,
f.end_line,
content,
))
})
.collect();
if sites.len() < 2 {
return None;
}
sites.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)));
let lines = sites[0].2 - sites[0].1 + 1;
let tokens = count_tokens(&sites[0].3);
let hash = near_miss_hash(&sites);
let locations = sites
.into_iter()
.map(|(file, start_line, end_line, content)| {
let preview = content.lines().take(3).collect::<Vec<_>>().join("\n");
DuplicateLocation {
file,
start_line,
end_line,
content_preview: if content.lines().count() > 3 {
format!("{preview}...")
} else {
preview
},
}
})
.collect();
Some(DuplicateBlock {
hash,
locations,
lines,
tokens,
#[allow(clippy::cast_possible_truncation)]
similarity: group.average_similarity as f32,
clone_type: CloneType::NearMiss,
})
}
fn fragment_text(
contents: &BTreeMap<&Path, &str>,
file: &Path,
start_line: usize,
end_line: usize,
min_lines: usize,
) -> Option<String> {
let content = contents.get(file)?;
let lines: Vec<&str> = content.lines().collect();
let start = start_line.checked_sub(1)?;
let end = end_line.min(lines.len());
if start >= end {
return None;
}
if substantive_lines(&lines[start..end]).len() < min_lines.max(1) {
return None;
}
Some(normalize_block(&lines[start..end]))
}
fn near_miss_hash(sites: &[(String, usize, usize, String)]) -> String {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let mut hasher = DefaultHasher::new();
for (file, start, end, _) in sites {
file.hash(&mut hasher);
start.hash(&mut hasher);
end.hash(&mut hasher);
}
format!("n{:x}", hasher.finish())
}
fn calculate_duplicate_statistics(
duplicate_blocks: &[DuplicateBlock],
file_stats: &mut BTreeMap<String, FileStats>,
) -> usize {
let mut duplicated: HashMap<&str, std::collections::HashSet<usize>> = HashMap::new();
for block in duplicate_blocks {
for loc in &block.locations {
let lines = duplicated.entry(loc.file.as_str()).or_default();
for line in loc.start_line..=loc.end_line {
lines.insert(line);
}
}
}
for (path, stats) in file_stats.iter_mut() {
let counted = duplicated.get(path.as_str()).map_or(0, |set| {
set.iter()
.filter(|line| **line <= stats.total_lines)
.count()
});
stats.duplicate_lines = counted;
stats.duplication_percentage = if stats.total_lines > 0 {
#[allow(clippy::cast_precision_loss)]
let pct = (counted as f32 / stats.total_lines as f32) * 100.0;
pct.min(100.0)
} else {
0.0
};
}
file_stats.values().map(|s| s.duplicate_lines).sum()
}
fn calculate_duplication_percentage(duplicate_lines: usize, total_lines: usize) -> f32 {
if total_lines > 0 {
#[allow(clippy::cast_precision_loss)]
let pct = (duplicate_lines as f32 / total_lines as f32) * 100.0;
pct.min(100.0)
} else {
0.0
}
}
fn build_duplicate_report(
duplicate_blocks: Vec<DuplicateBlock>,
duplicate_lines: usize,
total_lines: usize,
duplication_percentage: f32,
file_stats: BTreeMap<String, FileStats>,
) -> DuplicateReport {
DuplicateReport {
total_duplicates: duplicate_blocks.len(),
duplicate_lines,
total_lines,
duplication_percentage,
duplicate_blocks,
file_statistics: file_stats,
}
}