debtmap 0.23.0

Code complexity and technical debt analyzer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

use crate::core::{DuplicationBlock, DuplicationLocation};
use rayon::prelude::*;
use std::path::PathBuf;

mod similarity;
#[cfg(test)]
mod threshold_tests;

use similarity::{SimilarityChunk, group_similar_chunks};

/// Detects code duplication across multiple files using parallel processing.
///
/// Uses rayon for parallel chunk extraction and deterministic similarity grouping.
/// A threshold of `1.0` requires normalized text equality. Lower thresholds use
/// exact set-Jaccard token similarity and emit only directly verified pairs.
pub fn detect_duplication(
    files: Vec<(PathBuf, String)>,
    min_lines: usize,
    similarity_threshold: f64,
) -> Vec<DuplicationBlock> {
    if min_lines == 0
        || !similarity_threshold.is_finite()
        || !(0.0 < similarity_threshold && similarity_threshold <= 1.0)
    {
        return Vec::new();
    }

    let chunks = extract_project_chunks(&files, min_lines);
    group_similar_chunks(chunks, min_lines, similarity_threshold)
}

fn extract_project_chunks(files: &[(PathBuf, String)], chunk_size: usize) -> Vec<SimilarityChunk> {
    let mut chunks: Vec<_> = files
        .par_iter()
        .flat_map_iter(|(path, content)| {
            extract_chunks(content, chunk_size)
                .into_iter()
                .map(|(start_line, normalized)| {
                    SimilarityChunk::new(
                        normalized,
                        DuplicationLocation {
                            file: path.clone(),
                            start_line,
                            end_line: start_line + chunk_size - 1,
                        },
                    )
                })
        })
        .collect();
    chunks.sort_by(|left, right| {
        left.location
            .file
            .cmp(&right.location.file)
            .then(left.location.start_line.cmp(&right.location.start_line))
            .then(left.location.end_line.cmp(&right.location.end_line))
    });
    chunks
}

fn extract_chunks(content: &str, chunk_size: usize) -> Vec<(usize, String)> {
    let lines: Vec<&str> = content.lines().collect();

    if lines.len() < chunk_size {
        return Vec::new();
    }

    (0..=lines.len() - chunk_size)
        .map(|i| {
            let chunk = lines[i..i + chunk_size].join("\n");
            (i + 1, normalize_chunk(&chunk))
        })
        .collect()
}

fn normalize_chunk(chunk: &str) -> String {
    chunk
        .lines()
        .map(|line| line.trim())
        .filter(|line| !line.is_empty() && !line.starts_with("//") && !line.starts_with('#'))
        .collect::<Vec<_>>()
        .join("\n")
}

pub fn calculate_similarity(chunk1: &str, chunk2: &str) -> f64 {
    similarity::calculate_similarity(chunk1, chunk2)
}

pub fn merge_adjacent_duplications(mut blocks: Vec<DuplicationBlock>) -> Vec<DuplicationBlock> {
    blocks.sort_by_key(|b| (b.locations[0].file.clone(), b.locations[0].start_line));

    blocks.into_iter().fold(Vec::new(), |mut merged, block| {
        match merged.last_mut() {
            Some(last) if can_merge(last, &block) => {
                merge_blocks(last, block);
            }
            _ => merged.push(block),
        }
        merged
    })
}

fn can_merge(block1: &DuplicationBlock, block2: &DuplicationBlock) -> bool {
    block1.locations.iter().any(|loc1| {
        block2
            .locations
            .iter()
            .any(|loc2| loc1.file == loc2.file && loc1.end_line + 1 == loc2.start_line)
    })
}

fn merge_blocks(target: &mut DuplicationBlock, source: DuplicationBlock) {
    target.lines += source.lines;

    for loc in target.locations.iter_mut() {
        if let Some(src_loc) = source
            .locations
            .iter()
            .find(|src_loc| loc.file == src_loc.file && loc.end_line + 1 == src_loc.start_line)
        {
            loc.end_line = src_loc.end_line;
        }
    }
}