kimun 0.20.0

Code metrics tool — health score, complexity, duplication, hotspots, ownership
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

/// Phase 2: validate candidate hash groups by verifying text equality.
///
/// Filters out FNV collisions and boilerplate patterns (>100 occurrences).
/// Builds a reverse lookup from LocationSet hash to window hash, used
/// by the extension phase to merge adjacent duplicate windows.
use std::collections::HashMap;

use super::hashing::hash_location_set;
use super::{LocationSet, MAX_OCCURRENCES, NormalizedFile};

/// Filter candidate hashes to those with 2+ truly identical locations.
///
/// After dedup, verifies each hash group by comparing actual text content
/// (guards against FNV collisions). Skips patterns with more than
/// `MAX_OCCURRENCES` hits, which are likely boilerplate. Returns both a
/// reverse lookup (`LocationSet → hash`) for the extension phase, and the
/// sorted list of valid `(hash, locations)` entries.
pub fn validate_hashes(
    hash_map: HashMap<u64, LocationSet>,
    files: &[NormalizedFile],
    min_lines: usize,
    quiet: bool,
) -> (HashMap<u64, u64>, Vec<(u64, LocationSet)>) {
    let mut location_to_hash: HashMap<u64, u64> = HashMap::new();
    let mut valid_hashes: Vec<(u64, LocationSet)> = Vec::new();
    let mut skipped_common = 0usize;

    for (hash, mut locations) in hash_map {
        if locations.len() < 2 || locations.len() > MAX_OCCURRENCES {
            if locations.len() > MAX_OCCURRENCES {
                skipped_common += 1;
            }
            continue;
        }
        locations.sort();
        locations.dedup();
        if locations.len() < 2 {
            continue;
        }

        // Post-hash verification: confirm windows have identical text content
        let first = &locations[0];
        let first_window: Vec<&str> = files[first.0].lines[first.1..first.1 + min_lines]
            .iter()
            .map(|l| l.content.as_str())
            .collect();
        let all_match = locations[1..].iter().all(|(fi, off)| {
            files[*fi].lines[*off..*off + min_lines]
                .iter()
                .map(|l| l.content.as_str())
                .eq(first_window.iter().copied())
        });
        if !all_match {
            continue;
        }

        location_to_hash.insert(hash_location_set(&locations), hash);
        valid_hashes.push((hash, locations));
    }

    if skipped_common > 0 && !quiet {
        eprintln!(
            "note: skipped {skipped_common} patterns with >{MAX_OCCURRENCES} occurrences (likely boilerplate)"
        );
    }

    valid_hashes.sort_by(|a, b| a.1.cmp(&b.1));
    (location_to_hash, valid_hashes)
}