codehelion_core/
structural.rs

1//! Structural-mode analysis: the whole Type-2/Type-3 pipeline, wired end to
2//! end over already-parsed IR.
3//!
4//! The stages each live in their own module; this one composes them into the
5//! funnel the mode runs:
6//!
7//! 1. [`crate::features`] turns each file's IR into candidate-extraction
8//!    features (statement windows, subtrees, characteristic vector, approximate
9//!    CFG, API calls);
10//! 2. [`crate::candidate`] seeds exact-hash fragment pairs and [`crate::near_match`]
11//!    proposes near-clone unit pairs by MinHash/LSH — both over-approximate
12//!    cheaply, and both are lifted here to *unit* pairs;
13//! 3. [`crate::maximal`] folds the sliding-window seeds back into the maximal
14//!    shared statement runs they describe, so a duplicated block is one region
15//!    rather than a fan of overlapping window matches;
16//! 4. [`crate::verify`] judges each distinct unit pair precisely, keeping only
17//!    the ones that clear the clone threshold;
18//! 5. [`crate::grouping`] turns the surviving pairs into cohesive medoid groups,
19//!    so non-transitive Type-3 chains do not fuse.
20//!
21//! Regions and groups answer different questions and neither replaces the
22//! other: a group says two whole units are copies of each other, a region says
23//! one stretch of statements is shared, which happens between units that are
24//! not copies at all.
25//!
26//! Every unit carries its raw [`UnitFingerprint`] as its stable, position-free
27//! grouping key ([`crate::stable_id`]). The whole function is deterministic: the
28//! unit order follows the IR walk, candidate pairs are deduplicated through an
29//! ordered set, and grouping orders its own output. Nothing here executes target
30//! code — it only reads IR that was already produced from source.
31
32use std::collections::{BTreeMap, BTreeSet};
33
34use crate::boilerplate::{self, Boilerplate};
35use crate::candidate::{self, CandidateConfig, CandidateStats};
36use crate::clone_class::CloneClass;
37use crate::conditional::ArmPath;
38use crate::control_flow::{self, ControlFlowConfig, ControlFlowStats};
39use crate::discovery::{BuildVariant, Language};
40use crate::engine::{LiteralNorm, normalize::Resolution};
41use crate::features::{self, FileFeatures};
42use crate::frontend::{Lexeme, Token, TokenKind, UnitKind};
43use crate::grouping::{
44    self, GroupingConfig, GroupingSet, GroupingStats, GroupingUnit, SimilarityEdge,
45};
46use crate::ir::{ByteRange, IrNode, Shape, SyntaxIrFile};
47use crate::maximal::{self, MaximalConfig, RegionSide, RegionStats, SharedRegion};
48use crate::near_match::{self, NearMatchConfig, NearMatchStats};
49use crate::stable_id::{
50    self, CloneGroupFingerprint, ContentNorm, CrossVariantComparisonId, CrossVariantGroupId,
51    FileContext, FragmentFingerprint, UnitFingerprint,
52};
53use crate::substitution;
54use crate::test_code::{self, TestCodeEvidence};
55use crate::types::{ApiEvidence, TypeEvidence, TypeTag};
56use crate::verify::{self, SimilarityBreakdown, UnitView, VerifyConfig};
57
58mod analysis;
59mod evidence;
60mod model;
61mod pairs;
62mod regions;
63mod reporting;
64mod siblings;
65mod units;
66
67use evidence::{UnitEvidence, token_count_meets_minimum, unit_evidence, unit_meets_minimum};
68use pairs::{lift_to_unit_pairs, unrepresented_pairs};
69use regions::{confirm_regions, drop_subsumed, grow_runs};
70use reporting::{dominant_boilerplate_members, group_detail, written_once_per_width_members};
71use siblings::sweep_siblings;
72use units::{flatten_units, line_range, view};
73
74#[cfg(test)]
75use regions::{Confirmed, covers_run, merge_adjacent};
76#[cfg(test)]
77use reporting::{dominant_boilerplate, is_allocation_api, set_jaccard};
78
79pub use analysis::*;
80pub use evidence::*;
81pub use model::*;
82pub use reporting::span_identifier_jaccard;
83
84/// One analysed unit's data, held together for verification and grouping.
85struct Unit {
86    file: usize,
87    local: usize,
88    kind: UnitKind,
89    statements: Vec<crate::ir::StatementSummary>,
90    fingerprint: UnitFingerprint,
91    content: FragmentFingerprint,
92    normalized_content: FragmentFingerprint,
93    range: ByteRange,
94    lines: (u32, u32),
95    tokens: (usize, usize),
96    name: Option<Lexeme>,
97    boilerplate: Option<Boilerplate>,
98    test_code: bool,
99    test_code_evidence: Option<TestCodeEvidence>,
100    /// The preprocessor conditionals the unit sits under, if any.
101    arms: ArmPath,
102}
103
104impl Unit {
105    /// Content domain used to identify a clone relation of `class`.
106    const fn group_content(&self, class: CloneClass) -> FragmentFingerprint {
107        if matches!(class, CloneClass::Type1) {
108            self.content
109        } else {
110            self.normalized_content
111        }
112    }
113}
114
115#[cfg(test)]
116#[allow(clippy::expect_used)]
117mod tests;
codehelion_core/structural.rs

codehelion_core/
structural.rs
