codehelion_core/grouping.rs
1//! Structural-mode clone grouping: turning verified pairs into cohesive groups.
2//!
3//! Type-3 similarity is *not* transitive: A resembles B and B resembles C does
4//! not make A resemble C. Feeding verified pairs straight into a union-find and
5//! emitting the connected components would therefore fuse a chain of drifting
6//! near-clones into one incoherent group (AGENTS.md §2-9). Union-find is used
7//! here for one thing only — carving the pair graph into independent components
8//! so the expensive per-group work is bounded — and its components are never
9//! output as groups. Every component is then refined:
10//!
11//! 1. a **medoid** (canonical instance) is chosen as the member with the
12//! greatest total similarity to the rest, ties broken by the smallest stable
13//! key so the choice is deterministic;
14//! 2. the **medoid constraint** ejects any member too far from the medoid; the
15//! ejected members are regrouped among themselves rather than dropped;
16//! 3. **complete-linkage** refinement then removes members until the weakest
17//! pair inside the group clears the cohesion floor, so every pair in a
18//! reported group — not merely every member-to-medoid edge — is similar.
19//!
20//! Refinement is quadratic in component size, so a component past
21//! [`GroupingConfig::max_component`] is cut into pieces first and each piece
22//! refined on its own. That costs recall and never soundness — the rules that
23//! make a group cohesive are unchanged — and the count of components it fired
24//! on is reported rather than left to be inferred from the timing.
25//!
26//! A pair that verification never proposed has no edge here; its similarity is
27//! taken as zero, which is what makes the complete-linkage floor split a chain
28//! whose ends were never compared. Singletons are not clone groups. The whole
29//! module is a pure, deterministic function of its inputs: components,
30//! candidate medoids under sampling, and every output collection are ordered by
31//! stable key, never by discovery order.
32//!
33//! # What this asks of the stages above
34//!
35//! Reading an absent edge as a similarity of zero is the same as saying the two
36//! were weighed and found apart. That holds while the stages above are complete
37//! *per set*: a family they decline to propose at all is a family nothing here
38//! claims anything about, and a family they propose is one every pair of which
39//! they proposed. It stops holding the moment a ceiling leaves a family half
40//! proposed — then a set of copies arrives looking like a set that disagrees,
41//! refinement breaks it up, and the comparisons that did survive are carried
42//! out one at a time as pairs no group holds both halves of. One duplication
43//! comes back as many, and the report grows as the allowance shrinks.
44//!
45//! So a ceiling upstream of here has to cut between sets and never inside one.
46//! Two have been found doing otherwise — the candidate-pair budget, which used
47//! to stop in the middle of a posting list, and this module's own
48//! [`GroupingConfig::max_component`], which cuts a component it cannot refine
49//! whole. The first was changed to stop between posting lists; the second
50//! cannot be, since cutting is the whole point of it, so it reports which
51//! members it put apart ([`GroupingSet::severed_by_the_ceiling`]) and the
52//! caller counts those relations rather than stating them.
53//!
54//! A ceiling that drops a whole set is fine and needs none of this: the
55//! high-frequency posting cap drops entire lists, and lowering it onto the
56//! labelled corpora only ever costs findings, never multiplies them. The
57//! distinction is not how much a ceiling removes but whether what it leaves is
58//! a set that was compared with itself.
59
60use std::collections::{BTreeMap, BTreeSet};
61
62use crate::clone_class::CloneClass;
63use crate::verify::Confidence;
64
65/// Version of the rules that decide which occurrences sit in one group.
66///
67/// Recorded beside every run so a later one can say whether two results were
68/// grouped alike. Raising it does not move a member's content id — the same
69/// code still hashes the same — but it can move a group's, because a group
70/// fingerprint folds in the set of contents its members hold.
71///
72/// It stays at v1 until the first release tag, along with every other version
73/// this build records. A second number would only describe an audit database
74/// somebody still has on disk, and re-running the scan is the whole of the
75/// recovery; changing medoid selection, the cohesion floors or the refinement
76/// order therefore leaves this constant alone.
77pub const GROUPING_VERSION: &str = "grouping-v1";
78
79/// Tuning for grouping. Similarities are in `[0, 1]`; the defaults are
80/// provisional and calibrated against the chain corpus.
81#[derive(Debug, Clone, PartialEq)]
82pub struct GroupingConfig {
83 /// Smallest similarity a member may have to the medoid and stay in the
84 /// group; members below it are ejected and regrouped.
85 pub medoid_min_similarity: f64,
86 /// Complete-linkage floor: the smallest similarity any pair inside a
87 /// reported group may have. Below it the group is split.
88 pub min_pairwise_similarity: f64,
89 /// Component size above which medoid selection samples candidates rather
90 /// than scoring every member, to avoid quadratic blow-up on huge
91 /// components. The sample is deterministic and key-diverse, so repeated
92 /// content cannot occupy every medoid candidate.
93 pub sampling_threshold: usize,
94 /// Number of candidate medoids scored when a component exceeds
95 /// [`Self::sampling_threshold`].
96 pub sample_size: usize,
97 /// Largest component refined as one piece. A component above this is cut
98 /// into key-ordered pieces, each refined on its own. Content-identical
99 /// units are never separated: one equivalence class can therefore exceed
100 /// this limit by itself.
101 ///
102 /// Refinement materializes the component's pairwise similarity matrix and
103 /// orders it once, so it costs O(k² log k) time and O(k²) memory. A
104 /// codebase of thousands of structurally interchangeable units — generated
105 /// code, or a repository built to make the scan expensive — still produces
106 /// exactly that component, which is why the ceiling exists at all
107 /// (AGENTS.md §2-10, §7).
108 ///
109 /// Cutting costs recall, never soundness: each piece is refined by the
110 /// same medoid and complete-linkage rules, so every reported group is
111 /// still cohesive. What is lost is the chance that two members landing in
112 /// different pieces would have grouped. The cut is by stable key, so it is
113 /// deterministic, and the count of components it fired on is reported.
114 /// Keeping equal-key units together prevents independently cut pieces
115 /// from minting the same content-derived group and finding identifiers.
116 ///
117 /// Which members the cut put apart is reported too, through
118 /// [`GroupingSet::severed_by_the_ceiling`]. A caller that carries out the
119 /// verified relations no group expresses needs it: a relation across the
120 /// cut is not one refinement weighed and declined, and carrying it out
121 /// would restate the set once per crossing — at the size that makes this
122 /// ceiling fire, that is the whole report.
123 pub max_component: usize,
124}
125
126impl Default for GroupingConfig {
127 fn default() -> Self {
128 Self {
129 medoid_min_similarity: 0.60,
130 min_pairwise_similarity: 0.60,
131 sampling_threshold: 256,
132 sample_size: 32,
133 // Above the sampling threshold, so a component between the two is
134 // still refined whole with a sampled medoid.
135 max_component: 1024,
136 }
137 }
138}
139
140/// One verified similarity relation between two units, as produced by
141/// [`crate::verify`]. Endpoints are indices into the unit slice passed to
142/// [`group`]; the pair is undirected and `a != b` is required.
143#[derive(Debug, Clone, Copy, PartialEq)]
144pub struct SimilarityEdge {
145 /// One endpoint (a unit index).
146 pub a: usize,
147 /// The other endpoint (a unit index).
148 pub b: usize,
149 /// The pair's grouping similarity, in `[0, 1]` (the verdict composite).
150 pub similarity: f64,
151 /// Per-dimension evidence behind `similarity`, when the verifier measured
152 /// it. Generic grouping clients that have only a scalar may leave this
153 /// absent; Structural mode always preserves its verifier breakdown.
154 pub breakdown: Option<crate::verify::SimilarityBreakdown>,
155 /// The pair's clone classification.
156 pub class: CloneClass,
157 /// The pair's confidence.
158 pub confidence: Confidence,
159}
160
161/// A unit as seen by grouping: only its stable key matters here.
162///
163/// The key is used for deterministic tie-breaking and ordering. It is the
164/// unit's content fingerprint bytes; grouping never interprets it beyond
165/// ordering.
166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
167pub struct GroupingUnit {
168 /// Stable, position-free key (a content fingerprint's bytes).
169 pub key: [u8; 16],
170}
171
172/// A cohesive clone group: a medoid plus the members that cleared both the
173/// medoid constraint and the complete-linkage floor.
174#[derive(Debug, Clone, PartialEq)]
175pub struct StructuralGroup {
176 /// The weakest clone class among the group's internal edges (a group is no
177 /// stronger than its loosest accepted pair).
178 pub clone_type: CloneClass,
179 /// The weakest confidence among the group's internal edges.
180 pub confidence: Confidence,
181 /// The medoid: the group's canonical instance (a unit index).
182 pub canonical: usize,
183 /// Member unit indices, the medoid first, then the rest by ascending key.
184 pub members: Vec<usize>,
185 /// Similarity of each member to the medoid, parallel to [`Self::members`]
186 /// (the medoid's own entry is `1.0`).
187 pub medoid_similarities: Vec<f64>,
188 /// The weakest pairwise similarity inside the group: its cohesion, at or
189 /// above [`GroupingConfig::min_pairwise_similarity`].
190 pub min_pairwise: f64,
191}
192
193/// Counters describing what grouping saw and did.
194#[derive(Debug, Clone, Default, PartialEq, Eq)]
195pub struct GroupingStats {
196 /// Units considered (the input length).
197 pub units: usize,
198 /// Verified edges considered.
199 pub edges: usize,
200 /// Initial connected components carved by union-find.
201 pub components: usize,
202 /// Components too large to refine as one piece, cut into pieces of
203 /// [`GroupingConfig::max_component`]. Reported because the cut can leave
204 /// clones of each other in separate groups.
205 pub oversized_components: usize,
206 /// Groups emitted after medoid and complete-linkage refinement.
207 pub groups: usize,
208 /// Members ejected by the medoid constraint (and regrouped elsewhere).
209 pub medoid_ejections: usize,
210 /// Components whose medoid candidates were sampled rather than exhaustively
211 /// scored.
212 pub sampled_medoids: usize,
213 /// Total distinct-content medoid candidates scored in sampled components.
214 pub sampled_medoid_candidates: usize,
215 /// Members removed by complete-linkage splitting.
216 pub linkage_splits: usize,
217 /// Members left ungrouped as singletons after refinement.
218 pub singletons: usize,
219}
220
221/// The grouping result: refined groups plus statistics.
222#[derive(Debug, Clone, PartialEq)]
223pub struct GroupingSet {
224 /// Cohesive groups, ordered by their medoid's key.
225 pub groups: Vec<StructuralGroup>,
226 /// Which piece each unit of a cut component landed in.
227 ///
228 /// Empty unless [`GroupingConfig::max_component`] fired. Units of a
229 /// component small enough to refine whole are absent, because nothing
230 /// about them was decided by the ceiling.
231 piece_of: BTreeMap<usize, u32>,
232 /// What grouping saw and did.
233 pub stats: GroupingStats,
234}
235
236impl GroupingSet {
237 /// Whether the component ceiling is why these two were never weighed
238 /// against each other.
239 ///
240 /// Two units in one component that the ceiling cut into pieces, and in
241 /// different pieces, were never candidates for the same group — not
242 /// because refinement judged them apart but because refinement never saw
243 /// them together. A caller carrying out the relations no group expresses
244 /// has to tell that apart from the ones a group declined to hold, which
245 /// are a fact about the code rather than about a ceiling.
246 #[must_use]
247 pub fn severed_by_the_ceiling(&self, a: usize, b: usize) -> bool {
248 match (self.piece_of.get(&a), self.piece_of.get(&b)) {
249 (Some(left), Some(right)) => left != right,
250 _ => false,
251 }
252 }
253}
254
255/// Group verified pairs into cohesive clone groups.
256///
257/// The result is a pure function of the inputs: neither the edge order nor the
258/// unit order (beyond what the indices name) changes the groups or their order.
259#[must_use]
260pub fn group(
261 units: &[GroupingUnit],
262 edges: &[SimilarityEdge],
263 config: &GroupingConfig,
264) -> GroupingSet {
265 let sim = SimilarityGraph::build(units.len(), edges);
266 let mut stats = GroupingStats {
267 units: units.len(),
268 edges: edges.len(),
269 ..GroupingStats::default()
270 };
271
272 let components = connected_components(units.len(), edges);
273 stats.components = components.len();
274
275 let mut groups = Vec::new();
276 // Which piece a unit landed in, recorded only where the ceiling cut, so a
277 // relation the cut prevented can later be told from one refinement weighed
278 // and declined.
279 let mut piece_of: BTreeMap<usize, u32> = BTreeMap::new();
280 let mut next_piece = 0u32;
281 for component in &components {
282 let cut = component.len() > piece_limit(config);
283 for piece in refinable_pieces(component, units, config, &mut stats) {
284 if cut {
285 for &member in &piece {
286 piece_of.insert(member, next_piece);
287 }
288 next_piece += 1;
289 }
290 refine_component(&piece, units, &sim, config, &mut groups, &mut stats);
291 }
292 }
293
294 // Deterministic output order: by the medoid's key, then by group content.
295 groups.sort_by(|left, right| {
296 units[left.canonical]
297 .key
298 .cmp(&units[right.canonical].key)
299 .then(left.members.len().cmp(&right.members.len()))
300 .then_with(|| {
301 left.members
302 .iter()
303 .map(|&member| units[member].key)
304 .cmp(right.members.iter().map(|&member| units[member].key))
305 })
306 });
307 stats.groups = groups.len();
308 GroupingSet {
309 groups,
310 piece_of,
311 stats,
312 }
313}
314
315/// The largest set refinement runs on as one piece.
316///
317/// At least two, because a ceiling of one would cut every pair apart and leave
318/// nothing that could group at all.
319const fn piece_limit(config: &GroupingConfig) -> usize {
320 if config.max_component > 2 {
321 config.max_component
322 } else {
323 2
324 }
325}
326
327/// Symmetric similarity lookup over the verified edges. Absent pairs read as
328/// zero — units verification never compared are treated as dissimilar.
329struct SimilarityGraph {
330 edges: BTreeMap<(usize, usize), EdgeData>,
331}
332
333#[derive(Debug, Clone, Copy)]
334struct EdgeData {
335 similarity: f64,
336 class: CloneClass,
337 confidence: Confidence,
338}
339
340impl SimilarityGraph {
341 fn build(_unit_count: usize, edges: &[SimilarityEdge]) -> Self {
342 let mut map = BTreeMap::new();
343 for edge in edges {
344 if edge.a == edge.b {
345 continue;
346 }
347 let key = ordered(edge.a, edge.b);
348 // Keep the strongest edge if a pair is listed more than once, so
349 // the result never depends on input order.
350 let data = EdgeData {
351 similarity: edge.similarity,
352 class: edge.class,
353 confidence: edge.confidence,
354 };
355 map.entry(key)
356 .and_modify(|existing: &mut EdgeData| {
357 if edge.similarity > existing.similarity {
358 *existing = data;
359 }
360 })
361 .or_insert(data);
362 }
363 Self { edges: map }
364 }
365
366 fn similarity(&self, a: usize, b: usize) -> f64 {
367 if a == b {
368 return 1.0;
369 }
370 self.edges
371 .get(&ordered(a, b))
372 .map_or(0.0, |data| data.similarity)
373 }
374
375 fn edge(&self, a: usize, b: usize) -> Option<EdgeData> {
376 if a == b {
377 return None;
378 }
379 self.edges.get(&ordered(a, b)).copied()
380 }
381}
382
383/// Normalize an undirected endpoint pair to `(min, max)`.
384const fn ordered(a: usize, b: usize) -> (usize, usize) {
385 if a <= b { (a, b) } else { (b, a) }
386}
387
388/// Carve the pair graph into connected components. This is the *only* use of
389/// union-find here: its components seed the per-component refinement and are
390/// never emitted as groups (a chain of near-clones is one component but many
391/// groups). Members are returned sorted by key-independent index; refinement
392/// re-sorts by key.
393fn connected_components(unit_count: usize, edges: &[SimilarityEdge]) -> Vec<Vec<usize>> {
394 let mut parent: Vec<usize> = (0..unit_count).collect();
395 let mut connected = BTreeSet::new();
396 for edge in edges {
397 if edge.a != edge.b {
398 union(&mut parent, edge.a, edge.b);
399 connected.insert(edge.a);
400 connected.insert(edge.b);
401 }
402 }
403 let mut buckets: BTreeMap<usize, Vec<usize>> = BTreeMap::new();
404 for node in connected {
405 let root = find(&mut parent, node);
406 buckets.entry(root).or_default().push(node);
407 }
408 buckets.into_values().collect()
409}
410
411fn find(parent: &mut [usize], node: usize) -> usize {
412 let mut root = node;
413 while parent[root] != root {
414 root = parent[root];
415 }
416 // Path compression.
417 let mut current = node;
418 while parent[current] != root {
419 let next = parent[current];
420 parent[current] = root;
421 current = next;
422 }
423 root
424}
425
426fn union(parent: &mut [usize], a: usize, b: usize) {
427 let ra = find(parent, a);
428 let rb = find(parent, b);
429 if ra != rb {
430 // Attach the larger root under the smaller for a deterministic forest.
431 if ra < rb {
432 parent[rb] = ra;
433 } else {
434 parent[ra] = rb;
435 }
436 }
437}
438
439/// The pieces of a component that refinement runs on: the component itself
440/// when it fits under [`GroupingConfig::max_component`], otherwise key-ordered
441/// pieces. An equal-key equivalence class is atomic: splitting it would create
442/// separate groups with the same content-derived identity.
443fn refinable_pieces(
444 component: &[usize],
445 units: &[GroupingUnit],
446 config: &GroupingConfig,
447 stats: &mut GroupingStats,
448) -> Vec<Vec<usize>> {
449 let limit = piece_limit(config);
450 if component.len() <= limit {
451 return vec![component.to_vec()];
452 }
453 stats.oversized_components += 1;
454 let mut ordered = component.to_vec();
455 ordered.sort_by_key(|&member| units[member].key);
456 let mut pieces = Vec::new();
457 let mut current = Vec::new();
458 let mut class_start = 0;
459 while class_start < ordered.len() {
460 let key = units[ordered[class_start]].key;
461 let class_end = ordered[class_start..]
462 .iter()
463 .position(|&member| units[member].key != key)
464 .map_or(ordered.len(), |offset| class_start + offset);
465 let class = &ordered[class_start..class_end];
466 if !current.is_empty() && current.len() + class.len() > limit {
467 pieces.push(std::mem::take(&mut current));
468 }
469 current.extend_from_slice(class);
470 // A content class larger than the ceiling is indivisible. Emit it as
471 // one oversized piece rather than minting identical groups from its
472 // arbitrary sub-pieces.
473 if current.len() > limit {
474 pieces.push(std::mem::take(&mut current));
475 }
476 class_start = class_end;
477 }
478 if !current.is_empty() {
479 pieces.push(current);
480 }
481 pieces
482}
483
484/// Refine one component into cohesive groups, appending them to `groups`.
485///
486/// Terminates because each recursion runs on a strictly smaller set: a member
487/// is only ejected into `rest`, and the group built from `kept` never re-enters
488/// refinement.
489fn refine_component(
490 component: &[usize],
491 units: &[GroupingUnit],
492 sim: &SimilarityGraph,
493 config: &GroupingConfig,
494 groups: &mut Vec<StructuralGroup>,
495 stats: &mut GroupingStats,
496) {
497 if component.len() < 2 {
498 stats.singletons += component.len();
499 return;
500 }
501
502 let medoid = select_medoid(component, units, sim, config, stats);
503
504 // Medoid constraint: keep members close enough to the medoid, eject the
505 // rest for independent regrouping.
506 let mut kept = Vec::new();
507 let mut rest = Vec::new();
508 for &member in component {
509 if member == medoid || sim.similarity(member, medoid) >= config.medoid_min_similarity {
510 kept.push(member);
511 } else {
512 rest.push(member);
513 }
514 }
515 stats.medoid_ejections += rest.len();
516
517 // Complete-linkage: remove members until the weakest pair clears the floor.
518 complete_linkage_trim(medoid, &mut kept, &mut rest, units, sim, config, stats);
519
520 if let Some(built) = build_group(medoid, &kept, units, sim) {
521 groups.push(built);
522 } else {
523 stats.singletons += kept.len();
524 }
525
526 if !rest.is_empty() {
527 // Regroup the ejected members; deterministic order for recursion.
528 rest.sort_by_key(|&m| units[m].key);
529 refine_component(&rest, units, sim, config, groups, stats);
530 }
531}
532
533/// Choose the medoid: the member with the greatest total similarity to the
534/// others, ties broken by the smallest key. On components past the sampling
535/// threshold, candidates are selected evenly from distinct content keys. This
536/// keeps the cost bounded without allowing one repeated content to occupy the
537/// whole sample.
538fn select_medoid(
539 component: &[usize],
540 units: &[GroupingUnit],
541 sim: &SimilarityGraph,
542 config: &GroupingConfig,
543 stats: &mut GroupingStats,
544) -> usize {
545 let mut candidates: Vec<usize> = component.to_vec();
546 candidates.sort_by_key(|&m| units[m].key);
547 if candidates.len() > config.sampling_threshold {
548 candidates.dedup_by_key(|member| units[*member].key);
549 let sample_size = config.sample_size.max(1).min(candidates.len());
550 if candidates.len() > sample_size {
551 let last = candidates.len() - 1;
552 candidates = if sample_size == 1 {
553 vec![candidates[last / 2]]
554 } else {
555 (0..sample_size)
556 .map(|index| candidates[index * last / (sample_size - 1)])
557 .collect()
558 };
559 }
560 stats.sampled_medoids += 1;
561 stats.sampled_medoid_candidates += candidates.len();
562 }
563
564 let mut best = candidates[0];
565 let mut best_total = total_similarity(best, component, sim);
566 for &candidate in &candidates[1..] {
567 let total = total_similarity(candidate, component, sim);
568 // Greater total wins; on a tie the smaller key wins, keeping the pick
569 // deterministic without an exact float comparison.
570 let better = match total.total_cmp(&best_total) {
571 std::cmp::Ordering::Greater => true,
572 std::cmp::Ordering::Equal => units[candidate].key < units[best].key,
573 std::cmp::Ordering::Less => false,
574 };
575 if better {
576 best = candidate;
577 best_total = total;
578 }
579 }
580 best
581}
582
583/// Sum of a member's similarity to every other member of the set.
584fn total_similarity(member: usize, set: &[usize], sim: &SimilarityGraph) -> f64 {
585 set.iter()
586 .filter(|&&other| other != member)
587 .map(|&other| sim.similarity(member, other))
588 .sum()
589}
590
591/// Trim `kept` until its weakest pair reaches the complete-linkage floor,
592/// moving each removed member into `rest`. The medoid is never removed. The
593/// removed member of the weakest pair is the non-medoid one with the lower
594/// total similarity inside `kept` (ties broken by the larger key), so the
595/// choice is deterministic and progress is guaranteed.
596///
597/// Pair similarities do not change while a component is refined. Sort that
598/// matrix once, then discard inactive endpoints as members leave the set.
599/// Totals are a row cache: removing one member subtracts its row from every
600/// survivor. The old implementation re-scanned the whole matrix and then
601/// re-summed two rows for every ejection, which made this O(k³). This keeps
602/// the same decision rule in O(k² log k) time and O(k²) bounded memory.
603fn complete_linkage_trim(
604 medoid: usize,
605 kept: &mut Vec<usize>,
606 rest: &mut Vec<usize>,
607 units: &[GroupingUnit],
608 sim: &SimilarityGraph,
609 config: &GroupingConfig,
610 stats: &mut GroupingStats,
611) {
612 let members = kept.clone();
613 let mut active = vec![true; members.len()];
614 let mut totals = vec![0.0; members.len()];
615 let mut pairs = Vec::with_capacity(kept.len().saturating_mul(kept.len().saturating_sub(1)) / 2);
616 for (index, &left) in members.iter().enumerate() {
617 for (right_index, &right) in members.iter().enumerate().skip(index + 1) {
618 let similarity = sim.similarity(left, right);
619 totals[index] += similarity;
620 totals[right_index] += similarity;
621 pairs.push((
622 similarity,
623 canonical_pair(left, right, units),
624 index,
625 right_index,
626 ));
627 }
628 }
629 pairs.sort_by(|left, right| {
630 left.0
631 .total_cmp(&right.0)
632 .then_with(|| left.1.cmp(&right.1))
633 });
634 let mut next_pair = 0;
635
636 let mut active_count = members.len();
637 while active_count >= 2 {
638 while pairs
639 .get(next_pair)
640 .is_some_and(|(_, _, left, right)| !active[*left] || !active[*right])
641 {
642 next_pair += 1;
643 }
644 let Some(&(worst_sim, _, left, right)) = pairs.get(next_pair) else {
645 break;
646 };
647 if worst_sim >= config.min_pairwise_similarity {
648 break;
649 }
650 let victim = if members[left] == medoid {
651 right
652 } else if members[right] == medoid {
653 left
654 } else {
655 match totals[left].total_cmp(&totals[right]) {
656 std::cmp::Ordering::Less => left,
657 std::cmp::Ordering::Equal
658 if units[members[left]].key >= units[members[right]].key =>
659 {
660 left
661 }
662 std::cmp::Ordering::Greater | std::cmp::Ordering::Equal => right,
663 }
664 };
665 active[victim] = false;
666 for (index, &member) in members.iter().enumerate() {
667 if active[index] {
668 totals[index] -= sim.similarity(member, members[victim]);
669 }
670 }
671 active_count -= 1;
672 rest.push(members[victim]);
673 stats.linkage_splits += 1;
674 }
675 *kept = members
676 .into_iter()
677 .zip(active)
678 .filter_map(|(member, active)| active.then_some(member))
679 .collect();
680}
681
682/// An endpoint pair ordered by the units' stable keys. This is only used for
683/// deterministic tie-breaking; equal keys represent interchangeable content.
684fn canonical_pair(left: usize, right: usize, units: &[GroupingUnit]) -> ([u8; 16], [u8; 16]) {
685 let left_key = units[left].key;
686 let right_key = units[right].key;
687 if left_key <= right_key {
688 (left_key, right_key)
689 } else {
690 (right_key, left_key)
691 }
692}
693
694/// Assemble a group from a medoid and its kept members, or `None` when fewer
695/// than two members remain (a singleton is not a group).
696fn build_group(
697 medoid: usize,
698 kept: &[usize],
699 units: &[GroupingUnit],
700 sim: &SimilarityGraph,
701) -> Option<StructuralGroup> {
702 if kept.len() < 2 {
703 return None;
704 }
705 let mut ordered_members: Vec<usize> = kept.iter().copied().filter(|&m| m != medoid).collect();
706 ordered_members.sort_by_key(|&m| units[m].key);
707 ordered_members.insert(0, medoid);
708
709 let medoid_similarities: Vec<f64> = ordered_members
710 .iter()
711 .map(|&member| sim.similarity(medoid, member))
712 .collect();
713
714 // Weakest class, confidence and pairwise similarity across internal edges.
715 let mut clone_type = CloneClass::Type1;
716 let mut confidence = Confidence::High;
717 let mut min_pairwise = 1.0_f64;
718 for (i, &left) in ordered_members.iter().enumerate() {
719 for &right in &ordered_members[i + 1..] {
720 min_pairwise = min_pairwise.min(sim.similarity(left, right));
721 if let Some(data) = sim.edge(left, right) {
722 clone_type = weaker_class(clone_type, data.class);
723 confidence = weaker_confidence(confidence, data.confidence);
724 }
725 }
726 }
727
728 Some(StructuralGroup {
729 clone_type,
730 confidence,
731 canonical: medoid,
732 members: ordered_members,
733 medoid_similarities,
734 min_pairwise,
735 })
736}
737
738/// The looser of two classes: Type-3 is weakest, Type-1 strongest.
739const fn weaker_class(a: CloneClass, b: CloneClass) -> CloneClass {
740 match (a, b) {
741 (CloneClass::Type3, _) | (_, CloneClass::Type3) => CloneClass::Type3,
742 (CloneClass::Type2, _) | (_, CloneClass::Type2) => CloneClass::Type2,
743 _ => CloneClass::Type1,
744 }
745}
746
747/// The lower of two confidences.
748const fn weaker_confidence(a: Confidence, b: Confidence) -> Confidence {
749 match (a, b) {
750 (Confidence::Low, _) | (_, Confidence::Low) => Confidence::Low,
751 (Confidence::Medium, _) | (_, Confidence::Medium) => Confidence::Medium,
752 _ => Confidence::High,
753 }
754}
755
756#[cfg(test)]
757#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
758mod tests;