santh-dedup 0.1.2

High-performance dataset deduplication for ML training data using MinHash + LSH
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

/// A cluster of duplicate documents.
#[derive(Debug, Clone, PartialEq)]
pub struct DuplicateCluster {
    /// Cluster ID (sequential from 0).
    pub id: usize,
    /// Document indices in this cluster.
    pub indices: Vec<usize>,
    /// Representative index (first in cluster).
    pub representative: usize,
}

impl DuplicateCluster {
    /// Create a new duplicate cluster.
    pub fn new(id: usize, representative: usize) -> Self {
        Self {
            id,
            indices: vec![representative],
            representative,
        }
    }

    /// Add an index to the cluster.
    pub fn add(&mut self, index: usize) {
        self.indices.push(index);
    }

    /// Get the number of documents.
    #[must_use]
    pub fn len(&self) -> usize {
        self.indices.len()
    }

    /// Check if cluster has no documents.
    #[must_use]
    pub fn is_empty(&self) -> bool {
        self.indices.is_empty()
    }

    /// Check if cluster has duplicates.
    #[must_use]
    pub fn is_duplicate(&self) -> bool {
        self.len() > 1
    }

    /// Check if this cluster contains a document.
    #[must_use]
    pub fn contains(&self, index: usize) -> bool {
        self.indices.contains(&index)
    }
}