Struct ContentDedup 

pub struct ContentDedup { /* private fields */ }

Deduplication by content (hash + simhash)

Implementations

impl ContentDedup

pub fn new(hamming_threshold: u32) -> Self

New deduplicator with Hamming threshold

pub fn hash_content(content: &str) -> String

SHA-256 hash of content

pub fn simhash(content: &str) -> u64

Simhash of content (for near-duplicates)

pub fn hamming_distance(a: u64, b: u64) -> u32

Hamming distance between two simhashes

pub fn is_exact_duplicate(&self, content: &str) -> bool

Check if content is an exact duplicate

pub fn is_near_duplicate(&self, content: &str) -> bool

Check if content is a near-duplicate

pub fn is_duplicate(&self, content: &str) -> bool

Check if duplicate (exact or near)

pub fn mark_seen(&self, content: &str)

Mark content as seen

pub fn check_and_mark(&self, content: &str) -> bool

Check and mark in one operation

pub fn count(&self) -> usize

Number of contents seen (exact)

pub fn clear(&self)

Clear the deduplicator

Trait Implementations

impl Default for ContentDedup

fn default() -> Self

Returns the “default value” for a type.