bgustcrawler 0.2.0

Distributed web crawler and scheduler integrated with Kafka, SurrealDB, and S3 storage
Documentation
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::sync::RwLock;

pub struct BloomFilter {
    bit_vec: Vec<bool>,
    num_hashes: usize,
}

impl BloomFilter {
    pub fn new(size: usize, num_hashes: usize) -> Self {
        Self {
            bit_vec: vec![false; size],
            num_hashes,
        }
    }

    fn hashes<T: Hash>(&self, item: &T) -> Vec<usize> {
        let mut hashes = Vec::with_capacity(self.num_hashes);
        for i in 0..self.num_hashes {
            let mut hasher = DefaultHasher::new();
            item.hash(&mut hasher);
            i.hash(&mut hasher);
            let hash = hasher.finish() as usize;
            hashes.push(hash % self.bit_vec.len());
        }
        hashes
    }

    pub fn insert<T: Hash>(&mut self, item: &T) {
        for index in self.hashes(item) {
            self.bit_vec[index] = true;
        }
    }

    pub fn contains<T: Hash>(&self, item: &T) -> bool {
        for index in self.hashes(item) {
            if !self.bit_vec[index] {
                return false;
            }
        }
        true
    }
}

pub struct ThreadSafeBloomFilter {
    inner: RwLock<BloomFilter>,
}

impl ThreadSafeBloomFilter {
    pub fn new(size: usize, num_hashes: usize) -> Self {
        Self {
            inner: RwLock::new(BloomFilter::new(size, num_hashes)),
        }
    }

    pub fn insert<T: Hash>(&self, item: &T) {
        let mut writer = self.inner.write().unwrap();
        writer.insert(item);
    }

    pub fn contains<T: Hash>(&self, item: &T) -> bool {
        let reader = self.inner.read().unwrap();
        reader.contains(item)
    }
}