use std::collections::BTreeSet;
#[cfg(feature = "rng")]
use rand::{Rng, rng};
use sha2::{Digest, Sha256};
use super::{blocking::BlockingKey, component::Hash32};
const NUM_PERM: usize = 16;
const ROWS: usize = 2;
const BANDS: usize = NUM_PERM / ROWS;
const ODD_MASK: u64 = 1;
#[derive(Debug)]
pub struct MinHashLsh {
coefficients: [(u64, u64); NUM_PERM],
}
#[cfg(feature = "rng")]
impl Default for MinHashLsh {
fn default() -> Self {
Self::new()
}
}
impl MinHashLsh {
#[cfg(feature = "rng")]
pub fn new() -> Self {
let mut rng = rng();
let mut coefficients = [(0u64, 0u64); NUM_PERM];
for slot in &mut coefficients {
*slot = (rng.next_u64() | ODD_MASK, rng.next_u64());
}
Self { coefficients }
}
pub fn from_seed(seed: &[u8]) -> Self {
let mut coefficients = [(0u64, 0u64); NUM_PERM];
for (index, slot) in coefficients.iter_mut().enumerate() {
let mut hasher = Sha256::new();
hasher.update(seed);
hasher.update((index as u64).to_le_bytes());
let digest = hasher.finalize();
let a = u64::from_le_bytes(digest[..8].try_into().unwrap_or_default()) | ODD_MASK;
let b = u64::from_le_bytes(digest[8..16].try_into().unwrap_or_default());
*slot = (a, b);
}
Self { coefficients }
}
pub fn band_keys(&self, elements: &BTreeSet<Hash32>) -> Vec<BlockingKey> {
if elements.is_empty() {
return Vec::new();
}
let signature = self.signature(elements);
(0..BANDS)
.map(|band| {
let start = band * ROWS;
band_key(band, &signature[start..start + ROWS])
})
.collect()
}
fn signature(&self, elements: &BTreeSet<Hash32>) -> [u64; NUM_PERM] {
let mut signature = [u64::MAX; NUM_PERM];
for element in elements {
let x = leading_u64(element);
for (slot, &(a, b)) in signature.iter_mut().zip(self.coefficients.iter()) {
let hashed = a.wrapping_mul(x).wrapping_add(b);
if hashed < *slot {
*slot = hashed;
}
}
}
signature
}
}
fn leading_u64(hash: &Hash32) -> u64 {
let mut buf = [0u8; 8];
buf.copy_from_slice(&hash[..8]);
u64::from_le_bytes(buf)
}
fn band_key(band: usize, rows: &[u64]) -> BlockingKey {
let mut hasher = Sha256::new();
hasher.update(b"minhash-band");
hasher.update((band as u64).to_le_bytes());
for &row in rows {
hasher.update(row.to_le_bytes());
}
hasher.finalize().into()
}
#[cfg(test)]
mod tests {
use super::MinHashLsh;
use crate::fuzzy::component::Salt;
#[test]
fn similar_sets_share_a_band() {
let salt = Salt::random();
let lsh = MinHashLsh::new();
let base: Vec<String> = (0..30).map(|i| format!("font-{i}")).collect();
let mut changed = base.clone();
changed[0] = "font-new".to_string();
let a = lsh.band_keys(&salt.hash_set(&base));
let b = lsh.band_keys(&salt.hash_set(&changed));
assert!(
a.iter().any(|k| b.contains(k)),
"near-identical sets must collide in at least one band",
);
}
#[test]
fn identical_sets_share_every_band() {
let salt = Salt::random();
let lsh = MinHashLsh::new();
let set = salt.hash_set(["a", "b", "c", "d"]);
assert_eq!(lsh.band_keys(&set), lsh.band_keys(&set));
}
#[test]
fn disjoint_sets_do_not_collide() {
let salt = Salt::random();
let lsh = MinHashLsh::new();
let a = lsh.band_keys(&salt.hash_set(["a1", "a2", "a3", "a4"]));
let b = lsh.band_keys(&salt.hash_set(["z1", "z2", "z3", "z4"]));
assert!(a.iter().all(|k| !b.contains(k)));
}
#[test]
fn empty_set_yields_no_keys() {
let lsh = MinHashLsh::new();
let keys = lsh.band_keys(&Salt::random().hash_set(Vec::<String>::new()));
assert_eq!(keys.len(), 0);
}
}