pub mod agreement;
pub mod blocking;
mod canonical;
pub mod component;
pub mod engine;
#[cfg(feature = "rng")]
pub mod eval;
pub mod frequency;
pub mod minhash;
pub mod record;
pub mod velocity;
pub use agreement::AgreementStore;
pub use blocking::CandidateSource;
pub use component::{FieldSpec, Kind, classify};
pub use engine::{Decision, MatchOutcome};
pub use frequency::FrequencyStore;
pub use record::FingerprintStore;
pub use velocity::{VelocityBand, VelocityStore};
use std::{
collections::{BTreeMap, HashSet},
fmt,
};
use serde_json::Value;
use self::{
agreement::AgreementTable,
blocking::{BlockingIndex, BlockingKey, DEFAULT_MAX_BLOCK},
canonical::{all_scalar_key, canonical_scalar, group_key, object_entries, value_to_i64},
component::{Salt, Stored},
frequency::FrequencyTable,
minhash::MinHashLsh,
record::{FingerprintRecord, RecordStore},
velocity::VelocityTable,
};
#[derive(Debug, Clone, Copy)]
pub struct EvictionPolicy {
pub max_records: Option<usize>,
pub record_ttl_ms: Option<u64>,
pub max_frequency_values: Option<usize>,
pub max_agreement_components: Option<usize>,
pub max_velocity_keys: Option<usize>,
pub max_block: usize,
}
impl Default for EvictionPolicy {
fn default() -> Self {
Self {
max_records: Some(1_000_000),
record_ttl_ms: None,
max_frequency_values: Some(1_000_000),
max_agreement_components: Some(1024),
max_velocity_keys: Some(1_000_000),
max_block: DEFAULT_MAX_BLOCK,
}
}
}
pub struct FuzzyStore {
salt: Salt,
records: Box<dyn FingerprintStore>,
blocking: Box<dyn CandidateSource>,
minhash: MinHashLsh,
frequency: Box<dyn FrequencyStore>,
agreement: Box<dyn AgreementStore>,
velocity: Box<dyn VelocityStore>,
identify_lock: std::sync::Mutex<()>,
}
impl fmt::Debug for FuzzyStore {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("FuzzyStore")
.field("salt", &self.salt)
.field("minhash", &self.minhash)
.finish_non_exhaustive()
}
}
#[cfg(feature = "rng")]
impl Default for FuzzyStore {
fn default() -> Self {
Self::new()
}
}
impl FuzzyStore {
#[cfg(feature = "rng")]
pub fn new() -> Self {
Self::new_with_policy(EvictionPolicy::default())
}
#[cfg(feature = "rng")]
pub fn new_with_policy(policy: EvictionPolicy) -> Self {
Self {
salt: Salt::random(),
records: Box::new(RecordStore::with_capacity(
policy.max_records,
policy.record_ttl_ms,
)),
blocking: Box::new(BlockingIndex::with_max_block(policy.max_block)),
minhash: MinHashLsh::new(),
frequency: Box::new(FrequencyTable::with_capacity(policy.max_frequency_values)),
agreement: Box::new(AgreementTable::with_capacity(
policy.max_agreement_components,
)),
velocity: Box::new(VelocityTable::with_capacity(policy.max_velocity_keys)),
identify_lock: std::sync::Mutex::new(()),
}
}
pub fn deterministic(secret: &[u8]) -> Self {
Self {
salt: Salt::from_secret(secret),
records: Box::new(RecordStore::new()),
blocking: Box::new(BlockingIndex::new()),
minhash: MinHashLsh::from_seed(secret),
frequency: Box::new(FrequencyTable::new()),
agreement: Box::new(AgreementTable::new()),
velocity: Box::new(VelocityTable::new()),
identify_lock: std::sync::Mutex::new(()),
}
}
pub fn from_backends(
salt: Salt,
minhash: MinHashLsh,
records: Box<dyn FingerprintStore>,
blocking: Box<dyn CandidateSource>,
frequency: Box<dyn FrequencyStore>,
agreement: Box<dyn AgreementStore>,
velocity: Box<dyn VelocityStore>,
) -> Self {
Self {
salt,
records,
blocking,
minhash,
frequency,
agreement,
velocity,
identify_lock: std::sync::Mutex::new(()),
}
}
pub fn observe(&self, visitor: &str, components: &Value, now_ms: u64) -> bool {
let stored = self.stored_map(components);
for value in stored.values() {
match value {
Stored::Category(hash) => self.frequency.record(*hash),
Stored::Numeric(n) => self.frequency.record(self.salt.hash(&n.to_string())),
Stored::Set(_) => {}
}
}
for key in self.blocking_keys(&stored) {
self.blocking.insert(key, visitor);
}
self.records.observe(visitor, stored, now_ms)
}
pub fn erase(&self, visitor: &str) -> bool {
let _guard = self
.identify_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let existed = self.records.remove(visitor);
self.blocking.remove_visitor(visitor);
existed
}
pub fn purge_expired(&self, now_ms: u64, max_age_ms: u64) -> u64 {
let _guard = self
.identify_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
self.records.purge_older_than(now_ms, max_age_ms)
}
pub fn candidates(&self, components: &Value) -> HashSet<String> {
let stored = self.stored_map(components);
self.blocking.candidates(&self.blocking_keys(&stored))
}
pub fn blocking_key_hexes(&self, components: &Value) -> Vec<String> {
let stored = self.stored_map(components);
self.blocking_keys(&stored)
.iter()
.map(hex::encode)
.collect()
}
fn stored_map(&self, components: &Value) -> BTreeMap<String, Stored> {
let mut stored = BTreeMap::new();
for (name, value) in object_entries(components) {
if let Some(value) = self.to_stored(name, value) {
stored.insert(name.clone(), value);
}
}
stored
}
pub fn record(&self, visitor: &str) -> Option<FingerprintRecord> {
self.records.get(visitor)
}
pub fn frequency(&self) -> &dyn FrequencyStore {
self.frequency.as_ref()
}
pub fn blocking(&self) -> &dyn CandidateSource {
self.blocking.as_ref()
}
fn to_stored(&self, name: &str, value: &Value) -> Option<Stored> {
match classify(name).kind {
Kind::Category => canonical_scalar(value).map(|s| Stored::Category(self.salt.hash(&s))),
Kind::Numeric => value_to_i64(value).map(Stored::Numeric),
Kind::Set => value.as_array().map(|items| {
Stored::Set(self.salt.hash_set(items.iter().filter_map(Value::as_str)))
}),
}
}
fn blocking_keys(&self, stored: &BTreeMap<String, Stored>) -> Vec<BlockingKey> {
let mut keys = Vec::new();
if let Some(key) = all_scalar_key(stored) {
keys.push(key);
}
if let Some(key) = group_key(stored, b"K1", &["webgl", "platform", "timezone"]) {
keys.push(key);
}
if let Some(key) = group_key(stored, b"K2", &["audio", "cpu_cores", "device_memory"]) {
keys.push(key);
}
if let Some(Stored::Set(fonts)) = stored.get("fonts") {
keys.extend(self.minhash.band_keys(fonts));
}
keys
}
}
#[cfg(test)]
mod tests {
use super::{EvictionPolicy, FuzzyStore};
use serde_json::json;
fn full_probe() -> serde_json::Value {
json!({
"webgl": "ANGLE (Intel)",
"platform": "Linux x86_64",
"timezone": "Asia/Shanghai",
"audio": "124.04",
"cpu_cores": 8,
"device_memory": 8,
"fonts": ["Arial", "Helvetica", "Courier", "Times", "Verdana"],
"user_agent": "Chrome/120",
})
}
#[test]
fn observe_populates_record_and_recalls_self() {
let store = FuzzyStore::new();
let probe = full_probe();
assert!(store.observe("v1", &probe, 1_000));
let record = store.record("v1").unwrap();
assert_eq!(record.observation_count, 1);
assert_eq!(record.first_seen, 1_000);
assert!(record.components.contains_key("webgl"));
assert!(record.components.contains_key("fonts"));
assert!(store.candidates(&probe).contains("v1"));
}
#[test]
fn recall_survives_a_changed_ua_and_one_font() {
let store = FuzzyStore::new();
store.observe("v1", &full_probe(), 1_000);
let mut drifted = full_probe();
drifted["user_agent"] = json!("Chrome/121");
drifted["fonts"] = json!(["Arial", "Helvetica", "Courier", "Times", "Segoe"]);
assert!(store.candidates(&drifted).contains("v1"));
}
#[test]
fn unrelated_probe_is_not_recalled() {
let store = FuzzyStore::new();
store.observe("v1", &full_probe(), 1_000);
let other = json!({
"webgl": "Apple GPU",
"platform": "iPhone",
"timezone": "America/New_York",
"audio": "35.7",
"cpu_cores": 6,
"device_memory": 4,
"fonts": ["SF Pro", "Menlo", "Georgia", "Palatino"],
"user_agent": "Safari/17",
});
assert!(!store.candidates(&other).contains("v1"));
}
#[test]
fn frequency_tracks_category_values() {
let store = FuzzyStore::new();
store.observe("v1", &full_probe(), 1_000);
store.observe("v2", &full_probe(), 2_000);
let hash = store.salt.hash("Linux x86_64");
assert_eq!(store.frequency().count(hash), 2);
}
#[test]
fn missing_components_are_skipped_not_stored() {
let store = FuzzyStore::new();
store.observe("v1", &json!({ "webgl": "x", "canvas": null }), 1_000);
let record = store.record("v1").unwrap();
assert!(record.components.contains_key("webgl"));
assert!(!record.components.contains_key("canvas"));
}
#[test]
fn new_with_policy_bounds_the_record_library() {
let policy = EvictionPolicy {
max_records: Some(2),
..EvictionPolicy::default()
};
let store = FuzzyStore::new_with_policy(policy);
store.observe("v1", &full_probe(), 1_000);
store.observe("v2", &full_probe(), 2_000);
store.observe("v3", &full_probe(), 3_000);
assert!(store.record("v1").is_none()); assert!(store.record("v2").is_some());
assert!(store.record("v3").is_some());
}
#[test]
fn erase_removes_record_and_blocking_so_device_is_new_again() {
let store = FuzzyStore::new();
let probe = full_probe();
store.observe("v1", &probe, 1_000);
assert!(store.candidates(&probe).contains("v1"));
assert!(store.erase("v1"));
assert!(store.record("v1").is_none());
assert!(!store.candidates(&probe).contains("v1"));
assert!(!store.erase("v1"));
let outcome = store.identify(&probe, 2_000);
assert!(outcome.is_new_device);
}
#[test]
fn purge_expired_drops_aged_records_under_lock() {
let store = FuzzyStore::new();
store.observe("old", &full_probe(), 1_000);
store.observe("fresh", &full_probe(), 9_000);
assert_eq!(store.purge_expired(10_000, 5_000), 1);
assert!(store.record("old").is_none());
assert!(store.record("fresh").is_some());
assert_eq!(store.purge_expired(10_000, 5_000), 0);
}
#[test]
fn non_object_probe_records_visitor_with_no_components() {
let store = FuzzyStore::new();
assert!(store.observe("v1", &json!("not-an-object"), 1_000));
assert!(store.record("v1").unwrap().components.is_empty());
}
#[derive(Debug)]
struct CountingRecordStore {
inner: super::record::RecordStore,
observes: std::sync::Arc<std::sync::atomic::AtomicU64>,
}
impl super::FingerprintStore for CountingRecordStore {
fn observe(
&self,
visitor: &str,
components: std::collections::BTreeMap<String, super::component::Stored>,
now_ms: u64,
) -> bool {
self.observes
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
self.inner.observe(visitor, components, now_ms)
}
fn get(&self, visitor: &str) -> Option<super::record::FingerprintRecord> {
self.inner.get(visitor)
}
fn remove(&self, visitor: &str) -> bool {
self.inner.remove(visitor)
}
}
#[test]
fn injected_record_backend_is_actually_used() {
use std::sync::{
Arc,
atomic::{AtomicU64, Ordering},
};
let observes = Arc::new(AtomicU64::new(0));
let store = FuzzyStore::from_backends(
super::Salt::from_secret(b"seam-test"),
super::MinHashLsh::from_seed(b"seam-test"),
Box::new(CountingRecordStore {
inner: super::record::RecordStore::new(),
observes: Arc::clone(&observes),
}),
Box::new(super::BlockingIndex::new()),
Box::new(super::FrequencyTable::new()),
Box::new(super::AgreementTable::new()),
Box::new(super::VelocityTable::new()),
);
assert!(store.observe("v1", &full_probe(), 1_000));
assert_eq!(observes.load(Ordering::Relaxed), 1);
assert!(store.candidates(&full_probe()).contains("v1"));
assert_eq!(store.record("v1").unwrap().observation_count, 1);
let matched = store.identify(&full_probe(), 2_000);
assert_eq!(matched.decision, super::Decision::Match);
assert_eq!(matched.visitor_id, "v1");
assert_eq!(observes.load(Ordering::Relaxed), 2);
}
}