use std::collections::HashMap;
pub struct DedupConfig {
pub ttl_ms: u64,
pub capacity: usize,
}
impl Default for DedupConfig {
fn default() -> Self {
DedupConfig {
ttl_ms: 300_000,
capacity: 1_024,
}
}
}
#[derive(Debug, Clone)]
pub struct CacheEntry {
pub fingerprint: String,
pub value: String,
pub inserted_ms: u64,
pub hit_count: u32,
}
pub struct SemanticDedup {
config: DedupConfig,
cache: HashMap<String, CacheEntry>,
}
impl SemanticDedup {
pub fn new(config: DedupConfig) -> Self {
SemanticDedup {
config,
cache: HashMap::new(),
}
}
pub fn fingerprint(text: &str) -> String {
let lowered = text.to_lowercase();
let mut result = String::with_capacity(lowered.len().min(128));
let mut last_was_space = false;
for ch in lowered.chars() {
if ch.is_whitespace() {
if !last_was_space {
result.push(' ');
last_was_space = true;
}
} else if ch.is_alphanumeric() {
result.push(ch);
last_was_space = false;
}
}
let trimmed = result.trim_matches(' ');
if trimmed.chars().count() > 128 {
trimmed.chars().take(128).collect()
} else {
trimmed.to_string()
}
}
pub fn check(&mut self, text: &str, now_ms: u64) -> Option<&CacheEntry> {
self.evict_expired(now_ms);
let fp = Self::fingerprint(text);
if let Some(entry) = self.cache.get_mut(&fp) {
entry.hit_count += 1;
return self.cache.get(&fp);
}
None
}
pub fn register(&mut self, text: &str, value: String, now_ms: u64) {
self.evict_expired(now_ms);
let fp = Self::fingerprint(text);
if !self.cache.contains_key(&fp) && self.cache.len() >= self.config.capacity {
self.evict_lru();
}
self.cache.insert(
fp.clone(),
CacheEntry {
fingerprint: fp,
value,
inserted_ms: now_ms,
hit_count: 0,
},
);
}
pub fn evict_expired(&mut self, now_ms: u64) {
let ttl = self.config.ttl_ms;
self.cache
.retain(|_, entry| now_ms.saturating_sub(entry.inserted_ms) < ttl);
}
pub fn len(&self) -> usize {
self.cache.len()
}
pub fn is_empty(&self) -> bool {
self.cache.is_empty()
}
pub fn live_keys(&self, now_ms: u64) -> Vec<String> {
let ttl = self.config.ttl_ms;
let mut keys: Vec<String> = self
.cache
.iter()
.filter(|(_, entry)| now_ms.saturating_sub(entry.inserted_ms) < ttl)
.map(|(k, _)| k.clone())
.collect();
keys.sort();
keys
}
pub fn clear(&mut self) {
self.cache.clear();
}
fn evict_lru(&mut self) {
if self.cache.is_empty() {
return;
}
let oldest_key = self
.cache
.iter()
.min_by_key(|(_, entry)| entry.inserted_ms)
.map(|(k, _)| k.clone());
if let Some(key) = oldest_key {
self.cache.remove(&key);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_dedup() -> SemanticDedup {
SemanticDedup::new(DedupConfig::default())
}
fn make_dedup_with(ttl_ms: u64, capacity: usize) -> SemanticDedup {
SemanticDedup::new(DedupConfig { ttl_ms, capacity })
}
#[test]
fn test_fingerprint_lowercases_input() {
let fp = SemanticDedup::fingerprint("Hello WORLD");
assert_eq!(fp, "hello world");
}
#[test]
fn test_fingerprint_collapses_whitespace() {
let fp = SemanticDedup::fingerprint("hello world\t\nfoo");
assert_eq!(fp, "hello world foo");
}
#[test]
fn test_fingerprint_removes_punctuation() {
let fp = SemanticDedup::fingerprint("hello, world! How are you?");
assert_eq!(fp, "hello world how are you");
}
#[test]
fn test_fingerprint_trims_to_128_chars() {
let long_input: String = "a ".repeat(200); let fp = SemanticDedup::fingerprint(&long_input);
assert!(
fp.chars().count() <= 128,
"fingerprint must be at most 128 chars"
);
}
#[test]
fn test_fingerprint_empty_input_gives_empty() {
let fp = SemanticDedup::fingerprint("");
assert_eq!(fp, "");
}
#[test]
fn test_fingerprint_only_whitespace_gives_empty_or_single_space() {
let fp = SemanticDedup::fingerprint(" \t\n ");
assert!(fp.is_empty(), "unexpected: {:?}", fp);
}
#[test]
fn test_fingerprint_alphanumeric_unchanged() {
let fp = SemanticDedup::fingerprint("abc123");
assert_eq!(fp, "abc123");
}
#[test]
fn test_fingerprint_same_for_minor_variations_extra_spaces() {
let fp1 = SemanticDedup::fingerprint("hello world");
let fp2 = SemanticDedup::fingerprint("hello world");
assert_eq!(fp1, fp2);
}
#[test]
fn test_fingerprint_same_for_minor_variations_mixed_case() {
let fp1 = SemanticDedup::fingerprint("Hello World");
let fp2 = SemanticDedup::fingerprint("hello world");
assert_eq!(fp1, fp2);
}
#[test]
fn test_fingerprint_same_for_minor_variations_combined() {
let fp1 = SemanticDedup::fingerprint(" Hello, World! ");
let fp2 = SemanticDedup::fingerprint("hello world");
assert_eq!(fp1, fp2);
}
#[test]
fn test_fingerprint_different_for_different_content() {
let fp1 = SemanticDedup::fingerprint("hello world");
let fp2 = SemanticDedup::fingerprint("goodbye world");
assert_ne!(fp1, fp2);
}
#[test]
fn test_fingerprint_max_length_exactly_128() {
let input: String = "a".repeat(128);
let fp = SemanticDedup::fingerprint(&input);
assert_eq!(fp.chars().count(), 128);
}
#[test]
fn test_fingerprint_longer_than_128_truncated() {
let input: String = "a".repeat(200);
let fp = SemanticDedup::fingerprint(&input);
assert_eq!(fp.chars().count(), 128);
}
#[test]
fn test_fingerprint_is_deterministic() {
let text = "This IS a Test Prompt!!";
let fp1 = SemanticDedup::fingerprint(text);
let fp2 = SemanticDedup::fingerprint(text);
assert_eq!(fp1, fp2);
}
#[test]
fn test_new_cache_is_empty() {
let d = make_dedup();
assert!(d.is_empty());
assert_eq!(d.len(), 0);
}
#[test]
fn test_dedup_config_defaults() {
let cfg = DedupConfig::default();
assert_eq!(cfg.ttl_ms, 300_000);
assert_eq!(cfg.capacity, 1_024);
}
#[test]
fn test_register_adds_entry() {
let mut d = make_dedup();
d.register("hello world", "response1".to_string(), 1_000);
assert_eq!(d.len(), 1);
}
#[test]
fn test_register_second_time_same_key_overwrites() {
let mut d = make_dedup();
d.register("hello world", "first".to_string(), 1_000);
d.register("hello world", "second".to_string(), 2_000);
assert_eq!(d.len(), 1);
let entry = d.check("hello world", 2_000).expect("entry should exist");
assert_eq!(entry.value, "second");
}
#[test]
fn test_register_stores_inserted_ms() {
let mut d = make_dedup();
d.register("test prompt", "val".to_string(), 9_999);
let entry = d.check("test prompt", 9_999).expect("entry");
assert_eq!(entry.inserted_ms, 9_999);
}
#[test]
fn test_register_increments_len() {
let mut d = make_dedup();
d.register("first", "a".to_string(), 0);
assert_eq!(d.len(), 1);
d.register("second", "b".to_string(), 0);
assert_eq!(d.len(), 2);
}
#[test]
fn test_two_different_texts_two_entries() {
let mut d = make_dedup();
d.register("prompt one", "v1".to_string(), 0);
d.register("prompt two", "v2".to_string(), 0);
assert_eq!(d.len(), 2);
}
#[test]
fn test_check_returns_none_for_unknown_text() {
let mut d = make_dedup();
assert!(d.check("nothing here", 0).is_none());
}
#[test]
fn test_check_returns_some_for_known_text() {
let mut d = make_dedup();
d.register("hello", "world".to_string(), 0);
assert!(d.check("hello", 0).is_some());
}
#[test]
fn test_check_returns_stored_value() {
let mut d = make_dedup();
d.register("my prompt", "the answer".to_string(), 0);
let entry = d.check("my prompt", 0).expect("entry");
assert_eq!(entry.value, "the answer");
}
#[test]
fn test_check_hit_count_starts_at_zero() {
let mut d = make_dedup();
d.register("prompt", "val".to_string(), 0);
let fp = SemanticDedup::fingerprint("prompt");
assert_eq!(d.cache[&fp].hit_count, 0);
}
#[test]
fn test_check_increments_hit_count() {
let mut d = make_dedup();
d.register("prompt", "val".to_string(), 0);
d.check("prompt", 0);
let fp = SemanticDedup::fingerprint("prompt");
assert_eq!(d.cache[&fp].hit_count, 1);
}
#[test]
fn test_entry_hit_count_accumulates_across_checks() {
let mut d = make_dedup();
d.register("some text", "resp".to_string(), 0);
d.check("some text", 0);
d.check("some text", 0);
d.check("some text", 0);
let fp = SemanticDedup::fingerprint("some text");
assert_eq!(d.cache[&fp].hit_count, 3);
}
#[test]
fn test_check_returns_none_for_expired_entry() {
let mut d = make_dedup_with(1_000, 128);
d.register("old prompt", "val".to_string(), 0);
assert!(d.check("old prompt", 2_000).is_none());
}
#[test]
fn test_check_returns_some_for_live_entry() {
let mut d = make_dedup_with(5_000, 128);
d.register("live prompt", "val".to_string(), 1_000);
assert!(d.check("live prompt", 3_000).is_some());
}
#[test]
fn test_check_after_ttl_expires_returns_none() {
let mut d = make_dedup_with(500, 64);
d.register("prompt", "val".to_string(), 100);
assert!(d.check("prompt", 700).is_none());
}
#[test]
fn test_check_evicts_expired_entries_on_call() {
let mut d = make_dedup_with(100, 64);
d.register("stale", "v".to_string(), 0);
d.register("fresh", "v".to_string(), 200);
d.check("anything", 150);
assert_eq!(d.len(), 1, "expired entry should have been evicted");
}
#[test]
fn test_evict_expired_removes_old_entries() {
let mut d = make_dedup_with(1_000, 128);
d.register("old", "v".to_string(), 0);
d.register("also old", "v".to_string(), 100);
d.evict_expired(2_000); assert!(d.is_empty());
}
#[test]
fn test_evict_expired_keeps_live_entries() {
let mut d = make_dedup_with(1_000, 128);
d.register("old", "v".to_string(), 0);
d.register("fresh", "v".to_string(), 1_500);
d.evict_expired(2_000); assert_eq!(d.len(), 1);
assert!(d.cache.contains_key(&SemanticDedup::fingerprint("fresh")));
}
#[test]
fn test_evict_expired_with_zero_now_removes_nothing() {
let mut d = make_dedup_with(1_000, 128);
d.register("a", "v".to_string(), 0);
d.register("b", "v".to_string(), 0);
d.evict_expired(0);
assert_eq!(d.len(), 2);
}
#[test]
fn test_len_matches_registered_entries() {
let mut d = make_dedup();
d.register("one", "v".to_string(), 0);
d.register("two", "v".to_string(), 0);
d.register("three", "v".to_string(), 0);
assert_eq!(d.len(), 3);
}
#[test]
fn test_is_empty_true_when_empty() {
let d = make_dedup();
assert!(d.is_empty());
}
#[test]
fn test_is_empty_false_when_has_entries() {
let mut d = make_dedup();
d.register("entry", "v".to_string(), 0);
assert!(!d.is_empty());
}
#[test]
fn test_capacity_evicts_lru_on_insert() {
let mut d = make_dedup_with(1_000_000, 2);
d.register("alpha", "v1".to_string(), 1);
d.register("beta", "v2".to_string(), 2);
d.register("gamma", "v3".to_string(), 3);
assert_eq!(d.len(), 2);
}
#[test]
fn test_capacity_evicts_oldest_not_newest() {
let mut d = make_dedup_with(1_000_000, 2);
d.register("alpha", "v1".to_string(), 10);
d.register("beta", "v2".to_string(), 20);
d.register("gamma", "v3".to_string(), 30);
assert!(
d.cache.get(&SemanticDedup::fingerprint("alpha")).is_none(),
"alpha should have been evicted as LRU"
);
assert!(d.cache.contains_key(&SemanticDedup::fingerprint("beta")));
assert!(d.cache.contains_key(&SemanticDedup::fingerprint("gamma")));
}
#[test]
fn test_register_at_capacity_replaces_oldest() {
let mut d = make_dedup_with(1_000_000, 3);
d.register("p1", "v1".to_string(), 5);
d.register("p2", "v2".to_string(), 10);
d.register("p3", "v3".to_string(), 15);
d.register("p4", "v4".to_string(), 20);
assert!(d.cache.get(&SemanticDedup::fingerprint("p1")).is_none());
assert_eq!(d.len(), 3);
}
#[test]
fn test_live_keys_excludes_expired() {
let mut d = make_dedup_with(500, 64);
d.register("old", "v".to_string(), 0);
d.register("new", "v".to_string(), 600);
let keys = d.live_keys(700);
assert!(!keys.contains(&SemanticDedup::fingerprint("old")));
assert!(keys.contains(&SemanticDedup::fingerprint("new")));
}
#[test]
fn test_live_keys_sorted() {
let mut d = make_dedup();
d.register("zebra", "v".to_string(), 0);
d.register("apple", "v".to_string(), 0);
d.register("mango", "v".to_string(), 0);
let keys = d.live_keys(0);
let mut sorted = keys.clone();
sorted.sort();
assert_eq!(keys, sorted, "live_keys must return a sorted slice");
}
#[test]
fn test_clear_empties_cache() {
let mut d = make_dedup();
d.register("a", "v".to_string(), 0);
d.register("b", "v".to_string(), 0);
d.register("c", "v".to_string(), 0);
d.clear();
assert!(d.is_empty());
assert_eq!(d.len(), 0);
}
#[test]
fn test_clear_then_reinsert_works() {
let mut d = make_dedup();
d.register("prompt", "v1".to_string(), 0);
d.clear();
d.register("prompt", "v2".to_string(), 100);
let entry = d.check("prompt", 100).expect("entry after reinsert");
assert_eq!(entry.value, "v2");
}
#[test]
fn test_check_normalises_before_lookup() {
let mut d = make_dedup();
d.register("hello world", "result".to_string(), 0);
let entry = d.check(" Hello, WORLD! ", 0).expect("should hit");
assert_eq!(entry.value, "result");
}
#[test]
fn test_register_normalises_key() {
let mut d = make_dedup();
d.register(" HELLO world ", "v1".to_string(), 0);
assert!(d.check("hello world", 0).is_some());
}
#[test]
fn test_cache_entry_clone() {
let entry = CacheEntry {
fingerprint: "fp".to_string(),
value: "val".to_string(),
inserted_ms: 42,
hit_count: 7,
};
let cloned = entry.clone();
assert_eq!(cloned.fingerprint, entry.fingerprint);
assert_eq!(cloned.value, entry.value);
assert_eq!(cloned.inserted_ms, entry.inserted_ms);
assert_eq!(cloned.hit_count, entry.hit_count);
}
#[test]
fn test_capacity_one_always_evicts_previous() {
let mut d = make_dedup_with(1_000_000, 1);
d.register("first", "v1".to_string(), 1);
d.register("second", "v2".to_string(), 2);
assert_eq!(d.len(), 1);
assert!(d.cache.contains_key(&SemanticDedup::fingerprint("second")));
}
#[test]
fn test_overwrite_does_not_grow_beyond_capacity() {
let mut d = make_dedup_with(1_000_000, 2);
d.register("a", "v1".to_string(), 1);
d.register("b", "v2".to_string(), 2);
d.register("a", "v3".to_string(), 3);
assert_eq!(d.len(), 2);
let entry = d.check("a", 3).expect("a should still be present");
assert_eq!(entry.value, "v3");
}
#[test]
fn test_fingerprint_removes_special_chars_only() {
let fp = SemanticDedup::fingerprint("café résumé");
for ch in fp.chars() {
assert!(
ch.is_alphanumeric() || ch == ' ',
"unexpected char: {:?}",
ch
);
}
}
#[test]
fn test_evict_expired_empty_cache_is_noop() {
let mut d = make_dedup();
d.evict_expired(1_000_000); assert!(d.is_empty());
}
#[test]
fn test_live_keys_empty_cache_gives_empty_vec() {
let d = make_dedup();
assert!(d.live_keys(0).is_empty());
}
#[test]
fn test_register_capacity_zero_is_noop() {
let mut d = make_dedup_with(1_000_000, 0);
d.register("test", "v".to_string(), 0);
let _len = d.len();
}
}