use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use moka::sync::Cache;
use parking_lot::RwLock;
use sha2::{Digest, Sha256};
use crate::types::{ChunkId, TenantId};
#[derive(Debug, Clone)]
pub struct SemanticCacheConfig {
pub similarity_threshold: f32,
pub ttl_seconds: u64,
pub max_entries: u64,
pub confidence_boost_on_hit: f32,
pub confidence_decay_rate: f32,
}
impl Default for SemanticCacheConfig {
fn default() -> Self {
Self {
similarity_threshold: 0.85,
ttl_seconds: 2700, max_entries: 10_000,
confidence_boost_on_hit: 0.1,
confidence_decay_rate: 0.05,
}
}
}
#[derive(Debug, Clone)]
pub struct CachedResult {
pub chunk_id: ChunkId,
pub score: f32,
pub text_preview: String,
}
#[derive(Debug, Clone)]
pub struct CacheEntry {
pub query_embedding: Vec<f32>,
pub results: Vec<CachedResult>,
pub tenant_id: TenantId,
pub project_id: Option<String>,
pub memory_version: u64,
pub created_at: i64,
pub last_hit: i64,
pub hit_count: u32,
pub confidence: f32,
}
#[derive(Debug, Clone)]
pub struct CacheHit {
pub results: Vec<CachedResult>,
pub similarity: f32,
pub confidence: f32,
pub age_ms: u64,
pub cache_key: String,
}
#[derive(Debug, Clone)]
struct CacheRecord {
entry: CacheEntry,
retrieval_limit: usize,
}
struct QueryIndexEntry {
cache_key: String,
embedding: Vec<f32>,
tenant_id: TenantId,
project_id: Option<String>,
}
#[derive(Debug, Clone, Default)]
pub struct CacheStats {
pub total_lookups: u64,
pub cache_hits: u64,
pub cache_misses: u64,
pub version_invalidations: u64,
pub ttl_expirations: u64,
pub avg_confidence: f32,
pub entry_count: usize,
}
struct AtomicStats {
total_lookups: AtomicU64,
cache_hits: AtomicU64,
cache_misses: AtomicU64,
version_invalidations: AtomicU64,
ttl_expirations: AtomicU64,
}
impl Default for AtomicStats {
fn default() -> Self {
Self {
total_lookups: AtomicU64::new(0),
cache_hits: AtomicU64::new(0),
cache_misses: AtomicU64::new(0),
version_invalidations: AtomicU64::new(0),
ttl_expirations: AtomicU64::new(0),
}
}
}
pub struct SemanticCache {
entries: Cache<String, CacheRecord>,
query_index: RwLock<Vec<QueryIndexEntry>>,
config: SemanticCacheConfig,
stats: AtomicStats,
}
impl SemanticCache {
pub fn new(config: SemanticCacheConfig) -> Self {
let entries = Cache::builder()
.max_capacity(config.max_entries)
.time_to_live(Duration::from_secs(config.ttl_seconds))
.build();
Self {
entries,
query_index: RwLock::new(Vec::new()),
config,
stats: AtomicStats::default(),
}
}
pub fn lookup(
&self,
query_embedding: &[f32],
tenant_id: &TenantId,
project_id: Option<&str>,
current_version: u64,
) -> Option<CacheHit> {
self.lookup_with_limit(query_embedding, tenant_id, project_id, current_version, 0)
}
pub(crate) fn lookup_with_limit(
&self,
query_embedding: &[f32],
tenant_id: &TenantId,
project_id: Option<&str>,
current_version: u64,
requested_limit: usize,
) -> Option<CacheHit> {
self.stats.total_lookups.fetch_add(1, Ordering::Relaxed);
let now_ms = current_time_ms();
let query_index = self.query_index.read();
let mut best_match: Option<(String, f32)> = None;
for entry in query_index.iter() {
if entry.tenant_id != *tenant_id {
continue;
}
match (&entry.project_id, project_id) {
(None, None) => {}
(Some(a), Some(b)) if a == b => {}
_ => continue,
}
let similarity = cosine_similarity(query_embedding, &entry.embedding);
if similarity >= self.config.similarity_threshold
&& best_match.as_ref().is_none_or(|(_, s)| similarity > *s)
{
best_match = Some((entry.cache_key.clone(), similarity));
}
}
drop(query_index);
let (cache_key, similarity) = best_match?;
let mut record = self.entries.get(&cache_key)?;
if record.entry.memory_version < current_version {
self.stats
.version_invalidations
.fetch_add(1, Ordering::Relaxed);
self.stats.cache_misses.fetch_add(1, Ordering::Relaxed);
return None;
}
if record.retrieval_limit < requested_limit {
self.stats.cache_misses.fetch_add(1, Ordering::Relaxed);
return None;
}
record.entry.last_hit = now_ms;
record.entry.hit_count += 1;
record.entry.confidence =
(record.entry.confidence + self.config.confidence_boost_on_hit).min(1.0);
self.entries.insert(cache_key.clone(), record.clone());
self.stats.cache_hits.fetch_add(1, Ordering::Relaxed);
Some(CacheHit {
results: record.entry.results.clone(),
similarity,
confidence: record.entry.confidence,
age_ms: (now_ms - record.entry.created_at).max(0) as u64,
cache_key,
})
}
pub fn insert(
&self,
query_embedding: Vec<f32>,
tenant_id: TenantId,
project_id: Option<String>,
results: Vec<CachedResult>,
memory_version: u64,
) {
let retrieval_limit = results.len();
self.insert_with_limit(
query_embedding,
tenant_id,
project_id,
results,
memory_version,
retrieval_limit,
);
}
pub(crate) fn insert_with_limit(
&self,
query_embedding: Vec<f32>,
tenant_id: TenantId,
project_id: Option<String>,
results: Vec<CachedResult>,
memory_version: u64,
retrieval_limit: usize,
) {
let cache_key = embedding_hash(&query_embedding, &tenant_id, project_id.as_deref());
let now_ms = current_time_ms();
let entry = CacheEntry {
query_embedding: query_embedding.clone(),
results,
tenant_id: tenant_id.clone(),
project_id: project_id.clone(),
memory_version,
created_at: now_ms,
last_hit: now_ms,
hit_count: 0,
confidence: 0.5, };
self.entries.insert(
cache_key.clone(),
CacheRecord {
entry,
retrieval_limit,
},
);
let index_entry = QueryIndexEntry {
cache_key,
embedding: query_embedding,
tenant_id,
project_id,
};
let mut query_index = self.query_index.write();
if let Some(existing) = query_index
.iter_mut()
.find(|existing| existing.cache_key == index_entry.cache_key)
{
*existing = index_entry;
} else {
query_index.push(index_entry);
}
}
pub fn invalidate_tenant(&self, tenant_id: &TenantId) {
let mut query_index = self.query_index.write();
let keys_to_remove: Vec<String> = query_index
.iter()
.filter(|e| e.tenant_id == *tenant_id)
.map(|e| e.cache_key.clone())
.collect();
for key in &keys_to_remove {
self.entries.invalidate(key);
}
query_index.retain(|e| e.tenant_id != *tenant_id);
}
pub fn invalidate_by_version(&self, tenant_id: &TenantId, min_version: u64) {
let mut query_index = self.query_index.write();
let mut keys_to_remove = Vec::new();
for entry in query_index.iter() {
if entry.tenant_id == *tenant_id {
if let Some(cached) = self.entries.get(&entry.cache_key) {
if cached.entry.memory_version < min_version {
keys_to_remove.push(entry.cache_key.clone());
}
}
}
}
for key in &keys_to_remove {
self.entries.invalidate(key);
self.stats
.version_invalidations
.fetch_add(1, Ordering::Relaxed);
}
query_index.retain(|e| !keys_to_remove.contains(&e.cache_key));
}
pub fn invalidate_chunks(&self, chunk_ids: &[ChunkId]) {
let mut query_index = self.query_index.write();
let mut keys_to_remove = Vec::new();
for entry in query_index.iter() {
if let Some(cached) = self.entries.get(&entry.cache_key) {
let contains_chunk = cached
.entry
.results
.iter()
.any(|r| chunk_ids.contains(&r.chunk_id));
if contains_chunk {
keys_to_remove.push(entry.cache_key.clone());
}
}
}
for key in &keys_to_remove {
self.entries.invalidate(key);
}
query_index.retain(|e| !keys_to_remove.contains(&e.cache_key));
}
pub fn get_stats(&self) -> CacheStats {
let query_index = self.query_index.read();
let entry_count = query_index.len();
let mut total_confidence = 0.0f32;
let mut count = 0u32;
for entry in query_index.iter() {
if let Some(cached) = self.entries.get(&entry.cache_key) {
total_confidence += cached.entry.confidence;
count += 1;
}
}
let avg_confidence = if count > 0 {
total_confidence / count as f32
} else {
0.0
};
CacheStats {
total_lookups: self.stats.total_lookups.load(Ordering::Relaxed),
cache_hits: self.stats.cache_hits.load(Ordering::Relaxed),
cache_misses: self.stats.cache_misses.load(Ordering::Relaxed),
version_invalidations: self.stats.version_invalidations.load(Ordering::Relaxed),
ttl_expirations: self.stats.ttl_expirations.load(Ordering::Relaxed),
avg_confidence,
entry_count,
}
}
pub fn prune_index(&self) {
let mut query_index = self.query_index.write();
let before_count = query_index.len();
query_index.retain(|e| self.entries.contains_key(&e.cache_key));
let removed = before_count - query_index.len();
if removed > 0 {
self.stats
.ttl_expirations
.fetch_add(removed as u64, Ordering::Relaxed);
}
}
}
fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
if a.len() != b.len() || a.is_empty() {
return 0.0;
}
let mut dot_product = 0.0f32;
let mut norm_a = 0.0f32;
let mut norm_b = 0.0f32;
for (x, y) in a.iter().zip(b.iter()) {
dot_product += x * y;
norm_a += x * x;
norm_b += y * y;
}
let norm_a = norm_a.sqrt();
let norm_b = norm_b.sqrt();
if norm_a == 0.0 || norm_b == 0.0 {
return 0.0;
}
dot_product / (norm_a * norm_b)
}
fn embedding_hash(embedding: &[f32], tenant_id: &TenantId, project_id: Option<&str>) -> String {
let mut hasher = Sha256::new();
let tenant = tenant_id.as_str().as_bytes();
hasher.update((tenant.len() as u64).to_le_bytes());
hasher.update(tenant);
match project_id {
None => hasher.update([0]),
Some(project) => {
hasher.update([1]);
hasher.update((project.len() as u64).to_le_bytes());
hasher.update(project.as_bytes());
}
}
for val in embedding {
hasher.update(val.to_le_bytes());
}
let result = hasher.finalize();
hex::encode(&result[..16])
}
fn current_time_ms() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0)
}
mod hex {
pub fn encode(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{:02x}", b)).collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_tenant(name: &str) -> TenantId {
TenantId::new(name).unwrap()
}
fn make_embedding(seed: u64) -> Vec<f32> {
let mut vec: Vec<f32> = (0..384)
.map(|i| ((seed + i) % 100) as f32 / 100.0)
.collect();
let norm: f32 = vec.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm > 0.0 {
for v in &mut vec {
*v /= norm;
}
}
vec
}
fn make_similar_embedding(base: &[f32], noise: f32) -> Vec<f32> {
let mut vec = base.to_vec();
for (i, v) in vec.iter_mut().enumerate() {
*v += (i as f32 * noise).sin() * noise;
}
let norm: f32 = vec.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm > 0.0 {
for v in &mut vec {
*v /= norm;
}
}
vec
}
fn make_chunk_id() -> ChunkId {
ChunkId::new()
}
fn make_result(chunk_id: &ChunkId) -> Vec<CachedResult> {
vec![CachedResult {
chunk_id: chunk_id.clone(),
score: 1.0,
text_preview: String::new(),
}]
}
#[test]
fn test_cache_hit_flow() {
let config = SemanticCacheConfig::default();
let cache = SemanticCache::new(config);
let tenant = make_tenant("test_tenant");
let embedding = make_embedding(42);
let chunk_id = make_chunk_id();
let results = vec![CachedResult {
chunk_id: chunk_id.clone(),
score: 0.95,
text_preview: "Test content".to_string(),
}];
cache.insert(embedding.clone(), tenant.clone(), None, results.clone(), 1);
let similar = make_similar_embedding(&embedding, 0.01);
let hit = cache.lookup(&similar, &tenant, None, 1);
assert!(hit.is_some(), "Similar query should hit cache");
let hit = hit.unwrap();
assert!(
hit.similarity > 0.85,
"Similarity should be above threshold"
);
assert_eq!(hit.results.len(), 1);
let dissimilar = make_embedding(999);
let miss = cache.lookup(&dissimilar, &tenant, None, 1);
assert!(miss.is_none(), "Dissimilar query should miss cache");
let stats = cache.get_stats();
assert_eq!(stats.cache_hits, 1);
assert_eq!(stats.cache_misses, 0);
assert_eq!(stats.total_lookups, 2);
}
#[test]
fn test_version_invalidation() {
let config = SemanticCacheConfig::default();
let cache = SemanticCache::new(config);
let tenant = make_tenant("test_tenant");
let embedding = make_embedding(42);
let chunk_id = make_chunk_id();
let results = vec![CachedResult {
chunk_id,
score: 0.95,
text_preview: "Test content".to_string(),
}];
cache.insert(embedding.clone(), tenant.clone(), None, results, 5);
let hit = cache.lookup(&embedding, &tenant, None, 5);
assert!(hit.is_some(), "Version 5 lookup should hit");
let miss = cache.lookup(&embedding, &tenant, None, 6);
assert!(miss.is_none(), "Version 6 lookup should miss (stale entry)");
cache.invalidate_by_version(&tenant, 6);
let stats = cache.get_stats();
assert_eq!(stats.entry_count, 0);
}
#[test]
fn test_tenant_isolation() {
let config = SemanticCacheConfig::default();
let cache = SemanticCache::new(config);
let tenant_a = make_tenant("tenant_a");
let tenant_b = make_tenant("tenant_b");
let embedding = make_embedding(42);
let chunk_id = make_chunk_id();
let results = vec![CachedResult {
chunk_id,
score: 0.95,
text_preview: "Test content".to_string(),
}];
cache.insert(embedding.clone(), tenant_a.clone(), None, results, 1);
let miss = cache.lookup(&embedding, &tenant_b, None, 1);
assert!(miss.is_none(), "Tenant B should not see Tenant A's cache");
let hit = cache.lookup(&embedding, &tenant_a, None, 1);
assert!(hit.is_some(), "Tenant A should see its own cache");
}
#[test]
fn test_confidence_mechanics() {
let config = SemanticCacheConfig {
confidence_boost_on_hit: 0.1,
..Default::default()
};
let cache = SemanticCache::new(config);
let tenant = make_tenant("test_tenant");
let embedding = make_embedding(42);
let chunk_id = make_chunk_id();
let results = vec![CachedResult {
chunk_id,
score: 0.95,
text_preview: "Test content".to_string(),
}];
cache.insert(embedding.clone(), tenant.clone(), None, results, 1);
let hit1 = cache.lookup(&embedding, &tenant, None, 1).unwrap();
assert!(
(hit1.confidence - 0.6).abs() < 0.01,
"Confidence should be 0.6 after first hit"
);
let hit2 = cache.lookup(&embedding, &tenant, None, 1).unwrap();
assert!(
(hit2.confidence - 0.7).abs() < 0.01,
"Confidence should be 0.7 after second hit"
);
let hit3 = cache.lookup(&embedding, &tenant, None, 1).unwrap();
assert!(
(hit3.confidence - 0.8).abs() < 0.01,
"Confidence should be 0.8 after third hit"
);
}
#[test]
fn test_result_limit_controls_cache_acceptance() {
let cache = SemanticCache::new(SemanticCacheConfig::default());
let tenant = make_tenant("test_tenant");
let embedding = make_embedding(42);
let results = [make_chunk_id(), make_chunk_id()]
.iter()
.map(|chunk_id| CachedResult {
chunk_id: chunk_id.clone(),
score: 0.9,
text_preview: String::new(),
})
.collect();
cache.insert_with_limit(embedding.clone(), tenant.clone(), None, results, 1, 3);
let repeat = cache
.lookup_with_limit(&embedding, &tenant, None, 1, 3)
.unwrap();
assert_eq!(repeat.results.len(), 2);
let smaller = cache
.lookup_with_limit(&embedding, &tenant, None, 1, 2)
.unwrap();
assert_eq!(smaller.results.len(), 2);
let confidence_before_miss = cache.get_stats().avg_confidence;
assert!(cache
.lookup_with_limit(&embedding, &tenant, None, 1, 5)
.is_none());
let stats = cache.get_stats();
assert_eq!(stats.total_lookups, 3);
assert_eq!(stats.cache_hits, 2);
assert_eq!(stats.cache_misses, 1);
assert!((stats.avg_confidence - confidence_before_miss).abs() < f32::EPSILON);
}
#[test]
fn test_chunk_invalidation() {
let config = SemanticCacheConfig::default();
let cache = SemanticCache::new(config);
let tenant = make_tenant("test_tenant");
let embedding = make_embedding(42);
let chunk_a = make_chunk_id();
let chunk_b = make_chunk_id();
let results = vec![
CachedResult {
chunk_id: chunk_a.clone(),
score: 0.95,
text_preview: "Content A".to_string(),
},
CachedResult {
chunk_id: chunk_b.clone(),
score: 0.85,
text_preview: "Content B".to_string(),
},
];
cache.insert(embedding.clone(), tenant.clone(), None, results, 1);
let hit = cache.lookup(&embedding, &tenant, None, 1);
assert!(hit.is_some());
cache.invalidate_chunks(&[chunk_a]);
let miss = cache.lookup(&embedding, &tenant, None, 1);
assert!(
miss.is_none(),
"Entry should be removed after chunk invalidation"
);
let stats = cache.get_stats();
assert_eq!(stats.entry_count, 0);
}
#[test]
fn test_project_isolation() {
let config = SemanticCacheConfig::default();
let cache = SemanticCache::new(config);
let tenant = make_tenant("test_tenant");
let embedding = make_embedding(42);
let chunk_id = make_chunk_id();
let results = vec![CachedResult {
chunk_id,
score: 0.95,
text_preview: "Test content".to_string(),
}];
cache.insert(
embedding.clone(),
tenant.clone(),
Some("project_a".to_string()),
results.clone(),
1,
);
let miss = cache.lookup(&embedding, &tenant, Some("project_b"), 1);
assert!(miss.is_none(), "Project B should not see Project A's cache");
let hit = cache.lookup(&embedding, &tenant, Some("project_a"), 1);
assert!(hit.is_some(), "Project A should see its own cache");
let miss_no_project = cache.lookup(&embedding, &tenant, None, 1);
assert!(
miss_no_project.is_none(),
"No project should not see Project A's cache"
);
}
#[test]
fn test_cache_keys_isolate_ambiguous_scopes() {
let cache = SemanticCache::new(SemanticCacheConfig::default());
let embedding = make_embedding(42);
let scopes = [
(make_tenant("ab"), Some("c".to_string()), make_chunk_id()),
(make_tenant("a"), Some("bc".to_string()), make_chunk_id()),
(make_tenant("scope"), None, make_chunk_id()),
(make_tenant("scope"), Some(String::new()), make_chunk_id()),
];
for (tenant, project, chunk_id) in &scopes {
cache.insert(
embedding.clone(),
tenant.clone(),
project.clone(),
make_result(chunk_id),
1,
);
}
assert_eq!(cache.get_stats().entry_count, 4);
for (tenant, project, chunk_id) in &scopes {
let hit = cache
.lookup(&embedding, tenant, project.as_deref(), 1)
.unwrap();
assert_eq!(&hit.results[0].chunk_id, chunk_id);
}
}
#[test]
fn test_cache_replacement_keeps_index_bounded() {
let cache = SemanticCache::new(SemanticCacheConfig::default());
let embedding = make_embedding(42);
let tenant = make_tenant("test_tenant");
let first = make_chunk_id();
let replacement = make_chunk_id();
cache.insert(
embedding.clone(),
tenant.clone(),
None,
make_result(&first),
1,
);
cache.insert(
embedding.clone(),
tenant.clone(),
None,
make_result(&replacement),
1,
);
assert_eq!(cache.get_stats().entry_count, 1);
let hit = cache.lookup(&embedding, &tenant, None, 1).unwrap();
assert_eq!(hit.results[0].chunk_id, replacement);
}
#[test]
fn test_cosine_similarity() {
let a = vec![1.0, 0.0, 0.0];
assert!((cosine_similarity(&a, &a) - 1.0).abs() < 0.001);
let b = vec![0.0, 1.0, 0.0];
assert!(cosine_similarity(&a, &b).abs() < 0.001);
let c = vec![-1.0, 0.0, 0.0];
assert!((cosine_similarity(&a, &c) - (-1.0)).abs() < 0.001);
let zero = vec![0.0, 0.0, 0.0];
assert_eq!(cosine_similarity(&a, &zero), 0.0);
assert_eq!(cosine_similarity(&zero, &a), 0.0);
let d = vec![1.0, 0.0];
assert_eq!(cosine_similarity(&a, &d), 0.0);
}
#[test]
fn test_prune_index() {
let config = SemanticCacheConfig {
ttl_seconds: 1,
..Default::default()
};
let cache = SemanticCache::new(config);
let tenant = make_tenant("test_tenant");
let embedding = make_embedding(42);
let chunk_id = make_chunk_id();
let results = vec![CachedResult {
chunk_id,
score: 0.95,
text_preview: "Test content".to_string(),
}];
cache.insert(embedding, tenant, None, results, 1);
std::thread::sleep(std::time::Duration::from_secs(2));
cache.prune_index();
let stats = cache.get_stats();
assert_eq!(stats.entry_count, 0, "Expired entries should be pruned");
}
#[test]
fn test_stats_tracking() {
let config = SemanticCacheConfig::default();
let cache = SemanticCache::new(config);
let tenant = make_tenant("test_tenant");
let embedding1 = make_embedding(42);
let embedding2 = make_embedding(999);
let chunk_id = make_chunk_id();
let results = vec![CachedResult {
chunk_id,
score: 0.95,
text_preview: "Test content".to_string(),
}];
cache.insert(embedding1.clone(), tenant.clone(), None, results, 1);
cache.lookup(&embedding1, &tenant, None, 1);
cache.lookup(&embedding2, &tenant, None, 1);
let stats = cache.get_stats();
assert_eq!(stats.total_lookups, 2);
assert_eq!(stats.cache_hits, 1);
assert_eq!(stats.cache_misses, 0); assert_eq!(stats.entry_count, 1);
}
}