use crate::model::ModelConfig;
use crate::service::EmbeddingRole;
use lru::LruCache;
use parking_lot::RwLock;
use sha2::{Digest, Sha256};
use std::num::NonZeroUsize;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use tracing::debug;
pub type CacheKey = [u8; 32];
pub const DEFAULT_CACHE_CAPACITY: usize = 4000;
const NUM_SHARDS: usize = 16;
const SHARD_MASK: usize = NUM_SHARDS - 1;
const _: () = assert!(
NUM_SHARDS.is_power_of_two(),
"NUM_SHARDS must be a power of 2"
);
struct CacheShard {
lru: RwLock<LruCache<CacheKey, Arc<[f32]>>>,
hits: AtomicU64,
misses: AtomicU64,
}
impl CacheShard {
fn new(capacity: NonZeroUsize) -> Self {
Self {
lru: RwLock::new(LruCache::new(capacity)),
hits: AtomicU64::new(0),
misses: AtomicU64::new(0),
}
}
#[inline]
fn get(&self, key: &CacheKey) -> Option<Arc<[f32]>> {
let mut lru = self.lru.write();
let result = lru.get(key).cloned();
if result.is_some() {
self.hits.fetch_add(1, Ordering::Relaxed);
} else {
self.misses.fetch_add(1, Ordering::Relaxed);
}
result
}
#[inline]
fn put(&self, key: CacheKey, embedding: Arc<[f32]>) {
let mut lru = self.lru.write();
lru.put(key, embedding);
}
fn len(&self) -> usize {
self.lru.read().len()
}
fn clear(&self) {
self.lru.write().clear();
}
fn hits(&self) -> u64 {
self.hits.load(Ordering::Relaxed)
}
fn misses(&self) -> u64 {
self.misses.load(Ordering::Relaxed)
}
}
pub struct EmbeddingCache {
shards: Vec<CacheShard>,
enabled: bool,
capacity: usize,
}
#[inline(always)]
fn shard_index(key: &CacheKey) -> usize {
key[0] as usize & SHARD_MASK
}
impl EmbeddingCache {
pub fn new(capacity: usize) -> Self {
let enabled = capacity != 0;
let per_shard = if enabled {
let base = capacity.div_ceil(NUM_SHARDS);
if base == 0 { 1 } else { base }
} else {
1 };
#[expect(
clippy::expect_used,
reason = "per_shard is clamped to at least 1 on the lines above"
)]
let per_shard_nz = NonZeroUsize::new(per_shard).expect("per_shard is always >= 1");
let shards = (0..NUM_SHARDS)
.map(|_| CacheShard::new(per_shard_nz))
.collect();
Self {
shards,
enabled,
capacity,
}
}
pub fn with_default_capacity() -> Self {
Self::new(DEFAULT_CACHE_CAPACITY)
}
pub fn compute_key(
&self,
text: &str,
model_config: ModelConfig,
role: EmbeddingRole,
) -> CacheKey {
let mut hasher = Sha256::new();
hasher.update(text.as_bytes());
let model_key = format!(
"{}:{}:{}:{}",
model_config.model,
model_config.model.key_version(),
model_config.dimensions(),
role.cache_tag(),
);
hasher.update(model_key.as_bytes());
hasher.finalize().into()
}
pub fn get(&self, key: &CacheKey) -> Option<Arc<[f32]>> {
if !self.enabled {
return None;
}
let idx = shard_index(key);
let result = self.shards[idx].get(key);
if result.is_some() {
debug!("cache hit for key {:?}", &key[..8]);
}
result
}
pub fn put(&self, key: CacheKey, embedding: Vec<f32>) {
if !self.enabled {
return;
}
let idx = shard_index(&key);
self.shards[idx].put(key, Arc::from(embedding));
debug!("cached embedding for key {:?}", &key[..8]);
}
pub fn get_many(&self, keys: &[CacheKey]) -> Vec<Option<Arc<[f32]>>> {
if !self.enabled {
return vec![None; keys.len()];
}
keys.iter()
.map(|key| {
let idx = shard_index(key);
self.shards[idx].get(key)
})
.collect()
}
pub fn put_many(&self, entries: Vec<(CacheKey, Vec<f32>)>) {
if !self.enabled {
return;
}
for (key, embedding) in entries {
let idx = shard_index(&key);
self.shards[idx].put(key, Arc::from(embedding));
}
}
pub fn stats(&self) -> CacheStats {
if !self.enabled {
let (hits, misses) = self.aggregate_counters();
return CacheStats {
size: 0,
capacity: 0,
hits,
misses,
};
}
let size: usize = self.shards.iter().map(CacheShard::len).sum();
let (hits, misses) = self.aggregate_counters();
CacheStats {
size,
capacity: self.capacity,
hits,
misses,
}
}
pub fn per_shard_stats(&self) -> Vec<ShardStats> {
self.shards
.iter()
.enumerate()
.map(|(i, s)| ShardStats {
shard_id: i,
size: s.len(),
hits: s.hits(),
misses: s.misses(),
})
.collect()
}
pub fn clear(&self) {
if !self.enabled {
return;
}
for shard in &self.shards {
shard.clear();
}
debug!("cache cleared");
}
#[inline]
pub fn is_enabled(&self) -> bool {
self.enabled
}
fn aggregate_counters(&self) -> (u64, u64) {
let hits: u64 = self.shards.iter().map(CacheShard::hits).sum();
let misses: u64 = self.shards.iter().map(CacheShard::misses).sum();
(hits, misses)
}
}
impl Default for EmbeddingCache {
fn default() -> Self {
Self::with_default_capacity()
}
}
#[derive(Debug, Clone, Copy)]
pub struct CacheStats {
pub size: usize,
pub capacity: usize,
pub hits: u64,
pub misses: u64,
}
impl CacheStats {
pub fn hit_rate(&self) -> f64 {
let total = self.hits + self.misses;
if total == 0 {
0.0
} else {
self.hits as f64 / total as f64
}
}
}
#[derive(Debug, Clone, Copy)]
pub struct ShardStats {
pub shard_id: usize,
pub size: usize,
pub hits: u64,
pub misses: u64,
}
impl ShardStats {
pub fn hit_rate(&self) -> f64 {
let total = self.hits + self.misses;
if total == 0 {
0.0
} else {
self.hits as f64 / total as f64
}
}
}
#[cfg(test)]
mod tests;