semantic_commands/caches/
in_memory.rs

1use std::sync::Arc;
2
3use crate::Cache;
4use anyhow::Result;
5use moka::future::Cache as MokaCache;
6
7pub struct InMemoryCache {
8	cache: MokaCache<String, Arc<Vec<f32>>>,
9}
10
11impl InMemoryCache {
12	/// Unbounded cache (grows indefinitely)
13	///
14	/// Suitable for:
15	/// - Testing/development
16	/// - Short-lived processes
17	/// - Known input sets
18	pub fn unbounded() -> Self {
19		Self {
20			cache: MokaCache::builder().build(),
21		}
22	}
23
24	/// Bounded by entry count
25	///
26	/// Evicts least-recently-used entries when full.
27	///
28	/// Example: 10,000 entries ≈ 60 MB (at 1536 dims/embedding)
29	pub fn with_max_entries(max_entries: u64) -> Self {
30		Self {
31			cache: MokaCache::builder().max_capacity(max_entries).build(),
32		}
33	}
34
35	/// Bounded by approximate memory usage
36	///
37	/// Weights entries by `vector.len() * 4` bytes (f32 size).
38	///
39	/// Example: 100 MB for ~17,000 embeddings (at 1536 dims)
40	pub fn with_max_memory_mb(mb: u64) -> Self {
41		let max_bytes = mb * 1024 * 1024;
42		Self {
43			cache: MokaCache::builder()
44				.weigher(|_key: &String, value: &Arc<Vec<f32>>| (value.len() * 4) as u32)
45				.max_capacity(max_bytes)
46				.build(),
47		}
48	}
49}
50
51impl Default for InMemoryCache {
52	/// Default: bounded to 10,000 entries (~60 MB)
53	fn default() -> Self {
54		Self::with_max_entries(10_000)
55	}
56}
57
58#[async_trait::async_trait]
59impl Cache for InMemoryCache {
60	async fn get(&self, input: &str) -> Result<Option<Vec<f32>>> {
61		Ok(self.cache.get(input).await.map(|arc| (*arc).clone()))
62	}
63
64	async fn put(&self, input: &str, embedding: Vec<f32>) -> Result<()> {
65		self.cache.insert(input.to_string(), Arc::new(embedding)).await;
66		Ok(())
67	}
68
69	async fn init(&self) -> Result<()> {
70		Ok(())
71	}
72}
semantic_commands/caches/in_memory.rs

semantic_commands/caches/
in_memory.rs
