langfuse/prompts/
cache.rs

1//! Thread-safe prompt cache with per-entry TTL.
2
3use std::time::{Duration, Instant};
4
5use dashmap::DashMap;
6
7use crate::prompts::chat::ChatPromptClient;
8use crate::prompts::text::TextPromptClient;
9
10/// Internal cache entry wrapping a value with insertion time and TTL.
11struct CacheEntry<T> {
12    value: T,
13    inserted_at: Instant,
14    ttl: Duration,
15}
16
17impl<T> CacheEntry<T> {
18    fn is_expired(&self) -> bool {
19        self.inserted_at.elapsed() >= self.ttl
20    }
21}
22
23/// Thread-safe prompt cache backed by [`DashMap`].
24///
25/// Each entry carries its own TTL (defaulting to `default_ttl`). Expired entries are
26/// lazily evicted on access.
27pub struct PromptCache {
28    text_entries: DashMap<String, CacheEntry<TextPromptClient>>,
29    chat_entries: DashMap<String, CacheEntry<ChatPromptClient>>,
30    default_ttl: Duration,
31}
32
33impl PromptCache {
34    /// Create a new cache with the given default TTL for entries.
35    pub fn new(default_ttl: Duration) -> Self {
36        Self {
37            text_entries: DashMap::new(),
38            chat_entries: DashMap::new(),
39            default_ttl,
40        }
41    }
42
43    // ── Text prompts ──────────────────────────────────────────────────
44
45    /// Retrieve a cached text prompt if it exists and has not expired.
46    ///
47    /// Cache key format: `"{name}:{version}"` or `"{name}:latest"`.
48    pub fn get_text(&self, key: &str) -> Option<TextPromptClient> {
49        let entry = self.text_entries.get(key)?;
50        if entry.is_expired() {
51            drop(entry);
52            self.text_entries.remove(key);
53            return None;
54        }
55        Some(entry.value.clone())
56    }
57
58    /// Insert (or replace) a text prompt in the cache using the default TTL.
59    pub fn put_text(&self, key: &str, prompt: TextPromptClient) {
60        self.text_entries.insert(
61            key.to_owned(),
62            CacheEntry {
63                value: prompt,
64                inserted_at: Instant::now(),
65                ttl: self.default_ttl,
66            },
67        );
68    }
69
70    /// Retrieve a cached text prompt even if it has expired.
71    ///
72    /// Used for fallback behavior: when the API is unreachable, an expired cached
73    /// entry is better than no entry at all.
74    pub fn get_text_expired(&self, key: &str) -> Option<TextPromptClient> {
75        self.text_entries.get(key).map(|entry| entry.value.clone())
76    }
77
78    // ── Chat prompts ──────────────────────────────────────────────────
79
80    /// Retrieve a cached chat prompt if it exists and has not expired.
81    pub fn get_chat(&self, key: &str) -> Option<ChatPromptClient> {
82        let entry = self.chat_entries.get(key)?;
83        if entry.is_expired() {
84            drop(entry);
85            self.chat_entries.remove(key);
86            return None;
87        }
88        Some(entry.value.clone())
89    }
90
91    /// Insert (or replace) a chat prompt in the cache using the default TTL.
92    pub fn put_chat(&self, key: &str, prompt: ChatPromptClient) {
93        self.chat_entries.insert(
94            key.to_owned(),
95            CacheEntry {
96                value: prompt,
97                inserted_at: Instant::now(),
98                ttl: self.default_ttl,
99            },
100        );
101    }
102
103    /// Retrieve a cached chat prompt even if it has expired.
104    ///
105    /// Used for fallback behavior: when the API is unreachable, an expired cached
106    /// entry is better than no entry at all.
107    pub fn get_chat_expired(&self, key: &str) -> Option<ChatPromptClient> {
108        self.chat_entries.get(key).map(|entry| entry.value.clone())
109    }
110
111    // ── Maintenance ───────────────────────────────────────────────────
112
113    /// Remove all entries from the cache.
114    pub fn clear(&self) {
115        self.text_entries.clear();
116        self.chat_entries.clear();
117    }
118
119    /// Remove all entries whose key starts with the given prefix.
120    ///
121    /// Used to invalidate all cached versions/labels for a prompt name after
122    /// a create or update operation.
123    pub fn invalidate_by_prefix(&self, prefix: &str) {
124        self.text_entries.retain(|k, _| !k.starts_with(prefix));
125        self.chat_entries.retain(|k, _| !k.starts_with(prefix));
126    }
127}
langfuse/prompts/cache.rs

langfuse/prompts/
cache.rs
