Skip to main content

graphrt_llm/credentials/
cached.rs

1//! Caching wrapper with single-flight refresh, fetch timeouts and shared failures.
2
3use std::collections::HashMap;
4use std::fmt;
5use std::sync::atomic::{AtomicU64, Ordering};
6use std::sync::{Arc, Mutex, PoisonError};
7use std::time::Duration;
8
9use futures::future::BoxFuture;
10use futures::lock::Mutex as AsyncMutex;
11
12use super::{Credential, CredentialSource, unix_now_ms};
13use crate::error::LlmError;
14
15/// Credentials are refreshed this long before they expire (60 s).
16pub const DEFAULT_REFRESH_SKEW_MS: u64 = 60_000;
17
18/// A fetch from the wrapped source that takes longer than this fails (15 s).
19pub const DEFAULT_FETCH_TIMEOUT: Duration = Duration::from_secs(15);
20
21#[derive(Default)]
22struct SlotState {
23    /// Cached credential and the source generation it was fetched under.
24    credential: Option<(Credential, u64)>,
25    /// Last failed fetch and its fetch sequence number.
26    failure: Option<(LlmError, u64)>,
27}
28
29/// One provider's cache slot. Holding the async lock while fetching makes
30/// concurrent requests wait for that single fetch instead of starting their own.
31#[derive(Default)]
32struct Slot {
33    state: AsyncMutex<SlotState>,
34    /// Number of fetches completed in this slot (success or failure).
35    completed: AtomicU64,
36}
37
38/// Caches credentials per provider until shortly before they expire.
39///
40/// - Credentials without `expires_at_ms` are cached until [`CredentialSource::invalidate`]
41///   or until the wrapped source's [`CredentialSource::generation`] changes (e.g. the
42///   host registered a different key source or signed the user out).
43/// - Concurrent requests for the same provider share one fetch. If that fetch
44///   fails, every request that was waiting behind it gets the same error instead of
45///   retrying the failing fetch one after another.
46/// - A fetch that does not finish within the fetch timeout (default 15 s) fails
47///   with [`LlmError::Auth`]. The timeout needs a Tokio runtime; elsewhere fetches
48///   are not time-limited.
49/// - `invalidate` drops the cached value and forwards to the inner source.
50pub struct CachedCredentials<S> {
51    inner: S,
52    skew_ms: u64,
53    fetch_timeout: Duration,
54    slots: Mutex<HashMap<String, Arc<Slot>>>,
55}
56
57impl<S: CredentialSource> CachedCredentials<S> {
58    pub fn new(inner: S) -> Self {
59        Self {
60            inner,
61            skew_ms: DEFAULT_REFRESH_SKEW_MS,
62            fetch_timeout: DEFAULT_FETCH_TIMEOUT,
63            slots: Mutex::new(HashMap::new()),
64        }
65    }
66
67    /// Refresh this many milliseconds before expiry.
68    pub fn refresh_skew_ms(mut self, skew_ms: u64) -> Self {
69        self.skew_ms = skew_ms;
70        self
71    }
72
73    /// Fail a fetch from the wrapped source after this long.
74    pub fn fetch_timeout(mut self, timeout: Duration) -> Self {
75        self.fetch_timeout = timeout;
76        self
77    }
78
79    /// The wrapped source.
80    pub fn inner(&self) -> &S {
81        &self.inner
82    }
83
84    /// Drop every cached credential (all providers).
85    pub fn clear(&self) {
86        let mut slots = self.slots.lock().unwrap_or_else(PoisonError::into_inner);
87        slots.clear();
88    }
89
90    fn slot(&self, provider: &str) -> Arc<Slot> {
91        let mut slots = self.slots.lock().unwrap_or_else(PoisonError::into_inner);
92        slots.entry(provider.to_owned()).or_default().clone()
93    }
94
95    async fn fetch(&self, provider: &str) -> Result<Credential, LlmError> {
96        let fetch = self.inner.credential(provider);
97        #[cfg(feature = "http")]
98        {
99            if tokio::runtime::Handle::try_current().is_ok() {
100                return match tokio::time::timeout(self.fetch_timeout, fetch).await {
101                    Ok(result) => result,
102                    Err(_) => Err(LlmError::Auth(format!(
103                        "credential source for `{provider}` did not answer within {} ms",
104                        self.fetch_timeout.as_millis()
105                    ))),
106                };
107            }
108        }
109        fetch.await
110    }
111}
112
113impl<S> fmt::Debug for CachedCredentials<S>
114where
115    S: fmt::Debug,
116{
117    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
118        f.debug_struct("CachedCredentials")
119            .field("inner", &self.inner)
120            .field("skew_ms", &self.skew_ms)
121            .field("fetch_timeout", &self.fetch_timeout)
122            .finish()
123    }
124}
125
126impl<S: CredentialSource> CredentialSource for CachedCredentials<S> {
127    fn credential<'a>(&'a self, provider: &'a str) -> BoxFuture<'a, Result<Credential, LlmError>> {
128        Box::pin(async move {
129            let slot = self.slot(provider);
130            // Fetches completed before we started waiting; a failure recorded after
131            // this point belongs to a fetch we were queued behind.
132            let ticket = slot.completed.load(Ordering::SeqCst);
133            let mut state = slot.state.lock().await;
134            if let Some((error, seq)) = &state.failure {
135                if *seq > ticket {
136                    return Err(error.clone());
137                }
138            }
139            let generation = self.inner.generation();
140            if let Some((credential, cached_generation)) = &state.credential {
141                let fresh = *cached_generation == generation && !credential.is_expired_at(unix_now_ms(), self.skew_ms);
142                if fresh {
143                    return Ok(credential.clone());
144                }
145            }
146            let result = self.fetch(provider).await;
147            let seq = slot.completed.fetch_add(1, Ordering::SeqCst).saturating_add(1);
148            match result {
149                Ok(credential) => {
150                    state.credential = Some((credential.clone(), generation));
151                    state.failure = None;
152                    Ok(credential)
153                }
154                Err(error) => {
155                    state.credential = None;
156                    state.failure = Some((error.clone(), seq));
157                    Err(error)
158                }
159            }
160        })
161    }
162
163    fn invalidate(&self, provider: &str) {
164        // Replace the slot: requests already waiting on the old slot finish with
165        // whatever it holds; new requests fetch again.
166        {
167            let mut slots = self.slots.lock().unwrap_or_else(PoisonError::into_inner);
168            slots.insert(provider.to_owned(), Arc::new(Slot::default()));
169        }
170        self.inner.invalidate(provider);
171    }
172
173    fn generation(&self) -> u64 {
174        self.inner.generation()
175    }
176}
177
178impl<S: CredentialSource + ?Sized> CredentialSource for Arc<S> {
179    fn credential<'a>(&'a self, provider: &'a str) -> BoxFuture<'a, Result<Credential, LlmError>> {
180        (**self).credential(provider)
181    }
182
183    fn invalidate(&self, provider: &str) {
184        (**self).invalidate(provider);
185    }
186
187    fn generation(&self) -> u64 {
188        (**self).generation()
189    }
190}
191
192#[cfg(test)]
193mod tests {
194    use std::sync::atomic::{AtomicUsize, Ordering};
195
196    use super::*;
197
198    #[derive(Debug, Default)]
199    struct Counting {
200        calls: AtomicUsize,
201        ttl_ms: Option<u64>,
202        generation: AtomicU64,
203    }
204
205    impl CredentialSource for Counting {
206        fn credential<'a>(&'a self, _provider: &'a str) -> BoxFuture<'a, Result<Credential, LlmError>> {
207            let n = self.calls.fetch_add(1, Ordering::SeqCst) + 1;
208            let mut c = Credential::new(format!("token-{n}"));
209            if let Some(ttl) = self.ttl_ms {
210                c = c.expires_at(unix_now_ms() + ttl);
211            }
212            Box::pin(async move { Ok(c) })
213        }
214
215        fn generation(&self) -> u64 {
216            self.generation.load(Ordering::SeqCst)
217        }
218    }
219
220    #[tokio::test]
221    async fn caches_until_invalidated() {
222        let cache = CachedCredentials::new(Counting::default());
223        let a = cache.credential("p").await.unwrap();
224        let b = cache.credential("p").await.unwrap();
225        assert_eq!(a.secret, "token-1");
226        assert_eq!(b.secret, "token-1");
227        cache.invalidate("p");
228        let c = cache.credential("p").await.unwrap();
229        assert_eq!(c.secret, "token-2");
230        // Providers are cached independently.
231        let other = cache.credential("q").await.unwrap();
232        assert_eq!(other.secret, "token-3");
233    }
234
235    #[tokio::test]
236    async fn source_generation_change_drops_cached_values() {
237        let cache = CachedCredentials::new(Counting::default());
238        assert_eq!(cache.credential("p").await.unwrap().secret, "token-1");
239        // e.g. the host signed user A out and registered user B's key source.
240        cache.inner().generation.fetch_add(1, Ordering::SeqCst);
241        assert_eq!(cache.credential("p").await.unwrap().secret, "token-2");
242        cache.clear();
243        assert_eq!(cache.credential("p").await.unwrap().secret, "token-3");
244    }
245
246    #[tokio::test]
247    async fn refreshes_near_expiry() {
248        // Expires in 30 s, refresh skew 60 s: always considered stale.
249        let cache = CachedCredentials::new(Counting { ttl_ms: Some(30_000), ..Counting::default() });
250        let a = cache.credential("p").await.unwrap();
251        let b = cache.credential("p").await.unwrap();
252        assert_ne!(a.secret, b.secret);
253        // With a smaller skew the same credential is reused.
254        let cache =
255            CachedCredentials::new(Counting { ttl_ms: Some(30_000), ..Counting::default() }).refresh_skew_ms(1_000);
256        let a = cache.credential("p").await.unwrap();
257        let b = cache.credential("p").await.unwrap();
258        assert_eq!(a.secret, b.secret);
259    }
260
261    #[tokio::test]
262    async fn concurrent_requests_share_one_fetch() {
263        #[derive(Debug, Default)]
264        struct Slow(AtomicUsize);
265        impl CredentialSource for Slow {
266            fn credential<'a>(&'a self, _p: &'a str) -> BoxFuture<'a, Result<Credential, LlmError>> {
267                Box::pin(async move {
268                    self.0.fetch_add(1, Ordering::SeqCst);
269                    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
270                    Ok(Credential::new("t"))
271                })
272            }
273        }
274        let cache = Arc::new(CachedCredentials::new(Slow::default()));
275        let futs = (0..8).map(|_| {
276            let c = cache.clone();
277            async move { c.credential("p").await }
278        });
279        let results = futures::future::join_all(futs).await;
280        assert!(results.iter().all(Result::is_ok));
281        assert_eq!(cache.inner().0.load(Ordering::SeqCst), 1);
282    }
283
284    #[tokio::test]
285    async fn waiters_share_one_failure_and_hung_sources_time_out() {
286        #[derive(Debug, Default)]
287        struct Hangs(AtomicUsize);
288        impl CredentialSource for Hangs {
289            fn credential<'a>(&'a self, _p: &'a str) -> BoxFuture<'a, Result<Credential, LlmError>> {
290                Box::pin(async move {
291                    self.0.fetch_add(1, Ordering::SeqCst);
292                    futures::future::pending::<()>().await;
293                    Ok(Credential::new("never"))
294                })
295            }
296        }
297        let cache = Arc::new(CachedCredentials::new(Hangs::default()).fetch_timeout(Duration::from_millis(50)));
298        let started = std::time::Instant::now();
299        let futs = (0..6).map(|_| {
300            let c = cache.clone();
301            async move { c.credential("p").await }
302        });
303        let results = futures::future::join_all(futs).await;
304        assert!(results.iter().all(|r| matches!(r, Err(LlmError::Auth(_)))));
305        // One fetch timed out and every waiter got its error: not 6 serial timeouts.
306        assert_eq!(cache.inner().0.load(Ordering::SeqCst), 1);
307        assert!(started.elapsed() < Duration::from_millis(250), "{:?}", started.elapsed());
308        // A later request tries again.
309        assert!(cache.credential("p").await.is_err());
310        assert_eq!(cache.inner().0.load(Ordering::SeqCst), 2);
311    }
312}