1use std::collections::HashMap;
4use std::fmt;
5use std::sync::atomic::{AtomicU64, Ordering};
6use std::sync::{Arc, Mutex, PoisonError};
7use std::time::Duration;
8
9use futures::future::BoxFuture;
10use futures::lock::Mutex as AsyncMutex;
11
12use super::{Credential, CredentialSource, unix_now_ms};
13use crate::error::LlmError;
14
15pub const DEFAULT_REFRESH_SKEW_MS: u64 = 60_000;
17
18pub const DEFAULT_FETCH_TIMEOUT: Duration = Duration::from_secs(15);
20
21#[derive(Default)]
22struct SlotState {
23 credential: Option<(Credential, u64)>,
25 failure: Option<(LlmError, u64)>,
27}
28
29#[derive(Default)]
32struct Slot {
33 state: AsyncMutex<SlotState>,
34 completed: AtomicU64,
36}
37
38pub struct CachedCredentials<S> {
51 inner: S,
52 skew_ms: u64,
53 fetch_timeout: Duration,
54 slots: Mutex<HashMap<String, Arc<Slot>>>,
55}
56
57impl<S: CredentialSource> CachedCredentials<S> {
58 pub fn new(inner: S) -> Self {
59 Self {
60 inner,
61 skew_ms: DEFAULT_REFRESH_SKEW_MS,
62 fetch_timeout: DEFAULT_FETCH_TIMEOUT,
63 slots: Mutex::new(HashMap::new()),
64 }
65 }
66
67 pub fn refresh_skew_ms(mut self, skew_ms: u64) -> Self {
69 self.skew_ms = skew_ms;
70 self
71 }
72
73 pub fn fetch_timeout(mut self, timeout: Duration) -> Self {
75 self.fetch_timeout = timeout;
76 self
77 }
78
79 pub fn inner(&self) -> &S {
81 &self.inner
82 }
83
84 pub fn clear(&self) {
86 let mut slots = self.slots.lock().unwrap_or_else(PoisonError::into_inner);
87 slots.clear();
88 }
89
90 fn slot(&self, provider: &str) -> Arc<Slot> {
91 let mut slots = self.slots.lock().unwrap_or_else(PoisonError::into_inner);
92 slots.entry(provider.to_owned()).or_default().clone()
93 }
94
95 async fn fetch(&self, provider: &str) -> Result<Credential, LlmError> {
96 let fetch = self.inner.credential(provider);
97 #[cfg(feature = "http")]
98 {
99 if tokio::runtime::Handle::try_current().is_ok() {
100 return match tokio::time::timeout(self.fetch_timeout, fetch).await {
101 Ok(result) => result,
102 Err(_) => Err(LlmError::Auth(format!(
103 "credential source for `{provider}` did not answer within {} ms",
104 self.fetch_timeout.as_millis()
105 ))),
106 };
107 }
108 }
109 fetch.await
110 }
111}
112
113impl<S> fmt::Debug for CachedCredentials<S>
114where
115 S: fmt::Debug,
116{
117 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
118 f.debug_struct("CachedCredentials")
119 .field("inner", &self.inner)
120 .field("skew_ms", &self.skew_ms)
121 .field("fetch_timeout", &self.fetch_timeout)
122 .finish()
123 }
124}
125
126impl<S: CredentialSource> CredentialSource for CachedCredentials<S> {
127 fn credential<'a>(&'a self, provider: &'a str) -> BoxFuture<'a, Result<Credential, LlmError>> {
128 Box::pin(async move {
129 let slot = self.slot(provider);
130 let ticket = slot.completed.load(Ordering::SeqCst);
133 let mut state = slot.state.lock().await;
134 if let Some((error, seq)) = &state.failure {
135 if *seq > ticket {
136 return Err(error.clone());
137 }
138 }
139 let generation = self.inner.generation();
140 if let Some((credential, cached_generation)) = &state.credential {
141 let fresh = *cached_generation == generation && !credential.is_expired_at(unix_now_ms(), self.skew_ms);
142 if fresh {
143 return Ok(credential.clone());
144 }
145 }
146 let result = self.fetch(provider).await;
147 let seq = slot.completed.fetch_add(1, Ordering::SeqCst).saturating_add(1);
148 match result {
149 Ok(credential) => {
150 state.credential = Some((credential.clone(), generation));
151 state.failure = None;
152 Ok(credential)
153 }
154 Err(error) => {
155 state.credential = None;
156 state.failure = Some((error.clone(), seq));
157 Err(error)
158 }
159 }
160 })
161 }
162
163 fn invalidate(&self, provider: &str) {
164 {
167 let mut slots = self.slots.lock().unwrap_or_else(PoisonError::into_inner);
168 slots.insert(provider.to_owned(), Arc::new(Slot::default()));
169 }
170 self.inner.invalidate(provider);
171 }
172
173 fn generation(&self) -> u64 {
174 self.inner.generation()
175 }
176}
177
178impl<S: CredentialSource + ?Sized> CredentialSource for Arc<S> {
179 fn credential<'a>(&'a self, provider: &'a str) -> BoxFuture<'a, Result<Credential, LlmError>> {
180 (**self).credential(provider)
181 }
182
183 fn invalidate(&self, provider: &str) {
184 (**self).invalidate(provider);
185 }
186
187 fn generation(&self) -> u64 {
188 (**self).generation()
189 }
190}
191
192#[cfg(test)]
193mod tests {
194 use std::sync::atomic::{AtomicUsize, Ordering};
195
196 use super::*;
197
198 #[derive(Debug, Default)]
199 struct Counting {
200 calls: AtomicUsize,
201 ttl_ms: Option<u64>,
202 generation: AtomicU64,
203 }
204
205 impl CredentialSource for Counting {
206 fn credential<'a>(&'a self, _provider: &'a str) -> BoxFuture<'a, Result<Credential, LlmError>> {
207 let n = self.calls.fetch_add(1, Ordering::SeqCst) + 1;
208 let mut c = Credential::new(format!("token-{n}"));
209 if let Some(ttl) = self.ttl_ms {
210 c = c.expires_at(unix_now_ms() + ttl);
211 }
212 Box::pin(async move { Ok(c) })
213 }
214
215 fn generation(&self) -> u64 {
216 self.generation.load(Ordering::SeqCst)
217 }
218 }
219
220 #[tokio::test]
221 async fn caches_until_invalidated() {
222 let cache = CachedCredentials::new(Counting::default());
223 let a = cache.credential("p").await.unwrap();
224 let b = cache.credential("p").await.unwrap();
225 assert_eq!(a.secret, "token-1");
226 assert_eq!(b.secret, "token-1");
227 cache.invalidate("p");
228 let c = cache.credential("p").await.unwrap();
229 assert_eq!(c.secret, "token-2");
230 let other = cache.credential("q").await.unwrap();
232 assert_eq!(other.secret, "token-3");
233 }
234
235 #[tokio::test]
236 async fn source_generation_change_drops_cached_values() {
237 let cache = CachedCredentials::new(Counting::default());
238 assert_eq!(cache.credential("p").await.unwrap().secret, "token-1");
239 cache.inner().generation.fetch_add(1, Ordering::SeqCst);
241 assert_eq!(cache.credential("p").await.unwrap().secret, "token-2");
242 cache.clear();
243 assert_eq!(cache.credential("p").await.unwrap().secret, "token-3");
244 }
245
246 #[tokio::test]
247 async fn refreshes_near_expiry() {
248 let cache = CachedCredentials::new(Counting { ttl_ms: Some(30_000), ..Counting::default() });
250 let a = cache.credential("p").await.unwrap();
251 let b = cache.credential("p").await.unwrap();
252 assert_ne!(a.secret, b.secret);
253 let cache =
255 CachedCredentials::new(Counting { ttl_ms: Some(30_000), ..Counting::default() }).refresh_skew_ms(1_000);
256 let a = cache.credential("p").await.unwrap();
257 let b = cache.credential("p").await.unwrap();
258 assert_eq!(a.secret, b.secret);
259 }
260
261 #[tokio::test]
262 async fn concurrent_requests_share_one_fetch() {
263 #[derive(Debug, Default)]
264 struct Slow(AtomicUsize);
265 impl CredentialSource for Slow {
266 fn credential<'a>(&'a self, _p: &'a str) -> BoxFuture<'a, Result<Credential, LlmError>> {
267 Box::pin(async move {
268 self.0.fetch_add(1, Ordering::SeqCst);
269 tokio::time::sleep(std::time::Duration::from_millis(30)).await;
270 Ok(Credential::new("t"))
271 })
272 }
273 }
274 let cache = Arc::new(CachedCredentials::new(Slow::default()));
275 let futs = (0..8).map(|_| {
276 let c = cache.clone();
277 async move { c.credential("p").await }
278 });
279 let results = futures::future::join_all(futs).await;
280 assert!(results.iter().all(Result::is_ok));
281 assert_eq!(cache.inner().0.load(Ordering::SeqCst), 1);
282 }
283
284 #[tokio::test]
285 async fn waiters_share_one_failure_and_hung_sources_time_out() {
286 #[derive(Debug, Default)]
287 struct Hangs(AtomicUsize);
288 impl CredentialSource for Hangs {
289 fn credential<'a>(&'a self, _p: &'a str) -> BoxFuture<'a, Result<Credential, LlmError>> {
290 Box::pin(async move {
291 self.0.fetch_add(1, Ordering::SeqCst);
292 futures::future::pending::<()>().await;
293 Ok(Credential::new("never"))
294 })
295 }
296 }
297 let cache = Arc::new(CachedCredentials::new(Hangs::default()).fetch_timeout(Duration::from_millis(50)));
298 let started = std::time::Instant::now();
299 let futs = (0..6).map(|_| {
300 let c = cache.clone();
301 async move { c.credential("p").await }
302 });
303 let results = futures::future::join_all(futs).await;
304 assert!(results.iter().all(|r| matches!(r, Err(LlmError::Auth(_)))));
305 assert_eq!(cache.inner().0.load(Ordering::SeqCst), 1);
307 assert!(started.elapsed() < Duration::from_millis(250), "{:?}", started.elapsed());
308 assert!(cache.credential("p").await.is_err());
310 assert_eq!(cache.inner().0.load(Ordering::SeqCst), 2);
311 }
312}