codewhale_runtime/
retry_status.rs1use std::collections::HashMap;
32use std::sync::{Mutex, OnceLock};
33use std::time::{Duration, Instant};
34
35type RateLimitPauses = HashMap<String, (Instant, Duration)>;
41
42#[derive(Debug, Clone)]
46pub struct RetryBanner {
47 pub attempt: u32,
49 pub deadline: Instant,
51 pub reason: String,
53}
54
55#[derive(Debug, Clone, Default)]
57pub enum RetryState {
58 #[default]
60 Idle,
61 Active(RetryBanner),
63 Failed { reason: String, since: Instant },
68}
69
70impl RetryState {
71 #[cfg(any(test, feature = "test-support"))]
75 #[must_use]
76 pub fn seconds_remaining(&self) -> Option<u64> {
77 match self {
78 Self::Active(banner) => Some(
79 banner
80 .deadline
81 .saturating_duration_since(Instant::now())
82 .as_secs(),
83 ),
84 _ => None,
85 }
86 }
87
88 #[cfg(any(test, feature = "test-support"))]
92 #[must_use]
93 pub fn is_failed(&self) -> bool {
94 matches!(self, Self::Failed { .. })
95 }
96}
97
98#[cfg(not(any(test, all(feature = "test-thread-scoped-state", debug_assertions))))]
101fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R {
102 static STATE: OnceLock<Mutex<RetryState>> = OnceLock::new();
103 let mut state = STATE
104 .get_or_init(|| Mutex::new(RetryState::Idle))
105 .lock()
106 .unwrap_or_else(|error| error.into_inner());
107 f(&mut state)
108}
109
110#[cfg(not(any(test, all(feature = "test-thread-scoped-state", debug_assertions))))]
111fn with_rate_limit<R>(f: impl FnOnce(&mut RateLimitPauses) -> R) -> R {
112 static STATE: OnceLock<Mutex<RateLimitPauses>> = OnceLock::new();
113 let mut state = STATE
114 .get_or_init(|| Mutex::new(HashMap::new()))
115 .lock()
116 .unwrap_or_else(|error| error.into_inner());
117 f(&mut state)
118}
119
120#[cfg(any(test, all(feature = "test-thread-scoped-state", debug_assertions)))]
140fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R {
141 #[allow(clippy::type_complexity)]
142 static STATE: OnceLock<Mutex<std::collections::HashMap<std::thread::ThreadId, RetryState>>> =
143 OnceLock::new();
144 let mut by_thread = STATE
145 .get_or_init(|| Mutex::new(std::collections::HashMap::new()))
146 .lock()
147 .unwrap_or_else(|error| error.into_inner());
148 f(by_thread
149 .entry(std::thread::current().id())
150 .or_insert(RetryState::Idle))
151}
152
153#[cfg(any(test, all(feature = "test-thread-scoped-state", debug_assertions)))]
154fn with_rate_limit<R>(f: impl FnOnce(&mut RateLimitPauses) -> R) -> R {
155 static STATE: OnceLock<Mutex<HashMap<std::thread::ThreadId, RateLimitPauses>>> =
156 OnceLock::new();
157 let mut by_thread = STATE
158 .get_or_init(|| Mutex::new(HashMap::new()))
159 .lock()
160 .unwrap_or_else(|error| error.into_inner());
161 f(by_thread.entry(std::thread::current().id()).or_default())
162}
163
164#[cfg(any(test, feature = "test-support"))]
167#[must_use]
168pub fn snapshot() -> RetryState {
169 with_state(|state| state.clone())
170}
171
172pub const MAX_PROVIDER_PAUSE: Duration = Duration::from_secs(3600);
173
174pub fn note_rate_limit(scope: &str, delay: Duration) {
181 let delay = delay.min(MAX_PROVIDER_PAUSE);
182 with_rate_limit(|pauses| {
183 let now = Instant::now();
184 pauses.retain(|_, (observed, delay)| {
185 !delay
186 .saturating_sub(now.saturating_duration_since(*observed))
187 .is_zero()
188 });
189 let current = pauses.entry(scope.to_string()).or_insert((now, delay));
190 if current
191 .1
192 .saturating_sub(now.saturating_duration_since(current.0))
193 < delay
194 {
195 *current = (now, delay);
196 }
197 });
198}
199
200#[must_use]
202pub fn rate_limit_remaining(scope: &str) -> Option<Duration> {
203 let now = Instant::now();
204 with_rate_limit(|pauses| {
205 let (observed, delay) = pauses.get(scope).copied()?;
206 let remaining = delay.saturating_sub(now.saturating_duration_since(observed));
207 if remaining.is_zero() {
208 pauses.remove(scope);
209 None
210 } else {
211 Some(remaining)
212 }
213 })
214}
215
216pub fn start(attempt: u32, delay: Duration, reason: impl Into<String>) {
220 let delay = delay.min(MAX_PROVIDER_PAUSE);
221 let banner = RetryBanner {
222 attempt,
223 deadline: Instant::now() + delay,
224 reason: reason.into(),
225 };
226 with_state(|state| *state = RetryState::Active(banner));
227}
228
229pub fn succeeded() {
231 with_state(|state| *state = RetryState::Idle);
232}
233
234pub fn failed(reason: impl Into<String>) {
237 with_state(|state| {
238 *state = RetryState::Failed {
239 reason: reason.into(),
240 since: Instant::now(),
241 };
242 });
243}
244
245pub fn clear() {
248 with_state(|state| *state = RetryState::Idle);
249}
250
251#[cfg(any(test, feature = "test-support"))]
253pub fn clear_rate_limit() {
254 with_rate_limit(HashMap::clear);
255}
256
257#[cfg(any(test, feature = "test-support"))]
262pub fn test_guard() -> std::sync::MutexGuard<'static, ()> {
263 static GUARD: Mutex<()> = Mutex::new(());
264 GUARD.lock().unwrap_or_else(|e| e.into_inner())
265}
266
267#[cfg(test)]
268mod tests {
269 use super::*;
270
271 fn setup() -> std::sync::MutexGuard<'static, ()> {
274 let g = test_guard();
275 clear();
276 clear_rate_limit();
277 g
278 }
279
280 #[test]
281 fn idle_by_default_after_clear() {
282 let _g = setup();
283 assert!(matches!(snapshot(), RetryState::Idle));
284 assert_eq!(snapshot().seconds_remaining(), None);
285 }
286
287 #[test]
288 fn start_then_succeeded_returns_to_idle() {
289 let _g = setup();
290 start(1, Duration::from_secs(5), "rate limited");
291 let s = snapshot();
292 assert!(matches!(s, RetryState::Active(_)));
293 let remaining = s.seconds_remaining().unwrap();
294 assert!(remaining <= 5, "{remaining}");
295 succeeded();
296 assert!(matches!(snapshot(), RetryState::Idle));
297 }
298
299 #[test]
300 fn failed_persists_until_clear() {
301 let _g = setup();
302 failed("upstream 500");
303 let s = snapshot();
304 assert!(s.is_failed());
305 if let RetryState::Failed { reason, .. } = s {
306 assert_eq!(reason, "upstream 500");
307 } else {
308 panic!("expected Failed");
309 }
310 clear();
311 assert!(matches!(snapshot(), RetryState::Idle));
312 }
313
314 #[test]
315 fn deadline_in_past_yields_zero_remaining() {
316 let _g = setup();
317 with_state(|state| {
319 *state = RetryState::Active(RetryBanner {
320 attempt: 2,
321 deadline: Instant::now() - Duration::from_secs(1),
322 reason: "test".into(),
323 });
324 });
325 assert_eq!(snapshot().seconds_remaining(), Some(0));
326 clear();
327 }
328
329 #[test]
330 fn rate_limit_deadline_survives_banner_clear() {
331 let _g = setup();
332 note_rate_limit("provider-a", Duration::from_secs(5));
333 start(1, Duration::from_secs(5), "rate limited");
334 succeeded();
335 assert!(
336 rate_limit_remaining("provider-a").is_some(),
337 "provider rate limit pause must not be cleared by an unrelated success"
338 );
339 clear_rate_limit();
340 }
341
342 #[test]
343 fn rate_limit_bounds_unrepresentable_instant_minimum() {
344 let _g = setup();
345 note_rate_limit("extreme-minimum", Duration::from_secs(u64::MAX));
346 note_rate_limit("extreme-minimum", Duration::from_secs(1));
347 let remaining = rate_limit_remaining("extreme-minimum").unwrap();
348 assert!(remaining <= MAX_PROVIDER_PAUSE);
349 assert!(remaining > Duration::from_secs(3598));
350 clear_rate_limit();
351 }
352
353 #[test]
354 fn one_providers_rate_limit_does_not_pause_another() {
355 let _g = setup();
356 note_rate_limit("provider-a@api.a.example", Duration::from_secs(30));
357 assert!(rate_limit_remaining("provider-a@api.a.example").is_some());
358 assert_eq!(rate_limit_remaining("ollama@127.0.0.1:11434"), None);
359
360 note_rate_limit("provider-a@api.a.example", Duration::from_secs(1));
362 assert!(
363 rate_limit_remaining("provider-a@api.a.example").unwrap() > Duration::from_secs(20)
364 );
365 clear_rate_limit();
366 assert_eq!(rate_limit_remaining("provider-a@api.a.example"), None);
367 }
368}