Skip to main content

codewhale_runtime/
retry_status.rs

1//! Process-wide retry-state surface (#499).
2//!
3//! Read-side caveat (0.9.4): the renderer this module was written for was
4//! the legacy footer's retry banner, which went with `FooterWidget`. The
5//! *producer* — `client::send_with_retry` — is still live and still records
6//! every retry, and `client`'s own tests read it back through `snapshot`.
7//! The read surface (`snapshot`, the countdown, the banner fields) is
8//! therefore test-gated (`cfg(any(test, feature = "test-support"))` where
9//! possible, so the TUI's tests reach it through the `test-support`
10//! feature; `cfg_attr(not(test))` allows where prod constructs); a renderer
11//! restores it by dropping the gates. Give the banner a renderer, or
12//! delete the producer too — but not half of it.
13//!
14//! The HTTP retry path in `client::send_with_retry` already times its
15//! waits and knows the error category. This module gives the TUI a way
16//! to observe that state — `start`, `succeeded`, and `failed` flip a
17//! global `RetryState` that the footer / status panel reads each frame.
18//!
19//! Why a process-wide global: the user-facing TUI runs as one engine
20//! per process, and the only retry state we want to surface is the one
21//! the user is staring at. Sub-agent retries in background tasks
22//! deliberately do **not** light up the foreground banner — they're
23//! supposed to be invisible. If a future feature ever needs per-engine
24//! retry surfaces, swap this for an `Arc<RwLock<...>>` carried on the
25//! `EngineHandle`; the public API stays the same.
26//!
27//! The `Retry-After` pause is process-wide too, but keyed by provider scope:
28//! every request to the rate-limited route waits it out, and requests to any
29//! other route (another provider, a local runtime) do not.
30
31use std::collections::HashMap;
32use std::sync::{Mutex, OnceLock};
33use std::time::{Duration, Instant};
34
35/// Rate-limit pause deadlines, one per provider scope (see
36/// [`note_rate_limit`]). A 429 from one provider must not stall requests to
37/// another provider, a local runtime, or a sub-agent on a different route.
38// Keep a duration relative to its observation time: even a valid u64-second
39// server minimum may be outside the representable Instant range.
40type RateLimitPauses = HashMap<String, (Instant, Duration)>;
41
42/// One in-flight retry attempt. `deadline` is the wall-clock time the
43/// next request will fire — the UI subtracts `Instant::now()` from it
44/// to render a live countdown.
45#[derive(Debug, Clone)]
46pub struct RetryBanner {
47    /// 1-indexed retry attempt number (the first retry is attempt 1).
48    pub attempt: u32,
49    /// Time at which the next request will be sent.
50    pub deadline: Instant,
51    /// Short human-readable reason ("rate limited", "server error", …).
52    pub reason: String,
53}
54
55/// Snapshot of the retry surface for the UI to render.
56#[derive(Debug, Clone, Default)]
57pub enum RetryState {
58    /// No retry in flight. Banner hidden.
59    #[default]
60    Idle,
61    /// A request is sleeping before retrying. Show countdown banner.
62    Active(RetryBanner),
63    /// All retries exhausted; show failure row until the next turn
64    /// starts. `since` records when the row was set so a future polish
65    /// pass can age it out automatically; today the engine clears it on
66    /// `TurnStarted`.
67    Failed { reason: String, since: Instant },
68}
69
70impl RetryState {
71    /// Wall-clock seconds remaining on the active banner, or `None` if
72    /// not active. Saturates at zero — the renderer should treat any
73    /// negative remaining as "firing now".
74    #[cfg(any(test, feature = "test-support"))]
75    #[must_use]
76    pub fn seconds_remaining(&self) -> Option<u64> {
77        match self {
78            Self::Active(banner) => Some(
79                banner
80                    .deadline
81                    .saturating_duration_since(Instant::now())
82                    .as_secs(),
83            ),
84            _ => None,
85        }
86    }
87
88    /// Whether the failure row should still be shown. Mirrors the
89    /// "until next turn" rule in the issue spec; the engine clears it
90    /// explicitly via [`clear`] on `TurnStarted`.
91    #[cfg(any(test, feature = "test-support"))]
92    #[must_use]
93    pub fn is_failed(&self) -> bool {
94        matches!(self, Self::Failed { .. })
95    }
96}
97
98/// Lazy-init the cell on first read so callers don't have to initialize
99/// process-wide state at boot.
100#[cfg(not(any(test, all(feature = "test-thread-scoped-state", debug_assertions))))]
101fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R {
102    static STATE: OnceLock<Mutex<RetryState>> = OnceLock::new();
103    let mut state = STATE
104        .get_or_init(|| Mutex::new(RetryState::Idle))
105        .lock()
106        .unwrap_or_else(|error| error.into_inner());
107    f(&mut state)
108}
109
110#[cfg(not(any(test, all(feature = "test-thread-scoped-state", debug_assertions))))]
111fn with_rate_limit<R>(f: impl FnOnce(&mut RateLimitPauses) -> R) -> R {
112    static STATE: OnceLock<Mutex<RateLimitPauses>> = OnceLock::new();
113    let mut state = STATE
114        .get_or_init(|| Mutex::new(HashMap::new()))
115        .lock()
116        .unwrap_or_else(|error| error.into_inner());
117    f(&mut state)
118}
119
120/// Under test, this state is per-thread.
121///
122/// Production has exactly one foreground engine per process, so a global is the
123/// right model there. The test harness does not: retry state is written as a
124/// side effect of *any* client request, by production code that has no test
125/// guard to take, so a real request in one test could publish a banner or a
126/// provider pause into another test's assertions. Scoping by thread removes the
127/// race at its source rather than asking every future test that happens to
128/// perform HTTP to remember a lock.
129///
130/// This changes behavior (a `Retry-After` pause no longer holds across tokio
131/// worker threads), so it is *not* part of `test-support`, which only adds
132/// helpers. The TUI's tests opt in with `test-thread-scoped-state`, and even
133/// then only a build with debug assertions gets it: an optimized build, e.g.
134/// `cargo build --release --workspace --all-features`, keeps the process-wide
135/// pause. Cargo unifies features across one build, so under
136/// `cargo test --workspace` (or a debug `--all-features` build) the debug
137/// `codewhale` binary that integration tests spawn gets the per-thread pause
138/// too; release and optimized binaries never do.
139#[cfg(any(test, all(feature = "test-thread-scoped-state", debug_assertions)))]
140fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R {
141    #[allow(clippy::type_complexity)]
142    static STATE: OnceLock<Mutex<std::collections::HashMap<std::thread::ThreadId, RetryState>>> =
143        OnceLock::new();
144    let mut by_thread = STATE
145        .get_or_init(|| Mutex::new(std::collections::HashMap::new()))
146        .lock()
147        .unwrap_or_else(|error| error.into_inner());
148    f(by_thread
149        .entry(std::thread::current().id())
150        .or_insert(RetryState::Idle))
151}
152
153#[cfg(any(test, all(feature = "test-thread-scoped-state", debug_assertions)))]
154fn with_rate_limit<R>(f: impl FnOnce(&mut RateLimitPauses) -> R) -> R {
155    static STATE: OnceLock<Mutex<HashMap<std::thread::ThreadId, RateLimitPauses>>> =
156        OnceLock::new();
157    let mut by_thread = STATE
158        .get_or_init(|| Mutex::new(HashMap::new()))
159        .lock()
160        .unwrap_or_else(|error| error.into_inner());
161    f(by_thread.entry(std::thread::current().id()).or_default())
162}
163
164/// Read snapshot for renderers. No production renderer exists since the
165/// legacy footer went away; `client` retry tests are the only readers.
166#[cfg(any(test, feature = "test-support"))]
167#[must_use]
168pub fn snapshot() -> RetryState {
169    with_state(|state| state.clone())
170}
171
172pub const MAX_PROVIDER_PAUSE: Duration = Duration::from_secs(3600);
173
174/// Extend the rate-limit pause window for one provider `scope` (the caller's
175/// route identity and host). This is separate from the footer banner so one
176/// successful concurrent request cannot clear another request's active
177/// `Retry-After` window, and it is keyed so a 429 from one provider never
178/// pauses requests to another. Expired scopes are dropped here, so the map
179/// holds at most the providers currently rate limited.
180pub fn note_rate_limit(scope: &str, delay: Duration) {
181    let delay = delay.min(MAX_PROVIDER_PAUSE);
182    with_rate_limit(|pauses| {
183        let now = Instant::now();
184        pauses.retain(|_, (observed, delay)| {
185            !delay
186                .saturating_sub(now.saturating_duration_since(*observed))
187                .is_zero()
188        });
189        let current = pauses.entry(scope.to_string()).or_insert((now, delay));
190        if current
191            .1
192            .saturating_sub(now.saturating_duration_since(current.0))
193            < delay
194        {
195            *current = (now, delay);
196        }
197    });
198}
199
200/// Remaining rate-limit pause for `scope`, if any.
201#[must_use]
202pub fn rate_limit_remaining(scope: &str) -> Option<Duration> {
203    let now = Instant::now();
204    with_rate_limit(|pauses| {
205        let (observed, delay) = pauses.get(scope).copied()?;
206        let remaining = delay.saturating_sub(now.saturating_duration_since(observed));
207        if remaining.is_zero() {
208            pauses.remove(scope);
209            None
210        } else {
211            Some(remaining)
212        }
213    })
214}
215
216/// Mark an in-flight retry. `attempt` is the number of the *upcoming*
217/// retry (1 for the first); `delay` is how long the client will sleep
218/// before firing.
219pub fn start(attempt: u32, delay: Duration, reason: impl Into<String>) {
220    let delay = delay.min(MAX_PROVIDER_PAUSE);
221    let banner = RetryBanner {
222        attempt,
223        deadline: Instant::now() + delay,
224        reason: reason.into(),
225    };
226    with_state(|state| *state = RetryState::Active(banner));
227}
228
229/// Mark the retry chain as having succeeded. Hides the banner.
230pub fn succeeded() {
231    with_state(|state| *state = RetryState::Idle);
232}
233
234/// Mark the retry chain as having exhausted retries. The renderer keeps
235/// the failure row until [`clear`] (typically called on `TurnStarted`).
236pub fn failed(reason: impl Into<String>) {
237    with_state(|state| {
238        *state = RetryState::Failed {
239            reason: reason.into(),
240            since: Instant::now(),
241        };
242    });
243}
244
245/// Reset to idle. Called on `TurnStarted` so the previous turn's
246/// failure row doesn't bleed into the next turn.
247pub fn clear() {
248    with_state(|state| *state = RetryState::Idle);
249}
250
251/// Drop every scope's rate-limit pause.
252#[cfg(any(test, feature = "test-support"))]
253pub fn clear_rate_limit() {
254    with_rate_limit(HashMap::clear);
255}
256
257/// Test helper: serialize tests that touch the global state so cargo's
258/// parallel runner can't observe a torn read. The guard is exported so
259/// tests in *other* modules (e.g. footer rendering tests) can hold the
260/// same lock as the ones in `retry_status::tests`.
261#[cfg(any(test, feature = "test-support"))]
262pub fn test_guard() -> std::sync::MutexGuard<'static, ()> {
263    static GUARD: Mutex<()> = Mutex::new(());
264    GUARD.lock().unwrap_or_else(|e| e.into_inner())
265}
266
267#[cfg(test)]
268mod tests {
269    use super::*;
270
271    /// Acquire the cross-module test guard from [`super::test_guard`] and
272    /// reset state to `Idle` before yielding to the test body.
273    fn setup() -> std::sync::MutexGuard<'static, ()> {
274        let g = test_guard();
275        clear();
276        clear_rate_limit();
277        g
278    }
279
280    #[test]
281    fn idle_by_default_after_clear() {
282        let _g = setup();
283        assert!(matches!(snapshot(), RetryState::Idle));
284        assert_eq!(snapshot().seconds_remaining(), None);
285    }
286
287    #[test]
288    fn start_then_succeeded_returns_to_idle() {
289        let _g = setup();
290        start(1, Duration::from_secs(5), "rate limited");
291        let s = snapshot();
292        assert!(matches!(s, RetryState::Active(_)));
293        let remaining = s.seconds_remaining().unwrap();
294        assert!(remaining <= 5, "{remaining}");
295        succeeded();
296        assert!(matches!(snapshot(), RetryState::Idle));
297    }
298
299    #[test]
300    fn failed_persists_until_clear() {
301        let _g = setup();
302        failed("upstream 500");
303        let s = snapshot();
304        assert!(s.is_failed());
305        if let RetryState::Failed { reason, .. } = s {
306            assert_eq!(reason, "upstream 500");
307        } else {
308            panic!("expected Failed");
309        }
310        clear();
311        assert!(matches!(snapshot(), RetryState::Idle));
312    }
313
314    #[test]
315    fn deadline_in_past_yields_zero_remaining() {
316        let _g = setup();
317        // Bypass `start` so we can plant a deadline already in the past.
318        with_state(|state| {
319            *state = RetryState::Active(RetryBanner {
320                attempt: 2,
321                deadline: Instant::now() - Duration::from_secs(1),
322                reason: "test".into(),
323            });
324        });
325        assert_eq!(snapshot().seconds_remaining(), Some(0));
326        clear();
327    }
328
329    #[test]
330    fn rate_limit_deadline_survives_banner_clear() {
331        let _g = setup();
332        note_rate_limit("provider-a", Duration::from_secs(5));
333        start(1, Duration::from_secs(5), "rate limited");
334        succeeded();
335        assert!(
336            rate_limit_remaining("provider-a").is_some(),
337            "provider rate limit pause must not be cleared by an unrelated success"
338        );
339        clear_rate_limit();
340    }
341
342    #[test]
343    fn rate_limit_bounds_unrepresentable_instant_minimum() {
344        let _g = setup();
345        note_rate_limit("extreme-minimum", Duration::from_secs(u64::MAX));
346        note_rate_limit("extreme-minimum", Duration::from_secs(1));
347        let remaining = rate_limit_remaining("extreme-minimum").unwrap();
348        assert!(remaining <= MAX_PROVIDER_PAUSE);
349        assert!(remaining > Duration::from_secs(3598));
350        clear_rate_limit();
351    }
352
353    #[test]
354    fn one_providers_rate_limit_does_not_pause_another() {
355        let _g = setup();
356        note_rate_limit("provider-a@api.a.example", Duration::from_secs(30));
357        assert!(rate_limit_remaining("provider-a@api.a.example").is_some());
358        assert_eq!(rate_limit_remaining("ollama@127.0.0.1:11434"), None);
359
360        // A shorter Retry-After never shortens an existing window.
361        note_rate_limit("provider-a@api.a.example", Duration::from_secs(1));
362        assert!(
363            rate_limit_remaining("provider-a@api.a.example").unwrap() > Duration::from_secs(20)
364        );
365        clear_rate_limit();
366        assert_eq!(rate_limit_remaining("provider-a@api.a.example"), None);
367    }
368}