1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
//! Process-wide retry-state surface (#499).
//!
//! Read-side caveat (0.9.4): the renderer this module was written for was
//! the legacy footer's retry banner, which went with `FooterWidget`. The
//! *producer* — `client::send_with_retry` — is still live and still records
//! every retry, and `client`'s own tests read it back through [`snapshot`].
//! The read surface below therefore carries `#[allow(dead_code)]` rather
//! than being deleted: removing it would mean changing `start`/`failed`'s
//! signatures at their live call sites in `client.rs`. Give the banner a
//! renderer, or delete the producer too — but not half of it.
//!
//! The HTTP retry path in `client::send_with_retry` already times its
//! waits and knows the error category. This module gives the TUI a way
//! to observe that state — `start`, `succeeded`, and `failed` flip a
//! global `RetryState` that the footer / status panel reads each frame.
//!
//! Why a process-wide global: the user-facing TUI runs as one engine
//! per process, and the only retry state we want to surface is the one
//! the user is staring at. Sub-agent retries in background tasks
//! deliberately do **not** light up the foreground banner — they're
//! supposed to be invisible. If a future feature ever needs per-engine
//! retry surfaces, swap this for an `Arc<RwLock<...>>` carried on the
//! `EngineHandle`; the public API stays the same.
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};
/// One in-flight retry attempt. `deadline` is the wall-clock time the
/// next request will fire — the UI subtracts `Instant::now()` from it
/// to render a live countdown.
#[derive(Debug, Clone)]
#[allow(dead_code)] // written by client::send_with_retry; see the read-side caveat above
pub struct RetryBanner {
/// 1-indexed retry attempt number (the first retry is attempt 1).
pub attempt: u32,
/// Time at which the next request will be sent.
pub deadline: Instant,
/// Short human-readable reason ("rate limited", "server error", …).
pub reason: String,
}
/// Snapshot of the retry surface for the UI to render.
#[derive(Debug, Clone, Default)]
pub enum RetryState {
/// No retry in flight. Banner hidden.
#[default]
Idle,
/// A request is sleeping before retrying. Show countdown banner.
Active(#[allow(dead_code)] RetryBanner),
/// All retries exhausted; show failure row until the next turn
/// starts. `since` records when the row was set so a future polish
/// pass can age it out automatically; today the engine clears it on
/// `TurnStarted`.
Failed {
#[allow(dead_code)]
reason: String,
#[allow(dead_code)]
since: Instant,
},
}
impl RetryState {
/// Wall-clock seconds remaining on the active banner, or `None` if
/// not active. Saturates at zero — the renderer should treat any
/// negative remaining as "firing now".
#[must_use]
#[allow(dead_code)] // no renderer since the legacy footer banner went
pub fn seconds_remaining(&self) -> Option<u64> {
match self {
Self::Active(banner) => Some(
banner
.deadline
.saturating_duration_since(Instant::now())
.as_secs(),
),
_ => None,
}
}
/// Whether the failure row should still be shown. Mirrors the
/// "until next turn" rule in the issue spec; the engine clears it
/// explicitly via [`clear`] on `TurnStarted`.
#[cfg(test)]
#[must_use]
pub fn is_failed(&self) -> bool {
matches!(self, Self::Failed { .. })
}
}
/// Lazy-init the cell on first read so callers don't have to initialize
/// process-wide state at boot.
#[cfg(not(test))]
fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R {
static STATE: OnceLock<Mutex<RetryState>> = OnceLock::new();
let mut state = STATE
.get_or_init(|| Mutex::new(RetryState::Idle))
.lock()
.unwrap_or_else(|error| error.into_inner());
f(&mut state)
}
#[cfg(not(test))]
fn with_rate_limit<R>(f: impl FnOnce(&mut Option<Instant>) -> R) -> R {
static STATE: OnceLock<Mutex<Option<Instant>>> = OnceLock::new();
let mut state = STATE
.get_or_init(|| Mutex::new(None))
.lock()
.unwrap_or_else(|error| error.into_inner());
f(&mut state)
}
/// Under test, this state is per-thread.
///
/// Production has exactly one foreground engine per process, so a global is the
/// right model there. The test harness does not: retry state is written as a
/// side effect of *any* client request, by production code that has no test
/// guard to take, so a real request in one test could publish a banner or a
/// provider pause into another test's assertions. Scoping by thread removes the
/// race at its source rather than asking every future test that happens to
/// perform HTTP to remember a lock.
#[cfg(test)]
fn with_state<R>(f: impl FnOnce(&mut RetryState) -> R) -> R {
#[allow(clippy::type_complexity)]
static STATE: OnceLock<Mutex<std::collections::HashMap<std::thread::ThreadId, RetryState>>> =
OnceLock::new();
let mut by_thread = STATE
.get_or_init(|| Mutex::new(std::collections::HashMap::new()))
.lock()
.unwrap_or_else(|error| error.into_inner());
f(by_thread
.entry(std::thread::current().id())
.or_insert(RetryState::Idle))
}
#[cfg(test)]
fn with_rate_limit<R>(f: impl FnOnce(&mut Option<Instant>) -> R) -> R {
#[allow(clippy::type_complexity)]
static STATE: OnceLock<
Mutex<std::collections::HashMap<std::thread::ThreadId, Option<Instant>>>,
> = OnceLock::new();
let mut by_thread = STATE
.get_or_init(|| Mutex::new(std::collections::HashMap::new()))
.lock()
.unwrap_or_else(|error| error.into_inner());
f(by_thread.entry(std::thread::current().id()).or_default())
}
/// Public read snapshot for renderers.
#[must_use]
#[allow(dead_code)] // read by client.rs's retry tests; no production renderer today
pub fn snapshot() -> RetryState {
with_state(|state| state.clone())
}
/// Extend the provider-wide rate-limit pause window. This is separate from
/// the footer banner so one successful concurrent request cannot clear another
/// request's active `Retry-After` window.
pub fn note_rate_limit(delay: Duration) {
let deadline = Instant::now() + delay;
with_rate_limit(|current| {
if current.is_none_or(|existing| existing < deadline) {
*current = Some(deadline);
}
});
}
/// Remaining provider-wide rate-limit pause, if any.
#[must_use]
pub fn rate_limit_remaining() -> Option<Duration> {
let now = Instant::now();
with_rate_limit(|current| match *current {
Some(deadline) if deadline > now => Some(deadline.duration_since(now)),
Some(_) => {
*current = None;
None
}
None => None,
})
}
/// Mark an in-flight retry. `attempt` is the number of the *upcoming*
/// retry (1 for the first); `delay` is how long the client will sleep
/// before firing.
pub fn start(attempt: u32, delay: Duration, reason: impl Into<String>) {
let banner = RetryBanner {
attempt,
deadline: Instant::now() + delay,
reason: reason.into(),
};
with_state(|state| *state = RetryState::Active(banner));
}
/// Mark the retry chain as having succeeded. Hides the banner.
pub fn succeeded() {
with_state(|state| *state = RetryState::Idle);
}
/// Mark the retry chain as having exhausted retries. The renderer keeps
/// the failure row until [`clear`] (typically called on `TurnStarted`).
pub fn failed(reason: impl Into<String>) {
with_state(|state| {
*state = RetryState::Failed {
reason: reason.into(),
since: Instant::now(),
};
});
}
/// Reset to idle. Called on `TurnStarted` so the previous turn's
/// failure row doesn't bleed into the next turn.
pub fn clear() {
with_state(|state| *state = RetryState::Idle);
}
#[cfg(test)]
pub fn clear_rate_limit() {
with_rate_limit(|current| *current = None);
}
/// Test helper: serialize tests that touch the global state so cargo's
/// parallel runner can't observe a torn read. The guard is exported so
/// tests in *other* modules (e.g. footer rendering tests) can hold the
/// same lock as the ones in `retry_status::tests`.
#[cfg(test)]
pub fn test_guard() -> std::sync::MutexGuard<'static, ()> {
static GUARD: Mutex<()> = Mutex::new(());
GUARD.lock().unwrap_or_else(|e| e.into_inner())
}
#[cfg(test)]
mod tests {
use super::*;
/// Acquire the cross-module test guard from [`super::test_guard`] and
/// reset state to `Idle` before yielding to the test body.
fn setup() -> std::sync::MutexGuard<'static, ()> {
let g = test_guard();
clear();
clear_rate_limit();
g
}
#[test]
fn idle_by_default_after_clear() {
let _g = setup();
assert!(matches!(snapshot(), RetryState::Idle));
assert_eq!(snapshot().seconds_remaining(), None);
}
#[test]
fn start_then_succeeded_returns_to_idle() {
let _g = setup();
start(1, Duration::from_secs(5), "rate limited");
let s = snapshot();
assert!(matches!(s, RetryState::Active(_)));
let remaining = s.seconds_remaining().unwrap();
assert!(remaining <= 5, "{remaining}");
succeeded();
assert!(matches!(snapshot(), RetryState::Idle));
}
#[test]
fn failed_persists_until_clear() {
let _g = setup();
failed("upstream 500");
let s = snapshot();
assert!(s.is_failed());
if let RetryState::Failed { reason, .. } = s {
assert_eq!(reason, "upstream 500");
} else {
panic!("expected Failed");
}
clear();
assert!(matches!(snapshot(), RetryState::Idle));
}
#[test]
fn deadline_in_past_yields_zero_remaining() {
let _g = setup();
// Bypass `start` so we can plant a deadline already in the past.
with_state(|state| {
*state = RetryState::Active(RetryBanner {
attempt: 2,
deadline: Instant::now() - Duration::from_secs(1),
reason: "test".into(),
});
});
assert_eq!(snapshot().seconds_remaining(), Some(0));
clear();
}
#[test]
fn rate_limit_deadline_survives_banner_clear() {
let _g = setup();
note_rate_limit(Duration::from_secs(5));
start(1, Duration::from_secs(5), "rate limited");
succeeded();
assert!(
rate_limit_remaining().is_some(),
"provider-wide rate limit pause must not be cleared by an unrelated success"
);
clear_rate_limit();
}
}