Skip to main content

orch/engine/
cooldown.rs

1//! Per-agent and per-model cooldown tracking.
2//!
3//! Shared by `engine::runner::response` (recording failures) and
4//! `engine::router::config` (avoiding cooled models during pool selection).
5//!
6//! # Persistence
7//!
8//! All cooldowns are persisted to the KV store in SQLite so they survive
9//! service restarts. Call [`init_cooldown_store`] once at startup (after
10//! the `TaskStore` is open) to register the store and pre-load unexpired
11//! cooldowns.
12//!
13//! When a rate limit includes a "try again at {date}" message, the cooldown
14//! is set to that specific timestamp instead of the default duration.
15//! Relative usage windows (e.g. "5-hour usage limit") are also detected and
16//! used to set a model-specific cooldown matching the advertised window.
17
18use chrono::TimeZone;
19use regex::Regex;
20use std::collections::HashMap;
21use std::collections::HashSet;
22use std::sync::LazyLock;
23use std::sync::MutexGuard;
24use std::sync::{Arc, Mutex, OnceLock};
25use tokio::sync;
26
27use crate::engine::router::config;
28#[cfg(test)]
29use crate::engine::runner::agents::opencode;
30use crate::store::TaskStore;
31
32/// Default fallback cooldown for agents when no store is available: 30 minutes.
33///
34/// Used in contexts that need a concrete duration and cannot compute exponential
35/// backoff (e.g. computing how long to wait before retrying when a cooldown
36/// expiry timestamp is unknown). New code should use [`BACKOFF_BASE_SECS`] instead.
37pub const AGENT_COOLDOWN_SECS: i64 = 30 * 60;
38
39/// Base backoff for generic agent/model failures: 5 minutes.
40pub const BACKOFF_BASE_SECS: i64 = 5 * 60;
41
42/// Maximum backoff for generic agent/model failures: 4 hours.
43pub const BACKOFF_MAX_SECS: i64 = 4 * 60 * 60;
44
45/// Base backoff for persistent model failures (silent exit/provider errors): 4 hours.
46pub const PERSISTENT_MODEL_BACKOFF_BASE_SECS: i64 = 4 * 60 * 60;
47
48/// Maximum backoff for persistent model failures: 7 days.
49pub const PERSISTENT_MODEL_BACKOFF_MAX_SECS: i64 = 7 * 24 * 60 * 60;
50
51/// Base backoff for a one-off model failure that does not mean the model is gone: 5 minutes.
52pub const TRANSIENT_MODEL_BACKOFF_BASE_SECS: i64 = 5 * 60;
53
54/// Maximum backoff for a one-off model failure: 15 minutes.
55pub const TRANSIENT_MODEL_BACKOFF_MAX_SECS: i64 = 15 * 60;
56
57/// Base backoff for credit exhaustion (out_of_credits): 1 hour.
58pub const CREDIT_BACKOFF_BASE_SECS: i64 = 60 * 60;
59
60/// Base backoff for org-level disabling: 2 hours.
61pub const ORG_BACKOFF_BASE_SECS: i64 = 2 * 60 * 60;
62
63/// Standard cap for credit exhaustion and org-level disabling: 8 hours.
64/// Repeated failures at the cap move to the extended tiers (48h, then 96h)
65pub const CREDIT_BACKOFF_MAX_SECS: i64 = 8 * 60 * 60;
66
67/// Base cooldown for billing cycle exhaustion: 24 hours.
68///
69/// Unlike per-request rate limits, billing cycle exhaustion is a calendar event
70/// that won't self-resolve for days or weeks. The first failure starts at 24h;
71/// subsequent failures escalate via `compute_backoff()` up to
72/// `BILLING_CYCLE_MAX_SECS` (7 days). This prevents daily retry-and-fail cycles
73/// when a monthly billing quota is exhausted.
74pub const BILLING_CYCLE_COOLDOWN_SECS: i64 = 24 * 60 * 60;
75
76/// Maximum cooldown for billing cycle exhaustion: 7 days.
77///
78/// Monthly billing cycles typically reset on a fixed date. 7 days is a
79/// reasonable cap — the cooldown will expire before the next monthly reset
80/// in most cases, allowing a single probe to re-check availability.
81pub const BILLING_CYCLE_MAX_SECS: i64 = 7 * 24 * 60 * 60;
82
83/// Credit exhaustion reason detected from error message.
84#[derive(Debug, Clone, Copy, PartialEq, Eq)]
85pub enum CreditExhaustionReason {
86    /// Per-model credit exhaustion (out_of_credits).
87    OutOfCredits,
88    /// Organization-level disabling (org_level_disabled).
89    OrgLevelDisabled,
90    /// Monthly billing cycle exhaustion — escalating cooldown (24h base, 7d cap).
91    BillingCycleExhausted,
92}
93
94impl CreditExhaustionReason {
95    /// Stable string for `task_runs.outcome`, distinct from the cooldown reason
96    /// string used by `record_credit_exhaustion`. Lets callers persist the actual
97    /// failure mode instead of collapsing it into a generic "rate_limit"/"failed".
98    pub fn outcome_str(self) -> &'static str {
99        match self {
100            CreditExhaustionReason::OutOfCredits => "credit_exhausted",
101            CreditExhaustionReason::OrgLevelDisabled => "org_level_disabled",
102            CreditExhaustionReason::BillingCycleExhausted => "billing_cycle_exhausted",
103        }
104    }
105}
106
107/// Short agent cooldown applied on silence detection (120 seconds).
108///
109/// Forces the router to pick a different agent immediately on re-route,
110/// without long-term blocking. The model cooldown (exponential, starting at
111/// 5 min) keeps the dead model out while this short cooldown just breaks
112/// the same-agent loop.
113pub const SILENCE_AGENT_COOLDOWN_SECS: u64 = 120;
114
115/// Silence detections before applying an extended cooldown (rolling window).
116pub const SILENCE_COUNT_THRESHOLD: usize = 5;
117
118/// Rolling window for silence detections (24 hours).
119pub const SILENCE_COUNT_WINDOW_SECS: i64 = 24 * 60 * 60;
120
121/// Extended cooldown applied after repeated silence detections (4 hours).
122pub const SILENCE_EXTENDED_COOLDOWN_SECS: u64 = 4 * 60 * 60;
123
124/// KV key prefix for persisted cooldowns (both agent and model).
125const KV_PREFIX: &str = "cooldown:";
126
127/// KV key prefix for rolling silence counters.
128const SILENCE_COUNT_PREFIX: &str = "silence_count:";
129
130/// KV key prefix for per-agent and per-agent:model failure counts (drives exponential backoff).
131const FAILURE_COUNT_PREFIX: &str = "failure_count:";
132
133/// KV key prefix for credit-exhaustion failure counts (separate from generic failures to prevent
134/// cross-contamination of backoff escalation — credit exhaustion uses different base/max durations).
135const CREDIT_FAILURE_COUNT_PREFIX: &str = "credit_failure_count:";
136
137/// KV key prefix for persistent model failure counts (separate from [`FAILURE_COUNT_PREFIX`] so
138/// `record_agent_success()` resetting the generic counter on an intervening success doesn't erase
139/// escalation progress for structurally unreliable models — see `record_persistent_model_failure`).
140const PERSISTENT_FAILURE_COUNT_PREFIX: &str = "persistent_failure_count:";
141
142/// KV key prefix for the unix timestamp after which `rate_limits` rows count toward degraded
143const RATE_LIMITS_CLEARED_AT_PREFIX: &str = "rate_limits_cleared_at:";
144
145/// Agent name from a cooldown key (`claude:sonnet` -> `claude`)
146fn agent_from_key(key: &str) -> &str {
147    key.split_once(':').map_or(key, |(agent, _)| agent)
148}
149
150struct CooldownEntry {
151    /// Unix timestamp when the cooldown expires.
152    cooldown_until: i64,
153    reason: String,
154}
155
156#[derive(Debug)]
157pub struct SilenceCountResult {
158    pub count: usize,
159    pub extended_cooldown_applied: bool,
160}
161
162/// Global in-memory cooldown map, protected by a Mutex.
163///
164/// Uses `std::sync::Mutex` because this map is also read from sync helper
165/// functions (`is_in_cooldown`, `is_model_in_cooldown`, etc.).  All lock
166/// holders perform only in-memory HashMap operations — **no `.await` is ever
167/// called while a guard is held**.  This invariant must be preserved: if a
168/// future caller needs to await while holding this lock, switch to
169/// `tokio::sync::Mutex` and make the helpers async.
170fn cooldowns() -> &'static Mutex<HashMap<String, CooldownEntry>> {
171    static COOLDOWNS: OnceLock<Mutex<HashMap<String, CooldownEntry>>> = OnceLock::new();
172    COOLDOWNS.get_or_init(|| Mutex::new(HashMap::new()))
173}
174
175/// Global store reference used for background KV writes.
176///
177/// Uses `tokio::sync::Mutex` because this value is only ever accessed from
178/// async functions. A `tokio::sync::Mutex` yields the Tokio task instead of
179/// blocking the worker thread if the lock is contended.
180fn cooldown_store() -> &'static sync::Mutex<Option<Arc<TaskStore>>> {
181    static STORE: OnceLock<sync::Mutex<Option<Arc<TaskStore>>>> = OnceLock::new();
182    STORE.get_or_init(|| sync::Mutex::new(None))
183}
184
185/// Initialise persistent cooldowns.
186///
187/// Must be called once at engine startup, after the `TaskStore` is opened.
188/// Loads unexpired cooldowns from KV and registers the store for future writes.
189/* Parses the `{"until": N, "reason": "..."}` cooldown KV format, falling back
190to the legacy bare-integer format (reason unknown) so existing rows keep
191working across the upgrade without a migration. */
192fn parse_cooldown_value(value: &str) -> Option<(i64, String)> {
193    if let Ok(serde_json::Value::Object(map)) = serde_json::from_str(value) {
194        let until = map.get("until")?.as_i64()?;
195        let reason = map
196            .get("reason")
197            .and_then(|r| r.as_str())
198            .unwrap_or("persisted")
199            .to_string();
200        return Some((until, reason));
201    }
202    value
203        .parse::<i64>()
204        .ok()
205        .map(|until| (until, "persisted".to_string()))
206}
207
208pub async fn init_cooldown_store(store: Arc<TaskStore>) {
209    match store.kv_list_prefix(KV_PREFIX).await {
210        Ok(rows) => {
211            let now = chrono::Utc::now().timestamp();
212            let mut map = cooldown_lock();
213            let mut loaded = 0;
214            for (key, value) in rows {
215                let cooldown_key = key.trim_start_matches(KV_PREFIX);
216                if let Some((cooldown_until, reason)) = parse_cooldown_value(&value)
217                    && now < cooldown_until
218                {
219                    map.insert(
220                        cooldown_key.to_string(),
221                        CooldownEntry {
222                            cooldown_until,
223                            reason,
224                        },
225                    );
226                    loaded += 1;
227                }
228                // Expired entries are simply not loaded — they'll be overwritten
229                // on the next failure or cleaned up by future writes.
230            }
231            if loaded > 0 {
232                tracing::info!(loaded, "loaded persisted cooldowns from KV store");
233            }
234        }
235        Err(e) => {
236            tracing::warn!(err = %e, "failed to load cooldowns from KV store");
237        }
238    }
239
240    // Load GitHub 5xx circuit breaker state if it was persisted before restart.
241    if let Ok(Some(raw)) = store.kv_get("cooldown:github:5xx").await
242        && let Ok(cooldown_until) = raw.parse::<i64>()
243    {
244        let now = chrono::Utc::now().timestamp();
245        if now < cooldown_until {
246            let mut open = github_5xx_circuit_open_lock();
247            *open = Some(cooldown_until);
248            let remaining = cooldown_until - now;
249            tracing::warn!(
250                remaining_secs = remaining,
251                "GitHub 5xx circuit breaker restored from KV (still open)"
252            );
253        }
254    }
255
256    *cooldown_store().lock().await = Some(store);
257}
258
259/// Reconcile the in-memory cooldown map with the persisted KV state.
260///
261/// KV is the source of truth. This is called once per tick by
262/// `router.refresh_health` so the running service notices state mutated by a
263/// separate process (e.g. `orch cooldown clear`). Without this sync, the CLI
264/// clears KV but the engine's in-memory map keeps the stale entries until they
265/// expire on their own, producing the "all agents cooled" + "No active
266/// cooldowns" inconsistency.
267///
268/// Reconciliation rules (KV authoritative for every key it has):
269/// - KV row missing for an in-memory key: leave memory alone — likely a
270///   `set_cooldown_async` race where memory was written but the KV row isn't
271///   yet visible. The next tick will catch up.
272/// - KV row present with `cooldown_until <= now` (including the "0" tombstone
273///   written by `clear_cooldown`): drop from memory.
274/// - KV row present with `cooldown_until > now`: insert/update memory to match.
275pub async fn sync_from_kv(store: &Arc<TaskStore>) {
276    let rows = match store.kv_list_prefix(KV_PREFIX).await {
277        Ok(rows) => rows,
278        Err(e) => {
279            tracing::warn!(err = %e, "cooldown sync_from_kv: KV list failed");
280            return;
281        }
282    };
283    let now = chrono::Utc::now().timestamp();
284
285    // Some((ts, reason)) for active cooldowns, None for cleared/expired rows.
286    // Missing key means no KV row at all, left alone below.
287    let mut explicit_kv: HashMap<String, Option<(i64, String)>> =
288        HashMap::with_capacity(rows.len());
289    for (kv_key, value) in &rows {
290        let cooldown_key = kv_key.trim_start_matches(KV_PREFIX);
291        if let Some((ts, reason)) = parse_cooldown_value(value) {
292            let normalized = if ts > now { Some((ts, reason)) } else { None };
293            explicit_kv.insert(cooldown_key.to_string(), normalized);
294        }
295    }
296
297    // The 5xx circuit breaker lives in its own statics, so mirror its KV row there
298    match explicit_kv.get("github:5xx") {
299        Some(None) => {
300            let mut open = github_5xx_circuit_open_lock();
301            if open.take().is_some() {
302                tracing::info!("GitHub 5xx circuit breaker closed by KV clear");
303                github_5xx_timestamps_lock().clear();
304            }
305        }
306        Some(Some((ts, _))) => {
307            *github_5xx_circuit_open_lock() = Some(*ts);
308        }
309        None => {}
310    }
311
312    let mut map = cooldown_lock();
313    let mut dropped = 0usize;
314    let mut updated = 0usize;
315
316    // Drop in-memory entries that KV explicitly marks cleared/expired.
317    map.retain(|key, _| {
318        if matches!(explicit_kv.get(key), Some(None)) {
319            dropped += 1;
320            false
321        } else {
322            true
323        }
324    });
325
326    // Adopt KV's authoritative timestamp for active cooldowns.
327    for (key, normalized) in explicit_kv {
328        if let Some((ts, reason)) = normalized {
329            if let Some(entry) = map.get_mut(&key) {
330                if entry.cooldown_until != ts {
331                    entry.cooldown_until = ts;
332                    updated += 1;
333                }
334            } else {
335                map.insert(
336                    key,
337                    CooldownEntry {
338                        cooldown_until: ts,
339                        reason,
340                    },
341                );
342                updated += 1;
343            }
344        }
345    }
346
347    if dropped > 0 || updated > 0 {
348        tracing::info!(
349            dropped,
350            updated,
351            "cooldown sync_from_kv: reconciled in-memory map with KV"
352        );
353    }
354}
355
356/// Failure count threshold (times at max backoff) for first extended tier (24h).
357///
358/// After a model has been at the max cap this many times, apply a 6x multiplier.
359const EXTENDED_TIER_1_THRESHOLD: u64 = 2;
360
361/// Multiplier for first extended tier (6x = 24h when max=4h).
362const EXTENDED_TIER_1_MULTIPLIER: i64 = 6;
363
364/// Failure count threshold (times at max backoff) for second extended tier (48h).
365///
366/// After a model has been at the max cap this many times, apply a 12x multiplier.
367const EXTENDED_TIER_2_THRESHOLD: u64 = 5;
368
369/// Multiplier for second extended tier (12x = 48h when max=4h).
370const EXTENDED_TIER_2_MULTIPLIER: i64 = 12;
371
372/// Compute exponential backoff duration using base-3 growth.
373///
374/// Returns `min(base * 3^(count-1), max)` for count >= 1.
375/// For count == 0 or 1 (first failure), returns `base`.
376///
377/// | count | base=300s | result |
378/// |-------|-----------|--------|
379/// | 0     | 300       | 300    |
380/// | 1     | 300       | 300    |
381/// | 2     | 300       | 900    |
382/// | 3     | 300       | 2700   |
383/// | 4     | 300       | 8100   |
384/// | 5-6   | 14400     | 14400 (max, not yet extended) |
385/// | 7-9   | 86400     | 86400 (24h extended tier) |
386/// | 10+   | 172800    | 172800 (48h extended tier) |
387///
388/// The extended tier applies when the model has hit the max cap multiple times,
389/// indicating a persistently broken model that won't self-resolve quickly.
390pub fn compute_backoff(count: u64, base: i64, max: i64, allow_extended: bool) -> i64 {
391    if count <= 1 {
392        return base;
393    }
394    let exp = u32::try_from(count.saturating_sub(1)).unwrap_or(u32::MAX);
395    let factor = 3_i64.saturating_pow(exp);
396    let standard = base.saturating_mul(factor).min(max);
397
398    // Extended tier: after N failures at max backoff, apply longer window
399    // to reduce noise from persistently broken models.
400    //
401    // First find the count at which exponential growth first hits max:
402    // solve: base * 3^(first_max-1) >= max
403    // => 3^(first_max-1) >= max / base
404    // => first_max-1 >= log_3(max / base)
405    // => first_max >= 1 + log_3(max / base)
406    let first_max = if base <= 0 {
407        0
408    } else {
409        // Use integer approximation: count where base * 3^(count-1) would first exceed max
410        let mut accumulated = base;
411        let mut first_max_count = 1u64;
412        while accumulated < max && first_max_count < count {
413            accumulated = accumulated.saturating_mul(3);
414            first_max_count += 1;
415        }
416        first_max_count
417    };
418
419    // Times we've been at the max cap (excluding the first time we hit it)
420    let times_at_max = count.saturating_sub(first_max);
421
422    if !allow_extended {
423        return standard;
424    }
425
426    if standard == max && times_at_max >= EXTENDED_TIER_2_THRESHOLD {
427        max.saturating_mul(EXTENDED_TIER_2_MULTIPLIER)
428    } else if standard == max && times_at_max >= EXTENDED_TIER_1_THRESHOLD {
429        max.saturating_mul(EXTENDED_TIER_1_MULTIPLIER)
430    } else {
431        standard
432    }
433}
434
435/// Read the failure count for a key from KV, increment it, write it back, and return the new count.
436///
437/// Always increments — the backoff formula's `max` parameter prevents the
438/// cooldown duration from growing beyond the cap, so unbounded counts are safe.
439///
440/// Returns 1 when no store is configured (unit-test contexts without a store).
441/// Returns `u64::MAX` on store errors (e.g. lock contention) so backoff applies the cap duration
442/// instead of resetting to the base, preventing rapid re-dispatch during store outages.
443async fn read_and_increment_failure_count(store_opt: Option<&Arc<TaskStore>>, key: &str) -> u64 {
444    read_and_increment_failure_count_with_prefix(store_opt, FAILURE_COUNT_PREFIX, key).await
445}
446
447/// Like [`read_and_increment_failure_count`] but with an explicit KV prefix.
448///
449/// Credit exhaustion uses [`CREDIT_FAILURE_COUNT_PREFIX`] so its backoff escalation
450/// is independent of generic agent failures (which use [`FAILURE_COUNT_PREFIX`]).
451async fn read_and_increment_failure_count_with_prefix(
452    store_opt: Option<&Arc<TaskStore>>,
453    prefix: &str,
454    key: &str,
455) -> u64 {
456    let kv_key = format!("{prefix}{key}");
457    if let Some(store) = store_opt {
458        match store.kv_increment(&kv_key).await {
459            Ok(n) => n,
460            Err(e) => {
461                tracing::warn!(key = %kv_key, err = %e, "failed to increment failure count — treating as high count for safe backoff");
462                u64::MAX
463            }
464        }
465    } else {
466        1
467    }
468}
469
470/// Backoff schedule for one kind of per-model failure.
471struct ModelBackoff {
472    prefix: &'static str,
473    base: i64,
474    max: i64,
475    allow_extended: bool,
476    reason: &'static str,
477}
478
479/// Bump the failure counter for `agent:model` and cool the model for the backoff it maps to.
480async fn record_model_backoff(agent_name: &str, model: &str, schedule: &ModelBackoff) {
481    let key = format!("{agent_name}:{model}");
482    let store_opt = cooldown_store().lock().await.clone();
483    let count =
484        read_and_increment_failure_count_with_prefix(store_opt.as_ref(), schedule.prefix, &key)
485            .await;
486    let cooldown_secs =
487        compute_backoff(count, schedule.base, schedule.max, schedule.allow_extended);
488    let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
489    set_cooldown_async(&key, cooldown_until, schedule.reason).await;
490}
491
492/// Reset failure counts for an agent and agent:model combo after a successful run.
493///
494/// Call this from the runner's success path so that the next failure starts
495/// backoff from the base duration again, not from wherever it left off.
496/// Resets both generic and credit-exhaustion failure counters.
497pub async fn record_agent_success(agent_name: &str, model: &str) {
498    let store_opt = cooldown_store().lock().await.clone();
499    if let Some(store) = store_opt {
500        let agent_key = format!("{FAILURE_COUNT_PREFIX}{agent_name}");
501        let credit_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{agent_name}");
502        if let Err(e) = store.kv_set(&agent_key, "0").await {
503            tracing::warn!(key = agent_key, err = %e, "failed to reset failure count");
504        }
505        if !model.is_empty() {
506            let model_key = format!("{FAILURE_COUNT_PREFIX}{agent_name}:{model}");
507            if let Err(e) = store.kv_set(&model_key, "0").await {
508                tracing::warn!(key = model_key, err = %e, "failed to reset failure count");
509            }
510        }
511        if let Err(e) = store.kv_set(&credit_key, "0").await {
512            tracing::warn!(key = credit_key, err = %e, "failed to reset failure count");
513        }
514    }
515}
516
517/// Record that an agent has failed and should be temporarily avoided.
518///
519/// Applies exponential backoff based on the agent's failure count in KV.
520/// If `error_message` contains "try again at {date}", that vendor date is used
521/// instead of the computed backoff (vendor dates are always authoritative).
522pub async fn record_agent_failure_with_message(agent_name: &str, error_message: &str) {
523    // Vendor-specified retry date takes priority over backoff — but only
524    // if it is in the present or future. If the parsed timestamp is in the
525    // past (stale vendor message), fall through to the exponential backoff
526    // path instead of returning early and bypassing backoff entirely.
527    if let Some(cooldown_until) = parse_retry_at(error_message) {
528        let now = chrono::Utc::now().timestamp();
529        if cooldown_until > now {
530            set_cooldown_async(agent_name, cooldown_until, "agent_error").await;
531            return;
532        }
533        tracing::debug!(
534            agent = agent_name,
535            cooldown_until,
536            "vendor retry_at is in the past — using backoff instead"
537        );
538    }
539
540    let store_opt = cooldown_store().lock().await.clone();
541    let count = read_and_increment_failure_count(store_opt.as_ref(), agent_name).await;
542    let base = config::get_agent_backoff_base(agent_name);
543    let cooldown_until =
544        chrono::Utc::now().timestamp() + compute_backoff(count, base, BACKOFF_MAX_SECS, true);
545    set_cooldown_async(agent_name, cooldown_until, "agent_error").await;
546}
547
548/// Detect credit exhaustion from an error message and return the stable
549/// `task_runs.outcome` string for it (see `CreditExhaustionReason::outcome_str`).
550///
551/// Callers that classify `AgentError::RateLimit`/`AgentError::Auth` into an
552/// outcome string should consult this first — generic rate-limit patterns like
553/// "usage limit" also match billing-cycle exhaustion messages (e.g. Kimi's
554/// "You've reached your usage limit for this billing cycle"), which would
555/// otherwise be stored as a plain "rate_limit" outcome.
556pub fn detect_credit_exhaustion_outcome(error_message: &str) -> Option<&'static str> {
557    detect_credit_exhaustion(error_message).map(CreditExhaustionReason::outcome_str)
558}
559
560/// Detect credit exhaustion reasons from error message.
561///
562/// Returns `Some(CreditExhaustionReason)` if the error indicates
563/// out_of_credits or org_level_disabled, which require longer agent-wide cooldowns.
564pub fn detect_credit_exhaustion(error_message: &str) -> Option<CreditExhaustionReason> {
565    let lower = error_message.to_lowercase();
566
567    let overage_patterns = [
568        "overagedisabledreason",
569        "overage_disabled_reason",
570        "overage disabled reason",
571    ];
572
573    if overage_patterns.iter().any(|p| lower.contains(p)) {
574        if lower.contains("out_of_credits") || lower.contains("outofcredits") {
575            return Some(CreditExhaustionReason::OutOfCredits);
576        }
577        if lower.contains("org_level_disabled") || lower.contains("orgdisabled") {
578            return Some(CreditExhaustionReason::OrgLevelDisabled);
579        }
580    }
581
582    let billing_patterns = [
583        "credit balance too low",
584        "credit balance insufficient",
585        "insufficient credit balance",
586        "billing quota exceeded",
587        "billing limit exceeded",
588        "organization has been disabled",
589        "org disabled",
590    ];
591
592    if billing_patterns.iter().any(|p| lower.contains(p)) {
593        return Some(CreditExhaustionReason::OrgLevelDisabled);
594    }
595
596    // Billing cycle exhaustion: monthly quota reset — check before generic quota
597    // patterns because "monthly quota exceeded" would otherwise match "quota exceeded".
598    let billing_cycle_patterns = [
599        "billing cycle",
600        "quota refreshed next cycle",
601        "monthly quota",
602        "quota will be refreshed",
603        "next billing cycle",
604        "refreshed in the next cycle",
605        "weekly/monthly limit exhausted",
606        "limit will reset at",
607        "upgrade to pro", // codex/chatgpt plan usage cap: requires billing upgrade
608        "upgrade your token plan", // minimax: Token Plan usage limit reached
609        "token plan usage limit", // minimax: Token Plan usage limit reached
610        "purchase credits for more", // minimax: ...purchase Credits for more usage
611    ];
612
613    if billing_cycle_patterns.iter().any(|p| lower.contains(p)) {
614        return Some(CreditExhaustionReason::BillingCycleExhausted);
615    }
616
617    if lower.contains("out of credits")
618        || lower.contains("outofcredits")
619        || lower.contains("insufficient funds")
620        || lower.contains("no credits remaining")
621        || lower.contains("insufficient_quota")
622        || lower.contains("quota exceeded")
623        || lower.contains("insufficient balance")
624        || lower.contains("please recharge")
625        || lower.contains("no resource package")
626        || lower.contains("balance exhausted")
627        || lower.contains("402 payment required")
628    {
629        return Some(CreditExhaustionReason::OutOfCredits);
630    }
631
632    None
633}
634
635/// Record an agent-level cooldown for credit exhaustion.
636///
637/// Applies exponential backoff based on the agent's failure count.
638/// All variants use escalating backoff:
639/// - `OutOfCredits`: 1h, 3h, then 8h (credits can be refilled any time), 48h from the 5th failure, 96h from the 8th
640/// - `OrgLevelDisabled`: 2h, 6h, then 8h, 48h from the 5th failure, 96h from the 8th
641/// - `BillingCycleExhausted`: starts at 24h, caps at 7 days (monthly event)
642///
643/// Billing cycle exhaustion was previously a flat 24h, but this caused daily
644/// retry-and-fail cycles when a monthly quota was exhausted (the 24h cooldown
645/// would expire, kimi would be retried, fail immediately, and get another 24h).
646/// Now each recurrence escalates: 24h → 72h → 7d (capped).
647pub async fn record_credit_exhaustion(agent_name: &str, reason: CreditExhaustionReason) {
648    let reason_str = match reason {
649        CreditExhaustionReason::OutOfCredits => "credit_exhaustion_out_of_credits",
650        CreditExhaustionReason::OrgLevelDisabled => "credit_exhaustion_org_level_disabled",
651        CreditExhaustionReason::BillingCycleExhausted => "billing_cycle_exhausted",
652    };
653
654    // A failure from a run that started before the cooldown is the same event
655    if is_in_cooldown_with_reason(agent_name, reason_str) {
656        tracing::debug!(
657            agent = agent_name,
658            reason = reason_str,
659            "credit exhaustion already cooled, not escalating"
660        );
661        return;
662    }
663
664    let store_opt = cooldown_store().lock().await.clone();
665    // Use a distinct failure counter (CREDIT_FAILURE_COUNT_PREFIX) so credit-
666    // exhaustion backoff escalation is independent of generic agent failures.
667    // Previously both paths shared FAILURE_COUNT_PREFIX, causing generic failures
668    // to inflate credit-exhaustion backoff (and vice versa).
669    let count = read_and_increment_failure_count_with_prefix(
670        store_opt.as_ref(),
671        CREDIT_FAILURE_COUNT_PREFIX,
672        agent_name,
673    )
674    .await;
675
676    let (base, max) = match reason {
677        CreditExhaustionReason::OutOfCredits => (CREDIT_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS),
678        CreditExhaustionReason::OrgLevelDisabled => {
679            (ORG_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS)
680        }
681        CreditExhaustionReason::BillingCycleExhausted => {
682            (BILLING_CYCLE_COOLDOWN_SECS, BILLING_CYCLE_MAX_SECS)
683        }
684    };
685    let cooldown_secs = compute_backoff(
686        count,
687        base,
688        max,
689        !matches!(reason, CreditExhaustionReason::BillingCycleExhausted),
690    );
691    let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
692    set_cooldown_async(agent_name, cooldown_until, reason_str).await;
693    tracing::warn!(
694        agent = agent_name,
695        reason = reason_str,
696        attempt = count,
697        cooldown_secs,
698        "credit exhaustion detected: applying exponential agent-wide cooldown"
699    );
700}
701
702// Returns false when the message is no credit exhaustion. Model-scoped billing cycle cools the model only
703pub async fn record_credit_exhaustion_from_message(
704    agent_name: &str,
705    model: Option<&str>,
706    message: &str,
707) -> bool {
708    let Some(reason) = detect_credit_exhaustion(message) else {
709        return false;
710    };
711    match (reason, model) {
712        (CreditExhaustionReason::BillingCycleExhausted, Some(model)) => {
713            record_persistent_model_failure(agent_name, model).await;
714        }
715        _ => record_credit_exhaustion(agent_name, reason).await,
716    }
717    true
718}
719
720/// Record that a specific agent+model combo has failed.
721///
722/// Applies exponential backoff based on the model's failure count in KV.
723pub async fn record_model_failure(agent_name: &str, model: &str) {
724    let base = config::get_agent_backoff_base(agent_name);
725    let schedule = ModelBackoff {
726        prefix: FAILURE_COUNT_PREFIX,
727        base,
728        max: BACKOFF_MAX_SECS,
729        allow_extended: true,
730        reason: "model_error",
731    };
732    record_model_backoff(agent_name, model, &schedule).await;
733}
734
735/// Record a rate-limit event for an agent+model.
736///
737/// If the error message advertises a relative usage window (e.g.
738/// "5-hour usage limit"), the cooldown is applied to the specific model for
739/// that window length. The quota is model-scoped, so the whole agent is not
740/// cooled. If no window is detected, fall back to the generic agent+model
741/// exponential backoff path.
742pub async fn record_rate_limit(agent_name: &str, model: Option<&str>, error_message: &str) {
743    if let Some(window_secs) = parse_relative_usage_window(error_message)
744        && let Some(m) = model
745    {
746        let cooldown_until = chrono::Utc::now()
747            .timestamp()
748            .saturating_add(i64::try_from(window_secs).unwrap_or(i64::MAX));
749        set_model_cooldown(agent_name, m, window_secs, "rate_limit_usage_window").await;
750        tracing::info!(
751            agent = agent_name,
752            model = m,
753            window_secs,
754            cooldown_until,
755            "relative usage window detected: applying model-specific cooldown"
756        );
757        return;
758    }
759
760    record_agent_failure_with_message(agent_name, error_message).await;
761    if let Some(m) = model {
762        record_model_failure(agent_name, m).await;
763    }
764}
765
766/// Record a persistent model-specific failure and apply longer exponential backoff.
767///
768/// Used for failure modes that are commonly broken for long periods (for example
769/// silent exit-0 or opaque provider errors). Starts at 4h and escalates with
770/// the generic backoff formula up to 7 days.
771///
772/// Uses [`PERSISTENT_FAILURE_COUNT_PREFIX`], a counter independent of the generic
773/// `failure_count:` key that `record_agent_success()` resets on every successful
774/// run. Otherwise a single intervening success between persistent failures would
775/// reset escalation back to the 4h base, letting structurally unreliable models
776/// (intermittent parse errors, truncated runs) stay in rotation indefinitely.
777pub async fn record_persistent_model_failure(agent_name: &str, model: &str) {
778    let schedule = ModelBackoff {
779        prefix: PERSISTENT_FAILURE_COUNT_PREFIX,
780        base: PERSISTENT_MODEL_BACKOFF_BASE_SECS,
781        max: PERSISTENT_MODEL_BACKOFF_MAX_SECS,
782        allow_extended: false,
783        reason: "persistent_model_error",
784    };
785    record_model_backoff(agent_name, model, &schedule).await;
786}
787
788/// Record a `ModelUnavailable` failure for an agent+model.
789///
790/// "Not found", "not supported" and region blocks never lift on their own, so they get the
791/// persistent backoff (4h base, 7d max). Any other message gets [`record_model_failure`]
792pub async fn record_model_unavailable(agent_name: &str, model: &str, message: &str) {
793    if is_model_permanently_gone(message) {
794        record_persistent_model_failure(agent_name, model).await;
795    } else {
796        record_model_failure(agent_name, model).await;
797    }
798}
799
800fn is_model_permanently_gone(message: &str) -> bool {
801    let lower = message.to_lowercase();
802    lower.contains("not found")
803        || lower.contains("not supported")
804        || lower.contains("not available in your country")
805        || lower.contains("not available in your region")
806}
807
808/// Record a run that failed without showing the model is gone: empty or cut off output, an
809/// unparseable review, an opaque provider error.
810///
811/// A stopped session or one odd answer causes these, so the backoff is 5 then 15 minutes. Billing
812/// exhaustion and "model not found" keep [`record_persistent_model_failure`]
813pub async fn record_transient_model_failure(agent_name: &str, model: &str) {
814    let schedule = ModelBackoff {
815        prefix: FAILURE_COUNT_PREFIX,
816        base: TRANSIENT_MODEL_BACKOFF_BASE_SECS,
817        max: TRANSIENT_MODEL_BACKOFF_MAX_SECS,
818        allow_extended: false,
819        reason: "transient_model_error",
820    };
821    record_model_backoff(agent_name, model, &schedule).await;
822}
823
824/// Set a model cooldown with a custom duration (in seconds) and reason.
825///
826/// Generic model-cooldown setter shared by silence detection and the
827/// rate-limit relative-usage-window path. The reason is persisted with
828/// the cooldown (KV + in-memory entry) so operators can tell the two
829/// apart in `orch cooldown list`.
830pub async fn set_model_cooldown(agent_name: &str, model: &str, duration_secs: u64, reason: &str) {
831    let key = format!("{agent_name}:{model}");
832    let cooldown_until = chrono::Utc::now()
833        .timestamp()
834        .saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX));
835    set_cooldown_async(&key, cooldown_until, reason).await;
836}
837
838/// Set a model cooldown that ends at a vendor-provided Unix timestamp.
839pub async fn set_model_cooldown_until(agent_name: &str, model: &str, until: i64) {
840    let key = format!("{agent_name}:{model}");
841    set_cooldown_async(&key, until, "vendor_reset_at").await;
842}
843
844/// Set a short agent-level cooldown (in seconds).
845///
846/// Used by silence detection to temporarily block the whole agent so the
847/// router picks a different one on re-route. Unlike `record_agent_failure`
848/// (exponential backoff), this uses a short duration (typically 120s) —
849/// just enough to force one re-route cycle to a different agent.
850pub async fn set_agent_cooldown(agent_name: &str, duration_secs: u64) {
851    let cooldown_until = chrono::Utc::now()
852        .timestamp()
853        .saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX));
854    set_cooldown_async(agent_name, cooldown_until, "silence_agent_cooldown").await;
855}
856
857/// KV prefix for the latest quota window of each type an agent reported: `rate_window:{agent}:{type}`
858pub const RATE_WINDOW_KV_PREFIX: &str = "rate_window:";
859
860/// One quota window from a vendor `rate_limit_event`, with the vendor window type kept as sent
861#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
862pub struct RateLimitWindow {
863    pub status: String,
864    pub window_type: String,
865    pub utilization: Option<f64>,
866    pub resets_at: Option<i64>,
867}
868
869/// Store the latest window for an agent. A rejected window cools the agent until its reset time
870pub async fn record_rate_limit_window(agent_name: &str, window: &RateLimitWindow) {
871    let store_opt = cooldown_store().lock().await.clone();
872    if let Some(store) = store_opt
873        && let Ok(value) = serde_json::to_string(window)
874        && let Err(e) = store
875            .kv_set(
876                &format!("{RATE_WINDOW_KV_PREFIX}{agent_name}:{}", window.window_type),
877                &value,
878            )
879            .await
880    {
881        tracing::warn!(agent = agent_name, err = %e, "failed to persist rate limit window");
882    }
883
884    let now = chrono::Utc::now().timestamp();
885    if window.status == "rejected"
886        && let Some(reset) = window.resets_at
887        && reset > now
888    {
889        let until = reset.min(now + PERSISTENT_MODEL_BACKOFF_MAX_SECS);
890        set_cooldown_async(agent_name, until, "rate_limit_window").await;
891        tracing::info!(
892            agent = agent_name,
893            window = %window.window_type,
894            cooldown_until = until,
895            "rate limit window rejected: cooling agent until vendor reset"
896        );
897    }
898}
899
900/// Latest stored quota window per agent, sorted by agent name
901pub async fn list_rate_limit_windows(store: &Arc<TaskStore>) -> Vec<(String, RateLimitWindow)> {
902    let rows = store
903        .kv_list_prefix(RATE_WINDOW_KV_PREFIX)
904        .await
905        .unwrap_or_default();
906    let mut windows: Vec<(String, RateLimitWindow)> = rows
907        .into_iter()
908        .filter_map(|(key, value)| {
909            let (agent, _) = key.strip_prefix(RATE_WINDOW_KV_PREFIX)?.rsplit_once(':')?;
910            let window = serde_json::from_str(&value).ok()?;
911            Some((agent.to_string(), window))
912        })
913        .collect();
914    windows.sort_by(|a, b| (&a.0, &a.1.window_type).cmp(&(&b.0, &b.1.window_type)));
915    windows
916}
917
918/// Record a silence detection for an agent+model and apply extended cooldowns
919/// when repeated silences exceed the threshold within the rolling window.
920pub async fn record_silence_detection(agent_name: &str, model: &str) -> Option<SilenceCountResult> {
921    let store_opt = cooldown_store().lock().await.clone();
922    let Some(store) = store_opt else {
923        tracing::debug!(
924            agent = agent_name,
925            model,
926            "skipping silence count record (no KV store)"
927        );
928        return None;
929    };
930
931    let key = format!("{SILENCE_COUNT_PREFIX}{agent_name}:{model}");
932    let now = chrono::Utc::now().timestamp();
933    let window_start = now - SILENCE_COUNT_WINDOW_SECS;
934
935    let mut timestamps = match store.kv_get(&key).await {
936        Ok(Some(raw)) => match serde_json::from_str::<Vec<i64>>(&raw) {
937            Ok(ts) => ts,
938            Err(e) => {
939                tracing::warn!(
940                    kv_key = %key,
941                    err = %e,
942                    "failed to parse silence timestamps from KV — resetting window"
943                );
944                Vec::new()
945            }
946        },
947        Ok(None) => Vec::new(),
948        Err(err) => {
949            tracing::warn!(
950                kv_key = %key,
951                err = %err,
952                "failed to load silence count from KV"
953            );
954            Vec::new()
955        }
956    };
957
958    timestamps.retain(|ts| *ts >= window_start);
959    timestamps.push(now);
960
961    let count = timestamps.len();
962    let mut extended_cooldown_applied = false;
963    if count >= SILENCE_COUNT_THRESHOLD {
964        set_model_cooldown(
965            agent_name,
966            model,
967            SILENCE_EXTENDED_COOLDOWN_SECS,
968            "silence_detected",
969        )
970        .await;
971        extended_cooldown_applied = true;
972        timestamps.clear();
973    }
974
975    let value = serde_json::to_string(&timestamps).unwrap_or_else(|_| "[]".to_string());
976    if let Err(err) = store.kv_set(&key, &value).await {
977        tracing::warn!(kv_key = key, err = %err, "failed to persist silence count");
978    }
979
980    Some(SilenceCountResult {
981        count,
982        extended_cooldown_applied,
983    })
984}
985
986/// Check if a specific agent+model combo is in cooldown.
987pub fn is_model_in_cooldown(agent_name: &str, model: &str) -> bool {
988    let key = format!("{agent_name}:{model}");
989    is_in_cooldown(&key)
990}
991
992/// Check if an agent is currently in cooldown period.
993pub fn is_agent_in_cooldown(agent_name: &str) -> bool {
994    is_in_cooldown(agent_name)
995}
996
997/// Return the cooldown expiry timestamp for a key (agent or agent:model), if active.
998pub fn cooldown_until(key: &str) -> Option<i64> {
999    let map = cooldown_lock();
1000    map.get(key).and_then(|entry| {
1001        let now = chrono::Utc::now().timestamp();
1002        if now < entry.cooldown_until {
1003            Some(entry.cooldown_until)
1004        } else {
1005            None
1006        }
1007    })
1008}
1009
1010/// Return the reason string recorded when a cooldown was set, if the cooldown is still active.
1011pub fn cooldown_reason(key: &str) -> Option<String> {
1012    let map = cooldown_lock();
1013    map.get(key).and_then(|entry| {
1014        let now = chrono::Utc::now().timestamp();
1015        if now < entry.cooldown_until {
1016            Some(entry.reason.clone())
1017        } else {
1018            None
1019        }
1020    })
1021}
1022
1023/// Active cooldowns as `(key, cooldown_until_unix, reason)` from the store, so a process other
1024/// than the daemon sees the same list as `orch cooldown list`
1025pub async fn list_persisted_cooldowns(store: &TaskStore) -> Vec<(String, i64, String)> {
1026    let now = chrono::Utc::now().timestamp();
1027    let mut out: Vec<(String, i64, String)> = store
1028        .kv_list_prefix(KV_PREFIX)
1029        .await
1030        .unwrap_or_default()
1031        .into_iter()
1032        .filter_map(|(key, value)| {
1033            let (until, reason) = parse_cooldown_value(&value)?;
1034            (now < until).then(|| (key.trim_start_matches(KV_PREFIX).to_string(), until, reason))
1035        })
1036        .collect();
1037    out.sort_by(|a, b| a.0.cmp(&b.0));
1038    out
1039}
1040
1041/// Return all currently active cooldowns as `(key, cooldown_until_unix, reason)`.
1042///
1043/// Reads from the in-memory map only — always fast, no async needed.
1044/// Expired entries are filtered out. Used by `orch cooldown list`.
1045pub fn list_all_cooldowns() -> Vec<(String, i64, String)> {
1046    let now = chrono::Utc::now().timestamp();
1047    let map = cooldown_lock();
1048    let mut result: Vec<(String, i64, String)> = map
1049        .iter()
1050        .filter(|(_, entry)| now < entry.cooldown_until)
1051        .map(|(key, entry)| (key.clone(), entry.cooldown_until, entry.reason.clone()))
1052        .collect();
1053    result.sort_by(|a, b| a.0.cmp(&b.0));
1054    result
1055}
1056
1057/// Write `rate_limits_cleared_at:{agent}` = now so the next health check
1058/// ignores older `rate_limits` rows. Rows stay in the table for metrics
1059pub async fn mark_rate_limits_cleared(store: &Arc<TaskStore>, agents: &[String]) {
1060    let now = chrono::Utc::now().timestamp().to_string();
1061    let mut seen = HashSet::new();
1062    for key in agents {
1063        let agent = agent_from_key(key);
1064        if agent.is_empty() || !seen.insert(agent.to_string()) {
1065            continue;
1066        }
1067        let kv_key = format!("{RATE_LIMITS_CLEARED_AT_PREFIX}{agent}");
1068        if let Err(e) = store.kv_set(&kv_key, &now).await {
1069            tracing::warn!(agent, err = %e, "failed to mark rate limits cleared");
1070        }
1071    }
1072}
1073
1074/// Clear a specific cooldown by key, or all cooldowns when `key == "*"`.
1075///
1076/// Removes from the in-memory map, writes a past timestamp to KV, and resets
1077/// the corresponding `failure_count:` entries so the next failure starts
1078/// backoff from the base duration. Also writes `rate_limits_cleared_at:{agent}`
1079/// so a degraded agent held only by rate rows recovers on the next health check
1080pub async fn clear_cooldown(key: &str, store: &Arc<TaskStore>) {
1081    let mut agents_to_mark: Vec<String> = Vec::new();
1082    if key == "*" {
1083        let keys: Vec<String> = {
1084            let mut map = cooldown_lock();
1085            let keys: Vec<String> = map.keys().cloned().collect();
1086            map.clear();
1087            keys
1088        };
1089        for k in &keys {
1090            let kv_key = format!("{KV_PREFIX}{k}");
1091            if let Err(e) = store.kv_set(&kv_key, "0").await {
1092                tracing::warn!(key = kv_key, err = %e, "failed to clear cooldown");
1093            }
1094            // Reset failure count so backoff restarts from base.
1095            let fc_key = format!("{FAILURE_COUNT_PREFIX}{k}");
1096            if let Err(e) = store.kv_set(&fc_key, "0").await {
1097                tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1098            }
1099            // Also reset credit-specific failure count.
1100            let credit_fc_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{k}");
1101            if let Err(e) = store.kv_set(&credit_fc_key, "0").await {
1102                tracing::warn!(key = credit_fc_key, err = %e, "failed to reset credit failure count");
1103            }
1104            // Also reset the persistent-model-failure counter.
1105            let persistent_fc_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{k}");
1106            if let Err(e) = store.kv_set(&persistent_fc_key, "0").await {
1107                tracing::warn!(key = persistent_fc_key, err = %e, "failed to reset persistent failure count");
1108            }
1109        }
1110        // Also reset any persisted failure_count keys that are not in the
1111        // in-memory map (e.g. survived a restart or were set without a
1112        // corresponding in-memory cooldown entry).
1113        if let Ok(fc_entries) = store.kv_list_prefix(FAILURE_COUNT_PREFIX).await {
1114            for (fc_key, _) in fc_entries {
1115                if let Err(e) = store.kv_set(&fc_key, "0").await {
1116                    tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1117                }
1118            }
1119        }
1120        if let Ok(cfc_entries) = store.kv_list_prefix(CREDIT_FAILURE_COUNT_PREFIX).await {
1121            for (cfc_key, _) in cfc_entries {
1122                if let Err(e) = store.kv_set(&cfc_key, "0").await {
1123                    tracing::warn!(key = cfc_key, err = %e, "failed to reset credit failure count");
1124                }
1125            }
1126        }
1127        if let Ok(pfc_entries) = store.kv_list_prefix(PERSISTENT_FAILURE_COUNT_PREFIX).await {
1128            for (pfc_key, _) in pfc_entries {
1129                if let Err(e) = store.kv_set(&pfc_key, "0").await {
1130                    tracing::warn!(key = pfc_key, err = %e, "failed to reset persistent failure count");
1131                }
1132            }
1133        }
1134        tracing::info!(
1135            count = keys.len(),
1136            "cleared all cooldowns and failure counts"
1137        );
1138        agents_to_mark.extend(keys.iter().map(|k| agent_from_key(k).to_string()));
1139        match store.rate_limit_agents().await {
1140            Ok(rl_agents) => agents_to_mark.extend(rl_agents),
1141            Err(e) => tracing::warn!(err = %e, "failed to list rate_limits agents while clearing"),
1142        }
1143        match store.kv_list_prefix(RATE_LIMITS_CLEARED_AT_PREFIX).await {
1144            Ok(marks) => {
1145                for (k, _) in marks {
1146                    let agent = k.trim_start_matches(RATE_LIMITS_CLEARED_AT_PREFIX);
1147                    if !agent.is_empty() {
1148                        agents_to_mark.push(agent.to_string());
1149                    }
1150                }
1151            }
1152            Err(e) => tracing::warn!(err = %e, "failed to list rate_limits_cleared_at keys"),
1153        }
1154    } else {
1155        {
1156            let mut map = cooldown_lock();
1157            map.remove(key);
1158        }
1159        let kv_key = format!("{KV_PREFIX}{key}");
1160        if let Err(e) = store.kv_set(&kv_key, "0").await {
1161            tracing::warn!(key = kv_key, err = %e, "failed to clear cooldown");
1162        }
1163        // Reset failure count so backoff restarts from base.
1164        let fc_key = format!("{FAILURE_COUNT_PREFIX}{key}");
1165        if let Err(e) = store.kv_set(&fc_key, "0").await {
1166            tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1167        }
1168        // Also reset credit-specific failure count.
1169        let credit_fc_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{key}");
1170        if let Err(e) = store.kv_set(&credit_fc_key, "0").await {
1171            tracing::warn!(key = credit_fc_key, err = %e, "failed to reset credit failure count");
1172        }
1173        // Also reset the persistent-model-failure counter.
1174        let persistent_fc_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{key}");
1175        if let Err(e) = store.kv_set(&persistent_fc_key, "0").await {
1176            tracing::warn!(key = persistent_fc_key, err = %e, "failed to reset persistent failure count");
1177        }
1178        tracing::info!(key, "cleared cooldown and failure count");
1179        agents_to_mark.push(agent_from_key(key).to_string());
1180    }
1181    mark_rate_limits_cleared(store, &agents_to_mark).await;
1182}
1183
1184/// Set a cooldown with a specific expiry timestamp. Updates in-memory state only.
1185///
1186/// Returns `true` if the cooldown was applied, `false` if an existing longer
1187/// cooldown prevented the update (never-shorten rule).
1188///
1189/// **Callers are responsible for persisting to KV.** Use [`set_cooldown_async`]
1190/// for critical cooldowns that must survive a crash, or this function combined
1191/// with a fire-and-forget `tokio::spawn` for non-critical short cooldowns.
1192fn set_cooldown_in_memory(key: &str, cooldown_until: i64, reason: &str) -> bool {
1193    let mut map = cooldown_lock();
1194    if let Some(existing) = map.get(key) {
1195        // Never shorten an existing cooldown. This prevents a short
1196        // retry window (e.g., generic rate limit) from overriding a longer
1197        // billing-cycle cooldown.
1198        if existing.cooldown_until >= cooldown_until {
1199            return false;
1200        }
1201    }
1202    map.insert(
1203        key.to_string(),
1204        CooldownEntry {
1205            cooldown_until,
1206            reason: reason.to_string(),
1207        },
1208    );
1209    true
1210}
1211
1212/// Set a cooldown with a specific expiry timestamp and persist to KV inline.
1213///
1214/// This is the preferred function for critical cooldowns (credit exhaustion,
1215/// billing cycle, agent/model failures) that must survive a service crash.
1216/// The KV write is awaited before returning, so the caller can be confident
1217/// the cooldown is durable once this function returns `true`.
1218///
1219/// Returns `true` if the cooldown was applied, `false` if an existing longer
1220/// cooldown prevented the update (never-shorten rule).
1221async fn set_cooldown_async(key: &str, cooldown_until: i64, reason: &str) -> bool {
1222    if !set_cooldown_in_memory(key, cooldown_until, reason) {
1223        return false;
1224    }
1225    let store_opt = cooldown_store().lock().await.clone();
1226    if let Some(store) = store_opt {
1227        let kv_key = format!("{KV_PREFIX}{key}");
1228        let value = serde_json::json!({"until": cooldown_until, "reason": reason}).to_string();
1229        if let Err(e) = store.kv_set(&kv_key, &value).await {
1230            tracing::warn!(
1231                kv_key,
1232                err = %e,
1233                "failed to persist cooldown to KV store — rolling back in-memory entry"
1234            );
1235            let mut map = cooldown_lock();
1236            // Only roll back if nothing has superseded our write.
1237            // A racing thread may have set a longer cooldown in the meantime;
1238            // removing unconditionally would erase that valid entry.
1239            if map.get(key).map(|e| e.cooldown_until) == Some(cooldown_until) {
1240                map.remove(key);
1241            }
1242            return false;
1243        }
1244    }
1245    true
1246}
1247
1248/// Helper to acquire the cooldown map lock, logging and recovering on poison.
1249fn cooldown_lock() -> MutexGuard<'static, HashMap<String, CooldownEntry>> {
1250    cooldowns().lock().unwrap_or_else(|e| {
1251        tracing::error!("cooldown map lock poisoned — recovering with partial state; backoff and degraded-agent decisions may be incorrect");
1252        e.into_inner()
1253    })
1254}
1255
1256/// Helper to acquire the degraded agents lock, logging and recovering on poison.
1257fn degraded_agents_lock() -> MutexGuard<'static, HashMap<String, String>> {
1258    degraded_agents().lock().unwrap_or_else(|e| {
1259        tracing::error!("degraded agents lock poisoned — recovering with partial state; health check decisions may be incorrect");
1260        e.into_inner()
1261    })
1262}
1263
1264/// Helper to acquire the GitHub 5xx timestamps lock, logging and recovering on poison.
1265fn github_5xx_timestamps_lock() -> MutexGuard<'static, Vec<i64>> {
1266    github_5xx_timestamps().lock().unwrap_or_else(|e| {
1267        tracing::error!("github 5xx timestamps lock poisoned — recovering with partial state; circuit breaker may operate incorrectly");
1268        e.into_inner()
1269    })
1270}
1271
1272/// Helper to acquire the GitHub 5xx circuit open lock, logging and recovering on poison.
1273fn github_5xx_circuit_open_lock() -> MutexGuard<'static, Option<i64>> {
1274    github_5xx_circuit_open().lock().unwrap_or_else(|e| {
1275        tracing::error!("github 5xx circuit open lock poisoned — recovering with partial state; circuit breaker may operate incorrectly");
1276        e.into_inner()
1277    })
1278}
1279
1280/// Check if a key is in cooldown with the given reason.
1281fn is_in_cooldown_with_reason(key: &str, reason: &str) -> bool {
1282    let map = cooldown_lock();
1283    map.get(key).is_some_and(|entry| {
1284        entry.reason == reason && chrono::Utc::now().timestamp() < entry.cooldown_until
1285    })
1286}
1287
1288/// Check if a key is currently in cooldown.
1289fn is_in_cooldown(key: &str) -> bool {
1290    let map = cooldown_lock();
1291    if let Some(entry) = map.get(key) {
1292        let now = chrono::Utc::now().timestamp();
1293        return now < entry.cooldown_until;
1294    }
1295    false
1296}
1297
1298// ---------------------------------------------------------------------------
1299// Degraded-agent tracking (pre-emptive health check)
1300// ---------------------------------------------------------------------------
1301
1302/// Default health check window for recent rate-limit events (hours).
1303pub const DEFAULT_HEALTH_CHECK_WINDOW_HOURS: u32 = 6;
1304
1305/// Default threshold: minimum rate-limit events within the window to consider
1306/// an agent degraded.  A single transient 429 shouldn't mark an agent; we need
1307/// a pattern of repeated failures.
1308pub const DEFAULT_DEGRADED_RATE_LIMIT_THRESHOLD: i64 = 3;
1309
1310/// In-memory map of agents currently flagged as degraded, keyed by agent
1311/// with the reason that holds now
1312fn degraded_agents() -> &'static Mutex<HashMap<String, String>> {
1313    static DEGRADED: OnceLock<Mutex<HashMap<String, String>>> = OnceLock::new();
1314    DEGRADED.get_or_init(|| Mutex::new(HashMap::new()))
1315}
1316
1317/// Check if an agent is currently flagged as degraded by the health check.
1318///
1319/// Degraded means: all configured models are cooled **or** the agent has
1320/// exceeded the rate-limit threshold within the configured lookback window.
1321pub fn is_agent_degraded(agent: &str) -> bool {
1322    let set = degraded_agents_lock();
1323    set.contains_key(agent)
1324}
1325
1326/// Reason the agent is currently degraded, if it is
1327pub fn agent_degraded_reason(agent: &str) -> Option<String> {
1328    degraded_agents_lock().get(agent).cloned()
1329}
1330
1331/// Active degraded agents as `(agent, reason)`, sorted by agent name
1332pub fn list_degraded_agents() -> Vec<(String, String)> {
1333    let mut entries: Vec<(String, String)> = degraded_agents_lock()
1334        .iter()
1335        .map(|(agent, reason)| (agent.clone(), reason.clone()))
1336        .collect();
1337    entries.sort_by(|a, b| a.0.cmp(&b.0));
1338    entries
1339}
1340
1341/// Mark an agent as degraded (called by [`refresh_degraded_agents`]).
1342pub fn mark_agent_degraded(agent: &str) {
1343    mark_agent_degraded_with_reason(agent, "degraded");
1344}
1345
1346fn mark_agent_degraded_with_reason(agent: &str, reason: &str) {
1347    let mut set = degraded_agents_lock();
1348    set.insert(agent.to_string(), reason.to_string());
1349}
1350
1351/// Clear the degraded flag for an agent.
1352pub fn clear_agent_degraded(agent: &str) {
1353    let mut set = degraded_agents_lock();
1354    set.remove(agent);
1355}
1356
1357/// Refresh the degraded-agent set from the rate_limits table and cooldown state.
1358///
1359/// An agent is marked degraded when:
1360/// 1. It is in agent-level cooldown (explicit cooldown KV key exists and hasn't expired), **or**
1361/// 2. All its configured models are in cooldown, **or**
1362/// 3. It has >= `threshold` `limit_type = 'rate'` events within `window_hours`.
1363///
1364/// Failures keep their own cooldowns and do not count toward the threshold
1365/// Agents that no longer meet these criteria are cleared.
1366///
1367/// Note: This function calls `clear_expired_cooldowns()` first to ensure the
1368/// in-memory cooldown map reflects current state. An agent should NOT be marked
1369/// degraded simply because an expired cooldown entry hasn't been cleaned up yet.
1370pub async fn refresh_degraded_agents(
1371    store: &Arc<TaskStore>,
1372    available_agents: &[String],
1373    model_checker: &(dyn Fn(&str) -> bool + Send + Sync),
1374    window_hours: u32,
1375    threshold: i64,
1376) {
1377    // Clear expired cooldowns from in-memory map before evaluating agent state.
1378    // This ensures we don't incorrectly mark agents as degraded due to stale
1379    // in-memory entries that should have expired but weren't cleaned up.
1380    clear_expired_cooldowns();
1381
1382    // Time the DB query to surface slow health-checks that can dominate the
1383    // engine tick latency. If the query fails, log and return early.
1384    let start = chrono::Utc::now();
1385    let counts = match store.recent_rate_limit_counts(window_hours).await {
1386        Ok(c) => {
1387            let dur_ms = (chrono::Utc::now() - start).num_milliseconds();
1388            tracing::info!(
1389                duration_ms = dur_ms,
1390                window_hours,
1391                "recent_rate_limit_counts query completed"
1392            );
1393            c
1394        }
1395        Err(e) => {
1396            let dur_ms = (chrono::Utc::now() - start).num_milliseconds();
1397            tracing::warn!(
1398                err = %e,
1399                duration_ms = dur_ms,
1400                "failed to query recent rate limit counts for health check"
1401            );
1402            return;
1403        }
1404    };
1405
1406    for agent in available_agents {
1407        let in_cooldown = is_agent_in_cooldown(agent);
1408        let no_models = !model_checker(agent);
1409        let rate_limit_count = counts.get(agent.as_str()).copied().unwrap_or(0);
1410        let over_threshold = rate_limit_count >= threshold;
1411
1412        let degraded = in_cooldown || no_models || over_threshold;
1413
1414        if degraded {
1415            let reason = if in_cooldown {
1416                "agent in cooldown"
1417            } else if no_models {
1418                "all models cooled"
1419            } else {
1420                "rate limit threshold exceeded"
1421            };
1422            let previous = agent_degraded_reason(agent);
1423            match previous.as_deref() {
1424                None => tracing::warn!(
1425                    agent,
1426                    reason,
1427                    rate_limit_count,
1428                    window_hours,
1429                    threshold,
1430                    "pre-emptive health check: marking agent as degraded"
1431                ),
1432                Some(prev) if prev != reason => tracing::warn!(
1433                    agent,
1434                    reason,
1435                    previous = prev,
1436                    rate_limit_count,
1437                    window_hours,
1438                    threshold,
1439                    "pre-emptive health check: degraded reason changed"
1440                ),
1441                Some(_) => {}
1442            }
1443            mark_agent_degraded_with_reason(agent, reason);
1444        } else {
1445            if is_agent_degraded(agent) {
1446                tracing::info!(
1447                    agent,
1448                    rate_limit_count,
1449                    "pre-emptive health check: agent recovered, clearing degraded flag"
1450                );
1451            }
1452            clear_agent_degraded(agent);
1453        }
1454    }
1455}
1456
1457/// Clear expired cooldowns from the in-memory map.
1458pub fn clear_expired_cooldowns() {
1459    let mut map = cooldown_lock();
1460    let now = chrono::Utc::now().timestamp();
1461    map.retain(|_key, entry| now < entry.cooldown_until);
1462}
1463
1464// ---------------------------------------------------------------------------
1465// GitHub 5xx circuit breaker
1466// ---------------------------------------------------------------------------
1467
1468/// Sliding window for counting GitHub 5xx errors (seconds).
1469const GITHUB_5XX_WINDOW_SECS: i64 = 120;
1470
1471/// Number of 5xx errors within the window to trip the circuit breaker.
1472const GITHUB_5XX_THRESHOLD: usize = 5;
1473
1474/// Duration the circuit breaker stays open after tripping (seconds).
1475const GITHUB_5XX_COOLDOWN_SECS: i64 = 180;
1476
1477/// Global sliding window of 5xx error timestamps.
1478fn github_5xx_timestamps() -> &'static Mutex<Vec<i64>> {
1479    static TS: OnceLock<Mutex<Vec<i64>>> = OnceLock::new();
1480    TS.get_or_init(|| Mutex::new(Vec::new()))
1481}
1482
1483/// Whether the GitHub 5xx circuit breaker is currently open (tripped).
1484fn github_5xx_circuit_open() -> &'static Mutex<Option<i64>> {
1485    static OPEN: OnceLock<Mutex<Option<i64>>> = OnceLock::new();
1486    OPEN.get_or_init(|| Mutex::new(None))
1487}
1488
1489/// Record a GitHub 5xx server error and trip the circuit breaker if the
1490/// sliding window threshold is exceeded.
1491///
1492/// Emits a single high-level log entry when the circuit transitions to open.
1493pub async fn record_github_5xx() {
1494    let now = chrono::Utc::now().timestamp();
1495    let window_start = now - GITHUB_5XX_WINDOW_SECS;
1496
1497    let should_trip = {
1498        let mut ts = github_5xx_timestamps_lock();
1499        ts.retain(|t| *t >= window_start);
1500        ts.push(now);
1501        ts.len() >= GITHUB_5XX_THRESHOLD
1502    };
1503
1504    if should_trip {
1505        let cooldown_until = {
1506            let mut open = github_5xx_circuit_open_lock();
1507            if open.is_none() {
1508                let cd = now + GITHUB_5XX_COOLDOWN_SECS;
1509                *open = Some(cd);
1510                tracing::warn!(
1511                    errors_in_window = GITHUB_5XX_THRESHOLD,
1512                    window_secs = GITHUB_5XX_WINDOW_SECS,
1513                    cooldown_secs = GITHUB_5XX_COOLDOWN_SECS,
1514                    "GitHub 5xx circuit breaker OPEN — throttling non-critical work"
1515                );
1516                Some(cd)
1517            } else {
1518                None
1519            }
1520        };
1521        // Persist to KV (outside the mutex lock so the future is Send).
1522        if let Some(cd) = cooldown_until {
1523            let store_opt = cooldown_store().lock().await.clone();
1524            if let Some(store) = store_opt {
1525                let _ = store.kv_set("cooldown:github:5xx", &cd.to_string()).await;
1526            }
1527            // Also set the generic cooldown so is_agent_in_cooldown("github:5xx")
1528            // returns true immediately, avoiding a race window where concurrent
1529            // requests bypass the circuit breaker.
1530            set_agent_cooldown("github:5xx", GITHUB_5XX_COOLDOWN_SECS as u64).await;
1531        }
1532        // Clear the sliding window after tripping so we don't re-trip immediately
1533        let mut ts = github_5xx_timestamps_lock();
1534        ts.clear();
1535    }
1536}
1537
1538/// Check if the GitHub 5xx circuit breaker is currently open.
1539///
1540/// When open, non-critical background work (cleanup, polling, mention scans)
1541/// should be skipped. Critical operations (PR creation) should still attempt
1542/// with their existing exponential backoff.
1543///
1544/// Automatically recovers (logs a recovery message) when the cooldown expires.
1545pub fn is_github_circuit_open() -> bool {
1546    // Check the dedicated in-memory circuit flag first. If present, honour it
1547    // and perform auto-recovery when it expires (logging once on close).
1548    {
1549        let mut open = github_5xx_circuit_open_lock();
1550        if let Some(until) = *open {
1551            let now = chrono::Utc::now().timestamp();
1552            if now >= until {
1553                *open = None;
1554                tracing::info!("GitHub 5xx circuit breaker CLOSED — resuming normal operations");
1555                return false;
1556            }
1557            return true;
1558        }
1559    }
1560
1561    // If the dedicated flag is not set, consult the generic cooldown map so
1562    // callers see a consistent view regardless of which code path set the
1563    // persisted cooldown (e.g. send_with_retries -> set_agent_cooldown).
1564    if let Some(until) = cooldown_until("github:5xx") {
1565        // Populate the dedicated in-memory flag so future checks and the
1566        // auto-recovery path go through the same code and logging.
1567        let mut open = github_5xx_circuit_open_lock();
1568        if open.is_none() {
1569            *open = Some(until);
1570            let now = chrono::Utc::now().timestamp();
1571            if now < until {
1572                let remaining = until - now;
1573                tracing::warn!(
1574                    remaining_secs = remaining,
1575                    "GitHub 5xx circuit breaker OPEN (discovered via generic cooldown) — throttling non-critical work"
1576                );
1577            }
1578        }
1579        return true;
1580    }
1581
1582    false
1583}
1584
1585/// Returns the remaining seconds of the GitHub 5xx circuit breaker cooldown,
1586/// or 0 if the circuit is closed.
1587pub fn github_circuit_remaining_secs() -> u64 {
1588    // Prefer the dedicated in-memory value when present.
1589    {
1590        let open = github_5xx_circuit_open_lock();
1591        if let Some(until) = *open {
1592            let now = chrono::Utc::now().timestamp();
1593            if now < until {
1594                return u64::try_from(until - now).unwrap_or(0);
1595            }
1596            return 0;
1597        }
1598    }
1599
1600    // Fall back to the generic cooldown map if the dedicated flag isn't set.
1601    if let Some(until) = cooldown_until("github:5xx") {
1602        let now = chrono::Utc::now().timestamp();
1603        if now < until {
1604            return u64::try_from(until - now).unwrap_or(0);
1605        }
1606    }
1607    0
1608}
1609
1610/// Parse a "try again at {date}" or "reset at {date}" from an error message.
1611///
1612/// Returns a Unix timestamp if a retry-at date is found.
1613pub(crate) fn parse_retry_at(error_message: &str) -> Option<i64> {
1614    if error_message.is_empty() {
1615        return None;
1616    }
1617
1618    let lower = error_message.to_lowercase();
1619    // Accept a wider set of vendor-provided markers. Claude commonly uses
1620    // variations like "resets 10:30am" (no explicit "at"), so include
1621    // both "resets" and the older "reset at"/"try again at" markers.
1622    for retry_marker in [
1623        "try again at ",
1624        "reset at ",
1625        "resets at ",
1626        "resets ",
1627        "reset ",
1628    ] {
1629        if let Some(idx) = lower.find(retry_marker) {
1630            let date_str = &lower[idx + retry_marker.len()..];
1631            // Take until period, newline, or end
1632            let date_str = date_str
1633                .split(['.', '\n'])
1634                .next()
1635                .unwrap_or(date_str)
1636                .trim();
1637
1638            // Try common date formats
1639            // "Mar 26th, 2026 5:55 AM" → strip ordinal suffixes
1640            let cleaned = date_str
1641                .replace("st,", ",")
1642                .replace("nd,", ",")
1643                .replace("rd,", ",")
1644                .replace("th,", ",");
1645
1646            for fmt in &[
1647                "%b %d, %Y %I:%M %p", // Mar 26, 2026 5:55 AM
1648                "%b %d, %Y %I:%M%p",  // Mar 26, 2026 5:55AM
1649                "%B %d, %Y %I:%M %p", // March 26, 2026 5:55 AM
1650                "%Y-%m-%dT%H:%M:%S",  // 2026-03-26T05:55:00
1651                "%Y-%m-%d %H:%M:%S",  // 2026-03-26 05:55:00
1652                "%Y-%m-%d %H:%M",     // 2026-03-26 05:55
1653            ] {
1654                if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(&cleaned, fmt) {
1655                    // Vendor messages (e.g. Codex/GLM) emit times in the machine's
1656                    // local timezone, not UTC. Interpret naive datetime as local time.
1657                    let utc = match chrono::Local.from_local_datetime(&dt) {
1658                        chrono::LocalResult::Single(local_dt) => {
1659                            local_dt.with_timezone(&chrono::Utc)
1660                        }
1661                        chrono::LocalResult::Ambiguous(earliest, _) => {
1662                            // DST fold: pick the earlier (conservative) interpretation
1663                            earliest.with_timezone(&chrono::Utc)
1664                        }
1665                        chrono::LocalResult::None => {
1666                            // DST gap: fall back to treating as UTC to avoid panic
1667                            dt.and_utc()
1668                        }
1669                    };
1670                    tracing::info!(
1671                        retry_at = %utc,
1672                        raw = date_str,
1673                        marker = retry_marker,
1674                        "parsed rate limit retry-at date"
1675                    );
1676                    return Some(utc.timestamp());
1677                }
1678            }
1679            tracing::debug!(
1680                raw = date_str,
1681                marker = retry_marker,
1682                "could not parse retry-at date"
1683            );
1684
1685            // If the vendor provided only a time (e.g. "resets 10:30am"),
1686            // attempt to parse a time-only and interpret it as the next
1687            // occurrence of that time in the local timezone.
1688            // Strip parenthesised timezone hints like "(America/Sao_Paulo)".
1689            let time_only = date_str.split_once('(').map_or(date_str, |(t, _)| t).trim();
1690            // Hour-only values like "8am" have no minute field, which chrono needs, so insert ":00"
1691            let mut compact = time_only.replace(" am", "am").replace(" pm", "pm");
1692            let split = compact.rfind(' ').map_or(0, |i| i + 1);
1693            let last = &compact[split..];
1694            if !last.contains(':')
1695                && (last.ends_with("am") || last.ends_with("pm"))
1696                && last.len() > 2
1697            {
1698                let at = split + last.len() - 2;
1699                compact.insert_str(at, ":00");
1700            }
1701            // "Jun 9 at 1am" has no year, so take the next future occurrence
1702            let md = compact.replace(" at ", " ");
1703            let now_local = chrono::Local::now();
1704            let this_year = chrono::Datelike::year(&now_local);
1705            for year in [this_year, this_year + 1] {
1706                let with_year = format!("{year} {md}");
1707                if let Ok(dt) =
1708                    chrono::NaiveDateTime::parse_from_str(&with_year, "%Y %b %d %I:%M%p")
1709                    && let Some(local_dt) = chrono::Local.from_local_datetime(&dt).earliest()
1710                    && local_dt > now_local
1711                {
1712                    return Some(local_dt.timestamp());
1713                }
1714            }
1715            let time_only = compact.as_str();
1716            for tf in &["%I:%M %p", "%I:%M%p", "%I %p", "%H:%M"] {
1717                if let Ok(t) = chrono::NaiveTime::parse_from_str(time_only, tf) {
1718                    // Build a NaiveDateTime for today at that time, then choose
1719                    // today or tomorrow depending on whether the time has passed.
1720                    let local_now = chrono::Local::now();
1721                    let today = local_now.date_naive();
1722                    let candidate = chrono::NaiveDateTime::new(today, t);
1723                    // Use the TimeZone trait methods; annotate types so the
1724                    // compiler can infer the concrete DateTime types.
1725                    let candidate_local: chrono::DateTime<chrono::Local> =
1726                        match chrono::Local.from_local_datetime(&candidate) {
1727                            chrono::LocalResult::Single(dt) => dt,
1728                            chrono::LocalResult::Ambiguous(earliest, _) => earliest,
1729                            chrono::LocalResult::None => {
1730                                // DST gap — fall back to UTC interpretation
1731                                let utc_dt = candidate.and_utc();
1732                                return Some(utc_dt.timestamp());
1733                            }
1734                        };
1735                    let chosen = if candidate_local <= local_now {
1736                        // Use tomorrow
1737                        let tomorrow = today.succ_opt().unwrap_or(today);
1738                        let ndt = chrono::NaiveDateTime::new(tomorrow, t);
1739                        match chrono::Local.from_local_datetime(&ndt) {
1740                            chrono::LocalResult::Single(dt) => dt.with_timezone(&chrono::Utc),
1741                            chrono::LocalResult::Ambiguous(earliest, _) => {
1742                                earliest.with_timezone(&chrono::Utc)
1743                            }
1744                            chrono::LocalResult::None => ndt.and_utc(),
1745                        }
1746                    } else {
1747                        candidate_local.with_timezone(&chrono::Utc)
1748                    };
1749                    tracing::info!(
1750                        retry_at = %chosen,
1751                        raw = date_str,
1752                        marker = retry_marker,
1753                        "parsed rate limit retry-at time-only; interpreted as next occurrence"
1754                    );
1755                    return Some(chosen.timestamp());
1756                }
1757            }
1758        }
1759    }
1760
1761    // Without a specific retry/reset date, return None and let the caller use
1762    // its default cooldown (exponential backoff). A blanket long fallback here
1763    // would cause false billing-cycle cooldowns on agents without those limits.
1764    None
1765}
1766
1767/// Parse a relative usage window (e.g. "5-hour usage limit") from an error
1768/// message.
1769///
1770/// Returns the window length in seconds. Only matches in usage/quota/limit
1771/// contexts ("usage limit", "quota", "window", "limit") to avoid interpreting
1772/// unrelated numbers as cooldown durations.
1773///
1774/// `pub(crate)` so `engine::runner::fallback::handle_error()` can apply the
1775/// same window-detection the `needs_review` call site
1776/// (`record_rate_limit()`, below) already uses — see issue #3580.
1777pub(crate) fn parse_relative_usage_window(error_message: &str) -> Option<u64> {
1778    static WINDOW_RE: LazyLock<Regex> = LazyLock::new(|| {
1779        Regex::new(
1780            r"(?i)(?:^|\s|[\(\)\.,;:!?])(?:(\d+)\s*-?\s*(hour|hours|hr|hrs|minute|minutes|min|mins|day|days)|(hourly|daily|weekly|monthly))(?:$|\s|[\(\)\.,;:!?])"
1781        )
1782        .expect("usage window regex is valid")
1783    });
1784
1785    if error_message.is_empty() {
1786        return None;
1787    }
1788
1789    let lower = error_message.to_lowercase();
1790    let has_context = lower.contains("usage limit")
1791        || lower.contains("quota")
1792        || lower.contains("window")
1793        || lower.contains("limit");
1794    if !has_context {
1795        return None;
1796    }
1797
1798    let mut max_secs: Option<u64> = None;
1799    for caps in WINDOW_RE.captures_iter(&lower) {
1800        let secs = if let Some(num) = caps.get(1) {
1801            let n: u64 = num.as_str().parse().ok()?;
1802            let unit = caps.get(2)?.as_str().to_lowercase();
1803            let multiplier = match unit.as_str() {
1804                "hour" | "hours" | "hr" | "hrs" => 3600,
1805                "minute" | "minutes" | "min" | "mins" => 60,
1806                "day" | "days" => 86400,
1807                _ => return None,
1808            };
1809            n.checked_mul(multiplier)?
1810        } else if let Some(word) = caps.get(3) {
1811            match word.as_str().to_lowercase().as_str() {
1812                "hourly" => 3600,
1813                "daily" => 86400,
1814                "weekly" => 604_800,
1815                "monthly" => 2_592_000,
1816                _ => return None,
1817            }
1818        } else {
1819            continue;
1820        };
1821        max_secs = Some(max_secs.map_or(secs, |m| m.max(secs)));
1822    }
1823
1824    max_secs
1825}
1826
1827#[cfg(test)]
1828pub(crate) async fn reset_global_state() {
1829    cooldowns().lock().unwrap().clear();
1830    *cooldown_store().lock().await = None;
1831    github_5xx_timestamps().lock().unwrap().clear();
1832    *github_5xx_circuit_open().lock().unwrap() = None;
1833    degraded_agents().lock().unwrap().clear();
1834    opencode::reset_model_caches_for_test();
1835}
1836
1837#[cfg(test)]
1838mod tests {
1839    use std::time::Duration;
1840
1841    use super::*;
1842    use chrono::Datelike;
1843    use chrono::TimeZone;
1844    use serial_test::serial;
1845    use tokio::time;
1846
1847    async fn test_store() -> Arc<TaskStore> {
1848        Arc::new(TaskStore::open_memory().await.expect("in-memory store"))
1849    }
1850
1851    fn sqlite_datetime_offset(secs: i64) -> String {
1852        (chrono::Utc::now() + chrono::Duration::seconds(secs))
1853            .format("%Y-%m-%d %H:%M:%S")
1854            .to_string()
1855    }
1856
1857    #[serial(cooldown_state)]
1858    #[tokio::test]
1859    async fn init_loads_unexpired_cooldowns_from_kv() {
1860        super::reset_global_state().await;
1861        let store = test_store().await;
1862        let cooldown_until = chrono::Utc::now().timestamp() + 3600; // 1 hour from now
1863
1864        store
1865            .kv_set(
1866                &format!("{KV_PREFIX}kimi:k2p5"),
1867                &cooldown_until.to_string(),
1868            )
1869            .await
1870            .unwrap();
1871
1872        // Also add an already-expired entry
1873        let expired = chrono::Utc::now().timestamp() - 1;
1874        store
1875            .kv_set(
1876                &format!("{KV_PREFIX}opencode:old-free"),
1877                &expired.to_string(),
1878            )
1879            .await
1880            .unwrap();
1881
1882        {
1883            let mut map = cooldown_lock();
1884            map.clear();
1885        }
1886
1887        init_cooldown_store(store).await;
1888
1889        assert!(
1890            is_model_in_cooldown("kimi", "k2p5"),
1891            "unexpired cooldown should be loaded"
1892        );
1893        assert!(
1894            !is_model_in_cooldown("opencode", "old-free"),
1895            "expired cooldown should not be loaded"
1896        );
1897    }
1898
1899    #[serial(cooldown_state)]
1900    #[tokio::test]
1901    async fn credit_exhaustion_inside_active_cooldown_keeps_counter_at_one() {
1902        super::reset_global_state().await;
1903        let store = test_store().await;
1904        *cooldown_store().lock().await = Some(Arc::clone(&store));
1905
1906        let msg =
1907            "API error (status 402 Payment Required): usage balance exhausted, out of credits";
1908        assert!(record_credit_exhaustion_from_message("credit_dup_agent", Some("m"), msg).await);
1909        assert!(record_credit_exhaustion_from_message("credit_dup_agent", Some("m"), msg).await);
1910
1911        assert!(is_agent_in_cooldown("credit_dup_agent"));
1912        let count = store
1913            .kv_get(&format!("{CREDIT_FAILURE_COUNT_PREFIX}credit_dup_agent"))
1914            .await
1915            .unwrap();
1916        assert_eq!(count.as_deref(), Some("1"));
1917    }
1918
1919    #[serial(cooldown_state)]
1920    #[tokio::test]
1921    async fn non_credit_message_is_not_recorded_as_credit_exhaustion() {
1922        super::reset_global_state().await;
1923        assert!(!record_credit_exhaustion_from_message("x_agent", Some("m"), "boom").await);
1924        assert!(!is_agent_in_cooldown("x_agent"));
1925    }
1926
1927    #[serial(cooldown_state)]
1928    #[tokio::test]
1929    async fn record_model_failure_writes_to_kv() {
1930        super::reset_global_state().await;
1931        let store = test_store().await;
1932
1933        *cooldown_store().lock().await = Some(Arc::clone(&store));
1934
1935        record_model_failure("testagent_persist", "testmodel_persist").await;
1936
1937        // Give the async KV write a moment to settle before querying.
1938        time::sleep(Duration::from_millis(50)).await;
1939
1940        let kv_key = format!("{KV_PREFIX}testagent_persist:testmodel_persist");
1941        let stored = store.kv_get(&kv_key).await.unwrap();
1942        assert!(stored.is_some(), "model failure should be persisted to KV");
1943        let (ts, reason) = parse_cooldown_value(&stored.unwrap()).expect("cooldown value");
1944        let now = chrono::Utc::now().timestamp();
1945        // Should be in the future (cooldown_until, not failed_at)
1946        assert!(ts > now, "persisted timestamp should be in the future");
1947        assert_eq!(reason, "model_error");
1948    }
1949
1950    #[serial(cooldown_state)]
1951    #[tokio::test]
1952    async fn record_transient_model_failure_backs_off_5_then_15_minutes() {
1953        super::reset_global_state().await;
1954        let store = test_store().await;
1955        *cooldown_store().lock().await = Some(Arc::clone(&store));
1956
1957        let agent = "testagent_transient";
1958        let model = "testmodel_transient";
1959        let key = format!("{agent}:{model}");
1960
1961        record_transient_model_failure(agent, model).await;
1962        let first = cooldown_until(&key)
1963            .expect("first cooldown should exist")
1964            .saturating_sub(chrono::Utc::now().timestamp());
1965        assert!(
1966            (TRANSIENT_MODEL_BACKOFF_BASE_SECS - 5..=TRANSIENT_MODEL_BACKOFF_BASE_SECS + 5)
1967                .contains(&first),
1968            "first transient cooldown should be ~5m, got {first}s"
1969        );
1970
1971        record_transient_model_failure(agent, model).await;
1972        let second = cooldown_until(&key)
1973            .expect("second cooldown should exist")
1974            .saturating_sub(chrono::Utc::now().timestamp());
1975        assert!(
1976            (TRANSIENT_MODEL_BACKOFF_MAX_SECS - 5..=TRANSIENT_MODEL_BACKOFF_MAX_SECS + 5)
1977                .contains(&second),
1978            "second transient cooldown should be capped at ~15m, got {second}s"
1979        );
1980
1981        let (_, reason) = parse_cooldown_value(
1982            &store
1983                .kv_get(&format!("{KV_PREFIX}{key}"))
1984                .await
1985                .unwrap()
1986                .expect("cooldown persisted"),
1987        )
1988        .expect("cooldown value");
1989        assert_eq!(reason, "transient_model_error");
1990
1991        let count = store
1992            .kv_get(&format!("{FAILURE_COUNT_PREFIX}{key}"))
1993            .await
1994            .unwrap();
1995        assert_eq!(count.as_deref(), Some("2"));
1996    }
1997
1998    #[serial(cooldown_state)]
1999    #[tokio::test]
2000    async fn record_model_failure_and_persistent_failure_use_separate_counters() {
2001        super::reset_global_state().await;
2002        let store = test_store().await;
2003        *cooldown_store().lock().await = Some(Arc::clone(&store));
2004
2005        let agent = "testagent_counters";
2006        let model = "testmodel_counters";
2007        let key = format!("{agent}:{model}");
2008
2009        record_model_failure(agent, model).await;
2010        record_persistent_model_failure(agent, model).await;
2011        record_persistent_model_failure(agent, model).await;
2012
2013        let generic = store
2014            .kv_get(&format!("{FAILURE_COUNT_PREFIX}{key}"))
2015            .await
2016            .unwrap();
2017        let persistent = store
2018            .kv_get(&format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{key}"))
2019            .await
2020            .unwrap();
2021        assert_eq!(generic.as_deref(), Some("1"));
2022        assert_eq!(persistent.as_deref(), Some("2"));
2023
2024        let (_, reason) = parse_cooldown_value(
2025            &store
2026                .kv_get(&format!("{KV_PREFIX}{key}"))
2027                .await
2028                .unwrap()
2029                .expect("cooldown persisted"),
2030        )
2031        .expect("cooldown value");
2032        assert_eq!(reason, "persistent_model_error");
2033    }
2034
2035    #[serial(cooldown_state)]
2036    #[tokio::test]
2037    async fn cooldown_reason_survives_restart_via_kv() {
2038        super::reset_global_state().await;
2039        let store = test_store().await;
2040        let agent = "test_reason_restart";
2041        let future = chrono::Utc::now().timestamp() + 600;
2042
2043        init_cooldown_store(Arc::clone(&store)).await;
2044        set_cooldown_async(agent, future, "billing_cycle_exhausted").await;
2045
2046        // Simulate a service restart: drop the in-memory map, reload from KV.
2047        {
2048            let mut map = cooldown_lock();
2049            map.clear();
2050        }
2051        init_cooldown_store(store).await;
2052
2053        let entries = list_all_cooldowns();
2054        let entry = entries.iter().find(|(key, ..)| key == agent);
2055        assert_eq!(
2056            entry.map(|(_, _, reason)| reason.as_str()),
2057            Some("billing_cycle_exhausted"),
2058            "cooldown reason must survive a restart, not collapse to 'persisted'"
2059        );
2060    }
2061
2062    #[serial(cooldown_state)]
2063    #[tokio::test]
2064    async fn record_persistent_model_failure_escalates_and_increments_failure_count() {
2065        super::reset_global_state().await;
2066        let store = test_store().await;
2067        *cooldown_store().lock().await = Some(Arc::clone(&store));
2068
2069        let agent = "testagent_persistent";
2070        let model = "testmodel_persistent";
2071        let key = format!("{agent}:{model}");
2072
2073        // First failure -> base 4h.
2074        record_persistent_model_failure(agent, model).await;
2075        let now = chrono::Utc::now().timestamp();
2076        let first_until = cooldown_until(&key).expect("first cooldown should exist");
2077        let first_remaining = first_until.saturating_sub(now);
2078        assert!(
2079            first_remaining >= PERSISTENT_MODEL_BACKOFF_BASE_SECS - 5,
2080            "first persistent model cooldown should be ~4h, got {first_remaining}s"
2081        );
2082
2083        // Second failure -> 12h (3x growth).
2084        record_persistent_model_failure(agent, model).await;
2085        let now2 = chrono::Utc::now().timestamp();
2086        let second_until = cooldown_until(&key).expect("second cooldown should exist");
2087        let second_remaining = second_until.saturating_sub(now2);
2088        assert!(
2089            second_remaining >= (PERSISTENT_MODEL_BACKOFF_BASE_SECS * 3) - 5,
2090            "second persistent model cooldown should be ~12h, got {second_remaining}s"
2091        );
2092
2093        let count_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{agent}:{model}");
2094        let stored_count = store
2095            .kv_get(&count_key)
2096            .await
2097            .expect("kv read should succeed")
2098            .expect("failure count should be written");
2099        assert_eq!(stored_count, "2");
2100    }
2101
2102    #[serial(cooldown_state)]
2103    #[tokio::test]
2104    async fn record_agent_success_does_not_reset_persistent_failure_count() {
2105        super::reset_global_state().await;
2106        let store = test_store().await;
2107        *cooldown_store().lock().await = Some(Arc::clone(&store));
2108
2109        let agent = "testagent_persistent_success";
2110        let model = "testmodel_persistent_success";
2111        let key = format!("{agent}:{model}");
2112
2113        // Two persistent failures escalate the counter to 2 (~12h cooldown).
2114        record_persistent_model_failure(agent, model).await;
2115        record_persistent_model_failure(agent, model).await;
2116
2117        let count_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{agent}:{model}");
2118        let stored_count = store
2119            .kv_get(&count_key)
2120            .await
2121            .expect("kv read should succeed")
2122            .expect("failure count should be written");
2123        assert_eq!(stored_count, "2");
2124
2125        // A successful run resets the generic failure_count but must NOT touch
2126        // the persistent counter — otherwise an intervening success would erase
2127        // escalation progress for a structurally unreliable model.
2128        record_agent_success(agent, model).await;
2129
2130        let stored_count_after_success = store
2131            .kv_get(&count_key)
2132            .await
2133            .expect("kv read should succeed")
2134            .expect("persistent failure count should still exist");
2135        assert_eq!(
2136            stored_count_after_success, "2",
2137            "persistent failure count must survive an intervening success"
2138        );
2139
2140        // The next persistent failure should escalate from 3 (~36h), not reset to base (4h).
2141        record_persistent_model_failure(agent, model).await;
2142        let now = chrono::Utc::now().timestamp();
2143        let third_until = cooldown_until(&key).expect("third cooldown should exist");
2144        let third_remaining = third_until.saturating_sub(now);
2145        assert!(
2146            third_remaining > PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2147            "third persistent model failure should escalate past the 4h base, got {third_remaining}s"
2148        );
2149    }
2150
2151    #[serial(cooldown_state)]
2152    #[tokio::test]
2153    async fn record_persistent_model_failure_stays_capped_at_7_days() {
2154        super::reset_global_state().await;
2155        let store = test_store().await;
2156        *cooldown_store().lock().await = Some(Arc::clone(&store));
2157
2158        let agent = "testagent_persistent_capped";
2159        let model = "testmodel_persistent_capped";
2160        let key = format!("{agent}:{model}");
2161
2162        for _ in 0..9 {
2163            record_persistent_model_failure(agent, model).await;
2164        }
2165
2166        let remaining = cooldown_until(&key)
2167            .expect("cooldown should exist")
2168            .saturating_sub(chrono::Utc::now().timestamp());
2169        assert!(
2170            remaining >= PERSISTENT_MODEL_BACKOFF_MAX_SECS - 5,
2171            "persistent model cooldown should stay capped at 7 days, got {remaining}s"
2172        );
2173        assert!(
2174            remaining <= PERSISTENT_MODEL_BACKOFF_MAX_SECS + 5,
2175            "persistent model cooldown should not exceed 7 days, got {remaining}s"
2176        );
2177    }
2178
2179    #[serial(cooldown_state)]
2180    #[test]
2181    fn parse_retry_at_codex_format() {
2182        let msg = "You've hit your usage limit. Upgrade to Pro, visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Mar 26th, 2026 5:55 AM.";
2183        let ts = parse_retry_at(msg);
2184        assert!(ts.is_some(), "should parse codex retry-at date");
2185
2186        // The parsed timestamp must equal "Mar 26 2026 05:55 local" in UTC.
2187        // We reconstruct what the function should produce and compare directly.
2188        let naive = chrono::NaiveDateTime::new(
2189            chrono::NaiveDate::from_ymd_opt(2026, 3, 26).unwrap(),
2190            chrono::NaiveTime::from_hms_opt(5, 55, 0).unwrap(),
2191        );
2192        let expected_utc = match chrono::Local.from_local_datetime(&naive) {
2193            chrono::LocalResult::Single(local_dt) => local_dt.with_timezone(&chrono::Utc),
2194            chrono::LocalResult::Ambiguous(earliest, _) => earliest.with_timezone(&chrono::Utc),
2195            chrono::LocalResult::None => naive.and_utc(),
2196        };
2197        assert_eq!(
2198            ts.unwrap(),
2199            expected_utc.timestamp(),
2200            "timestamp should reflect local-timezone interpretation of the vendor date"
2201        );
2202
2203        // Sanity: the UTC date must still be March 26 (true for all timezones within ±14h)
2204        let dt = chrono::DateTime::from_timestamp(ts.unwrap(), 0).unwrap();
2205        assert_eq!(dt.month(), 3);
2206        assert_eq!(dt.day(), 26);
2207    }
2208
2209    #[serial(cooldown_state)]
2210    #[test]
2211    fn parse_retry_at_hour_only_and_month_day() {
2212        let now = chrono::Utc::now().timestamp();
2213        for msg in [
2214            "You've hit your session limit · resets 8am (America/Sao_Paulo)",
2215            "You've hit your session limit · resets 6pm (America/Sao_Paulo)",
2216            "You've hit your session limit · resets 12am (America/Sao_Paulo)",
2217            "You've hit your weekly limit · resets Jun 9 at 1am (America/Sao_Paulo)",
2218        ] {
2219            let ts = parse_retry_at(msg).unwrap_or_else(|| panic!("should parse: {msg}"));
2220            assert!(ts > now, "{msg}");
2221        }
2222    }
2223
2224    #[serial(cooldown_state)]
2225    #[test]
2226    fn parse_retry_at_weekly_resets_without_year() {
2227        let t = chrono::Local::now() + chrono::Duration::days(2);
2228        let msg = format!(
2229            "You've hit your weekly limit · resets {} {} at 1am (America/Sao_Paulo)",
2230            t.format("%b"),
2231            t.day()
2232        );
2233        let ts = parse_retry_at(&msg).expect("should parse year-less reset date");
2234        let remaining = ts - chrono::Utc::now().timestamp();
2235        assert!(
2236            remaining > 3600 && remaining < 3 * 86400,
2237            "expected ~2d retry-at, got {remaining}s"
2238        );
2239    }
2240
2241    #[serial(cooldown_state)]
2242    #[test]
2243    fn parse_retry_at_no_date() {
2244        assert!(parse_retry_at("generic rate limit error").is_none());
2245        assert!(parse_retry_at("").is_none());
2246    }
2247
2248    #[serial(cooldown_state)]
2249    #[test]
2250    fn parse_retry_at_glm_reset_format_with_seconds() {
2251        let msg = "Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-04-23 18:33:48";
2252        let ts = parse_retry_at(msg).expect("should parse GLM reset-at date");
2253
2254        let naive = chrono::NaiveDateTime::new(
2255            chrono::NaiveDate::from_ymd_opt(2026, 4, 23).expect("valid date"),
2256            chrono::NaiveTime::from_hms_opt(18, 33, 48).expect("valid time"),
2257        );
2258        let expected_utc = match chrono::Local.from_local_datetime(&naive) {
2259            chrono::LocalResult::Single(local_dt) => local_dt.with_timezone(&chrono::Utc),
2260            chrono::LocalResult::Ambiguous(earliest, _) => earliest.with_timezone(&chrono::Utc),
2261            chrono::LocalResult::None => naive.and_utc(),
2262        };
2263
2264        assert_eq!(
2265            ts,
2266            expected_utc.timestamp(),
2267            "timestamp should reflect local-timezone interpretation of vendor reset date"
2268        );
2269    }
2270
2271    #[serial(cooldown_state)]
2272    #[test]
2273    fn parse_retry_at_billing_cycle_without_date_returns_none() {
2274        // Without a specific "try again at" date, parse_retry_at should return
2275        // None so callers use their default cooldown (30m agent / 1h model).
2276        let msg = "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle. Upgrade to get more.";
2277        assert!(
2278            parse_retry_at(msg).is_none(),
2279            "billing cycle without date should return None"
2280        );
2281    }
2282
2283    #[serial(cooldown_state)]
2284    #[test]
2285    fn parse_relative_usage_window_extracts_kimi_five_hour_window() {
2286        let msg = "Failed to authenticate. API Error: 403 You've reached your 5-hour usage limit. Your quota will reset when the current 5-hour window expires.";
2287        assert_eq!(
2288            parse_relative_usage_window(msg),
2289            Some(5 * 3600),
2290            "should extract the 5-hour window as seconds"
2291        );
2292    }
2293
2294    #[serial(cooldown_state)]
2295    #[test]
2296    fn parse_relative_usage_window_handles_common_durations() {
2297        assert_eq!(
2298            parse_relative_usage_window("You've reached your hourly usage limit."),
2299            Some(3600)
2300        );
2301        assert_eq!(
2302            parse_relative_usage_window("You've reached your daily usage limit."),
2303            Some(86400)
2304        );
2305        assert_eq!(
2306            parse_relative_usage_window("You've reached your weekly limit."),
2307            Some(604_800)
2308        );
2309        assert_eq!(
2310            parse_relative_usage_window("You've reached your 30-minute usage limit."),
2311            Some(1800)
2312        );
2313        assert_eq!(
2314            parse_relative_usage_window("You've reached your 2-day quota."),
2315            Some(172_800)
2316        );
2317    }
2318
2319    #[serial(cooldown_state)]
2320    #[test]
2321    fn parse_relative_usage_window_returns_none_without_usage_context() {
2322        // A bare rate-limit message with a number should not be interpreted as a
2323        // relative usage window unless usage/quota/window context is present.
2324        assert!(parse_relative_usage_window("429 Too Many Requests").is_none());
2325        assert!(parse_relative_usage_window("rate limit exceeded").is_none());
2326        assert!(parse_relative_usage_window("").is_none());
2327    }
2328
2329    #[serial(cooldown_state)]
2330    #[tokio::test]
2331    async fn record_rate_limit_uses_model_specific_window_cooldown() {
2332        super::reset_global_state().await;
2333        let store = test_store().await;
2334        *cooldown_store().lock().await = Some(Arc::clone(&store));
2335
2336        let agent = "test_record_rl_window_agent";
2337        let model = "test_record_rl_window_model";
2338        let msg = "You've reached your 5-hour usage limit. Your quota will reset when the current 5-hour window expires.";
2339
2340        record_rate_limit(agent, Some(model), msg).await;
2341
2342        assert!(
2343            is_model_in_cooldown(agent, model),
2344            "model should be cooled for the relative usage window"
2345        );
2346        assert!(
2347            !is_agent_in_cooldown(agent),
2348            "agent should not be cooled for a model-scoped quota"
2349        );
2350
2351        let key = format!("{agent}:{model}");
2352        let remaining = cooldown_until(&key)
2353            .expect("model cooldown should exist")
2354            .saturating_sub(chrono::Utc::now().timestamp());
2355        assert!(
2356            remaining >= 5 * 3600 - 5,
2357            "model cooldown should be at least ~5 hours, got {remaining}s"
2358        );
2359        assert_eq!(
2360            cooldown_reason(&key).as_deref(),
2361            Some("rate_limit_usage_window"),
2362            "relative usage-window cooldown must be labeled as a rate limit, not silence"
2363        );
2364    }
2365
2366    #[serial(cooldown_state)]
2367    #[tokio::test]
2368    async fn record_rate_limit_without_window_falls_back_to_agent_and_model() {
2369        super::reset_global_state().await;
2370        let store = test_store().await;
2371        *cooldown_store().lock().await = Some(Arc::clone(&store));
2372
2373        let agent = "test_record_rl_fallback_agent";
2374        let model = "test_record_rl_fallback_model";
2375        let msg = "429 Too Many Requests";
2376
2377        record_rate_limit(agent, Some(model), msg).await;
2378
2379        assert!(
2380            is_agent_in_cooldown(agent),
2381            "agent should be cooled when no relative window is detected"
2382        );
2383        assert!(
2384            is_model_in_cooldown(agent, model),
2385            "model should be cooled when no relative window is detected"
2386        );
2387    }
2388
2389    #[serial(cooldown_state)]
2390    #[tokio::test]
2391    async fn agent_cooldown_persists_to_kv() {
2392        super::reset_global_state().await;
2393        // Agent-level cooldowns persist to KV via set_cooldown_async
2394        let agent = "test_agent_persist_check";
2395        record_agent_failure_with_message(agent, "").await;
2396        assert!(is_agent_in_cooldown(agent));
2397    }
2398
2399    #[serial(cooldown_state)]
2400    #[tokio::test]
2401    async fn cooldown_never_shortens() {
2402        super::reset_global_state().await;
2403        let agent = "test_agent_no_shorten";
2404        set_agent_cooldown(agent, 24 * 60 * 60).await;
2405        let initial = {
2406            let map = cooldowns().lock().unwrap();
2407            map.get(agent)
2408                .expect("cooldown entry should exist")
2409                .cooldown_until
2410        };
2411
2412        // Shorter cooldown attempt should not override the existing one.
2413        record_agent_failure_with_message(agent, "rate limit").await;
2414        let after = {
2415            let map = cooldowns().lock().unwrap();
2416            map.get(agent)
2417                .expect("cooldown entry should exist")
2418                .cooldown_until
2419        };
2420        assert_eq!(
2421            initial, after,
2422            "cooldown should not be shortened by later failures"
2423        );
2424    }
2425
2426    #[serial(cooldown_state)]
2427    #[tokio::test]
2428    async fn silence_agent_cooldown_is_short_lived() {
2429        super::reset_global_state().await;
2430        let agent = "test_silence_agent_cd";
2431        assert!(!is_agent_in_cooldown(agent));
2432
2433        set_agent_cooldown(agent, SILENCE_AGENT_COOLDOWN_SECS).await;
2434        assert!(is_agent_in_cooldown(agent));
2435
2436        // Verify it's a short cooldown (120s), not the exponential backoff one
2437        let map = cooldowns().lock().unwrap();
2438        let entry = map.get(agent).expect("should have cooldown entry");
2439        let now = chrono::Utc::now().timestamp();
2440        let remaining = entry.cooldown_until - now;
2441        assert!(
2442            remaining <= i64::try_from(SILENCE_AGENT_COOLDOWN_SECS).unwrap(),
2443            "silence agent cooldown should be <= {SILENCE_AGENT_COOLDOWN_SECS}s, got {remaining}s"
2444        );
2445        assert!(remaining > 0, "cooldown should still be active");
2446    }
2447
2448    #[serial(cooldown_state)]
2449    #[tokio::test]
2450    async fn record_silence_detection_applies_extended_cooldown() {
2451        super::reset_global_state().await;
2452        let store = test_store().await;
2453        *cooldown_store().lock().await = Some(Arc::clone(&store));
2454
2455        let agent = "test_silence_count_agent";
2456        let model = "test_silence_count_model";
2457        assert!(!is_model_in_cooldown(agent, model));
2458
2459        for _ in 0..SILENCE_COUNT_THRESHOLD {
2460            let result = record_silence_detection(agent, model).await;
2461            assert!(result.is_some());
2462        }
2463
2464        assert!(is_model_in_cooldown(agent, model));
2465        let key = format!("{agent}:{model}");
2466        let remaining = {
2467            let map = cooldowns().lock().unwrap();
2468            let entry = map.get(&key).expect("cooldown entry should exist");
2469            entry.cooldown_until - chrono::Utc::now().timestamp()
2470        };
2471        assert!(
2472            remaining >= i64::try_from(SILENCE_EXTENDED_COOLDOWN_SECS).unwrap() - 5,
2473            "extended cooldown should be applied, got {remaining}s"
2474        );
2475
2476        let kv_key = format!("{SILENCE_COUNT_PREFIX}{agent}:{model}");
2477        let stored = store.kv_get(&kv_key).await.unwrap().unwrap();
2478        assert_eq!(stored, "[]");
2479    }
2480
2481    #[serial(cooldown_state)]
2482    #[test]
2483    fn detect_credit_exhaustion_out_of_credits() {
2484        let msg = "Error: API error - {\"error\":{\"message\":\"You exceeded your current quota\",\"type\":\"insufficient_quota\",\"param\":null,\"code\":\"insufficient_quota\"}}";
2485        assert_eq!(
2486            detect_credit_exhaustion(msg),
2487            Some(CreditExhaustionReason::OutOfCredits)
2488        );
2489    }
2490
2491    #[serial(cooldown_state)]
2492    #[test]
2493    fn detect_credit_exhaustion_out_of_credits_variations() {
2494        assert_eq!(
2495            detect_credit_exhaustion("out of credits"),
2496            Some(CreditExhaustionReason::OutOfCredits)
2497        );
2498        assert_eq!(
2499            detect_credit_exhaustion("outofcredits"),
2500            Some(CreditExhaustionReason::OutOfCredits)
2501        );
2502        assert_eq!(
2503            detect_credit_exhaustion("no credits remaining"),
2504            Some(CreditExhaustionReason::OutOfCredits)
2505        );
2506        assert_eq!(
2507            detect_credit_exhaustion("insufficient funds"),
2508            Some(CreditExhaustionReason::OutOfCredits)
2509        );
2510    }
2511
2512    #[serial(cooldown_state)]
2513    #[test]
2514    fn detect_credit_exhaustion_org_level_disabled() {
2515        let msg = "Error: API error - {\"error\":{\"message\":\"Organization has been disabled\",\"type\":\"org_disabled\",\"code\":\"overageDisabledReason:org_level_disabled\"}}";
2516        assert_eq!(
2517            detect_credit_exhaustion(msg),
2518            Some(CreditExhaustionReason::OrgLevelDisabled)
2519        );
2520    }
2521
2522    #[serial(cooldown_state)]
2523    #[test]
2524    fn detect_credit_exhaustion_org_level_disabled_variations() {
2525        assert_eq!(
2526            detect_credit_exhaustion("organization has been disabled"),
2527            Some(CreditExhaustionReason::OrgLevelDisabled)
2528        );
2529        assert_eq!(
2530            detect_credit_exhaustion("org disabled"),
2531            Some(CreditExhaustionReason::OrgLevelDisabled)
2532        );
2533        assert_eq!(
2534            detect_credit_exhaustion("billing quota exceeded"),
2535            Some(CreditExhaustionReason::OrgLevelDisabled)
2536        );
2537        assert_eq!(
2538            detect_credit_exhaustion("credit balance too low"),
2539            Some(CreditExhaustionReason::OrgLevelDisabled)
2540        );
2541    }
2542
2543    #[serial(cooldown_state)]
2544    #[test]
2545    fn detect_credit_exhaustion_overage_disabled_reason_parsing() {
2546        let msg = "overageDisabledReason: out_of_credits";
2547        assert_eq!(
2548            detect_credit_exhaustion(msg),
2549            Some(CreditExhaustionReason::OutOfCredits)
2550        );
2551
2552        let msg = "overageDisabledReason: org_level_disabled";
2553        assert_eq!(
2554            detect_credit_exhaustion(msg),
2555            Some(CreditExhaustionReason::OrgLevelDisabled)
2556        );
2557    }
2558
2559    #[serial(cooldown_state)]
2560    #[test]
2561    fn detect_credit_exhaustion_glm_insufficient_balance() {
2562        // GLM/ZhipuAI 429: "Insufficient balance or no resource package. Please recharge."
2563        assert_eq!(
2564            detect_credit_exhaustion(
2565                "API Error: Request rejected (429) · Insufficient balance or no resource package. Please recharge."
2566            ),
2567            Some(CreditExhaustionReason::OutOfCredits)
2568        );
2569        assert_eq!(
2570            detect_credit_exhaustion("Insufficient balance"),
2571            Some(CreditExhaustionReason::OutOfCredits)
2572        );
2573        assert_eq!(
2574            detect_credit_exhaustion("please recharge your account"),
2575            Some(CreditExhaustionReason::OutOfCredits)
2576        );
2577        assert_eq!(
2578            detect_credit_exhaustion("no resource package available"),
2579            Some(CreditExhaustionReason::OutOfCredits)
2580        );
2581    }
2582
2583    #[serial(cooldown_state)]
2584    #[test]
2585    fn detect_credit_exhaustion_none_for_regular_rate_limit() {
2586        assert!(detect_credit_exhaustion("rate limit exceeded").is_none());
2587        assert!(detect_credit_exhaustion("too many requests").is_none());
2588        assert!(detect_credit_exhaustion("429 Too Many Requests").is_none());
2589        assert!(detect_credit_exhaustion("you've hit your usage limit").is_none());
2590        assert!(
2591            detect_credit_exhaustion("You've hit your usage limit, try again in a few minutes.")
2592                .is_none()
2593        );
2594    }
2595
2596    #[serial(cooldown_state)]
2597    #[tokio::test]
2598    async fn record_credit_exhaustion_applies_agent_cooldown() {
2599        super::reset_global_state().await;
2600        let agent = "test_credit_exhaust_agent";
2601        assert!(!is_agent_in_cooldown(agent));
2602
2603        record_credit_exhaustion(agent, CreditExhaustionReason::OutOfCredits).await;
2604        assert!(is_agent_in_cooldown(agent));
2605
2606        let remaining = {
2607            let map = cooldowns().lock().unwrap();
2608            let entry = map.get(agent).expect("cooldown entry should exist");
2609            entry.cooldown_until - chrono::Utc::now().timestamp()
2610        };
2611        // First failure: base = 1h
2612        assert!(
2613            remaining >= CREDIT_BACKOFF_BASE_SECS - 5,
2614            "first out_of_credits cooldown should be ~1 hour, got {remaining}s"
2615        );
2616        assert!(
2617            remaining <= CREDIT_BACKOFF_MAX_SECS,
2618            "cooldown should not exceed cap of 8h, got {remaining}s"
2619        );
2620    }
2621
2622    #[serial(cooldown_state)]
2623    #[tokio::test]
2624    async fn record_credit_exhaustion_org_level_applies_longer_cooldown() {
2625        super::reset_global_state().await;
2626        let agent = "test_org_disabled_agent";
2627        assert!(!is_agent_in_cooldown(agent));
2628
2629        record_credit_exhaustion(agent, CreditExhaustionReason::OrgLevelDisabled).await;
2630        assert!(is_agent_in_cooldown(agent));
2631
2632        let remaining = {
2633            let map = cooldowns().lock().unwrap();
2634            let entry = map.get(agent).expect("cooldown entry should exist");
2635            entry.cooldown_until - chrono::Utc::now().timestamp()
2636        };
2637        // First failure: base = 2h (ORG_BACKOFF_BASE_SECS)
2638        assert!(
2639            remaining >= ORG_BACKOFF_BASE_SECS - 5,
2640            "first org-level disabled cooldown should be ~2 hours, got {remaining}s"
2641        );
2642        assert!(
2643            remaining <= CREDIT_BACKOFF_MAX_SECS,
2644            "cooldown should not exceed cap of 8h, got {remaining}s"
2645        );
2646    }
2647
2648    #[serial(cooldown_state)]
2649    #[tokio::test]
2650    async fn record_credit_exhaustion_billing_cycle_applies_24h() {
2651        super::reset_global_state().await;
2652        let store = test_store().await;
2653        *cooldown_store().lock().await = Some(Arc::clone(&store));
2654
2655        let agent = "test_billing_cycle_agent";
2656        assert!(!is_agent_in_cooldown(agent));
2657
2658        record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2659        assert!(is_agent_in_cooldown(agent));
2660
2661        let remaining = {
2662            let map = cooldowns().lock().unwrap();
2663            let entry = map.get(agent).expect("cooldown entry should exist");
2664            entry.cooldown_until - chrono::Utc::now().timestamp()
2665        };
2666        // First failure: base = 24h
2667        assert!(
2668            remaining >= BILLING_CYCLE_COOLDOWN_SECS - 5,
2669            "first billing cycle cooldown should be ~24 hours, got {remaining}s"
2670        );
2671        assert!(
2672            remaining <= BILLING_CYCLE_MAX_SECS,
2673            "cooldown should not exceed cap of 7 days, got {remaining}s"
2674        );
2675    }
2676
2677    #[serial(cooldown_state)]
2678    #[tokio::test]
2679    async fn record_credit_exhaustion_billing_cycle_escalates() {
2680        super::reset_global_state().await;
2681        let store = test_store().await;
2682        *cooldown_store().lock().await = Some(Arc::clone(&store));
2683
2684        let agent = "test_billing_cycle_escalation";
2685
2686        // First failure: 24h
2687        record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2688        let remaining_1 = {
2689            let map = cooldowns().lock().unwrap();
2690            let entry = map.get(agent).expect("cooldown entry should exist");
2691            entry.cooldown_until - chrono::Utc::now().timestamp()
2692        };
2693        assert!(
2694            (BILLING_CYCLE_COOLDOWN_SECS - 5..=BILLING_CYCLE_COOLDOWN_SECS + 5)
2695                .contains(&remaining_1),
2696            "first billing cycle cooldown should be ~24h, got {remaining_1}s"
2697        );
2698
2699        // Clear in-memory cooldown (but NOT failure count) to allow next set_cooldown_async
2700        {
2701            let mut map = cooldowns().lock().unwrap();
2702            map.remove(agent);
2703        }
2704
2705        // Second failure: 24h * 3 = 72h (3 days)
2706        record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2707        let remaining_2 = {
2708            let map = cooldowns().lock().unwrap();
2709            let entry = map.get(agent).expect("cooldown entry should exist");
2710            entry.cooldown_until - chrono::Utc::now().timestamp()
2711        };
2712        let expected_2 = BILLING_CYCLE_COOLDOWN_SECS * 3; // 72h
2713        assert!(
2714            remaining_2 >= expected_2 - 5,
2715            "second billing cycle cooldown should be ~72h, got {remaining_2}s"
2716        );
2717
2718        // Clear in-memory cooldown again
2719        {
2720            let mut map = cooldowns().lock().unwrap();
2721            map.remove(agent);
2722        }
2723
2724        // Third failure: 24h * 9 = 216h → capped at 7 days (168h)
2725        record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2726        let remaining_3 = {
2727            let map = cooldowns().lock().unwrap();
2728            let entry = map.get(agent).expect("cooldown entry should exist");
2729            entry.cooldown_until - chrono::Utc::now().timestamp()
2730        };
2731        assert!(
2732            remaining_3 >= BILLING_CYCLE_MAX_SECS - 5,
2733            "third billing cycle cooldown should be capped at 7 days, got {remaining_3}s"
2734        );
2735        assert!(
2736            remaining_3 <= BILLING_CYCLE_MAX_SECS + 5,
2737            "third billing cycle cooldown should not exceed 7 days, got {remaining_3}s"
2738        );
2739    }
2740
2741    #[serial(cooldown_state)]
2742    #[tokio::test]
2743    async fn record_credit_exhaustion_billing_cycle_stays_capped_after_repeats() {
2744        super::reset_global_state().await;
2745        let store = test_store().await;
2746        *cooldown_store().lock().await = Some(Arc::clone(&store));
2747
2748        let agent = "test_billing_cycle_agent_capped";
2749
2750        for i in 0..5 {
2751            if i > 0 {
2752                // Each repeat is a new exhaustion after the previous cooldown ended
2753                cooldowns().lock().unwrap().remove(agent);
2754            }
2755            record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2756        }
2757
2758        let remaining = {
2759            let map = cooldowns().lock().unwrap();
2760            let entry = map.get(agent).expect("cooldown entry should exist");
2761            entry.cooldown_until - chrono::Utc::now().timestamp()
2762        };
2763        assert!(
2764            remaining >= BILLING_CYCLE_MAX_SECS - 5,
2765            "billing cycle cooldown should stay capped at 7 days, got {remaining}s"
2766        );
2767        assert!(
2768            remaining <= BILLING_CYCLE_MAX_SECS + 5,
2769            "billing cycle cooldown should not exceed 7 days, got {remaining}s"
2770        );
2771    }
2772
2773    #[serial(cooldown_state)]
2774    #[test]
2775    fn credit_exhaustion_backoff_constants() {
2776        // OutOfCredits first failure: 1h base
2777        assert_eq!(
2778            compute_backoff(1, CREDIT_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS, true),
2779            CREDIT_BACKOFF_BASE_SECS
2780        );
2781        // OrgLevelDisabled first failure: 2h base
2782        assert_eq!(
2783            compute_backoff(1, ORG_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS, true),
2784            ORG_BACKOFF_BASE_SECS
2785        );
2786        // BillingCycleExhausted: 24h base, 7d cap
2787        assert_eq!(BILLING_CYCLE_COOLDOWN_SECS, 24 * 60 * 60);
2788        assert_eq!(BILLING_CYCLE_MAX_SECS, 7 * 24 * 60 * 60);
2789    }
2790
2791    #[serial(cooldown_state)]
2792    #[test]
2793    fn compute_backoff_long_cap_without_extended_stays_capped() {
2794        assert_eq!(
2795            compute_backoff(
2796                3,
2797                BILLING_CYCLE_COOLDOWN_SECS,
2798                BILLING_CYCLE_MAX_SECS,
2799                false
2800            ),
2801            BILLING_CYCLE_MAX_SECS,
2802            "7-day capped paths should stop at the documented max"
2803        );
2804        assert_eq!(
2805            compute_backoff(
2806                9,
2807                PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2808                PERSISTENT_MODEL_BACKOFF_MAX_SECS,
2809                false,
2810            ),
2811            PERSISTENT_MODEL_BACKOFF_MAX_SECS,
2812            "persistent model failures should not extend past 7 days"
2813        );
2814    }
2815
2816    // ---- Degraded-agent tracking tests ----
2817
2818    #[serial(cooldown_state)]
2819    #[test]
2820    fn mark_and_check_agent_degraded() {
2821        let agent = "test_degraded_mark";
2822        assert!(!is_agent_degraded(agent));
2823        mark_agent_degraded(agent);
2824        assert!(is_agent_degraded(agent));
2825        clear_agent_degraded(agent);
2826        assert!(!is_agent_degraded(agent));
2827    }
2828
2829    #[serial(cooldown_state)]
2830    #[tokio::test]
2831    async fn sync_from_kv_drops_in_memory_entry_cleared_externally() {
2832        super::reset_global_state().await;
2833        let store = test_store().await;
2834        let agent = "test_sync_external_clear";
2835        let future = chrono::Utc::now().timestamp() + 600;
2836
2837        // Register the store and put a cooldown in both memory and KV.
2838        init_cooldown_store(Arc::clone(&store)).await;
2839        set_cooldown_async(agent, future, "test").await;
2840        assert!(is_agent_in_cooldown(agent));
2841
2842        // Simulate `orch cooldown clear --all` from a separate CLI process: it
2843        // can only write KV (to "0"); the running engine's in-memory map is
2844        // untouched.
2845        store
2846            .kv_set(&format!("{KV_PREFIX}{agent}"), "0")
2847            .await
2848            .expect("kv_set tombstone");
2849        assert!(
2850            is_agent_in_cooldown(agent),
2851            "in-memory cooldown should still be present before sync"
2852        );
2853
2854        sync_from_kv(&store).await;
2855
2856        assert!(
2857            !is_agent_in_cooldown(agent),
2858            "sync_from_kv must drop in-memory entries the KV tombstoned"
2859        );
2860    }
2861
2862    #[serial(cooldown_state)]
2863    #[tokio::test]
2864    async fn sync_from_kv_closes_github_circuit_when_row_cleared() {
2865        super::reset_global_state().await;
2866        let store = test_store().await;
2867        {
2868            *github_5xx_circuit_open_lock() = Some(chrono::Utc::now().timestamp() + 600);
2869            github_5xx_timestamps_lock().push(chrono::Utc::now().timestamp());
2870        }
2871        assert!(is_github_circuit_open());
2872
2873        store
2874            .kv_set("cooldown:github:5xx", "0")
2875            .await
2876            .expect("kv_set tombstone");
2877        sync_from_kv(&store).await;
2878
2879        assert!(!is_github_circuit_open());
2880        assert!(github_5xx_timestamps_lock().is_empty());
2881    }
2882
2883    #[serial(cooldown_state)]
2884    #[tokio::test]
2885    async fn sync_from_kv_keeps_5xx_window_when_expired_row_and_breaker_closed() {
2886        super::reset_global_state().await;
2887        let store = test_store().await;
2888        let now = chrono::Utc::now().timestamp();
2889        {
2890            github_5xx_timestamps_lock().push(now);
2891            github_5xx_timestamps_lock().push(now);
2892        }
2893
2894        store
2895            .kv_set("cooldown:github:5xx", &(now - 600).to_string())
2896            .await
2897            .expect("kv_set past");
2898        sync_from_kv(&store).await;
2899
2900        assert!(!is_github_circuit_open());
2901        assert_eq!(github_5xx_timestamps_lock().len(), 2);
2902
2903        github_5xx_timestamps_lock().clear();
2904    }
2905
2906    #[serial(cooldown_state)]
2907    #[tokio::test]
2908    async fn sync_from_kv_keeps_github_circuit_open_for_future_row() {
2909        super::reset_global_state().await;
2910        let store = test_store().await;
2911        let future = chrono::Utc::now().timestamp() + 600;
2912        *github_5xx_circuit_open_lock() = Some(future);
2913
2914        store
2915            .kv_set("cooldown:github:5xx", &future.to_string())
2916            .await
2917            .expect("kv_set future");
2918        sync_from_kv(&store).await;
2919
2920        assert!(is_github_circuit_open());
2921
2922        *github_5xx_circuit_open_lock() = None;
2923    }
2924
2925    #[serial(cooldown_state)]
2926    #[tokio::test]
2927    async fn sync_from_kv_keeps_in_memory_entry_when_kv_row_missing() {
2928        // Race window: set_cooldown_async writes memory first, then KV. If a
2929        // sync runs between those two writes, the KV row is missing — we must
2930        // leave the in-memory entry alone, not drop it.
2931        super::reset_global_state().await;
2932        let store = test_store().await;
2933        let agent = "test_sync_race_window";
2934        let future = chrono::Utc::now().timestamp() + 600;
2935
2936        // Simulate the race: memory has the entry, KV does not.
2937        set_cooldown_in_memory(agent, future, "raced");
2938        assert!(is_agent_in_cooldown(agent));
2939
2940        sync_from_kv(&store).await;
2941
2942        assert!(
2943            is_agent_in_cooldown(agent),
2944            "sync_from_kv must NOT drop entries whose KV row is missing — that is the set_cooldown_async race window"
2945        );
2946    }
2947
2948    #[serial(cooldown_state)]
2949    #[tokio::test]
2950    async fn sync_from_kv_adopts_kv_timestamp() {
2951        // KV is the source of truth: if KV has a different active timestamp,
2952        // memory adopts it (covers external manual KV edits).
2953        super::reset_global_state().await;
2954        let store = test_store().await;
2955        let agent = "test_sync_adopt_kv";
2956        let original = chrono::Utc::now().timestamp() + 600;
2957        let updated = chrono::Utc::now().timestamp() + 1200;
2958
2959        init_cooldown_store(Arc::clone(&store)).await;
2960        set_cooldown_async(agent, original, "initial").await;
2961
2962        store
2963            .kv_set(&format!("{KV_PREFIX}{agent}"), &updated.to_string())
2964            .await
2965            .expect("kv_set updated");
2966
2967        sync_from_kv(&store).await;
2968
2969        let until = cooldown_until(agent).expect("cooldown still active");
2970        assert_eq!(
2971            until, updated,
2972            "sync_from_kv must adopt KV's authoritative timestamp"
2973        );
2974    }
2975
2976    #[serial(cooldown_state)]
2977    #[tokio::test]
2978    async fn refresh_degraded_agents_marks_cooled_agent() {
2979        super::reset_global_state().await;
2980        let store = test_store().await;
2981        let agent = "test_refresh_cooled";
2982
2983        // Put agent in cooldown
2984        record_agent_failure_with_message(agent, "").await;
2985        assert!(is_agent_in_cooldown(agent));
2986
2987        // model_checker always returns true (agent has models)
2988        let agents = vec![agent.to_string()];
2989        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
2990
2991        assert!(
2992            is_agent_degraded(agent),
2993            "agent in cooldown should be marked degraded"
2994        );
2995
2996        // Cleanup
2997        clear_agent_degraded(agent);
2998    }
2999
3000    #[serial(cooldown_state)]
3001    #[tokio::test]
3002    async fn refresh_degraded_agents_marks_no_models_agent() {
3003        super::reset_global_state().await;
3004        let store = test_store().await;
3005        let agent = "test_refresh_no_models";
3006
3007        // model_checker returns false (no available models)
3008        let agents = vec![agent.to_string()];
3009        refresh_degraded_agents(&store, &agents, &|_| false, 6, 3).await;
3010
3011        assert!(
3012            is_agent_degraded(agent),
3013            "agent with no available models should be marked degraded"
3014        );
3015
3016        // Cleanup
3017        clear_agent_degraded(agent);
3018    }
3019
3020    #[serial(cooldown_state)]
3021    #[tokio::test]
3022    async fn refresh_degraded_agents_marks_rate_limited_agent() {
3023        super::reset_global_state().await;
3024        let store = test_store().await;
3025        let agent = "test_refresh_rate_limited";
3026
3027        // Insert rate limit events exceeding threshold
3028        for _ in 0..4 {
3029            store.record_rate_limit(agent, "rate", None).await.unwrap();
3030        }
3031
3032        let agents = vec![agent.to_string()];
3033        // threshold=3, window=6h — 4 events should trigger degraded
3034        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3035
3036        assert!(
3037            is_agent_degraded(agent),
3038            "agent with rate limit count >= threshold should be marked degraded"
3039        );
3040
3041        // Cleanup
3042        clear_agent_degraded(agent);
3043    }
3044
3045    #[serial(cooldown_state)]
3046    #[tokio::test]
3047    async fn refresh_degraded_agents_clears_healthy_agent() {
3048        super::reset_global_state().await;
3049        let store = test_store().await;
3050        let agent = "test_refresh_healthy";
3051
3052        // Pre-mark as degraded
3053        mark_agent_degraded(agent);
3054        assert!(is_agent_degraded(agent));
3055
3056        // No cooldown, has models, no rate limit events
3057        let agents = vec![agent.to_string()];
3058        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3059
3060        assert!(
3061            !is_agent_degraded(agent),
3062            "healthy agent should have degraded flag cleared"
3063        );
3064    }
3065
3066    #[serial(cooldown_state)]
3067    #[tokio::test]
3068    async fn refresh_degraded_agents_ignores_expired_cooldowns() {
3069        super::reset_global_state().await;
3070        let store = test_store().await;
3071        let agent = "test_refresh_expired_cooldown";
3072
3073        // Manually insert an already-expired cooldown entry directly into the map
3074        // (simulating a stale entry that wasn't cleaned up)
3075        let expired_ts = chrono::Utc::now().timestamp() - 100; // 100 seconds ago
3076        {
3077            let mut map = cooldowns().lock().unwrap();
3078            map.insert(
3079                agent.to_string(),
3080                CooldownEntry {
3081                    cooldown_until: expired_ts,
3082                    reason: "expired_test".to_string(),
3083                },
3084            );
3085        }
3086
3087        // Verify the expired entry is in the map but should not cause degraded status
3088        assert!(
3089            !is_agent_in_cooldown(agent),
3090            "expired cooldown should not be active"
3091        );
3092
3093        // Pre-mark as degraded to verify it gets cleared
3094        mark_agent_degraded(agent);
3095        assert!(is_agent_degraded(agent));
3096
3097        // Run refresh — should clear degraded because cooldown is expired
3098        let agents = vec![agent.to_string()];
3099        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3100
3101        assert!(
3102            !is_agent_degraded(agent),
3103            "agent with only expired cooldown should NOT be marked degraded"
3104        );
3105
3106        // Cleanup
3107        clear_agent_degraded(agent);
3108        {
3109            let mut map = cooldowns().lock().unwrap();
3110            map.remove(agent);
3111        }
3112    }
3113
3114    #[serial(cooldown_state)]
3115    #[test]
3116    fn compute_backoff_grows_exponentially() {
3117        // count=0 or 1: base
3118        assert_eq!(compute_backoff(0, 300, 14400, true), 300);
3119        assert_eq!(compute_backoff(1, 300, 14400, true), 300);
3120        // count=2: base * 3
3121        assert_eq!(compute_backoff(2, 300, 14400, true), 900);
3122        // count=3: base * 9
3123        assert_eq!(compute_backoff(3, 300, 14400, true), 2700);
3124        // count=4: base * 27 = 8100 < 14400
3125        assert_eq!(compute_backoff(4, 300, 14400, true), 8100);
3126        // count=5: base * 81 = 24300 → capped at 14400
3127        assert_eq!(compute_backoff(5, 300, 14400, true), 14400);
3128    }
3129
3130    #[serial(cooldown_state)]
3131    #[test]
3132    fn compute_backoff_extended_tier_1_applies_24h() {
3133        // With base=300, max=14400:
3134        // count=4: 8100s (not yet at max)
3135        // count=5: 14400s (first time at max, times_at_max=0, no extension)
3136        // count=6: 14400s (times_at_max=1, no extension yet, need >=2)
3137        // count=7: 86400s (times_at_max=2, extended tier 1: 6x = 24h)
3138        assert_eq!(
3139            compute_backoff(5, 300, 14400, true),
3140            14400,
3141            "first at max, not extended"
3142        );
3143        assert_eq!(
3144            compute_backoff(6, 300, 14400, true),
3145            14400,
3146            "times_at_max=1, not extended"
3147        );
3148        assert_eq!(
3149            compute_backoff(7, 300, 14400, true),
3150            86400,
3151            "times_at_max=2, extended 6x"
3152        );
3153        assert_eq!(
3154            compute_backoff(8, 300, 14400, true),
3155            86400,
3156            "times_at_max=3, still extended 6x"
3157        );
3158    }
3159
3160    #[serial(cooldown_state)]
3161    #[test]
3162    fn compute_backoff_extended_tier_2_applies_48h() {
3163        // With base=300, max=14400:
3164        // count=10: times_at_max=5 (first_max=5, count=10), extended tier 2: 12x = 48h
3165        assert_eq!(
3166            compute_backoff(9, 300, 14400, true),
3167            86400,
3168            "times_at_max=4, still tier 1"
3169        );
3170        assert_eq!(
3171            compute_backoff(10, 300, 14400, true),
3172            172_800,
3173            "times_at_max=5, extended 12x"
3174        );
3175        assert_eq!(
3176            compute_backoff(15, 300, 14400, true),
3177            172_800,
3178            "times_at_max=10, still tier 2"
3179        );
3180        assert_eq!(
3181            compute_backoff(20, 300, 14400, true),
3182            172_800,
3183            "times_at_max=15, still tier 2"
3184        );
3185    }
3186
3187    #[serial(cooldown_state)]
3188    #[test]
3189    fn compute_backoff_extended_tier_with_different_base_max() {
3190        // With base=60, max=3600:
3191        // count=3: 60*9=540 < 3600
3192        // count=4: 60*27=1620 < 3600
3193        // count=5: 60*81=4860 → capped at 3600
3194        // count=6: times_at_max=1, still 3600
3195        // count=7: times_at_max=2, extended 6x = 21600 (6h)
3196        assert_eq!(compute_backoff(5, 60, 3600, true), 3600, "first at max");
3197        assert_eq!(compute_backoff(6, 60, 3600, true), 3600, "times_at_max=1");
3198        assert_eq!(
3199            compute_backoff(7, 60, 3600, true),
3200            21600,
3201            "times_at_max=2, extended 6x"
3202        );
3203    }
3204
3205    #[serial(cooldown_state)]
3206    #[test]
3207    fn compute_backoff_extended_tier_persistent_fails_example() {
3208        // Real-world example from issue: glm:haiku with 21 failures
3209        // With base=300, max=14400:
3210        // count=21 → times_at_max=16 (first_max=5), extended tier 2: 12x = 48h
3211        assert_eq!(
3212            compute_backoff(21, 300, 14400, true),
3213            172_800,
3214            "21 failures should get 48h extended cooldown"
3215        );
3216
3217        // kimi:haiku with 36 failures
3218        assert_eq!(
3219            compute_backoff(36, 300, 14400, true),
3220            172_800,
3221            "36 failures should also get 48h extended cooldown"
3222        );
3223    }
3224
3225    #[serial(cooldown_state)]
3226    #[test]
3227    fn compute_backoff_extended_tier_zero_base() {
3228        // Zero base should still work without panicking
3229        assert_eq!(compute_backoff(1, 0, 14400, true), 0);
3230        assert_eq!(compute_backoff(5, 0, 14400, true), 0);
3231        assert_eq!(compute_backoff(10, 0, 14400, true), 0);
3232    }
3233
3234    #[serial(cooldown_state)]
3235    #[test]
3236    fn detect_credit_exhaustion_billing_cycle() {
3237        assert_eq!(
3238            detect_credit_exhaustion("billing cycle"),
3239            Some(CreditExhaustionReason::BillingCycleExhausted)
3240        );
3241        assert_eq!(
3242            detect_credit_exhaustion("quota refreshed next cycle"),
3243            Some(CreditExhaustionReason::BillingCycleExhausted)
3244        );
3245        assert_eq!(
3246            detect_credit_exhaustion(
3247                "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle."
3248            ),
3249            Some(CreditExhaustionReason::BillingCycleExhausted)
3250        );
3251        assert_eq!(
3252            detect_credit_exhaustion("monthly quota exceeded"),
3253            Some(CreditExhaustionReason::BillingCycleExhausted)
3254        );
3255        assert_eq!(
3256            detect_credit_exhaustion(
3257                "Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-04-23 18:33:48"
3258            ),
3259            Some(CreditExhaustionReason::BillingCycleExhausted)
3260        );
3261    }
3262
3263    #[serial(cooldown_state)]
3264    #[test]
3265    fn detect_credit_exhaustion_billing_cycle_minimax() {
3266        assert_eq!(
3267            detect_credit_exhaustion(
3268                "minimax rate limit: API Error: Request rejected (429) · Token Plan usage limit reached: Upgrade your Token Plan or purchase Credits for more usage."
3269            ),
3270            Some(CreditExhaustionReason::BillingCycleExhausted)
3271        );
3272        assert_eq!(
3273            detect_credit_exhaustion("Upgrade your Token Plan"),
3274            Some(CreditExhaustionReason::BillingCycleExhausted)
3275        );
3276        assert_eq!(
3277            detect_credit_exhaustion("Token Plan usage limit reached"),
3278            Some(CreditExhaustionReason::BillingCycleExhausted)
3279        );
3280        assert_eq!(
3281            detect_credit_exhaustion("purchase Credits for more usage"),
3282            Some(CreditExhaustionReason::BillingCycleExhausted)
3283        );
3284        assert_eq!(
3285            detect_credit_exhaustion(
3286                "You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro), visit https://chatgpt.com/codex/settings/usage to purchase more credits"
3287            ),
3288            Some(CreditExhaustionReason::BillingCycleExhausted)
3289        );
3290    }
3291
3292    #[serial(cooldown_state)]
3293    #[test]
3294    fn detect_credit_exhaustion_outcome_kimi_billing_cycle() {
3295        // Regression test for issue #3529: the exact Kimi billing-cycle exhaustion
3296        // message from task_runs (tasks 156734/156678) must resolve to the
3297        // "billing_cycle_exhausted" outcome string, not fall through to a generic
3298        // rate-limit/auth classification.
3299        let msg = "Failed to authenticate. API Error: 403 You've reached your usage limit for \
3300                    this billing cycle. Your quota will be refreshed soon.";
3301        assert_eq!(
3302            detect_credit_exhaustion_outcome(msg),
3303            Some("billing_cycle_exhausted")
3304        );
3305    }
3306
3307    #[serial(cooldown_state)]
3308    #[test]
3309    fn detect_credit_exhaustion_outcome_maps_all_reasons() {
3310        assert_eq!(
3311            detect_credit_exhaustion_outcome("out of credits"),
3312            Some("credit_exhausted")
3313        );
3314        assert_eq!(
3315            detect_credit_exhaustion_outcome("organization has been disabled"),
3316            Some("org_level_disabled")
3317        );
3318        assert_eq!(
3319            detect_credit_exhaustion_outcome("monthly quota exceeded"),
3320            Some("billing_cycle_exhausted")
3321        );
3322    }
3323
3324    #[test]
3325    fn detect_credit_exhaustion_outcome_returns_none_for_unrelated_errors() {
3326        assert_eq!(
3327            detect_credit_exhaustion_outcome("429 Too Many Requests"),
3328            None
3329        );
3330        assert_eq!(detect_credit_exhaustion_outcome("connection reset"), None);
3331    }
3332
3333    #[serial(cooldown_state)]
3334    #[test]
3335    fn detect_credit_exhaustion_overage_takes_priority_over_billing_cycle() {
3336        // A message containing both overage and billing cycle language should resolve
3337        // to OrgLevelDisabled/OutOfCredits (detected first) not BillingCycleExhausted.
3338        let msg = "overageDisabledReason: org_level_disabled for billing cycle";
3339        assert_eq!(
3340            detect_credit_exhaustion(msg),
3341            Some(CreditExhaustionReason::OrgLevelDisabled)
3342        );
3343    }
3344
3345    #[serial(cooldown_state)]
3346    #[tokio::test]
3347    async fn refresh_degraded_agents_below_threshold_not_degraded() {
3348        super::reset_global_state().await;
3349        let store = test_store().await;
3350        let agent = "test_refresh_below_threshold";
3351
3352        // Insert rate limit events below threshold
3353        for _ in 0..2 {
3354            store.record_rate_limit(agent, "rate", None).await.unwrap();
3355        }
3356
3357        let agents = vec![agent.to_string()];
3358        // threshold=3, only 2 events — should NOT be degraded
3359        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3360
3361        assert!(
3362            !is_agent_degraded(agent),
3363            "agent with rate limit count < threshold should not be degraded"
3364        );
3365    }
3366
3367    #[serial(cooldown_state)]
3368    #[tokio::test]
3369    async fn refresh_degraded_agents_ignores_error_limit_type() {
3370        super::reset_global_state().await;
3371        let store = test_store().await;
3372        let agent = "test_refresh_error_rows";
3373
3374        for _ in 0..3 {
3375            store.record_rate_limit(agent, "error", None).await.unwrap();
3376        }
3377
3378        let agents = vec![agent.to_string()];
3379        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3380
3381        assert!(
3382            !is_agent_degraded(agent),
3383            "three error rows in 6 hours must not mark an agent degraded"
3384        );
3385    }
3386
3387    #[serial(cooldown_state)]
3388    #[tokio::test]
3389    async fn refresh_degraded_agents_marks_three_rate_rows() {
3390        super::reset_global_state().await;
3391        let store = test_store().await;
3392        let agent = "test_refresh_three_rate_rows";
3393
3394        for _ in 0..3 {
3395            store.record_rate_limit(agent, "rate", None).await.unwrap();
3396        }
3397
3398        let agents = vec![agent.to_string()];
3399        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3400
3401        assert!(
3402            is_agent_degraded(agent),
3403            "three rate rows in 6 hours must mark an agent degraded"
3404        );
3405        assert_eq!(
3406            agent_degraded_reason(agent).as_deref(),
3407            Some("rate limit threshold exceeded")
3408        );
3409        let listed = list_degraded_agents();
3410        assert!(
3411            listed
3412                .iter()
3413                .any(|(a, r)| a == agent && r == "rate limit threshold exceeded"),
3414            "orch cooldown list must show the degraded reason, got {listed:?}"
3415        );
3416
3417        clear_agent_degraded(agent);
3418    }
3419
3420    #[serial(cooldown_state)]
3421    #[tokio::test]
3422    async fn refresh_degraded_agents_logs_reason_change() {
3423        super::reset_global_state().await;
3424        let store = test_store().await;
3425        let agent = "test_refresh_reason_change";
3426        *cooldown_store().lock().await = Some(Arc::clone(&store));
3427
3428        set_agent_cooldown(agent, 600).await;
3429        let agents = vec![agent.to_string()];
3430        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3431        assert_eq!(
3432            agent_degraded_reason(agent).as_deref(),
3433            Some("agent in cooldown")
3434        );
3435
3436        clear_cooldown(agent, &store).await;
3437        let after_clear = sqlite_datetime_offset(60);
3438        for _ in 0..3 {
3439            store
3440                .record_rate_limit_at(agent, "rate", None, &after_clear)
3441                .await
3442                .unwrap();
3443        }
3444        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3445        assert_eq!(
3446            agent_degraded_reason(agent).as_deref(),
3447            Some("rate limit threshold exceeded"),
3448            "degraded reason must update when the hold changes"
3449        );
3450
3451        clear_agent_degraded(agent);
3452    }
3453
3454    #[serial(cooldown_state)]
3455    #[tokio::test]
3456    async fn clear_un_degrades_on_next_health_check() {
3457        super::reset_global_state().await;
3458        let store = test_store().await;
3459        let agent = "test_clear_un_degrades";
3460        let before = sqlite_datetime_offset(-30);
3461        let after = sqlite_datetime_offset(60);
3462
3463        for _ in 0..3 {
3464            store
3465                .record_rate_limit_at(agent, "rate", None, &before)
3466                .await
3467                .unwrap();
3468        }
3469
3470        let agents = vec![agent.to_string()];
3471        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3472        assert!(
3473            is_agent_degraded(agent),
3474            "three rate rows must mark an agent degraded"
3475        );
3476
3477        clear_cooldown(agent, &store).await;
3478        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3479        assert!(
3480            !is_agent_degraded(agent),
3481            "clear must un-degrade on the next health check"
3482        );
3483
3484        store
3485            .record_rate_limit_at(agent, "rate", None, &after)
3486            .await
3487            .unwrap();
3488        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3489        assert!(
3490            !is_agent_degraded(agent),
3491            "one rate row after the mark must not degrade again"
3492        );
3493
3494        for _ in 0..2 {
3495            store
3496                .record_rate_limit_at(agent, "rate", None, &after)
3497                .await
3498                .unwrap();
3499        }
3500        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3501        assert!(
3502            is_agent_degraded(agent),
3503            "three rate rows after the mark must degrade again"
3504        );
3505
3506        clear_agent_degraded(agent);
3507    }
3508
3509    #[serial(cooldown_state)]
3510    #[tokio::test]
3511    async fn clear_all_un_degrades_rate_limited_agent() {
3512        super::reset_global_state().await;
3513        let store = test_store().await;
3514        let agent = "test_clear_all_un_degrades";
3515        let before = sqlite_datetime_offset(-30);
3516
3517        for _ in 0..3 {
3518            store
3519                .record_rate_limit_at(agent, "rate", None, &before)
3520                .await
3521                .unwrap();
3522        }
3523
3524        let agents = vec![agent.to_string()];
3525        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3526        assert!(is_agent_degraded(agent));
3527
3528        clear_cooldown("*", &store).await;
3529        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3530        assert!(
3531            !is_agent_degraded(agent),
3532            "clear --all must un-degrade an agent held only by rate rows"
3533        );
3534    }
3535
3536    // ---- Exponential backoff integration tests ----
3537
3538    #[serial(cooldown_state)]
3539    #[tokio::test]
3540    async fn repeated_agent_failures_escalate_backoff() {
3541        super::reset_global_state().await;
3542        let store = test_store().await;
3543        *cooldown_store().lock().await = Some(Arc::clone(&store));
3544
3545        let agent = "test_escalation_agent";
3546
3547        // First failure: should be ~5 min (BACKOFF_BASE_SECS)
3548        record_agent_failure_with_message(agent, "").await;
3549        let remaining_1 = {
3550            let map = cooldowns().lock().unwrap();
3551            let entry = map.get(agent).expect("should have cooldown");
3552            entry.cooldown_until - chrono::Utc::now().timestamp()
3553        };
3554        assert!(
3555            (BACKOFF_BASE_SECS - 5..=BACKOFF_BASE_SECS + 5).contains(&remaining_1),
3556            "first failure should be ~5 min, got {remaining_1}s"
3557        );
3558
3559        // Clear the cooldown (but NOT failure count) to allow the next set_cooldown_async to apply
3560        {
3561            let mut map = cooldowns().lock().unwrap();
3562            map.remove(agent);
3563        }
3564
3565        // Second failure: should be ~15 min (5 * 3)
3566        record_agent_failure_with_message(agent, "").await;
3567        let remaining_2 = {
3568            let map = cooldowns().lock().unwrap();
3569            let entry = map.get(agent).expect("should have cooldown");
3570            entry.cooldown_until - chrono::Utc::now().timestamp()
3571        };
3572        assert!(
3573            remaining_2 >= BACKOFF_BASE_SECS * 3 - 5,
3574            "second failure should be ~15 min, got {remaining_2}s"
3575        );
3576    }
3577
3578    #[serial(cooldown_state)]
3579    #[tokio::test]
3580    async fn vendor_past_retry_falls_back_to_backoff() {
3581        super::reset_global_state().await;
3582        // If a vendor-provided "try again at" date is in the past, the
3583        // code must NOT early-return and must instead apply the normal
3584        // exponential backoff (increment failure count and set a future cooldown).
3585        let store = test_store().await;
3586        *cooldown_store().lock().await = Some(Arc::clone(&store));
3587
3588        let agent = "test_vendor_past_retry";
3589
3590        // Ensure no pre-existing cooldown
3591        {
3592            let mut map = cooldowns().lock().unwrap();
3593            map.remove(agent);
3594        }
3595
3596        // A clearly past date that parse_retry_at can parse
3597        let past_msg = "You've hit the limit; try again at Jan 01, 2000 01:00 AM.";
3598
3599        record_agent_failure_with_message(agent, past_msg).await;
3600
3601        // After handling a past vendor timestamp, we should have applied
3602        // exponential backoff (base = BACKOFF_BASE_SECS) and a persisted
3603        // failure count of 1.
3604        assert!(
3605            is_agent_in_cooldown(agent),
3606            "agent should be in cooldown after failure"
3607        );
3608
3609        let remaining = {
3610            let map = cooldowns().lock().unwrap();
3611            let entry = map.get(agent).expect("cooldown entry should exist");
3612            entry.cooldown_until - chrono::Utc::now().timestamp()
3613        };
3614
3615        assert!(
3616            remaining >= BACKOFF_BASE_SECS - 5,
3617            "expected backoff of ~{BACKOFF_BASE_SECS}s when vendor timestamp is past, got {remaining}s"
3618        );
3619
3620        // Verify the failure count was incremented and persisted to KV
3621        let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3622        let stored = store
3623            .kv_get(&kv_key)
3624            .await
3625            .unwrap()
3626            .expect("failure count should be stored");
3627        assert_eq!(stored, "1");
3628    }
3629
3630    fn window(status: &str, resets_at: Option<i64>) -> RateLimitWindow {
3631        RateLimitWindow {
3632            status: status.to_string(),
3633            window_type: "five_hour".to_string(),
3634            utilization: Some(1.0),
3635            resets_at,
3636        }
3637    }
3638
3639    #[serial(cooldown_state)]
3640    #[tokio::test]
3641    async fn rejected_window_cools_agent_until_reset() {
3642        super::reset_global_state().await;
3643        let store = test_store().await;
3644        *cooldown_store().lock().await = Some(Arc::clone(&store));
3645
3646        let agent = "test_window_rejected_agent";
3647        let reset = chrono::Utc::now().timestamp() + 7200;
3648        record_rate_limit_window(agent, &window("rejected", Some(reset))).await;
3649
3650        assert_eq!(cooldown_until(agent), Some(reset));
3651        let stored = list_rate_limit_windows(&store).await;
3652        assert_eq!(
3653            stored,
3654            vec![(agent.to_string(), window("rejected", Some(reset)))]
3655        );
3656    }
3657
3658    #[serial(cooldown_state)]
3659    #[tokio::test]
3660    async fn allowed_window_changes_no_cooldown() {
3661        super::reset_global_state().await;
3662        let store = test_store().await;
3663        *cooldown_store().lock().await = Some(Arc::clone(&store));
3664
3665        let agent = "test_window_allowed_agent";
3666        let reset = chrono::Utc::now().timestamp() + 7200;
3667        record_rate_limit_window(agent, &window("allowed", Some(reset))).await;
3668        record_rate_limit_window(agent, &window("allowed_warning", Some(reset))).await;
3669
3670        assert_eq!(cooldown_until(agent), None);
3671        assert_eq!(list_rate_limit_windows(&store).await.len(), 1);
3672    }
3673
3674    #[serial(cooldown_state)]
3675    #[tokio::test]
3676    async fn record_agent_success_resets_backoff() {
3677        super::reset_global_state().await;
3678        let store = test_store().await;
3679        *cooldown_store().lock().await = Some(Arc::clone(&store));
3680
3681        let agent = "test_success_reset_agent";
3682        let model = "test_model";
3683
3684        // Accumulate 3 failures to escalate the backoff
3685        for _ in 0..3 {
3686            let _ = read_and_increment_failure_count(Some(&store), agent).await;
3687        }
3688        // Verify count is 3
3689        let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3690        let count: u32 = store
3691            .kv_get(&kv_key)
3692            .await
3693            .unwrap()
3694            .unwrap()
3695            .parse()
3696            .unwrap();
3697        assert_eq!(count, 3);
3698
3699        // Success should reset to 0
3700        record_agent_success(agent, model).await;
3701
3702        let count_after: u32 = store
3703            .kv_get(&kv_key)
3704            .await
3705            .unwrap()
3706            .unwrap()
3707            .parse()
3708            .unwrap();
3709        assert_eq!(count_after, 0, "success should reset failure count to 0");
3710
3711        // Model-specific count should also be reset
3712        let model_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}:{model}");
3713        let model_count: u32 = store
3714            .kv_get(&model_kv_key)
3715            .await
3716            .unwrap()
3717            .unwrap()
3718            .parse()
3719            .unwrap();
3720        assert_eq!(
3721            model_count, 0,
3722            "success should reset model failure count to 0"
3723        );
3724    }
3725
3726    #[serial(cooldown_state)]
3727    #[tokio::test]
3728    async fn record_agent_success_without_model_skips_empty_model_key() {
3729        super::reset_global_state().await;
3730        let store = test_store().await;
3731        *cooldown_store().lock().await = Some(Arc::clone(&store));
3732
3733        let agent = "test_success_without_model";
3734
3735        record_agent_success(agent, "").await;
3736
3737        let agent_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3738        let agent_count = store
3739            .kv_get(&agent_kv_key)
3740            .await
3741            .unwrap()
3742            .expect("agent failure count should be stored");
3743        assert_eq!(agent_count, "0");
3744
3745        let empty_model_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}:");
3746        let empty_model_count = store.kv_get(&empty_model_kv_key).await.unwrap();
3747        assert!(
3748            empty_model_count.is_none(),
3749            "success without model should not create an empty model failure count key"
3750        );
3751    }
3752
3753    #[serial(cooldown_state)]
3754    #[tokio::test]
3755    async fn clear_cooldown_resets_failure_counts() {
3756        super::reset_global_state().await;
3757        let store = test_store().await;
3758        *cooldown_store().lock().await = Some(Arc::clone(&store));
3759
3760        let agent = "test_clear_fc_agent";
3761
3762        // Accumulate failures
3763        for _ in 0..5 {
3764            let _ = read_and_increment_failure_count(Some(&store), agent).await;
3765        }
3766        // Set a cooldown so clear_cooldown has something to clear
3767        set_agent_cooldown(agent, 3600).await;
3768
3769        // Verify failure count is 5
3770        let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3771        let count: u32 = store
3772            .kv_get(&kv_key)
3773            .await
3774            .unwrap()
3775            .unwrap()
3776            .parse()
3777            .unwrap();
3778        assert_eq!(count, 5);
3779
3780        // Clear the cooldown
3781        clear_cooldown(agent, &store).await;
3782
3783        // Failure count should be reset to 0
3784        let count_after: u32 = store
3785            .kv_get(&kv_key)
3786            .await
3787            .unwrap()
3788            .unwrap()
3789            .parse()
3790            .unwrap();
3791        assert_eq!(
3792            count_after, 0,
3793            "clear_cooldown should reset failure count to 0"
3794        );
3795
3796        // Cooldown should be removed
3797        assert!(
3798            !is_agent_in_cooldown(agent),
3799            "agent should not be in cooldown after clear"
3800        );
3801    }
3802
3803    #[serial(cooldown_state)]
3804    #[tokio::test]
3805    async fn clear_all_cooldowns_resets_all_failure_counts() {
3806        super::reset_global_state().await;
3807        let store = test_store().await;
3808        *cooldown_store().lock().await = Some(Arc::clone(&store));
3809
3810        let agents = ["test_clear_all_a", "test_clear_all_b"];
3811        for agent in &agents {
3812            let _ = read_and_increment_failure_count(Some(&store), agent).await;
3813            set_agent_cooldown(agent, 3600).await;
3814        }
3815
3816        clear_cooldown("*", &store).await;
3817
3818        for agent in &agents {
3819            assert!(!is_agent_in_cooldown(agent));
3820            let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3821            let count: u32 = store
3822                .kv_get(&kv_key)
3823                .await
3824                .unwrap()
3825                .unwrap()
3826                .parse()
3827                .unwrap();
3828            assert_eq!(
3829                count, 0,
3830                "clear --all should reset failure count for {agent}"
3831            );
3832        }
3833    }
3834
3835    #[serial(cooldown_state)]
3836    #[test]
3837    fn compute_backoff_saturates_on_large_counts() {
3838        // Very large count should not overflow — extended tier applies 48h
3839        assert_eq!(
3840            compute_backoff(100, BACKOFF_BASE_SECS, BACKOFF_MAX_SECS, true),
3841            BACKOFF_MAX_SECS * 12
3842        );
3843        assert_eq!(
3844            compute_backoff(u64::from(u32::MAX), 300, 14400, true),
3845            172_800
3846        );
3847        assert_eq!(compute_backoff(u64::MAX, 300, 14400, true), 172_800);
3848    }
3849
3850    #[serial(cooldown_state)]
3851    #[test]
3852    fn compute_backoff_zero_base_returns_zero() {
3853        assert_eq!(compute_backoff(1, 0, 14400, true), 0);
3854        assert_eq!(compute_backoff(5, 0, 14400, true), 0);
3855    }
3856
3857    // ---- GitHub 5xx circuit breaker tests ----
3858
3859    #[serial(cooldown_state)]
3860    #[tokio::test]
3861    async fn github_circuit_stays_closed_below_threshold() {
3862        super::reset_global_state().await;
3863        // Reset circuit breaker state
3864        {
3865            let mut open = github_5xx_circuit_open().lock().unwrap();
3866            *open = None;
3867            let mut ts = github_5xx_timestamps().lock().unwrap();
3868            ts.clear();
3869            // Also clear any generic cooldown map entry for the github 5xx
3870            // circuit so tests are isolated from other tests that may have
3871            // set the generic cooldown via set_agent_cooldown("github:5xx", ...).
3872            let mut map = cooldown_lock();
3873            map.remove("github:5xx");
3874        }
3875
3876        // Record fewer errors than threshold
3877        for _ in 0..GITHUB_5XX_THRESHOLD - 1 {
3878            record_github_5xx().await;
3879        }
3880
3881        assert!(
3882            !is_github_circuit_open(),
3883            "circuit should stay closed below threshold"
3884        );
3885        assert_eq!(github_circuit_remaining_secs(), 0);
3886    }
3887
3888    #[serial(cooldown_state)]
3889    #[tokio::test]
3890    async fn github_circuit_trips_at_threshold() {
3891        super::reset_global_state().await;
3892        // Reset circuit breaker state
3893        {
3894            let mut open = github_5xx_circuit_open_lock();
3895            *open = None;
3896            let mut ts = github_5xx_timestamps_lock();
3897            ts.clear();
3898            let mut map = cooldown_lock();
3899            map.remove("github:5xx");
3900        }
3901
3902        // Record exactly threshold errors
3903        for _ in 0..GITHUB_5XX_THRESHOLD {
3904            record_github_5xx().await;
3905        }
3906
3907        assert!(is_github_circuit_open(), "circuit should trip at threshold");
3908        assert!(
3909            github_circuit_remaining_secs() > 0,
3910            "remaining should be > 0 when open"
3911        );
3912        assert!(
3913            github_circuit_remaining_secs() <= GITHUB_5XX_COOLDOWN_SECS as u64,
3914            "remaining should not exceed cooldown duration"
3915        );
3916
3917        // Clean up
3918        {
3919            let mut open = github_5xx_circuit_open().lock().unwrap();
3920            *open = None;
3921        }
3922    }
3923
3924    #[serial(cooldown_state)]
3925    #[tokio::test]
3926    async fn github_circuit_cannot_double_trip() {
3927        super::reset_global_state().await;
3928        // Reset circuit breaker state
3929        {
3930            let mut open = github_5xx_circuit_open_lock();
3931            *open = None;
3932            let mut ts = github_5xx_timestamps_lock();
3933            ts.clear();
3934            let mut map = cooldown_lock();
3935            map.remove("github:5xx");
3936        }
3937
3938        // Trip the circuit
3939        for _ in 0..GITHUB_5XX_THRESHOLD {
3940            record_github_5xx().await;
3941        }
3942        assert!(is_github_circuit_open());
3943
3944        let first_remaining = github_circuit_remaining_secs();
3945
3946        // More 5xx errors should NOT reset the cooldown (never-shorten)
3947        for _ in 0..GITHUB_5XX_THRESHOLD {
3948            record_github_5xx().await;
3949        }
3950
3951        let second_remaining = github_circuit_remaining_secs();
3952        assert!(
3953            second_remaining <= first_remaining,
3954            "additional errors should not extend cooldown"
3955        );
3956
3957        // Clean up
3958        {
3959            let mut open = github_5xx_circuit_open().lock().unwrap();
3960            *open = None;
3961        }
3962    }
3963
3964    #[serial(cooldown_state)]
3965    #[tokio::test]
3966    async fn github_circuit_recovers_after_cooldown() {
3967        super::reset_global_state().await;
3968        // Reset circuit breaker state
3969        {
3970            let mut open = github_5xx_circuit_open_lock();
3971            *open = None;
3972            let mut ts = github_5xx_timestamps_lock();
3973            ts.clear();
3974            let mut map = cooldown_lock();
3975            map.remove("github:5xx");
3976        }
3977
3978        // Set the circuit to an expired cooldown
3979        {
3980            let mut open = github_5xx_circuit_open().lock().unwrap();
3981            *open = Some(chrono::Utc::now().timestamp() - 1);
3982        }
3983
3984        // Should be closed now (auto-recovery)
3985        assert!(
3986            !is_github_circuit_open(),
3987            "circuit should auto-recover after cooldown"
3988        );
3989    }
3990
3991    #[serial(cooldown_state)]
3992    #[tokio::test]
3993    async fn github_circuit_sliding_window_ages_out() {
3994        super::reset_global_state().await;
3995        // Reset circuit breaker state
3996        {
3997            let mut open = github_5xx_circuit_open_lock();
3998            *open = None;
3999            let mut ts = github_5xx_timestamps_lock();
4000            ts.clear();
4001            let mut map = cooldown_lock();
4002            map.remove("github:5xx");
4003        }
4004
4005        // Record errors that are outside the window
4006        {
4007            let mut ts = github_5xx_timestamps().lock().unwrap();
4008            let old = chrono::Utc::now().timestamp() - GITHUB_5XX_WINDOW_SECS - 10;
4009            for _ in 0..GITHUB_5XX_THRESHOLD {
4010                ts.push(old);
4011            }
4012        }
4013
4014        // A new error should clear old ones and not trip the circuit
4015        record_github_5xx().await;
4016
4017        assert!(
4018            !is_github_circuit_open(),
4019            "old errors outside window should not trip circuit"
4020        );
4021    }
4022}