Skip to main content

orch/engine/
cooldown.rs

1//! Per-agent and per-model cooldown tracking.
2//!
3//! Shared by `engine::runner::response` (recording failures) and
4//! `engine::router::config` (avoiding cooled models during pool selection).
5//!
6//! # Persistence
7//!
8//! All cooldowns are persisted to the KV store in SQLite so they survive
9//! service restarts. Call [`init_cooldown_store`] once at startup (after
10//! the `TaskStore` is open) to register the store and pre-load unexpired
11//! cooldowns.
12//!
13//! When a rate limit includes a "try again at {date}" message, the cooldown
14//! is set to that specific timestamp instead of the default duration.
15//! Relative usage windows (e.g. "5-hour usage limit") are also detected and
16//! used to set a model-specific cooldown matching the advertised window.
17
18use chrono::TimeZone;
19use regex::Regex;
20use std::collections::HashMap;
21use std::collections::HashSet;
22use std::sync::LazyLock;
23use std::sync::MutexGuard;
24use std::sync::{Arc, Mutex, OnceLock};
25use tokio::sync;
26
27use crate::engine::router::config;
28#[cfg(test)]
29use crate::engine::runner::agents::opencode;
30use crate::store::TaskStore;
31
32/// Default fallback cooldown for agents when no store is available: 30 minutes.
33///
34/// Used in contexts that need a concrete duration and cannot compute exponential
35/// backoff (e.g. computing how long to wait before retrying when a cooldown
36/// expiry timestamp is unknown). New code should use [`BACKOFF_BASE_SECS`] instead.
37pub const AGENT_COOLDOWN_SECS: i64 = 30 * 60;
38
39/// Base backoff for generic agent/model failures: 5 minutes.
40pub const BACKOFF_BASE_SECS: i64 = 5 * 60;
41
42/// Maximum backoff for generic agent/model failures: 4 hours.
43pub const BACKOFF_MAX_SECS: i64 = 4 * 60 * 60;
44
45/// Base backoff for persistent model failures (silent exit/provider errors): 4 hours.
46pub const PERSISTENT_MODEL_BACKOFF_BASE_SECS: i64 = 4 * 60 * 60;
47
48/// Maximum backoff for persistent model failures: 7 days.
49pub const PERSISTENT_MODEL_BACKOFF_MAX_SECS: i64 = 7 * 24 * 60 * 60;
50
51/// Base backoff for a one-off model failure that does not mean the model is gone: 5 minutes.
52pub const TRANSIENT_MODEL_BACKOFF_BASE_SECS: i64 = 5 * 60;
53
54/// Maximum backoff for a one-off model failure: 15 minutes.
55pub const TRANSIENT_MODEL_BACKOFF_MAX_SECS: i64 = 15 * 60;
56
57/// Base backoff for credit exhaustion (out_of_credits): 1 hour.
58pub const CREDIT_BACKOFF_BASE_SECS: i64 = 60 * 60;
59
60/// Base backoff for org-level disabling: 2 hours.
61pub const ORG_BACKOFF_BASE_SECS: i64 = 2 * 60 * 60;
62
63/// Standard cap for credit exhaustion and org-level disabling: 8 hours.
64/// Repeated failures at the cap move to the extended tiers (48h, then 96h)
65pub const CREDIT_BACKOFF_MAX_SECS: i64 = 8 * 60 * 60;
66
67/// Base cooldown for billing cycle exhaustion: 24 hours.
68///
69/// Unlike per-request rate limits, billing cycle exhaustion is a calendar event
70/// that won't self-resolve for days or weeks. The first failure starts at 24h;
71/// subsequent failures escalate via `compute_backoff()` up to
72/// `BILLING_CYCLE_MAX_SECS` (7 days). This prevents daily retry-and-fail cycles
73/// when a monthly billing quota is exhausted.
74pub const BILLING_CYCLE_COOLDOWN_SECS: i64 = 24 * 60 * 60;
75
76/// Maximum cooldown for billing cycle exhaustion: 7 days.
77///
78/// Monthly billing cycles typically reset on a fixed date. 7 days is a
79/// reasonable cap — the cooldown will expire before the next monthly reset
80/// in most cases, allowing a single probe to re-check availability.
81pub const BILLING_CYCLE_MAX_SECS: i64 = 7 * 24 * 60 * 60;
82
83/// Credit exhaustion reason detected from error message.
84#[derive(Debug, Clone, Copy, PartialEq, Eq)]
85pub enum CreditExhaustionReason {
86    /// Per-model credit exhaustion (out_of_credits).
87    OutOfCredits,
88    /// Organization-level disabling (org_level_disabled).
89    OrgLevelDisabled,
90    /// Monthly billing cycle exhaustion — escalating cooldown (24h base, 7d cap).
91    BillingCycleExhausted,
92}
93
94impl CreditExhaustionReason {
95    /// Stable string for `task_runs.outcome`, distinct from the cooldown reason
96    /// string used by `record_credit_exhaustion`. Lets callers persist the actual
97    /// failure mode instead of collapsing it into a generic "rate_limit"/"failed".
98    pub fn outcome_str(self) -> &'static str {
99        match self {
100            CreditExhaustionReason::OutOfCredits => "credit_exhausted",
101            CreditExhaustionReason::OrgLevelDisabled => "org_level_disabled",
102            CreditExhaustionReason::BillingCycleExhausted => "billing_cycle_exhausted",
103        }
104    }
105}
106
107/// Short agent cooldown applied on silence detection (120 seconds).
108///
109/// Forces the router to pick a different agent immediately on re-route,
110/// without long-term blocking. The model cooldown (exponential, starting at
111/// 5 min) keeps the dead model out while this short cooldown just breaks
112/// the same-agent loop.
113pub const SILENCE_AGENT_COOLDOWN_SECS: u64 = 120;
114
115/// Silence detections before applying an extended cooldown (rolling window).
116pub const SILENCE_COUNT_THRESHOLD: usize = 5;
117
118/// Rolling window for silence detections (24 hours).
119pub const SILENCE_COUNT_WINDOW_SECS: i64 = 24 * 60 * 60;
120
121/// Extended cooldown applied after repeated silence detections (4 hours).
122pub const SILENCE_EXTENDED_COOLDOWN_SECS: u64 = 4 * 60 * 60;
123
124/// KV key prefix for persisted cooldowns (both agent and model).
125const KV_PREFIX: &str = "cooldown:";
126
127/// KV key prefix for rolling silence counters.
128const SILENCE_COUNT_PREFIX: &str = "silence_count:";
129
130/// KV key prefix for per-agent and per-agent:model failure counts (drives exponential backoff).
131const FAILURE_COUNT_PREFIX: &str = "failure_count:";
132
133/// KV key prefix for credit-exhaustion failure counts (separate from generic failures to prevent
134/// cross-contamination of backoff escalation — credit exhaustion uses different base/max durations).
135const CREDIT_FAILURE_COUNT_PREFIX: &str = "credit_failure_count:";
136
137/// KV key prefix for persistent model failure counts (separate from [`FAILURE_COUNT_PREFIX`] so
138/// `record_agent_success()` resetting the generic counter on an intervening success doesn't erase
139/// escalation progress for structurally unreliable models — see `record_persistent_model_failure`).
140const PERSISTENT_FAILURE_COUNT_PREFIX: &str = "persistent_failure_count:";
141
142/// KV key prefix for the unix timestamp after which `rate_limits` rows count toward degraded
143const RATE_LIMITS_CLEARED_AT_PREFIX: &str = "rate_limits_cleared_at:";
144
145/// Agent name from a cooldown key (`claude:sonnet` -> `claude`)
146fn agent_from_key(key: &str) -> &str {
147    key.split_once(':').map_or(key, |(agent, _)| agent)
148}
149
150struct CooldownEntry {
151    /// Unix timestamp when the cooldown expires.
152    cooldown_until: i64,
153    reason: String,
154}
155
156#[derive(Debug)]
157pub struct SilenceCountResult {
158    pub count: usize,
159    pub extended_cooldown_applied: bool,
160}
161
162/// Global in-memory cooldown map, protected by a Mutex.
163///
164/// Uses `std::sync::Mutex` because this map is also read from sync helper
165/// functions (`is_in_cooldown`, `is_model_in_cooldown`, etc.).  All lock
166/// holders perform only in-memory HashMap operations — **no `.await` is ever
167/// called while a guard is held**.  This invariant must be preserved: if a
168/// future caller needs to await while holding this lock, switch to
169/// `tokio::sync::Mutex` and make the helpers async.
170fn cooldowns() -> &'static Mutex<HashMap<String, CooldownEntry>> {
171    static COOLDOWNS: OnceLock<Mutex<HashMap<String, CooldownEntry>>> = OnceLock::new();
172    COOLDOWNS.get_or_init(|| Mutex::new(HashMap::new()))
173}
174
175/// Global store reference used for background KV writes.
176///
177/// Uses `tokio::sync::Mutex` because this value is only ever accessed from
178/// async functions. A `tokio::sync::Mutex` yields the Tokio task instead of
179/// blocking the worker thread if the lock is contended.
180fn cooldown_store() -> &'static sync::Mutex<Option<Arc<TaskStore>>> {
181    static STORE: OnceLock<sync::Mutex<Option<Arc<TaskStore>>>> = OnceLock::new();
182    STORE.get_or_init(|| sync::Mutex::new(None))
183}
184
185/// Initialise persistent cooldowns.
186///
187/// Must be called once at engine startup, after the `TaskStore` is opened.
188/// Loads unexpired cooldowns from KV and registers the store for future writes.
189/* Parses the `{"until": N, "reason": "..."}` cooldown KV format, falling back
190to the legacy bare-integer format (reason unknown) so existing rows keep
191working across the upgrade without a migration. */
192fn parse_cooldown_value(value: &str) -> Option<(i64, String)> {
193    if let Ok(serde_json::Value::Object(map)) = serde_json::from_str(value) {
194        let until = map.get("until")?.as_i64()?;
195        let reason = map
196            .get("reason")
197            .and_then(|r| r.as_str())
198            .unwrap_or("persisted")
199            .to_string();
200        return Some((until, reason));
201    }
202    value
203        .parse::<i64>()
204        .ok()
205        .map(|until| (until, "persisted".to_string()))
206}
207
208pub async fn init_cooldown_store(store: Arc<TaskStore>) {
209    match store.kv_list_prefix(KV_PREFIX).await {
210        Ok(rows) => {
211            let now = chrono::Utc::now().timestamp();
212            let mut map = cooldown_lock();
213            let mut loaded = 0;
214            for (key, value) in rows {
215                let cooldown_key = key.trim_start_matches(KV_PREFIX);
216                if let Some((cooldown_until, reason)) = parse_cooldown_value(&value)
217                    && now < cooldown_until
218                {
219                    map.insert(
220                        cooldown_key.to_string(),
221                        CooldownEntry {
222                            cooldown_until,
223                            reason,
224                        },
225                    );
226                    loaded += 1;
227                }
228                // Expired entries are simply not loaded — they'll be overwritten
229                // on the next failure or cleaned up by future writes.
230            }
231            if loaded > 0 {
232                tracing::info!(loaded, "loaded persisted cooldowns from KV store");
233            }
234        }
235        Err(e) => {
236            tracing::warn!(err = %e, "failed to load cooldowns from KV store");
237        }
238    }
239
240    // Load GitHub 5xx circuit breaker state if it was persisted before restart.
241    if let Ok(Some(raw)) = store.kv_get("cooldown:github:5xx").await
242        && let Ok(cooldown_until) = raw.parse::<i64>()
243    {
244        let now = chrono::Utc::now().timestamp();
245        if now < cooldown_until {
246            let mut open = github_5xx_circuit_open_lock();
247            *open = Some(cooldown_until);
248            let remaining = cooldown_until - now;
249            tracing::warn!(
250                remaining_secs = remaining,
251                "GitHub 5xx circuit breaker restored from KV (still open)"
252            );
253        }
254    }
255
256    *cooldown_store().lock().await = Some(store);
257}
258
259/// Reconcile the in-memory cooldown map with the persisted KV state.
260///
261/// KV is the source of truth. This is called once per tick by
262/// `router.refresh_health` so the running service notices state mutated by a
263/// separate process (e.g. `orch cooldown clear`). Without this sync, the CLI
264/// clears KV but the engine's in-memory map keeps the stale entries until they
265/// expire on their own, producing the "all agents cooled" + "No active
266/// cooldowns" inconsistency.
267///
268/// Reconciliation rules (KV authoritative for every key it has):
269/// - KV row missing for an in-memory key: leave memory alone — likely a
270///   `set_cooldown_async` race where memory was written but the KV row isn't
271///   yet visible. The next tick will catch up.
272/// - KV row present with `cooldown_until <= now` (including the "0" tombstone
273///   written by `clear_cooldown`): drop from memory.
274/// - KV row present with `cooldown_until > now`: insert/update memory to match.
275pub async fn sync_from_kv(store: &Arc<TaskStore>) {
276    let rows = match store.kv_list_prefix(KV_PREFIX).await {
277        Ok(rows) => rows,
278        Err(e) => {
279            tracing::warn!(err = %e, "cooldown sync_from_kv: KV list failed");
280            return;
281        }
282    };
283    let now = chrono::Utc::now().timestamp();
284
285    // Some((ts, reason)) for active cooldowns, None for cleared/expired rows.
286    // Missing key means no KV row at all, left alone below.
287    let mut explicit_kv: HashMap<String, Option<(i64, String)>> =
288        HashMap::with_capacity(rows.len());
289    for (kv_key, value) in &rows {
290        let cooldown_key = kv_key.trim_start_matches(KV_PREFIX);
291        if let Some((ts, reason)) = parse_cooldown_value(value) {
292            let normalized = if ts > now { Some((ts, reason)) } else { None };
293            explicit_kv.insert(cooldown_key.to_string(), normalized);
294        }
295    }
296
297    // The 5xx circuit breaker lives in its own statics, so mirror its KV row there
298    match explicit_kv.get("github:5xx") {
299        Some(None) => {
300            let mut open = github_5xx_circuit_open_lock();
301            if open.take().is_some() {
302                tracing::info!("GitHub 5xx circuit breaker closed by KV clear");
303                github_5xx_timestamps_lock().clear();
304            }
305        }
306        Some(Some((ts, _))) => {
307            *github_5xx_circuit_open_lock() = Some(*ts);
308        }
309        None => {}
310    }
311
312    let mut map = cooldown_lock();
313    let mut dropped = 0usize;
314    let mut updated = 0usize;
315
316    // Drop in-memory entries that KV explicitly marks cleared/expired.
317    map.retain(|key, _| {
318        if matches!(explicit_kv.get(key), Some(None)) {
319            dropped += 1;
320            false
321        } else {
322            true
323        }
324    });
325
326    // Adopt KV's authoritative timestamp for active cooldowns.
327    for (key, normalized) in explicit_kv {
328        if let Some((ts, reason)) = normalized {
329            if let Some(entry) = map.get_mut(&key) {
330                if entry.cooldown_until != ts {
331                    entry.cooldown_until = ts;
332                    updated += 1;
333                }
334            } else {
335                map.insert(
336                    key,
337                    CooldownEntry {
338                        cooldown_until: ts,
339                        reason,
340                    },
341                );
342                updated += 1;
343            }
344        }
345    }
346
347    if dropped > 0 || updated > 0 {
348        tracing::info!(
349            dropped,
350            updated,
351            "cooldown sync_from_kv: reconciled in-memory map with KV"
352        );
353    }
354}
355
356/// Failure count threshold (times at max backoff) for first extended tier (24h).
357///
358/// After a model has been at the max cap this many times, apply a 6x multiplier.
359const EXTENDED_TIER_1_THRESHOLD: u64 = 2;
360
361/// Multiplier for first extended tier (6x = 24h when max=4h).
362const EXTENDED_TIER_1_MULTIPLIER: i64 = 6;
363
364/// Failure count threshold (times at max backoff) for second extended tier (48h).
365///
366/// After a model has been at the max cap this many times, apply a 12x multiplier.
367const EXTENDED_TIER_2_THRESHOLD: u64 = 5;
368
369/// Multiplier for second extended tier (12x = 48h when max=4h).
370const EXTENDED_TIER_2_MULTIPLIER: i64 = 12;
371
372/// Compute exponential backoff duration using base-3 growth.
373///
374/// Returns `min(base * 3^(count-1), max)` for count >= 1.
375/// For count == 0 or 1 (first failure), returns `base`.
376///
377/// | count | base=300s | result |
378/// |-------|-----------|--------|
379/// | 0     | 300       | 300    |
380/// | 1     | 300       | 300    |
381/// | 2     | 300       | 900    |
382/// | 3     | 300       | 2700   |
383/// | 4     | 300       | 8100   |
384/// | 5-6   | 14400     | 14400 (max, not yet extended) |
385/// | 7-9   | 86400     | 86400 (24h extended tier) |
386/// | 10+   | 172800    | 172800 (48h extended tier) |
387///
388/// The extended tier applies when the model has hit the max cap multiple times,
389/// indicating a persistently broken model that won't self-resolve quickly.
390pub fn compute_backoff(count: u64, base: i64, max: i64, allow_extended: bool) -> i64 {
391    if count <= 1 {
392        return base;
393    }
394    let exp = u32::try_from(count.saturating_sub(1)).unwrap_or(u32::MAX);
395    let factor = 3_i64.saturating_pow(exp);
396    let standard = base.saturating_mul(factor).min(max);
397
398    // Extended tier: after N failures at max backoff, apply longer window
399    // to reduce noise from persistently broken models.
400    //
401    // First find the count at which exponential growth first hits max:
402    // solve: base * 3^(first_max-1) >= max
403    // => 3^(first_max-1) >= max / base
404    // => first_max-1 >= log_3(max / base)
405    // => first_max >= 1 + log_3(max / base)
406    let first_max = if base <= 0 {
407        0
408    } else {
409        // Use integer approximation: count where base * 3^(count-1) would first exceed max
410        let mut accumulated = base;
411        let mut first_max_count = 1u64;
412        while accumulated < max && first_max_count < count {
413            accumulated = accumulated.saturating_mul(3);
414            first_max_count += 1;
415        }
416        first_max_count
417    };
418
419    // Times we've been at the max cap (excluding the first time we hit it)
420    let times_at_max = count.saturating_sub(first_max);
421
422    if !allow_extended {
423        return standard;
424    }
425
426    if standard == max && times_at_max >= EXTENDED_TIER_2_THRESHOLD {
427        max.saturating_mul(EXTENDED_TIER_2_MULTIPLIER)
428    } else if standard == max && times_at_max >= EXTENDED_TIER_1_THRESHOLD {
429        max.saturating_mul(EXTENDED_TIER_1_MULTIPLIER)
430    } else {
431        standard
432    }
433}
434
435/// Read the failure count for a key from KV, increment it, write it back, and return the new count.
436///
437/// Always increments — the backoff formula's `max` parameter prevents the
438/// cooldown duration from growing beyond the cap, so unbounded counts are safe.
439///
440/// Returns 1 when no store is configured (unit-test contexts without a store).
441/// Returns `u64::MAX` on store errors (e.g. lock contention) so backoff applies the cap duration
442/// instead of resetting to the base, preventing rapid re-dispatch during store outages.
443async fn read_and_increment_failure_count(store_opt: Option<&Arc<TaskStore>>, key: &str) -> u64 {
444    read_and_increment_failure_count_with_prefix(store_opt, FAILURE_COUNT_PREFIX, key).await
445}
446
447/// Like [`read_and_increment_failure_count`] but with an explicit KV prefix.
448///
449/// Credit exhaustion uses [`CREDIT_FAILURE_COUNT_PREFIX`] so its backoff escalation
450/// is independent of generic agent failures (which use [`FAILURE_COUNT_PREFIX`]).
451async fn read_and_increment_failure_count_with_prefix(
452    store_opt: Option<&Arc<TaskStore>>,
453    prefix: &str,
454    key: &str,
455) -> u64 {
456    let kv_key = format!("{prefix}{key}");
457    if let Some(store) = store_opt {
458        match store.kv_increment(&kv_key).await {
459            Ok(n) => n,
460            Err(e) => {
461                tracing::warn!(key = %kv_key, err = %e, "failed to increment failure count — treating as high count for safe backoff");
462                u64::MAX
463            }
464        }
465    } else {
466        1
467    }
468}
469
470/// Backoff schedule for one kind of per-model failure.
471struct ModelBackoff {
472    prefix: &'static str,
473    base: i64,
474    max: i64,
475    allow_extended: bool,
476    reason: &'static str,
477}
478
479/// Bump the failure counter for `agent:model` and cool the model for the backoff it maps to.
480async fn record_model_backoff(agent_name: &str, model: &str, schedule: &ModelBackoff) {
481    let key = format!("{agent_name}:{model}");
482    let store_opt = cooldown_store().lock().await.clone();
483    let count =
484        read_and_increment_failure_count_with_prefix(store_opt.as_ref(), schedule.prefix, &key)
485            .await;
486    let cooldown_secs =
487        compute_backoff(count, schedule.base, schedule.max, schedule.allow_extended);
488    let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
489    set_cooldown_async(&key, cooldown_until, schedule.reason).await;
490}
491
492/// Reset failure counts for an agent and agent:model combo after a successful run.
493///
494/// Call this from the runner's success path so that the next failure starts
495/// backoff from the base duration again, not from wherever it left off.
496/// Resets both generic and credit-exhaustion failure counters.
497pub async fn record_agent_success(agent_name: &str, model: &str) {
498    let store_opt = cooldown_store().lock().await.clone();
499    if let Some(store) = store_opt {
500        let agent_key = format!("{FAILURE_COUNT_PREFIX}{agent_name}");
501        let credit_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{agent_name}");
502        if let Err(e) = store.kv_set(&agent_key, "0").await {
503            tracing::warn!(key = agent_key, err = %e, "failed to reset failure count");
504        }
505        if !model.is_empty() {
506            let model_key = format!("{FAILURE_COUNT_PREFIX}{agent_name}:{model}");
507            if let Err(e) = store.kv_set(&model_key, "0").await {
508                tracing::warn!(key = model_key, err = %e, "failed to reset failure count");
509            }
510        }
511        if let Err(e) = store.kv_set(&credit_key, "0").await {
512            tracing::warn!(key = credit_key, err = %e, "failed to reset failure count");
513        }
514    }
515}
516
517/// Record that an agent has failed and should be temporarily avoided.
518///
519/// Applies exponential backoff based on the agent's failure count in KV.
520/// If `error_message` contains "try again at {date}", that vendor date is used
521/// instead of the computed backoff (vendor dates are always authoritative).
522pub async fn record_agent_failure_with_message(agent_name: &str, error_message: &str) {
523    // Vendor-specified retry date takes priority over backoff — but only
524    // if it is in the present or future. If the parsed timestamp is in the
525    // past (stale vendor message), fall through to the exponential backoff
526    // path instead of returning early and bypassing backoff entirely.
527    if let Some(cooldown_until) = parse_retry_at(error_message) {
528        let now = chrono::Utc::now().timestamp();
529        if cooldown_until > now {
530            set_cooldown_async(agent_name, cooldown_until, "agent_error").await;
531            return;
532        }
533        tracing::debug!(
534            agent = agent_name,
535            cooldown_until,
536            "vendor retry_at is in the past — using backoff instead"
537        );
538    }
539
540    let store_opt = cooldown_store().lock().await.clone();
541    let count = read_and_increment_failure_count(store_opt.as_ref(), agent_name).await;
542    let base = config::get_agent_backoff_base(agent_name);
543    let cooldown_until =
544        chrono::Utc::now().timestamp() + compute_backoff(count, base, BACKOFF_MAX_SECS, true);
545    set_cooldown_async(agent_name, cooldown_until, "agent_error").await;
546}
547
548/// Detect credit exhaustion from an error message and return the stable
549/// `task_runs.outcome` string for it (see `CreditExhaustionReason::outcome_str`).
550///
551/// Callers that classify `AgentError::RateLimit`/`AgentError::Auth` into an
552/// outcome string should consult this first — generic rate-limit patterns like
553/// "usage limit" also match billing-cycle exhaustion messages (e.g. Kimi's
554/// "You've reached your usage limit for this billing cycle"), which would
555/// otherwise be stored as a plain "rate_limit" outcome.
556pub fn detect_credit_exhaustion_outcome(error_message: &str) -> Option<&'static str> {
557    detect_credit_exhaustion(error_message).map(CreditExhaustionReason::outcome_str)
558}
559
560/// Detect credit exhaustion reasons from error message.
561///
562/// Returns `Some(CreditExhaustionReason)` if the error indicates
563/// out_of_credits or org_level_disabled, which require longer agent-wide cooldowns.
564pub fn detect_credit_exhaustion(error_message: &str) -> Option<CreditExhaustionReason> {
565    let lower = error_message.to_lowercase();
566
567    let overage_patterns = [
568        "overagedisabledreason",
569        "overage_disabled_reason",
570        "overage disabled reason",
571    ];
572
573    if overage_patterns.iter().any(|p| lower.contains(p)) {
574        if lower.contains("out_of_credits") || lower.contains("outofcredits") {
575            return Some(CreditExhaustionReason::OutOfCredits);
576        }
577        if lower.contains("org_level_disabled") || lower.contains("orgdisabled") {
578            return Some(CreditExhaustionReason::OrgLevelDisabled);
579        }
580    }
581
582    let billing_patterns = [
583        "credit balance too low",
584        "credit balance insufficient",
585        "insufficient credit balance",
586        "billing quota exceeded",
587        "billing limit exceeded",
588        "organization has been disabled",
589        "org disabled",
590    ];
591
592    if billing_patterns.iter().any(|p| lower.contains(p)) {
593        return Some(CreditExhaustionReason::OrgLevelDisabled);
594    }
595
596    // Billing cycle exhaustion: monthly quota reset — check before generic quota
597    // patterns because "monthly quota exceeded" would otherwise match "quota exceeded".
598    let billing_cycle_patterns = [
599        "billing cycle",
600        "quota refreshed next cycle",
601        "monthly quota",
602        "quota will be refreshed",
603        "next billing cycle",
604        "refreshed in the next cycle",
605        "weekly/monthly limit exhausted",
606        "limit will reset at",
607        "upgrade to pro", // codex/chatgpt plan usage cap: requires billing upgrade
608        "upgrade your token plan", // minimax: Token Plan usage limit reached
609        "token plan usage limit", // minimax: Token Plan usage limit reached
610        "purchase credits for more", // minimax: ...purchase Credits for more usage
611    ];
612
613    if billing_cycle_patterns.iter().any(|p| lower.contains(p)) {
614        return Some(CreditExhaustionReason::BillingCycleExhausted);
615    }
616
617    if lower.contains("out of credits")
618        || lower.contains("outofcredits")
619        || lower.contains("insufficient funds")
620        || lower.contains("no credits remaining")
621        || lower.contains("insufficient_quota")
622        || lower.contains("quota exceeded")
623        || lower.contains("insufficient balance")
624        || lower.contains("please recharge")
625        || lower.contains("no resource package")
626        || lower.contains("balance exhausted")
627        || lower.contains("402 payment required")
628    {
629        return Some(CreditExhaustionReason::OutOfCredits);
630    }
631
632    None
633}
634
635/// Record an agent-level cooldown for credit exhaustion.
636///
637/// Applies exponential backoff based on the agent's failure count.
638/// All variants use escalating backoff:
639/// - `OutOfCredits`: 1h, 3h, then 8h (credits can be refilled any time), 48h from the 5th failure, 96h from the 8th
640/// - `OrgLevelDisabled`: 2h, 6h, then 8h, 48h from the 5th failure, 96h from the 8th
641/// - `BillingCycleExhausted`: starts at 24h, caps at 7 days (monthly event)
642///
643/// Billing cycle exhaustion was previously a flat 24h, but this caused daily
644/// retry-and-fail cycles when a monthly quota was exhausted (the 24h cooldown
645/// would expire, kimi would be retried, fail immediately, and get another 24h).
646/// Now each recurrence escalates: 24h → 72h → 7d (capped).
647pub async fn record_credit_exhaustion(agent_name: &str, reason: CreditExhaustionReason) {
648    let reason_str = match reason {
649        CreditExhaustionReason::OutOfCredits => "credit_exhaustion_out_of_credits",
650        CreditExhaustionReason::OrgLevelDisabled => "credit_exhaustion_org_level_disabled",
651        CreditExhaustionReason::BillingCycleExhausted => "billing_cycle_exhausted",
652    };
653
654    // A failure from a run that started before the cooldown is the same event
655    if is_in_cooldown_with_reason(agent_name, reason_str) {
656        tracing::debug!(
657            agent = agent_name,
658            reason = reason_str,
659            "credit exhaustion already cooled, not escalating"
660        );
661        return;
662    }
663
664    let store_opt = cooldown_store().lock().await.clone();
665    // Use a distinct failure counter (CREDIT_FAILURE_COUNT_PREFIX) so credit-
666    // exhaustion backoff escalation is independent of generic agent failures.
667    // Previously both paths shared FAILURE_COUNT_PREFIX, causing generic failures
668    // to inflate credit-exhaustion backoff (and vice versa).
669    let count = read_and_increment_failure_count_with_prefix(
670        store_opt.as_ref(),
671        CREDIT_FAILURE_COUNT_PREFIX,
672        agent_name,
673    )
674    .await;
675
676    let (base, max) = match reason {
677        CreditExhaustionReason::OutOfCredits => (CREDIT_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS),
678        CreditExhaustionReason::OrgLevelDisabled => {
679            (ORG_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS)
680        }
681        CreditExhaustionReason::BillingCycleExhausted => {
682            (BILLING_CYCLE_COOLDOWN_SECS, BILLING_CYCLE_MAX_SECS)
683        }
684    };
685    let cooldown_secs = compute_backoff(
686        count,
687        base,
688        max,
689        !matches!(reason, CreditExhaustionReason::BillingCycleExhausted),
690    );
691    let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
692    set_cooldown_async(agent_name, cooldown_until, reason_str).await;
693    tracing::warn!(
694        agent = agent_name,
695        reason = reason_str,
696        attempt = count,
697        cooldown_secs,
698        "credit exhaustion detected: applying exponential agent-wide cooldown"
699    );
700}
701
702// Returns false when the message is no credit exhaustion. Model-scoped billing cycle cools the model only
703pub async fn record_credit_exhaustion_from_message(
704    agent_name: &str,
705    model: Option<&str>,
706    message: &str,
707) -> bool {
708    let Some(reason) = detect_credit_exhaustion(message) else {
709        return false;
710    };
711    match (reason, model) {
712        (CreditExhaustionReason::BillingCycleExhausted, Some(model)) => {
713            record_persistent_model_failure(agent_name, model).await;
714        }
715        _ => record_credit_exhaustion(agent_name, reason).await,
716    }
717    true
718}
719
720/// Record that a specific agent+model combo has failed.
721///
722/// Applies exponential backoff based on the model's failure count in KV.
723pub async fn record_model_failure(agent_name: &str, model: &str) {
724    let base = config::get_agent_backoff_base(agent_name);
725    let schedule = ModelBackoff {
726        prefix: FAILURE_COUNT_PREFIX,
727        base,
728        max: BACKOFF_MAX_SECS,
729        allow_extended: true,
730        reason: "model_error",
731    };
732    record_model_backoff(agent_name, model, &schedule).await;
733}
734
735/// Record a rate-limit event for an agent+model.
736///
737/// If the error message advertises a relative usage window (e.g.
738/// "5-hour usage limit"), the cooldown is applied to the specific model for
739/// that window length. The quota is model-scoped, so the whole agent is not
740/// cooled. If no window is detected, fall back to the generic agent+model
741/// exponential backoff path.
742pub async fn record_rate_limit(agent_name: &str, model: Option<&str>, error_message: &str) {
743    if let Some(window_secs) = parse_relative_usage_window(error_message)
744        && let Some(m) = model
745    {
746        let cooldown_until = chrono::Utc::now()
747            .timestamp()
748            .saturating_add(i64::try_from(window_secs).unwrap_or(i64::MAX));
749        set_model_cooldown(agent_name, m, window_secs, "rate_limit_usage_window").await;
750        tracing::info!(
751            agent = agent_name,
752            model = m,
753            window_secs,
754            cooldown_until,
755            "relative usage window detected: applying model-specific cooldown"
756        );
757        return;
758    }
759
760    record_agent_failure_with_message(agent_name, error_message).await;
761    if let Some(m) = model {
762        record_model_failure(agent_name, m).await;
763    }
764}
765
766/// Record a persistent model-specific failure and apply longer exponential backoff.
767///
768/// Used for failure modes that are commonly broken for long periods (for example
769/// silent exit-0 or opaque provider errors). Starts at 4h and escalates with
770/// the generic backoff formula up to 7 days.
771///
772/// Uses [`PERSISTENT_FAILURE_COUNT_PREFIX`], a counter independent of the generic
773/// `failure_count:` key that `record_agent_success()` resets on every successful
774/// run. Otherwise a single intervening success between persistent failures would
775/// reset escalation back to the 4h base, letting structurally unreliable models
776/// (intermittent parse errors, truncated runs) stay in rotation indefinitely.
777pub async fn record_persistent_model_failure(agent_name: &str, model: &str) {
778    let schedule = ModelBackoff {
779        prefix: PERSISTENT_FAILURE_COUNT_PREFIX,
780        base: PERSISTENT_MODEL_BACKOFF_BASE_SECS,
781        max: PERSISTENT_MODEL_BACKOFF_MAX_SECS,
782        allow_extended: false,
783        reason: "persistent_model_error",
784    };
785    record_model_backoff(agent_name, model, &schedule).await;
786}
787
788/// Record a `ModelUnavailable` failure for an agent+model.
789///
790/// "Not found", "not supported" and region blocks never lift on their own, so they get the
791/// persistent backoff (4h base, 7d max). Any other message gets [`record_model_failure`]
792pub async fn record_model_unavailable(agent_name: &str, model: &str, message: &str) {
793    if is_model_permanently_gone(message) {
794        record_persistent_model_failure(agent_name, model).await;
795    } else {
796        record_model_failure(agent_name, model).await;
797    }
798}
799
800fn is_model_permanently_gone(message: &str) -> bool {
801    let lower = message.to_lowercase();
802    lower.contains("not found")
803        || lower.contains("not supported")
804        || lower.contains("not available in your country")
805        || lower.contains("not available in your region")
806}
807
808/// Record a run that failed without showing the model is gone: empty or cut off output, an
809/// unparseable review, an opaque provider error.
810///
811/// A stopped session or one odd answer causes these, so the backoff is 5 then 15 minutes. Billing
812/// exhaustion and "model not found" keep [`record_persistent_model_failure`]
813pub async fn record_transient_model_failure(agent_name: &str, model: &str) {
814    let schedule = ModelBackoff {
815        prefix: FAILURE_COUNT_PREFIX,
816        base: TRANSIENT_MODEL_BACKOFF_BASE_SECS,
817        max: TRANSIENT_MODEL_BACKOFF_MAX_SECS,
818        allow_extended: false,
819        reason: "transient_model_error",
820    };
821    record_model_backoff(agent_name, model, &schedule).await;
822}
823
824/// Set a model cooldown with a custom duration (in seconds) and reason.
825///
826/// Generic model-cooldown setter shared by silence detection and the
827/// rate-limit relative-usage-window path. The reason is persisted with
828/// the cooldown (KV + in-memory entry) so operators can tell the two
829/// apart in `orch cooldown list`.
830pub async fn set_model_cooldown(agent_name: &str, model: &str, duration_secs: u64, reason: &str) {
831    let key = format!("{agent_name}:{model}");
832    let cooldown_until = chrono::Utc::now()
833        .timestamp()
834        .saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX));
835    set_cooldown_async(&key, cooldown_until, reason).await;
836}
837
838/// Set a model cooldown that ends at a vendor-provided Unix timestamp.
839pub async fn set_model_cooldown_until(agent_name: &str, model: &str, until: i64) {
840    let key = format!("{agent_name}:{model}");
841    set_cooldown_async(&key, until, "vendor_reset_at").await;
842}
843
844/// Set a short agent-level cooldown (in seconds).
845///
846/// Used by silence detection to temporarily block the whole agent so the
847/// router picks a different one on re-route. Unlike `record_agent_failure`
848/// (exponential backoff), this uses a short duration (typically 120s) —
849/// just enough to force one re-route cycle to a different agent.
850pub async fn set_agent_cooldown(agent_name: &str, duration_secs: u64) {
851    let cooldown_until = chrono::Utc::now()
852        .timestamp()
853        .saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX));
854    set_cooldown_async(agent_name, cooldown_until, "silence_agent_cooldown").await;
855}
856
857/// KV prefix for the latest quota window of each type an agent reported: `rate_window:{agent}:{type}`
858pub const RATE_WINDOW_KV_PREFIX: &str = "rate_window:";
859
860/// One quota window from a vendor `rate_limit_event`, with the vendor window type kept as sent
861#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
862pub struct RateLimitWindow {
863    pub status: String,
864    pub window_type: String,
865    pub utilization: Option<f64>,
866    pub resets_at: Option<i64>,
867}
868
869/// Store the latest window for an agent. A rejected window cools the agent until its reset time
870pub async fn record_rate_limit_window(agent_name: &str, window: &RateLimitWindow) {
871    let store_opt = cooldown_store().lock().await.clone();
872    if let Some(store) = store_opt
873        && let Ok(value) = serde_json::to_string(window)
874        && let Err(e) = store
875            .kv_set(
876                &format!("{RATE_WINDOW_KV_PREFIX}{agent_name}:{}", window.window_type),
877                &value,
878            )
879            .await
880    {
881        tracing::warn!(agent = agent_name, err = %e, "failed to persist rate limit window");
882    }
883
884    let now = chrono::Utc::now().timestamp();
885    if window.status == "rejected"
886        && let Some(reset) = window.resets_at
887        && reset > now
888    {
889        let until = reset.min(now + PERSISTENT_MODEL_BACKOFF_MAX_SECS);
890        set_cooldown_async(agent_name, until, "rate_limit_window").await;
891        tracing::info!(
892            agent = agent_name,
893            window = %window.window_type,
894            cooldown_until = until,
895            "rate limit window rejected: cooling agent until vendor reset"
896        );
897    }
898}
899
900/// Latest stored quota window per agent, sorted by agent name
901pub async fn list_rate_limit_windows(store: &Arc<TaskStore>) -> Vec<(String, RateLimitWindow)> {
902    let rows = store
903        .kv_list_prefix(RATE_WINDOW_KV_PREFIX)
904        .await
905        .unwrap_or_default();
906    let mut windows: Vec<(String, RateLimitWindow)> = rows
907        .into_iter()
908        .filter_map(|(key, value)| {
909            let (agent, _) = key.strip_prefix(RATE_WINDOW_KV_PREFIX)?.rsplit_once(':')?;
910            let window = serde_json::from_str(&value).ok()?;
911            Some((agent.to_string(), window))
912        })
913        .collect();
914    windows.sort_by(|a, b| (&a.0, &a.1.window_type).cmp(&(&b.0, &b.1.window_type)));
915    windows
916}
917
918/// Record a silence detection for an agent+model and apply extended cooldowns
919/// when repeated silences exceed the threshold within the rolling window.
920pub async fn record_silence_detection(agent_name: &str, model: &str) -> Option<SilenceCountResult> {
921    let store_opt = cooldown_store().lock().await.clone();
922    let Some(store) = store_opt else {
923        tracing::debug!(
924            agent = agent_name,
925            model,
926            "skipping silence count record (no KV store)"
927        );
928        return None;
929    };
930
931    let key = format!("{SILENCE_COUNT_PREFIX}{agent_name}:{model}");
932    let now = chrono::Utc::now().timestamp();
933    let window_start = now - SILENCE_COUNT_WINDOW_SECS;
934
935    let mut timestamps = match store.kv_get(&key).await {
936        Ok(Some(raw)) => match serde_json::from_str::<Vec<i64>>(&raw) {
937            Ok(ts) => ts,
938            Err(e) => {
939                tracing::warn!(
940                    kv_key = %key,
941                    err = %e,
942                    "failed to parse silence timestamps from KV — resetting window"
943                );
944                Vec::new()
945            }
946        },
947        Ok(None) => Vec::new(),
948        Err(err) => {
949            tracing::warn!(
950                kv_key = %key,
951                err = %err,
952                "failed to load silence count from KV"
953            );
954            Vec::new()
955        }
956    };
957
958    timestamps.retain(|ts| *ts >= window_start);
959    timestamps.push(now);
960
961    let count = timestamps.len();
962    let mut extended_cooldown_applied = false;
963    if count >= SILENCE_COUNT_THRESHOLD {
964        set_model_cooldown(
965            agent_name,
966            model,
967            SILENCE_EXTENDED_COOLDOWN_SECS,
968            "silence_detected",
969        )
970        .await;
971        extended_cooldown_applied = true;
972        timestamps.clear();
973    }
974
975    let value = serde_json::to_string(&timestamps).unwrap_or_else(|_| "[]".to_string());
976    if let Err(err) = store.kv_set(&key, &value).await {
977        tracing::warn!(kv_key = key, err = %err, "failed to persist silence count");
978    }
979
980    Some(SilenceCountResult {
981        count,
982        extended_cooldown_applied,
983    })
984}
985
986/// Check if a specific agent+model combo is in cooldown.
987pub fn is_model_in_cooldown(agent_name: &str, model: &str) -> bool {
988    let key = format!("{agent_name}:{model}");
989    is_in_cooldown(&key)
990}
991
992/// Check if an agent is currently in cooldown period.
993pub fn is_agent_in_cooldown(agent_name: &str) -> bool {
994    is_in_cooldown(agent_name)
995}
996
997/// Return the cooldown expiry timestamp for a key (agent or agent:model), if active.
998pub fn cooldown_until(key: &str) -> Option<i64> {
999    let map = cooldown_lock();
1000    map.get(key).and_then(|entry| {
1001        let now = chrono::Utc::now().timestamp();
1002        if now < entry.cooldown_until {
1003            Some(entry.cooldown_until)
1004        } else {
1005            None
1006        }
1007    })
1008}
1009
1010/// Return the reason string recorded when a cooldown was set, if the cooldown is still active.
1011pub fn cooldown_reason(key: &str) -> Option<String> {
1012    let map = cooldown_lock();
1013    map.get(key).and_then(|entry| {
1014        let now = chrono::Utc::now().timestamp();
1015        if now < entry.cooldown_until {
1016            Some(entry.reason.clone())
1017        } else {
1018            None
1019        }
1020    })
1021}
1022
1023/// Return all currently active cooldowns as `(key, cooldown_until_unix, reason)`.
1024///
1025/// Reads from the in-memory map only — always fast, no async needed.
1026/// Expired entries are filtered out. Used by `orch cooldown list`.
1027pub fn list_all_cooldowns() -> Vec<(String, i64, String)> {
1028    let now = chrono::Utc::now().timestamp();
1029    let map = cooldown_lock();
1030    let mut result: Vec<(String, i64, String)> = map
1031        .iter()
1032        .filter(|(_, entry)| now < entry.cooldown_until)
1033        .map(|(key, entry)| (key.clone(), entry.cooldown_until, entry.reason.clone()))
1034        .collect();
1035    result.sort_by(|a, b| a.0.cmp(&b.0));
1036    result
1037}
1038
1039/// Write `rate_limits_cleared_at:{agent}` = now so the next health check
1040/// ignores older `rate_limits` rows. Rows stay in the table for metrics
1041pub async fn mark_rate_limits_cleared(store: &Arc<TaskStore>, agents: &[String]) {
1042    let now = chrono::Utc::now().timestamp().to_string();
1043    let mut seen = HashSet::new();
1044    for key in agents {
1045        let agent = agent_from_key(key);
1046        if agent.is_empty() || !seen.insert(agent.to_string()) {
1047            continue;
1048        }
1049        let kv_key = format!("{RATE_LIMITS_CLEARED_AT_PREFIX}{agent}");
1050        if let Err(e) = store.kv_set(&kv_key, &now).await {
1051            tracing::warn!(agent, err = %e, "failed to mark rate limits cleared");
1052        }
1053    }
1054}
1055
1056/// Clear a specific cooldown by key, or all cooldowns when `key == "*"`.
1057///
1058/// Removes from the in-memory map, writes a past timestamp to KV, and resets
1059/// the corresponding `failure_count:` entries so the next failure starts
1060/// backoff from the base duration. Also writes `rate_limits_cleared_at:{agent}`
1061/// so a degraded agent held only by rate rows recovers on the next health check
1062pub async fn clear_cooldown(key: &str, store: &Arc<TaskStore>) {
1063    let mut agents_to_mark: Vec<String> = Vec::new();
1064    if key == "*" {
1065        let keys: Vec<String> = {
1066            let mut map = cooldown_lock();
1067            let keys: Vec<String> = map.keys().cloned().collect();
1068            map.clear();
1069            keys
1070        };
1071        for k in &keys {
1072            let kv_key = format!("{KV_PREFIX}{k}");
1073            if let Err(e) = store.kv_set(&kv_key, "0").await {
1074                tracing::warn!(key = kv_key, err = %e, "failed to clear cooldown");
1075            }
1076            // Reset failure count so backoff restarts from base.
1077            let fc_key = format!("{FAILURE_COUNT_PREFIX}{k}");
1078            if let Err(e) = store.kv_set(&fc_key, "0").await {
1079                tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1080            }
1081            // Also reset credit-specific failure count.
1082            let credit_fc_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{k}");
1083            if let Err(e) = store.kv_set(&credit_fc_key, "0").await {
1084                tracing::warn!(key = credit_fc_key, err = %e, "failed to reset credit failure count");
1085            }
1086            // Also reset the persistent-model-failure counter.
1087            let persistent_fc_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{k}");
1088            if let Err(e) = store.kv_set(&persistent_fc_key, "0").await {
1089                tracing::warn!(key = persistent_fc_key, err = %e, "failed to reset persistent failure count");
1090            }
1091        }
1092        // Also reset any persisted failure_count keys that are not in the
1093        // in-memory map (e.g. survived a restart or were set without a
1094        // corresponding in-memory cooldown entry).
1095        if let Ok(fc_entries) = store.kv_list_prefix(FAILURE_COUNT_PREFIX).await {
1096            for (fc_key, _) in fc_entries {
1097                if let Err(e) = store.kv_set(&fc_key, "0").await {
1098                    tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1099                }
1100            }
1101        }
1102        if let Ok(cfc_entries) = store.kv_list_prefix(CREDIT_FAILURE_COUNT_PREFIX).await {
1103            for (cfc_key, _) in cfc_entries {
1104                if let Err(e) = store.kv_set(&cfc_key, "0").await {
1105                    tracing::warn!(key = cfc_key, err = %e, "failed to reset credit failure count");
1106                }
1107            }
1108        }
1109        if let Ok(pfc_entries) = store.kv_list_prefix(PERSISTENT_FAILURE_COUNT_PREFIX).await {
1110            for (pfc_key, _) in pfc_entries {
1111                if let Err(e) = store.kv_set(&pfc_key, "0").await {
1112                    tracing::warn!(key = pfc_key, err = %e, "failed to reset persistent failure count");
1113                }
1114            }
1115        }
1116        tracing::info!(
1117            count = keys.len(),
1118            "cleared all cooldowns and failure counts"
1119        );
1120        agents_to_mark.extend(keys.iter().map(|k| agent_from_key(k).to_string()));
1121        match store.rate_limit_agents().await {
1122            Ok(rl_agents) => agents_to_mark.extend(rl_agents),
1123            Err(e) => tracing::warn!(err = %e, "failed to list rate_limits agents while clearing"),
1124        }
1125        match store.kv_list_prefix(RATE_LIMITS_CLEARED_AT_PREFIX).await {
1126            Ok(marks) => {
1127                for (k, _) in marks {
1128                    let agent = k.trim_start_matches(RATE_LIMITS_CLEARED_AT_PREFIX);
1129                    if !agent.is_empty() {
1130                        agents_to_mark.push(agent.to_string());
1131                    }
1132                }
1133            }
1134            Err(e) => tracing::warn!(err = %e, "failed to list rate_limits_cleared_at keys"),
1135        }
1136    } else {
1137        {
1138            let mut map = cooldown_lock();
1139            map.remove(key);
1140        }
1141        let kv_key = format!("{KV_PREFIX}{key}");
1142        if let Err(e) = store.kv_set(&kv_key, "0").await {
1143            tracing::warn!(key = kv_key, err = %e, "failed to clear cooldown");
1144        }
1145        // Reset failure count so backoff restarts from base.
1146        let fc_key = format!("{FAILURE_COUNT_PREFIX}{key}");
1147        if let Err(e) = store.kv_set(&fc_key, "0").await {
1148            tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1149        }
1150        // Also reset credit-specific failure count.
1151        let credit_fc_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{key}");
1152        if let Err(e) = store.kv_set(&credit_fc_key, "0").await {
1153            tracing::warn!(key = credit_fc_key, err = %e, "failed to reset credit failure count");
1154        }
1155        // Also reset the persistent-model-failure counter.
1156        let persistent_fc_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{key}");
1157        if let Err(e) = store.kv_set(&persistent_fc_key, "0").await {
1158            tracing::warn!(key = persistent_fc_key, err = %e, "failed to reset persistent failure count");
1159        }
1160        tracing::info!(key, "cleared cooldown and failure count");
1161        agents_to_mark.push(agent_from_key(key).to_string());
1162    }
1163    mark_rate_limits_cleared(store, &agents_to_mark).await;
1164}
1165
1166/// Set a cooldown with a specific expiry timestamp. Updates in-memory state only.
1167///
1168/// Returns `true` if the cooldown was applied, `false` if an existing longer
1169/// cooldown prevented the update (never-shorten rule).
1170///
1171/// **Callers are responsible for persisting to KV.** Use [`set_cooldown_async`]
1172/// for critical cooldowns that must survive a crash, or this function combined
1173/// with a fire-and-forget `tokio::spawn` for non-critical short cooldowns.
1174fn set_cooldown_in_memory(key: &str, cooldown_until: i64, reason: &str) -> bool {
1175    let mut map = cooldown_lock();
1176    if let Some(existing) = map.get(key) {
1177        // Never shorten an existing cooldown. This prevents a short
1178        // retry window (e.g., generic rate limit) from overriding a longer
1179        // billing-cycle cooldown.
1180        if existing.cooldown_until >= cooldown_until {
1181            return false;
1182        }
1183    }
1184    map.insert(
1185        key.to_string(),
1186        CooldownEntry {
1187            cooldown_until,
1188            reason: reason.to_string(),
1189        },
1190    );
1191    true
1192}
1193
1194/// Set a cooldown with a specific expiry timestamp and persist to KV inline.
1195///
1196/// This is the preferred function for critical cooldowns (credit exhaustion,
1197/// billing cycle, agent/model failures) that must survive a service crash.
1198/// The KV write is awaited before returning, so the caller can be confident
1199/// the cooldown is durable once this function returns `true`.
1200///
1201/// Returns `true` if the cooldown was applied, `false` if an existing longer
1202/// cooldown prevented the update (never-shorten rule).
1203async fn set_cooldown_async(key: &str, cooldown_until: i64, reason: &str) -> bool {
1204    if !set_cooldown_in_memory(key, cooldown_until, reason) {
1205        return false;
1206    }
1207    let store_opt = cooldown_store().lock().await.clone();
1208    if let Some(store) = store_opt {
1209        let kv_key = format!("{KV_PREFIX}{key}");
1210        let value = serde_json::json!({"until": cooldown_until, "reason": reason}).to_string();
1211        if let Err(e) = store.kv_set(&kv_key, &value).await {
1212            tracing::warn!(
1213                kv_key,
1214                err = %e,
1215                "failed to persist cooldown to KV store — rolling back in-memory entry"
1216            );
1217            let mut map = cooldown_lock();
1218            // Only roll back if nothing has superseded our write.
1219            // A racing thread may have set a longer cooldown in the meantime;
1220            // removing unconditionally would erase that valid entry.
1221            if map.get(key).map(|e| e.cooldown_until) == Some(cooldown_until) {
1222                map.remove(key);
1223            }
1224            return false;
1225        }
1226    }
1227    true
1228}
1229
1230/// Helper to acquire the cooldown map lock, logging and recovering on poison.
1231fn cooldown_lock() -> MutexGuard<'static, HashMap<String, CooldownEntry>> {
1232    cooldowns().lock().unwrap_or_else(|e| {
1233        tracing::error!("cooldown map lock poisoned — recovering with partial state; backoff and degraded-agent decisions may be incorrect");
1234        e.into_inner()
1235    })
1236}
1237
1238/// Helper to acquire the degraded agents lock, logging and recovering on poison.
1239fn degraded_agents_lock() -> MutexGuard<'static, HashMap<String, String>> {
1240    degraded_agents().lock().unwrap_or_else(|e| {
1241        tracing::error!("degraded agents lock poisoned — recovering with partial state; health check decisions may be incorrect");
1242        e.into_inner()
1243    })
1244}
1245
1246/// Helper to acquire the GitHub 5xx timestamps lock, logging and recovering on poison.
1247fn github_5xx_timestamps_lock() -> MutexGuard<'static, Vec<i64>> {
1248    github_5xx_timestamps().lock().unwrap_or_else(|e| {
1249        tracing::error!("github 5xx timestamps lock poisoned — recovering with partial state; circuit breaker may operate incorrectly");
1250        e.into_inner()
1251    })
1252}
1253
1254/// Helper to acquire the GitHub 5xx circuit open lock, logging and recovering on poison.
1255fn github_5xx_circuit_open_lock() -> MutexGuard<'static, Option<i64>> {
1256    github_5xx_circuit_open().lock().unwrap_or_else(|e| {
1257        tracing::error!("github 5xx circuit open lock poisoned — recovering with partial state; circuit breaker may operate incorrectly");
1258        e.into_inner()
1259    })
1260}
1261
1262/// Check if a key is in cooldown with the given reason.
1263fn is_in_cooldown_with_reason(key: &str, reason: &str) -> bool {
1264    let map = cooldown_lock();
1265    map.get(key).is_some_and(|entry| {
1266        entry.reason == reason && chrono::Utc::now().timestamp() < entry.cooldown_until
1267    })
1268}
1269
1270/// Check if a key is currently in cooldown.
1271fn is_in_cooldown(key: &str) -> bool {
1272    let map = cooldown_lock();
1273    if let Some(entry) = map.get(key) {
1274        let now = chrono::Utc::now().timestamp();
1275        return now < entry.cooldown_until;
1276    }
1277    false
1278}
1279
1280// ---------------------------------------------------------------------------
1281// Degraded-agent tracking (pre-emptive health check)
1282// ---------------------------------------------------------------------------
1283
1284/// Default health check window for recent rate-limit events (hours).
1285pub const DEFAULT_HEALTH_CHECK_WINDOW_HOURS: u32 = 6;
1286
1287/// Default threshold: minimum rate-limit events within the window to consider
1288/// an agent degraded.  A single transient 429 shouldn't mark an agent; we need
1289/// a pattern of repeated failures.
1290pub const DEFAULT_DEGRADED_RATE_LIMIT_THRESHOLD: i64 = 3;
1291
1292/// In-memory map of agents currently flagged as degraded, keyed by agent
1293/// with the reason that holds now
1294fn degraded_agents() -> &'static Mutex<HashMap<String, String>> {
1295    static DEGRADED: OnceLock<Mutex<HashMap<String, String>>> = OnceLock::new();
1296    DEGRADED.get_or_init(|| Mutex::new(HashMap::new()))
1297}
1298
1299/// Check if an agent is currently flagged as degraded by the health check.
1300///
1301/// Degraded means: all configured models are cooled **or** the agent has
1302/// exceeded the rate-limit threshold within the configured lookback window.
1303pub fn is_agent_degraded(agent: &str) -> bool {
1304    let set = degraded_agents_lock();
1305    set.contains_key(agent)
1306}
1307
1308/// Reason the agent is currently degraded, if it is
1309pub fn agent_degraded_reason(agent: &str) -> Option<String> {
1310    degraded_agents_lock().get(agent).cloned()
1311}
1312
1313/// Active degraded agents as `(agent, reason)`, sorted by agent name
1314pub fn list_degraded_agents() -> Vec<(String, String)> {
1315    let mut entries: Vec<(String, String)> = degraded_agents_lock()
1316        .iter()
1317        .map(|(agent, reason)| (agent.clone(), reason.clone()))
1318        .collect();
1319    entries.sort_by(|a, b| a.0.cmp(&b.0));
1320    entries
1321}
1322
1323/// Mark an agent as degraded (called by [`refresh_degraded_agents`]).
1324pub fn mark_agent_degraded(agent: &str) {
1325    mark_agent_degraded_with_reason(agent, "degraded");
1326}
1327
1328fn mark_agent_degraded_with_reason(agent: &str, reason: &str) {
1329    let mut set = degraded_agents_lock();
1330    set.insert(agent.to_string(), reason.to_string());
1331}
1332
1333/// Clear the degraded flag for an agent.
1334pub fn clear_agent_degraded(agent: &str) {
1335    let mut set = degraded_agents_lock();
1336    set.remove(agent);
1337}
1338
1339/// Refresh the degraded-agent set from the rate_limits table and cooldown state.
1340///
1341/// An agent is marked degraded when:
1342/// 1. It is in agent-level cooldown (explicit cooldown KV key exists and hasn't expired), **or**
1343/// 2. All its configured models are in cooldown, **or**
1344/// 3. It has >= `threshold` `limit_type = 'rate'` events within `window_hours`.
1345///
1346/// Failures keep their own cooldowns and do not count toward the threshold
1347/// Agents that no longer meet these criteria are cleared.
1348///
1349/// Note: This function calls `clear_expired_cooldowns()` first to ensure the
1350/// in-memory cooldown map reflects current state. An agent should NOT be marked
1351/// degraded simply because an expired cooldown entry hasn't been cleaned up yet.
1352pub async fn refresh_degraded_agents(
1353    store: &Arc<TaskStore>,
1354    available_agents: &[String],
1355    model_checker: &(dyn Fn(&str) -> bool + Send + Sync),
1356    window_hours: u32,
1357    threshold: i64,
1358) {
1359    // Clear expired cooldowns from in-memory map before evaluating agent state.
1360    // This ensures we don't incorrectly mark agents as degraded due to stale
1361    // in-memory entries that should have expired but weren't cleaned up.
1362    clear_expired_cooldowns();
1363
1364    // Time the DB query to surface slow health-checks that can dominate the
1365    // engine tick latency. If the query fails, log and return early.
1366    let start = chrono::Utc::now();
1367    let counts = match store.recent_rate_limit_counts(window_hours).await {
1368        Ok(c) => {
1369            let dur_ms = (chrono::Utc::now() - start).num_milliseconds();
1370            tracing::info!(
1371                duration_ms = dur_ms,
1372                window_hours,
1373                "recent_rate_limit_counts query completed"
1374            );
1375            c
1376        }
1377        Err(e) => {
1378            let dur_ms = (chrono::Utc::now() - start).num_milliseconds();
1379            tracing::warn!(
1380                err = %e,
1381                duration_ms = dur_ms,
1382                "failed to query recent rate limit counts for health check"
1383            );
1384            return;
1385        }
1386    };
1387
1388    for agent in available_agents {
1389        let in_cooldown = is_agent_in_cooldown(agent);
1390        let no_models = !model_checker(agent);
1391        let rate_limit_count = counts.get(agent.as_str()).copied().unwrap_or(0);
1392        let over_threshold = rate_limit_count >= threshold;
1393
1394        let degraded = in_cooldown || no_models || over_threshold;
1395
1396        if degraded {
1397            let reason = if in_cooldown {
1398                "agent in cooldown"
1399            } else if no_models {
1400                "all models cooled"
1401            } else {
1402                "rate limit threshold exceeded"
1403            };
1404            let previous = agent_degraded_reason(agent);
1405            match previous.as_deref() {
1406                None => tracing::warn!(
1407                    agent,
1408                    reason,
1409                    rate_limit_count,
1410                    window_hours,
1411                    threshold,
1412                    "pre-emptive health check: marking agent as degraded"
1413                ),
1414                Some(prev) if prev != reason => tracing::warn!(
1415                    agent,
1416                    reason,
1417                    previous = prev,
1418                    rate_limit_count,
1419                    window_hours,
1420                    threshold,
1421                    "pre-emptive health check: degraded reason changed"
1422                ),
1423                Some(_) => {}
1424            }
1425            mark_agent_degraded_with_reason(agent, reason);
1426        } else {
1427            if is_agent_degraded(agent) {
1428                tracing::info!(
1429                    agent,
1430                    rate_limit_count,
1431                    "pre-emptive health check: agent recovered, clearing degraded flag"
1432                );
1433            }
1434            clear_agent_degraded(agent);
1435        }
1436    }
1437}
1438
1439/// Clear expired cooldowns from the in-memory map.
1440pub fn clear_expired_cooldowns() {
1441    let mut map = cooldown_lock();
1442    let now = chrono::Utc::now().timestamp();
1443    map.retain(|_key, entry| now < entry.cooldown_until);
1444}
1445
1446// ---------------------------------------------------------------------------
1447// GitHub 5xx circuit breaker
1448// ---------------------------------------------------------------------------
1449
1450/// Sliding window for counting GitHub 5xx errors (seconds).
1451const GITHUB_5XX_WINDOW_SECS: i64 = 120;
1452
1453/// Number of 5xx errors within the window to trip the circuit breaker.
1454const GITHUB_5XX_THRESHOLD: usize = 5;
1455
1456/// Duration the circuit breaker stays open after tripping (seconds).
1457const GITHUB_5XX_COOLDOWN_SECS: i64 = 180;
1458
1459/// Global sliding window of 5xx error timestamps.
1460fn github_5xx_timestamps() -> &'static Mutex<Vec<i64>> {
1461    static TS: OnceLock<Mutex<Vec<i64>>> = OnceLock::new();
1462    TS.get_or_init(|| Mutex::new(Vec::new()))
1463}
1464
1465/// Whether the GitHub 5xx circuit breaker is currently open (tripped).
1466fn github_5xx_circuit_open() -> &'static Mutex<Option<i64>> {
1467    static OPEN: OnceLock<Mutex<Option<i64>>> = OnceLock::new();
1468    OPEN.get_or_init(|| Mutex::new(None))
1469}
1470
1471/// Record a GitHub 5xx server error and trip the circuit breaker if the
1472/// sliding window threshold is exceeded.
1473///
1474/// Emits a single high-level log entry when the circuit transitions to open.
1475pub async fn record_github_5xx() {
1476    let now = chrono::Utc::now().timestamp();
1477    let window_start = now - GITHUB_5XX_WINDOW_SECS;
1478
1479    let should_trip = {
1480        let mut ts = github_5xx_timestamps_lock();
1481        ts.retain(|t| *t >= window_start);
1482        ts.push(now);
1483        ts.len() >= GITHUB_5XX_THRESHOLD
1484    };
1485
1486    if should_trip {
1487        let cooldown_until = {
1488            let mut open = github_5xx_circuit_open_lock();
1489            if open.is_none() {
1490                let cd = now + GITHUB_5XX_COOLDOWN_SECS;
1491                *open = Some(cd);
1492                tracing::warn!(
1493                    errors_in_window = GITHUB_5XX_THRESHOLD,
1494                    window_secs = GITHUB_5XX_WINDOW_SECS,
1495                    cooldown_secs = GITHUB_5XX_COOLDOWN_SECS,
1496                    "GitHub 5xx circuit breaker OPEN — throttling non-critical work"
1497                );
1498                Some(cd)
1499            } else {
1500                None
1501            }
1502        };
1503        // Persist to KV (outside the mutex lock so the future is Send).
1504        if let Some(cd) = cooldown_until {
1505            let store_opt = cooldown_store().lock().await.clone();
1506            if let Some(store) = store_opt {
1507                let _ = store.kv_set("cooldown:github:5xx", &cd.to_string()).await;
1508            }
1509            // Also set the generic cooldown so is_agent_in_cooldown("github:5xx")
1510            // returns true immediately, avoiding a race window where concurrent
1511            // requests bypass the circuit breaker.
1512            set_agent_cooldown("github:5xx", GITHUB_5XX_COOLDOWN_SECS as u64).await;
1513        }
1514        // Clear the sliding window after tripping so we don't re-trip immediately
1515        let mut ts = github_5xx_timestamps_lock();
1516        ts.clear();
1517    }
1518}
1519
1520/// Check if the GitHub 5xx circuit breaker is currently open.
1521///
1522/// When open, non-critical background work (cleanup, polling, mention scans)
1523/// should be skipped. Critical operations (PR creation) should still attempt
1524/// with their existing exponential backoff.
1525///
1526/// Automatically recovers (logs a recovery message) when the cooldown expires.
1527pub fn is_github_circuit_open() -> bool {
1528    // Check the dedicated in-memory circuit flag first. If present, honour it
1529    // and perform auto-recovery when it expires (logging once on close).
1530    {
1531        let mut open = github_5xx_circuit_open_lock();
1532        if let Some(until) = *open {
1533            let now = chrono::Utc::now().timestamp();
1534            if now >= until {
1535                *open = None;
1536                tracing::info!("GitHub 5xx circuit breaker CLOSED — resuming normal operations");
1537                return false;
1538            }
1539            return true;
1540        }
1541    }
1542
1543    // If the dedicated flag is not set, consult the generic cooldown map so
1544    // callers see a consistent view regardless of which code path set the
1545    // persisted cooldown (e.g. send_with_retries -> set_agent_cooldown).
1546    if let Some(until) = cooldown_until("github:5xx") {
1547        // Populate the dedicated in-memory flag so future checks and the
1548        // auto-recovery path go through the same code and logging.
1549        let mut open = github_5xx_circuit_open_lock();
1550        if open.is_none() {
1551            *open = Some(until);
1552            let now = chrono::Utc::now().timestamp();
1553            if now < until {
1554                let remaining = until - now;
1555                tracing::warn!(
1556                    remaining_secs = remaining,
1557                    "GitHub 5xx circuit breaker OPEN (discovered via generic cooldown) — throttling non-critical work"
1558                );
1559            }
1560        }
1561        return true;
1562    }
1563
1564    false
1565}
1566
1567/// Returns the remaining seconds of the GitHub 5xx circuit breaker cooldown,
1568/// or 0 if the circuit is closed.
1569pub fn github_circuit_remaining_secs() -> u64 {
1570    // Prefer the dedicated in-memory value when present.
1571    {
1572        let open = github_5xx_circuit_open_lock();
1573        if let Some(until) = *open {
1574            let now = chrono::Utc::now().timestamp();
1575            if now < until {
1576                return u64::try_from(until - now).unwrap_or(0);
1577            }
1578            return 0;
1579        }
1580    }
1581
1582    // Fall back to the generic cooldown map if the dedicated flag isn't set.
1583    if let Some(until) = cooldown_until("github:5xx") {
1584        let now = chrono::Utc::now().timestamp();
1585        if now < until {
1586            return u64::try_from(until - now).unwrap_or(0);
1587        }
1588    }
1589    0
1590}
1591
1592/// Parse a "try again at {date}" or "reset at {date}" from an error message.
1593///
1594/// Returns a Unix timestamp if a retry-at date is found.
1595pub(crate) fn parse_retry_at(error_message: &str) -> Option<i64> {
1596    if error_message.is_empty() {
1597        return None;
1598    }
1599
1600    let lower = error_message.to_lowercase();
1601    // Accept a wider set of vendor-provided markers. Claude commonly uses
1602    // variations like "resets 10:30am" (no explicit "at"), so include
1603    // both "resets" and the older "reset at"/"try again at" markers.
1604    for retry_marker in [
1605        "try again at ",
1606        "reset at ",
1607        "resets at ",
1608        "resets ",
1609        "reset ",
1610    ] {
1611        if let Some(idx) = lower.find(retry_marker) {
1612            let date_str = &lower[idx + retry_marker.len()..];
1613            // Take until period, newline, or end
1614            let date_str = date_str
1615                .split(['.', '\n'])
1616                .next()
1617                .unwrap_or(date_str)
1618                .trim();
1619
1620            // Try common date formats
1621            // "Mar 26th, 2026 5:55 AM" → strip ordinal suffixes
1622            let cleaned = date_str
1623                .replace("st,", ",")
1624                .replace("nd,", ",")
1625                .replace("rd,", ",")
1626                .replace("th,", ",");
1627
1628            for fmt in &[
1629                "%b %d, %Y %I:%M %p", // Mar 26, 2026 5:55 AM
1630                "%b %d, %Y %I:%M%p",  // Mar 26, 2026 5:55AM
1631                "%B %d, %Y %I:%M %p", // March 26, 2026 5:55 AM
1632                "%Y-%m-%dT%H:%M:%S",  // 2026-03-26T05:55:00
1633                "%Y-%m-%d %H:%M:%S",  // 2026-03-26 05:55:00
1634                "%Y-%m-%d %H:%M",     // 2026-03-26 05:55
1635            ] {
1636                if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(&cleaned, fmt) {
1637                    // Vendor messages (e.g. Codex/GLM) emit times in the machine's
1638                    // local timezone, not UTC. Interpret naive datetime as local time.
1639                    let utc = match chrono::Local.from_local_datetime(&dt) {
1640                        chrono::LocalResult::Single(local_dt) => {
1641                            local_dt.with_timezone(&chrono::Utc)
1642                        }
1643                        chrono::LocalResult::Ambiguous(earliest, _) => {
1644                            // DST fold: pick the earlier (conservative) interpretation
1645                            earliest.with_timezone(&chrono::Utc)
1646                        }
1647                        chrono::LocalResult::None => {
1648                            // DST gap: fall back to treating as UTC to avoid panic
1649                            dt.and_utc()
1650                        }
1651                    };
1652                    tracing::info!(
1653                        retry_at = %utc,
1654                        raw = date_str,
1655                        marker = retry_marker,
1656                        "parsed rate limit retry-at date"
1657                    );
1658                    return Some(utc.timestamp());
1659                }
1660            }
1661            tracing::debug!(
1662                raw = date_str,
1663                marker = retry_marker,
1664                "could not parse retry-at date"
1665            );
1666
1667            // If the vendor provided only a time (e.g. "resets 10:30am"),
1668            // attempt to parse a time-only and interpret it as the next
1669            // occurrence of that time in the local timezone.
1670            // Strip parenthesised timezone hints like "(America/Sao_Paulo)".
1671            let time_only = date_str.split_once('(').map_or(date_str, |(t, _)| t).trim();
1672            // Hour-only values like "8am" have no minute field, which chrono needs, so insert ":00"
1673            let mut compact = time_only.replace(" am", "am").replace(" pm", "pm");
1674            let split = compact.rfind(' ').map_or(0, |i| i + 1);
1675            let last = &compact[split..];
1676            if !last.contains(':')
1677                && (last.ends_with("am") || last.ends_with("pm"))
1678                && last.len() > 2
1679            {
1680                let at = split + last.len() - 2;
1681                compact.insert_str(at, ":00");
1682            }
1683            // "Jun 9 at 1am" has no year, so take the next future occurrence
1684            let md = compact.replace(" at ", " ");
1685            let now_local = chrono::Local::now();
1686            let this_year = chrono::Datelike::year(&now_local);
1687            for year in [this_year, this_year + 1] {
1688                let with_year = format!("{year} {md}");
1689                if let Ok(dt) =
1690                    chrono::NaiveDateTime::parse_from_str(&with_year, "%Y %b %d %I:%M%p")
1691                    && let Some(local_dt) = chrono::Local.from_local_datetime(&dt).earliest()
1692                    && local_dt > now_local
1693                {
1694                    return Some(local_dt.timestamp());
1695                }
1696            }
1697            let time_only = compact.as_str();
1698            for tf in &["%I:%M %p", "%I:%M%p", "%I %p", "%H:%M"] {
1699                if let Ok(t) = chrono::NaiveTime::parse_from_str(time_only, tf) {
1700                    // Build a NaiveDateTime for today at that time, then choose
1701                    // today or tomorrow depending on whether the time has passed.
1702                    let local_now = chrono::Local::now();
1703                    let today = local_now.date_naive();
1704                    let candidate = chrono::NaiveDateTime::new(today, t);
1705                    // Use the TimeZone trait methods; annotate types so the
1706                    // compiler can infer the concrete DateTime types.
1707                    let candidate_local: chrono::DateTime<chrono::Local> =
1708                        match chrono::Local.from_local_datetime(&candidate) {
1709                            chrono::LocalResult::Single(dt) => dt,
1710                            chrono::LocalResult::Ambiguous(earliest, _) => earliest,
1711                            chrono::LocalResult::None => {
1712                                // DST gap — fall back to UTC interpretation
1713                                let utc_dt = candidate.and_utc();
1714                                return Some(utc_dt.timestamp());
1715                            }
1716                        };
1717                    let chosen = if candidate_local <= local_now {
1718                        // Use tomorrow
1719                        let tomorrow = today.succ_opt().unwrap_or(today);
1720                        let ndt = chrono::NaiveDateTime::new(tomorrow, t);
1721                        match chrono::Local.from_local_datetime(&ndt) {
1722                            chrono::LocalResult::Single(dt) => dt.with_timezone(&chrono::Utc),
1723                            chrono::LocalResult::Ambiguous(earliest, _) => {
1724                                earliest.with_timezone(&chrono::Utc)
1725                            }
1726                            chrono::LocalResult::None => ndt.and_utc(),
1727                        }
1728                    } else {
1729                        candidate_local.with_timezone(&chrono::Utc)
1730                    };
1731                    tracing::info!(
1732                        retry_at = %chosen,
1733                        raw = date_str,
1734                        marker = retry_marker,
1735                        "parsed rate limit retry-at time-only; interpreted as next occurrence"
1736                    );
1737                    return Some(chosen.timestamp());
1738                }
1739            }
1740        }
1741    }
1742
1743    // Without a specific retry/reset date, return None and let the caller use
1744    // its default cooldown (exponential backoff). A blanket long fallback here
1745    // would cause false billing-cycle cooldowns on agents without those limits.
1746    None
1747}
1748
1749/// Parse a relative usage window (e.g. "5-hour usage limit") from an error
1750/// message.
1751///
1752/// Returns the window length in seconds. Only matches in usage/quota/limit
1753/// contexts ("usage limit", "quota", "window", "limit") to avoid interpreting
1754/// unrelated numbers as cooldown durations.
1755///
1756/// `pub(crate)` so `engine::runner::fallback::handle_error()` can apply the
1757/// same window-detection the `needs_review` call site
1758/// (`record_rate_limit()`, below) already uses — see issue #3580.
1759pub(crate) fn parse_relative_usage_window(error_message: &str) -> Option<u64> {
1760    static WINDOW_RE: LazyLock<Regex> = LazyLock::new(|| {
1761        Regex::new(
1762            r"(?i)(?:^|\s|[\(\)\.,;:!?])(?:(\d+)\s*-?\s*(hour|hours|hr|hrs|minute|minutes|min|mins|day|days)|(hourly|daily|weekly|monthly))(?:$|\s|[\(\)\.,;:!?])"
1763        )
1764        .expect("usage window regex is valid")
1765    });
1766
1767    if error_message.is_empty() {
1768        return None;
1769    }
1770
1771    let lower = error_message.to_lowercase();
1772    let has_context = lower.contains("usage limit")
1773        || lower.contains("quota")
1774        || lower.contains("window")
1775        || lower.contains("limit");
1776    if !has_context {
1777        return None;
1778    }
1779
1780    let mut max_secs: Option<u64> = None;
1781    for caps in WINDOW_RE.captures_iter(&lower) {
1782        let secs = if let Some(num) = caps.get(1) {
1783            let n: u64 = num.as_str().parse().ok()?;
1784            let unit = caps.get(2)?.as_str().to_lowercase();
1785            let multiplier = match unit.as_str() {
1786                "hour" | "hours" | "hr" | "hrs" => 3600,
1787                "minute" | "minutes" | "min" | "mins" => 60,
1788                "day" | "days" => 86400,
1789                _ => return None,
1790            };
1791            n.checked_mul(multiplier)?
1792        } else if let Some(word) = caps.get(3) {
1793            match word.as_str().to_lowercase().as_str() {
1794                "hourly" => 3600,
1795                "daily" => 86400,
1796                "weekly" => 604_800,
1797                "monthly" => 2_592_000,
1798                _ => return None,
1799            }
1800        } else {
1801            continue;
1802        };
1803        max_secs = Some(max_secs.map_or(secs, |m| m.max(secs)));
1804    }
1805
1806    max_secs
1807}
1808
1809#[cfg(test)]
1810pub(crate) async fn reset_global_state() {
1811    cooldowns().lock().unwrap().clear();
1812    *cooldown_store().lock().await = None;
1813    github_5xx_timestamps().lock().unwrap().clear();
1814    *github_5xx_circuit_open().lock().unwrap() = None;
1815    degraded_agents().lock().unwrap().clear();
1816    opencode::reset_model_caches_for_test();
1817}
1818
1819#[cfg(test)]
1820mod tests {
1821    use std::time::Duration;
1822
1823    use super::*;
1824    use chrono::Datelike;
1825    use chrono::TimeZone;
1826    use serial_test::serial;
1827    use tokio::time;
1828
1829    async fn test_store() -> Arc<TaskStore> {
1830        Arc::new(TaskStore::open_memory().await.expect("in-memory store"))
1831    }
1832
1833    fn sqlite_datetime_offset(secs: i64) -> String {
1834        (chrono::Utc::now() + chrono::Duration::seconds(secs))
1835            .format("%Y-%m-%d %H:%M:%S")
1836            .to_string()
1837    }
1838
1839    #[serial(cooldown_state)]
1840    #[tokio::test]
1841    async fn init_loads_unexpired_cooldowns_from_kv() {
1842        super::reset_global_state().await;
1843        let store = test_store().await;
1844        let cooldown_until = chrono::Utc::now().timestamp() + 3600; // 1 hour from now
1845
1846        store
1847            .kv_set(
1848                &format!("{KV_PREFIX}kimi:k2p5"),
1849                &cooldown_until.to_string(),
1850            )
1851            .await
1852            .unwrap();
1853
1854        // Also add an already-expired entry
1855        let expired = chrono::Utc::now().timestamp() - 1;
1856        store
1857            .kv_set(
1858                &format!("{KV_PREFIX}opencode:old-free"),
1859                &expired.to_string(),
1860            )
1861            .await
1862            .unwrap();
1863
1864        {
1865            let mut map = cooldown_lock();
1866            map.clear();
1867        }
1868
1869        init_cooldown_store(store).await;
1870
1871        assert!(
1872            is_model_in_cooldown("kimi", "k2p5"),
1873            "unexpired cooldown should be loaded"
1874        );
1875        assert!(
1876            !is_model_in_cooldown("opencode", "old-free"),
1877            "expired cooldown should not be loaded"
1878        );
1879    }
1880
1881    #[serial(cooldown_state)]
1882    #[tokio::test]
1883    async fn credit_exhaustion_inside_active_cooldown_keeps_counter_at_one() {
1884        super::reset_global_state().await;
1885        let store = test_store().await;
1886        *cooldown_store().lock().await = Some(Arc::clone(&store));
1887
1888        let msg =
1889            "API error (status 402 Payment Required): usage balance exhausted, out of credits";
1890        assert!(record_credit_exhaustion_from_message("credit_dup_agent", Some("m"), msg).await);
1891        assert!(record_credit_exhaustion_from_message("credit_dup_agent", Some("m"), msg).await);
1892
1893        assert!(is_agent_in_cooldown("credit_dup_agent"));
1894        let count = store
1895            .kv_get(&format!("{CREDIT_FAILURE_COUNT_PREFIX}credit_dup_agent"))
1896            .await
1897            .unwrap();
1898        assert_eq!(count.as_deref(), Some("1"));
1899    }
1900
1901    #[serial(cooldown_state)]
1902    #[tokio::test]
1903    async fn non_credit_message_is_not_recorded_as_credit_exhaustion() {
1904        super::reset_global_state().await;
1905        assert!(!record_credit_exhaustion_from_message("x_agent", Some("m"), "boom").await);
1906        assert!(!is_agent_in_cooldown("x_agent"));
1907    }
1908
1909    #[serial(cooldown_state)]
1910    #[tokio::test]
1911    async fn record_model_failure_writes_to_kv() {
1912        super::reset_global_state().await;
1913        let store = test_store().await;
1914
1915        *cooldown_store().lock().await = Some(Arc::clone(&store));
1916
1917        record_model_failure("testagent_persist", "testmodel_persist").await;
1918
1919        // Give the async KV write a moment to settle before querying.
1920        time::sleep(Duration::from_millis(50)).await;
1921
1922        let kv_key = format!("{KV_PREFIX}testagent_persist:testmodel_persist");
1923        let stored = store.kv_get(&kv_key).await.unwrap();
1924        assert!(stored.is_some(), "model failure should be persisted to KV");
1925        let (ts, reason) = parse_cooldown_value(&stored.unwrap()).expect("cooldown value");
1926        let now = chrono::Utc::now().timestamp();
1927        // Should be in the future (cooldown_until, not failed_at)
1928        assert!(ts > now, "persisted timestamp should be in the future");
1929        assert_eq!(reason, "model_error");
1930    }
1931
1932    #[serial(cooldown_state)]
1933    #[tokio::test]
1934    async fn record_transient_model_failure_backs_off_5_then_15_minutes() {
1935        super::reset_global_state().await;
1936        let store = test_store().await;
1937        *cooldown_store().lock().await = Some(Arc::clone(&store));
1938
1939        let agent = "testagent_transient";
1940        let model = "testmodel_transient";
1941        let key = format!("{agent}:{model}");
1942
1943        record_transient_model_failure(agent, model).await;
1944        let first = cooldown_until(&key)
1945            .expect("first cooldown should exist")
1946            .saturating_sub(chrono::Utc::now().timestamp());
1947        assert!(
1948            (TRANSIENT_MODEL_BACKOFF_BASE_SECS - 5..=TRANSIENT_MODEL_BACKOFF_BASE_SECS + 5)
1949                .contains(&first),
1950            "first transient cooldown should be ~5m, got {first}s"
1951        );
1952
1953        record_transient_model_failure(agent, model).await;
1954        let second = cooldown_until(&key)
1955            .expect("second cooldown should exist")
1956            .saturating_sub(chrono::Utc::now().timestamp());
1957        assert!(
1958            (TRANSIENT_MODEL_BACKOFF_MAX_SECS - 5..=TRANSIENT_MODEL_BACKOFF_MAX_SECS + 5)
1959                .contains(&second),
1960            "second transient cooldown should be capped at ~15m, got {second}s"
1961        );
1962
1963        let (_, reason) = parse_cooldown_value(
1964            &store
1965                .kv_get(&format!("{KV_PREFIX}{key}"))
1966                .await
1967                .unwrap()
1968                .expect("cooldown persisted"),
1969        )
1970        .expect("cooldown value");
1971        assert_eq!(reason, "transient_model_error");
1972
1973        let count = store
1974            .kv_get(&format!("{FAILURE_COUNT_PREFIX}{key}"))
1975            .await
1976            .unwrap();
1977        assert_eq!(count.as_deref(), Some("2"));
1978    }
1979
1980    #[serial(cooldown_state)]
1981    #[tokio::test]
1982    async fn record_model_failure_and_persistent_failure_use_separate_counters() {
1983        super::reset_global_state().await;
1984        let store = test_store().await;
1985        *cooldown_store().lock().await = Some(Arc::clone(&store));
1986
1987        let agent = "testagent_counters";
1988        let model = "testmodel_counters";
1989        let key = format!("{agent}:{model}");
1990
1991        record_model_failure(agent, model).await;
1992        record_persistent_model_failure(agent, model).await;
1993        record_persistent_model_failure(agent, model).await;
1994
1995        let generic = store
1996            .kv_get(&format!("{FAILURE_COUNT_PREFIX}{key}"))
1997            .await
1998            .unwrap();
1999        let persistent = store
2000            .kv_get(&format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{key}"))
2001            .await
2002            .unwrap();
2003        assert_eq!(generic.as_deref(), Some("1"));
2004        assert_eq!(persistent.as_deref(), Some("2"));
2005
2006        let (_, reason) = parse_cooldown_value(
2007            &store
2008                .kv_get(&format!("{KV_PREFIX}{key}"))
2009                .await
2010                .unwrap()
2011                .expect("cooldown persisted"),
2012        )
2013        .expect("cooldown value");
2014        assert_eq!(reason, "persistent_model_error");
2015    }
2016
2017    #[serial(cooldown_state)]
2018    #[tokio::test]
2019    async fn cooldown_reason_survives_restart_via_kv() {
2020        super::reset_global_state().await;
2021        let store = test_store().await;
2022        let agent = "test_reason_restart";
2023        let future = chrono::Utc::now().timestamp() + 600;
2024
2025        init_cooldown_store(Arc::clone(&store)).await;
2026        set_cooldown_async(agent, future, "billing_cycle_exhausted").await;
2027
2028        // Simulate a service restart: drop the in-memory map, reload from KV.
2029        {
2030            let mut map = cooldown_lock();
2031            map.clear();
2032        }
2033        init_cooldown_store(store).await;
2034
2035        let entries = list_all_cooldowns();
2036        let entry = entries.iter().find(|(key, ..)| key == agent);
2037        assert_eq!(
2038            entry.map(|(_, _, reason)| reason.as_str()),
2039            Some("billing_cycle_exhausted"),
2040            "cooldown reason must survive a restart, not collapse to 'persisted'"
2041        );
2042    }
2043
2044    #[serial(cooldown_state)]
2045    #[tokio::test]
2046    async fn record_persistent_model_failure_escalates_and_increments_failure_count() {
2047        super::reset_global_state().await;
2048        let store = test_store().await;
2049        *cooldown_store().lock().await = Some(Arc::clone(&store));
2050
2051        let agent = "testagent_persistent";
2052        let model = "testmodel_persistent";
2053        let key = format!("{agent}:{model}");
2054
2055        // First failure -> base 4h.
2056        record_persistent_model_failure(agent, model).await;
2057        let now = chrono::Utc::now().timestamp();
2058        let first_until = cooldown_until(&key).expect("first cooldown should exist");
2059        let first_remaining = first_until.saturating_sub(now);
2060        assert!(
2061            first_remaining >= PERSISTENT_MODEL_BACKOFF_BASE_SECS - 5,
2062            "first persistent model cooldown should be ~4h, got {first_remaining}s"
2063        );
2064
2065        // Second failure -> 12h (3x growth).
2066        record_persistent_model_failure(agent, model).await;
2067        let now2 = chrono::Utc::now().timestamp();
2068        let second_until = cooldown_until(&key).expect("second cooldown should exist");
2069        let second_remaining = second_until.saturating_sub(now2);
2070        assert!(
2071            second_remaining >= (PERSISTENT_MODEL_BACKOFF_BASE_SECS * 3) - 5,
2072            "second persistent model cooldown should be ~12h, got {second_remaining}s"
2073        );
2074
2075        let count_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{agent}:{model}");
2076        let stored_count = store
2077            .kv_get(&count_key)
2078            .await
2079            .expect("kv read should succeed")
2080            .expect("failure count should be written");
2081        assert_eq!(stored_count, "2");
2082    }
2083
2084    #[serial(cooldown_state)]
2085    #[tokio::test]
2086    async fn record_agent_success_does_not_reset_persistent_failure_count() {
2087        super::reset_global_state().await;
2088        let store = test_store().await;
2089        *cooldown_store().lock().await = Some(Arc::clone(&store));
2090
2091        let agent = "testagent_persistent_success";
2092        let model = "testmodel_persistent_success";
2093        let key = format!("{agent}:{model}");
2094
2095        // Two persistent failures escalate the counter to 2 (~12h cooldown).
2096        record_persistent_model_failure(agent, model).await;
2097        record_persistent_model_failure(agent, model).await;
2098
2099        let count_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{agent}:{model}");
2100        let stored_count = store
2101            .kv_get(&count_key)
2102            .await
2103            .expect("kv read should succeed")
2104            .expect("failure count should be written");
2105        assert_eq!(stored_count, "2");
2106
2107        // A successful run resets the generic failure_count but must NOT touch
2108        // the persistent counter — otherwise an intervening success would erase
2109        // escalation progress for a structurally unreliable model.
2110        record_agent_success(agent, model).await;
2111
2112        let stored_count_after_success = store
2113            .kv_get(&count_key)
2114            .await
2115            .expect("kv read should succeed")
2116            .expect("persistent failure count should still exist");
2117        assert_eq!(
2118            stored_count_after_success, "2",
2119            "persistent failure count must survive an intervening success"
2120        );
2121
2122        // The next persistent failure should escalate from 3 (~36h), not reset to base (4h).
2123        record_persistent_model_failure(agent, model).await;
2124        let now = chrono::Utc::now().timestamp();
2125        let third_until = cooldown_until(&key).expect("third cooldown should exist");
2126        let third_remaining = third_until.saturating_sub(now);
2127        assert!(
2128            third_remaining > PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2129            "third persistent model failure should escalate past the 4h base, got {third_remaining}s"
2130        );
2131    }
2132
2133    #[serial(cooldown_state)]
2134    #[tokio::test]
2135    async fn record_persistent_model_failure_stays_capped_at_7_days() {
2136        super::reset_global_state().await;
2137        let store = test_store().await;
2138        *cooldown_store().lock().await = Some(Arc::clone(&store));
2139
2140        let agent = "testagent_persistent_capped";
2141        let model = "testmodel_persistent_capped";
2142        let key = format!("{agent}:{model}");
2143
2144        for _ in 0..9 {
2145            record_persistent_model_failure(agent, model).await;
2146        }
2147
2148        let remaining = cooldown_until(&key)
2149            .expect("cooldown should exist")
2150            .saturating_sub(chrono::Utc::now().timestamp());
2151        assert!(
2152            remaining >= PERSISTENT_MODEL_BACKOFF_MAX_SECS - 5,
2153            "persistent model cooldown should stay capped at 7 days, got {remaining}s"
2154        );
2155        assert!(
2156            remaining <= PERSISTENT_MODEL_BACKOFF_MAX_SECS + 5,
2157            "persistent model cooldown should not exceed 7 days, got {remaining}s"
2158        );
2159    }
2160
2161    #[serial(cooldown_state)]
2162    #[test]
2163    fn parse_retry_at_codex_format() {
2164        let msg = "You've hit your usage limit. Upgrade to Pro, visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Mar 26th, 2026 5:55 AM.";
2165        let ts = parse_retry_at(msg);
2166        assert!(ts.is_some(), "should parse codex retry-at date");
2167
2168        // The parsed timestamp must equal "Mar 26 2026 05:55 local" in UTC.
2169        // We reconstruct what the function should produce and compare directly.
2170        let naive = chrono::NaiveDateTime::new(
2171            chrono::NaiveDate::from_ymd_opt(2026, 3, 26).unwrap(),
2172            chrono::NaiveTime::from_hms_opt(5, 55, 0).unwrap(),
2173        );
2174        let expected_utc = match chrono::Local.from_local_datetime(&naive) {
2175            chrono::LocalResult::Single(local_dt) => local_dt.with_timezone(&chrono::Utc),
2176            chrono::LocalResult::Ambiguous(earliest, _) => earliest.with_timezone(&chrono::Utc),
2177            chrono::LocalResult::None => naive.and_utc(),
2178        };
2179        assert_eq!(
2180            ts.unwrap(),
2181            expected_utc.timestamp(),
2182            "timestamp should reflect local-timezone interpretation of the vendor date"
2183        );
2184
2185        // Sanity: the UTC date must still be March 26 (true for all timezones within ±14h)
2186        let dt = chrono::DateTime::from_timestamp(ts.unwrap(), 0).unwrap();
2187        assert_eq!(dt.month(), 3);
2188        assert_eq!(dt.day(), 26);
2189    }
2190
2191    #[serial(cooldown_state)]
2192    #[test]
2193    fn parse_retry_at_hour_only_and_month_day() {
2194        let now = chrono::Utc::now().timestamp();
2195        for msg in [
2196            "You've hit your session limit · resets 8am (America/Sao_Paulo)",
2197            "You've hit your session limit · resets 6pm (America/Sao_Paulo)",
2198            "You've hit your session limit · resets 12am (America/Sao_Paulo)",
2199            "You've hit your weekly limit · resets Jun 9 at 1am (America/Sao_Paulo)",
2200        ] {
2201            let ts = parse_retry_at(msg).unwrap_or_else(|| panic!("should parse: {msg}"));
2202            assert!(ts > now, "{msg}");
2203        }
2204    }
2205
2206    #[serial(cooldown_state)]
2207    #[test]
2208    fn parse_retry_at_weekly_resets_without_year() {
2209        let t = chrono::Local::now() + chrono::Duration::days(2);
2210        let msg = format!(
2211            "You've hit your weekly limit · resets {} {} at 1am (America/Sao_Paulo)",
2212            t.format("%b"),
2213            t.day()
2214        );
2215        let ts = parse_retry_at(&msg).expect("should parse year-less reset date");
2216        let remaining = ts - chrono::Utc::now().timestamp();
2217        assert!(
2218            remaining > 3600 && remaining < 3 * 86400,
2219            "expected ~2d retry-at, got {remaining}s"
2220        );
2221    }
2222
2223    #[serial(cooldown_state)]
2224    #[test]
2225    fn parse_retry_at_no_date() {
2226        assert!(parse_retry_at("generic rate limit error").is_none());
2227        assert!(parse_retry_at("").is_none());
2228    }
2229
2230    #[serial(cooldown_state)]
2231    #[test]
2232    fn parse_retry_at_glm_reset_format_with_seconds() {
2233        let msg = "Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-04-23 18:33:48";
2234        let ts = parse_retry_at(msg).expect("should parse GLM reset-at date");
2235
2236        let naive = chrono::NaiveDateTime::new(
2237            chrono::NaiveDate::from_ymd_opt(2026, 4, 23).expect("valid date"),
2238            chrono::NaiveTime::from_hms_opt(18, 33, 48).expect("valid time"),
2239        );
2240        let expected_utc = match chrono::Local.from_local_datetime(&naive) {
2241            chrono::LocalResult::Single(local_dt) => local_dt.with_timezone(&chrono::Utc),
2242            chrono::LocalResult::Ambiguous(earliest, _) => earliest.with_timezone(&chrono::Utc),
2243            chrono::LocalResult::None => naive.and_utc(),
2244        };
2245
2246        assert_eq!(
2247            ts,
2248            expected_utc.timestamp(),
2249            "timestamp should reflect local-timezone interpretation of vendor reset date"
2250        );
2251    }
2252
2253    #[serial(cooldown_state)]
2254    #[test]
2255    fn parse_retry_at_billing_cycle_without_date_returns_none() {
2256        // Without a specific "try again at" date, parse_retry_at should return
2257        // None so callers use their default cooldown (30m agent / 1h model).
2258        let msg = "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle. Upgrade to get more.";
2259        assert!(
2260            parse_retry_at(msg).is_none(),
2261            "billing cycle without date should return None"
2262        );
2263    }
2264
2265    #[serial(cooldown_state)]
2266    #[test]
2267    fn parse_relative_usage_window_extracts_kimi_five_hour_window() {
2268        let msg = "Failed to authenticate. API Error: 403 You've reached your 5-hour usage limit. Your quota will reset when the current 5-hour window expires.";
2269        assert_eq!(
2270            parse_relative_usage_window(msg),
2271            Some(5 * 3600),
2272            "should extract the 5-hour window as seconds"
2273        );
2274    }
2275
2276    #[serial(cooldown_state)]
2277    #[test]
2278    fn parse_relative_usage_window_handles_common_durations() {
2279        assert_eq!(
2280            parse_relative_usage_window("You've reached your hourly usage limit."),
2281            Some(3600)
2282        );
2283        assert_eq!(
2284            parse_relative_usage_window("You've reached your daily usage limit."),
2285            Some(86400)
2286        );
2287        assert_eq!(
2288            parse_relative_usage_window("You've reached your weekly limit."),
2289            Some(604_800)
2290        );
2291        assert_eq!(
2292            parse_relative_usage_window("You've reached your 30-minute usage limit."),
2293            Some(1800)
2294        );
2295        assert_eq!(
2296            parse_relative_usage_window("You've reached your 2-day quota."),
2297            Some(172_800)
2298        );
2299    }
2300
2301    #[serial(cooldown_state)]
2302    #[test]
2303    fn parse_relative_usage_window_returns_none_without_usage_context() {
2304        // A bare rate-limit message with a number should not be interpreted as a
2305        // relative usage window unless usage/quota/window context is present.
2306        assert!(parse_relative_usage_window("429 Too Many Requests").is_none());
2307        assert!(parse_relative_usage_window("rate limit exceeded").is_none());
2308        assert!(parse_relative_usage_window("").is_none());
2309    }
2310
2311    #[serial(cooldown_state)]
2312    #[tokio::test]
2313    async fn record_rate_limit_uses_model_specific_window_cooldown() {
2314        super::reset_global_state().await;
2315        let store = test_store().await;
2316        *cooldown_store().lock().await = Some(Arc::clone(&store));
2317
2318        let agent = "test_record_rl_window_agent";
2319        let model = "test_record_rl_window_model";
2320        let msg = "You've reached your 5-hour usage limit. Your quota will reset when the current 5-hour window expires.";
2321
2322        record_rate_limit(agent, Some(model), msg).await;
2323
2324        assert!(
2325            is_model_in_cooldown(agent, model),
2326            "model should be cooled for the relative usage window"
2327        );
2328        assert!(
2329            !is_agent_in_cooldown(agent),
2330            "agent should not be cooled for a model-scoped quota"
2331        );
2332
2333        let key = format!("{agent}:{model}");
2334        let remaining = cooldown_until(&key)
2335            .expect("model cooldown should exist")
2336            .saturating_sub(chrono::Utc::now().timestamp());
2337        assert!(
2338            remaining >= 5 * 3600 - 5,
2339            "model cooldown should be at least ~5 hours, got {remaining}s"
2340        );
2341        assert_eq!(
2342            cooldown_reason(&key).as_deref(),
2343            Some("rate_limit_usage_window"),
2344            "relative usage-window cooldown must be labeled as a rate limit, not silence"
2345        );
2346    }
2347
2348    #[serial(cooldown_state)]
2349    #[tokio::test]
2350    async fn record_rate_limit_without_window_falls_back_to_agent_and_model() {
2351        super::reset_global_state().await;
2352        let store = test_store().await;
2353        *cooldown_store().lock().await = Some(Arc::clone(&store));
2354
2355        let agent = "test_record_rl_fallback_agent";
2356        let model = "test_record_rl_fallback_model";
2357        let msg = "429 Too Many Requests";
2358
2359        record_rate_limit(agent, Some(model), msg).await;
2360
2361        assert!(
2362            is_agent_in_cooldown(agent),
2363            "agent should be cooled when no relative window is detected"
2364        );
2365        assert!(
2366            is_model_in_cooldown(agent, model),
2367            "model should be cooled when no relative window is detected"
2368        );
2369    }
2370
2371    #[serial(cooldown_state)]
2372    #[tokio::test]
2373    async fn agent_cooldown_persists_to_kv() {
2374        super::reset_global_state().await;
2375        // Agent-level cooldowns persist to KV via set_cooldown_async
2376        let agent = "test_agent_persist_check";
2377        record_agent_failure_with_message(agent, "").await;
2378        assert!(is_agent_in_cooldown(agent));
2379    }
2380
2381    #[serial(cooldown_state)]
2382    #[tokio::test]
2383    async fn cooldown_never_shortens() {
2384        super::reset_global_state().await;
2385        let agent = "test_agent_no_shorten";
2386        set_agent_cooldown(agent, 24 * 60 * 60).await;
2387        let initial = {
2388            let map = cooldowns().lock().unwrap();
2389            map.get(agent)
2390                .expect("cooldown entry should exist")
2391                .cooldown_until
2392        };
2393
2394        // Shorter cooldown attempt should not override the existing one.
2395        record_agent_failure_with_message(agent, "rate limit").await;
2396        let after = {
2397            let map = cooldowns().lock().unwrap();
2398            map.get(agent)
2399                .expect("cooldown entry should exist")
2400                .cooldown_until
2401        };
2402        assert_eq!(
2403            initial, after,
2404            "cooldown should not be shortened by later failures"
2405        );
2406    }
2407
2408    #[serial(cooldown_state)]
2409    #[tokio::test]
2410    async fn silence_agent_cooldown_is_short_lived() {
2411        super::reset_global_state().await;
2412        let agent = "test_silence_agent_cd";
2413        assert!(!is_agent_in_cooldown(agent));
2414
2415        set_agent_cooldown(agent, SILENCE_AGENT_COOLDOWN_SECS).await;
2416        assert!(is_agent_in_cooldown(agent));
2417
2418        // Verify it's a short cooldown (120s), not the exponential backoff one
2419        let map = cooldowns().lock().unwrap();
2420        let entry = map.get(agent).expect("should have cooldown entry");
2421        let now = chrono::Utc::now().timestamp();
2422        let remaining = entry.cooldown_until - now;
2423        assert!(
2424            remaining <= i64::try_from(SILENCE_AGENT_COOLDOWN_SECS).unwrap(),
2425            "silence agent cooldown should be <= {SILENCE_AGENT_COOLDOWN_SECS}s, got {remaining}s"
2426        );
2427        assert!(remaining > 0, "cooldown should still be active");
2428    }
2429
2430    #[serial(cooldown_state)]
2431    #[tokio::test]
2432    async fn record_silence_detection_applies_extended_cooldown() {
2433        super::reset_global_state().await;
2434        let store = test_store().await;
2435        *cooldown_store().lock().await = Some(Arc::clone(&store));
2436
2437        let agent = "test_silence_count_agent";
2438        let model = "test_silence_count_model";
2439        assert!(!is_model_in_cooldown(agent, model));
2440
2441        for _ in 0..SILENCE_COUNT_THRESHOLD {
2442            let result = record_silence_detection(agent, model).await;
2443            assert!(result.is_some());
2444        }
2445
2446        assert!(is_model_in_cooldown(agent, model));
2447        let key = format!("{agent}:{model}");
2448        let remaining = {
2449            let map = cooldowns().lock().unwrap();
2450            let entry = map.get(&key).expect("cooldown entry should exist");
2451            entry.cooldown_until - chrono::Utc::now().timestamp()
2452        };
2453        assert!(
2454            remaining >= i64::try_from(SILENCE_EXTENDED_COOLDOWN_SECS).unwrap() - 5,
2455            "extended cooldown should be applied, got {remaining}s"
2456        );
2457
2458        let kv_key = format!("{SILENCE_COUNT_PREFIX}{agent}:{model}");
2459        let stored = store.kv_get(&kv_key).await.unwrap().unwrap();
2460        assert_eq!(stored, "[]");
2461    }
2462
2463    #[serial(cooldown_state)]
2464    #[test]
2465    fn detect_credit_exhaustion_out_of_credits() {
2466        let msg = "Error: API error - {\"error\":{\"message\":\"You exceeded your current quota\",\"type\":\"insufficient_quota\",\"param\":null,\"code\":\"insufficient_quota\"}}";
2467        assert_eq!(
2468            detect_credit_exhaustion(msg),
2469            Some(CreditExhaustionReason::OutOfCredits)
2470        );
2471    }
2472
2473    #[serial(cooldown_state)]
2474    #[test]
2475    fn detect_credit_exhaustion_out_of_credits_variations() {
2476        assert_eq!(
2477            detect_credit_exhaustion("out of credits"),
2478            Some(CreditExhaustionReason::OutOfCredits)
2479        );
2480        assert_eq!(
2481            detect_credit_exhaustion("outofcredits"),
2482            Some(CreditExhaustionReason::OutOfCredits)
2483        );
2484        assert_eq!(
2485            detect_credit_exhaustion("no credits remaining"),
2486            Some(CreditExhaustionReason::OutOfCredits)
2487        );
2488        assert_eq!(
2489            detect_credit_exhaustion("insufficient funds"),
2490            Some(CreditExhaustionReason::OutOfCredits)
2491        );
2492    }
2493
2494    #[serial(cooldown_state)]
2495    #[test]
2496    fn detect_credit_exhaustion_org_level_disabled() {
2497        let msg = "Error: API error - {\"error\":{\"message\":\"Organization has been disabled\",\"type\":\"org_disabled\",\"code\":\"overageDisabledReason:org_level_disabled\"}}";
2498        assert_eq!(
2499            detect_credit_exhaustion(msg),
2500            Some(CreditExhaustionReason::OrgLevelDisabled)
2501        );
2502    }
2503
2504    #[serial(cooldown_state)]
2505    #[test]
2506    fn detect_credit_exhaustion_org_level_disabled_variations() {
2507        assert_eq!(
2508            detect_credit_exhaustion("organization has been disabled"),
2509            Some(CreditExhaustionReason::OrgLevelDisabled)
2510        );
2511        assert_eq!(
2512            detect_credit_exhaustion("org disabled"),
2513            Some(CreditExhaustionReason::OrgLevelDisabled)
2514        );
2515        assert_eq!(
2516            detect_credit_exhaustion("billing quota exceeded"),
2517            Some(CreditExhaustionReason::OrgLevelDisabled)
2518        );
2519        assert_eq!(
2520            detect_credit_exhaustion("credit balance too low"),
2521            Some(CreditExhaustionReason::OrgLevelDisabled)
2522        );
2523    }
2524
2525    #[serial(cooldown_state)]
2526    #[test]
2527    fn detect_credit_exhaustion_overage_disabled_reason_parsing() {
2528        let msg = "overageDisabledReason: out_of_credits";
2529        assert_eq!(
2530            detect_credit_exhaustion(msg),
2531            Some(CreditExhaustionReason::OutOfCredits)
2532        );
2533
2534        let msg = "overageDisabledReason: org_level_disabled";
2535        assert_eq!(
2536            detect_credit_exhaustion(msg),
2537            Some(CreditExhaustionReason::OrgLevelDisabled)
2538        );
2539    }
2540
2541    #[serial(cooldown_state)]
2542    #[test]
2543    fn detect_credit_exhaustion_glm_insufficient_balance() {
2544        // GLM/ZhipuAI 429: "Insufficient balance or no resource package. Please recharge."
2545        assert_eq!(
2546            detect_credit_exhaustion(
2547                "API Error: Request rejected (429) · Insufficient balance or no resource package. Please recharge."
2548            ),
2549            Some(CreditExhaustionReason::OutOfCredits)
2550        );
2551        assert_eq!(
2552            detect_credit_exhaustion("Insufficient balance"),
2553            Some(CreditExhaustionReason::OutOfCredits)
2554        );
2555        assert_eq!(
2556            detect_credit_exhaustion("please recharge your account"),
2557            Some(CreditExhaustionReason::OutOfCredits)
2558        );
2559        assert_eq!(
2560            detect_credit_exhaustion("no resource package available"),
2561            Some(CreditExhaustionReason::OutOfCredits)
2562        );
2563    }
2564
2565    #[serial(cooldown_state)]
2566    #[test]
2567    fn detect_credit_exhaustion_none_for_regular_rate_limit() {
2568        assert!(detect_credit_exhaustion("rate limit exceeded").is_none());
2569        assert!(detect_credit_exhaustion("too many requests").is_none());
2570        assert!(detect_credit_exhaustion("429 Too Many Requests").is_none());
2571        assert!(detect_credit_exhaustion("you've hit your usage limit").is_none());
2572        assert!(
2573            detect_credit_exhaustion("You've hit your usage limit, try again in a few minutes.")
2574                .is_none()
2575        );
2576    }
2577
2578    #[serial(cooldown_state)]
2579    #[tokio::test]
2580    async fn record_credit_exhaustion_applies_agent_cooldown() {
2581        super::reset_global_state().await;
2582        let agent = "test_credit_exhaust_agent";
2583        assert!(!is_agent_in_cooldown(agent));
2584
2585        record_credit_exhaustion(agent, CreditExhaustionReason::OutOfCredits).await;
2586        assert!(is_agent_in_cooldown(agent));
2587
2588        let remaining = {
2589            let map = cooldowns().lock().unwrap();
2590            let entry = map.get(agent).expect("cooldown entry should exist");
2591            entry.cooldown_until - chrono::Utc::now().timestamp()
2592        };
2593        // First failure: base = 1h
2594        assert!(
2595            remaining >= CREDIT_BACKOFF_BASE_SECS - 5,
2596            "first out_of_credits cooldown should be ~1 hour, got {remaining}s"
2597        );
2598        assert!(
2599            remaining <= CREDIT_BACKOFF_MAX_SECS,
2600            "cooldown should not exceed cap of 8h, got {remaining}s"
2601        );
2602    }
2603
2604    #[serial(cooldown_state)]
2605    #[tokio::test]
2606    async fn record_credit_exhaustion_org_level_applies_longer_cooldown() {
2607        super::reset_global_state().await;
2608        let agent = "test_org_disabled_agent";
2609        assert!(!is_agent_in_cooldown(agent));
2610
2611        record_credit_exhaustion(agent, CreditExhaustionReason::OrgLevelDisabled).await;
2612        assert!(is_agent_in_cooldown(agent));
2613
2614        let remaining = {
2615            let map = cooldowns().lock().unwrap();
2616            let entry = map.get(agent).expect("cooldown entry should exist");
2617            entry.cooldown_until - chrono::Utc::now().timestamp()
2618        };
2619        // First failure: base = 2h (ORG_BACKOFF_BASE_SECS)
2620        assert!(
2621            remaining >= ORG_BACKOFF_BASE_SECS - 5,
2622            "first org-level disabled cooldown should be ~2 hours, got {remaining}s"
2623        );
2624        assert!(
2625            remaining <= CREDIT_BACKOFF_MAX_SECS,
2626            "cooldown should not exceed cap of 8h, got {remaining}s"
2627        );
2628    }
2629
2630    #[serial(cooldown_state)]
2631    #[tokio::test]
2632    async fn record_credit_exhaustion_billing_cycle_applies_24h() {
2633        super::reset_global_state().await;
2634        let store = test_store().await;
2635        *cooldown_store().lock().await = Some(Arc::clone(&store));
2636
2637        let agent = "test_billing_cycle_agent";
2638        assert!(!is_agent_in_cooldown(agent));
2639
2640        record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2641        assert!(is_agent_in_cooldown(agent));
2642
2643        let remaining = {
2644            let map = cooldowns().lock().unwrap();
2645            let entry = map.get(agent).expect("cooldown entry should exist");
2646            entry.cooldown_until - chrono::Utc::now().timestamp()
2647        };
2648        // First failure: base = 24h
2649        assert!(
2650            remaining >= BILLING_CYCLE_COOLDOWN_SECS - 5,
2651            "first billing cycle cooldown should be ~24 hours, got {remaining}s"
2652        );
2653        assert!(
2654            remaining <= BILLING_CYCLE_MAX_SECS,
2655            "cooldown should not exceed cap of 7 days, got {remaining}s"
2656        );
2657    }
2658
2659    #[serial(cooldown_state)]
2660    #[tokio::test]
2661    async fn record_credit_exhaustion_billing_cycle_escalates() {
2662        super::reset_global_state().await;
2663        let store = test_store().await;
2664        *cooldown_store().lock().await = Some(Arc::clone(&store));
2665
2666        let agent = "test_billing_cycle_escalation";
2667
2668        // First failure: 24h
2669        record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2670        let remaining_1 = {
2671            let map = cooldowns().lock().unwrap();
2672            let entry = map.get(agent).expect("cooldown entry should exist");
2673            entry.cooldown_until - chrono::Utc::now().timestamp()
2674        };
2675        assert!(
2676            (BILLING_CYCLE_COOLDOWN_SECS - 5..=BILLING_CYCLE_COOLDOWN_SECS + 5)
2677                .contains(&remaining_1),
2678            "first billing cycle cooldown should be ~24h, got {remaining_1}s"
2679        );
2680
2681        // Clear in-memory cooldown (but NOT failure count) to allow next set_cooldown_async
2682        {
2683            let mut map = cooldowns().lock().unwrap();
2684            map.remove(agent);
2685        }
2686
2687        // Second failure: 24h * 3 = 72h (3 days)
2688        record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2689        let remaining_2 = {
2690            let map = cooldowns().lock().unwrap();
2691            let entry = map.get(agent).expect("cooldown entry should exist");
2692            entry.cooldown_until - chrono::Utc::now().timestamp()
2693        };
2694        let expected_2 = BILLING_CYCLE_COOLDOWN_SECS * 3; // 72h
2695        assert!(
2696            remaining_2 >= expected_2 - 5,
2697            "second billing cycle cooldown should be ~72h, got {remaining_2}s"
2698        );
2699
2700        // Clear in-memory cooldown again
2701        {
2702            let mut map = cooldowns().lock().unwrap();
2703            map.remove(agent);
2704        }
2705
2706        // Third failure: 24h * 9 = 216h → capped at 7 days (168h)
2707        record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2708        let remaining_3 = {
2709            let map = cooldowns().lock().unwrap();
2710            let entry = map.get(agent).expect("cooldown entry should exist");
2711            entry.cooldown_until - chrono::Utc::now().timestamp()
2712        };
2713        assert!(
2714            remaining_3 >= BILLING_CYCLE_MAX_SECS - 5,
2715            "third billing cycle cooldown should be capped at 7 days, got {remaining_3}s"
2716        );
2717        assert!(
2718            remaining_3 <= BILLING_CYCLE_MAX_SECS + 5,
2719            "third billing cycle cooldown should not exceed 7 days, got {remaining_3}s"
2720        );
2721    }
2722
2723    #[serial(cooldown_state)]
2724    #[tokio::test]
2725    async fn record_credit_exhaustion_billing_cycle_stays_capped_after_repeats() {
2726        super::reset_global_state().await;
2727        let store = test_store().await;
2728        *cooldown_store().lock().await = Some(Arc::clone(&store));
2729
2730        let agent = "test_billing_cycle_agent_capped";
2731
2732        for i in 0..5 {
2733            if i > 0 {
2734                // Each repeat is a new exhaustion after the previous cooldown ended
2735                cooldowns().lock().unwrap().remove(agent);
2736            }
2737            record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2738        }
2739
2740        let remaining = {
2741            let map = cooldowns().lock().unwrap();
2742            let entry = map.get(agent).expect("cooldown entry should exist");
2743            entry.cooldown_until - chrono::Utc::now().timestamp()
2744        };
2745        assert!(
2746            remaining >= BILLING_CYCLE_MAX_SECS - 5,
2747            "billing cycle cooldown should stay capped at 7 days, got {remaining}s"
2748        );
2749        assert!(
2750            remaining <= BILLING_CYCLE_MAX_SECS + 5,
2751            "billing cycle cooldown should not exceed 7 days, got {remaining}s"
2752        );
2753    }
2754
2755    #[serial(cooldown_state)]
2756    #[test]
2757    fn credit_exhaustion_backoff_constants() {
2758        // OutOfCredits first failure: 1h base
2759        assert_eq!(
2760            compute_backoff(1, CREDIT_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS, true),
2761            CREDIT_BACKOFF_BASE_SECS
2762        );
2763        // OrgLevelDisabled first failure: 2h base
2764        assert_eq!(
2765            compute_backoff(1, ORG_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS, true),
2766            ORG_BACKOFF_BASE_SECS
2767        );
2768        // BillingCycleExhausted: 24h base, 7d cap
2769        assert_eq!(BILLING_CYCLE_COOLDOWN_SECS, 24 * 60 * 60);
2770        assert_eq!(BILLING_CYCLE_MAX_SECS, 7 * 24 * 60 * 60);
2771    }
2772
2773    #[serial(cooldown_state)]
2774    #[test]
2775    fn compute_backoff_long_cap_without_extended_stays_capped() {
2776        assert_eq!(
2777            compute_backoff(
2778                3,
2779                BILLING_CYCLE_COOLDOWN_SECS,
2780                BILLING_CYCLE_MAX_SECS,
2781                false
2782            ),
2783            BILLING_CYCLE_MAX_SECS,
2784            "7-day capped paths should stop at the documented max"
2785        );
2786        assert_eq!(
2787            compute_backoff(
2788                9,
2789                PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2790                PERSISTENT_MODEL_BACKOFF_MAX_SECS,
2791                false,
2792            ),
2793            PERSISTENT_MODEL_BACKOFF_MAX_SECS,
2794            "persistent model failures should not extend past 7 days"
2795        );
2796    }
2797
2798    // ---- Degraded-agent tracking tests ----
2799
2800    #[serial(cooldown_state)]
2801    #[test]
2802    fn mark_and_check_agent_degraded() {
2803        let agent = "test_degraded_mark";
2804        assert!(!is_agent_degraded(agent));
2805        mark_agent_degraded(agent);
2806        assert!(is_agent_degraded(agent));
2807        clear_agent_degraded(agent);
2808        assert!(!is_agent_degraded(agent));
2809    }
2810
2811    #[serial(cooldown_state)]
2812    #[tokio::test]
2813    async fn sync_from_kv_drops_in_memory_entry_cleared_externally() {
2814        super::reset_global_state().await;
2815        let store = test_store().await;
2816        let agent = "test_sync_external_clear";
2817        let future = chrono::Utc::now().timestamp() + 600;
2818
2819        // Register the store and put a cooldown in both memory and KV.
2820        init_cooldown_store(Arc::clone(&store)).await;
2821        set_cooldown_async(agent, future, "test").await;
2822        assert!(is_agent_in_cooldown(agent));
2823
2824        // Simulate `orch cooldown clear --all` from a separate CLI process: it
2825        // can only write KV (to "0"); the running engine's in-memory map is
2826        // untouched.
2827        store
2828            .kv_set(&format!("{KV_PREFIX}{agent}"), "0")
2829            .await
2830            .expect("kv_set tombstone");
2831        assert!(
2832            is_agent_in_cooldown(agent),
2833            "in-memory cooldown should still be present before sync"
2834        );
2835
2836        sync_from_kv(&store).await;
2837
2838        assert!(
2839            !is_agent_in_cooldown(agent),
2840            "sync_from_kv must drop in-memory entries the KV tombstoned"
2841        );
2842    }
2843
2844    #[serial(cooldown_state)]
2845    #[tokio::test]
2846    async fn sync_from_kv_closes_github_circuit_when_row_cleared() {
2847        super::reset_global_state().await;
2848        let store = test_store().await;
2849        {
2850            *github_5xx_circuit_open_lock() = Some(chrono::Utc::now().timestamp() + 600);
2851            github_5xx_timestamps_lock().push(chrono::Utc::now().timestamp());
2852        }
2853        assert!(is_github_circuit_open());
2854
2855        store
2856            .kv_set("cooldown:github:5xx", "0")
2857            .await
2858            .expect("kv_set tombstone");
2859        sync_from_kv(&store).await;
2860
2861        assert!(!is_github_circuit_open());
2862        assert!(github_5xx_timestamps_lock().is_empty());
2863    }
2864
2865    #[serial(cooldown_state)]
2866    #[tokio::test]
2867    async fn sync_from_kv_keeps_5xx_window_when_expired_row_and_breaker_closed() {
2868        super::reset_global_state().await;
2869        let store = test_store().await;
2870        let now = chrono::Utc::now().timestamp();
2871        {
2872            github_5xx_timestamps_lock().push(now);
2873            github_5xx_timestamps_lock().push(now);
2874        }
2875
2876        store
2877            .kv_set("cooldown:github:5xx", &(now - 600).to_string())
2878            .await
2879            .expect("kv_set past");
2880        sync_from_kv(&store).await;
2881
2882        assert!(!is_github_circuit_open());
2883        assert_eq!(github_5xx_timestamps_lock().len(), 2);
2884
2885        github_5xx_timestamps_lock().clear();
2886    }
2887
2888    #[serial(cooldown_state)]
2889    #[tokio::test]
2890    async fn sync_from_kv_keeps_github_circuit_open_for_future_row() {
2891        super::reset_global_state().await;
2892        let store = test_store().await;
2893        let future = chrono::Utc::now().timestamp() + 600;
2894        *github_5xx_circuit_open_lock() = Some(future);
2895
2896        store
2897            .kv_set("cooldown:github:5xx", &future.to_string())
2898            .await
2899            .expect("kv_set future");
2900        sync_from_kv(&store).await;
2901
2902        assert!(is_github_circuit_open());
2903
2904        *github_5xx_circuit_open_lock() = None;
2905    }
2906
2907    #[serial(cooldown_state)]
2908    #[tokio::test]
2909    async fn sync_from_kv_keeps_in_memory_entry_when_kv_row_missing() {
2910        // Race window: set_cooldown_async writes memory first, then KV. If a
2911        // sync runs between those two writes, the KV row is missing — we must
2912        // leave the in-memory entry alone, not drop it.
2913        super::reset_global_state().await;
2914        let store = test_store().await;
2915        let agent = "test_sync_race_window";
2916        let future = chrono::Utc::now().timestamp() + 600;
2917
2918        // Simulate the race: memory has the entry, KV does not.
2919        set_cooldown_in_memory(agent, future, "raced");
2920        assert!(is_agent_in_cooldown(agent));
2921
2922        sync_from_kv(&store).await;
2923
2924        assert!(
2925            is_agent_in_cooldown(agent),
2926            "sync_from_kv must NOT drop entries whose KV row is missing — that is the set_cooldown_async race window"
2927        );
2928    }
2929
2930    #[serial(cooldown_state)]
2931    #[tokio::test]
2932    async fn sync_from_kv_adopts_kv_timestamp() {
2933        // KV is the source of truth: if KV has a different active timestamp,
2934        // memory adopts it (covers external manual KV edits).
2935        super::reset_global_state().await;
2936        let store = test_store().await;
2937        let agent = "test_sync_adopt_kv";
2938        let original = chrono::Utc::now().timestamp() + 600;
2939        let updated = chrono::Utc::now().timestamp() + 1200;
2940
2941        init_cooldown_store(Arc::clone(&store)).await;
2942        set_cooldown_async(agent, original, "initial").await;
2943
2944        store
2945            .kv_set(&format!("{KV_PREFIX}{agent}"), &updated.to_string())
2946            .await
2947            .expect("kv_set updated");
2948
2949        sync_from_kv(&store).await;
2950
2951        let until = cooldown_until(agent).expect("cooldown still active");
2952        assert_eq!(
2953            until, updated,
2954            "sync_from_kv must adopt KV's authoritative timestamp"
2955        );
2956    }
2957
2958    #[serial(cooldown_state)]
2959    #[tokio::test]
2960    async fn refresh_degraded_agents_marks_cooled_agent() {
2961        super::reset_global_state().await;
2962        let store = test_store().await;
2963        let agent = "test_refresh_cooled";
2964
2965        // Put agent in cooldown
2966        record_agent_failure_with_message(agent, "").await;
2967        assert!(is_agent_in_cooldown(agent));
2968
2969        // model_checker always returns true (agent has models)
2970        let agents = vec![agent.to_string()];
2971        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
2972
2973        assert!(
2974            is_agent_degraded(agent),
2975            "agent in cooldown should be marked degraded"
2976        );
2977
2978        // Cleanup
2979        clear_agent_degraded(agent);
2980    }
2981
2982    #[serial(cooldown_state)]
2983    #[tokio::test]
2984    async fn refresh_degraded_agents_marks_no_models_agent() {
2985        super::reset_global_state().await;
2986        let store = test_store().await;
2987        let agent = "test_refresh_no_models";
2988
2989        // model_checker returns false (no available models)
2990        let agents = vec![agent.to_string()];
2991        refresh_degraded_agents(&store, &agents, &|_| false, 6, 3).await;
2992
2993        assert!(
2994            is_agent_degraded(agent),
2995            "agent with no available models should be marked degraded"
2996        );
2997
2998        // Cleanup
2999        clear_agent_degraded(agent);
3000    }
3001
3002    #[serial(cooldown_state)]
3003    #[tokio::test]
3004    async fn refresh_degraded_agents_marks_rate_limited_agent() {
3005        super::reset_global_state().await;
3006        let store = test_store().await;
3007        let agent = "test_refresh_rate_limited";
3008
3009        // Insert rate limit events exceeding threshold
3010        for _ in 0..4 {
3011            store.record_rate_limit(agent, "rate", None).await.unwrap();
3012        }
3013
3014        let agents = vec![agent.to_string()];
3015        // threshold=3, window=6h — 4 events should trigger degraded
3016        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3017
3018        assert!(
3019            is_agent_degraded(agent),
3020            "agent with rate limit count >= threshold should be marked degraded"
3021        );
3022
3023        // Cleanup
3024        clear_agent_degraded(agent);
3025    }
3026
3027    #[serial(cooldown_state)]
3028    #[tokio::test]
3029    async fn refresh_degraded_agents_clears_healthy_agent() {
3030        super::reset_global_state().await;
3031        let store = test_store().await;
3032        let agent = "test_refresh_healthy";
3033
3034        // Pre-mark as degraded
3035        mark_agent_degraded(agent);
3036        assert!(is_agent_degraded(agent));
3037
3038        // No cooldown, has models, no rate limit events
3039        let agents = vec![agent.to_string()];
3040        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3041
3042        assert!(
3043            !is_agent_degraded(agent),
3044            "healthy agent should have degraded flag cleared"
3045        );
3046    }
3047
3048    #[serial(cooldown_state)]
3049    #[tokio::test]
3050    async fn refresh_degraded_agents_ignores_expired_cooldowns() {
3051        super::reset_global_state().await;
3052        let store = test_store().await;
3053        let agent = "test_refresh_expired_cooldown";
3054
3055        // Manually insert an already-expired cooldown entry directly into the map
3056        // (simulating a stale entry that wasn't cleaned up)
3057        let expired_ts = chrono::Utc::now().timestamp() - 100; // 100 seconds ago
3058        {
3059            let mut map = cooldowns().lock().unwrap();
3060            map.insert(
3061                agent.to_string(),
3062                CooldownEntry {
3063                    cooldown_until: expired_ts,
3064                    reason: "expired_test".to_string(),
3065                },
3066            );
3067        }
3068
3069        // Verify the expired entry is in the map but should not cause degraded status
3070        assert!(
3071            !is_agent_in_cooldown(agent),
3072            "expired cooldown should not be active"
3073        );
3074
3075        // Pre-mark as degraded to verify it gets cleared
3076        mark_agent_degraded(agent);
3077        assert!(is_agent_degraded(agent));
3078
3079        // Run refresh — should clear degraded because cooldown is expired
3080        let agents = vec![agent.to_string()];
3081        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3082
3083        assert!(
3084            !is_agent_degraded(agent),
3085            "agent with only expired cooldown should NOT be marked degraded"
3086        );
3087
3088        // Cleanup
3089        clear_agent_degraded(agent);
3090        {
3091            let mut map = cooldowns().lock().unwrap();
3092            map.remove(agent);
3093        }
3094    }
3095
3096    #[serial(cooldown_state)]
3097    #[test]
3098    fn compute_backoff_grows_exponentially() {
3099        // count=0 or 1: base
3100        assert_eq!(compute_backoff(0, 300, 14400, true), 300);
3101        assert_eq!(compute_backoff(1, 300, 14400, true), 300);
3102        // count=2: base * 3
3103        assert_eq!(compute_backoff(2, 300, 14400, true), 900);
3104        // count=3: base * 9
3105        assert_eq!(compute_backoff(3, 300, 14400, true), 2700);
3106        // count=4: base * 27 = 8100 < 14400
3107        assert_eq!(compute_backoff(4, 300, 14400, true), 8100);
3108        // count=5: base * 81 = 24300 → capped at 14400
3109        assert_eq!(compute_backoff(5, 300, 14400, true), 14400);
3110    }
3111
3112    #[serial(cooldown_state)]
3113    #[test]
3114    fn compute_backoff_extended_tier_1_applies_24h() {
3115        // With base=300, max=14400:
3116        // count=4: 8100s (not yet at max)
3117        // count=5: 14400s (first time at max, times_at_max=0, no extension)
3118        // count=6: 14400s (times_at_max=1, no extension yet, need >=2)
3119        // count=7: 86400s (times_at_max=2, extended tier 1: 6x = 24h)
3120        assert_eq!(
3121            compute_backoff(5, 300, 14400, true),
3122            14400,
3123            "first at max, not extended"
3124        );
3125        assert_eq!(
3126            compute_backoff(6, 300, 14400, true),
3127            14400,
3128            "times_at_max=1, not extended"
3129        );
3130        assert_eq!(
3131            compute_backoff(7, 300, 14400, true),
3132            86400,
3133            "times_at_max=2, extended 6x"
3134        );
3135        assert_eq!(
3136            compute_backoff(8, 300, 14400, true),
3137            86400,
3138            "times_at_max=3, still extended 6x"
3139        );
3140    }
3141
3142    #[serial(cooldown_state)]
3143    #[test]
3144    fn compute_backoff_extended_tier_2_applies_48h() {
3145        // With base=300, max=14400:
3146        // count=10: times_at_max=5 (first_max=5, count=10), extended tier 2: 12x = 48h
3147        assert_eq!(
3148            compute_backoff(9, 300, 14400, true),
3149            86400,
3150            "times_at_max=4, still tier 1"
3151        );
3152        assert_eq!(
3153            compute_backoff(10, 300, 14400, true),
3154            172_800,
3155            "times_at_max=5, extended 12x"
3156        );
3157        assert_eq!(
3158            compute_backoff(15, 300, 14400, true),
3159            172_800,
3160            "times_at_max=10, still tier 2"
3161        );
3162        assert_eq!(
3163            compute_backoff(20, 300, 14400, true),
3164            172_800,
3165            "times_at_max=15, still tier 2"
3166        );
3167    }
3168
3169    #[serial(cooldown_state)]
3170    #[test]
3171    fn compute_backoff_extended_tier_with_different_base_max() {
3172        // With base=60, max=3600:
3173        // count=3: 60*9=540 < 3600
3174        // count=4: 60*27=1620 < 3600
3175        // count=5: 60*81=4860 → capped at 3600
3176        // count=6: times_at_max=1, still 3600
3177        // count=7: times_at_max=2, extended 6x = 21600 (6h)
3178        assert_eq!(compute_backoff(5, 60, 3600, true), 3600, "first at max");
3179        assert_eq!(compute_backoff(6, 60, 3600, true), 3600, "times_at_max=1");
3180        assert_eq!(
3181            compute_backoff(7, 60, 3600, true),
3182            21600,
3183            "times_at_max=2, extended 6x"
3184        );
3185    }
3186
3187    #[serial(cooldown_state)]
3188    #[test]
3189    fn compute_backoff_extended_tier_persistent_fails_example() {
3190        // Real-world example from issue: glm:haiku with 21 failures
3191        // With base=300, max=14400:
3192        // count=21 → times_at_max=16 (first_max=5), extended tier 2: 12x = 48h
3193        assert_eq!(
3194            compute_backoff(21, 300, 14400, true),
3195            172_800,
3196            "21 failures should get 48h extended cooldown"
3197        );
3198
3199        // kimi:haiku with 36 failures
3200        assert_eq!(
3201            compute_backoff(36, 300, 14400, true),
3202            172_800,
3203            "36 failures should also get 48h extended cooldown"
3204        );
3205    }
3206
3207    #[serial(cooldown_state)]
3208    #[test]
3209    fn compute_backoff_extended_tier_zero_base() {
3210        // Zero base should still work without panicking
3211        assert_eq!(compute_backoff(1, 0, 14400, true), 0);
3212        assert_eq!(compute_backoff(5, 0, 14400, true), 0);
3213        assert_eq!(compute_backoff(10, 0, 14400, true), 0);
3214    }
3215
3216    #[serial(cooldown_state)]
3217    #[test]
3218    fn detect_credit_exhaustion_billing_cycle() {
3219        assert_eq!(
3220            detect_credit_exhaustion("billing cycle"),
3221            Some(CreditExhaustionReason::BillingCycleExhausted)
3222        );
3223        assert_eq!(
3224            detect_credit_exhaustion("quota refreshed next cycle"),
3225            Some(CreditExhaustionReason::BillingCycleExhausted)
3226        );
3227        assert_eq!(
3228            detect_credit_exhaustion(
3229                "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle."
3230            ),
3231            Some(CreditExhaustionReason::BillingCycleExhausted)
3232        );
3233        assert_eq!(
3234            detect_credit_exhaustion("monthly quota exceeded"),
3235            Some(CreditExhaustionReason::BillingCycleExhausted)
3236        );
3237        assert_eq!(
3238            detect_credit_exhaustion(
3239                "Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-04-23 18:33:48"
3240            ),
3241            Some(CreditExhaustionReason::BillingCycleExhausted)
3242        );
3243    }
3244
3245    #[serial(cooldown_state)]
3246    #[test]
3247    fn detect_credit_exhaustion_billing_cycle_minimax() {
3248        assert_eq!(
3249            detect_credit_exhaustion(
3250                "minimax rate limit: API Error: Request rejected (429) · Token Plan usage limit reached: Upgrade your Token Plan or purchase Credits for more usage."
3251            ),
3252            Some(CreditExhaustionReason::BillingCycleExhausted)
3253        );
3254        assert_eq!(
3255            detect_credit_exhaustion("Upgrade your Token Plan"),
3256            Some(CreditExhaustionReason::BillingCycleExhausted)
3257        );
3258        assert_eq!(
3259            detect_credit_exhaustion("Token Plan usage limit reached"),
3260            Some(CreditExhaustionReason::BillingCycleExhausted)
3261        );
3262        assert_eq!(
3263            detect_credit_exhaustion("purchase Credits for more usage"),
3264            Some(CreditExhaustionReason::BillingCycleExhausted)
3265        );
3266        assert_eq!(
3267            detect_credit_exhaustion(
3268                "You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro), visit https://chatgpt.com/codex/settings/usage to purchase more credits"
3269            ),
3270            Some(CreditExhaustionReason::BillingCycleExhausted)
3271        );
3272    }
3273
3274    #[serial(cooldown_state)]
3275    #[test]
3276    fn detect_credit_exhaustion_outcome_kimi_billing_cycle() {
3277        // Regression test for issue #3529: the exact Kimi billing-cycle exhaustion
3278        // message from task_runs (tasks 156734/156678) must resolve to the
3279        // "billing_cycle_exhausted" outcome string, not fall through to a generic
3280        // rate-limit/auth classification.
3281        let msg = "Failed to authenticate. API Error: 403 You've reached your usage limit for \
3282                    this billing cycle. Your quota will be refreshed soon.";
3283        assert_eq!(
3284            detect_credit_exhaustion_outcome(msg),
3285            Some("billing_cycle_exhausted")
3286        );
3287    }
3288
3289    #[serial(cooldown_state)]
3290    #[test]
3291    fn detect_credit_exhaustion_outcome_maps_all_reasons() {
3292        assert_eq!(
3293            detect_credit_exhaustion_outcome("out of credits"),
3294            Some("credit_exhausted")
3295        );
3296        assert_eq!(
3297            detect_credit_exhaustion_outcome("organization has been disabled"),
3298            Some("org_level_disabled")
3299        );
3300        assert_eq!(
3301            detect_credit_exhaustion_outcome("monthly quota exceeded"),
3302            Some("billing_cycle_exhausted")
3303        );
3304    }
3305
3306    #[test]
3307    fn detect_credit_exhaustion_outcome_returns_none_for_unrelated_errors() {
3308        assert_eq!(
3309            detect_credit_exhaustion_outcome("429 Too Many Requests"),
3310            None
3311        );
3312        assert_eq!(detect_credit_exhaustion_outcome("connection reset"), None);
3313    }
3314
3315    #[serial(cooldown_state)]
3316    #[test]
3317    fn detect_credit_exhaustion_overage_takes_priority_over_billing_cycle() {
3318        // A message containing both overage and billing cycle language should resolve
3319        // to OrgLevelDisabled/OutOfCredits (detected first) not BillingCycleExhausted.
3320        let msg = "overageDisabledReason: org_level_disabled for billing cycle";
3321        assert_eq!(
3322            detect_credit_exhaustion(msg),
3323            Some(CreditExhaustionReason::OrgLevelDisabled)
3324        );
3325    }
3326
3327    #[serial(cooldown_state)]
3328    #[tokio::test]
3329    async fn refresh_degraded_agents_below_threshold_not_degraded() {
3330        super::reset_global_state().await;
3331        let store = test_store().await;
3332        let agent = "test_refresh_below_threshold";
3333
3334        // Insert rate limit events below threshold
3335        for _ in 0..2 {
3336            store.record_rate_limit(agent, "rate", None).await.unwrap();
3337        }
3338
3339        let agents = vec![agent.to_string()];
3340        // threshold=3, only 2 events — should NOT be degraded
3341        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3342
3343        assert!(
3344            !is_agent_degraded(agent),
3345            "agent with rate limit count < threshold should not be degraded"
3346        );
3347    }
3348
3349    #[serial(cooldown_state)]
3350    #[tokio::test]
3351    async fn refresh_degraded_agents_ignores_error_limit_type() {
3352        super::reset_global_state().await;
3353        let store = test_store().await;
3354        let agent = "test_refresh_error_rows";
3355
3356        for _ in 0..3 {
3357            store.record_rate_limit(agent, "error", None).await.unwrap();
3358        }
3359
3360        let agents = vec![agent.to_string()];
3361        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3362
3363        assert!(
3364            !is_agent_degraded(agent),
3365            "three error rows in 6 hours must not mark an agent degraded"
3366        );
3367    }
3368
3369    #[serial(cooldown_state)]
3370    #[tokio::test]
3371    async fn refresh_degraded_agents_marks_three_rate_rows() {
3372        super::reset_global_state().await;
3373        let store = test_store().await;
3374        let agent = "test_refresh_three_rate_rows";
3375
3376        for _ in 0..3 {
3377            store.record_rate_limit(agent, "rate", None).await.unwrap();
3378        }
3379
3380        let agents = vec![agent.to_string()];
3381        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3382
3383        assert!(
3384            is_agent_degraded(agent),
3385            "three rate rows in 6 hours must mark an agent degraded"
3386        );
3387        assert_eq!(
3388            agent_degraded_reason(agent).as_deref(),
3389            Some("rate limit threshold exceeded")
3390        );
3391        let listed = list_degraded_agents();
3392        assert!(
3393            listed
3394                .iter()
3395                .any(|(a, r)| a == agent && r == "rate limit threshold exceeded"),
3396            "orch cooldown list must show the degraded reason, got {listed:?}"
3397        );
3398
3399        clear_agent_degraded(agent);
3400    }
3401
3402    #[serial(cooldown_state)]
3403    #[tokio::test]
3404    async fn refresh_degraded_agents_logs_reason_change() {
3405        super::reset_global_state().await;
3406        let store = test_store().await;
3407        let agent = "test_refresh_reason_change";
3408        *cooldown_store().lock().await = Some(Arc::clone(&store));
3409
3410        set_agent_cooldown(agent, 600).await;
3411        let agents = vec![agent.to_string()];
3412        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3413        assert_eq!(
3414            agent_degraded_reason(agent).as_deref(),
3415            Some("agent in cooldown")
3416        );
3417
3418        clear_cooldown(agent, &store).await;
3419        let after_clear = sqlite_datetime_offset(60);
3420        for _ in 0..3 {
3421            store
3422                .record_rate_limit_at(agent, "rate", None, &after_clear)
3423                .await
3424                .unwrap();
3425        }
3426        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3427        assert_eq!(
3428            agent_degraded_reason(agent).as_deref(),
3429            Some("rate limit threshold exceeded"),
3430            "degraded reason must update when the hold changes"
3431        );
3432
3433        clear_agent_degraded(agent);
3434    }
3435
3436    #[serial(cooldown_state)]
3437    #[tokio::test]
3438    async fn clear_un_degrades_on_next_health_check() {
3439        super::reset_global_state().await;
3440        let store = test_store().await;
3441        let agent = "test_clear_un_degrades";
3442        let before = sqlite_datetime_offset(-30);
3443        let after = sqlite_datetime_offset(60);
3444
3445        for _ in 0..3 {
3446            store
3447                .record_rate_limit_at(agent, "rate", None, &before)
3448                .await
3449                .unwrap();
3450        }
3451
3452        let agents = vec![agent.to_string()];
3453        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3454        assert!(
3455            is_agent_degraded(agent),
3456            "three rate rows must mark an agent degraded"
3457        );
3458
3459        clear_cooldown(agent, &store).await;
3460        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3461        assert!(
3462            !is_agent_degraded(agent),
3463            "clear must un-degrade on the next health check"
3464        );
3465
3466        store
3467            .record_rate_limit_at(agent, "rate", None, &after)
3468            .await
3469            .unwrap();
3470        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3471        assert!(
3472            !is_agent_degraded(agent),
3473            "one rate row after the mark must not degrade again"
3474        );
3475
3476        for _ in 0..2 {
3477            store
3478                .record_rate_limit_at(agent, "rate", None, &after)
3479                .await
3480                .unwrap();
3481        }
3482        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3483        assert!(
3484            is_agent_degraded(agent),
3485            "three rate rows after the mark must degrade again"
3486        );
3487
3488        clear_agent_degraded(agent);
3489    }
3490
3491    #[serial(cooldown_state)]
3492    #[tokio::test]
3493    async fn clear_all_un_degrades_rate_limited_agent() {
3494        super::reset_global_state().await;
3495        let store = test_store().await;
3496        let agent = "test_clear_all_un_degrades";
3497        let before = sqlite_datetime_offset(-30);
3498
3499        for _ in 0..3 {
3500            store
3501                .record_rate_limit_at(agent, "rate", None, &before)
3502                .await
3503                .unwrap();
3504        }
3505
3506        let agents = vec![agent.to_string()];
3507        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3508        assert!(is_agent_degraded(agent));
3509
3510        clear_cooldown("*", &store).await;
3511        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3512        assert!(
3513            !is_agent_degraded(agent),
3514            "clear --all must un-degrade an agent held only by rate rows"
3515        );
3516    }
3517
3518    // ---- Exponential backoff integration tests ----
3519
3520    #[serial(cooldown_state)]
3521    #[tokio::test]
3522    async fn repeated_agent_failures_escalate_backoff() {
3523        super::reset_global_state().await;
3524        let store = test_store().await;
3525        *cooldown_store().lock().await = Some(Arc::clone(&store));
3526
3527        let agent = "test_escalation_agent";
3528
3529        // First failure: should be ~5 min (BACKOFF_BASE_SECS)
3530        record_agent_failure_with_message(agent, "").await;
3531        let remaining_1 = {
3532            let map = cooldowns().lock().unwrap();
3533            let entry = map.get(agent).expect("should have cooldown");
3534            entry.cooldown_until - chrono::Utc::now().timestamp()
3535        };
3536        assert!(
3537            (BACKOFF_BASE_SECS - 5..=BACKOFF_BASE_SECS + 5).contains(&remaining_1),
3538            "first failure should be ~5 min, got {remaining_1}s"
3539        );
3540
3541        // Clear the cooldown (but NOT failure count) to allow the next set_cooldown_async to apply
3542        {
3543            let mut map = cooldowns().lock().unwrap();
3544            map.remove(agent);
3545        }
3546
3547        // Second failure: should be ~15 min (5 * 3)
3548        record_agent_failure_with_message(agent, "").await;
3549        let remaining_2 = {
3550            let map = cooldowns().lock().unwrap();
3551            let entry = map.get(agent).expect("should have cooldown");
3552            entry.cooldown_until - chrono::Utc::now().timestamp()
3553        };
3554        assert!(
3555            remaining_2 >= BACKOFF_BASE_SECS * 3 - 5,
3556            "second failure should be ~15 min, got {remaining_2}s"
3557        );
3558    }
3559
3560    #[serial(cooldown_state)]
3561    #[tokio::test]
3562    async fn vendor_past_retry_falls_back_to_backoff() {
3563        super::reset_global_state().await;
3564        // If a vendor-provided "try again at" date is in the past, the
3565        // code must NOT early-return and must instead apply the normal
3566        // exponential backoff (increment failure count and set a future cooldown).
3567        let store = test_store().await;
3568        *cooldown_store().lock().await = Some(Arc::clone(&store));
3569
3570        let agent = "test_vendor_past_retry";
3571
3572        // Ensure no pre-existing cooldown
3573        {
3574            let mut map = cooldowns().lock().unwrap();
3575            map.remove(agent);
3576        }
3577
3578        // A clearly past date that parse_retry_at can parse
3579        let past_msg = "You've hit the limit; try again at Jan 01, 2000 01:00 AM.";
3580
3581        record_agent_failure_with_message(agent, past_msg).await;
3582
3583        // After handling a past vendor timestamp, we should have applied
3584        // exponential backoff (base = BACKOFF_BASE_SECS) and a persisted
3585        // failure count of 1.
3586        assert!(
3587            is_agent_in_cooldown(agent),
3588            "agent should be in cooldown after failure"
3589        );
3590
3591        let remaining = {
3592            let map = cooldowns().lock().unwrap();
3593            let entry = map.get(agent).expect("cooldown entry should exist");
3594            entry.cooldown_until - chrono::Utc::now().timestamp()
3595        };
3596
3597        assert!(
3598            remaining >= BACKOFF_BASE_SECS - 5,
3599            "expected backoff of ~{BACKOFF_BASE_SECS}s when vendor timestamp is past, got {remaining}s"
3600        );
3601
3602        // Verify the failure count was incremented and persisted to KV
3603        let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3604        let stored = store
3605            .kv_get(&kv_key)
3606            .await
3607            .unwrap()
3608            .expect("failure count should be stored");
3609        assert_eq!(stored, "1");
3610    }
3611
3612    fn window(status: &str, resets_at: Option<i64>) -> RateLimitWindow {
3613        RateLimitWindow {
3614            status: status.to_string(),
3615            window_type: "five_hour".to_string(),
3616            utilization: Some(1.0),
3617            resets_at,
3618        }
3619    }
3620
3621    #[serial(cooldown_state)]
3622    #[tokio::test]
3623    async fn rejected_window_cools_agent_until_reset() {
3624        super::reset_global_state().await;
3625        let store = test_store().await;
3626        *cooldown_store().lock().await = Some(Arc::clone(&store));
3627
3628        let agent = "test_window_rejected_agent";
3629        let reset = chrono::Utc::now().timestamp() + 7200;
3630        record_rate_limit_window(agent, &window("rejected", Some(reset))).await;
3631
3632        assert_eq!(cooldown_until(agent), Some(reset));
3633        let stored = list_rate_limit_windows(&store).await;
3634        assert_eq!(
3635            stored,
3636            vec![(agent.to_string(), window("rejected", Some(reset)))]
3637        );
3638    }
3639
3640    #[serial(cooldown_state)]
3641    #[tokio::test]
3642    async fn allowed_window_changes_no_cooldown() {
3643        super::reset_global_state().await;
3644        let store = test_store().await;
3645        *cooldown_store().lock().await = Some(Arc::clone(&store));
3646
3647        let agent = "test_window_allowed_agent";
3648        let reset = chrono::Utc::now().timestamp() + 7200;
3649        record_rate_limit_window(agent, &window("allowed", Some(reset))).await;
3650        record_rate_limit_window(agent, &window("allowed_warning", Some(reset))).await;
3651
3652        assert_eq!(cooldown_until(agent), None);
3653        assert_eq!(list_rate_limit_windows(&store).await.len(), 1);
3654    }
3655
3656    #[serial(cooldown_state)]
3657    #[tokio::test]
3658    async fn record_agent_success_resets_backoff() {
3659        super::reset_global_state().await;
3660        let store = test_store().await;
3661        *cooldown_store().lock().await = Some(Arc::clone(&store));
3662
3663        let agent = "test_success_reset_agent";
3664        let model = "test_model";
3665
3666        // Accumulate 3 failures to escalate the backoff
3667        for _ in 0..3 {
3668            let _ = read_and_increment_failure_count(Some(&store), agent).await;
3669        }
3670        // Verify count is 3
3671        let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3672        let count: u32 = store
3673            .kv_get(&kv_key)
3674            .await
3675            .unwrap()
3676            .unwrap()
3677            .parse()
3678            .unwrap();
3679        assert_eq!(count, 3);
3680
3681        // Success should reset to 0
3682        record_agent_success(agent, model).await;
3683
3684        let count_after: u32 = store
3685            .kv_get(&kv_key)
3686            .await
3687            .unwrap()
3688            .unwrap()
3689            .parse()
3690            .unwrap();
3691        assert_eq!(count_after, 0, "success should reset failure count to 0");
3692
3693        // Model-specific count should also be reset
3694        let model_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}:{model}");
3695        let model_count: u32 = store
3696            .kv_get(&model_kv_key)
3697            .await
3698            .unwrap()
3699            .unwrap()
3700            .parse()
3701            .unwrap();
3702        assert_eq!(
3703            model_count, 0,
3704            "success should reset model failure count to 0"
3705        );
3706    }
3707
3708    #[serial(cooldown_state)]
3709    #[tokio::test]
3710    async fn record_agent_success_without_model_skips_empty_model_key() {
3711        super::reset_global_state().await;
3712        let store = test_store().await;
3713        *cooldown_store().lock().await = Some(Arc::clone(&store));
3714
3715        let agent = "test_success_without_model";
3716
3717        record_agent_success(agent, "").await;
3718
3719        let agent_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3720        let agent_count = store
3721            .kv_get(&agent_kv_key)
3722            .await
3723            .unwrap()
3724            .expect("agent failure count should be stored");
3725        assert_eq!(agent_count, "0");
3726
3727        let empty_model_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}:");
3728        let empty_model_count = store.kv_get(&empty_model_kv_key).await.unwrap();
3729        assert!(
3730            empty_model_count.is_none(),
3731            "success without model should not create an empty model failure count key"
3732        );
3733    }
3734
3735    #[serial(cooldown_state)]
3736    #[tokio::test]
3737    async fn clear_cooldown_resets_failure_counts() {
3738        super::reset_global_state().await;
3739        let store = test_store().await;
3740        *cooldown_store().lock().await = Some(Arc::clone(&store));
3741
3742        let agent = "test_clear_fc_agent";
3743
3744        // Accumulate failures
3745        for _ in 0..5 {
3746            let _ = read_and_increment_failure_count(Some(&store), agent).await;
3747        }
3748        // Set a cooldown so clear_cooldown has something to clear
3749        set_agent_cooldown(agent, 3600).await;
3750
3751        // Verify failure count is 5
3752        let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3753        let count: u32 = store
3754            .kv_get(&kv_key)
3755            .await
3756            .unwrap()
3757            .unwrap()
3758            .parse()
3759            .unwrap();
3760        assert_eq!(count, 5);
3761
3762        // Clear the cooldown
3763        clear_cooldown(agent, &store).await;
3764
3765        // Failure count should be reset to 0
3766        let count_after: u32 = store
3767            .kv_get(&kv_key)
3768            .await
3769            .unwrap()
3770            .unwrap()
3771            .parse()
3772            .unwrap();
3773        assert_eq!(
3774            count_after, 0,
3775            "clear_cooldown should reset failure count to 0"
3776        );
3777
3778        // Cooldown should be removed
3779        assert!(
3780            !is_agent_in_cooldown(agent),
3781            "agent should not be in cooldown after clear"
3782        );
3783    }
3784
3785    #[serial(cooldown_state)]
3786    #[tokio::test]
3787    async fn clear_all_cooldowns_resets_all_failure_counts() {
3788        super::reset_global_state().await;
3789        let store = test_store().await;
3790        *cooldown_store().lock().await = Some(Arc::clone(&store));
3791
3792        let agents = ["test_clear_all_a", "test_clear_all_b"];
3793        for agent in &agents {
3794            let _ = read_and_increment_failure_count(Some(&store), agent).await;
3795            set_agent_cooldown(agent, 3600).await;
3796        }
3797
3798        clear_cooldown("*", &store).await;
3799
3800        for agent in &agents {
3801            assert!(!is_agent_in_cooldown(agent));
3802            let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3803            let count: u32 = store
3804                .kv_get(&kv_key)
3805                .await
3806                .unwrap()
3807                .unwrap()
3808                .parse()
3809                .unwrap();
3810            assert_eq!(
3811                count, 0,
3812                "clear --all should reset failure count for {agent}"
3813            );
3814        }
3815    }
3816
3817    #[serial(cooldown_state)]
3818    #[test]
3819    fn compute_backoff_saturates_on_large_counts() {
3820        // Very large count should not overflow — extended tier applies 48h
3821        assert_eq!(
3822            compute_backoff(100, BACKOFF_BASE_SECS, BACKOFF_MAX_SECS, true),
3823            BACKOFF_MAX_SECS * 12
3824        );
3825        assert_eq!(
3826            compute_backoff(u64::from(u32::MAX), 300, 14400, true),
3827            172_800
3828        );
3829        assert_eq!(compute_backoff(u64::MAX, 300, 14400, true), 172_800);
3830    }
3831
3832    #[serial(cooldown_state)]
3833    #[test]
3834    fn compute_backoff_zero_base_returns_zero() {
3835        assert_eq!(compute_backoff(1, 0, 14400, true), 0);
3836        assert_eq!(compute_backoff(5, 0, 14400, true), 0);
3837    }
3838
3839    // ---- GitHub 5xx circuit breaker tests ----
3840
3841    #[serial(cooldown_state)]
3842    #[tokio::test]
3843    async fn github_circuit_stays_closed_below_threshold() {
3844        super::reset_global_state().await;
3845        // Reset circuit breaker state
3846        {
3847            let mut open = github_5xx_circuit_open().lock().unwrap();
3848            *open = None;
3849            let mut ts = github_5xx_timestamps().lock().unwrap();
3850            ts.clear();
3851            // Also clear any generic cooldown map entry for the github 5xx
3852            // circuit so tests are isolated from other tests that may have
3853            // set the generic cooldown via set_agent_cooldown("github:5xx", ...).
3854            let mut map = cooldown_lock();
3855            map.remove("github:5xx");
3856        }
3857
3858        // Record fewer errors than threshold
3859        for _ in 0..GITHUB_5XX_THRESHOLD - 1 {
3860            record_github_5xx().await;
3861        }
3862
3863        assert!(
3864            !is_github_circuit_open(),
3865            "circuit should stay closed below threshold"
3866        );
3867        assert_eq!(github_circuit_remaining_secs(), 0);
3868    }
3869
3870    #[serial(cooldown_state)]
3871    #[tokio::test]
3872    async fn github_circuit_trips_at_threshold() {
3873        super::reset_global_state().await;
3874        // Reset circuit breaker state
3875        {
3876            let mut open = github_5xx_circuit_open_lock();
3877            *open = None;
3878            let mut ts = github_5xx_timestamps_lock();
3879            ts.clear();
3880            let mut map = cooldown_lock();
3881            map.remove("github:5xx");
3882        }
3883
3884        // Record exactly threshold errors
3885        for _ in 0..GITHUB_5XX_THRESHOLD {
3886            record_github_5xx().await;
3887        }
3888
3889        assert!(is_github_circuit_open(), "circuit should trip at threshold");
3890        assert!(
3891            github_circuit_remaining_secs() > 0,
3892            "remaining should be > 0 when open"
3893        );
3894        assert!(
3895            github_circuit_remaining_secs() <= GITHUB_5XX_COOLDOWN_SECS as u64,
3896            "remaining should not exceed cooldown duration"
3897        );
3898
3899        // Clean up
3900        {
3901            let mut open = github_5xx_circuit_open().lock().unwrap();
3902            *open = None;
3903        }
3904    }
3905
3906    #[serial(cooldown_state)]
3907    #[tokio::test]
3908    async fn github_circuit_cannot_double_trip() {
3909        super::reset_global_state().await;
3910        // Reset circuit breaker state
3911        {
3912            let mut open = github_5xx_circuit_open_lock();
3913            *open = None;
3914            let mut ts = github_5xx_timestamps_lock();
3915            ts.clear();
3916            let mut map = cooldown_lock();
3917            map.remove("github:5xx");
3918        }
3919
3920        // Trip the circuit
3921        for _ in 0..GITHUB_5XX_THRESHOLD {
3922            record_github_5xx().await;
3923        }
3924        assert!(is_github_circuit_open());
3925
3926        let first_remaining = github_circuit_remaining_secs();
3927
3928        // More 5xx errors should NOT reset the cooldown (never-shorten)
3929        for _ in 0..GITHUB_5XX_THRESHOLD {
3930            record_github_5xx().await;
3931        }
3932
3933        let second_remaining = github_circuit_remaining_secs();
3934        assert!(
3935            second_remaining <= first_remaining,
3936            "additional errors should not extend cooldown"
3937        );
3938
3939        // Clean up
3940        {
3941            let mut open = github_5xx_circuit_open().lock().unwrap();
3942            *open = None;
3943        }
3944    }
3945
3946    #[serial(cooldown_state)]
3947    #[tokio::test]
3948    async fn github_circuit_recovers_after_cooldown() {
3949        super::reset_global_state().await;
3950        // Reset circuit breaker state
3951        {
3952            let mut open = github_5xx_circuit_open_lock();
3953            *open = None;
3954            let mut ts = github_5xx_timestamps_lock();
3955            ts.clear();
3956            let mut map = cooldown_lock();
3957            map.remove("github:5xx");
3958        }
3959
3960        // Set the circuit to an expired cooldown
3961        {
3962            let mut open = github_5xx_circuit_open().lock().unwrap();
3963            *open = Some(chrono::Utc::now().timestamp() - 1);
3964        }
3965
3966        // Should be closed now (auto-recovery)
3967        assert!(
3968            !is_github_circuit_open(),
3969            "circuit should auto-recover after cooldown"
3970        );
3971    }
3972
3973    #[serial(cooldown_state)]
3974    #[tokio::test]
3975    async fn github_circuit_sliding_window_ages_out() {
3976        super::reset_global_state().await;
3977        // Reset circuit breaker state
3978        {
3979            let mut open = github_5xx_circuit_open_lock();
3980            *open = None;
3981            let mut ts = github_5xx_timestamps_lock();
3982            ts.clear();
3983            let mut map = cooldown_lock();
3984            map.remove("github:5xx");
3985        }
3986
3987        // Record errors that are outside the window
3988        {
3989            let mut ts = github_5xx_timestamps().lock().unwrap();
3990            let old = chrono::Utc::now().timestamp() - GITHUB_5XX_WINDOW_SECS - 10;
3991            for _ in 0..GITHUB_5XX_THRESHOLD {
3992                ts.push(old);
3993            }
3994        }
3995
3996        // A new error should clear old ones and not trip the circuit
3997        record_github_5xx().await;
3998
3999        assert!(
4000            !is_github_circuit_open(),
4001            "old errors outside window should not trip circuit"
4002        );
4003    }
4004}