Skip to main content

orch/engine/
cooldown.rs

1//! Per-agent and per-model cooldown tracking.
2//!
3//! Shared by `engine::runner::response` (recording failures) and
4//! `engine::router::config` (avoiding cooled models during pool selection).
5//!
6//! # Persistence
7//!
8//! All cooldowns are persisted to the KV store in SQLite so they survive
9//! service restarts. Call [`init_cooldown_store`] once at startup (after
10//! the `TaskStore` is open) to register the store and pre-load unexpired
11//! cooldowns.
12//!
13//! When a rate limit includes a "try again at {date}" message, the cooldown
14//! is set to that specific timestamp instead of the default duration.
15//! Relative usage windows (e.g. "5-hour usage limit") are also detected and
16//! used to set a model-specific cooldown matching the advertised window.
17
18use chrono::TimeZone;
19use regex::Regex;
20use std::collections::HashMap;
21use std::collections::HashSet;
22use std::sync::LazyLock;
23use std::sync::MutexGuard;
24use std::sync::{Arc, Mutex, OnceLock};
25use tokio::sync;
26
27use crate::engine::router::config;
28#[cfg(test)]
29use crate::engine::runner::agents::opencode;
30use crate::store::TaskStore;
31
32/// Default fallback cooldown for agents when no store is available: 30 minutes.
33///
34/// Used in contexts that need a concrete duration and cannot compute exponential
35/// backoff (e.g. computing how long to wait before retrying when a cooldown
36/// expiry timestamp is unknown). New code should use [`BACKOFF_BASE_SECS`] instead.
37pub const AGENT_COOLDOWN_SECS: i64 = 30 * 60;
38
39/// Base backoff for generic agent/model failures: 5 minutes.
40pub const BACKOFF_BASE_SECS: i64 = 5 * 60;
41
42/// Maximum backoff for generic agent/model failures: 4 hours.
43pub const BACKOFF_MAX_SECS: i64 = 4 * 60 * 60;
44
45/// Base backoff for persistent model failures (silent exit/provider errors): 4 hours.
46pub const PERSISTENT_MODEL_BACKOFF_BASE_SECS: i64 = 4 * 60 * 60;
47
48/// Maximum backoff for persistent model failures: 7 days.
49pub const PERSISTENT_MODEL_BACKOFF_MAX_SECS: i64 = 7 * 24 * 60 * 60;
50
51/// Base backoff for a one-off model failure that does not mean the model is gone: 5 minutes.
52pub const TRANSIENT_MODEL_BACKOFF_BASE_SECS: i64 = 5 * 60;
53
54/// Maximum backoff for a one-off model failure: 15 minutes.
55pub const TRANSIENT_MODEL_BACKOFF_MAX_SECS: i64 = 15 * 60;
56
57/// Base backoff for credit exhaustion (out_of_credits): 1 hour.
58pub const CREDIT_BACKOFF_BASE_SECS: i64 = 60 * 60;
59
60/// Base backoff for org-level disabling: 2 hours.
61pub const ORG_BACKOFF_BASE_SECS: i64 = 2 * 60 * 60;
62
63/// Standard cap for credit exhaustion and org-level disabling: 8 hours.
64/// Repeated failures at the cap move to the extended tiers (48h, then 96h)
65pub const CREDIT_BACKOFF_MAX_SECS: i64 = 8 * 60 * 60;
66
67/// Base cooldown for billing cycle exhaustion: 24 hours.
68///
69/// Unlike per-request rate limits, billing cycle exhaustion is a calendar event
70/// that won't self-resolve for days or weeks. The first failure starts at 24h;
71/// subsequent failures escalate via `compute_backoff()` up to
72/// `BILLING_CYCLE_MAX_SECS` (7 days). This prevents daily retry-and-fail cycles
73/// when a monthly billing quota is exhausted.
74pub const BILLING_CYCLE_COOLDOWN_SECS: i64 = 24 * 60 * 60;
75
76/// Maximum cooldown for billing cycle exhaustion: 7 days.
77///
78/// Monthly billing cycles typically reset on a fixed date. 7 days is a
79/// reasonable cap — the cooldown will expire before the next monthly reset
80/// in most cases, allowing a single probe to re-check availability.
81pub const BILLING_CYCLE_MAX_SECS: i64 = 7 * 24 * 60 * 60;
82
83/// Credit exhaustion reason detected from error message.
84#[derive(Debug, Clone, Copy, PartialEq, Eq)]
85pub enum CreditExhaustionReason {
86    /// Per-model credit exhaustion (out_of_credits).
87    OutOfCredits,
88    /// Organization-level disabling (org_level_disabled).
89    OrgLevelDisabled,
90    /// Monthly billing cycle exhaustion — escalating cooldown (24h base, 7d cap).
91    BillingCycleExhausted,
92}
93
94impl CreditExhaustionReason {
95    /// Stable string for `task_runs.outcome`, distinct from the cooldown reason
96    /// string used by `record_credit_exhaustion`. Lets callers persist the actual
97    /// failure mode instead of collapsing it into a generic "rate_limit"/"failed".
98    pub fn outcome_str(self) -> &'static str {
99        match self {
100            CreditExhaustionReason::OutOfCredits => "credit_exhausted",
101            CreditExhaustionReason::OrgLevelDisabled => "org_level_disabled",
102            CreditExhaustionReason::BillingCycleExhausted => "billing_cycle_exhausted",
103        }
104    }
105}
106
107/// Short agent cooldown applied on silence detection (120 seconds).
108///
109/// Forces the router to pick a different agent immediately on re-route,
110/// without long-term blocking. The model cooldown (exponential, starting at
111/// 5 min) keeps the dead model out while this short cooldown just breaks
112/// the same-agent loop.
113pub const SILENCE_AGENT_COOLDOWN_SECS: u64 = 120;
114
115/// Silence detections before applying an extended cooldown (rolling window).
116pub const SILENCE_COUNT_THRESHOLD: usize = 5;
117
118/// Rolling window for silence detections (24 hours).
119pub const SILENCE_COUNT_WINDOW_SECS: i64 = 24 * 60 * 60;
120
121/// Extended cooldown applied after repeated silence detections (4 hours).
122pub const SILENCE_EXTENDED_COOLDOWN_SECS: u64 = 4 * 60 * 60;
123
124/// KV key prefix for persisted cooldowns (both agent and model).
125const KV_PREFIX: &str = "cooldown:";
126
127/// KV key prefix for rolling silence counters.
128const SILENCE_COUNT_PREFIX: &str = "silence_count:";
129
130/// KV key prefix for per-agent and per-agent:model failure counts (drives exponential backoff).
131const FAILURE_COUNT_PREFIX: &str = "failure_count:";
132
133/// KV key prefix for credit-exhaustion failure counts (separate from generic failures to prevent
134/// cross-contamination of backoff escalation — credit exhaustion uses different base/max durations).
135const CREDIT_FAILURE_COUNT_PREFIX: &str = "credit_failure_count:";
136
137/// KV key prefix for persistent model failure counts (separate from [`FAILURE_COUNT_PREFIX`] so
138/// `record_agent_success()` resetting the generic counter on an intervening success doesn't erase
139/// escalation progress for structurally unreliable models — see `record_persistent_model_failure`).
140const PERSISTENT_FAILURE_COUNT_PREFIX: &str = "persistent_failure_count:";
141
142/// KV key prefix for the unix timestamp after which `rate_limits` rows count toward degraded
143const RATE_LIMITS_CLEARED_AT_PREFIX: &str = "rate_limits_cleared_at:";
144
145/// Agent name from a cooldown key (`claude:sonnet` -> `claude`)
146fn agent_from_key(key: &str) -> &str {
147    key.split_once(':').map_or(key, |(agent, _)| agent)
148}
149
150struct CooldownEntry {
151    /// Unix timestamp when the cooldown expires.
152    cooldown_until: i64,
153    reason: String,
154}
155
156#[derive(Debug)]
157pub struct SilenceCountResult {
158    pub count: usize,
159    pub extended_cooldown_applied: bool,
160}
161
162/// Global in-memory cooldown map, protected by a Mutex.
163///
164/// Uses `std::sync::Mutex` because this map is also read from sync helper
165/// functions (`is_in_cooldown`, `is_model_in_cooldown`, etc.).  All lock
166/// holders perform only in-memory HashMap operations — **no `.await` is ever
167/// called while a guard is held**.  This invariant must be preserved: if a
168/// future caller needs to await while holding this lock, switch to
169/// `tokio::sync::Mutex` and make the helpers async.
170fn cooldowns() -> &'static Mutex<HashMap<String, CooldownEntry>> {
171    static COOLDOWNS: OnceLock<Mutex<HashMap<String, CooldownEntry>>> = OnceLock::new();
172    COOLDOWNS.get_or_init(|| Mutex::new(HashMap::new()))
173}
174
175/// Global store reference used for background KV writes.
176///
177/// Uses `tokio::sync::Mutex` because this value is only ever accessed from
178/// async functions. A `tokio::sync::Mutex` yields the Tokio task instead of
179/// blocking the worker thread if the lock is contended.
180fn cooldown_store() -> &'static sync::Mutex<Option<Arc<TaskStore>>> {
181    static STORE: OnceLock<sync::Mutex<Option<Arc<TaskStore>>>> = OnceLock::new();
182    STORE.get_or_init(|| sync::Mutex::new(None))
183}
184
185/// Initialise persistent cooldowns.
186///
187/// Must be called once at engine startup, after the `TaskStore` is opened.
188/// Loads unexpired cooldowns from KV and registers the store for future writes.
189/* Parses the `{"until": N, "reason": "..."}` cooldown KV format, falling back
190to the legacy bare-integer format (reason unknown) so existing rows keep
191working across the upgrade without a migration. */
192fn parse_cooldown_value(value: &str) -> Option<(i64, String)> {
193    if let Ok(serde_json::Value::Object(map)) = serde_json::from_str(value) {
194        let until = map.get("until")?.as_i64()?;
195        let reason = map
196            .get("reason")
197            .and_then(|r| r.as_str())
198            .unwrap_or("persisted")
199            .to_string();
200        return Some((until, reason));
201    }
202    value
203        .parse::<i64>()
204        .ok()
205        .map(|until| (until, "persisted".to_string()))
206}
207
208pub async fn init_cooldown_store(store: Arc<TaskStore>) {
209    match store.kv_list_prefix(KV_PREFIX).await {
210        Ok(rows) => {
211            let now = chrono::Utc::now().timestamp();
212            let mut map = cooldown_lock();
213            let mut loaded = 0;
214            for (key, value) in rows {
215                let cooldown_key = key.trim_start_matches(KV_PREFIX);
216                if let Some((cooldown_until, reason)) = parse_cooldown_value(&value)
217                    && now < cooldown_until
218                {
219                    map.insert(
220                        cooldown_key.to_string(),
221                        CooldownEntry {
222                            cooldown_until,
223                            reason,
224                        },
225                    );
226                    loaded += 1;
227                }
228                // Expired entries are simply not loaded — they'll be overwritten
229                // on the next failure or cleaned up by future writes.
230            }
231            if loaded > 0 {
232                tracing::info!(loaded, "loaded persisted cooldowns from KV store");
233            }
234        }
235        Err(e) => {
236            tracing::warn!(err = %e, "failed to load cooldowns from KV store");
237        }
238    }
239
240    // Load GitHub 5xx circuit breaker state if it was persisted before restart.
241    if let Ok(Some(raw)) = store.kv_get("cooldown:github:5xx").await
242        && let Ok(cooldown_until) = raw.parse::<i64>()
243    {
244        let now = chrono::Utc::now().timestamp();
245        if now < cooldown_until {
246            let mut open = github_5xx_circuit_open_lock();
247            *open = Some(cooldown_until);
248            let remaining = cooldown_until - now;
249            tracing::warn!(
250                remaining_secs = remaining,
251                "GitHub 5xx circuit breaker restored from KV (still open)"
252            );
253        }
254    }
255
256    *cooldown_store().lock().await = Some(store);
257}
258
259/// Reconcile the in-memory cooldown map with the persisted KV state.
260///
261/// KV is the source of truth. This is called once per tick by
262/// `router.refresh_health` so the running service notices state mutated by a
263/// separate process (e.g. `orch cooldown clear`). Without this sync, the CLI
264/// clears KV but the engine's in-memory map keeps the stale entries until they
265/// expire on their own, producing the "all agents cooled" + "No active
266/// cooldowns" inconsistency.
267///
268/// Reconciliation rules (KV authoritative for every key it has):
269/// - KV row missing for an in-memory key: leave memory alone — likely a
270///   `set_cooldown_async` race where memory was written but the KV row isn't
271///   yet visible. The next tick will catch up.
272/// - KV row present with `cooldown_until <= now` (including the "0" tombstone
273///   written by `clear_cooldown`): drop from memory.
274/// - KV row present with `cooldown_until > now`: insert/update memory to match.
275pub async fn sync_from_kv(store: &Arc<TaskStore>) {
276    let rows = match store.kv_list_prefix(KV_PREFIX).await {
277        Ok(rows) => rows,
278        Err(e) => {
279            tracing::warn!(err = %e, "cooldown sync_from_kv: KV list failed");
280            return;
281        }
282    };
283    let now = chrono::Utc::now().timestamp();
284
285    // Some((ts, reason)) for active cooldowns, None for cleared/expired rows.
286    // Missing key means no KV row at all, left alone below.
287    let mut explicit_kv: HashMap<String, Option<(i64, String)>> =
288        HashMap::with_capacity(rows.len());
289    for (kv_key, value) in &rows {
290        let cooldown_key = kv_key.trim_start_matches(KV_PREFIX);
291        if let Some((ts, reason)) = parse_cooldown_value(value) {
292            let normalized = if ts > now { Some((ts, reason)) } else { None };
293            explicit_kv.insert(cooldown_key.to_string(), normalized);
294        }
295    }
296
297    // The 5xx circuit breaker lives in its own statics, so mirror its KV row there
298    match explicit_kv.get("github:5xx") {
299        Some(None) => {
300            let mut open = github_5xx_circuit_open_lock();
301            if open.take().is_some() {
302                tracing::info!("GitHub 5xx circuit breaker closed by KV clear");
303                github_5xx_timestamps_lock().clear();
304            }
305        }
306        Some(Some((ts, _))) => {
307            *github_5xx_circuit_open_lock() = Some(*ts);
308        }
309        None => {}
310    }
311
312    let mut map = cooldown_lock();
313    let mut dropped = 0usize;
314    let mut updated = 0usize;
315
316    // Drop in-memory entries that KV explicitly marks cleared/expired.
317    map.retain(|key, _| {
318        if matches!(explicit_kv.get(key), Some(None)) {
319            dropped += 1;
320            false
321        } else {
322            true
323        }
324    });
325
326    // Adopt KV's authoritative timestamp for active cooldowns.
327    for (key, normalized) in explicit_kv {
328        if let Some((ts, reason)) = normalized {
329            if let Some(entry) = map.get_mut(&key) {
330                if entry.cooldown_until != ts {
331                    entry.cooldown_until = ts;
332                    updated += 1;
333                }
334            } else {
335                map.insert(
336                    key,
337                    CooldownEntry {
338                        cooldown_until: ts,
339                        reason,
340                    },
341                );
342                updated += 1;
343            }
344        }
345    }
346
347    if dropped > 0 || updated > 0 {
348        tracing::info!(
349            dropped,
350            updated,
351            "cooldown sync_from_kv: reconciled in-memory map with KV"
352        );
353    }
354}
355
356/// Failure count threshold (times at max backoff) for first extended tier (24h).
357///
358/// After a model has been at the max cap this many times, apply a 6x multiplier.
359const EXTENDED_TIER_1_THRESHOLD: u64 = 2;
360
361/// Multiplier for first extended tier (6x = 24h when max=4h).
362const EXTENDED_TIER_1_MULTIPLIER: i64 = 6;
363
364/// Failure count threshold (times at max backoff) for second extended tier (48h).
365///
366/// After a model has been at the max cap this many times, apply a 12x multiplier.
367const EXTENDED_TIER_2_THRESHOLD: u64 = 5;
368
369/// Multiplier for second extended tier (12x = 48h when max=4h).
370const EXTENDED_TIER_2_MULTIPLIER: i64 = 12;
371
372/// Compute exponential backoff duration using base-3 growth.
373///
374/// Returns `min(base * 3^(count-1), max)` for count >= 1.
375/// For count == 0 or 1 (first failure), returns `base`.
376///
377/// | count | base=300s | result |
378/// |-------|-----------|--------|
379/// | 0     | 300       | 300    |
380/// | 1     | 300       | 300    |
381/// | 2     | 300       | 900    |
382/// | 3     | 300       | 2700   |
383/// | 4     | 300       | 8100   |
384/// | 5-6   | 14400     | 14400 (max, not yet extended) |
385/// | 7-9   | 86400     | 86400 (24h extended tier) |
386/// | 10+   | 172800    | 172800 (48h extended tier) |
387///
388/// The extended tier applies when the model has hit the max cap multiple times,
389/// indicating a persistently broken model that won't self-resolve quickly.
390pub fn compute_backoff(count: u64, base: i64, max: i64, allow_extended: bool) -> i64 {
391    if count <= 1 {
392        return base;
393    }
394    let exp = u32::try_from(count.saturating_sub(1)).unwrap_or(u32::MAX);
395    let factor = 3_i64.saturating_pow(exp);
396    let standard = base.saturating_mul(factor).min(max);
397
398    // Extended tier: after N failures at max backoff, apply longer window
399    // to reduce noise from persistently broken models.
400    //
401    // First find the count at which exponential growth first hits max:
402    // solve: base * 3^(first_max-1) >= max
403    // => 3^(first_max-1) >= max / base
404    // => first_max-1 >= log_3(max / base)
405    // => first_max >= 1 + log_3(max / base)
406    let first_max = if base <= 0 {
407        0
408    } else {
409        // Use integer approximation: count where base * 3^(count-1) would first exceed max
410        let mut accumulated = base;
411        let mut first_max_count = 1u64;
412        while accumulated < max && first_max_count < count {
413            accumulated = accumulated.saturating_mul(3);
414            first_max_count += 1;
415        }
416        first_max_count
417    };
418
419    // Times we've been at the max cap (excluding the first time we hit it)
420    let times_at_max = count.saturating_sub(first_max);
421
422    if !allow_extended {
423        return standard;
424    }
425
426    if standard == max && times_at_max >= EXTENDED_TIER_2_THRESHOLD {
427        max.saturating_mul(EXTENDED_TIER_2_MULTIPLIER)
428    } else if standard == max && times_at_max >= EXTENDED_TIER_1_THRESHOLD {
429        max.saturating_mul(EXTENDED_TIER_1_MULTIPLIER)
430    } else {
431        standard
432    }
433}
434
435/// Read the failure count for a key from KV, increment it, write it back, and return the new count.
436///
437/// Always increments — the backoff formula's `max` parameter prevents the
438/// cooldown duration from growing beyond the cap, so unbounded counts are safe.
439///
440/// Returns 1 when no store is configured (unit-test contexts without a store).
441/// Returns `u64::MAX` on store errors (e.g. lock contention) so backoff applies the cap duration
442/// instead of resetting to the base, preventing rapid re-dispatch during store outages.
443async fn read_and_increment_failure_count(store_opt: Option<&Arc<TaskStore>>, key: &str) -> u64 {
444    read_and_increment_failure_count_with_prefix(store_opt, FAILURE_COUNT_PREFIX, key).await
445}
446
447/// Like [`read_and_increment_failure_count`] but with an explicit KV prefix.
448///
449/// Credit exhaustion uses [`CREDIT_FAILURE_COUNT_PREFIX`] so its backoff escalation
450/// is independent of generic agent failures (which use [`FAILURE_COUNT_PREFIX`]).
451async fn read_and_increment_failure_count_with_prefix(
452    store_opt: Option<&Arc<TaskStore>>,
453    prefix: &str,
454    key: &str,
455) -> u64 {
456    let kv_key = format!("{prefix}{key}");
457    if let Some(store) = store_opt {
458        match store.kv_increment(&kv_key).await {
459            Ok(n) => n,
460            Err(e) => {
461                tracing::warn!(key = %kv_key, err = %e, "failed to increment failure count — treating as high count for safe backoff");
462                u64::MAX
463            }
464        }
465    } else {
466        1
467    }
468}
469
470/// Reset failure counts for an agent and agent:model combo after a successful run.
471///
472/// Call this from the runner's success path so that the next failure starts
473/// backoff from the base duration again, not from wherever it left off.
474/// Resets both generic and credit-exhaustion failure counters.
475pub async fn record_agent_success(agent_name: &str, model: &str) {
476    let store_opt = cooldown_store().lock().await.clone();
477    if let Some(store) = store_opt {
478        let agent_key = format!("{FAILURE_COUNT_PREFIX}{agent_name}");
479        let credit_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{agent_name}");
480        if let Err(e) = store.kv_set(&agent_key, "0").await {
481            tracing::warn!(key = agent_key, err = %e, "failed to reset failure count");
482        }
483        if !model.is_empty() {
484            let model_key = format!("{FAILURE_COUNT_PREFIX}{agent_name}:{model}");
485            if let Err(e) = store.kv_set(&model_key, "0").await {
486                tracing::warn!(key = model_key, err = %e, "failed to reset failure count");
487            }
488        }
489        if let Err(e) = store.kv_set(&credit_key, "0").await {
490            tracing::warn!(key = credit_key, err = %e, "failed to reset failure count");
491        }
492    }
493}
494
495/// Record that an agent has failed and should be temporarily avoided.
496///
497/// Applies exponential backoff based on the agent's failure count in KV.
498/// If `error_message` contains "try again at {date}", that vendor date is used
499/// instead of the computed backoff (vendor dates are always authoritative).
500pub async fn record_agent_failure_with_message(agent_name: &str, error_message: &str) {
501    // Vendor-specified retry date takes priority over backoff — but only
502    // if it is in the present or future. If the parsed timestamp is in the
503    // past (stale vendor message), fall through to the exponential backoff
504    // path instead of returning early and bypassing backoff entirely.
505    if let Some(cooldown_until) = parse_retry_at(error_message) {
506        let now = chrono::Utc::now().timestamp();
507        if cooldown_until > now {
508            set_cooldown_async(agent_name, cooldown_until, "agent_error").await;
509            return;
510        }
511        tracing::debug!(
512            agent = agent_name,
513            cooldown_until,
514            "vendor retry_at is in the past — using backoff instead"
515        );
516    }
517
518    let store_opt = cooldown_store().lock().await.clone();
519    let count = read_and_increment_failure_count(store_opt.as_ref(), agent_name).await;
520    let base = config::get_agent_backoff_base(agent_name);
521    let cooldown_until =
522        chrono::Utc::now().timestamp() + compute_backoff(count, base, BACKOFF_MAX_SECS, true);
523    set_cooldown_async(agent_name, cooldown_until, "agent_error").await;
524}
525
526/// Detect credit exhaustion from an error message and return the stable
527/// `task_runs.outcome` string for it (see `CreditExhaustionReason::outcome_str`).
528///
529/// Callers that classify `AgentError::RateLimit`/`AgentError::Auth` into an
530/// outcome string should consult this first — generic rate-limit patterns like
531/// "usage limit" also match billing-cycle exhaustion messages (e.g. Kimi's
532/// "You've reached your usage limit for this billing cycle"), which would
533/// otherwise be stored as a plain "rate_limit" outcome.
534pub fn detect_credit_exhaustion_outcome(error_message: &str) -> Option<&'static str> {
535    detect_credit_exhaustion(error_message).map(CreditExhaustionReason::outcome_str)
536}
537
538/// Detect credit exhaustion reasons from error message.
539///
540/// Returns `Some(CreditExhaustionReason)` if the error indicates
541/// out_of_credits or org_level_disabled, which require longer agent-wide cooldowns.
542pub fn detect_credit_exhaustion(error_message: &str) -> Option<CreditExhaustionReason> {
543    let lower = error_message.to_lowercase();
544
545    let overage_patterns = [
546        "overagedisabledreason",
547        "overage_disabled_reason",
548        "overage disabled reason",
549    ];
550
551    if overage_patterns.iter().any(|p| lower.contains(p)) {
552        if lower.contains("out_of_credits") || lower.contains("outofcredits") {
553            return Some(CreditExhaustionReason::OutOfCredits);
554        }
555        if lower.contains("org_level_disabled") || lower.contains("orgdisabled") {
556            return Some(CreditExhaustionReason::OrgLevelDisabled);
557        }
558    }
559
560    let billing_patterns = [
561        "credit balance too low",
562        "credit balance insufficient",
563        "insufficient credit balance",
564        "billing quota exceeded",
565        "billing limit exceeded",
566        "organization has been disabled",
567        "org disabled",
568    ];
569
570    if billing_patterns.iter().any(|p| lower.contains(p)) {
571        return Some(CreditExhaustionReason::OrgLevelDisabled);
572    }
573
574    // Billing cycle exhaustion: monthly quota reset — check before generic quota
575    // patterns because "monthly quota exceeded" would otherwise match "quota exceeded".
576    let billing_cycle_patterns = [
577        "billing cycle",
578        "quota refreshed next cycle",
579        "monthly quota",
580        "quota will be refreshed",
581        "next billing cycle",
582        "refreshed in the next cycle",
583        "weekly/monthly limit exhausted",
584        "limit will reset at",
585        "upgrade to pro", // codex/chatgpt plan usage cap: requires billing upgrade
586        "upgrade your token plan", // minimax: Token Plan usage limit reached
587        "token plan usage limit", // minimax: Token Plan usage limit reached
588        "purchase credits for more", // minimax: ...purchase Credits for more usage
589    ];
590
591    if billing_cycle_patterns.iter().any(|p| lower.contains(p)) {
592        return Some(CreditExhaustionReason::BillingCycleExhausted);
593    }
594
595    if lower.contains("out of credits")
596        || lower.contains("outofcredits")
597        || lower.contains("insufficient funds")
598        || lower.contains("no credits remaining")
599        || lower.contains("insufficient_quota")
600        || lower.contains("quota exceeded")
601        || lower.contains("insufficient balance")
602        || lower.contains("please recharge")
603        || lower.contains("no resource package")
604        || lower.contains("balance exhausted")
605        || lower.contains("402 payment required")
606    {
607        return Some(CreditExhaustionReason::OutOfCredits);
608    }
609
610    None
611}
612
613/// Record an agent-level cooldown for credit exhaustion.
614///
615/// Applies exponential backoff based on the agent's failure count.
616/// All variants use escalating backoff:
617/// - `OutOfCredits`: 1h, 3h, then 8h (credits can be refilled any time), 48h from the 5th failure, 96h from the 8th
618/// - `OrgLevelDisabled`: 2h, 6h, then 8h, 48h from the 5th failure, 96h from the 8th
619/// - `BillingCycleExhausted`: starts at 24h, caps at 7 days (monthly event)
620///
621/// Billing cycle exhaustion was previously a flat 24h, but this caused daily
622/// retry-and-fail cycles when a monthly quota was exhausted (the 24h cooldown
623/// would expire, kimi would be retried, fail immediately, and get another 24h).
624/// Now each recurrence escalates: 24h → 72h → 7d (capped).
625pub async fn record_credit_exhaustion(agent_name: &str, reason: CreditExhaustionReason) {
626    let reason_str = match reason {
627        CreditExhaustionReason::OutOfCredits => "credit_exhaustion_out_of_credits",
628        CreditExhaustionReason::OrgLevelDisabled => "credit_exhaustion_org_level_disabled",
629        CreditExhaustionReason::BillingCycleExhausted => "billing_cycle_exhausted",
630    };
631
632    // A failure from a run that started before the cooldown is the same event
633    if is_in_cooldown_with_reason(agent_name, reason_str) {
634        tracing::debug!(
635            agent = agent_name,
636            reason = reason_str,
637            "credit exhaustion already cooled, not escalating"
638        );
639        return;
640    }
641
642    let store_opt = cooldown_store().lock().await.clone();
643    // Use a distinct failure counter (CREDIT_FAILURE_COUNT_PREFIX) so credit-
644    // exhaustion backoff escalation is independent of generic agent failures.
645    // Previously both paths shared FAILURE_COUNT_PREFIX, causing generic failures
646    // to inflate credit-exhaustion backoff (and vice versa).
647    let count = read_and_increment_failure_count_with_prefix(
648        store_opt.as_ref(),
649        CREDIT_FAILURE_COUNT_PREFIX,
650        agent_name,
651    )
652    .await;
653
654    let (base, max) = match reason {
655        CreditExhaustionReason::OutOfCredits => (CREDIT_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS),
656        CreditExhaustionReason::OrgLevelDisabled => {
657            (ORG_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS)
658        }
659        CreditExhaustionReason::BillingCycleExhausted => {
660            (BILLING_CYCLE_COOLDOWN_SECS, BILLING_CYCLE_MAX_SECS)
661        }
662    };
663    let cooldown_secs = compute_backoff(
664        count,
665        base,
666        max,
667        !matches!(reason, CreditExhaustionReason::BillingCycleExhausted),
668    );
669    let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
670    set_cooldown_async(agent_name, cooldown_until, reason_str).await;
671    tracing::warn!(
672        agent = agent_name,
673        reason = reason_str,
674        attempt = count,
675        cooldown_secs,
676        "credit exhaustion detected: applying exponential agent-wide cooldown"
677    );
678}
679
680// Returns false when the message is no credit exhaustion. Model-scoped billing cycle cools the model only
681pub async fn record_credit_exhaustion_from_message(
682    agent_name: &str,
683    model: Option<&str>,
684    message: &str,
685) -> bool {
686    let Some(reason) = detect_credit_exhaustion(message) else {
687        return false;
688    };
689    match (reason, model) {
690        (CreditExhaustionReason::BillingCycleExhausted, Some(model)) => {
691            record_persistent_model_failure(agent_name, model).await;
692        }
693        _ => record_credit_exhaustion(agent_name, reason).await,
694    }
695    true
696}
697
698/// Record that a specific agent+model combo has failed.
699///
700/// Applies exponential backoff based on the model's failure count in KV.
701pub async fn record_model_failure(agent_name: &str, model: &str) {
702    let key = format!("{agent_name}:{model}");
703    let store_opt = cooldown_store().lock().await.clone();
704    let count = read_and_increment_failure_count(store_opt.as_ref(), &key).await;
705    let base = config::get_agent_backoff_base(agent_name);
706    let cooldown_until =
707        chrono::Utc::now().timestamp() + compute_backoff(count, base, BACKOFF_MAX_SECS, true);
708    set_cooldown_async(&key, cooldown_until, "model_error").await;
709}
710
711/// Record a rate-limit event for an agent+model.
712///
713/// If the error message advertises a relative usage window (e.g.
714/// "5-hour usage limit"), the cooldown is applied to the specific model for
715/// that window length. The quota is model-scoped, so the whole agent is not
716/// cooled. If no window is detected, fall back to the generic agent+model
717/// exponential backoff path.
718pub async fn record_rate_limit(agent_name: &str, model: Option<&str>, error_message: &str) {
719    if let Some(window_secs) = parse_relative_usage_window(error_message)
720        && let Some(m) = model
721    {
722        let cooldown_until = chrono::Utc::now()
723            .timestamp()
724            .saturating_add(i64::try_from(window_secs).unwrap_or(i64::MAX));
725        set_model_cooldown(agent_name, m, window_secs, "rate_limit_usage_window").await;
726        tracing::info!(
727            agent = agent_name,
728            model = m,
729            window_secs,
730            cooldown_until,
731            "relative usage window detected: applying model-specific cooldown"
732        );
733        return;
734    }
735
736    record_agent_failure_with_message(agent_name, error_message).await;
737    if let Some(m) = model {
738        record_model_failure(agent_name, m).await;
739    }
740}
741
742/// Record a persistent model-specific failure and apply longer exponential backoff.
743///
744/// Used for failure modes that are commonly broken for long periods (for example
745/// silent exit-0 or opaque provider errors). Starts at 4h and escalates with
746/// the generic backoff formula up to 7 days.
747///
748/// Uses [`PERSISTENT_FAILURE_COUNT_PREFIX`], a counter independent of the generic
749/// `failure_count:` key that `record_agent_success()` resets on every successful
750/// run. Otherwise a single intervening success between persistent failures would
751/// reset escalation back to the 4h base, letting structurally unreliable models
752/// (intermittent parse errors, truncated runs) stay in rotation indefinitely.
753pub async fn record_persistent_model_failure(agent_name: &str, model: &str) {
754    let key = format!("{agent_name}:{model}");
755    let store_opt = cooldown_store().lock().await.clone();
756    let count = read_and_increment_failure_count_with_prefix(
757        store_opt.as_ref(),
758        PERSISTENT_FAILURE_COUNT_PREFIX,
759        &key,
760    )
761    .await;
762    let cooldown_secs = compute_backoff(
763        count,
764        PERSISTENT_MODEL_BACKOFF_BASE_SECS,
765        PERSISTENT_MODEL_BACKOFF_MAX_SECS,
766        false,
767    );
768    let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
769    set_cooldown_async(&key, cooldown_until, "persistent_model_error").await;
770}
771
772/// Record a `ModelUnavailable` failure for an agent+model.
773///
774/// "Not found", "not supported" and region blocks never lift on their own, so they get the
775/// persistent backoff (4h base, 7d max). Any other message gets [`record_model_failure`]
776pub async fn record_model_unavailable(agent_name: &str, model: &str, message: &str) {
777    if is_model_permanently_gone(message) {
778        record_persistent_model_failure(agent_name, model).await;
779    } else {
780        record_model_failure(agent_name, model).await;
781    }
782}
783
784fn is_model_permanently_gone(message: &str) -> bool {
785    let lower = message.to_lowercase();
786    lower.contains("not found")
787        || lower.contains("not supported")
788        || lower.contains("not available in your country")
789        || lower.contains("not available in your region")
790}
791
792/// Record a run that failed without showing the model is gone: empty or cut off output, an
793/// unparseable review, an opaque provider error.
794///
795/// A stopped session or one odd answer causes these, so the backoff is 5 then 15 minutes. Billing
796/// exhaustion and "model not found" keep [`record_persistent_model_failure`]
797pub async fn record_transient_model_failure(agent_name: &str, model: &str) {
798    let key = format!("{agent_name}:{model}");
799    let store_opt = cooldown_store().lock().await.clone();
800    let count = read_and_increment_failure_count(store_opt.as_ref(), &key).await;
801    let cooldown_secs = compute_backoff(
802        count,
803        TRANSIENT_MODEL_BACKOFF_BASE_SECS,
804        TRANSIENT_MODEL_BACKOFF_MAX_SECS,
805        false,
806    );
807    let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
808    set_cooldown_async(&key, cooldown_until, "transient_model_error").await;
809}
810
811/// Set a model cooldown with a custom duration (in seconds) and reason.
812///
813/// Generic model-cooldown setter shared by silence detection and the
814/// rate-limit relative-usage-window path. The reason is persisted with
815/// the cooldown (KV + in-memory entry) so operators can tell the two
816/// apart in `orch cooldown list`.
817pub async fn set_model_cooldown(agent_name: &str, model: &str, duration_secs: u64, reason: &str) {
818    let key = format!("{agent_name}:{model}");
819    let cooldown_until = chrono::Utc::now()
820        .timestamp()
821        .saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX));
822    set_cooldown_async(&key, cooldown_until, reason).await;
823}
824
825/// Set a model cooldown that ends at a vendor-provided Unix timestamp.
826pub async fn set_model_cooldown_until(agent_name: &str, model: &str, until: i64) {
827    let key = format!("{agent_name}:{model}");
828    set_cooldown_async(&key, until, "vendor_reset_at").await;
829}
830
831/// Set a short agent-level cooldown (in seconds).
832///
833/// Used by silence detection to temporarily block the whole agent so the
834/// router picks a different one on re-route. Unlike `record_agent_failure`
835/// (exponential backoff), this uses a short duration (typically 120s) —
836/// just enough to force one re-route cycle to a different agent.
837pub async fn set_agent_cooldown(agent_name: &str, duration_secs: u64) {
838    let cooldown_until = chrono::Utc::now()
839        .timestamp()
840        .saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX));
841    set_cooldown_async(agent_name, cooldown_until, "silence_agent_cooldown").await;
842}
843
844/// KV prefix for the latest quota window of each type an agent reported: `rate_window:{agent}:{type}`
845pub const RATE_WINDOW_KV_PREFIX: &str = "rate_window:";
846
847/// One quota window from a vendor `rate_limit_event`, with the vendor window type kept as sent
848#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
849pub struct RateLimitWindow {
850    pub status: String,
851    pub window_type: String,
852    pub utilization: Option<f64>,
853    pub resets_at: Option<i64>,
854}
855
856/// Store the latest window for an agent. A rejected window cools the agent until its reset time
857pub async fn record_rate_limit_window(agent_name: &str, window: &RateLimitWindow) {
858    let store_opt = cooldown_store().lock().await.clone();
859    if let Some(store) = store_opt
860        && let Ok(value) = serde_json::to_string(window)
861        && let Err(e) = store
862            .kv_set(
863                &format!("{RATE_WINDOW_KV_PREFIX}{agent_name}:{}", window.window_type),
864                &value,
865            )
866            .await
867    {
868        tracing::warn!(agent = agent_name, err = %e, "failed to persist rate limit window");
869    }
870
871    let now = chrono::Utc::now().timestamp();
872    if window.status == "rejected"
873        && let Some(reset) = window.resets_at
874        && reset > now
875    {
876        let until = reset.min(now + PERSISTENT_MODEL_BACKOFF_MAX_SECS);
877        set_cooldown_async(agent_name, until, "rate_limit_window").await;
878        tracing::info!(
879            agent = agent_name,
880            window = %window.window_type,
881            cooldown_until = until,
882            "rate limit window rejected: cooling agent until vendor reset"
883        );
884    }
885}
886
887/// Latest stored quota window per agent, sorted by agent name
888pub async fn list_rate_limit_windows(store: &Arc<TaskStore>) -> Vec<(String, RateLimitWindow)> {
889    let rows = store
890        .kv_list_prefix(RATE_WINDOW_KV_PREFIX)
891        .await
892        .unwrap_or_default();
893    let mut windows: Vec<(String, RateLimitWindow)> = rows
894        .into_iter()
895        .filter_map(|(key, value)| {
896            let (agent, _) = key.strip_prefix(RATE_WINDOW_KV_PREFIX)?.rsplit_once(':')?;
897            let window = serde_json::from_str(&value).ok()?;
898            Some((agent.to_string(), window))
899        })
900        .collect();
901    windows.sort_by(|a, b| (&a.0, &a.1.window_type).cmp(&(&b.0, &b.1.window_type)));
902    windows
903}
904
905/// Record a silence detection for an agent+model and apply extended cooldowns
906/// when repeated silences exceed the threshold within the rolling window.
907pub async fn record_silence_detection(agent_name: &str, model: &str) -> Option<SilenceCountResult> {
908    let store_opt = cooldown_store().lock().await.clone();
909    let Some(store) = store_opt else {
910        tracing::debug!(
911            agent = agent_name,
912            model,
913            "skipping silence count record (no KV store)"
914        );
915        return None;
916    };
917
918    let key = format!("{SILENCE_COUNT_PREFIX}{agent_name}:{model}");
919    let now = chrono::Utc::now().timestamp();
920    let window_start = now - SILENCE_COUNT_WINDOW_SECS;
921
922    let mut timestamps = match store.kv_get(&key).await {
923        Ok(Some(raw)) => match serde_json::from_str::<Vec<i64>>(&raw) {
924            Ok(ts) => ts,
925            Err(e) => {
926                tracing::warn!(
927                    kv_key = %key,
928                    err = %e,
929                    "failed to parse silence timestamps from KV — resetting window"
930                );
931                Vec::new()
932            }
933        },
934        Ok(None) => Vec::new(),
935        Err(err) => {
936            tracing::warn!(
937                kv_key = %key,
938                err = %err,
939                "failed to load silence count from KV"
940            );
941            Vec::new()
942        }
943    };
944
945    timestamps.retain(|ts| *ts >= window_start);
946    timestamps.push(now);
947
948    let count = timestamps.len();
949    let mut extended_cooldown_applied = false;
950    if count >= SILENCE_COUNT_THRESHOLD {
951        set_model_cooldown(
952            agent_name,
953            model,
954            SILENCE_EXTENDED_COOLDOWN_SECS,
955            "silence_detected",
956        )
957        .await;
958        extended_cooldown_applied = true;
959        timestamps.clear();
960    }
961
962    let value = serde_json::to_string(&timestamps).unwrap_or_else(|_| "[]".to_string());
963    if let Err(err) = store.kv_set(&key, &value).await {
964        tracing::warn!(kv_key = key, err = %err, "failed to persist silence count");
965    }
966
967    Some(SilenceCountResult {
968        count,
969        extended_cooldown_applied,
970    })
971}
972
973/// Check if a specific agent+model combo is in cooldown.
974pub fn is_model_in_cooldown(agent_name: &str, model: &str) -> bool {
975    let key = format!("{agent_name}:{model}");
976    is_in_cooldown(&key)
977}
978
979/// Check if an agent is currently in cooldown period.
980pub fn is_agent_in_cooldown(agent_name: &str) -> bool {
981    is_in_cooldown(agent_name)
982}
983
984/// Return the cooldown expiry timestamp for a key (agent or agent:model), if active.
985pub fn cooldown_until(key: &str) -> Option<i64> {
986    let map = cooldown_lock();
987    map.get(key).and_then(|entry| {
988        let now = chrono::Utc::now().timestamp();
989        if now < entry.cooldown_until {
990            Some(entry.cooldown_until)
991        } else {
992            None
993        }
994    })
995}
996
997/// Return the reason string recorded when a cooldown was set, if the cooldown is still active.
998pub fn cooldown_reason(key: &str) -> Option<String> {
999    let map = cooldown_lock();
1000    map.get(key).and_then(|entry| {
1001        let now = chrono::Utc::now().timestamp();
1002        if now < entry.cooldown_until {
1003            Some(entry.reason.clone())
1004        } else {
1005            None
1006        }
1007    })
1008}
1009
1010/// Return all currently active cooldowns as `(key, cooldown_until_unix, reason)`.
1011///
1012/// Reads from the in-memory map only — always fast, no async needed.
1013/// Expired entries are filtered out. Used by `orch cooldown list`.
1014pub fn list_all_cooldowns() -> Vec<(String, i64, String)> {
1015    let now = chrono::Utc::now().timestamp();
1016    let map = cooldown_lock();
1017    let mut result: Vec<(String, i64, String)> = map
1018        .iter()
1019        .filter(|(_, entry)| now < entry.cooldown_until)
1020        .map(|(key, entry)| (key.clone(), entry.cooldown_until, entry.reason.clone()))
1021        .collect();
1022    result.sort_by(|a, b| a.0.cmp(&b.0));
1023    result
1024}
1025
1026/// Write `rate_limits_cleared_at:{agent}` = now so the next health check
1027/// ignores older `rate_limits` rows. Rows stay in the table for metrics
1028pub async fn mark_rate_limits_cleared(store: &Arc<TaskStore>, agents: &[String]) {
1029    let now = chrono::Utc::now().timestamp().to_string();
1030    let mut seen = HashSet::new();
1031    for key in agents {
1032        let agent = agent_from_key(key);
1033        if agent.is_empty() || !seen.insert(agent.to_string()) {
1034            continue;
1035        }
1036        let kv_key = format!("{RATE_LIMITS_CLEARED_AT_PREFIX}{agent}");
1037        if let Err(e) = store.kv_set(&kv_key, &now).await {
1038            tracing::warn!(agent, err = %e, "failed to mark rate limits cleared");
1039        }
1040    }
1041}
1042
1043/// Clear a specific cooldown by key, or all cooldowns when `key == "*"`.
1044///
1045/// Removes from the in-memory map, writes a past timestamp to KV, and resets
1046/// the corresponding `failure_count:` entries so the next failure starts
1047/// backoff from the base duration. Also writes `rate_limits_cleared_at:{agent}`
1048/// so a degraded agent held only by rate rows recovers on the next health check
1049pub async fn clear_cooldown(key: &str, store: &Arc<TaskStore>) {
1050    let mut agents_to_mark: Vec<String> = Vec::new();
1051    if key == "*" {
1052        let keys: Vec<String> = {
1053            let mut map = cooldown_lock();
1054            let keys: Vec<String> = map.keys().cloned().collect();
1055            map.clear();
1056            keys
1057        };
1058        for k in &keys {
1059            let kv_key = format!("{KV_PREFIX}{k}");
1060            if let Err(e) = store.kv_set(&kv_key, "0").await {
1061                tracing::warn!(key = kv_key, err = %e, "failed to clear cooldown");
1062            }
1063            // Reset failure count so backoff restarts from base.
1064            let fc_key = format!("{FAILURE_COUNT_PREFIX}{k}");
1065            if let Err(e) = store.kv_set(&fc_key, "0").await {
1066                tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1067            }
1068            // Also reset credit-specific failure count.
1069            let credit_fc_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{k}");
1070            if let Err(e) = store.kv_set(&credit_fc_key, "0").await {
1071                tracing::warn!(key = credit_fc_key, err = %e, "failed to reset credit failure count");
1072            }
1073            // Also reset the persistent-model-failure counter.
1074            let persistent_fc_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{k}");
1075            if let Err(e) = store.kv_set(&persistent_fc_key, "0").await {
1076                tracing::warn!(key = persistent_fc_key, err = %e, "failed to reset persistent failure count");
1077            }
1078        }
1079        // Also reset any persisted failure_count keys that are not in the
1080        // in-memory map (e.g. survived a restart or were set without a
1081        // corresponding in-memory cooldown entry).
1082        if let Ok(fc_entries) = store.kv_list_prefix(FAILURE_COUNT_PREFIX).await {
1083            for (fc_key, _) in fc_entries {
1084                if let Err(e) = store.kv_set(&fc_key, "0").await {
1085                    tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1086                }
1087            }
1088        }
1089        if let Ok(cfc_entries) = store.kv_list_prefix(CREDIT_FAILURE_COUNT_PREFIX).await {
1090            for (cfc_key, _) in cfc_entries {
1091                if let Err(e) = store.kv_set(&cfc_key, "0").await {
1092                    tracing::warn!(key = cfc_key, err = %e, "failed to reset credit failure count");
1093                }
1094            }
1095        }
1096        if let Ok(pfc_entries) = store.kv_list_prefix(PERSISTENT_FAILURE_COUNT_PREFIX).await {
1097            for (pfc_key, _) in pfc_entries {
1098                if let Err(e) = store.kv_set(&pfc_key, "0").await {
1099                    tracing::warn!(key = pfc_key, err = %e, "failed to reset persistent failure count");
1100                }
1101            }
1102        }
1103        tracing::info!(
1104            count = keys.len(),
1105            "cleared all cooldowns and failure counts"
1106        );
1107        agents_to_mark.extend(keys.iter().map(|k| agent_from_key(k).to_string()));
1108        match store.rate_limit_agents().await {
1109            Ok(rl_agents) => agents_to_mark.extend(rl_agents),
1110            Err(e) => tracing::warn!(err = %e, "failed to list rate_limits agents while clearing"),
1111        }
1112        match store.kv_list_prefix(RATE_LIMITS_CLEARED_AT_PREFIX).await {
1113            Ok(marks) => {
1114                for (k, _) in marks {
1115                    let agent = k.trim_start_matches(RATE_LIMITS_CLEARED_AT_PREFIX);
1116                    if !agent.is_empty() {
1117                        agents_to_mark.push(agent.to_string());
1118                    }
1119                }
1120            }
1121            Err(e) => tracing::warn!(err = %e, "failed to list rate_limits_cleared_at keys"),
1122        }
1123    } else {
1124        {
1125            let mut map = cooldown_lock();
1126            map.remove(key);
1127        }
1128        let kv_key = format!("{KV_PREFIX}{key}");
1129        if let Err(e) = store.kv_set(&kv_key, "0").await {
1130            tracing::warn!(key = kv_key, err = %e, "failed to clear cooldown");
1131        }
1132        // Reset failure count so backoff restarts from base.
1133        let fc_key = format!("{FAILURE_COUNT_PREFIX}{key}");
1134        if let Err(e) = store.kv_set(&fc_key, "0").await {
1135            tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1136        }
1137        // Also reset credit-specific failure count.
1138        let credit_fc_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{key}");
1139        if let Err(e) = store.kv_set(&credit_fc_key, "0").await {
1140            tracing::warn!(key = credit_fc_key, err = %e, "failed to reset credit failure count");
1141        }
1142        // Also reset the persistent-model-failure counter.
1143        let persistent_fc_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{key}");
1144        if let Err(e) = store.kv_set(&persistent_fc_key, "0").await {
1145            tracing::warn!(key = persistent_fc_key, err = %e, "failed to reset persistent failure count");
1146        }
1147        tracing::info!(key, "cleared cooldown and failure count");
1148        agents_to_mark.push(agent_from_key(key).to_string());
1149    }
1150    mark_rate_limits_cleared(store, &agents_to_mark).await;
1151}
1152
1153/// Set a cooldown with a specific expiry timestamp. Updates in-memory state only.
1154///
1155/// Returns `true` if the cooldown was applied, `false` if an existing longer
1156/// cooldown prevented the update (never-shorten rule).
1157///
1158/// **Callers are responsible for persisting to KV.** Use [`set_cooldown_async`]
1159/// for critical cooldowns that must survive a crash, or this function combined
1160/// with a fire-and-forget `tokio::spawn` for non-critical short cooldowns.
1161fn set_cooldown_in_memory(key: &str, cooldown_until: i64, reason: &str) -> bool {
1162    let mut map = cooldown_lock();
1163    if let Some(existing) = map.get(key) {
1164        // Never shorten an existing cooldown. This prevents a short
1165        // retry window (e.g., generic rate limit) from overriding a longer
1166        // billing-cycle cooldown.
1167        if existing.cooldown_until >= cooldown_until {
1168            return false;
1169        }
1170    }
1171    map.insert(
1172        key.to_string(),
1173        CooldownEntry {
1174            cooldown_until,
1175            reason: reason.to_string(),
1176        },
1177    );
1178    true
1179}
1180
1181/// Set a cooldown with a specific expiry timestamp and persist to KV inline.
1182///
1183/// This is the preferred function for critical cooldowns (credit exhaustion,
1184/// billing cycle, agent/model failures) that must survive a service crash.
1185/// The KV write is awaited before returning, so the caller can be confident
1186/// the cooldown is durable once this function returns `true`.
1187///
1188/// Returns `true` if the cooldown was applied, `false` if an existing longer
1189/// cooldown prevented the update (never-shorten rule).
1190async fn set_cooldown_async(key: &str, cooldown_until: i64, reason: &str) -> bool {
1191    if !set_cooldown_in_memory(key, cooldown_until, reason) {
1192        return false;
1193    }
1194    let store_opt = cooldown_store().lock().await.clone();
1195    if let Some(store) = store_opt {
1196        let kv_key = format!("{KV_PREFIX}{key}");
1197        let value = serde_json::json!({"until": cooldown_until, "reason": reason}).to_string();
1198        if let Err(e) = store.kv_set(&kv_key, &value).await {
1199            tracing::warn!(
1200                kv_key,
1201                err = %e,
1202                "failed to persist cooldown to KV store — rolling back in-memory entry"
1203            );
1204            let mut map = cooldown_lock();
1205            // Only roll back if nothing has superseded our write.
1206            // A racing thread may have set a longer cooldown in the meantime;
1207            // removing unconditionally would erase that valid entry.
1208            if map.get(key).map(|e| e.cooldown_until) == Some(cooldown_until) {
1209                map.remove(key);
1210            }
1211            return false;
1212        }
1213    }
1214    true
1215}
1216
1217/// Helper to acquire the cooldown map lock, logging and recovering on poison.
1218fn cooldown_lock() -> MutexGuard<'static, HashMap<String, CooldownEntry>> {
1219    cooldowns().lock().unwrap_or_else(|e| {
1220        tracing::error!("cooldown map lock poisoned — recovering with partial state; backoff and degraded-agent decisions may be incorrect");
1221        e.into_inner()
1222    })
1223}
1224
1225/// Helper to acquire the degraded agents lock, logging and recovering on poison.
1226fn degraded_agents_lock() -> MutexGuard<'static, HashMap<String, String>> {
1227    degraded_agents().lock().unwrap_or_else(|e| {
1228        tracing::error!("degraded agents lock poisoned — recovering with partial state; health check decisions may be incorrect");
1229        e.into_inner()
1230    })
1231}
1232
1233/// Helper to acquire the GitHub 5xx timestamps lock, logging and recovering on poison.
1234fn github_5xx_timestamps_lock() -> MutexGuard<'static, Vec<i64>> {
1235    github_5xx_timestamps().lock().unwrap_or_else(|e| {
1236        tracing::error!("github 5xx timestamps lock poisoned — recovering with partial state; circuit breaker may operate incorrectly");
1237        e.into_inner()
1238    })
1239}
1240
1241/// Helper to acquire the GitHub 5xx circuit open lock, logging and recovering on poison.
1242fn github_5xx_circuit_open_lock() -> MutexGuard<'static, Option<i64>> {
1243    github_5xx_circuit_open().lock().unwrap_or_else(|e| {
1244        tracing::error!("github 5xx circuit open lock poisoned — recovering with partial state; circuit breaker may operate incorrectly");
1245        e.into_inner()
1246    })
1247}
1248
1249/// Check if a key is in cooldown with the given reason.
1250fn is_in_cooldown_with_reason(key: &str, reason: &str) -> bool {
1251    let map = cooldown_lock();
1252    map.get(key).is_some_and(|entry| {
1253        entry.reason == reason && chrono::Utc::now().timestamp() < entry.cooldown_until
1254    })
1255}
1256
1257/// Check if a key is currently in cooldown.
1258fn is_in_cooldown(key: &str) -> bool {
1259    let map = cooldown_lock();
1260    if let Some(entry) = map.get(key) {
1261        let now = chrono::Utc::now().timestamp();
1262        return now < entry.cooldown_until;
1263    }
1264    false
1265}
1266
1267// ---------------------------------------------------------------------------
1268// Degraded-agent tracking (pre-emptive health check)
1269// ---------------------------------------------------------------------------
1270
1271/// Default health check window for recent rate-limit events (hours).
1272pub const DEFAULT_HEALTH_CHECK_WINDOW_HOURS: u32 = 6;
1273
1274/// Default threshold: minimum rate-limit events within the window to consider
1275/// an agent degraded.  A single transient 429 shouldn't mark an agent; we need
1276/// a pattern of repeated failures.
1277pub const DEFAULT_DEGRADED_RATE_LIMIT_THRESHOLD: i64 = 3;
1278
1279/// In-memory map of agents currently flagged as degraded, keyed by agent
1280/// with the reason that holds now
1281fn degraded_agents() -> &'static Mutex<HashMap<String, String>> {
1282    static DEGRADED: OnceLock<Mutex<HashMap<String, String>>> = OnceLock::new();
1283    DEGRADED.get_or_init(|| Mutex::new(HashMap::new()))
1284}
1285
1286/// Check if an agent is currently flagged as degraded by the health check.
1287///
1288/// Degraded means: all configured models are cooled **or** the agent has
1289/// exceeded the rate-limit threshold within the configured lookback window.
1290pub fn is_agent_degraded(agent: &str) -> bool {
1291    let set = degraded_agents_lock();
1292    set.contains_key(agent)
1293}
1294
1295/// Reason the agent is currently degraded, if it is
1296pub fn agent_degraded_reason(agent: &str) -> Option<String> {
1297    degraded_agents_lock().get(agent).cloned()
1298}
1299
1300/// Active degraded agents as `(agent, reason)`, sorted by agent name
1301pub fn list_degraded_agents() -> Vec<(String, String)> {
1302    let mut entries: Vec<(String, String)> = degraded_agents_lock()
1303        .iter()
1304        .map(|(agent, reason)| (agent.clone(), reason.clone()))
1305        .collect();
1306    entries.sort_by(|a, b| a.0.cmp(&b.0));
1307    entries
1308}
1309
1310/// Mark an agent as degraded (called by [`refresh_degraded_agents`]).
1311pub fn mark_agent_degraded(agent: &str) {
1312    mark_agent_degraded_with_reason(agent, "degraded");
1313}
1314
1315fn mark_agent_degraded_with_reason(agent: &str, reason: &str) {
1316    let mut set = degraded_agents_lock();
1317    set.insert(agent.to_string(), reason.to_string());
1318}
1319
1320/// Clear the degraded flag for an agent.
1321pub fn clear_agent_degraded(agent: &str) {
1322    let mut set = degraded_agents_lock();
1323    set.remove(agent);
1324}
1325
1326/// Refresh the degraded-agent set from the rate_limits table and cooldown state.
1327///
1328/// An agent is marked degraded when:
1329/// 1. It is in agent-level cooldown (explicit cooldown KV key exists and hasn't expired), **or**
1330/// 2. All its configured models are in cooldown, **or**
1331/// 3. It has >= `threshold` `limit_type = 'rate'` events within `window_hours`.
1332///
1333/// Failures keep their own cooldowns and do not count toward the threshold
1334/// Agents that no longer meet these criteria are cleared.
1335///
1336/// Note: This function calls `clear_expired_cooldowns()` first to ensure the
1337/// in-memory cooldown map reflects current state. An agent should NOT be marked
1338/// degraded simply because an expired cooldown entry hasn't been cleaned up yet.
1339pub async fn refresh_degraded_agents(
1340    store: &Arc<TaskStore>,
1341    available_agents: &[String],
1342    model_checker: &(dyn Fn(&str) -> bool + Send + Sync),
1343    window_hours: u32,
1344    threshold: i64,
1345) {
1346    // Clear expired cooldowns from in-memory map before evaluating agent state.
1347    // This ensures we don't incorrectly mark agents as degraded due to stale
1348    // in-memory entries that should have expired but weren't cleaned up.
1349    clear_expired_cooldowns();
1350
1351    // Time the DB query to surface slow health-checks that can dominate the
1352    // engine tick latency. If the query fails, log and return early.
1353    let start = chrono::Utc::now();
1354    let counts = match store.recent_rate_limit_counts(window_hours).await {
1355        Ok(c) => {
1356            let dur_ms = (chrono::Utc::now() - start).num_milliseconds();
1357            tracing::info!(
1358                duration_ms = dur_ms,
1359                window_hours,
1360                "recent_rate_limit_counts query completed"
1361            );
1362            c
1363        }
1364        Err(e) => {
1365            let dur_ms = (chrono::Utc::now() - start).num_milliseconds();
1366            tracing::warn!(
1367                err = %e,
1368                duration_ms = dur_ms,
1369                "failed to query recent rate limit counts for health check"
1370            );
1371            return;
1372        }
1373    };
1374
1375    for agent in available_agents {
1376        let in_cooldown = is_agent_in_cooldown(agent);
1377        let no_models = !model_checker(agent);
1378        let rate_limit_count = counts.get(agent.as_str()).copied().unwrap_or(0);
1379        let over_threshold = rate_limit_count >= threshold;
1380
1381        let degraded = in_cooldown || no_models || over_threshold;
1382
1383        if degraded {
1384            let reason = if in_cooldown {
1385                "agent in cooldown"
1386            } else if no_models {
1387                "all models cooled"
1388            } else {
1389                "rate limit threshold exceeded"
1390            };
1391            let previous = agent_degraded_reason(agent);
1392            match previous.as_deref() {
1393                None => tracing::warn!(
1394                    agent,
1395                    reason,
1396                    rate_limit_count,
1397                    window_hours,
1398                    threshold,
1399                    "pre-emptive health check: marking agent as degraded"
1400                ),
1401                Some(prev) if prev != reason => tracing::warn!(
1402                    agent,
1403                    reason,
1404                    previous = prev,
1405                    rate_limit_count,
1406                    window_hours,
1407                    threshold,
1408                    "pre-emptive health check: degraded reason changed"
1409                ),
1410                Some(_) => {}
1411            }
1412            mark_agent_degraded_with_reason(agent, reason);
1413        } else {
1414            if is_agent_degraded(agent) {
1415                tracing::info!(
1416                    agent,
1417                    rate_limit_count,
1418                    "pre-emptive health check: agent recovered, clearing degraded flag"
1419                );
1420            }
1421            clear_agent_degraded(agent);
1422        }
1423    }
1424}
1425
1426/// Clear expired cooldowns from the in-memory map.
1427pub fn clear_expired_cooldowns() {
1428    let mut map = cooldown_lock();
1429    let now = chrono::Utc::now().timestamp();
1430    map.retain(|_key, entry| now < entry.cooldown_until);
1431}
1432
1433// ---------------------------------------------------------------------------
1434// GitHub 5xx circuit breaker
1435// ---------------------------------------------------------------------------
1436
1437/// Sliding window for counting GitHub 5xx errors (seconds).
1438const GITHUB_5XX_WINDOW_SECS: i64 = 120;
1439
1440/// Number of 5xx errors within the window to trip the circuit breaker.
1441const GITHUB_5XX_THRESHOLD: usize = 5;
1442
1443/// Duration the circuit breaker stays open after tripping (seconds).
1444const GITHUB_5XX_COOLDOWN_SECS: i64 = 180;
1445
1446/// Global sliding window of 5xx error timestamps.
1447fn github_5xx_timestamps() -> &'static Mutex<Vec<i64>> {
1448    static TS: OnceLock<Mutex<Vec<i64>>> = OnceLock::new();
1449    TS.get_or_init(|| Mutex::new(Vec::new()))
1450}
1451
1452/// Whether the GitHub 5xx circuit breaker is currently open (tripped).
1453fn github_5xx_circuit_open() -> &'static Mutex<Option<i64>> {
1454    static OPEN: OnceLock<Mutex<Option<i64>>> = OnceLock::new();
1455    OPEN.get_or_init(|| Mutex::new(None))
1456}
1457
1458/// Record a GitHub 5xx server error and trip the circuit breaker if the
1459/// sliding window threshold is exceeded.
1460///
1461/// Emits a single high-level log entry when the circuit transitions to open.
1462pub async fn record_github_5xx() {
1463    let now = chrono::Utc::now().timestamp();
1464    let window_start = now - GITHUB_5XX_WINDOW_SECS;
1465
1466    let should_trip = {
1467        let mut ts = github_5xx_timestamps_lock();
1468        ts.retain(|t| *t >= window_start);
1469        ts.push(now);
1470        ts.len() >= GITHUB_5XX_THRESHOLD
1471    };
1472
1473    if should_trip {
1474        let cooldown_until = {
1475            let mut open = github_5xx_circuit_open_lock();
1476            if open.is_none() {
1477                let cd = now + GITHUB_5XX_COOLDOWN_SECS;
1478                *open = Some(cd);
1479                tracing::warn!(
1480                    errors_in_window = GITHUB_5XX_THRESHOLD,
1481                    window_secs = GITHUB_5XX_WINDOW_SECS,
1482                    cooldown_secs = GITHUB_5XX_COOLDOWN_SECS,
1483                    "GitHub 5xx circuit breaker OPEN — throttling non-critical work"
1484                );
1485                Some(cd)
1486            } else {
1487                None
1488            }
1489        };
1490        // Persist to KV (outside the mutex lock so the future is Send).
1491        if let Some(cd) = cooldown_until {
1492            let store_opt = cooldown_store().lock().await.clone();
1493            if let Some(store) = store_opt {
1494                let _ = store.kv_set("cooldown:github:5xx", &cd.to_string()).await;
1495            }
1496            // Also set the generic cooldown so is_agent_in_cooldown("github:5xx")
1497            // returns true immediately, avoiding a race window where concurrent
1498            // requests bypass the circuit breaker.
1499            set_agent_cooldown("github:5xx", GITHUB_5XX_COOLDOWN_SECS as u64).await;
1500        }
1501        // Clear the sliding window after tripping so we don't re-trip immediately
1502        let mut ts = github_5xx_timestamps_lock();
1503        ts.clear();
1504    }
1505}
1506
1507/// Check if the GitHub 5xx circuit breaker is currently open.
1508///
1509/// When open, non-critical background work (cleanup, polling, mention scans)
1510/// should be skipped. Critical operations (PR creation) should still attempt
1511/// with their existing exponential backoff.
1512///
1513/// Automatically recovers (logs a recovery message) when the cooldown expires.
1514pub fn is_github_circuit_open() -> bool {
1515    // Check the dedicated in-memory circuit flag first. If present, honour it
1516    // and perform auto-recovery when it expires (logging once on close).
1517    {
1518        let mut open = github_5xx_circuit_open_lock();
1519        if let Some(until) = *open {
1520            let now = chrono::Utc::now().timestamp();
1521            if now >= until {
1522                *open = None;
1523                tracing::info!("GitHub 5xx circuit breaker CLOSED — resuming normal operations");
1524                return false;
1525            }
1526            return true;
1527        }
1528    }
1529
1530    // If the dedicated flag is not set, consult the generic cooldown map so
1531    // callers see a consistent view regardless of which code path set the
1532    // persisted cooldown (e.g. send_with_retries -> set_agent_cooldown).
1533    if let Some(until) = cooldown_until("github:5xx") {
1534        // Populate the dedicated in-memory flag so future checks and the
1535        // auto-recovery path go through the same code and logging.
1536        let mut open = github_5xx_circuit_open_lock();
1537        if open.is_none() {
1538            *open = Some(until);
1539            let now = chrono::Utc::now().timestamp();
1540            if now < until {
1541                let remaining = until - now;
1542                tracing::warn!(
1543                    remaining_secs = remaining,
1544                    "GitHub 5xx circuit breaker OPEN (discovered via generic cooldown) — throttling non-critical work"
1545                );
1546            }
1547        }
1548        return true;
1549    }
1550
1551    false
1552}
1553
1554/// Returns the remaining seconds of the GitHub 5xx circuit breaker cooldown,
1555/// or 0 if the circuit is closed.
1556pub fn github_circuit_remaining_secs() -> u64 {
1557    // Prefer the dedicated in-memory value when present.
1558    {
1559        let open = github_5xx_circuit_open_lock();
1560        if let Some(until) = *open {
1561            let now = chrono::Utc::now().timestamp();
1562            if now < until {
1563                return u64::try_from(until - now).unwrap_or(0);
1564            }
1565            return 0;
1566        }
1567    }
1568
1569    // Fall back to the generic cooldown map if the dedicated flag isn't set.
1570    if let Some(until) = cooldown_until("github:5xx") {
1571        let now = chrono::Utc::now().timestamp();
1572        if now < until {
1573            return u64::try_from(until - now).unwrap_or(0);
1574        }
1575    }
1576    0
1577}
1578
1579/// Parse a "try again at {date}" or "reset at {date}" from an error message.
1580///
1581/// Returns a Unix timestamp if a retry-at date is found.
1582pub(crate) fn parse_retry_at(error_message: &str) -> Option<i64> {
1583    if error_message.is_empty() {
1584        return None;
1585    }
1586
1587    let lower = error_message.to_lowercase();
1588    // Accept a wider set of vendor-provided markers. Claude commonly uses
1589    // variations like "resets 10:30am" (no explicit "at"), so include
1590    // both "resets" and the older "reset at"/"try again at" markers.
1591    for retry_marker in [
1592        "try again at ",
1593        "reset at ",
1594        "resets at ",
1595        "resets ",
1596        "reset ",
1597    ] {
1598        if let Some(idx) = lower.find(retry_marker) {
1599            let date_str = &lower[idx + retry_marker.len()..];
1600            // Take until period, newline, or end
1601            let date_str = date_str
1602                .split(['.', '\n'])
1603                .next()
1604                .unwrap_or(date_str)
1605                .trim();
1606
1607            // Try common date formats
1608            // "Mar 26th, 2026 5:55 AM" → strip ordinal suffixes
1609            let cleaned = date_str
1610                .replace("st,", ",")
1611                .replace("nd,", ",")
1612                .replace("rd,", ",")
1613                .replace("th,", ",");
1614
1615            for fmt in &[
1616                "%b %d, %Y %I:%M %p", // Mar 26, 2026 5:55 AM
1617                "%b %d, %Y %I:%M%p",  // Mar 26, 2026 5:55AM
1618                "%B %d, %Y %I:%M %p", // March 26, 2026 5:55 AM
1619                "%Y-%m-%dT%H:%M:%S",  // 2026-03-26T05:55:00
1620                "%Y-%m-%d %H:%M:%S",  // 2026-03-26 05:55:00
1621                "%Y-%m-%d %H:%M",     // 2026-03-26 05:55
1622            ] {
1623                if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(&cleaned, fmt) {
1624                    // Vendor messages (e.g. Codex/GLM) emit times in the machine's
1625                    // local timezone, not UTC. Interpret naive datetime as local time.
1626                    let utc = match chrono::Local.from_local_datetime(&dt) {
1627                        chrono::LocalResult::Single(local_dt) => {
1628                            local_dt.with_timezone(&chrono::Utc)
1629                        }
1630                        chrono::LocalResult::Ambiguous(earliest, _) => {
1631                            // DST fold: pick the earlier (conservative) interpretation
1632                            earliest.with_timezone(&chrono::Utc)
1633                        }
1634                        chrono::LocalResult::None => {
1635                            // DST gap: fall back to treating as UTC to avoid panic
1636                            dt.and_utc()
1637                        }
1638                    };
1639                    tracing::info!(
1640                        retry_at = %utc,
1641                        raw = date_str,
1642                        marker = retry_marker,
1643                        "parsed rate limit retry-at date"
1644                    );
1645                    return Some(utc.timestamp());
1646                }
1647            }
1648            tracing::debug!(
1649                raw = date_str,
1650                marker = retry_marker,
1651                "could not parse retry-at date"
1652            );
1653
1654            // If the vendor provided only a time (e.g. "resets 10:30am"),
1655            // attempt to parse a time-only and interpret it as the next
1656            // occurrence of that time in the local timezone.
1657            // Strip parenthesised timezone hints like "(America/Sao_Paulo)".
1658            let time_only = date_str.split_once('(').map_or(date_str, |(t, _)| t).trim();
1659            // Hour-only values like "8am" have no minute field, which chrono needs, so insert ":00"
1660            let mut compact = time_only.replace(" am", "am").replace(" pm", "pm");
1661            let split = compact.rfind(' ').map_or(0, |i| i + 1);
1662            let last = &compact[split..];
1663            if !last.contains(':')
1664                && (last.ends_with("am") || last.ends_with("pm"))
1665                && last.len() > 2
1666            {
1667                let at = split + last.len() - 2;
1668                compact.insert_str(at, ":00");
1669            }
1670            // "Jun 9 at 1am" has no year, so take the next future occurrence
1671            let md = compact.replace(" at ", " ");
1672            let now_local = chrono::Local::now();
1673            let this_year = chrono::Datelike::year(&now_local);
1674            for year in [this_year, this_year + 1] {
1675                let with_year = format!("{year} {md}");
1676                if let Ok(dt) =
1677                    chrono::NaiveDateTime::parse_from_str(&with_year, "%Y %b %d %I:%M%p")
1678                    && let Some(local_dt) = chrono::Local.from_local_datetime(&dt).earliest()
1679                    && local_dt > now_local
1680                {
1681                    return Some(local_dt.timestamp());
1682                }
1683            }
1684            let time_only = compact.as_str();
1685            for tf in &["%I:%M %p", "%I:%M%p", "%I %p", "%H:%M"] {
1686                if let Ok(t) = chrono::NaiveTime::parse_from_str(time_only, tf) {
1687                    // Build a NaiveDateTime for today at that time, then choose
1688                    // today or tomorrow depending on whether the time has passed.
1689                    let local_now = chrono::Local::now();
1690                    let today = local_now.date_naive();
1691                    let candidate = chrono::NaiveDateTime::new(today, t);
1692                    // Use the TimeZone trait methods; annotate types so the
1693                    // compiler can infer the concrete DateTime types.
1694                    let candidate_local: chrono::DateTime<chrono::Local> =
1695                        match chrono::Local.from_local_datetime(&candidate) {
1696                            chrono::LocalResult::Single(dt) => dt,
1697                            chrono::LocalResult::Ambiguous(earliest, _) => earliest,
1698                            chrono::LocalResult::None => {
1699                                // DST gap — fall back to UTC interpretation
1700                                let utc_dt = candidate.and_utc();
1701                                return Some(utc_dt.timestamp());
1702                            }
1703                        };
1704                    let chosen = if candidate_local <= local_now {
1705                        // Use tomorrow
1706                        let tomorrow = today.succ_opt().unwrap_or(today);
1707                        let ndt = chrono::NaiveDateTime::new(tomorrow, t);
1708                        match chrono::Local.from_local_datetime(&ndt) {
1709                            chrono::LocalResult::Single(dt) => dt.with_timezone(&chrono::Utc),
1710                            chrono::LocalResult::Ambiguous(earliest, _) => {
1711                                earliest.with_timezone(&chrono::Utc)
1712                            }
1713                            chrono::LocalResult::None => ndt.and_utc(),
1714                        }
1715                    } else {
1716                        candidate_local.with_timezone(&chrono::Utc)
1717                    };
1718                    tracing::info!(
1719                        retry_at = %chosen,
1720                        raw = date_str,
1721                        marker = retry_marker,
1722                        "parsed rate limit retry-at time-only; interpreted as next occurrence"
1723                    );
1724                    return Some(chosen.timestamp());
1725                }
1726            }
1727        }
1728    }
1729
1730    // Without a specific retry/reset date, return None and let the caller use
1731    // its default cooldown (exponential backoff). A blanket long fallback here
1732    // would cause false billing-cycle cooldowns on agents without those limits.
1733    None
1734}
1735
1736/// Parse a relative usage window (e.g. "5-hour usage limit") from an error
1737/// message.
1738///
1739/// Returns the window length in seconds. Only matches in usage/quota/limit
1740/// contexts ("usage limit", "quota", "window", "limit") to avoid interpreting
1741/// unrelated numbers as cooldown durations.
1742///
1743/// `pub(crate)` so `engine::runner::fallback::handle_error()` can apply the
1744/// same window-detection the `needs_review` call site
1745/// (`record_rate_limit()`, below) already uses — see issue #3580.
1746pub(crate) fn parse_relative_usage_window(error_message: &str) -> Option<u64> {
1747    static WINDOW_RE: LazyLock<Regex> = LazyLock::new(|| {
1748        Regex::new(
1749            r"(?i)(?:^|\s|[\(\)\.,;:!?])(?:(\d+)\s*-?\s*(hour|hours|hr|hrs|minute|minutes|min|mins|day|days)|(hourly|daily|weekly|monthly))(?:$|\s|[\(\)\.,;:!?])"
1750        )
1751        .expect("usage window regex is valid")
1752    });
1753
1754    if error_message.is_empty() {
1755        return None;
1756    }
1757
1758    let lower = error_message.to_lowercase();
1759    let has_context = lower.contains("usage limit")
1760        || lower.contains("quota")
1761        || lower.contains("window")
1762        || lower.contains("limit");
1763    if !has_context {
1764        return None;
1765    }
1766
1767    let mut max_secs: Option<u64> = None;
1768    for caps in WINDOW_RE.captures_iter(&lower) {
1769        let secs = if let Some(num) = caps.get(1) {
1770            let n: u64 = num.as_str().parse().ok()?;
1771            let unit = caps.get(2)?.as_str().to_lowercase();
1772            let multiplier = match unit.as_str() {
1773                "hour" | "hours" | "hr" | "hrs" => 3600,
1774                "minute" | "minutes" | "min" | "mins" => 60,
1775                "day" | "days" => 86400,
1776                _ => return None,
1777            };
1778            n.checked_mul(multiplier)?
1779        } else if let Some(word) = caps.get(3) {
1780            match word.as_str().to_lowercase().as_str() {
1781                "hourly" => 3600,
1782                "daily" => 86400,
1783                "weekly" => 604_800,
1784                "monthly" => 2_592_000,
1785                _ => return None,
1786            }
1787        } else {
1788            continue;
1789        };
1790        max_secs = Some(max_secs.map_or(secs, |m| m.max(secs)));
1791    }
1792
1793    max_secs
1794}
1795
1796#[cfg(test)]
1797pub(crate) async fn reset_global_state() {
1798    cooldowns().lock().unwrap().clear();
1799    *cooldown_store().lock().await = None;
1800    github_5xx_timestamps().lock().unwrap().clear();
1801    *github_5xx_circuit_open().lock().unwrap() = None;
1802    degraded_agents().lock().unwrap().clear();
1803    opencode::reset_model_caches_for_test();
1804}
1805
1806#[cfg(test)]
1807mod tests {
1808    use std::time::Duration;
1809
1810    use super::*;
1811    use chrono::Datelike;
1812    use chrono::TimeZone;
1813    use serial_test::serial;
1814    use tokio::time;
1815
1816    async fn test_store() -> Arc<TaskStore> {
1817        Arc::new(TaskStore::open_memory().await.expect("in-memory store"))
1818    }
1819
1820    fn sqlite_datetime_offset(secs: i64) -> String {
1821        (chrono::Utc::now() + chrono::Duration::seconds(secs))
1822            .format("%Y-%m-%d %H:%M:%S")
1823            .to_string()
1824    }
1825
1826    #[serial(cooldown_state)]
1827    #[tokio::test]
1828    async fn init_loads_unexpired_cooldowns_from_kv() {
1829        super::reset_global_state().await;
1830        let store = test_store().await;
1831        let cooldown_until = chrono::Utc::now().timestamp() + 3600; // 1 hour from now
1832
1833        store
1834            .kv_set(
1835                &format!("{KV_PREFIX}kimi:k2p5"),
1836                &cooldown_until.to_string(),
1837            )
1838            .await
1839            .unwrap();
1840
1841        // Also add an already-expired entry
1842        let expired = chrono::Utc::now().timestamp() - 1;
1843        store
1844            .kv_set(
1845                &format!("{KV_PREFIX}opencode:old-free"),
1846                &expired.to_string(),
1847            )
1848            .await
1849            .unwrap();
1850
1851        {
1852            let mut map = cooldown_lock();
1853            map.clear();
1854        }
1855
1856        init_cooldown_store(store).await;
1857
1858        assert!(
1859            is_model_in_cooldown("kimi", "k2p5"),
1860            "unexpired cooldown should be loaded"
1861        );
1862        assert!(
1863            !is_model_in_cooldown("opencode", "old-free"),
1864            "expired cooldown should not be loaded"
1865        );
1866    }
1867
1868    #[serial(cooldown_state)]
1869    #[tokio::test]
1870    async fn credit_exhaustion_inside_active_cooldown_keeps_counter_at_one() {
1871        super::reset_global_state().await;
1872        let store = test_store().await;
1873        *cooldown_store().lock().await = Some(Arc::clone(&store));
1874
1875        let msg =
1876            "API error (status 402 Payment Required): usage balance exhausted, out of credits";
1877        assert!(record_credit_exhaustion_from_message("credit_dup_agent", Some("m"), msg).await);
1878        assert!(record_credit_exhaustion_from_message("credit_dup_agent", Some("m"), msg).await);
1879
1880        assert!(is_agent_in_cooldown("credit_dup_agent"));
1881        let count = store
1882            .kv_get(&format!("{CREDIT_FAILURE_COUNT_PREFIX}credit_dup_agent"))
1883            .await
1884            .unwrap();
1885        assert_eq!(count.as_deref(), Some("1"));
1886    }
1887
1888    #[serial(cooldown_state)]
1889    #[tokio::test]
1890    async fn non_credit_message_is_not_recorded_as_credit_exhaustion() {
1891        super::reset_global_state().await;
1892        assert!(!record_credit_exhaustion_from_message("x_agent", Some("m"), "boom").await);
1893        assert!(!is_agent_in_cooldown("x_agent"));
1894    }
1895
1896    #[serial(cooldown_state)]
1897    #[tokio::test]
1898    async fn record_model_failure_writes_to_kv() {
1899        super::reset_global_state().await;
1900        let store = test_store().await;
1901
1902        *cooldown_store().lock().await = Some(Arc::clone(&store));
1903
1904        record_model_failure("testagent_persist", "testmodel_persist").await;
1905
1906        // Give the async KV write a moment to settle before querying.
1907        time::sleep(Duration::from_millis(50)).await;
1908
1909        let kv_key = format!("{KV_PREFIX}testagent_persist:testmodel_persist");
1910        let stored = store.kv_get(&kv_key).await.unwrap();
1911        assert!(stored.is_some(), "model failure should be persisted to KV");
1912        let (ts, reason) = parse_cooldown_value(&stored.unwrap()).expect("cooldown value");
1913        let now = chrono::Utc::now().timestamp();
1914        // Should be in the future (cooldown_until, not failed_at)
1915        assert!(ts > now, "persisted timestamp should be in the future");
1916        assert_eq!(reason, "model_error");
1917    }
1918
1919    #[serial(cooldown_state)]
1920    #[tokio::test]
1921    async fn cooldown_reason_survives_restart_via_kv() {
1922        super::reset_global_state().await;
1923        let store = test_store().await;
1924        let agent = "test_reason_restart";
1925        let future = chrono::Utc::now().timestamp() + 600;
1926
1927        init_cooldown_store(Arc::clone(&store)).await;
1928        set_cooldown_async(agent, future, "billing_cycle_exhausted").await;
1929
1930        // Simulate a service restart: drop the in-memory map, reload from KV.
1931        {
1932            let mut map = cooldown_lock();
1933            map.clear();
1934        }
1935        init_cooldown_store(store).await;
1936
1937        let entries = list_all_cooldowns();
1938        let entry = entries.iter().find(|(key, ..)| key == agent);
1939        assert_eq!(
1940            entry.map(|(_, _, reason)| reason.as_str()),
1941            Some("billing_cycle_exhausted"),
1942            "cooldown reason must survive a restart, not collapse to 'persisted'"
1943        );
1944    }
1945
1946    #[serial(cooldown_state)]
1947    #[tokio::test]
1948    async fn record_persistent_model_failure_escalates_and_increments_failure_count() {
1949        super::reset_global_state().await;
1950        let store = test_store().await;
1951        *cooldown_store().lock().await = Some(Arc::clone(&store));
1952
1953        let agent = "testagent_persistent";
1954        let model = "testmodel_persistent";
1955        let key = format!("{agent}:{model}");
1956
1957        // First failure -> base 4h.
1958        record_persistent_model_failure(agent, model).await;
1959        let now = chrono::Utc::now().timestamp();
1960        let first_until = cooldown_until(&key).expect("first cooldown should exist");
1961        let first_remaining = first_until.saturating_sub(now);
1962        assert!(
1963            first_remaining >= PERSISTENT_MODEL_BACKOFF_BASE_SECS - 5,
1964            "first persistent model cooldown should be ~4h, got {first_remaining}s"
1965        );
1966
1967        // Second failure -> 12h (3x growth).
1968        record_persistent_model_failure(agent, model).await;
1969        let now2 = chrono::Utc::now().timestamp();
1970        let second_until = cooldown_until(&key).expect("second cooldown should exist");
1971        let second_remaining = second_until.saturating_sub(now2);
1972        assert!(
1973            second_remaining >= (PERSISTENT_MODEL_BACKOFF_BASE_SECS * 3) - 5,
1974            "second persistent model cooldown should be ~12h, got {second_remaining}s"
1975        );
1976
1977        let count_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{agent}:{model}");
1978        let stored_count = store
1979            .kv_get(&count_key)
1980            .await
1981            .expect("kv read should succeed")
1982            .expect("failure count should be written");
1983        assert_eq!(stored_count, "2");
1984    }
1985
1986    #[serial(cooldown_state)]
1987    #[tokio::test]
1988    async fn record_agent_success_does_not_reset_persistent_failure_count() {
1989        super::reset_global_state().await;
1990        let store = test_store().await;
1991        *cooldown_store().lock().await = Some(Arc::clone(&store));
1992
1993        let agent = "testagent_persistent_success";
1994        let model = "testmodel_persistent_success";
1995        let key = format!("{agent}:{model}");
1996
1997        // Two persistent failures escalate the counter to 2 (~12h cooldown).
1998        record_persistent_model_failure(agent, model).await;
1999        record_persistent_model_failure(agent, model).await;
2000
2001        let count_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{agent}:{model}");
2002        let stored_count = store
2003            .kv_get(&count_key)
2004            .await
2005            .expect("kv read should succeed")
2006            .expect("failure count should be written");
2007        assert_eq!(stored_count, "2");
2008
2009        // A successful run resets the generic failure_count but must NOT touch
2010        // the persistent counter — otherwise an intervening success would erase
2011        // escalation progress for a structurally unreliable model.
2012        record_agent_success(agent, model).await;
2013
2014        let stored_count_after_success = store
2015            .kv_get(&count_key)
2016            .await
2017            .expect("kv read should succeed")
2018            .expect("persistent failure count should still exist");
2019        assert_eq!(
2020            stored_count_after_success, "2",
2021            "persistent failure count must survive an intervening success"
2022        );
2023
2024        // The next persistent failure should escalate from 3 (~36h), not reset to base (4h).
2025        record_persistent_model_failure(agent, model).await;
2026        let now = chrono::Utc::now().timestamp();
2027        let third_until = cooldown_until(&key).expect("third cooldown should exist");
2028        let third_remaining = third_until.saturating_sub(now);
2029        assert!(
2030            third_remaining > PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2031            "third persistent model failure should escalate past the 4h base, got {third_remaining}s"
2032        );
2033    }
2034
2035    #[serial(cooldown_state)]
2036    #[tokio::test]
2037    async fn record_persistent_model_failure_stays_capped_at_7_days() {
2038        super::reset_global_state().await;
2039        let store = test_store().await;
2040        *cooldown_store().lock().await = Some(Arc::clone(&store));
2041
2042        let agent = "testagent_persistent_capped";
2043        let model = "testmodel_persistent_capped";
2044        let key = format!("{agent}:{model}");
2045
2046        for _ in 0..9 {
2047            record_persistent_model_failure(agent, model).await;
2048        }
2049
2050        let remaining = cooldown_until(&key)
2051            .expect("cooldown should exist")
2052            .saturating_sub(chrono::Utc::now().timestamp());
2053        assert!(
2054            remaining >= PERSISTENT_MODEL_BACKOFF_MAX_SECS - 5,
2055            "persistent model cooldown should stay capped at 7 days, got {remaining}s"
2056        );
2057        assert!(
2058            remaining <= PERSISTENT_MODEL_BACKOFF_MAX_SECS + 5,
2059            "persistent model cooldown should not exceed 7 days, got {remaining}s"
2060        );
2061    }
2062
2063    #[serial(cooldown_state)]
2064    #[test]
2065    fn parse_retry_at_codex_format() {
2066        let msg = "You've hit your usage limit. Upgrade to Pro, visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Mar 26th, 2026 5:55 AM.";
2067        let ts = parse_retry_at(msg);
2068        assert!(ts.is_some(), "should parse codex retry-at date");
2069
2070        // The parsed timestamp must equal "Mar 26 2026 05:55 local" in UTC.
2071        // We reconstruct what the function should produce and compare directly.
2072        let naive = chrono::NaiveDateTime::new(
2073            chrono::NaiveDate::from_ymd_opt(2026, 3, 26).unwrap(),
2074            chrono::NaiveTime::from_hms_opt(5, 55, 0).unwrap(),
2075        );
2076        let expected_utc = match chrono::Local.from_local_datetime(&naive) {
2077            chrono::LocalResult::Single(local_dt) => local_dt.with_timezone(&chrono::Utc),
2078            chrono::LocalResult::Ambiguous(earliest, _) => earliest.with_timezone(&chrono::Utc),
2079            chrono::LocalResult::None => naive.and_utc(),
2080        };
2081        assert_eq!(
2082            ts.unwrap(),
2083            expected_utc.timestamp(),
2084            "timestamp should reflect local-timezone interpretation of the vendor date"
2085        );
2086
2087        // Sanity: the UTC date must still be March 26 (true for all timezones within ±14h)
2088        let dt = chrono::DateTime::from_timestamp(ts.unwrap(), 0).unwrap();
2089        assert_eq!(dt.month(), 3);
2090        assert_eq!(dt.day(), 26);
2091    }
2092
2093    #[serial(cooldown_state)]
2094    #[test]
2095    fn parse_retry_at_hour_only_and_month_day() {
2096        let now = chrono::Utc::now().timestamp();
2097        for msg in [
2098            "You've hit your session limit · resets 8am (America/Sao_Paulo)",
2099            "You've hit your session limit · resets 6pm (America/Sao_Paulo)",
2100            "You've hit your session limit · resets 12am (America/Sao_Paulo)",
2101            "You've hit your weekly limit · resets Jun 9 at 1am (America/Sao_Paulo)",
2102        ] {
2103            let ts = parse_retry_at(msg).unwrap_or_else(|| panic!("should parse: {msg}"));
2104            assert!(ts > now, "{msg}");
2105        }
2106    }
2107
2108    #[serial(cooldown_state)]
2109    #[test]
2110    fn parse_retry_at_weekly_resets_without_year() {
2111        let t = chrono::Local::now() + chrono::Duration::days(2);
2112        let msg = format!(
2113            "You've hit your weekly limit · resets {} {} at 1am (America/Sao_Paulo)",
2114            t.format("%b"),
2115            t.day()
2116        );
2117        let ts = parse_retry_at(&msg).expect("should parse year-less reset date");
2118        let remaining = ts - chrono::Utc::now().timestamp();
2119        assert!(
2120            remaining > 3600 && remaining < 3 * 86400,
2121            "expected ~2d retry-at, got {remaining}s"
2122        );
2123    }
2124
2125    #[serial(cooldown_state)]
2126    #[test]
2127    fn parse_retry_at_no_date() {
2128        assert!(parse_retry_at("generic rate limit error").is_none());
2129        assert!(parse_retry_at("").is_none());
2130    }
2131
2132    #[serial(cooldown_state)]
2133    #[test]
2134    fn parse_retry_at_glm_reset_format_with_seconds() {
2135        let msg = "Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-04-23 18:33:48";
2136        let ts = parse_retry_at(msg).expect("should parse GLM reset-at date");
2137
2138        let naive = chrono::NaiveDateTime::new(
2139            chrono::NaiveDate::from_ymd_opt(2026, 4, 23).expect("valid date"),
2140            chrono::NaiveTime::from_hms_opt(18, 33, 48).expect("valid time"),
2141        );
2142        let expected_utc = match chrono::Local.from_local_datetime(&naive) {
2143            chrono::LocalResult::Single(local_dt) => local_dt.with_timezone(&chrono::Utc),
2144            chrono::LocalResult::Ambiguous(earliest, _) => earliest.with_timezone(&chrono::Utc),
2145            chrono::LocalResult::None => naive.and_utc(),
2146        };
2147
2148        assert_eq!(
2149            ts,
2150            expected_utc.timestamp(),
2151            "timestamp should reflect local-timezone interpretation of vendor reset date"
2152        );
2153    }
2154
2155    #[serial(cooldown_state)]
2156    #[test]
2157    fn parse_retry_at_billing_cycle_without_date_returns_none() {
2158        // Without a specific "try again at" date, parse_retry_at should return
2159        // None so callers use their default cooldown (30m agent / 1h model).
2160        let msg = "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle. Upgrade to get more.";
2161        assert!(
2162            parse_retry_at(msg).is_none(),
2163            "billing cycle without date should return None"
2164        );
2165    }
2166
2167    #[serial(cooldown_state)]
2168    #[test]
2169    fn parse_relative_usage_window_extracts_kimi_five_hour_window() {
2170        let msg = "Failed to authenticate. API Error: 403 You've reached your 5-hour usage limit. Your quota will reset when the current 5-hour window expires.";
2171        assert_eq!(
2172            parse_relative_usage_window(msg),
2173            Some(5 * 3600),
2174            "should extract the 5-hour window as seconds"
2175        );
2176    }
2177
2178    #[serial(cooldown_state)]
2179    #[test]
2180    fn parse_relative_usage_window_handles_common_durations() {
2181        assert_eq!(
2182            parse_relative_usage_window("You've reached your hourly usage limit."),
2183            Some(3600)
2184        );
2185        assert_eq!(
2186            parse_relative_usage_window("You've reached your daily usage limit."),
2187            Some(86400)
2188        );
2189        assert_eq!(
2190            parse_relative_usage_window("You've reached your weekly limit."),
2191            Some(604_800)
2192        );
2193        assert_eq!(
2194            parse_relative_usage_window("You've reached your 30-minute usage limit."),
2195            Some(1800)
2196        );
2197        assert_eq!(
2198            parse_relative_usage_window("You've reached your 2-day quota."),
2199            Some(172_800)
2200        );
2201    }
2202
2203    #[serial(cooldown_state)]
2204    #[test]
2205    fn parse_relative_usage_window_returns_none_without_usage_context() {
2206        // A bare rate-limit message with a number should not be interpreted as a
2207        // relative usage window unless usage/quota/window context is present.
2208        assert!(parse_relative_usage_window("429 Too Many Requests").is_none());
2209        assert!(parse_relative_usage_window("rate limit exceeded").is_none());
2210        assert!(parse_relative_usage_window("").is_none());
2211    }
2212
2213    #[serial(cooldown_state)]
2214    #[tokio::test]
2215    async fn record_rate_limit_uses_model_specific_window_cooldown() {
2216        super::reset_global_state().await;
2217        let store = test_store().await;
2218        *cooldown_store().lock().await = Some(Arc::clone(&store));
2219
2220        let agent = "test_record_rl_window_agent";
2221        let model = "test_record_rl_window_model";
2222        let msg = "You've reached your 5-hour usage limit. Your quota will reset when the current 5-hour window expires.";
2223
2224        record_rate_limit(agent, Some(model), msg).await;
2225
2226        assert!(
2227            is_model_in_cooldown(agent, model),
2228            "model should be cooled for the relative usage window"
2229        );
2230        assert!(
2231            !is_agent_in_cooldown(agent),
2232            "agent should not be cooled for a model-scoped quota"
2233        );
2234
2235        let key = format!("{agent}:{model}");
2236        let remaining = cooldown_until(&key)
2237            .expect("model cooldown should exist")
2238            .saturating_sub(chrono::Utc::now().timestamp());
2239        assert!(
2240            remaining >= 5 * 3600 - 5,
2241            "model cooldown should be at least ~5 hours, got {remaining}s"
2242        );
2243        assert_eq!(
2244            cooldown_reason(&key).as_deref(),
2245            Some("rate_limit_usage_window"),
2246            "relative usage-window cooldown must be labeled as a rate limit, not silence"
2247        );
2248    }
2249
2250    #[serial(cooldown_state)]
2251    #[tokio::test]
2252    async fn record_rate_limit_without_window_falls_back_to_agent_and_model() {
2253        super::reset_global_state().await;
2254        let store = test_store().await;
2255        *cooldown_store().lock().await = Some(Arc::clone(&store));
2256
2257        let agent = "test_record_rl_fallback_agent";
2258        let model = "test_record_rl_fallback_model";
2259        let msg = "429 Too Many Requests";
2260
2261        record_rate_limit(agent, Some(model), msg).await;
2262
2263        assert!(
2264            is_agent_in_cooldown(agent),
2265            "agent should be cooled when no relative window is detected"
2266        );
2267        assert!(
2268            is_model_in_cooldown(agent, model),
2269            "model should be cooled when no relative window is detected"
2270        );
2271    }
2272
2273    #[serial(cooldown_state)]
2274    #[tokio::test]
2275    async fn agent_cooldown_persists_to_kv() {
2276        super::reset_global_state().await;
2277        // Agent-level cooldowns persist to KV via set_cooldown_async
2278        let agent = "test_agent_persist_check";
2279        record_agent_failure_with_message(agent, "").await;
2280        assert!(is_agent_in_cooldown(agent));
2281    }
2282
2283    #[serial(cooldown_state)]
2284    #[tokio::test]
2285    async fn cooldown_never_shortens() {
2286        super::reset_global_state().await;
2287        let agent = "test_agent_no_shorten";
2288        set_agent_cooldown(agent, 24 * 60 * 60).await;
2289        let initial = {
2290            let map = cooldowns().lock().unwrap();
2291            map.get(agent)
2292                .expect("cooldown entry should exist")
2293                .cooldown_until
2294        };
2295
2296        // Shorter cooldown attempt should not override the existing one.
2297        record_agent_failure_with_message(agent, "rate limit").await;
2298        let after = {
2299            let map = cooldowns().lock().unwrap();
2300            map.get(agent)
2301                .expect("cooldown entry should exist")
2302                .cooldown_until
2303        };
2304        assert_eq!(
2305            initial, after,
2306            "cooldown should not be shortened by later failures"
2307        );
2308    }
2309
2310    #[serial(cooldown_state)]
2311    #[tokio::test]
2312    async fn silence_agent_cooldown_is_short_lived() {
2313        super::reset_global_state().await;
2314        let agent = "test_silence_agent_cd";
2315        assert!(!is_agent_in_cooldown(agent));
2316
2317        set_agent_cooldown(agent, SILENCE_AGENT_COOLDOWN_SECS).await;
2318        assert!(is_agent_in_cooldown(agent));
2319
2320        // Verify it's a short cooldown (120s), not the exponential backoff one
2321        let map = cooldowns().lock().unwrap();
2322        let entry = map.get(agent).expect("should have cooldown entry");
2323        let now = chrono::Utc::now().timestamp();
2324        let remaining = entry.cooldown_until - now;
2325        assert!(
2326            remaining <= i64::try_from(SILENCE_AGENT_COOLDOWN_SECS).unwrap(),
2327            "silence agent cooldown should be <= {SILENCE_AGENT_COOLDOWN_SECS}s, got {remaining}s"
2328        );
2329        assert!(remaining > 0, "cooldown should still be active");
2330    }
2331
2332    #[serial(cooldown_state)]
2333    #[tokio::test]
2334    async fn record_silence_detection_applies_extended_cooldown() {
2335        super::reset_global_state().await;
2336        let store = test_store().await;
2337        *cooldown_store().lock().await = Some(Arc::clone(&store));
2338
2339        let agent = "test_silence_count_agent";
2340        let model = "test_silence_count_model";
2341        assert!(!is_model_in_cooldown(agent, model));
2342
2343        for _ in 0..SILENCE_COUNT_THRESHOLD {
2344            let result = record_silence_detection(agent, model).await;
2345            assert!(result.is_some());
2346        }
2347
2348        assert!(is_model_in_cooldown(agent, model));
2349        let key = format!("{agent}:{model}");
2350        let remaining = {
2351            let map = cooldowns().lock().unwrap();
2352            let entry = map.get(&key).expect("cooldown entry should exist");
2353            entry.cooldown_until - chrono::Utc::now().timestamp()
2354        };
2355        assert!(
2356            remaining >= i64::try_from(SILENCE_EXTENDED_COOLDOWN_SECS).unwrap() - 5,
2357            "extended cooldown should be applied, got {remaining}s"
2358        );
2359
2360        let kv_key = format!("{SILENCE_COUNT_PREFIX}{agent}:{model}");
2361        let stored = store.kv_get(&kv_key).await.unwrap().unwrap();
2362        assert_eq!(stored, "[]");
2363    }
2364
2365    #[serial(cooldown_state)]
2366    #[test]
2367    fn detect_credit_exhaustion_out_of_credits() {
2368        let msg = "Error: API error - {\"error\":{\"message\":\"You exceeded your current quota\",\"type\":\"insufficient_quota\",\"param\":null,\"code\":\"insufficient_quota\"}}";
2369        assert_eq!(
2370            detect_credit_exhaustion(msg),
2371            Some(CreditExhaustionReason::OutOfCredits)
2372        );
2373    }
2374
2375    #[serial(cooldown_state)]
2376    #[test]
2377    fn detect_credit_exhaustion_out_of_credits_variations() {
2378        assert_eq!(
2379            detect_credit_exhaustion("out of credits"),
2380            Some(CreditExhaustionReason::OutOfCredits)
2381        );
2382        assert_eq!(
2383            detect_credit_exhaustion("outofcredits"),
2384            Some(CreditExhaustionReason::OutOfCredits)
2385        );
2386        assert_eq!(
2387            detect_credit_exhaustion("no credits remaining"),
2388            Some(CreditExhaustionReason::OutOfCredits)
2389        );
2390        assert_eq!(
2391            detect_credit_exhaustion("insufficient funds"),
2392            Some(CreditExhaustionReason::OutOfCredits)
2393        );
2394    }
2395
2396    #[serial(cooldown_state)]
2397    #[test]
2398    fn detect_credit_exhaustion_org_level_disabled() {
2399        let msg = "Error: API error - {\"error\":{\"message\":\"Organization has been disabled\",\"type\":\"org_disabled\",\"code\":\"overageDisabledReason:org_level_disabled\"}}";
2400        assert_eq!(
2401            detect_credit_exhaustion(msg),
2402            Some(CreditExhaustionReason::OrgLevelDisabled)
2403        );
2404    }
2405
2406    #[serial(cooldown_state)]
2407    #[test]
2408    fn detect_credit_exhaustion_org_level_disabled_variations() {
2409        assert_eq!(
2410            detect_credit_exhaustion("organization has been disabled"),
2411            Some(CreditExhaustionReason::OrgLevelDisabled)
2412        );
2413        assert_eq!(
2414            detect_credit_exhaustion("org disabled"),
2415            Some(CreditExhaustionReason::OrgLevelDisabled)
2416        );
2417        assert_eq!(
2418            detect_credit_exhaustion("billing quota exceeded"),
2419            Some(CreditExhaustionReason::OrgLevelDisabled)
2420        );
2421        assert_eq!(
2422            detect_credit_exhaustion("credit balance too low"),
2423            Some(CreditExhaustionReason::OrgLevelDisabled)
2424        );
2425    }
2426
2427    #[serial(cooldown_state)]
2428    #[test]
2429    fn detect_credit_exhaustion_overage_disabled_reason_parsing() {
2430        let msg = "overageDisabledReason: out_of_credits";
2431        assert_eq!(
2432            detect_credit_exhaustion(msg),
2433            Some(CreditExhaustionReason::OutOfCredits)
2434        );
2435
2436        let msg = "overageDisabledReason: org_level_disabled";
2437        assert_eq!(
2438            detect_credit_exhaustion(msg),
2439            Some(CreditExhaustionReason::OrgLevelDisabled)
2440        );
2441    }
2442
2443    #[serial(cooldown_state)]
2444    #[test]
2445    fn detect_credit_exhaustion_glm_insufficient_balance() {
2446        // GLM/ZhipuAI 429: "Insufficient balance or no resource package. Please recharge."
2447        assert_eq!(
2448            detect_credit_exhaustion(
2449                "API Error: Request rejected (429) · Insufficient balance or no resource package. Please recharge."
2450            ),
2451            Some(CreditExhaustionReason::OutOfCredits)
2452        );
2453        assert_eq!(
2454            detect_credit_exhaustion("Insufficient balance"),
2455            Some(CreditExhaustionReason::OutOfCredits)
2456        );
2457        assert_eq!(
2458            detect_credit_exhaustion("please recharge your account"),
2459            Some(CreditExhaustionReason::OutOfCredits)
2460        );
2461        assert_eq!(
2462            detect_credit_exhaustion("no resource package available"),
2463            Some(CreditExhaustionReason::OutOfCredits)
2464        );
2465    }
2466
2467    #[serial(cooldown_state)]
2468    #[test]
2469    fn detect_credit_exhaustion_none_for_regular_rate_limit() {
2470        assert!(detect_credit_exhaustion("rate limit exceeded").is_none());
2471        assert!(detect_credit_exhaustion("too many requests").is_none());
2472        assert!(detect_credit_exhaustion("429 Too Many Requests").is_none());
2473        assert!(detect_credit_exhaustion("you've hit your usage limit").is_none());
2474        assert!(
2475            detect_credit_exhaustion("You've hit your usage limit, try again in a few minutes.")
2476                .is_none()
2477        );
2478    }
2479
2480    #[serial(cooldown_state)]
2481    #[tokio::test]
2482    async fn record_credit_exhaustion_applies_agent_cooldown() {
2483        super::reset_global_state().await;
2484        let agent = "test_credit_exhaust_agent";
2485        assert!(!is_agent_in_cooldown(agent));
2486
2487        record_credit_exhaustion(agent, CreditExhaustionReason::OutOfCredits).await;
2488        assert!(is_agent_in_cooldown(agent));
2489
2490        let remaining = {
2491            let map = cooldowns().lock().unwrap();
2492            let entry = map.get(agent).expect("cooldown entry should exist");
2493            entry.cooldown_until - chrono::Utc::now().timestamp()
2494        };
2495        // First failure: base = 1h
2496        assert!(
2497            remaining >= CREDIT_BACKOFF_BASE_SECS - 5,
2498            "first out_of_credits cooldown should be ~1 hour, got {remaining}s"
2499        );
2500        assert!(
2501            remaining <= CREDIT_BACKOFF_MAX_SECS,
2502            "cooldown should not exceed cap of 8h, got {remaining}s"
2503        );
2504    }
2505
2506    #[serial(cooldown_state)]
2507    #[tokio::test]
2508    async fn record_credit_exhaustion_org_level_applies_longer_cooldown() {
2509        super::reset_global_state().await;
2510        let agent = "test_org_disabled_agent";
2511        assert!(!is_agent_in_cooldown(agent));
2512
2513        record_credit_exhaustion(agent, CreditExhaustionReason::OrgLevelDisabled).await;
2514        assert!(is_agent_in_cooldown(agent));
2515
2516        let remaining = {
2517            let map = cooldowns().lock().unwrap();
2518            let entry = map.get(agent).expect("cooldown entry should exist");
2519            entry.cooldown_until - chrono::Utc::now().timestamp()
2520        };
2521        // First failure: base = 2h (ORG_BACKOFF_BASE_SECS)
2522        assert!(
2523            remaining >= ORG_BACKOFF_BASE_SECS - 5,
2524            "first org-level disabled cooldown should be ~2 hours, got {remaining}s"
2525        );
2526        assert!(
2527            remaining <= CREDIT_BACKOFF_MAX_SECS,
2528            "cooldown should not exceed cap of 8h, got {remaining}s"
2529        );
2530    }
2531
2532    #[serial(cooldown_state)]
2533    #[tokio::test]
2534    async fn record_credit_exhaustion_billing_cycle_applies_24h() {
2535        super::reset_global_state().await;
2536        let store = test_store().await;
2537        *cooldown_store().lock().await = Some(Arc::clone(&store));
2538
2539        let agent = "test_billing_cycle_agent";
2540        assert!(!is_agent_in_cooldown(agent));
2541
2542        record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2543        assert!(is_agent_in_cooldown(agent));
2544
2545        let remaining = {
2546            let map = cooldowns().lock().unwrap();
2547            let entry = map.get(agent).expect("cooldown entry should exist");
2548            entry.cooldown_until - chrono::Utc::now().timestamp()
2549        };
2550        // First failure: base = 24h
2551        assert!(
2552            remaining >= BILLING_CYCLE_COOLDOWN_SECS - 5,
2553            "first billing cycle cooldown should be ~24 hours, got {remaining}s"
2554        );
2555        assert!(
2556            remaining <= BILLING_CYCLE_MAX_SECS,
2557            "cooldown should not exceed cap of 7 days, got {remaining}s"
2558        );
2559    }
2560
2561    #[serial(cooldown_state)]
2562    #[tokio::test]
2563    async fn record_credit_exhaustion_billing_cycle_escalates() {
2564        super::reset_global_state().await;
2565        let store = test_store().await;
2566        *cooldown_store().lock().await = Some(Arc::clone(&store));
2567
2568        let agent = "test_billing_cycle_escalation";
2569
2570        // First failure: 24h
2571        record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2572        let remaining_1 = {
2573            let map = cooldowns().lock().unwrap();
2574            let entry = map.get(agent).expect("cooldown entry should exist");
2575            entry.cooldown_until - chrono::Utc::now().timestamp()
2576        };
2577        assert!(
2578            (BILLING_CYCLE_COOLDOWN_SECS - 5..=BILLING_CYCLE_COOLDOWN_SECS + 5)
2579                .contains(&remaining_1),
2580            "first billing cycle cooldown should be ~24h, got {remaining_1}s"
2581        );
2582
2583        // Clear in-memory cooldown (but NOT failure count) to allow next set_cooldown_async
2584        {
2585            let mut map = cooldowns().lock().unwrap();
2586            map.remove(agent);
2587        }
2588
2589        // Second failure: 24h * 3 = 72h (3 days)
2590        record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2591        let remaining_2 = {
2592            let map = cooldowns().lock().unwrap();
2593            let entry = map.get(agent).expect("cooldown entry should exist");
2594            entry.cooldown_until - chrono::Utc::now().timestamp()
2595        };
2596        let expected_2 = BILLING_CYCLE_COOLDOWN_SECS * 3; // 72h
2597        assert!(
2598            remaining_2 >= expected_2 - 5,
2599            "second billing cycle cooldown should be ~72h, got {remaining_2}s"
2600        );
2601
2602        // Clear in-memory cooldown again
2603        {
2604            let mut map = cooldowns().lock().unwrap();
2605            map.remove(agent);
2606        }
2607
2608        // Third failure: 24h * 9 = 216h → capped at 7 days (168h)
2609        record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2610        let remaining_3 = {
2611            let map = cooldowns().lock().unwrap();
2612            let entry = map.get(agent).expect("cooldown entry should exist");
2613            entry.cooldown_until - chrono::Utc::now().timestamp()
2614        };
2615        assert!(
2616            remaining_3 >= BILLING_CYCLE_MAX_SECS - 5,
2617            "third billing cycle cooldown should be capped at 7 days, got {remaining_3}s"
2618        );
2619        assert!(
2620            remaining_3 <= BILLING_CYCLE_MAX_SECS + 5,
2621            "third billing cycle cooldown should not exceed 7 days, got {remaining_3}s"
2622        );
2623    }
2624
2625    #[serial(cooldown_state)]
2626    #[tokio::test]
2627    async fn record_credit_exhaustion_billing_cycle_stays_capped_after_repeats() {
2628        super::reset_global_state().await;
2629        let store = test_store().await;
2630        *cooldown_store().lock().await = Some(Arc::clone(&store));
2631
2632        let agent = "test_billing_cycle_agent_capped";
2633
2634        for i in 0..5 {
2635            if i > 0 {
2636                // Each repeat is a new exhaustion after the previous cooldown ended
2637                cooldowns().lock().unwrap().remove(agent);
2638            }
2639            record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2640        }
2641
2642        let remaining = {
2643            let map = cooldowns().lock().unwrap();
2644            let entry = map.get(agent).expect("cooldown entry should exist");
2645            entry.cooldown_until - chrono::Utc::now().timestamp()
2646        };
2647        assert!(
2648            remaining >= BILLING_CYCLE_MAX_SECS - 5,
2649            "billing cycle cooldown should stay capped at 7 days, got {remaining}s"
2650        );
2651        assert!(
2652            remaining <= BILLING_CYCLE_MAX_SECS + 5,
2653            "billing cycle cooldown should not exceed 7 days, got {remaining}s"
2654        );
2655    }
2656
2657    #[serial(cooldown_state)]
2658    #[test]
2659    fn credit_exhaustion_backoff_constants() {
2660        // OutOfCredits first failure: 1h base
2661        assert_eq!(
2662            compute_backoff(1, CREDIT_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS, true),
2663            CREDIT_BACKOFF_BASE_SECS
2664        );
2665        // OrgLevelDisabled first failure: 2h base
2666        assert_eq!(
2667            compute_backoff(1, ORG_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS, true),
2668            ORG_BACKOFF_BASE_SECS
2669        );
2670        // BillingCycleExhausted: 24h base, 7d cap
2671        assert_eq!(BILLING_CYCLE_COOLDOWN_SECS, 24 * 60 * 60);
2672        assert_eq!(BILLING_CYCLE_MAX_SECS, 7 * 24 * 60 * 60);
2673    }
2674
2675    #[serial(cooldown_state)]
2676    #[test]
2677    fn compute_backoff_long_cap_without_extended_stays_capped() {
2678        assert_eq!(
2679            compute_backoff(
2680                3,
2681                BILLING_CYCLE_COOLDOWN_SECS,
2682                BILLING_CYCLE_MAX_SECS,
2683                false
2684            ),
2685            BILLING_CYCLE_MAX_SECS,
2686            "7-day capped paths should stop at the documented max"
2687        );
2688        assert_eq!(
2689            compute_backoff(
2690                9,
2691                PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2692                PERSISTENT_MODEL_BACKOFF_MAX_SECS,
2693                false,
2694            ),
2695            PERSISTENT_MODEL_BACKOFF_MAX_SECS,
2696            "persistent model failures should not extend past 7 days"
2697        );
2698    }
2699
2700    // ---- Degraded-agent tracking tests ----
2701
2702    #[serial(cooldown_state)]
2703    #[test]
2704    fn mark_and_check_agent_degraded() {
2705        let agent = "test_degraded_mark";
2706        assert!(!is_agent_degraded(agent));
2707        mark_agent_degraded(agent);
2708        assert!(is_agent_degraded(agent));
2709        clear_agent_degraded(agent);
2710        assert!(!is_agent_degraded(agent));
2711    }
2712
2713    #[serial(cooldown_state)]
2714    #[tokio::test]
2715    async fn sync_from_kv_drops_in_memory_entry_cleared_externally() {
2716        super::reset_global_state().await;
2717        let store = test_store().await;
2718        let agent = "test_sync_external_clear";
2719        let future = chrono::Utc::now().timestamp() + 600;
2720
2721        // Register the store and put a cooldown in both memory and KV.
2722        init_cooldown_store(Arc::clone(&store)).await;
2723        set_cooldown_async(agent, future, "test").await;
2724        assert!(is_agent_in_cooldown(agent));
2725
2726        // Simulate `orch cooldown clear --all` from a separate CLI process: it
2727        // can only write KV (to "0"); the running engine's in-memory map is
2728        // untouched.
2729        store
2730            .kv_set(&format!("{KV_PREFIX}{agent}"), "0")
2731            .await
2732            .expect("kv_set tombstone");
2733        assert!(
2734            is_agent_in_cooldown(agent),
2735            "in-memory cooldown should still be present before sync"
2736        );
2737
2738        sync_from_kv(&store).await;
2739
2740        assert!(
2741            !is_agent_in_cooldown(agent),
2742            "sync_from_kv must drop in-memory entries the KV tombstoned"
2743        );
2744    }
2745
2746    #[serial(cooldown_state)]
2747    #[tokio::test]
2748    async fn sync_from_kv_closes_github_circuit_when_row_cleared() {
2749        super::reset_global_state().await;
2750        let store = test_store().await;
2751        {
2752            *github_5xx_circuit_open_lock() = Some(chrono::Utc::now().timestamp() + 600);
2753            github_5xx_timestamps_lock().push(chrono::Utc::now().timestamp());
2754        }
2755        assert!(is_github_circuit_open());
2756
2757        store
2758            .kv_set("cooldown:github:5xx", "0")
2759            .await
2760            .expect("kv_set tombstone");
2761        sync_from_kv(&store).await;
2762
2763        assert!(!is_github_circuit_open());
2764        assert!(github_5xx_timestamps_lock().is_empty());
2765    }
2766
2767    #[serial(cooldown_state)]
2768    #[tokio::test]
2769    async fn sync_from_kv_keeps_5xx_window_when_expired_row_and_breaker_closed() {
2770        super::reset_global_state().await;
2771        let store = test_store().await;
2772        let now = chrono::Utc::now().timestamp();
2773        {
2774            github_5xx_timestamps_lock().push(now);
2775            github_5xx_timestamps_lock().push(now);
2776        }
2777
2778        store
2779            .kv_set("cooldown:github:5xx", &(now - 600).to_string())
2780            .await
2781            .expect("kv_set past");
2782        sync_from_kv(&store).await;
2783
2784        assert!(!is_github_circuit_open());
2785        assert_eq!(github_5xx_timestamps_lock().len(), 2);
2786
2787        github_5xx_timestamps_lock().clear();
2788    }
2789
2790    #[serial(cooldown_state)]
2791    #[tokio::test]
2792    async fn sync_from_kv_keeps_github_circuit_open_for_future_row() {
2793        super::reset_global_state().await;
2794        let store = test_store().await;
2795        let future = chrono::Utc::now().timestamp() + 600;
2796        *github_5xx_circuit_open_lock() = Some(future);
2797
2798        store
2799            .kv_set("cooldown:github:5xx", &future.to_string())
2800            .await
2801            .expect("kv_set future");
2802        sync_from_kv(&store).await;
2803
2804        assert!(is_github_circuit_open());
2805
2806        *github_5xx_circuit_open_lock() = None;
2807    }
2808
2809    #[serial(cooldown_state)]
2810    #[tokio::test]
2811    async fn sync_from_kv_keeps_in_memory_entry_when_kv_row_missing() {
2812        // Race window: set_cooldown_async writes memory first, then KV. If a
2813        // sync runs between those two writes, the KV row is missing — we must
2814        // leave the in-memory entry alone, not drop it.
2815        super::reset_global_state().await;
2816        let store = test_store().await;
2817        let agent = "test_sync_race_window";
2818        let future = chrono::Utc::now().timestamp() + 600;
2819
2820        // Simulate the race: memory has the entry, KV does not.
2821        set_cooldown_in_memory(agent, future, "raced");
2822        assert!(is_agent_in_cooldown(agent));
2823
2824        sync_from_kv(&store).await;
2825
2826        assert!(
2827            is_agent_in_cooldown(agent),
2828            "sync_from_kv must NOT drop entries whose KV row is missing — that is the set_cooldown_async race window"
2829        );
2830    }
2831
2832    #[serial(cooldown_state)]
2833    #[tokio::test]
2834    async fn sync_from_kv_adopts_kv_timestamp() {
2835        // KV is the source of truth: if KV has a different active timestamp,
2836        // memory adopts it (covers external manual KV edits).
2837        super::reset_global_state().await;
2838        let store = test_store().await;
2839        let agent = "test_sync_adopt_kv";
2840        let original = chrono::Utc::now().timestamp() + 600;
2841        let updated = chrono::Utc::now().timestamp() + 1200;
2842
2843        init_cooldown_store(Arc::clone(&store)).await;
2844        set_cooldown_async(agent, original, "initial").await;
2845
2846        store
2847            .kv_set(&format!("{KV_PREFIX}{agent}"), &updated.to_string())
2848            .await
2849            .expect("kv_set updated");
2850
2851        sync_from_kv(&store).await;
2852
2853        let until = cooldown_until(agent).expect("cooldown still active");
2854        assert_eq!(
2855            until, updated,
2856            "sync_from_kv must adopt KV's authoritative timestamp"
2857        );
2858    }
2859
2860    #[serial(cooldown_state)]
2861    #[tokio::test]
2862    async fn refresh_degraded_agents_marks_cooled_agent() {
2863        super::reset_global_state().await;
2864        let store = test_store().await;
2865        let agent = "test_refresh_cooled";
2866
2867        // Put agent in cooldown
2868        record_agent_failure_with_message(agent, "").await;
2869        assert!(is_agent_in_cooldown(agent));
2870
2871        // model_checker always returns true (agent has models)
2872        let agents = vec![agent.to_string()];
2873        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
2874
2875        assert!(
2876            is_agent_degraded(agent),
2877            "agent in cooldown should be marked degraded"
2878        );
2879
2880        // Cleanup
2881        clear_agent_degraded(agent);
2882    }
2883
2884    #[serial(cooldown_state)]
2885    #[tokio::test]
2886    async fn refresh_degraded_agents_marks_no_models_agent() {
2887        super::reset_global_state().await;
2888        let store = test_store().await;
2889        let agent = "test_refresh_no_models";
2890
2891        // model_checker returns false (no available models)
2892        let agents = vec![agent.to_string()];
2893        refresh_degraded_agents(&store, &agents, &|_| false, 6, 3).await;
2894
2895        assert!(
2896            is_agent_degraded(agent),
2897            "agent with no available models should be marked degraded"
2898        );
2899
2900        // Cleanup
2901        clear_agent_degraded(agent);
2902    }
2903
2904    #[serial(cooldown_state)]
2905    #[tokio::test]
2906    async fn refresh_degraded_agents_marks_rate_limited_agent() {
2907        super::reset_global_state().await;
2908        let store = test_store().await;
2909        let agent = "test_refresh_rate_limited";
2910
2911        // Insert rate limit events exceeding threshold
2912        for _ in 0..4 {
2913            store.record_rate_limit(agent, "rate", None).await.unwrap();
2914        }
2915
2916        let agents = vec![agent.to_string()];
2917        // threshold=3, window=6h — 4 events should trigger degraded
2918        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
2919
2920        assert!(
2921            is_agent_degraded(agent),
2922            "agent with rate limit count >= threshold should be marked degraded"
2923        );
2924
2925        // Cleanup
2926        clear_agent_degraded(agent);
2927    }
2928
2929    #[serial(cooldown_state)]
2930    #[tokio::test]
2931    async fn refresh_degraded_agents_clears_healthy_agent() {
2932        super::reset_global_state().await;
2933        let store = test_store().await;
2934        let agent = "test_refresh_healthy";
2935
2936        // Pre-mark as degraded
2937        mark_agent_degraded(agent);
2938        assert!(is_agent_degraded(agent));
2939
2940        // No cooldown, has models, no rate limit events
2941        let agents = vec![agent.to_string()];
2942        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
2943
2944        assert!(
2945            !is_agent_degraded(agent),
2946            "healthy agent should have degraded flag cleared"
2947        );
2948    }
2949
2950    #[serial(cooldown_state)]
2951    #[tokio::test]
2952    async fn refresh_degraded_agents_ignores_expired_cooldowns() {
2953        super::reset_global_state().await;
2954        let store = test_store().await;
2955        let agent = "test_refresh_expired_cooldown";
2956
2957        // Manually insert an already-expired cooldown entry directly into the map
2958        // (simulating a stale entry that wasn't cleaned up)
2959        let expired_ts = chrono::Utc::now().timestamp() - 100; // 100 seconds ago
2960        {
2961            let mut map = cooldowns().lock().unwrap();
2962            map.insert(
2963                agent.to_string(),
2964                CooldownEntry {
2965                    cooldown_until: expired_ts,
2966                    reason: "expired_test".to_string(),
2967                },
2968            );
2969        }
2970
2971        // Verify the expired entry is in the map but should not cause degraded status
2972        assert!(
2973            !is_agent_in_cooldown(agent),
2974            "expired cooldown should not be active"
2975        );
2976
2977        // Pre-mark as degraded to verify it gets cleared
2978        mark_agent_degraded(agent);
2979        assert!(is_agent_degraded(agent));
2980
2981        // Run refresh — should clear degraded because cooldown is expired
2982        let agents = vec![agent.to_string()];
2983        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
2984
2985        assert!(
2986            !is_agent_degraded(agent),
2987            "agent with only expired cooldown should NOT be marked degraded"
2988        );
2989
2990        // Cleanup
2991        clear_agent_degraded(agent);
2992        {
2993            let mut map = cooldowns().lock().unwrap();
2994            map.remove(agent);
2995        }
2996    }
2997
2998    #[serial(cooldown_state)]
2999    #[test]
3000    fn compute_backoff_grows_exponentially() {
3001        // count=0 or 1: base
3002        assert_eq!(compute_backoff(0, 300, 14400, true), 300);
3003        assert_eq!(compute_backoff(1, 300, 14400, true), 300);
3004        // count=2: base * 3
3005        assert_eq!(compute_backoff(2, 300, 14400, true), 900);
3006        // count=3: base * 9
3007        assert_eq!(compute_backoff(3, 300, 14400, true), 2700);
3008        // count=4: base * 27 = 8100 < 14400
3009        assert_eq!(compute_backoff(4, 300, 14400, true), 8100);
3010        // count=5: base * 81 = 24300 → capped at 14400
3011        assert_eq!(compute_backoff(5, 300, 14400, true), 14400);
3012    }
3013
3014    #[serial(cooldown_state)]
3015    #[test]
3016    fn compute_backoff_extended_tier_1_applies_24h() {
3017        // With base=300, max=14400:
3018        // count=4: 8100s (not yet at max)
3019        // count=5: 14400s (first time at max, times_at_max=0, no extension)
3020        // count=6: 14400s (times_at_max=1, no extension yet, need >=2)
3021        // count=7: 86400s (times_at_max=2, extended tier 1: 6x = 24h)
3022        assert_eq!(
3023            compute_backoff(5, 300, 14400, true),
3024            14400,
3025            "first at max, not extended"
3026        );
3027        assert_eq!(
3028            compute_backoff(6, 300, 14400, true),
3029            14400,
3030            "times_at_max=1, not extended"
3031        );
3032        assert_eq!(
3033            compute_backoff(7, 300, 14400, true),
3034            86400,
3035            "times_at_max=2, extended 6x"
3036        );
3037        assert_eq!(
3038            compute_backoff(8, 300, 14400, true),
3039            86400,
3040            "times_at_max=3, still extended 6x"
3041        );
3042    }
3043
3044    #[serial(cooldown_state)]
3045    #[test]
3046    fn compute_backoff_extended_tier_2_applies_48h() {
3047        // With base=300, max=14400:
3048        // count=10: times_at_max=5 (first_max=5, count=10), extended tier 2: 12x = 48h
3049        assert_eq!(
3050            compute_backoff(9, 300, 14400, true),
3051            86400,
3052            "times_at_max=4, still tier 1"
3053        );
3054        assert_eq!(
3055            compute_backoff(10, 300, 14400, true),
3056            172_800,
3057            "times_at_max=5, extended 12x"
3058        );
3059        assert_eq!(
3060            compute_backoff(15, 300, 14400, true),
3061            172_800,
3062            "times_at_max=10, still tier 2"
3063        );
3064        assert_eq!(
3065            compute_backoff(20, 300, 14400, true),
3066            172_800,
3067            "times_at_max=15, still tier 2"
3068        );
3069    }
3070
3071    #[serial(cooldown_state)]
3072    #[test]
3073    fn compute_backoff_extended_tier_with_different_base_max() {
3074        // With base=60, max=3600:
3075        // count=3: 60*9=540 < 3600
3076        // count=4: 60*27=1620 < 3600
3077        // count=5: 60*81=4860 → capped at 3600
3078        // count=6: times_at_max=1, still 3600
3079        // count=7: times_at_max=2, extended 6x = 21600 (6h)
3080        assert_eq!(compute_backoff(5, 60, 3600, true), 3600, "first at max");
3081        assert_eq!(compute_backoff(6, 60, 3600, true), 3600, "times_at_max=1");
3082        assert_eq!(
3083            compute_backoff(7, 60, 3600, true),
3084            21600,
3085            "times_at_max=2, extended 6x"
3086        );
3087    }
3088
3089    #[serial(cooldown_state)]
3090    #[test]
3091    fn compute_backoff_extended_tier_persistent_fails_example() {
3092        // Real-world example from issue: glm:haiku with 21 failures
3093        // With base=300, max=14400:
3094        // count=21 → times_at_max=16 (first_max=5), extended tier 2: 12x = 48h
3095        assert_eq!(
3096            compute_backoff(21, 300, 14400, true),
3097            172_800,
3098            "21 failures should get 48h extended cooldown"
3099        );
3100
3101        // kimi:haiku with 36 failures
3102        assert_eq!(
3103            compute_backoff(36, 300, 14400, true),
3104            172_800,
3105            "36 failures should also get 48h extended cooldown"
3106        );
3107    }
3108
3109    #[serial(cooldown_state)]
3110    #[test]
3111    fn compute_backoff_extended_tier_zero_base() {
3112        // Zero base should still work without panicking
3113        assert_eq!(compute_backoff(1, 0, 14400, true), 0);
3114        assert_eq!(compute_backoff(5, 0, 14400, true), 0);
3115        assert_eq!(compute_backoff(10, 0, 14400, true), 0);
3116    }
3117
3118    #[serial(cooldown_state)]
3119    #[test]
3120    fn detect_credit_exhaustion_billing_cycle() {
3121        assert_eq!(
3122            detect_credit_exhaustion("billing cycle"),
3123            Some(CreditExhaustionReason::BillingCycleExhausted)
3124        );
3125        assert_eq!(
3126            detect_credit_exhaustion("quota refreshed next cycle"),
3127            Some(CreditExhaustionReason::BillingCycleExhausted)
3128        );
3129        assert_eq!(
3130            detect_credit_exhaustion(
3131                "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle."
3132            ),
3133            Some(CreditExhaustionReason::BillingCycleExhausted)
3134        );
3135        assert_eq!(
3136            detect_credit_exhaustion("monthly quota exceeded"),
3137            Some(CreditExhaustionReason::BillingCycleExhausted)
3138        );
3139        assert_eq!(
3140            detect_credit_exhaustion(
3141                "Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-04-23 18:33:48"
3142            ),
3143            Some(CreditExhaustionReason::BillingCycleExhausted)
3144        );
3145    }
3146
3147    #[serial(cooldown_state)]
3148    #[test]
3149    fn detect_credit_exhaustion_billing_cycle_minimax() {
3150        assert_eq!(
3151            detect_credit_exhaustion(
3152                "minimax rate limit: API Error: Request rejected (429) · Token Plan usage limit reached: Upgrade your Token Plan or purchase Credits for more usage."
3153            ),
3154            Some(CreditExhaustionReason::BillingCycleExhausted)
3155        );
3156        assert_eq!(
3157            detect_credit_exhaustion("Upgrade your Token Plan"),
3158            Some(CreditExhaustionReason::BillingCycleExhausted)
3159        );
3160        assert_eq!(
3161            detect_credit_exhaustion("Token Plan usage limit reached"),
3162            Some(CreditExhaustionReason::BillingCycleExhausted)
3163        );
3164        assert_eq!(
3165            detect_credit_exhaustion("purchase Credits for more usage"),
3166            Some(CreditExhaustionReason::BillingCycleExhausted)
3167        );
3168        assert_eq!(
3169            detect_credit_exhaustion(
3170                "You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro), visit https://chatgpt.com/codex/settings/usage to purchase more credits"
3171            ),
3172            Some(CreditExhaustionReason::BillingCycleExhausted)
3173        );
3174    }
3175
3176    #[serial(cooldown_state)]
3177    #[test]
3178    fn detect_credit_exhaustion_outcome_kimi_billing_cycle() {
3179        // Regression test for issue #3529: the exact Kimi billing-cycle exhaustion
3180        // message from task_runs (tasks 156734/156678) must resolve to the
3181        // "billing_cycle_exhausted" outcome string, not fall through to a generic
3182        // rate-limit/auth classification.
3183        let msg = "Failed to authenticate. API Error: 403 You've reached your usage limit for \
3184                    this billing cycle. Your quota will be refreshed soon.";
3185        assert_eq!(
3186            detect_credit_exhaustion_outcome(msg),
3187            Some("billing_cycle_exhausted")
3188        );
3189    }
3190
3191    #[serial(cooldown_state)]
3192    #[test]
3193    fn detect_credit_exhaustion_outcome_maps_all_reasons() {
3194        assert_eq!(
3195            detect_credit_exhaustion_outcome("out of credits"),
3196            Some("credit_exhausted")
3197        );
3198        assert_eq!(
3199            detect_credit_exhaustion_outcome("organization has been disabled"),
3200            Some("org_level_disabled")
3201        );
3202        assert_eq!(
3203            detect_credit_exhaustion_outcome("monthly quota exceeded"),
3204            Some("billing_cycle_exhausted")
3205        );
3206    }
3207
3208    #[test]
3209    fn detect_credit_exhaustion_outcome_returns_none_for_unrelated_errors() {
3210        assert_eq!(
3211            detect_credit_exhaustion_outcome("429 Too Many Requests"),
3212            None
3213        );
3214        assert_eq!(detect_credit_exhaustion_outcome("connection reset"), None);
3215    }
3216
3217    #[serial(cooldown_state)]
3218    #[test]
3219    fn detect_credit_exhaustion_overage_takes_priority_over_billing_cycle() {
3220        // A message containing both overage and billing cycle language should resolve
3221        // to OrgLevelDisabled/OutOfCredits (detected first) not BillingCycleExhausted.
3222        let msg = "overageDisabledReason: org_level_disabled for billing cycle";
3223        assert_eq!(
3224            detect_credit_exhaustion(msg),
3225            Some(CreditExhaustionReason::OrgLevelDisabled)
3226        );
3227    }
3228
3229    #[serial(cooldown_state)]
3230    #[tokio::test]
3231    async fn refresh_degraded_agents_below_threshold_not_degraded() {
3232        super::reset_global_state().await;
3233        let store = test_store().await;
3234        let agent = "test_refresh_below_threshold";
3235
3236        // Insert rate limit events below threshold
3237        for _ in 0..2 {
3238            store.record_rate_limit(agent, "rate", None).await.unwrap();
3239        }
3240
3241        let agents = vec![agent.to_string()];
3242        // threshold=3, only 2 events — should NOT be degraded
3243        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3244
3245        assert!(
3246            !is_agent_degraded(agent),
3247            "agent with rate limit count < threshold should not be degraded"
3248        );
3249    }
3250
3251    #[serial(cooldown_state)]
3252    #[tokio::test]
3253    async fn refresh_degraded_agents_ignores_error_limit_type() {
3254        super::reset_global_state().await;
3255        let store = test_store().await;
3256        let agent = "test_refresh_error_rows";
3257
3258        for _ in 0..3 {
3259            store.record_rate_limit(agent, "error", None).await.unwrap();
3260        }
3261
3262        let agents = vec![agent.to_string()];
3263        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3264
3265        assert!(
3266            !is_agent_degraded(agent),
3267            "three error rows in 6 hours must not mark an agent degraded"
3268        );
3269    }
3270
3271    #[serial(cooldown_state)]
3272    #[tokio::test]
3273    async fn refresh_degraded_agents_marks_three_rate_rows() {
3274        super::reset_global_state().await;
3275        let store = test_store().await;
3276        let agent = "test_refresh_three_rate_rows";
3277
3278        for _ in 0..3 {
3279            store.record_rate_limit(agent, "rate", None).await.unwrap();
3280        }
3281
3282        let agents = vec![agent.to_string()];
3283        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3284
3285        assert!(
3286            is_agent_degraded(agent),
3287            "three rate rows in 6 hours must mark an agent degraded"
3288        );
3289        assert_eq!(
3290            agent_degraded_reason(agent).as_deref(),
3291            Some("rate limit threshold exceeded")
3292        );
3293        let listed = list_degraded_agents();
3294        assert!(
3295            listed
3296                .iter()
3297                .any(|(a, r)| a == agent && r == "rate limit threshold exceeded"),
3298            "orch cooldown list must show the degraded reason, got {listed:?}"
3299        );
3300
3301        clear_agent_degraded(agent);
3302    }
3303
3304    #[serial(cooldown_state)]
3305    #[tokio::test]
3306    async fn refresh_degraded_agents_logs_reason_change() {
3307        super::reset_global_state().await;
3308        let store = test_store().await;
3309        let agent = "test_refresh_reason_change";
3310        *cooldown_store().lock().await = Some(Arc::clone(&store));
3311
3312        set_agent_cooldown(agent, 600).await;
3313        let agents = vec![agent.to_string()];
3314        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3315        assert_eq!(
3316            agent_degraded_reason(agent).as_deref(),
3317            Some("agent in cooldown")
3318        );
3319
3320        clear_cooldown(agent, &store).await;
3321        let after_clear = sqlite_datetime_offset(60);
3322        for _ in 0..3 {
3323            store
3324                .record_rate_limit_at(agent, "rate", None, &after_clear)
3325                .await
3326                .unwrap();
3327        }
3328        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3329        assert_eq!(
3330            agent_degraded_reason(agent).as_deref(),
3331            Some("rate limit threshold exceeded"),
3332            "degraded reason must update when the hold changes"
3333        );
3334
3335        clear_agent_degraded(agent);
3336    }
3337
3338    #[serial(cooldown_state)]
3339    #[tokio::test]
3340    async fn clear_un_degrades_on_next_health_check() {
3341        super::reset_global_state().await;
3342        let store = test_store().await;
3343        let agent = "test_clear_un_degrades";
3344        let before = sqlite_datetime_offset(-30);
3345        let after = sqlite_datetime_offset(60);
3346
3347        for _ in 0..3 {
3348            store
3349                .record_rate_limit_at(agent, "rate", None, &before)
3350                .await
3351                .unwrap();
3352        }
3353
3354        let agents = vec![agent.to_string()];
3355        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3356        assert!(
3357            is_agent_degraded(agent),
3358            "three rate rows must mark an agent degraded"
3359        );
3360
3361        clear_cooldown(agent, &store).await;
3362        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3363        assert!(
3364            !is_agent_degraded(agent),
3365            "clear must un-degrade on the next health check"
3366        );
3367
3368        store
3369            .record_rate_limit_at(agent, "rate", None, &after)
3370            .await
3371            .unwrap();
3372        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3373        assert!(
3374            !is_agent_degraded(agent),
3375            "one rate row after the mark must not degrade again"
3376        );
3377
3378        for _ in 0..2 {
3379            store
3380                .record_rate_limit_at(agent, "rate", None, &after)
3381                .await
3382                .unwrap();
3383        }
3384        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3385        assert!(
3386            is_agent_degraded(agent),
3387            "three rate rows after the mark must degrade again"
3388        );
3389
3390        clear_agent_degraded(agent);
3391    }
3392
3393    #[serial(cooldown_state)]
3394    #[tokio::test]
3395    async fn clear_all_un_degrades_rate_limited_agent() {
3396        super::reset_global_state().await;
3397        let store = test_store().await;
3398        let agent = "test_clear_all_un_degrades";
3399        let before = sqlite_datetime_offset(-30);
3400
3401        for _ in 0..3 {
3402            store
3403                .record_rate_limit_at(agent, "rate", None, &before)
3404                .await
3405                .unwrap();
3406        }
3407
3408        let agents = vec![agent.to_string()];
3409        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3410        assert!(is_agent_degraded(agent));
3411
3412        clear_cooldown("*", &store).await;
3413        refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3414        assert!(
3415            !is_agent_degraded(agent),
3416            "clear --all must un-degrade an agent held only by rate rows"
3417        );
3418    }
3419
3420    // ---- Exponential backoff integration tests ----
3421
3422    #[serial(cooldown_state)]
3423    #[tokio::test]
3424    async fn repeated_agent_failures_escalate_backoff() {
3425        super::reset_global_state().await;
3426        let store = test_store().await;
3427        *cooldown_store().lock().await = Some(Arc::clone(&store));
3428
3429        let agent = "test_escalation_agent";
3430
3431        // First failure: should be ~5 min (BACKOFF_BASE_SECS)
3432        record_agent_failure_with_message(agent, "").await;
3433        let remaining_1 = {
3434            let map = cooldowns().lock().unwrap();
3435            let entry = map.get(agent).expect("should have cooldown");
3436            entry.cooldown_until - chrono::Utc::now().timestamp()
3437        };
3438        assert!(
3439            (BACKOFF_BASE_SECS - 5..=BACKOFF_BASE_SECS + 5).contains(&remaining_1),
3440            "first failure should be ~5 min, got {remaining_1}s"
3441        );
3442
3443        // Clear the cooldown (but NOT failure count) to allow the next set_cooldown_async to apply
3444        {
3445            let mut map = cooldowns().lock().unwrap();
3446            map.remove(agent);
3447        }
3448
3449        // Second failure: should be ~15 min (5 * 3)
3450        record_agent_failure_with_message(agent, "").await;
3451        let remaining_2 = {
3452            let map = cooldowns().lock().unwrap();
3453            let entry = map.get(agent).expect("should have cooldown");
3454            entry.cooldown_until - chrono::Utc::now().timestamp()
3455        };
3456        assert!(
3457            remaining_2 >= BACKOFF_BASE_SECS * 3 - 5,
3458            "second failure should be ~15 min, got {remaining_2}s"
3459        );
3460    }
3461
3462    #[serial(cooldown_state)]
3463    #[tokio::test]
3464    async fn vendor_past_retry_falls_back_to_backoff() {
3465        super::reset_global_state().await;
3466        // If a vendor-provided "try again at" date is in the past, the
3467        // code must NOT early-return and must instead apply the normal
3468        // exponential backoff (increment failure count and set a future cooldown).
3469        let store = test_store().await;
3470        *cooldown_store().lock().await = Some(Arc::clone(&store));
3471
3472        let agent = "test_vendor_past_retry";
3473
3474        // Ensure no pre-existing cooldown
3475        {
3476            let mut map = cooldowns().lock().unwrap();
3477            map.remove(agent);
3478        }
3479
3480        // A clearly past date that parse_retry_at can parse
3481        let past_msg = "You've hit the limit; try again at Jan 01, 2000 01:00 AM.";
3482
3483        record_agent_failure_with_message(agent, past_msg).await;
3484
3485        // After handling a past vendor timestamp, we should have applied
3486        // exponential backoff (base = BACKOFF_BASE_SECS) and a persisted
3487        // failure count of 1.
3488        assert!(
3489            is_agent_in_cooldown(agent),
3490            "agent should be in cooldown after failure"
3491        );
3492
3493        let remaining = {
3494            let map = cooldowns().lock().unwrap();
3495            let entry = map.get(agent).expect("cooldown entry should exist");
3496            entry.cooldown_until - chrono::Utc::now().timestamp()
3497        };
3498
3499        assert!(
3500            remaining >= BACKOFF_BASE_SECS - 5,
3501            "expected backoff of ~{BACKOFF_BASE_SECS}s when vendor timestamp is past, got {remaining}s"
3502        );
3503
3504        // Verify the failure count was incremented and persisted to KV
3505        let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3506        let stored = store
3507            .kv_get(&kv_key)
3508            .await
3509            .unwrap()
3510            .expect("failure count should be stored");
3511        assert_eq!(stored, "1");
3512    }
3513
3514    fn window(status: &str, resets_at: Option<i64>) -> RateLimitWindow {
3515        RateLimitWindow {
3516            status: status.to_string(),
3517            window_type: "five_hour".to_string(),
3518            utilization: Some(1.0),
3519            resets_at,
3520        }
3521    }
3522
3523    #[serial(cooldown_state)]
3524    #[tokio::test]
3525    async fn rejected_window_cools_agent_until_reset() {
3526        super::reset_global_state().await;
3527        let store = test_store().await;
3528        *cooldown_store().lock().await = Some(Arc::clone(&store));
3529
3530        let agent = "test_window_rejected_agent";
3531        let reset = chrono::Utc::now().timestamp() + 7200;
3532        record_rate_limit_window(agent, &window("rejected", Some(reset))).await;
3533
3534        assert_eq!(cooldown_until(agent), Some(reset));
3535        let stored = list_rate_limit_windows(&store).await;
3536        assert_eq!(
3537            stored,
3538            vec![(agent.to_string(), window("rejected", Some(reset)))]
3539        );
3540    }
3541
3542    #[serial(cooldown_state)]
3543    #[tokio::test]
3544    async fn allowed_window_changes_no_cooldown() {
3545        super::reset_global_state().await;
3546        let store = test_store().await;
3547        *cooldown_store().lock().await = Some(Arc::clone(&store));
3548
3549        let agent = "test_window_allowed_agent";
3550        let reset = chrono::Utc::now().timestamp() + 7200;
3551        record_rate_limit_window(agent, &window("allowed", Some(reset))).await;
3552        record_rate_limit_window(agent, &window("allowed_warning", Some(reset))).await;
3553
3554        assert_eq!(cooldown_until(agent), None);
3555        assert_eq!(list_rate_limit_windows(&store).await.len(), 1);
3556    }
3557
3558    #[serial(cooldown_state)]
3559    #[tokio::test]
3560    async fn record_agent_success_resets_backoff() {
3561        super::reset_global_state().await;
3562        let store = test_store().await;
3563        *cooldown_store().lock().await = Some(Arc::clone(&store));
3564
3565        let agent = "test_success_reset_agent";
3566        let model = "test_model";
3567
3568        // Accumulate 3 failures to escalate the backoff
3569        for _ in 0..3 {
3570            let _ = read_and_increment_failure_count(Some(&store), agent).await;
3571        }
3572        // Verify count is 3
3573        let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3574        let count: u32 = store
3575            .kv_get(&kv_key)
3576            .await
3577            .unwrap()
3578            .unwrap()
3579            .parse()
3580            .unwrap();
3581        assert_eq!(count, 3);
3582
3583        // Success should reset to 0
3584        record_agent_success(agent, model).await;
3585
3586        let count_after: u32 = store
3587            .kv_get(&kv_key)
3588            .await
3589            .unwrap()
3590            .unwrap()
3591            .parse()
3592            .unwrap();
3593        assert_eq!(count_after, 0, "success should reset failure count to 0");
3594
3595        // Model-specific count should also be reset
3596        let model_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}:{model}");
3597        let model_count: u32 = store
3598            .kv_get(&model_kv_key)
3599            .await
3600            .unwrap()
3601            .unwrap()
3602            .parse()
3603            .unwrap();
3604        assert_eq!(
3605            model_count, 0,
3606            "success should reset model failure count to 0"
3607        );
3608    }
3609
3610    #[serial(cooldown_state)]
3611    #[tokio::test]
3612    async fn record_agent_success_without_model_skips_empty_model_key() {
3613        super::reset_global_state().await;
3614        let store = test_store().await;
3615        *cooldown_store().lock().await = Some(Arc::clone(&store));
3616
3617        let agent = "test_success_without_model";
3618
3619        record_agent_success(agent, "").await;
3620
3621        let agent_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3622        let agent_count = store
3623            .kv_get(&agent_kv_key)
3624            .await
3625            .unwrap()
3626            .expect("agent failure count should be stored");
3627        assert_eq!(agent_count, "0");
3628
3629        let empty_model_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}:");
3630        let empty_model_count = store.kv_get(&empty_model_kv_key).await.unwrap();
3631        assert!(
3632            empty_model_count.is_none(),
3633            "success without model should not create an empty model failure count key"
3634        );
3635    }
3636
3637    #[serial(cooldown_state)]
3638    #[tokio::test]
3639    async fn clear_cooldown_resets_failure_counts() {
3640        super::reset_global_state().await;
3641        let store = test_store().await;
3642        *cooldown_store().lock().await = Some(Arc::clone(&store));
3643
3644        let agent = "test_clear_fc_agent";
3645
3646        // Accumulate failures
3647        for _ in 0..5 {
3648            let _ = read_and_increment_failure_count(Some(&store), agent).await;
3649        }
3650        // Set a cooldown so clear_cooldown has something to clear
3651        set_agent_cooldown(agent, 3600).await;
3652
3653        // Verify failure count is 5
3654        let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3655        let count: u32 = store
3656            .kv_get(&kv_key)
3657            .await
3658            .unwrap()
3659            .unwrap()
3660            .parse()
3661            .unwrap();
3662        assert_eq!(count, 5);
3663
3664        // Clear the cooldown
3665        clear_cooldown(agent, &store).await;
3666
3667        // Failure count should be reset to 0
3668        let count_after: u32 = store
3669            .kv_get(&kv_key)
3670            .await
3671            .unwrap()
3672            .unwrap()
3673            .parse()
3674            .unwrap();
3675        assert_eq!(
3676            count_after, 0,
3677            "clear_cooldown should reset failure count to 0"
3678        );
3679
3680        // Cooldown should be removed
3681        assert!(
3682            !is_agent_in_cooldown(agent),
3683            "agent should not be in cooldown after clear"
3684        );
3685    }
3686
3687    #[serial(cooldown_state)]
3688    #[tokio::test]
3689    async fn clear_all_cooldowns_resets_all_failure_counts() {
3690        super::reset_global_state().await;
3691        let store = test_store().await;
3692        *cooldown_store().lock().await = Some(Arc::clone(&store));
3693
3694        let agents = ["test_clear_all_a", "test_clear_all_b"];
3695        for agent in &agents {
3696            let _ = read_and_increment_failure_count(Some(&store), agent).await;
3697            set_agent_cooldown(agent, 3600).await;
3698        }
3699
3700        clear_cooldown("*", &store).await;
3701
3702        for agent in &agents {
3703            assert!(!is_agent_in_cooldown(agent));
3704            let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3705            let count: u32 = store
3706                .kv_get(&kv_key)
3707                .await
3708                .unwrap()
3709                .unwrap()
3710                .parse()
3711                .unwrap();
3712            assert_eq!(
3713                count, 0,
3714                "clear --all should reset failure count for {agent}"
3715            );
3716        }
3717    }
3718
3719    #[serial(cooldown_state)]
3720    #[test]
3721    fn compute_backoff_saturates_on_large_counts() {
3722        // Very large count should not overflow — extended tier applies 48h
3723        assert_eq!(
3724            compute_backoff(100, BACKOFF_BASE_SECS, BACKOFF_MAX_SECS, true),
3725            BACKOFF_MAX_SECS * 12
3726        );
3727        assert_eq!(
3728            compute_backoff(u64::from(u32::MAX), 300, 14400, true),
3729            172_800
3730        );
3731        assert_eq!(compute_backoff(u64::MAX, 300, 14400, true), 172_800);
3732    }
3733
3734    #[serial(cooldown_state)]
3735    #[test]
3736    fn compute_backoff_zero_base_returns_zero() {
3737        assert_eq!(compute_backoff(1, 0, 14400, true), 0);
3738        assert_eq!(compute_backoff(5, 0, 14400, true), 0);
3739    }
3740
3741    // ---- GitHub 5xx circuit breaker tests ----
3742
3743    #[serial(cooldown_state)]
3744    #[tokio::test]
3745    async fn github_circuit_stays_closed_below_threshold() {
3746        super::reset_global_state().await;
3747        // Reset circuit breaker state
3748        {
3749            let mut open = github_5xx_circuit_open().lock().unwrap();
3750            *open = None;
3751            let mut ts = github_5xx_timestamps().lock().unwrap();
3752            ts.clear();
3753            // Also clear any generic cooldown map entry for the github 5xx
3754            // circuit so tests are isolated from other tests that may have
3755            // set the generic cooldown via set_agent_cooldown("github:5xx", ...).
3756            let mut map = cooldown_lock();
3757            map.remove("github:5xx");
3758        }
3759
3760        // Record fewer errors than threshold
3761        for _ in 0..GITHUB_5XX_THRESHOLD - 1 {
3762            record_github_5xx().await;
3763        }
3764
3765        assert!(
3766            !is_github_circuit_open(),
3767            "circuit should stay closed below threshold"
3768        );
3769        assert_eq!(github_circuit_remaining_secs(), 0);
3770    }
3771
3772    #[serial(cooldown_state)]
3773    #[tokio::test]
3774    async fn github_circuit_trips_at_threshold() {
3775        super::reset_global_state().await;
3776        // Reset circuit breaker state
3777        {
3778            let mut open = github_5xx_circuit_open_lock();
3779            *open = None;
3780            let mut ts = github_5xx_timestamps_lock();
3781            ts.clear();
3782            let mut map = cooldown_lock();
3783            map.remove("github:5xx");
3784        }
3785
3786        // Record exactly threshold errors
3787        for _ in 0..GITHUB_5XX_THRESHOLD {
3788            record_github_5xx().await;
3789        }
3790
3791        assert!(is_github_circuit_open(), "circuit should trip at threshold");
3792        assert!(
3793            github_circuit_remaining_secs() > 0,
3794            "remaining should be > 0 when open"
3795        );
3796        assert!(
3797            github_circuit_remaining_secs() <= GITHUB_5XX_COOLDOWN_SECS as u64,
3798            "remaining should not exceed cooldown duration"
3799        );
3800
3801        // Clean up
3802        {
3803            let mut open = github_5xx_circuit_open().lock().unwrap();
3804            *open = None;
3805        }
3806    }
3807
3808    #[serial(cooldown_state)]
3809    #[tokio::test]
3810    async fn github_circuit_cannot_double_trip() {
3811        super::reset_global_state().await;
3812        // Reset circuit breaker state
3813        {
3814            let mut open = github_5xx_circuit_open_lock();
3815            *open = None;
3816            let mut ts = github_5xx_timestamps_lock();
3817            ts.clear();
3818            let mut map = cooldown_lock();
3819            map.remove("github:5xx");
3820        }
3821
3822        // Trip the circuit
3823        for _ in 0..GITHUB_5XX_THRESHOLD {
3824            record_github_5xx().await;
3825        }
3826        assert!(is_github_circuit_open());
3827
3828        let first_remaining = github_circuit_remaining_secs();
3829
3830        // More 5xx errors should NOT reset the cooldown (never-shorten)
3831        for _ in 0..GITHUB_5XX_THRESHOLD {
3832            record_github_5xx().await;
3833        }
3834
3835        let second_remaining = github_circuit_remaining_secs();
3836        assert!(
3837            second_remaining <= first_remaining,
3838            "additional errors should not extend cooldown"
3839        );
3840
3841        // Clean up
3842        {
3843            let mut open = github_5xx_circuit_open().lock().unwrap();
3844            *open = None;
3845        }
3846    }
3847
3848    #[serial(cooldown_state)]
3849    #[tokio::test]
3850    async fn github_circuit_recovers_after_cooldown() {
3851        super::reset_global_state().await;
3852        // Reset circuit breaker state
3853        {
3854            let mut open = github_5xx_circuit_open_lock();
3855            *open = None;
3856            let mut ts = github_5xx_timestamps_lock();
3857            ts.clear();
3858            let mut map = cooldown_lock();
3859            map.remove("github:5xx");
3860        }
3861
3862        // Set the circuit to an expired cooldown
3863        {
3864            let mut open = github_5xx_circuit_open().lock().unwrap();
3865            *open = Some(chrono::Utc::now().timestamp() - 1);
3866        }
3867
3868        // Should be closed now (auto-recovery)
3869        assert!(
3870            !is_github_circuit_open(),
3871            "circuit should auto-recover after cooldown"
3872        );
3873    }
3874
3875    #[serial(cooldown_state)]
3876    #[tokio::test]
3877    async fn github_circuit_sliding_window_ages_out() {
3878        super::reset_global_state().await;
3879        // Reset circuit breaker state
3880        {
3881            let mut open = github_5xx_circuit_open_lock();
3882            *open = None;
3883            let mut ts = github_5xx_timestamps_lock();
3884            ts.clear();
3885            let mut map = cooldown_lock();
3886            map.remove("github:5xx");
3887        }
3888
3889        // Record errors that are outside the window
3890        {
3891            let mut ts = github_5xx_timestamps().lock().unwrap();
3892            let old = chrono::Utc::now().timestamp() - GITHUB_5XX_WINDOW_SECS - 10;
3893            for _ in 0..GITHUB_5XX_THRESHOLD {
3894                ts.push(old);
3895            }
3896        }
3897
3898        // A new error should clear old ones and not trip the circuit
3899        record_github_5xx().await;
3900
3901        assert!(
3902            !is_github_circuit_open(),
3903            "old errors outside window should not trip circuit"
3904        );
3905    }
3906}