1use chrono::TimeZone;
19use regex::Regex;
20use std::collections::HashMap;
21use std::collections::HashSet;
22use std::sync::LazyLock;
23use std::sync::MutexGuard;
24use std::sync::{Arc, Mutex, OnceLock};
25use tokio::sync;
26
27use crate::engine::router::config;
28#[cfg(test)]
29use crate::engine::runner::agents::opencode;
30use crate::store::TaskStore;
31
32pub const AGENT_COOLDOWN_SECS: i64 = 30 * 60;
38
39pub const BACKOFF_BASE_SECS: i64 = 5 * 60;
41
42pub const BACKOFF_MAX_SECS: i64 = 4 * 60 * 60;
44
45pub const PERSISTENT_MODEL_BACKOFF_BASE_SECS: i64 = 4 * 60 * 60;
47
48pub const PERSISTENT_MODEL_BACKOFF_MAX_SECS: i64 = 7 * 24 * 60 * 60;
50
51pub const TRANSIENT_MODEL_BACKOFF_BASE_SECS: i64 = 5 * 60;
53
54pub const TRANSIENT_MODEL_BACKOFF_MAX_SECS: i64 = 15 * 60;
56
57pub const CREDIT_BACKOFF_BASE_SECS: i64 = 60 * 60;
59
60pub const ORG_BACKOFF_BASE_SECS: i64 = 2 * 60 * 60;
62
63pub const CREDIT_BACKOFF_MAX_SECS: i64 = 8 * 60 * 60;
66
67pub const BILLING_CYCLE_COOLDOWN_SECS: i64 = 24 * 60 * 60;
75
76pub const BILLING_CYCLE_MAX_SECS: i64 = 7 * 24 * 60 * 60;
82
83#[derive(Debug, Clone, Copy, PartialEq, Eq)]
85pub enum CreditExhaustionReason {
86 OutOfCredits,
88 OrgLevelDisabled,
90 BillingCycleExhausted,
92}
93
94impl CreditExhaustionReason {
95 pub fn outcome_str(self) -> &'static str {
99 match self {
100 CreditExhaustionReason::OutOfCredits => "credit_exhausted",
101 CreditExhaustionReason::OrgLevelDisabled => "org_level_disabled",
102 CreditExhaustionReason::BillingCycleExhausted => "billing_cycle_exhausted",
103 }
104 }
105}
106
107pub const SILENCE_AGENT_COOLDOWN_SECS: u64 = 120;
114
115pub const SILENCE_COUNT_THRESHOLD: usize = 5;
117
118pub const SILENCE_COUNT_WINDOW_SECS: i64 = 24 * 60 * 60;
120
121pub const SILENCE_EXTENDED_COOLDOWN_SECS: u64 = 4 * 60 * 60;
123
124const KV_PREFIX: &str = "cooldown:";
126
127const SILENCE_COUNT_PREFIX: &str = "silence_count:";
129
130const FAILURE_COUNT_PREFIX: &str = "failure_count:";
132
133const CREDIT_FAILURE_COUNT_PREFIX: &str = "credit_failure_count:";
136
137const PERSISTENT_FAILURE_COUNT_PREFIX: &str = "persistent_failure_count:";
141
142const RATE_LIMITS_CLEARED_AT_PREFIX: &str = "rate_limits_cleared_at:";
144
145fn agent_from_key(key: &str) -> &str {
147 key.split_once(':').map_or(key, |(agent, _)| agent)
148}
149
150struct CooldownEntry {
151 cooldown_until: i64,
153 reason: String,
154}
155
156#[derive(Debug)]
157pub struct SilenceCountResult {
158 pub count: usize,
159 pub extended_cooldown_applied: bool,
160}
161
162fn cooldowns() -> &'static Mutex<HashMap<String, CooldownEntry>> {
171 static COOLDOWNS: OnceLock<Mutex<HashMap<String, CooldownEntry>>> = OnceLock::new();
172 COOLDOWNS.get_or_init(|| Mutex::new(HashMap::new()))
173}
174
175fn cooldown_store() -> &'static sync::Mutex<Option<Arc<TaskStore>>> {
181 static STORE: OnceLock<sync::Mutex<Option<Arc<TaskStore>>>> = OnceLock::new();
182 STORE.get_or_init(|| sync::Mutex::new(None))
183}
184
185fn parse_cooldown_value(value: &str) -> Option<(i64, String)> {
193 if let Ok(serde_json::Value::Object(map)) = serde_json::from_str(value) {
194 let until = map.get("until")?.as_i64()?;
195 let reason = map
196 .get("reason")
197 .and_then(|r| r.as_str())
198 .unwrap_or("persisted")
199 .to_string();
200 return Some((until, reason));
201 }
202 value
203 .parse::<i64>()
204 .ok()
205 .map(|until| (until, "persisted".to_string()))
206}
207
208pub async fn init_cooldown_store(store: Arc<TaskStore>) {
209 match store.kv_list_prefix(KV_PREFIX).await {
210 Ok(rows) => {
211 let now = chrono::Utc::now().timestamp();
212 let mut map = cooldown_lock();
213 let mut loaded = 0;
214 for (key, value) in rows {
215 let cooldown_key = key.trim_start_matches(KV_PREFIX);
216 if let Some((cooldown_until, reason)) = parse_cooldown_value(&value)
217 && now < cooldown_until
218 {
219 map.insert(
220 cooldown_key.to_string(),
221 CooldownEntry {
222 cooldown_until,
223 reason,
224 },
225 );
226 loaded += 1;
227 }
228 }
231 if loaded > 0 {
232 tracing::info!(loaded, "loaded persisted cooldowns from KV store");
233 }
234 }
235 Err(e) => {
236 tracing::warn!(err = %e, "failed to load cooldowns from KV store");
237 }
238 }
239
240 if let Ok(Some(raw)) = store.kv_get("cooldown:github:5xx").await
242 && let Ok(cooldown_until) = raw.parse::<i64>()
243 {
244 let now = chrono::Utc::now().timestamp();
245 if now < cooldown_until {
246 let mut open = github_5xx_circuit_open_lock();
247 *open = Some(cooldown_until);
248 let remaining = cooldown_until - now;
249 tracing::warn!(
250 remaining_secs = remaining,
251 "GitHub 5xx circuit breaker restored from KV (still open)"
252 );
253 }
254 }
255
256 *cooldown_store().lock().await = Some(store);
257}
258
259pub async fn sync_from_kv(store: &Arc<TaskStore>) {
276 let rows = match store.kv_list_prefix(KV_PREFIX).await {
277 Ok(rows) => rows,
278 Err(e) => {
279 tracing::warn!(err = %e, "cooldown sync_from_kv: KV list failed");
280 return;
281 }
282 };
283 let now = chrono::Utc::now().timestamp();
284
285 let mut explicit_kv: HashMap<String, Option<(i64, String)>> =
288 HashMap::with_capacity(rows.len());
289 for (kv_key, value) in &rows {
290 let cooldown_key = kv_key.trim_start_matches(KV_PREFIX);
291 if let Some((ts, reason)) = parse_cooldown_value(value) {
292 let normalized = if ts > now { Some((ts, reason)) } else { None };
293 explicit_kv.insert(cooldown_key.to_string(), normalized);
294 }
295 }
296
297 match explicit_kv.get("github:5xx") {
299 Some(None) => {
300 let mut open = github_5xx_circuit_open_lock();
301 if open.take().is_some() {
302 tracing::info!("GitHub 5xx circuit breaker closed by KV clear");
303 github_5xx_timestamps_lock().clear();
304 }
305 }
306 Some(Some((ts, _))) => {
307 *github_5xx_circuit_open_lock() = Some(*ts);
308 }
309 None => {}
310 }
311
312 let mut map = cooldown_lock();
313 let mut dropped = 0usize;
314 let mut updated = 0usize;
315
316 map.retain(|key, _| {
318 if matches!(explicit_kv.get(key), Some(None)) {
319 dropped += 1;
320 false
321 } else {
322 true
323 }
324 });
325
326 for (key, normalized) in explicit_kv {
328 if let Some((ts, reason)) = normalized {
329 if let Some(entry) = map.get_mut(&key) {
330 if entry.cooldown_until != ts {
331 entry.cooldown_until = ts;
332 updated += 1;
333 }
334 } else {
335 map.insert(
336 key,
337 CooldownEntry {
338 cooldown_until: ts,
339 reason,
340 },
341 );
342 updated += 1;
343 }
344 }
345 }
346
347 if dropped > 0 || updated > 0 {
348 tracing::info!(
349 dropped,
350 updated,
351 "cooldown sync_from_kv: reconciled in-memory map with KV"
352 );
353 }
354}
355
356const EXTENDED_TIER_1_THRESHOLD: u64 = 2;
360
361const EXTENDED_TIER_1_MULTIPLIER: i64 = 6;
363
364const EXTENDED_TIER_2_THRESHOLD: u64 = 5;
368
369const EXTENDED_TIER_2_MULTIPLIER: i64 = 12;
371
372pub fn compute_backoff(count: u64, base: i64, max: i64, allow_extended: bool) -> i64 {
391 if count <= 1 {
392 return base;
393 }
394 let exp = u32::try_from(count.saturating_sub(1)).unwrap_or(u32::MAX);
395 let factor = 3_i64.saturating_pow(exp);
396 let standard = base.saturating_mul(factor).min(max);
397
398 let first_max = if base <= 0 {
407 0
408 } else {
409 let mut accumulated = base;
411 let mut first_max_count = 1u64;
412 while accumulated < max && first_max_count < count {
413 accumulated = accumulated.saturating_mul(3);
414 first_max_count += 1;
415 }
416 first_max_count
417 };
418
419 let times_at_max = count.saturating_sub(first_max);
421
422 if !allow_extended {
423 return standard;
424 }
425
426 if standard == max && times_at_max >= EXTENDED_TIER_2_THRESHOLD {
427 max.saturating_mul(EXTENDED_TIER_2_MULTIPLIER)
428 } else if standard == max && times_at_max >= EXTENDED_TIER_1_THRESHOLD {
429 max.saturating_mul(EXTENDED_TIER_1_MULTIPLIER)
430 } else {
431 standard
432 }
433}
434
435async fn read_and_increment_failure_count(store_opt: Option<&Arc<TaskStore>>, key: &str) -> u64 {
444 read_and_increment_failure_count_with_prefix(store_opt, FAILURE_COUNT_PREFIX, key).await
445}
446
447async fn read_and_increment_failure_count_with_prefix(
452 store_opt: Option<&Arc<TaskStore>>,
453 prefix: &str,
454 key: &str,
455) -> u64 {
456 let kv_key = format!("{prefix}{key}");
457 if let Some(store) = store_opt {
458 match store.kv_increment(&kv_key).await {
459 Ok(n) => n,
460 Err(e) => {
461 tracing::warn!(key = %kv_key, err = %e, "failed to increment failure count — treating as high count for safe backoff");
462 u64::MAX
463 }
464 }
465 } else {
466 1
467 }
468}
469
470struct ModelBackoff {
472 prefix: &'static str,
473 base: i64,
474 max: i64,
475 allow_extended: bool,
476 reason: &'static str,
477}
478
479async fn record_model_backoff(agent_name: &str, model: &str, schedule: &ModelBackoff) {
481 let key = format!("{agent_name}:{model}");
482 let store_opt = cooldown_store().lock().await.clone();
483 let count =
484 read_and_increment_failure_count_with_prefix(store_opt.as_ref(), schedule.prefix, &key)
485 .await;
486 let cooldown_secs =
487 compute_backoff(count, schedule.base, schedule.max, schedule.allow_extended);
488 let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
489 set_cooldown_async(&key, cooldown_until, schedule.reason).await;
490}
491
492pub async fn record_agent_success(agent_name: &str, model: &str) {
498 let store_opt = cooldown_store().lock().await.clone();
499 if let Some(store) = store_opt {
500 let agent_key = format!("{FAILURE_COUNT_PREFIX}{agent_name}");
501 let credit_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{agent_name}");
502 if let Err(e) = store.kv_set(&agent_key, "0").await {
503 tracing::warn!(key = agent_key, err = %e, "failed to reset failure count");
504 }
505 if !model.is_empty() {
506 let model_key = format!("{FAILURE_COUNT_PREFIX}{agent_name}:{model}");
507 if let Err(e) = store.kv_set(&model_key, "0").await {
508 tracing::warn!(key = model_key, err = %e, "failed to reset failure count");
509 }
510 }
511 if let Err(e) = store.kv_set(&credit_key, "0").await {
512 tracing::warn!(key = credit_key, err = %e, "failed to reset failure count");
513 }
514 }
515}
516
517pub async fn record_agent_failure_with_message(agent_name: &str, error_message: &str) {
523 if let Some(cooldown_until) = parse_retry_at(error_message) {
528 let now = chrono::Utc::now().timestamp();
529 if cooldown_until > now {
530 set_cooldown_async(agent_name, cooldown_until, "agent_error").await;
531 return;
532 }
533 tracing::debug!(
534 agent = agent_name,
535 cooldown_until,
536 "vendor retry_at is in the past — using backoff instead"
537 );
538 }
539
540 let store_opt = cooldown_store().lock().await.clone();
541 let count = read_and_increment_failure_count(store_opt.as_ref(), agent_name).await;
542 let base = config::get_agent_backoff_base(agent_name);
543 let cooldown_until =
544 chrono::Utc::now().timestamp() + compute_backoff(count, base, BACKOFF_MAX_SECS, true);
545 set_cooldown_async(agent_name, cooldown_until, "agent_error").await;
546}
547
548pub fn detect_credit_exhaustion_outcome(error_message: &str) -> Option<&'static str> {
557 detect_credit_exhaustion(error_message).map(CreditExhaustionReason::outcome_str)
558}
559
560pub fn detect_credit_exhaustion(error_message: &str) -> Option<CreditExhaustionReason> {
565 let lower = error_message.to_lowercase();
566
567 let overage_patterns = [
568 "overagedisabledreason",
569 "overage_disabled_reason",
570 "overage disabled reason",
571 ];
572
573 if overage_patterns.iter().any(|p| lower.contains(p)) {
574 if lower.contains("out_of_credits") || lower.contains("outofcredits") {
575 return Some(CreditExhaustionReason::OutOfCredits);
576 }
577 if lower.contains("org_level_disabled") || lower.contains("orgdisabled") {
578 return Some(CreditExhaustionReason::OrgLevelDisabled);
579 }
580 }
581
582 let billing_patterns = [
583 "credit balance too low",
584 "credit balance insufficient",
585 "insufficient credit balance",
586 "billing quota exceeded",
587 "billing limit exceeded",
588 "organization has been disabled",
589 "org disabled",
590 ];
591
592 if billing_patterns.iter().any(|p| lower.contains(p)) {
593 return Some(CreditExhaustionReason::OrgLevelDisabled);
594 }
595
596 let billing_cycle_patterns = [
599 "billing cycle",
600 "quota refreshed next cycle",
601 "monthly quota",
602 "quota will be refreshed",
603 "next billing cycle",
604 "refreshed in the next cycle",
605 "weekly/monthly limit exhausted",
606 "limit will reset at",
607 "upgrade to pro", "upgrade your token plan", "token plan usage limit", "purchase credits for more", ];
612
613 if billing_cycle_patterns.iter().any(|p| lower.contains(p)) {
614 return Some(CreditExhaustionReason::BillingCycleExhausted);
615 }
616
617 if lower.contains("out of credits")
618 || lower.contains("outofcredits")
619 || lower.contains("insufficient funds")
620 || lower.contains("no credits remaining")
621 || lower.contains("insufficient_quota")
622 || lower.contains("quota exceeded")
623 || lower.contains("insufficient balance")
624 || lower.contains("please recharge")
625 || lower.contains("no resource package")
626 || lower.contains("balance exhausted")
627 || lower.contains("402 payment required")
628 {
629 return Some(CreditExhaustionReason::OutOfCredits);
630 }
631
632 None
633}
634
635pub async fn record_credit_exhaustion(agent_name: &str, reason: CreditExhaustionReason) {
648 let reason_str = match reason {
649 CreditExhaustionReason::OutOfCredits => "credit_exhaustion_out_of_credits",
650 CreditExhaustionReason::OrgLevelDisabled => "credit_exhaustion_org_level_disabled",
651 CreditExhaustionReason::BillingCycleExhausted => "billing_cycle_exhausted",
652 };
653
654 if is_in_cooldown_with_reason(agent_name, reason_str) {
656 tracing::debug!(
657 agent = agent_name,
658 reason = reason_str,
659 "credit exhaustion already cooled, not escalating"
660 );
661 return;
662 }
663
664 let store_opt = cooldown_store().lock().await.clone();
665 let count = read_and_increment_failure_count_with_prefix(
670 store_opt.as_ref(),
671 CREDIT_FAILURE_COUNT_PREFIX,
672 agent_name,
673 )
674 .await;
675
676 let (base, max) = match reason {
677 CreditExhaustionReason::OutOfCredits => (CREDIT_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS),
678 CreditExhaustionReason::OrgLevelDisabled => {
679 (ORG_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS)
680 }
681 CreditExhaustionReason::BillingCycleExhausted => {
682 (BILLING_CYCLE_COOLDOWN_SECS, BILLING_CYCLE_MAX_SECS)
683 }
684 };
685 let cooldown_secs = compute_backoff(
686 count,
687 base,
688 max,
689 !matches!(reason, CreditExhaustionReason::BillingCycleExhausted),
690 );
691 let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
692 set_cooldown_async(agent_name, cooldown_until, reason_str).await;
693 tracing::warn!(
694 agent = agent_name,
695 reason = reason_str,
696 attempt = count,
697 cooldown_secs,
698 "credit exhaustion detected: applying exponential agent-wide cooldown"
699 );
700}
701
702pub async fn record_credit_exhaustion_from_message(
704 agent_name: &str,
705 model: Option<&str>,
706 message: &str,
707) -> bool {
708 let Some(reason) = detect_credit_exhaustion(message) else {
709 return false;
710 };
711 match (reason, model) {
712 (CreditExhaustionReason::BillingCycleExhausted, Some(model)) => {
713 record_persistent_model_failure(agent_name, model).await;
714 }
715 _ => record_credit_exhaustion(agent_name, reason).await,
716 }
717 true
718}
719
720pub async fn record_model_failure(agent_name: &str, model: &str) {
724 let base = config::get_agent_backoff_base(agent_name);
725 let schedule = ModelBackoff {
726 prefix: FAILURE_COUNT_PREFIX,
727 base,
728 max: BACKOFF_MAX_SECS,
729 allow_extended: true,
730 reason: "model_error",
731 };
732 record_model_backoff(agent_name, model, &schedule).await;
733}
734
735pub async fn record_rate_limit(agent_name: &str, model: Option<&str>, error_message: &str) {
743 if let Some(window_secs) = parse_relative_usage_window(error_message)
744 && let Some(m) = model
745 {
746 let cooldown_until = chrono::Utc::now()
747 .timestamp()
748 .saturating_add(i64::try_from(window_secs).unwrap_or(i64::MAX));
749 set_model_cooldown(agent_name, m, window_secs, "rate_limit_usage_window").await;
750 tracing::info!(
751 agent = agent_name,
752 model = m,
753 window_secs,
754 cooldown_until,
755 "relative usage window detected: applying model-specific cooldown"
756 );
757 return;
758 }
759
760 record_agent_failure_with_message(agent_name, error_message).await;
761 if let Some(m) = model {
762 record_model_failure(agent_name, m).await;
763 }
764}
765
766pub async fn record_persistent_model_failure(agent_name: &str, model: &str) {
778 let schedule = ModelBackoff {
779 prefix: PERSISTENT_FAILURE_COUNT_PREFIX,
780 base: PERSISTENT_MODEL_BACKOFF_BASE_SECS,
781 max: PERSISTENT_MODEL_BACKOFF_MAX_SECS,
782 allow_extended: false,
783 reason: "persistent_model_error",
784 };
785 record_model_backoff(agent_name, model, &schedule).await;
786}
787
788pub async fn record_model_unavailable(agent_name: &str, model: &str, message: &str) {
793 if is_model_permanently_gone(message) {
794 record_persistent_model_failure(agent_name, model).await;
795 } else {
796 record_model_failure(agent_name, model).await;
797 }
798}
799
800fn is_model_permanently_gone(message: &str) -> bool {
801 let lower = message.to_lowercase();
802 lower.contains("not found")
803 || lower.contains("not supported")
804 || lower.contains("not available in your country")
805 || lower.contains("not available in your region")
806}
807
808pub async fn record_transient_model_failure(agent_name: &str, model: &str) {
814 let schedule = ModelBackoff {
815 prefix: FAILURE_COUNT_PREFIX,
816 base: TRANSIENT_MODEL_BACKOFF_BASE_SECS,
817 max: TRANSIENT_MODEL_BACKOFF_MAX_SECS,
818 allow_extended: false,
819 reason: "transient_model_error",
820 };
821 record_model_backoff(agent_name, model, &schedule).await;
822}
823
824pub async fn set_model_cooldown(agent_name: &str, model: &str, duration_secs: u64, reason: &str) {
831 let key = format!("{agent_name}:{model}");
832 let cooldown_until = chrono::Utc::now()
833 .timestamp()
834 .saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX));
835 set_cooldown_async(&key, cooldown_until, reason).await;
836}
837
838pub async fn set_model_cooldown_until(agent_name: &str, model: &str, until: i64) {
840 let key = format!("{agent_name}:{model}");
841 set_cooldown_async(&key, until, "vendor_reset_at").await;
842}
843
844pub async fn set_agent_cooldown(agent_name: &str, duration_secs: u64) {
851 let cooldown_until = chrono::Utc::now()
852 .timestamp()
853 .saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX));
854 set_cooldown_async(agent_name, cooldown_until, "silence_agent_cooldown").await;
855}
856
857pub const RATE_WINDOW_KV_PREFIX: &str = "rate_window:";
859
860#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
862pub struct RateLimitWindow {
863 pub status: String,
864 pub window_type: String,
865 pub utilization: Option<f64>,
866 pub resets_at: Option<i64>,
867}
868
869pub async fn record_rate_limit_window(agent_name: &str, window: &RateLimitWindow) {
871 let store_opt = cooldown_store().lock().await.clone();
872 if let Some(store) = store_opt
873 && let Ok(value) = serde_json::to_string(window)
874 && let Err(e) = store
875 .kv_set(
876 &format!("{RATE_WINDOW_KV_PREFIX}{agent_name}:{}", window.window_type),
877 &value,
878 )
879 .await
880 {
881 tracing::warn!(agent = agent_name, err = %e, "failed to persist rate limit window");
882 }
883
884 let now = chrono::Utc::now().timestamp();
885 if window.status == "rejected"
886 && let Some(reset) = window.resets_at
887 && reset > now
888 {
889 let until = reset.min(now + PERSISTENT_MODEL_BACKOFF_MAX_SECS);
890 set_cooldown_async(agent_name, until, "rate_limit_window").await;
891 tracing::info!(
892 agent = agent_name,
893 window = %window.window_type,
894 cooldown_until = until,
895 "rate limit window rejected: cooling agent until vendor reset"
896 );
897 }
898}
899
900pub async fn list_rate_limit_windows(store: &Arc<TaskStore>) -> Vec<(String, RateLimitWindow)> {
902 let rows = store
903 .kv_list_prefix(RATE_WINDOW_KV_PREFIX)
904 .await
905 .unwrap_or_default();
906 let mut windows: Vec<(String, RateLimitWindow)> = rows
907 .into_iter()
908 .filter_map(|(key, value)| {
909 let (agent, _) = key.strip_prefix(RATE_WINDOW_KV_PREFIX)?.rsplit_once(':')?;
910 let window = serde_json::from_str(&value).ok()?;
911 Some((agent.to_string(), window))
912 })
913 .collect();
914 windows.sort_by(|a, b| (&a.0, &a.1.window_type).cmp(&(&b.0, &b.1.window_type)));
915 windows
916}
917
918pub async fn record_silence_detection(agent_name: &str, model: &str) -> Option<SilenceCountResult> {
921 let store_opt = cooldown_store().lock().await.clone();
922 let Some(store) = store_opt else {
923 tracing::debug!(
924 agent = agent_name,
925 model,
926 "skipping silence count record (no KV store)"
927 );
928 return None;
929 };
930
931 let key = format!("{SILENCE_COUNT_PREFIX}{agent_name}:{model}");
932 let now = chrono::Utc::now().timestamp();
933 let window_start = now - SILENCE_COUNT_WINDOW_SECS;
934
935 let mut timestamps = match store.kv_get(&key).await {
936 Ok(Some(raw)) => match serde_json::from_str::<Vec<i64>>(&raw) {
937 Ok(ts) => ts,
938 Err(e) => {
939 tracing::warn!(
940 kv_key = %key,
941 err = %e,
942 "failed to parse silence timestamps from KV — resetting window"
943 );
944 Vec::new()
945 }
946 },
947 Ok(None) => Vec::new(),
948 Err(err) => {
949 tracing::warn!(
950 kv_key = %key,
951 err = %err,
952 "failed to load silence count from KV"
953 );
954 Vec::new()
955 }
956 };
957
958 timestamps.retain(|ts| *ts >= window_start);
959 timestamps.push(now);
960
961 let count = timestamps.len();
962 let mut extended_cooldown_applied = false;
963 if count >= SILENCE_COUNT_THRESHOLD {
964 set_model_cooldown(
965 agent_name,
966 model,
967 SILENCE_EXTENDED_COOLDOWN_SECS,
968 "silence_detected",
969 )
970 .await;
971 extended_cooldown_applied = true;
972 timestamps.clear();
973 }
974
975 let value = serde_json::to_string(×tamps).unwrap_or_else(|_| "[]".to_string());
976 if let Err(err) = store.kv_set(&key, &value).await {
977 tracing::warn!(kv_key = key, err = %err, "failed to persist silence count");
978 }
979
980 Some(SilenceCountResult {
981 count,
982 extended_cooldown_applied,
983 })
984}
985
986pub fn is_model_in_cooldown(agent_name: &str, model: &str) -> bool {
988 let key = format!("{agent_name}:{model}");
989 is_in_cooldown(&key)
990}
991
992pub fn is_agent_in_cooldown(agent_name: &str) -> bool {
994 is_in_cooldown(agent_name)
995}
996
997pub fn cooldown_until(key: &str) -> Option<i64> {
999 let map = cooldown_lock();
1000 map.get(key).and_then(|entry| {
1001 let now = chrono::Utc::now().timestamp();
1002 if now < entry.cooldown_until {
1003 Some(entry.cooldown_until)
1004 } else {
1005 None
1006 }
1007 })
1008}
1009
1010pub fn cooldown_reason(key: &str) -> Option<String> {
1012 let map = cooldown_lock();
1013 map.get(key).and_then(|entry| {
1014 let now = chrono::Utc::now().timestamp();
1015 if now < entry.cooldown_until {
1016 Some(entry.reason.clone())
1017 } else {
1018 None
1019 }
1020 })
1021}
1022
1023pub async fn list_persisted_cooldowns(store: &TaskStore) -> Vec<(String, i64, String)> {
1026 let now = chrono::Utc::now().timestamp();
1027 let mut out: Vec<(String, i64, String)> = store
1028 .kv_list_prefix(KV_PREFIX)
1029 .await
1030 .unwrap_or_default()
1031 .into_iter()
1032 .filter_map(|(key, value)| {
1033 let (until, reason) = parse_cooldown_value(&value)?;
1034 (now < until).then(|| (key.trim_start_matches(KV_PREFIX).to_string(), until, reason))
1035 })
1036 .collect();
1037 out.sort_by(|a, b| a.0.cmp(&b.0));
1038 out
1039}
1040
1041pub fn list_all_cooldowns() -> Vec<(String, i64, String)> {
1046 let now = chrono::Utc::now().timestamp();
1047 let map = cooldown_lock();
1048 let mut result: Vec<(String, i64, String)> = map
1049 .iter()
1050 .filter(|(_, entry)| now < entry.cooldown_until)
1051 .map(|(key, entry)| (key.clone(), entry.cooldown_until, entry.reason.clone()))
1052 .collect();
1053 result.sort_by(|a, b| a.0.cmp(&b.0));
1054 result
1055}
1056
1057pub async fn mark_rate_limits_cleared(store: &Arc<TaskStore>, agents: &[String]) {
1060 let now = chrono::Utc::now().timestamp().to_string();
1061 let mut seen = HashSet::new();
1062 for key in agents {
1063 let agent = agent_from_key(key);
1064 if agent.is_empty() || !seen.insert(agent.to_string()) {
1065 continue;
1066 }
1067 let kv_key = format!("{RATE_LIMITS_CLEARED_AT_PREFIX}{agent}");
1068 if let Err(e) = store.kv_set(&kv_key, &now).await {
1069 tracing::warn!(agent, err = %e, "failed to mark rate limits cleared");
1070 }
1071 }
1072}
1073
1074pub async fn clear_cooldown(key: &str, store: &Arc<TaskStore>) {
1081 let mut agents_to_mark: Vec<String> = Vec::new();
1082 if key == "*" {
1083 let keys: Vec<String> = {
1084 let mut map = cooldown_lock();
1085 let keys: Vec<String> = map.keys().cloned().collect();
1086 map.clear();
1087 keys
1088 };
1089 for k in &keys {
1090 let kv_key = format!("{KV_PREFIX}{k}");
1091 if let Err(e) = store.kv_set(&kv_key, "0").await {
1092 tracing::warn!(key = kv_key, err = %e, "failed to clear cooldown");
1093 }
1094 let fc_key = format!("{FAILURE_COUNT_PREFIX}{k}");
1096 if let Err(e) = store.kv_set(&fc_key, "0").await {
1097 tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1098 }
1099 let credit_fc_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{k}");
1101 if let Err(e) = store.kv_set(&credit_fc_key, "0").await {
1102 tracing::warn!(key = credit_fc_key, err = %e, "failed to reset credit failure count");
1103 }
1104 let persistent_fc_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{k}");
1106 if let Err(e) = store.kv_set(&persistent_fc_key, "0").await {
1107 tracing::warn!(key = persistent_fc_key, err = %e, "failed to reset persistent failure count");
1108 }
1109 }
1110 if let Ok(fc_entries) = store.kv_list_prefix(FAILURE_COUNT_PREFIX).await {
1114 for (fc_key, _) in fc_entries {
1115 if let Err(e) = store.kv_set(&fc_key, "0").await {
1116 tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1117 }
1118 }
1119 }
1120 if let Ok(cfc_entries) = store.kv_list_prefix(CREDIT_FAILURE_COUNT_PREFIX).await {
1121 for (cfc_key, _) in cfc_entries {
1122 if let Err(e) = store.kv_set(&cfc_key, "0").await {
1123 tracing::warn!(key = cfc_key, err = %e, "failed to reset credit failure count");
1124 }
1125 }
1126 }
1127 if let Ok(pfc_entries) = store.kv_list_prefix(PERSISTENT_FAILURE_COUNT_PREFIX).await {
1128 for (pfc_key, _) in pfc_entries {
1129 if let Err(e) = store.kv_set(&pfc_key, "0").await {
1130 tracing::warn!(key = pfc_key, err = %e, "failed to reset persistent failure count");
1131 }
1132 }
1133 }
1134 tracing::info!(
1135 count = keys.len(),
1136 "cleared all cooldowns and failure counts"
1137 );
1138 agents_to_mark.extend(keys.iter().map(|k| agent_from_key(k).to_string()));
1139 match store.rate_limit_agents().await {
1140 Ok(rl_agents) => agents_to_mark.extend(rl_agents),
1141 Err(e) => tracing::warn!(err = %e, "failed to list rate_limits agents while clearing"),
1142 }
1143 match store.kv_list_prefix(RATE_LIMITS_CLEARED_AT_PREFIX).await {
1144 Ok(marks) => {
1145 for (k, _) in marks {
1146 let agent = k.trim_start_matches(RATE_LIMITS_CLEARED_AT_PREFIX);
1147 if !agent.is_empty() {
1148 agents_to_mark.push(agent.to_string());
1149 }
1150 }
1151 }
1152 Err(e) => tracing::warn!(err = %e, "failed to list rate_limits_cleared_at keys"),
1153 }
1154 } else {
1155 {
1156 let mut map = cooldown_lock();
1157 map.remove(key);
1158 }
1159 let kv_key = format!("{KV_PREFIX}{key}");
1160 if let Err(e) = store.kv_set(&kv_key, "0").await {
1161 tracing::warn!(key = kv_key, err = %e, "failed to clear cooldown");
1162 }
1163 let fc_key = format!("{FAILURE_COUNT_PREFIX}{key}");
1165 if let Err(e) = store.kv_set(&fc_key, "0").await {
1166 tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1167 }
1168 let credit_fc_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{key}");
1170 if let Err(e) = store.kv_set(&credit_fc_key, "0").await {
1171 tracing::warn!(key = credit_fc_key, err = %e, "failed to reset credit failure count");
1172 }
1173 let persistent_fc_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{key}");
1175 if let Err(e) = store.kv_set(&persistent_fc_key, "0").await {
1176 tracing::warn!(key = persistent_fc_key, err = %e, "failed to reset persistent failure count");
1177 }
1178 tracing::info!(key, "cleared cooldown and failure count");
1179 agents_to_mark.push(agent_from_key(key).to_string());
1180 }
1181 mark_rate_limits_cleared(store, &agents_to_mark).await;
1182}
1183
1184fn set_cooldown_in_memory(key: &str, cooldown_until: i64, reason: &str) -> bool {
1193 let mut map = cooldown_lock();
1194 if let Some(existing) = map.get(key) {
1195 if existing.cooldown_until >= cooldown_until {
1199 return false;
1200 }
1201 }
1202 map.insert(
1203 key.to_string(),
1204 CooldownEntry {
1205 cooldown_until,
1206 reason: reason.to_string(),
1207 },
1208 );
1209 true
1210}
1211
1212async fn set_cooldown_async(key: &str, cooldown_until: i64, reason: &str) -> bool {
1222 if !set_cooldown_in_memory(key, cooldown_until, reason) {
1223 return false;
1224 }
1225 let store_opt = cooldown_store().lock().await.clone();
1226 if let Some(store) = store_opt {
1227 let kv_key = format!("{KV_PREFIX}{key}");
1228 let value = serde_json::json!({"until": cooldown_until, "reason": reason}).to_string();
1229 if let Err(e) = store.kv_set(&kv_key, &value).await {
1230 tracing::warn!(
1231 kv_key,
1232 err = %e,
1233 "failed to persist cooldown to KV store — rolling back in-memory entry"
1234 );
1235 let mut map = cooldown_lock();
1236 if map.get(key).map(|e| e.cooldown_until) == Some(cooldown_until) {
1240 map.remove(key);
1241 }
1242 return false;
1243 }
1244 }
1245 true
1246}
1247
1248fn cooldown_lock() -> MutexGuard<'static, HashMap<String, CooldownEntry>> {
1250 cooldowns().lock().unwrap_or_else(|e| {
1251 tracing::error!("cooldown map lock poisoned — recovering with partial state; backoff and degraded-agent decisions may be incorrect");
1252 e.into_inner()
1253 })
1254}
1255
1256fn degraded_agents_lock() -> MutexGuard<'static, HashMap<String, String>> {
1258 degraded_agents().lock().unwrap_or_else(|e| {
1259 tracing::error!("degraded agents lock poisoned — recovering with partial state; health check decisions may be incorrect");
1260 e.into_inner()
1261 })
1262}
1263
1264fn github_5xx_timestamps_lock() -> MutexGuard<'static, Vec<i64>> {
1266 github_5xx_timestamps().lock().unwrap_or_else(|e| {
1267 tracing::error!("github 5xx timestamps lock poisoned — recovering with partial state; circuit breaker may operate incorrectly");
1268 e.into_inner()
1269 })
1270}
1271
1272fn github_5xx_circuit_open_lock() -> MutexGuard<'static, Option<i64>> {
1274 github_5xx_circuit_open().lock().unwrap_or_else(|e| {
1275 tracing::error!("github 5xx circuit open lock poisoned — recovering with partial state; circuit breaker may operate incorrectly");
1276 e.into_inner()
1277 })
1278}
1279
1280fn is_in_cooldown_with_reason(key: &str, reason: &str) -> bool {
1282 let map = cooldown_lock();
1283 map.get(key).is_some_and(|entry| {
1284 entry.reason == reason && chrono::Utc::now().timestamp() < entry.cooldown_until
1285 })
1286}
1287
1288fn is_in_cooldown(key: &str) -> bool {
1290 let map = cooldown_lock();
1291 if let Some(entry) = map.get(key) {
1292 let now = chrono::Utc::now().timestamp();
1293 return now < entry.cooldown_until;
1294 }
1295 false
1296}
1297
1298pub const DEFAULT_HEALTH_CHECK_WINDOW_HOURS: u32 = 6;
1304
1305pub const DEFAULT_DEGRADED_RATE_LIMIT_THRESHOLD: i64 = 3;
1309
1310fn degraded_agents() -> &'static Mutex<HashMap<String, String>> {
1313 static DEGRADED: OnceLock<Mutex<HashMap<String, String>>> = OnceLock::new();
1314 DEGRADED.get_or_init(|| Mutex::new(HashMap::new()))
1315}
1316
1317pub fn is_agent_degraded(agent: &str) -> bool {
1322 let set = degraded_agents_lock();
1323 set.contains_key(agent)
1324}
1325
1326pub fn agent_degraded_reason(agent: &str) -> Option<String> {
1328 degraded_agents_lock().get(agent).cloned()
1329}
1330
1331pub fn list_degraded_agents() -> Vec<(String, String)> {
1333 let mut entries: Vec<(String, String)> = degraded_agents_lock()
1334 .iter()
1335 .map(|(agent, reason)| (agent.clone(), reason.clone()))
1336 .collect();
1337 entries.sort_by(|a, b| a.0.cmp(&b.0));
1338 entries
1339}
1340
1341pub fn mark_agent_degraded(agent: &str) {
1343 mark_agent_degraded_with_reason(agent, "degraded");
1344}
1345
1346fn mark_agent_degraded_with_reason(agent: &str, reason: &str) {
1347 let mut set = degraded_agents_lock();
1348 set.insert(agent.to_string(), reason.to_string());
1349}
1350
1351pub fn clear_agent_degraded(agent: &str) {
1353 let mut set = degraded_agents_lock();
1354 set.remove(agent);
1355}
1356
1357pub async fn refresh_degraded_agents(
1371 store: &Arc<TaskStore>,
1372 available_agents: &[String],
1373 model_checker: &(dyn Fn(&str) -> bool + Send + Sync),
1374 window_hours: u32,
1375 threshold: i64,
1376) {
1377 clear_expired_cooldowns();
1381
1382 let start = chrono::Utc::now();
1385 let counts = match store.recent_rate_limit_counts(window_hours).await {
1386 Ok(c) => {
1387 let dur_ms = (chrono::Utc::now() - start).num_milliseconds();
1388 tracing::info!(
1389 duration_ms = dur_ms,
1390 window_hours,
1391 "recent_rate_limit_counts query completed"
1392 );
1393 c
1394 }
1395 Err(e) => {
1396 let dur_ms = (chrono::Utc::now() - start).num_milliseconds();
1397 tracing::warn!(
1398 err = %e,
1399 duration_ms = dur_ms,
1400 "failed to query recent rate limit counts for health check"
1401 );
1402 return;
1403 }
1404 };
1405
1406 for agent in available_agents {
1407 let in_cooldown = is_agent_in_cooldown(agent);
1408 let no_models = !model_checker(agent);
1409 let rate_limit_count = counts.get(agent.as_str()).copied().unwrap_or(0);
1410 let over_threshold = rate_limit_count >= threshold;
1411
1412 let degraded = in_cooldown || no_models || over_threshold;
1413
1414 if degraded {
1415 let reason = if in_cooldown {
1416 "agent in cooldown"
1417 } else if no_models {
1418 "all models cooled"
1419 } else {
1420 "rate limit threshold exceeded"
1421 };
1422 let previous = agent_degraded_reason(agent);
1423 match previous.as_deref() {
1424 None => tracing::warn!(
1425 agent,
1426 reason,
1427 rate_limit_count,
1428 window_hours,
1429 threshold,
1430 "pre-emptive health check: marking agent as degraded"
1431 ),
1432 Some(prev) if prev != reason => tracing::warn!(
1433 agent,
1434 reason,
1435 previous = prev,
1436 rate_limit_count,
1437 window_hours,
1438 threshold,
1439 "pre-emptive health check: degraded reason changed"
1440 ),
1441 Some(_) => {}
1442 }
1443 mark_agent_degraded_with_reason(agent, reason);
1444 } else {
1445 if is_agent_degraded(agent) {
1446 tracing::info!(
1447 agent,
1448 rate_limit_count,
1449 "pre-emptive health check: agent recovered, clearing degraded flag"
1450 );
1451 }
1452 clear_agent_degraded(agent);
1453 }
1454 }
1455}
1456
1457pub fn clear_expired_cooldowns() {
1459 let mut map = cooldown_lock();
1460 let now = chrono::Utc::now().timestamp();
1461 map.retain(|_key, entry| now < entry.cooldown_until);
1462}
1463
1464const GITHUB_5XX_WINDOW_SECS: i64 = 120;
1470
1471const GITHUB_5XX_THRESHOLD: usize = 5;
1473
1474const GITHUB_5XX_COOLDOWN_SECS: i64 = 180;
1476
1477fn github_5xx_timestamps() -> &'static Mutex<Vec<i64>> {
1479 static TS: OnceLock<Mutex<Vec<i64>>> = OnceLock::new();
1480 TS.get_or_init(|| Mutex::new(Vec::new()))
1481}
1482
1483fn github_5xx_circuit_open() -> &'static Mutex<Option<i64>> {
1485 static OPEN: OnceLock<Mutex<Option<i64>>> = OnceLock::new();
1486 OPEN.get_or_init(|| Mutex::new(None))
1487}
1488
1489pub async fn record_github_5xx() {
1494 let now = chrono::Utc::now().timestamp();
1495 let window_start = now - GITHUB_5XX_WINDOW_SECS;
1496
1497 let should_trip = {
1498 let mut ts = github_5xx_timestamps_lock();
1499 ts.retain(|t| *t >= window_start);
1500 ts.push(now);
1501 ts.len() >= GITHUB_5XX_THRESHOLD
1502 };
1503
1504 if should_trip {
1505 let cooldown_until = {
1506 let mut open = github_5xx_circuit_open_lock();
1507 if open.is_none() {
1508 let cd = now + GITHUB_5XX_COOLDOWN_SECS;
1509 *open = Some(cd);
1510 tracing::warn!(
1511 errors_in_window = GITHUB_5XX_THRESHOLD,
1512 window_secs = GITHUB_5XX_WINDOW_SECS,
1513 cooldown_secs = GITHUB_5XX_COOLDOWN_SECS,
1514 "GitHub 5xx circuit breaker OPEN — throttling non-critical work"
1515 );
1516 Some(cd)
1517 } else {
1518 None
1519 }
1520 };
1521 if let Some(cd) = cooldown_until {
1523 let store_opt = cooldown_store().lock().await.clone();
1524 if let Some(store) = store_opt {
1525 let _ = store.kv_set("cooldown:github:5xx", &cd.to_string()).await;
1526 }
1527 set_agent_cooldown("github:5xx", GITHUB_5XX_COOLDOWN_SECS as u64).await;
1531 }
1532 let mut ts = github_5xx_timestamps_lock();
1534 ts.clear();
1535 }
1536}
1537
1538pub fn is_github_circuit_open() -> bool {
1546 {
1549 let mut open = github_5xx_circuit_open_lock();
1550 if let Some(until) = *open {
1551 let now = chrono::Utc::now().timestamp();
1552 if now >= until {
1553 *open = None;
1554 tracing::info!("GitHub 5xx circuit breaker CLOSED — resuming normal operations");
1555 return false;
1556 }
1557 return true;
1558 }
1559 }
1560
1561 if let Some(until) = cooldown_until("github:5xx") {
1565 let mut open = github_5xx_circuit_open_lock();
1568 if open.is_none() {
1569 *open = Some(until);
1570 let now = chrono::Utc::now().timestamp();
1571 if now < until {
1572 let remaining = until - now;
1573 tracing::warn!(
1574 remaining_secs = remaining,
1575 "GitHub 5xx circuit breaker OPEN (discovered via generic cooldown) — throttling non-critical work"
1576 );
1577 }
1578 }
1579 return true;
1580 }
1581
1582 false
1583}
1584
1585pub fn github_circuit_remaining_secs() -> u64 {
1588 {
1590 let open = github_5xx_circuit_open_lock();
1591 if let Some(until) = *open {
1592 let now = chrono::Utc::now().timestamp();
1593 if now < until {
1594 return u64::try_from(until - now).unwrap_or(0);
1595 }
1596 return 0;
1597 }
1598 }
1599
1600 if let Some(until) = cooldown_until("github:5xx") {
1602 let now = chrono::Utc::now().timestamp();
1603 if now < until {
1604 return u64::try_from(until - now).unwrap_or(0);
1605 }
1606 }
1607 0
1608}
1609
1610pub(crate) fn parse_retry_at(error_message: &str) -> Option<i64> {
1614 if error_message.is_empty() {
1615 return None;
1616 }
1617
1618 let lower = error_message.to_lowercase();
1619 for retry_marker in [
1623 "try again at ",
1624 "reset at ",
1625 "resets at ",
1626 "resets ",
1627 "reset ",
1628 ] {
1629 if let Some(idx) = lower.find(retry_marker) {
1630 let date_str = &lower[idx + retry_marker.len()..];
1631 let date_str = date_str
1633 .split(['.', '\n'])
1634 .next()
1635 .unwrap_or(date_str)
1636 .trim();
1637
1638 let cleaned = date_str
1641 .replace("st,", ",")
1642 .replace("nd,", ",")
1643 .replace("rd,", ",")
1644 .replace("th,", ",");
1645
1646 for fmt in &[
1647 "%b %d, %Y %I:%M %p", "%b %d, %Y %I:%M%p", "%B %d, %Y %I:%M %p", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", ] {
1654 if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(&cleaned, fmt) {
1655 let utc = match chrono::Local.from_local_datetime(&dt) {
1658 chrono::LocalResult::Single(local_dt) => {
1659 local_dt.with_timezone(&chrono::Utc)
1660 }
1661 chrono::LocalResult::Ambiguous(earliest, _) => {
1662 earliest.with_timezone(&chrono::Utc)
1664 }
1665 chrono::LocalResult::None => {
1666 dt.and_utc()
1668 }
1669 };
1670 tracing::info!(
1671 retry_at = %utc,
1672 raw = date_str,
1673 marker = retry_marker,
1674 "parsed rate limit retry-at date"
1675 );
1676 return Some(utc.timestamp());
1677 }
1678 }
1679 tracing::debug!(
1680 raw = date_str,
1681 marker = retry_marker,
1682 "could not parse retry-at date"
1683 );
1684
1685 let time_only = date_str.split_once('(').map_or(date_str, |(t, _)| t).trim();
1690 let mut compact = time_only.replace(" am", "am").replace(" pm", "pm");
1692 let split = compact.rfind(' ').map_or(0, |i| i + 1);
1693 let last = &compact[split..];
1694 if !last.contains(':')
1695 && (last.ends_with("am") || last.ends_with("pm"))
1696 && last.len() > 2
1697 {
1698 let at = split + last.len() - 2;
1699 compact.insert_str(at, ":00");
1700 }
1701 let md = compact.replace(" at ", " ");
1703 let now_local = chrono::Local::now();
1704 let this_year = chrono::Datelike::year(&now_local);
1705 for year in [this_year, this_year + 1] {
1706 let with_year = format!("{year} {md}");
1707 if let Ok(dt) =
1708 chrono::NaiveDateTime::parse_from_str(&with_year, "%Y %b %d %I:%M%p")
1709 && let Some(local_dt) = chrono::Local.from_local_datetime(&dt).earliest()
1710 && local_dt > now_local
1711 {
1712 return Some(local_dt.timestamp());
1713 }
1714 }
1715 let time_only = compact.as_str();
1716 for tf in &["%I:%M %p", "%I:%M%p", "%I %p", "%H:%M"] {
1717 if let Ok(t) = chrono::NaiveTime::parse_from_str(time_only, tf) {
1718 let local_now = chrono::Local::now();
1721 let today = local_now.date_naive();
1722 let candidate = chrono::NaiveDateTime::new(today, t);
1723 let candidate_local: chrono::DateTime<chrono::Local> =
1726 match chrono::Local.from_local_datetime(&candidate) {
1727 chrono::LocalResult::Single(dt) => dt,
1728 chrono::LocalResult::Ambiguous(earliest, _) => earliest,
1729 chrono::LocalResult::None => {
1730 let utc_dt = candidate.and_utc();
1732 return Some(utc_dt.timestamp());
1733 }
1734 };
1735 let chosen = if candidate_local <= local_now {
1736 let tomorrow = today.succ_opt().unwrap_or(today);
1738 let ndt = chrono::NaiveDateTime::new(tomorrow, t);
1739 match chrono::Local.from_local_datetime(&ndt) {
1740 chrono::LocalResult::Single(dt) => dt.with_timezone(&chrono::Utc),
1741 chrono::LocalResult::Ambiguous(earliest, _) => {
1742 earliest.with_timezone(&chrono::Utc)
1743 }
1744 chrono::LocalResult::None => ndt.and_utc(),
1745 }
1746 } else {
1747 candidate_local.with_timezone(&chrono::Utc)
1748 };
1749 tracing::info!(
1750 retry_at = %chosen,
1751 raw = date_str,
1752 marker = retry_marker,
1753 "parsed rate limit retry-at time-only; interpreted as next occurrence"
1754 );
1755 return Some(chosen.timestamp());
1756 }
1757 }
1758 }
1759 }
1760
1761 None
1765}
1766
1767pub(crate) fn parse_relative_usage_window(error_message: &str) -> Option<u64> {
1778 static WINDOW_RE: LazyLock<Regex> = LazyLock::new(|| {
1779 Regex::new(
1780 r"(?i)(?:^|\s|[\(\)\.,;:!?])(?:(\d+)\s*-?\s*(hour|hours|hr|hrs|minute|minutes|min|mins|day|days)|(hourly|daily|weekly|monthly))(?:$|\s|[\(\)\.,;:!?])"
1781 )
1782 .expect("usage window regex is valid")
1783 });
1784
1785 if error_message.is_empty() {
1786 return None;
1787 }
1788
1789 let lower = error_message.to_lowercase();
1790 let has_context = lower.contains("usage limit")
1791 || lower.contains("quota")
1792 || lower.contains("window")
1793 || lower.contains("limit");
1794 if !has_context {
1795 return None;
1796 }
1797
1798 let mut max_secs: Option<u64> = None;
1799 for caps in WINDOW_RE.captures_iter(&lower) {
1800 let secs = if let Some(num) = caps.get(1) {
1801 let n: u64 = num.as_str().parse().ok()?;
1802 let unit = caps.get(2)?.as_str().to_lowercase();
1803 let multiplier = match unit.as_str() {
1804 "hour" | "hours" | "hr" | "hrs" => 3600,
1805 "minute" | "minutes" | "min" | "mins" => 60,
1806 "day" | "days" => 86400,
1807 _ => return None,
1808 };
1809 n.checked_mul(multiplier)?
1810 } else if let Some(word) = caps.get(3) {
1811 match word.as_str().to_lowercase().as_str() {
1812 "hourly" => 3600,
1813 "daily" => 86400,
1814 "weekly" => 604_800,
1815 "monthly" => 2_592_000,
1816 _ => return None,
1817 }
1818 } else {
1819 continue;
1820 };
1821 max_secs = Some(max_secs.map_or(secs, |m| m.max(secs)));
1822 }
1823
1824 max_secs
1825}
1826
1827#[cfg(test)]
1828pub(crate) async fn reset_global_state() {
1829 cooldowns().lock().unwrap().clear();
1830 *cooldown_store().lock().await = None;
1831 github_5xx_timestamps().lock().unwrap().clear();
1832 *github_5xx_circuit_open().lock().unwrap() = None;
1833 degraded_agents().lock().unwrap().clear();
1834 opencode::reset_model_caches_for_test();
1835}
1836
1837#[cfg(test)]
1838mod tests {
1839 use std::time::Duration;
1840
1841 use super::*;
1842 use chrono::Datelike;
1843 use chrono::TimeZone;
1844 use serial_test::serial;
1845 use tokio::time;
1846
1847 async fn test_store() -> Arc<TaskStore> {
1848 Arc::new(TaskStore::open_memory().await.expect("in-memory store"))
1849 }
1850
1851 fn sqlite_datetime_offset(secs: i64) -> String {
1852 (chrono::Utc::now() + chrono::Duration::seconds(secs))
1853 .format("%Y-%m-%d %H:%M:%S")
1854 .to_string()
1855 }
1856
1857 #[serial(cooldown_state)]
1858 #[tokio::test]
1859 async fn init_loads_unexpired_cooldowns_from_kv() {
1860 super::reset_global_state().await;
1861 let store = test_store().await;
1862 let cooldown_until = chrono::Utc::now().timestamp() + 3600; store
1865 .kv_set(
1866 &format!("{KV_PREFIX}kimi:k2p5"),
1867 &cooldown_until.to_string(),
1868 )
1869 .await
1870 .unwrap();
1871
1872 let expired = chrono::Utc::now().timestamp() - 1;
1874 store
1875 .kv_set(
1876 &format!("{KV_PREFIX}opencode:old-free"),
1877 &expired.to_string(),
1878 )
1879 .await
1880 .unwrap();
1881
1882 {
1883 let mut map = cooldown_lock();
1884 map.clear();
1885 }
1886
1887 init_cooldown_store(store).await;
1888
1889 assert!(
1890 is_model_in_cooldown("kimi", "k2p5"),
1891 "unexpired cooldown should be loaded"
1892 );
1893 assert!(
1894 !is_model_in_cooldown("opencode", "old-free"),
1895 "expired cooldown should not be loaded"
1896 );
1897 }
1898
1899 #[serial(cooldown_state)]
1900 #[tokio::test]
1901 async fn credit_exhaustion_inside_active_cooldown_keeps_counter_at_one() {
1902 super::reset_global_state().await;
1903 let store = test_store().await;
1904 *cooldown_store().lock().await = Some(Arc::clone(&store));
1905
1906 let msg =
1907 "API error (status 402 Payment Required): usage balance exhausted, out of credits";
1908 assert!(record_credit_exhaustion_from_message("credit_dup_agent", Some("m"), msg).await);
1909 assert!(record_credit_exhaustion_from_message("credit_dup_agent", Some("m"), msg).await);
1910
1911 assert!(is_agent_in_cooldown("credit_dup_agent"));
1912 let count = store
1913 .kv_get(&format!("{CREDIT_FAILURE_COUNT_PREFIX}credit_dup_agent"))
1914 .await
1915 .unwrap();
1916 assert_eq!(count.as_deref(), Some("1"));
1917 }
1918
1919 #[serial(cooldown_state)]
1920 #[tokio::test]
1921 async fn non_credit_message_is_not_recorded_as_credit_exhaustion() {
1922 super::reset_global_state().await;
1923 assert!(!record_credit_exhaustion_from_message("x_agent", Some("m"), "boom").await);
1924 assert!(!is_agent_in_cooldown("x_agent"));
1925 }
1926
1927 #[serial(cooldown_state)]
1928 #[tokio::test]
1929 async fn record_model_failure_writes_to_kv() {
1930 super::reset_global_state().await;
1931 let store = test_store().await;
1932
1933 *cooldown_store().lock().await = Some(Arc::clone(&store));
1934
1935 record_model_failure("testagent_persist", "testmodel_persist").await;
1936
1937 time::sleep(Duration::from_millis(50)).await;
1939
1940 let kv_key = format!("{KV_PREFIX}testagent_persist:testmodel_persist");
1941 let stored = store.kv_get(&kv_key).await.unwrap();
1942 assert!(stored.is_some(), "model failure should be persisted to KV");
1943 let (ts, reason) = parse_cooldown_value(&stored.unwrap()).expect("cooldown value");
1944 let now = chrono::Utc::now().timestamp();
1945 assert!(ts > now, "persisted timestamp should be in the future");
1947 assert_eq!(reason, "model_error");
1948 }
1949
1950 #[serial(cooldown_state)]
1951 #[tokio::test]
1952 async fn record_transient_model_failure_backs_off_5_then_15_minutes() {
1953 super::reset_global_state().await;
1954 let store = test_store().await;
1955 *cooldown_store().lock().await = Some(Arc::clone(&store));
1956
1957 let agent = "testagent_transient";
1958 let model = "testmodel_transient";
1959 let key = format!("{agent}:{model}");
1960
1961 record_transient_model_failure(agent, model).await;
1962 let first = cooldown_until(&key)
1963 .expect("first cooldown should exist")
1964 .saturating_sub(chrono::Utc::now().timestamp());
1965 assert!(
1966 (TRANSIENT_MODEL_BACKOFF_BASE_SECS - 5..=TRANSIENT_MODEL_BACKOFF_BASE_SECS + 5)
1967 .contains(&first),
1968 "first transient cooldown should be ~5m, got {first}s"
1969 );
1970
1971 record_transient_model_failure(agent, model).await;
1972 let second = cooldown_until(&key)
1973 .expect("second cooldown should exist")
1974 .saturating_sub(chrono::Utc::now().timestamp());
1975 assert!(
1976 (TRANSIENT_MODEL_BACKOFF_MAX_SECS - 5..=TRANSIENT_MODEL_BACKOFF_MAX_SECS + 5)
1977 .contains(&second),
1978 "second transient cooldown should be capped at ~15m, got {second}s"
1979 );
1980
1981 let (_, reason) = parse_cooldown_value(
1982 &store
1983 .kv_get(&format!("{KV_PREFIX}{key}"))
1984 .await
1985 .unwrap()
1986 .expect("cooldown persisted"),
1987 )
1988 .expect("cooldown value");
1989 assert_eq!(reason, "transient_model_error");
1990
1991 let count = store
1992 .kv_get(&format!("{FAILURE_COUNT_PREFIX}{key}"))
1993 .await
1994 .unwrap();
1995 assert_eq!(count.as_deref(), Some("2"));
1996 }
1997
1998 #[serial(cooldown_state)]
1999 #[tokio::test]
2000 async fn record_model_failure_and_persistent_failure_use_separate_counters() {
2001 super::reset_global_state().await;
2002 let store = test_store().await;
2003 *cooldown_store().lock().await = Some(Arc::clone(&store));
2004
2005 let agent = "testagent_counters";
2006 let model = "testmodel_counters";
2007 let key = format!("{agent}:{model}");
2008
2009 record_model_failure(agent, model).await;
2010 record_persistent_model_failure(agent, model).await;
2011 record_persistent_model_failure(agent, model).await;
2012
2013 let generic = store
2014 .kv_get(&format!("{FAILURE_COUNT_PREFIX}{key}"))
2015 .await
2016 .unwrap();
2017 let persistent = store
2018 .kv_get(&format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{key}"))
2019 .await
2020 .unwrap();
2021 assert_eq!(generic.as_deref(), Some("1"));
2022 assert_eq!(persistent.as_deref(), Some("2"));
2023
2024 let (_, reason) = parse_cooldown_value(
2025 &store
2026 .kv_get(&format!("{KV_PREFIX}{key}"))
2027 .await
2028 .unwrap()
2029 .expect("cooldown persisted"),
2030 )
2031 .expect("cooldown value");
2032 assert_eq!(reason, "persistent_model_error");
2033 }
2034
2035 #[serial(cooldown_state)]
2036 #[tokio::test]
2037 async fn cooldown_reason_survives_restart_via_kv() {
2038 super::reset_global_state().await;
2039 let store = test_store().await;
2040 let agent = "test_reason_restart";
2041 let future = chrono::Utc::now().timestamp() + 600;
2042
2043 init_cooldown_store(Arc::clone(&store)).await;
2044 set_cooldown_async(agent, future, "billing_cycle_exhausted").await;
2045
2046 {
2048 let mut map = cooldown_lock();
2049 map.clear();
2050 }
2051 init_cooldown_store(store).await;
2052
2053 let entries = list_all_cooldowns();
2054 let entry = entries.iter().find(|(key, ..)| key == agent);
2055 assert_eq!(
2056 entry.map(|(_, _, reason)| reason.as_str()),
2057 Some("billing_cycle_exhausted"),
2058 "cooldown reason must survive a restart, not collapse to 'persisted'"
2059 );
2060 }
2061
2062 #[serial(cooldown_state)]
2063 #[tokio::test]
2064 async fn record_persistent_model_failure_escalates_and_increments_failure_count() {
2065 super::reset_global_state().await;
2066 let store = test_store().await;
2067 *cooldown_store().lock().await = Some(Arc::clone(&store));
2068
2069 let agent = "testagent_persistent";
2070 let model = "testmodel_persistent";
2071 let key = format!("{agent}:{model}");
2072
2073 record_persistent_model_failure(agent, model).await;
2075 let now = chrono::Utc::now().timestamp();
2076 let first_until = cooldown_until(&key).expect("first cooldown should exist");
2077 let first_remaining = first_until.saturating_sub(now);
2078 assert!(
2079 first_remaining >= PERSISTENT_MODEL_BACKOFF_BASE_SECS - 5,
2080 "first persistent model cooldown should be ~4h, got {first_remaining}s"
2081 );
2082
2083 record_persistent_model_failure(agent, model).await;
2085 let now2 = chrono::Utc::now().timestamp();
2086 let second_until = cooldown_until(&key).expect("second cooldown should exist");
2087 let second_remaining = second_until.saturating_sub(now2);
2088 assert!(
2089 second_remaining >= (PERSISTENT_MODEL_BACKOFF_BASE_SECS * 3) - 5,
2090 "second persistent model cooldown should be ~12h, got {second_remaining}s"
2091 );
2092
2093 let count_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{agent}:{model}");
2094 let stored_count = store
2095 .kv_get(&count_key)
2096 .await
2097 .expect("kv read should succeed")
2098 .expect("failure count should be written");
2099 assert_eq!(stored_count, "2");
2100 }
2101
2102 #[serial(cooldown_state)]
2103 #[tokio::test]
2104 async fn record_agent_success_does_not_reset_persistent_failure_count() {
2105 super::reset_global_state().await;
2106 let store = test_store().await;
2107 *cooldown_store().lock().await = Some(Arc::clone(&store));
2108
2109 let agent = "testagent_persistent_success";
2110 let model = "testmodel_persistent_success";
2111 let key = format!("{agent}:{model}");
2112
2113 record_persistent_model_failure(agent, model).await;
2115 record_persistent_model_failure(agent, model).await;
2116
2117 let count_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{agent}:{model}");
2118 let stored_count = store
2119 .kv_get(&count_key)
2120 .await
2121 .expect("kv read should succeed")
2122 .expect("failure count should be written");
2123 assert_eq!(stored_count, "2");
2124
2125 record_agent_success(agent, model).await;
2129
2130 let stored_count_after_success = store
2131 .kv_get(&count_key)
2132 .await
2133 .expect("kv read should succeed")
2134 .expect("persistent failure count should still exist");
2135 assert_eq!(
2136 stored_count_after_success, "2",
2137 "persistent failure count must survive an intervening success"
2138 );
2139
2140 record_persistent_model_failure(agent, model).await;
2142 let now = chrono::Utc::now().timestamp();
2143 let third_until = cooldown_until(&key).expect("third cooldown should exist");
2144 let third_remaining = third_until.saturating_sub(now);
2145 assert!(
2146 third_remaining > PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2147 "third persistent model failure should escalate past the 4h base, got {third_remaining}s"
2148 );
2149 }
2150
2151 #[serial(cooldown_state)]
2152 #[tokio::test]
2153 async fn record_persistent_model_failure_stays_capped_at_7_days() {
2154 super::reset_global_state().await;
2155 let store = test_store().await;
2156 *cooldown_store().lock().await = Some(Arc::clone(&store));
2157
2158 let agent = "testagent_persistent_capped";
2159 let model = "testmodel_persistent_capped";
2160 let key = format!("{agent}:{model}");
2161
2162 for _ in 0..9 {
2163 record_persistent_model_failure(agent, model).await;
2164 }
2165
2166 let remaining = cooldown_until(&key)
2167 .expect("cooldown should exist")
2168 .saturating_sub(chrono::Utc::now().timestamp());
2169 assert!(
2170 remaining >= PERSISTENT_MODEL_BACKOFF_MAX_SECS - 5,
2171 "persistent model cooldown should stay capped at 7 days, got {remaining}s"
2172 );
2173 assert!(
2174 remaining <= PERSISTENT_MODEL_BACKOFF_MAX_SECS + 5,
2175 "persistent model cooldown should not exceed 7 days, got {remaining}s"
2176 );
2177 }
2178
2179 #[serial(cooldown_state)]
2180 #[test]
2181 fn parse_retry_at_codex_format() {
2182 let msg = "You've hit your usage limit. Upgrade to Pro, visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Mar 26th, 2026 5:55 AM.";
2183 let ts = parse_retry_at(msg);
2184 assert!(ts.is_some(), "should parse codex retry-at date");
2185
2186 let naive = chrono::NaiveDateTime::new(
2189 chrono::NaiveDate::from_ymd_opt(2026, 3, 26).unwrap(),
2190 chrono::NaiveTime::from_hms_opt(5, 55, 0).unwrap(),
2191 );
2192 let expected_utc = match chrono::Local.from_local_datetime(&naive) {
2193 chrono::LocalResult::Single(local_dt) => local_dt.with_timezone(&chrono::Utc),
2194 chrono::LocalResult::Ambiguous(earliest, _) => earliest.with_timezone(&chrono::Utc),
2195 chrono::LocalResult::None => naive.and_utc(),
2196 };
2197 assert_eq!(
2198 ts.unwrap(),
2199 expected_utc.timestamp(),
2200 "timestamp should reflect local-timezone interpretation of the vendor date"
2201 );
2202
2203 let dt = chrono::DateTime::from_timestamp(ts.unwrap(), 0).unwrap();
2205 assert_eq!(dt.month(), 3);
2206 assert_eq!(dt.day(), 26);
2207 }
2208
2209 #[serial(cooldown_state)]
2210 #[test]
2211 fn parse_retry_at_hour_only_and_month_day() {
2212 let now = chrono::Utc::now().timestamp();
2213 for msg in [
2214 "You've hit your session limit · resets 8am (America/Sao_Paulo)",
2215 "You've hit your session limit · resets 6pm (America/Sao_Paulo)",
2216 "You've hit your session limit · resets 12am (America/Sao_Paulo)",
2217 "You've hit your weekly limit · resets Jun 9 at 1am (America/Sao_Paulo)",
2218 ] {
2219 let ts = parse_retry_at(msg).unwrap_or_else(|| panic!("should parse: {msg}"));
2220 assert!(ts > now, "{msg}");
2221 }
2222 }
2223
2224 #[serial(cooldown_state)]
2225 #[test]
2226 fn parse_retry_at_weekly_resets_without_year() {
2227 let t = chrono::Local::now() + chrono::Duration::days(2);
2228 let msg = format!(
2229 "You've hit your weekly limit · resets {} {} at 1am (America/Sao_Paulo)",
2230 t.format("%b"),
2231 t.day()
2232 );
2233 let ts = parse_retry_at(&msg).expect("should parse year-less reset date");
2234 let remaining = ts - chrono::Utc::now().timestamp();
2235 assert!(
2236 remaining > 3600 && remaining < 3 * 86400,
2237 "expected ~2d retry-at, got {remaining}s"
2238 );
2239 }
2240
2241 #[serial(cooldown_state)]
2242 #[test]
2243 fn parse_retry_at_no_date() {
2244 assert!(parse_retry_at("generic rate limit error").is_none());
2245 assert!(parse_retry_at("").is_none());
2246 }
2247
2248 #[serial(cooldown_state)]
2249 #[test]
2250 fn parse_retry_at_glm_reset_format_with_seconds() {
2251 let msg = "Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-04-23 18:33:48";
2252 let ts = parse_retry_at(msg).expect("should parse GLM reset-at date");
2253
2254 let naive = chrono::NaiveDateTime::new(
2255 chrono::NaiveDate::from_ymd_opt(2026, 4, 23).expect("valid date"),
2256 chrono::NaiveTime::from_hms_opt(18, 33, 48).expect("valid time"),
2257 );
2258 let expected_utc = match chrono::Local.from_local_datetime(&naive) {
2259 chrono::LocalResult::Single(local_dt) => local_dt.with_timezone(&chrono::Utc),
2260 chrono::LocalResult::Ambiguous(earliest, _) => earliest.with_timezone(&chrono::Utc),
2261 chrono::LocalResult::None => naive.and_utc(),
2262 };
2263
2264 assert_eq!(
2265 ts,
2266 expected_utc.timestamp(),
2267 "timestamp should reflect local-timezone interpretation of vendor reset date"
2268 );
2269 }
2270
2271 #[serial(cooldown_state)]
2272 #[test]
2273 fn parse_retry_at_billing_cycle_without_date_returns_none() {
2274 let msg = "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle. Upgrade to get more.";
2277 assert!(
2278 parse_retry_at(msg).is_none(),
2279 "billing cycle without date should return None"
2280 );
2281 }
2282
2283 #[serial(cooldown_state)]
2284 #[test]
2285 fn parse_relative_usage_window_extracts_kimi_five_hour_window() {
2286 let msg = "Failed to authenticate. API Error: 403 You've reached your 5-hour usage limit. Your quota will reset when the current 5-hour window expires.";
2287 assert_eq!(
2288 parse_relative_usage_window(msg),
2289 Some(5 * 3600),
2290 "should extract the 5-hour window as seconds"
2291 );
2292 }
2293
2294 #[serial(cooldown_state)]
2295 #[test]
2296 fn parse_relative_usage_window_handles_common_durations() {
2297 assert_eq!(
2298 parse_relative_usage_window("You've reached your hourly usage limit."),
2299 Some(3600)
2300 );
2301 assert_eq!(
2302 parse_relative_usage_window("You've reached your daily usage limit."),
2303 Some(86400)
2304 );
2305 assert_eq!(
2306 parse_relative_usage_window("You've reached your weekly limit."),
2307 Some(604_800)
2308 );
2309 assert_eq!(
2310 parse_relative_usage_window("You've reached your 30-minute usage limit."),
2311 Some(1800)
2312 );
2313 assert_eq!(
2314 parse_relative_usage_window("You've reached your 2-day quota."),
2315 Some(172_800)
2316 );
2317 }
2318
2319 #[serial(cooldown_state)]
2320 #[test]
2321 fn parse_relative_usage_window_returns_none_without_usage_context() {
2322 assert!(parse_relative_usage_window("429 Too Many Requests").is_none());
2325 assert!(parse_relative_usage_window("rate limit exceeded").is_none());
2326 assert!(parse_relative_usage_window("").is_none());
2327 }
2328
2329 #[serial(cooldown_state)]
2330 #[tokio::test]
2331 async fn record_rate_limit_uses_model_specific_window_cooldown() {
2332 super::reset_global_state().await;
2333 let store = test_store().await;
2334 *cooldown_store().lock().await = Some(Arc::clone(&store));
2335
2336 let agent = "test_record_rl_window_agent";
2337 let model = "test_record_rl_window_model";
2338 let msg = "You've reached your 5-hour usage limit. Your quota will reset when the current 5-hour window expires.";
2339
2340 record_rate_limit(agent, Some(model), msg).await;
2341
2342 assert!(
2343 is_model_in_cooldown(agent, model),
2344 "model should be cooled for the relative usage window"
2345 );
2346 assert!(
2347 !is_agent_in_cooldown(agent),
2348 "agent should not be cooled for a model-scoped quota"
2349 );
2350
2351 let key = format!("{agent}:{model}");
2352 let remaining = cooldown_until(&key)
2353 .expect("model cooldown should exist")
2354 .saturating_sub(chrono::Utc::now().timestamp());
2355 assert!(
2356 remaining >= 5 * 3600 - 5,
2357 "model cooldown should be at least ~5 hours, got {remaining}s"
2358 );
2359 assert_eq!(
2360 cooldown_reason(&key).as_deref(),
2361 Some("rate_limit_usage_window"),
2362 "relative usage-window cooldown must be labeled as a rate limit, not silence"
2363 );
2364 }
2365
2366 #[serial(cooldown_state)]
2367 #[tokio::test]
2368 async fn record_rate_limit_without_window_falls_back_to_agent_and_model() {
2369 super::reset_global_state().await;
2370 let store = test_store().await;
2371 *cooldown_store().lock().await = Some(Arc::clone(&store));
2372
2373 let agent = "test_record_rl_fallback_agent";
2374 let model = "test_record_rl_fallback_model";
2375 let msg = "429 Too Many Requests";
2376
2377 record_rate_limit(agent, Some(model), msg).await;
2378
2379 assert!(
2380 is_agent_in_cooldown(agent),
2381 "agent should be cooled when no relative window is detected"
2382 );
2383 assert!(
2384 is_model_in_cooldown(agent, model),
2385 "model should be cooled when no relative window is detected"
2386 );
2387 }
2388
2389 #[serial(cooldown_state)]
2390 #[tokio::test]
2391 async fn agent_cooldown_persists_to_kv() {
2392 super::reset_global_state().await;
2393 let agent = "test_agent_persist_check";
2395 record_agent_failure_with_message(agent, "").await;
2396 assert!(is_agent_in_cooldown(agent));
2397 }
2398
2399 #[serial(cooldown_state)]
2400 #[tokio::test]
2401 async fn cooldown_never_shortens() {
2402 super::reset_global_state().await;
2403 let agent = "test_agent_no_shorten";
2404 set_agent_cooldown(agent, 24 * 60 * 60).await;
2405 let initial = {
2406 let map = cooldowns().lock().unwrap();
2407 map.get(agent)
2408 .expect("cooldown entry should exist")
2409 .cooldown_until
2410 };
2411
2412 record_agent_failure_with_message(agent, "rate limit").await;
2414 let after = {
2415 let map = cooldowns().lock().unwrap();
2416 map.get(agent)
2417 .expect("cooldown entry should exist")
2418 .cooldown_until
2419 };
2420 assert_eq!(
2421 initial, after,
2422 "cooldown should not be shortened by later failures"
2423 );
2424 }
2425
2426 #[serial(cooldown_state)]
2427 #[tokio::test]
2428 async fn silence_agent_cooldown_is_short_lived() {
2429 super::reset_global_state().await;
2430 let agent = "test_silence_agent_cd";
2431 assert!(!is_agent_in_cooldown(agent));
2432
2433 set_agent_cooldown(agent, SILENCE_AGENT_COOLDOWN_SECS).await;
2434 assert!(is_agent_in_cooldown(agent));
2435
2436 let map = cooldowns().lock().unwrap();
2438 let entry = map.get(agent).expect("should have cooldown entry");
2439 let now = chrono::Utc::now().timestamp();
2440 let remaining = entry.cooldown_until - now;
2441 assert!(
2442 remaining <= i64::try_from(SILENCE_AGENT_COOLDOWN_SECS).unwrap(),
2443 "silence agent cooldown should be <= {SILENCE_AGENT_COOLDOWN_SECS}s, got {remaining}s"
2444 );
2445 assert!(remaining > 0, "cooldown should still be active");
2446 }
2447
2448 #[serial(cooldown_state)]
2449 #[tokio::test]
2450 async fn record_silence_detection_applies_extended_cooldown() {
2451 super::reset_global_state().await;
2452 let store = test_store().await;
2453 *cooldown_store().lock().await = Some(Arc::clone(&store));
2454
2455 let agent = "test_silence_count_agent";
2456 let model = "test_silence_count_model";
2457 assert!(!is_model_in_cooldown(agent, model));
2458
2459 for _ in 0..SILENCE_COUNT_THRESHOLD {
2460 let result = record_silence_detection(agent, model).await;
2461 assert!(result.is_some());
2462 }
2463
2464 assert!(is_model_in_cooldown(agent, model));
2465 let key = format!("{agent}:{model}");
2466 let remaining = {
2467 let map = cooldowns().lock().unwrap();
2468 let entry = map.get(&key).expect("cooldown entry should exist");
2469 entry.cooldown_until - chrono::Utc::now().timestamp()
2470 };
2471 assert!(
2472 remaining >= i64::try_from(SILENCE_EXTENDED_COOLDOWN_SECS).unwrap() - 5,
2473 "extended cooldown should be applied, got {remaining}s"
2474 );
2475
2476 let kv_key = format!("{SILENCE_COUNT_PREFIX}{agent}:{model}");
2477 let stored = store.kv_get(&kv_key).await.unwrap().unwrap();
2478 assert_eq!(stored, "[]");
2479 }
2480
2481 #[serial(cooldown_state)]
2482 #[test]
2483 fn detect_credit_exhaustion_out_of_credits() {
2484 let msg = "Error: API error - {\"error\":{\"message\":\"You exceeded your current quota\",\"type\":\"insufficient_quota\",\"param\":null,\"code\":\"insufficient_quota\"}}";
2485 assert_eq!(
2486 detect_credit_exhaustion(msg),
2487 Some(CreditExhaustionReason::OutOfCredits)
2488 );
2489 }
2490
2491 #[serial(cooldown_state)]
2492 #[test]
2493 fn detect_credit_exhaustion_out_of_credits_variations() {
2494 assert_eq!(
2495 detect_credit_exhaustion("out of credits"),
2496 Some(CreditExhaustionReason::OutOfCredits)
2497 );
2498 assert_eq!(
2499 detect_credit_exhaustion("outofcredits"),
2500 Some(CreditExhaustionReason::OutOfCredits)
2501 );
2502 assert_eq!(
2503 detect_credit_exhaustion("no credits remaining"),
2504 Some(CreditExhaustionReason::OutOfCredits)
2505 );
2506 assert_eq!(
2507 detect_credit_exhaustion("insufficient funds"),
2508 Some(CreditExhaustionReason::OutOfCredits)
2509 );
2510 }
2511
2512 #[serial(cooldown_state)]
2513 #[test]
2514 fn detect_credit_exhaustion_org_level_disabled() {
2515 let msg = "Error: API error - {\"error\":{\"message\":\"Organization has been disabled\",\"type\":\"org_disabled\",\"code\":\"overageDisabledReason:org_level_disabled\"}}";
2516 assert_eq!(
2517 detect_credit_exhaustion(msg),
2518 Some(CreditExhaustionReason::OrgLevelDisabled)
2519 );
2520 }
2521
2522 #[serial(cooldown_state)]
2523 #[test]
2524 fn detect_credit_exhaustion_org_level_disabled_variations() {
2525 assert_eq!(
2526 detect_credit_exhaustion("organization has been disabled"),
2527 Some(CreditExhaustionReason::OrgLevelDisabled)
2528 );
2529 assert_eq!(
2530 detect_credit_exhaustion("org disabled"),
2531 Some(CreditExhaustionReason::OrgLevelDisabled)
2532 );
2533 assert_eq!(
2534 detect_credit_exhaustion("billing quota exceeded"),
2535 Some(CreditExhaustionReason::OrgLevelDisabled)
2536 );
2537 assert_eq!(
2538 detect_credit_exhaustion("credit balance too low"),
2539 Some(CreditExhaustionReason::OrgLevelDisabled)
2540 );
2541 }
2542
2543 #[serial(cooldown_state)]
2544 #[test]
2545 fn detect_credit_exhaustion_overage_disabled_reason_parsing() {
2546 let msg = "overageDisabledReason: out_of_credits";
2547 assert_eq!(
2548 detect_credit_exhaustion(msg),
2549 Some(CreditExhaustionReason::OutOfCredits)
2550 );
2551
2552 let msg = "overageDisabledReason: org_level_disabled";
2553 assert_eq!(
2554 detect_credit_exhaustion(msg),
2555 Some(CreditExhaustionReason::OrgLevelDisabled)
2556 );
2557 }
2558
2559 #[serial(cooldown_state)]
2560 #[test]
2561 fn detect_credit_exhaustion_glm_insufficient_balance() {
2562 assert_eq!(
2564 detect_credit_exhaustion(
2565 "API Error: Request rejected (429) · Insufficient balance or no resource package. Please recharge."
2566 ),
2567 Some(CreditExhaustionReason::OutOfCredits)
2568 );
2569 assert_eq!(
2570 detect_credit_exhaustion("Insufficient balance"),
2571 Some(CreditExhaustionReason::OutOfCredits)
2572 );
2573 assert_eq!(
2574 detect_credit_exhaustion("please recharge your account"),
2575 Some(CreditExhaustionReason::OutOfCredits)
2576 );
2577 assert_eq!(
2578 detect_credit_exhaustion("no resource package available"),
2579 Some(CreditExhaustionReason::OutOfCredits)
2580 );
2581 }
2582
2583 #[serial(cooldown_state)]
2584 #[test]
2585 fn detect_credit_exhaustion_none_for_regular_rate_limit() {
2586 assert!(detect_credit_exhaustion("rate limit exceeded").is_none());
2587 assert!(detect_credit_exhaustion("too many requests").is_none());
2588 assert!(detect_credit_exhaustion("429 Too Many Requests").is_none());
2589 assert!(detect_credit_exhaustion("you've hit your usage limit").is_none());
2590 assert!(
2591 detect_credit_exhaustion("You've hit your usage limit, try again in a few minutes.")
2592 .is_none()
2593 );
2594 }
2595
2596 #[serial(cooldown_state)]
2597 #[tokio::test]
2598 async fn record_credit_exhaustion_applies_agent_cooldown() {
2599 super::reset_global_state().await;
2600 let agent = "test_credit_exhaust_agent";
2601 assert!(!is_agent_in_cooldown(agent));
2602
2603 record_credit_exhaustion(agent, CreditExhaustionReason::OutOfCredits).await;
2604 assert!(is_agent_in_cooldown(agent));
2605
2606 let remaining = {
2607 let map = cooldowns().lock().unwrap();
2608 let entry = map.get(agent).expect("cooldown entry should exist");
2609 entry.cooldown_until - chrono::Utc::now().timestamp()
2610 };
2611 assert!(
2613 remaining >= CREDIT_BACKOFF_BASE_SECS - 5,
2614 "first out_of_credits cooldown should be ~1 hour, got {remaining}s"
2615 );
2616 assert!(
2617 remaining <= CREDIT_BACKOFF_MAX_SECS,
2618 "cooldown should not exceed cap of 8h, got {remaining}s"
2619 );
2620 }
2621
2622 #[serial(cooldown_state)]
2623 #[tokio::test]
2624 async fn record_credit_exhaustion_org_level_applies_longer_cooldown() {
2625 super::reset_global_state().await;
2626 let agent = "test_org_disabled_agent";
2627 assert!(!is_agent_in_cooldown(agent));
2628
2629 record_credit_exhaustion(agent, CreditExhaustionReason::OrgLevelDisabled).await;
2630 assert!(is_agent_in_cooldown(agent));
2631
2632 let remaining = {
2633 let map = cooldowns().lock().unwrap();
2634 let entry = map.get(agent).expect("cooldown entry should exist");
2635 entry.cooldown_until - chrono::Utc::now().timestamp()
2636 };
2637 assert!(
2639 remaining >= ORG_BACKOFF_BASE_SECS - 5,
2640 "first org-level disabled cooldown should be ~2 hours, got {remaining}s"
2641 );
2642 assert!(
2643 remaining <= CREDIT_BACKOFF_MAX_SECS,
2644 "cooldown should not exceed cap of 8h, got {remaining}s"
2645 );
2646 }
2647
2648 #[serial(cooldown_state)]
2649 #[tokio::test]
2650 async fn record_credit_exhaustion_billing_cycle_applies_24h() {
2651 super::reset_global_state().await;
2652 let store = test_store().await;
2653 *cooldown_store().lock().await = Some(Arc::clone(&store));
2654
2655 let agent = "test_billing_cycle_agent";
2656 assert!(!is_agent_in_cooldown(agent));
2657
2658 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2659 assert!(is_agent_in_cooldown(agent));
2660
2661 let remaining = {
2662 let map = cooldowns().lock().unwrap();
2663 let entry = map.get(agent).expect("cooldown entry should exist");
2664 entry.cooldown_until - chrono::Utc::now().timestamp()
2665 };
2666 assert!(
2668 remaining >= BILLING_CYCLE_COOLDOWN_SECS - 5,
2669 "first billing cycle cooldown should be ~24 hours, got {remaining}s"
2670 );
2671 assert!(
2672 remaining <= BILLING_CYCLE_MAX_SECS,
2673 "cooldown should not exceed cap of 7 days, got {remaining}s"
2674 );
2675 }
2676
2677 #[serial(cooldown_state)]
2678 #[tokio::test]
2679 async fn record_credit_exhaustion_billing_cycle_escalates() {
2680 super::reset_global_state().await;
2681 let store = test_store().await;
2682 *cooldown_store().lock().await = Some(Arc::clone(&store));
2683
2684 let agent = "test_billing_cycle_escalation";
2685
2686 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2688 let remaining_1 = {
2689 let map = cooldowns().lock().unwrap();
2690 let entry = map.get(agent).expect("cooldown entry should exist");
2691 entry.cooldown_until - chrono::Utc::now().timestamp()
2692 };
2693 assert!(
2694 (BILLING_CYCLE_COOLDOWN_SECS - 5..=BILLING_CYCLE_COOLDOWN_SECS + 5)
2695 .contains(&remaining_1),
2696 "first billing cycle cooldown should be ~24h, got {remaining_1}s"
2697 );
2698
2699 {
2701 let mut map = cooldowns().lock().unwrap();
2702 map.remove(agent);
2703 }
2704
2705 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2707 let remaining_2 = {
2708 let map = cooldowns().lock().unwrap();
2709 let entry = map.get(agent).expect("cooldown entry should exist");
2710 entry.cooldown_until - chrono::Utc::now().timestamp()
2711 };
2712 let expected_2 = BILLING_CYCLE_COOLDOWN_SECS * 3; assert!(
2714 remaining_2 >= expected_2 - 5,
2715 "second billing cycle cooldown should be ~72h, got {remaining_2}s"
2716 );
2717
2718 {
2720 let mut map = cooldowns().lock().unwrap();
2721 map.remove(agent);
2722 }
2723
2724 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2726 let remaining_3 = {
2727 let map = cooldowns().lock().unwrap();
2728 let entry = map.get(agent).expect("cooldown entry should exist");
2729 entry.cooldown_until - chrono::Utc::now().timestamp()
2730 };
2731 assert!(
2732 remaining_3 >= BILLING_CYCLE_MAX_SECS - 5,
2733 "third billing cycle cooldown should be capped at 7 days, got {remaining_3}s"
2734 );
2735 assert!(
2736 remaining_3 <= BILLING_CYCLE_MAX_SECS + 5,
2737 "third billing cycle cooldown should not exceed 7 days, got {remaining_3}s"
2738 );
2739 }
2740
2741 #[serial(cooldown_state)]
2742 #[tokio::test]
2743 async fn record_credit_exhaustion_billing_cycle_stays_capped_after_repeats() {
2744 super::reset_global_state().await;
2745 let store = test_store().await;
2746 *cooldown_store().lock().await = Some(Arc::clone(&store));
2747
2748 let agent = "test_billing_cycle_agent_capped";
2749
2750 for i in 0..5 {
2751 if i > 0 {
2752 cooldowns().lock().unwrap().remove(agent);
2754 }
2755 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2756 }
2757
2758 let remaining = {
2759 let map = cooldowns().lock().unwrap();
2760 let entry = map.get(agent).expect("cooldown entry should exist");
2761 entry.cooldown_until - chrono::Utc::now().timestamp()
2762 };
2763 assert!(
2764 remaining >= BILLING_CYCLE_MAX_SECS - 5,
2765 "billing cycle cooldown should stay capped at 7 days, got {remaining}s"
2766 );
2767 assert!(
2768 remaining <= BILLING_CYCLE_MAX_SECS + 5,
2769 "billing cycle cooldown should not exceed 7 days, got {remaining}s"
2770 );
2771 }
2772
2773 #[serial(cooldown_state)]
2774 #[test]
2775 fn credit_exhaustion_backoff_constants() {
2776 assert_eq!(
2778 compute_backoff(1, CREDIT_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS, true),
2779 CREDIT_BACKOFF_BASE_SECS
2780 );
2781 assert_eq!(
2783 compute_backoff(1, ORG_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS, true),
2784 ORG_BACKOFF_BASE_SECS
2785 );
2786 assert_eq!(BILLING_CYCLE_COOLDOWN_SECS, 24 * 60 * 60);
2788 assert_eq!(BILLING_CYCLE_MAX_SECS, 7 * 24 * 60 * 60);
2789 }
2790
2791 #[serial(cooldown_state)]
2792 #[test]
2793 fn compute_backoff_long_cap_without_extended_stays_capped() {
2794 assert_eq!(
2795 compute_backoff(
2796 3,
2797 BILLING_CYCLE_COOLDOWN_SECS,
2798 BILLING_CYCLE_MAX_SECS,
2799 false
2800 ),
2801 BILLING_CYCLE_MAX_SECS,
2802 "7-day capped paths should stop at the documented max"
2803 );
2804 assert_eq!(
2805 compute_backoff(
2806 9,
2807 PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2808 PERSISTENT_MODEL_BACKOFF_MAX_SECS,
2809 false,
2810 ),
2811 PERSISTENT_MODEL_BACKOFF_MAX_SECS,
2812 "persistent model failures should not extend past 7 days"
2813 );
2814 }
2815
2816 #[serial(cooldown_state)]
2819 #[test]
2820 fn mark_and_check_agent_degraded() {
2821 let agent = "test_degraded_mark";
2822 assert!(!is_agent_degraded(agent));
2823 mark_agent_degraded(agent);
2824 assert!(is_agent_degraded(agent));
2825 clear_agent_degraded(agent);
2826 assert!(!is_agent_degraded(agent));
2827 }
2828
2829 #[serial(cooldown_state)]
2830 #[tokio::test]
2831 async fn sync_from_kv_drops_in_memory_entry_cleared_externally() {
2832 super::reset_global_state().await;
2833 let store = test_store().await;
2834 let agent = "test_sync_external_clear";
2835 let future = chrono::Utc::now().timestamp() + 600;
2836
2837 init_cooldown_store(Arc::clone(&store)).await;
2839 set_cooldown_async(agent, future, "test").await;
2840 assert!(is_agent_in_cooldown(agent));
2841
2842 store
2846 .kv_set(&format!("{KV_PREFIX}{agent}"), "0")
2847 .await
2848 .expect("kv_set tombstone");
2849 assert!(
2850 is_agent_in_cooldown(agent),
2851 "in-memory cooldown should still be present before sync"
2852 );
2853
2854 sync_from_kv(&store).await;
2855
2856 assert!(
2857 !is_agent_in_cooldown(agent),
2858 "sync_from_kv must drop in-memory entries the KV tombstoned"
2859 );
2860 }
2861
2862 #[serial(cooldown_state)]
2863 #[tokio::test]
2864 async fn sync_from_kv_closes_github_circuit_when_row_cleared() {
2865 super::reset_global_state().await;
2866 let store = test_store().await;
2867 {
2868 *github_5xx_circuit_open_lock() = Some(chrono::Utc::now().timestamp() + 600);
2869 github_5xx_timestamps_lock().push(chrono::Utc::now().timestamp());
2870 }
2871 assert!(is_github_circuit_open());
2872
2873 store
2874 .kv_set("cooldown:github:5xx", "0")
2875 .await
2876 .expect("kv_set tombstone");
2877 sync_from_kv(&store).await;
2878
2879 assert!(!is_github_circuit_open());
2880 assert!(github_5xx_timestamps_lock().is_empty());
2881 }
2882
2883 #[serial(cooldown_state)]
2884 #[tokio::test]
2885 async fn sync_from_kv_keeps_5xx_window_when_expired_row_and_breaker_closed() {
2886 super::reset_global_state().await;
2887 let store = test_store().await;
2888 let now = chrono::Utc::now().timestamp();
2889 {
2890 github_5xx_timestamps_lock().push(now);
2891 github_5xx_timestamps_lock().push(now);
2892 }
2893
2894 store
2895 .kv_set("cooldown:github:5xx", &(now - 600).to_string())
2896 .await
2897 .expect("kv_set past");
2898 sync_from_kv(&store).await;
2899
2900 assert!(!is_github_circuit_open());
2901 assert_eq!(github_5xx_timestamps_lock().len(), 2);
2902
2903 github_5xx_timestamps_lock().clear();
2904 }
2905
2906 #[serial(cooldown_state)]
2907 #[tokio::test]
2908 async fn sync_from_kv_keeps_github_circuit_open_for_future_row() {
2909 super::reset_global_state().await;
2910 let store = test_store().await;
2911 let future = chrono::Utc::now().timestamp() + 600;
2912 *github_5xx_circuit_open_lock() = Some(future);
2913
2914 store
2915 .kv_set("cooldown:github:5xx", &future.to_string())
2916 .await
2917 .expect("kv_set future");
2918 sync_from_kv(&store).await;
2919
2920 assert!(is_github_circuit_open());
2921
2922 *github_5xx_circuit_open_lock() = None;
2923 }
2924
2925 #[serial(cooldown_state)]
2926 #[tokio::test]
2927 async fn sync_from_kv_keeps_in_memory_entry_when_kv_row_missing() {
2928 super::reset_global_state().await;
2932 let store = test_store().await;
2933 let agent = "test_sync_race_window";
2934 let future = chrono::Utc::now().timestamp() + 600;
2935
2936 set_cooldown_in_memory(agent, future, "raced");
2938 assert!(is_agent_in_cooldown(agent));
2939
2940 sync_from_kv(&store).await;
2941
2942 assert!(
2943 is_agent_in_cooldown(agent),
2944 "sync_from_kv must NOT drop entries whose KV row is missing — that is the set_cooldown_async race window"
2945 );
2946 }
2947
2948 #[serial(cooldown_state)]
2949 #[tokio::test]
2950 async fn sync_from_kv_adopts_kv_timestamp() {
2951 super::reset_global_state().await;
2954 let store = test_store().await;
2955 let agent = "test_sync_adopt_kv";
2956 let original = chrono::Utc::now().timestamp() + 600;
2957 let updated = chrono::Utc::now().timestamp() + 1200;
2958
2959 init_cooldown_store(Arc::clone(&store)).await;
2960 set_cooldown_async(agent, original, "initial").await;
2961
2962 store
2963 .kv_set(&format!("{KV_PREFIX}{agent}"), &updated.to_string())
2964 .await
2965 .expect("kv_set updated");
2966
2967 sync_from_kv(&store).await;
2968
2969 let until = cooldown_until(agent).expect("cooldown still active");
2970 assert_eq!(
2971 until, updated,
2972 "sync_from_kv must adopt KV's authoritative timestamp"
2973 );
2974 }
2975
2976 #[serial(cooldown_state)]
2977 #[tokio::test]
2978 async fn refresh_degraded_agents_marks_cooled_agent() {
2979 super::reset_global_state().await;
2980 let store = test_store().await;
2981 let agent = "test_refresh_cooled";
2982
2983 record_agent_failure_with_message(agent, "").await;
2985 assert!(is_agent_in_cooldown(agent));
2986
2987 let agents = vec![agent.to_string()];
2989 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
2990
2991 assert!(
2992 is_agent_degraded(agent),
2993 "agent in cooldown should be marked degraded"
2994 );
2995
2996 clear_agent_degraded(agent);
2998 }
2999
3000 #[serial(cooldown_state)]
3001 #[tokio::test]
3002 async fn refresh_degraded_agents_marks_no_models_agent() {
3003 super::reset_global_state().await;
3004 let store = test_store().await;
3005 let agent = "test_refresh_no_models";
3006
3007 let agents = vec![agent.to_string()];
3009 refresh_degraded_agents(&store, &agents, &|_| false, 6, 3).await;
3010
3011 assert!(
3012 is_agent_degraded(agent),
3013 "agent with no available models should be marked degraded"
3014 );
3015
3016 clear_agent_degraded(agent);
3018 }
3019
3020 #[serial(cooldown_state)]
3021 #[tokio::test]
3022 async fn refresh_degraded_agents_marks_rate_limited_agent() {
3023 super::reset_global_state().await;
3024 let store = test_store().await;
3025 let agent = "test_refresh_rate_limited";
3026
3027 for _ in 0..4 {
3029 store.record_rate_limit(agent, "rate", None).await.unwrap();
3030 }
3031
3032 let agents = vec![agent.to_string()];
3033 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3035
3036 assert!(
3037 is_agent_degraded(agent),
3038 "agent with rate limit count >= threshold should be marked degraded"
3039 );
3040
3041 clear_agent_degraded(agent);
3043 }
3044
3045 #[serial(cooldown_state)]
3046 #[tokio::test]
3047 async fn refresh_degraded_agents_clears_healthy_agent() {
3048 super::reset_global_state().await;
3049 let store = test_store().await;
3050 let agent = "test_refresh_healthy";
3051
3052 mark_agent_degraded(agent);
3054 assert!(is_agent_degraded(agent));
3055
3056 let agents = vec![agent.to_string()];
3058 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3059
3060 assert!(
3061 !is_agent_degraded(agent),
3062 "healthy agent should have degraded flag cleared"
3063 );
3064 }
3065
3066 #[serial(cooldown_state)]
3067 #[tokio::test]
3068 async fn refresh_degraded_agents_ignores_expired_cooldowns() {
3069 super::reset_global_state().await;
3070 let store = test_store().await;
3071 let agent = "test_refresh_expired_cooldown";
3072
3073 let expired_ts = chrono::Utc::now().timestamp() - 100; {
3077 let mut map = cooldowns().lock().unwrap();
3078 map.insert(
3079 agent.to_string(),
3080 CooldownEntry {
3081 cooldown_until: expired_ts,
3082 reason: "expired_test".to_string(),
3083 },
3084 );
3085 }
3086
3087 assert!(
3089 !is_agent_in_cooldown(agent),
3090 "expired cooldown should not be active"
3091 );
3092
3093 mark_agent_degraded(agent);
3095 assert!(is_agent_degraded(agent));
3096
3097 let agents = vec![agent.to_string()];
3099 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3100
3101 assert!(
3102 !is_agent_degraded(agent),
3103 "agent with only expired cooldown should NOT be marked degraded"
3104 );
3105
3106 clear_agent_degraded(agent);
3108 {
3109 let mut map = cooldowns().lock().unwrap();
3110 map.remove(agent);
3111 }
3112 }
3113
3114 #[serial(cooldown_state)]
3115 #[test]
3116 fn compute_backoff_grows_exponentially() {
3117 assert_eq!(compute_backoff(0, 300, 14400, true), 300);
3119 assert_eq!(compute_backoff(1, 300, 14400, true), 300);
3120 assert_eq!(compute_backoff(2, 300, 14400, true), 900);
3122 assert_eq!(compute_backoff(3, 300, 14400, true), 2700);
3124 assert_eq!(compute_backoff(4, 300, 14400, true), 8100);
3126 assert_eq!(compute_backoff(5, 300, 14400, true), 14400);
3128 }
3129
3130 #[serial(cooldown_state)]
3131 #[test]
3132 fn compute_backoff_extended_tier_1_applies_24h() {
3133 assert_eq!(
3139 compute_backoff(5, 300, 14400, true),
3140 14400,
3141 "first at max, not extended"
3142 );
3143 assert_eq!(
3144 compute_backoff(6, 300, 14400, true),
3145 14400,
3146 "times_at_max=1, not extended"
3147 );
3148 assert_eq!(
3149 compute_backoff(7, 300, 14400, true),
3150 86400,
3151 "times_at_max=2, extended 6x"
3152 );
3153 assert_eq!(
3154 compute_backoff(8, 300, 14400, true),
3155 86400,
3156 "times_at_max=3, still extended 6x"
3157 );
3158 }
3159
3160 #[serial(cooldown_state)]
3161 #[test]
3162 fn compute_backoff_extended_tier_2_applies_48h() {
3163 assert_eq!(
3166 compute_backoff(9, 300, 14400, true),
3167 86400,
3168 "times_at_max=4, still tier 1"
3169 );
3170 assert_eq!(
3171 compute_backoff(10, 300, 14400, true),
3172 172_800,
3173 "times_at_max=5, extended 12x"
3174 );
3175 assert_eq!(
3176 compute_backoff(15, 300, 14400, true),
3177 172_800,
3178 "times_at_max=10, still tier 2"
3179 );
3180 assert_eq!(
3181 compute_backoff(20, 300, 14400, true),
3182 172_800,
3183 "times_at_max=15, still tier 2"
3184 );
3185 }
3186
3187 #[serial(cooldown_state)]
3188 #[test]
3189 fn compute_backoff_extended_tier_with_different_base_max() {
3190 assert_eq!(compute_backoff(5, 60, 3600, true), 3600, "first at max");
3197 assert_eq!(compute_backoff(6, 60, 3600, true), 3600, "times_at_max=1");
3198 assert_eq!(
3199 compute_backoff(7, 60, 3600, true),
3200 21600,
3201 "times_at_max=2, extended 6x"
3202 );
3203 }
3204
3205 #[serial(cooldown_state)]
3206 #[test]
3207 fn compute_backoff_extended_tier_persistent_fails_example() {
3208 assert_eq!(
3212 compute_backoff(21, 300, 14400, true),
3213 172_800,
3214 "21 failures should get 48h extended cooldown"
3215 );
3216
3217 assert_eq!(
3219 compute_backoff(36, 300, 14400, true),
3220 172_800,
3221 "36 failures should also get 48h extended cooldown"
3222 );
3223 }
3224
3225 #[serial(cooldown_state)]
3226 #[test]
3227 fn compute_backoff_extended_tier_zero_base() {
3228 assert_eq!(compute_backoff(1, 0, 14400, true), 0);
3230 assert_eq!(compute_backoff(5, 0, 14400, true), 0);
3231 assert_eq!(compute_backoff(10, 0, 14400, true), 0);
3232 }
3233
3234 #[serial(cooldown_state)]
3235 #[test]
3236 fn detect_credit_exhaustion_billing_cycle() {
3237 assert_eq!(
3238 detect_credit_exhaustion("billing cycle"),
3239 Some(CreditExhaustionReason::BillingCycleExhausted)
3240 );
3241 assert_eq!(
3242 detect_credit_exhaustion("quota refreshed next cycle"),
3243 Some(CreditExhaustionReason::BillingCycleExhausted)
3244 );
3245 assert_eq!(
3246 detect_credit_exhaustion(
3247 "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle."
3248 ),
3249 Some(CreditExhaustionReason::BillingCycleExhausted)
3250 );
3251 assert_eq!(
3252 detect_credit_exhaustion("monthly quota exceeded"),
3253 Some(CreditExhaustionReason::BillingCycleExhausted)
3254 );
3255 assert_eq!(
3256 detect_credit_exhaustion(
3257 "Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-04-23 18:33:48"
3258 ),
3259 Some(CreditExhaustionReason::BillingCycleExhausted)
3260 );
3261 }
3262
3263 #[serial(cooldown_state)]
3264 #[test]
3265 fn detect_credit_exhaustion_billing_cycle_minimax() {
3266 assert_eq!(
3267 detect_credit_exhaustion(
3268 "minimax rate limit: API Error: Request rejected (429) · Token Plan usage limit reached: Upgrade your Token Plan or purchase Credits for more usage."
3269 ),
3270 Some(CreditExhaustionReason::BillingCycleExhausted)
3271 );
3272 assert_eq!(
3273 detect_credit_exhaustion("Upgrade your Token Plan"),
3274 Some(CreditExhaustionReason::BillingCycleExhausted)
3275 );
3276 assert_eq!(
3277 detect_credit_exhaustion("Token Plan usage limit reached"),
3278 Some(CreditExhaustionReason::BillingCycleExhausted)
3279 );
3280 assert_eq!(
3281 detect_credit_exhaustion("purchase Credits for more usage"),
3282 Some(CreditExhaustionReason::BillingCycleExhausted)
3283 );
3284 assert_eq!(
3285 detect_credit_exhaustion(
3286 "You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro), visit https://chatgpt.com/codex/settings/usage to purchase more credits"
3287 ),
3288 Some(CreditExhaustionReason::BillingCycleExhausted)
3289 );
3290 }
3291
3292 #[serial(cooldown_state)]
3293 #[test]
3294 fn detect_credit_exhaustion_outcome_kimi_billing_cycle() {
3295 let msg = "Failed to authenticate. API Error: 403 You've reached your usage limit for \
3300 this billing cycle. Your quota will be refreshed soon.";
3301 assert_eq!(
3302 detect_credit_exhaustion_outcome(msg),
3303 Some("billing_cycle_exhausted")
3304 );
3305 }
3306
3307 #[serial(cooldown_state)]
3308 #[test]
3309 fn detect_credit_exhaustion_outcome_maps_all_reasons() {
3310 assert_eq!(
3311 detect_credit_exhaustion_outcome("out of credits"),
3312 Some("credit_exhausted")
3313 );
3314 assert_eq!(
3315 detect_credit_exhaustion_outcome("organization has been disabled"),
3316 Some("org_level_disabled")
3317 );
3318 assert_eq!(
3319 detect_credit_exhaustion_outcome("monthly quota exceeded"),
3320 Some("billing_cycle_exhausted")
3321 );
3322 }
3323
3324 #[test]
3325 fn detect_credit_exhaustion_outcome_returns_none_for_unrelated_errors() {
3326 assert_eq!(
3327 detect_credit_exhaustion_outcome("429 Too Many Requests"),
3328 None
3329 );
3330 assert_eq!(detect_credit_exhaustion_outcome("connection reset"), None);
3331 }
3332
3333 #[serial(cooldown_state)]
3334 #[test]
3335 fn detect_credit_exhaustion_overage_takes_priority_over_billing_cycle() {
3336 let msg = "overageDisabledReason: org_level_disabled for billing cycle";
3339 assert_eq!(
3340 detect_credit_exhaustion(msg),
3341 Some(CreditExhaustionReason::OrgLevelDisabled)
3342 );
3343 }
3344
3345 #[serial(cooldown_state)]
3346 #[tokio::test]
3347 async fn refresh_degraded_agents_below_threshold_not_degraded() {
3348 super::reset_global_state().await;
3349 let store = test_store().await;
3350 let agent = "test_refresh_below_threshold";
3351
3352 for _ in 0..2 {
3354 store.record_rate_limit(agent, "rate", None).await.unwrap();
3355 }
3356
3357 let agents = vec![agent.to_string()];
3358 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3360
3361 assert!(
3362 !is_agent_degraded(agent),
3363 "agent with rate limit count < threshold should not be degraded"
3364 );
3365 }
3366
3367 #[serial(cooldown_state)]
3368 #[tokio::test]
3369 async fn refresh_degraded_agents_ignores_error_limit_type() {
3370 super::reset_global_state().await;
3371 let store = test_store().await;
3372 let agent = "test_refresh_error_rows";
3373
3374 for _ in 0..3 {
3375 store.record_rate_limit(agent, "error", None).await.unwrap();
3376 }
3377
3378 let agents = vec![agent.to_string()];
3379 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3380
3381 assert!(
3382 !is_agent_degraded(agent),
3383 "three error rows in 6 hours must not mark an agent degraded"
3384 );
3385 }
3386
3387 #[serial(cooldown_state)]
3388 #[tokio::test]
3389 async fn refresh_degraded_agents_marks_three_rate_rows() {
3390 super::reset_global_state().await;
3391 let store = test_store().await;
3392 let agent = "test_refresh_three_rate_rows";
3393
3394 for _ in 0..3 {
3395 store.record_rate_limit(agent, "rate", None).await.unwrap();
3396 }
3397
3398 let agents = vec![agent.to_string()];
3399 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3400
3401 assert!(
3402 is_agent_degraded(agent),
3403 "three rate rows in 6 hours must mark an agent degraded"
3404 );
3405 assert_eq!(
3406 agent_degraded_reason(agent).as_deref(),
3407 Some("rate limit threshold exceeded")
3408 );
3409 let listed = list_degraded_agents();
3410 assert!(
3411 listed
3412 .iter()
3413 .any(|(a, r)| a == agent && r == "rate limit threshold exceeded"),
3414 "orch cooldown list must show the degraded reason, got {listed:?}"
3415 );
3416
3417 clear_agent_degraded(agent);
3418 }
3419
3420 #[serial(cooldown_state)]
3421 #[tokio::test]
3422 async fn refresh_degraded_agents_logs_reason_change() {
3423 super::reset_global_state().await;
3424 let store = test_store().await;
3425 let agent = "test_refresh_reason_change";
3426 *cooldown_store().lock().await = Some(Arc::clone(&store));
3427
3428 set_agent_cooldown(agent, 600).await;
3429 let agents = vec![agent.to_string()];
3430 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3431 assert_eq!(
3432 agent_degraded_reason(agent).as_deref(),
3433 Some("agent in cooldown")
3434 );
3435
3436 clear_cooldown(agent, &store).await;
3437 let after_clear = sqlite_datetime_offset(60);
3438 for _ in 0..3 {
3439 store
3440 .record_rate_limit_at(agent, "rate", None, &after_clear)
3441 .await
3442 .unwrap();
3443 }
3444 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3445 assert_eq!(
3446 agent_degraded_reason(agent).as_deref(),
3447 Some("rate limit threshold exceeded"),
3448 "degraded reason must update when the hold changes"
3449 );
3450
3451 clear_agent_degraded(agent);
3452 }
3453
3454 #[serial(cooldown_state)]
3455 #[tokio::test]
3456 async fn clear_un_degrades_on_next_health_check() {
3457 super::reset_global_state().await;
3458 let store = test_store().await;
3459 let agent = "test_clear_un_degrades";
3460 let before = sqlite_datetime_offset(-30);
3461 let after = sqlite_datetime_offset(60);
3462
3463 for _ in 0..3 {
3464 store
3465 .record_rate_limit_at(agent, "rate", None, &before)
3466 .await
3467 .unwrap();
3468 }
3469
3470 let agents = vec![agent.to_string()];
3471 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3472 assert!(
3473 is_agent_degraded(agent),
3474 "three rate rows must mark an agent degraded"
3475 );
3476
3477 clear_cooldown(agent, &store).await;
3478 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3479 assert!(
3480 !is_agent_degraded(agent),
3481 "clear must un-degrade on the next health check"
3482 );
3483
3484 store
3485 .record_rate_limit_at(agent, "rate", None, &after)
3486 .await
3487 .unwrap();
3488 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3489 assert!(
3490 !is_agent_degraded(agent),
3491 "one rate row after the mark must not degrade again"
3492 );
3493
3494 for _ in 0..2 {
3495 store
3496 .record_rate_limit_at(agent, "rate", None, &after)
3497 .await
3498 .unwrap();
3499 }
3500 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3501 assert!(
3502 is_agent_degraded(agent),
3503 "three rate rows after the mark must degrade again"
3504 );
3505
3506 clear_agent_degraded(agent);
3507 }
3508
3509 #[serial(cooldown_state)]
3510 #[tokio::test]
3511 async fn clear_all_un_degrades_rate_limited_agent() {
3512 super::reset_global_state().await;
3513 let store = test_store().await;
3514 let agent = "test_clear_all_un_degrades";
3515 let before = sqlite_datetime_offset(-30);
3516
3517 for _ in 0..3 {
3518 store
3519 .record_rate_limit_at(agent, "rate", None, &before)
3520 .await
3521 .unwrap();
3522 }
3523
3524 let agents = vec![agent.to_string()];
3525 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3526 assert!(is_agent_degraded(agent));
3527
3528 clear_cooldown("*", &store).await;
3529 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3530 assert!(
3531 !is_agent_degraded(agent),
3532 "clear --all must un-degrade an agent held only by rate rows"
3533 );
3534 }
3535
3536 #[serial(cooldown_state)]
3539 #[tokio::test]
3540 async fn repeated_agent_failures_escalate_backoff() {
3541 super::reset_global_state().await;
3542 let store = test_store().await;
3543 *cooldown_store().lock().await = Some(Arc::clone(&store));
3544
3545 let agent = "test_escalation_agent";
3546
3547 record_agent_failure_with_message(agent, "").await;
3549 let remaining_1 = {
3550 let map = cooldowns().lock().unwrap();
3551 let entry = map.get(agent).expect("should have cooldown");
3552 entry.cooldown_until - chrono::Utc::now().timestamp()
3553 };
3554 assert!(
3555 (BACKOFF_BASE_SECS - 5..=BACKOFF_BASE_SECS + 5).contains(&remaining_1),
3556 "first failure should be ~5 min, got {remaining_1}s"
3557 );
3558
3559 {
3561 let mut map = cooldowns().lock().unwrap();
3562 map.remove(agent);
3563 }
3564
3565 record_agent_failure_with_message(agent, "").await;
3567 let remaining_2 = {
3568 let map = cooldowns().lock().unwrap();
3569 let entry = map.get(agent).expect("should have cooldown");
3570 entry.cooldown_until - chrono::Utc::now().timestamp()
3571 };
3572 assert!(
3573 remaining_2 >= BACKOFF_BASE_SECS * 3 - 5,
3574 "second failure should be ~15 min, got {remaining_2}s"
3575 );
3576 }
3577
3578 #[serial(cooldown_state)]
3579 #[tokio::test]
3580 async fn vendor_past_retry_falls_back_to_backoff() {
3581 super::reset_global_state().await;
3582 let store = test_store().await;
3586 *cooldown_store().lock().await = Some(Arc::clone(&store));
3587
3588 let agent = "test_vendor_past_retry";
3589
3590 {
3592 let mut map = cooldowns().lock().unwrap();
3593 map.remove(agent);
3594 }
3595
3596 let past_msg = "You've hit the limit; try again at Jan 01, 2000 01:00 AM.";
3598
3599 record_agent_failure_with_message(agent, past_msg).await;
3600
3601 assert!(
3605 is_agent_in_cooldown(agent),
3606 "agent should be in cooldown after failure"
3607 );
3608
3609 let remaining = {
3610 let map = cooldowns().lock().unwrap();
3611 let entry = map.get(agent).expect("cooldown entry should exist");
3612 entry.cooldown_until - chrono::Utc::now().timestamp()
3613 };
3614
3615 assert!(
3616 remaining >= BACKOFF_BASE_SECS - 5,
3617 "expected backoff of ~{BACKOFF_BASE_SECS}s when vendor timestamp is past, got {remaining}s"
3618 );
3619
3620 let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3622 let stored = store
3623 .kv_get(&kv_key)
3624 .await
3625 .unwrap()
3626 .expect("failure count should be stored");
3627 assert_eq!(stored, "1");
3628 }
3629
3630 fn window(status: &str, resets_at: Option<i64>) -> RateLimitWindow {
3631 RateLimitWindow {
3632 status: status.to_string(),
3633 window_type: "five_hour".to_string(),
3634 utilization: Some(1.0),
3635 resets_at,
3636 }
3637 }
3638
3639 #[serial(cooldown_state)]
3640 #[tokio::test]
3641 async fn rejected_window_cools_agent_until_reset() {
3642 super::reset_global_state().await;
3643 let store = test_store().await;
3644 *cooldown_store().lock().await = Some(Arc::clone(&store));
3645
3646 let agent = "test_window_rejected_agent";
3647 let reset = chrono::Utc::now().timestamp() + 7200;
3648 record_rate_limit_window(agent, &window("rejected", Some(reset))).await;
3649
3650 assert_eq!(cooldown_until(agent), Some(reset));
3651 let stored = list_rate_limit_windows(&store).await;
3652 assert_eq!(
3653 stored,
3654 vec![(agent.to_string(), window("rejected", Some(reset)))]
3655 );
3656 }
3657
3658 #[serial(cooldown_state)]
3659 #[tokio::test]
3660 async fn allowed_window_changes_no_cooldown() {
3661 super::reset_global_state().await;
3662 let store = test_store().await;
3663 *cooldown_store().lock().await = Some(Arc::clone(&store));
3664
3665 let agent = "test_window_allowed_agent";
3666 let reset = chrono::Utc::now().timestamp() + 7200;
3667 record_rate_limit_window(agent, &window("allowed", Some(reset))).await;
3668 record_rate_limit_window(agent, &window("allowed_warning", Some(reset))).await;
3669
3670 assert_eq!(cooldown_until(agent), None);
3671 assert_eq!(list_rate_limit_windows(&store).await.len(), 1);
3672 }
3673
3674 #[serial(cooldown_state)]
3675 #[tokio::test]
3676 async fn record_agent_success_resets_backoff() {
3677 super::reset_global_state().await;
3678 let store = test_store().await;
3679 *cooldown_store().lock().await = Some(Arc::clone(&store));
3680
3681 let agent = "test_success_reset_agent";
3682 let model = "test_model";
3683
3684 for _ in 0..3 {
3686 let _ = read_and_increment_failure_count(Some(&store), agent).await;
3687 }
3688 let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3690 let count: u32 = store
3691 .kv_get(&kv_key)
3692 .await
3693 .unwrap()
3694 .unwrap()
3695 .parse()
3696 .unwrap();
3697 assert_eq!(count, 3);
3698
3699 record_agent_success(agent, model).await;
3701
3702 let count_after: u32 = store
3703 .kv_get(&kv_key)
3704 .await
3705 .unwrap()
3706 .unwrap()
3707 .parse()
3708 .unwrap();
3709 assert_eq!(count_after, 0, "success should reset failure count to 0");
3710
3711 let model_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}:{model}");
3713 let model_count: u32 = store
3714 .kv_get(&model_kv_key)
3715 .await
3716 .unwrap()
3717 .unwrap()
3718 .parse()
3719 .unwrap();
3720 assert_eq!(
3721 model_count, 0,
3722 "success should reset model failure count to 0"
3723 );
3724 }
3725
3726 #[serial(cooldown_state)]
3727 #[tokio::test]
3728 async fn record_agent_success_without_model_skips_empty_model_key() {
3729 super::reset_global_state().await;
3730 let store = test_store().await;
3731 *cooldown_store().lock().await = Some(Arc::clone(&store));
3732
3733 let agent = "test_success_without_model";
3734
3735 record_agent_success(agent, "").await;
3736
3737 let agent_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3738 let agent_count = store
3739 .kv_get(&agent_kv_key)
3740 .await
3741 .unwrap()
3742 .expect("agent failure count should be stored");
3743 assert_eq!(agent_count, "0");
3744
3745 let empty_model_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}:");
3746 let empty_model_count = store.kv_get(&empty_model_kv_key).await.unwrap();
3747 assert!(
3748 empty_model_count.is_none(),
3749 "success without model should not create an empty model failure count key"
3750 );
3751 }
3752
3753 #[serial(cooldown_state)]
3754 #[tokio::test]
3755 async fn clear_cooldown_resets_failure_counts() {
3756 super::reset_global_state().await;
3757 let store = test_store().await;
3758 *cooldown_store().lock().await = Some(Arc::clone(&store));
3759
3760 let agent = "test_clear_fc_agent";
3761
3762 for _ in 0..5 {
3764 let _ = read_and_increment_failure_count(Some(&store), agent).await;
3765 }
3766 set_agent_cooldown(agent, 3600).await;
3768
3769 let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3771 let count: u32 = store
3772 .kv_get(&kv_key)
3773 .await
3774 .unwrap()
3775 .unwrap()
3776 .parse()
3777 .unwrap();
3778 assert_eq!(count, 5);
3779
3780 clear_cooldown(agent, &store).await;
3782
3783 let count_after: u32 = store
3785 .kv_get(&kv_key)
3786 .await
3787 .unwrap()
3788 .unwrap()
3789 .parse()
3790 .unwrap();
3791 assert_eq!(
3792 count_after, 0,
3793 "clear_cooldown should reset failure count to 0"
3794 );
3795
3796 assert!(
3798 !is_agent_in_cooldown(agent),
3799 "agent should not be in cooldown after clear"
3800 );
3801 }
3802
3803 #[serial(cooldown_state)]
3804 #[tokio::test]
3805 async fn clear_all_cooldowns_resets_all_failure_counts() {
3806 super::reset_global_state().await;
3807 let store = test_store().await;
3808 *cooldown_store().lock().await = Some(Arc::clone(&store));
3809
3810 let agents = ["test_clear_all_a", "test_clear_all_b"];
3811 for agent in &agents {
3812 let _ = read_and_increment_failure_count(Some(&store), agent).await;
3813 set_agent_cooldown(agent, 3600).await;
3814 }
3815
3816 clear_cooldown("*", &store).await;
3817
3818 for agent in &agents {
3819 assert!(!is_agent_in_cooldown(agent));
3820 let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3821 let count: u32 = store
3822 .kv_get(&kv_key)
3823 .await
3824 .unwrap()
3825 .unwrap()
3826 .parse()
3827 .unwrap();
3828 assert_eq!(
3829 count, 0,
3830 "clear --all should reset failure count for {agent}"
3831 );
3832 }
3833 }
3834
3835 #[serial(cooldown_state)]
3836 #[test]
3837 fn compute_backoff_saturates_on_large_counts() {
3838 assert_eq!(
3840 compute_backoff(100, BACKOFF_BASE_SECS, BACKOFF_MAX_SECS, true),
3841 BACKOFF_MAX_SECS * 12
3842 );
3843 assert_eq!(
3844 compute_backoff(u64::from(u32::MAX), 300, 14400, true),
3845 172_800
3846 );
3847 assert_eq!(compute_backoff(u64::MAX, 300, 14400, true), 172_800);
3848 }
3849
3850 #[serial(cooldown_state)]
3851 #[test]
3852 fn compute_backoff_zero_base_returns_zero() {
3853 assert_eq!(compute_backoff(1, 0, 14400, true), 0);
3854 assert_eq!(compute_backoff(5, 0, 14400, true), 0);
3855 }
3856
3857 #[serial(cooldown_state)]
3860 #[tokio::test]
3861 async fn github_circuit_stays_closed_below_threshold() {
3862 super::reset_global_state().await;
3863 {
3865 let mut open = github_5xx_circuit_open().lock().unwrap();
3866 *open = None;
3867 let mut ts = github_5xx_timestamps().lock().unwrap();
3868 ts.clear();
3869 let mut map = cooldown_lock();
3873 map.remove("github:5xx");
3874 }
3875
3876 for _ in 0..GITHUB_5XX_THRESHOLD - 1 {
3878 record_github_5xx().await;
3879 }
3880
3881 assert!(
3882 !is_github_circuit_open(),
3883 "circuit should stay closed below threshold"
3884 );
3885 assert_eq!(github_circuit_remaining_secs(), 0);
3886 }
3887
3888 #[serial(cooldown_state)]
3889 #[tokio::test]
3890 async fn github_circuit_trips_at_threshold() {
3891 super::reset_global_state().await;
3892 {
3894 let mut open = github_5xx_circuit_open_lock();
3895 *open = None;
3896 let mut ts = github_5xx_timestamps_lock();
3897 ts.clear();
3898 let mut map = cooldown_lock();
3899 map.remove("github:5xx");
3900 }
3901
3902 for _ in 0..GITHUB_5XX_THRESHOLD {
3904 record_github_5xx().await;
3905 }
3906
3907 assert!(is_github_circuit_open(), "circuit should trip at threshold");
3908 assert!(
3909 github_circuit_remaining_secs() > 0,
3910 "remaining should be > 0 when open"
3911 );
3912 assert!(
3913 github_circuit_remaining_secs() <= GITHUB_5XX_COOLDOWN_SECS as u64,
3914 "remaining should not exceed cooldown duration"
3915 );
3916
3917 {
3919 let mut open = github_5xx_circuit_open().lock().unwrap();
3920 *open = None;
3921 }
3922 }
3923
3924 #[serial(cooldown_state)]
3925 #[tokio::test]
3926 async fn github_circuit_cannot_double_trip() {
3927 super::reset_global_state().await;
3928 {
3930 let mut open = github_5xx_circuit_open_lock();
3931 *open = None;
3932 let mut ts = github_5xx_timestamps_lock();
3933 ts.clear();
3934 let mut map = cooldown_lock();
3935 map.remove("github:5xx");
3936 }
3937
3938 for _ in 0..GITHUB_5XX_THRESHOLD {
3940 record_github_5xx().await;
3941 }
3942 assert!(is_github_circuit_open());
3943
3944 let first_remaining = github_circuit_remaining_secs();
3945
3946 for _ in 0..GITHUB_5XX_THRESHOLD {
3948 record_github_5xx().await;
3949 }
3950
3951 let second_remaining = github_circuit_remaining_secs();
3952 assert!(
3953 second_remaining <= first_remaining,
3954 "additional errors should not extend cooldown"
3955 );
3956
3957 {
3959 let mut open = github_5xx_circuit_open().lock().unwrap();
3960 *open = None;
3961 }
3962 }
3963
3964 #[serial(cooldown_state)]
3965 #[tokio::test]
3966 async fn github_circuit_recovers_after_cooldown() {
3967 super::reset_global_state().await;
3968 {
3970 let mut open = github_5xx_circuit_open_lock();
3971 *open = None;
3972 let mut ts = github_5xx_timestamps_lock();
3973 ts.clear();
3974 let mut map = cooldown_lock();
3975 map.remove("github:5xx");
3976 }
3977
3978 {
3980 let mut open = github_5xx_circuit_open().lock().unwrap();
3981 *open = Some(chrono::Utc::now().timestamp() - 1);
3982 }
3983
3984 assert!(
3986 !is_github_circuit_open(),
3987 "circuit should auto-recover after cooldown"
3988 );
3989 }
3990
3991 #[serial(cooldown_state)]
3992 #[tokio::test]
3993 async fn github_circuit_sliding_window_ages_out() {
3994 super::reset_global_state().await;
3995 {
3997 let mut open = github_5xx_circuit_open_lock();
3998 *open = None;
3999 let mut ts = github_5xx_timestamps_lock();
4000 ts.clear();
4001 let mut map = cooldown_lock();
4002 map.remove("github:5xx");
4003 }
4004
4005 {
4007 let mut ts = github_5xx_timestamps().lock().unwrap();
4008 let old = chrono::Utc::now().timestamp() - GITHUB_5XX_WINDOW_SECS - 10;
4009 for _ in 0..GITHUB_5XX_THRESHOLD {
4010 ts.push(old);
4011 }
4012 }
4013
4014 record_github_5xx().await;
4016
4017 assert!(
4018 !is_github_circuit_open(),
4019 "old errors outside window should not trip circuit"
4020 );
4021 }
4022}