1use chrono::TimeZone;
19use regex::Regex;
20use std::collections::HashMap;
21use std::collections::HashSet;
22use std::sync::LazyLock;
23use std::sync::MutexGuard;
24use std::sync::{Arc, Mutex, OnceLock};
25use tokio::sync;
26
27use crate::engine::router::config;
28#[cfg(test)]
29use crate::engine::runner::agents::opencode;
30use crate::store::TaskStore;
31
32pub const AGENT_COOLDOWN_SECS: i64 = 30 * 60;
38
39pub const BACKOFF_BASE_SECS: i64 = 5 * 60;
41
42pub const BACKOFF_MAX_SECS: i64 = 4 * 60 * 60;
44
45pub const PERSISTENT_MODEL_BACKOFF_BASE_SECS: i64 = 4 * 60 * 60;
47
48pub const PERSISTENT_MODEL_BACKOFF_MAX_SECS: i64 = 7 * 24 * 60 * 60;
50
51pub const TRANSIENT_MODEL_BACKOFF_BASE_SECS: i64 = 5 * 60;
53
54pub const TRANSIENT_MODEL_BACKOFF_MAX_SECS: i64 = 15 * 60;
56
57pub const CREDIT_BACKOFF_BASE_SECS: i64 = 60 * 60;
59
60pub const ORG_BACKOFF_BASE_SECS: i64 = 2 * 60 * 60;
62
63pub const CREDIT_BACKOFF_MAX_SECS: i64 = 8 * 60 * 60;
66
67pub const BILLING_CYCLE_COOLDOWN_SECS: i64 = 24 * 60 * 60;
75
76pub const BILLING_CYCLE_MAX_SECS: i64 = 7 * 24 * 60 * 60;
82
83#[derive(Debug, Clone, Copy, PartialEq, Eq)]
85pub enum CreditExhaustionReason {
86 OutOfCredits,
88 OrgLevelDisabled,
90 BillingCycleExhausted,
92}
93
94impl CreditExhaustionReason {
95 pub fn outcome_str(self) -> &'static str {
99 match self {
100 CreditExhaustionReason::OutOfCredits => "credit_exhausted",
101 CreditExhaustionReason::OrgLevelDisabled => "org_level_disabled",
102 CreditExhaustionReason::BillingCycleExhausted => "billing_cycle_exhausted",
103 }
104 }
105}
106
107pub const SILENCE_AGENT_COOLDOWN_SECS: u64 = 120;
114
115pub const SILENCE_COUNT_THRESHOLD: usize = 5;
117
118pub const SILENCE_COUNT_WINDOW_SECS: i64 = 24 * 60 * 60;
120
121pub const SILENCE_EXTENDED_COOLDOWN_SECS: u64 = 4 * 60 * 60;
123
124const KV_PREFIX: &str = "cooldown:";
126
127const SILENCE_COUNT_PREFIX: &str = "silence_count:";
129
130const FAILURE_COUNT_PREFIX: &str = "failure_count:";
132
133const CREDIT_FAILURE_COUNT_PREFIX: &str = "credit_failure_count:";
136
137const PERSISTENT_FAILURE_COUNT_PREFIX: &str = "persistent_failure_count:";
141
142const RATE_LIMITS_CLEARED_AT_PREFIX: &str = "rate_limits_cleared_at:";
144
145fn agent_from_key(key: &str) -> &str {
147 key.split_once(':').map_or(key, |(agent, _)| agent)
148}
149
150struct CooldownEntry {
151 cooldown_until: i64,
153 reason: String,
154}
155
156#[derive(Debug)]
157pub struct SilenceCountResult {
158 pub count: usize,
159 pub extended_cooldown_applied: bool,
160}
161
162fn cooldowns() -> &'static Mutex<HashMap<String, CooldownEntry>> {
171 static COOLDOWNS: OnceLock<Mutex<HashMap<String, CooldownEntry>>> = OnceLock::new();
172 COOLDOWNS.get_or_init(|| Mutex::new(HashMap::new()))
173}
174
175fn cooldown_store() -> &'static sync::Mutex<Option<Arc<TaskStore>>> {
181 static STORE: OnceLock<sync::Mutex<Option<Arc<TaskStore>>>> = OnceLock::new();
182 STORE.get_or_init(|| sync::Mutex::new(None))
183}
184
185fn parse_cooldown_value(value: &str) -> Option<(i64, String)> {
193 if let Ok(serde_json::Value::Object(map)) = serde_json::from_str(value) {
194 let until = map.get("until")?.as_i64()?;
195 let reason = map
196 .get("reason")
197 .and_then(|r| r.as_str())
198 .unwrap_or("persisted")
199 .to_string();
200 return Some((until, reason));
201 }
202 value
203 .parse::<i64>()
204 .ok()
205 .map(|until| (until, "persisted".to_string()))
206}
207
208pub async fn init_cooldown_store(store: Arc<TaskStore>) {
209 match store.kv_list_prefix(KV_PREFIX).await {
210 Ok(rows) => {
211 let now = chrono::Utc::now().timestamp();
212 let mut map = cooldown_lock();
213 let mut loaded = 0;
214 for (key, value) in rows {
215 let cooldown_key = key.trim_start_matches(KV_PREFIX);
216 if let Some((cooldown_until, reason)) = parse_cooldown_value(&value)
217 && now < cooldown_until
218 {
219 map.insert(
220 cooldown_key.to_string(),
221 CooldownEntry {
222 cooldown_until,
223 reason,
224 },
225 );
226 loaded += 1;
227 }
228 }
231 if loaded > 0 {
232 tracing::info!(loaded, "loaded persisted cooldowns from KV store");
233 }
234 }
235 Err(e) => {
236 tracing::warn!(err = %e, "failed to load cooldowns from KV store");
237 }
238 }
239
240 if let Ok(Some(raw)) = store.kv_get("cooldown:github:5xx").await
242 && let Ok(cooldown_until) = raw.parse::<i64>()
243 {
244 let now = chrono::Utc::now().timestamp();
245 if now < cooldown_until {
246 let mut open = github_5xx_circuit_open_lock();
247 *open = Some(cooldown_until);
248 let remaining = cooldown_until - now;
249 tracing::warn!(
250 remaining_secs = remaining,
251 "GitHub 5xx circuit breaker restored from KV (still open)"
252 );
253 }
254 }
255
256 *cooldown_store().lock().await = Some(store);
257}
258
259pub async fn sync_from_kv(store: &Arc<TaskStore>) {
276 let rows = match store.kv_list_prefix(KV_PREFIX).await {
277 Ok(rows) => rows,
278 Err(e) => {
279 tracing::warn!(err = %e, "cooldown sync_from_kv: KV list failed");
280 return;
281 }
282 };
283 let now = chrono::Utc::now().timestamp();
284
285 let mut explicit_kv: HashMap<String, Option<(i64, String)>> =
288 HashMap::with_capacity(rows.len());
289 for (kv_key, value) in &rows {
290 let cooldown_key = kv_key.trim_start_matches(KV_PREFIX);
291 if let Some((ts, reason)) = parse_cooldown_value(value) {
292 let normalized = if ts > now { Some((ts, reason)) } else { None };
293 explicit_kv.insert(cooldown_key.to_string(), normalized);
294 }
295 }
296
297 match explicit_kv.get("github:5xx") {
299 Some(None) => {
300 let mut open = github_5xx_circuit_open_lock();
301 if open.take().is_some() {
302 tracing::info!("GitHub 5xx circuit breaker closed by KV clear");
303 github_5xx_timestamps_lock().clear();
304 }
305 }
306 Some(Some((ts, _))) => {
307 *github_5xx_circuit_open_lock() = Some(*ts);
308 }
309 None => {}
310 }
311
312 let mut map = cooldown_lock();
313 let mut dropped = 0usize;
314 let mut updated = 0usize;
315
316 map.retain(|key, _| {
318 if matches!(explicit_kv.get(key), Some(None)) {
319 dropped += 1;
320 false
321 } else {
322 true
323 }
324 });
325
326 for (key, normalized) in explicit_kv {
328 if let Some((ts, reason)) = normalized {
329 if let Some(entry) = map.get_mut(&key) {
330 if entry.cooldown_until != ts {
331 entry.cooldown_until = ts;
332 updated += 1;
333 }
334 } else {
335 map.insert(
336 key,
337 CooldownEntry {
338 cooldown_until: ts,
339 reason,
340 },
341 );
342 updated += 1;
343 }
344 }
345 }
346
347 if dropped > 0 || updated > 0 {
348 tracing::info!(
349 dropped,
350 updated,
351 "cooldown sync_from_kv: reconciled in-memory map with KV"
352 );
353 }
354}
355
356const EXTENDED_TIER_1_THRESHOLD: u64 = 2;
360
361const EXTENDED_TIER_1_MULTIPLIER: i64 = 6;
363
364const EXTENDED_TIER_2_THRESHOLD: u64 = 5;
368
369const EXTENDED_TIER_2_MULTIPLIER: i64 = 12;
371
372pub fn compute_backoff(count: u64, base: i64, max: i64, allow_extended: bool) -> i64 {
391 if count <= 1 {
392 return base;
393 }
394 let exp = u32::try_from(count.saturating_sub(1)).unwrap_or(u32::MAX);
395 let factor = 3_i64.saturating_pow(exp);
396 let standard = base.saturating_mul(factor).min(max);
397
398 let first_max = if base <= 0 {
407 0
408 } else {
409 let mut accumulated = base;
411 let mut first_max_count = 1u64;
412 while accumulated < max && first_max_count < count {
413 accumulated = accumulated.saturating_mul(3);
414 first_max_count += 1;
415 }
416 first_max_count
417 };
418
419 let times_at_max = count.saturating_sub(first_max);
421
422 if !allow_extended {
423 return standard;
424 }
425
426 if standard == max && times_at_max >= EXTENDED_TIER_2_THRESHOLD {
427 max.saturating_mul(EXTENDED_TIER_2_MULTIPLIER)
428 } else if standard == max && times_at_max >= EXTENDED_TIER_1_THRESHOLD {
429 max.saturating_mul(EXTENDED_TIER_1_MULTIPLIER)
430 } else {
431 standard
432 }
433}
434
435async fn read_and_increment_failure_count(store_opt: Option<&Arc<TaskStore>>, key: &str) -> u64 {
444 read_and_increment_failure_count_with_prefix(store_opt, FAILURE_COUNT_PREFIX, key).await
445}
446
447async fn read_and_increment_failure_count_with_prefix(
452 store_opt: Option<&Arc<TaskStore>>,
453 prefix: &str,
454 key: &str,
455) -> u64 {
456 let kv_key = format!("{prefix}{key}");
457 if let Some(store) = store_opt {
458 match store.kv_increment(&kv_key).await {
459 Ok(n) => n,
460 Err(e) => {
461 tracing::warn!(key = %kv_key, err = %e, "failed to increment failure count — treating as high count for safe backoff");
462 u64::MAX
463 }
464 }
465 } else {
466 1
467 }
468}
469
470struct ModelBackoff {
472 prefix: &'static str,
473 base: i64,
474 max: i64,
475 allow_extended: bool,
476 reason: &'static str,
477}
478
479async fn record_model_backoff(agent_name: &str, model: &str, schedule: &ModelBackoff) {
481 let key = format!("{agent_name}:{model}");
482 let store_opt = cooldown_store().lock().await.clone();
483 let count =
484 read_and_increment_failure_count_with_prefix(store_opt.as_ref(), schedule.prefix, &key)
485 .await;
486 let cooldown_secs =
487 compute_backoff(count, schedule.base, schedule.max, schedule.allow_extended);
488 let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
489 set_cooldown_async(&key, cooldown_until, schedule.reason).await;
490}
491
492pub async fn record_agent_success(agent_name: &str, model: &str) {
498 let store_opt = cooldown_store().lock().await.clone();
499 if let Some(store) = store_opt {
500 let agent_key = format!("{FAILURE_COUNT_PREFIX}{agent_name}");
501 let credit_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{agent_name}");
502 if let Err(e) = store.kv_set(&agent_key, "0").await {
503 tracing::warn!(key = agent_key, err = %e, "failed to reset failure count");
504 }
505 if !model.is_empty() {
506 let model_key = format!("{FAILURE_COUNT_PREFIX}{agent_name}:{model}");
507 if let Err(e) = store.kv_set(&model_key, "0").await {
508 tracing::warn!(key = model_key, err = %e, "failed to reset failure count");
509 }
510 }
511 if let Err(e) = store.kv_set(&credit_key, "0").await {
512 tracing::warn!(key = credit_key, err = %e, "failed to reset failure count");
513 }
514 }
515}
516
517pub async fn record_agent_failure_with_message(agent_name: &str, error_message: &str) {
523 if let Some(cooldown_until) = parse_retry_at(error_message) {
528 let now = chrono::Utc::now().timestamp();
529 if cooldown_until > now {
530 set_cooldown_async(agent_name, cooldown_until, "agent_error").await;
531 return;
532 }
533 tracing::debug!(
534 agent = agent_name,
535 cooldown_until,
536 "vendor retry_at is in the past — using backoff instead"
537 );
538 }
539
540 let store_opt = cooldown_store().lock().await.clone();
541 let count = read_and_increment_failure_count(store_opt.as_ref(), agent_name).await;
542 let base = config::get_agent_backoff_base(agent_name);
543 let cooldown_until =
544 chrono::Utc::now().timestamp() + compute_backoff(count, base, BACKOFF_MAX_SECS, true);
545 set_cooldown_async(agent_name, cooldown_until, "agent_error").await;
546}
547
548pub fn detect_credit_exhaustion_outcome(error_message: &str) -> Option<&'static str> {
557 detect_credit_exhaustion(error_message).map(CreditExhaustionReason::outcome_str)
558}
559
560pub fn detect_credit_exhaustion(error_message: &str) -> Option<CreditExhaustionReason> {
565 let lower = error_message.to_lowercase();
566
567 let overage_patterns = [
568 "overagedisabledreason",
569 "overage_disabled_reason",
570 "overage disabled reason",
571 ];
572
573 if overage_patterns.iter().any(|p| lower.contains(p)) {
574 if lower.contains("out_of_credits") || lower.contains("outofcredits") {
575 return Some(CreditExhaustionReason::OutOfCredits);
576 }
577 if lower.contains("org_level_disabled") || lower.contains("orgdisabled") {
578 return Some(CreditExhaustionReason::OrgLevelDisabled);
579 }
580 }
581
582 let billing_patterns = [
583 "credit balance too low",
584 "credit balance insufficient",
585 "insufficient credit balance",
586 "billing quota exceeded",
587 "billing limit exceeded",
588 "organization has been disabled",
589 "org disabled",
590 ];
591
592 if billing_patterns.iter().any(|p| lower.contains(p)) {
593 return Some(CreditExhaustionReason::OrgLevelDisabled);
594 }
595
596 let billing_cycle_patterns = [
599 "billing cycle",
600 "quota refreshed next cycle",
601 "monthly quota",
602 "quota will be refreshed",
603 "next billing cycle",
604 "refreshed in the next cycle",
605 "weekly/monthly limit exhausted",
606 "limit will reset at",
607 "upgrade to pro", "upgrade your token plan", "token plan usage limit", "purchase credits for more", ];
612
613 if billing_cycle_patterns.iter().any(|p| lower.contains(p)) {
614 return Some(CreditExhaustionReason::BillingCycleExhausted);
615 }
616
617 if lower.contains("out of credits")
618 || lower.contains("outofcredits")
619 || lower.contains("insufficient funds")
620 || lower.contains("no credits remaining")
621 || lower.contains("insufficient_quota")
622 || lower.contains("quota exceeded")
623 || lower.contains("insufficient balance")
624 || lower.contains("please recharge")
625 || lower.contains("no resource package")
626 || lower.contains("balance exhausted")
627 || lower.contains("402 payment required")
628 {
629 return Some(CreditExhaustionReason::OutOfCredits);
630 }
631
632 None
633}
634
635pub async fn record_credit_exhaustion(agent_name: &str, reason: CreditExhaustionReason) {
648 let reason_str = match reason {
649 CreditExhaustionReason::OutOfCredits => "credit_exhaustion_out_of_credits",
650 CreditExhaustionReason::OrgLevelDisabled => "credit_exhaustion_org_level_disabled",
651 CreditExhaustionReason::BillingCycleExhausted => "billing_cycle_exhausted",
652 };
653
654 if is_in_cooldown_with_reason(agent_name, reason_str) {
656 tracing::debug!(
657 agent = agent_name,
658 reason = reason_str,
659 "credit exhaustion already cooled, not escalating"
660 );
661 return;
662 }
663
664 let store_opt = cooldown_store().lock().await.clone();
665 let count = read_and_increment_failure_count_with_prefix(
670 store_opt.as_ref(),
671 CREDIT_FAILURE_COUNT_PREFIX,
672 agent_name,
673 )
674 .await;
675
676 let (base, max) = match reason {
677 CreditExhaustionReason::OutOfCredits => (CREDIT_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS),
678 CreditExhaustionReason::OrgLevelDisabled => {
679 (ORG_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS)
680 }
681 CreditExhaustionReason::BillingCycleExhausted => {
682 (BILLING_CYCLE_COOLDOWN_SECS, BILLING_CYCLE_MAX_SECS)
683 }
684 };
685 let cooldown_secs = compute_backoff(
686 count,
687 base,
688 max,
689 !matches!(reason, CreditExhaustionReason::BillingCycleExhausted),
690 );
691 let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
692 set_cooldown_async(agent_name, cooldown_until, reason_str).await;
693 tracing::warn!(
694 agent = agent_name,
695 reason = reason_str,
696 attempt = count,
697 cooldown_secs,
698 "credit exhaustion detected: applying exponential agent-wide cooldown"
699 );
700}
701
702pub async fn record_credit_exhaustion_from_message(
704 agent_name: &str,
705 model: Option<&str>,
706 message: &str,
707) -> bool {
708 let Some(reason) = detect_credit_exhaustion(message) else {
709 return false;
710 };
711 match (reason, model) {
712 (CreditExhaustionReason::BillingCycleExhausted, Some(model)) => {
713 record_persistent_model_failure(agent_name, model).await;
714 }
715 _ => record_credit_exhaustion(agent_name, reason).await,
716 }
717 true
718}
719
720pub async fn record_model_failure(agent_name: &str, model: &str) {
724 let base = config::get_agent_backoff_base(agent_name);
725 let schedule = ModelBackoff {
726 prefix: FAILURE_COUNT_PREFIX,
727 base,
728 max: BACKOFF_MAX_SECS,
729 allow_extended: true,
730 reason: "model_error",
731 };
732 record_model_backoff(agent_name, model, &schedule).await;
733}
734
735pub async fn record_rate_limit(agent_name: &str, model: Option<&str>, error_message: &str) {
743 if let Some(window_secs) = parse_relative_usage_window(error_message)
744 && let Some(m) = model
745 {
746 let cooldown_until = chrono::Utc::now()
747 .timestamp()
748 .saturating_add(i64::try_from(window_secs).unwrap_or(i64::MAX));
749 set_model_cooldown(agent_name, m, window_secs, "rate_limit_usage_window").await;
750 tracing::info!(
751 agent = agent_name,
752 model = m,
753 window_secs,
754 cooldown_until,
755 "relative usage window detected: applying model-specific cooldown"
756 );
757 return;
758 }
759
760 record_agent_failure_with_message(agent_name, error_message).await;
761 if let Some(m) = model {
762 record_model_failure(agent_name, m).await;
763 }
764}
765
766pub async fn record_persistent_model_failure(agent_name: &str, model: &str) {
778 let schedule = ModelBackoff {
779 prefix: PERSISTENT_FAILURE_COUNT_PREFIX,
780 base: PERSISTENT_MODEL_BACKOFF_BASE_SECS,
781 max: PERSISTENT_MODEL_BACKOFF_MAX_SECS,
782 allow_extended: false,
783 reason: "persistent_model_error",
784 };
785 record_model_backoff(agent_name, model, &schedule).await;
786}
787
788pub async fn record_model_unavailable(agent_name: &str, model: &str, message: &str) {
793 if is_model_permanently_gone(message) {
794 record_persistent_model_failure(agent_name, model).await;
795 } else {
796 record_model_failure(agent_name, model).await;
797 }
798}
799
800fn is_model_permanently_gone(message: &str) -> bool {
801 let lower = message.to_lowercase();
802 lower.contains("not found")
803 || lower.contains("not supported")
804 || lower.contains("not available in your country")
805 || lower.contains("not available in your region")
806}
807
808pub async fn record_transient_model_failure(agent_name: &str, model: &str) {
814 let schedule = ModelBackoff {
815 prefix: FAILURE_COUNT_PREFIX,
816 base: TRANSIENT_MODEL_BACKOFF_BASE_SECS,
817 max: TRANSIENT_MODEL_BACKOFF_MAX_SECS,
818 allow_extended: false,
819 reason: "transient_model_error",
820 };
821 record_model_backoff(agent_name, model, &schedule).await;
822}
823
824pub async fn set_model_cooldown(agent_name: &str, model: &str, duration_secs: u64, reason: &str) {
831 let key = format!("{agent_name}:{model}");
832 let cooldown_until = chrono::Utc::now()
833 .timestamp()
834 .saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX));
835 set_cooldown_async(&key, cooldown_until, reason).await;
836}
837
838pub async fn set_model_cooldown_until(agent_name: &str, model: &str, until: i64) {
840 let key = format!("{agent_name}:{model}");
841 set_cooldown_async(&key, until, "vendor_reset_at").await;
842}
843
844pub async fn set_agent_cooldown(agent_name: &str, duration_secs: u64) {
851 let cooldown_until = chrono::Utc::now()
852 .timestamp()
853 .saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX));
854 set_cooldown_async(agent_name, cooldown_until, "silence_agent_cooldown").await;
855}
856
857pub const RATE_WINDOW_KV_PREFIX: &str = "rate_window:";
859
860#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
862pub struct RateLimitWindow {
863 pub status: String,
864 pub window_type: String,
865 pub utilization: Option<f64>,
866 pub resets_at: Option<i64>,
867}
868
869pub async fn record_rate_limit_window(agent_name: &str, window: &RateLimitWindow) {
871 let store_opt = cooldown_store().lock().await.clone();
872 if let Some(store) = store_opt
873 && let Ok(value) = serde_json::to_string(window)
874 && let Err(e) = store
875 .kv_set(
876 &format!("{RATE_WINDOW_KV_PREFIX}{agent_name}:{}", window.window_type),
877 &value,
878 )
879 .await
880 {
881 tracing::warn!(agent = agent_name, err = %e, "failed to persist rate limit window");
882 }
883
884 let now = chrono::Utc::now().timestamp();
885 if window.status == "rejected"
886 && let Some(reset) = window.resets_at
887 && reset > now
888 {
889 let until = reset.min(now + PERSISTENT_MODEL_BACKOFF_MAX_SECS);
890 set_cooldown_async(agent_name, until, "rate_limit_window").await;
891 tracing::info!(
892 agent = agent_name,
893 window = %window.window_type,
894 cooldown_until = until,
895 "rate limit window rejected: cooling agent until vendor reset"
896 );
897 }
898}
899
900pub async fn list_rate_limit_windows(store: &Arc<TaskStore>) -> Vec<(String, RateLimitWindow)> {
902 let rows = store
903 .kv_list_prefix(RATE_WINDOW_KV_PREFIX)
904 .await
905 .unwrap_or_default();
906 let mut windows: Vec<(String, RateLimitWindow)> = rows
907 .into_iter()
908 .filter_map(|(key, value)| {
909 let (agent, _) = key.strip_prefix(RATE_WINDOW_KV_PREFIX)?.rsplit_once(':')?;
910 let window = serde_json::from_str(&value).ok()?;
911 Some((agent.to_string(), window))
912 })
913 .collect();
914 windows.sort_by(|a, b| (&a.0, &a.1.window_type).cmp(&(&b.0, &b.1.window_type)));
915 windows
916}
917
918pub async fn record_silence_detection(agent_name: &str, model: &str) -> Option<SilenceCountResult> {
921 let store_opt = cooldown_store().lock().await.clone();
922 let Some(store) = store_opt else {
923 tracing::debug!(
924 agent = agent_name,
925 model,
926 "skipping silence count record (no KV store)"
927 );
928 return None;
929 };
930
931 let key = format!("{SILENCE_COUNT_PREFIX}{agent_name}:{model}");
932 let now = chrono::Utc::now().timestamp();
933 let window_start = now - SILENCE_COUNT_WINDOW_SECS;
934
935 let mut timestamps = match store.kv_get(&key).await {
936 Ok(Some(raw)) => match serde_json::from_str::<Vec<i64>>(&raw) {
937 Ok(ts) => ts,
938 Err(e) => {
939 tracing::warn!(
940 kv_key = %key,
941 err = %e,
942 "failed to parse silence timestamps from KV — resetting window"
943 );
944 Vec::new()
945 }
946 },
947 Ok(None) => Vec::new(),
948 Err(err) => {
949 tracing::warn!(
950 kv_key = %key,
951 err = %err,
952 "failed to load silence count from KV"
953 );
954 Vec::new()
955 }
956 };
957
958 timestamps.retain(|ts| *ts >= window_start);
959 timestamps.push(now);
960
961 let count = timestamps.len();
962 let mut extended_cooldown_applied = false;
963 if count >= SILENCE_COUNT_THRESHOLD {
964 set_model_cooldown(
965 agent_name,
966 model,
967 SILENCE_EXTENDED_COOLDOWN_SECS,
968 "silence_detected",
969 )
970 .await;
971 extended_cooldown_applied = true;
972 timestamps.clear();
973 }
974
975 let value = serde_json::to_string(×tamps).unwrap_or_else(|_| "[]".to_string());
976 if let Err(err) = store.kv_set(&key, &value).await {
977 tracing::warn!(kv_key = key, err = %err, "failed to persist silence count");
978 }
979
980 Some(SilenceCountResult {
981 count,
982 extended_cooldown_applied,
983 })
984}
985
986pub fn is_model_in_cooldown(agent_name: &str, model: &str) -> bool {
988 let key = format!("{agent_name}:{model}");
989 is_in_cooldown(&key)
990}
991
992pub fn is_agent_in_cooldown(agent_name: &str) -> bool {
994 is_in_cooldown(agent_name)
995}
996
997pub fn cooldown_until(key: &str) -> Option<i64> {
999 let map = cooldown_lock();
1000 map.get(key).and_then(|entry| {
1001 let now = chrono::Utc::now().timestamp();
1002 if now < entry.cooldown_until {
1003 Some(entry.cooldown_until)
1004 } else {
1005 None
1006 }
1007 })
1008}
1009
1010pub fn cooldown_reason(key: &str) -> Option<String> {
1012 let map = cooldown_lock();
1013 map.get(key).and_then(|entry| {
1014 let now = chrono::Utc::now().timestamp();
1015 if now < entry.cooldown_until {
1016 Some(entry.reason.clone())
1017 } else {
1018 None
1019 }
1020 })
1021}
1022
1023pub fn list_all_cooldowns() -> Vec<(String, i64, String)> {
1028 let now = chrono::Utc::now().timestamp();
1029 let map = cooldown_lock();
1030 let mut result: Vec<(String, i64, String)> = map
1031 .iter()
1032 .filter(|(_, entry)| now < entry.cooldown_until)
1033 .map(|(key, entry)| (key.clone(), entry.cooldown_until, entry.reason.clone()))
1034 .collect();
1035 result.sort_by(|a, b| a.0.cmp(&b.0));
1036 result
1037}
1038
1039pub async fn mark_rate_limits_cleared(store: &Arc<TaskStore>, agents: &[String]) {
1042 let now = chrono::Utc::now().timestamp().to_string();
1043 let mut seen = HashSet::new();
1044 for key in agents {
1045 let agent = agent_from_key(key);
1046 if agent.is_empty() || !seen.insert(agent.to_string()) {
1047 continue;
1048 }
1049 let kv_key = format!("{RATE_LIMITS_CLEARED_AT_PREFIX}{agent}");
1050 if let Err(e) = store.kv_set(&kv_key, &now).await {
1051 tracing::warn!(agent, err = %e, "failed to mark rate limits cleared");
1052 }
1053 }
1054}
1055
1056pub async fn clear_cooldown(key: &str, store: &Arc<TaskStore>) {
1063 let mut agents_to_mark: Vec<String> = Vec::new();
1064 if key == "*" {
1065 let keys: Vec<String> = {
1066 let mut map = cooldown_lock();
1067 let keys: Vec<String> = map.keys().cloned().collect();
1068 map.clear();
1069 keys
1070 };
1071 for k in &keys {
1072 let kv_key = format!("{KV_PREFIX}{k}");
1073 if let Err(e) = store.kv_set(&kv_key, "0").await {
1074 tracing::warn!(key = kv_key, err = %e, "failed to clear cooldown");
1075 }
1076 let fc_key = format!("{FAILURE_COUNT_PREFIX}{k}");
1078 if let Err(e) = store.kv_set(&fc_key, "0").await {
1079 tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1080 }
1081 let credit_fc_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{k}");
1083 if let Err(e) = store.kv_set(&credit_fc_key, "0").await {
1084 tracing::warn!(key = credit_fc_key, err = %e, "failed to reset credit failure count");
1085 }
1086 let persistent_fc_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{k}");
1088 if let Err(e) = store.kv_set(&persistent_fc_key, "0").await {
1089 tracing::warn!(key = persistent_fc_key, err = %e, "failed to reset persistent failure count");
1090 }
1091 }
1092 if let Ok(fc_entries) = store.kv_list_prefix(FAILURE_COUNT_PREFIX).await {
1096 for (fc_key, _) in fc_entries {
1097 if let Err(e) = store.kv_set(&fc_key, "0").await {
1098 tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1099 }
1100 }
1101 }
1102 if let Ok(cfc_entries) = store.kv_list_prefix(CREDIT_FAILURE_COUNT_PREFIX).await {
1103 for (cfc_key, _) in cfc_entries {
1104 if let Err(e) = store.kv_set(&cfc_key, "0").await {
1105 tracing::warn!(key = cfc_key, err = %e, "failed to reset credit failure count");
1106 }
1107 }
1108 }
1109 if let Ok(pfc_entries) = store.kv_list_prefix(PERSISTENT_FAILURE_COUNT_PREFIX).await {
1110 for (pfc_key, _) in pfc_entries {
1111 if let Err(e) = store.kv_set(&pfc_key, "0").await {
1112 tracing::warn!(key = pfc_key, err = %e, "failed to reset persistent failure count");
1113 }
1114 }
1115 }
1116 tracing::info!(
1117 count = keys.len(),
1118 "cleared all cooldowns and failure counts"
1119 );
1120 agents_to_mark.extend(keys.iter().map(|k| agent_from_key(k).to_string()));
1121 match store.rate_limit_agents().await {
1122 Ok(rl_agents) => agents_to_mark.extend(rl_agents),
1123 Err(e) => tracing::warn!(err = %e, "failed to list rate_limits agents while clearing"),
1124 }
1125 match store.kv_list_prefix(RATE_LIMITS_CLEARED_AT_PREFIX).await {
1126 Ok(marks) => {
1127 for (k, _) in marks {
1128 let agent = k.trim_start_matches(RATE_LIMITS_CLEARED_AT_PREFIX);
1129 if !agent.is_empty() {
1130 agents_to_mark.push(agent.to_string());
1131 }
1132 }
1133 }
1134 Err(e) => tracing::warn!(err = %e, "failed to list rate_limits_cleared_at keys"),
1135 }
1136 } else {
1137 {
1138 let mut map = cooldown_lock();
1139 map.remove(key);
1140 }
1141 let kv_key = format!("{KV_PREFIX}{key}");
1142 if let Err(e) = store.kv_set(&kv_key, "0").await {
1143 tracing::warn!(key = kv_key, err = %e, "failed to clear cooldown");
1144 }
1145 let fc_key = format!("{FAILURE_COUNT_PREFIX}{key}");
1147 if let Err(e) = store.kv_set(&fc_key, "0").await {
1148 tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1149 }
1150 let credit_fc_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{key}");
1152 if let Err(e) = store.kv_set(&credit_fc_key, "0").await {
1153 tracing::warn!(key = credit_fc_key, err = %e, "failed to reset credit failure count");
1154 }
1155 let persistent_fc_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{key}");
1157 if let Err(e) = store.kv_set(&persistent_fc_key, "0").await {
1158 tracing::warn!(key = persistent_fc_key, err = %e, "failed to reset persistent failure count");
1159 }
1160 tracing::info!(key, "cleared cooldown and failure count");
1161 agents_to_mark.push(agent_from_key(key).to_string());
1162 }
1163 mark_rate_limits_cleared(store, &agents_to_mark).await;
1164}
1165
1166fn set_cooldown_in_memory(key: &str, cooldown_until: i64, reason: &str) -> bool {
1175 let mut map = cooldown_lock();
1176 if let Some(existing) = map.get(key) {
1177 if existing.cooldown_until >= cooldown_until {
1181 return false;
1182 }
1183 }
1184 map.insert(
1185 key.to_string(),
1186 CooldownEntry {
1187 cooldown_until,
1188 reason: reason.to_string(),
1189 },
1190 );
1191 true
1192}
1193
1194async fn set_cooldown_async(key: &str, cooldown_until: i64, reason: &str) -> bool {
1204 if !set_cooldown_in_memory(key, cooldown_until, reason) {
1205 return false;
1206 }
1207 let store_opt = cooldown_store().lock().await.clone();
1208 if let Some(store) = store_opt {
1209 let kv_key = format!("{KV_PREFIX}{key}");
1210 let value = serde_json::json!({"until": cooldown_until, "reason": reason}).to_string();
1211 if let Err(e) = store.kv_set(&kv_key, &value).await {
1212 tracing::warn!(
1213 kv_key,
1214 err = %e,
1215 "failed to persist cooldown to KV store — rolling back in-memory entry"
1216 );
1217 let mut map = cooldown_lock();
1218 if map.get(key).map(|e| e.cooldown_until) == Some(cooldown_until) {
1222 map.remove(key);
1223 }
1224 return false;
1225 }
1226 }
1227 true
1228}
1229
1230fn cooldown_lock() -> MutexGuard<'static, HashMap<String, CooldownEntry>> {
1232 cooldowns().lock().unwrap_or_else(|e| {
1233 tracing::error!("cooldown map lock poisoned — recovering with partial state; backoff and degraded-agent decisions may be incorrect");
1234 e.into_inner()
1235 })
1236}
1237
1238fn degraded_agents_lock() -> MutexGuard<'static, HashMap<String, String>> {
1240 degraded_agents().lock().unwrap_or_else(|e| {
1241 tracing::error!("degraded agents lock poisoned — recovering with partial state; health check decisions may be incorrect");
1242 e.into_inner()
1243 })
1244}
1245
1246fn github_5xx_timestamps_lock() -> MutexGuard<'static, Vec<i64>> {
1248 github_5xx_timestamps().lock().unwrap_or_else(|e| {
1249 tracing::error!("github 5xx timestamps lock poisoned — recovering with partial state; circuit breaker may operate incorrectly");
1250 e.into_inner()
1251 })
1252}
1253
1254fn github_5xx_circuit_open_lock() -> MutexGuard<'static, Option<i64>> {
1256 github_5xx_circuit_open().lock().unwrap_or_else(|e| {
1257 tracing::error!("github 5xx circuit open lock poisoned — recovering with partial state; circuit breaker may operate incorrectly");
1258 e.into_inner()
1259 })
1260}
1261
1262fn is_in_cooldown_with_reason(key: &str, reason: &str) -> bool {
1264 let map = cooldown_lock();
1265 map.get(key).is_some_and(|entry| {
1266 entry.reason == reason && chrono::Utc::now().timestamp() < entry.cooldown_until
1267 })
1268}
1269
1270fn is_in_cooldown(key: &str) -> bool {
1272 let map = cooldown_lock();
1273 if let Some(entry) = map.get(key) {
1274 let now = chrono::Utc::now().timestamp();
1275 return now < entry.cooldown_until;
1276 }
1277 false
1278}
1279
1280pub const DEFAULT_HEALTH_CHECK_WINDOW_HOURS: u32 = 6;
1286
1287pub const DEFAULT_DEGRADED_RATE_LIMIT_THRESHOLD: i64 = 3;
1291
1292fn degraded_agents() -> &'static Mutex<HashMap<String, String>> {
1295 static DEGRADED: OnceLock<Mutex<HashMap<String, String>>> = OnceLock::new();
1296 DEGRADED.get_or_init(|| Mutex::new(HashMap::new()))
1297}
1298
1299pub fn is_agent_degraded(agent: &str) -> bool {
1304 let set = degraded_agents_lock();
1305 set.contains_key(agent)
1306}
1307
1308pub fn agent_degraded_reason(agent: &str) -> Option<String> {
1310 degraded_agents_lock().get(agent).cloned()
1311}
1312
1313pub fn list_degraded_agents() -> Vec<(String, String)> {
1315 let mut entries: Vec<(String, String)> = degraded_agents_lock()
1316 .iter()
1317 .map(|(agent, reason)| (agent.clone(), reason.clone()))
1318 .collect();
1319 entries.sort_by(|a, b| a.0.cmp(&b.0));
1320 entries
1321}
1322
1323pub fn mark_agent_degraded(agent: &str) {
1325 mark_agent_degraded_with_reason(agent, "degraded");
1326}
1327
1328fn mark_agent_degraded_with_reason(agent: &str, reason: &str) {
1329 let mut set = degraded_agents_lock();
1330 set.insert(agent.to_string(), reason.to_string());
1331}
1332
1333pub fn clear_agent_degraded(agent: &str) {
1335 let mut set = degraded_agents_lock();
1336 set.remove(agent);
1337}
1338
1339pub async fn refresh_degraded_agents(
1353 store: &Arc<TaskStore>,
1354 available_agents: &[String],
1355 model_checker: &(dyn Fn(&str) -> bool + Send + Sync),
1356 window_hours: u32,
1357 threshold: i64,
1358) {
1359 clear_expired_cooldowns();
1363
1364 let start = chrono::Utc::now();
1367 let counts = match store.recent_rate_limit_counts(window_hours).await {
1368 Ok(c) => {
1369 let dur_ms = (chrono::Utc::now() - start).num_milliseconds();
1370 tracing::info!(
1371 duration_ms = dur_ms,
1372 window_hours,
1373 "recent_rate_limit_counts query completed"
1374 );
1375 c
1376 }
1377 Err(e) => {
1378 let dur_ms = (chrono::Utc::now() - start).num_milliseconds();
1379 tracing::warn!(
1380 err = %e,
1381 duration_ms = dur_ms,
1382 "failed to query recent rate limit counts for health check"
1383 );
1384 return;
1385 }
1386 };
1387
1388 for agent in available_agents {
1389 let in_cooldown = is_agent_in_cooldown(agent);
1390 let no_models = !model_checker(agent);
1391 let rate_limit_count = counts.get(agent.as_str()).copied().unwrap_or(0);
1392 let over_threshold = rate_limit_count >= threshold;
1393
1394 let degraded = in_cooldown || no_models || over_threshold;
1395
1396 if degraded {
1397 let reason = if in_cooldown {
1398 "agent in cooldown"
1399 } else if no_models {
1400 "all models cooled"
1401 } else {
1402 "rate limit threshold exceeded"
1403 };
1404 let previous = agent_degraded_reason(agent);
1405 match previous.as_deref() {
1406 None => tracing::warn!(
1407 agent,
1408 reason,
1409 rate_limit_count,
1410 window_hours,
1411 threshold,
1412 "pre-emptive health check: marking agent as degraded"
1413 ),
1414 Some(prev) if prev != reason => tracing::warn!(
1415 agent,
1416 reason,
1417 previous = prev,
1418 rate_limit_count,
1419 window_hours,
1420 threshold,
1421 "pre-emptive health check: degraded reason changed"
1422 ),
1423 Some(_) => {}
1424 }
1425 mark_agent_degraded_with_reason(agent, reason);
1426 } else {
1427 if is_agent_degraded(agent) {
1428 tracing::info!(
1429 agent,
1430 rate_limit_count,
1431 "pre-emptive health check: agent recovered, clearing degraded flag"
1432 );
1433 }
1434 clear_agent_degraded(agent);
1435 }
1436 }
1437}
1438
1439pub fn clear_expired_cooldowns() {
1441 let mut map = cooldown_lock();
1442 let now = chrono::Utc::now().timestamp();
1443 map.retain(|_key, entry| now < entry.cooldown_until);
1444}
1445
1446const GITHUB_5XX_WINDOW_SECS: i64 = 120;
1452
1453const GITHUB_5XX_THRESHOLD: usize = 5;
1455
1456const GITHUB_5XX_COOLDOWN_SECS: i64 = 180;
1458
1459fn github_5xx_timestamps() -> &'static Mutex<Vec<i64>> {
1461 static TS: OnceLock<Mutex<Vec<i64>>> = OnceLock::new();
1462 TS.get_or_init(|| Mutex::new(Vec::new()))
1463}
1464
1465fn github_5xx_circuit_open() -> &'static Mutex<Option<i64>> {
1467 static OPEN: OnceLock<Mutex<Option<i64>>> = OnceLock::new();
1468 OPEN.get_or_init(|| Mutex::new(None))
1469}
1470
1471pub async fn record_github_5xx() {
1476 let now = chrono::Utc::now().timestamp();
1477 let window_start = now - GITHUB_5XX_WINDOW_SECS;
1478
1479 let should_trip = {
1480 let mut ts = github_5xx_timestamps_lock();
1481 ts.retain(|t| *t >= window_start);
1482 ts.push(now);
1483 ts.len() >= GITHUB_5XX_THRESHOLD
1484 };
1485
1486 if should_trip {
1487 let cooldown_until = {
1488 let mut open = github_5xx_circuit_open_lock();
1489 if open.is_none() {
1490 let cd = now + GITHUB_5XX_COOLDOWN_SECS;
1491 *open = Some(cd);
1492 tracing::warn!(
1493 errors_in_window = GITHUB_5XX_THRESHOLD,
1494 window_secs = GITHUB_5XX_WINDOW_SECS,
1495 cooldown_secs = GITHUB_5XX_COOLDOWN_SECS,
1496 "GitHub 5xx circuit breaker OPEN — throttling non-critical work"
1497 );
1498 Some(cd)
1499 } else {
1500 None
1501 }
1502 };
1503 if let Some(cd) = cooldown_until {
1505 let store_opt = cooldown_store().lock().await.clone();
1506 if let Some(store) = store_opt {
1507 let _ = store.kv_set("cooldown:github:5xx", &cd.to_string()).await;
1508 }
1509 set_agent_cooldown("github:5xx", GITHUB_5XX_COOLDOWN_SECS as u64).await;
1513 }
1514 let mut ts = github_5xx_timestamps_lock();
1516 ts.clear();
1517 }
1518}
1519
1520pub fn is_github_circuit_open() -> bool {
1528 {
1531 let mut open = github_5xx_circuit_open_lock();
1532 if let Some(until) = *open {
1533 let now = chrono::Utc::now().timestamp();
1534 if now >= until {
1535 *open = None;
1536 tracing::info!("GitHub 5xx circuit breaker CLOSED — resuming normal operations");
1537 return false;
1538 }
1539 return true;
1540 }
1541 }
1542
1543 if let Some(until) = cooldown_until("github:5xx") {
1547 let mut open = github_5xx_circuit_open_lock();
1550 if open.is_none() {
1551 *open = Some(until);
1552 let now = chrono::Utc::now().timestamp();
1553 if now < until {
1554 let remaining = until - now;
1555 tracing::warn!(
1556 remaining_secs = remaining,
1557 "GitHub 5xx circuit breaker OPEN (discovered via generic cooldown) — throttling non-critical work"
1558 );
1559 }
1560 }
1561 return true;
1562 }
1563
1564 false
1565}
1566
1567pub fn github_circuit_remaining_secs() -> u64 {
1570 {
1572 let open = github_5xx_circuit_open_lock();
1573 if let Some(until) = *open {
1574 let now = chrono::Utc::now().timestamp();
1575 if now < until {
1576 return u64::try_from(until - now).unwrap_or(0);
1577 }
1578 return 0;
1579 }
1580 }
1581
1582 if let Some(until) = cooldown_until("github:5xx") {
1584 let now = chrono::Utc::now().timestamp();
1585 if now < until {
1586 return u64::try_from(until - now).unwrap_or(0);
1587 }
1588 }
1589 0
1590}
1591
1592pub(crate) fn parse_retry_at(error_message: &str) -> Option<i64> {
1596 if error_message.is_empty() {
1597 return None;
1598 }
1599
1600 let lower = error_message.to_lowercase();
1601 for retry_marker in [
1605 "try again at ",
1606 "reset at ",
1607 "resets at ",
1608 "resets ",
1609 "reset ",
1610 ] {
1611 if let Some(idx) = lower.find(retry_marker) {
1612 let date_str = &lower[idx + retry_marker.len()..];
1613 let date_str = date_str
1615 .split(['.', '\n'])
1616 .next()
1617 .unwrap_or(date_str)
1618 .trim();
1619
1620 let cleaned = date_str
1623 .replace("st,", ",")
1624 .replace("nd,", ",")
1625 .replace("rd,", ",")
1626 .replace("th,", ",");
1627
1628 for fmt in &[
1629 "%b %d, %Y %I:%M %p", "%b %d, %Y %I:%M%p", "%B %d, %Y %I:%M %p", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", ] {
1636 if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(&cleaned, fmt) {
1637 let utc = match chrono::Local.from_local_datetime(&dt) {
1640 chrono::LocalResult::Single(local_dt) => {
1641 local_dt.with_timezone(&chrono::Utc)
1642 }
1643 chrono::LocalResult::Ambiguous(earliest, _) => {
1644 earliest.with_timezone(&chrono::Utc)
1646 }
1647 chrono::LocalResult::None => {
1648 dt.and_utc()
1650 }
1651 };
1652 tracing::info!(
1653 retry_at = %utc,
1654 raw = date_str,
1655 marker = retry_marker,
1656 "parsed rate limit retry-at date"
1657 );
1658 return Some(utc.timestamp());
1659 }
1660 }
1661 tracing::debug!(
1662 raw = date_str,
1663 marker = retry_marker,
1664 "could not parse retry-at date"
1665 );
1666
1667 let time_only = date_str.split_once('(').map_or(date_str, |(t, _)| t).trim();
1672 let mut compact = time_only.replace(" am", "am").replace(" pm", "pm");
1674 let split = compact.rfind(' ').map_or(0, |i| i + 1);
1675 let last = &compact[split..];
1676 if !last.contains(':')
1677 && (last.ends_with("am") || last.ends_with("pm"))
1678 && last.len() > 2
1679 {
1680 let at = split + last.len() - 2;
1681 compact.insert_str(at, ":00");
1682 }
1683 let md = compact.replace(" at ", " ");
1685 let now_local = chrono::Local::now();
1686 let this_year = chrono::Datelike::year(&now_local);
1687 for year in [this_year, this_year + 1] {
1688 let with_year = format!("{year} {md}");
1689 if let Ok(dt) =
1690 chrono::NaiveDateTime::parse_from_str(&with_year, "%Y %b %d %I:%M%p")
1691 && let Some(local_dt) = chrono::Local.from_local_datetime(&dt).earliest()
1692 && local_dt > now_local
1693 {
1694 return Some(local_dt.timestamp());
1695 }
1696 }
1697 let time_only = compact.as_str();
1698 for tf in &["%I:%M %p", "%I:%M%p", "%I %p", "%H:%M"] {
1699 if let Ok(t) = chrono::NaiveTime::parse_from_str(time_only, tf) {
1700 let local_now = chrono::Local::now();
1703 let today = local_now.date_naive();
1704 let candidate = chrono::NaiveDateTime::new(today, t);
1705 let candidate_local: chrono::DateTime<chrono::Local> =
1708 match chrono::Local.from_local_datetime(&candidate) {
1709 chrono::LocalResult::Single(dt) => dt,
1710 chrono::LocalResult::Ambiguous(earliest, _) => earliest,
1711 chrono::LocalResult::None => {
1712 let utc_dt = candidate.and_utc();
1714 return Some(utc_dt.timestamp());
1715 }
1716 };
1717 let chosen = if candidate_local <= local_now {
1718 let tomorrow = today.succ_opt().unwrap_or(today);
1720 let ndt = chrono::NaiveDateTime::new(tomorrow, t);
1721 match chrono::Local.from_local_datetime(&ndt) {
1722 chrono::LocalResult::Single(dt) => dt.with_timezone(&chrono::Utc),
1723 chrono::LocalResult::Ambiguous(earliest, _) => {
1724 earliest.with_timezone(&chrono::Utc)
1725 }
1726 chrono::LocalResult::None => ndt.and_utc(),
1727 }
1728 } else {
1729 candidate_local.with_timezone(&chrono::Utc)
1730 };
1731 tracing::info!(
1732 retry_at = %chosen,
1733 raw = date_str,
1734 marker = retry_marker,
1735 "parsed rate limit retry-at time-only; interpreted as next occurrence"
1736 );
1737 return Some(chosen.timestamp());
1738 }
1739 }
1740 }
1741 }
1742
1743 None
1747}
1748
1749pub(crate) fn parse_relative_usage_window(error_message: &str) -> Option<u64> {
1760 static WINDOW_RE: LazyLock<Regex> = LazyLock::new(|| {
1761 Regex::new(
1762 r"(?i)(?:^|\s|[\(\)\.,;:!?])(?:(\d+)\s*-?\s*(hour|hours|hr|hrs|minute|minutes|min|mins|day|days)|(hourly|daily|weekly|monthly))(?:$|\s|[\(\)\.,;:!?])"
1763 )
1764 .expect("usage window regex is valid")
1765 });
1766
1767 if error_message.is_empty() {
1768 return None;
1769 }
1770
1771 let lower = error_message.to_lowercase();
1772 let has_context = lower.contains("usage limit")
1773 || lower.contains("quota")
1774 || lower.contains("window")
1775 || lower.contains("limit");
1776 if !has_context {
1777 return None;
1778 }
1779
1780 let mut max_secs: Option<u64> = None;
1781 for caps in WINDOW_RE.captures_iter(&lower) {
1782 let secs = if let Some(num) = caps.get(1) {
1783 let n: u64 = num.as_str().parse().ok()?;
1784 let unit = caps.get(2)?.as_str().to_lowercase();
1785 let multiplier = match unit.as_str() {
1786 "hour" | "hours" | "hr" | "hrs" => 3600,
1787 "minute" | "minutes" | "min" | "mins" => 60,
1788 "day" | "days" => 86400,
1789 _ => return None,
1790 };
1791 n.checked_mul(multiplier)?
1792 } else if let Some(word) = caps.get(3) {
1793 match word.as_str().to_lowercase().as_str() {
1794 "hourly" => 3600,
1795 "daily" => 86400,
1796 "weekly" => 604_800,
1797 "monthly" => 2_592_000,
1798 _ => return None,
1799 }
1800 } else {
1801 continue;
1802 };
1803 max_secs = Some(max_secs.map_or(secs, |m| m.max(secs)));
1804 }
1805
1806 max_secs
1807}
1808
1809#[cfg(test)]
1810pub(crate) async fn reset_global_state() {
1811 cooldowns().lock().unwrap().clear();
1812 *cooldown_store().lock().await = None;
1813 github_5xx_timestamps().lock().unwrap().clear();
1814 *github_5xx_circuit_open().lock().unwrap() = None;
1815 degraded_agents().lock().unwrap().clear();
1816 opencode::reset_model_caches_for_test();
1817}
1818
1819#[cfg(test)]
1820mod tests {
1821 use std::time::Duration;
1822
1823 use super::*;
1824 use chrono::Datelike;
1825 use chrono::TimeZone;
1826 use serial_test::serial;
1827 use tokio::time;
1828
1829 async fn test_store() -> Arc<TaskStore> {
1830 Arc::new(TaskStore::open_memory().await.expect("in-memory store"))
1831 }
1832
1833 fn sqlite_datetime_offset(secs: i64) -> String {
1834 (chrono::Utc::now() + chrono::Duration::seconds(secs))
1835 .format("%Y-%m-%d %H:%M:%S")
1836 .to_string()
1837 }
1838
1839 #[serial(cooldown_state)]
1840 #[tokio::test]
1841 async fn init_loads_unexpired_cooldowns_from_kv() {
1842 super::reset_global_state().await;
1843 let store = test_store().await;
1844 let cooldown_until = chrono::Utc::now().timestamp() + 3600; store
1847 .kv_set(
1848 &format!("{KV_PREFIX}kimi:k2p5"),
1849 &cooldown_until.to_string(),
1850 )
1851 .await
1852 .unwrap();
1853
1854 let expired = chrono::Utc::now().timestamp() - 1;
1856 store
1857 .kv_set(
1858 &format!("{KV_PREFIX}opencode:old-free"),
1859 &expired.to_string(),
1860 )
1861 .await
1862 .unwrap();
1863
1864 {
1865 let mut map = cooldown_lock();
1866 map.clear();
1867 }
1868
1869 init_cooldown_store(store).await;
1870
1871 assert!(
1872 is_model_in_cooldown("kimi", "k2p5"),
1873 "unexpired cooldown should be loaded"
1874 );
1875 assert!(
1876 !is_model_in_cooldown("opencode", "old-free"),
1877 "expired cooldown should not be loaded"
1878 );
1879 }
1880
1881 #[serial(cooldown_state)]
1882 #[tokio::test]
1883 async fn credit_exhaustion_inside_active_cooldown_keeps_counter_at_one() {
1884 super::reset_global_state().await;
1885 let store = test_store().await;
1886 *cooldown_store().lock().await = Some(Arc::clone(&store));
1887
1888 let msg =
1889 "API error (status 402 Payment Required): usage balance exhausted, out of credits";
1890 assert!(record_credit_exhaustion_from_message("credit_dup_agent", Some("m"), msg).await);
1891 assert!(record_credit_exhaustion_from_message("credit_dup_agent", Some("m"), msg).await);
1892
1893 assert!(is_agent_in_cooldown("credit_dup_agent"));
1894 let count = store
1895 .kv_get(&format!("{CREDIT_FAILURE_COUNT_PREFIX}credit_dup_agent"))
1896 .await
1897 .unwrap();
1898 assert_eq!(count.as_deref(), Some("1"));
1899 }
1900
1901 #[serial(cooldown_state)]
1902 #[tokio::test]
1903 async fn non_credit_message_is_not_recorded_as_credit_exhaustion() {
1904 super::reset_global_state().await;
1905 assert!(!record_credit_exhaustion_from_message("x_agent", Some("m"), "boom").await);
1906 assert!(!is_agent_in_cooldown("x_agent"));
1907 }
1908
1909 #[serial(cooldown_state)]
1910 #[tokio::test]
1911 async fn record_model_failure_writes_to_kv() {
1912 super::reset_global_state().await;
1913 let store = test_store().await;
1914
1915 *cooldown_store().lock().await = Some(Arc::clone(&store));
1916
1917 record_model_failure("testagent_persist", "testmodel_persist").await;
1918
1919 time::sleep(Duration::from_millis(50)).await;
1921
1922 let kv_key = format!("{KV_PREFIX}testagent_persist:testmodel_persist");
1923 let stored = store.kv_get(&kv_key).await.unwrap();
1924 assert!(stored.is_some(), "model failure should be persisted to KV");
1925 let (ts, reason) = parse_cooldown_value(&stored.unwrap()).expect("cooldown value");
1926 let now = chrono::Utc::now().timestamp();
1927 assert!(ts > now, "persisted timestamp should be in the future");
1929 assert_eq!(reason, "model_error");
1930 }
1931
1932 #[serial(cooldown_state)]
1933 #[tokio::test]
1934 async fn record_transient_model_failure_backs_off_5_then_15_minutes() {
1935 super::reset_global_state().await;
1936 let store = test_store().await;
1937 *cooldown_store().lock().await = Some(Arc::clone(&store));
1938
1939 let agent = "testagent_transient";
1940 let model = "testmodel_transient";
1941 let key = format!("{agent}:{model}");
1942
1943 record_transient_model_failure(agent, model).await;
1944 let first = cooldown_until(&key)
1945 .expect("first cooldown should exist")
1946 .saturating_sub(chrono::Utc::now().timestamp());
1947 assert!(
1948 (TRANSIENT_MODEL_BACKOFF_BASE_SECS - 5..=TRANSIENT_MODEL_BACKOFF_BASE_SECS + 5)
1949 .contains(&first),
1950 "first transient cooldown should be ~5m, got {first}s"
1951 );
1952
1953 record_transient_model_failure(agent, model).await;
1954 let second = cooldown_until(&key)
1955 .expect("second cooldown should exist")
1956 .saturating_sub(chrono::Utc::now().timestamp());
1957 assert!(
1958 (TRANSIENT_MODEL_BACKOFF_MAX_SECS - 5..=TRANSIENT_MODEL_BACKOFF_MAX_SECS + 5)
1959 .contains(&second),
1960 "second transient cooldown should be capped at ~15m, got {second}s"
1961 );
1962
1963 let (_, reason) = parse_cooldown_value(
1964 &store
1965 .kv_get(&format!("{KV_PREFIX}{key}"))
1966 .await
1967 .unwrap()
1968 .expect("cooldown persisted"),
1969 )
1970 .expect("cooldown value");
1971 assert_eq!(reason, "transient_model_error");
1972
1973 let count = store
1974 .kv_get(&format!("{FAILURE_COUNT_PREFIX}{key}"))
1975 .await
1976 .unwrap();
1977 assert_eq!(count.as_deref(), Some("2"));
1978 }
1979
1980 #[serial(cooldown_state)]
1981 #[tokio::test]
1982 async fn record_model_failure_and_persistent_failure_use_separate_counters() {
1983 super::reset_global_state().await;
1984 let store = test_store().await;
1985 *cooldown_store().lock().await = Some(Arc::clone(&store));
1986
1987 let agent = "testagent_counters";
1988 let model = "testmodel_counters";
1989 let key = format!("{agent}:{model}");
1990
1991 record_model_failure(agent, model).await;
1992 record_persistent_model_failure(agent, model).await;
1993 record_persistent_model_failure(agent, model).await;
1994
1995 let generic = store
1996 .kv_get(&format!("{FAILURE_COUNT_PREFIX}{key}"))
1997 .await
1998 .unwrap();
1999 let persistent = store
2000 .kv_get(&format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{key}"))
2001 .await
2002 .unwrap();
2003 assert_eq!(generic.as_deref(), Some("1"));
2004 assert_eq!(persistent.as_deref(), Some("2"));
2005
2006 let (_, reason) = parse_cooldown_value(
2007 &store
2008 .kv_get(&format!("{KV_PREFIX}{key}"))
2009 .await
2010 .unwrap()
2011 .expect("cooldown persisted"),
2012 )
2013 .expect("cooldown value");
2014 assert_eq!(reason, "persistent_model_error");
2015 }
2016
2017 #[serial(cooldown_state)]
2018 #[tokio::test]
2019 async fn cooldown_reason_survives_restart_via_kv() {
2020 super::reset_global_state().await;
2021 let store = test_store().await;
2022 let agent = "test_reason_restart";
2023 let future = chrono::Utc::now().timestamp() + 600;
2024
2025 init_cooldown_store(Arc::clone(&store)).await;
2026 set_cooldown_async(agent, future, "billing_cycle_exhausted").await;
2027
2028 {
2030 let mut map = cooldown_lock();
2031 map.clear();
2032 }
2033 init_cooldown_store(store).await;
2034
2035 let entries = list_all_cooldowns();
2036 let entry = entries.iter().find(|(key, ..)| key == agent);
2037 assert_eq!(
2038 entry.map(|(_, _, reason)| reason.as_str()),
2039 Some("billing_cycle_exhausted"),
2040 "cooldown reason must survive a restart, not collapse to 'persisted'"
2041 );
2042 }
2043
2044 #[serial(cooldown_state)]
2045 #[tokio::test]
2046 async fn record_persistent_model_failure_escalates_and_increments_failure_count() {
2047 super::reset_global_state().await;
2048 let store = test_store().await;
2049 *cooldown_store().lock().await = Some(Arc::clone(&store));
2050
2051 let agent = "testagent_persistent";
2052 let model = "testmodel_persistent";
2053 let key = format!("{agent}:{model}");
2054
2055 record_persistent_model_failure(agent, model).await;
2057 let now = chrono::Utc::now().timestamp();
2058 let first_until = cooldown_until(&key).expect("first cooldown should exist");
2059 let first_remaining = first_until.saturating_sub(now);
2060 assert!(
2061 first_remaining >= PERSISTENT_MODEL_BACKOFF_BASE_SECS - 5,
2062 "first persistent model cooldown should be ~4h, got {first_remaining}s"
2063 );
2064
2065 record_persistent_model_failure(agent, model).await;
2067 let now2 = chrono::Utc::now().timestamp();
2068 let second_until = cooldown_until(&key).expect("second cooldown should exist");
2069 let second_remaining = second_until.saturating_sub(now2);
2070 assert!(
2071 second_remaining >= (PERSISTENT_MODEL_BACKOFF_BASE_SECS * 3) - 5,
2072 "second persistent model cooldown should be ~12h, got {second_remaining}s"
2073 );
2074
2075 let count_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{agent}:{model}");
2076 let stored_count = store
2077 .kv_get(&count_key)
2078 .await
2079 .expect("kv read should succeed")
2080 .expect("failure count should be written");
2081 assert_eq!(stored_count, "2");
2082 }
2083
2084 #[serial(cooldown_state)]
2085 #[tokio::test]
2086 async fn record_agent_success_does_not_reset_persistent_failure_count() {
2087 super::reset_global_state().await;
2088 let store = test_store().await;
2089 *cooldown_store().lock().await = Some(Arc::clone(&store));
2090
2091 let agent = "testagent_persistent_success";
2092 let model = "testmodel_persistent_success";
2093 let key = format!("{agent}:{model}");
2094
2095 record_persistent_model_failure(agent, model).await;
2097 record_persistent_model_failure(agent, model).await;
2098
2099 let count_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{agent}:{model}");
2100 let stored_count = store
2101 .kv_get(&count_key)
2102 .await
2103 .expect("kv read should succeed")
2104 .expect("failure count should be written");
2105 assert_eq!(stored_count, "2");
2106
2107 record_agent_success(agent, model).await;
2111
2112 let stored_count_after_success = store
2113 .kv_get(&count_key)
2114 .await
2115 .expect("kv read should succeed")
2116 .expect("persistent failure count should still exist");
2117 assert_eq!(
2118 stored_count_after_success, "2",
2119 "persistent failure count must survive an intervening success"
2120 );
2121
2122 record_persistent_model_failure(agent, model).await;
2124 let now = chrono::Utc::now().timestamp();
2125 let third_until = cooldown_until(&key).expect("third cooldown should exist");
2126 let third_remaining = third_until.saturating_sub(now);
2127 assert!(
2128 third_remaining > PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2129 "third persistent model failure should escalate past the 4h base, got {third_remaining}s"
2130 );
2131 }
2132
2133 #[serial(cooldown_state)]
2134 #[tokio::test]
2135 async fn record_persistent_model_failure_stays_capped_at_7_days() {
2136 super::reset_global_state().await;
2137 let store = test_store().await;
2138 *cooldown_store().lock().await = Some(Arc::clone(&store));
2139
2140 let agent = "testagent_persistent_capped";
2141 let model = "testmodel_persistent_capped";
2142 let key = format!("{agent}:{model}");
2143
2144 for _ in 0..9 {
2145 record_persistent_model_failure(agent, model).await;
2146 }
2147
2148 let remaining = cooldown_until(&key)
2149 .expect("cooldown should exist")
2150 .saturating_sub(chrono::Utc::now().timestamp());
2151 assert!(
2152 remaining >= PERSISTENT_MODEL_BACKOFF_MAX_SECS - 5,
2153 "persistent model cooldown should stay capped at 7 days, got {remaining}s"
2154 );
2155 assert!(
2156 remaining <= PERSISTENT_MODEL_BACKOFF_MAX_SECS + 5,
2157 "persistent model cooldown should not exceed 7 days, got {remaining}s"
2158 );
2159 }
2160
2161 #[serial(cooldown_state)]
2162 #[test]
2163 fn parse_retry_at_codex_format() {
2164 let msg = "You've hit your usage limit. Upgrade to Pro, visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Mar 26th, 2026 5:55 AM.";
2165 let ts = parse_retry_at(msg);
2166 assert!(ts.is_some(), "should parse codex retry-at date");
2167
2168 let naive = chrono::NaiveDateTime::new(
2171 chrono::NaiveDate::from_ymd_opt(2026, 3, 26).unwrap(),
2172 chrono::NaiveTime::from_hms_opt(5, 55, 0).unwrap(),
2173 );
2174 let expected_utc = match chrono::Local.from_local_datetime(&naive) {
2175 chrono::LocalResult::Single(local_dt) => local_dt.with_timezone(&chrono::Utc),
2176 chrono::LocalResult::Ambiguous(earliest, _) => earliest.with_timezone(&chrono::Utc),
2177 chrono::LocalResult::None => naive.and_utc(),
2178 };
2179 assert_eq!(
2180 ts.unwrap(),
2181 expected_utc.timestamp(),
2182 "timestamp should reflect local-timezone interpretation of the vendor date"
2183 );
2184
2185 let dt = chrono::DateTime::from_timestamp(ts.unwrap(), 0).unwrap();
2187 assert_eq!(dt.month(), 3);
2188 assert_eq!(dt.day(), 26);
2189 }
2190
2191 #[serial(cooldown_state)]
2192 #[test]
2193 fn parse_retry_at_hour_only_and_month_day() {
2194 let now = chrono::Utc::now().timestamp();
2195 for msg in [
2196 "You've hit your session limit · resets 8am (America/Sao_Paulo)",
2197 "You've hit your session limit · resets 6pm (America/Sao_Paulo)",
2198 "You've hit your session limit · resets 12am (America/Sao_Paulo)",
2199 "You've hit your weekly limit · resets Jun 9 at 1am (America/Sao_Paulo)",
2200 ] {
2201 let ts = parse_retry_at(msg).unwrap_or_else(|| panic!("should parse: {msg}"));
2202 assert!(ts > now, "{msg}");
2203 }
2204 }
2205
2206 #[serial(cooldown_state)]
2207 #[test]
2208 fn parse_retry_at_weekly_resets_without_year() {
2209 let t = chrono::Local::now() + chrono::Duration::days(2);
2210 let msg = format!(
2211 "You've hit your weekly limit · resets {} {} at 1am (America/Sao_Paulo)",
2212 t.format("%b"),
2213 t.day()
2214 );
2215 let ts = parse_retry_at(&msg).expect("should parse year-less reset date");
2216 let remaining = ts - chrono::Utc::now().timestamp();
2217 assert!(
2218 remaining > 3600 && remaining < 3 * 86400,
2219 "expected ~2d retry-at, got {remaining}s"
2220 );
2221 }
2222
2223 #[serial(cooldown_state)]
2224 #[test]
2225 fn parse_retry_at_no_date() {
2226 assert!(parse_retry_at("generic rate limit error").is_none());
2227 assert!(parse_retry_at("").is_none());
2228 }
2229
2230 #[serial(cooldown_state)]
2231 #[test]
2232 fn parse_retry_at_glm_reset_format_with_seconds() {
2233 let msg = "Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-04-23 18:33:48";
2234 let ts = parse_retry_at(msg).expect("should parse GLM reset-at date");
2235
2236 let naive = chrono::NaiveDateTime::new(
2237 chrono::NaiveDate::from_ymd_opt(2026, 4, 23).expect("valid date"),
2238 chrono::NaiveTime::from_hms_opt(18, 33, 48).expect("valid time"),
2239 );
2240 let expected_utc = match chrono::Local.from_local_datetime(&naive) {
2241 chrono::LocalResult::Single(local_dt) => local_dt.with_timezone(&chrono::Utc),
2242 chrono::LocalResult::Ambiguous(earliest, _) => earliest.with_timezone(&chrono::Utc),
2243 chrono::LocalResult::None => naive.and_utc(),
2244 };
2245
2246 assert_eq!(
2247 ts,
2248 expected_utc.timestamp(),
2249 "timestamp should reflect local-timezone interpretation of vendor reset date"
2250 );
2251 }
2252
2253 #[serial(cooldown_state)]
2254 #[test]
2255 fn parse_retry_at_billing_cycle_without_date_returns_none() {
2256 let msg = "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle. Upgrade to get more.";
2259 assert!(
2260 parse_retry_at(msg).is_none(),
2261 "billing cycle without date should return None"
2262 );
2263 }
2264
2265 #[serial(cooldown_state)]
2266 #[test]
2267 fn parse_relative_usage_window_extracts_kimi_five_hour_window() {
2268 let msg = "Failed to authenticate. API Error: 403 You've reached your 5-hour usage limit. Your quota will reset when the current 5-hour window expires.";
2269 assert_eq!(
2270 parse_relative_usage_window(msg),
2271 Some(5 * 3600),
2272 "should extract the 5-hour window as seconds"
2273 );
2274 }
2275
2276 #[serial(cooldown_state)]
2277 #[test]
2278 fn parse_relative_usage_window_handles_common_durations() {
2279 assert_eq!(
2280 parse_relative_usage_window("You've reached your hourly usage limit."),
2281 Some(3600)
2282 );
2283 assert_eq!(
2284 parse_relative_usage_window("You've reached your daily usage limit."),
2285 Some(86400)
2286 );
2287 assert_eq!(
2288 parse_relative_usage_window("You've reached your weekly limit."),
2289 Some(604_800)
2290 );
2291 assert_eq!(
2292 parse_relative_usage_window("You've reached your 30-minute usage limit."),
2293 Some(1800)
2294 );
2295 assert_eq!(
2296 parse_relative_usage_window("You've reached your 2-day quota."),
2297 Some(172_800)
2298 );
2299 }
2300
2301 #[serial(cooldown_state)]
2302 #[test]
2303 fn parse_relative_usage_window_returns_none_without_usage_context() {
2304 assert!(parse_relative_usage_window("429 Too Many Requests").is_none());
2307 assert!(parse_relative_usage_window("rate limit exceeded").is_none());
2308 assert!(parse_relative_usage_window("").is_none());
2309 }
2310
2311 #[serial(cooldown_state)]
2312 #[tokio::test]
2313 async fn record_rate_limit_uses_model_specific_window_cooldown() {
2314 super::reset_global_state().await;
2315 let store = test_store().await;
2316 *cooldown_store().lock().await = Some(Arc::clone(&store));
2317
2318 let agent = "test_record_rl_window_agent";
2319 let model = "test_record_rl_window_model";
2320 let msg = "You've reached your 5-hour usage limit. Your quota will reset when the current 5-hour window expires.";
2321
2322 record_rate_limit(agent, Some(model), msg).await;
2323
2324 assert!(
2325 is_model_in_cooldown(agent, model),
2326 "model should be cooled for the relative usage window"
2327 );
2328 assert!(
2329 !is_agent_in_cooldown(agent),
2330 "agent should not be cooled for a model-scoped quota"
2331 );
2332
2333 let key = format!("{agent}:{model}");
2334 let remaining = cooldown_until(&key)
2335 .expect("model cooldown should exist")
2336 .saturating_sub(chrono::Utc::now().timestamp());
2337 assert!(
2338 remaining >= 5 * 3600 - 5,
2339 "model cooldown should be at least ~5 hours, got {remaining}s"
2340 );
2341 assert_eq!(
2342 cooldown_reason(&key).as_deref(),
2343 Some("rate_limit_usage_window"),
2344 "relative usage-window cooldown must be labeled as a rate limit, not silence"
2345 );
2346 }
2347
2348 #[serial(cooldown_state)]
2349 #[tokio::test]
2350 async fn record_rate_limit_without_window_falls_back_to_agent_and_model() {
2351 super::reset_global_state().await;
2352 let store = test_store().await;
2353 *cooldown_store().lock().await = Some(Arc::clone(&store));
2354
2355 let agent = "test_record_rl_fallback_agent";
2356 let model = "test_record_rl_fallback_model";
2357 let msg = "429 Too Many Requests";
2358
2359 record_rate_limit(agent, Some(model), msg).await;
2360
2361 assert!(
2362 is_agent_in_cooldown(agent),
2363 "agent should be cooled when no relative window is detected"
2364 );
2365 assert!(
2366 is_model_in_cooldown(agent, model),
2367 "model should be cooled when no relative window is detected"
2368 );
2369 }
2370
2371 #[serial(cooldown_state)]
2372 #[tokio::test]
2373 async fn agent_cooldown_persists_to_kv() {
2374 super::reset_global_state().await;
2375 let agent = "test_agent_persist_check";
2377 record_agent_failure_with_message(agent, "").await;
2378 assert!(is_agent_in_cooldown(agent));
2379 }
2380
2381 #[serial(cooldown_state)]
2382 #[tokio::test]
2383 async fn cooldown_never_shortens() {
2384 super::reset_global_state().await;
2385 let agent = "test_agent_no_shorten";
2386 set_agent_cooldown(agent, 24 * 60 * 60).await;
2387 let initial = {
2388 let map = cooldowns().lock().unwrap();
2389 map.get(agent)
2390 .expect("cooldown entry should exist")
2391 .cooldown_until
2392 };
2393
2394 record_agent_failure_with_message(agent, "rate limit").await;
2396 let after = {
2397 let map = cooldowns().lock().unwrap();
2398 map.get(agent)
2399 .expect("cooldown entry should exist")
2400 .cooldown_until
2401 };
2402 assert_eq!(
2403 initial, after,
2404 "cooldown should not be shortened by later failures"
2405 );
2406 }
2407
2408 #[serial(cooldown_state)]
2409 #[tokio::test]
2410 async fn silence_agent_cooldown_is_short_lived() {
2411 super::reset_global_state().await;
2412 let agent = "test_silence_agent_cd";
2413 assert!(!is_agent_in_cooldown(agent));
2414
2415 set_agent_cooldown(agent, SILENCE_AGENT_COOLDOWN_SECS).await;
2416 assert!(is_agent_in_cooldown(agent));
2417
2418 let map = cooldowns().lock().unwrap();
2420 let entry = map.get(agent).expect("should have cooldown entry");
2421 let now = chrono::Utc::now().timestamp();
2422 let remaining = entry.cooldown_until - now;
2423 assert!(
2424 remaining <= i64::try_from(SILENCE_AGENT_COOLDOWN_SECS).unwrap(),
2425 "silence agent cooldown should be <= {SILENCE_AGENT_COOLDOWN_SECS}s, got {remaining}s"
2426 );
2427 assert!(remaining > 0, "cooldown should still be active");
2428 }
2429
2430 #[serial(cooldown_state)]
2431 #[tokio::test]
2432 async fn record_silence_detection_applies_extended_cooldown() {
2433 super::reset_global_state().await;
2434 let store = test_store().await;
2435 *cooldown_store().lock().await = Some(Arc::clone(&store));
2436
2437 let agent = "test_silence_count_agent";
2438 let model = "test_silence_count_model";
2439 assert!(!is_model_in_cooldown(agent, model));
2440
2441 for _ in 0..SILENCE_COUNT_THRESHOLD {
2442 let result = record_silence_detection(agent, model).await;
2443 assert!(result.is_some());
2444 }
2445
2446 assert!(is_model_in_cooldown(agent, model));
2447 let key = format!("{agent}:{model}");
2448 let remaining = {
2449 let map = cooldowns().lock().unwrap();
2450 let entry = map.get(&key).expect("cooldown entry should exist");
2451 entry.cooldown_until - chrono::Utc::now().timestamp()
2452 };
2453 assert!(
2454 remaining >= i64::try_from(SILENCE_EXTENDED_COOLDOWN_SECS).unwrap() - 5,
2455 "extended cooldown should be applied, got {remaining}s"
2456 );
2457
2458 let kv_key = format!("{SILENCE_COUNT_PREFIX}{agent}:{model}");
2459 let stored = store.kv_get(&kv_key).await.unwrap().unwrap();
2460 assert_eq!(stored, "[]");
2461 }
2462
2463 #[serial(cooldown_state)]
2464 #[test]
2465 fn detect_credit_exhaustion_out_of_credits() {
2466 let msg = "Error: API error - {\"error\":{\"message\":\"You exceeded your current quota\",\"type\":\"insufficient_quota\",\"param\":null,\"code\":\"insufficient_quota\"}}";
2467 assert_eq!(
2468 detect_credit_exhaustion(msg),
2469 Some(CreditExhaustionReason::OutOfCredits)
2470 );
2471 }
2472
2473 #[serial(cooldown_state)]
2474 #[test]
2475 fn detect_credit_exhaustion_out_of_credits_variations() {
2476 assert_eq!(
2477 detect_credit_exhaustion("out of credits"),
2478 Some(CreditExhaustionReason::OutOfCredits)
2479 );
2480 assert_eq!(
2481 detect_credit_exhaustion("outofcredits"),
2482 Some(CreditExhaustionReason::OutOfCredits)
2483 );
2484 assert_eq!(
2485 detect_credit_exhaustion("no credits remaining"),
2486 Some(CreditExhaustionReason::OutOfCredits)
2487 );
2488 assert_eq!(
2489 detect_credit_exhaustion("insufficient funds"),
2490 Some(CreditExhaustionReason::OutOfCredits)
2491 );
2492 }
2493
2494 #[serial(cooldown_state)]
2495 #[test]
2496 fn detect_credit_exhaustion_org_level_disabled() {
2497 let msg = "Error: API error - {\"error\":{\"message\":\"Organization has been disabled\",\"type\":\"org_disabled\",\"code\":\"overageDisabledReason:org_level_disabled\"}}";
2498 assert_eq!(
2499 detect_credit_exhaustion(msg),
2500 Some(CreditExhaustionReason::OrgLevelDisabled)
2501 );
2502 }
2503
2504 #[serial(cooldown_state)]
2505 #[test]
2506 fn detect_credit_exhaustion_org_level_disabled_variations() {
2507 assert_eq!(
2508 detect_credit_exhaustion("organization has been disabled"),
2509 Some(CreditExhaustionReason::OrgLevelDisabled)
2510 );
2511 assert_eq!(
2512 detect_credit_exhaustion("org disabled"),
2513 Some(CreditExhaustionReason::OrgLevelDisabled)
2514 );
2515 assert_eq!(
2516 detect_credit_exhaustion("billing quota exceeded"),
2517 Some(CreditExhaustionReason::OrgLevelDisabled)
2518 );
2519 assert_eq!(
2520 detect_credit_exhaustion("credit balance too low"),
2521 Some(CreditExhaustionReason::OrgLevelDisabled)
2522 );
2523 }
2524
2525 #[serial(cooldown_state)]
2526 #[test]
2527 fn detect_credit_exhaustion_overage_disabled_reason_parsing() {
2528 let msg = "overageDisabledReason: out_of_credits";
2529 assert_eq!(
2530 detect_credit_exhaustion(msg),
2531 Some(CreditExhaustionReason::OutOfCredits)
2532 );
2533
2534 let msg = "overageDisabledReason: org_level_disabled";
2535 assert_eq!(
2536 detect_credit_exhaustion(msg),
2537 Some(CreditExhaustionReason::OrgLevelDisabled)
2538 );
2539 }
2540
2541 #[serial(cooldown_state)]
2542 #[test]
2543 fn detect_credit_exhaustion_glm_insufficient_balance() {
2544 assert_eq!(
2546 detect_credit_exhaustion(
2547 "API Error: Request rejected (429) · Insufficient balance or no resource package. Please recharge."
2548 ),
2549 Some(CreditExhaustionReason::OutOfCredits)
2550 );
2551 assert_eq!(
2552 detect_credit_exhaustion("Insufficient balance"),
2553 Some(CreditExhaustionReason::OutOfCredits)
2554 );
2555 assert_eq!(
2556 detect_credit_exhaustion("please recharge your account"),
2557 Some(CreditExhaustionReason::OutOfCredits)
2558 );
2559 assert_eq!(
2560 detect_credit_exhaustion("no resource package available"),
2561 Some(CreditExhaustionReason::OutOfCredits)
2562 );
2563 }
2564
2565 #[serial(cooldown_state)]
2566 #[test]
2567 fn detect_credit_exhaustion_none_for_regular_rate_limit() {
2568 assert!(detect_credit_exhaustion("rate limit exceeded").is_none());
2569 assert!(detect_credit_exhaustion("too many requests").is_none());
2570 assert!(detect_credit_exhaustion("429 Too Many Requests").is_none());
2571 assert!(detect_credit_exhaustion("you've hit your usage limit").is_none());
2572 assert!(
2573 detect_credit_exhaustion("You've hit your usage limit, try again in a few minutes.")
2574 .is_none()
2575 );
2576 }
2577
2578 #[serial(cooldown_state)]
2579 #[tokio::test]
2580 async fn record_credit_exhaustion_applies_agent_cooldown() {
2581 super::reset_global_state().await;
2582 let agent = "test_credit_exhaust_agent";
2583 assert!(!is_agent_in_cooldown(agent));
2584
2585 record_credit_exhaustion(agent, CreditExhaustionReason::OutOfCredits).await;
2586 assert!(is_agent_in_cooldown(agent));
2587
2588 let remaining = {
2589 let map = cooldowns().lock().unwrap();
2590 let entry = map.get(agent).expect("cooldown entry should exist");
2591 entry.cooldown_until - chrono::Utc::now().timestamp()
2592 };
2593 assert!(
2595 remaining >= CREDIT_BACKOFF_BASE_SECS - 5,
2596 "first out_of_credits cooldown should be ~1 hour, got {remaining}s"
2597 );
2598 assert!(
2599 remaining <= CREDIT_BACKOFF_MAX_SECS,
2600 "cooldown should not exceed cap of 8h, got {remaining}s"
2601 );
2602 }
2603
2604 #[serial(cooldown_state)]
2605 #[tokio::test]
2606 async fn record_credit_exhaustion_org_level_applies_longer_cooldown() {
2607 super::reset_global_state().await;
2608 let agent = "test_org_disabled_agent";
2609 assert!(!is_agent_in_cooldown(agent));
2610
2611 record_credit_exhaustion(agent, CreditExhaustionReason::OrgLevelDisabled).await;
2612 assert!(is_agent_in_cooldown(agent));
2613
2614 let remaining = {
2615 let map = cooldowns().lock().unwrap();
2616 let entry = map.get(agent).expect("cooldown entry should exist");
2617 entry.cooldown_until - chrono::Utc::now().timestamp()
2618 };
2619 assert!(
2621 remaining >= ORG_BACKOFF_BASE_SECS - 5,
2622 "first org-level disabled cooldown should be ~2 hours, got {remaining}s"
2623 );
2624 assert!(
2625 remaining <= CREDIT_BACKOFF_MAX_SECS,
2626 "cooldown should not exceed cap of 8h, got {remaining}s"
2627 );
2628 }
2629
2630 #[serial(cooldown_state)]
2631 #[tokio::test]
2632 async fn record_credit_exhaustion_billing_cycle_applies_24h() {
2633 super::reset_global_state().await;
2634 let store = test_store().await;
2635 *cooldown_store().lock().await = Some(Arc::clone(&store));
2636
2637 let agent = "test_billing_cycle_agent";
2638 assert!(!is_agent_in_cooldown(agent));
2639
2640 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2641 assert!(is_agent_in_cooldown(agent));
2642
2643 let remaining = {
2644 let map = cooldowns().lock().unwrap();
2645 let entry = map.get(agent).expect("cooldown entry should exist");
2646 entry.cooldown_until - chrono::Utc::now().timestamp()
2647 };
2648 assert!(
2650 remaining >= BILLING_CYCLE_COOLDOWN_SECS - 5,
2651 "first billing cycle cooldown should be ~24 hours, got {remaining}s"
2652 );
2653 assert!(
2654 remaining <= BILLING_CYCLE_MAX_SECS,
2655 "cooldown should not exceed cap of 7 days, got {remaining}s"
2656 );
2657 }
2658
2659 #[serial(cooldown_state)]
2660 #[tokio::test]
2661 async fn record_credit_exhaustion_billing_cycle_escalates() {
2662 super::reset_global_state().await;
2663 let store = test_store().await;
2664 *cooldown_store().lock().await = Some(Arc::clone(&store));
2665
2666 let agent = "test_billing_cycle_escalation";
2667
2668 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2670 let remaining_1 = {
2671 let map = cooldowns().lock().unwrap();
2672 let entry = map.get(agent).expect("cooldown entry should exist");
2673 entry.cooldown_until - chrono::Utc::now().timestamp()
2674 };
2675 assert!(
2676 (BILLING_CYCLE_COOLDOWN_SECS - 5..=BILLING_CYCLE_COOLDOWN_SECS + 5)
2677 .contains(&remaining_1),
2678 "first billing cycle cooldown should be ~24h, got {remaining_1}s"
2679 );
2680
2681 {
2683 let mut map = cooldowns().lock().unwrap();
2684 map.remove(agent);
2685 }
2686
2687 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2689 let remaining_2 = {
2690 let map = cooldowns().lock().unwrap();
2691 let entry = map.get(agent).expect("cooldown entry should exist");
2692 entry.cooldown_until - chrono::Utc::now().timestamp()
2693 };
2694 let expected_2 = BILLING_CYCLE_COOLDOWN_SECS * 3; assert!(
2696 remaining_2 >= expected_2 - 5,
2697 "second billing cycle cooldown should be ~72h, got {remaining_2}s"
2698 );
2699
2700 {
2702 let mut map = cooldowns().lock().unwrap();
2703 map.remove(agent);
2704 }
2705
2706 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2708 let remaining_3 = {
2709 let map = cooldowns().lock().unwrap();
2710 let entry = map.get(agent).expect("cooldown entry should exist");
2711 entry.cooldown_until - chrono::Utc::now().timestamp()
2712 };
2713 assert!(
2714 remaining_3 >= BILLING_CYCLE_MAX_SECS - 5,
2715 "third billing cycle cooldown should be capped at 7 days, got {remaining_3}s"
2716 );
2717 assert!(
2718 remaining_3 <= BILLING_CYCLE_MAX_SECS + 5,
2719 "third billing cycle cooldown should not exceed 7 days, got {remaining_3}s"
2720 );
2721 }
2722
2723 #[serial(cooldown_state)]
2724 #[tokio::test]
2725 async fn record_credit_exhaustion_billing_cycle_stays_capped_after_repeats() {
2726 super::reset_global_state().await;
2727 let store = test_store().await;
2728 *cooldown_store().lock().await = Some(Arc::clone(&store));
2729
2730 let agent = "test_billing_cycle_agent_capped";
2731
2732 for i in 0..5 {
2733 if i > 0 {
2734 cooldowns().lock().unwrap().remove(agent);
2736 }
2737 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2738 }
2739
2740 let remaining = {
2741 let map = cooldowns().lock().unwrap();
2742 let entry = map.get(agent).expect("cooldown entry should exist");
2743 entry.cooldown_until - chrono::Utc::now().timestamp()
2744 };
2745 assert!(
2746 remaining >= BILLING_CYCLE_MAX_SECS - 5,
2747 "billing cycle cooldown should stay capped at 7 days, got {remaining}s"
2748 );
2749 assert!(
2750 remaining <= BILLING_CYCLE_MAX_SECS + 5,
2751 "billing cycle cooldown should not exceed 7 days, got {remaining}s"
2752 );
2753 }
2754
2755 #[serial(cooldown_state)]
2756 #[test]
2757 fn credit_exhaustion_backoff_constants() {
2758 assert_eq!(
2760 compute_backoff(1, CREDIT_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS, true),
2761 CREDIT_BACKOFF_BASE_SECS
2762 );
2763 assert_eq!(
2765 compute_backoff(1, ORG_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS, true),
2766 ORG_BACKOFF_BASE_SECS
2767 );
2768 assert_eq!(BILLING_CYCLE_COOLDOWN_SECS, 24 * 60 * 60);
2770 assert_eq!(BILLING_CYCLE_MAX_SECS, 7 * 24 * 60 * 60);
2771 }
2772
2773 #[serial(cooldown_state)]
2774 #[test]
2775 fn compute_backoff_long_cap_without_extended_stays_capped() {
2776 assert_eq!(
2777 compute_backoff(
2778 3,
2779 BILLING_CYCLE_COOLDOWN_SECS,
2780 BILLING_CYCLE_MAX_SECS,
2781 false
2782 ),
2783 BILLING_CYCLE_MAX_SECS,
2784 "7-day capped paths should stop at the documented max"
2785 );
2786 assert_eq!(
2787 compute_backoff(
2788 9,
2789 PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2790 PERSISTENT_MODEL_BACKOFF_MAX_SECS,
2791 false,
2792 ),
2793 PERSISTENT_MODEL_BACKOFF_MAX_SECS,
2794 "persistent model failures should not extend past 7 days"
2795 );
2796 }
2797
2798 #[serial(cooldown_state)]
2801 #[test]
2802 fn mark_and_check_agent_degraded() {
2803 let agent = "test_degraded_mark";
2804 assert!(!is_agent_degraded(agent));
2805 mark_agent_degraded(agent);
2806 assert!(is_agent_degraded(agent));
2807 clear_agent_degraded(agent);
2808 assert!(!is_agent_degraded(agent));
2809 }
2810
2811 #[serial(cooldown_state)]
2812 #[tokio::test]
2813 async fn sync_from_kv_drops_in_memory_entry_cleared_externally() {
2814 super::reset_global_state().await;
2815 let store = test_store().await;
2816 let agent = "test_sync_external_clear";
2817 let future = chrono::Utc::now().timestamp() + 600;
2818
2819 init_cooldown_store(Arc::clone(&store)).await;
2821 set_cooldown_async(agent, future, "test").await;
2822 assert!(is_agent_in_cooldown(agent));
2823
2824 store
2828 .kv_set(&format!("{KV_PREFIX}{agent}"), "0")
2829 .await
2830 .expect("kv_set tombstone");
2831 assert!(
2832 is_agent_in_cooldown(agent),
2833 "in-memory cooldown should still be present before sync"
2834 );
2835
2836 sync_from_kv(&store).await;
2837
2838 assert!(
2839 !is_agent_in_cooldown(agent),
2840 "sync_from_kv must drop in-memory entries the KV tombstoned"
2841 );
2842 }
2843
2844 #[serial(cooldown_state)]
2845 #[tokio::test]
2846 async fn sync_from_kv_closes_github_circuit_when_row_cleared() {
2847 super::reset_global_state().await;
2848 let store = test_store().await;
2849 {
2850 *github_5xx_circuit_open_lock() = Some(chrono::Utc::now().timestamp() + 600);
2851 github_5xx_timestamps_lock().push(chrono::Utc::now().timestamp());
2852 }
2853 assert!(is_github_circuit_open());
2854
2855 store
2856 .kv_set("cooldown:github:5xx", "0")
2857 .await
2858 .expect("kv_set tombstone");
2859 sync_from_kv(&store).await;
2860
2861 assert!(!is_github_circuit_open());
2862 assert!(github_5xx_timestamps_lock().is_empty());
2863 }
2864
2865 #[serial(cooldown_state)]
2866 #[tokio::test]
2867 async fn sync_from_kv_keeps_5xx_window_when_expired_row_and_breaker_closed() {
2868 super::reset_global_state().await;
2869 let store = test_store().await;
2870 let now = chrono::Utc::now().timestamp();
2871 {
2872 github_5xx_timestamps_lock().push(now);
2873 github_5xx_timestamps_lock().push(now);
2874 }
2875
2876 store
2877 .kv_set("cooldown:github:5xx", &(now - 600).to_string())
2878 .await
2879 .expect("kv_set past");
2880 sync_from_kv(&store).await;
2881
2882 assert!(!is_github_circuit_open());
2883 assert_eq!(github_5xx_timestamps_lock().len(), 2);
2884
2885 github_5xx_timestamps_lock().clear();
2886 }
2887
2888 #[serial(cooldown_state)]
2889 #[tokio::test]
2890 async fn sync_from_kv_keeps_github_circuit_open_for_future_row() {
2891 super::reset_global_state().await;
2892 let store = test_store().await;
2893 let future = chrono::Utc::now().timestamp() + 600;
2894 *github_5xx_circuit_open_lock() = Some(future);
2895
2896 store
2897 .kv_set("cooldown:github:5xx", &future.to_string())
2898 .await
2899 .expect("kv_set future");
2900 sync_from_kv(&store).await;
2901
2902 assert!(is_github_circuit_open());
2903
2904 *github_5xx_circuit_open_lock() = None;
2905 }
2906
2907 #[serial(cooldown_state)]
2908 #[tokio::test]
2909 async fn sync_from_kv_keeps_in_memory_entry_when_kv_row_missing() {
2910 super::reset_global_state().await;
2914 let store = test_store().await;
2915 let agent = "test_sync_race_window";
2916 let future = chrono::Utc::now().timestamp() + 600;
2917
2918 set_cooldown_in_memory(agent, future, "raced");
2920 assert!(is_agent_in_cooldown(agent));
2921
2922 sync_from_kv(&store).await;
2923
2924 assert!(
2925 is_agent_in_cooldown(agent),
2926 "sync_from_kv must NOT drop entries whose KV row is missing — that is the set_cooldown_async race window"
2927 );
2928 }
2929
2930 #[serial(cooldown_state)]
2931 #[tokio::test]
2932 async fn sync_from_kv_adopts_kv_timestamp() {
2933 super::reset_global_state().await;
2936 let store = test_store().await;
2937 let agent = "test_sync_adopt_kv";
2938 let original = chrono::Utc::now().timestamp() + 600;
2939 let updated = chrono::Utc::now().timestamp() + 1200;
2940
2941 init_cooldown_store(Arc::clone(&store)).await;
2942 set_cooldown_async(agent, original, "initial").await;
2943
2944 store
2945 .kv_set(&format!("{KV_PREFIX}{agent}"), &updated.to_string())
2946 .await
2947 .expect("kv_set updated");
2948
2949 sync_from_kv(&store).await;
2950
2951 let until = cooldown_until(agent).expect("cooldown still active");
2952 assert_eq!(
2953 until, updated,
2954 "sync_from_kv must adopt KV's authoritative timestamp"
2955 );
2956 }
2957
2958 #[serial(cooldown_state)]
2959 #[tokio::test]
2960 async fn refresh_degraded_agents_marks_cooled_agent() {
2961 super::reset_global_state().await;
2962 let store = test_store().await;
2963 let agent = "test_refresh_cooled";
2964
2965 record_agent_failure_with_message(agent, "").await;
2967 assert!(is_agent_in_cooldown(agent));
2968
2969 let agents = vec![agent.to_string()];
2971 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
2972
2973 assert!(
2974 is_agent_degraded(agent),
2975 "agent in cooldown should be marked degraded"
2976 );
2977
2978 clear_agent_degraded(agent);
2980 }
2981
2982 #[serial(cooldown_state)]
2983 #[tokio::test]
2984 async fn refresh_degraded_agents_marks_no_models_agent() {
2985 super::reset_global_state().await;
2986 let store = test_store().await;
2987 let agent = "test_refresh_no_models";
2988
2989 let agents = vec![agent.to_string()];
2991 refresh_degraded_agents(&store, &agents, &|_| false, 6, 3).await;
2992
2993 assert!(
2994 is_agent_degraded(agent),
2995 "agent with no available models should be marked degraded"
2996 );
2997
2998 clear_agent_degraded(agent);
3000 }
3001
3002 #[serial(cooldown_state)]
3003 #[tokio::test]
3004 async fn refresh_degraded_agents_marks_rate_limited_agent() {
3005 super::reset_global_state().await;
3006 let store = test_store().await;
3007 let agent = "test_refresh_rate_limited";
3008
3009 for _ in 0..4 {
3011 store.record_rate_limit(agent, "rate", None).await.unwrap();
3012 }
3013
3014 let agents = vec![agent.to_string()];
3015 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3017
3018 assert!(
3019 is_agent_degraded(agent),
3020 "agent with rate limit count >= threshold should be marked degraded"
3021 );
3022
3023 clear_agent_degraded(agent);
3025 }
3026
3027 #[serial(cooldown_state)]
3028 #[tokio::test]
3029 async fn refresh_degraded_agents_clears_healthy_agent() {
3030 super::reset_global_state().await;
3031 let store = test_store().await;
3032 let agent = "test_refresh_healthy";
3033
3034 mark_agent_degraded(agent);
3036 assert!(is_agent_degraded(agent));
3037
3038 let agents = vec![agent.to_string()];
3040 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3041
3042 assert!(
3043 !is_agent_degraded(agent),
3044 "healthy agent should have degraded flag cleared"
3045 );
3046 }
3047
3048 #[serial(cooldown_state)]
3049 #[tokio::test]
3050 async fn refresh_degraded_agents_ignores_expired_cooldowns() {
3051 super::reset_global_state().await;
3052 let store = test_store().await;
3053 let agent = "test_refresh_expired_cooldown";
3054
3055 let expired_ts = chrono::Utc::now().timestamp() - 100; {
3059 let mut map = cooldowns().lock().unwrap();
3060 map.insert(
3061 agent.to_string(),
3062 CooldownEntry {
3063 cooldown_until: expired_ts,
3064 reason: "expired_test".to_string(),
3065 },
3066 );
3067 }
3068
3069 assert!(
3071 !is_agent_in_cooldown(agent),
3072 "expired cooldown should not be active"
3073 );
3074
3075 mark_agent_degraded(agent);
3077 assert!(is_agent_degraded(agent));
3078
3079 let agents = vec![agent.to_string()];
3081 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3082
3083 assert!(
3084 !is_agent_degraded(agent),
3085 "agent with only expired cooldown should NOT be marked degraded"
3086 );
3087
3088 clear_agent_degraded(agent);
3090 {
3091 let mut map = cooldowns().lock().unwrap();
3092 map.remove(agent);
3093 }
3094 }
3095
3096 #[serial(cooldown_state)]
3097 #[test]
3098 fn compute_backoff_grows_exponentially() {
3099 assert_eq!(compute_backoff(0, 300, 14400, true), 300);
3101 assert_eq!(compute_backoff(1, 300, 14400, true), 300);
3102 assert_eq!(compute_backoff(2, 300, 14400, true), 900);
3104 assert_eq!(compute_backoff(3, 300, 14400, true), 2700);
3106 assert_eq!(compute_backoff(4, 300, 14400, true), 8100);
3108 assert_eq!(compute_backoff(5, 300, 14400, true), 14400);
3110 }
3111
3112 #[serial(cooldown_state)]
3113 #[test]
3114 fn compute_backoff_extended_tier_1_applies_24h() {
3115 assert_eq!(
3121 compute_backoff(5, 300, 14400, true),
3122 14400,
3123 "first at max, not extended"
3124 );
3125 assert_eq!(
3126 compute_backoff(6, 300, 14400, true),
3127 14400,
3128 "times_at_max=1, not extended"
3129 );
3130 assert_eq!(
3131 compute_backoff(7, 300, 14400, true),
3132 86400,
3133 "times_at_max=2, extended 6x"
3134 );
3135 assert_eq!(
3136 compute_backoff(8, 300, 14400, true),
3137 86400,
3138 "times_at_max=3, still extended 6x"
3139 );
3140 }
3141
3142 #[serial(cooldown_state)]
3143 #[test]
3144 fn compute_backoff_extended_tier_2_applies_48h() {
3145 assert_eq!(
3148 compute_backoff(9, 300, 14400, true),
3149 86400,
3150 "times_at_max=4, still tier 1"
3151 );
3152 assert_eq!(
3153 compute_backoff(10, 300, 14400, true),
3154 172_800,
3155 "times_at_max=5, extended 12x"
3156 );
3157 assert_eq!(
3158 compute_backoff(15, 300, 14400, true),
3159 172_800,
3160 "times_at_max=10, still tier 2"
3161 );
3162 assert_eq!(
3163 compute_backoff(20, 300, 14400, true),
3164 172_800,
3165 "times_at_max=15, still tier 2"
3166 );
3167 }
3168
3169 #[serial(cooldown_state)]
3170 #[test]
3171 fn compute_backoff_extended_tier_with_different_base_max() {
3172 assert_eq!(compute_backoff(5, 60, 3600, true), 3600, "first at max");
3179 assert_eq!(compute_backoff(6, 60, 3600, true), 3600, "times_at_max=1");
3180 assert_eq!(
3181 compute_backoff(7, 60, 3600, true),
3182 21600,
3183 "times_at_max=2, extended 6x"
3184 );
3185 }
3186
3187 #[serial(cooldown_state)]
3188 #[test]
3189 fn compute_backoff_extended_tier_persistent_fails_example() {
3190 assert_eq!(
3194 compute_backoff(21, 300, 14400, true),
3195 172_800,
3196 "21 failures should get 48h extended cooldown"
3197 );
3198
3199 assert_eq!(
3201 compute_backoff(36, 300, 14400, true),
3202 172_800,
3203 "36 failures should also get 48h extended cooldown"
3204 );
3205 }
3206
3207 #[serial(cooldown_state)]
3208 #[test]
3209 fn compute_backoff_extended_tier_zero_base() {
3210 assert_eq!(compute_backoff(1, 0, 14400, true), 0);
3212 assert_eq!(compute_backoff(5, 0, 14400, true), 0);
3213 assert_eq!(compute_backoff(10, 0, 14400, true), 0);
3214 }
3215
3216 #[serial(cooldown_state)]
3217 #[test]
3218 fn detect_credit_exhaustion_billing_cycle() {
3219 assert_eq!(
3220 detect_credit_exhaustion("billing cycle"),
3221 Some(CreditExhaustionReason::BillingCycleExhausted)
3222 );
3223 assert_eq!(
3224 detect_credit_exhaustion("quota refreshed next cycle"),
3225 Some(CreditExhaustionReason::BillingCycleExhausted)
3226 );
3227 assert_eq!(
3228 detect_credit_exhaustion(
3229 "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle."
3230 ),
3231 Some(CreditExhaustionReason::BillingCycleExhausted)
3232 );
3233 assert_eq!(
3234 detect_credit_exhaustion("monthly quota exceeded"),
3235 Some(CreditExhaustionReason::BillingCycleExhausted)
3236 );
3237 assert_eq!(
3238 detect_credit_exhaustion(
3239 "Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-04-23 18:33:48"
3240 ),
3241 Some(CreditExhaustionReason::BillingCycleExhausted)
3242 );
3243 }
3244
3245 #[serial(cooldown_state)]
3246 #[test]
3247 fn detect_credit_exhaustion_billing_cycle_minimax() {
3248 assert_eq!(
3249 detect_credit_exhaustion(
3250 "minimax rate limit: API Error: Request rejected (429) · Token Plan usage limit reached: Upgrade your Token Plan or purchase Credits for more usage."
3251 ),
3252 Some(CreditExhaustionReason::BillingCycleExhausted)
3253 );
3254 assert_eq!(
3255 detect_credit_exhaustion("Upgrade your Token Plan"),
3256 Some(CreditExhaustionReason::BillingCycleExhausted)
3257 );
3258 assert_eq!(
3259 detect_credit_exhaustion("Token Plan usage limit reached"),
3260 Some(CreditExhaustionReason::BillingCycleExhausted)
3261 );
3262 assert_eq!(
3263 detect_credit_exhaustion("purchase Credits for more usage"),
3264 Some(CreditExhaustionReason::BillingCycleExhausted)
3265 );
3266 assert_eq!(
3267 detect_credit_exhaustion(
3268 "You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro), visit https://chatgpt.com/codex/settings/usage to purchase more credits"
3269 ),
3270 Some(CreditExhaustionReason::BillingCycleExhausted)
3271 );
3272 }
3273
3274 #[serial(cooldown_state)]
3275 #[test]
3276 fn detect_credit_exhaustion_outcome_kimi_billing_cycle() {
3277 let msg = "Failed to authenticate. API Error: 403 You've reached your usage limit for \
3282 this billing cycle. Your quota will be refreshed soon.";
3283 assert_eq!(
3284 detect_credit_exhaustion_outcome(msg),
3285 Some("billing_cycle_exhausted")
3286 );
3287 }
3288
3289 #[serial(cooldown_state)]
3290 #[test]
3291 fn detect_credit_exhaustion_outcome_maps_all_reasons() {
3292 assert_eq!(
3293 detect_credit_exhaustion_outcome("out of credits"),
3294 Some("credit_exhausted")
3295 );
3296 assert_eq!(
3297 detect_credit_exhaustion_outcome("organization has been disabled"),
3298 Some("org_level_disabled")
3299 );
3300 assert_eq!(
3301 detect_credit_exhaustion_outcome("monthly quota exceeded"),
3302 Some("billing_cycle_exhausted")
3303 );
3304 }
3305
3306 #[test]
3307 fn detect_credit_exhaustion_outcome_returns_none_for_unrelated_errors() {
3308 assert_eq!(
3309 detect_credit_exhaustion_outcome("429 Too Many Requests"),
3310 None
3311 );
3312 assert_eq!(detect_credit_exhaustion_outcome("connection reset"), None);
3313 }
3314
3315 #[serial(cooldown_state)]
3316 #[test]
3317 fn detect_credit_exhaustion_overage_takes_priority_over_billing_cycle() {
3318 let msg = "overageDisabledReason: org_level_disabled for billing cycle";
3321 assert_eq!(
3322 detect_credit_exhaustion(msg),
3323 Some(CreditExhaustionReason::OrgLevelDisabled)
3324 );
3325 }
3326
3327 #[serial(cooldown_state)]
3328 #[tokio::test]
3329 async fn refresh_degraded_agents_below_threshold_not_degraded() {
3330 super::reset_global_state().await;
3331 let store = test_store().await;
3332 let agent = "test_refresh_below_threshold";
3333
3334 for _ in 0..2 {
3336 store.record_rate_limit(agent, "rate", None).await.unwrap();
3337 }
3338
3339 let agents = vec![agent.to_string()];
3340 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3342
3343 assert!(
3344 !is_agent_degraded(agent),
3345 "agent with rate limit count < threshold should not be degraded"
3346 );
3347 }
3348
3349 #[serial(cooldown_state)]
3350 #[tokio::test]
3351 async fn refresh_degraded_agents_ignores_error_limit_type() {
3352 super::reset_global_state().await;
3353 let store = test_store().await;
3354 let agent = "test_refresh_error_rows";
3355
3356 for _ in 0..3 {
3357 store.record_rate_limit(agent, "error", None).await.unwrap();
3358 }
3359
3360 let agents = vec![agent.to_string()];
3361 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3362
3363 assert!(
3364 !is_agent_degraded(agent),
3365 "three error rows in 6 hours must not mark an agent degraded"
3366 );
3367 }
3368
3369 #[serial(cooldown_state)]
3370 #[tokio::test]
3371 async fn refresh_degraded_agents_marks_three_rate_rows() {
3372 super::reset_global_state().await;
3373 let store = test_store().await;
3374 let agent = "test_refresh_three_rate_rows";
3375
3376 for _ in 0..3 {
3377 store.record_rate_limit(agent, "rate", None).await.unwrap();
3378 }
3379
3380 let agents = vec![agent.to_string()];
3381 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3382
3383 assert!(
3384 is_agent_degraded(agent),
3385 "three rate rows in 6 hours must mark an agent degraded"
3386 );
3387 assert_eq!(
3388 agent_degraded_reason(agent).as_deref(),
3389 Some("rate limit threshold exceeded")
3390 );
3391 let listed = list_degraded_agents();
3392 assert!(
3393 listed
3394 .iter()
3395 .any(|(a, r)| a == agent && r == "rate limit threshold exceeded"),
3396 "orch cooldown list must show the degraded reason, got {listed:?}"
3397 );
3398
3399 clear_agent_degraded(agent);
3400 }
3401
3402 #[serial(cooldown_state)]
3403 #[tokio::test]
3404 async fn refresh_degraded_agents_logs_reason_change() {
3405 super::reset_global_state().await;
3406 let store = test_store().await;
3407 let agent = "test_refresh_reason_change";
3408 *cooldown_store().lock().await = Some(Arc::clone(&store));
3409
3410 set_agent_cooldown(agent, 600).await;
3411 let agents = vec![agent.to_string()];
3412 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3413 assert_eq!(
3414 agent_degraded_reason(agent).as_deref(),
3415 Some("agent in cooldown")
3416 );
3417
3418 clear_cooldown(agent, &store).await;
3419 let after_clear = sqlite_datetime_offset(60);
3420 for _ in 0..3 {
3421 store
3422 .record_rate_limit_at(agent, "rate", None, &after_clear)
3423 .await
3424 .unwrap();
3425 }
3426 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3427 assert_eq!(
3428 agent_degraded_reason(agent).as_deref(),
3429 Some("rate limit threshold exceeded"),
3430 "degraded reason must update when the hold changes"
3431 );
3432
3433 clear_agent_degraded(agent);
3434 }
3435
3436 #[serial(cooldown_state)]
3437 #[tokio::test]
3438 async fn clear_un_degrades_on_next_health_check() {
3439 super::reset_global_state().await;
3440 let store = test_store().await;
3441 let agent = "test_clear_un_degrades";
3442 let before = sqlite_datetime_offset(-30);
3443 let after = sqlite_datetime_offset(60);
3444
3445 for _ in 0..3 {
3446 store
3447 .record_rate_limit_at(agent, "rate", None, &before)
3448 .await
3449 .unwrap();
3450 }
3451
3452 let agents = vec![agent.to_string()];
3453 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3454 assert!(
3455 is_agent_degraded(agent),
3456 "three rate rows must mark an agent degraded"
3457 );
3458
3459 clear_cooldown(agent, &store).await;
3460 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3461 assert!(
3462 !is_agent_degraded(agent),
3463 "clear must un-degrade on the next health check"
3464 );
3465
3466 store
3467 .record_rate_limit_at(agent, "rate", None, &after)
3468 .await
3469 .unwrap();
3470 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3471 assert!(
3472 !is_agent_degraded(agent),
3473 "one rate row after the mark must not degrade again"
3474 );
3475
3476 for _ in 0..2 {
3477 store
3478 .record_rate_limit_at(agent, "rate", None, &after)
3479 .await
3480 .unwrap();
3481 }
3482 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3483 assert!(
3484 is_agent_degraded(agent),
3485 "three rate rows after the mark must degrade again"
3486 );
3487
3488 clear_agent_degraded(agent);
3489 }
3490
3491 #[serial(cooldown_state)]
3492 #[tokio::test]
3493 async fn clear_all_un_degrades_rate_limited_agent() {
3494 super::reset_global_state().await;
3495 let store = test_store().await;
3496 let agent = "test_clear_all_un_degrades";
3497 let before = sqlite_datetime_offset(-30);
3498
3499 for _ in 0..3 {
3500 store
3501 .record_rate_limit_at(agent, "rate", None, &before)
3502 .await
3503 .unwrap();
3504 }
3505
3506 let agents = vec![agent.to_string()];
3507 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3508 assert!(is_agent_degraded(agent));
3509
3510 clear_cooldown("*", &store).await;
3511 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3512 assert!(
3513 !is_agent_degraded(agent),
3514 "clear --all must un-degrade an agent held only by rate rows"
3515 );
3516 }
3517
3518 #[serial(cooldown_state)]
3521 #[tokio::test]
3522 async fn repeated_agent_failures_escalate_backoff() {
3523 super::reset_global_state().await;
3524 let store = test_store().await;
3525 *cooldown_store().lock().await = Some(Arc::clone(&store));
3526
3527 let agent = "test_escalation_agent";
3528
3529 record_agent_failure_with_message(agent, "").await;
3531 let remaining_1 = {
3532 let map = cooldowns().lock().unwrap();
3533 let entry = map.get(agent).expect("should have cooldown");
3534 entry.cooldown_until - chrono::Utc::now().timestamp()
3535 };
3536 assert!(
3537 (BACKOFF_BASE_SECS - 5..=BACKOFF_BASE_SECS + 5).contains(&remaining_1),
3538 "first failure should be ~5 min, got {remaining_1}s"
3539 );
3540
3541 {
3543 let mut map = cooldowns().lock().unwrap();
3544 map.remove(agent);
3545 }
3546
3547 record_agent_failure_with_message(agent, "").await;
3549 let remaining_2 = {
3550 let map = cooldowns().lock().unwrap();
3551 let entry = map.get(agent).expect("should have cooldown");
3552 entry.cooldown_until - chrono::Utc::now().timestamp()
3553 };
3554 assert!(
3555 remaining_2 >= BACKOFF_BASE_SECS * 3 - 5,
3556 "second failure should be ~15 min, got {remaining_2}s"
3557 );
3558 }
3559
3560 #[serial(cooldown_state)]
3561 #[tokio::test]
3562 async fn vendor_past_retry_falls_back_to_backoff() {
3563 super::reset_global_state().await;
3564 let store = test_store().await;
3568 *cooldown_store().lock().await = Some(Arc::clone(&store));
3569
3570 let agent = "test_vendor_past_retry";
3571
3572 {
3574 let mut map = cooldowns().lock().unwrap();
3575 map.remove(agent);
3576 }
3577
3578 let past_msg = "You've hit the limit; try again at Jan 01, 2000 01:00 AM.";
3580
3581 record_agent_failure_with_message(agent, past_msg).await;
3582
3583 assert!(
3587 is_agent_in_cooldown(agent),
3588 "agent should be in cooldown after failure"
3589 );
3590
3591 let remaining = {
3592 let map = cooldowns().lock().unwrap();
3593 let entry = map.get(agent).expect("cooldown entry should exist");
3594 entry.cooldown_until - chrono::Utc::now().timestamp()
3595 };
3596
3597 assert!(
3598 remaining >= BACKOFF_BASE_SECS - 5,
3599 "expected backoff of ~{BACKOFF_BASE_SECS}s when vendor timestamp is past, got {remaining}s"
3600 );
3601
3602 let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3604 let stored = store
3605 .kv_get(&kv_key)
3606 .await
3607 .unwrap()
3608 .expect("failure count should be stored");
3609 assert_eq!(stored, "1");
3610 }
3611
3612 fn window(status: &str, resets_at: Option<i64>) -> RateLimitWindow {
3613 RateLimitWindow {
3614 status: status.to_string(),
3615 window_type: "five_hour".to_string(),
3616 utilization: Some(1.0),
3617 resets_at,
3618 }
3619 }
3620
3621 #[serial(cooldown_state)]
3622 #[tokio::test]
3623 async fn rejected_window_cools_agent_until_reset() {
3624 super::reset_global_state().await;
3625 let store = test_store().await;
3626 *cooldown_store().lock().await = Some(Arc::clone(&store));
3627
3628 let agent = "test_window_rejected_agent";
3629 let reset = chrono::Utc::now().timestamp() + 7200;
3630 record_rate_limit_window(agent, &window("rejected", Some(reset))).await;
3631
3632 assert_eq!(cooldown_until(agent), Some(reset));
3633 let stored = list_rate_limit_windows(&store).await;
3634 assert_eq!(
3635 stored,
3636 vec![(agent.to_string(), window("rejected", Some(reset)))]
3637 );
3638 }
3639
3640 #[serial(cooldown_state)]
3641 #[tokio::test]
3642 async fn allowed_window_changes_no_cooldown() {
3643 super::reset_global_state().await;
3644 let store = test_store().await;
3645 *cooldown_store().lock().await = Some(Arc::clone(&store));
3646
3647 let agent = "test_window_allowed_agent";
3648 let reset = chrono::Utc::now().timestamp() + 7200;
3649 record_rate_limit_window(agent, &window("allowed", Some(reset))).await;
3650 record_rate_limit_window(agent, &window("allowed_warning", Some(reset))).await;
3651
3652 assert_eq!(cooldown_until(agent), None);
3653 assert_eq!(list_rate_limit_windows(&store).await.len(), 1);
3654 }
3655
3656 #[serial(cooldown_state)]
3657 #[tokio::test]
3658 async fn record_agent_success_resets_backoff() {
3659 super::reset_global_state().await;
3660 let store = test_store().await;
3661 *cooldown_store().lock().await = Some(Arc::clone(&store));
3662
3663 let agent = "test_success_reset_agent";
3664 let model = "test_model";
3665
3666 for _ in 0..3 {
3668 let _ = read_and_increment_failure_count(Some(&store), agent).await;
3669 }
3670 let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3672 let count: u32 = store
3673 .kv_get(&kv_key)
3674 .await
3675 .unwrap()
3676 .unwrap()
3677 .parse()
3678 .unwrap();
3679 assert_eq!(count, 3);
3680
3681 record_agent_success(agent, model).await;
3683
3684 let count_after: u32 = store
3685 .kv_get(&kv_key)
3686 .await
3687 .unwrap()
3688 .unwrap()
3689 .parse()
3690 .unwrap();
3691 assert_eq!(count_after, 0, "success should reset failure count to 0");
3692
3693 let model_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}:{model}");
3695 let model_count: u32 = store
3696 .kv_get(&model_kv_key)
3697 .await
3698 .unwrap()
3699 .unwrap()
3700 .parse()
3701 .unwrap();
3702 assert_eq!(
3703 model_count, 0,
3704 "success should reset model failure count to 0"
3705 );
3706 }
3707
3708 #[serial(cooldown_state)]
3709 #[tokio::test]
3710 async fn record_agent_success_without_model_skips_empty_model_key() {
3711 super::reset_global_state().await;
3712 let store = test_store().await;
3713 *cooldown_store().lock().await = Some(Arc::clone(&store));
3714
3715 let agent = "test_success_without_model";
3716
3717 record_agent_success(agent, "").await;
3718
3719 let agent_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3720 let agent_count = store
3721 .kv_get(&agent_kv_key)
3722 .await
3723 .unwrap()
3724 .expect("agent failure count should be stored");
3725 assert_eq!(agent_count, "0");
3726
3727 let empty_model_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}:");
3728 let empty_model_count = store.kv_get(&empty_model_kv_key).await.unwrap();
3729 assert!(
3730 empty_model_count.is_none(),
3731 "success without model should not create an empty model failure count key"
3732 );
3733 }
3734
3735 #[serial(cooldown_state)]
3736 #[tokio::test]
3737 async fn clear_cooldown_resets_failure_counts() {
3738 super::reset_global_state().await;
3739 let store = test_store().await;
3740 *cooldown_store().lock().await = Some(Arc::clone(&store));
3741
3742 let agent = "test_clear_fc_agent";
3743
3744 for _ in 0..5 {
3746 let _ = read_and_increment_failure_count(Some(&store), agent).await;
3747 }
3748 set_agent_cooldown(agent, 3600).await;
3750
3751 let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3753 let count: u32 = store
3754 .kv_get(&kv_key)
3755 .await
3756 .unwrap()
3757 .unwrap()
3758 .parse()
3759 .unwrap();
3760 assert_eq!(count, 5);
3761
3762 clear_cooldown(agent, &store).await;
3764
3765 let count_after: u32 = store
3767 .kv_get(&kv_key)
3768 .await
3769 .unwrap()
3770 .unwrap()
3771 .parse()
3772 .unwrap();
3773 assert_eq!(
3774 count_after, 0,
3775 "clear_cooldown should reset failure count to 0"
3776 );
3777
3778 assert!(
3780 !is_agent_in_cooldown(agent),
3781 "agent should not be in cooldown after clear"
3782 );
3783 }
3784
3785 #[serial(cooldown_state)]
3786 #[tokio::test]
3787 async fn clear_all_cooldowns_resets_all_failure_counts() {
3788 super::reset_global_state().await;
3789 let store = test_store().await;
3790 *cooldown_store().lock().await = Some(Arc::clone(&store));
3791
3792 let agents = ["test_clear_all_a", "test_clear_all_b"];
3793 for agent in &agents {
3794 let _ = read_and_increment_failure_count(Some(&store), agent).await;
3795 set_agent_cooldown(agent, 3600).await;
3796 }
3797
3798 clear_cooldown("*", &store).await;
3799
3800 for agent in &agents {
3801 assert!(!is_agent_in_cooldown(agent));
3802 let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3803 let count: u32 = store
3804 .kv_get(&kv_key)
3805 .await
3806 .unwrap()
3807 .unwrap()
3808 .parse()
3809 .unwrap();
3810 assert_eq!(
3811 count, 0,
3812 "clear --all should reset failure count for {agent}"
3813 );
3814 }
3815 }
3816
3817 #[serial(cooldown_state)]
3818 #[test]
3819 fn compute_backoff_saturates_on_large_counts() {
3820 assert_eq!(
3822 compute_backoff(100, BACKOFF_BASE_SECS, BACKOFF_MAX_SECS, true),
3823 BACKOFF_MAX_SECS * 12
3824 );
3825 assert_eq!(
3826 compute_backoff(u64::from(u32::MAX), 300, 14400, true),
3827 172_800
3828 );
3829 assert_eq!(compute_backoff(u64::MAX, 300, 14400, true), 172_800);
3830 }
3831
3832 #[serial(cooldown_state)]
3833 #[test]
3834 fn compute_backoff_zero_base_returns_zero() {
3835 assert_eq!(compute_backoff(1, 0, 14400, true), 0);
3836 assert_eq!(compute_backoff(5, 0, 14400, true), 0);
3837 }
3838
3839 #[serial(cooldown_state)]
3842 #[tokio::test]
3843 async fn github_circuit_stays_closed_below_threshold() {
3844 super::reset_global_state().await;
3845 {
3847 let mut open = github_5xx_circuit_open().lock().unwrap();
3848 *open = None;
3849 let mut ts = github_5xx_timestamps().lock().unwrap();
3850 ts.clear();
3851 let mut map = cooldown_lock();
3855 map.remove("github:5xx");
3856 }
3857
3858 for _ in 0..GITHUB_5XX_THRESHOLD - 1 {
3860 record_github_5xx().await;
3861 }
3862
3863 assert!(
3864 !is_github_circuit_open(),
3865 "circuit should stay closed below threshold"
3866 );
3867 assert_eq!(github_circuit_remaining_secs(), 0);
3868 }
3869
3870 #[serial(cooldown_state)]
3871 #[tokio::test]
3872 async fn github_circuit_trips_at_threshold() {
3873 super::reset_global_state().await;
3874 {
3876 let mut open = github_5xx_circuit_open_lock();
3877 *open = None;
3878 let mut ts = github_5xx_timestamps_lock();
3879 ts.clear();
3880 let mut map = cooldown_lock();
3881 map.remove("github:5xx");
3882 }
3883
3884 for _ in 0..GITHUB_5XX_THRESHOLD {
3886 record_github_5xx().await;
3887 }
3888
3889 assert!(is_github_circuit_open(), "circuit should trip at threshold");
3890 assert!(
3891 github_circuit_remaining_secs() > 0,
3892 "remaining should be > 0 when open"
3893 );
3894 assert!(
3895 github_circuit_remaining_secs() <= GITHUB_5XX_COOLDOWN_SECS as u64,
3896 "remaining should not exceed cooldown duration"
3897 );
3898
3899 {
3901 let mut open = github_5xx_circuit_open().lock().unwrap();
3902 *open = None;
3903 }
3904 }
3905
3906 #[serial(cooldown_state)]
3907 #[tokio::test]
3908 async fn github_circuit_cannot_double_trip() {
3909 super::reset_global_state().await;
3910 {
3912 let mut open = github_5xx_circuit_open_lock();
3913 *open = None;
3914 let mut ts = github_5xx_timestamps_lock();
3915 ts.clear();
3916 let mut map = cooldown_lock();
3917 map.remove("github:5xx");
3918 }
3919
3920 for _ in 0..GITHUB_5XX_THRESHOLD {
3922 record_github_5xx().await;
3923 }
3924 assert!(is_github_circuit_open());
3925
3926 let first_remaining = github_circuit_remaining_secs();
3927
3928 for _ in 0..GITHUB_5XX_THRESHOLD {
3930 record_github_5xx().await;
3931 }
3932
3933 let second_remaining = github_circuit_remaining_secs();
3934 assert!(
3935 second_remaining <= first_remaining,
3936 "additional errors should not extend cooldown"
3937 );
3938
3939 {
3941 let mut open = github_5xx_circuit_open().lock().unwrap();
3942 *open = None;
3943 }
3944 }
3945
3946 #[serial(cooldown_state)]
3947 #[tokio::test]
3948 async fn github_circuit_recovers_after_cooldown() {
3949 super::reset_global_state().await;
3950 {
3952 let mut open = github_5xx_circuit_open_lock();
3953 *open = None;
3954 let mut ts = github_5xx_timestamps_lock();
3955 ts.clear();
3956 let mut map = cooldown_lock();
3957 map.remove("github:5xx");
3958 }
3959
3960 {
3962 let mut open = github_5xx_circuit_open().lock().unwrap();
3963 *open = Some(chrono::Utc::now().timestamp() - 1);
3964 }
3965
3966 assert!(
3968 !is_github_circuit_open(),
3969 "circuit should auto-recover after cooldown"
3970 );
3971 }
3972
3973 #[serial(cooldown_state)]
3974 #[tokio::test]
3975 async fn github_circuit_sliding_window_ages_out() {
3976 super::reset_global_state().await;
3977 {
3979 let mut open = github_5xx_circuit_open_lock();
3980 *open = None;
3981 let mut ts = github_5xx_timestamps_lock();
3982 ts.clear();
3983 let mut map = cooldown_lock();
3984 map.remove("github:5xx");
3985 }
3986
3987 {
3989 let mut ts = github_5xx_timestamps().lock().unwrap();
3990 let old = chrono::Utc::now().timestamp() - GITHUB_5XX_WINDOW_SECS - 10;
3991 for _ in 0..GITHUB_5XX_THRESHOLD {
3992 ts.push(old);
3993 }
3994 }
3995
3996 record_github_5xx().await;
3998
3999 assert!(
4000 !is_github_circuit_open(),
4001 "old errors outside window should not trip circuit"
4002 );
4003 }
4004}