1use chrono::TimeZone;
19use regex::Regex;
20use std::collections::HashMap;
21use std::collections::HashSet;
22use std::sync::LazyLock;
23use std::sync::MutexGuard;
24use std::sync::{Arc, Mutex, OnceLock};
25use tokio::sync;
26
27use crate::engine::router::config;
28#[cfg(test)]
29use crate::engine::runner::agents::opencode;
30use crate::store::TaskStore;
31
32pub const AGENT_COOLDOWN_SECS: i64 = 30 * 60;
38
39pub const BACKOFF_BASE_SECS: i64 = 5 * 60;
41
42pub const BACKOFF_MAX_SECS: i64 = 4 * 60 * 60;
44
45pub const PERSISTENT_MODEL_BACKOFF_BASE_SECS: i64 = 4 * 60 * 60;
47
48pub const PERSISTENT_MODEL_BACKOFF_MAX_SECS: i64 = 7 * 24 * 60 * 60;
50
51pub const TRANSIENT_MODEL_BACKOFF_BASE_SECS: i64 = 5 * 60;
53
54pub const TRANSIENT_MODEL_BACKOFF_MAX_SECS: i64 = 15 * 60;
56
57pub const CREDIT_BACKOFF_BASE_SECS: i64 = 60 * 60;
59
60pub const ORG_BACKOFF_BASE_SECS: i64 = 2 * 60 * 60;
62
63pub const CREDIT_BACKOFF_MAX_SECS: i64 = 8 * 60 * 60;
66
67pub const BILLING_CYCLE_COOLDOWN_SECS: i64 = 24 * 60 * 60;
75
76pub const BILLING_CYCLE_MAX_SECS: i64 = 7 * 24 * 60 * 60;
82
83#[derive(Debug, Clone, Copy, PartialEq, Eq)]
85pub enum CreditExhaustionReason {
86 OutOfCredits,
88 OrgLevelDisabled,
90 BillingCycleExhausted,
92}
93
94impl CreditExhaustionReason {
95 pub fn outcome_str(self) -> &'static str {
99 match self {
100 CreditExhaustionReason::OutOfCredits => "credit_exhausted",
101 CreditExhaustionReason::OrgLevelDisabled => "org_level_disabled",
102 CreditExhaustionReason::BillingCycleExhausted => "billing_cycle_exhausted",
103 }
104 }
105}
106
107pub const SILENCE_AGENT_COOLDOWN_SECS: u64 = 120;
114
115pub const SILENCE_COUNT_THRESHOLD: usize = 5;
117
118pub const SILENCE_COUNT_WINDOW_SECS: i64 = 24 * 60 * 60;
120
121pub const SILENCE_EXTENDED_COOLDOWN_SECS: u64 = 4 * 60 * 60;
123
124const KV_PREFIX: &str = "cooldown:";
126
127const SILENCE_COUNT_PREFIX: &str = "silence_count:";
129
130const FAILURE_COUNT_PREFIX: &str = "failure_count:";
132
133const CREDIT_FAILURE_COUNT_PREFIX: &str = "credit_failure_count:";
136
137const PERSISTENT_FAILURE_COUNT_PREFIX: &str = "persistent_failure_count:";
141
142const RATE_LIMITS_CLEARED_AT_PREFIX: &str = "rate_limits_cleared_at:";
144
145fn agent_from_key(key: &str) -> &str {
147 key.split_once(':').map_or(key, |(agent, _)| agent)
148}
149
150struct CooldownEntry {
151 cooldown_until: i64,
153 reason: String,
154}
155
156#[derive(Debug)]
157pub struct SilenceCountResult {
158 pub count: usize,
159 pub extended_cooldown_applied: bool,
160}
161
162fn cooldowns() -> &'static Mutex<HashMap<String, CooldownEntry>> {
171 static COOLDOWNS: OnceLock<Mutex<HashMap<String, CooldownEntry>>> = OnceLock::new();
172 COOLDOWNS.get_or_init(|| Mutex::new(HashMap::new()))
173}
174
175fn cooldown_store() -> &'static sync::Mutex<Option<Arc<TaskStore>>> {
181 static STORE: OnceLock<sync::Mutex<Option<Arc<TaskStore>>>> = OnceLock::new();
182 STORE.get_or_init(|| sync::Mutex::new(None))
183}
184
185fn parse_cooldown_value(value: &str) -> Option<(i64, String)> {
193 if let Ok(serde_json::Value::Object(map)) = serde_json::from_str(value) {
194 let until = map.get("until")?.as_i64()?;
195 let reason = map
196 .get("reason")
197 .and_then(|r| r.as_str())
198 .unwrap_or("persisted")
199 .to_string();
200 return Some((until, reason));
201 }
202 value
203 .parse::<i64>()
204 .ok()
205 .map(|until| (until, "persisted".to_string()))
206}
207
208pub async fn init_cooldown_store(store: Arc<TaskStore>) {
209 match store.kv_list_prefix(KV_PREFIX).await {
210 Ok(rows) => {
211 let now = chrono::Utc::now().timestamp();
212 let mut map = cooldown_lock();
213 let mut loaded = 0;
214 for (key, value) in rows {
215 let cooldown_key = key.trim_start_matches(KV_PREFIX);
216 if let Some((cooldown_until, reason)) = parse_cooldown_value(&value)
217 && now < cooldown_until
218 {
219 map.insert(
220 cooldown_key.to_string(),
221 CooldownEntry {
222 cooldown_until,
223 reason,
224 },
225 );
226 loaded += 1;
227 }
228 }
231 if loaded > 0 {
232 tracing::info!(loaded, "loaded persisted cooldowns from KV store");
233 }
234 }
235 Err(e) => {
236 tracing::warn!(err = %e, "failed to load cooldowns from KV store");
237 }
238 }
239
240 if let Ok(Some(raw)) = store.kv_get("cooldown:github:5xx").await
242 && let Ok(cooldown_until) = raw.parse::<i64>()
243 {
244 let now = chrono::Utc::now().timestamp();
245 if now < cooldown_until {
246 let mut open = github_5xx_circuit_open_lock();
247 *open = Some(cooldown_until);
248 let remaining = cooldown_until - now;
249 tracing::warn!(
250 remaining_secs = remaining,
251 "GitHub 5xx circuit breaker restored from KV (still open)"
252 );
253 }
254 }
255
256 *cooldown_store().lock().await = Some(store);
257}
258
259pub async fn sync_from_kv(store: &Arc<TaskStore>) {
276 let rows = match store.kv_list_prefix(KV_PREFIX).await {
277 Ok(rows) => rows,
278 Err(e) => {
279 tracing::warn!(err = %e, "cooldown sync_from_kv: KV list failed");
280 return;
281 }
282 };
283 let now = chrono::Utc::now().timestamp();
284
285 let mut explicit_kv: HashMap<String, Option<(i64, String)>> =
288 HashMap::with_capacity(rows.len());
289 for (kv_key, value) in &rows {
290 let cooldown_key = kv_key.trim_start_matches(KV_PREFIX);
291 if let Some((ts, reason)) = parse_cooldown_value(value) {
292 let normalized = if ts > now { Some((ts, reason)) } else { None };
293 explicit_kv.insert(cooldown_key.to_string(), normalized);
294 }
295 }
296
297 match explicit_kv.get("github:5xx") {
299 Some(None) => {
300 let mut open = github_5xx_circuit_open_lock();
301 if open.take().is_some() {
302 tracing::info!("GitHub 5xx circuit breaker closed by KV clear");
303 github_5xx_timestamps_lock().clear();
304 }
305 }
306 Some(Some((ts, _))) => {
307 *github_5xx_circuit_open_lock() = Some(*ts);
308 }
309 None => {}
310 }
311
312 let mut map = cooldown_lock();
313 let mut dropped = 0usize;
314 let mut updated = 0usize;
315
316 map.retain(|key, _| {
318 if matches!(explicit_kv.get(key), Some(None)) {
319 dropped += 1;
320 false
321 } else {
322 true
323 }
324 });
325
326 for (key, normalized) in explicit_kv {
328 if let Some((ts, reason)) = normalized {
329 if let Some(entry) = map.get_mut(&key) {
330 if entry.cooldown_until != ts {
331 entry.cooldown_until = ts;
332 updated += 1;
333 }
334 } else {
335 map.insert(
336 key,
337 CooldownEntry {
338 cooldown_until: ts,
339 reason,
340 },
341 );
342 updated += 1;
343 }
344 }
345 }
346
347 if dropped > 0 || updated > 0 {
348 tracing::info!(
349 dropped,
350 updated,
351 "cooldown sync_from_kv: reconciled in-memory map with KV"
352 );
353 }
354}
355
356const EXTENDED_TIER_1_THRESHOLD: u64 = 2;
360
361const EXTENDED_TIER_1_MULTIPLIER: i64 = 6;
363
364const EXTENDED_TIER_2_THRESHOLD: u64 = 5;
368
369const EXTENDED_TIER_2_MULTIPLIER: i64 = 12;
371
372pub fn compute_backoff(count: u64, base: i64, max: i64, allow_extended: bool) -> i64 {
391 if count <= 1 {
392 return base;
393 }
394 let exp = u32::try_from(count.saturating_sub(1)).unwrap_or(u32::MAX);
395 let factor = 3_i64.saturating_pow(exp);
396 let standard = base.saturating_mul(factor).min(max);
397
398 let first_max = if base <= 0 {
407 0
408 } else {
409 let mut accumulated = base;
411 let mut first_max_count = 1u64;
412 while accumulated < max && first_max_count < count {
413 accumulated = accumulated.saturating_mul(3);
414 first_max_count += 1;
415 }
416 first_max_count
417 };
418
419 let times_at_max = count.saturating_sub(first_max);
421
422 if !allow_extended {
423 return standard;
424 }
425
426 if standard == max && times_at_max >= EXTENDED_TIER_2_THRESHOLD {
427 max.saturating_mul(EXTENDED_TIER_2_MULTIPLIER)
428 } else if standard == max && times_at_max >= EXTENDED_TIER_1_THRESHOLD {
429 max.saturating_mul(EXTENDED_TIER_1_MULTIPLIER)
430 } else {
431 standard
432 }
433}
434
435async fn read_and_increment_failure_count(store_opt: Option<&Arc<TaskStore>>, key: &str) -> u64 {
444 read_and_increment_failure_count_with_prefix(store_opt, FAILURE_COUNT_PREFIX, key).await
445}
446
447async fn read_and_increment_failure_count_with_prefix(
452 store_opt: Option<&Arc<TaskStore>>,
453 prefix: &str,
454 key: &str,
455) -> u64 {
456 let kv_key = format!("{prefix}{key}");
457 if let Some(store) = store_opt {
458 match store.kv_increment(&kv_key).await {
459 Ok(n) => n,
460 Err(e) => {
461 tracing::warn!(key = %kv_key, err = %e, "failed to increment failure count — treating as high count for safe backoff");
462 u64::MAX
463 }
464 }
465 } else {
466 1
467 }
468}
469
470pub async fn record_agent_success(agent_name: &str, model: &str) {
476 let store_opt = cooldown_store().lock().await.clone();
477 if let Some(store) = store_opt {
478 let agent_key = format!("{FAILURE_COUNT_PREFIX}{agent_name}");
479 let credit_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{agent_name}");
480 if let Err(e) = store.kv_set(&agent_key, "0").await {
481 tracing::warn!(key = agent_key, err = %e, "failed to reset failure count");
482 }
483 if !model.is_empty() {
484 let model_key = format!("{FAILURE_COUNT_PREFIX}{agent_name}:{model}");
485 if let Err(e) = store.kv_set(&model_key, "0").await {
486 tracing::warn!(key = model_key, err = %e, "failed to reset failure count");
487 }
488 }
489 if let Err(e) = store.kv_set(&credit_key, "0").await {
490 tracing::warn!(key = credit_key, err = %e, "failed to reset failure count");
491 }
492 }
493}
494
495pub async fn record_agent_failure_with_message(agent_name: &str, error_message: &str) {
501 if let Some(cooldown_until) = parse_retry_at(error_message) {
506 let now = chrono::Utc::now().timestamp();
507 if cooldown_until > now {
508 set_cooldown_async(agent_name, cooldown_until, "agent_error").await;
509 return;
510 }
511 tracing::debug!(
512 agent = agent_name,
513 cooldown_until,
514 "vendor retry_at is in the past — using backoff instead"
515 );
516 }
517
518 let store_opt = cooldown_store().lock().await.clone();
519 let count = read_and_increment_failure_count(store_opt.as_ref(), agent_name).await;
520 let base = config::get_agent_backoff_base(agent_name);
521 let cooldown_until =
522 chrono::Utc::now().timestamp() + compute_backoff(count, base, BACKOFF_MAX_SECS, true);
523 set_cooldown_async(agent_name, cooldown_until, "agent_error").await;
524}
525
526pub fn detect_credit_exhaustion_outcome(error_message: &str) -> Option<&'static str> {
535 detect_credit_exhaustion(error_message).map(CreditExhaustionReason::outcome_str)
536}
537
538pub fn detect_credit_exhaustion(error_message: &str) -> Option<CreditExhaustionReason> {
543 let lower = error_message.to_lowercase();
544
545 let overage_patterns = [
546 "overagedisabledreason",
547 "overage_disabled_reason",
548 "overage disabled reason",
549 ];
550
551 if overage_patterns.iter().any(|p| lower.contains(p)) {
552 if lower.contains("out_of_credits") || lower.contains("outofcredits") {
553 return Some(CreditExhaustionReason::OutOfCredits);
554 }
555 if lower.contains("org_level_disabled") || lower.contains("orgdisabled") {
556 return Some(CreditExhaustionReason::OrgLevelDisabled);
557 }
558 }
559
560 let billing_patterns = [
561 "credit balance too low",
562 "credit balance insufficient",
563 "insufficient credit balance",
564 "billing quota exceeded",
565 "billing limit exceeded",
566 "organization has been disabled",
567 "org disabled",
568 ];
569
570 if billing_patterns.iter().any(|p| lower.contains(p)) {
571 return Some(CreditExhaustionReason::OrgLevelDisabled);
572 }
573
574 let billing_cycle_patterns = [
577 "billing cycle",
578 "quota refreshed next cycle",
579 "monthly quota",
580 "quota will be refreshed",
581 "next billing cycle",
582 "refreshed in the next cycle",
583 "weekly/monthly limit exhausted",
584 "limit will reset at",
585 "upgrade to pro", "upgrade your token plan", "token plan usage limit", "purchase credits for more", ];
590
591 if billing_cycle_patterns.iter().any(|p| lower.contains(p)) {
592 return Some(CreditExhaustionReason::BillingCycleExhausted);
593 }
594
595 if lower.contains("out of credits")
596 || lower.contains("outofcredits")
597 || lower.contains("insufficient funds")
598 || lower.contains("no credits remaining")
599 || lower.contains("insufficient_quota")
600 || lower.contains("quota exceeded")
601 || lower.contains("insufficient balance")
602 || lower.contains("please recharge")
603 || lower.contains("no resource package")
604 || lower.contains("balance exhausted")
605 || lower.contains("402 payment required")
606 {
607 return Some(CreditExhaustionReason::OutOfCredits);
608 }
609
610 None
611}
612
613pub async fn record_credit_exhaustion(agent_name: &str, reason: CreditExhaustionReason) {
626 let reason_str = match reason {
627 CreditExhaustionReason::OutOfCredits => "credit_exhaustion_out_of_credits",
628 CreditExhaustionReason::OrgLevelDisabled => "credit_exhaustion_org_level_disabled",
629 CreditExhaustionReason::BillingCycleExhausted => "billing_cycle_exhausted",
630 };
631
632 if is_in_cooldown_with_reason(agent_name, reason_str) {
634 tracing::debug!(
635 agent = agent_name,
636 reason = reason_str,
637 "credit exhaustion already cooled, not escalating"
638 );
639 return;
640 }
641
642 let store_opt = cooldown_store().lock().await.clone();
643 let count = read_and_increment_failure_count_with_prefix(
648 store_opt.as_ref(),
649 CREDIT_FAILURE_COUNT_PREFIX,
650 agent_name,
651 )
652 .await;
653
654 let (base, max) = match reason {
655 CreditExhaustionReason::OutOfCredits => (CREDIT_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS),
656 CreditExhaustionReason::OrgLevelDisabled => {
657 (ORG_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS)
658 }
659 CreditExhaustionReason::BillingCycleExhausted => {
660 (BILLING_CYCLE_COOLDOWN_SECS, BILLING_CYCLE_MAX_SECS)
661 }
662 };
663 let cooldown_secs = compute_backoff(
664 count,
665 base,
666 max,
667 !matches!(reason, CreditExhaustionReason::BillingCycleExhausted),
668 );
669 let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
670 set_cooldown_async(agent_name, cooldown_until, reason_str).await;
671 tracing::warn!(
672 agent = agent_name,
673 reason = reason_str,
674 attempt = count,
675 cooldown_secs,
676 "credit exhaustion detected: applying exponential agent-wide cooldown"
677 );
678}
679
680pub async fn record_credit_exhaustion_from_message(
682 agent_name: &str,
683 model: Option<&str>,
684 message: &str,
685) -> bool {
686 let Some(reason) = detect_credit_exhaustion(message) else {
687 return false;
688 };
689 match (reason, model) {
690 (CreditExhaustionReason::BillingCycleExhausted, Some(model)) => {
691 record_persistent_model_failure(agent_name, model).await;
692 }
693 _ => record_credit_exhaustion(agent_name, reason).await,
694 }
695 true
696}
697
698pub async fn record_model_failure(agent_name: &str, model: &str) {
702 let key = format!("{agent_name}:{model}");
703 let store_opt = cooldown_store().lock().await.clone();
704 let count = read_and_increment_failure_count(store_opt.as_ref(), &key).await;
705 let base = config::get_agent_backoff_base(agent_name);
706 let cooldown_until =
707 chrono::Utc::now().timestamp() + compute_backoff(count, base, BACKOFF_MAX_SECS, true);
708 set_cooldown_async(&key, cooldown_until, "model_error").await;
709}
710
711pub async fn record_rate_limit(agent_name: &str, model: Option<&str>, error_message: &str) {
719 if let Some(window_secs) = parse_relative_usage_window(error_message)
720 && let Some(m) = model
721 {
722 let cooldown_until = chrono::Utc::now()
723 .timestamp()
724 .saturating_add(i64::try_from(window_secs).unwrap_or(i64::MAX));
725 set_model_cooldown(agent_name, m, window_secs, "rate_limit_usage_window").await;
726 tracing::info!(
727 agent = agent_name,
728 model = m,
729 window_secs,
730 cooldown_until,
731 "relative usage window detected: applying model-specific cooldown"
732 );
733 return;
734 }
735
736 record_agent_failure_with_message(agent_name, error_message).await;
737 if let Some(m) = model {
738 record_model_failure(agent_name, m).await;
739 }
740}
741
742pub async fn record_persistent_model_failure(agent_name: &str, model: &str) {
754 let key = format!("{agent_name}:{model}");
755 let store_opt = cooldown_store().lock().await.clone();
756 let count = read_and_increment_failure_count_with_prefix(
757 store_opt.as_ref(),
758 PERSISTENT_FAILURE_COUNT_PREFIX,
759 &key,
760 )
761 .await;
762 let cooldown_secs = compute_backoff(
763 count,
764 PERSISTENT_MODEL_BACKOFF_BASE_SECS,
765 PERSISTENT_MODEL_BACKOFF_MAX_SECS,
766 false,
767 );
768 let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
769 set_cooldown_async(&key, cooldown_until, "persistent_model_error").await;
770}
771
772pub async fn record_model_unavailable(agent_name: &str, model: &str, message: &str) {
777 if is_model_permanently_gone(message) {
778 record_persistent_model_failure(agent_name, model).await;
779 } else {
780 record_model_failure(agent_name, model).await;
781 }
782}
783
784fn is_model_permanently_gone(message: &str) -> bool {
785 let lower = message.to_lowercase();
786 lower.contains("not found")
787 || lower.contains("not supported")
788 || lower.contains("not available in your country")
789 || lower.contains("not available in your region")
790}
791
792pub async fn record_transient_model_failure(agent_name: &str, model: &str) {
798 let key = format!("{agent_name}:{model}");
799 let store_opt = cooldown_store().lock().await.clone();
800 let count = read_and_increment_failure_count(store_opt.as_ref(), &key).await;
801 let cooldown_secs = compute_backoff(
802 count,
803 TRANSIENT_MODEL_BACKOFF_BASE_SECS,
804 TRANSIENT_MODEL_BACKOFF_MAX_SECS,
805 false,
806 );
807 let cooldown_until = chrono::Utc::now().timestamp() + cooldown_secs;
808 set_cooldown_async(&key, cooldown_until, "transient_model_error").await;
809}
810
811pub async fn set_model_cooldown(agent_name: &str, model: &str, duration_secs: u64, reason: &str) {
818 let key = format!("{agent_name}:{model}");
819 let cooldown_until = chrono::Utc::now()
820 .timestamp()
821 .saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX));
822 set_cooldown_async(&key, cooldown_until, reason).await;
823}
824
825pub async fn set_model_cooldown_until(agent_name: &str, model: &str, until: i64) {
827 let key = format!("{agent_name}:{model}");
828 set_cooldown_async(&key, until, "vendor_reset_at").await;
829}
830
831pub async fn set_agent_cooldown(agent_name: &str, duration_secs: u64) {
838 let cooldown_until = chrono::Utc::now()
839 .timestamp()
840 .saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX));
841 set_cooldown_async(agent_name, cooldown_until, "silence_agent_cooldown").await;
842}
843
844pub const RATE_WINDOW_KV_PREFIX: &str = "rate_window:";
846
847#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
849pub struct RateLimitWindow {
850 pub status: String,
851 pub window_type: String,
852 pub utilization: Option<f64>,
853 pub resets_at: Option<i64>,
854}
855
856pub async fn record_rate_limit_window(agent_name: &str, window: &RateLimitWindow) {
858 let store_opt = cooldown_store().lock().await.clone();
859 if let Some(store) = store_opt
860 && let Ok(value) = serde_json::to_string(window)
861 && let Err(e) = store
862 .kv_set(
863 &format!("{RATE_WINDOW_KV_PREFIX}{agent_name}:{}", window.window_type),
864 &value,
865 )
866 .await
867 {
868 tracing::warn!(agent = agent_name, err = %e, "failed to persist rate limit window");
869 }
870
871 let now = chrono::Utc::now().timestamp();
872 if window.status == "rejected"
873 && let Some(reset) = window.resets_at
874 && reset > now
875 {
876 let until = reset.min(now + PERSISTENT_MODEL_BACKOFF_MAX_SECS);
877 set_cooldown_async(agent_name, until, "rate_limit_window").await;
878 tracing::info!(
879 agent = agent_name,
880 window = %window.window_type,
881 cooldown_until = until,
882 "rate limit window rejected: cooling agent until vendor reset"
883 );
884 }
885}
886
887pub async fn list_rate_limit_windows(store: &Arc<TaskStore>) -> Vec<(String, RateLimitWindow)> {
889 let rows = store
890 .kv_list_prefix(RATE_WINDOW_KV_PREFIX)
891 .await
892 .unwrap_or_default();
893 let mut windows: Vec<(String, RateLimitWindow)> = rows
894 .into_iter()
895 .filter_map(|(key, value)| {
896 let (agent, _) = key.strip_prefix(RATE_WINDOW_KV_PREFIX)?.rsplit_once(':')?;
897 let window = serde_json::from_str(&value).ok()?;
898 Some((agent.to_string(), window))
899 })
900 .collect();
901 windows.sort_by(|a, b| (&a.0, &a.1.window_type).cmp(&(&b.0, &b.1.window_type)));
902 windows
903}
904
905pub async fn record_silence_detection(agent_name: &str, model: &str) -> Option<SilenceCountResult> {
908 let store_opt = cooldown_store().lock().await.clone();
909 let Some(store) = store_opt else {
910 tracing::debug!(
911 agent = agent_name,
912 model,
913 "skipping silence count record (no KV store)"
914 );
915 return None;
916 };
917
918 let key = format!("{SILENCE_COUNT_PREFIX}{agent_name}:{model}");
919 let now = chrono::Utc::now().timestamp();
920 let window_start = now - SILENCE_COUNT_WINDOW_SECS;
921
922 let mut timestamps = match store.kv_get(&key).await {
923 Ok(Some(raw)) => match serde_json::from_str::<Vec<i64>>(&raw) {
924 Ok(ts) => ts,
925 Err(e) => {
926 tracing::warn!(
927 kv_key = %key,
928 err = %e,
929 "failed to parse silence timestamps from KV — resetting window"
930 );
931 Vec::new()
932 }
933 },
934 Ok(None) => Vec::new(),
935 Err(err) => {
936 tracing::warn!(
937 kv_key = %key,
938 err = %err,
939 "failed to load silence count from KV"
940 );
941 Vec::new()
942 }
943 };
944
945 timestamps.retain(|ts| *ts >= window_start);
946 timestamps.push(now);
947
948 let count = timestamps.len();
949 let mut extended_cooldown_applied = false;
950 if count >= SILENCE_COUNT_THRESHOLD {
951 set_model_cooldown(
952 agent_name,
953 model,
954 SILENCE_EXTENDED_COOLDOWN_SECS,
955 "silence_detected",
956 )
957 .await;
958 extended_cooldown_applied = true;
959 timestamps.clear();
960 }
961
962 let value = serde_json::to_string(×tamps).unwrap_or_else(|_| "[]".to_string());
963 if let Err(err) = store.kv_set(&key, &value).await {
964 tracing::warn!(kv_key = key, err = %err, "failed to persist silence count");
965 }
966
967 Some(SilenceCountResult {
968 count,
969 extended_cooldown_applied,
970 })
971}
972
973pub fn is_model_in_cooldown(agent_name: &str, model: &str) -> bool {
975 let key = format!("{agent_name}:{model}");
976 is_in_cooldown(&key)
977}
978
979pub fn is_agent_in_cooldown(agent_name: &str) -> bool {
981 is_in_cooldown(agent_name)
982}
983
984pub fn cooldown_until(key: &str) -> Option<i64> {
986 let map = cooldown_lock();
987 map.get(key).and_then(|entry| {
988 let now = chrono::Utc::now().timestamp();
989 if now < entry.cooldown_until {
990 Some(entry.cooldown_until)
991 } else {
992 None
993 }
994 })
995}
996
997pub fn cooldown_reason(key: &str) -> Option<String> {
999 let map = cooldown_lock();
1000 map.get(key).and_then(|entry| {
1001 let now = chrono::Utc::now().timestamp();
1002 if now < entry.cooldown_until {
1003 Some(entry.reason.clone())
1004 } else {
1005 None
1006 }
1007 })
1008}
1009
1010pub fn list_all_cooldowns() -> Vec<(String, i64, String)> {
1015 let now = chrono::Utc::now().timestamp();
1016 let map = cooldown_lock();
1017 let mut result: Vec<(String, i64, String)> = map
1018 .iter()
1019 .filter(|(_, entry)| now < entry.cooldown_until)
1020 .map(|(key, entry)| (key.clone(), entry.cooldown_until, entry.reason.clone()))
1021 .collect();
1022 result.sort_by(|a, b| a.0.cmp(&b.0));
1023 result
1024}
1025
1026pub async fn mark_rate_limits_cleared(store: &Arc<TaskStore>, agents: &[String]) {
1029 let now = chrono::Utc::now().timestamp().to_string();
1030 let mut seen = HashSet::new();
1031 for key in agents {
1032 let agent = agent_from_key(key);
1033 if agent.is_empty() || !seen.insert(agent.to_string()) {
1034 continue;
1035 }
1036 let kv_key = format!("{RATE_LIMITS_CLEARED_AT_PREFIX}{agent}");
1037 if let Err(e) = store.kv_set(&kv_key, &now).await {
1038 tracing::warn!(agent, err = %e, "failed to mark rate limits cleared");
1039 }
1040 }
1041}
1042
1043pub async fn clear_cooldown(key: &str, store: &Arc<TaskStore>) {
1050 let mut agents_to_mark: Vec<String> = Vec::new();
1051 if key == "*" {
1052 let keys: Vec<String> = {
1053 let mut map = cooldown_lock();
1054 let keys: Vec<String> = map.keys().cloned().collect();
1055 map.clear();
1056 keys
1057 };
1058 for k in &keys {
1059 let kv_key = format!("{KV_PREFIX}{k}");
1060 if let Err(e) = store.kv_set(&kv_key, "0").await {
1061 tracing::warn!(key = kv_key, err = %e, "failed to clear cooldown");
1062 }
1063 let fc_key = format!("{FAILURE_COUNT_PREFIX}{k}");
1065 if let Err(e) = store.kv_set(&fc_key, "0").await {
1066 tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1067 }
1068 let credit_fc_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{k}");
1070 if let Err(e) = store.kv_set(&credit_fc_key, "0").await {
1071 tracing::warn!(key = credit_fc_key, err = %e, "failed to reset credit failure count");
1072 }
1073 let persistent_fc_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{k}");
1075 if let Err(e) = store.kv_set(&persistent_fc_key, "0").await {
1076 tracing::warn!(key = persistent_fc_key, err = %e, "failed to reset persistent failure count");
1077 }
1078 }
1079 if let Ok(fc_entries) = store.kv_list_prefix(FAILURE_COUNT_PREFIX).await {
1083 for (fc_key, _) in fc_entries {
1084 if let Err(e) = store.kv_set(&fc_key, "0").await {
1085 tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1086 }
1087 }
1088 }
1089 if let Ok(cfc_entries) = store.kv_list_prefix(CREDIT_FAILURE_COUNT_PREFIX).await {
1090 for (cfc_key, _) in cfc_entries {
1091 if let Err(e) = store.kv_set(&cfc_key, "0").await {
1092 tracing::warn!(key = cfc_key, err = %e, "failed to reset credit failure count");
1093 }
1094 }
1095 }
1096 if let Ok(pfc_entries) = store.kv_list_prefix(PERSISTENT_FAILURE_COUNT_PREFIX).await {
1097 for (pfc_key, _) in pfc_entries {
1098 if let Err(e) = store.kv_set(&pfc_key, "0").await {
1099 tracing::warn!(key = pfc_key, err = %e, "failed to reset persistent failure count");
1100 }
1101 }
1102 }
1103 tracing::info!(
1104 count = keys.len(),
1105 "cleared all cooldowns and failure counts"
1106 );
1107 agents_to_mark.extend(keys.iter().map(|k| agent_from_key(k).to_string()));
1108 match store.rate_limit_agents().await {
1109 Ok(rl_agents) => agents_to_mark.extend(rl_agents),
1110 Err(e) => tracing::warn!(err = %e, "failed to list rate_limits agents while clearing"),
1111 }
1112 match store.kv_list_prefix(RATE_LIMITS_CLEARED_AT_PREFIX).await {
1113 Ok(marks) => {
1114 for (k, _) in marks {
1115 let agent = k.trim_start_matches(RATE_LIMITS_CLEARED_AT_PREFIX);
1116 if !agent.is_empty() {
1117 agents_to_mark.push(agent.to_string());
1118 }
1119 }
1120 }
1121 Err(e) => tracing::warn!(err = %e, "failed to list rate_limits_cleared_at keys"),
1122 }
1123 } else {
1124 {
1125 let mut map = cooldown_lock();
1126 map.remove(key);
1127 }
1128 let kv_key = format!("{KV_PREFIX}{key}");
1129 if let Err(e) = store.kv_set(&kv_key, "0").await {
1130 tracing::warn!(key = kv_key, err = %e, "failed to clear cooldown");
1131 }
1132 let fc_key = format!("{FAILURE_COUNT_PREFIX}{key}");
1134 if let Err(e) = store.kv_set(&fc_key, "0").await {
1135 tracing::warn!(key = fc_key, err = %e, "failed to reset failure count");
1136 }
1137 let credit_fc_key = format!("{CREDIT_FAILURE_COUNT_PREFIX}{key}");
1139 if let Err(e) = store.kv_set(&credit_fc_key, "0").await {
1140 tracing::warn!(key = credit_fc_key, err = %e, "failed to reset credit failure count");
1141 }
1142 let persistent_fc_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{key}");
1144 if let Err(e) = store.kv_set(&persistent_fc_key, "0").await {
1145 tracing::warn!(key = persistent_fc_key, err = %e, "failed to reset persistent failure count");
1146 }
1147 tracing::info!(key, "cleared cooldown and failure count");
1148 agents_to_mark.push(agent_from_key(key).to_string());
1149 }
1150 mark_rate_limits_cleared(store, &agents_to_mark).await;
1151}
1152
1153fn set_cooldown_in_memory(key: &str, cooldown_until: i64, reason: &str) -> bool {
1162 let mut map = cooldown_lock();
1163 if let Some(existing) = map.get(key) {
1164 if existing.cooldown_until >= cooldown_until {
1168 return false;
1169 }
1170 }
1171 map.insert(
1172 key.to_string(),
1173 CooldownEntry {
1174 cooldown_until,
1175 reason: reason.to_string(),
1176 },
1177 );
1178 true
1179}
1180
1181async fn set_cooldown_async(key: &str, cooldown_until: i64, reason: &str) -> bool {
1191 if !set_cooldown_in_memory(key, cooldown_until, reason) {
1192 return false;
1193 }
1194 let store_opt = cooldown_store().lock().await.clone();
1195 if let Some(store) = store_opt {
1196 let kv_key = format!("{KV_PREFIX}{key}");
1197 let value = serde_json::json!({"until": cooldown_until, "reason": reason}).to_string();
1198 if let Err(e) = store.kv_set(&kv_key, &value).await {
1199 tracing::warn!(
1200 kv_key,
1201 err = %e,
1202 "failed to persist cooldown to KV store — rolling back in-memory entry"
1203 );
1204 let mut map = cooldown_lock();
1205 if map.get(key).map(|e| e.cooldown_until) == Some(cooldown_until) {
1209 map.remove(key);
1210 }
1211 return false;
1212 }
1213 }
1214 true
1215}
1216
1217fn cooldown_lock() -> MutexGuard<'static, HashMap<String, CooldownEntry>> {
1219 cooldowns().lock().unwrap_or_else(|e| {
1220 tracing::error!("cooldown map lock poisoned — recovering with partial state; backoff and degraded-agent decisions may be incorrect");
1221 e.into_inner()
1222 })
1223}
1224
1225fn degraded_agents_lock() -> MutexGuard<'static, HashMap<String, String>> {
1227 degraded_agents().lock().unwrap_or_else(|e| {
1228 tracing::error!("degraded agents lock poisoned — recovering with partial state; health check decisions may be incorrect");
1229 e.into_inner()
1230 })
1231}
1232
1233fn github_5xx_timestamps_lock() -> MutexGuard<'static, Vec<i64>> {
1235 github_5xx_timestamps().lock().unwrap_or_else(|e| {
1236 tracing::error!("github 5xx timestamps lock poisoned — recovering with partial state; circuit breaker may operate incorrectly");
1237 e.into_inner()
1238 })
1239}
1240
1241fn github_5xx_circuit_open_lock() -> MutexGuard<'static, Option<i64>> {
1243 github_5xx_circuit_open().lock().unwrap_or_else(|e| {
1244 tracing::error!("github 5xx circuit open lock poisoned — recovering with partial state; circuit breaker may operate incorrectly");
1245 e.into_inner()
1246 })
1247}
1248
1249fn is_in_cooldown_with_reason(key: &str, reason: &str) -> bool {
1251 let map = cooldown_lock();
1252 map.get(key).is_some_and(|entry| {
1253 entry.reason == reason && chrono::Utc::now().timestamp() < entry.cooldown_until
1254 })
1255}
1256
1257fn is_in_cooldown(key: &str) -> bool {
1259 let map = cooldown_lock();
1260 if let Some(entry) = map.get(key) {
1261 let now = chrono::Utc::now().timestamp();
1262 return now < entry.cooldown_until;
1263 }
1264 false
1265}
1266
1267pub const DEFAULT_HEALTH_CHECK_WINDOW_HOURS: u32 = 6;
1273
1274pub const DEFAULT_DEGRADED_RATE_LIMIT_THRESHOLD: i64 = 3;
1278
1279fn degraded_agents() -> &'static Mutex<HashMap<String, String>> {
1282 static DEGRADED: OnceLock<Mutex<HashMap<String, String>>> = OnceLock::new();
1283 DEGRADED.get_or_init(|| Mutex::new(HashMap::new()))
1284}
1285
1286pub fn is_agent_degraded(agent: &str) -> bool {
1291 let set = degraded_agents_lock();
1292 set.contains_key(agent)
1293}
1294
1295pub fn agent_degraded_reason(agent: &str) -> Option<String> {
1297 degraded_agents_lock().get(agent).cloned()
1298}
1299
1300pub fn list_degraded_agents() -> Vec<(String, String)> {
1302 let mut entries: Vec<(String, String)> = degraded_agents_lock()
1303 .iter()
1304 .map(|(agent, reason)| (agent.clone(), reason.clone()))
1305 .collect();
1306 entries.sort_by(|a, b| a.0.cmp(&b.0));
1307 entries
1308}
1309
1310pub fn mark_agent_degraded(agent: &str) {
1312 mark_agent_degraded_with_reason(agent, "degraded");
1313}
1314
1315fn mark_agent_degraded_with_reason(agent: &str, reason: &str) {
1316 let mut set = degraded_agents_lock();
1317 set.insert(agent.to_string(), reason.to_string());
1318}
1319
1320pub fn clear_agent_degraded(agent: &str) {
1322 let mut set = degraded_agents_lock();
1323 set.remove(agent);
1324}
1325
1326pub async fn refresh_degraded_agents(
1340 store: &Arc<TaskStore>,
1341 available_agents: &[String],
1342 model_checker: &(dyn Fn(&str) -> bool + Send + Sync),
1343 window_hours: u32,
1344 threshold: i64,
1345) {
1346 clear_expired_cooldowns();
1350
1351 let start = chrono::Utc::now();
1354 let counts = match store.recent_rate_limit_counts(window_hours).await {
1355 Ok(c) => {
1356 let dur_ms = (chrono::Utc::now() - start).num_milliseconds();
1357 tracing::info!(
1358 duration_ms = dur_ms,
1359 window_hours,
1360 "recent_rate_limit_counts query completed"
1361 );
1362 c
1363 }
1364 Err(e) => {
1365 let dur_ms = (chrono::Utc::now() - start).num_milliseconds();
1366 tracing::warn!(
1367 err = %e,
1368 duration_ms = dur_ms,
1369 "failed to query recent rate limit counts for health check"
1370 );
1371 return;
1372 }
1373 };
1374
1375 for agent in available_agents {
1376 let in_cooldown = is_agent_in_cooldown(agent);
1377 let no_models = !model_checker(agent);
1378 let rate_limit_count = counts.get(agent.as_str()).copied().unwrap_or(0);
1379 let over_threshold = rate_limit_count >= threshold;
1380
1381 let degraded = in_cooldown || no_models || over_threshold;
1382
1383 if degraded {
1384 let reason = if in_cooldown {
1385 "agent in cooldown"
1386 } else if no_models {
1387 "all models cooled"
1388 } else {
1389 "rate limit threshold exceeded"
1390 };
1391 let previous = agent_degraded_reason(agent);
1392 match previous.as_deref() {
1393 None => tracing::warn!(
1394 agent,
1395 reason,
1396 rate_limit_count,
1397 window_hours,
1398 threshold,
1399 "pre-emptive health check: marking agent as degraded"
1400 ),
1401 Some(prev) if prev != reason => tracing::warn!(
1402 agent,
1403 reason,
1404 previous = prev,
1405 rate_limit_count,
1406 window_hours,
1407 threshold,
1408 "pre-emptive health check: degraded reason changed"
1409 ),
1410 Some(_) => {}
1411 }
1412 mark_agent_degraded_with_reason(agent, reason);
1413 } else {
1414 if is_agent_degraded(agent) {
1415 tracing::info!(
1416 agent,
1417 rate_limit_count,
1418 "pre-emptive health check: agent recovered, clearing degraded flag"
1419 );
1420 }
1421 clear_agent_degraded(agent);
1422 }
1423 }
1424}
1425
1426pub fn clear_expired_cooldowns() {
1428 let mut map = cooldown_lock();
1429 let now = chrono::Utc::now().timestamp();
1430 map.retain(|_key, entry| now < entry.cooldown_until);
1431}
1432
1433const GITHUB_5XX_WINDOW_SECS: i64 = 120;
1439
1440const GITHUB_5XX_THRESHOLD: usize = 5;
1442
1443const GITHUB_5XX_COOLDOWN_SECS: i64 = 180;
1445
1446fn github_5xx_timestamps() -> &'static Mutex<Vec<i64>> {
1448 static TS: OnceLock<Mutex<Vec<i64>>> = OnceLock::new();
1449 TS.get_or_init(|| Mutex::new(Vec::new()))
1450}
1451
1452fn github_5xx_circuit_open() -> &'static Mutex<Option<i64>> {
1454 static OPEN: OnceLock<Mutex<Option<i64>>> = OnceLock::new();
1455 OPEN.get_or_init(|| Mutex::new(None))
1456}
1457
1458pub async fn record_github_5xx() {
1463 let now = chrono::Utc::now().timestamp();
1464 let window_start = now - GITHUB_5XX_WINDOW_SECS;
1465
1466 let should_trip = {
1467 let mut ts = github_5xx_timestamps_lock();
1468 ts.retain(|t| *t >= window_start);
1469 ts.push(now);
1470 ts.len() >= GITHUB_5XX_THRESHOLD
1471 };
1472
1473 if should_trip {
1474 let cooldown_until = {
1475 let mut open = github_5xx_circuit_open_lock();
1476 if open.is_none() {
1477 let cd = now + GITHUB_5XX_COOLDOWN_SECS;
1478 *open = Some(cd);
1479 tracing::warn!(
1480 errors_in_window = GITHUB_5XX_THRESHOLD,
1481 window_secs = GITHUB_5XX_WINDOW_SECS,
1482 cooldown_secs = GITHUB_5XX_COOLDOWN_SECS,
1483 "GitHub 5xx circuit breaker OPEN — throttling non-critical work"
1484 );
1485 Some(cd)
1486 } else {
1487 None
1488 }
1489 };
1490 if let Some(cd) = cooldown_until {
1492 let store_opt = cooldown_store().lock().await.clone();
1493 if let Some(store) = store_opt {
1494 let _ = store.kv_set("cooldown:github:5xx", &cd.to_string()).await;
1495 }
1496 set_agent_cooldown("github:5xx", GITHUB_5XX_COOLDOWN_SECS as u64).await;
1500 }
1501 let mut ts = github_5xx_timestamps_lock();
1503 ts.clear();
1504 }
1505}
1506
1507pub fn is_github_circuit_open() -> bool {
1515 {
1518 let mut open = github_5xx_circuit_open_lock();
1519 if let Some(until) = *open {
1520 let now = chrono::Utc::now().timestamp();
1521 if now >= until {
1522 *open = None;
1523 tracing::info!("GitHub 5xx circuit breaker CLOSED — resuming normal operations");
1524 return false;
1525 }
1526 return true;
1527 }
1528 }
1529
1530 if let Some(until) = cooldown_until("github:5xx") {
1534 let mut open = github_5xx_circuit_open_lock();
1537 if open.is_none() {
1538 *open = Some(until);
1539 let now = chrono::Utc::now().timestamp();
1540 if now < until {
1541 let remaining = until - now;
1542 tracing::warn!(
1543 remaining_secs = remaining,
1544 "GitHub 5xx circuit breaker OPEN (discovered via generic cooldown) — throttling non-critical work"
1545 );
1546 }
1547 }
1548 return true;
1549 }
1550
1551 false
1552}
1553
1554pub fn github_circuit_remaining_secs() -> u64 {
1557 {
1559 let open = github_5xx_circuit_open_lock();
1560 if let Some(until) = *open {
1561 let now = chrono::Utc::now().timestamp();
1562 if now < until {
1563 return u64::try_from(until - now).unwrap_or(0);
1564 }
1565 return 0;
1566 }
1567 }
1568
1569 if let Some(until) = cooldown_until("github:5xx") {
1571 let now = chrono::Utc::now().timestamp();
1572 if now < until {
1573 return u64::try_from(until - now).unwrap_or(0);
1574 }
1575 }
1576 0
1577}
1578
1579pub(crate) fn parse_retry_at(error_message: &str) -> Option<i64> {
1583 if error_message.is_empty() {
1584 return None;
1585 }
1586
1587 let lower = error_message.to_lowercase();
1588 for retry_marker in [
1592 "try again at ",
1593 "reset at ",
1594 "resets at ",
1595 "resets ",
1596 "reset ",
1597 ] {
1598 if let Some(idx) = lower.find(retry_marker) {
1599 let date_str = &lower[idx + retry_marker.len()..];
1600 let date_str = date_str
1602 .split(['.', '\n'])
1603 .next()
1604 .unwrap_or(date_str)
1605 .trim();
1606
1607 let cleaned = date_str
1610 .replace("st,", ",")
1611 .replace("nd,", ",")
1612 .replace("rd,", ",")
1613 .replace("th,", ",");
1614
1615 for fmt in &[
1616 "%b %d, %Y %I:%M %p", "%b %d, %Y %I:%M%p", "%B %d, %Y %I:%M %p", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", ] {
1623 if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(&cleaned, fmt) {
1624 let utc = match chrono::Local.from_local_datetime(&dt) {
1627 chrono::LocalResult::Single(local_dt) => {
1628 local_dt.with_timezone(&chrono::Utc)
1629 }
1630 chrono::LocalResult::Ambiguous(earliest, _) => {
1631 earliest.with_timezone(&chrono::Utc)
1633 }
1634 chrono::LocalResult::None => {
1635 dt.and_utc()
1637 }
1638 };
1639 tracing::info!(
1640 retry_at = %utc,
1641 raw = date_str,
1642 marker = retry_marker,
1643 "parsed rate limit retry-at date"
1644 );
1645 return Some(utc.timestamp());
1646 }
1647 }
1648 tracing::debug!(
1649 raw = date_str,
1650 marker = retry_marker,
1651 "could not parse retry-at date"
1652 );
1653
1654 let time_only = date_str.split_once('(').map_or(date_str, |(t, _)| t).trim();
1659 let mut compact = time_only.replace(" am", "am").replace(" pm", "pm");
1661 let split = compact.rfind(' ').map_or(0, |i| i + 1);
1662 let last = &compact[split..];
1663 if !last.contains(':')
1664 && (last.ends_with("am") || last.ends_with("pm"))
1665 && last.len() > 2
1666 {
1667 let at = split + last.len() - 2;
1668 compact.insert_str(at, ":00");
1669 }
1670 let md = compact.replace(" at ", " ");
1672 let now_local = chrono::Local::now();
1673 let this_year = chrono::Datelike::year(&now_local);
1674 for year in [this_year, this_year + 1] {
1675 let with_year = format!("{year} {md}");
1676 if let Ok(dt) =
1677 chrono::NaiveDateTime::parse_from_str(&with_year, "%Y %b %d %I:%M%p")
1678 && let Some(local_dt) = chrono::Local.from_local_datetime(&dt).earliest()
1679 && local_dt > now_local
1680 {
1681 return Some(local_dt.timestamp());
1682 }
1683 }
1684 let time_only = compact.as_str();
1685 for tf in &["%I:%M %p", "%I:%M%p", "%I %p", "%H:%M"] {
1686 if let Ok(t) = chrono::NaiveTime::parse_from_str(time_only, tf) {
1687 let local_now = chrono::Local::now();
1690 let today = local_now.date_naive();
1691 let candidate = chrono::NaiveDateTime::new(today, t);
1692 let candidate_local: chrono::DateTime<chrono::Local> =
1695 match chrono::Local.from_local_datetime(&candidate) {
1696 chrono::LocalResult::Single(dt) => dt,
1697 chrono::LocalResult::Ambiguous(earliest, _) => earliest,
1698 chrono::LocalResult::None => {
1699 let utc_dt = candidate.and_utc();
1701 return Some(utc_dt.timestamp());
1702 }
1703 };
1704 let chosen = if candidate_local <= local_now {
1705 let tomorrow = today.succ_opt().unwrap_or(today);
1707 let ndt = chrono::NaiveDateTime::new(tomorrow, t);
1708 match chrono::Local.from_local_datetime(&ndt) {
1709 chrono::LocalResult::Single(dt) => dt.with_timezone(&chrono::Utc),
1710 chrono::LocalResult::Ambiguous(earliest, _) => {
1711 earliest.with_timezone(&chrono::Utc)
1712 }
1713 chrono::LocalResult::None => ndt.and_utc(),
1714 }
1715 } else {
1716 candidate_local.with_timezone(&chrono::Utc)
1717 };
1718 tracing::info!(
1719 retry_at = %chosen,
1720 raw = date_str,
1721 marker = retry_marker,
1722 "parsed rate limit retry-at time-only; interpreted as next occurrence"
1723 );
1724 return Some(chosen.timestamp());
1725 }
1726 }
1727 }
1728 }
1729
1730 None
1734}
1735
1736pub(crate) fn parse_relative_usage_window(error_message: &str) -> Option<u64> {
1747 static WINDOW_RE: LazyLock<Regex> = LazyLock::new(|| {
1748 Regex::new(
1749 r"(?i)(?:^|\s|[\(\)\.,;:!?])(?:(\d+)\s*-?\s*(hour|hours|hr|hrs|minute|minutes|min|mins|day|days)|(hourly|daily|weekly|monthly))(?:$|\s|[\(\)\.,;:!?])"
1750 )
1751 .expect("usage window regex is valid")
1752 });
1753
1754 if error_message.is_empty() {
1755 return None;
1756 }
1757
1758 let lower = error_message.to_lowercase();
1759 let has_context = lower.contains("usage limit")
1760 || lower.contains("quota")
1761 || lower.contains("window")
1762 || lower.contains("limit");
1763 if !has_context {
1764 return None;
1765 }
1766
1767 let mut max_secs: Option<u64> = None;
1768 for caps in WINDOW_RE.captures_iter(&lower) {
1769 let secs = if let Some(num) = caps.get(1) {
1770 let n: u64 = num.as_str().parse().ok()?;
1771 let unit = caps.get(2)?.as_str().to_lowercase();
1772 let multiplier = match unit.as_str() {
1773 "hour" | "hours" | "hr" | "hrs" => 3600,
1774 "minute" | "minutes" | "min" | "mins" => 60,
1775 "day" | "days" => 86400,
1776 _ => return None,
1777 };
1778 n.checked_mul(multiplier)?
1779 } else if let Some(word) = caps.get(3) {
1780 match word.as_str().to_lowercase().as_str() {
1781 "hourly" => 3600,
1782 "daily" => 86400,
1783 "weekly" => 604_800,
1784 "monthly" => 2_592_000,
1785 _ => return None,
1786 }
1787 } else {
1788 continue;
1789 };
1790 max_secs = Some(max_secs.map_or(secs, |m| m.max(secs)));
1791 }
1792
1793 max_secs
1794}
1795
1796#[cfg(test)]
1797pub(crate) async fn reset_global_state() {
1798 cooldowns().lock().unwrap().clear();
1799 *cooldown_store().lock().await = None;
1800 github_5xx_timestamps().lock().unwrap().clear();
1801 *github_5xx_circuit_open().lock().unwrap() = None;
1802 degraded_agents().lock().unwrap().clear();
1803 opencode::reset_model_caches_for_test();
1804}
1805
1806#[cfg(test)]
1807mod tests {
1808 use std::time::Duration;
1809
1810 use super::*;
1811 use chrono::Datelike;
1812 use chrono::TimeZone;
1813 use serial_test::serial;
1814 use tokio::time;
1815
1816 async fn test_store() -> Arc<TaskStore> {
1817 Arc::new(TaskStore::open_memory().await.expect("in-memory store"))
1818 }
1819
1820 fn sqlite_datetime_offset(secs: i64) -> String {
1821 (chrono::Utc::now() + chrono::Duration::seconds(secs))
1822 .format("%Y-%m-%d %H:%M:%S")
1823 .to_string()
1824 }
1825
1826 #[serial(cooldown_state)]
1827 #[tokio::test]
1828 async fn init_loads_unexpired_cooldowns_from_kv() {
1829 super::reset_global_state().await;
1830 let store = test_store().await;
1831 let cooldown_until = chrono::Utc::now().timestamp() + 3600; store
1834 .kv_set(
1835 &format!("{KV_PREFIX}kimi:k2p5"),
1836 &cooldown_until.to_string(),
1837 )
1838 .await
1839 .unwrap();
1840
1841 let expired = chrono::Utc::now().timestamp() - 1;
1843 store
1844 .kv_set(
1845 &format!("{KV_PREFIX}opencode:old-free"),
1846 &expired.to_string(),
1847 )
1848 .await
1849 .unwrap();
1850
1851 {
1852 let mut map = cooldown_lock();
1853 map.clear();
1854 }
1855
1856 init_cooldown_store(store).await;
1857
1858 assert!(
1859 is_model_in_cooldown("kimi", "k2p5"),
1860 "unexpired cooldown should be loaded"
1861 );
1862 assert!(
1863 !is_model_in_cooldown("opencode", "old-free"),
1864 "expired cooldown should not be loaded"
1865 );
1866 }
1867
1868 #[serial(cooldown_state)]
1869 #[tokio::test]
1870 async fn credit_exhaustion_inside_active_cooldown_keeps_counter_at_one() {
1871 super::reset_global_state().await;
1872 let store = test_store().await;
1873 *cooldown_store().lock().await = Some(Arc::clone(&store));
1874
1875 let msg =
1876 "API error (status 402 Payment Required): usage balance exhausted, out of credits";
1877 assert!(record_credit_exhaustion_from_message("credit_dup_agent", Some("m"), msg).await);
1878 assert!(record_credit_exhaustion_from_message("credit_dup_agent", Some("m"), msg).await);
1879
1880 assert!(is_agent_in_cooldown("credit_dup_agent"));
1881 let count = store
1882 .kv_get(&format!("{CREDIT_FAILURE_COUNT_PREFIX}credit_dup_agent"))
1883 .await
1884 .unwrap();
1885 assert_eq!(count.as_deref(), Some("1"));
1886 }
1887
1888 #[serial(cooldown_state)]
1889 #[tokio::test]
1890 async fn non_credit_message_is_not_recorded_as_credit_exhaustion() {
1891 super::reset_global_state().await;
1892 assert!(!record_credit_exhaustion_from_message("x_agent", Some("m"), "boom").await);
1893 assert!(!is_agent_in_cooldown("x_agent"));
1894 }
1895
1896 #[serial(cooldown_state)]
1897 #[tokio::test]
1898 async fn record_model_failure_writes_to_kv() {
1899 super::reset_global_state().await;
1900 let store = test_store().await;
1901
1902 *cooldown_store().lock().await = Some(Arc::clone(&store));
1903
1904 record_model_failure("testagent_persist", "testmodel_persist").await;
1905
1906 time::sleep(Duration::from_millis(50)).await;
1908
1909 let kv_key = format!("{KV_PREFIX}testagent_persist:testmodel_persist");
1910 let stored = store.kv_get(&kv_key).await.unwrap();
1911 assert!(stored.is_some(), "model failure should be persisted to KV");
1912 let (ts, reason) = parse_cooldown_value(&stored.unwrap()).expect("cooldown value");
1913 let now = chrono::Utc::now().timestamp();
1914 assert!(ts > now, "persisted timestamp should be in the future");
1916 assert_eq!(reason, "model_error");
1917 }
1918
1919 #[serial(cooldown_state)]
1920 #[tokio::test]
1921 async fn cooldown_reason_survives_restart_via_kv() {
1922 super::reset_global_state().await;
1923 let store = test_store().await;
1924 let agent = "test_reason_restart";
1925 let future = chrono::Utc::now().timestamp() + 600;
1926
1927 init_cooldown_store(Arc::clone(&store)).await;
1928 set_cooldown_async(agent, future, "billing_cycle_exhausted").await;
1929
1930 {
1932 let mut map = cooldown_lock();
1933 map.clear();
1934 }
1935 init_cooldown_store(store).await;
1936
1937 let entries = list_all_cooldowns();
1938 let entry = entries.iter().find(|(key, ..)| key == agent);
1939 assert_eq!(
1940 entry.map(|(_, _, reason)| reason.as_str()),
1941 Some("billing_cycle_exhausted"),
1942 "cooldown reason must survive a restart, not collapse to 'persisted'"
1943 );
1944 }
1945
1946 #[serial(cooldown_state)]
1947 #[tokio::test]
1948 async fn record_persistent_model_failure_escalates_and_increments_failure_count() {
1949 super::reset_global_state().await;
1950 let store = test_store().await;
1951 *cooldown_store().lock().await = Some(Arc::clone(&store));
1952
1953 let agent = "testagent_persistent";
1954 let model = "testmodel_persistent";
1955 let key = format!("{agent}:{model}");
1956
1957 record_persistent_model_failure(agent, model).await;
1959 let now = chrono::Utc::now().timestamp();
1960 let first_until = cooldown_until(&key).expect("first cooldown should exist");
1961 let first_remaining = first_until.saturating_sub(now);
1962 assert!(
1963 first_remaining >= PERSISTENT_MODEL_BACKOFF_BASE_SECS - 5,
1964 "first persistent model cooldown should be ~4h, got {first_remaining}s"
1965 );
1966
1967 record_persistent_model_failure(agent, model).await;
1969 let now2 = chrono::Utc::now().timestamp();
1970 let second_until = cooldown_until(&key).expect("second cooldown should exist");
1971 let second_remaining = second_until.saturating_sub(now2);
1972 assert!(
1973 second_remaining >= (PERSISTENT_MODEL_BACKOFF_BASE_SECS * 3) - 5,
1974 "second persistent model cooldown should be ~12h, got {second_remaining}s"
1975 );
1976
1977 let count_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{agent}:{model}");
1978 let stored_count = store
1979 .kv_get(&count_key)
1980 .await
1981 .expect("kv read should succeed")
1982 .expect("failure count should be written");
1983 assert_eq!(stored_count, "2");
1984 }
1985
1986 #[serial(cooldown_state)]
1987 #[tokio::test]
1988 async fn record_agent_success_does_not_reset_persistent_failure_count() {
1989 super::reset_global_state().await;
1990 let store = test_store().await;
1991 *cooldown_store().lock().await = Some(Arc::clone(&store));
1992
1993 let agent = "testagent_persistent_success";
1994 let model = "testmodel_persistent_success";
1995 let key = format!("{agent}:{model}");
1996
1997 record_persistent_model_failure(agent, model).await;
1999 record_persistent_model_failure(agent, model).await;
2000
2001 let count_key = format!("{PERSISTENT_FAILURE_COUNT_PREFIX}{agent}:{model}");
2002 let stored_count = store
2003 .kv_get(&count_key)
2004 .await
2005 .expect("kv read should succeed")
2006 .expect("failure count should be written");
2007 assert_eq!(stored_count, "2");
2008
2009 record_agent_success(agent, model).await;
2013
2014 let stored_count_after_success = store
2015 .kv_get(&count_key)
2016 .await
2017 .expect("kv read should succeed")
2018 .expect("persistent failure count should still exist");
2019 assert_eq!(
2020 stored_count_after_success, "2",
2021 "persistent failure count must survive an intervening success"
2022 );
2023
2024 record_persistent_model_failure(agent, model).await;
2026 let now = chrono::Utc::now().timestamp();
2027 let third_until = cooldown_until(&key).expect("third cooldown should exist");
2028 let third_remaining = third_until.saturating_sub(now);
2029 assert!(
2030 third_remaining > PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2031 "third persistent model failure should escalate past the 4h base, got {third_remaining}s"
2032 );
2033 }
2034
2035 #[serial(cooldown_state)]
2036 #[tokio::test]
2037 async fn record_persistent_model_failure_stays_capped_at_7_days() {
2038 super::reset_global_state().await;
2039 let store = test_store().await;
2040 *cooldown_store().lock().await = Some(Arc::clone(&store));
2041
2042 let agent = "testagent_persistent_capped";
2043 let model = "testmodel_persistent_capped";
2044 let key = format!("{agent}:{model}");
2045
2046 for _ in 0..9 {
2047 record_persistent_model_failure(agent, model).await;
2048 }
2049
2050 let remaining = cooldown_until(&key)
2051 .expect("cooldown should exist")
2052 .saturating_sub(chrono::Utc::now().timestamp());
2053 assert!(
2054 remaining >= PERSISTENT_MODEL_BACKOFF_MAX_SECS - 5,
2055 "persistent model cooldown should stay capped at 7 days, got {remaining}s"
2056 );
2057 assert!(
2058 remaining <= PERSISTENT_MODEL_BACKOFF_MAX_SECS + 5,
2059 "persistent model cooldown should not exceed 7 days, got {remaining}s"
2060 );
2061 }
2062
2063 #[serial(cooldown_state)]
2064 #[test]
2065 fn parse_retry_at_codex_format() {
2066 let msg = "You've hit your usage limit. Upgrade to Pro, visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Mar 26th, 2026 5:55 AM.";
2067 let ts = parse_retry_at(msg);
2068 assert!(ts.is_some(), "should parse codex retry-at date");
2069
2070 let naive = chrono::NaiveDateTime::new(
2073 chrono::NaiveDate::from_ymd_opt(2026, 3, 26).unwrap(),
2074 chrono::NaiveTime::from_hms_opt(5, 55, 0).unwrap(),
2075 );
2076 let expected_utc = match chrono::Local.from_local_datetime(&naive) {
2077 chrono::LocalResult::Single(local_dt) => local_dt.with_timezone(&chrono::Utc),
2078 chrono::LocalResult::Ambiguous(earliest, _) => earliest.with_timezone(&chrono::Utc),
2079 chrono::LocalResult::None => naive.and_utc(),
2080 };
2081 assert_eq!(
2082 ts.unwrap(),
2083 expected_utc.timestamp(),
2084 "timestamp should reflect local-timezone interpretation of the vendor date"
2085 );
2086
2087 let dt = chrono::DateTime::from_timestamp(ts.unwrap(), 0).unwrap();
2089 assert_eq!(dt.month(), 3);
2090 assert_eq!(dt.day(), 26);
2091 }
2092
2093 #[serial(cooldown_state)]
2094 #[test]
2095 fn parse_retry_at_hour_only_and_month_day() {
2096 let now = chrono::Utc::now().timestamp();
2097 for msg in [
2098 "You've hit your session limit · resets 8am (America/Sao_Paulo)",
2099 "You've hit your session limit · resets 6pm (America/Sao_Paulo)",
2100 "You've hit your session limit · resets 12am (America/Sao_Paulo)",
2101 "You've hit your weekly limit · resets Jun 9 at 1am (America/Sao_Paulo)",
2102 ] {
2103 let ts = parse_retry_at(msg).unwrap_or_else(|| panic!("should parse: {msg}"));
2104 assert!(ts > now, "{msg}");
2105 }
2106 }
2107
2108 #[serial(cooldown_state)]
2109 #[test]
2110 fn parse_retry_at_weekly_resets_without_year() {
2111 let t = chrono::Local::now() + chrono::Duration::days(2);
2112 let msg = format!(
2113 "You've hit your weekly limit · resets {} {} at 1am (America/Sao_Paulo)",
2114 t.format("%b"),
2115 t.day()
2116 );
2117 let ts = parse_retry_at(&msg).expect("should parse year-less reset date");
2118 let remaining = ts - chrono::Utc::now().timestamp();
2119 assert!(
2120 remaining > 3600 && remaining < 3 * 86400,
2121 "expected ~2d retry-at, got {remaining}s"
2122 );
2123 }
2124
2125 #[serial(cooldown_state)]
2126 #[test]
2127 fn parse_retry_at_no_date() {
2128 assert!(parse_retry_at("generic rate limit error").is_none());
2129 assert!(parse_retry_at("").is_none());
2130 }
2131
2132 #[serial(cooldown_state)]
2133 #[test]
2134 fn parse_retry_at_glm_reset_format_with_seconds() {
2135 let msg = "Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-04-23 18:33:48";
2136 let ts = parse_retry_at(msg).expect("should parse GLM reset-at date");
2137
2138 let naive = chrono::NaiveDateTime::new(
2139 chrono::NaiveDate::from_ymd_opt(2026, 4, 23).expect("valid date"),
2140 chrono::NaiveTime::from_hms_opt(18, 33, 48).expect("valid time"),
2141 );
2142 let expected_utc = match chrono::Local.from_local_datetime(&naive) {
2143 chrono::LocalResult::Single(local_dt) => local_dt.with_timezone(&chrono::Utc),
2144 chrono::LocalResult::Ambiguous(earliest, _) => earliest.with_timezone(&chrono::Utc),
2145 chrono::LocalResult::None => naive.and_utc(),
2146 };
2147
2148 assert_eq!(
2149 ts,
2150 expected_utc.timestamp(),
2151 "timestamp should reflect local-timezone interpretation of vendor reset date"
2152 );
2153 }
2154
2155 #[serial(cooldown_state)]
2156 #[test]
2157 fn parse_retry_at_billing_cycle_without_date_returns_none() {
2158 let msg = "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle. Upgrade to get more.";
2161 assert!(
2162 parse_retry_at(msg).is_none(),
2163 "billing cycle without date should return None"
2164 );
2165 }
2166
2167 #[serial(cooldown_state)]
2168 #[test]
2169 fn parse_relative_usage_window_extracts_kimi_five_hour_window() {
2170 let msg = "Failed to authenticate. API Error: 403 You've reached your 5-hour usage limit. Your quota will reset when the current 5-hour window expires.";
2171 assert_eq!(
2172 parse_relative_usage_window(msg),
2173 Some(5 * 3600),
2174 "should extract the 5-hour window as seconds"
2175 );
2176 }
2177
2178 #[serial(cooldown_state)]
2179 #[test]
2180 fn parse_relative_usage_window_handles_common_durations() {
2181 assert_eq!(
2182 parse_relative_usage_window("You've reached your hourly usage limit."),
2183 Some(3600)
2184 );
2185 assert_eq!(
2186 parse_relative_usage_window("You've reached your daily usage limit."),
2187 Some(86400)
2188 );
2189 assert_eq!(
2190 parse_relative_usage_window("You've reached your weekly limit."),
2191 Some(604_800)
2192 );
2193 assert_eq!(
2194 parse_relative_usage_window("You've reached your 30-minute usage limit."),
2195 Some(1800)
2196 );
2197 assert_eq!(
2198 parse_relative_usage_window("You've reached your 2-day quota."),
2199 Some(172_800)
2200 );
2201 }
2202
2203 #[serial(cooldown_state)]
2204 #[test]
2205 fn parse_relative_usage_window_returns_none_without_usage_context() {
2206 assert!(parse_relative_usage_window("429 Too Many Requests").is_none());
2209 assert!(parse_relative_usage_window("rate limit exceeded").is_none());
2210 assert!(parse_relative_usage_window("").is_none());
2211 }
2212
2213 #[serial(cooldown_state)]
2214 #[tokio::test]
2215 async fn record_rate_limit_uses_model_specific_window_cooldown() {
2216 super::reset_global_state().await;
2217 let store = test_store().await;
2218 *cooldown_store().lock().await = Some(Arc::clone(&store));
2219
2220 let agent = "test_record_rl_window_agent";
2221 let model = "test_record_rl_window_model";
2222 let msg = "You've reached your 5-hour usage limit. Your quota will reset when the current 5-hour window expires.";
2223
2224 record_rate_limit(agent, Some(model), msg).await;
2225
2226 assert!(
2227 is_model_in_cooldown(agent, model),
2228 "model should be cooled for the relative usage window"
2229 );
2230 assert!(
2231 !is_agent_in_cooldown(agent),
2232 "agent should not be cooled for a model-scoped quota"
2233 );
2234
2235 let key = format!("{agent}:{model}");
2236 let remaining = cooldown_until(&key)
2237 .expect("model cooldown should exist")
2238 .saturating_sub(chrono::Utc::now().timestamp());
2239 assert!(
2240 remaining >= 5 * 3600 - 5,
2241 "model cooldown should be at least ~5 hours, got {remaining}s"
2242 );
2243 assert_eq!(
2244 cooldown_reason(&key).as_deref(),
2245 Some("rate_limit_usage_window"),
2246 "relative usage-window cooldown must be labeled as a rate limit, not silence"
2247 );
2248 }
2249
2250 #[serial(cooldown_state)]
2251 #[tokio::test]
2252 async fn record_rate_limit_without_window_falls_back_to_agent_and_model() {
2253 super::reset_global_state().await;
2254 let store = test_store().await;
2255 *cooldown_store().lock().await = Some(Arc::clone(&store));
2256
2257 let agent = "test_record_rl_fallback_agent";
2258 let model = "test_record_rl_fallback_model";
2259 let msg = "429 Too Many Requests";
2260
2261 record_rate_limit(agent, Some(model), msg).await;
2262
2263 assert!(
2264 is_agent_in_cooldown(agent),
2265 "agent should be cooled when no relative window is detected"
2266 );
2267 assert!(
2268 is_model_in_cooldown(agent, model),
2269 "model should be cooled when no relative window is detected"
2270 );
2271 }
2272
2273 #[serial(cooldown_state)]
2274 #[tokio::test]
2275 async fn agent_cooldown_persists_to_kv() {
2276 super::reset_global_state().await;
2277 let agent = "test_agent_persist_check";
2279 record_agent_failure_with_message(agent, "").await;
2280 assert!(is_agent_in_cooldown(agent));
2281 }
2282
2283 #[serial(cooldown_state)]
2284 #[tokio::test]
2285 async fn cooldown_never_shortens() {
2286 super::reset_global_state().await;
2287 let agent = "test_agent_no_shorten";
2288 set_agent_cooldown(agent, 24 * 60 * 60).await;
2289 let initial = {
2290 let map = cooldowns().lock().unwrap();
2291 map.get(agent)
2292 .expect("cooldown entry should exist")
2293 .cooldown_until
2294 };
2295
2296 record_agent_failure_with_message(agent, "rate limit").await;
2298 let after = {
2299 let map = cooldowns().lock().unwrap();
2300 map.get(agent)
2301 .expect("cooldown entry should exist")
2302 .cooldown_until
2303 };
2304 assert_eq!(
2305 initial, after,
2306 "cooldown should not be shortened by later failures"
2307 );
2308 }
2309
2310 #[serial(cooldown_state)]
2311 #[tokio::test]
2312 async fn silence_agent_cooldown_is_short_lived() {
2313 super::reset_global_state().await;
2314 let agent = "test_silence_agent_cd";
2315 assert!(!is_agent_in_cooldown(agent));
2316
2317 set_agent_cooldown(agent, SILENCE_AGENT_COOLDOWN_SECS).await;
2318 assert!(is_agent_in_cooldown(agent));
2319
2320 let map = cooldowns().lock().unwrap();
2322 let entry = map.get(agent).expect("should have cooldown entry");
2323 let now = chrono::Utc::now().timestamp();
2324 let remaining = entry.cooldown_until - now;
2325 assert!(
2326 remaining <= i64::try_from(SILENCE_AGENT_COOLDOWN_SECS).unwrap(),
2327 "silence agent cooldown should be <= {SILENCE_AGENT_COOLDOWN_SECS}s, got {remaining}s"
2328 );
2329 assert!(remaining > 0, "cooldown should still be active");
2330 }
2331
2332 #[serial(cooldown_state)]
2333 #[tokio::test]
2334 async fn record_silence_detection_applies_extended_cooldown() {
2335 super::reset_global_state().await;
2336 let store = test_store().await;
2337 *cooldown_store().lock().await = Some(Arc::clone(&store));
2338
2339 let agent = "test_silence_count_agent";
2340 let model = "test_silence_count_model";
2341 assert!(!is_model_in_cooldown(agent, model));
2342
2343 for _ in 0..SILENCE_COUNT_THRESHOLD {
2344 let result = record_silence_detection(agent, model).await;
2345 assert!(result.is_some());
2346 }
2347
2348 assert!(is_model_in_cooldown(agent, model));
2349 let key = format!("{agent}:{model}");
2350 let remaining = {
2351 let map = cooldowns().lock().unwrap();
2352 let entry = map.get(&key).expect("cooldown entry should exist");
2353 entry.cooldown_until - chrono::Utc::now().timestamp()
2354 };
2355 assert!(
2356 remaining >= i64::try_from(SILENCE_EXTENDED_COOLDOWN_SECS).unwrap() - 5,
2357 "extended cooldown should be applied, got {remaining}s"
2358 );
2359
2360 let kv_key = format!("{SILENCE_COUNT_PREFIX}{agent}:{model}");
2361 let stored = store.kv_get(&kv_key).await.unwrap().unwrap();
2362 assert_eq!(stored, "[]");
2363 }
2364
2365 #[serial(cooldown_state)]
2366 #[test]
2367 fn detect_credit_exhaustion_out_of_credits() {
2368 let msg = "Error: API error - {\"error\":{\"message\":\"You exceeded your current quota\",\"type\":\"insufficient_quota\",\"param\":null,\"code\":\"insufficient_quota\"}}";
2369 assert_eq!(
2370 detect_credit_exhaustion(msg),
2371 Some(CreditExhaustionReason::OutOfCredits)
2372 );
2373 }
2374
2375 #[serial(cooldown_state)]
2376 #[test]
2377 fn detect_credit_exhaustion_out_of_credits_variations() {
2378 assert_eq!(
2379 detect_credit_exhaustion("out of credits"),
2380 Some(CreditExhaustionReason::OutOfCredits)
2381 );
2382 assert_eq!(
2383 detect_credit_exhaustion("outofcredits"),
2384 Some(CreditExhaustionReason::OutOfCredits)
2385 );
2386 assert_eq!(
2387 detect_credit_exhaustion("no credits remaining"),
2388 Some(CreditExhaustionReason::OutOfCredits)
2389 );
2390 assert_eq!(
2391 detect_credit_exhaustion("insufficient funds"),
2392 Some(CreditExhaustionReason::OutOfCredits)
2393 );
2394 }
2395
2396 #[serial(cooldown_state)]
2397 #[test]
2398 fn detect_credit_exhaustion_org_level_disabled() {
2399 let msg = "Error: API error - {\"error\":{\"message\":\"Organization has been disabled\",\"type\":\"org_disabled\",\"code\":\"overageDisabledReason:org_level_disabled\"}}";
2400 assert_eq!(
2401 detect_credit_exhaustion(msg),
2402 Some(CreditExhaustionReason::OrgLevelDisabled)
2403 );
2404 }
2405
2406 #[serial(cooldown_state)]
2407 #[test]
2408 fn detect_credit_exhaustion_org_level_disabled_variations() {
2409 assert_eq!(
2410 detect_credit_exhaustion("organization has been disabled"),
2411 Some(CreditExhaustionReason::OrgLevelDisabled)
2412 );
2413 assert_eq!(
2414 detect_credit_exhaustion("org disabled"),
2415 Some(CreditExhaustionReason::OrgLevelDisabled)
2416 );
2417 assert_eq!(
2418 detect_credit_exhaustion("billing quota exceeded"),
2419 Some(CreditExhaustionReason::OrgLevelDisabled)
2420 );
2421 assert_eq!(
2422 detect_credit_exhaustion("credit balance too low"),
2423 Some(CreditExhaustionReason::OrgLevelDisabled)
2424 );
2425 }
2426
2427 #[serial(cooldown_state)]
2428 #[test]
2429 fn detect_credit_exhaustion_overage_disabled_reason_parsing() {
2430 let msg = "overageDisabledReason: out_of_credits";
2431 assert_eq!(
2432 detect_credit_exhaustion(msg),
2433 Some(CreditExhaustionReason::OutOfCredits)
2434 );
2435
2436 let msg = "overageDisabledReason: org_level_disabled";
2437 assert_eq!(
2438 detect_credit_exhaustion(msg),
2439 Some(CreditExhaustionReason::OrgLevelDisabled)
2440 );
2441 }
2442
2443 #[serial(cooldown_state)]
2444 #[test]
2445 fn detect_credit_exhaustion_glm_insufficient_balance() {
2446 assert_eq!(
2448 detect_credit_exhaustion(
2449 "API Error: Request rejected (429) · Insufficient balance or no resource package. Please recharge."
2450 ),
2451 Some(CreditExhaustionReason::OutOfCredits)
2452 );
2453 assert_eq!(
2454 detect_credit_exhaustion("Insufficient balance"),
2455 Some(CreditExhaustionReason::OutOfCredits)
2456 );
2457 assert_eq!(
2458 detect_credit_exhaustion("please recharge your account"),
2459 Some(CreditExhaustionReason::OutOfCredits)
2460 );
2461 assert_eq!(
2462 detect_credit_exhaustion("no resource package available"),
2463 Some(CreditExhaustionReason::OutOfCredits)
2464 );
2465 }
2466
2467 #[serial(cooldown_state)]
2468 #[test]
2469 fn detect_credit_exhaustion_none_for_regular_rate_limit() {
2470 assert!(detect_credit_exhaustion("rate limit exceeded").is_none());
2471 assert!(detect_credit_exhaustion("too many requests").is_none());
2472 assert!(detect_credit_exhaustion("429 Too Many Requests").is_none());
2473 assert!(detect_credit_exhaustion("you've hit your usage limit").is_none());
2474 assert!(
2475 detect_credit_exhaustion("You've hit your usage limit, try again in a few minutes.")
2476 .is_none()
2477 );
2478 }
2479
2480 #[serial(cooldown_state)]
2481 #[tokio::test]
2482 async fn record_credit_exhaustion_applies_agent_cooldown() {
2483 super::reset_global_state().await;
2484 let agent = "test_credit_exhaust_agent";
2485 assert!(!is_agent_in_cooldown(agent));
2486
2487 record_credit_exhaustion(agent, CreditExhaustionReason::OutOfCredits).await;
2488 assert!(is_agent_in_cooldown(agent));
2489
2490 let remaining = {
2491 let map = cooldowns().lock().unwrap();
2492 let entry = map.get(agent).expect("cooldown entry should exist");
2493 entry.cooldown_until - chrono::Utc::now().timestamp()
2494 };
2495 assert!(
2497 remaining >= CREDIT_BACKOFF_BASE_SECS - 5,
2498 "first out_of_credits cooldown should be ~1 hour, got {remaining}s"
2499 );
2500 assert!(
2501 remaining <= CREDIT_BACKOFF_MAX_SECS,
2502 "cooldown should not exceed cap of 8h, got {remaining}s"
2503 );
2504 }
2505
2506 #[serial(cooldown_state)]
2507 #[tokio::test]
2508 async fn record_credit_exhaustion_org_level_applies_longer_cooldown() {
2509 super::reset_global_state().await;
2510 let agent = "test_org_disabled_agent";
2511 assert!(!is_agent_in_cooldown(agent));
2512
2513 record_credit_exhaustion(agent, CreditExhaustionReason::OrgLevelDisabled).await;
2514 assert!(is_agent_in_cooldown(agent));
2515
2516 let remaining = {
2517 let map = cooldowns().lock().unwrap();
2518 let entry = map.get(agent).expect("cooldown entry should exist");
2519 entry.cooldown_until - chrono::Utc::now().timestamp()
2520 };
2521 assert!(
2523 remaining >= ORG_BACKOFF_BASE_SECS - 5,
2524 "first org-level disabled cooldown should be ~2 hours, got {remaining}s"
2525 );
2526 assert!(
2527 remaining <= CREDIT_BACKOFF_MAX_SECS,
2528 "cooldown should not exceed cap of 8h, got {remaining}s"
2529 );
2530 }
2531
2532 #[serial(cooldown_state)]
2533 #[tokio::test]
2534 async fn record_credit_exhaustion_billing_cycle_applies_24h() {
2535 super::reset_global_state().await;
2536 let store = test_store().await;
2537 *cooldown_store().lock().await = Some(Arc::clone(&store));
2538
2539 let agent = "test_billing_cycle_agent";
2540 assert!(!is_agent_in_cooldown(agent));
2541
2542 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2543 assert!(is_agent_in_cooldown(agent));
2544
2545 let remaining = {
2546 let map = cooldowns().lock().unwrap();
2547 let entry = map.get(agent).expect("cooldown entry should exist");
2548 entry.cooldown_until - chrono::Utc::now().timestamp()
2549 };
2550 assert!(
2552 remaining >= BILLING_CYCLE_COOLDOWN_SECS - 5,
2553 "first billing cycle cooldown should be ~24 hours, got {remaining}s"
2554 );
2555 assert!(
2556 remaining <= BILLING_CYCLE_MAX_SECS,
2557 "cooldown should not exceed cap of 7 days, got {remaining}s"
2558 );
2559 }
2560
2561 #[serial(cooldown_state)]
2562 #[tokio::test]
2563 async fn record_credit_exhaustion_billing_cycle_escalates() {
2564 super::reset_global_state().await;
2565 let store = test_store().await;
2566 *cooldown_store().lock().await = Some(Arc::clone(&store));
2567
2568 let agent = "test_billing_cycle_escalation";
2569
2570 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2572 let remaining_1 = {
2573 let map = cooldowns().lock().unwrap();
2574 let entry = map.get(agent).expect("cooldown entry should exist");
2575 entry.cooldown_until - chrono::Utc::now().timestamp()
2576 };
2577 assert!(
2578 (BILLING_CYCLE_COOLDOWN_SECS - 5..=BILLING_CYCLE_COOLDOWN_SECS + 5)
2579 .contains(&remaining_1),
2580 "first billing cycle cooldown should be ~24h, got {remaining_1}s"
2581 );
2582
2583 {
2585 let mut map = cooldowns().lock().unwrap();
2586 map.remove(agent);
2587 }
2588
2589 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2591 let remaining_2 = {
2592 let map = cooldowns().lock().unwrap();
2593 let entry = map.get(agent).expect("cooldown entry should exist");
2594 entry.cooldown_until - chrono::Utc::now().timestamp()
2595 };
2596 let expected_2 = BILLING_CYCLE_COOLDOWN_SECS * 3; assert!(
2598 remaining_2 >= expected_2 - 5,
2599 "second billing cycle cooldown should be ~72h, got {remaining_2}s"
2600 );
2601
2602 {
2604 let mut map = cooldowns().lock().unwrap();
2605 map.remove(agent);
2606 }
2607
2608 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2610 let remaining_3 = {
2611 let map = cooldowns().lock().unwrap();
2612 let entry = map.get(agent).expect("cooldown entry should exist");
2613 entry.cooldown_until - chrono::Utc::now().timestamp()
2614 };
2615 assert!(
2616 remaining_3 >= BILLING_CYCLE_MAX_SECS - 5,
2617 "third billing cycle cooldown should be capped at 7 days, got {remaining_3}s"
2618 );
2619 assert!(
2620 remaining_3 <= BILLING_CYCLE_MAX_SECS + 5,
2621 "third billing cycle cooldown should not exceed 7 days, got {remaining_3}s"
2622 );
2623 }
2624
2625 #[serial(cooldown_state)]
2626 #[tokio::test]
2627 async fn record_credit_exhaustion_billing_cycle_stays_capped_after_repeats() {
2628 super::reset_global_state().await;
2629 let store = test_store().await;
2630 *cooldown_store().lock().await = Some(Arc::clone(&store));
2631
2632 let agent = "test_billing_cycle_agent_capped";
2633
2634 for i in 0..5 {
2635 if i > 0 {
2636 cooldowns().lock().unwrap().remove(agent);
2638 }
2639 record_credit_exhaustion(agent, CreditExhaustionReason::BillingCycleExhausted).await;
2640 }
2641
2642 let remaining = {
2643 let map = cooldowns().lock().unwrap();
2644 let entry = map.get(agent).expect("cooldown entry should exist");
2645 entry.cooldown_until - chrono::Utc::now().timestamp()
2646 };
2647 assert!(
2648 remaining >= BILLING_CYCLE_MAX_SECS - 5,
2649 "billing cycle cooldown should stay capped at 7 days, got {remaining}s"
2650 );
2651 assert!(
2652 remaining <= BILLING_CYCLE_MAX_SECS + 5,
2653 "billing cycle cooldown should not exceed 7 days, got {remaining}s"
2654 );
2655 }
2656
2657 #[serial(cooldown_state)]
2658 #[test]
2659 fn credit_exhaustion_backoff_constants() {
2660 assert_eq!(
2662 compute_backoff(1, CREDIT_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS, true),
2663 CREDIT_BACKOFF_BASE_SECS
2664 );
2665 assert_eq!(
2667 compute_backoff(1, ORG_BACKOFF_BASE_SECS, CREDIT_BACKOFF_MAX_SECS, true),
2668 ORG_BACKOFF_BASE_SECS
2669 );
2670 assert_eq!(BILLING_CYCLE_COOLDOWN_SECS, 24 * 60 * 60);
2672 assert_eq!(BILLING_CYCLE_MAX_SECS, 7 * 24 * 60 * 60);
2673 }
2674
2675 #[serial(cooldown_state)]
2676 #[test]
2677 fn compute_backoff_long_cap_without_extended_stays_capped() {
2678 assert_eq!(
2679 compute_backoff(
2680 3,
2681 BILLING_CYCLE_COOLDOWN_SECS,
2682 BILLING_CYCLE_MAX_SECS,
2683 false
2684 ),
2685 BILLING_CYCLE_MAX_SECS,
2686 "7-day capped paths should stop at the documented max"
2687 );
2688 assert_eq!(
2689 compute_backoff(
2690 9,
2691 PERSISTENT_MODEL_BACKOFF_BASE_SECS,
2692 PERSISTENT_MODEL_BACKOFF_MAX_SECS,
2693 false,
2694 ),
2695 PERSISTENT_MODEL_BACKOFF_MAX_SECS,
2696 "persistent model failures should not extend past 7 days"
2697 );
2698 }
2699
2700 #[serial(cooldown_state)]
2703 #[test]
2704 fn mark_and_check_agent_degraded() {
2705 let agent = "test_degraded_mark";
2706 assert!(!is_agent_degraded(agent));
2707 mark_agent_degraded(agent);
2708 assert!(is_agent_degraded(agent));
2709 clear_agent_degraded(agent);
2710 assert!(!is_agent_degraded(agent));
2711 }
2712
2713 #[serial(cooldown_state)]
2714 #[tokio::test]
2715 async fn sync_from_kv_drops_in_memory_entry_cleared_externally() {
2716 super::reset_global_state().await;
2717 let store = test_store().await;
2718 let agent = "test_sync_external_clear";
2719 let future = chrono::Utc::now().timestamp() + 600;
2720
2721 init_cooldown_store(Arc::clone(&store)).await;
2723 set_cooldown_async(agent, future, "test").await;
2724 assert!(is_agent_in_cooldown(agent));
2725
2726 store
2730 .kv_set(&format!("{KV_PREFIX}{agent}"), "0")
2731 .await
2732 .expect("kv_set tombstone");
2733 assert!(
2734 is_agent_in_cooldown(agent),
2735 "in-memory cooldown should still be present before sync"
2736 );
2737
2738 sync_from_kv(&store).await;
2739
2740 assert!(
2741 !is_agent_in_cooldown(agent),
2742 "sync_from_kv must drop in-memory entries the KV tombstoned"
2743 );
2744 }
2745
2746 #[serial(cooldown_state)]
2747 #[tokio::test]
2748 async fn sync_from_kv_closes_github_circuit_when_row_cleared() {
2749 super::reset_global_state().await;
2750 let store = test_store().await;
2751 {
2752 *github_5xx_circuit_open_lock() = Some(chrono::Utc::now().timestamp() + 600);
2753 github_5xx_timestamps_lock().push(chrono::Utc::now().timestamp());
2754 }
2755 assert!(is_github_circuit_open());
2756
2757 store
2758 .kv_set("cooldown:github:5xx", "0")
2759 .await
2760 .expect("kv_set tombstone");
2761 sync_from_kv(&store).await;
2762
2763 assert!(!is_github_circuit_open());
2764 assert!(github_5xx_timestamps_lock().is_empty());
2765 }
2766
2767 #[serial(cooldown_state)]
2768 #[tokio::test]
2769 async fn sync_from_kv_keeps_5xx_window_when_expired_row_and_breaker_closed() {
2770 super::reset_global_state().await;
2771 let store = test_store().await;
2772 let now = chrono::Utc::now().timestamp();
2773 {
2774 github_5xx_timestamps_lock().push(now);
2775 github_5xx_timestamps_lock().push(now);
2776 }
2777
2778 store
2779 .kv_set("cooldown:github:5xx", &(now - 600).to_string())
2780 .await
2781 .expect("kv_set past");
2782 sync_from_kv(&store).await;
2783
2784 assert!(!is_github_circuit_open());
2785 assert_eq!(github_5xx_timestamps_lock().len(), 2);
2786
2787 github_5xx_timestamps_lock().clear();
2788 }
2789
2790 #[serial(cooldown_state)]
2791 #[tokio::test]
2792 async fn sync_from_kv_keeps_github_circuit_open_for_future_row() {
2793 super::reset_global_state().await;
2794 let store = test_store().await;
2795 let future = chrono::Utc::now().timestamp() + 600;
2796 *github_5xx_circuit_open_lock() = Some(future);
2797
2798 store
2799 .kv_set("cooldown:github:5xx", &future.to_string())
2800 .await
2801 .expect("kv_set future");
2802 sync_from_kv(&store).await;
2803
2804 assert!(is_github_circuit_open());
2805
2806 *github_5xx_circuit_open_lock() = None;
2807 }
2808
2809 #[serial(cooldown_state)]
2810 #[tokio::test]
2811 async fn sync_from_kv_keeps_in_memory_entry_when_kv_row_missing() {
2812 super::reset_global_state().await;
2816 let store = test_store().await;
2817 let agent = "test_sync_race_window";
2818 let future = chrono::Utc::now().timestamp() + 600;
2819
2820 set_cooldown_in_memory(agent, future, "raced");
2822 assert!(is_agent_in_cooldown(agent));
2823
2824 sync_from_kv(&store).await;
2825
2826 assert!(
2827 is_agent_in_cooldown(agent),
2828 "sync_from_kv must NOT drop entries whose KV row is missing — that is the set_cooldown_async race window"
2829 );
2830 }
2831
2832 #[serial(cooldown_state)]
2833 #[tokio::test]
2834 async fn sync_from_kv_adopts_kv_timestamp() {
2835 super::reset_global_state().await;
2838 let store = test_store().await;
2839 let agent = "test_sync_adopt_kv";
2840 let original = chrono::Utc::now().timestamp() + 600;
2841 let updated = chrono::Utc::now().timestamp() + 1200;
2842
2843 init_cooldown_store(Arc::clone(&store)).await;
2844 set_cooldown_async(agent, original, "initial").await;
2845
2846 store
2847 .kv_set(&format!("{KV_PREFIX}{agent}"), &updated.to_string())
2848 .await
2849 .expect("kv_set updated");
2850
2851 sync_from_kv(&store).await;
2852
2853 let until = cooldown_until(agent).expect("cooldown still active");
2854 assert_eq!(
2855 until, updated,
2856 "sync_from_kv must adopt KV's authoritative timestamp"
2857 );
2858 }
2859
2860 #[serial(cooldown_state)]
2861 #[tokio::test]
2862 async fn refresh_degraded_agents_marks_cooled_agent() {
2863 super::reset_global_state().await;
2864 let store = test_store().await;
2865 let agent = "test_refresh_cooled";
2866
2867 record_agent_failure_with_message(agent, "").await;
2869 assert!(is_agent_in_cooldown(agent));
2870
2871 let agents = vec![agent.to_string()];
2873 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
2874
2875 assert!(
2876 is_agent_degraded(agent),
2877 "agent in cooldown should be marked degraded"
2878 );
2879
2880 clear_agent_degraded(agent);
2882 }
2883
2884 #[serial(cooldown_state)]
2885 #[tokio::test]
2886 async fn refresh_degraded_agents_marks_no_models_agent() {
2887 super::reset_global_state().await;
2888 let store = test_store().await;
2889 let agent = "test_refresh_no_models";
2890
2891 let agents = vec![agent.to_string()];
2893 refresh_degraded_agents(&store, &agents, &|_| false, 6, 3).await;
2894
2895 assert!(
2896 is_agent_degraded(agent),
2897 "agent with no available models should be marked degraded"
2898 );
2899
2900 clear_agent_degraded(agent);
2902 }
2903
2904 #[serial(cooldown_state)]
2905 #[tokio::test]
2906 async fn refresh_degraded_agents_marks_rate_limited_agent() {
2907 super::reset_global_state().await;
2908 let store = test_store().await;
2909 let agent = "test_refresh_rate_limited";
2910
2911 for _ in 0..4 {
2913 store.record_rate_limit(agent, "rate", None).await.unwrap();
2914 }
2915
2916 let agents = vec![agent.to_string()];
2917 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
2919
2920 assert!(
2921 is_agent_degraded(agent),
2922 "agent with rate limit count >= threshold should be marked degraded"
2923 );
2924
2925 clear_agent_degraded(agent);
2927 }
2928
2929 #[serial(cooldown_state)]
2930 #[tokio::test]
2931 async fn refresh_degraded_agents_clears_healthy_agent() {
2932 super::reset_global_state().await;
2933 let store = test_store().await;
2934 let agent = "test_refresh_healthy";
2935
2936 mark_agent_degraded(agent);
2938 assert!(is_agent_degraded(agent));
2939
2940 let agents = vec![agent.to_string()];
2942 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
2943
2944 assert!(
2945 !is_agent_degraded(agent),
2946 "healthy agent should have degraded flag cleared"
2947 );
2948 }
2949
2950 #[serial(cooldown_state)]
2951 #[tokio::test]
2952 async fn refresh_degraded_agents_ignores_expired_cooldowns() {
2953 super::reset_global_state().await;
2954 let store = test_store().await;
2955 let agent = "test_refresh_expired_cooldown";
2956
2957 let expired_ts = chrono::Utc::now().timestamp() - 100; {
2961 let mut map = cooldowns().lock().unwrap();
2962 map.insert(
2963 agent.to_string(),
2964 CooldownEntry {
2965 cooldown_until: expired_ts,
2966 reason: "expired_test".to_string(),
2967 },
2968 );
2969 }
2970
2971 assert!(
2973 !is_agent_in_cooldown(agent),
2974 "expired cooldown should not be active"
2975 );
2976
2977 mark_agent_degraded(agent);
2979 assert!(is_agent_degraded(agent));
2980
2981 let agents = vec![agent.to_string()];
2983 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
2984
2985 assert!(
2986 !is_agent_degraded(agent),
2987 "agent with only expired cooldown should NOT be marked degraded"
2988 );
2989
2990 clear_agent_degraded(agent);
2992 {
2993 let mut map = cooldowns().lock().unwrap();
2994 map.remove(agent);
2995 }
2996 }
2997
2998 #[serial(cooldown_state)]
2999 #[test]
3000 fn compute_backoff_grows_exponentially() {
3001 assert_eq!(compute_backoff(0, 300, 14400, true), 300);
3003 assert_eq!(compute_backoff(1, 300, 14400, true), 300);
3004 assert_eq!(compute_backoff(2, 300, 14400, true), 900);
3006 assert_eq!(compute_backoff(3, 300, 14400, true), 2700);
3008 assert_eq!(compute_backoff(4, 300, 14400, true), 8100);
3010 assert_eq!(compute_backoff(5, 300, 14400, true), 14400);
3012 }
3013
3014 #[serial(cooldown_state)]
3015 #[test]
3016 fn compute_backoff_extended_tier_1_applies_24h() {
3017 assert_eq!(
3023 compute_backoff(5, 300, 14400, true),
3024 14400,
3025 "first at max, not extended"
3026 );
3027 assert_eq!(
3028 compute_backoff(6, 300, 14400, true),
3029 14400,
3030 "times_at_max=1, not extended"
3031 );
3032 assert_eq!(
3033 compute_backoff(7, 300, 14400, true),
3034 86400,
3035 "times_at_max=2, extended 6x"
3036 );
3037 assert_eq!(
3038 compute_backoff(8, 300, 14400, true),
3039 86400,
3040 "times_at_max=3, still extended 6x"
3041 );
3042 }
3043
3044 #[serial(cooldown_state)]
3045 #[test]
3046 fn compute_backoff_extended_tier_2_applies_48h() {
3047 assert_eq!(
3050 compute_backoff(9, 300, 14400, true),
3051 86400,
3052 "times_at_max=4, still tier 1"
3053 );
3054 assert_eq!(
3055 compute_backoff(10, 300, 14400, true),
3056 172_800,
3057 "times_at_max=5, extended 12x"
3058 );
3059 assert_eq!(
3060 compute_backoff(15, 300, 14400, true),
3061 172_800,
3062 "times_at_max=10, still tier 2"
3063 );
3064 assert_eq!(
3065 compute_backoff(20, 300, 14400, true),
3066 172_800,
3067 "times_at_max=15, still tier 2"
3068 );
3069 }
3070
3071 #[serial(cooldown_state)]
3072 #[test]
3073 fn compute_backoff_extended_tier_with_different_base_max() {
3074 assert_eq!(compute_backoff(5, 60, 3600, true), 3600, "first at max");
3081 assert_eq!(compute_backoff(6, 60, 3600, true), 3600, "times_at_max=1");
3082 assert_eq!(
3083 compute_backoff(7, 60, 3600, true),
3084 21600,
3085 "times_at_max=2, extended 6x"
3086 );
3087 }
3088
3089 #[serial(cooldown_state)]
3090 #[test]
3091 fn compute_backoff_extended_tier_persistent_fails_example() {
3092 assert_eq!(
3096 compute_backoff(21, 300, 14400, true),
3097 172_800,
3098 "21 failures should get 48h extended cooldown"
3099 );
3100
3101 assert_eq!(
3103 compute_backoff(36, 300, 14400, true),
3104 172_800,
3105 "36 failures should also get 48h extended cooldown"
3106 );
3107 }
3108
3109 #[serial(cooldown_state)]
3110 #[test]
3111 fn compute_backoff_extended_tier_zero_base() {
3112 assert_eq!(compute_backoff(1, 0, 14400, true), 0);
3114 assert_eq!(compute_backoff(5, 0, 14400, true), 0);
3115 assert_eq!(compute_backoff(10, 0, 14400, true), 0);
3116 }
3117
3118 #[serial(cooldown_state)]
3119 #[test]
3120 fn detect_credit_exhaustion_billing_cycle() {
3121 assert_eq!(
3122 detect_credit_exhaustion("billing cycle"),
3123 Some(CreditExhaustionReason::BillingCycleExhausted)
3124 );
3125 assert_eq!(
3126 detect_credit_exhaustion("quota refreshed next cycle"),
3127 Some(CreditExhaustionReason::BillingCycleExhausted)
3128 );
3129 assert_eq!(
3130 detect_credit_exhaustion(
3131 "You've reached your usage limit for this billing cycle. Your quota will be refreshed in the next cycle."
3132 ),
3133 Some(CreditExhaustionReason::BillingCycleExhausted)
3134 );
3135 assert_eq!(
3136 detect_credit_exhaustion("monthly quota exceeded"),
3137 Some(CreditExhaustionReason::BillingCycleExhausted)
3138 );
3139 assert_eq!(
3140 detect_credit_exhaustion(
3141 "Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-04-23 18:33:48"
3142 ),
3143 Some(CreditExhaustionReason::BillingCycleExhausted)
3144 );
3145 }
3146
3147 #[serial(cooldown_state)]
3148 #[test]
3149 fn detect_credit_exhaustion_billing_cycle_minimax() {
3150 assert_eq!(
3151 detect_credit_exhaustion(
3152 "minimax rate limit: API Error: Request rejected (429) · Token Plan usage limit reached: Upgrade your Token Plan or purchase Credits for more usage."
3153 ),
3154 Some(CreditExhaustionReason::BillingCycleExhausted)
3155 );
3156 assert_eq!(
3157 detect_credit_exhaustion("Upgrade your Token Plan"),
3158 Some(CreditExhaustionReason::BillingCycleExhausted)
3159 );
3160 assert_eq!(
3161 detect_credit_exhaustion("Token Plan usage limit reached"),
3162 Some(CreditExhaustionReason::BillingCycleExhausted)
3163 );
3164 assert_eq!(
3165 detect_credit_exhaustion("purchase Credits for more usage"),
3166 Some(CreditExhaustionReason::BillingCycleExhausted)
3167 );
3168 assert_eq!(
3169 detect_credit_exhaustion(
3170 "You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro), visit https://chatgpt.com/codex/settings/usage to purchase more credits"
3171 ),
3172 Some(CreditExhaustionReason::BillingCycleExhausted)
3173 );
3174 }
3175
3176 #[serial(cooldown_state)]
3177 #[test]
3178 fn detect_credit_exhaustion_outcome_kimi_billing_cycle() {
3179 let msg = "Failed to authenticate. API Error: 403 You've reached your usage limit for \
3184 this billing cycle. Your quota will be refreshed soon.";
3185 assert_eq!(
3186 detect_credit_exhaustion_outcome(msg),
3187 Some("billing_cycle_exhausted")
3188 );
3189 }
3190
3191 #[serial(cooldown_state)]
3192 #[test]
3193 fn detect_credit_exhaustion_outcome_maps_all_reasons() {
3194 assert_eq!(
3195 detect_credit_exhaustion_outcome("out of credits"),
3196 Some("credit_exhausted")
3197 );
3198 assert_eq!(
3199 detect_credit_exhaustion_outcome("organization has been disabled"),
3200 Some("org_level_disabled")
3201 );
3202 assert_eq!(
3203 detect_credit_exhaustion_outcome("monthly quota exceeded"),
3204 Some("billing_cycle_exhausted")
3205 );
3206 }
3207
3208 #[test]
3209 fn detect_credit_exhaustion_outcome_returns_none_for_unrelated_errors() {
3210 assert_eq!(
3211 detect_credit_exhaustion_outcome("429 Too Many Requests"),
3212 None
3213 );
3214 assert_eq!(detect_credit_exhaustion_outcome("connection reset"), None);
3215 }
3216
3217 #[serial(cooldown_state)]
3218 #[test]
3219 fn detect_credit_exhaustion_overage_takes_priority_over_billing_cycle() {
3220 let msg = "overageDisabledReason: org_level_disabled for billing cycle";
3223 assert_eq!(
3224 detect_credit_exhaustion(msg),
3225 Some(CreditExhaustionReason::OrgLevelDisabled)
3226 );
3227 }
3228
3229 #[serial(cooldown_state)]
3230 #[tokio::test]
3231 async fn refresh_degraded_agents_below_threshold_not_degraded() {
3232 super::reset_global_state().await;
3233 let store = test_store().await;
3234 let agent = "test_refresh_below_threshold";
3235
3236 for _ in 0..2 {
3238 store.record_rate_limit(agent, "rate", None).await.unwrap();
3239 }
3240
3241 let agents = vec![agent.to_string()];
3242 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3244
3245 assert!(
3246 !is_agent_degraded(agent),
3247 "agent with rate limit count < threshold should not be degraded"
3248 );
3249 }
3250
3251 #[serial(cooldown_state)]
3252 #[tokio::test]
3253 async fn refresh_degraded_agents_ignores_error_limit_type() {
3254 super::reset_global_state().await;
3255 let store = test_store().await;
3256 let agent = "test_refresh_error_rows";
3257
3258 for _ in 0..3 {
3259 store.record_rate_limit(agent, "error", None).await.unwrap();
3260 }
3261
3262 let agents = vec![agent.to_string()];
3263 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3264
3265 assert!(
3266 !is_agent_degraded(agent),
3267 "three error rows in 6 hours must not mark an agent degraded"
3268 );
3269 }
3270
3271 #[serial(cooldown_state)]
3272 #[tokio::test]
3273 async fn refresh_degraded_agents_marks_three_rate_rows() {
3274 super::reset_global_state().await;
3275 let store = test_store().await;
3276 let agent = "test_refresh_three_rate_rows";
3277
3278 for _ in 0..3 {
3279 store.record_rate_limit(agent, "rate", None).await.unwrap();
3280 }
3281
3282 let agents = vec![agent.to_string()];
3283 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3284
3285 assert!(
3286 is_agent_degraded(agent),
3287 "three rate rows in 6 hours must mark an agent degraded"
3288 );
3289 assert_eq!(
3290 agent_degraded_reason(agent).as_deref(),
3291 Some("rate limit threshold exceeded")
3292 );
3293 let listed = list_degraded_agents();
3294 assert!(
3295 listed
3296 .iter()
3297 .any(|(a, r)| a == agent && r == "rate limit threshold exceeded"),
3298 "orch cooldown list must show the degraded reason, got {listed:?}"
3299 );
3300
3301 clear_agent_degraded(agent);
3302 }
3303
3304 #[serial(cooldown_state)]
3305 #[tokio::test]
3306 async fn refresh_degraded_agents_logs_reason_change() {
3307 super::reset_global_state().await;
3308 let store = test_store().await;
3309 let agent = "test_refresh_reason_change";
3310 *cooldown_store().lock().await = Some(Arc::clone(&store));
3311
3312 set_agent_cooldown(agent, 600).await;
3313 let agents = vec![agent.to_string()];
3314 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3315 assert_eq!(
3316 agent_degraded_reason(agent).as_deref(),
3317 Some("agent in cooldown")
3318 );
3319
3320 clear_cooldown(agent, &store).await;
3321 let after_clear = sqlite_datetime_offset(60);
3322 for _ in 0..3 {
3323 store
3324 .record_rate_limit_at(agent, "rate", None, &after_clear)
3325 .await
3326 .unwrap();
3327 }
3328 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3329 assert_eq!(
3330 agent_degraded_reason(agent).as_deref(),
3331 Some("rate limit threshold exceeded"),
3332 "degraded reason must update when the hold changes"
3333 );
3334
3335 clear_agent_degraded(agent);
3336 }
3337
3338 #[serial(cooldown_state)]
3339 #[tokio::test]
3340 async fn clear_un_degrades_on_next_health_check() {
3341 super::reset_global_state().await;
3342 let store = test_store().await;
3343 let agent = "test_clear_un_degrades";
3344 let before = sqlite_datetime_offset(-30);
3345 let after = sqlite_datetime_offset(60);
3346
3347 for _ in 0..3 {
3348 store
3349 .record_rate_limit_at(agent, "rate", None, &before)
3350 .await
3351 .unwrap();
3352 }
3353
3354 let agents = vec![agent.to_string()];
3355 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3356 assert!(
3357 is_agent_degraded(agent),
3358 "three rate rows must mark an agent degraded"
3359 );
3360
3361 clear_cooldown(agent, &store).await;
3362 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3363 assert!(
3364 !is_agent_degraded(agent),
3365 "clear must un-degrade on the next health check"
3366 );
3367
3368 store
3369 .record_rate_limit_at(agent, "rate", None, &after)
3370 .await
3371 .unwrap();
3372 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3373 assert!(
3374 !is_agent_degraded(agent),
3375 "one rate row after the mark must not degrade again"
3376 );
3377
3378 for _ in 0..2 {
3379 store
3380 .record_rate_limit_at(agent, "rate", None, &after)
3381 .await
3382 .unwrap();
3383 }
3384 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3385 assert!(
3386 is_agent_degraded(agent),
3387 "three rate rows after the mark must degrade again"
3388 );
3389
3390 clear_agent_degraded(agent);
3391 }
3392
3393 #[serial(cooldown_state)]
3394 #[tokio::test]
3395 async fn clear_all_un_degrades_rate_limited_agent() {
3396 super::reset_global_state().await;
3397 let store = test_store().await;
3398 let agent = "test_clear_all_un_degrades";
3399 let before = sqlite_datetime_offset(-30);
3400
3401 for _ in 0..3 {
3402 store
3403 .record_rate_limit_at(agent, "rate", None, &before)
3404 .await
3405 .unwrap();
3406 }
3407
3408 let agents = vec![agent.to_string()];
3409 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3410 assert!(is_agent_degraded(agent));
3411
3412 clear_cooldown("*", &store).await;
3413 refresh_degraded_agents(&store, &agents, &|_| true, 6, 3).await;
3414 assert!(
3415 !is_agent_degraded(agent),
3416 "clear --all must un-degrade an agent held only by rate rows"
3417 );
3418 }
3419
3420 #[serial(cooldown_state)]
3423 #[tokio::test]
3424 async fn repeated_agent_failures_escalate_backoff() {
3425 super::reset_global_state().await;
3426 let store = test_store().await;
3427 *cooldown_store().lock().await = Some(Arc::clone(&store));
3428
3429 let agent = "test_escalation_agent";
3430
3431 record_agent_failure_with_message(agent, "").await;
3433 let remaining_1 = {
3434 let map = cooldowns().lock().unwrap();
3435 let entry = map.get(agent).expect("should have cooldown");
3436 entry.cooldown_until - chrono::Utc::now().timestamp()
3437 };
3438 assert!(
3439 (BACKOFF_BASE_SECS - 5..=BACKOFF_BASE_SECS + 5).contains(&remaining_1),
3440 "first failure should be ~5 min, got {remaining_1}s"
3441 );
3442
3443 {
3445 let mut map = cooldowns().lock().unwrap();
3446 map.remove(agent);
3447 }
3448
3449 record_agent_failure_with_message(agent, "").await;
3451 let remaining_2 = {
3452 let map = cooldowns().lock().unwrap();
3453 let entry = map.get(agent).expect("should have cooldown");
3454 entry.cooldown_until - chrono::Utc::now().timestamp()
3455 };
3456 assert!(
3457 remaining_2 >= BACKOFF_BASE_SECS * 3 - 5,
3458 "second failure should be ~15 min, got {remaining_2}s"
3459 );
3460 }
3461
3462 #[serial(cooldown_state)]
3463 #[tokio::test]
3464 async fn vendor_past_retry_falls_back_to_backoff() {
3465 super::reset_global_state().await;
3466 let store = test_store().await;
3470 *cooldown_store().lock().await = Some(Arc::clone(&store));
3471
3472 let agent = "test_vendor_past_retry";
3473
3474 {
3476 let mut map = cooldowns().lock().unwrap();
3477 map.remove(agent);
3478 }
3479
3480 let past_msg = "You've hit the limit; try again at Jan 01, 2000 01:00 AM.";
3482
3483 record_agent_failure_with_message(agent, past_msg).await;
3484
3485 assert!(
3489 is_agent_in_cooldown(agent),
3490 "agent should be in cooldown after failure"
3491 );
3492
3493 let remaining = {
3494 let map = cooldowns().lock().unwrap();
3495 let entry = map.get(agent).expect("cooldown entry should exist");
3496 entry.cooldown_until - chrono::Utc::now().timestamp()
3497 };
3498
3499 assert!(
3500 remaining >= BACKOFF_BASE_SECS - 5,
3501 "expected backoff of ~{BACKOFF_BASE_SECS}s when vendor timestamp is past, got {remaining}s"
3502 );
3503
3504 let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3506 let stored = store
3507 .kv_get(&kv_key)
3508 .await
3509 .unwrap()
3510 .expect("failure count should be stored");
3511 assert_eq!(stored, "1");
3512 }
3513
3514 fn window(status: &str, resets_at: Option<i64>) -> RateLimitWindow {
3515 RateLimitWindow {
3516 status: status.to_string(),
3517 window_type: "five_hour".to_string(),
3518 utilization: Some(1.0),
3519 resets_at,
3520 }
3521 }
3522
3523 #[serial(cooldown_state)]
3524 #[tokio::test]
3525 async fn rejected_window_cools_agent_until_reset() {
3526 super::reset_global_state().await;
3527 let store = test_store().await;
3528 *cooldown_store().lock().await = Some(Arc::clone(&store));
3529
3530 let agent = "test_window_rejected_agent";
3531 let reset = chrono::Utc::now().timestamp() + 7200;
3532 record_rate_limit_window(agent, &window("rejected", Some(reset))).await;
3533
3534 assert_eq!(cooldown_until(agent), Some(reset));
3535 let stored = list_rate_limit_windows(&store).await;
3536 assert_eq!(
3537 stored,
3538 vec![(agent.to_string(), window("rejected", Some(reset)))]
3539 );
3540 }
3541
3542 #[serial(cooldown_state)]
3543 #[tokio::test]
3544 async fn allowed_window_changes_no_cooldown() {
3545 super::reset_global_state().await;
3546 let store = test_store().await;
3547 *cooldown_store().lock().await = Some(Arc::clone(&store));
3548
3549 let agent = "test_window_allowed_agent";
3550 let reset = chrono::Utc::now().timestamp() + 7200;
3551 record_rate_limit_window(agent, &window("allowed", Some(reset))).await;
3552 record_rate_limit_window(agent, &window("allowed_warning", Some(reset))).await;
3553
3554 assert_eq!(cooldown_until(agent), None);
3555 assert_eq!(list_rate_limit_windows(&store).await.len(), 1);
3556 }
3557
3558 #[serial(cooldown_state)]
3559 #[tokio::test]
3560 async fn record_agent_success_resets_backoff() {
3561 super::reset_global_state().await;
3562 let store = test_store().await;
3563 *cooldown_store().lock().await = Some(Arc::clone(&store));
3564
3565 let agent = "test_success_reset_agent";
3566 let model = "test_model";
3567
3568 for _ in 0..3 {
3570 let _ = read_and_increment_failure_count(Some(&store), agent).await;
3571 }
3572 let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3574 let count: u32 = store
3575 .kv_get(&kv_key)
3576 .await
3577 .unwrap()
3578 .unwrap()
3579 .parse()
3580 .unwrap();
3581 assert_eq!(count, 3);
3582
3583 record_agent_success(agent, model).await;
3585
3586 let count_after: u32 = store
3587 .kv_get(&kv_key)
3588 .await
3589 .unwrap()
3590 .unwrap()
3591 .parse()
3592 .unwrap();
3593 assert_eq!(count_after, 0, "success should reset failure count to 0");
3594
3595 let model_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}:{model}");
3597 let model_count: u32 = store
3598 .kv_get(&model_kv_key)
3599 .await
3600 .unwrap()
3601 .unwrap()
3602 .parse()
3603 .unwrap();
3604 assert_eq!(
3605 model_count, 0,
3606 "success should reset model failure count to 0"
3607 );
3608 }
3609
3610 #[serial(cooldown_state)]
3611 #[tokio::test]
3612 async fn record_agent_success_without_model_skips_empty_model_key() {
3613 super::reset_global_state().await;
3614 let store = test_store().await;
3615 *cooldown_store().lock().await = Some(Arc::clone(&store));
3616
3617 let agent = "test_success_without_model";
3618
3619 record_agent_success(agent, "").await;
3620
3621 let agent_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3622 let agent_count = store
3623 .kv_get(&agent_kv_key)
3624 .await
3625 .unwrap()
3626 .expect("agent failure count should be stored");
3627 assert_eq!(agent_count, "0");
3628
3629 let empty_model_kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}:");
3630 let empty_model_count = store.kv_get(&empty_model_kv_key).await.unwrap();
3631 assert!(
3632 empty_model_count.is_none(),
3633 "success without model should not create an empty model failure count key"
3634 );
3635 }
3636
3637 #[serial(cooldown_state)]
3638 #[tokio::test]
3639 async fn clear_cooldown_resets_failure_counts() {
3640 super::reset_global_state().await;
3641 let store = test_store().await;
3642 *cooldown_store().lock().await = Some(Arc::clone(&store));
3643
3644 let agent = "test_clear_fc_agent";
3645
3646 for _ in 0..5 {
3648 let _ = read_and_increment_failure_count(Some(&store), agent).await;
3649 }
3650 set_agent_cooldown(agent, 3600).await;
3652
3653 let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3655 let count: u32 = store
3656 .kv_get(&kv_key)
3657 .await
3658 .unwrap()
3659 .unwrap()
3660 .parse()
3661 .unwrap();
3662 assert_eq!(count, 5);
3663
3664 clear_cooldown(agent, &store).await;
3666
3667 let count_after: u32 = store
3669 .kv_get(&kv_key)
3670 .await
3671 .unwrap()
3672 .unwrap()
3673 .parse()
3674 .unwrap();
3675 assert_eq!(
3676 count_after, 0,
3677 "clear_cooldown should reset failure count to 0"
3678 );
3679
3680 assert!(
3682 !is_agent_in_cooldown(agent),
3683 "agent should not be in cooldown after clear"
3684 );
3685 }
3686
3687 #[serial(cooldown_state)]
3688 #[tokio::test]
3689 async fn clear_all_cooldowns_resets_all_failure_counts() {
3690 super::reset_global_state().await;
3691 let store = test_store().await;
3692 *cooldown_store().lock().await = Some(Arc::clone(&store));
3693
3694 let agents = ["test_clear_all_a", "test_clear_all_b"];
3695 for agent in &agents {
3696 let _ = read_and_increment_failure_count(Some(&store), agent).await;
3697 set_agent_cooldown(agent, 3600).await;
3698 }
3699
3700 clear_cooldown("*", &store).await;
3701
3702 for agent in &agents {
3703 assert!(!is_agent_in_cooldown(agent));
3704 let kv_key = format!("{FAILURE_COUNT_PREFIX}{agent}");
3705 let count: u32 = store
3706 .kv_get(&kv_key)
3707 .await
3708 .unwrap()
3709 .unwrap()
3710 .parse()
3711 .unwrap();
3712 assert_eq!(
3713 count, 0,
3714 "clear --all should reset failure count for {agent}"
3715 );
3716 }
3717 }
3718
3719 #[serial(cooldown_state)]
3720 #[test]
3721 fn compute_backoff_saturates_on_large_counts() {
3722 assert_eq!(
3724 compute_backoff(100, BACKOFF_BASE_SECS, BACKOFF_MAX_SECS, true),
3725 BACKOFF_MAX_SECS * 12
3726 );
3727 assert_eq!(
3728 compute_backoff(u64::from(u32::MAX), 300, 14400, true),
3729 172_800
3730 );
3731 assert_eq!(compute_backoff(u64::MAX, 300, 14400, true), 172_800);
3732 }
3733
3734 #[serial(cooldown_state)]
3735 #[test]
3736 fn compute_backoff_zero_base_returns_zero() {
3737 assert_eq!(compute_backoff(1, 0, 14400, true), 0);
3738 assert_eq!(compute_backoff(5, 0, 14400, true), 0);
3739 }
3740
3741 #[serial(cooldown_state)]
3744 #[tokio::test]
3745 async fn github_circuit_stays_closed_below_threshold() {
3746 super::reset_global_state().await;
3747 {
3749 let mut open = github_5xx_circuit_open().lock().unwrap();
3750 *open = None;
3751 let mut ts = github_5xx_timestamps().lock().unwrap();
3752 ts.clear();
3753 let mut map = cooldown_lock();
3757 map.remove("github:5xx");
3758 }
3759
3760 for _ in 0..GITHUB_5XX_THRESHOLD - 1 {
3762 record_github_5xx().await;
3763 }
3764
3765 assert!(
3766 !is_github_circuit_open(),
3767 "circuit should stay closed below threshold"
3768 );
3769 assert_eq!(github_circuit_remaining_secs(), 0);
3770 }
3771
3772 #[serial(cooldown_state)]
3773 #[tokio::test]
3774 async fn github_circuit_trips_at_threshold() {
3775 super::reset_global_state().await;
3776 {
3778 let mut open = github_5xx_circuit_open_lock();
3779 *open = None;
3780 let mut ts = github_5xx_timestamps_lock();
3781 ts.clear();
3782 let mut map = cooldown_lock();
3783 map.remove("github:5xx");
3784 }
3785
3786 for _ in 0..GITHUB_5XX_THRESHOLD {
3788 record_github_5xx().await;
3789 }
3790
3791 assert!(is_github_circuit_open(), "circuit should trip at threshold");
3792 assert!(
3793 github_circuit_remaining_secs() > 0,
3794 "remaining should be > 0 when open"
3795 );
3796 assert!(
3797 github_circuit_remaining_secs() <= GITHUB_5XX_COOLDOWN_SECS as u64,
3798 "remaining should not exceed cooldown duration"
3799 );
3800
3801 {
3803 let mut open = github_5xx_circuit_open().lock().unwrap();
3804 *open = None;
3805 }
3806 }
3807
3808 #[serial(cooldown_state)]
3809 #[tokio::test]
3810 async fn github_circuit_cannot_double_trip() {
3811 super::reset_global_state().await;
3812 {
3814 let mut open = github_5xx_circuit_open_lock();
3815 *open = None;
3816 let mut ts = github_5xx_timestamps_lock();
3817 ts.clear();
3818 let mut map = cooldown_lock();
3819 map.remove("github:5xx");
3820 }
3821
3822 for _ in 0..GITHUB_5XX_THRESHOLD {
3824 record_github_5xx().await;
3825 }
3826 assert!(is_github_circuit_open());
3827
3828 let first_remaining = github_circuit_remaining_secs();
3829
3830 for _ in 0..GITHUB_5XX_THRESHOLD {
3832 record_github_5xx().await;
3833 }
3834
3835 let second_remaining = github_circuit_remaining_secs();
3836 assert!(
3837 second_remaining <= first_remaining,
3838 "additional errors should not extend cooldown"
3839 );
3840
3841 {
3843 let mut open = github_5xx_circuit_open().lock().unwrap();
3844 *open = None;
3845 }
3846 }
3847
3848 #[serial(cooldown_state)]
3849 #[tokio::test]
3850 async fn github_circuit_recovers_after_cooldown() {
3851 super::reset_global_state().await;
3852 {
3854 let mut open = github_5xx_circuit_open_lock();
3855 *open = None;
3856 let mut ts = github_5xx_timestamps_lock();
3857 ts.clear();
3858 let mut map = cooldown_lock();
3859 map.remove("github:5xx");
3860 }
3861
3862 {
3864 let mut open = github_5xx_circuit_open().lock().unwrap();
3865 *open = Some(chrono::Utc::now().timestamp() - 1);
3866 }
3867
3868 assert!(
3870 !is_github_circuit_open(),
3871 "circuit should auto-recover after cooldown"
3872 );
3873 }
3874
3875 #[serial(cooldown_state)]
3876 #[tokio::test]
3877 async fn github_circuit_sliding_window_ages_out() {
3878 super::reset_global_state().await;
3879 {
3881 let mut open = github_5xx_circuit_open_lock();
3882 *open = None;
3883 let mut ts = github_5xx_timestamps_lock();
3884 ts.clear();
3885 let mut map = cooldown_lock();
3886 map.remove("github:5xx");
3887 }
3888
3889 {
3891 let mut ts = github_5xx_timestamps().lock().unwrap();
3892 let old = chrono::Utc::now().timestamp() - GITHUB_5XX_WINDOW_SECS - 10;
3893 for _ in 0..GITHUB_5XX_THRESHOLD {
3894 ts.push(old);
3895 }
3896 }
3897
3898 record_github_5xx().await;
3900
3901 assert!(
3902 !is_github_circuit_open(),
3903 "old errors outside window should not trip circuit"
3904 );
3905 }
3906}