1use std::collections::HashMap;
5use std::time::{Duration, Instant, SystemTime};
6
7use cpa_common::gostr::GoStr;
8use cpa_core::credential::Credential;
9use cpa_core::exec::ExecError;
10#[cfg(test)]
11use cpa_core::exec::FailureScope;
12use serde::Deserialize;
13use serde_json::Value;
14
15use crate::runtime::{Outcome, Selection};
16
17#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
18pub enum Strategy {
19 #[default]
20 RoundRobin,
21 FillFirst,
22 WeightedRoundRobin,
23 SoonestReset,
28}
29
30#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
34pub struct Windows {
35 pub weekly_reset: Option<SystemTime>,
37 pub rolled_over: Option<Rollover>,
42 pub exhausted: bool,
45}
46
47#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
50pub enum WindowId {
51 Claude7d,
52 Claude5h,
53 CodexPrimary,
54 CodexSecondary,
55}
56
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub struct Rollover {
60 pub window: WindowId,
61 pub at: SystemTime,
63 pub gap: Duration,
67}
68
69const FALLBACK_WINDOW: Duration = Duration::from_secs(5 * 3600);
71
72#[derive(Debug, Clone, Copy)]
74struct Window {
75 id: WindowId,
76 used_up: bool,
77 reset: Option<SystemTime>,
78 stated_reset: Option<SystemTime>,
80 length: Option<Duration>,
82}
83
84impl Window {
85 fn exhausted_until(&self, observed_at: SystemTime) -> Option<SystemTime> {
88 if !self.used_up {
89 return None;
90 }
91 self.reset.or_else(|| {
92 self.length
93 .and_then(|l| observed_at.checked_add(l))
94 .or_else(|| observed_at.checked_add(FALLBACK_WINDOW))
95 })
96 }
97}
98
99fn window_length(minutes: f64, observed_at: SystemTime) -> Option<Duration> {
102 if !minutes.is_finite() || minutes <= 0.0 {
103 return None;
104 }
105 let length = Duration::try_from_secs_f64(minutes * 60.0).ok()?;
106 observed_at.checked_add(length).map(|_| length)
107}
108
109fn reset_time(raw: &str) -> Option<SystemTime> {
111 let raw = raw.trim();
112 if let Ok(seconds) = raw.parse::<f64>() {
113 return (seconds.is_finite() && seconds > 0.0)
114 .then(|| Duration::try_from_secs_f64(seconds).ok())
115 .flatten()
116 .and_then(|d| SystemTime::UNIX_EPOCH.checked_add(d));
117 }
118 chrono::DateTime::parse_from_rfc3339(raw).ok().map(SystemTime::from)
119}
120
121impl Windows {
122 pub fn observed(
125 provider: &str,
126 signals: &std::collections::BTreeMap<String, String>,
127 observed_at: SystemTime,
128 now: SystemTime,
129 ) -> Self {
130 let get = |name: &str| {
131 signals
132 .iter()
133 .find(|(k, _)| k.eq_ignore_ascii_case(name))
134 .map(|(_, v)| v.trim())
135 };
136 let number = |name: &str| get(name).and_then(|v| v.parse::<f64>().ok()).filter(|n| n.is_finite());
137 let codex = provider.trim().eq_ignore_ascii_case("codex");
138 let (long, short) = match provider.trim().to_ascii_lowercase().as_str() {
139 "claude" => {
140 let window = |id: WindowId, label: &str, minutes: f64| Window {
141 id,
142 used_up: get(&format!("anthropic-ratelimit-unified-{label}-status"))
143 .is_some_and(|s| s.eq_ignore_ascii_case("rejected"))
144 || number(&format!("anthropic-ratelimit-unified-{label}-utilization"))
145 .is_some_and(|u| u >= 1.0),
146 reset: get(&format!("anthropic-ratelimit-unified-{label}-reset")).and_then(reset_time),
147 stated_reset: get(&format!("anthropic-ratelimit-unified-{label}-reset")).and_then(reset_time),
148 length: window_length(minutes, observed_at),
149 };
150 (
151 Some(window(WindowId::Claude7d, "7d", 7.0 * 24.0 * 60.0)),
152 Some(window(WindowId::Claude5h, "5h", 300.0)),
153 )
154 }
155 "codex" => {
156 let window = |id: WindowId, label: &str| {
159 let p = format!("x-codex-{label}-");
160 let used = number(&format!("{p}used-percent"));
161 let absolute = get(&format!("{p}reset-at")).and_then(reset_time);
162 let after = number(&format!("{p}reset-after-seconds")).filter(|s| *s >= 0.0);
163 let relative = after.and_then(|a| observed_at.checked_add(Duration::try_from_secs_f64(a).ok()?));
164 let reset = absolute.or(relative);
165 let stated_reset = absolute.or(relative.filter(|_| after.is_some_and(|a| a > 0.0)));
168 let length = number(&format!("{p}window-minutes")).and_then(|m| window_length(m, observed_at));
169 (used.is_some() || reset.is_some() || length.is_some()).then_some(Window {
170 id,
171 used_up: used.is_some_and(|u| u >= 100.0),
172 reset,
173 stated_reset,
174 length,
175 })
176 };
177 let day = Duration::from_secs(24 * 3600);
178 match (
179 window(WindowId::CodexPrimary, "primary"),
180 window(WindowId::CodexSecondary, "secondary"),
181 ) {
182 (Some(p), Some(s)) if p.length.unwrap_or(Duration::ZERO) > s.length.unwrap_or(Duration::MAX) => {
185 (Some(p), Some(s))
186 }
187 (Some(p), Some(s)) => (Some(s), Some(p)),
188 (Some(p), None) if p.length.is_some_and(|l| l >= day) => (Some(p), None),
189 (None, Some(s)) if s.length.is_none_or(|l| l >= day) => (Some(s), None),
190 (only, None) | (None, only) => (None, only),
191 }
192 }
193 _ => (None, None),
194 };
195 let windows = [long, short];
196 let until = windows
197 .iter()
198 .flatten()
199 .filter_map(|w| w.exhausted_until(observed_at))
200 .max();
201 let flag_until = (codex && get("x-codex-limit-reached").is_some_and(|v| v.eq_ignore_ascii_case("true")))
205 .then(|| {
206 until
207 .or_else(|| windows.iter().flatten().filter_map(|w| w.reset).min())
208 .or_else(|| observed_at.checked_add(FALLBACK_WINDOW))
209 })
210 .flatten();
211 let long_reset = long.and_then(|w| w.reset);
212 Self {
213 weekly_reset: long_reset.filter(|r| *r > now),
214 rolled_over: long.and_then(|w| {
215 let at = w.stated_reset.filter(|r| *r <= now)?;
216 Some(Rollover {
217 window: w.id,
218 at,
219 gap: w.length.unwrap_or(FALLBACK_WINDOW) / 2,
220 })
221 }),
222 exhausted: until.max(flag_until).is_some_and(|u| u > now),
223 }
224 }
225
226 fn order(&self, probed: bool) -> (bool, u8, Option<SystemTime>) {
231 let class = match (self.weekly_reset, probed) {
232 (Some(_), _) => 1,
233 (None, false) => 0,
234 (None, true) => 2,
235 };
236 (self.exhausted, class, self.weekly_reset)
237 }
238}
239
240type Probes = HashMap<WindowId, SystemTime>;
243
244fn probed(record: Option<&Probes>, rolled_over: Option<Rollover>) -> bool {
248 match (record, rolled_over) {
249 (None, _) => false,
250 (Some(_), None) => true,
251 (Some(markers), Some(r)) => markers
252 .get(&r.window)
253 .is_some_and(|at| at.checked_add(r.gap).is_none_or(|limit| r.at <= limit)),
254 }
255}
256
257pub type Ranks<'a> = dyn Fn(&Credential) -> Windows + 'a;
259
260#[derive(Debug, Clone)]
261pub struct Policy {
262 pub strategy: Strategy,
263 pub request_retry: usize,
265 pub max_retry_credentials: usize,
267 pub max_retry_interval: Duration,
269 pub disable_cooling: bool,
271 pub transient_error_cooldown_seconds: i64,
273 pub session_affinity: bool,
275 pub session_affinity_ttl: Duration,
277 pub session_affinity_subagents: bool,
278 pub save_cooldown_status: bool,
281 pub max_trusted_cooldown: Duration,
284 pub force_model_prefix: bool,
286 pub compat_disable_cooling: Vec<(String, Option<bool>)>,
290 pub oauth_request_scoped_errors: HashMap<String, Vec<ErrorRule>>,
292}
293
294impl Default for Policy {
295 fn default() -> Self {
296 Self {
297 strategy: Strategy::RoundRobin,
298 request_retry: 0,
299 max_retry_credentials: 0,
300 max_retry_interval: Duration::ZERO,
301 disable_cooling: false,
302 transient_error_cooldown_seconds: 0,
303 session_affinity: false,
304 session_affinity_ttl: Duration::from_secs(3600),
305 session_affinity_subagents: true,
306 save_cooldown_status: false,
307 max_trusted_cooldown: DEFAULT_TRUSTED_COOLDOWN,
308 force_model_prefix: false,
309 compat_disable_cooling: Vec::new(),
310 oauth_request_scoped_errors: HashMap::new(),
311 }
312 }
313}
314
315#[derive(Default)]
317pub struct RawRouting<'a> {
318 pub strategy: &'a str,
319 pub force_model_prefix: bool,
320 pub session_affinity: bool,
321 pub session_affinity_ttl: &'a str,
322 pub session_affinity_subagents: Option<bool>,
323 pub request_retry: i64,
324 pub max_retry_credentials: i64,
325 pub max_retry_interval: i64,
326 pub disable_cooling: bool,
327 pub save_cooldown_status: bool,
328 pub transient_error_cooldown_seconds: i64,
329 pub max_trusted_cooldown: &'a str,
330}
331
332impl From<&cpa_core::config::RoutingConfig> for Policy {
335 fn from(r: &cpa_core::config::RoutingConfig) -> Self {
336 normalize(RawRouting {
337 strategy: &r.strategy,
338 force_model_prefix: r.force_model_prefix,
339 session_affinity: r.session_affinity,
340 session_affinity_ttl: &r.session_affinity_ttl,
341 session_affinity_subagents: r.session_affinity_subagents,
342 request_retry: r.retry.request_retry,
343 max_retry_credentials: r.retry.max_retry_credentials,
344 max_retry_interval: r.retry.max_retry_interval,
345 disable_cooling: r.cooldown.disable_cooling,
346 save_cooldown_status: r.cooldown.save_cooldown_status,
347 transient_error_cooldown_seconds: r.cooldown.transient_error_cooldown_seconds,
348 max_trusted_cooldown: &r.cooldown.max_trusted_cooldown,
349 })
350 }
351}
352
353pub fn normalize(raw: RawRouting<'_>) -> Policy {
354 let ttl = cpa_core::config::parse_duration(raw.session_affinity_ttl.trim())
355 .filter(|d| *d > 0)
356 .map(|n| Duration::from_nanos(n as u64).max(Duration::from_secs(1)))
357 .unwrap_or(Duration::from_secs(3600));
358 Policy {
359 strategy: match raw.strategy.trim().to_ascii_lowercase().as_str() {
360 "weighted-round-robin" | "weightedroundrobin" | "wrr" => Strategy::WeightedRoundRobin,
361 "fill-first" | "fillfirst" | "ff" => Strategy::FillFirst,
362 "" | "soonest-reset" | "soonestreset" | "reset-first" | "resetfirst" => Strategy::SoonestReset,
364 _ => Strategy::RoundRobin,
365 },
366 force_model_prefix: raw.force_model_prefix,
367 session_affinity: raw.session_affinity,
368 session_affinity_ttl: ttl,
369 session_affinity_subagents: !raw.session_affinity || raw.session_affinity_subagents.unwrap_or(true),
370 request_retry: raw.request_retry.max(0) as usize,
371 max_retry_credentials: raw.max_retry_credentials.max(0) as usize,
372 max_retry_interval: Duration::from_secs(raw.max_retry_interval.max(0) as u64),
373 disable_cooling: raw.disable_cooling,
374 save_cooldown_status: raw.save_cooldown_status,
375 transient_error_cooldown_seconds: raw.transient_error_cooldown_seconds,
376 max_trusted_cooldown: trusted_cooldown(raw.max_trusted_cooldown),
377 ..Policy::default()
378 }
379}
380
381const DEFAULT_TRUSTED_COOLDOWN: Duration = Duration::from_secs(3600);
382
383fn trusted_cooldown(raw: &str) -> Duration {
388 let raw = raw.trim();
389 if raw.is_empty() {
390 return DEFAULT_TRUSTED_COOLDOWN;
391 }
392 let nanos = match raw.parse::<i64>() {
393 Ok(seconds) => seconds.saturating_mul(1_000_000_000),
394 Err(_) => match cpa_core::config::parse_duration(raw) {
395 Some(nanos) => nanos,
396 None => {
397 static WARNED: std::sync::Mutex<String> = std::sync::Mutex::new(String::new());
398 let mut last = WARNED.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
399 if *last != raw {
400 raw.clone_into(&mut last);
401 tracing::warn!(
402 "routing.cooldown.max-trusted-cooldown {raw:?} is not a duration (\"90m\") or a \
403 number of seconds; using 1h"
404 );
405 }
406 return DEFAULT_TRUSTED_COOLDOWN;
407 }
408 },
409 };
410 match nanos {
411 ..=0 => Duration::ZERO,
412 n => Duration::from_nanos(n as u64).max(Duration::from_secs(10)),
413 }
414}
415
416#[derive(Debug, Clone, Deserialize)]
417pub struct ErrorRule {
418 #[serde(default)]
419 pub status: i64,
420 #[serde(default)]
421 pub r#match: Vec<String>,
422 #[serde(default, rename = "match-regexr")]
423 pub match_regex: Vec<String>,
424 #[serde(default)]
425 pub action: String,
426}
427
428#[derive(Debug, Clone, Copy)]
429pub struct ErrorAction {
430 pub matched: bool,
432 pub stop: bool,
433 pub cooldown: bool,
434 pub force_cooldown: bool,
435}
436
437fn metadata<'a>(c: &'a Credential, key: &str) -> Option<&'a Value> {
438 c.metadata
440 .get(key)
441 .or_else(|| c.metadata.get(key.replace('_', "-").as_str()))
442}
443
444pub(crate) fn priority(c: &Credential) -> i64 {
446 integer(c, "priority").unwrap_or(0)
447}
448
449fn integer(c: &Credential, key: &str) -> Option<i64> {
450 c.attributes
451 .get(key)
452 .filter(|_| key == "priority")
453 .and_then(|s| s.trim().parse().ok())
454 .or_else(|| {
455 [key.to_owned(), key.replace('_', "-")].iter().find_map(|key| {
456 let v = c.metadata.get(key)?;
457 v.as_i64()
458 .or_else(|| v.as_str()?.trim().parse().ok())
459 .or_else(|| v.as_f64().map(|n| n as i64))
460 })
461 })
462}
463
464fn boolean(c: &Credential, key: &str) -> Option<bool> {
465 let parse = |s: &str| match s.trim() {
466 "true" | "True" | "TRUE" | "1" | "t" | "T" => Some(true),
467 "false" | "False" | "FALSE" | "0" | "f" | "F" => Some(false),
468 _ => None,
469 };
470 [key.to_owned(), key.replace('_', "-")].iter().find_map(|key| {
471 let v = c.metadata.get(key)?;
472 v.as_bool()
473 .or_else(|| parse(v.as_str()?))
474 .or_else(|| v.as_f64().map(|n| n != 0.0))
475 })
476}
477
478fn oauth(c: &Credential) -> bool {
479 c.str("api_key").is_none() && !c.attributes.contains_key("api_key")
480}
481
482impl Policy {
483 pub fn retry_limit(&self, c: &Credential) -> usize {
484 integer(c, "request_retry")
485 .filter(|n| *n >= 0)
486 .map(|n| n as usize)
487 .unwrap_or(self.request_retry)
488 }
489
490 pub(crate) fn cooling_disabled(&self, c: &Credential) -> bool {
493 boolean(c, "disable_cooling")
494 .or_else(|| self.provider_cooling(c))
495 .unwrap_or(self.disable_cooling)
496 }
497
498 fn provider_cooling(&self, c: &Credential) -> Option<bool> {
502 let provider = c.provider.trim().go_lower();
503 let attr = |k: &str| c.attributes.get(k).map(|v| v.trim()).unwrap_or_default();
504 let (compat_name, mut provider_key) = (attr("compat_name"), attr("provider_key"));
505 if provider.is_empty()
506 || (provider_key.is_empty() && compat_name.is_empty() && provider != "openai-compatibility")
507 {
508 return None;
509 }
510 if provider_key.is_empty() {
511 provider_key = &provider;
512 }
513 let candidates = [compat_name, provider_key, provider.as_str()];
514 self.compat_disable_cooling
515 .iter()
516 .find(|(name, _)| candidates.iter().any(|c| !c.is_empty() && c.go_eq_fold(name)))
517 .and_then(|(_, disable)| *disable)
518 }
519
520 pub fn error_action(&self, c: &Credential, error: &ExecError) -> ErrorAction {
521 let file_rules: Option<Vec<ErrorRule>> =
522 metadata(c, "request_scoped_errors").and_then(|v| serde_json::from_value(v.clone()).ok());
523 let rules = file_rules.as_ref().filter(|r| !r.is_empty()).or_else(|| {
524 oauth(c)
525 .then(|| self.oauth_request_scoped_errors.get(&c.provider.trim().go_lower()))
526 .flatten()
527 });
528 if let Some(rules) = rules {
529 let body = crate::classify::rule_text(error);
530 let status = i64::from(crate::classify::go_status(error));
531 for rule in rules {
532 if rule.status <= 0 || rule.status != status {
533 continue;
534 }
535 let matches = rule.r#match.iter().any(|s| !s.is_empty() && body.contains(s))
536 || rule
537 .match_regex
538 .iter()
539 .any(|s| !s.is_empty() && regex::Regex::new(s).is_ok_and(|re| re.is_match(&body)));
540 if !matches {
541 continue;
542 }
543 let (stop, cooldown) = match rule.action.trim().to_ascii_lowercase().as_str() {
544 "stop" => (true, false),
545 "stop-and-cooldown" => (true, true),
546 "continue" => (false, false),
547 "continue-and-cooldown" => (false, true),
548 _ => continue,
549 };
550 return ErrorAction {
551 matched: true,
552 stop,
553 cooldown,
554 force_cooldown: cooldown,
555 };
556 }
557 }
558 let request = crate::classify::is_request_invalid(error);
559 ErrorAction {
560 matched: false,
561 stop: request,
562 cooldown: !crate::classify::skips_cooldown(error),
563 force_cooldown: false,
564 }
565 }
566}
567
568pub fn retry_status(status: u16) -> bool {
570 matches!(status, 403 | 408 | 429 | 500 | 502 | 503 | 504)
571}
572
573fn weight(c: &Credential) -> i64 {
574 let value = if let Some(s) = c.attributes.get("weight").filter(|s| !s.trim().is_empty()) {
575 s.trim().parse::<i64>().ok()
576 } else if let Some(v) = metadata(c, "weight") {
577 v.as_i64().or_else(|| v.as_str()?.trim().parse().ok()).or_else(|| {
578 let n = v.as_f64()?;
579 (n.fract() == 0.0 && n <= 1_000_000.0).then_some(n as i64)
580 })
581 } else {
582 Some(1)
583 };
584 value.filter(|n| *n <= 1_000_000).unwrap_or(0).max(0)
585}
586
587pub use cpa_core::registry::dynamic::canonical_model;
588
589#[derive(Default)]
590struct Rotation {
591 last: String,
592 weights: HashMap<String, i64>,
593 current: HashMap<String, i64>,
594}
595
596pub(crate) struct Cooldown {
597 pub deadline: Instant,
598 pub level: u32,
599 pub status: u16,
600 pub quota: bool,
602 pub error: String,
604 pub since: SystemTime,
606 pub credential: bool,
609 pub trust: Trust,
611 pub quota_kind: Option<QuotaKind>,
614 pub quota_under: Option<(Instant, QuotaKind)>,
619 pub recorded: u64,
623}
624
625#[derive(Debug, Clone, Copy, PartialEq, Eq)]
626pub(crate) struct QuotaKind {
627 pub status: u16,
628 pub reason: &'static str,
629}
630
631impl Cooldown {
632 fn quota_block(&self, now: Instant) -> Option<(Instant, QuotaKind)> {
634 let block = if self.quota {
635 self.quota_kind.map(|k| (self.deadline, k))
636 } else {
637 self.quota_under
638 };
639 block.filter(|(end, _)| *end > now || self.trust.stated.is_some_and(|r| r > now))
640 }
641}
642
643#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
647pub(crate) struct Trust {
648 pub window: Option<u32>,
651 pub generation: u32,
656 pub bounded: bool,
658 pub accepted: bool,
662 pub stated: Option<Instant>,
664 pub probe: Option<Instant>,
666}
667
668impl Trust {
669 fn accept(&mut self) {
671 self.bounded = false;
672 self.accepted = true;
673 self.probe = None;
674 }
675}
676
677const PROBE_HOLD: Duration = Duration::from_secs(30);
680const MAX_TRUST_WINDOW: u32 = 20;
682
683impl Cooldown {
684 fn until(&self, now: Instant) -> Option<Instant> {
686 let live = (self.deadline > now).then_some(self.deadline);
687 live.max(self.trust.probe.filter(|p| *p > now))
688 }
689
690 fn newer_than(&self, picked: u32) -> bool {
694 (self.trust.bounded || self.trust.accepted) && (self.trust.generation.wrapping_sub(picked) as i32) > 0
697 }
698}
699
700#[derive(Default)]
701pub(crate) struct Scheduler {
702 rotations: HashMap<(String, String), Rotation>,
703 cursors: HashMap<(String, String), usize>,
705 pub(crate) cooldowns: HashMap<(String, String), Cooldown>,
706 affinity: crate::affinity::Cache,
708 lcp: crate::lcp::Matcher,
711 probes: HashMap<String, Probes>,
714 affinity_off: bool,
717 generation: u32,
720 dispatched: u64,
723 rejected: HashMap<String, std::sync::Arc<Credential>>,
728}
729
730pub fn compat_cooling(cfg: &cpa_core::config::Config) -> Vec<(String, Option<bool>)> {
733 cfg.document
734 .get("api-keys")
735 .and_then(|k| k.get("openai-compatibility"))
736 .and_then(serde_yaml_ng::Value::as_sequence)
737 .into_iter()
738 .flatten()
739 .filter(|g| {
740 g.get("base-url")
741 .and_then(serde_yaml_ng::Value::as_str)
742 .is_some_and(|b| !b.trim().is_empty())
743 && g.get("disabled").and_then(serde_yaml_ng::Value::as_bool) != Some(true)
744 })
745 .map(|g| {
746 (
747 g.get("name")
748 .and_then(serde_yaml_ng::Value::as_str)
749 .unwrap_or_default()
750 .trim()
751 .to_owned(),
752 g.get("disable-cooling").and_then(serde_yaml_ng::Value::as_bool),
753 )
754 })
755 .collect()
756}
757
758struct SessionKeys {
760 primary: crate::affinity::Key,
761 fallback: Option<crate::affinity::Key>,
762 parent: String,
764}
765
766fn session_keys(selection: &Selection) -> Option<SessionKeys> {
767 let primary = selection.session.as_deref().filter(|s| !s.is_empty())?;
768 let scope = selection.provider_keys().join(",");
769 let model = canonical_model(&selection.model).to_owned();
770 let parent = selection.session_parent.clone().unwrap_or_default();
771 let fallback = (!parent.is_empty() && parent != primary).then(|| (scope.clone(), model.clone(), parent.clone()));
772 Some(SessionKeys {
773 primary: (scope, model, primary.to_owned()),
774 fallback,
775 parent,
776 })
777}
778
779fn quota_backoff(level: u32) -> (Duration, u32) {
781 let seconds = (1u64 << level.min(11)).min(1800);
782 if seconds >= 1800 {
783 (Duration::from_secs(1800), level)
784 } else {
785 (Duration::from_secs(seconds), level + 1)
786 }
787}
788
789fn bound(stated: Duration, cap: Duration, window: Option<u32>) -> Option<(Duration, u32)> {
796 if cap.is_zero() {
797 return None;
798 }
799 let step = window.map_or(0, |w| w.saturating_add(1)).min(MAX_TRUST_WINDOW);
800 let bound = cap.saturating_mul(1 << step);
801 (stated > bound).then_some((bound, step))
802}
803
804fn transient(policy: &Policy, hint: Option<Duration>, disabled: bool) -> Option<Duration> {
806 if disabled || policy.transient_error_cooldown_seconds < 0 {
807 return None;
808 }
809 if let Some(hint) = hint.filter(|d| !d.is_zero()) {
810 return Some(hint);
811 }
812 Some(Duration::from_secs(match policy.transient_error_cooldown_seconds {
813 0 => 60,
814 s => s as u64,
815 }))
816}
817
818impl Scheduler {
819 pub fn configure(&mut self, previous: &Policy, next: &Policy) {
820 self.affinity_off = !next.session_affinity;
821 if previous.strategy != next.strategy
822 || previous.session_affinity != next.session_affinity
823 || previous.session_affinity_ttl != next.session_affinity_ttl
824 || previous.session_affinity_subagents != next.session_affinity_subagents
825 {
826 self.rotations.clear();
827 self.cursors.clear();
828 self.affinity.clear();
829 self.lcp.reset(crate::lcp::Limits {
832 ttl: next.session_affinity_ttl,
833 ..Default::default()
834 });
835 self.probes.clear();
836 }
837 }
838
839 fn affinity(&self, policy: &Policy) -> bool {
841 policy.session_affinity && !self.affinity_off
842 }
843
844 pub fn quota_cooling(&self, c: &Credential, model: &str, now: Instant) -> bool {
845 let model = canonical_model(model);
846 [model, ""].into_iter().any(|m| {
847 self.cooldowns
848 .get(&(c.id.clone(), m.to_owned()))
849 .is_some_and(|s| s.quota && s.until(now).is_some())
850 })
851 }
852
853 pub fn wait(&self, c: &Credential, model: &str, now: Instant) -> Option<Duration> {
854 let model = canonical_model(model);
855 [model, ""]
856 .into_iter()
857 .filter_map(|model| {
858 self.cooldowns
859 .get(&(c.id.clone(), model.to_owned()))
860 .and_then(|s| s.until(now)?.checked_duration_since(now))
861 .filter(|d| !d.is_zero())
862 })
863 .max()
864 }
865
866 pub fn cooling(&self, c: &Credential, model: &str, now: Instant) -> bool {
868 let model = canonical_model(model);
869 [model, ""].into_iter().any(|m| {
870 self.cooldowns
871 .get(&(c.id.clone(), m.to_owned()))
872 .is_some_and(|s| s.deadline > now)
873 })
874 }
875
876 pub fn last_error(&self, c: &Credential, model: &str) -> Option<&Cooldown> {
878 let model = canonical_model(model);
879 self.cooldowns
880 .get(&(c.id.clone(), model.to_owned()))
881 .or_else(|| self.cooldowns.get(&(c.id.clone(), String::new())))
882 }
883
884 pub fn retry_eligible(&self, c: &Credential, model: &str, now: Instant) -> bool {
885 let model = canonical_model(model);
886 [model, ""].into_iter().all(|model| {
890 self.cooldowns.get(&(c.id.clone(), model.to_owned())).is_none_or(|s| {
891 let probing = s.trust.probe.is_some_and(|p| p > now);
892 !probing && (s.deadline <= now || (retry_status(s.status) && !s.trust.bounded))
893 })
894 })
895 }
896
897 #[cfg(test)]
899 pub fn accept_probe(&mut self, c: &Credential, model: &str, now: Instant) {
900 self.accept_probe_picked(c, model, self.generation, now);
901 }
902
903 pub fn accept_probe_picked(&mut self, c: &Credential, model: &str, picked: u32, now: Instant) {
912 let model = canonical_model(model);
913 let mut credential_window = None;
914 for m in [model, ""] {
915 if let Some(s) = self.cooldowns.get_mut(&(c.id.clone(), m.to_owned()))
916 && s.trust.bounded
917 && s.deadline <= now
918 && !s.newer_than(picked)
919 {
920 s.trust.accept();
921 if m.is_empty() {
922 credential_window = Some(s.deadline);
923 }
924 }
925 }
926 if let Some(ended) = credential_window {
927 for ((id, m), s) in &mut self.cooldowns {
928 if *id == c.id && !m.is_empty() && s.trust.bounded && s.deadline == ended && !s.newer_than(picked) {
929 s.trust.accept();
930 }
931 }
932 }
933 }
934
935 pub fn outside_selection_generation(&self) -> u32 {
942 self.generation.wrapping_sub(1 << 30)
943 }
944
945 pub fn reserve_probe(&mut self, c: &Credential, model: &str, now: Instant) -> u32 {
951 let model = canonical_model(model);
952 let next = self.generation.wrapping_add(1);
953 for m in [model, ""] {
954 if let Some(s) = self.cooldowns.get_mut(&(c.id.clone(), m.to_owned()))
955 && s.trust.bounded
956 && s.deadline <= now
957 {
958 s.trust.probe = now.checked_add(PROBE_HOLD);
959 s.trust.generation = next;
960 self.generation = next;
961 }
962 }
963 self.generation
964 }
965
966 #[cfg(test)]
973 pub fn pick<'a>(
974 &mut self,
975 candidates: &[(&'a Credential, &str)],
976 selection: &Selection,
977 policy: &Policy,
978 now: Instant,
979 ) -> Option<&'a Credential> {
980 self.pick_ranked(candidates, selection, policy, &|_| Windows::default(), now)
981 }
982
983 #[cfg(test)]
985 pub fn pick_ranked<'a>(
986 &mut self,
987 candidates: &[(&'a Credential, &str)],
988 selection: &Selection,
989 policy: &Policy,
990 ranks: &Ranks<'_>,
991 now: Instant,
992 ) -> Option<&'a Credential> {
993 self.pick_session(candidates, selection, policy, ranks, now)
994 .map(|(c, _)| c)
995 }
996
997 pub fn pick_session<'a>(
1000 &mut self,
1001 candidates: &[(&'a Credential, &str)],
1002 selection: &Selection,
1003 policy: &Policy,
1004 ranks: &Ranks<'_>,
1005 now: Instant,
1006 ) -> Option<(&'a Credential, Option<crate::lcp::Match>)> {
1007 if candidates.is_empty() {
1008 return None;
1009 }
1010 if let Some(request) = selection.lcp.as_deref().filter(|_| self.affinity(policy)) {
1011 let namespace = request.namespace(&selection.model);
1012 if let Some(found) = self.lcp.find(&namespace, request.prepared(), now)
1014 && let Some((c, _)) = candidates.iter().find(|(c, _)| c.id == found.auth)
1015 {
1016 return Some((c, Some(found)));
1017 }
1018 let picked = self.pick_unbound(candidates, selection, policy.strategy, ranks)?;
1019 let bound = self.lcp.bind(&namespace, request.prepared(), &picked.id, now);
1020 return Some((picked, bound));
1021 }
1022 self.pick_legacy(candidates, selection, policy, ranks, now)
1023 .map(|c| (c, None))
1024 }
1025
1026 fn pick_legacy<'a>(
1028 &mut self,
1029 candidates: &[(&'a Credential, &str)],
1030 selection: &Selection,
1031 policy: &Policy,
1032 ranks: &Ranks<'_>,
1033 now: Instant,
1034 ) -> Option<&'a Credential> {
1035 let ttl = policy.session_affinity_ttl;
1036 let Some(keys) = self.affinity(policy).then(|| session_keys(selection)).flatten() else {
1037 return self.pick_unbound(candidates, selection, policy.strategy, ranks);
1038 };
1039 self.affinity.sweep(now, ttl);
1040 let fork = selection.session_fork;
1041 let subagent = !fork && cpa_common::session::is_subagent_session(&keys.primary.2, &keys.parent);
1042 let bind_keys = match &keys.fallback {
1043 Some(fallback) if !subagent && !fork => vec![keys.primary.clone(), fallback.clone()],
1044 _ => vec![keys.primary.clone()],
1045 };
1046 let find = |id: &str| candidates.iter().find(|(c, _)| c.id == id).map(|(c, _)| *c);
1047 let reuse = match self.affinity.get_and_refresh(&keys.primary, now, ttl) {
1048 Some(id) => find(&id),
1050 None => keys
1051 .fallback
1052 .as_ref()
1053 .and_then(|fallback| self.affinity.get(fallback, now))
1054 .filter(|_| !subagent || policy.session_affinity_subagents)
1055 .and_then(|id| find(&id)),
1056 };
1057 let picked = match reuse {
1058 Some(c) => c,
1059 None => self.pick_unbound(candidates, selection, policy.strategy, ranks)?,
1060 };
1061 self.affinity.bind(&picked.id, &bind_keys, now, ttl);
1062 Some(picked)
1063 }
1064
1065 pub fn session_result(
1073 &mut self,
1074 c: &Credential,
1075 selection: &Selection,
1076 outcome: &Outcome,
1077 policy: &Policy,
1078 lcp: Option<&crate::lcp::Match>,
1079 now: Instant,
1080 ) {
1081 if !self.affinity(policy) {
1082 return;
1083 }
1084 let success = match outcome {
1085 Outcome::Success => true,
1086 Outcome::Failure(error) if policy.error_action(c, error).cooldown => false,
1087 _ => return,
1088 };
1089 if let Some(request) = selection.lcp.as_deref() {
1090 let namespace = request.namespace(&selection.model);
1091 if success {
1092 self.lcp.touch(&namespace, request.prepared(), &c.id, now);
1093 } else {
1094 let generation = lcp.map_or(0, |m| m.access);
1095 self.lcp
1096 .remove(&namespace, &request.prepared().fingerprints, &c.id, generation, now);
1097 }
1098 if lcp.is_some() {
1099 return;
1100 }
1101 }
1102 let Some(keys) = session_keys(selection) else {
1103 return;
1104 };
1105 let mut targets = vec![keys.primary.clone()];
1106 if let Some(fallback) = keys.fallback
1107 && !cpa_common::session::is_subagent_session(&keys.primary.2, &keys.parent)
1108 {
1109 targets.push(fallback);
1110 }
1111 for key in &targets {
1112 if success {
1113 self.affinity.touch(key, &c.id, now, policy.session_affinity_ttl);
1114 } else {
1115 self.affinity.compare_and_delete(key, &c.id);
1116 }
1117 }
1118 }
1119
1120 pub fn pick_builtin<'a>(
1127 &mut self,
1128 candidates: &[(&'a Credential, &str)],
1129 selection: &Selection,
1130 strategy: Strategy,
1131 ranks: &Ranks<'_>,
1132 ) -> Option<&'a Credential> {
1133 self.pick_unbound(candidates, selection, strategy, ranks)
1134 }
1135
1136 fn pick_unbound<'a>(
1137 &mut self,
1138 candidates: &[(&'a Credential, &str)],
1139 selection: &Selection,
1140 strategy: Strategy,
1141 ranks: &Ranks<'_>,
1142 ) -> Option<&'a Credential> {
1143 let model = canonical_model(&selection.model).to_owned();
1144 let provider_keys = selection.provider_keys();
1145 let providers_key = provider_keys.join(",");
1146 let tier = candidates
1147 .iter()
1148 .map(|(c, _)| integer(c, "priority").unwrap_or(0))
1149 .max()?;
1150 let mut ready: Vec<(&Credential, &str)> = candidates
1151 .iter()
1152 .copied()
1153 .filter(|(c, _)| integer(c, "priority").unwrap_or(0) == tier)
1154 .collect();
1155 ready.sort_by(|a, b| a.0.id.cmp(&b.0.id));
1156 let mut groups: Vec<(&str, Vec<&Credential>)> = Vec::new();
1158 for provider in provider_keys
1159 .iter()
1160 .map(String::as_str)
1161 .chain(ready.iter().map(|(_, p)| *p))
1162 {
1163 if groups.iter().any(|(p, _)| *p == provider) {
1164 continue;
1165 }
1166 let members: Vec<&Credential> = ready.iter().filter(|(_, p)| *p == provider).map(|(c, _)| *c).collect();
1167 if !members.is_empty() {
1168 groups.push((provider, members));
1169 }
1170 }
1171 if self.rotations.len() >= 4096 {
1172 self.rotations.clear();
1173 self.cursors.clear();
1174 }
1175 let picked = if groups.len() == 1 {
1176 let (provider, members) = &groups[0];
1177 self.pick_within(provider, &model, members, strategy, ranks)
1178 } else {
1179 match strategy {
1180 Strategy::FillFirst => groups[0].1[0],
1181 Strategy::SoonestReset => {
1183 let (provider, members) = &groups[0];
1184 self.pick_within(provider, &model, members, Strategy::SoonestReset, ranks)
1185 }
1186 Strategy::WeightedRoundRobin => {
1187 let all: Vec<&Credential> = ready.iter().map(|(c, _)| *c).collect();
1188 self.pick_within(&providers_key, &model, &all, Strategy::WeightedRoundRobin, ranks)
1189 }
1190 Strategy::RoundRobin => {
1191 let total: usize = groups.iter().map(|(_, m)| m.len()).sum();
1194 let cursor = self.cursors.entry((providers_key.clone(), model.clone())).or_default();
1195 let slot = *cursor % total;
1196 *cursor = slot + 1;
1197 let mut start = 0;
1198 let mut index = 0;
1199 for (i, (_, members)) in groups.iter().enumerate() {
1200 if slot < start + members.len() {
1201 index = i;
1202 break;
1203 }
1204 start += members.len();
1205 }
1206 let (provider, members) = &groups[index];
1207 self.pick_within(provider, &model, members, Strategy::RoundRobin, ranks)
1208 }
1209 }
1210 };
1211 Some(picked)
1212 }
1213
1214 fn pick_within<'a>(
1215 &mut self,
1216 scope: &str,
1217 model: &str,
1218 members: &[&'a Credential],
1219 strategy: Strategy,
1220 ranks: &Ranks<'_>,
1221 ) -> &'a Credential {
1222 let state = self.rotations.entry((scope.to_owned(), model.to_owned())).or_default();
1223 match strategy {
1224 Strategy::FillFirst => members[0],
1225 Strategy::SoonestReset => {
1226 let probes = &mut self.probes;
1227 let ranked: Vec<_> = members
1228 .iter()
1229 .map(|c| {
1230 let windows = ranks(c);
1231 let order = windows.order(probed(probes.get(&c.id), windows.rolled_over));
1232 (order, windows.rolled_over, *c)
1233 })
1234 .collect();
1235 let best = ranked.iter().map(|(order, ..)| *order).min().expect("members");
1236 let tied: Vec<_> = ranked.iter().filter(|(o, ..)| *o == best).collect();
1238 let (order, rolled_over, picked) = *tied
1239 .iter()
1240 .find(|(.., c)| c.id > state.last)
1241 .copied()
1242 .unwrap_or(tied[0]);
1243 if order.1 == 0 {
1244 let markers = probes.entry(picked.id.clone()).or_default();
1247 if let Some(r) = rolled_over {
1248 markers.insert(r.window, r.at);
1249 }
1250 }
1251 state.last.clone_from(&picked.id);
1252 picked
1253 }
1254 Strategy::RoundRobin => {
1255 let picked = members
1256 .iter()
1257 .find(|c| c.id > state.last)
1258 .copied()
1259 .unwrap_or(members[0]);
1260 state.last.clone_from(&picked.id);
1261 picked
1262 }
1263 Strategy::WeightedRoundRobin => {
1264 if members
1265 .iter()
1266 .any(|c| state.weights.get(&c.id).is_some_and(|w| *w != weight(c)))
1267 {
1268 state.current.clear();
1269 }
1270 if state.weights.len() > 1024 || state.current.len() > 1024 {
1271 state.weights.retain(|id, _| members.iter().any(|c| c.id == *id));
1272 state.current.retain(|id, _| members.iter().any(|c| c.id == *id));
1273 }
1274 let mut total = 0i64;
1275 let mut best = i64::MIN;
1276 let mut picked = members[0];
1277 for c in members {
1278 let w = weight(c);
1279 state.weights.insert(c.id.clone(), w);
1280 let current = state.current.entry(c.id.clone()).or_default();
1281 *current = current.saturating_add(w);
1282 total = total.saturating_add(w);
1283 if *current > best {
1284 best = *current;
1285 picked = c;
1286 }
1287 }
1288 let current = state.current.get_mut(&picked.id).unwrap();
1289 *current = current.saturating_sub(total);
1290 picked
1291 }
1292 }
1293 }
1294
1295 pub fn admits(&self, c: &Credential, policy: &Policy) -> bool {
1296 policy.strategy != Strategy::WeightedRoundRobin || weight(c) > 0
1297 }
1298
1299 pub(crate) fn reject_tokens(&mut self, c: std::sync::Arc<Credential>) {
1301 self.rejected.insert(c.id.clone(), c);
1302 }
1303
1304 pub(crate) fn tokens_rejected(&self, c: &Credential) -> bool {
1307 self.rejected
1308 .get(&c.id)
1309 .is_some_and(|r| !crate::runtime::credentials_changed(r, c))
1310 }
1311
1312 #[cfg(test)]
1314 pub fn record(&mut self, c: &Credential, model: &str, outcome: &Outcome, policy: &Policy, now: Instant) {
1315 self.record_picked(c, model, outcome, policy, self.generation, now);
1316 }
1317
1318 #[cfg(test)]
1320 pub fn record_picked(
1321 &mut self,
1322 c: &Credential,
1323 model: &str,
1324 outcome: &Outcome,
1325 policy: &Policy,
1326 picked: u32,
1327 now: Instant,
1328 ) {
1329 self.record_dispatched(c, model, outcome, policy, picked, 0, now);
1330 }
1331
1332 pub fn dispatch(&mut self) -> u64 {
1334 self.dispatched += 1;
1335 self.dispatched
1336 }
1337
1338 #[allow(clippy::too_many_arguments)]
1344 pub fn record_dispatched(
1345 &mut self,
1346 c: &Credential,
1347 model: &str,
1348 outcome: &Outcome,
1349 policy: &Policy,
1350 picked: u32,
1351 seq: u64,
1352 now: Instant,
1353 ) {
1354 let model = canonical_model(model);
1355 self.apply(c, model, outcome, policy, picked, seq, now);
1356 if let Outcome::Failure(error) = outcome {
1361 let action = policy.error_action(c, error);
1362 if action.cooldown || model.is_empty() {
1363 let disabled = policy.cooling_disabled(c) && !action.force_cooldown;
1364 let all = crate::classify::credential_scoped(error) && !disabled;
1365 for ((id, m), s) in &mut self.cooldowns {
1366 if *id == c.id && (all || m == model) {
1367 s.recorded = self.dispatched;
1368 }
1369 }
1370 }
1371 }
1372 }
1373
1374 #[allow(clippy::too_many_arguments)]
1375 fn apply(
1376 &mut self,
1377 c: &Credential,
1378 model: &str,
1379 outcome: &Outcome,
1380 policy: &Policy,
1381 picked: u32,
1382 seq: u64,
1383 now: Instant,
1384 ) {
1385 let key = (c.id.clone(), model.to_owned());
1386 for m in [model, ""] {
1389 if let Some(s) = self.cooldowns.get_mut(&(c.id.clone(), m.to_owned()))
1390 && !s.newer_than(picked)
1391 {
1392 s.trust.probe = None;
1393 }
1394 }
1395 let error = match outcome {
1396 Outcome::Success => {
1397 let global = (c.id.clone(), String::new());
1399 if self.cooldowns.get(&global).is_some_and(|s| s.quota && s.deadline > now) {
1400 return;
1401 }
1402 if seq > 0
1406 && self
1407 .cooldowns
1408 .get(&key)
1409 .is_some_and(|s| s.deadline > now && s.recorded >= seq)
1410 {
1411 return;
1412 }
1413 if let Some(ended) = self
1416 .cooldowns
1417 .get(&global)
1418 .filter(|s| s.deadline <= now && !s.newer_than(picked))
1419 .map(|s| s.deadline)
1420 {
1421 self.cooldowns.remove(&global);
1422 for ((id, _), s) in &mut self.cooldowns {
1427 if *id == c.id
1428 && s.deadline <= now
1429 && (!s.trust.bounded || s.deadline == ended)
1430 && !s.newer_than(picked)
1431 {
1432 s.trust = Trust::default();
1433 }
1434 }
1435 }
1436 if !self.cooldowns.get(&key).is_some_and(|s| s.newer_than(picked)) {
1437 self.cooldowns.remove(&key);
1438 }
1439 return;
1440 }
1441 Outcome::Cancelled | Outcome::Neutral(_) => return,
1442 Outcome::Failure(error) => error,
1443 };
1444 let action = policy.error_action(c, error);
1445 if !action.cooldown {
1446 return;
1447 }
1448 let credential_quota = crate::classify::credential_scoped(error);
1449 let key = if credential_quota {
1450 (c.id.clone(), String::new())
1451 } else {
1452 key
1453 };
1454 let disabled = policy.cooling_disabled(c) && !action.force_cooldown;
1456 let status = crate::classify::go_status(error);
1457 let text = crate::classify::error_text(error);
1458 let prev = self.cooldowns.get(&key);
1459 if prev.is_some_and(|s| s.deadline <= now && s.newer_than(picked)) {
1463 return;
1464 }
1465 let prev_deadline = prev.map(|s| s.deadline);
1466 let prev_live = prev.filter(|s| s.deadline > now).map(|s| s.deadline);
1467 let prev_trust = prev.map(|s| s.trust).unwrap_or_default();
1468 let quota_under = prev.and_then(|p| p.quota_block(now));
1470 let mut level = prev.filter(|s| s.quota).map(|s| s.level).unwrap_or(0);
1471 let mut trust = Trust {
1473 window: prev_trust.window,
1474 ..Trust::default()
1475 };
1476 let hint = error.retry_after.filter(|d| !d.is_zero());
1477 let mut quota = false;
1478 let mut cloudflare = false;
1479 let duration = if crate::classify::is_model_support(status, &text) {
1480 (!disabled).then(|| hint.unwrap_or(Duration::from_secs(43200)))
1481 } else if crate::classify::is_cloudflare(status, &text) {
1482 quota = true;
1483 cloudflare = true;
1484 (!disabled).then(|| {
1485 let (d, next) = quota_backoff(level);
1486 level = next;
1487 d.max(Duration::from_secs(10))
1488 })
1489 } else if crate::classify::is_invalid_grant(status, &text) {
1490 (!disabled).then_some(Duration::from_secs(1800))
1491 } else {
1492 match status {
1493 401..=403 => (!disabled).then_some(Duration::from_secs(1800)),
1494 404 => (!disabled).then(|| hint.unwrap_or(Duration::from_secs(43200))),
1495 429 => {
1496 quota = true;
1497 if credential_quota && prev.is_none_or(|s| !s.quota) {
1498 level = 0;
1499 }
1500 if disabled {
1501 None
1502 } else if let Some(hint) = error.retry_after {
1503 let stated = hint.max(Duration::from_secs(10));
1506 let cap = policy.max_trusted_cooldown;
1507 if !cap.is_zero()
1508 && let Some(window) = prev.filter(|s| s.trust.bounded && s.deadline > now)
1509 {
1510 let reset = now.checked_add(stated);
1514 let old = window.deadline;
1515 let deadline = reset.map_or(old, |r| old.min(r));
1516 let mut window_trust = window.trust;
1517 if let Some(slot) = self.cooldowns.get_mut(&key) {
1518 slot.deadline = deadline;
1519 slot.error = text;
1520 slot.trust.stated = reset.or(slot.trust.stated);
1521 window_trust = slot.trust;
1522 }
1523 if credential_quota {
1524 for ((id, m), state) in &mut self.cooldowns {
1528 if *id == c.id && !m.is_empty() && state.trust.bounded && state.deadline == old {
1529 state.deadline = deadline;
1530 state.trust.stated = window_trust.stated;
1531 state.trust.window = state.trust.window.max(window_trust.window);
1532 }
1533 }
1534 self.extend_siblings(c, deadline, None, now);
1535 }
1536 return;
1537 }
1538 Some(match bound(stated, cap, prev_trust.window) {
1539 Some((bound, window)) => {
1540 self.generation = self.generation.wrapping_add(1);
1541 trust = Trust {
1542 window: Some(window),
1543 generation: self.generation,
1544 bounded: true,
1545 accepted: false,
1546 stated: now.checked_add(stated),
1547 probe: None,
1548 };
1549 bound
1550 }
1551 None => stated,
1552 })
1553 } else if let Some(prev) = prev.filter(|s| s.quota && s.deadline > now) {
1554 let deadline = prev.deadline;
1557 if let Some(slot) = self.cooldowns.get_mut(&key) {
1558 slot.error = text;
1559 }
1560 if credential_quota {
1561 self.extend_siblings(c, deadline, None, now);
1562 }
1563 return;
1564 } else {
1565 let (d, next) = quota_backoff(level);
1566 level = next;
1567 Some(d)
1568 }
1569 }
1570 408 | 500 | 502..=504 | 520..=526 => transient(policy, hint, disabled),
1571 _ => transient(policy, None, disabled),
1572 }
1573 };
1574 let duration = match duration {
1577 None if action.force_cooldown => Some(Duration::from_secs(60)),
1578 d => d,
1579 };
1580 let Some(next) = duration.and_then(|d| now.checked_add(d)) else {
1581 self.cooldowns.remove(&key);
1582 return;
1583 };
1584 let deadline = prev_live.map_or(next, |p| p.max(next));
1585 if prev_live == Some(deadline)
1590 && let Some(slot) = self.cooldowns.get_mut(&key).filter(|s| s.quota && s.trust.bounded)
1591 {
1592 slot.error = text;
1593 slot.level = level;
1595 return;
1596 }
1597 let provenance = |deadline: Instant| {
1600 if deadline == next {
1601 trust
1602 } else if prev_live == Some(deadline) {
1603 Trust {
1604 probe: None,
1605 ..prev_trust
1606 }
1607 } else {
1608 Trust {
1609 window: trust.window,
1610 ..Trust::default()
1611 }
1612 }
1613 };
1614 let kind = |reason| {
1615 quota.then_some(QuotaKind {
1616 status: error.status,
1617 reason,
1618 })
1619 };
1620 let own_reason = if cloudflare { "cloudflare challenge" } else { "quota" };
1621 let state = |deadline, reason| Cooldown {
1622 deadline,
1623 level,
1624 status: error.status,
1625 quota,
1626 error: text.clone(),
1627 since: SystemTime::now(),
1628 credential: false,
1629 trust: provenance(deadline),
1630 quota_kind: kind(reason),
1631 quota_under: quota_under.filter(|_| !quota),
1632 recorded: 0,
1633 };
1634 let reason = if credential_quota {
1635 "credential_quota"
1636 } else {
1637 own_reason
1638 };
1639 self.cooldowns.insert(key, state(deadline, reason));
1640 if credential_quota {
1641 self.extend_siblings(c, deadline, prev_deadline, now);
1642 if !model.is_empty() {
1644 let own = (c.id.clone(), model.to_owned());
1645 let own_deadline = self
1649 .cooldowns
1650 .get(&own)
1651 .filter(|s| s.deadline > now)
1652 .map_or(deadline, |s| s.deadline);
1653 let own_trust = self.cooldowns.get(&own).map(|s| s.trust).unwrap_or_default();
1654 let own_window = own_trust.window;
1655 let mut own_state = state(own_deadline, own_reason);
1656 if own_deadline != deadline {
1657 own_state.trust = Trust {
1660 probe: None,
1661 ..own_trust
1662 };
1663 } else {
1664 own_state.trust = provenance(deadline);
1665 }
1666 own_state.trust.window = own_state.trust.window.max(own_window);
1667 self.cooldowns.insert(own, own_state);
1668 }
1669 }
1670 }
1671
1672 fn extend_siblings(&mut self, c: &Credential, deadline: Instant, renews: Option<Instant>, now: Instant) {
1682 let (level, trust, status) = self
1683 .cooldowns
1684 .get(&(c.id.clone(), String::new()))
1685 .map_or((0, Trust::default(), 429), |s| (s.level, s.trust, s.status));
1686 for ((id, model), state) in &mut self.cooldowns {
1687 if *id != c.id || model.is_empty() {
1688 continue;
1689 }
1690 let copy = state.trust.bounded && (state.deadline == deadline || Some(state.deadline) == renews);
1691 if state.deadline > now || copy {
1692 let window = state.trust.window.max(trust.window);
1693 if copy || !(state.trust.bounded || trust.bounded) {
1694 if deadline >= state.deadline {
1695 state.trust = Trust { probe: None, ..trust };
1696 }
1697 state.deadline = state.deadline.max(deadline);
1698 }
1699 state.trust.window = window;
1701 state.quota = true;
1702 state.credential = true;
1703 state.level = level;
1704 state.quota_kind = Some(QuotaKind {
1705 status,
1706 reason: "credential_quota",
1707 });
1708 }
1709 }
1710 }
1711
1712 pub fn reset(&mut self, id: &str) -> Vec<String> {
1715 let mut models: Vec<String> = self
1716 .cooldowns
1717 .keys()
1718 .filter(|(cid, _)| cid == id)
1719 .map(|(_, m)| m.clone())
1720 .collect();
1721 self.cooldowns.retain(|(cid, _), _| cid != id);
1722 self.affinity.invalidate(id);
1723 self.lcp.invalidate(id);
1724 self.probes.remove(id);
1725 models.sort();
1726 models
1727 }
1728
1729 pub fn records(&self, c: &Credential, now: Instant, wall: SystemTime) -> Vec<crate::cooldown_store::Record> {
1732 use crate::cooldown_store::{LastError, Quota, Record};
1733 let mut out: Vec<Record> = self
1734 .cooldowns
1735 .iter()
1736 .filter(|((id, _), state)| *id == c.id && state.deadline > now)
1737 .map(|((_, model), state)| {
1738 let at = wall + (state.deadline - now);
1739 let recover = state
1741 .trust
1742 .stated
1743 .filter(|r| state.trust.bounded && *r > state.deadline)
1744 .map_or(at, |r| wall + (r - now));
1745 let reason = match (model.is_empty() || state.credential, state.quota, state.status) {
1746 (true, true, _) => "credential_quota".to_owned(),
1747 (false, true, 429) => "quota".to_owned(),
1748 (false, true, _) => "cloudflare challenge".to_owned(),
1749 (_, false, _) => state.error.clone(),
1750 };
1751 Record {
1752 provider: c.provider.trim().to_owned(),
1753 auth_id: c.id.clone(),
1754 model: model.clone(),
1755 status: "cooling".into(),
1756 next_retry_after: Some(at),
1757 quota: if state.quota {
1758 Quota {
1759 exceeded: true,
1760 reason: reason.clone(),
1761 next_recover_at: Some(recover),
1762 backoff_level: state.level,
1763 observed_at: None,
1764 trust_windows: Some(state.trust.window.map_or(0, |w| w.saturating_add(1))),
1765 }
1766 } else {
1767 Quota::default()
1768 },
1769 reason,
1770 last_error: Some(LastError {
1771 message: state.error.clone(),
1772 http_status: u32::from(state.status),
1773 ..LastError::default()
1774 }),
1775 updated_at: Some(state.since),
1776 auth_file: match &c.source {
1777 cpa_core::credential::Source::File(path) => Some(path.clone()),
1778 _ => None,
1779 },
1780 }
1781 })
1782 .collect();
1783 out.sort_by(|a, b| a.model.cmp(&b.model));
1784 out
1785 }
1786
1787 pub fn restore(
1791 &mut self,
1792 c: &Credential,
1793 record: &crate::cooldown_store::Record,
1794 has_model_records: bool,
1795 cap: Duration,
1796 now: Instant,
1797 wall: SystemTime,
1798 ) -> bool {
1799 let Some(retry) = record.next_retry_after.filter(|at| *at > wall) else {
1803 return false;
1804 };
1805 let recover = record.quota.next_recover_at.filter(|_| record.quota.exceeded);
1806 let saved = record.quota.trust_windows.map(|n| n.min(MAX_TRUST_WINDOW + 1));
1807 let model = record.model.trim();
1808 if model.is_empty() && has_model_records && record.quota.reason != "credential_quota" {
1809 return false;
1810 }
1811 let since = |at: SystemTime| at.duration_since(wall).unwrap_or_default();
1812 let (remaining, trust) = match (record.quota.exceeded && !cap.is_zero(), saved) {
1813 (true, Some(spent)) => {
1818 let stated = recover.filter(|r| *r > retry).map(|r| now + since(r));
1819 let trust = Trust {
1820 window: spent.checked_sub(1),
1821 generation: 0,
1822 bounded: stated.is_some(),
1823 accepted: false,
1824 stated,
1825 probe: None,
1826 };
1827 (since(retry), trust)
1828 }
1829 (true, None) => {
1832 let at = recover.map_or(retry, |r| retry.max(r));
1833 match bound(since(at), cap, None) {
1834 Some((bound, window)) => (
1835 bound,
1836 Trust {
1837 window: Some(window),
1838 generation: 0,
1839 bounded: true,
1840 accepted: false,
1841 stated: Some(now + since(at)),
1842 probe: None,
1843 },
1844 ),
1845 None => (since(at), Trust::default()),
1846 }
1847 }
1848 (false, _) => (since(recover.map_or(retry, |r| retry.max(r))), Trust::default()),
1849 };
1850 if remaining.is_zero() {
1851 return false;
1852 }
1853 let deadline = now + remaining;
1854 let error = record.last_error.clone().unwrap_or_default();
1855 let status = match error.http_status {
1856 0 if record.quota.exceeded => 429,
1857 s => u16::try_from(s).unwrap_or(0),
1858 };
1859 let key = (c.id.clone(), canonical_model(model).to_owned());
1860 let recorded = self.cooldowns.get(&key).map_or(0, |prev| prev.recorded);
1861 let prev = self.cooldowns.get(&key).filter(|prev| prev.deadline > now);
1862 let prev_wins = prev.is_some_and(|p| p.deadline > deadline);
1863 let deadline = prev.map_or(deadline, |prev| prev.deadline.max(deadline));
1864 let mut quota = (
1865 record.quota.exceeded,
1866 status,
1867 !model.is_empty() && record.quota.reason == "credential_quota",
1868 );
1869 let reason = match record.quota.reason.as_str() {
1870 "credential_quota" => "credential_quota",
1871 "cloudflare challenge" => "cloudflare challenge",
1872 _ => "quota",
1873 };
1874 let mut quota_kind = record.quota.exceeded.then_some(QuotaKind { status, reason });
1875 let quota_under = prev.and_then(|p| p.quota_block(now));
1878 let mut trust = trust;
1879 if let Some(prev) = prev.filter(|p| p.trust != Trust::default() || trust != Trust::default()) {
1883 let window = prev.trust.window.max(trust.window);
1884 if prev_wins {
1885 trust = Trust {
1886 probe: None,
1887 ..prev.trust
1888 };
1889 quota = (prev.quota, prev.status, prev.credential);
1890 quota_kind = prev.quota_kind;
1891 }
1892 trust.window = window;
1893 }
1894 if trust.bounded && !prev_wins {
1898 self.generation = self.generation.wrapping_add(1);
1899 trust.generation = self.generation;
1900 }
1901 let (quota, status, credential) = quota;
1902 self.cooldowns.insert(
1903 key,
1904 Cooldown {
1905 deadline,
1906 level: record.quota.backoff_level,
1907 status,
1908 quota,
1909 error: if error.message.is_empty() {
1910 record.reason.clone()
1911 } else {
1912 error.message
1913 },
1914 since: record.updated_at.unwrap_or(wall),
1915 credential,
1916 trust,
1917 quota_kind: quota_kind.filter(|_| quota),
1918 quota_under: quota_under.filter(|_| !quota),
1919 recorded,
1921 },
1922 );
1923 true
1924 }
1925
1926 pub fn clear_disabled(&mut self, credentials: &[std::sync::Arc<Credential>], policy: &Policy) -> bool {
1929 let before = self.cooldowns.len();
1930 self.cooldowns.retain(|(id, _), _| {
1931 credentials
1932 .iter()
1933 .find(|c| c.id == *id)
1934 .is_none_or(|c| !c.disabled && !policy.cooling_disabled(c))
1935 });
1936 self.cooldowns.len() != before
1937 }
1938
1939 pub fn reconcile(&mut self, credentials: &[std::sync::Arc<Credential>]) {
1940 self.cooldowns
1941 .retain(|(id, _), _| credentials.iter().any(|c| c.id == *id));
1942 self.affinity.retain(|id| credentials.iter().any(|c| c.id == id));
1943 self.lcp.retain(|id| credentials.iter().any(|c| c.id == id));
1944 self.probes.retain(|id, _| credentials.iter().any(|c| c.id == *id));
1945 self.rejected.retain(|id, _| credentials.iter().any(|c| c.id == *id));
1946 }
1947}
1948
1949fn recover_after(s: &Cooldown, now: Instant) -> Option<Duration> {
1951 let stated = s
1952 .trust
1953 .stated
1954 .filter(|r| s.trust.bounded && *r > now && *r > s.deadline)?;
1955 Some(stated - now)
1956}
1957
1958#[derive(Debug, Clone, PartialEq, Eq)]
1961pub struct CooldownState {
1962 pub model: String,
1963 pub remaining: Duration,
1966 pub recover_in: Option<Duration>,
1968 pub level: u32,
1969 pub status: u16,
1970 pub quota: bool,
1971}
1972
1973impl Scheduler {
1974 pub(crate) fn cooldowns_of(&self, id: &str, now: Instant) -> Vec<CooldownState> {
1976 let mut out: Vec<CooldownState> = self
1977 .cooldowns
1978 .iter()
1979 .filter(|((cid, _), s)| cid == id && (s.deadline > now || recover_after(s, now).is_some()))
1980 .map(|((_, model), s)| CooldownState {
1981 model: model.clone(),
1982 remaining: s.deadline.saturating_duration_since(now),
1983 recover_in: recover_after(s, now),
1984 level: s.level,
1985 status: s.status,
1986 quota: s.quota,
1987 })
1988 .collect();
1989 out.sort_by(|a, b| a.model.cmp(&b.model));
1990 out
1991 }
1992
1993 pub(crate) fn reset_cooldowns(&mut self, id: &str) -> Vec<String> {
1995 let mut models: Vec<String> = self
1996 .cooldowns
1997 .keys()
1998 .filter(|(cid, model)| cid == id && !model.is_empty())
1999 .map(|(_, model)| model.clone())
2000 .collect();
2001 models.sort();
2002 self.cooldowns.retain(|(cid, _), _| cid != id);
2003 self.probes.remove(id);
2005 models
2006 }
2007
2008 pub(crate) fn clear_unauthorized(&mut self, id: &str, now: Instant) -> bool {
2018 self.rejected.remove(id);
2019 let mut changed = false;
2020 self.cooldowns.retain(|(cid, model), c| {
2021 let error = c.error.to_ascii_lowercase();
2022 let unauthorized = c.status == 401
2023 || error.contains("status 401")
2024 || error.contains("401 unauthorized")
2025 || (!model.is_empty() && error.contains("unauthorized"));
2026 if cid != id || !unauthorized {
2027 return true;
2028 }
2029 changed = true;
2030 let Some((end, kind)) = c.quota_block(now) else {
2031 return false;
2032 };
2033 c.quota = true;
2034 c.quota_kind = Some(kind);
2035 c.quota_under = None;
2036 c.deadline = c.deadline.max(end);
2037 c.status = kind.status;
2038 if error.contains("unauthorized") {
2041 c.error = kind.reason.into();
2042 }
2043 true
2044 });
2045 changed
2046 }
2047}
2048
2049#[cfg(test)]
2050mod tests {
2051 use super::*;
2052 use std::path::Path;
2053
2054 #[test]
2059 fn a_quota_under_a_401_keeps_its_kind_through_restore_and_recovery() {
2060 let (c, d) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
2061 let p = Policy {
2062 max_trusted_cooldown: Duration::ZERO,
2063 ..Policy::default()
2064 };
2065 let (start, wall) = (Instant::now(), SystemTime::now());
2066 let unauthorized = |text: &str| Outcome::Failure(ExecError::local(401, FailureScope::Credential, text));
2067 let mut challenge = ExecError::local(429, FailureScope::Model, "<title>Just a moment...</title> cloudflare");
2068 challenge.retry_after = Some(secs(2 * H));
2069 let mut s = Scheduler::default();
2070 s.record(&c, "restored", &hinted(FailureScope::Model, secs(2 * H)), &p, start);
2071 s.record(&c, "cloudflare", &Outcome::Failure(challenge), &p, start);
2072 s.record(&c, "plain", &hinted(FailureScope::Model, secs(2 * H)), &p, start);
2073 s.record(&d, "sibling", &unauthorized("unauthorized"), &p, start);
2074 let answered = start + secs(2);
2076 for model in ["restored", "cloudflare"] {
2077 s.record(&c, model, &unauthorized("unauthorized"), &p, answered);
2078 }
2079 s.record(&c, "plain", &unauthorized("status 401"), &p, answered);
2080 s.record(
2081 &d,
2082 "other",
2083 &hinted(FailureScope::Credential, secs(2 * H)),
2084 &p,
2085 answered,
2086 );
2087 assert!(
2088 entry(&s, &d, "sibling").quota,
2089 "the credential-wide quota reached the 401"
2090 );
2091 let restored = s
2092 .records(&c, answered, wall)
2093 .into_iter()
2094 .find(|r| r.model == "restored")
2095 .unwrap();
2096 assert!(!restored.quota.exceeded, "the saved record is the 401's");
2097 assert!(s.restore(&c, &restored, true, Duration::ZERO, answered, wall));
2098 let cases = [
2099 (&c, "restored", "quota"),
2100 (&c, "cloudflare", "cloudflare challenge"),
2101 (&c, "plain", "status 401"),
2102 (&d, "sibling", "credential_quota"),
2103 ];
2104 let blocked: Vec<_> = cases.iter().map(|(x, m, _)| s.wait(x, m, answered)).collect();
2105 assert!(s.clear_unauthorized("a", answered));
2106 assert!(s.clear_unauthorized("b", answered));
2107 for (x, model, error) in cases {
2108 let e = entry(&s, x, model);
2109 assert_eq!((e.quota, e.status, e.error.as_str()), (true, 429, error), "{model}");
2110 }
2111 let after: Vec<_> = cases.iter().map(|(x, m, _)| s.wait(x, m, answered)).collect();
2112 assert_eq!(after, blocked, "each stays blocked until the same time");
2113 assert!(after.iter().all(Option::is_some));
2114 }
2115
2116 #[test]
2120 fn clear_unauthorized_matches_go_per_key_kind() {
2121 let now = Instant::now();
2122 let cooldown = |status: u16, error: &str| Cooldown {
2123 deadline: now + Duration::from_secs(600),
2124 level: 0,
2125 status,
2126 quota: false,
2127 error: error.into(),
2128 since: SystemTime::now(),
2129 credential: false,
2130 trust: Trust::default(),
2131 quota_kind: None,
2132 quota_under: None,
2133 recorded: 0,
2134 };
2135 let mut s = Scheduler::default();
2136 for (id, model, c) in [
2137 ("a", "401", cooldown(401, "upstream said no")),
2138 ("a", "403-text", cooldown(403, "unauthorized operation")),
2139 ("a", "403", cooldown(403, "forbidden")),
2140 ("a", "", cooldown(403, "unauthorized operation")),
2141 ("b", "401", cooldown(401, "upstream said no")),
2142 ] {
2143 s.cooldowns.insert((id.into(), model.into()), c);
2144 }
2145 assert!(s.clear_unauthorized("a", now));
2146 let mut left: Vec<_> = s.cooldowns.keys().map(|(id, m)| (id.as_str(), m.as_str())).collect();
2147 left.sort();
2148 assert_eq!(left, [("a", ""), ("a", "403"), ("b", "401")]);
2149 for (status, error) in [(401, "x"), (403, "Status 401: revoked"), (403, "HTTP 401 Unauthorized")] {
2150 s.cooldowns.insert(("a".into(), String::new()), cooldown(status, error));
2151 assert!(s.clear_unauthorized("a", now), "{status} {error}");
2152 assert!(
2153 !s.cooldowns.contains_key(&("a".into(), String::new())),
2154 "{status} {error}"
2155 );
2156 }
2157 assert!(!s.clear_unauthorized("a", now), "nothing left to clear");
2158 }
2159
2160 #[test]
2164 fn a_401_over_a_live_quota_keeps_the_quota_through_recovery() {
2165 let c = cred("a", serde_json::json!({}));
2166 let start = Instant::now();
2167 let unauthorized = Outcome::Failure(ExecError::local(401, FailureScope::Credential, "unauthorized"));
2168 for (cap, replaced) in [(Duration::ZERO, 401), (Policy::default().max_trusted_cooldown, 429)] {
2171 let p = Policy {
2172 max_trusted_cooldown: cap,
2173 ..Policy::default()
2174 };
2175 let mut s = Scheduler::default();
2176 s.record(&c, "long", &hinted(FailureScope::Model, secs(2 * H)), &p, start);
2177 s.record(&c, "short", &hinted(FailureScope::Model, secs(MIN)), &p, start);
2178 let answered = start + secs(MIN / 2);
2179 s.record(&c, "long", &unauthorized, &p, answered);
2180 s.record(&c, "short", &unauthorized, &p, answered);
2181 assert_eq!(entry(&s, &c, "long").status, replaced, "{cap:?}");
2182 let blocked = s.wait(&c, "long", answered);
2183 let bind = start + secs(5 * MIN);
2184 assert!(s.clear_unauthorized("a", bind), "{cap:?}");
2185 let long = entry(&s, &c, "long");
2186 assert_eq!(
2187 (long.quota, long.status, long.error.as_str()),
2188 (true, 429, "quota"),
2189 "{cap:?}"
2190 );
2191 assert_eq!(
2192 s.wait(&c, "long", bind),
2193 blocked.map(|b| b - (bind - answered)),
2194 "{cap:?}: still blocked until the same time"
2195 );
2196 assert!(s.wait(&c, "long", bind).is_some_and(|w| w > secs(30 * MIN)), "{cap:?}");
2197 assert_eq!(s.wait(&c, "short", bind), None, "{cap:?}: its quota had ended");
2198 }
2199 }
2200
2201 fn tag<'a>(creds: &[&'a Credential]) -> Vec<(&'a Credential, &'static str)> {
2202 creds.iter().map(|c| (*c, "claude")).collect()
2203 }
2204
2205 fn cred(id: &str, extra: Value) -> Credential {
2206 let mut metadata = extra.as_object().unwrap().clone();
2207 metadata.insert("type".into(), "claude".into());
2208 Credential::from_file(Path::new("/mock"), &Path::new("/mock").join(id), metadata).unwrap()
2209 }
2210
2211 fn selection(model: &str) -> Selection {
2212 Selection {
2213 provider: "claude".into(),
2214 model: model.into(),
2215 ..Default::default()
2216 }
2217 }
2218
2219 #[test]
2220 fn normalization_matches_go_runtime_not_example_config() {
2221 for (raw, expected) in [
2222 (" WRR ", Strategy::WeightedRoundRobin),
2223 ("weightedroundrobin", Strategy::WeightedRoundRobin),
2224 ("weighted-round-robin", Strategy::WeightedRoundRobin),
2225 (" FF", Strategy::FillFirst),
2226 ("fillfirst", Strategy::FillFirst),
2227 ("fill-first", Strategy::FillFirst),
2228 ("soonest-reset", Strategy::SoonestReset),
2229 (" Reset-First ", Strategy::SoonestReset),
2230 ("resetfirst", Strategy::SoonestReset),
2231 ("", Strategy::SoonestReset),
2232 (" ", Strategy::SoonestReset),
2233 ("round-robin", Strategy::RoundRobin),
2234 ("anything", Strategy::RoundRobin),
2235 ] {
2236 assert_eq!(
2237 normalize(RawRouting {
2238 strategy: raw,
2239 ..Default::default()
2240 })
2241 .strategy,
2242 expected
2243 );
2244 }
2245 for (ttl, seconds) in [
2246 ("", 3600),
2247 ("bad", 3600),
2248 ("-1h", 3600),
2249 ("0", 3600),
2250 ("0.5s", 1),
2251 ("0.0000000001s", 3600),
2252 ("1h30m", 5400),
2253 (" 2m ", 120),
2254 ] {
2255 assert_eq!(
2256 normalize(RawRouting {
2257 session_affinity_ttl: ttl,
2258 ..Default::default()
2259 })
2260 .session_affinity_ttl,
2261 Duration::from_secs(seconds),
2262 "{ttl}"
2263 );
2264 }
2265 let policy = normalize(RawRouting {
2266 request_retry: -1,
2267 max_retry_credentials: -5,
2268 max_retry_interval: -3,
2269 session_affinity_subagents: Some(false),
2270 ..Default::default()
2271 });
2272 assert_eq!(policy.request_retry, 0);
2273 assert_eq!(policy.max_retry_credentials, 0);
2274 assert_eq!(policy.max_retry_interval, Duration::ZERO);
2275 assert!(policy.session_affinity_subagents, "ignored when affinity disabled");
2276 assert!(
2277 !normalize(RawRouting {
2278 session_affinity: true,
2279 session_affinity_subagents: Some(false),
2280 ..Default::default()
2281 })
2282 .session_affinity_subagents
2283 );
2284 assert!(
2285 normalize(RawRouting {
2286 session_affinity: true,
2287 ..Default::default()
2288 })
2289 .session_affinity_subagents
2290 );
2291 }
2292
2293 #[test]
2294 fn go_duration_decimal_compound_units_and_overflow() {
2295 for (raw, expected) in [
2296 ("1h2m3.4s", 3_723_400_000_000),
2297 (".5s", 500_000_000),
2298 ("+1.000000001s", 1_000_000_001),
2299 ("1.s", 1_000_000_000),
2300 ("1µs2μs3us", 6000),
2301 ("0.3333333333333333333h", 1_200_000_000_000),
2302 ("0.100000000000000000000h", 360_000_000_000),
2303 ("-9223372036854775808ns", i64::MIN),
2304 ("9223372036854775807ns", i64::MAX),
2305 ("0.9ns", 0),
2306 ] {
2307 assert_eq!(cpa_core::config::parse_duration(raw), Some(expected), "{raw}");
2308 }
2309 for raw in [
2310 "1d",
2311 ".s",
2312 "1",
2313 "1h 2m",
2314 "1s-2s",
2315 "1e3s",
2316 "9223372036854775808ns",
2317 "-9223372036854775809ns",
2318 ] {
2319 assert_eq!(cpa_core::config::parse_duration(raw), None, "{raw}");
2320 }
2321 }
2322
2323 #[test]
2324 fn rotation_tracks_previous_identity_and_provider_model_keys() {
2325 let a = cred("a", serde_json::json!({}));
2326 let b = cred("b", serde_json::json!({}));
2327 let c = cred("c", serde_json::json!({}));
2328 let p = Policy::default();
2329 let now = Instant::now();
2330 let mut s = Scheduler::default();
2331 assert_eq!(s.pick(&tag(&[&c, &b, &a]), &selection("m"), &p, now).unwrap().id, "a");
2332 assert_eq!(s.pick(&tag(&[&a, &c]), &selection("m"), &p, now).unwrap().id, "c");
2333 assert_eq!(s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now).unwrap().id, "a");
2334 assert_eq!(s.pick(&tag(&[&a, &b]), &selection("other"), &p, now).unwrap().id, "a");
2335 assert_eq!(s.pick(&tag(&[&a, &b]), &selection("m(high)"), &p, now).unwrap().id, "b");
2336 }
2337
2338 #[test]
2339 fn smooth_weights_keep_signed_credits_across_temporary_exclusions() {
2340 let a = cred("a", serde_json::json!({"weight":5}));
2341 let b = cred("b", serde_json::json!({"weight":1}));
2342 let c = cred("c", serde_json::json!({"weight":1}));
2343 let p = Policy {
2344 strategy: Strategy::WeightedRoundRobin,
2345 ..Default::default()
2346 };
2347 let now = Instant::now();
2348 let mut s = Scheduler::default();
2349 let picks: Vec<_> = (0..7)
2350 .map(|_| {
2351 s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now)
2352 .unwrap()
2353 .id
2354 .clone()
2355 })
2356 .collect();
2357 assert_eq!(picks, ["a", "a", "b", "a", "c", "a", "a"]);
2358 let mut s = Scheduler::default();
2359 assert_eq!(s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now).unwrap().id, "a");
2360 assert_eq!(s.pick(&tag(&[&b, &c]), &selection("m"), &p, now).unwrap().id, "b");
2361 assert_eq!(s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now).unwrap().id, "a");
2362 assert_eq!(s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now).unwrap().id, "c");
2363 for v in [
2364 serde_json::json!(0),
2365 serde_json::json!(-3),
2366 serde_json::json!(1_000_001),
2367 serde_json::json!(1.5),
2368 serde_json::json!("bad"),
2369 ] {
2370 assert!(!s.admits(&cred("off", serde_json::json!({"weight":v})), &p));
2371 }
2372 assert_eq!(weight(&cred("max", serde_json::json!({"weight":1_000_000}))), 1_000_000);
2373 assert_eq!(weight(&cred("float", serde_json::json!({"weight":2.0}))), 2);
2374 }
2375
2376 #[test]
2377 fn affinity_beats_recovered_priority_until_unusable_or_expired() {
2378 let low = cred("a", serde_json::json!({"priority":-1}));
2379 let high = cred("b", serde_json::json!({"priority":9}));
2380 let p = Policy {
2381 strategy: Strategy::FillFirst,
2382 session_affinity: true,
2383 session_affinity_ttl: Duration::from_secs(10),
2384 ..Default::default()
2385 };
2386 let sel = Selection {
2387 session: Some("session".into()),
2388 ..selection("m")
2389 };
2390 let now = Instant::now();
2391 let mut s = Scheduler::default();
2392 assert_eq!(s.pick(&tag(&[&low]), &sel, &p, now).unwrap().id, "a");
2393 assert_eq!(
2394 s.pick(&tag(&[&low, &high]), &sel, &p, now + Duration::from_secs(5))
2395 .unwrap()
2396 .id,
2397 "a"
2398 );
2399 assert_eq!(
2400 s.pick(&tag(&[&low, &high]), &sel, &p, now + Duration::from_secs(15))
2401 .unwrap()
2402 .id,
2403 "b"
2404 );
2405 assert_eq!(
2406 s.pick(&tag(&[&low]), &sel, &p, now + Duration::from_secs(16))
2407 .unwrap()
2408 .id,
2409 "a"
2410 );
2411 assert_eq!(s.pick(&tag(&[&low, &high]), &selection("m"), &p, now).unwrap().id, "b");
2412 }
2413
2414 #[test]
2415 fn max_trusted_cooldown_reads_durations_seconds_and_off() {
2416 let read = |raw| {
2417 normalize(RawRouting {
2418 max_trusted_cooldown: raw,
2419 ..Default::default()
2420 })
2421 .max_trusted_cooldown
2422 };
2423 assert_eq!(read(""), Duration::from_secs(3600));
2424 assert_eq!(read("90m"), Duration::from_secs(5400));
2425 assert_eq!(read("120"), Duration::from_secs(120));
2426 assert_eq!(read("0"), Duration::ZERO);
2427 assert_eq!(read("0s"), Duration::ZERO);
2428 assert_eq!(read("-5"), Duration::ZERO);
2429 for unreadable in ["soon", "1d", "false", "1.5"] {
2430 assert_eq!(read(unreadable), Duration::from_secs(3600), "{unreadable}");
2431 }
2432 assert_eq!(read("3"), Duration::from_secs(10), "clamped to Go's floor");
2433 assert_eq!(read("2s"), Duration::from_secs(10), "clamped to Go's floor");
2434 }
2435
2436 const H: u64 = 3600;
2437 const MIN: u64 = 60;
2438
2439 fn secs(n: u64) -> Duration {
2440 Duration::from_secs(n)
2441 }
2442
2443 fn hinted(scope: FailureScope, stated: Duration) -> Outcome {
2445 let mut e = ExecError::local(429, scope, "usage_limit_reached");
2446 e.retry_after = Some(stated);
2447 Outcome::Failure(e)
2448 }
2449
2450 fn unhinted(scope: FailureScope) -> Outcome {
2451 Outcome::Failure(ExecError::local(429, scope, "rate limited"))
2452 }
2453
2454 fn entry<'a>(s: &'a Scheduler, c: &Credential, model: &str) -> &'a Cooldown {
2455 &s.cooldowns[&(c.id.clone(), model.to_owned())]
2456 }
2457
2458 #[test]
2462 fn quota_resets_are_trusted_up_to_a_doubling_bound() {
2463 let c = cred("a", serde_json::json!({}));
2464 let p = Policy::default();
2465 let start = Instant::now();
2466 let reset = start + secs(6 * 24 * H);
2467 for (scope, key) in [(FailureScope::Model, "m"), (FailureScope::Credential, "")] {
2468 let mut s = Scheduler::default();
2469 let mut now = start;
2470 for window in [1, 2, 4, 8, 16, 32, 64] {
2471 s.record(&c, "m", &hinted(scope, reset - now), &p, now);
2472 assert_eq!(s.wait(&c, "m", now), Some(secs(window * H)), "{scope:?}");
2473 assert_eq!(entry(&s, &c, key).trust.stated, Some(reset), "{scope:?}: stated kept");
2474 assert_eq!(entry(&s, &c, key).level, 0, "{scope:?}: Go's level untouched");
2475 now += secs(window * H);
2476 assert!(s.retry_eligible(&c, "m", now));
2477 }
2478 s.record(&c, "m", &hinted(scope, reset - now), &p, now);
2480 assert_eq!(
2481 s.wait(&c, "m", now),
2482 Some(reset - now),
2483 "{scope:?}: capped by the stated reset"
2484 );
2485 let mut s = Scheduler::default();
2487 s.record(&c, "m", &hinted(scope, reset - start), &p, start);
2488 let then = start + secs(H);
2489 s.record(&c, "m", &hinted(scope, reset - then), &p, then);
2490 assert_eq!(s.wait(&c, "m", then), Some(secs(2 * H)));
2491 let later = start + secs(3 * H);
2492 s.record(&c, "m", &Outcome::Success, &p, later);
2493 s.record(&c, "m", &hinted(scope, reset - later), &p, later);
2494 assert_eq!(
2495 s.wait(&c, "m", later),
2496 Some(secs(H)),
2497 "{scope:?}: back to the first bound"
2498 );
2499 }
2500 let mut s = Scheduler::default();
2502 s.record(&c, "m", &hinted(FailureScope::Model, secs(30 * MIN)), &p, start);
2503 assert_eq!(s.wait(&c, "m", start), Some(secs(30 * MIN)));
2504 assert!(!entry(&s, &c, "m").trust.bounded);
2505 let go = Policy {
2507 max_trusted_cooldown: Duration::ZERO,
2508 ..Policy::default()
2509 };
2510 let mut s = Scheduler::default();
2511 s.record(&c, "m", &hinted(FailureScope::Model, reset - start), &go, start);
2512 assert_eq!(s.wait(&c, "m", start), Some(reset - start));
2513 }
2514
2515 #[test]
2518 fn trust_count_and_go_backoff_level_are_separate() {
2519 let c = cred("a", serde_json::json!({}));
2520 let p = Policy::default();
2521 let days = secs(6 * 24 * H);
2522 let mut s = Scheduler::default();
2524 let mut now = Instant::now();
2525 for _ in 0..9 {
2526 s.record(&c, "m", &unhinted(FailureScope::Model), &p, now);
2527 now += secs(600);
2528 }
2529 assert_eq!(entry(&s, &c, "m").level, 9);
2530 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
2531 assert_eq!(s.wait(&c, "m", now), Some(secs(H)), "the first bound, not 2^9 hours");
2532 assert_eq!(entry(&s, &c, "m").level, 9, "Go's level keeps its meaning");
2533 let mut s = Scheduler::default();
2536 let mut now = Instant::now();
2537 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
2538 now += secs(H);
2539 s.record(&c, "m", &unhinted(FailureScope::Model), &p, now);
2540 assert_eq!(s.wait(&c, "m", now), Some(secs(1)), "Go's first backoff step");
2541 now += secs(1);
2542 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
2543 assert_eq!(s.wait(&c, "m", now), Some(secs(2 * H)), "the trust count survived");
2544 }
2545
2546 #[test]
2549 fn a_reset_inside_another_cooldown_gets_the_bound() {
2550 let c = cred("a", serde_json::json!({}));
2551 let p = Policy::default();
2552 let days = secs(6 * 24 * H);
2553 let start = Instant::now();
2554 let first = |status: u16, scope| {
2555 let mut s = Scheduler::default();
2556 s.record(
2557 &c,
2558 "m",
2559 &Outcome::Failure(ExecError::local(status, scope, "earlier")),
2560 &p,
2561 start,
2562 );
2563 s
2564 };
2565 for (status, scope, earlier) in [
2566 (503, FailureScope::Credential, secs(60)),
2567 (429, FailureScope::Model, secs(1)),
2568 ] {
2569 let mut s = first(status, scope);
2570 assert_eq!(s.wait(&c, "m", start), Some(earlier));
2571 let at = start + Duration::from_millis(500);
2572 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, at);
2573 assert_eq!(s.wait(&c, "m", at), Some(secs(H)), "after {status}");
2574 assert!(entry(&s, &c, "m").trust.bounded, "after {status}");
2575 let later = at + secs(60);
2577 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, later);
2578 assert_eq!(s.wait(&c, "m", at), Some(secs(H)), "after {status}");
2579 assert_eq!(entry(&s, &c, "m").trust.window, Some(0), "after {status}");
2580 }
2581 let mut s = first(401, FailureScope::Credential);
2583 let cap = Policy {
2584 max_trusted_cooldown: secs(600),
2585 ..Policy::default()
2586 };
2587 s.record(&c, "m", &hinted(FailureScope::Model, days), &cap, start);
2588 assert_eq!(s.wait(&c, "m", start), Some(secs(30 * MIN)));
2589 assert!(!entry(&s, &c, "m").trust.bounded);
2590 }
2591
2592 #[test]
2595 fn concurrent_resets_keep_or_shorten_the_window() {
2596 let c = cred("a", serde_json::json!({}));
2597 let p = Policy::default();
2598 let start = Instant::now();
2599 let window = |s: &mut Scheduler, policy: &Policy| {
2600 s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), policy, start);
2601 };
2602 let mut s = Scheduler::default();
2604 window(&mut s, &p);
2605 let at = start + secs(30 * MIN);
2606 s.record(&c, "m", &hinted(FailureScope::Model, secs(45 * MIN)), &p, at);
2607 assert_eq!(s.wait(&c, "m", start), Some(secs(H)));
2608 let mut s = Scheduler::default();
2610 window(&mut s, &p);
2611 let at = start + secs(MIN);
2612 s.record(&c, "m", &hinted(FailureScope::Model, secs(30 * MIN)), &p, at);
2613 assert_eq!(s.wait(&c, "m", start), Some(secs(31 * MIN)));
2614 assert_eq!(entry(&s, &c, "m").trust.window, Some(0), "not a new window");
2615 let go = Policy {
2617 max_trusted_cooldown: Duration::ZERO,
2618 ..Policy::default()
2619 };
2620 let mut s = Scheduler::default();
2621 s.record(&c, "m", &hinted(FailureScope::Model, secs(H)), &go, start);
2622 let at = start + secs(30 * MIN);
2623 s.record(&c, "m", &hinted(FailureScope::Model, secs(45 * MIN)), &go, at);
2624 assert_eq!(s.wait(&c, "m", start), Some(secs(75 * MIN)));
2625 }
2626
2627 #[test]
2630 fn one_probe_per_ended_window() {
2631 let c = cred("a", serde_json::json!({}));
2632 let p = Policy::default();
2633 let start = Instant::now();
2634 for (scope, key) in [(FailureScope::Model, "m"), (FailureScope::Credential, "")] {
2635 let mut s = Scheduler::default();
2636 s.record(&c, "m", &hinted(scope, secs(6 * 24 * H)), &p, start);
2637 let near = start + secs(H) - secs(5);
2638 assert!(
2639 !s.retry_eligible(&c, "m", near),
2640 "{scope:?}: a bounded window is not waited for"
2641 );
2642 let end = start + secs(H);
2643 assert_eq!(s.wait(&c, "m", end), None, "{scope:?}: the probe is due");
2644 s.reserve_probe(&c, "m", end);
2645 assert_eq!(s.wait(&c, "m", end), Some(PROBE_HOLD), "{scope:?}: reserved");
2646 assert!(s.quota_cooling(&c, "m", end), "{scope:?}");
2647 assert!(!s.retry_eligible(&c, "m", end), "{scope:?}");
2648 assert_eq!(s.wait(&c, "other", end).is_some(), key.is_empty(), "{scope:?}");
2651 s.record(&c, "m", &hinted(scope, secs(5 * 24 * H)), &p, end + secs(2));
2653 assert_eq!(s.wait(&c, "m", end + secs(2)), Some(secs(2 * H)), "{scope:?}");
2654 assert_eq!(entry(&s, &c, key).trust.probe, None);
2655 let next = end + secs(2) + secs(2 * H);
2657 s.reserve_probe(&c, "m", next);
2658 s.record(&c, "m", &Outcome::Cancelled, &p, next + secs(1));
2659 assert_eq!(s.wait(&c, "m", next + secs(1)), None, "{scope:?}");
2660 s.reserve_probe(&c, "m", next + secs(1));
2661 assert_eq!(s.wait(&c, "m", next + secs(1) + PROBE_HOLD), None, "{scope:?}");
2662 }
2663 let mut s = Scheduler::default();
2665 s.record(&c, "m", &unhinted(FailureScope::Model), &p, start);
2666 s.reserve_probe(&c, "m", start + secs(2));
2667 assert_eq!(s.wait(&c, "m", start + secs(2)), None);
2668 }
2669
2670 #[test]
2673 fn a_success_on_another_model_clears_the_escalation() {
2674 let c = cred("a", serde_json::json!({}));
2675 let p = Policy::default();
2676 let start = Instant::now();
2677 let days = secs(6 * 24 * H);
2678 let mut s = Scheduler::default();
2679 s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, start);
2680 assert_eq!(entry(&s, &c, "a").trust.window, Some(0));
2681 let end = start + secs(H);
2682 s.record(&c, "b", &Outcome::Success, &p, end);
2683 s.record(&c, "a", &hinted(FailureScope::Model, days), &p, end);
2684 assert_eq!(s.wait(&c, "a", end), Some(secs(H)), "the first bound again");
2685 }
2686
2687 #[test]
2692 fn an_accepted_probe_frees_the_account_and_keeps_the_count() {
2693 let c = cred("a", serde_json::json!({}));
2694 let p = Policy::default();
2695 let days = secs(6 * 24 * H);
2696 let start = Instant::now();
2697 let end = start + secs(H);
2698 let mut s = Scheduler::default();
2700 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
2701 s.reserve_probe(&c, "m", end);
2702 assert!(s.wait(&c, "m", end).is_some(), "reserved");
2703 s.accept_probe(&c, "m", end);
2704 assert_eq!(s.wait(&c, "m", end), None, "the next pick goes through");
2705 s.reserve_probe(&c, "m", end + secs(1));
2706 assert_eq!(s.wait(&c, "m", end + secs(1)), None, "no second reservation");
2707 let later = end + secs(60);
2708 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, later);
2709 assert_eq!(s.wait(&c, "m", later), Some(secs(2 * H)), "the count survived");
2710 s.accept_probe(&c, "m", later + secs(1));
2712 assert!(entry(&s, &c, "m").trust.bounded);
2713 let mut s = Scheduler::default();
2715 s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, start);
2716 s.reserve_probe(&c, "b", end);
2717 s.accept_probe(&c, "b", end);
2718 assert_eq!(s.wait(&c, "b", end), None);
2719 s.reserve_probe(&c, "a", end);
2720 assert_eq!(s.wait(&c, "a", end), None, "A's copy does not reserve again");
2721 assert_eq!(entry(&s, &c, "a").trust.window, Some(0), "count kept");
2722 }
2723
2724 #[test]
2729 fn a_credential_probe_leaves_a_models_own_ended_window() {
2730 let c = cred("a", serde_json::json!({}));
2731 let p = Policy::default();
2732 let days = secs(6 * 24 * H);
2733 let start = Instant::now();
2734 for streamed in [true, false] {
2735 let mut s = Scheduler::default();
2736 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
2737 let opened = start + secs(H) + secs(MIN);
2738 s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, opened);
2739 let end = opened + secs(H);
2740 s.reserve_probe(&c, "b", end);
2741 if streamed {
2742 s.accept_probe(&c, "b", end);
2743 } else {
2744 s.record(&c, "b", &Outcome::Success, &p, end);
2745 }
2746 s.reserve_probe(&c, "a", end);
2747 assert_eq!(s.wait(&c, "a", end), None, "streamed {streamed}: A's copy is answered");
2748 s.reserve_probe(&c, "m", end);
2749 assert_eq!(
2750 s.wait(&c, "m", end),
2751 Some(PROBE_HOLD),
2752 "streamed {streamed}: m reserves its own probe"
2753 );
2754 }
2755 let mut s = Scheduler::default();
2760 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
2761 let opened = start + secs(H) + secs(MIN);
2762 s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, opened);
2763 let end = opened + secs(H);
2764 let credential_probe = s.reserve_probe(&c, "b", end);
2765 s.accept_probe_picked(&c, "b", credential_probe, end);
2766 let first = s.reserve_probe(&c, "m", end);
2767 let lapsed = end + PROBE_HOLD + secs(1);
2768 let second = s.reserve_probe(&c, "m", lapsed);
2769 assert!(credential_probe != first && first != second, "real generations");
2770 s.accept_probe_picked(&c, "m", second, lapsed);
2771 let done = lapsed + secs(1);
2772 s.record_picked(&c, "b", &Outcome::Success, &p, credential_probe, done);
2773 s.record_picked(&c, "m", &hinted(FailureScope::Model, days), &p, first, done);
2774 assert_eq!(s.wait(&c, "m", done), None, "no window from the superseded probe");
2775 s.record_picked(&c, "m", &Outcome::Success, &p, second, done + secs(1));
2776 assert!(
2777 !s.cooldowns.contains_key(&(c.id.clone(), "m".to_owned())),
2778 "m recovered"
2779 );
2780 }
2781
2782 #[test]
2786 fn a_superseded_probes_429_leaves_the_window_to_the_current_probe() {
2787 let c = cred("a", serde_json::json!({}));
2788 let p = Policy::default();
2789 let days = secs(6 * 24 * H);
2790 let start = Instant::now();
2791 for scope in [FailureScope::Model, FailureScope::Credential] {
2792 for accepted in [false, true] {
2795 let mut s = Scheduler::default();
2796 s.record(&c, "m", &hinted(scope, days), &p, start);
2797 let end = start + secs(H);
2798 let first = s.reserve_probe(&c, "m", end);
2799 let lapsed = end + PROBE_HOLD + secs(1);
2800 let second = s.reserve_probe(&c, "m", lapsed);
2801 if accepted {
2802 s.accept_probe_picked(&c, "m", second, lapsed);
2803 }
2804 let late = lapsed + secs(1);
2805 s.record_picked(&c, "m", &hinted(scope, days - secs(H)), &p, first, late);
2806 assert_eq!(
2807 s.wait(&c, "m", late),
2808 (!accepted).then(|| PROBE_HOLD - secs(1)),
2809 "{scope:?}, accepted {accepted}: still the second probe's"
2810 );
2811 s.record_picked(&c, "m", &Outcome::Success, &p, second, late + secs(1));
2812 assert_eq!(
2813 s.wait(&c, "m", late + secs(1)),
2814 None,
2815 "{scope:?}, accepted {accepted}: recovered"
2816 );
2817 }
2818 }
2819 }
2820
2821 #[test]
2825 fn a_renewed_credential_window_carries_its_ended_copies() {
2826 let c = cred("a", serde_json::json!({}));
2827 let p = Policy::default();
2828 let days = secs(6 * 24 * H);
2829 let start = Instant::now();
2830 for streamed in [true, false] {
2831 let mut s = Scheduler::default();
2832 s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, start);
2833 let first_end = start + secs(H);
2834 s.reserve_probe(&c, "b", first_end);
2835 let renewed = first_end + secs(1);
2836 s.record(&c, "b", &hinted(FailureScope::Credential, days - secs(H)), &p, renewed);
2837 let second_end = renewed + secs(2 * H);
2838 s.reserve_probe(&c, "c", second_end);
2839 if streamed {
2840 s.accept_probe(&c, "c", second_end);
2841 } else {
2842 s.record(&c, "c", &Outcome::Success, &p, second_end);
2843 }
2844 s.reserve_probe(&c, "a", second_end);
2845 assert_eq!(s.wait(&c, "a", second_end), None, "streamed {streamed}: no probe for A");
2846 if !streamed {
2847 s.record(&c, "a", &hinted(FailureScope::Model, days), &p, second_end);
2848 assert_eq!(s.wait(&c, "a", second_end), Some(secs(H)), "the count is cleared");
2849 }
2850 }
2851 }
2852
2853 #[test]
2857 fn outside_selection_leases_answer_no_bounded_window() {
2858 let c = cred("a", serde_json::json!({}));
2859 let p = Policy::default();
2860 let start = Instant::now();
2861 let mut s = Scheduler::default();
2862 s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, start);
2863 let ended = start + secs(H);
2864 let probe = s.reserve_probe(&c, "m", ended);
2865 let credits = s.outside_selection_generation();
2866 for outcome in [Outcome::Cancelled, Outcome::Success] {
2867 s.record_picked(&c, "m", &outcome, &p, credits, ended);
2868 assert!(s.wait(&c, "m", ended).is_some(), "the reserved probe holds the window");
2869 }
2870 s.accept_probe_picked(&c, "m", credits, ended);
2871 assert!(s.wait(&c, "m", ended).is_some(), "a credits stream accepts nothing");
2872 s.record_picked(&c, "m", &Outcome::Success, &p, probe, ended);
2873 assert_eq!(s.wait(&c, "m", ended), None, "the probe's own success ends it");
2874 let credits = s.outside_selection_generation();
2876 let later = ended + secs(MIN);
2877 s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, later);
2878 let reopened = later + secs(H);
2879 s.reserve_probe(&c, "m", reopened);
2880 s.record_picked(&c, "m", &Outcome::Cancelled, &p, credits, reopened);
2881 assert!(s.wait(&c, "m", reopened).is_some(), "the later probe still holds");
2882 }
2883
2884 #[test]
2887 fn generations_compare_across_the_wrap() {
2888 let c = cred("a", serde_json::json!({}));
2889 let p = Policy::default();
2890 let start = Instant::now();
2891 let mut s = Scheduler {
2892 generation: u32::MAX,
2893 ..Scheduler::default()
2894 };
2895 let before = s.reserve_probe(&c, "m", start);
2896 assert_eq!(before, u32::MAX);
2897 s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, start);
2898 assert_eq!(entry(&s, &c, "m").trust.generation, 0, "wrapped");
2899 let at = start + secs(MIN);
2900 s.record_picked(&c, "m", &Outcome::Success, &p, before, at);
2901 assert_eq!(s.wait(&c, "m", at), Some(secs(H - MIN)), "an older success leaves it");
2902 s.record_picked(&c, "m", &Outcome::Success, &p, 0, at);
2903 assert_eq!(s.wait(&c, "m", at), None, "a newer success ends it");
2904 }
2905
2906 #[test]
2909 fn an_earlier_credential_reset_shortens_inherited_model_windows() {
2910 let c = cred("a", serde_json::json!({}));
2911 let p = Policy::default();
2912 let start = Instant::now();
2913 let mut s = Scheduler::default();
2914 let support = ExecError::local(400, FailureScope::Model, "model is not supported");
2915 s.record(&c, "support", &Outcome::Failure(support), &p, start);
2916 s.record(&c, "sibling", &unhinted(FailureScope::Model), &p, start);
2917 s.record(&c, "a", &hinted(FailureScope::Credential, secs(6 * 24 * H)), &p, start);
2918 let at = start + secs(MIN);
2919 s.record(&c, "a", &hinted(FailureScope::Credential, secs(30 * MIN)), &p, at);
2920 for model in ["a", "sibling", "unrelated"] {
2921 assert_eq!(s.wait(&c, model, start), Some(secs(31 * MIN)), "{model}");
2922 }
2923 assert_eq!(s.wait(&c, "support", start), Some(secs(12 * H)));
2924 }
2925
2926 #[test]
2931 fn an_earlier_credential_reset_never_moves_a_models_own_probe_earlier() {
2932 let c = cred("a", serde_json::json!({}));
2933 let p = Policy::default();
2934 let days = secs(6 * 24 * H);
2935 let start = Instant::now();
2936 let own_end = start + secs(H);
2937 for failing in ["a", "m"] {
2938 let mut s = Scheduler::default();
2939 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
2940 let opened = start + secs(30 * MIN);
2941 s.record(&c, failing, &hinted(FailureScope::Credential, days), &p, opened);
2942 let at = opened + secs(MIN);
2943 s.record(&c, failing, &hinted(FailureScope::Credential, secs(10 * MIN)), &p, at);
2944 let moved = at + secs(10 * MIN);
2945 assert_eq!(
2946 s.wait(&c, "other", moved),
2947 None,
2948 "{failing}: the credential's probe is due"
2949 );
2950 assert_eq!(
2951 s.wait(&c, "m", moved),
2952 Some(own_end - moved),
2953 "{failing}: m waits for its own bound"
2954 );
2955 }
2956 }
2957
2958 #[test]
2964 fn an_earlier_credential_reset_never_shortens_a_models_own_cooldown() {
2965 let c = cred("a", serde_json::json!({}));
2966 let p = Policy::default();
2967 let start = Instant::now();
2968 let unauthorized = Outcome::Failure(ExecError::local(401, FailureScope::Credential, "unauthorized"));
2969 for failing in ["a", "m"] {
2970 let mut s = Scheduler::default();
2971 s.record(&c, "m", &unauthorized, &p, start);
2972 assert_eq!(s.wait(&c, "m", start), Some(secs(30 * MIN)));
2973 let opened = start + secs(MIN);
2974 s.record(
2975 &c,
2976 failing,
2977 &hinted(FailureScope::Credential, secs(6 * 24 * H)),
2978 &p,
2979 opened,
2980 );
2981 let at = start + secs(2 * MIN);
2982 s.record(&c, failing, &hinted(FailureScope::Credential, secs(MIN)), &p, at);
2983 let moved = start + secs(3 * MIN);
2984 assert_eq!(
2985 s.wait(&c, "other", moved),
2986 None,
2987 "{failing}: the credential's probe is due"
2988 );
2989 assert_eq!(
2990 s.wait(&c, "m", moved),
2991 Some(secs(27 * MIN)),
2992 "{failing}: m keeps its own 30 minutes"
2993 );
2994 }
2995 }
2996
2997 #[test]
3000 fn the_own_key_keeps_its_window_through_a_credential_merge() {
3001 let c = cred("a", serde_json::json!({}));
3002 let p = Policy::default();
3003 let days = secs(6 * 24 * H);
3004 let start = Instant::now();
3005 let mut s = Scheduler::default();
3006 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
3007 s.record(
3008 &c,
3009 "m",
3010 &hinted(FailureScope::Model, days - secs(H)),
3011 &p,
3012 start + secs(H),
3013 );
3014 let own_end = start + secs(3 * H);
3015 let at = start + secs(H) + secs(MIN);
3016 s.record(&c, "m", &hinted(FailureScope::Credential, days), &p, at);
3017 let own = entry(&s, &c, "m");
3018 assert_eq!(own.deadline, own_end);
3019 assert!(own.trust.bounded && own.trust.stated == Some(start + days));
3020 s.reserve_probe(&c, "m", own_end);
3021 assert!(
3022 entry(&s, &c, "m").trust.probe.is_some(),
3023 "it reserves a probe at its end"
3024 );
3025 }
3026
3027 #[test]
3030 fn a_restore_keeps_the_winner_trust() {
3031 use crate::cooldown_store::{Quota, Record};
3032 let c = cred("a", serde_json::json!({}));
3033 let p = Policy::default();
3034 let (now, wall) = (Instant::now(), SystemTime::now());
3035 let days = secs(6 * 24 * H);
3036 let mut s = Scheduler::default();
3037 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
3038 let live = entry(&s, &c, "m").trust;
3039 let shorter = Record {
3040 auth_id: "a".into(),
3041 model: "m".into(),
3042 next_retry_after: Some(wall + secs(10 * MIN)),
3043 quota: Quota {
3044 exceeded: true,
3045 reason: "quota".into(),
3046 next_recover_at: Some(wall + secs(10 * MIN)),
3047 trust_windows: Some(3),
3048 ..Default::default()
3049 },
3050 ..Default::default()
3051 };
3052 assert!(s.restore(&c, &shorter, true, secs(H), now, wall));
3053 let merged = entry(&s, &c, "m");
3054 assert_eq!(merged.deadline, now + secs(H));
3055 assert!(merged.trust.bounded && merged.trust.stated == live.stated);
3056 assert_eq!(merged.trust.window, Some(2), "counts merge");
3057 assert_eq!(merged.trust.generation, live.generation, "the live window's generation");
3058 }
3059
3060 #[test]
3064 fn a_success_dispatched_before_a_cooldown_keeps_it() {
3065 let c = cred("a", serde_json::json!({}));
3066 let p = Policy {
3067 max_trusted_cooldown: Duration::ZERO,
3068 ..Policy::default()
3069 };
3070 let (now, wall) = (Instant::now(), SystemTime::now());
3071 let g = 0;
3072 let quota = || hinted(FailureScope::Model, secs(H));
3073 let unavailable = || {
3076 let mut e = ExecError::local(503, FailureScope::Model, "unavailable");
3077 e.retry_after = Some(secs(H));
3078 Outcome::Failure(e)
3079 };
3080 for (model, failure) in [("m", quota()), ("", unavailable())] {
3081 let mut s = Scheduler::default();
3082 let stale = s.dispatch();
3083 let failed = s.dispatch();
3084 s.record_dispatched(&c, model, &failure, &p, g, failed, now);
3085 s.record_dispatched(&c, model, &Outcome::Success, &p, g, stale, now);
3086 s.record_dispatched(&c, model, &Outcome::Success, &p, g, failed, now);
3087 assert!(s.cooling(&c, model, now), "{model:?}: the stale successes keep it");
3088 let after = s.dispatch();
3089 s.record_dispatched(&c, model, &Outcome::Success, &p, g, after, now);
3090 assert!(!s.cooling(&c, model, now), "{model:?}: a later dispatch clears it");
3091
3092 let again = s.dispatch();
3093 s.record_dispatched(&c, model, &failure, &p, g, again, now);
3094 s.record_dispatched(&c, model, &Outcome::Success, &p, g, 0, now);
3095 assert!(!s.cooling(&c, model, now), "{model:?}: an unstamped success clears it");
3096 }
3097 let mut s = Scheduler::default();
3100 let first = s.dispatch();
3101 s.record_dispatched(&c, "", &unavailable(), &p, g, first, now);
3102 let success = s.dispatch();
3103 let dropped = Outcome::Failure(ExecError::local(0, FailureScope::Model, "http2: client conn is closed"));
3104 let third = s.dispatch();
3105 s.record_dispatched(&c, "", &dropped, &p, g, third, now);
3106 s.record_dispatched(&c, "", &Outcome::Success, &p, g, success, now);
3107 assert!(s.cooling(&c, "", now), "the skipped failure stamped the credential");
3108 let mut s = Scheduler::default();
3111 let stale = s.dispatch();
3112 let failed = s.dispatch();
3113 s.record_dispatched(&c, "m", "a(), &p, g, failed, now);
3114 let saved = s.records(&c, now, wall);
3115 assert!(s.restore(&c, &saved[0], true, Duration::ZERO, now, wall));
3116 s.record_dispatched(&c, "m", &Outcome::Success, &p, g, stale, now);
3117 assert!(s.cooling(&c, "m", now), "the restore kept the watermark");
3118 let mut restarted = Scheduler::default();
3119 assert!(restarted.restore(&c, &saved[0], true, Duration::ZERO, now, wall));
3120 let first = restarted.dispatch();
3121 restarted.record_dispatched(&c, "m", &Outcome::Success, &p, g, first, now);
3122 assert!(!restarted.cooling(&c, "m", now), "no watermark after a restart");
3123 let mut s = Scheduler::default();
3125 let other = s.dispatch();
3126 s.record_dispatched(&c, "other", "a(), &p, g, other, now);
3127 let stale = s.dispatch();
3128 let wide = s.dispatch();
3129 s.record_dispatched(&c, "m", &hinted(FailureScope::Credential, secs(H)), &p, g, wide, now);
3130 s.cooldowns.remove(&(c.id.clone(), String::new()));
3131 s.record_dispatched(&c, "other", &Outcome::Success, &p, g, stale, now);
3132 assert!(
3133 s.cooling(&c, "other", now),
3134 "the sibling carries the credential's watermark"
3135 );
3136 }
3137
3138 #[test]
3142 fn a_restore_while_serving_keeps_the_window_newer_than_older_leases() {
3143 let c = cred("a", serde_json::json!({}));
3144 let p = Policy::default();
3145 let (now, wall) = (Instant::now(), SystemTime::now());
3146 let mut s = Scheduler::default();
3147 let stream = s.reserve_probe(&c, "m", now);
3148 s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, now);
3149 let saved = s.records(&c, now, wall);
3150 assert!(s.restore(&c, &saved[0], true, secs(H), now, wall));
3151 assert!(entry(&s, &c, "m").trust.bounded);
3152 let at = now + secs(MIN);
3153 s.record_picked(&c, "m", &Outcome::Success, &p, stream, at);
3154 assert_eq!(s.wait(&c, "m", at), Some(secs(H - MIN)), "the window stands");
3155 }
3156
3157 #[test]
3160 fn a_503_inside_a_bounded_window_keeps_it() {
3161 let c = cred("a", serde_json::json!({}));
3162 let p = Policy::default();
3163 let (now, wall) = (Instant::now(), SystemTime::now());
3164 let days = secs(6 * 24 * H);
3165 let mut s = Scheduler::default();
3166 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
3167 let at = now + secs(30 * MIN);
3168 let unavailable = ExecError::local(503, FailureScope::Model, "overloaded");
3169 s.record(&c, "m", &Outcome::Failure(unavailable), &p, at);
3170 let saved = &s.records(&c, at, wall + secs(30 * MIN))[0];
3171 assert!(saved.quota.exceeded);
3172 assert_eq!(saved.quota.next_recover_at, Some(wall + days));
3173 assert_eq!(saved.quota.trust_windows, Some(1));
3174 assert_eq!(saved.reason, "quota");
3175 }
3176
3177 #[test]
3180 fn a_cloudflare_challenge_inside_a_bounded_window_raises_the_level() {
3181 let c = cred("a", serde_json::json!({}));
3182 let p = Policy::default();
3183 let start = Instant::now();
3184 let mut s = Scheduler::default();
3185 s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, start);
3186 let challenge = Outcome::Failure(ExecError::local(403, FailureScope::Model, "cloudflare challenge"));
3187 for (n, level) in [(1, 1), (2, 2)] {
3188 s.record(&c, "m", &challenge, &p, start + secs(n * MIN));
3189 assert_eq!(entry(&s, &c, "m").level, level);
3190 assert_eq!(s.wait(&c, "m", start), Some(secs(H)), "the window stands");
3191 assert!(entry(&s, &c, "m").trust.bounded);
3192 }
3193 }
3194
3195 #[test]
3199 fn restored_quota_resets_are_bounded_too() {
3200 use crate::cooldown_store::{Quota, Record};
3201 let c = cred("a", serde_json::json!({}));
3202 let p = Policy::default();
3203 let (now, wall) = (Instant::now(), SystemTime::now());
3204 let cap = secs(H);
3205 let days = secs(6 * 24 * H);
3206 let mut s = Scheduler::default();
3207 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
3208 let saved = s.records(&c, now, wall);
3209 assert_eq!(saved[0].next_retry_after, Some(wall + secs(H)));
3210 assert_eq!(saved[0].quota.next_recover_at, Some(wall + days));
3211 assert_eq!(
3212 (saved[0].quota.trust_windows, saved[0].quota.backoff_level),
3213 (Some(1), 0)
3214 );
3215 let mut back = Scheduler::default();
3217 assert!(back.restore(&c, &saved[0], true, cap, now, wall));
3218 assert_eq!(back.wait(&c, "m", now), Some(secs(H)));
3219 assert_eq!(back.records(&c, now, wall), saved);
3220 let record_at = |retry: Duration, trust_windows, backoff_level| Record {
3221 auth_id: "a".into(),
3222 model: "m".into(),
3223 next_retry_after: Some(wall + retry),
3224 quota: Quota {
3225 exceeded: true,
3226 reason: "quota".into(),
3227 next_recover_at: Some(wall + days),
3228 backoff_level,
3229 trust_windows,
3230 ..Default::default()
3231 },
3232 ..Default::default()
3233 };
3234 let mut s = Scheduler::default();
3236 assert!(s.restore(&c, &record_at(days, None, 2), true, cap, now, wall));
3237 assert_eq!(s.wait(&c, "m", now), Some(secs(H)));
3238 assert_eq!(entry(&s, &c, "m").level, 2);
3239 let mut s = Scheduler::default();
3242 assert!(s.restore(&c, &record_at(secs(4 * H), Some(3), 0), true, cap, now, wall));
3243 assert_eq!(s.wait(&c, "m", now), Some(secs(4 * H)));
3244 let trust = entry(&s, &c, "m").trust;
3245 assert!(trust.bounded && trust.window == Some(2));
3246 assert_eq!(trust.stated, Some(now + days));
3247 let later = now + secs(4 * H);
3248 s.record(&c, "m", &hinted(FailureScope::Model, days - secs(4 * H)), &p, later);
3249 assert_eq!(s.wait(&c, "m", later), Some(secs(8 * H)));
3250 let ninety = secs(90 * MIN);
3252 let trusted = Record {
3253 quota: Quota {
3254 next_recover_at: Some(wall + ninety),
3255 ..record_at(ninety, Some(1), 0).quota
3256 },
3257 ..record_at(ninety, Some(1), 0)
3258 };
3259 let mut s = Scheduler::default();
3260 assert!(s.restore(&c, &trusted, true, cap, now, wall));
3261 assert_eq!(s.wait(&c, "m", now), Some(ninety));
3262 let back = &s.records(&c, now, wall)[0];
3263 assert_eq!(
3264 (
3265 back.next_retry_after,
3266 back.quota.next_recover_at,
3267 back.quota.trust_windows
3268 ),
3269 (trusted.next_retry_after, trusted.quota.next_recover_at, Some(1))
3270 );
3271 assert!(!entry(&s, &c, "m").trust.bounded);
3272 let mut s = Scheduler::default();
3274 assert!(s.restore(&c, &record_at(secs(H), Some(u32::MAX), 0), true, cap, now, wall));
3275 assert_eq!(entry(&s, &c, "m").trust.window, Some(MAX_TRUST_WINDOW));
3276 let mut s = Scheduler::default();
3278 assert!(s.restore(&c, &record_at(days, Some(3), 0), true, Duration::ZERO, now, wall));
3279 assert_eq!(s.wait(&c, "m", now), Some(days));
3280 }
3281
3282 #[test]
3286 fn a_marked_sibling_restores_its_own_deadline() {
3287 let c = cred("a", serde_json::json!({}));
3288 let p = Policy::default();
3289 let (now, wall) = (Instant::now(), SystemTime::now());
3290 let mut s = Scheduler::default();
3291 let support = ExecError::local(400, FailureScope::Model, "model is not supported");
3292 s.record(&c, "b", &Outcome::Failure(support), &p, now);
3293 assert_eq!(s.wait(&c, "b", now), Some(secs(12 * H)), "Go's model-support cooldown");
3294 s.record(&c, "a", &hinted(FailureScope::Credential, secs(6 * 24 * H)), &p, now);
3295 let records = s.records(&c, now, wall);
3296 let b = records.iter().find(|r| r.model == "b").unwrap();
3297 assert!(b.quota.exceeded && b.quota.trust_windows.is_some(), "{b:?}");
3298 let mut back = Scheduler::default();
3299 for r in &records {
3300 back.restore(&c, r, true, secs(H), now, wall);
3301 }
3302 assert_eq!(back.wait(&c, "b", now), Some(secs(12 * H)));
3303 assert!(!entry(&back, &c, "b").trust.bounded, "not cut, so not a bounded window");
3304 }
3305
3306 #[test]
3311 fn credential_wide_windows_keep_the_larger_model_count() {
3312 let c = cred("a", serde_json::json!({}));
3313 let p = Policy::default();
3314 let days = secs(6 * 24 * H);
3315 for failing in ["other", "m"] {
3316 let mut now = Instant::now();
3317 let mut s = Scheduler::default();
3318 for window in [1, 2, 4, 8] {
3319 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
3320 assert_eq!(s.wait(&c, "m", now), Some(secs(window * H)));
3321 if window < 8 {
3322 now += secs(window * H);
3323 }
3324 }
3325 let late = now + secs(8 * H) - secs(10 * MIN);
3328 s.record(&c, failing, &hinted(FailureScope::Credential, days), &p, late);
3329 assert_eq!(entry(&s, &c, "m").trust.window, Some(3), "{failing}: count kept");
3330 let end = late + secs(H);
3331 s.record(&c, "m", &hinted(FailureScope::Model, days), &p, end);
3332 assert_eq!(s.wait(&c, "m", end), Some(secs(16 * H)), "{failing}");
3333 }
3334 }
3335
3336 #[test]
3339 fn a_reset_inside_a_cloudflare_window_gets_the_first_bound() {
3340 let c = cred("a", serde_json::json!({}));
3341 let p = Policy::default();
3342 let mut now = Instant::now();
3343 let mut s = Scheduler::default();
3344 for _ in 0..5 {
3345 let challenge = ExecError::local(403, FailureScope::Model, "cloudflare challenge");
3346 s.record(&c, "m", &Outcome::Failure(challenge), &p, now);
3347 now += secs(1);
3348 }
3349 assert!(entry(&s, &c, "m").level >= 5);
3350 assert!(s.wait(&c, "m", now).is_some(), "inside the challenge window");
3351 s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, now);
3352 assert_eq!(s.wait(&c, "m", now), Some(secs(H)));
3353 }
3354
3355 #[test]
3356 fn cooldown_deadlines_floor_backoff_and_sibling_success() {
3357 let c = cred("a", serde_json::json!({}));
3358 let p = Policy::default();
3359 let now = Instant::now();
3360 let mut s = Scheduler::default();
3361 let fail = |status, scope, hint| {
3362 let mut e = ExecError::local(status, scope, "failure");
3363 e.retry_after = hint;
3364 Outcome::Failure(e)
3365 };
3366 s.record(&c, "sibling", &fail(404, FailureScope::Model, None), &p, now);
3367 s.record(
3368 &c,
3369 "m",
3370 &fail(429, FailureScope::Credential, Some(Duration::from_secs(2))),
3371 &p,
3372 now,
3373 );
3374 assert_eq!(s.wait(&c, "m", now), Some(Duration::from_secs(10)));
3375 assert_eq!(s.wait(&c, "new", now), Some(Duration::from_secs(10)));
3376 assert_eq!(
3377 s.wait(&c, "sibling", now),
3378 Some(Duration::from_secs(43200)),
3379 "quota must not shorten sibling"
3380 );
3381 s.record(&c, "sibling", &Outcome::Success, &p, now + Duration::from_secs(1));
3382 assert_eq!(
3383 s.wait(&c, "sibling", now),
3384 Some(Duration::from_secs(43200)),
3385 "inflight success cannot clear live quota"
3386 );
3387 assert_eq!(
3388 s.wait(&c, "new", now + Duration::from_secs(10)),
3389 None,
3390 "deadline boundary"
3391 );
3392 s.record(&c, "new", &Outcome::Success, &p, now + Duration::from_secs(10));
3393 assert!(s.wait(&c, "sibling", now + Duration::from_secs(10)).is_some());
3394 let mut s = Scheduler::default();
3395 s.record(&c, "m", &fail(429, FailureScope::Model, None), &p, now);
3396 assert_eq!(s.wait(&c, "m(high)", now), Some(Duration::from_secs(1)));
3397 s.record(&c, "m", &fail(429, FailureScope::Model, None), &p, now);
3398 assert_eq!(
3399 s.wait(&c, "m", now),
3400 Some(Duration::from_secs(1)),
3401 "active quota reused"
3402 );
3403 s.record(
3404 &c,
3405 "m",
3406 &fail(429, FailureScope::Model, None),
3407 &p,
3408 now + Duration::from_secs(1),
3409 );
3410 assert_eq!(
3411 s.wait(&c, "m", now + Duration::from_secs(1)),
3412 Some(Duration::from_secs(2))
3413 );
3414 assert_eq!(s.wait(&c, "other", now), None, "model failure stays local");
3415 for (status, seconds) in [
3416 (401, 1800),
3417 (402, 1800),
3418 (403, 1800),
3419 (404, 43200),
3420 (408, 60),
3421 (500, 60),
3422 (526, 60),
3423 ] {
3424 let mut s = Scheduler::default();
3425 s.record(&c, "m", &fail(status, FailureScope::Model, None), &p, now);
3426 assert_eq!(
3427 s.wait(&c, "m", now),
3428 Some(Duration::from_secs(seconds)),
3429 "status {status}"
3430 );
3431 }
3432 }
3433
3434 #[test]
3437 fn cooldowns_match_go_mark_result() {
3438 let fixture: Value = serde_json::from_str(include_str!("../tests/fixtures/server_go.json")).unwrap();
3439 let cases = fixture["cooldown"].as_array().unwrap();
3440 assert_eq!(cases.len(), 25);
3441 let policy = Policy {
3443 max_trusted_cooldown: Duration::ZERO,
3444 ..Policy::default()
3445 };
3446 for case in cases {
3447 let name = case["name"].as_str().unwrap();
3448 let c = cred(&format!("{name}.json"), serde_json::json!({}));
3449 let mut s = Scheduler::default();
3450 let now = Instant::now();
3451 for step in case["steps"].as_array().unwrap() {
3452 let status = step["status"].as_u64().unwrap() as u16;
3453 let scope = if step["credential_scope"].as_bool().unwrap() {
3454 FailureScope::Credential
3455 } else {
3456 match status {
3457 429 => FailureScope::Model,
3458 401..=403 | 408 | 500.. => FailureScope::Credential,
3459 _ => FailureScope::Request,
3460 }
3461 };
3462 let mut error = ExecError::local(status, scope, step["message"].as_str().unwrap());
3463 error
3464 .headers
3465 .insert("content-type", "application/json".parse().unwrap());
3466 let hint = step["retry_after_ms"].as_i64().unwrap();
3467 error.retry_after = (hint >= 0).then(|| Duration::from_millis(hint as u64));
3468 s.record(
3469 &c,
3470 step["model"].as_str().unwrap(),
3471 &Outcome::Failure(error),
3472 &policy,
3473 now,
3474 );
3475 }
3476 for (model, seconds) in case["seconds"].as_object().unwrap() {
3477 let wait = s.wait(&c, model, now).unwrap_or_default();
3478 assert_eq!(
3479 wait.as_secs_f64().round() as u64,
3480 seconds.as_u64().unwrap(),
3481 "{name} {model} wait"
3482 );
3483 let quota = case["quota"][model].as_bool().unwrap();
3484 assert_eq!(
3486 s.quota_cooling(&c, model, now),
3487 quota && wait > Duration::ZERO,
3488 "{name} {model} quota"
3489 );
3490 }
3491 }
3492 }
3493
3494 #[test]
3497 fn session_affinity_matches_go_selector() {
3498 let fixture: Value = serde_json::from_str(include_str!("../tests/fixtures/server_go.json")).unwrap();
3499 let cases = fixture["affinity"].as_array().unwrap();
3500 assert_eq!(cases.len(), 25);
3501 for case in cases {
3502 let name = case["name"].as_str().unwrap();
3503 let lcp_case = case["lcp"].as_bool().unwrap_or_default();
3504 let creds: Vec<Credential> = ["a", "b", "c"]
3505 .iter()
3506 .map(|id| {
3507 let mut c = cred(id, serde_json::json!({}));
3508 if let Some(p) = case["priorities"][*id].as_str() {
3509 c.attributes.insert("priority".into(), p.into());
3510 }
3511 c
3512 })
3513 .collect();
3514 let policy = Policy {
3515 strategy: Strategy::FillFirst,
3516 session_affinity: true,
3517 session_affinity_subagents: case["subagent_affinity"].as_bool().unwrap(),
3518 ..Default::default()
3519 };
3520 let mut s = Scheduler::default();
3521 let now = Instant::now();
3522 for (i, step) in case["steps"].as_array().unwrap().iter().enumerate() {
3523 let mut headers = axum::http::HeaderMap::new();
3524 for pair in step["headers"].as_array().into_iter().flatten() {
3525 headers.append(
3526 axum::http::HeaderName::from_bytes(pair[0].as_str().unwrap().as_bytes()).unwrap(),
3527 pair[1].as_str().unwrap().parse().unwrap(),
3528 );
3529 }
3530 let payload = step["payload"].as_str().unwrap_or_default().as_bytes();
3531 let format = cpa_core::format::Format::parse(case["format"].as_str().unwrap_or("openai")).unwrap();
3532 let caller = match step["caller"].as_str() {
3533 Some(c) => c,
3534 None => case["caller"].as_str().unwrap_or_default(),
3535 };
3536 let caller = if lcp_case { caller } else { "" };
3537 let session = crate::session::resolve(format, &headers, payload, None, caller);
3538 let lcp = (lcp_case && !session.explicit)
3540 .then(|| crate::lcp::Request::new(format.as_str(), payload, caller))
3541 .flatten()
3542 .map(std::sync::Arc::new);
3543 let sel = Selection {
3544 session: session.id,
3545 session_parent: session.parent,
3546 session_fork: session.fork,
3547 lcp,
3548 ..selection("m")
3549 };
3550 let find = |id: &str| creds.iter().find(|c| c.id == id).unwrap();
3551 let outcome = |op: &str| {
3552 if op.ends_with("ok") {
3553 Outcome::Success
3554 } else {
3555 let status = step["status"].as_u64().unwrap() as u16;
3556 let scope = if status >= 500 {
3557 FailureScope::Credential
3558 } else {
3559 FailureScope::Request
3560 };
3561 let mut e = ExecError::local(status, scope, step["message"].as_str().unwrap());
3562 e.headers.insert("content-type", "text/plain".parse().unwrap());
3563 Outcome::Failure(e)
3564 }
3565 };
3566 match step["op"].as_str().unwrap() {
3567 op @ ("pick" | "pick_ok" | "pick_fail") => {
3568 let available: Vec<&Credential> = step["available"]
3569 .as_array()
3570 .unwrap()
3571 .iter()
3572 .map(|id| find(id.as_str().unwrap()))
3573 .collect();
3574 let (picked, bound) = s
3575 .pick_session(&tag(&available), &sel, &policy, &|_| Windows::default(), now)
3576 .unwrap();
3577 assert_eq!(picked.id, step["picked"].as_str().unwrap(), "{name} step {i}");
3578 let got = bound.as_ref().map(|m| {
3580 let (node_kind, fork, compaction) = m.node();
3581 serde_json::json!({
3582 "session": m.session, "parent": m.parent, "node_kind": node_kind,
3583 "fork": fork, "compaction": compaction, "generation": m.access,
3584 })
3585 });
3586 assert_eq!(got.as_ref(), step.get("lcp"), "{name} step {i} lcp");
3587 if op != "pick" {
3588 let picked = picked.clone();
3589 s.session_result(&picked, &sel, &outcome(op), &policy, bound.as_ref(), now);
3590 }
3591 }
3592 op => {
3593 s.session_result(
3594 find(step["auth"].as_str().unwrap()),
3595 &sel,
3596 &outcome(op),
3597 &policy,
3598 None,
3599 now,
3600 );
3601 }
3602 }
3603 }
3604 }
3605 }
3606
3607 #[test]
3610 fn delayed_failure_keeps_binding_made_after_policy_change() {
3611 let a = cred("a", serde_json::json!({}));
3612 let p = Policy {
3613 session_affinity: true,
3614 ..Default::default()
3615 };
3616 let next = Policy {
3617 session_affinity_ttl: Duration::from_secs(600),
3618 ..p.clone()
3619 };
3620 let now = Instant::now();
3621 let mut s = Scheduler::default();
3622 let body = br#"{"messages":[{"role":"user","content":"hello"}]}"#;
3623 let request = crate::lcp::Request::new("openai", body, "client-key").unwrap();
3624 let sel = Selection {
3625 lcp: Some(std::sync::Arc::new(request.clone())),
3626 ..selection("m")
3627 };
3628 let ranks = |_: &Credential| Windows::default();
3629 let mut old = None;
3630 for _ in 0..5 {
3631 old = s.pick_session(&tag(&[&a]), &sel, &p, &ranks, now).unwrap().1;
3632 }
3633 let old = old.unwrap();
3634 s.configure(&p, &next);
3635 let fresh = s
3636 .pick_session(&tag(&[&a]), &sel, &next, &ranks, now)
3637 .unwrap()
3638 .1
3639 .unwrap();
3640 assert!(fresh.access > old.access, "{} after {}", fresh.access, old.access);
3641 let failure = Outcome::Failure(ExecError::local(500, FailureScope::Credential, "boom"));
3642 s.session_result(&a, &sel, &failure, &p, Some(&old), now);
3643 let namespace = request.namespace("m");
3644 let kept = s.lcp.find(&namespace, request.prepared(), now);
3645 assert_eq!(kept.map(|m| m.auth), Some("a".to_owned()));
3646 }
3647
3648 #[test]
3651 fn leases_from_before_disabling_affinity_bind_nothing() {
3652 let a = cred("a", serde_json::json!({}));
3653 let on = Policy {
3654 session_affinity: true,
3655 ..Default::default()
3656 };
3657 let off = Policy::default();
3658 let now = Instant::now();
3659 let mut s = Scheduler::default();
3660 s.configure(&off, &on);
3661 let body = br#"{"messages":[{"role":"user","content":"hello"}]}"#;
3662 let request = crate::lcp::Request::new("openai", body, "client-key").unwrap();
3663 let namespace = request.namespace("m");
3664 let lcp = Selection {
3665 lcp: Some(std::sync::Arc::new(request.clone())),
3666 ..selection("m")
3667 };
3668 let explicit = Selection {
3669 session: Some("s1".into()),
3670 ..selection("m")
3671 };
3672 let ranks = |_: &Credential| Windows::default();
3673 let bound = s.pick_session(&tag(&[&a]), &lcp, &on, &ranks, now).unwrap().1;
3674 assert!(bound.is_some());
3675 s.configure(&on, &off);
3676 assert!(s.lcp.find(&namespace, request.prepared(), now).is_none());
3677 s.session_result(&a, &lcp, &Outcome::Success, &on, bound.as_ref(), now);
3678 s.session_result(&a, &explicit, &Outcome::Success, &on, None, now);
3679 let retry = s.pick_session(&tag(&[&a]), &lcp, &on, &ranks, now).unwrap().1;
3680 assert!(retry.is_none());
3681 s.pick_session(&tag(&[&a]), &explicit, &on, &ranks, now);
3682 assert!(s.lcp.find(&namespace, request.prepared(), now).is_none());
3683 assert!(s.affinity.get(&session_keys(&explicit).unwrap().primary, now).is_none());
3684 s.configure(&off, &on);
3686 assert!(s.pick_session(&tag(&[&a]), &lcp, &on, &ranks, now).unwrap().1.is_some());
3687 }
3688
3689 #[test]
3690 fn policy_change_resets_rotation_and_affinity_but_keeps_cooldowns() {
3691 let a = cred("a", serde_json::json!({}));
3692 let b = cred("b", serde_json::json!({}));
3693 let p = Policy {
3694 session_affinity: true,
3695 ..Default::default()
3696 };
3697 let now = Instant::now();
3698 let mut s = Scheduler::default();
3699 let mut selection = selection("m");
3700 selection.session = Some("session".into());
3701 assert_eq!(s.pick(&tag(&[&a, &b]), &selection, &p, now).unwrap().id, "a");
3702 s.record(
3703 &a,
3704 "other",
3705 &Outcome::Failure(ExecError::local(429, FailureScope::Model, "quota")),
3706 &p,
3707 now,
3708 );
3709 let mut next = p.clone();
3710 next.request_retry = 2;
3711 s.configure(&p, &next);
3712 assert!(!s.rotations.is_empty(), "retry-only change preserves selector state");
3713 assert!(!s.affinity.is_empty());
3714 next.session_affinity_ttl = Duration::from_secs(5);
3715 s.configure(&p, &next);
3716 assert!(s.rotations.is_empty());
3717 assert!(s.affinity.is_empty());
3718 assert_eq!(s.wait(&a, "other", now), Some(Duration::from_secs(1)));
3719 }
3720
3721 #[test]
3722 fn forced_cooling_uses_fallback_and_quota_does_not_reuse_transient_state() {
3723 let now = Instant::now();
3724 for (status, seconds) in [(401, 1800), (404, 43200), (429, 1), (503, 60)] {
3727 let c = cred(
3728 "a",
3729 serde_json::json!({"disable_cooling":true,
3730 "request_scoped_errors":[{"status":status,"match":["forced"],"action":"stop-and-cooldown"}]}),
3731 );
3732 let mut s = Scheduler::default();
3733 s.record(
3734 &c,
3735 "m",
3736 &Outcome::Failure(ExecError::local(status, FailureScope::Credential, "forced")),
3737 &Policy::default(),
3738 now,
3739 );
3740 assert_eq!(
3741 s.wait(&c, "m", now),
3742 Some(Duration::from_secs(seconds)),
3743 "status {status}"
3744 );
3745 }
3746 let c = cred(
3748 "a",
3749 serde_json::json!({"request_scoped_errors":[{"status":503,"match":["forced"],"action":"continue-and-cooldown"}]}),
3750 );
3751 let no_transient = Policy {
3752 transient_error_cooldown_seconds: -1,
3753 ..Policy::default()
3754 };
3755 let mut s = Scheduler::default();
3756 let forced = Outcome::Failure(ExecError::local(503, FailureScope::Credential, "forced"));
3757 s.record(&c, "m", &forced, &no_transient, now);
3758 assert_eq!(s.wait(&c, "m", now), Some(Duration::from_secs(60)));
3759 let plain = Outcome::Failure(ExecError::local(503, FailureScope::Credential, "plain"));
3760 let mut s = Scheduler::default();
3761 s.record(&c, "m", &plain, &no_transient, now);
3762 assert_eq!(
3763 s.wait(&c, "m", now),
3764 None,
3765 "negative transient seconds disable 503 cooldowns"
3766 );
3767 let c = cred("a", serde_json::json!({}));
3768 let p = Policy::default();
3769 let mut s = Scheduler::default();
3770 s.record(
3771 &c,
3772 "m",
3773 &Outcome::Failure(ExecError::local(503, FailureScope::Model, "transient")),
3774 &p,
3775 now,
3776 );
3777 s.record(
3778 &c,
3779 "m",
3780 &Outcome::Failure(ExecError::local(429, FailureScope::Model, "quota")),
3781 &p,
3782 now,
3783 );
3784 assert!(
3785 s.quota_cooling(&c, "m", now),
3786 "quota must replace transient classification"
3787 );
3788 assert_eq!(
3789 s.wait(&c, "m", now),
3790 Some(Duration::from_secs(60)),
3791 "live deadline never shortened"
3792 );
3793 let mut hint = ExecError::local(418, FailureScope::Model, "other status");
3794 hint.retry_after = Some(Duration::from_secs(900));
3795 s.record(&c, "other", &Outcome::Failure(hint), &p, now);
3796 assert_eq!(
3797 s.wait(&c, "other", now),
3798 Some(Duration::from_secs(60)),
3799 "unlisted statuses ignore hint"
3800 );
3801 }
3802
3803 #[test]
3809 fn compat_groups_match_with_go_equal_fold() {
3810 let cfg = cpa_core::config::Config::parse(
3811 "openai-compatibility:\n - name: openai-compatible-ος\n base-url: http://127.0.0.1:9\n disable-cooling: true\n api-key-entries: [{api-key: k, proxy-url: 'http://127.0.0.1:8080'}]\n - name: ΟΣ\n base-url: http://127.0.0.1:9\n api-key-entries: [{api-key: k}]\n",
3812 )
3813 .unwrap();
3814 let creds = cpa_core::config::credentials::from_config(&cfg);
3815 let c = creds
3816 .iter()
3817 .find(|c| c.attributes.get("compat_name").is_some_and(|n| n == "ΟΣ"))
3818 .unwrap();
3819 assert_eq!(cpa_core::registry::dynamic::provider_key(c), "openai-compatible-οσ");
3820 assert_eq!(
3821 cpa_core::config::credentials::api_key_config_proxy(&cfg, c),
3822 "http://127.0.0.1:8080"
3823 );
3824 let mut policy = crate::management::policy(&cfg);
3825 policy.compat_disable_cooling = compat_cooling(&cfg);
3826 assert_eq!(policy.provider_cooling(c), Some(true));
3827 }
3828
3829 #[test]
3833 fn oauth_rule_channels_lowercase_like_go() {
3834 let cfg = cpa_core::config::Config::parse(
3835 "oauth:\n request-scoped-errors:\n HİGH:\n - {status: 503, match: [boom], action: stop}\n",
3836 )
3837 .unwrap();
3838 let c = cpa_core::config::credentials::from_file(
3839 &cfg,
3840 std::path::Path::new("/auth"),
3841 std::path::Path::new("/auth/a.json"),
3842 r#"{"type":"HİGH","access_token":"t"}"#.as_bytes(),
3843 )
3844 .unwrap()
3845 .unwrap();
3846 let action =
3847 crate::management::policy(&cfg).error_action(&c, &ExecError::local(503, FailureScope::Request, "boom"));
3848 assert!(action.matched && action.stop);
3849 }
3850
3851 #[test]
3854 fn rules_match_the_structured_body_of_a_missing_thread() {
3855 let c = cred(
3856 "a",
3857 serde_json::json!({"request_scoped_errors":[{"status":404,"match":["not_found_error"],"action":"continue"}]}),
3858 );
3859 let e = ExecError::local(404, FailureScope::Request, cpa_exec::claude::THREAD_NOT_FOUND);
3860 let action = Policy::default().error_action(&c, &e);
3861 assert!(action.matched && !action.stop);
3862 let other = ExecError::local(404, FailureScope::Request, "not found");
3865 assert!(!Policy::default().error_action(&c, &other).matched);
3866 let mut upstream = e.clone();
3867 upstream
3868 .headers
3869 .insert(axum::http::header::CONTENT_TYPE, "text/plain".parse().unwrap());
3870 assert!(!Policy::default().error_action(&c, &upstream).matched);
3871 }
3872
3873 #[test]
3874 fn rules_are_ordered_status_and_body_matches_with_canonical_precedence() {
3875 let c = cred(
3876 "a",
3877 serde_json::json!({"disable_cooling":true, "request_scoped_errors":[
3878 {"status":429,"match":[],"action":"stop"},
3879 {"status":500,"match":["quota"],"action":"stop"},
3880 {"status":429,"match-regexr":["[", "quota [0-9]+"],"action":" CONTINUE-AND-COOLDOWN "},
3881 {"status":429,"match":["quota"],"action":"stop"}]}),
3882 );
3883 let p = Policy::default();
3884 let e = ExecError::local(429, FailureScope::Request, "quota 12");
3885 let action = p.error_action(&c, &e);
3886 assert!(!action.stop);
3887 assert!(action.cooldown && action.force_cooldown);
3888 let now = Instant::now();
3889 let mut s = Scheduler::default();
3890 s.record(&c, "m", &Outcome::Failure(e), &p, now);
3891 assert!(
3892 s.wait(&c, "m", now).is_some(),
3893 "force cooling bypasses disabled cooling"
3894 );
3895 let c = cred(
3896 "a",
3897 serde_json::json!({"request_retry":0, "request-retry":5,
3898 "disable_cooling":false,"disable-cooling":true}),
3899 );
3900 assert_eq!(p.retry_limit(&c), 0);
3901 assert!(!p.cooling_disabled(&c));
3902 let legacy = cred(
3903 "legacy",
3904 serde_json::json!({"request_retry":null,"request-retry":2,
3905 "disable_cooling":"TrUe","disable-cooling":1}),
3906 );
3907 assert_eq!(
3908 p.retry_limit(&legacy),
3909 2,
3910 "invalid canonical override permits legacy fallback"
3911 );
3912 assert!(
3913 p.cooling_disabled(&legacy),
3914 "Go ParseBool rejects mixed case; numeric legacy accepted"
3915 );
3916 assert!(!p.cooling_disabled(&cred("numeric", serde_json::json!({"disable_cooling":0}))));
3917 assert_eq!(
3918 Policy {
3919 request_retry: 3,
3920 ..Default::default()
3921 }
3922 .retry_limit(&cred("b", serde_json::json!({"request_retry":"-1"}))),
3923 3
3924 );
3925 let cfg = cpa_core::config::Config::parse(
3928 "openai-compatibility:\n - {name: skipped, base-url: '', disable-cooling: false}\n - {name: Router, base-url: http://r.invalid, disable-cooling: true}\n - {name: plain, base-url: http://p.invalid}\n",
3929 )
3930 .unwrap();
3931 let mut p = Policy {
3932 compat_disable_cooling: compat_cooling(&cfg),
3933 ..Default::default()
3934 };
3935 assert_eq!(
3936 p.compat_disable_cooling,
3937 [("Router".to_owned(), Some(true)), ("plain".to_owned(), None)],
3938 "legacy openai-compatibility is read through the v8 document"
3939 );
3940 let compat = |attrs: &[(&str, &str)], meta: Value| {
3941 let mut c = cred("compat", meta);
3942 c.provider = "openai-compatibility".into();
3943 for (k, v) in attrs {
3944 c.attributes.insert((*k).into(), (*v).into());
3945 }
3946 c
3947 };
3948 assert!(p.cooling_disabled(&compat(&[("compat_name", "router")], serde_json::json!({}))));
3949 assert!(p.cooling_disabled(&compat(&[("provider_key", "ROUTER")], serde_json::json!({}))));
3950 assert!(!p.cooling_disabled(&compat(
3951 &[("compat_name", "router")],
3952 serde_json::json!({"disable_cooling": false})
3953 )));
3954 p.disable_cooling = true;
3955 assert!(
3956 p.cooling_disabled(&compat(&[("compat_name", "plain")], serde_json::json!({}))),
3957 "an entry without disable-cooling defers to global"
3958 );
3959 p.disable_cooling = false;
3960 assert!(
3961 !p.cooling_disabled(&cred("claude-file", serde_json::json!({}))),
3962 "credentials outside OpenAI compatibility ignore entries"
3963 );
3964 for scope in [FailureScope::Request, FailureScope::Transport] {
3965 let mut s = Scheduler::default();
3966 s.record(
3967 &c,
3968 "m",
3969 &Outcome::Failure(ExecError::local(502, scope, "fault")),
3970 &p,
3971 now,
3972 );
3973 assert_eq!(s.wait(&c, "m", now), None);
3974 }
3975 }
3976
3977 const DAY: u64 = 24 * 3600;
3980
3981 fn epoch(t: SystemTime) -> String {
3982 t.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs().to_string()
3983 }
3984
3985 fn claude(now: SystemTime, weekly: u64, five_hour: &str, five_hour_reset: SystemTime) -> Windows {
3988 claude_at(now, now, weekly, five_hour, five_hour_reset)
3989 }
3990
3991 fn claude_at(
3993 observed: SystemTime,
3994 now: SystemTime,
3995 weekly: u64,
3996 five_hour: &str,
3997 five_hour_reset: SystemTime,
3998 ) -> Windows {
3999 let signals: std::collections::BTreeMap<String, String> = [
4000 ("Anthropic-Ratelimit-Unified-7d-Status", "allowed".to_owned()),
4001 ("Anthropic-Ratelimit-Unified-7d-Utilization", "0.40".to_owned()),
4002 (
4003 "Anthropic-Ratelimit-Unified-7d-Reset",
4004 epoch(observed + Duration::from_secs(weekly)),
4005 ),
4006 ("Anthropic-Ratelimit-Unified-5h-Status", five_hour.to_owned()),
4007 ("Anthropic-Ratelimit-Unified-5h-Reset", epoch(five_hour_reset)),
4008 ]
4009 .into_iter()
4010 .map(|(k, v)| (k.to_owned(), v))
4011 .collect();
4012 Windows::observed("claude", &signals, observed, now)
4013 }
4014
4015 fn soonest() -> Policy {
4016 Policy {
4017 strategy: Strategy::SoonestReset,
4018 ..Default::default()
4019 }
4020 }
4021
4022 #[test]
4025 fn soonest_reset_spends_the_earliest_weekly_window_first() {
4026 let wall = SystemTime::now();
4027 let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4028 let in_an_hour = wall + Duration::from_secs(3600);
4029 let ranks_at = |b_five_hour: &'static str, now: SystemTime| {
4031 let (wa, wb) = (
4032 claude_at(wall, now, 4 * DAY, "allowed", in_an_hour),
4033 claude_at(wall, now, DAY, b_five_hour, in_an_hour),
4034 );
4035 move |cred: &Credential| if cred.id == "a" { wa } else { wb }
4036 };
4037 let policy = soonest();
4038 let mut s = Scheduler::default();
4039 let now = Instant::now();
4040 let ranks = ranks_at("allowed", wall);
4041 for _ in 0..3 {
4042 let picked = s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &policy, &ranks, now);
4043 assert_eq!(picked.unwrap().id, "b", "the sooner weekly reset is used up first");
4044 }
4045 s.record(
4047 &b,
4048 "m",
4049 &Outcome::Failure(ExecError::local(429, FailureScope::Credential, "quota")),
4050 &policy,
4051 now,
4052 );
4053 let ready: Vec<&Credential> = [&a, &b].into_iter().filter(|x| s.wait(x, "m", now).is_none()).collect();
4054 assert_eq!(ready.iter().map(|x| x.id.as_str()).collect::<Vec<_>>(), ["a"]);
4055 assert_eq!(
4056 s.pick_ranked(&tag(&ready), &selection("m"), &policy, &ranks, now)
4057 .unwrap()
4058 .id,
4059 "a"
4060 );
4061 let mut s = Scheduler::default();
4063 let exhausted = ranks_at("rejected", wall);
4064 for _ in 0..2 {
4065 assert_eq!(
4066 s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &policy, &exhausted, now)
4067 .unwrap()
4068 .id,
4069 "a"
4070 );
4071 }
4072 let reset = ranks_at("rejected", in_an_hour + Duration::from_secs(1));
4074 assert_eq!(
4075 s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &policy, &reset, now)
4076 .unwrap()
4077 .id,
4078 "b"
4079 );
4080 }
4081
4082 #[test]
4085 fn soonest_reset_probes_unknown_accounts_once_then_ranks_them() {
4086 let wall = SystemTime::now();
4087 let creds: Vec<Credential> = ["a", "b", "c", "d"]
4088 .iter()
4089 .map(|id| cred(id, serde_json::json!({})))
4090 .collect();
4091 let [a, b, c, d] = [&creds[0], &creds[1], &creds[2], &creds[3]];
4092 let in_an_hour = wall + Duration::from_secs(3600);
4093 let known = |x: &Credential| match x.id.as_str() {
4095 "c" => claude(wall, DAY, "rejected", in_an_hour),
4096 "d" => claude(wall, 3 * DAY, "allowed", in_an_hour),
4097 _ => Windows::default(),
4098 };
4099 let policy = soonest();
4100 let mut s = Scheduler::default();
4101 let now = Instant::now();
4102 let pick = |s: &mut Scheduler, set: &[&Credential], ranks: &Ranks<'_>| {
4103 s.pick_ranked(&tag(set), &selection("m"), &policy, ranks, now)
4104 .unwrap()
4105 .id
4106 .clone()
4107 };
4108 let order: Vec<String> = (0..4).map(|_| pick(&mut s, &[a, b, c, d], &known)).collect();
4110 assert_eq!(order, ["a", "b", "d", "d"]);
4111 let learned = |x: &Credential| match x.id.as_str() {
4114 "a" => claude(wall + Duration::from_secs(1), 12 * 3600, "allowed", in_an_hour),
4115 _ => known(x),
4116 };
4117 assert_eq!(pick(&mut s, &[a, b, c, d], &learned), "a");
4118 assert_eq!(pick(&mut s, &[b, c, d], &learned), "d");
4119 assert_eq!(
4120 pick(&mut s, &[b, c], &learned),
4121 "b",
4122 "probed and unknown, still before used-up"
4123 );
4124 assert_eq!(
4125 pick(&mut s, &[c], &learned),
4126 "c",
4127 "a used-up account is the last resort"
4128 );
4129 let passed = |x: &Credential| match x.id.as_str() {
4131 "d" => claude_at(
4132 wall,
4133 wall + Duration::from_secs(3 * DAY + 1),
4134 3 * DAY,
4135 "allowed",
4136 in_an_hour,
4137 ),
4138 _ => learned(x),
4139 };
4140 assert_eq!(pick(&mut s, &[a, b, d], &passed), "d");
4141 assert_eq!(pick(&mut s, &[a, b, d], &passed), "a");
4142 }
4143
4144 #[test]
4145 fn soonest_reset_rotates_equal_ranks() {
4146 let wall = SystemTime::now();
4147 let creds: Vec<Credential> = ["a", "b"].iter().map(|id| cred(id, serde_json::json!({}))).collect();
4148 let in_an_hour = wall + Duration::from_secs(3600);
4149 let ranks = |_: &Credential| claude(wall, 2 * DAY, "allowed", in_an_hour);
4151 let mut s = Scheduler::default();
4152 let now = Instant::now();
4153 let turns: Vec<String> = (0..4)
4154 .map(|_| {
4155 s.pick_ranked(&tag(&[&creds[0], &creds[1]]), &selection("m"), &soonest(), &ranks, now)
4156 .unwrap()
4157 .id
4158 .clone()
4159 })
4160 .collect();
4161 assert_eq!(turns, ["a", "b", "a", "b"]);
4162 }
4163
4164 #[test]
4165 fn soonest_reset_never_moves_a_session_pin() {
4166 let wall = SystemTime::now();
4167 let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4168 let in_an_hour = wall + Duration::from_secs(3600);
4169 let ranks = |x: &Credential| claude(wall, if x.id == "a" { 4 * DAY } else { DAY }, "allowed", in_an_hour);
4170 let policy = Policy {
4171 session_affinity: true,
4172 ..soonest()
4173 };
4174 let mut s = Scheduler::default();
4175 let now = Instant::now();
4176 let mut pinned = selection("m");
4177 pinned.session = Some("claude:thread-1".into());
4178 assert_eq!(
4180 s.pick_ranked(&tag(&[&a]), &pinned, &policy, &ranks, now).unwrap().id,
4181 "a"
4182 );
4183 for _ in 0..2 {
4184 assert_eq!(
4185 s.pick_ranked(&tag(&[&a, &b]), &pinned, &policy, &ranks, now)
4186 .unwrap()
4187 .id,
4188 "a"
4189 );
4190 }
4191 let mut other = selection("m");
4193 other.session = Some("claude:thread-2".into());
4194 assert_eq!(
4195 s.pick_ranked(&tag(&[&a, &b]), &other, &policy, &ranks, now).unwrap().id,
4196 "b"
4197 );
4198 }
4199
4200 #[test]
4201 fn default_strategy_stays_round_robin_and_ignores_resets() {
4202 assert_eq!(Policy::default().strategy, Strategy::RoundRobin);
4203 let wall = SystemTime::now();
4204 let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4205 let ranks = |x: &Credential| claude(wall, if x.id == "a" { 4 * DAY } else { DAY }, "allowed", wall);
4206 let mut s = Scheduler::default();
4207 let now = Instant::now();
4208 let turns: Vec<String> = (0..4)
4209 .map(|_| {
4210 s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &Policy::default(), &ranks, now)
4211 .unwrap()
4212 .id
4213 .clone()
4214 })
4215 .collect();
4216 assert_eq!(turns, ["a", "b", "a", "b"]);
4217 }
4218
4219 #[test]
4220 fn soonest_reset_in_mixed_providers_stays_in_the_first_group() {
4221 let wall = SystemTime::now();
4222 let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4223 let x = cred("x", serde_json::json!({}));
4224 let in_an_hour = wall + Duration::from_secs(3600);
4225 let ranks = |c: &Credential| {
4227 let days = match c.id.as_str() {
4228 "a" => 4,
4229 "b" => 2,
4230 _ => 1,
4231 };
4232 claude(wall, days * DAY, "allowed", in_an_hour)
4233 };
4234 let sel = Selection {
4235 providers: vec!["claude".into(), "codex".into()],
4236 model: "m".into(),
4237 ..Default::default()
4238 };
4239 let candidates = [(&a, "claude"), (&b, "claude"), (&x, "codex")];
4240 let mut s = Scheduler::default();
4241 let picked = s.pick_ranked(&candidates, &sel, &soonest(), &ranks, Instant::now());
4242 assert_eq!(picked.unwrap().id, "b");
4243 }
4244
4245 #[test]
4246 fn windows_read_claude_and_codex_signals() {
4247 let now = SystemTime::now();
4248 let map = |pairs: &[(&str, String)]| -> std::collections::BTreeMap<String, String> {
4249 pairs.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect()
4250 };
4251 let stale = map(&[(
4253 "Anthropic-Ratelimit-Unified-7d-Reset",
4254 epoch(now - Duration::from_secs(60)),
4255 )]);
4256 let w = Windows::observed("claude", &stale, now, now);
4257 assert_eq!(
4258 (
4259 w.weekly_reset,
4260 w.rolled_over.map(|r| (r.window, epoch(r.at))),
4261 w.exhausted
4262 ),
4263 (
4264 None,
4265 Some((WindowId::Claude7d, epoch(now - Duration::from_secs(60)))),
4266 false
4267 )
4268 );
4269 let full = map(&[
4271 ("Anthropic-Ratelimit-Unified-5h-Utilization", "1.0".into()),
4272 (
4273 "Anthropic-Ratelimit-Unified-5h-Reset",
4274 epoch(now + Duration::from_secs(600)),
4275 ),
4276 ]);
4277 assert!(Windows::observed("claude", &full, now, now).exhausted);
4278 assert!(!Windows::observed("claude", &full, now, now + Duration::from_secs(601)).exhausted);
4279 let observed = now - Duration::from_secs(100);
4282 let codex = map(&[
4283 ("X-Codex-Primary-Used-Percent", "100".into()),
4284 ("X-Codex-Primary-Window-Minutes", "300".into()),
4285 ("X-Codex-Primary-Reset-After-Seconds", "3600".into()),
4286 ("X-Codex-Secondary-Used-Percent", "40".into()),
4287 ("X-Codex-Secondary-Window-Minutes", "10080".into()),
4288 ("X-Codex-Secondary-Reset-At", epoch(now + Duration::from_secs(2 * DAY))),
4289 ]);
4290 let w = Windows::observed("codex", &codex, observed, now);
4291 assert!(w.exhausted, "the 5-hour primary is used up");
4292 assert_eq!(
4293 w.weekly_reset.map(epoch),
4294 Some(epoch(now + Duration::from_secs(2 * DAY)))
4295 );
4296 assert!(!Windows::observed("codex", &codex, observed, observed + Duration::from_secs(3601)).exhausted);
4297 let weekly_primary = map(&[
4298 ("X-Codex-Primary-Used-Percent", "51".into()),
4299 ("X-Codex-Primary-Window-Minutes", "10080".into()),
4300 ("X-Codex-Primary-Reset-At", epoch(now + Duration::from_secs(3 * DAY))),
4301 ]);
4302 let w = Windows::observed("codex", &weekly_primary, now, now);
4303 assert_eq!(
4304 (w.weekly_reset.map(epoch), w.exhausted),
4305 (Some(epoch(now + Duration::from_secs(3 * DAY))), false)
4306 );
4307 let reached = map(&[("X-Codex-Limit-Reached", "True".into())]);
4308 assert!(Windows::observed("codex", &reached, now, now).exhausted);
4309 let w = Windows::observed("gemini", &codex, now, now);
4311 assert_eq!((w.weekly_reset, w.exhausted), (None, false));
4312 }
4313
4314 fn signals(pairs: &[(&str, String)]) -> std::collections::BTreeMap<String, String> {
4317 pairs.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect()
4318 }
4319
4320 #[test]
4324 fn soonest_reset_answers_without_a_reset_do_not_re_arm_the_probe() {
4325 let wall = SystemTime::now();
4326 let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4327 let policy = soonest();
4328 let mut s = Scheduler::default();
4329 let now = Instant::now();
4330 let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
4331 let mut picks = Vec::new();
4332 for i in 0..5u64 {
4333 let a_seen = (i > 0).then(|| {
4335 let seen = wall + Duration::from_secs(i);
4336 Windows::observed(
4337 "claude",
4338 &signals(&[("Anthropic-Ratelimit-Unified-5h-Status", "allowed".into())]),
4339 seen,
4340 seen,
4341 )
4342 });
4343 let ranks = |x: &Credential| {
4344 if x.id == "a" {
4345 a_seen.unwrap_or_default()
4346 } else {
4347 b_known
4348 }
4349 };
4350 picks.push(
4351 s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &policy, &ranks, now)
4352 .unwrap()
4353 .id
4354 .clone(),
4355 );
4356 }
4357 assert_eq!(picks, ["a", "b", "b", "b", "b"]);
4358 }
4359
4360 #[test]
4363 fn soonest_reset_a_probe_that_fails_before_headers_is_spent() {
4364 let wall = SystemTime::now();
4365 let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4366 let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
4367 let ranks = |x: &Credential| if x.id == "a" { Windows::default() } else { b_known };
4368 let mut s = Scheduler::default();
4369 let now = Instant::now();
4370 let mut pick = || {
4371 s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
4372 .unwrap()
4373 .id
4374 .clone()
4375 };
4376 assert_eq!(pick(), "a", "the probe");
4377 assert_eq!([pick(), pick()], ["b", "b"]);
4378 }
4379
4380 #[test]
4382 fn soonest_reset_re_arms_only_on_a_rollover() {
4383 let wall = SystemTime::now();
4384 let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4385 let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
4386 let a_reset = wall + Duration::from_secs(60);
4387 let a_rolled = |seen: SystemTime| {
4389 Windows::observed(
4390 "claude",
4391 &signals(&[("Anthropic-Ratelimit-Unified-7d-Reset", epoch(a_reset))]),
4392 seen,
4393 wall + Duration::from_secs(61),
4394 )
4395 };
4396 let mut s = Scheduler::default();
4397 let now = Instant::now();
4398 let mut picks = Vec::new();
4399 for i in 0..3u64 {
4400 let a_now = a_rolled(wall + Duration::from_secs(i));
4401 let ranks = |x: &Credential| if x.id == "a" { a_now } else { b_known };
4402 picks.push(
4403 s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
4404 .unwrap()
4405 .id
4406 .clone(),
4407 );
4408 }
4409 assert_eq!(picks, ["a", "b", "b"], "one probe per rollover");
4410 let next = a_reset + Duration::from_secs(7 * DAY);
4412 let later = Windows::observed(
4413 "claude",
4414 &signals(&[("Anthropic-Ratelimit-Unified-7d-Reset", epoch(next))]),
4415 wall,
4416 next + Duration::from_secs(1),
4417 );
4418 let ranks = |x: &Credential| if x.id == "a" { later } else { b_known };
4419 let mut pick = || {
4420 s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
4421 .unwrap()
4422 .id
4423 .clone()
4424 };
4425 assert_eq!([pick(), pick()], ["a", "b"]);
4426 }
4427
4428 #[test]
4431 fn soonest_reset_zero_relative_resets_do_not_re_arm_the_probe() {
4432 let wall = SystemTime::now();
4433 let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4434 let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
4435 let zero = signals(&[
4436 ("X-Codex-Secondary-Used-Percent", "0".into()),
4437 ("X-Codex-Secondary-Window-Minutes", "10080".into()),
4438 ("X-Codex-Secondary-Reset-After-Seconds", "0".into()),
4439 ]);
4440 let mut s = Scheduler::default();
4441 let now = Instant::now();
4442 let mut picks = Vec::new();
4443 for i in 0..4u64 {
4444 let a_now = (i > 0).then(|| {
4447 let seen = wall + Duration::from_secs(2 * i);
4448 Windows::observed("codex", &zero, seen, seen + Duration::from_secs(1))
4449 });
4450 let ranks = |x: &Credential| {
4451 if x.id == "a" {
4452 a_now.unwrap_or_default()
4453 } else {
4454 b_known
4455 }
4456 };
4457 picks.push(
4458 s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
4459 .unwrap()
4460 .id
4461 .clone(),
4462 );
4463 }
4464 assert_eq!(picks, ["a", "b", "b", "b"]);
4465 let used_up = signals(&[
4467 ("X-Codex-Primary-Used-Percent", "100".into()),
4468 ("X-Codex-Primary-Reset-After-Seconds", "0".into()),
4469 ]);
4470 assert!(!Windows::observed("codex", &used_up, wall, wall).exhausted);
4471 }
4472
4473 #[test]
4476 fn soonest_reset_rollovers_within_half_a_window_are_one_rollover() {
4477 let wall = SystemTime::now();
4478 let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4479 let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
4480 let codex = |seen: SystemTime, after: u64, read: SystemTime| {
4482 Windows::observed(
4483 "codex",
4484 &signals(&[
4485 ("X-Codex-Secondary-Window-Minutes", "10080".into()),
4486 ("X-Codex-Secondary-Reset-After-Seconds", after.to_string()),
4487 ]),
4488 seen,
4489 read,
4490 )
4491 };
4492 let mut s = Scheduler::default();
4493 let now = Instant::now();
4494 let mut pick = |a_now: Windows| {
4495 let ranks = |x: &Credential| if x.id == "a" { a_now } else { b_known };
4496 s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
4497 .unwrap()
4498 .id
4499 .clone()
4500 };
4501 let boundary = wall + Duration::from_secs(600);
4502 assert_eq!(pick(codex(wall, 600, boundary + Duration::from_secs(1))), "a");
4504 for i in 1..4u64 {
4506 let seen = boundary + Duration::from_secs(i);
4507 assert_eq!(pick(codex(seen, 1, seen + Duration::from_secs(2))), "b", "answer {i}");
4508 }
4509 let next = boundary + Duration::from_secs(7 * DAY);
4511 assert_eq!(
4512 pick(codex(next - Duration::from_secs(5), 5, next + Duration::from_secs(1))),
4513 "a"
4514 );
4515 assert_eq!(
4516 pick(codex(next - Duration::from_secs(5), 5, next + Duration::from_secs(1))),
4517 "b"
4518 );
4519 }
4520
4521 #[test]
4524 fn soonest_reset_probe_markers_are_kept_per_window() {
4525 let wall = SystemTime::now();
4526 let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4527 let b_known = claude(wall, 5 * DAY, "allowed", wall + Duration::from_secs(3600));
4528 let r = wall + Duration::from_secs(600);
4529 let after = |t: SystemTime, secs: u64| t + Duration::from_secs(secs);
4530 let partial = signals(&[("X-Codex-Secondary-Reset-At", epoch(r))]);
4532 let complete = signals(&[
4535 ("X-Codex-Primary-Used-Percent", "10".into()),
4536 ("X-Codex-Primary-Window-Minutes", "10080".into()),
4537 ("X-Codex-Primary-Reset-At", epoch(after(r, DAY))),
4538 ("X-Codex-Secondary-Used-Percent", "10".into()),
4539 ("X-Codex-Secondary-Window-Minutes", "300".into()),
4540 ("X-Codex-Secondary-Reset-At", epoch(after(r, 5 * 3600))),
4541 ]);
4542 let mut s = Scheduler::default();
4543 let now = Instant::now();
4544 let mut pick = |a_now: Windows| {
4545 let ranks = |x: &Credential| if x.id == "a" { a_now } else { b_known };
4546 s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
4547 .unwrap()
4548 .id
4549 .clone()
4550 };
4551 let w = Windows::observed("codex", &partial, wall, after(r, 1));
4553 assert_eq!(w.rolled_over.map(|r| r.window), Some(WindowId::CodexSecondary));
4554 assert_eq!(pick(w), "a");
4555 assert_eq!(
4557 pick(Windows::observed("codex", &complete, after(r, 1), after(r, 2))),
4558 "a"
4559 );
4560 let w = Windows::observed("codex", &complete, after(r, 1), after(r, DAY + 1));
4563 assert_eq!(w.rolled_over.map(|r| r.window), Some(WindowId::CodexPrimary));
4564 assert_eq!(pick(w), "a");
4565 assert_eq!(pick(w), "b", "probed for that rollover");
4566 }
4567
4568 #[test]
4571 fn soonest_reset_management_reset_re_arms_the_probe_and_keeps_affinity() {
4572 let wall = SystemTime::now();
4573 let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4574 let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
4575 let ranks = |x: &Credential| if x.id == "a" { Windows::default() } else { b_known };
4576 let policy = Policy {
4577 session_affinity: true,
4578 ..soonest()
4579 };
4580 let mut s = Scheduler::default();
4581 let now = Instant::now();
4582 let mut thread = selection("m");
4583 thread.session = Some("claude:thread-1".into());
4584 let mut pick = |sel: &Selection, set: &[&Credential]| {
4585 s.pick_ranked(&tag(set), sel, &policy, &ranks, now).unwrap().id.clone()
4586 };
4587 assert_eq!(pick(&thread, &[&a]), "a");
4589 let fresh = selection("m");
4590 assert_eq!(pick(&fresh, &[&a, &b]), "b");
4591 s.reset_cooldowns("a");
4592 let mut pick = |sel: &Selection, set: &[&Credential]| {
4593 s.pick_ranked(&tag(set), sel, &policy, &ranks, now).unwrap().id.clone()
4594 };
4595 assert_eq!(pick(&fresh, &[&a, &b]), "a", "probed again");
4596 assert_eq!(pick(&fresh, &[&a, &b]), "b");
4597 assert_eq!(pick(&thread, &[&a, &b]), "a", "the pin is kept");
4598 }
4599
4600 #[test]
4603 fn windows_survive_unrepresentable_window_lengths() {
4604 let now = SystemTime::now();
4605 for (used, exhausted) in [("0", false), ("100", true)] {
4606 let huge = signals(&[
4607 ("X-Codex-Primary-Used-Percent", used.into()),
4608 ("X-Codex-Primary-Window-Minutes", "1e308".into()),
4609 ]);
4610 assert_eq!(
4611 Windows::observed("codex", &huge, now, now).exhausted,
4612 exhausted,
4613 "{used}%"
4614 );
4615 let later = now + Duration::from_secs(5 * 3600 + 1);
4616 assert!(
4617 !Windows::observed("codex", &huge, now, later).exhausted,
4618 "{used}% after five hours"
4619 );
4620 }
4621 let long = signals(&[
4623 ("X-Codex-Primary-Used-Percent", "100".into()),
4624 ("X-Codex-Primary-Window-Minutes", "1e9".into()),
4625 ]);
4626 assert!(Windows::observed("codex", &long, now, now + Duration::from_secs(DAY)).exhausted);
4627 }
4628
4629 #[test]
4632 fn codex_limit_reached_expires_with_the_window_reset() {
4633 let now = SystemTime::now();
4634 let flagged = signals(&[
4635 ("X-Codex-Limit-Reached", "true".into()),
4636 ("X-Codex-Primary-Used-Percent", "100".into()),
4637 ("X-Codex-Primary-Window-Minutes", "300".into()),
4638 ("X-Codex-Primary-Reset-At", epoch(now + Duration::from_secs(60))),
4639 ]);
4640 assert!(Windows::observed("codex", &flagged, now, now).exhausted);
4641 assert!(!Windows::observed("codex", &flagged, now, now + Duration::from_secs(61)).exhausted);
4642 let below = signals(&[
4644 ("X-Codex-Limit-Reached", "true".into()),
4645 ("X-Codex-Primary-Used-Percent", "40".into()),
4646 ("X-Codex-Primary-Reset-At", epoch(now + Duration::from_secs(600))),
4647 ]);
4648 assert!(Windows::observed("codex", &below, now, now + Duration::from_secs(599)).exhausted);
4649 assert!(!Windows::observed("codex", &below, now, now + Duration::from_secs(601)).exhausted);
4650 let bare = signals(&[("X-Codex-Limit-Reached", "true".into())]);
4652 assert!(Windows::observed("codex", &bare, now, now + Duration::from_secs(5 * 3600 - 1)).exhausted);
4653 assert!(!Windows::observed("codex", &bare, now, now + Duration::from_secs(5 * 3600 + 1)).exhausted);
4654 }
4655
4656 #[test]
4659 fn codex_windows_parse_usage_reset_and_length_independently() {
4660 let now = SystemTime::now();
4661 let weekly = now + Duration::from_secs(3 * DAY);
4662 let no_usage = signals(&[
4663 ("X-Codex-Secondary-Window-Minutes", "10080".into()),
4664 ("X-Codex-Secondary-Reset-At", epoch(weekly)),
4665 ]);
4666 let w = Windows::observed("codex", &no_usage, now, now);
4667 assert_eq!((w.weekly_reset.map(epoch), w.exhausted), (Some(epoch(weekly)), false));
4668 let reset_only = signals(&[("X-Codex-Secondary-Reset-After-Seconds", "7200".into())]);
4670 let w = Windows::observed("codex", &reset_only, now, now);
4671 assert_eq!(w.weekly_reset.map(epoch), Some(epoch(now + Duration::from_secs(7200))));
4672 }
4673
4674 #[test]
4676 fn codex_primary_longer_than_secondary_is_the_weekly_window() {
4677 let now = SystemTime::now();
4678 let weekly = now + Duration::from_secs(3 * DAY);
4679 let both = signals(&[
4680 ("X-Codex-Primary-Used-Percent", "51".into()),
4681 ("X-Codex-Primary-Window-Minutes", "10080".into()),
4682 ("X-Codex-Primary-Reset-At", epoch(weekly)),
4683 ("X-Codex-Secondary-Used-Percent", "100".into()),
4684 ("X-Codex-Secondary-Window-Minutes", "300".into()),
4685 ("X-Codex-Secondary-Reset-After-Seconds", "3600".into()),
4686 ]);
4687 let w = Windows::observed("codex", &both, now, now);
4688 assert_eq!((w.weekly_reset.map(epoch), w.exhausted), (Some(epoch(weekly)), true));
4689 let w = Windows::observed("codex", &both, now, now + Duration::from_secs(3601));
4690 assert_eq!((w.weekly_reset.map(epoch), w.exhausted), (Some(epoch(weekly)), false));
4691 }
4692}