use std::collections::HashMap;
use std::time::{Duration, Instant, SystemTime};
use cpa_common::gostr::GoStr;
use cpa_core::credential::Credential;
use cpa_core::exec::ExecError;
#[cfg(test)]
use cpa_core::exec::FailureScope;
use serde::Deserialize;
use serde_json::Value;
use crate::runtime::{Outcome, Selection};
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum Strategy {
#[default]
RoundRobin,
FillFirst,
WeightedRoundRobin,
SoonestReset,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Windows {
pub weekly_reset: Option<SystemTime>,
pub rolled_over: Option<Rollover>,
pub exhausted: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum WindowId {
Claude7d,
Claude5h,
CodexPrimary,
CodexSecondary,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Rollover {
pub window: WindowId,
pub at: SystemTime,
pub gap: Duration,
}
const FALLBACK_WINDOW: Duration = Duration::from_secs(5 * 3600);
#[derive(Debug, Clone, Copy)]
struct Window {
id: WindowId,
used_up: bool,
reset: Option<SystemTime>,
stated_reset: Option<SystemTime>,
length: Option<Duration>,
}
impl Window {
fn exhausted_until(&self, observed_at: SystemTime) -> Option<SystemTime> {
if !self.used_up {
return None;
}
self.reset.or_else(|| {
self.length
.and_then(|l| observed_at.checked_add(l))
.or_else(|| observed_at.checked_add(FALLBACK_WINDOW))
})
}
}
fn window_length(minutes: f64, observed_at: SystemTime) -> Option<Duration> {
if !minutes.is_finite() || minutes <= 0.0 {
return None;
}
let length = Duration::try_from_secs_f64(minutes * 60.0).ok()?;
observed_at.checked_add(length).map(|_| length)
}
fn reset_time(raw: &str) -> Option<SystemTime> {
let raw = raw.trim();
if let Ok(seconds) = raw.parse::<f64>() {
return (seconds.is_finite() && seconds > 0.0)
.then(|| Duration::try_from_secs_f64(seconds).ok())
.flatten()
.and_then(|d| SystemTime::UNIX_EPOCH.checked_add(d));
}
chrono::DateTime::parse_from_rfc3339(raw).ok().map(SystemTime::from)
}
impl Windows {
pub fn observed(
provider: &str,
signals: &std::collections::BTreeMap<String, String>,
observed_at: SystemTime,
now: SystemTime,
) -> Self {
let get = |name: &str| {
signals
.iter()
.find(|(k, _)| k.eq_ignore_ascii_case(name))
.map(|(_, v)| v.trim())
};
let number = |name: &str| get(name).and_then(|v| v.parse::<f64>().ok()).filter(|n| n.is_finite());
let codex = provider.trim().eq_ignore_ascii_case("codex");
let (long, short) = match provider.trim().to_ascii_lowercase().as_str() {
"claude" => {
let window = |id: WindowId, label: &str, minutes: f64| Window {
id,
used_up: get(&format!("anthropic-ratelimit-unified-{label}-status"))
.is_some_and(|s| s.eq_ignore_ascii_case("rejected"))
|| number(&format!("anthropic-ratelimit-unified-{label}-utilization"))
.is_some_and(|u| u >= 1.0),
reset: get(&format!("anthropic-ratelimit-unified-{label}-reset")).and_then(reset_time),
stated_reset: get(&format!("anthropic-ratelimit-unified-{label}-reset")).and_then(reset_time),
length: window_length(minutes, observed_at),
};
(
Some(window(WindowId::Claude7d, "7d", 7.0 * 24.0 * 60.0)),
Some(window(WindowId::Claude5h, "5h", 300.0)),
)
}
"codex" => {
let window = |id: WindowId, label: &str| {
let p = format!("x-codex-{label}-");
let used = number(&format!("{p}used-percent"));
let absolute = get(&format!("{p}reset-at")).and_then(reset_time);
let after = number(&format!("{p}reset-after-seconds")).filter(|s| *s >= 0.0);
let relative = after.and_then(|a| observed_at.checked_add(Duration::try_from_secs_f64(a).ok()?));
let reset = absolute.or(relative);
let stated_reset = absolute.or(relative.filter(|_| after.is_some_and(|a| a > 0.0)));
let length = number(&format!("{p}window-minutes")).and_then(|m| window_length(m, observed_at));
(used.is_some() || reset.is_some() || length.is_some()).then_some(Window {
id,
used_up: used.is_some_and(|u| u >= 100.0),
reset,
stated_reset,
length,
})
};
let day = Duration::from_secs(24 * 3600);
match (
window(WindowId::CodexPrimary, "primary"),
window(WindowId::CodexSecondary, "secondary"),
) {
(Some(p), Some(s)) if p.length.unwrap_or(Duration::ZERO) > s.length.unwrap_or(Duration::MAX) => {
(Some(p), Some(s))
}
(Some(p), Some(s)) => (Some(s), Some(p)),
(Some(p), None) if p.length.is_some_and(|l| l >= day) => (Some(p), None),
(None, Some(s)) if s.length.is_none_or(|l| l >= day) => (Some(s), None),
(only, None) | (None, only) => (None, only),
}
}
_ => (None, None),
};
let windows = [long, short];
let until = windows
.iter()
.flatten()
.filter_map(|w| w.exhausted_until(observed_at))
.max();
let flag_until = (codex && get("x-codex-limit-reached").is_some_and(|v| v.eq_ignore_ascii_case("true")))
.then(|| {
until
.or_else(|| windows.iter().flatten().filter_map(|w| w.reset).min())
.or_else(|| observed_at.checked_add(FALLBACK_WINDOW))
})
.flatten();
let long_reset = long.and_then(|w| w.reset);
Self {
weekly_reset: long_reset.filter(|r| *r > now),
rolled_over: long.and_then(|w| {
let at = w.stated_reset.filter(|r| *r <= now)?;
Some(Rollover {
window: w.id,
at,
gap: w.length.unwrap_or(FALLBACK_WINDOW) / 2,
})
}),
exhausted: until.max(flag_until).is_some_and(|u| u > now),
}
}
fn order(&self, probed: bool) -> (bool, u8, Option<SystemTime>) {
let class = match (self.weekly_reset, probed) {
(Some(_), _) => 1,
(None, false) => 0,
(None, true) => 2,
};
(self.exhausted, class, self.weekly_reset)
}
}
type Probes = HashMap<WindowId, SystemTime>;
fn probed(record: Option<&Probes>, rolled_over: Option<Rollover>) -> bool {
match (record, rolled_over) {
(None, _) => false,
(Some(_), None) => true,
(Some(markers), Some(r)) => markers
.get(&r.window)
.is_some_and(|at| at.checked_add(r.gap).is_none_or(|limit| r.at <= limit)),
}
}
pub type Ranks<'a> = dyn Fn(&Credential) -> Windows + 'a;
#[derive(Debug, Clone)]
pub struct Policy {
pub strategy: Strategy,
pub request_retry: usize,
pub max_retry_credentials: usize,
pub max_retry_interval: Duration,
pub disable_cooling: bool,
pub transient_error_cooldown_seconds: i64,
pub session_affinity: bool,
pub session_affinity_ttl: Duration,
pub session_affinity_subagents: bool,
pub save_cooldown_status: bool,
pub max_trusted_cooldown: Duration,
pub force_model_prefix: bool,
pub compat_disable_cooling: Vec<(String, Option<bool>)>,
pub oauth_request_scoped_errors: HashMap<String, Vec<ErrorRule>>,
}
impl Default for Policy {
fn default() -> Self {
Self {
strategy: Strategy::RoundRobin,
request_retry: 0,
max_retry_credentials: 0,
max_retry_interval: Duration::ZERO,
disable_cooling: false,
transient_error_cooldown_seconds: 0,
session_affinity: false,
session_affinity_ttl: Duration::from_secs(3600),
session_affinity_subagents: true,
save_cooldown_status: false,
max_trusted_cooldown: DEFAULT_TRUSTED_COOLDOWN,
force_model_prefix: false,
compat_disable_cooling: Vec::new(),
oauth_request_scoped_errors: HashMap::new(),
}
}
}
#[derive(Default)]
pub struct RawRouting<'a> {
pub strategy: &'a str,
pub force_model_prefix: bool,
pub session_affinity: bool,
pub session_affinity_ttl: &'a str,
pub session_affinity_subagents: Option<bool>,
pub request_retry: i64,
pub max_retry_credentials: i64,
pub max_retry_interval: i64,
pub disable_cooling: bool,
pub save_cooldown_status: bool,
pub transient_error_cooldown_seconds: i64,
pub max_trusted_cooldown: &'a str,
}
impl From<&cpa_core::config::RoutingConfig> for Policy {
fn from(r: &cpa_core::config::RoutingConfig) -> Self {
normalize(RawRouting {
strategy: &r.strategy,
force_model_prefix: r.force_model_prefix,
session_affinity: r.session_affinity,
session_affinity_ttl: &r.session_affinity_ttl,
session_affinity_subagents: r.session_affinity_subagents,
request_retry: r.retry.request_retry,
max_retry_credentials: r.retry.max_retry_credentials,
max_retry_interval: r.retry.max_retry_interval,
disable_cooling: r.cooldown.disable_cooling,
save_cooldown_status: r.cooldown.save_cooldown_status,
transient_error_cooldown_seconds: r.cooldown.transient_error_cooldown_seconds,
max_trusted_cooldown: &r.cooldown.max_trusted_cooldown,
})
}
}
pub fn normalize(raw: RawRouting<'_>) -> Policy {
let ttl = cpa_core::config::parse_duration(raw.session_affinity_ttl.trim())
.filter(|d| *d > 0)
.map(|n| Duration::from_nanos(n as u64).max(Duration::from_secs(1)))
.unwrap_or(Duration::from_secs(3600));
Policy {
strategy: match raw.strategy.trim().to_ascii_lowercase().as_str() {
"weighted-round-robin" | "weightedroundrobin" | "wrr" => Strategy::WeightedRoundRobin,
"fill-first" | "fillfirst" | "ff" => Strategy::FillFirst,
"soonest-reset" | "soonestreset" | "reset-first" | "resetfirst" => Strategy::SoonestReset,
_ => Strategy::RoundRobin,
},
force_model_prefix: raw.force_model_prefix,
session_affinity: raw.session_affinity,
session_affinity_ttl: ttl,
session_affinity_subagents: !raw.session_affinity || raw.session_affinity_subagents.unwrap_or(true),
request_retry: raw.request_retry.max(0) as usize,
max_retry_credentials: raw.max_retry_credentials.max(0) as usize,
max_retry_interval: Duration::from_secs(raw.max_retry_interval.max(0) as u64),
disable_cooling: raw.disable_cooling,
save_cooldown_status: raw.save_cooldown_status,
transient_error_cooldown_seconds: raw.transient_error_cooldown_seconds,
max_trusted_cooldown: trusted_cooldown(raw.max_trusted_cooldown),
..Policy::default()
}
}
const DEFAULT_TRUSTED_COOLDOWN: Duration = Duration::from_secs(3600);
fn trusted_cooldown(raw: &str) -> Duration {
let raw = raw.trim();
if raw.is_empty() {
return DEFAULT_TRUSTED_COOLDOWN;
}
let nanos = match raw.parse::<i64>() {
Ok(seconds) => seconds.saturating_mul(1_000_000_000),
Err(_) => match cpa_core::config::parse_duration(raw) {
Some(nanos) => nanos,
None => {
static WARNED: std::sync::Mutex<String> = std::sync::Mutex::new(String::new());
let mut last = WARNED.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if *last != raw {
raw.clone_into(&mut last);
tracing::warn!(
"routing.cooldown.max-trusted-cooldown {raw:?} is not a duration (\"90m\") or a \
number of seconds; using 1h"
);
}
return DEFAULT_TRUSTED_COOLDOWN;
}
},
};
match nanos {
..=0 => Duration::ZERO,
n => Duration::from_nanos(n as u64).max(Duration::from_secs(10)),
}
}
#[derive(Debug, Clone, Deserialize)]
pub struct ErrorRule {
#[serde(default)]
pub status: i64,
#[serde(default)]
pub r#match: Vec<String>,
#[serde(default, rename = "match-regexr")]
pub match_regex: Vec<String>,
#[serde(default)]
pub action: String,
}
#[derive(Debug, Clone, Copy)]
pub struct ErrorAction {
pub matched: bool,
pub stop: bool,
pub cooldown: bool,
pub force_cooldown: bool,
}
fn metadata<'a>(c: &'a Credential, key: &str) -> Option<&'a Value> {
c.metadata
.get(key)
.or_else(|| c.metadata.get(key.replace('_', "-").as_str()))
}
pub(crate) fn priority(c: &Credential) -> i64 {
integer(c, "priority").unwrap_or(0)
}
fn integer(c: &Credential, key: &str) -> Option<i64> {
c.attributes
.get(key)
.filter(|_| key == "priority")
.and_then(|s| s.trim().parse().ok())
.or_else(|| {
[key.to_owned(), key.replace('_', "-")].iter().find_map(|key| {
let v = c.metadata.get(key)?;
v.as_i64()
.or_else(|| v.as_str()?.trim().parse().ok())
.or_else(|| v.as_f64().map(|n| n as i64))
})
})
}
fn boolean(c: &Credential, key: &str) -> Option<bool> {
let parse = |s: &str| match s.trim() {
"true" | "True" | "TRUE" | "1" | "t" | "T" => Some(true),
"false" | "False" | "FALSE" | "0" | "f" | "F" => Some(false),
_ => None,
};
[key.to_owned(), key.replace('_', "-")].iter().find_map(|key| {
let v = c.metadata.get(key)?;
v.as_bool()
.or_else(|| parse(v.as_str()?))
.or_else(|| v.as_f64().map(|n| n != 0.0))
})
}
fn oauth(c: &Credential) -> bool {
c.str("api_key").is_none() && !c.attributes.contains_key("api_key")
}
impl Policy {
pub fn retry_limit(&self, c: &Credential) -> usize {
integer(c, "request_retry")
.filter(|n| *n >= 0)
.map(|n| n as usize)
.unwrap_or(self.request_retry)
}
pub(crate) fn cooling_disabled(&self, c: &Credential) -> bool {
boolean(c, "disable_cooling")
.or_else(|| self.provider_cooling(c))
.unwrap_or(self.disable_cooling)
}
fn provider_cooling(&self, c: &Credential) -> Option<bool> {
let provider = c.provider.trim().go_lower();
let attr = |k: &str| c.attributes.get(k).map(|v| v.trim()).unwrap_or_default();
let (compat_name, mut provider_key) = (attr("compat_name"), attr("provider_key"));
if provider.is_empty()
|| (provider_key.is_empty() && compat_name.is_empty() && provider != "openai-compatibility")
{
return None;
}
if provider_key.is_empty() {
provider_key = &provider;
}
let candidates = [compat_name, provider_key, provider.as_str()];
self.compat_disable_cooling
.iter()
.find(|(name, _)| candidates.iter().any(|c| !c.is_empty() && c.go_eq_fold(name)))
.and_then(|(_, disable)| *disable)
}
pub fn error_action(&self, c: &Credential, error: &ExecError) -> ErrorAction {
let file_rules: Option<Vec<ErrorRule>> =
metadata(c, "request_scoped_errors").and_then(|v| serde_json::from_value(v.clone()).ok());
let rules = file_rules.as_ref().filter(|r| !r.is_empty()).or_else(|| {
oauth(c)
.then(|| self.oauth_request_scoped_errors.get(&c.provider.trim().go_lower()))
.flatten()
});
if let Some(rules) = rules {
let body = crate::classify::rule_text(error);
let status = i64::from(crate::classify::go_status(error));
for rule in rules {
if rule.status <= 0 || rule.status != status {
continue;
}
let matches = rule.r#match.iter().any(|s| !s.is_empty() && body.contains(s))
|| rule
.match_regex
.iter()
.any(|s| !s.is_empty() && regex::Regex::new(s).is_ok_and(|re| re.is_match(&body)));
if !matches {
continue;
}
let (stop, cooldown) = match rule.action.trim().to_ascii_lowercase().as_str() {
"stop" => (true, false),
"stop-and-cooldown" => (true, true),
"continue" => (false, false),
"continue-and-cooldown" => (false, true),
_ => continue,
};
return ErrorAction {
matched: true,
stop,
cooldown,
force_cooldown: cooldown,
};
}
}
let request = crate::classify::is_request_invalid(error);
ErrorAction {
matched: false,
stop: request,
cooldown: !crate::classify::skips_cooldown(error),
force_cooldown: false,
}
}
}
pub fn retry_status(status: u16) -> bool {
matches!(status, 403 | 408 | 429 | 500 | 502 | 503 | 504)
}
fn weight(c: &Credential) -> i64 {
let value = if let Some(s) = c.attributes.get("weight").filter(|s| !s.trim().is_empty()) {
s.trim().parse::<i64>().ok()
} else if let Some(v) = metadata(c, "weight") {
v.as_i64().or_else(|| v.as_str()?.trim().parse().ok()).or_else(|| {
let n = v.as_f64()?;
(n.fract() == 0.0 && n <= 1_000_000.0).then_some(n as i64)
})
} else {
Some(1)
};
value.filter(|n| *n <= 1_000_000).unwrap_or(0).max(0)
}
pub use cpa_core::registry::dynamic::canonical_model;
#[derive(Default)]
struct Rotation {
last: String,
weights: HashMap<String, i64>,
current: HashMap<String, i64>,
}
pub(crate) struct Cooldown {
pub deadline: Instant,
pub level: u32,
pub status: u16,
pub quota: bool,
pub error: String,
pub since: SystemTime,
pub credential: bool,
pub trust: Trust,
pub quota_kind: Option<QuotaKind>,
pub quota_under: Option<(Instant, QuotaKind)>,
pub recorded: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct QuotaKind {
pub status: u16,
pub reason: &'static str,
}
impl Cooldown {
fn quota_block(&self, now: Instant) -> Option<(Instant, QuotaKind)> {
let block = if self.quota {
self.quota_kind.map(|k| (self.deadline, k))
} else {
self.quota_under
};
block.filter(|(end, _)| *end > now || self.trust.stated.is_some_and(|r| r > now))
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) struct Trust {
pub window: Option<u32>,
pub generation: u32,
pub bounded: bool,
pub accepted: bool,
pub stated: Option<Instant>,
pub probe: Option<Instant>,
}
impl Trust {
fn accept(&mut self) {
self.bounded = false;
self.accepted = true;
self.probe = None;
}
}
const PROBE_HOLD: Duration = Duration::from_secs(30);
const MAX_TRUST_WINDOW: u32 = 20;
impl Cooldown {
fn until(&self, now: Instant) -> Option<Instant> {
let live = (self.deadline > now).then_some(self.deadline);
live.max(self.trust.probe.filter(|p| *p > now))
}
fn newer_than(&self, picked: u32) -> bool {
(self.trust.bounded || self.trust.accepted) && (self.trust.generation.wrapping_sub(picked) as i32) > 0
}
}
#[derive(Default)]
pub(crate) struct Scheduler {
rotations: HashMap<(String, String), Rotation>,
cursors: HashMap<(String, String), usize>,
pub(crate) cooldowns: HashMap<(String, String), Cooldown>,
affinity: crate::affinity::Cache,
lcp: crate::lcp::Matcher,
probes: HashMap<String, Probes>,
affinity_off: bool,
generation: u32,
dispatched: u64,
}
pub fn compat_cooling(cfg: &cpa_core::config::Config) -> Vec<(String, Option<bool>)> {
cfg.document
.get("api-keys")
.and_then(|k| k.get("openai-compatibility"))
.and_then(serde_yaml_ng::Value::as_sequence)
.into_iter()
.flatten()
.filter(|g| {
g.get("base-url")
.and_then(serde_yaml_ng::Value::as_str)
.is_some_and(|b| !b.trim().is_empty())
&& g.get("disabled").and_then(serde_yaml_ng::Value::as_bool) != Some(true)
})
.map(|g| {
(
g.get("name")
.and_then(serde_yaml_ng::Value::as_str)
.unwrap_or_default()
.trim()
.to_owned(),
g.get("disable-cooling").and_then(serde_yaml_ng::Value::as_bool),
)
})
.collect()
}
struct SessionKeys {
primary: crate::affinity::Key,
fallback: Option<crate::affinity::Key>,
parent: String,
}
fn session_keys(selection: &Selection) -> Option<SessionKeys> {
let primary = selection.session.as_deref().filter(|s| !s.is_empty())?;
let scope = selection.provider_keys().join(",");
let model = canonical_model(&selection.model).to_owned();
let parent = selection.session_parent.clone().unwrap_or_default();
let fallback = (!parent.is_empty() && parent != primary).then(|| (scope.clone(), model.clone(), parent.clone()));
Some(SessionKeys {
primary: (scope, model, primary.to_owned()),
fallback,
parent,
})
}
fn quota_backoff(level: u32) -> (Duration, u32) {
let seconds = (1u64 << level.min(11)).min(1800);
if seconds >= 1800 {
(Duration::from_secs(1800), level)
} else {
(Duration::from_secs(seconds), level + 1)
}
}
fn bound(stated: Duration, cap: Duration, window: Option<u32>) -> Option<(Duration, u32)> {
if cap.is_zero() {
return None;
}
let step = window.map_or(0, |w| w.saturating_add(1)).min(MAX_TRUST_WINDOW);
let bound = cap.saturating_mul(1 << step);
(stated > bound).then_some((bound, step))
}
fn transient(policy: &Policy, hint: Option<Duration>, disabled: bool) -> Option<Duration> {
if disabled || policy.transient_error_cooldown_seconds < 0 {
return None;
}
if let Some(hint) = hint.filter(|d| !d.is_zero()) {
return Some(hint);
}
Some(Duration::from_secs(match policy.transient_error_cooldown_seconds {
0 => 60,
s => s as u64,
}))
}
impl Scheduler {
pub fn configure(&mut self, previous: &Policy, next: &Policy) {
self.affinity_off = !next.session_affinity;
if previous.strategy != next.strategy
|| previous.session_affinity != next.session_affinity
|| previous.session_affinity_ttl != next.session_affinity_ttl
|| previous.session_affinity_subagents != next.session_affinity_subagents
{
self.rotations.clear();
self.cursors.clear();
self.affinity.clear();
self.lcp.reset(crate::lcp::Limits {
ttl: next.session_affinity_ttl,
..Default::default()
});
self.probes.clear();
}
}
fn affinity(&self, policy: &Policy) -> bool {
policy.session_affinity && !self.affinity_off
}
pub fn quota_cooling(&self, c: &Credential, model: &str, now: Instant) -> bool {
let model = canonical_model(model);
[model, ""].into_iter().any(|m| {
self.cooldowns
.get(&(c.id.clone(), m.to_owned()))
.is_some_and(|s| s.quota && s.until(now).is_some())
})
}
pub fn wait(&self, c: &Credential, model: &str, now: Instant) -> Option<Duration> {
let model = canonical_model(model);
[model, ""]
.into_iter()
.filter_map(|model| {
self.cooldowns
.get(&(c.id.clone(), model.to_owned()))
.and_then(|s| s.until(now)?.checked_duration_since(now))
.filter(|d| !d.is_zero())
})
.max()
}
pub fn cooling(&self, c: &Credential, model: &str, now: Instant) -> bool {
let model = canonical_model(model);
[model, ""].into_iter().any(|m| {
self.cooldowns
.get(&(c.id.clone(), m.to_owned()))
.is_some_and(|s| s.deadline > now)
})
}
pub fn last_error(&self, c: &Credential, model: &str) -> Option<&Cooldown> {
let model = canonical_model(model);
self.cooldowns
.get(&(c.id.clone(), model.to_owned()))
.or_else(|| self.cooldowns.get(&(c.id.clone(), String::new())))
}
pub fn retry_eligible(&self, c: &Credential, model: &str, now: Instant) -> bool {
let model = canonical_model(model);
[model, ""].into_iter().all(|model| {
self.cooldowns.get(&(c.id.clone(), model.to_owned())).is_none_or(|s| {
let probing = s.trust.probe.is_some_and(|p| p > now);
!probing && (s.deadline <= now || (retry_status(s.status) && !s.trust.bounded))
})
})
}
#[cfg(test)]
pub fn accept_probe(&mut self, c: &Credential, model: &str, now: Instant) {
self.accept_probe_picked(c, model, self.generation, now);
}
pub fn accept_probe_picked(&mut self, c: &Credential, model: &str, picked: u32, now: Instant) {
let model = canonical_model(model);
let mut credential_window = None;
for m in [model, ""] {
if let Some(s) = self.cooldowns.get_mut(&(c.id.clone(), m.to_owned()))
&& s.trust.bounded
&& s.deadline <= now
&& !s.newer_than(picked)
{
s.trust.accept();
if m.is_empty() {
credential_window = Some(s.deadline);
}
}
}
if let Some(ended) = credential_window {
for ((id, m), s) in &mut self.cooldowns {
if *id == c.id && !m.is_empty() && s.trust.bounded && s.deadline == ended && !s.newer_than(picked) {
s.trust.accept();
}
}
}
}
pub fn outside_selection_generation(&self) -> u32 {
self.generation.wrapping_sub(1 << 30)
}
pub fn reserve_probe(&mut self, c: &Credential, model: &str, now: Instant) -> u32 {
let model = canonical_model(model);
let next = self.generation.wrapping_add(1);
for m in [model, ""] {
if let Some(s) = self.cooldowns.get_mut(&(c.id.clone(), m.to_owned()))
&& s.trust.bounded
&& s.deadline <= now
{
s.trust.probe = now.checked_add(PROBE_HOLD);
s.trust.generation = next;
self.generation = next;
}
}
self.generation
}
#[cfg(test)]
pub fn pick<'a>(
&mut self,
candidates: &[(&'a Credential, &str)],
selection: &Selection,
policy: &Policy,
now: Instant,
) -> Option<&'a Credential> {
self.pick_ranked(candidates, selection, policy, &|_| Windows::default(), now)
}
#[cfg(test)]
pub fn pick_ranked<'a>(
&mut self,
candidates: &[(&'a Credential, &str)],
selection: &Selection,
policy: &Policy,
ranks: &Ranks<'_>,
now: Instant,
) -> Option<&'a Credential> {
self.pick_session(candidates, selection, policy, ranks, now)
.map(|(c, _)| c)
}
pub fn pick_session<'a>(
&mut self,
candidates: &[(&'a Credential, &str)],
selection: &Selection,
policy: &Policy,
ranks: &Ranks<'_>,
now: Instant,
) -> Option<(&'a Credential, Option<crate::lcp::Match>)> {
if candidates.is_empty() {
return None;
}
if let Some(request) = selection.lcp.as_deref().filter(|_| self.affinity(policy)) {
let namespace = request.namespace(&selection.model);
if let Some(found) = self.lcp.find(&namespace, request.prepared(), now)
&& let Some((c, _)) = candidates.iter().find(|(c, _)| c.id == found.auth)
{
return Some((c, Some(found)));
}
let picked = self.pick_unbound(candidates, selection, policy.strategy, ranks)?;
let bound = self.lcp.bind(&namespace, request.prepared(), &picked.id, now);
return Some((picked, bound));
}
self.pick_legacy(candidates, selection, policy, ranks, now)
.map(|c| (c, None))
}
fn pick_legacy<'a>(
&mut self,
candidates: &[(&'a Credential, &str)],
selection: &Selection,
policy: &Policy,
ranks: &Ranks<'_>,
now: Instant,
) -> Option<&'a Credential> {
let ttl = policy.session_affinity_ttl;
let Some(keys) = self.affinity(policy).then(|| session_keys(selection)).flatten() else {
return self.pick_unbound(candidates, selection, policy.strategy, ranks);
};
self.affinity.sweep(now, ttl);
let fork = selection.session_fork;
let subagent = !fork && cpa_common::session::is_subagent_session(&keys.primary.2, &keys.parent);
let bind_keys = match &keys.fallback {
Some(fallback) if !subagent && !fork => vec![keys.primary.clone(), fallback.clone()],
_ => vec![keys.primary.clone()],
};
let find = |id: &str| candidates.iter().find(|(c, _)| c.id == id).map(|(c, _)| *c);
let reuse = match self.affinity.get_and_refresh(&keys.primary, now, ttl) {
Some(id) => find(&id),
None => keys
.fallback
.as_ref()
.and_then(|fallback| self.affinity.get(fallback, now))
.filter(|_| !subagent || policy.session_affinity_subagents)
.and_then(|id| find(&id)),
};
let picked = match reuse {
Some(c) => c,
None => self.pick_unbound(candidates, selection, policy.strategy, ranks)?,
};
self.affinity.bind(&picked.id, &bind_keys, now, ttl);
Some(picked)
}
pub fn session_result(
&mut self,
c: &Credential,
selection: &Selection,
outcome: &Outcome,
policy: &Policy,
lcp: Option<&crate::lcp::Match>,
now: Instant,
) {
if !self.affinity(policy) {
return;
}
let success = match outcome {
Outcome::Success => true,
Outcome::Failure(error) if policy.error_action(c, error).cooldown => false,
_ => return,
};
if let Some(request) = selection.lcp.as_deref() {
let namespace = request.namespace(&selection.model);
if success {
self.lcp.touch(&namespace, request.prepared(), &c.id, now);
} else {
let generation = lcp.map_or(0, |m| m.access);
self.lcp
.remove(&namespace, &request.prepared().fingerprints, &c.id, generation, now);
}
if lcp.is_some() {
return;
}
}
let Some(keys) = session_keys(selection) else {
return;
};
let mut targets = vec![keys.primary.clone()];
if let Some(fallback) = keys.fallback
&& !cpa_common::session::is_subagent_session(&keys.primary.2, &keys.parent)
{
targets.push(fallback);
}
for key in &targets {
if success {
self.affinity.touch(key, &c.id, now, policy.session_affinity_ttl);
} else {
self.affinity.compare_and_delete(key, &c.id);
}
}
}
pub fn pick_builtin<'a>(
&mut self,
candidates: &[(&'a Credential, &str)],
selection: &Selection,
strategy: Strategy,
ranks: &Ranks<'_>,
) -> Option<&'a Credential> {
self.pick_unbound(candidates, selection, strategy, ranks)
}
fn pick_unbound<'a>(
&mut self,
candidates: &[(&'a Credential, &str)],
selection: &Selection,
strategy: Strategy,
ranks: &Ranks<'_>,
) -> Option<&'a Credential> {
let model = canonical_model(&selection.model).to_owned();
let provider_keys = selection.provider_keys();
let providers_key = provider_keys.join(",");
let tier = candidates
.iter()
.map(|(c, _)| integer(c, "priority").unwrap_or(0))
.max()?;
let mut ready: Vec<(&Credential, &str)> = candidates
.iter()
.copied()
.filter(|(c, _)| integer(c, "priority").unwrap_or(0) == tier)
.collect();
ready.sort_by(|a, b| a.0.id.cmp(&b.0.id));
let mut groups: Vec<(&str, Vec<&Credential>)> = Vec::new();
for provider in provider_keys
.iter()
.map(String::as_str)
.chain(ready.iter().map(|(_, p)| *p))
{
if groups.iter().any(|(p, _)| *p == provider) {
continue;
}
let members: Vec<&Credential> = ready.iter().filter(|(_, p)| *p == provider).map(|(c, _)| *c).collect();
if !members.is_empty() {
groups.push((provider, members));
}
}
if self.rotations.len() >= 4096 {
self.rotations.clear();
self.cursors.clear();
}
let picked = if groups.len() == 1 {
let (provider, members) = &groups[0];
self.pick_within(provider, &model, members, strategy, ranks)
} else {
match strategy {
Strategy::FillFirst => groups[0].1[0],
Strategy::SoonestReset => {
let (provider, members) = &groups[0];
self.pick_within(provider, &model, members, Strategy::SoonestReset, ranks)
}
Strategy::WeightedRoundRobin => {
let all: Vec<&Credential> = ready.iter().map(|(c, _)| *c).collect();
self.pick_within(&providers_key, &model, &all, Strategy::WeightedRoundRobin, ranks)
}
Strategy::RoundRobin => {
let total: usize = groups.iter().map(|(_, m)| m.len()).sum();
let cursor = self.cursors.entry((providers_key.clone(), model.clone())).or_default();
let slot = *cursor % total;
*cursor = slot + 1;
let mut start = 0;
let mut index = 0;
for (i, (_, members)) in groups.iter().enumerate() {
if slot < start + members.len() {
index = i;
break;
}
start += members.len();
}
let (provider, members) = &groups[index];
self.pick_within(provider, &model, members, Strategy::RoundRobin, ranks)
}
}
};
Some(picked)
}
fn pick_within<'a>(
&mut self,
scope: &str,
model: &str,
members: &[&'a Credential],
strategy: Strategy,
ranks: &Ranks<'_>,
) -> &'a Credential {
let state = self.rotations.entry((scope.to_owned(), model.to_owned())).or_default();
match strategy {
Strategy::FillFirst => members[0],
Strategy::SoonestReset => {
let probes = &mut self.probes;
let ranked: Vec<_> = members
.iter()
.map(|c| {
let windows = ranks(c);
let order = windows.order(probed(probes.get(&c.id), windows.rolled_over));
(order, windows.rolled_over, *c)
})
.collect();
let best = ranked.iter().map(|(order, ..)| *order).min().expect("members");
let tied: Vec<_> = ranked.iter().filter(|(o, ..)| *o == best).collect();
let (order, rolled_over, picked) = *tied
.iter()
.find(|(.., c)| c.id > state.last)
.copied()
.unwrap_or(tied[0]);
if order.1 == 0 {
let markers = probes.entry(picked.id.clone()).or_default();
if let Some(r) = rolled_over {
markers.insert(r.window, r.at);
}
}
state.last.clone_from(&picked.id);
picked
}
Strategy::RoundRobin => {
let picked = members
.iter()
.find(|c| c.id > state.last)
.copied()
.unwrap_or(members[0]);
state.last.clone_from(&picked.id);
picked
}
Strategy::WeightedRoundRobin => {
if members
.iter()
.any(|c| state.weights.get(&c.id).is_some_and(|w| *w != weight(c)))
{
state.current.clear();
}
if state.weights.len() > 1024 || state.current.len() > 1024 {
state.weights.retain(|id, _| members.iter().any(|c| c.id == *id));
state.current.retain(|id, _| members.iter().any(|c| c.id == *id));
}
let mut total = 0i64;
let mut best = i64::MIN;
let mut picked = members[0];
for c in members {
let w = weight(c);
state.weights.insert(c.id.clone(), w);
let current = state.current.entry(c.id.clone()).or_default();
*current = current.saturating_add(w);
total = total.saturating_add(w);
if *current > best {
best = *current;
picked = c;
}
}
let current = state.current.get_mut(&picked.id).unwrap();
*current = current.saturating_sub(total);
picked
}
}
}
pub fn admits(&self, c: &Credential, policy: &Policy) -> bool {
policy.strategy != Strategy::WeightedRoundRobin || weight(c) > 0
}
#[cfg(test)]
pub fn record(&mut self, c: &Credential, model: &str, outcome: &Outcome, policy: &Policy, now: Instant) {
self.record_picked(c, model, outcome, policy, self.generation, now);
}
#[cfg(test)]
pub fn record_picked(
&mut self,
c: &Credential,
model: &str,
outcome: &Outcome,
policy: &Policy,
picked: u32,
now: Instant,
) {
self.record_dispatched(c, model, outcome, policy, picked, 0, now);
}
pub fn dispatch(&mut self) -> u64 {
self.dispatched += 1;
self.dispatched
}
#[allow(clippy::too_many_arguments)]
pub fn record_dispatched(
&mut self,
c: &Credential,
model: &str,
outcome: &Outcome,
policy: &Policy,
picked: u32,
seq: u64,
now: Instant,
) {
let model = canonical_model(model);
self.apply(c, model, outcome, policy, picked, seq, now);
if let Outcome::Failure(error) = outcome {
let action = policy.error_action(c, error);
if action.cooldown || model.is_empty() {
let disabled = policy.cooling_disabled(c) && !action.force_cooldown;
let all = crate::classify::credential_scoped(error) && !disabled;
for ((id, m), s) in &mut self.cooldowns {
if *id == c.id && (all || m == model) {
s.recorded = self.dispatched;
}
}
}
}
}
#[allow(clippy::too_many_arguments)]
fn apply(
&mut self,
c: &Credential,
model: &str,
outcome: &Outcome,
policy: &Policy,
picked: u32,
seq: u64,
now: Instant,
) {
let key = (c.id.clone(), model.to_owned());
for m in [model, ""] {
if let Some(s) = self.cooldowns.get_mut(&(c.id.clone(), m.to_owned()))
&& !s.newer_than(picked)
{
s.trust.probe = None;
}
}
let error = match outcome {
Outcome::Success => {
let global = (c.id.clone(), String::new());
if self.cooldowns.get(&global).is_some_and(|s| s.quota && s.deadline > now) {
return;
}
if seq > 0
&& self
.cooldowns
.get(&key)
.is_some_and(|s| s.deadline > now && s.recorded >= seq)
{
return;
}
if let Some(ended) = self
.cooldowns
.get(&global)
.filter(|s| s.deadline <= now && !s.newer_than(picked))
.map(|s| s.deadline)
{
self.cooldowns.remove(&global);
for ((id, _), s) in &mut self.cooldowns {
if *id == c.id
&& s.deadline <= now
&& (!s.trust.bounded || s.deadline == ended)
&& !s.newer_than(picked)
{
s.trust = Trust::default();
}
}
}
if !self.cooldowns.get(&key).is_some_and(|s| s.newer_than(picked)) {
self.cooldowns.remove(&key);
}
return;
}
Outcome::Cancelled | Outcome::Neutral(_) => return,
Outcome::Failure(error) => error,
};
let action = policy.error_action(c, error);
if !action.cooldown {
return;
}
let credential_quota = crate::classify::credential_scoped(error);
let key = if credential_quota {
(c.id.clone(), String::new())
} else {
key
};
let disabled = policy.cooling_disabled(c) && !action.force_cooldown;
let status = crate::classify::go_status(error);
let text = crate::classify::error_text(error);
let prev = self.cooldowns.get(&key);
if prev.is_some_and(|s| s.deadline <= now && s.newer_than(picked)) {
return;
}
let prev_deadline = prev.map(|s| s.deadline);
let prev_live = prev.filter(|s| s.deadline > now).map(|s| s.deadline);
let prev_trust = prev.map(|s| s.trust).unwrap_or_default();
let quota_under = prev.and_then(|p| p.quota_block(now));
let mut level = prev.filter(|s| s.quota).map(|s| s.level).unwrap_or(0);
let mut trust = Trust {
window: prev_trust.window,
..Trust::default()
};
let hint = error.retry_after.filter(|d| !d.is_zero());
let mut quota = false;
let mut cloudflare = false;
let duration = if crate::classify::is_model_support(status, &text) {
(!disabled).then(|| hint.unwrap_or(Duration::from_secs(43200)))
} else if crate::classify::is_cloudflare(status, &text) {
quota = true;
cloudflare = true;
(!disabled).then(|| {
let (d, next) = quota_backoff(level);
level = next;
d.max(Duration::from_secs(10))
})
} else if crate::classify::is_invalid_grant(status, &text) {
(!disabled).then_some(Duration::from_secs(1800))
} else {
match status {
401..=403 => (!disabled).then_some(Duration::from_secs(1800)),
404 => (!disabled).then(|| hint.unwrap_or(Duration::from_secs(43200))),
429 => {
quota = true;
if credential_quota && prev.is_none_or(|s| !s.quota) {
level = 0;
}
if disabled {
None
} else if let Some(hint) = error.retry_after {
let stated = hint.max(Duration::from_secs(10));
let cap = policy.max_trusted_cooldown;
if !cap.is_zero()
&& let Some(window) = prev.filter(|s| s.trust.bounded && s.deadline > now)
{
let reset = now.checked_add(stated);
let old = window.deadline;
let deadline = reset.map_or(old, |r| old.min(r));
let mut window_trust = window.trust;
if let Some(slot) = self.cooldowns.get_mut(&key) {
slot.deadline = deadline;
slot.error = text;
slot.trust.stated = reset.or(slot.trust.stated);
window_trust = slot.trust;
}
if credential_quota {
for ((id, m), state) in &mut self.cooldowns {
if *id == c.id && !m.is_empty() && state.trust.bounded && state.deadline == old {
state.deadline = deadline;
state.trust.stated = window_trust.stated;
state.trust.window = state.trust.window.max(window_trust.window);
}
}
self.extend_siblings(c, deadline, None, now);
}
return;
}
Some(match bound(stated, cap, prev_trust.window) {
Some((bound, window)) => {
self.generation = self.generation.wrapping_add(1);
trust = Trust {
window: Some(window),
generation: self.generation,
bounded: true,
accepted: false,
stated: now.checked_add(stated),
probe: None,
};
bound
}
None => stated,
})
} else if let Some(prev) = prev.filter(|s| s.quota && s.deadline > now) {
let deadline = prev.deadline;
if let Some(slot) = self.cooldowns.get_mut(&key) {
slot.error = text;
}
if credential_quota {
self.extend_siblings(c, deadline, None, now);
}
return;
} else {
let (d, next) = quota_backoff(level);
level = next;
Some(d)
}
}
408 | 500 | 502..=504 | 520..=526 => transient(policy, hint, disabled),
_ => transient(policy, None, disabled),
}
};
let duration = match duration {
None if action.force_cooldown => Some(Duration::from_secs(60)),
d => d,
};
let Some(next) = duration.and_then(|d| now.checked_add(d)) else {
self.cooldowns.remove(&key);
return;
};
let deadline = prev_live.map_or(next, |p| p.max(next));
if prev_live == Some(deadline)
&& let Some(slot) = self.cooldowns.get_mut(&key).filter(|s| s.quota && s.trust.bounded)
{
slot.error = text;
slot.level = level;
return;
}
let provenance = |deadline: Instant| {
if deadline == next {
trust
} else if prev_live == Some(deadline) {
Trust {
probe: None,
..prev_trust
}
} else {
Trust {
window: trust.window,
..Trust::default()
}
}
};
let kind = |reason| {
quota.then_some(QuotaKind {
status: error.status,
reason,
})
};
let own_reason = if cloudflare { "cloudflare challenge" } else { "quota" };
let state = |deadline, reason| Cooldown {
deadline,
level,
status: error.status,
quota,
error: text.clone(),
since: SystemTime::now(),
credential: false,
trust: provenance(deadline),
quota_kind: kind(reason),
quota_under: quota_under.filter(|_| !quota),
recorded: 0,
};
let reason = if credential_quota {
"credential_quota"
} else {
own_reason
};
self.cooldowns.insert(key, state(deadline, reason));
if credential_quota {
self.extend_siblings(c, deadline, prev_deadline, now);
if !model.is_empty() {
let own = (c.id.clone(), model.to_owned());
let own_deadline = self
.cooldowns
.get(&own)
.filter(|s| s.deadline > now)
.map_or(deadline, |s| s.deadline);
let own_trust = self.cooldowns.get(&own).map(|s| s.trust).unwrap_or_default();
let own_window = own_trust.window;
let mut own_state = state(own_deadline, own_reason);
if own_deadline != deadline {
own_state.trust = Trust {
probe: None,
..own_trust
};
} else {
own_state.trust = provenance(deadline);
}
own_state.trust.window = own_state.trust.window.max(own_window);
self.cooldowns.insert(own, own_state);
}
}
}
fn extend_siblings(&mut self, c: &Credential, deadline: Instant, renews: Option<Instant>, now: Instant) {
let (level, trust, status) = self
.cooldowns
.get(&(c.id.clone(), String::new()))
.map_or((0, Trust::default(), 429), |s| (s.level, s.trust, s.status));
for ((id, model), state) in &mut self.cooldowns {
if *id != c.id || model.is_empty() {
continue;
}
let copy = state.trust.bounded && (state.deadline == deadline || Some(state.deadline) == renews);
if state.deadline > now || copy {
let window = state.trust.window.max(trust.window);
if copy || !(state.trust.bounded || trust.bounded) {
if deadline >= state.deadline {
state.trust = Trust { probe: None, ..trust };
}
state.deadline = state.deadline.max(deadline);
}
state.trust.window = window;
state.quota = true;
state.credential = true;
state.level = level;
state.quota_kind = Some(QuotaKind {
status,
reason: "credential_quota",
});
}
}
}
pub fn reset(&mut self, id: &str) -> Vec<String> {
let mut models: Vec<String> = self
.cooldowns
.keys()
.filter(|(cid, _)| cid == id)
.map(|(_, m)| m.clone())
.collect();
self.cooldowns.retain(|(cid, _), _| cid != id);
self.affinity.invalidate(id);
self.lcp.invalidate(id);
self.probes.remove(id);
models.sort();
models
}
pub fn records(&self, c: &Credential, now: Instant, wall: SystemTime) -> Vec<crate::cooldown_store::Record> {
use crate::cooldown_store::{LastError, Quota, Record};
let mut out: Vec<Record> = self
.cooldowns
.iter()
.filter(|((id, _), state)| *id == c.id && state.deadline > now)
.map(|((_, model), state)| {
let at = wall + (state.deadline - now);
let recover = state
.trust
.stated
.filter(|r| state.trust.bounded && *r > state.deadline)
.map_or(at, |r| wall + (r - now));
let reason = match (model.is_empty() || state.credential, state.quota, state.status) {
(true, true, _) => "credential_quota".to_owned(),
(false, true, 429) => "quota".to_owned(),
(false, true, _) => "cloudflare challenge".to_owned(),
(_, false, _) => state.error.clone(),
};
Record {
provider: c.provider.trim().to_owned(),
auth_id: c.id.clone(),
model: model.clone(),
status: "cooling".into(),
next_retry_after: Some(at),
quota: if state.quota {
Quota {
exceeded: true,
reason: reason.clone(),
next_recover_at: Some(recover),
backoff_level: state.level,
observed_at: None,
trust_windows: Some(state.trust.window.map_or(0, |w| w.saturating_add(1))),
}
} else {
Quota::default()
},
reason,
last_error: Some(LastError {
message: state.error.clone(),
http_status: u32::from(state.status),
..LastError::default()
}),
updated_at: Some(state.since),
auth_file: match &c.source {
cpa_core::credential::Source::File(path) => Some(path.clone()),
_ => None,
},
}
})
.collect();
out.sort_by(|a, b| a.model.cmp(&b.model));
out
}
pub fn restore(
&mut self,
c: &Credential,
record: &crate::cooldown_store::Record,
has_model_records: bool,
cap: Duration,
now: Instant,
wall: SystemTime,
) -> bool {
let Some(retry) = record.next_retry_after.filter(|at| *at > wall) else {
return false;
};
let recover = record.quota.next_recover_at.filter(|_| record.quota.exceeded);
let saved = record.quota.trust_windows.map(|n| n.min(MAX_TRUST_WINDOW + 1));
let model = record.model.trim();
if model.is_empty() && has_model_records && record.quota.reason != "credential_quota" {
return false;
}
let since = |at: SystemTime| at.duration_since(wall).unwrap_or_default();
let (remaining, trust) = match (record.quota.exceeded && !cap.is_zero(), saved) {
(true, Some(spent)) => {
let stated = recover.filter(|r| *r > retry).map(|r| now + since(r));
let trust = Trust {
window: spent.checked_sub(1),
generation: 0,
bounded: stated.is_some(),
accepted: false,
stated,
probe: None,
};
(since(retry), trust)
}
(true, None) => {
let at = recover.map_or(retry, |r| retry.max(r));
match bound(since(at), cap, None) {
Some((bound, window)) => (
bound,
Trust {
window: Some(window),
generation: 0,
bounded: true,
accepted: false,
stated: Some(now + since(at)),
probe: None,
},
),
None => (since(at), Trust::default()),
}
}
(false, _) => (since(recover.map_or(retry, |r| retry.max(r))), Trust::default()),
};
if remaining.is_zero() {
return false;
}
let deadline = now + remaining;
let error = record.last_error.clone().unwrap_or_default();
let status = match error.http_status {
0 if record.quota.exceeded => 429,
s => u16::try_from(s).unwrap_or(0),
};
let key = (c.id.clone(), canonical_model(model).to_owned());
let recorded = self.cooldowns.get(&key).map_or(0, |prev| prev.recorded);
let prev = self.cooldowns.get(&key).filter(|prev| prev.deadline > now);
let prev_wins = prev.is_some_and(|p| p.deadline > deadline);
let deadline = prev.map_or(deadline, |prev| prev.deadline.max(deadline));
let mut quota = (
record.quota.exceeded,
status,
!model.is_empty() && record.quota.reason == "credential_quota",
);
let reason = match record.quota.reason.as_str() {
"credential_quota" => "credential_quota",
"cloudflare challenge" => "cloudflare challenge",
_ => "quota",
};
let mut quota_kind = record.quota.exceeded.then_some(QuotaKind { status, reason });
let quota_under = prev.and_then(|p| p.quota_block(now));
let mut trust = trust;
if let Some(prev) = prev.filter(|p| p.trust != Trust::default() || trust != Trust::default()) {
let window = prev.trust.window.max(trust.window);
if prev_wins {
trust = Trust {
probe: None,
..prev.trust
};
quota = (prev.quota, prev.status, prev.credential);
quota_kind = prev.quota_kind;
}
trust.window = window;
}
if trust.bounded && !prev_wins {
self.generation = self.generation.wrapping_add(1);
trust.generation = self.generation;
}
let (quota, status, credential) = quota;
self.cooldowns.insert(
key,
Cooldown {
deadline,
level: record.quota.backoff_level,
status,
quota,
error: if error.message.is_empty() {
record.reason.clone()
} else {
error.message
},
since: record.updated_at.unwrap_or(wall),
credential,
trust,
quota_kind: quota_kind.filter(|_| quota),
quota_under: quota_under.filter(|_| !quota),
recorded,
},
);
true
}
pub fn clear_disabled(&mut self, credentials: &[std::sync::Arc<Credential>], policy: &Policy) -> bool {
let before = self.cooldowns.len();
self.cooldowns.retain(|(id, _), _| {
credentials
.iter()
.find(|c| c.id == *id)
.is_none_or(|c| !c.disabled && !policy.cooling_disabled(c))
});
self.cooldowns.len() != before
}
pub fn reconcile(&mut self, credentials: &[std::sync::Arc<Credential>]) {
self.cooldowns
.retain(|(id, _), _| credentials.iter().any(|c| c.id == *id));
self.affinity.retain(|id| credentials.iter().any(|c| c.id == id));
self.lcp.retain(|id| credentials.iter().any(|c| c.id == id));
self.probes.retain(|id, _| credentials.iter().any(|c| c.id == *id));
}
}
fn recover_after(s: &Cooldown, now: Instant) -> Option<Duration> {
let stated = s
.trust
.stated
.filter(|r| s.trust.bounded && *r > now && *r > s.deadline)?;
Some(stated - now)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CooldownState {
pub model: String,
pub remaining: Duration,
pub recover_in: Option<Duration>,
pub level: u32,
pub status: u16,
pub quota: bool,
}
impl Scheduler {
pub(crate) fn cooldowns_of(&self, id: &str, now: Instant) -> Vec<CooldownState> {
let mut out: Vec<CooldownState> = self
.cooldowns
.iter()
.filter(|((cid, _), s)| cid == id && (s.deadline > now || recover_after(s, now).is_some()))
.map(|((_, model), s)| CooldownState {
model: model.clone(),
remaining: s.deadline.saturating_duration_since(now),
recover_in: recover_after(s, now),
level: s.level,
status: s.status,
quota: s.quota,
})
.collect();
out.sort_by(|a, b| a.model.cmp(&b.model));
out
}
pub(crate) fn reset_cooldowns(&mut self, id: &str) -> Vec<String> {
let mut models: Vec<String> = self
.cooldowns
.keys()
.filter(|(cid, model)| cid == id && !model.is_empty())
.map(|(_, model)| model.clone())
.collect();
models.sort();
self.cooldowns.retain(|(cid, _), _| cid != id);
self.probes.remove(id);
models
}
pub(crate) fn clear_unauthorized(&mut self, id: &str, now: Instant) -> bool {
let mut changed = false;
self.cooldowns.retain(|(cid, model), c| {
let error = c.error.to_ascii_lowercase();
let unauthorized = c.status == 401
|| error.contains("status 401")
|| error.contains("401 unauthorized")
|| (!model.is_empty() && error.contains("unauthorized"));
if cid != id || !unauthorized {
return true;
}
changed = true;
let Some((end, kind)) = c.quota_block(now) else {
return false;
};
c.quota = true;
c.quota_kind = Some(kind);
c.quota_under = None;
c.deadline = c.deadline.max(end);
c.status = kind.status;
if error.contains("unauthorized") {
c.error = kind.reason.into();
}
true
});
changed
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::Path;
#[test]
fn a_quota_under_a_401_keeps_its_kind_through_restore_and_recovery() {
let (c, d) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
let p = Policy {
max_trusted_cooldown: Duration::ZERO,
..Policy::default()
};
let (start, wall) = (Instant::now(), SystemTime::now());
let unauthorized = |text: &str| Outcome::Failure(ExecError::local(401, FailureScope::Credential, text));
let mut challenge = ExecError::local(429, FailureScope::Model, "<title>Just a moment...</title> cloudflare");
challenge.retry_after = Some(secs(2 * H));
let mut s = Scheduler::default();
s.record(&c, "restored", &hinted(FailureScope::Model, secs(2 * H)), &p, start);
s.record(&c, "cloudflare", &Outcome::Failure(challenge), &p, start);
s.record(&c, "plain", &hinted(FailureScope::Model, secs(2 * H)), &p, start);
s.record(&d, "sibling", &unauthorized("unauthorized"), &p, start);
let answered = start + secs(2);
for model in ["restored", "cloudflare"] {
s.record(&c, model, &unauthorized("unauthorized"), &p, answered);
}
s.record(&c, "plain", &unauthorized("status 401"), &p, answered);
s.record(
&d,
"other",
&hinted(FailureScope::Credential, secs(2 * H)),
&p,
answered,
);
assert!(
entry(&s, &d, "sibling").quota,
"the credential-wide quota reached the 401"
);
let restored = s
.records(&c, answered, wall)
.into_iter()
.find(|r| r.model == "restored")
.unwrap();
assert!(!restored.quota.exceeded, "the saved record is the 401's");
assert!(s.restore(&c, &restored, true, Duration::ZERO, answered, wall));
let cases = [
(&c, "restored", "quota"),
(&c, "cloudflare", "cloudflare challenge"),
(&c, "plain", "status 401"),
(&d, "sibling", "credential_quota"),
];
let blocked: Vec<_> = cases.iter().map(|(x, m, _)| s.wait(x, m, answered)).collect();
assert!(s.clear_unauthorized("a", answered));
assert!(s.clear_unauthorized("b", answered));
for (x, model, error) in cases {
let e = entry(&s, x, model);
assert_eq!((e.quota, e.status, e.error.as_str()), (true, 429, error), "{model}");
}
let after: Vec<_> = cases.iter().map(|(x, m, _)| s.wait(x, m, answered)).collect();
assert_eq!(after, blocked, "each stays blocked until the same time");
assert!(after.iter().all(Option::is_some));
}
#[test]
fn clear_unauthorized_matches_go_per_key_kind() {
let now = Instant::now();
let cooldown = |status: u16, error: &str| Cooldown {
deadline: now + Duration::from_secs(600),
level: 0,
status,
quota: false,
error: error.into(),
since: SystemTime::now(),
credential: false,
trust: Trust::default(),
quota_kind: None,
quota_under: None,
recorded: 0,
};
let mut s = Scheduler::default();
for (id, model, c) in [
("a", "401", cooldown(401, "upstream said no")),
("a", "403-text", cooldown(403, "unauthorized operation")),
("a", "403", cooldown(403, "forbidden")),
("a", "", cooldown(403, "unauthorized operation")),
("b", "401", cooldown(401, "upstream said no")),
] {
s.cooldowns.insert((id.into(), model.into()), c);
}
assert!(s.clear_unauthorized("a", now));
let mut left: Vec<_> = s.cooldowns.keys().map(|(id, m)| (id.as_str(), m.as_str())).collect();
left.sort();
assert_eq!(left, [("a", ""), ("a", "403"), ("b", "401")]);
for (status, error) in [(401, "x"), (403, "Status 401: revoked"), (403, "HTTP 401 Unauthorized")] {
s.cooldowns.insert(("a".into(), String::new()), cooldown(status, error));
assert!(s.clear_unauthorized("a", now), "{status} {error}");
assert!(
!s.cooldowns.contains_key(&("a".into(), String::new())),
"{status} {error}"
);
}
assert!(!s.clear_unauthorized("a", now), "nothing left to clear");
}
#[test]
fn a_401_over_a_live_quota_keeps_the_quota_through_recovery() {
let c = cred("a", serde_json::json!({}));
let start = Instant::now();
let unauthorized = Outcome::Failure(ExecError::local(401, FailureScope::Credential, "unauthorized"));
for (cap, replaced) in [(Duration::ZERO, 401), (Policy::default().max_trusted_cooldown, 429)] {
let p = Policy {
max_trusted_cooldown: cap,
..Policy::default()
};
let mut s = Scheduler::default();
s.record(&c, "long", &hinted(FailureScope::Model, secs(2 * H)), &p, start);
s.record(&c, "short", &hinted(FailureScope::Model, secs(MIN)), &p, start);
let answered = start + secs(MIN / 2);
s.record(&c, "long", &unauthorized, &p, answered);
s.record(&c, "short", &unauthorized, &p, answered);
assert_eq!(entry(&s, &c, "long").status, replaced, "{cap:?}");
let blocked = s.wait(&c, "long", answered);
let bind = start + secs(5 * MIN);
assert!(s.clear_unauthorized("a", bind), "{cap:?}");
let long = entry(&s, &c, "long");
assert_eq!(
(long.quota, long.status, long.error.as_str()),
(true, 429, "quota"),
"{cap:?}"
);
assert_eq!(
s.wait(&c, "long", bind),
blocked.map(|b| b - (bind - answered)),
"{cap:?}: still blocked until the same time"
);
assert!(s.wait(&c, "long", bind).is_some_and(|w| w > secs(30 * MIN)), "{cap:?}");
assert_eq!(s.wait(&c, "short", bind), None, "{cap:?}: its quota had ended");
}
}
fn tag<'a>(creds: &[&'a Credential]) -> Vec<(&'a Credential, &'static str)> {
creds.iter().map(|c| (*c, "claude")).collect()
}
fn cred(id: &str, extra: Value) -> Credential {
let mut metadata = extra.as_object().unwrap().clone();
metadata.insert("type".into(), "claude".into());
Credential::from_file(Path::new("/mock"), &Path::new("/mock").join(id), metadata).unwrap()
}
fn selection(model: &str) -> Selection {
Selection {
provider: "claude".into(),
model: model.into(),
..Default::default()
}
}
#[test]
fn normalization_matches_go_runtime_not_example_config() {
for (raw, expected) in [
(" WRR ", Strategy::WeightedRoundRobin),
("weightedroundrobin", Strategy::WeightedRoundRobin),
("weighted-round-robin", Strategy::WeightedRoundRobin),
(" FF", Strategy::FillFirst),
("fillfirst", Strategy::FillFirst),
("fill-first", Strategy::FillFirst),
("soonest-reset", Strategy::SoonestReset),
(" Reset-First ", Strategy::SoonestReset),
("resetfirst", Strategy::SoonestReset),
("", Strategy::RoundRobin),
("anything", Strategy::RoundRobin),
] {
assert_eq!(
normalize(RawRouting {
strategy: raw,
..Default::default()
})
.strategy,
expected
);
}
for (ttl, seconds) in [
("", 3600),
("bad", 3600),
("-1h", 3600),
("0", 3600),
("0.5s", 1),
("0.0000000001s", 3600),
("1h30m", 5400),
(" 2m ", 120),
] {
assert_eq!(
normalize(RawRouting {
session_affinity_ttl: ttl,
..Default::default()
})
.session_affinity_ttl,
Duration::from_secs(seconds),
"{ttl}"
);
}
let policy = normalize(RawRouting {
request_retry: -1,
max_retry_credentials: -5,
max_retry_interval: -3,
session_affinity_subagents: Some(false),
..Default::default()
});
assert_eq!(policy.request_retry, 0);
assert_eq!(policy.max_retry_credentials, 0);
assert_eq!(policy.max_retry_interval, Duration::ZERO);
assert!(policy.session_affinity_subagents, "ignored when affinity disabled");
assert!(
!normalize(RawRouting {
session_affinity: true,
session_affinity_subagents: Some(false),
..Default::default()
})
.session_affinity_subagents
);
assert!(
normalize(RawRouting {
session_affinity: true,
..Default::default()
})
.session_affinity_subagents
);
}
#[test]
fn go_duration_decimal_compound_units_and_overflow() {
for (raw, expected) in [
("1h2m3.4s", 3_723_400_000_000),
(".5s", 500_000_000),
("+1.000000001s", 1_000_000_001),
("1.s", 1_000_000_000),
("1µs2μs3us", 6000),
("0.3333333333333333333h", 1_200_000_000_000),
("0.100000000000000000000h", 360_000_000_000),
("-9223372036854775808ns", i64::MIN),
("9223372036854775807ns", i64::MAX),
("0.9ns", 0),
] {
assert_eq!(cpa_core::config::parse_duration(raw), Some(expected), "{raw}");
}
for raw in [
"1d",
".s",
"1",
"1h 2m",
"1s-2s",
"1e3s",
"9223372036854775808ns",
"-9223372036854775809ns",
] {
assert_eq!(cpa_core::config::parse_duration(raw), None, "{raw}");
}
}
#[test]
fn rotation_tracks_previous_identity_and_provider_model_keys() {
let a = cred("a", serde_json::json!({}));
let b = cred("b", serde_json::json!({}));
let c = cred("c", serde_json::json!({}));
let p = Policy::default();
let now = Instant::now();
let mut s = Scheduler::default();
assert_eq!(s.pick(&tag(&[&c, &b, &a]), &selection("m"), &p, now).unwrap().id, "a");
assert_eq!(s.pick(&tag(&[&a, &c]), &selection("m"), &p, now).unwrap().id, "c");
assert_eq!(s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now).unwrap().id, "a");
assert_eq!(s.pick(&tag(&[&a, &b]), &selection("other"), &p, now).unwrap().id, "a");
assert_eq!(s.pick(&tag(&[&a, &b]), &selection("m(high)"), &p, now).unwrap().id, "b");
}
#[test]
fn smooth_weights_keep_signed_credits_across_temporary_exclusions() {
let a = cred("a", serde_json::json!({"weight":5}));
let b = cred("b", serde_json::json!({"weight":1}));
let c = cred("c", serde_json::json!({"weight":1}));
let p = Policy {
strategy: Strategy::WeightedRoundRobin,
..Default::default()
};
let now = Instant::now();
let mut s = Scheduler::default();
let picks: Vec<_> = (0..7)
.map(|_| {
s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now)
.unwrap()
.id
.clone()
})
.collect();
assert_eq!(picks, ["a", "a", "b", "a", "c", "a", "a"]);
let mut s = Scheduler::default();
assert_eq!(s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now).unwrap().id, "a");
assert_eq!(s.pick(&tag(&[&b, &c]), &selection("m"), &p, now).unwrap().id, "b");
assert_eq!(s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now).unwrap().id, "a");
assert_eq!(s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now).unwrap().id, "c");
for v in [
serde_json::json!(0),
serde_json::json!(-3),
serde_json::json!(1_000_001),
serde_json::json!(1.5),
serde_json::json!("bad"),
] {
assert!(!s.admits(&cred("off", serde_json::json!({"weight":v})), &p));
}
assert_eq!(weight(&cred("max", serde_json::json!({"weight":1_000_000}))), 1_000_000);
assert_eq!(weight(&cred("float", serde_json::json!({"weight":2.0}))), 2);
}
#[test]
fn affinity_beats_recovered_priority_until_unusable_or_expired() {
let low = cred("a", serde_json::json!({"priority":-1}));
let high = cred("b", serde_json::json!({"priority":9}));
let p = Policy {
strategy: Strategy::FillFirst,
session_affinity: true,
session_affinity_ttl: Duration::from_secs(10),
..Default::default()
};
let sel = Selection {
session: Some("session".into()),
..selection("m")
};
let now = Instant::now();
let mut s = Scheduler::default();
assert_eq!(s.pick(&tag(&[&low]), &sel, &p, now).unwrap().id, "a");
assert_eq!(
s.pick(&tag(&[&low, &high]), &sel, &p, now + Duration::from_secs(5))
.unwrap()
.id,
"a"
);
assert_eq!(
s.pick(&tag(&[&low, &high]), &sel, &p, now + Duration::from_secs(15))
.unwrap()
.id,
"b"
);
assert_eq!(
s.pick(&tag(&[&low]), &sel, &p, now + Duration::from_secs(16))
.unwrap()
.id,
"a"
);
assert_eq!(s.pick(&tag(&[&low, &high]), &selection("m"), &p, now).unwrap().id, "b");
}
#[test]
fn max_trusted_cooldown_reads_durations_seconds_and_off() {
let read = |raw| {
normalize(RawRouting {
max_trusted_cooldown: raw,
..Default::default()
})
.max_trusted_cooldown
};
assert_eq!(read(""), Duration::from_secs(3600));
assert_eq!(read("90m"), Duration::from_secs(5400));
assert_eq!(read("120"), Duration::from_secs(120));
assert_eq!(read("0"), Duration::ZERO);
assert_eq!(read("0s"), Duration::ZERO);
assert_eq!(read("-5"), Duration::ZERO);
for unreadable in ["soon", "1d", "false", "1.5"] {
assert_eq!(read(unreadable), Duration::from_secs(3600), "{unreadable}");
}
assert_eq!(read("3"), Duration::from_secs(10), "clamped to Go's floor");
assert_eq!(read("2s"), Duration::from_secs(10), "clamped to Go's floor");
}
const H: u64 = 3600;
const MIN: u64 = 60;
fn secs(n: u64) -> Duration {
Duration::from_secs(n)
}
fn hinted(scope: FailureScope, stated: Duration) -> Outcome {
let mut e = ExecError::local(429, scope, "usage_limit_reached");
e.retry_after = Some(stated);
Outcome::Failure(e)
}
fn unhinted(scope: FailureScope) -> Outcome {
Outcome::Failure(ExecError::local(429, scope, "rate limited"))
}
fn entry<'a>(s: &'a Scheduler, c: &Credential, model: &str) -> &'a Cooldown {
&s.cooldowns[&(c.id.clone(), model.to_owned())]
}
#[test]
fn quota_resets_are_trusted_up_to_a_doubling_bound() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let start = Instant::now();
let reset = start + secs(6 * 24 * H);
for (scope, key) in [(FailureScope::Model, "m"), (FailureScope::Credential, "")] {
let mut s = Scheduler::default();
let mut now = start;
for window in [1, 2, 4, 8, 16, 32, 64] {
s.record(&c, "m", &hinted(scope, reset - now), &p, now);
assert_eq!(s.wait(&c, "m", now), Some(secs(window * H)), "{scope:?}");
assert_eq!(entry(&s, &c, key).trust.stated, Some(reset), "{scope:?}: stated kept");
assert_eq!(entry(&s, &c, key).level, 0, "{scope:?}: Go's level untouched");
now += secs(window * H);
assert!(s.retry_eligible(&c, "m", now));
}
s.record(&c, "m", &hinted(scope, reset - now), &p, now);
assert_eq!(
s.wait(&c, "m", now),
Some(reset - now),
"{scope:?}: capped by the stated reset"
);
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(scope, reset - start), &p, start);
let then = start + secs(H);
s.record(&c, "m", &hinted(scope, reset - then), &p, then);
assert_eq!(s.wait(&c, "m", then), Some(secs(2 * H)));
let later = start + secs(3 * H);
s.record(&c, "m", &Outcome::Success, &p, later);
s.record(&c, "m", &hinted(scope, reset - later), &p, later);
assert_eq!(
s.wait(&c, "m", later),
Some(secs(H)),
"{scope:?}: back to the first bound"
);
}
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, secs(30 * MIN)), &p, start);
assert_eq!(s.wait(&c, "m", start), Some(secs(30 * MIN)));
assert!(!entry(&s, &c, "m").trust.bounded);
let go = Policy {
max_trusted_cooldown: Duration::ZERO,
..Policy::default()
};
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, reset - start), &go, start);
assert_eq!(s.wait(&c, "m", start), Some(reset - start));
}
#[test]
fn trust_count_and_go_backoff_level_are_separate() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let days = secs(6 * 24 * H);
let mut s = Scheduler::default();
let mut now = Instant::now();
for _ in 0..9 {
s.record(&c, "m", &unhinted(FailureScope::Model), &p, now);
now += secs(600);
}
assert_eq!(entry(&s, &c, "m").level, 9);
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
assert_eq!(s.wait(&c, "m", now), Some(secs(H)), "the first bound, not 2^9 hours");
assert_eq!(entry(&s, &c, "m").level, 9, "Go's level keeps its meaning");
let mut s = Scheduler::default();
let mut now = Instant::now();
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
now += secs(H);
s.record(&c, "m", &unhinted(FailureScope::Model), &p, now);
assert_eq!(s.wait(&c, "m", now), Some(secs(1)), "Go's first backoff step");
now += secs(1);
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
assert_eq!(s.wait(&c, "m", now), Some(secs(2 * H)), "the trust count survived");
}
#[test]
fn a_reset_inside_another_cooldown_gets_the_bound() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let days = secs(6 * 24 * H);
let start = Instant::now();
let first = |status: u16, scope| {
let mut s = Scheduler::default();
s.record(
&c,
"m",
&Outcome::Failure(ExecError::local(status, scope, "earlier")),
&p,
start,
);
s
};
for (status, scope, earlier) in [
(503, FailureScope::Credential, secs(60)),
(429, FailureScope::Model, secs(1)),
] {
let mut s = first(status, scope);
assert_eq!(s.wait(&c, "m", start), Some(earlier));
let at = start + Duration::from_millis(500);
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, at);
assert_eq!(s.wait(&c, "m", at), Some(secs(H)), "after {status}");
assert!(entry(&s, &c, "m").trust.bounded, "after {status}");
let later = at + secs(60);
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, later);
assert_eq!(s.wait(&c, "m", at), Some(secs(H)), "after {status}");
assert_eq!(entry(&s, &c, "m").trust.window, Some(0), "after {status}");
}
let mut s = first(401, FailureScope::Credential);
let cap = Policy {
max_trusted_cooldown: secs(600),
..Policy::default()
};
s.record(&c, "m", &hinted(FailureScope::Model, days), &cap, start);
assert_eq!(s.wait(&c, "m", start), Some(secs(30 * MIN)));
assert!(!entry(&s, &c, "m").trust.bounded);
}
#[test]
fn concurrent_resets_keep_or_shorten_the_window() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let start = Instant::now();
let window = |s: &mut Scheduler, policy: &Policy| {
s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), policy, start);
};
let mut s = Scheduler::default();
window(&mut s, &p);
let at = start + secs(30 * MIN);
s.record(&c, "m", &hinted(FailureScope::Model, secs(45 * MIN)), &p, at);
assert_eq!(s.wait(&c, "m", start), Some(secs(H)));
let mut s = Scheduler::default();
window(&mut s, &p);
let at = start + secs(MIN);
s.record(&c, "m", &hinted(FailureScope::Model, secs(30 * MIN)), &p, at);
assert_eq!(s.wait(&c, "m", start), Some(secs(31 * MIN)));
assert_eq!(entry(&s, &c, "m").trust.window, Some(0), "not a new window");
let go = Policy {
max_trusted_cooldown: Duration::ZERO,
..Policy::default()
};
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, secs(H)), &go, start);
let at = start + secs(30 * MIN);
s.record(&c, "m", &hinted(FailureScope::Model, secs(45 * MIN)), &go, at);
assert_eq!(s.wait(&c, "m", start), Some(secs(75 * MIN)));
}
#[test]
fn one_probe_per_ended_window() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let start = Instant::now();
for (scope, key) in [(FailureScope::Model, "m"), (FailureScope::Credential, "")] {
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(scope, secs(6 * 24 * H)), &p, start);
let near = start + secs(H) - secs(5);
assert!(
!s.retry_eligible(&c, "m", near),
"{scope:?}: a bounded window is not waited for"
);
let end = start + secs(H);
assert_eq!(s.wait(&c, "m", end), None, "{scope:?}: the probe is due");
s.reserve_probe(&c, "m", end);
assert_eq!(s.wait(&c, "m", end), Some(PROBE_HOLD), "{scope:?}: reserved");
assert!(s.quota_cooling(&c, "m", end), "{scope:?}");
assert!(!s.retry_eligible(&c, "m", end), "{scope:?}");
assert_eq!(s.wait(&c, "other", end).is_some(), key.is_empty(), "{scope:?}");
s.record(&c, "m", &hinted(scope, secs(5 * 24 * H)), &p, end + secs(2));
assert_eq!(s.wait(&c, "m", end + secs(2)), Some(secs(2 * H)), "{scope:?}");
assert_eq!(entry(&s, &c, key).trust.probe, None);
let next = end + secs(2) + secs(2 * H);
s.reserve_probe(&c, "m", next);
s.record(&c, "m", &Outcome::Cancelled, &p, next + secs(1));
assert_eq!(s.wait(&c, "m", next + secs(1)), None, "{scope:?}");
s.reserve_probe(&c, "m", next + secs(1));
assert_eq!(s.wait(&c, "m", next + secs(1) + PROBE_HOLD), None, "{scope:?}");
}
let mut s = Scheduler::default();
s.record(&c, "m", &unhinted(FailureScope::Model), &p, start);
s.reserve_probe(&c, "m", start + secs(2));
assert_eq!(s.wait(&c, "m", start + secs(2)), None);
}
#[test]
fn a_success_on_another_model_clears_the_escalation() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let start = Instant::now();
let days = secs(6 * 24 * H);
let mut s = Scheduler::default();
s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, start);
assert_eq!(entry(&s, &c, "a").trust.window, Some(0));
let end = start + secs(H);
s.record(&c, "b", &Outcome::Success, &p, end);
s.record(&c, "a", &hinted(FailureScope::Model, days), &p, end);
assert_eq!(s.wait(&c, "a", end), Some(secs(H)), "the first bound again");
}
#[test]
fn an_accepted_probe_frees_the_account_and_keeps_the_count() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let days = secs(6 * 24 * H);
let start = Instant::now();
let end = start + secs(H);
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
s.reserve_probe(&c, "m", end);
assert!(s.wait(&c, "m", end).is_some(), "reserved");
s.accept_probe(&c, "m", end);
assert_eq!(s.wait(&c, "m", end), None, "the next pick goes through");
s.reserve_probe(&c, "m", end + secs(1));
assert_eq!(s.wait(&c, "m", end + secs(1)), None, "no second reservation");
let later = end + secs(60);
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, later);
assert_eq!(s.wait(&c, "m", later), Some(secs(2 * H)), "the count survived");
s.accept_probe(&c, "m", later + secs(1));
assert!(entry(&s, &c, "m").trust.bounded);
let mut s = Scheduler::default();
s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, start);
s.reserve_probe(&c, "b", end);
s.accept_probe(&c, "b", end);
assert_eq!(s.wait(&c, "b", end), None);
s.reserve_probe(&c, "a", end);
assert_eq!(s.wait(&c, "a", end), None, "A's copy does not reserve again");
assert_eq!(entry(&s, &c, "a").trust.window, Some(0), "count kept");
}
#[test]
fn a_credential_probe_leaves_a_models_own_ended_window() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let days = secs(6 * 24 * H);
let start = Instant::now();
for streamed in [true, false] {
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
let opened = start + secs(H) + secs(MIN);
s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, opened);
let end = opened + secs(H);
s.reserve_probe(&c, "b", end);
if streamed {
s.accept_probe(&c, "b", end);
} else {
s.record(&c, "b", &Outcome::Success, &p, end);
}
s.reserve_probe(&c, "a", end);
assert_eq!(s.wait(&c, "a", end), None, "streamed {streamed}: A's copy is answered");
s.reserve_probe(&c, "m", end);
assert_eq!(
s.wait(&c, "m", end),
Some(PROBE_HOLD),
"streamed {streamed}: m reserves its own probe"
);
}
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
let opened = start + secs(H) + secs(MIN);
s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, opened);
let end = opened + secs(H);
let credential_probe = s.reserve_probe(&c, "b", end);
s.accept_probe_picked(&c, "b", credential_probe, end);
let first = s.reserve_probe(&c, "m", end);
let lapsed = end + PROBE_HOLD + secs(1);
let second = s.reserve_probe(&c, "m", lapsed);
assert!(credential_probe != first && first != second, "real generations");
s.accept_probe_picked(&c, "m", second, lapsed);
let done = lapsed + secs(1);
s.record_picked(&c, "b", &Outcome::Success, &p, credential_probe, done);
s.record_picked(&c, "m", &hinted(FailureScope::Model, days), &p, first, done);
assert_eq!(s.wait(&c, "m", done), None, "no window from the superseded probe");
s.record_picked(&c, "m", &Outcome::Success, &p, second, done + secs(1));
assert!(
!s.cooldowns.contains_key(&(c.id.clone(), "m".to_owned())),
"m recovered"
);
}
#[test]
fn a_superseded_probes_429_leaves_the_window_to_the_current_probe() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let days = secs(6 * 24 * H);
let start = Instant::now();
for scope in [FailureScope::Model, FailureScope::Credential] {
for accepted in [false, true] {
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(scope, days), &p, start);
let end = start + secs(H);
let first = s.reserve_probe(&c, "m", end);
let lapsed = end + PROBE_HOLD + secs(1);
let second = s.reserve_probe(&c, "m", lapsed);
if accepted {
s.accept_probe_picked(&c, "m", second, lapsed);
}
let late = lapsed + secs(1);
s.record_picked(&c, "m", &hinted(scope, days - secs(H)), &p, first, late);
assert_eq!(
s.wait(&c, "m", late),
(!accepted).then(|| PROBE_HOLD - secs(1)),
"{scope:?}, accepted {accepted}: still the second probe's"
);
s.record_picked(&c, "m", &Outcome::Success, &p, second, late + secs(1));
assert_eq!(
s.wait(&c, "m", late + secs(1)),
None,
"{scope:?}, accepted {accepted}: recovered"
);
}
}
}
#[test]
fn a_renewed_credential_window_carries_its_ended_copies() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let days = secs(6 * 24 * H);
let start = Instant::now();
for streamed in [true, false] {
let mut s = Scheduler::default();
s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, start);
let first_end = start + secs(H);
s.reserve_probe(&c, "b", first_end);
let renewed = first_end + secs(1);
s.record(&c, "b", &hinted(FailureScope::Credential, days - secs(H)), &p, renewed);
let second_end = renewed + secs(2 * H);
s.reserve_probe(&c, "c", second_end);
if streamed {
s.accept_probe(&c, "c", second_end);
} else {
s.record(&c, "c", &Outcome::Success, &p, second_end);
}
s.reserve_probe(&c, "a", second_end);
assert_eq!(s.wait(&c, "a", second_end), None, "streamed {streamed}: no probe for A");
if !streamed {
s.record(&c, "a", &hinted(FailureScope::Model, days), &p, second_end);
assert_eq!(s.wait(&c, "a", second_end), Some(secs(H)), "the count is cleared");
}
}
}
#[test]
fn outside_selection_leases_answer_no_bounded_window() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let start = Instant::now();
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, start);
let ended = start + secs(H);
let probe = s.reserve_probe(&c, "m", ended);
let credits = s.outside_selection_generation();
for outcome in [Outcome::Cancelled, Outcome::Success] {
s.record_picked(&c, "m", &outcome, &p, credits, ended);
assert!(s.wait(&c, "m", ended).is_some(), "the reserved probe holds the window");
}
s.accept_probe_picked(&c, "m", credits, ended);
assert!(s.wait(&c, "m", ended).is_some(), "a credits stream accepts nothing");
s.record_picked(&c, "m", &Outcome::Success, &p, probe, ended);
assert_eq!(s.wait(&c, "m", ended), None, "the probe's own success ends it");
let credits = s.outside_selection_generation();
let later = ended + secs(MIN);
s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, later);
let reopened = later + secs(H);
s.reserve_probe(&c, "m", reopened);
s.record_picked(&c, "m", &Outcome::Cancelled, &p, credits, reopened);
assert!(s.wait(&c, "m", reopened).is_some(), "the later probe still holds");
}
#[test]
fn generations_compare_across_the_wrap() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let start = Instant::now();
let mut s = Scheduler {
generation: u32::MAX,
..Scheduler::default()
};
let before = s.reserve_probe(&c, "m", start);
assert_eq!(before, u32::MAX);
s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, start);
assert_eq!(entry(&s, &c, "m").trust.generation, 0, "wrapped");
let at = start + secs(MIN);
s.record_picked(&c, "m", &Outcome::Success, &p, before, at);
assert_eq!(s.wait(&c, "m", at), Some(secs(H - MIN)), "an older success leaves it");
s.record_picked(&c, "m", &Outcome::Success, &p, 0, at);
assert_eq!(s.wait(&c, "m", at), None, "a newer success ends it");
}
#[test]
fn an_earlier_credential_reset_shortens_inherited_model_windows() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let start = Instant::now();
let mut s = Scheduler::default();
let support = ExecError::local(400, FailureScope::Model, "model is not supported");
s.record(&c, "support", &Outcome::Failure(support), &p, start);
s.record(&c, "sibling", &unhinted(FailureScope::Model), &p, start);
s.record(&c, "a", &hinted(FailureScope::Credential, secs(6 * 24 * H)), &p, start);
let at = start + secs(MIN);
s.record(&c, "a", &hinted(FailureScope::Credential, secs(30 * MIN)), &p, at);
for model in ["a", "sibling", "unrelated"] {
assert_eq!(s.wait(&c, model, start), Some(secs(31 * MIN)), "{model}");
}
assert_eq!(s.wait(&c, "support", start), Some(secs(12 * H)));
}
#[test]
fn an_earlier_credential_reset_never_moves_a_models_own_probe_earlier() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let days = secs(6 * 24 * H);
let start = Instant::now();
let own_end = start + secs(H);
for failing in ["a", "m"] {
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
let opened = start + secs(30 * MIN);
s.record(&c, failing, &hinted(FailureScope::Credential, days), &p, opened);
let at = opened + secs(MIN);
s.record(&c, failing, &hinted(FailureScope::Credential, secs(10 * MIN)), &p, at);
let moved = at + secs(10 * MIN);
assert_eq!(
s.wait(&c, "other", moved),
None,
"{failing}: the credential's probe is due"
);
assert_eq!(
s.wait(&c, "m", moved),
Some(own_end - moved),
"{failing}: m waits for its own bound"
);
}
}
#[test]
fn an_earlier_credential_reset_never_shortens_a_models_own_cooldown() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let start = Instant::now();
let unauthorized = Outcome::Failure(ExecError::local(401, FailureScope::Credential, "unauthorized"));
for failing in ["a", "m"] {
let mut s = Scheduler::default();
s.record(&c, "m", &unauthorized, &p, start);
assert_eq!(s.wait(&c, "m", start), Some(secs(30 * MIN)));
let opened = start + secs(MIN);
s.record(
&c,
failing,
&hinted(FailureScope::Credential, secs(6 * 24 * H)),
&p,
opened,
);
let at = start + secs(2 * MIN);
s.record(&c, failing, &hinted(FailureScope::Credential, secs(MIN)), &p, at);
let moved = start + secs(3 * MIN);
assert_eq!(
s.wait(&c, "other", moved),
None,
"{failing}: the credential's probe is due"
);
assert_eq!(
s.wait(&c, "m", moved),
Some(secs(27 * MIN)),
"{failing}: m keeps its own 30 minutes"
);
}
}
#[test]
fn the_own_key_keeps_its_window_through_a_credential_merge() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let days = secs(6 * 24 * H);
let start = Instant::now();
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
s.record(
&c,
"m",
&hinted(FailureScope::Model, days - secs(H)),
&p,
start + secs(H),
);
let own_end = start + secs(3 * H);
let at = start + secs(H) + secs(MIN);
s.record(&c, "m", &hinted(FailureScope::Credential, days), &p, at);
let own = entry(&s, &c, "m");
assert_eq!(own.deadline, own_end);
assert!(own.trust.bounded && own.trust.stated == Some(start + days));
s.reserve_probe(&c, "m", own_end);
assert!(
entry(&s, &c, "m").trust.probe.is_some(),
"it reserves a probe at its end"
);
}
#[test]
fn a_restore_keeps_the_winner_trust() {
use crate::cooldown_store::{Quota, Record};
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let (now, wall) = (Instant::now(), SystemTime::now());
let days = secs(6 * 24 * H);
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
let live = entry(&s, &c, "m").trust;
let shorter = Record {
auth_id: "a".into(),
model: "m".into(),
next_retry_after: Some(wall + secs(10 * MIN)),
quota: Quota {
exceeded: true,
reason: "quota".into(),
next_recover_at: Some(wall + secs(10 * MIN)),
trust_windows: Some(3),
..Default::default()
},
..Default::default()
};
assert!(s.restore(&c, &shorter, true, secs(H), now, wall));
let merged = entry(&s, &c, "m");
assert_eq!(merged.deadline, now + secs(H));
assert!(merged.trust.bounded && merged.trust.stated == live.stated);
assert_eq!(merged.trust.window, Some(2), "counts merge");
assert_eq!(merged.trust.generation, live.generation, "the live window's generation");
}
#[test]
fn a_success_dispatched_before_a_cooldown_keeps_it() {
let c = cred("a", serde_json::json!({}));
let p = Policy {
max_trusted_cooldown: Duration::ZERO,
..Policy::default()
};
let (now, wall) = (Instant::now(), SystemTime::now());
let g = 0;
let quota = || hinted(FailureScope::Model, secs(H));
let unavailable = || {
let mut e = ExecError::local(503, FailureScope::Model, "unavailable");
e.retry_after = Some(secs(H));
Outcome::Failure(e)
};
for (model, failure) in [("m", quota()), ("", unavailable())] {
let mut s = Scheduler::default();
let stale = s.dispatch();
let failed = s.dispatch();
s.record_dispatched(&c, model, &failure, &p, g, failed, now);
s.record_dispatched(&c, model, &Outcome::Success, &p, g, stale, now);
s.record_dispatched(&c, model, &Outcome::Success, &p, g, failed, now);
assert!(s.cooling(&c, model, now), "{model:?}: the stale successes keep it");
let after = s.dispatch();
s.record_dispatched(&c, model, &Outcome::Success, &p, g, after, now);
assert!(!s.cooling(&c, model, now), "{model:?}: a later dispatch clears it");
let again = s.dispatch();
s.record_dispatched(&c, model, &failure, &p, g, again, now);
s.record_dispatched(&c, model, &Outcome::Success, &p, g, 0, now);
assert!(!s.cooling(&c, model, now), "{model:?}: an unstamped success clears it");
}
let mut s = Scheduler::default();
let first = s.dispatch();
s.record_dispatched(&c, "", &unavailable(), &p, g, first, now);
let success = s.dispatch();
let dropped = Outcome::Failure(ExecError::local(0, FailureScope::Model, "http2: client conn is closed"));
let third = s.dispatch();
s.record_dispatched(&c, "", &dropped, &p, g, third, now);
s.record_dispatched(&c, "", &Outcome::Success, &p, g, success, now);
assert!(s.cooling(&c, "", now), "the skipped failure stamped the credential");
let mut s = Scheduler::default();
let stale = s.dispatch();
let failed = s.dispatch();
s.record_dispatched(&c, "m", "a(), &p, g, failed, now);
let saved = s.records(&c, now, wall);
assert!(s.restore(&c, &saved[0], true, Duration::ZERO, now, wall));
s.record_dispatched(&c, "m", &Outcome::Success, &p, g, stale, now);
assert!(s.cooling(&c, "m", now), "the restore kept the watermark");
let mut restarted = Scheduler::default();
assert!(restarted.restore(&c, &saved[0], true, Duration::ZERO, now, wall));
let first = restarted.dispatch();
restarted.record_dispatched(&c, "m", &Outcome::Success, &p, g, first, now);
assert!(!restarted.cooling(&c, "m", now), "no watermark after a restart");
let mut s = Scheduler::default();
let other = s.dispatch();
s.record_dispatched(&c, "other", "a(), &p, g, other, now);
let stale = s.dispatch();
let wide = s.dispatch();
s.record_dispatched(&c, "m", &hinted(FailureScope::Credential, secs(H)), &p, g, wide, now);
s.cooldowns.remove(&(c.id.clone(), String::new()));
s.record_dispatched(&c, "other", &Outcome::Success, &p, g, stale, now);
assert!(
s.cooling(&c, "other", now),
"the sibling carries the credential's watermark"
);
}
#[test]
fn a_restore_while_serving_keeps_the_window_newer_than_older_leases() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let (now, wall) = (Instant::now(), SystemTime::now());
let mut s = Scheduler::default();
let stream = s.reserve_probe(&c, "m", now);
s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, now);
let saved = s.records(&c, now, wall);
assert!(s.restore(&c, &saved[0], true, secs(H), now, wall));
assert!(entry(&s, &c, "m").trust.bounded);
let at = now + secs(MIN);
s.record_picked(&c, "m", &Outcome::Success, &p, stream, at);
assert_eq!(s.wait(&c, "m", at), Some(secs(H - MIN)), "the window stands");
}
#[test]
fn a_503_inside_a_bounded_window_keeps_it() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let (now, wall) = (Instant::now(), SystemTime::now());
let days = secs(6 * 24 * H);
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
let at = now + secs(30 * MIN);
let unavailable = ExecError::local(503, FailureScope::Model, "overloaded");
s.record(&c, "m", &Outcome::Failure(unavailable), &p, at);
let saved = &s.records(&c, at, wall + secs(30 * MIN))[0];
assert!(saved.quota.exceeded);
assert_eq!(saved.quota.next_recover_at, Some(wall + days));
assert_eq!(saved.quota.trust_windows, Some(1));
assert_eq!(saved.reason, "quota");
}
#[test]
fn a_cloudflare_challenge_inside_a_bounded_window_raises_the_level() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let start = Instant::now();
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, start);
let challenge = Outcome::Failure(ExecError::local(403, FailureScope::Model, "cloudflare challenge"));
for (n, level) in [(1, 1), (2, 2)] {
s.record(&c, "m", &challenge, &p, start + secs(n * MIN));
assert_eq!(entry(&s, &c, "m").level, level);
assert_eq!(s.wait(&c, "m", start), Some(secs(H)), "the window stands");
assert!(entry(&s, &c, "m").trust.bounded);
}
}
#[test]
fn restored_quota_resets_are_bounded_too() {
use crate::cooldown_store::{Quota, Record};
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let (now, wall) = (Instant::now(), SystemTime::now());
let cap = secs(H);
let days = secs(6 * 24 * H);
let mut s = Scheduler::default();
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
let saved = s.records(&c, now, wall);
assert_eq!(saved[0].next_retry_after, Some(wall + secs(H)));
assert_eq!(saved[0].quota.next_recover_at, Some(wall + days));
assert_eq!(
(saved[0].quota.trust_windows, saved[0].quota.backoff_level),
(Some(1), 0)
);
let mut back = Scheduler::default();
assert!(back.restore(&c, &saved[0], true, cap, now, wall));
assert_eq!(back.wait(&c, "m", now), Some(secs(H)));
assert_eq!(back.records(&c, now, wall), saved);
let record_at = |retry: Duration, trust_windows, backoff_level| Record {
auth_id: "a".into(),
model: "m".into(),
next_retry_after: Some(wall + retry),
quota: Quota {
exceeded: true,
reason: "quota".into(),
next_recover_at: Some(wall + days),
backoff_level,
trust_windows,
..Default::default()
},
..Default::default()
};
let mut s = Scheduler::default();
assert!(s.restore(&c, &record_at(days, None, 2), true, cap, now, wall));
assert_eq!(s.wait(&c, "m", now), Some(secs(H)));
assert_eq!(entry(&s, &c, "m").level, 2);
let mut s = Scheduler::default();
assert!(s.restore(&c, &record_at(secs(4 * H), Some(3), 0), true, cap, now, wall));
assert_eq!(s.wait(&c, "m", now), Some(secs(4 * H)));
let trust = entry(&s, &c, "m").trust;
assert!(trust.bounded && trust.window == Some(2));
assert_eq!(trust.stated, Some(now + days));
let later = now + secs(4 * H);
s.record(&c, "m", &hinted(FailureScope::Model, days - secs(4 * H)), &p, later);
assert_eq!(s.wait(&c, "m", later), Some(secs(8 * H)));
let ninety = secs(90 * MIN);
let trusted = Record {
quota: Quota {
next_recover_at: Some(wall + ninety),
..record_at(ninety, Some(1), 0).quota
},
..record_at(ninety, Some(1), 0)
};
let mut s = Scheduler::default();
assert!(s.restore(&c, &trusted, true, cap, now, wall));
assert_eq!(s.wait(&c, "m", now), Some(ninety));
let back = &s.records(&c, now, wall)[0];
assert_eq!(
(
back.next_retry_after,
back.quota.next_recover_at,
back.quota.trust_windows
),
(trusted.next_retry_after, trusted.quota.next_recover_at, Some(1))
);
assert!(!entry(&s, &c, "m").trust.bounded);
let mut s = Scheduler::default();
assert!(s.restore(&c, &record_at(secs(H), Some(u32::MAX), 0), true, cap, now, wall));
assert_eq!(entry(&s, &c, "m").trust.window, Some(MAX_TRUST_WINDOW));
let mut s = Scheduler::default();
assert!(s.restore(&c, &record_at(days, Some(3), 0), true, Duration::ZERO, now, wall));
assert_eq!(s.wait(&c, "m", now), Some(days));
}
#[test]
fn a_marked_sibling_restores_its_own_deadline() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let (now, wall) = (Instant::now(), SystemTime::now());
let mut s = Scheduler::default();
let support = ExecError::local(400, FailureScope::Model, "model is not supported");
s.record(&c, "b", &Outcome::Failure(support), &p, now);
assert_eq!(s.wait(&c, "b", now), Some(secs(12 * H)), "Go's model-support cooldown");
s.record(&c, "a", &hinted(FailureScope::Credential, secs(6 * 24 * H)), &p, now);
let records = s.records(&c, now, wall);
let b = records.iter().find(|r| r.model == "b").unwrap();
assert!(b.quota.exceeded && b.quota.trust_windows.is_some(), "{b:?}");
let mut back = Scheduler::default();
for r in &records {
back.restore(&c, r, true, secs(H), now, wall);
}
assert_eq!(back.wait(&c, "b", now), Some(secs(12 * H)));
assert!(!entry(&back, &c, "b").trust.bounded, "not cut, so not a bounded window");
}
#[test]
fn credential_wide_windows_keep_the_larger_model_count() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let days = secs(6 * 24 * H);
for failing in ["other", "m"] {
let mut now = Instant::now();
let mut s = Scheduler::default();
for window in [1, 2, 4, 8] {
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
assert_eq!(s.wait(&c, "m", now), Some(secs(window * H)));
if window < 8 {
now += secs(window * H);
}
}
let late = now + secs(8 * H) - secs(10 * MIN);
s.record(&c, failing, &hinted(FailureScope::Credential, days), &p, late);
assert_eq!(entry(&s, &c, "m").trust.window, Some(3), "{failing}: count kept");
let end = late + secs(H);
s.record(&c, "m", &hinted(FailureScope::Model, days), &p, end);
assert_eq!(s.wait(&c, "m", end), Some(secs(16 * H)), "{failing}");
}
}
#[test]
fn a_reset_inside_a_cloudflare_window_gets_the_first_bound() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let mut now = Instant::now();
let mut s = Scheduler::default();
for _ in 0..5 {
let challenge = ExecError::local(403, FailureScope::Model, "cloudflare challenge");
s.record(&c, "m", &Outcome::Failure(challenge), &p, now);
now += secs(1);
}
assert!(entry(&s, &c, "m").level >= 5);
assert!(s.wait(&c, "m", now).is_some(), "inside the challenge window");
s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, now);
assert_eq!(s.wait(&c, "m", now), Some(secs(H)));
}
#[test]
fn cooldown_deadlines_floor_backoff_and_sibling_success() {
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let now = Instant::now();
let mut s = Scheduler::default();
let fail = |status, scope, hint| {
let mut e = ExecError::local(status, scope, "failure");
e.retry_after = hint;
Outcome::Failure(e)
};
s.record(&c, "sibling", &fail(404, FailureScope::Model, None), &p, now);
s.record(
&c,
"m",
&fail(429, FailureScope::Credential, Some(Duration::from_secs(2))),
&p,
now,
);
assert_eq!(s.wait(&c, "m", now), Some(Duration::from_secs(10)));
assert_eq!(s.wait(&c, "new", now), Some(Duration::from_secs(10)));
assert_eq!(
s.wait(&c, "sibling", now),
Some(Duration::from_secs(43200)),
"quota must not shorten sibling"
);
s.record(&c, "sibling", &Outcome::Success, &p, now + Duration::from_secs(1));
assert_eq!(
s.wait(&c, "sibling", now),
Some(Duration::from_secs(43200)),
"inflight success cannot clear live quota"
);
assert_eq!(
s.wait(&c, "new", now + Duration::from_secs(10)),
None,
"deadline boundary"
);
s.record(&c, "new", &Outcome::Success, &p, now + Duration::from_secs(10));
assert!(s.wait(&c, "sibling", now + Duration::from_secs(10)).is_some());
let mut s = Scheduler::default();
s.record(&c, "m", &fail(429, FailureScope::Model, None), &p, now);
assert_eq!(s.wait(&c, "m(high)", now), Some(Duration::from_secs(1)));
s.record(&c, "m", &fail(429, FailureScope::Model, None), &p, now);
assert_eq!(
s.wait(&c, "m", now),
Some(Duration::from_secs(1)),
"active quota reused"
);
s.record(
&c,
"m",
&fail(429, FailureScope::Model, None),
&p,
now + Duration::from_secs(1),
);
assert_eq!(
s.wait(&c, "m", now + Duration::from_secs(1)),
Some(Duration::from_secs(2))
);
assert_eq!(s.wait(&c, "other", now), None, "model failure stays local");
for (status, seconds) in [
(401, 1800),
(402, 1800),
(403, 1800),
(404, 43200),
(408, 60),
(500, 60),
(526, 60),
] {
let mut s = Scheduler::default();
s.record(&c, "m", &fail(status, FailureScope::Model, None), &p, now);
assert_eq!(
s.wait(&c, "m", now),
Some(Duration::from_secs(seconds)),
"status {status}"
);
}
}
#[test]
fn cooldowns_match_go_mark_result() {
let fixture: Value = serde_json::from_str(include_str!("../tests/fixtures/server_go.json")).unwrap();
let cases = fixture["cooldown"].as_array().unwrap();
assert_eq!(cases.len(), 25);
let policy = Policy {
max_trusted_cooldown: Duration::ZERO,
..Policy::default()
};
for case in cases {
let name = case["name"].as_str().unwrap();
let c = cred(&format!("{name}.json"), serde_json::json!({}));
let mut s = Scheduler::default();
let now = Instant::now();
for step in case["steps"].as_array().unwrap() {
let status = step["status"].as_u64().unwrap() as u16;
let scope = if step["credential_scope"].as_bool().unwrap() {
FailureScope::Credential
} else {
match status {
429 => FailureScope::Model,
401..=403 | 408 | 500.. => FailureScope::Credential,
_ => FailureScope::Request,
}
};
let mut error = ExecError::local(status, scope, step["message"].as_str().unwrap());
error
.headers
.insert("content-type", "application/json".parse().unwrap());
let hint = step["retry_after_ms"].as_i64().unwrap();
error.retry_after = (hint >= 0).then(|| Duration::from_millis(hint as u64));
s.record(
&c,
step["model"].as_str().unwrap(),
&Outcome::Failure(error),
&policy,
now,
);
}
for (model, seconds) in case["seconds"].as_object().unwrap() {
let wait = s.wait(&c, model, now).unwrap_or_default();
assert_eq!(
wait.as_secs_f64().round() as u64,
seconds.as_u64().unwrap(),
"{name} {model} wait"
);
let quota = case["quota"][model].as_bool().unwrap();
assert_eq!(
s.quota_cooling(&c, model, now),
quota && wait > Duration::ZERO,
"{name} {model} quota"
);
}
}
}
#[test]
fn session_affinity_matches_go_selector() {
let fixture: Value = serde_json::from_str(include_str!("../tests/fixtures/server_go.json")).unwrap();
let cases = fixture["affinity"].as_array().unwrap();
assert_eq!(cases.len(), 25);
for case in cases {
let name = case["name"].as_str().unwrap();
let lcp_case = case["lcp"].as_bool().unwrap_or_default();
let creds: Vec<Credential> = ["a", "b", "c"]
.iter()
.map(|id| {
let mut c = cred(id, serde_json::json!({}));
if let Some(p) = case["priorities"][*id].as_str() {
c.attributes.insert("priority".into(), p.into());
}
c
})
.collect();
let policy = Policy {
strategy: Strategy::FillFirst,
session_affinity: true,
session_affinity_subagents: case["subagent_affinity"].as_bool().unwrap(),
..Default::default()
};
let mut s = Scheduler::default();
let now = Instant::now();
for (i, step) in case["steps"].as_array().unwrap().iter().enumerate() {
let mut headers = axum::http::HeaderMap::new();
for pair in step["headers"].as_array().into_iter().flatten() {
headers.append(
axum::http::HeaderName::from_bytes(pair[0].as_str().unwrap().as_bytes()).unwrap(),
pair[1].as_str().unwrap().parse().unwrap(),
);
}
let payload = step["payload"].as_str().unwrap_or_default().as_bytes();
let format = cpa_core::format::Format::parse(case["format"].as_str().unwrap_or("openai")).unwrap();
let caller = match step["caller"].as_str() {
Some(c) => c,
None => case["caller"].as_str().unwrap_or_default(),
};
let caller = if lcp_case { caller } else { "" };
let session = crate::session::resolve(format, &headers, payload, None, caller);
let lcp = (lcp_case && !session.explicit)
.then(|| crate::lcp::Request::new(format.as_str(), payload, caller))
.flatten()
.map(std::sync::Arc::new);
let sel = Selection {
session: session.id,
session_parent: session.parent,
session_fork: session.fork,
lcp,
..selection("m")
};
let find = |id: &str| creds.iter().find(|c| c.id == id).unwrap();
let outcome = |op: &str| {
if op.ends_with("ok") {
Outcome::Success
} else {
let status = step["status"].as_u64().unwrap() as u16;
let scope = if status >= 500 {
FailureScope::Credential
} else {
FailureScope::Request
};
let mut e = ExecError::local(status, scope, step["message"].as_str().unwrap());
e.headers.insert("content-type", "text/plain".parse().unwrap());
Outcome::Failure(e)
}
};
match step["op"].as_str().unwrap() {
op @ ("pick" | "pick_ok" | "pick_fail") => {
let available: Vec<&Credential> = step["available"]
.as_array()
.unwrap()
.iter()
.map(|id| find(id.as_str().unwrap()))
.collect();
let (picked, bound) = s
.pick_session(&tag(&available), &sel, &policy, &|_| Windows::default(), now)
.unwrap();
assert_eq!(picked.id, step["picked"].as_str().unwrap(), "{name} step {i}");
let got = bound.as_ref().map(|m| {
let (node_kind, fork, compaction) = m.node();
serde_json::json!({
"session": m.session, "parent": m.parent, "node_kind": node_kind,
"fork": fork, "compaction": compaction, "generation": m.access,
})
});
assert_eq!(got.as_ref(), step.get("lcp"), "{name} step {i} lcp");
if op != "pick" {
let picked = picked.clone();
s.session_result(&picked, &sel, &outcome(op), &policy, bound.as_ref(), now);
}
}
op => {
s.session_result(
find(step["auth"].as_str().unwrap()),
&sel,
&outcome(op),
&policy,
None,
now,
);
}
}
}
}
}
#[test]
fn delayed_failure_keeps_binding_made_after_policy_change() {
let a = cred("a", serde_json::json!({}));
let p = Policy {
session_affinity: true,
..Default::default()
};
let next = Policy {
session_affinity_ttl: Duration::from_secs(600),
..p.clone()
};
let now = Instant::now();
let mut s = Scheduler::default();
let body = br#"{"messages":[{"role":"user","content":"hello"}]}"#;
let request = crate::lcp::Request::new("openai", body, "client-key").unwrap();
let sel = Selection {
lcp: Some(std::sync::Arc::new(request.clone())),
..selection("m")
};
let ranks = |_: &Credential| Windows::default();
let mut old = None;
for _ in 0..5 {
old = s.pick_session(&tag(&[&a]), &sel, &p, &ranks, now).unwrap().1;
}
let old = old.unwrap();
s.configure(&p, &next);
let fresh = s
.pick_session(&tag(&[&a]), &sel, &next, &ranks, now)
.unwrap()
.1
.unwrap();
assert!(fresh.access > old.access, "{} after {}", fresh.access, old.access);
let failure = Outcome::Failure(ExecError::local(500, FailureScope::Credential, "boom"));
s.session_result(&a, &sel, &failure, &p, Some(&old), now);
let namespace = request.namespace("m");
let kept = s.lcp.find(&namespace, request.prepared(), now);
assert_eq!(kept.map(|m| m.auth), Some("a".to_owned()));
}
#[test]
fn leases_from_before_disabling_affinity_bind_nothing() {
let a = cred("a", serde_json::json!({}));
let on = Policy {
session_affinity: true,
..Default::default()
};
let off = Policy::default();
let now = Instant::now();
let mut s = Scheduler::default();
s.configure(&off, &on);
let body = br#"{"messages":[{"role":"user","content":"hello"}]}"#;
let request = crate::lcp::Request::new("openai", body, "client-key").unwrap();
let namespace = request.namespace("m");
let lcp = Selection {
lcp: Some(std::sync::Arc::new(request.clone())),
..selection("m")
};
let explicit = Selection {
session: Some("s1".into()),
..selection("m")
};
let ranks = |_: &Credential| Windows::default();
let bound = s.pick_session(&tag(&[&a]), &lcp, &on, &ranks, now).unwrap().1;
assert!(bound.is_some());
s.configure(&on, &off);
assert!(s.lcp.find(&namespace, request.prepared(), now).is_none());
s.session_result(&a, &lcp, &Outcome::Success, &on, bound.as_ref(), now);
s.session_result(&a, &explicit, &Outcome::Success, &on, None, now);
let retry = s.pick_session(&tag(&[&a]), &lcp, &on, &ranks, now).unwrap().1;
assert!(retry.is_none());
s.pick_session(&tag(&[&a]), &explicit, &on, &ranks, now);
assert!(s.lcp.find(&namespace, request.prepared(), now).is_none());
assert!(s.affinity.get(&session_keys(&explicit).unwrap().primary, now).is_none());
s.configure(&off, &on);
assert!(s.pick_session(&tag(&[&a]), &lcp, &on, &ranks, now).unwrap().1.is_some());
}
#[test]
fn policy_change_resets_rotation_and_affinity_but_keeps_cooldowns() {
let a = cred("a", serde_json::json!({}));
let b = cred("b", serde_json::json!({}));
let p = Policy {
session_affinity: true,
..Default::default()
};
let now = Instant::now();
let mut s = Scheduler::default();
let mut selection = selection("m");
selection.session = Some("session".into());
assert_eq!(s.pick(&tag(&[&a, &b]), &selection, &p, now).unwrap().id, "a");
s.record(
&a,
"other",
&Outcome::Failure(ExecError::local(429, FailureScope::Model, "quota")),
&p,
now,
);
let mut next = p.clone();
next.request_retry = 2;
s.configure(&p, &next);
assert!(!s.rotations.is_empty(), "retry-only change preserves selector state");
assert!(!s.affinity.is_empty());
next.session_affinity_ttl = Duration::from_secs(5);
s.configure(&p, &next);
assert!(s.rotations.is_empty());
assert!(s.affinity.is_empty());
assert_eq!(s.wait(&a, "other", now), Some(Duration::from_secs(1)));
}
#[test]
fn forced_cooling_uses_fallback_and_quota_does_not_reuse_transient_state() {
let now = Instant::now();
for (status, seconds) in [(401, 1800), (404, 43200), (429, 1), (503, 60)] {
let c = cred(
"a",
serde_json::json!({"disable_cooling":true,
"request_scoped_errors":[{"status":status,"match":["forced"],"action":"stop-and-cooldown"}]}),
);
let mut s = Scheduler::default();
s.record(
&c,
"m",
&Outcome::Failure(ExecError::local(status, FailureScope::Credential, "forced")),
&Policy::default(),
now,
);
assert_eq!(
s.wait(&c, "m", now),
Some(Duration::from_secs(seconds)),
"status {status}"
);
}
let c = cred(
"a",
serde_json::json!({"request_scoped_errors":[{"status":503,"match":["forced"],"action":"continue-and-cooldown"}]}),
);
let no_transient = Policy {
transient_error_cooldown_seconds: -1,
..Policy::default()
};
let mut s = Scheduler::default();
let forced = Outcome::Failure(ExecError::local(503, FailureScope::Credential, "forced"));
s.record(&c, "m", &forced, &no_transient, now);
assert_eq!(s.wait(&c, "m", now), Some(Duration::from_secs(60)));
let plain = Outcome::Failure(ExecError::local(503, FailureScope::Credential, "plain"));
let mut s = Scheduler::default();
s.record(&c, "m", &plain, &no_transient, now);
assert_eq!(
s.wait(&c, "m", now),
None,
"negative transient seconds disable 503 cooldowns"
);
let c = cred("a", serde_json::json!({}));
let p = Policy::default();
let mut s = Scheduler::default();
s.record(
&c,
"m",
&Outcome::Failure(ExecError::local(503, FailureScope::Model, "transient")),
&p,
now,
);
s.record(
&c,
"m",
&Outcome::Failure(ExecError::local(429, FailureScope::Model, "quota")),
&p,
now,
);
assert!(
s.quota_cooling(&c, "m", now),
"quota must replace transient classification"
);
assert_eq!(
s.wait(&c, "m", now),
Some(Duration::from_secs(60)),
"live deadline never shortened"
);
let mut hint = ExecError::local(418, FailureScope::Model, "other status");
hint.retry_after = Some(Duration::from_secs(900));
s.record(&c, "other", &Outcome::Failure(hint), &p, now);
assert_eq!(
s.wait(&c, "other", now),
Some(Duration::from_secs(60)),
"unlisted statuses ignore hint"
);
}
#[test]
fn compat_groups_match_with_go_equal_fold() {
let cfg = cpa_core::config::Config::parse(
"openai-compatibility:\n - name: openai-compatible-ος\n base-url: http://127.0.0.1:9\n disable-cooling: true\n api-key-entries: [{api-key: k, proxy-url: 'http://127.0.0.1:8080'}]\n - name: ΟΣ\n base-url: http://127.0.0.1:9\n api-key-entries: [{api-key: k}]\n",
)
.unwrap();
let creds = cpa_core::config::credentials::from_config(&cfg);
let c = creds
.iter()
.find(|c| c.attributes.get("compat_name").is_some_and(|n| n == "ΟΣ"))
.unwrap();
assert_eq!(cpa_core::registry::dynamic::provider_key(c), "openai-compatible-οσ");
assert_eq!(
cpa_core::config::credentials::api_key_config_proxy(&cfg, c),
"http://127.0.0.1:8080"
);
let mut policy = crate::management::policy(&cfg);
policy.compat_disable_cooling = compat_cooling(&cfg);
assert_eq!(policy.provider_cooling(c), Some(true));
}
#[test]
fn oauth_rule_channels_lowercase_like_go() {
let cfg = cpa_core::config::Config::parse(
"oauth:\n request-scoped-errors:\n HİGH:\n - {status: 503, match: [boom], action: stop}\n",
)
.unwrap();
let c = cpa_core::config::credentials::from_file(
&cfg,
std::path::Path::new("/auth"),
std::path::Path::new("/auth/a.json"),
r#"{"type":"HİGH","access_token":"t"}"#.as_bytes(),
)
.unwrap()
.unwrap();
let action =
crate::management::policy(&cfg).error_action(&c, &ExecError::local(503, FailureScope::Request, "boom"));
assert!(action.matched && action.stop);
}
#[test]
fn rules_match_the_structured_body_of_a_missing_thread() {
let c = cred(
"a",
serde_json::json!({"request_scoped_errors":[{"status":404,"match":["not_found_error"],"action":"continue"}]}),
);
let e = ExecError::local(404, FailureScope::Request, cpa_exec::claude::THREAD_NOT_FOUND);
let action = Policy::default().error_action(&c, &e);
assert!(action.matched && !action.stop);
let other = ExecError::local(404, FailureScope::Request, "not found");
assert!(!Policy::default().error_action(&c, &other).matched);
let mut upstream = e.clone();
upstream
.headers
.insert(axum::http::header::CONTENT_TYPE, "text/plain".parse().unwrap());
assert!(!Policy::default().error_action(&c, &upstream).matched);
}
#[test]
fn rules_are_ordered_status_and_body_matches_with_canonical_precedence() {
let c = cred(
"a",
serde_json::json!({"disable_cooling":true, "request_scoped_errors":[
{"status":429,"match":[],"action":"stop"},
{"status":500,"match":["quota"],"action":"stop"},
{"status":429,"match-regexr":["[", "quota [0-9]+"],"action":" CONTINUE-AND-COOLDOWN "},
{"status":429,"match":["quota"],"action":"stop"}]}),
);
let p = Policy::default();
let e = ExecError::local(429, FailureScope::Request, "quota 12");
let action = p.error_action(&c, &e);
assert!(!action.stop);
assert!(action.cooldown && action.force_cooldown);
let now = Instant::now();
let mut s = Scheduler::default();
s.record(&c, "m", &Outcome::Failure(e), &p, now);
assert!(
s.wait(&c, "m", now).is_some(),
"force cooling bypasses disabled cooling"
);
let c = cred(
"a",
serde_json::json!({"request_retry":0, "request-retry":5,
"disable_cooling":false,"disable-cooling":true}),
);
assert_eq!(p.retry_limit(&c), 0);
assert!(!p.cooling_disabled(&c));
let legacy = cred(
"legacy",
serde_json::json!({"request_retry":null,"request-retry":2,
"disable_cooling":"TrUe","disable-cooling":1}),
);
assert_eq!(
p.retry_limit(&legacy),
2,
"invalid canonical override permits legacy fallback"
);
assert!(
p.cooling_disabled(&legacy),
"Go ParseBool rejects mixed case; numeric legacy accepted"
);
assert!(!p.cooling_disabled(&cred("numeric", serde_json::json!({"disable_cooling":0}))));
assert_eq!(
Policy {
request_retry: 3,
..Default::default()
}
.retry_limit(&cred("b", serde_json::json!({"request_retry":"-1"}))),
3
);
let cfg = cpa_core::config::Config::parse(
"openai-compatibility:\n - {name: skipped, base-url: '', disable-cooling: false}\n - {name: Router, base-url: http://r.invalid, disable-cooling: true}\n - {name: plain, base-url: http://p.invalid}\n",
)
.unwrap();
let mut p = Policy {
compat_disable_cooling: compat_cooling(&cfg),
..Default::default()
};
assert_eq!(
p.compat_disable_cooling,
[("Router".to_owned(), Some(true)), ("plain".to_owned(), None)],
"legacy openai-compatibility is read through the v8 document"
);
let compat = |attrs: &[(&str, &str)], meta: Value| {
let mut c = cred("compat", meta);
c.provider = "openai-compatibility".into();
for (k, v) in attrs {
c.attributes.insert((*k).into(), (*v).into());
}
c
};
assert!(p.cooling_disabled(&compat(&[("compat_name", "router")], serde_json::json!({}))));
assert!(p.cooling_disabled(&compat(&[("provider_key", "ROUTER")], serde_json::json!({}))));
assert!(!p.cooling_disabled(&compat(
&[("compat_name", "router")],
serde_json::json!({"disable_cooling": false})
)));
p.disable_cooling = true;
assert!(
p.cooling_disabled(&compat(&[("compat_name", "plain")], serde_json::json!({}))),
"an entry without disable-cooling defers to global"
);
p.disable_cooling = false;
assert!(
!p.cooling_disabled(&cred("claude-file", serde_json::json!({}))),
"credentials outside OpenAI compatibility ignore entries"
);
for scope in [FailureScope::Request, FailureScope::Transport] {
let mut s = Scheduler::default();
s.record(
&c,
"m",
&Outcome::Failure(ExecError::local(502, scope, "fault")),
&p,
now,
);
assert_eq!(s.wait(&c, "m", now), None);
}
}
const DAY: u64 = 24 * 3600;
fn epoch(t: SystemTime) -> String {
t.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs().to_string()
}
fn claude(now: SystemTime, weekly: u64, five_hour: &str, five_hour_reset: SystemTime) -> Windows {
claude_at(now, now, weekly, five_hour, five_hour_reset)
}
fn claude_at(
observed: SystemTime,
now: SystemTime,
weekly: u64,
five_hour: &str,
five_hour_reset: SystemTime,
) -> Windows {
let signals: std::collections::BTreeMap<String, String> = [
("Anthropic-Ratelimit-Unified-7d-Status", "allowed".to_owned()),
("Anthropic-Ratelimit-Unified-7d-Utilization", "0.40".to_owned()),
(
"Anthropic-Ratelimit-Unified-7d-Reset",
epoch(observed + Duration::from_secs(weekly)),
),
("Anthropic-Ratelimit-Unified-5h-Status", five_hour.to_owned()),
("Anthropic-Ratelimit-Unified-5h-Reset", epoch(five_hour_reset)),
]
.into_iter()
.map(|(k, v)| (k.to_owned(), v))
.collect();
Windows::observed("claude", &signals, observed, now)
}
fn soonest() -> Policy {
Policy {
strategy: Strategy::SoonestReset,
..Default::default()
}
}
#[test]
fn soonest_reset_spends_the_earliest_weekly_window_first() {
let wall = SystemTime::now();
let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
let in_an_hour = wall + Duration::from_secs(3600);
let ranks_at = |b_five_hour: &'static str, now: SystemTime| {
let (wa, wb) = (
claude_at(wall, now, 4 * DAY, "allowed", in_an_hour),
claude_at(wall, now, DAY, b_five_hour, in_an_hour),
);
move |cred: &Credential| if cred.id == "a" { wa } else { wb }
};
let policy = soonest();
let mut s = Scheduler::default();
let now = Instant::now();
let ranks = ranks_at("allowed", wall);
for _ in 0..3 {
let picked = s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &policy, &ranks, now);
assert_eq!(picked.unwrap().id, "b", "the sooner weekly reset is used up first");
}
s.record(
&b,
"m",
&Outcome::Failure(ExecError::local(429, FailureScope::Credential, "quota")),
&policy,
now,
);
let ready: Vec<&Credential> = [&a, &b].into_iter().filter(|x| s.wait(x, "m", now).is_none()).collect();
assert_eq!(ready.iter().map(|x| x.id.as_str()).collect::<Vec<_>>(), ["a"]);
assert_eq!(
s.pick_ranked(&tag(&ready), &selection("m"), &policy, &ranks, now)
.unwrap()
.id,
"a"
);
let mut s = Scheduler::default();
let exhausted = ranks_at("rejected", wall);
for _ in 0..2 {
assert_eq!(
s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &policy, &exhausted, now)
.unwrap()
.id,
"a"
);
}
let reset = ranks_at("rejected", in_an_hour + Duration::from_secs(1));
assert_eq!(
s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &policy, &reset, now)
.unwrap()
.id,
"b"
);
}
#[test]
fn soonest_reset_probes_unknown_accounts_once_then_ranks_them() {
let wall = SystemTime::now();
let creds: Vec<Credential> = ["a", "b", "c", "d"]
.iter()
.map(|id| cred(id, serde_json::json!({})))
.collect();
let [a, b, c, d] = [&creds[0], &creds[1], &creds[2], &creds[3]];
let in_an_hour = wall + Duration::from_secs(3600);
let known = |x: &Credential| match x.id.as_str() {
"c" => claude(wall, DAY, "rejected", in_an_hour),
"d" => claude(wall, 3 * DAY, "allowed", in_an_hour),
_ => Windows::default(),
};
let policy = soonest();
let mut s = Scheduler::default();
let now = Instant::now();
let pick = |s: &mut Scheduler, set: &[&Credential], ranks: &Ranks<'_>| {
s.pick_ranked(&tag(set), &selection("m"), &policy, ranks, now)
.unwrap()
.id
.clone()
};
let order: Vec<String> = (0..4).map(|_| pick(&mut s, &[a, b, c, d], &known)).collect();
assert_eq!(order, ["a", "b", "d", "d"]);
let learned = |x: &Credential| match x.id.as_str() {
"a" => claude(wall + Duration::from_secs(1), 12 * 3600, "allowed", in_an_hour),
_ => known(x),
};
assert_eq!(pick(&mut s, &[a, b, c, d], &learned), "a");
assert_eq!(pick(&mut s, &[b, c, d], &learned), "d");
assert_eq!(
pick(&mut s, &[b, c], &learned),
"b",
"probed and unknown, still before used-up"
);
assert_eq!(
pick(&mut s, &[c], &learned),
"c",
"a used-up account is the last resort"
);
let passed = |x: &Credential| match x.id.as_str() {
"d" => claude_at(
wall,
wall + Duration::from_secs(3 * DAY + 1),
3 * DAY,
"allowed",
in_an_hour,
),
_ => learned(x),
};
assert_eq!(pick(&mut s, &[a, b, d], &passed), "d");
assert_eq!(pick(&mut s, &[a, b, d], &passed), "a");
}
#[test]
fn soonest_reset_rotates_equal_ranks() {
let wall = SystemTime::now();
let creds: Vec<Credential> = ["a", "b"].iter().map(|id| cred(id, serde_json::json!({}))).collect();
let in_an_hour = wall + Duration::from_secs(3600);
let ranks = |_: &Credential| claude(wall, 2 * DAY, "allowed", in_an_hour);
let mut s = Scheduler::default();
let now = Instant::now();
let turns: Vec<String> = (0..4)
.map(|_| {
s.pick_ranked(&tag(&[&creds[0], &creds[1]]), &selection("m"), &soonest(), &ranks, now)
.unwrap()
.id
.clone()
})
.collect();
assert_eq!(turns, ["a", "b", "a", "b"]);
}
#[test]
fn soonest_reset_never_moves_a_session_pin() {
let wall = SystemTime::now();
let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
let in_an_hour = wall + Duration::from_secs(3600);
let ranks = |x: &Credential| claude(wall, if x.id == "a" { 4 * DAY } else { DAY }, "allowed", in_an_hour);
let policy = Policy {
session_affinity: true,
..soonest()
};
let mut s = Scheduler::default();
let now = Instant::now();
let mut pinned = selection("m");
pinned.session = Some("claude:thread-1".into());
assert_eq!(
s.pick_ranked(&tag(&[&a]), &pinned, &policy, &ranks, now).unwrap().id,
"a"
);
for _ in 0..2 {
assert_eq!(
s.pick_ranked(&tag(&[&a, &b]), &pinned, &policy, &ranks, now)
.unwrap()
.id,
"a"
);
}
let mut other = selection("m");
other.session = Some("claude:thread-2".into());
assert_eq!(
s.pick_ranked(&tag(&[&a, &b]), &other, &policy, &ranks, now).unwrap().id,
"b"
);
}
#[test]
fn default_strategy_stays_round_robin_and_ignores_resets() {
assert_eq!(Policy::default().strategy, Strategy::RoundRobin);
let wall = SystemTime::now();
let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
let ranks = |x: &Credential| claude(wall, if x.id == "a" { 4 * DAY } else { DAY }, "allowed", wall);
let mut s = Scheduler::default();
let now = Instant::now();
let turns: Vec<String> = (0..4)
.map(|_| {
s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &Policy::default(), &ranks, now)
.unwrap()
.id
.clone()
})
.collect();
assert_eq!(turns, ["a", "b", "a", "b"]);
}
#[test]
fn soonest_reset_in_mixed_providers_stays_in_the_first_group() {
let wall = SystemTime::now();
let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
let x = cred("x", serde_json::json!({}));
let in_an_hour = wall + Duration::from_secs(3600);
let ranks = |c: &Credential| {
let days = match c.id.as_str() {
"a" => 4,
"b" => 2,
_ => 1,
};
claude(wall, days * DAY, "allowed", in_an_hour)
};
let sel = Selection {
providers: vec!["claude".into(), "codex".into()],
model: "m".into(),
..Default::default()
};
let candidates = [(&a, "claude"), (&b, "claude"), (&x, "codex")];
let mut s = Scheduler::default();
let picked = s.pick_ranked(&candidates, &sel, &soonest(), &ranks, Instant::now());
assert_eq!(picked.unwrap().id, "b");
}
#[test]
fn windows_read_claude_and_codex_signals() {
let now = SystemTime::now();
let map = |pairs: &[(&str, String)]| -> std::collections::BTreeMap<String, String> {
pairs.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect()
};
let stale = map(&[(
"Anthropic-Ratelimit-Unified-7d-Reset",
epoch(now - Duration::from_secs(60)),
)]);
let w = Windows::observed("claude", &stale, now, now);
assert_eq!(
(
w.weekly_reset,
w.rolled_over.map(|r| (r.window, epoch(r.at))),
w.exhausted
),
(
None,
Some((WindowId::Claude7d, epoch(now - Duration::from_secs(60)))),
false
)
);
let full = map(&[
("Anthropic-Ratelimit-Unified-5h-Utilization", "1.0".into()),
(
"Anthropic-Ratelimit-Unified-5h-Reset",
epoch(now + Duration::from_secs(600)),
),
]);
assert!(Windows::observed("claude", &full, now, now).exhausted);
assert!(!Windows::observed("claude", &full, now, now + Duration::from_secs(601)).exhausted);
let observed = now - Duration::from_secs(100);
let codex = map(&[
("X-Codex-Primary-Used-Percent", "100".into()),
("X-Codex-Primary-Window-Minutes", "300".into()),
("X-Codex-Primary-Reset-After-Seconds", "3600".into()),
("X-Codex-Secondary-Used-Percent", "40".into()),
("X-Codex-Secondary-Window-Minutes", "10080".into()),
("X-Codex-Secondary-Reset-At", epoch(now + Duration::from_secs(2 * DAY))),
]);
let w = Windows::observed("codex", &codex, observed, now);
assert!(w.exhausted, "the 5-hour primary is used up");
assert_eq!(
w.weekly_reset.map(epoch),
Some(epoch(now + Duration::from_secs(2 * DAY)))
);
assert!(!Windows::observed("codex", &codex, observed, observed + Duration::from_secs(3601)).exhausted);
let weekly_primary = map(&[
("X-Codex-Primary-Used-Percent", "51".into()),
("X-Codex-Primary-Window-Minutes", "10080".into()),
("X-Codex-Primary-Reset-At", epoch(now + Duration::from_secs(3 * DAY))),
]);
let w = Windows::observed("codex", &weekly_primary, now, now);
assert_eq!(
(w.weekly_reset.map(epoch), w.exhausted),
(Some(epoch(now + Duration::from_secs(3 * DAY))), false)
);
let reached = map(&[("X-Codex-Limit-Reached", "True".into())]);
assert!(Windows::observed("codex", &reached, now, now).exhausted);
let w = Windows::observed("gemini", &codex, now, now);
assert_eq!((w.weekly_reset, w.exhausted), (None, false));
}
fn signals(pairs: &[(&str, String)]) -> std::collections::BTreeMap<String, String> {
pairs.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect()
}
#[test]
fn soonest_reset_answers_without_a_reset_do_not_re_arm_the_probe() {
let wall = SystemTime::now();
let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
let policy = soonest();
let mut s = Scheduler::default();
let now = Instant::now();
let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
let mut picks = Vec::new();
for i in 0..5u64 {
let a_seen = (i > 0).then(|| {
let seen = wall + Duration::from_secs(i);
Windows::observed(
"claude",
&signals(&[("Anthropic-Ratelimit-Unified-5h-Status", "allowed".into())]),
seen,
seen,
)
});
let ranks = |x: &Credential| {
if x.id == "a" {
a_seen.unwrap_or_default()
} else {
b_known
}
};
picks.push(
s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &policy, &ranks, now)
.unwrap()
.id
.clone(),
);
}
assert_eq!(picks, ["a", "b", "b", "b", "b"]);
}
#[test]
fn soonest_reset_a_probe_that_fails_before_headers_is_spent() {
let wall = SystemTime::now();
let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
let ranks = |x: &Credential| if x.id == "a" { Windows::default() } else { b_known };
let mut s = Scheduler::default();
let now = Instant::now();
let mut pick = || {
s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
.unwrap()
.id
.clone()
};
assert_eq!(pick(), "a", "the probe");
assert_eq!([pick(), pick()], ["b", "b"]);
}
#[test]
fn soonest_reset_re_arms_only_on_a_rollover() {
let wall = SystemTime::now();
let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
let a_reset = wall + Duration::from_secs(60);
let a_rolled = |seen: SystemTime| {
Windows::observed(
"claude",
&signals(&[("Anthropic-Ratelimit-Unified-7d-Reset", epoch(a_reset))]),
seen,
wall + Duration::from_secs(61),
)
};
let mut s = Scheduler::default();
let now = Instant::now();
let mut picks = Vec::new();
for i in 0..3u64 {
let a_now = a_rolled(wall + Duration::from_secs(i));
let ranks = |x: &Credential| if x.id == "a" { a_now } else { b_known };
picks.push(
s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
.unwrap()
.id
.clone(),
);
}
assert_eq!(picks, ["a", "b", "b"], "one probe per rollover");
let next = a_reset + Duration::from_secs(7 * DAY);
let later = Windows::observed(
"claude",
&signals(&[("Anthropic-Ratelimit-Unified-7d-Reset", epoch(next))]),
wall,
next + Duration::from_secs(1),
);
let ranks = |x: &Credential| if x.id == "a" { later } else { b_known };
let mut pick = || {
s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
.unwrap()
.id
.clone()
};
assert_eq!([pick(), pick()], ["a", "b"]);
}
#[test]
fn soonest_reset_zero_relative_resets_do_not_re_arm_the_probe() {
let wall = SystemTime::now();
let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
let zero = signals(&[
("X-Codex-Secondary-Used-Percent", "0".into()),
("X-Codex-Secondary-Window-Minutes", "10080".into()),
("X-Codex-Secondary-Reset-After-Seconds", "0".into()),
]);
let mut s = Scheduler::default();
let now = Instant::now();
let mut picks = Vec::new();
for i in 0..4u64 {
let a_now = (i > 0).then(|| {
let seen = wall + Duration::from_secs(2 * i);
Windows::observed("codex", &zero, seen, seen + Duration::from_secs(1))
});
let ranks = |x: &Credential| {
if x.id == "a" {
a_now.unwrap_or_default()
} else {
b_known
}
};
picks.push(
s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
.unwrap()
.id
.clone(),
);
}
assert_eq!(picks, ["a", "b", "b", "b"]);
let used_up = signals(&[
("X-Codex-Primary-Used-Percent", "100".into()),
("X-Codex-Primary-Reset-After-Seconds", "0".into()),
]);
assert!(!Windows::observed("codex", &used_up, wall, wall).exhausted);
}
#[test]
fn soonest_reset_rollovers_within_half_a_window_are_one_rollover() {
let wall = SystemTime::now();
let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
let codex = |seen: SystemTime, after: u64, read: SystemTime| {
Windows::observed(
"codex",
&signals(&[
("X-Codex-Secondary-Window-Minutes", "10080".into()),
("X-Codex-Secondary-Reset-After-Seconds", after.to_string()),
]),
seen,
read,
)
};
let mut s = Scheduler::default();
let now = Instant::now();
let mut pick = |a_now: Windows| {
let ranks = |x: &Credential| if x.id == "a" { a_now } else { b_known };
s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
.unwrap()
.id
.clone()
};
let boundary = wall + Duration::from_secs(600);
assert_eq!(pick(codex(wall, 600, boundary + Duration::from_secs(1))), "a");
for i in 1..4u64 {
let seen = boundary + Duration::from_secs(i);
assert_eq!(pick(codex(seen, 1, seen + Duration::from_secs(2))), "b", "answer {i}");
}
let next = boundary + Duration::from_secs(7 * DAY);
assert_eq!(
pick(codex(next - Duration::from_secs(5), 5, next + Duration::from_secs(1))),
"a"
);
assert_eq!(
pick(codex(next - Duration::from_secs(5), 5, next + Duration::from_secs(1))),
"b"
);
}
#[test]
fn soonest_reset_probe_markers_are_kept_per_window() {
let wall = SystemTime::now();
let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
let b_known = claude(wall, 5 * DAY, "allowed", wall + Duration::from_secs(3600));
let r = wall + Duration::from_secs(600);
let after = |t: SystemTime, secs: u64| t + Duration::from_secs(secs);
let partial = signals(&[("X-Codex-Secondary-Reset-At", epoch(r))]);
let complete = signals(&[
("X-Codex-Primary-Used-Percent", "10".into()),
("X-Codex-Primary-Window-Minutes", "10080".into()),
("X-Codex-Primary-Reset-At", epoch(after(r, DAY))),
("X-Codex-Secondary-Used-Percent", "10".into()),
("X-Codex-Secondary-Window-Minutes", "300".into()),
("X-Codex-Secondary-Reset-At", epoch(after(r, 5 * 3600))),
]);
let mut s = Scheduler::default();
let now = Instant::now();
let mut pick = |a_now: Windows| {
let ranks = |x: &Credential| if x.id == "a" { a_now } else { b_known };
s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
.unwrap()
.id
.clone()
};
let w = Windows::observed("codex", &partial, wall, after(r, 1));
assert_eq!(w.rolled_over.map(|r| r.window), Some(WindowId::CodexSecondary));
assert_eq!(pick(w), "a");
assert_eq!(
pick(Windows::observed("codex", &complete, after(r, 1), after(r, 2))),
"a"
);
let w = Windows::observed("codex", &complete, after(r, 1), after(r, DAY + 1));
assert_eq!(w.rolled_over.map(|r| r.window), Some(WindowId::CodexPrimary));
assert_eq!(pick(w), "a");
assert_eq!(pick(w), "b", "probed for that rollover");
}
#[test]
fn soonest_reset_management_reset_re_arms_the_probe_and_keeps_affinity() {
let wall = SystemTime::now();
let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
let ranks = |x: &Credential| if x.id == "a" { Windows::default() } else { b_known };
let policy = Policy {
session_affinity: true,
..soonest()
};
let mut s = Scheduler::default();
let now = Instant::now();
let mut thread = selection("m");
thread.session = Some("claude:thread-1".into());
let mut pick = |sel: &Selection, set: &[&Credential]| {
s.pick_ranked(&tag(set), sel, &policy, &ranks, now).unwrap().id.clone()
};
assert_eq!(pick(&thread, &[&a]), "a");
let fresh = selection("m");
assert_eq!(pick(&fresh, &[&a, &b]), "b");
s.reset_cooldowns("a");
let mut pick = |sel: &Selection, set: &[&Credential]| {
s.pick_ranked(&tag(set), sel, &policy, &ranks, now).unwrap().id.clone()
};
assert_eq!(pick(&fresh, &[&a, &b]), "a", "probed again");
assert_eq!(pick(&fresh, &[&a, &b]), "b");
assert_eq!(pick(&thread, &[&a, &b]), "a", "the pin is kept");
}
#[test]
fn windows_survive_unrepresentable_window_lengths() {
let now = SystemTime::now();
for (used, exhausted) in [("0", false), ("100", true)] {
let huge = signals(&[
("X-Codex-Primary-Used-Percent", used.into()),
("X-Codex-Primary-Window-Minutes", "1e308".into()),
]);
assert_eq!(
Windows::observed("codex", &huge, now, now).exhausted,
exhausted,
"{used}%"
);
let later = now + Duration::from_secs(5 * 3600 + 1);
assert!(
!Windows::observed("codex", &huge, now, later).exhausted,
"{used}% after five hours"
);
}
let long = signals(&[
("X-Codex-Primary-Used-Percent", "100".into()),
("X-Codex-Primary-Window-Minutes", "1e9".into()),
]);
assert!(Windows::observed("codex", &long, now, now + Duration::from_secs(DAY)).exhausted);
}
#[test]
fn codex_limit_reached_expires_with_the_window_reset() {
let now = SystemTime::now();
let flagged = signals(&[
("X-Codex-Limit-Reached", "true".into()),
("X-Codex-Primary-Used-Percent", "100".into()),
("X-Codex-Primary-Window-Minutes", "300".into()),
("X-Codex-Primary-Reset-At", epoch(now + Duration::from_secs(60))),
]);
assert!(Windows::observed("codex", &flagged, now, now).exhausted);
assert!(!Windows::observed("codex", &flagged, now, now + Duration::from_secs(61)).exhausted);
let below = signals(&[
("X-Codex-Limit-Reached", "true".into()),
("X-Codex-Primary-Used-Percent", "40".into()),
("X-Codex-Primary-Reset-At", epoch(now + Duration::from_secs(600))),
]);
assert!(Windows::observed("codex", &below, now, now + Duration::from_secs(599)).exhausted);
assert!(!Windows::observed("codex", &below, now, now + Duration::from_secs(601)).exhausted);
let bare = signals(&[("X-Codex-Limit-Reached", "true".into())]);
assert!(Windows::observed("codex", &bare, now, now + Duration::from_secs(5 * 3600 - 1)).exhausted);
assert!(!Windows::observed("codex", &bare, now, now + Duration::from_secs(5 * 3600 + 1)).exhausted);
}
#[test]
fn codex_windows_parse_usage_reset_and_length_independently() {
let now = SystemTime::now();
let weekly = now + Duration::from_secs(3 * DAY);
let no_usage = signals(&[
("X-Codex-Secondary-Window-Minutes", "10080".into()),
("X-Codex-Secondary-Reset-At", epoch(weekly)),
]);
let w = Windows::observed("codex", &no_usage, now, now);
assert_eq!((w.weekly_reset.map(epoch), w.exhausted), (Some(epoch(weekly)), false));
let reset_only = signals(&[("X-Codex-Secondary-Reset-After-Seconds", "7200".into())]);
let w = Windows::observed("codex", &reset_only, now, now);
assert_eq!(w.weekly_reset.map(epoch), Some(epoch(now + Duration::from_secs(7200))));
}
#[test]
fn codex_primary_longer_than_secondary_is_the_weekly_window() {
let now = SystemTime::now();
let weekly = now + Duration::from_secs(3 * DAY);
let both = signals(&[
("X-Codex-Primary-Used-Percent", "51".into()),
("X-Codex-Primary-Window-Minutes", "10080".into()),
("X-Codex-Primary-Reset-At", epoch(weekly)),
("X-Codex-Secondary-Used-Percent", "100".into()),
("X-Codex-Secondary-Window-Minutes", "300".into()),
("X-Codex-Secondary-Reset-After-Seconds", "3600".into()),
]);
let w = Windows::observed("codex", &both, now, now);
assert_eq!((w.weekly_reset.map(epoch), w.exhausted), (Some(epoch(weekly)), true));
let w = Windows::observed("codex", &both, now, now + Duration::from_secs(3601));
assert_eq!((w.weekly_reset.map(epoch), w.exhausted), (Some(epoch(weekly)), false));
}
}