Skip to main content

cpa_server/
scheduler.rs

1//! Credential routing policy. Config parsing remains owned by cpa-core; integration
2//! publishes this policy through Runtime::publish_policy.
3
4use std::collections::HashMap;
5use std::time::{Duration, Instant, SystemTime};
6
7use cpa_common::gostr::GoStr;
8use cpa_core::credential::Credential;
9use cpa_core::exec::ExecError;
10#[cfg(test)]
11use cpa_core::exec::FailureScope;
12use serde::Deserialize;
13use serde_json::Value;
14
15use crate::runtime::{Outcome, Selection};
16
17#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
18pub enum Strategy {
19    #[default]
20    RoundRobin,
21    FillFirst,
22    WeightedRoundRobin,
23    /// cliproxy-rs addition (`soonest-reset`, alias `reset-first`) and the default for
24    /// an unset strategy: spend the account whose weekly window resets soonest first,
25    /// until it cools or a usage window runs out, then the next soonest. Go has no such
26    /// strategy and reads it, like an unset one, as round-robin.
27    SoonestReset,
28}
29
30/// What `soonest-reset` knows about a credential's usage windows, from its latest
31/// passive quota observation (Claude `anthropic-ratelimit-unified-*`, Codex
32/// `x-codex-*`). The default is "nothing known".
33#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
34pub struct Windows {
35    /// When the long (weekly) window resets, while that is still ahead.
36    pub weekly_reset: Option<SystemTime>,
37    /// The long window's reset once it has passed: the window rolled over, so what was
38    /// learned about it is stale and the credential may be probed again. Only a reset
39    /// upstream stated (an absolute time, or a delay above zero) identifies a rollover;
40    /// a zero delay would make every answer's observation time a new one.
41    pub rolled_over: Option<Rollover>,
42    /// A window (Claude 5-hour or 7-day, Codex primary or secondary) is used up and has
43    /// not reset yet: upstream would refuse until it does.
44    pub exhausted: bool,
45}
46
47/// Which window a rollover belongs to. Probe markers are kept per window, so a
48/// rollover of one window is never compared with a probe made for another.
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
50pub enum WindowId {
51    Claude7d,
52    Claude5h,
53    CodexPrimary,
54    CodexSecondary,
55}
56
57/// A long window that rolled over.
58#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub struct Rollover {
60    pub window: WindowId,
61    /// The stated reset that passed.
62    pub at: SystemTime,
63    /// How far apart two rollovers of this window must be to count as different ones:
64    /// half its length (five hours when unknown), which absorbs the rounding of
65    /// relative resets around a boundary.
66    pub gap: Duration,
67}
68
69/// How long a used-up window without a usable reset or length is assumed to hold.
70const FALLBACK_WINDOW: Duration = Duration::from_secs(5 * 3600);
71
72/// One usage window read from quota signals; each part may be missing.
73#[derive(Debug, Clone, Copy)]
74struct Window {
75    id: WindowId,
76    used_up: bool,
77    reset: Option<SystemTime>,
78    /// `reset` when it can identify a rollover (not synthesized from a zero delay).
79    stated_reset: Option<SystemTime>,
80    /// The window length, when one was reported and can be represented.
81    length: Option<Duration>,
82}
83
84impl Window {
85    /// Until when a used-up window holds: its reset, else its length (or five hours)
86    /// after the observation. `None` when it is not used up.
87    fn exhausted_until(&self, observed_at: SystemTime) -> Option<SystemTime> {
88        if !self.used_up {
89            return None;
90        }
91        self.reset.or_else(|| {
92            self.length
93                .and_then(|l| observed_at.checked_add(l))
94                .or_else(|| observed_at.checked_add(FALLBACK_WINDOW))
95        })
96    }
97}
98
99/// A window length in minutes, when positive and representable as a deadline from
100/// `observed_at`; otherwise the missing-length policy applies.
101fn window_length(minutes: f64, observed_at: SystemTime) -> Option<Duration> {
102    if !minutes.is_finite() || minutes <= 0.0 {
103        return None;
104    }
105    let length = Duration::try_from_secs_f64(minutes * 60.0).ok()?;
106    observed_at.checked_add(length).map(|_| length)
107}
108
109/// A reset signal: epoch seconds (fractions allowed) or an RFC 3339 time.
110fn reset_time(raw: &str) -> Option<SystemTime> {
111    let raw = raw.trim();
112    if let Ok(seconds) = raw.parse::<f64>() {
113        return (seconds.is_finite() && seconds > 0.0)
114            .then(|| Duration::try_from_secs_f64(seconds).ok())
115            .flatten()
116            .and_then(|d| SystemTime::UNIX_EPOCH.checked_add(d));
117    }
118    chrono::DateTime::parse_from_rfc3339(raw).ok().map(SystemTime::from)
119}
120
121impl Windows {
122    /// Reads a snapshot's `signals` (header name to value), observed at `observed_at`,
123    /// for `provider` (`claude` or `codex`; anything else knows nothing).
124    pub fn observed(
125        provider: &str,
126        signals: &std::collections::BTreeMap<String, String>,
127        observed_at: SystemTime,
128        now: SystemTime,
129    ) -> Self {
130        let get = |name: &str| {
131            signals
132                .iter()
133                .find(|(k, _)| k.eq_ignore_ascii_case(name))
134                .map(|(_, v)| v.trim())
135        };
136        let number = |name: &str| get(name).and_then(|v| v.parse::<f64>().ok()).filter(|n| n.is_finite());
137        let codex = provider.trim().eq_ignore_ascii_case("codex");
138        let (long, short) = match provider.trim().to_ascii_lowercase().as_str() {
139            "claude" => {
140                let window = |id: WindowId, label: &str, minutes: f64| Window {
141                    id,
142                    used_up: get(&format!("anthropic-ratelimit-unified-{label}-status"))
143                        .is_some_and(|s| s.eq_ignore_ascii_case("rejected"))
144                        || number(&format!("anthropic-ratelimit-unified-{label}-utilization"))
145                            .is_some_and(|u| u >= 1.0),
146                    reset: get(&format!("anthropic-ratelimit-unified-{label}-reset")).and_then(reset_time),
147                    stated_reset: get(&format!("anthropic-ratelimit-unified-{label}-reset")).and_then(reset_time),
148                    length: window_length(minutes, observed_at),
149                };
150                (
151                    Some(window(WindowId::Claude7d, "7d", 7.0 * 24.0 * 60.0)),
152                    Some(window(WindowId::Claude5h, "5h", 300.0)),
153                )
154            }
155            "codex" => {
156                // Usage, reset and length are read independently; a window exists when
157                // any of them was reported.
158                let window = |id: WindowId, label: &str| {
159                    let p = format!("x-codex-{label}-");
160                    let used = number(&format!("{p}used-percent"));
161                    let absolute = get(&format!("{p}reset-at")).and_then(reset_time);
162                    let after = number(&format!("{p}reset-after-seconds")).filter(|s| *s >= 0.0);
163                    let relative = after.and_then(|a| observed_at.checked_add(Duration::try_from_secs_f64(a).ok()?));
164                    let reset = absolute.or(relative);
165                    // A zero delay still ends an exhausted window now, but its time is
166                    // just the observation's, so it names no rollover.
167                    let stated_reset = absolute.or(relative.filter(|_| after.is_some_and(|a| a > 0.0)));
168                    let length = number(&format!("{p}window-minutes")).and_then(|m| window_length(m, observed_at));
169                    (used.is_some() || reset.is_some() || length.is_some()).then_some(Window {
170                        id,
171                        used_up: used.is_some_and(|u| u >= 100.0),
172                        reset,
173                        stated_reset,
174                        length,
175                    })
176                };
177                let day = Duration::from_secs(24 * 3600);
178                match (
179                    window(WindowId::CodexPrimary, "primary"),
180                    window(WindowId::CodexSecondary, "secondary"),
181                ) {
182                    // The longer window is the weekly one, whichever header carries it;
183                    // without lengths, secondary is weekly.
184                    (Some(p), Some(s)) if p.length.unwrap_or(Duration::ZERO) > s.length.unwrap_or(Duration::MAX) => {
185                        (Some(p), Some(s))
186                    }
187                    (Some(p), Some(s)) => (Some(s), Some(p)),
188                    (Some(p), None) if p.length.is_some_and(|l| l >= day) => (Some(p), None),
189                    (None, Some(s)) if s.length.is_none_or(|l| l >= day) => (Some(s), None),
190                    (only, None) | (None, only) => (None, only),
191                }
192            }
193            _ => (None, None),
194        };
195        let windows = [long, short];
196        let until = windows
197            .iter()
198            .flatten()
199            .filter_map(|w| w.exhausted_until(observed_at))
200            .max();
201        // `X-Codex-Limit-Reached` lasts as long as the used-up windows do; when none is
202        // used up, until the soonest reset any window reports; with no reset at all,
203        // five hours from the observation.
204        let flag_until = (codex && get("x-codex-limit-reached").is_some_and(|v| v.eq_ignore_ascii_case("true")))
205            .then(|| {
206                until
207                    .or_else(|| windows.iter().flatten().filter_map(|w| w.reset).min())
208                    .or_else(|| observed_at.checked_add(FALLBACK_WINDOW))
209            })
210            .flatten();
211        let long_reset = long.and_then(|w| w.reset);
212        Self {
213            weekly_reset: long_reset.filter(|r| *r > now),
214            rolled_over: long.and_then(|w| {
215                let at = w.stated_reset.filter(|r| *r <= now)?;
216                Some(Rollover {
217                    window: w.id,
218                    at,
219                    gap: w.length.unwrap_or(FALLBACK_WINDOW) / 2,
220                })
221            }),
222            exhausted: until.max(flag_until).is_some_and(|u| u > now),
223        }
224    }
225
226    /// `soonest-reset` order. Usable accounts come before used-up ones. Among them, an
227    /// account without a known reset that has not been probed since its long window
228    /// last rolled over gets one probe request, then known resets go earliest first,
229    /// then accounts whose probe taught nothing.
230    fn order(&self, probed: bool) -> (bool, u8, Option<SystemTime>) {
231        let class = match (self.weekly_reset, probed) {
232            (Some(_), _) => 1,
233            (None, false) => 0,
234            (None, true) => 2,
235        };
236        (self.exhausted, class, self.weekly_reset)
237    }
238}
239
240/// A credential's `soonest-reset` probes: present once its probe for an unknown reset
241/// was spent, with the last rollover probed for, per window.
242type Probes = HashMap<WindowId, SystemTime>;
243
244/// Whether a credential was already probed for what it reports now. A probe holds
245/// until its long window rolls over after it: a reset of that same window that passed
246/// and is more than the window's gap later than the rollover last probed for it.
247fn probed(record: Option<&Probes>, rolled_over: Option<Rollover>) -> bool {
248    match (record, rolled_over) {
249        (None, _) => false,
250        (Some(_), None) => true,
251        (Some(markers), Some(r)) => markers
252            .get(&r.window)
253            .is_some_and(|at| at.checked_add(r.gap).is_none_or(|limit| r.at <= limit)),
254    }
255}
256
257/// A credential's [`Windows`] for one pick.
258pub type Ranks<'a> = dyn Fn(&Credential) -> Windows + 'a;
259
260#[derive(Debug, Clone)]
261pub struct Policy {
262    pub strategy: Strategy,
263    /// routing.retry.request-retry: int, default 0 (additional rounds).
264    pub request_retry: usize,
265    /// routing.retry.max-retry-credentials: int, default 0 (unlimited per round).
266    pub max_retry_credentials: usize,
267    /// routing.retry.max-retry-interval: int seconds, default 0 (no positive waits).
268    pub max_retry_interval: Duration,
269    /// routing.cooldown.disable-cooling: bool, default false.
270    pub disable_cooling: bool,
271    /// routing.cooldown.transient-error-cooldown-seconds: int, 0 = 60s, negative disables.
272    pub transient_error_cooldown_seconds: i64,
273    /// routing.session-affinity: bool, default false.
274    pub session_affinity: bool,
275    /// routing.session-affinity-ttl: duration string, default 1h.
276    pub session_affinity_ttl: Duration,
277    pub session_affinity_subagents: bool,
278    /// routing.cooldown.save-cooldown-status: persist cooldowns as `.cds` files in
279    /// `auth-dir` (cooldown_store.rs), never in credential files.
280    pub save_cooldown_status: bool,
281    /// routing.cooldown.max-trusted-cooldown (cliproxy-rs only): the first bound on a
282    /// quota reset an upstream states ([`trusted`]); zero trusts every reset, as Go does.
283    pub max_trusted_cooldown: Duration,
284    /// routing.force-model-prefix: bool, default false.
285    pub force_model_prefix: bool,
286    /// Enabled `openai-compatibility` entries in config order with their
287    /// `disable-cooling` (Go `providerCoolingOverrideForAuth`). Filled from the config on
288    /// every publish ([`compat_cooling`]).
289    pub compat_disable_cooling: Vec<(String, Option<bool>)>,
290    /// OAuth-only `oauth.request-scoped-errors` channels; API keys must not inherit these.
291    pub oauth_request_scoped_errors: HashMap<String, Vec<ErrorRule>>,
292}
293
294impl Default for Policy {
295    fn default() -> Self {
296        Self {
297            strategy: Strategy::RoundRobin,
298            request_retry: 0,
299            max_retry_credentials: 0,
300            max_retry_interval: Duration::ZERO,
301            disable_cooling: false,
302            transient_error_cooldown_seconds: 0,
303            session_affinity: false,
304            session_affinity_ttl: Duration::from_secs(3600),
305            session_affinity_subagents: true,
306            save_cooldown_status: false,
307            max_trusted_cooldown: DEFAULT_TRUSTED_COOLDOWN,
308            force_model_prefix: false,
309            compat_disable_cooling: Vec::new(),
310            oauth_request_scoped_errors: HashMap::new(),
311        }
312    }
313}
314
315/// Raw config values, independent of the config stream's RoutingConfig type.
316#[derive(Default)]
317pub struct RawRouting<'a> {
318    pub strategy: &'a str,
319    pub force_model_prefix: bool,
320    pub session_affinity: bool,
321    pub session_affinity_ttl: &'a str,
322    pub session_affinity_subagents: Option<bool>,
323    pub request_retry: i64,
324    pub max_retry_credentials: i64,
325    pub max_retry_interval: i64,
326    pub disable_cooling: bool,
327    pub save_cooldown_status: bool,
328    pub transient_error_cooldown_seconds: i64,
329    pub max_trusted_cooldown: &'a str,
330}
331
332// ponytail: OAuth-provider disable-cooling and request-scoped-error maps stay empty until
333// the config stream synthesizes provider overrides; per-credential metadata still applies.
334impl From<&cpa_core::config::RoutingConfig> for Policy {
335    fn from(r: &cpa_core::config::RoutingConfig) -> Self {
336        normalize(RawRouting {
337            strategy: &r.strategy,
338            force_model_prefix: r.force_model_prefix,
339            session_affinity: r.session_affinity,
340            session_affinity_ttl: &r.session_affinity_ttl,
341            session_affinity_subagents: r.session_affinity_subagents,
342            request_retry: r.retry.request_retry,
343            max_retry_credentials: r.retry.max_retry_credentials,
344            max_retry_interval: r.retry.max_retry_interval,
345            disable_cooling: r.cooldown.disable_cooling,
346            save_cooldown_status: r.cooldown.save_cooldown_status,
347            transient_error_cooldown_seconds: r.cooldown.transient_error_cooldown_seconds,
348            max_trusted_cooldown: &r.cooldown.max_trusted_cooldown,
349        })
350    }
351}
352
353pub fn normalize(raw: RawRouting<'_>) -> Policy {
354    let ttl = cpa_core::config::parse_duration(raw.session_affinity_ttl.trim())
355        .filter(|d| *d > 0)
356        .map(|n| Duration::from_nanos(n as u64).max(Duration::from_secs(1)))
357        .unwrap_or(Duration::from_secs(3600));
358    Policy {
359        strategy: match raw.strategy.trim().to_ascii_lowercase().as_str() {
360            "weighted-round-robin" | "weightedroundrobin" | "wrr" => Strategy::WeightedRoundRobin,
361            "fill-first" | "fillfirst" | "ff" => Strategy::FillFirst,
362            // An unset strategy is reset first; Go reads it as round-robin.
363            "" | "soonest-reset" | "soonestreset" | "reset-first" | "resetfirst" => Strategy::SoonestReset,
364            _ => Strategy::RoundRobin,
365        },
366        force_model_prefix: raw.force_model_prefix,
367        session_affinity: raw.session_affinity,
368        session_affinity_ttl: ttl,
369        session_affinity_subagents: !raw.session_affinity || raw.session_affinity_subagents.unwrap_or(true),
370        request_retry: raw.request_retry.max(0) as usize,
371        max_retry_credentials: raw.max_retry_credentials.max(0) as usize,
372        max_retry_interval: Duration::from_secs(raw.max_retry_interval.max(0) as u64),
373        disable_cooling: raw.disable_cooling,
374        save_cooldown_status: raw.save_cooldown_status,
375        transient_error_cooldown_seconds: raw.transient_error_cooldown_seconds,
376        max_trusted_cooldown: trusted_cooldown(raw.max_trusted_cooldown),
377        ..Policy::default()
378    }
379}
380
381const DEFAULT_TRUSTED_COOLDOWN: Duration = Duration::from_secs(3600);
382
383/// `max-trusted-cooldown`: a Go duration (`"90m"`) or a bare number of seconds. Empty
384/// means the one-hour default and zero or negative turns the bound off; a positive value
385/// is at least Go's 10 s quota floor. An unreadable value (`"1d"`, `false`, `"1.5"`)
386/// keeps the default and is logged once per value.
387fn trusted_cooldown(raw: &str) -> Duration {
388    let raw = raw.trim();
389    if raw.is_empty() {
390        return DEFAULT_TRUSTED_COOLDOWN;
391    }
392    let nanos = match raw.parse::<i64>() {
393        Ok(seconds) => seconds.saturating_mul(1_000_000_000),
394        Err(_) => match cpa_core::config::parse_duration(raw) {
395            Some(nanos) => nanos,
396            None => {
397                static WARNED: std::sync::Mutex<String> = std::sync::Mutex::new(String::new());
398                let mut last = WARNED.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
399                if *last != raw {
400                    raw.clone_into(&mut last);
401                    tracing::warn!(
402                        "routing.cooldown.max-trusted-cooldown {raw:?} is not a duration (\"90m\") or a \
403                         number of seconds; using 1h"
404                    );
405                }
406                return DEFAULT_TRUSTED_COOLDOWN;
407            }
408        },
409    };
410    match nanos {
411        ..=0 => Duration::ZERO,
412        n => Duration::from_nanos(n as u64).max(Duration::from_secs(10)),
413    }
414}
415
416#[derive(Debug, Clone, Deserialize)]
417pub struct ErrorRule {
418    #[serde(default)]
419    pub status: i64,
420    #[serde(default)]
421    pub r#match: Vec<String>,
422    #[serde(default, rename = "match-regexr")]
423    pub match_regex: Vec<String>,
424    #[serde(default)]
425    pub action: String,
426}
427
428#[derive(Debug, Clone, Copy)]
429pub struct ErrorAction {
430    /// A configured request-scoped rule decided this outcome.
431    pub matched: bool,
432    pub stop: bool,
433    pub cooldown: bool,
434    pub force_cooldown: bool,
435}
436
437fn metadata<'a>(c: &'a Credential, key: &str) -> Option<&'a Value> {
438    // Canonical presence wins even for false, zero and null.
439    c.metadata
440        .get(key)
441        .or_else(|| c.metadata.get(key.replace('_', "-").as_str()))
442}
443
444/// A credential's priority tier, as selection reads it.
445pub(crate) fn priority(c: &Credential) -> i64 {
446    integer(c, "priority").unwrap_or(0)
447}
448
449fn integer(c: &Credential, key: &str) -> Option<i64> {
450    c.attributes
451        .get(key)
452        .filter(|_| key == "priority")
453        .and_then(|s| s.trim().parse().ok())
454        .or_else(|| {
455            [key.to_owned(), key.replace('_', "-")].iter().find_map(|key| {
456                let v = c.metadata.get(key)?;
457                v.as_i64()
458                    .or_else(|| v.as_str()?.trim().parse().ok())
459                    .or_else(|| v.as_f64().map(|n| n as i64))
460            })
461        })
462}
463
464fn boolean(c: &Credential, key: &str) -> Option<bool> {
465    let parse = |s: &str| match s.trim() {
466        "true" | "True" | "TRUE" | "1" | "t" | "T" => Some(true),
467        "false" | "False" | "FALSE" | "0" | "f" | "F" => Some(false),
468        _ => None,
469    };
470    [key.to_owned(), key.replace('_', "-")].iter().find_map(|key| {
471        let v = c.metadata.get(key)?;
472        v.as_bool()
473            .or_else(|| parse(v.as_str()?))
474            .or_else(|| v.as_f64().map(|n| n != 0.0))
475    })
476}
477
478fn oauth(c: &Credential) -> bool {
479    c.str("api_key").is_none() && !c.attributes.contains_key("api_key")
480}
481
482impl Policy {
483    pub fn retry_limit(&self, c: &Credential) -> usize {
484        integer(c, "request_retry")
485            .filter(|n| *n >= 0)
486            .map(|n| n as usize)
487            .unwrap_or(self.request_retry)
488    }
489
490    /// Go `quotaCooldownDisabledForAuthWithConfig`: the credential's `disable_cooling`
491    /// metadata, then its OpenAI-compatibility entry's `disable-cooling`, then global.
492    pub(crate) fn cooling_disabled(&self, c: &Credential) -> bool {
493        boolean(c, "disable_cooling")
494            .or_else(|| self.provider_cooling(c))
495            .unwrap_or(self.disable_cooling)
496    }
497
498    /// Go `providerCoolingOverrideForAuth` with `resolveOpenAICompatConfig`: the first
499    /// enabled entry (config order) named by the credential's `compat_name`,
500    /// `provider_key` or provider.
501    fn provider_cooling(&self, c: &Credential) -> Option<bool> {
502        let provider = c.provider.trim().go_lower();
503        let attr = |k: &str| c.attributes.get(k).map(|v| v.trim()).unwrap_or_default();
504        let (compat_name, mut provider_key) = (attr("compat_name"), attr("provider_key"));
505        if provider.is_empty()
506            || (provider_key.is_empty() && compat_name.is_empty() && provider != "openai-compatibility")
507        {
508            return None;
509        }
510        if provider_key.is_empty() {
511            provider_key = &provider;
512        }
513        let candidates = [compat_name, provider_key, provider.as_str()];
514        self.compat_disable_cooling
515            .iter()
516            .find(|(name, _)| candidates.iter().any(|c| !c.is_empty() && c.go_eq_fold(name)))
517            .and_then(|(_, disable)| *disable)
518    }
519
520    pub fn error_action(&self, c: &Credential, error: &ExecError) -> ErrorAction {
521        let file_rules: Option<Vec<ErrorRule>> =
522            metadata(c, "request_scoped_errors").and_then(|v| serde_json::from_value(v.clone()).ok());
523        let rules = file_rules.as_ref().filter(|r| !r.is_empty()).or_else(|| {
524            oauth(c)
525                .then(|| self.oauth_request_scoped_errors.get(&c.provider.trim().go_lower()))
526                .flatten()
527        });
528        if let Some(rules) = rules {
529            let body = crate::classify::rule_text(error);
530            let status = i64::from(crate::classify::go_status(error));
531            for rule in rules {
532                if rule.status <= 0 || rule.status != status {
533                    continue;
534                }
535                let matches = rule.r#match.iter().any(|s| !s.is_empty() && body.contains(s))
536                    || rule
537                        .match_regex
538                        .iter()
539                        .any(|s| !s.is_empty() && regex::Regex::new(s).is_ok_and(|re| re.is_match(&body)));
540                if !matches {
541                    continue;
542                }
543                let (stop, cooldown) = match rule.action.trim().to_ascii_lowercase().as_str() {
544                    "stop" => (true, false),
545                    "stop-and-cooldown" => (true, true),
546                    "continue" => (false, false),
547                    "continue-and-cooldown" => (false, true),
548                    _ => continue,
549                };
550                return ErrorAction {
551                    matched: true,
552                    stop,
553                    cooldown,
554                    force_cooldown: cooldown,
555                };
556            }
557        }
558        let request = crate::classify::is_request_invalid(error);
559        ErrorAction {
560            matched: false,
561            stop: request,
562            cooldown: !crate::classify::skips_cooldown(error),
563            force_cooldown: false,
564        }
565    }
566}
567
568/// Go `isCredentialRetryRoundStatus`.
569pub fn retry_status(status: u16) -> bool {
570    matches!(status, 403 | 408 | 429 | 500 | 502 | 503 | 504)
571}
572
573fn weight(c: &Credential) -> i64 {
574    let value = if let Some(s) = c.attributes.get("weight").filter(|s| !s.trim().is_empty()) {
575        s.trim().parse::<i64>().ok()
576    } else if let Some(v) = metadata(c, "weight") {
577        v.as_i64().or_else(|| v.as_str()?.trim().parse().ok()).or_else(|| {
578            let n = v.as_f64()?;
579            (n.fract() == 0.0 && n <= 1_000_000.0).then_some(n as i64)
580        })
581    } else {
582        Some(1)
583    };
584    value.filter(|n| *n <= 1_000_000).unwrap_or(0).max(0)
585}
586
587pub use cpa_core::registry::dynamic::canonical_model;
588
589#[derive(Default)]
590struct Rotation {
591    last: String,
592    weights: HashMap<String, i64>,
593    current: HashMap<String, i64>,
594}
595
596pub(crate) struct Cooldown {
597    pub deadline: Instant,
598    pub level: u32,
599    pub status: u16,
600    /// Quota-style cooldown (429 or Cloudflare): reported as a model cooldown.
601    pub quota: bool,
602    /// The failure that set it (Go `ModelState.LastError`), for error summaries.
603    pub error: String,
604    /// When it was set (Go `ModelState.UpdatedAt`), for `save-cooldown-status`.
605    pub since: SystemTime,
606    /// A model quota inherited from a credential-wide quota (Go reason
607    /// `credential_quota` on sibling model states).
608    pub credential: bool,
609    /// Bounded trust of a stated quota reset (cliproxy-rs only); `level` stays Go's.
610    pub trust: Trust,
611    /// What the quota is when `quota` is set: the status that set it and its reason as
612    /// `records` names it (a Cloudflare challenge can answer 429).
613    pub quota_kind: Option<QuotaKind>,
614    /// A live quota block (its end and kind) that this non-quota failure answered: Go
615    /// keeps `ModelState.Quota` when it records a 401, and a recovery that clears the 401
616    /// keeps the block. Not saved: a record saved meanwhile is the failure's alone, and
617    /// a restore keeps a live block from the entry it merges with.
618    pub quota_under: Option<(Instant, QuotaKind)>,
619    /// The dispatch watermark when a failure last recorded this entry (Go
620    /// `cooldownRecordedSeq`): a success from a dispatch at or before it predates the
621    /// failure and leaves a live cooldown alone. Zero after a restart, as in Go.
622    pub recorded: u64,
623}
624
625#[derive(Debug, Clone, Copy, PartialEq, Eq)]
626pub(crate) struct QuotaKind {
627    pub status: u16,
628    pub reason: &'static str,
629}
630
631impl Cooldown {
632    /// The live quota block this entry is or carries.
633    fn quota_block(&self, now: Instant) -> Option<(Instant, QuotaKind)> {
634        let block = if self.quota {
635            self.quota_kind.map(|k| (self.deadline, k))
636        } else {
637            self.quota_under
638        };
639        block.filter(|(end, _)| *end > now || self.trust.stated.is_some_and(|r| r > now))
640    }
641}
642
643/// Bounded trust of one cooldown (`max-trusted-cooldown`, docs/DIFFERENCES-FROM-GO.md).
644/// Costs nothing until a stated reset is cut: then one comparison per 429 and, per
645/// recorded result, a probe check on the two keys the result touches.
646#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
647pub(crate) struct Trust {
648    /// Exponent of the latest bounded window, which lasted `cap << window`. Kept after
649    /// the window ends, so the next cut window doubles; a success clears it.
650    pub window: Option<u32>,
651    /// The scheduler's generation when this bounded window opened or its probe was
652    /// reserved ([`Scheduler::reserve_probe`]). A lease picked before it holds an older
653    /// answer, which neither ends the window nor frees the reservation. Cost: four
654    /// bytes here and on each lease, one comparison per key a result touches.
655    pub generation: u32,
656    /// The deadline is the bound's: a concurrent 429 inside it keeps it.
657    pub bounded: bool,
658    /// A probe the upstream accepted (a stream started) answered this ended bounded
659    /// window: it no longer reserves probes, but keeps its generation, so an older
660    /// lease's late answer still cannot replace it. Fits in existing padding.
661    pub accepted: bool,
662    /// The upstream's stated reset behind a bounded deadline (Go's recovery time).
663    pub stated: Option<Instant>,
664    /// An unanswered probe request holds the ended window until this instant.
665    pub probe: Option<Instant>,
666}
667
668impl Trust {
669    /// The window's probe was accepted ([`Scheduler::accept_probe_picked`]).
670    fn accept(&mut self) {
671        self.bounded = false;
672        self.accepted = true;
673        self.probe = None;
674    }
675}
676
677/// Longest a probe reserves an ended bounded window. A 429 arrives within seconds; a
678/// probe still unanswered after this is being served, and its result clears it anyway.
679const PROBE_HOLD: Duration = Duration::from_secs(30);
680/// Largest trust exponent: `cap << 20` is over a century for any cap of 1 h or more.
681const MAX_TRUST_WINDOW: u32 = 20;
682
683impl Cooldown {
684    /// When this entry stops blocking picks: its deadline, or an unanswered probe.
685    fn until(&self, now: Instant) -> Option<Instant> {
686        let live = (self.deadline > now).then_some(self.deadline);
687        live.max(self.trust.probe.filter(|p| *p > now))
688    }
689
690    /// A bounded window (still armed, or answered by an accepted probe) that opened, or
691    /// had its probe reserved, after the lease picked at generation `picked`: that
692    /// lease's answer does not answer it.
693    fn newer_than(&self, picked: u32) -> bool {
694        // Serial-number order, so the wrapping counter compares right while a lease is
695        // fewer than 2^31 generations old.
696        (self.trust.bounded || self.trust.accepted) && (self.trust.generation.wrapping_sub(picked) as i32) > 0
697    }
698}
699
700#[derive(Default)]
701pub(crate) struct Scheduler {
702    rotations: HashMap<(String, String), Rotation>,
703    /// Mixed-provider round-robin cursors (Go `mixedCursors`).
704    cursors: HashMap<(String, String), usize>,
705    pub(crate) cooldowns: HashMap<(String, String), Cooldown>,
706    /// Session affinity bindings (Go `SessionAffinitySelector.cache`).
707    affinity: crate::affinity::Cache,
708    /// Bindings of requests without an explicit session (Go
709    /// `SessionAffinitySelector.matcher`).
710    lcp: crate::lcp::Matcher,
711    /// `soonest-reset` probes by credential, reserved when the probe request is picked
712    /// (under the scheduler lock, so concurrent picks never probe twice). See [`probed`].
713    probes: HashMap<String, Probes>,
714    /// The live policy turned session affinity off: results of leases picked while it
715    /// was on no longer bind anything (Go's replacement selector has no affinity).
716    affinity_off: bool,
717    /// Bounded windows opened plus probes reserved; each lease carries the value at its
718    /// pick ([`Trust::generation`]).
719    generation: u32,
720    /// Upstream dispatches so far (Go `Manager.executionSeq`): each dispatch takes the
721    /// next value ([`Self::dispatch`]); a recorded failure stamps the current one.
722    dispatched: u64,
723    /// Credentials whose refresh after an upstream 401 failed with `invalid_grant`
724    /// while the refused tokens were still current, as they were then (Go
725    /// `hasUnauthorizedAuthFailure` from `refreshAuthForRequestAtEpoch`). Neither
726    /// selected nor refreshed until their tokens change ([`Self::tokens_rejected`]).
727    rejected: HashMap<String, std::sync::Arc<Credential>>,
728}
729
730/// Go `cfg.OpenAICompatibility` for cooling: enabled entries with a base URL (Go
731/// `SanitizeOpenAICompatibility`), in config order, with their `disable-cooling`.
732pub fn compat_cooling(cfg: &cpa_core::config::Config) -> Vec<(String, Option<bool>)> {
733    cfg.document
734        .get("api-keys")
735        .and_then(|k| k.get("openai-compatibility"))
736        .and_then(serde_yaml_ng::Value::as_sequence)
737        .into_iter()
738        .flatten()
739        .filter(|g| {
740            g.get("base-url")
741                .and_then(serde_yaml_ng::Value::as_str)
742                .is_some_and(|b| !b.trim().is_empty())
743                && g.get("disabled").and_then(serde_yaml_ng::Value::as_bool) != Some(true)
744        })
745        .map(|g| {
746            (
747                g.get("name")
748                    .and_then(serde_yaml_ng::Value::as_str)
749                    .unwrap_or_default()
750                    .trim()
751                    .to_owned(),
752                g.get("disable-cooling").and_then(serde_yaml_ng::Value::as_bool),
753            )
754        })
755        .collect()
756}
757
758/// A selection's affinity keys: the session, and its parent or alias when distinct.
759struct SessionKeys {
760    primary: crate::affinity::Key,
761    fallback: Option<crate::affinity::Key>,
762    /// The raw parent/alias session, for Go `isSubagentSession`.
763    parent: String,
764}
765
766fn session_keys(selection: &Selection) -> Option<SessionKeys> {
767    let primary = selection.session.as_deref().filter(|s| !s.is_empty())?;
768    let scope = selection.provider_keys().join(",");
769    let model = canonical_model(&selection.model).to_owned();
770    let parent = selection.session_parent.clone().unwrap_or_default();
771    let fallback = (!parent.is_empty() && parent != primary).then(|| (scope.clone(), model.clone(), parent.clone()));
772    Some(SessionKeys {
773        primary: (scope, model, primary.to_owned()),
774        fallback,
775        parent,
776    })
777}
778
779/// Go `nextQuotaCooldown`: 1s doubling to 30m.
780fn quota_backoff(level: u32) -> (Duration, u32) {
781    let seconds = (1u64 << level.min(11)).min(1800);
782    if seconds >= 1800 {
783        (Duration::from_secs(1800), level)
784    } else {
785        (Duration::from_secs(seconds), level + 1)
786    }
787}
788
789/// Bounded trust for a stated quota reset (docs/DIFFERENCES-FROM-GO.md): a reset longer
790/// than the window's bound is cut to it, so the next ordinary request after the bound
791/// probes the account and an early provider reset is noticed within the bound. The
792/// first window is `cap`; each cut window after one that ended without a success is
793/// twice the last (`window` is the last exponent). Returns the bound and its exponent,
794/// or `None` when the stated reset is trusted as it is (it fits, or `cap` is zero).
795fn bound(stated: Duration, cap: Duration, window: Option<u32>) -> Option<(Duration, u32)> {
796    if cap.is_zero() {
797        return None;
798    }
799    let step = window.map_or(0, |w| w.saturating_add(1)).min(MAX_TRUST_WINDOW);
800    let bound = cap.saturating_mul(1 << step);
801    (stated > bound).then_some((bound, step))
802}
803
804/// Go `recoverableFailureRetryAfterWithHint`.
805fn transient(policy: &Policy, hint: Option<Duration>, disabled: bool) -> Option<Duration> {
806    if disabled || policy.transient_error_cooldown_seconds < 0 {
807        return None;
808    }
809    if let Some(hint) = hint.filter(|d| !d.is_zero()) {
810        return Some(hint);
811    }
812    Some(Duration::from_secs(match policy.transient_error_cooldown_seconds {
813        0 => 60,
814        s => s as u64,
815    }))
816}
817
818impl Scheduler {
819    pub fn configure(&mut self, previous: &Policy, next: &Policy) {
820        self.affinity_off = !next.session_affinity;
821        if previous.strategy != next.strategy
822            || previous.session_affinity != next.session_affinity
823            || previous.session_affinity_ttl != next.session_affinity_ttl
824            || previous.session_affinity_subagents != next.session_affinity_subagents
825        {
826            self.rotations.clear();
827            self.cursors.clear();
828            self.affinity.clear();
829            // Go builds a new selector, and with it a new matcher on the new TTL. Unlike
830            // Go, access generations keep counting (docs/DIFFERENCES-FROM-GO.md).
831            self.lcp.reset(crate::lcp::Limits {
832                ttl: next.session_affinity_ttl,
833                ..Default::default()
834            });
835            self.probes.clear();
836        }
837    }
838
839    /// Session affinity under `policy`, unless the live policy has since turned it off.
840    fn affinity(&self, policy: &Policy) -> bool {
841        policy.session_affinity && !self.affinity_off
842    }
843
844    pub fn quota_cooling(&self, c: &Credential, model: &str, now: Instant) -> bool {
845        let model = canonical_model(model);
846        [model, ""].into_iter().any(|m| {
847            self.cooldowns
848                .get(&(c.id.clone(), m.to_owned()))
849                .is_some_and(|s| s.quota && s.until(now).is_some())
850        })
851    }
852
853    pub fn wait(&self, c: &Credential, model: &str, now: Instant) -> Option<Duration> {
854        let model = canonical_model(model);
855        [model, ""]
856            .into_iter()
857            .filter_map(|model| {
858                self.cooldowns
859                    .get(&(c.id.clone(), model.to_owned()))
860                    .and_then(|s| s.until(now)?.checked_duration_since(now))
861                    .filter(|d| !d.is_zero())
862            })
863            .max()
864    }
865
866    /// Whether a cooldown deadline (not a probe reservation) is ahead for `model`.
867    pub fn cooling(&self, c: &Credential, model: &str, now: Instant) -> bool {
868        let model = canonical_model(model);
869        [model, ""].into_iter().any(|m| {
870            self.cooldowns
871                .get(&(c.id.clone(), m.to_owned()))
872                .is_some_and(|s| s.deadline > now)
873        })
874    }
875
876    /// The last failure recorded for this credential and model, if any.
877    pub fn last_error(&self, c: &Credential, model: &str) -> Option<&Cooldown> {
878        let model = canonical_model(model);
879        self.cooldowns
880            .get(&(c.id.clone(), model.to_owned()))
881            .or_else(|| self.cooldowns.get(&(c.id.clone(), String::new())))
882    }
883
884    pub fn retry_eligible(&self, c: &Credential, model: &str, now: Instant) -> bool {
885        let model = canonical_model(model);
886        // A bounded window or an unanswered probe is not waited for: the probe answers
887        // for everyone, and holding requests to the window's end would release them
888        // together as a burst of probes.
889        [model, ""].into_iter().all(|model| {
890            self.cooldowns.get(&(c.id.clone(), model.to_owned())).is_none_or(|s| {
891                let probing = s.trust.probe.is_some_and(|p| p > now);
892                !probing && (s.deadline <= now || (retry_status(s.status) && !s.trust.bounded))
893            })
894        })
895    }
896
897    /// [`Self::accept_probe_picked`] for a lease picked just now.
898    #[cfg(test)]
899    pub fn accept_probe(&mut self, c: &Credential, model: &str, now: Instant) {
900        self.accept_probe_picked(c, model, self.generation, now);
901    }
902
903    /// The upstream accepted the attempt that probes `model` (a stream's first chunk),
904    /// picked at generation `picked`: the ended bounded window has its answer, so other
905    /// picks may go through while the stream runs. Only expired bounded entries that
906    /// are not newer than the lease change; their count stays until the stream's own
907    /// outcome is recorded. When the credential-wide window ended, the model keys that
908    /// share its deadline (its copies) are disarmed too; a model's own window is not.
909    /// Cost: two lookups, and a scan of the cooldown table only when the credential-wide
910    /// window ended.
911    pub fn accept_probe_picked(&mut self, c: &Credential, model: &str, picked: u32, now: Instant) {
912        let model = canonical_model(model);
913        let mut credential_window = None;
914        for m in [model, ""] {
915            if let Some(s) = self.cooldowns.get_mut(&(c.id.clone(), m.to_owned()))
916                && s.trust.bounded
917                && s.deadline <= now
918                && !s.newer_than(picked)
919            {
920                s.trust.accept();
921                if m.is_empty() {
922                    credential_window = Some(s.deadline);
923                }
924            }
925        }
926        if let Some(ended) = credential_window {
927            for ((id, m), s) in &mut self.cooldowns {
928                if *id == c.id && !m.is_empty() && s.trust.bounded && s.deadline == ended && !s.newer_than(picked) {
929                    s.trust.accept();
930                }
931            }
932        }
933    }
934
935    /// The generation of a lease picked outside selection (the Antigravity credits
936    /// fallback, which runs on credentials selection refused): one that every bounded
937    /// window and probe reservation within 2^30 generations of now, before or after, is
938    /// newer than. Such a lease answers none of them: a cancelled or successful credits
939    /// attempt neither frees another request's probe nor ends a window, which stays with
940    /// its own probe.
941    pub fn outside_selection_generation(&self) -> u32 {
942        self.generation.wrapping_sub(1 << 30)
943    }
944
945    /// Reserves the probe of `c` for `model` when its pick ends a bounded window: until
946    /// the probe's result is recorded (or [`PROBE_HOLD`] passes), the window's keys keep
947    /// cooling, so concurrent requests do not each go upstream. Returns the generation
948    /// the pick's lease carries; a reservation takes a new one, so only its own lease's
949    /// result frees it. Cost per pick: two lookups.
950    pub fn reserve_probe(&mut self, c: &Credential, model: &str, now: Instant) -> u32 {
951        let model = canonical_model(model);
952        let next = self.generation.wrapping_add(1);
953        for m in [model, ""] {
954            if let Some(s) = self.cooldowns.get_mut(&(c.id.clone(), m.to_owned()))
955                && s.trust.bounded
956                && s.deadline <= now
957            {
958                s.trust.probe = now.checked_add(PROBE_HOLD);
959                s.trust.generation = next;
960                self.generation = next;
961            }
962        }
963        self.generation
964    }
965
966    /// Picks among ready candidates, each paired with its provider key (Go
967    /// `SessionAffinitySelector.Pick` over the configured selector).
968    ///
969    /// An established binding outranks credential priority; a session's parent (or
970    /// prompt-cache conversation alias) binding is inherited by forks and, when
971    /// `session-affinity-subagents` allows, by subagents.
972    #[cfg(test)]
973    pub fn pick<'a>(
974        &mut self,
975        candidates: &[(&'a Credential, &str)],
976        selection: &Selection,
977        policy: &Policy,
978        now: Instant,
979    ) -> Option<&'a Credential> {
980        self.pick_ranked(candidates, selection, policy, &|_| Windows::default(), now)
981    }
982
983    /// [`Self::pick`] with each credential's usage windows, for `soonest-reset`.
984    #[cfg(test)]
985    pub fn pick_ranked<'a>(
986        &mut self,
987        candidates: &[(&'a Credential, &str)],
988        selection: &Selection,
989        policy: &Policy,
990        ranks: &Ranks<'_>,
991        now: Instant,
992    ) -> Option<&'a Credential> {
993        self.pick_session(candidates, selection, policy, ranks, now)
994            .map(|(c, _)| c)
995    }
996
997    /// [`Self::pick_ranked`], with the LCP binding of a request without an explicit
998    /// session (Go `pickLCP`): its session identity, lineage and access generation.
999    pub fn pick_session<'a>(
1000        &mut self,
1001        candidates: &[(&'a Credential, &str)],
1002        selection: &Selection,
1003        policy: &Policy,
1004        ranks: &Ranks<'_>,
1005        now: Instant,
1006    ) -> Option<(&'a Credential, Option<crate::lcp::Match>)> {
1007        if candidates.is_empty() {
1008            return None;
1009        }
1010        if let Some(request) = selection.lcp.as_deref().filter(|_| self.affinity(policy)) {
1011            let namespace = request.namespace(&selection.model);
1012            // A known trajectory stays on its credential while that one is available.
1013            if let Some(found) = self.lcp.find(&namespace, request.prepared(), now)
1014                && let Some((c, _)) = candidates.iter().find(|(c, _)| c.id == found.auth)
1015            {
1016                return Some((c, Some(found)));
1017            }
1018            let picked = self.pick_unbound(candidates, selection, policy.strategy, ranks)?;
1019            let bound = self.lcp.bind(&namespace, request.prepared(), &picked.id, now);
1020            return Some((picked, bound));
1021        }
1022        self.pick_legacy(candidates, selection, policy, ranks, now)
1023            .map(|c| (c, None))
1024    }
1025
1026    /// Go `SessionAffinitySelector.Pick` after the LCP step.
1027    fn pick_legacy<'a>(
1028        &mut self,
1029        candidates: &[(&'a Credential, &str)],
1030        selection: &Selection,
1031        policy: &Policy,
1032        ranks: &Ranks<'_>,
1033        now: Instant,
1034    ) -> Option<&'a Credential> {
1035        let ttl = policy.session_affinity_ttl;
1036        let Some(keys) = self.affinity(policy).then(|| session_keys(selection)).flatten() else {
1037            return self.pick_unbound(candidates, selection, policy.strategy, ranks);
1038        };
1039        self.affinity.sweep(now, ttl);
1040        let fork = selection.session_fork;
1041        let subagent = !fork && cpa_common::session::is_subagent_session(&keys.primary.2, &keys.parent);
1042        let bind_keys = match &keys.fallback {
1043            Some(fallback) if !subagent && !fork => vec![keys.primary.clone(), fallback.clone()],
1044            _ => vec![keys.primary.clone()],
1045        };
1046        let find = |id: &str| candidates.iter().find(|(c, _)| c.id == id).map(|(c, _)| *c);
1047        let reuse = match self.affinity.get_and_refresh(&keys.primary, now, ttl) {
1048            // A bound credential that is no longer available is replaced, not inherited.
1049            Some(id) => find(&id),
1050            None => keys
1051                .fallback
1052                .as_ref()
1053                .and_then(|fallback| self.affinity.get(fallback, now))
1054                .filter(|_| !subagent || policy.session_affinity_subagents)
1055                .and_then(|id| find(&id)),
1056        };
1057        let picked = match reuse {
1058            Some(c) => c,
1059            None => self.pick_unbound(candidates, selection, policy.strategy, ranks)?,
1060        };
1061        self.affinity.bind(&picked.id, &bind_keys, now, ttl);
1062        Some(picked)
1063    }
1064
1065    /// Go `SessionAffinitySelector.OnResult`: success refreshes the session's bindings to
1066    /// this credential, a credential-attributed failure releases them. Request-scoped
1067    /// and transport failures leave them alone.
1068    ///
1069    /// A request without an explicit session refreshes its LCP sequence on success; a
1070    /// credential failure removes that exact sequence unless a newer request refreshed
1071    /// it after `lcp`'s generation. Such a request never touches the session cache.
1072    pub fn session_result(
1073        &mut self,
1074        c: &Credential,
1075        selection: &Selection,
1076        outcome: &Outcome,
1077        policy: &Policy,
1078        lcp: Option<&crate::lcp::Match>,
1079        now: Instant,
1080    ) {
1081        if !self.affinity(policy) {
1082            return;
1083        }
1084        let success = match outcome {
1085            Outcome::Success => true,
1086            Outcome::Failure(error) if policy.error_action(c, error).cooldown => false,
1087            _ => return,
1088        };
1089        if let Some(request) = selection.lcp.as_deref() {
1090            let namespace = request.namespace(&selection.model);
1091            if success {
1092                self.lcp.touch(&namespace, request.prepared(), &c.id, now);
1093            } else {
1094                let generation = lcp.map_or(0, |m| m.access);
1095                self.lcp
1096                    .remove(&namespace, &request.prepared().fingerprints, &c.id, generation, now);
1097            }
1098            if lcp.is_some() {
1099                return;
1100            }
1101        }
1102        let Some(keys) = session_keys(selection) else {
1103            return;
1104        };
1105        let mut targets = vec![keys.primary.clone()];
1106        if let Some(fallback) = keys.fallback
1107            && !cpa_common::session::is_subagent_session(&keys.primary.2, &keys.parent)
1108        {
1109            targets.push(fallback);
1110        }
1111        for key in &targets {
1112            if success {
1113                self.affinity.touch(key, &c.id, now, policy.session_affinity_ttl);
1114            } else {
1115                self.affinity.compare_and_delete(key, &c.id);
1116            }
1117        }
1118    }
1119
1120    /// Go's built-in scheduler with `strategy` (`pickSingleWithStrategy` and
1121    /// `pickMixedWithStrategy`), without session affinity: what a plugin scheduler's
1122    /// `delegate_builtin` answer runs.
1123    /// ponytail: Go keeps the built-in scheduler's cursors apart from the configured
1124    /// selector's; here they share the rotation state. Separate the maps if a plugin
1125    /// scheduler mixing delegation and its own picks needs Go's exact rotation.
1126    pub fn pick_builtin<'a>(
1127        &mut self,
1128        candidates: &[(&'a Credential, &str)],
1129        selection: &Selection,
1130        strategy: Strategy,
1131        ranks: &Ranks<'_>,
1132    ) -> Option<&'a Credential> {
1133        self.pick_unbound(candidates, selection, strategy, ranks)
1134    }
1135
1136    fn pick_unbound<'a>(
1137        &mut self,
1138        candidates: &[(&'a Credential, &str)],
1139        selection: &Selection,
1140        strategy: Strategy,
1141        ranks: &Ranks<'_>,
1142    ) -> Option<&'a Credential> {
1143        let model = canonical_model(&selection.model).to_owned();
1144        let provider_keys = selection.provider_keys();
1145        let providers_key = provider_keys.join(",");
1146        let tier = candidates
1147            .iter()
1148            .map(|(c, _)| integer(c, "priority").unwrap_or(0))
1149            .max()?;
1150        let mut ready: Vec<(&Credential, &str)> = candidates
1151            .iter()
1152            .copied()
1153            .filter(|(c, _)| integer(c, "priority").unwrap_or(0) == tier)
1154            .collect();
1155        ready.sort_by(|a, b| a.0.id.cmp(&b.0.id));
1156        // Providers that have ready candidates, in the request's provider order.
1157        let mut groups: Vec<(&str, Vec<&Credential>)> = Vec::new();
1158        for provider in provider_keys
1159            .iter()
1160            .map(String::as_str)
1161            .chain(ready.iter().map(|(_, p)| *p))
1162        {
1163            if groups.iter().any(|(p, _)| *p == provider) {
1164                continue;
1165            }
1166            let members: Vec<&Credential> = ready.iter().filter(|(_, p)| *p == provider).map(|(c, _)| *c).collect();
1167            if !members.is_empty() {
1168                groups.push((provider, members));
1169            }
1170        }
1171        if self.rotations.len() >= 4096 {
1172            self.rotations.clear();
1173            self.cursors.clear();
1174        }
1175        let picked = if groups.len() == 1 {
1176            let (provider, members) = &groups[0];
1177            self.pick_within(provider, &model, members, strategy, ranks)
1178        } else {
1179            match strategy {
1180                Strategy::FillFirst => groups[0].1[0],
1181                // Like fill-first: the first provider group, then its soonest reset.
1182                Strategy::SoonestReset => {
1183                    let (provider, members) = &groups[0];
1184                    self.pick_within(provider, &model, members, Strategy::SoonestReset, ranks)
1185                }
1186                Strategy::WeightedRoundRobin => {
1187                    let all: Vec<&Credential> = ready.iter().map(|(c, _)| *c).collect();
1188                    self.pick_within(&providers_key, &model, &all, Strategy::WeightedRoundRobin, ranks)
1189                }
1190                Strategy::RoundRobin => {
1191                    // Go `pickMixed`: a cursor over provider segments sized by their ready
1192                    // counts, then that provider's own round-robin.
1193                    let total: usize = groups.iter().map(|(_, m)| m.len()).sum();
1194                    let cursor = self.cursors.entry((providers_key.clone(), model.clone())).or_default();
1195                    let slot = *cursor % total;
1196                    *cursor = slot + 1;
1197                    let mut start = 0;
1198                    let mut index = 0;
1199                    for (i, (_, members)) in groups.iter().enumerate() {
1200                        if slot < start + members.len() {
1201                            index = i;
1202                            break;
1203                        }
1204                        start += members.len();
1205                    }
1206                    let (provider, members) = &groups[index];
1207                    self.pick_within(provider, &model, members, Strategy::RoundRobin, ranks)
1208                }
1209            }
1210        };
1211        Some(picked)
1212    }
1213
1214    fn pick_within<'a>(
1215        &mut self,
1216        scope: &str,
1217        model: &str,
1218        members: &[&'a Credential],
1219        strategy: Strategy,
1220        ranks: &Ranks<'_>,
1221    ) -> &'a Credential {
1222        let state = self.rotations.entry((scope.to_owned(), model.to_owned())).or_default();
1223        match strategy {
1224            Strategy::FillFirst => members[0],
1225            Strategy::SoonestReset => {
1226                let probes = &mut self.probes;
1227                let ranked: Vec<_> = members
1228                    .iter()
1229                    .map(|c| {
1230                        let windows = ranks(c);
1231                        let order = windows.order(probed(probes.get(&c.id), windows.rolled_over));
1232                        (order, windows.rolled_over, *c)
1233                    })
1234                    .collect();
1235                let best = ranked.iter().map(|(order, ..)| *order).min().expect("members");
1236                // Equal ranks take turns.
1237                let tied: Vec<_> = ranked.iter().filter(|(o, ..)| *o == best).collect();
1238                let (order, rolled_over, picked) = *tied
1239                    .iter()
1240                    .find(|(.., c)| c.id > state.last)
1241                    .copied()
1242                    .unwrap_or(tied[0]);
1243                if order.1 == 0 {
1244                    // Its one probe, spent whatever the answer: the next pick ranks it
1245                    // by what the answer reports, if anything.
1246                    let markers = probes.entry(picked.id.clone()).or_default();
1247                    if let Some(r) = rolled_over {
1248                        markers.insert(r.window, r.at);
1249                    }
1250                }
1251                state.last.clone_from(&picked.id);
1252                picked
1253            }
1254            Strategy::RoundRobin => {
1255                let picked = members
1256                    .iter()
1257                    .find(|c| c.id > state.last)
1258                    .copied()
1259                    .unwrap_or(members[0]);
1260                state.last.clone_from(&picked.id);
1261                picked
1262            }
1263            Strategy::WeightedRoundRobin => {
1264                if members
1265                    .iter()
1266                    .any(|c| state.weights.get(&c.id).is_some_and(|w| *w != weight(c)))
1267                {
1268                    state.current.clear();
1269                }
1270                if state.weights.len() > 1024 || state.current.len() > 1024 {
1271                    state.weights.retain(|id, _| members.iter().any(|c| c.id == *id));
1272                    state.current.retain(|id, _| members.iter().any(|c| c.id == *id));
1273                }
1274                let mut total = 0i64;
1275                let mut best = i64::MIN;
1276                let mut picked = members[0];
1277                for c in members {
1278                    let w = weight(c);
1279                    state.weights.insert(c.id.clone(), w);
1280                    let current = state.current.entry(c.id.clone()).or_default();
1281                    *current = current.saturating_add(w);
1282                    total = total.saturating_add(w);
1283                    if *current > best {
1284                        best = *current;
1285                        picked = c;
1286                    }
1287                }
1288                let current = state.current.get_mut(&picked.id).unwrap();
1289                *current = current.saturating_sub(total);
1290                picked
1291            }
1292        }
1293    }
1294
1295    pub fn admits(&self, c: &Credential, policy: &Policy) -> bool {
1296        policy.strategy != Strategy::WeightedRoundRobin || weight(c) > 0
1297    }
1298
1299    /// Records that neither of `c`'s tokens can recover without a new sign-in.
1300    pub(crate) fn reject_tokens(&mut self, c: std::sync::Arc<Credential>) {
1301        self.rejected.insert(c.id.clone(), c);
1302    }
1303
1304    /// Whether `c` still holds the tokens [`Self::reject_tokens`] recorded (Go
1305    /// `updateInternalLocked` lifts the block once `CredentialsChanged`).
1306    pub(crate) fn tokens_rejected(&self, c: &Credential) -> bool {
1307        self.rejected
1308            .get(&c.id)
1309            .is_some_and(|r| !crate::runtime::credentials_changed(r, c))
1310    }
1311
1312    /// [`Self::record_picked`] for an attempt picked just now.
1313    #[cfg(test)]
1314    pub fn record(&mut self, c: &Credential, model: &str, outcome: &Outcome, policy: &Policy, now: Instant) {
1315        self.record_picked(c, model, outcome, policy, self.generation, now);
1316    }
1317
1318    /// [`Self::record_dispatched`] for a result without a dispatch stamp.
1319    #[cfg(test)]
1320    pub fn record_picked(
1321        &mut self,
1322        c: &Credential,
1323        model: &str,
1324        outcome: &Outcome,
1325        policy: &Policy,
1326        picked: u32,
1327        now: Instant,
1328    ) {
1329        self.record_dispatched(c, model, outcome, policy, picked, 0, now);
1330    }
1331
1332    /// Stamps one upstream dispatch (Go `nextExecutionSeq`).
1333    pub fn dispatch(&mut self) -> u64 {
1334        self.dispatched += 1;
1335        self.dispatched
1336    }
1337
1338    /// Applies one attempt's outcome (Go `MarkResult`). `picked` is the generation its
1339    /// lease was picked at ([`Self::reserve_probe`]): a bounded window or probe
1340    /// reservation newer than that is not answered by it. `seq` is its dispatch stamp
1341    /// ([`Self::dispatch`], 0 when unknown): a success leaves alone a live cooldown that
1342    /// a failure recorded after the dispatch (Go 35af74f9).
1343    #[allow(clippy::too_many_arguments)]
1344    pub fn record_dispatched(
1345        &mut self,
1346        c: &Credential,
1347        model: &str,
1348        outcome: &Outcome,
1349        policy: &Policy,
1350        picked: u32,
1351        seq: u64,
1352        now: Instant,
1353    ) {
1354        let model = canonical_model(model);
1355        self.apply(c, model, outcome, policy, picked, seq, now);
1356        // Go stamps the failing state with the dispatch watermark; a credential-wide 429
1357        // also stamps the credential and every sibling state. A credential-level failure
1358        // stamps the credential even when it skips the cooldown (Go's auth-level branch
1359        // has no skip).
1360        if let Outcome::Failure(error) = outcome {
1361            let action = policy.error_action(c, error);
1362            if action.cooldown || model.is_empty() {
1363                let disabled = policy.cooling_disabled(c) && !action.force_cooldown;
1364                let all = crate::classify::credential_scoped(error) && !disabled;
1365                for ((id, m), s) in &mut self.cooldowns {
1366                    if *id == c.id && (all || m == model) {
1367                        s.recorded = self.dispatched;
1368                    }
1369                }
1370            }
1371        }
1372    }
1373
1374    #[allow(clippy::too_many_arguments)]
1375    fn apply(
1376        &mut self,
1377        c: &Credential,
1378        model: &str,
1379        outcome: &Outcome,
1380        policy: &Policy,
1381        picked: u32,
1382        seq: u64,
1383        now: Instant,
1384    ) {
1385        let key = (c.id.clone(), model.to_owned());
1386        // Any answer ends a probe reservation on the keys it touches, except one taken
1387        // after its lease was picked (a later probe's, once this one's hold lapsed).
1388        for m in [model, ""] {
1389            if let Some(s) = self.cooldowns.get_mut(&(c.id.clone(), m.to_owned()))
1390                && !s.newer_than(picked)
1391            {
1392                s.trust.probe = None;
1393            }
1394        }
1395        let error = match outcome {
1396            Outcome::Success => {
1397                // Active credential quota survives success on an in-flight sibling.
1398                let global = (c.id.clone(), String::new());
1399                if self.cooldowns.get(&global).is_some_and(|s| s.quota && s.deadline > now) {
1400                    return;
1401                }
1402                // A live cooldown recorded after this attempt was dispatched: the success
1403                // predates the failure and proves no recovery (Go `retainModelCooldown`,
1404                // `retainAuthCooldown`).
1405                if seq > 0
1406                    && self
1407                        .cooldowns
1408                        .get(&key)
1409                        .is_some_and(|s| s.deadline > now && s.recorded >= seq)
1410                {
1411                    return;
1412                }
1413                // A request that started before a bounded window opened leaves it for the
1414                // window's own probe.
1415                if let Some(ended) = self
1416                    .cooldowns
1417                    .get(&global)
1418                    .filter(|s| s.deadline <= now && !s.newer_than(picked))
1419                    .map(|s| s.deadline)
1420                {
1421                    self.cooldowns.remove(&global);
1422                    // The credential recovered: model keys that a credential-wide window
1423                    // escalated forget it too (their Go state is left as Go keeps it). A
1424                    // model's own ended bounded window still waits for its probe, and one
1425                    // a newer probe reserved or answered keeps its state.
1426                    for ((id, _), s) in &mut self.cooldowns {
1427                        if *id == c.id
1428                            && s.deadline <= now
1429                            && (!s.trust.bounded || s.deadline == ended)
1430                            && !s.newer_than(picked)
1431                        {
1432                            s.trust = Trust::default();
1433                        }
1434                    }
1435                }
1436                if !self.cooldowns.get(&key).is_some_and(|s| s.newer_than(picked)) {
1437                    self.cooldowns.remove(&key);
1438                }
1439                return;
1440            }
1441            Outcome::Cancelled | Outcome::Neutral(_) => return,
1442            Outcome::Failure(error) => error,
1443        };
1444        let action = policy.error_action(c, error);
1445        if !action.cooldown {
1446            return;
1447        }
1448        let credential_quota = crate::classify::credential_scoped(error);
1449        let key = if credential_quota {
1450            (c.id.clone(), String::new())
1451        } else {
1452            key
1453        };
1454        // A forced cooldown applies the normal policy even when cooling is disabled.
1455        let disabled = policy.cooling_disabled(c) && !action.force_cooldown;
1456        let status = crate::classify::go_status(error);
1457        let text = crate::classify::error_text(error);
1458        let prev = self.cooldowns.get(&key);
1459        // An ended bounded window that a newer probe took over, or that opened after this
1460        // attempt was picked, waits for its own probe: this older answer neither renews
1461        // nor replaces it. Inside a live window the handling below applies.
1462        if prev.is_some_and(|s| s.deadline <= now && s.newer_than(picked)) {
1463            return;
1464        }
1465        let prev_deadline = prev.map(|s| s.deadline);
1466        let prev_live = prev.filter(|s| s.deadline > now).map(|s| s.deadline);
1467        let prev_trust = prev.map(|s| s.trust).unwrap_or_default();
1468        // A live quota block this failure answers, if it is not a quota failure itself.
1469        let quota_under = prev.and_then(|p| p.quota_block(now));
1470        let mut level = prev.filter(|s| s.quota).map(|s| s.level).unwrap_or(0);
1471        // Causes other than a cut reset keep the trust count: nothing recovered.
1472        let mut trust = Trust {
1473            window: prev_trust.window,
1474            ..Trust::default()
1475        };
1476        let hint = error.retry_after.filter(|d| !d.is_zero());
1477        let mut quota = false;
1478        let mut cloudflare = false;
1479        let duration = if crate::classify::is_model_support(status, &text) {
1480            (!disabled).then(|| hint.unwrap_or(Duration::from_secs(43200)))
1481        } else if crate::classify::is_cloudflare(status, &text) {
1482            quota = true;
1483            cloudflare = true;
1484            (!disabled).then(|| {
1485                let (d, next) = quota_backoff(level);
1486                level = next;
1487                d.max(Duration::from_secs(10))
1488            })
1489        } else if crate::classify::is_invalid_grant(status, &text) {
1490            (!disabled).then_some(Duration::from_secs(1800))
1491        } else {
1492            match status {
1493                401..=403 => (!disabled).then_some(Duration::from_secs(1800)),
1494                404 => (!disabled).then(|| hint.unwrap_or(Duration::from_secs(43200))),
1495                429 => {
1496                    quota = true;
1497                    if credential_quota && prev.is_none_or(|s| !s.quota) {
1498                        level = 0;
1499                    }
1500                    if disabled {
1501                        None
1502                    } else if let Some(hint) = error.retry_after {
1503                        // A present hint, even zero, gets the 10s floor (Go
1504                        // minQuotaCooldownFloor); only an absent one backs off.
1505                        let stated = hint.max(Duration::from_secs(10));
1506                        let cap = policy.max_trusted_cooldown;
1507                        if !cap.is_zero()
1508                            && let Some(window) = prev.filter(|s| s.trust.bounded && s.deadline > now)
1509                        {
1510                            // A concurrent answer inside a live bounded window: the probe
1511                            // deadline and the count stand, but an earlier stated reset
1512                            // moves the probe up to it.
1513                            let reset = now.checked_add(stated);
1514                            let old = window.deadline;
1515                            let deadline = reset.map_or(old, |r| old.min(r));
1516                            let mut window_trust = window.trust;
1517                            if let Some(slot) = self.cooldowns.get_mut(&key) {
1518                                slot.deadline = deadline;
1519                                slot.error = text;
1520                                slot.trust.stated = reset.or(slot.trust.stated);
1521                                window_trust = slot.trust;
1522                            }
1523                            if credential_quota {
1524                                // Model keys that inherited this credential window (same
1525                                // deadline, bounded) follow it when it moves up; longer
1526                                // independent cooldowns stand.
1527                                for ((id, m), state) in &mut self.cooldowns {
1528                                    if *id == c.id && !m.is_empty() && state.trust.bounded && state.deadline == old {
1529                                        state.deadline = deadline;
1530                                        state.trust.stated = window_trust.stated;
1531                                        state.trust.window = state.trust.window.max(window_trust.window);
1532                                    }
1533                                }
1534                                self.extend_siblings(c, deadline, None, now);
1535                            }
1536                            return;
1537                        }
1538                        Some(match bound(stated, cap, prev_trust.window) {
1539                            Some((bound, window)) => {
1540                                self.generation = self.generation.wrapping_add(1);
1541                                trust = Trust {
1542                                    window: Some(window),
1543                                    generation: self.generation,
1544                                    bounded: true,
1545                                    accepted: false,
1546                                    stated: now.checked_add(stated),
1547                                    probe: None,
1548                                };
1549                                bound
1550                            }
1551                            None => stated,
1552                        })
1553                    } else if let Some(prev) = prev.filter(|s| s.quota && s.deadline > now) {
1554                        // Go `quotaCooldownAfterFailure`: an active quota deadline is
1555                        // reused, not extended.
1556                        let deadline = prev.deadline;
1557                        if let Some(slot) = self.cooldowns.get_mut(&key) {
1558                            slot.error = text;
1559                        }
1560                        if credential_quota {
1561                            self.extend_siblings(c, deadline, None, now);
1562                        }
1563                        return;
1564                    } else {
1565                        let (d, next) = quota_backoff(level);
1566                        level = next;
1567                        Some(d)
1568                    }
1569                }
1570                408 | 500 | 502..=504 | 520..=526 => transient(policy, hint, disabled),
1571                _ => transient(policy, None, disabled),
1572            }
1573        };
1574        // Go falls back to the one-minute transient cooldown when a forced cooldown's
1575        // policy produced none (for example transient cooldowns disabled).
1576        let duration = match duration {
1577            None if action.force_cooldown => Some(Duration::from_secs(60)),
1578            d => d,
1579        };
1580        let Some(next) = duration.and_then(|d| now.checked_add(d)) else {
1581            self.cooldowns.remove(&key);
1582            return;
1583        };
1584        let deadline = prev_live.map_or(next, |p| p.max(next));
1585        // An answer from a request already in flight (a 503, a 401) inside a live bounded
1586        // quota window that outlasts it: the window stands as it is, quota and all, so
1587        // its saved record keeps the stated reset and count. Only bounded windows; Go's
1588        // own cooldowns merge as Go does.
1589        if prev_live == Some(deadline)
1590            && let Some(slot) = self.cooldowns.get_mut(&key).filter(|s| s.quota && s.trust.bounded)
1591        {
1592            slot.error = text;
1593            // Go's level still counts a Cloudflare challenge (unchanged for other causes).
1594            slot.level = level;
1595            return;
1596        }
1597        // Provenance follows the deadline: a longer earlier cooldown that stands keeps
1598        // its own bounded flag and stated reset.
1599        let provenance = |deadline: Instant| {
1600            if deadline == next {
1601                trust
1602            } else if prev_live == Some(deadline) {
1603                Trust {
1604                    probe: None,
1605                    ..prev_trust
1606                }
1607            } else {
1608                Trust {
1609                    window: trust.window,
1610                    ..Trust::default()
1611                }
1612            }
1613        };
1614        let kind = |reason| {
1615            quota.then_some(QuotaKind {
1616                status: error.status,
1617                reason,
1618            })
1619        };
1620        let own_reason = if cloudflare { "cloudflare challenge" } else { "quota" };
1621        let state = |deadline, reason| Cooldown {
1622            deadline,
1623            level,
1624            status: error.status,
1625            quota,
1626            error: text.clone(),
1627            since: SystemTime::now(),
1628            credential: false,
1629            trust: provenance(deadline),
1630            quota_kind: kind(reason),
1631            quota_under: quota_under.filter(|_| !quota),
1632            recorded: 0,
1633        };
1634        let reason = if credential_quota {
1635            "credential_quota"
1636        } else {
1637            own_reason
1638        };
1639        self.cooldowns.insert(key, state(deadline, reason));
1640        if credential_quota {
1641            self.extend_siblings(c, deadline, prev_deadline, now);
1642            // Go also records the failing model's own quota state (reason `quota`).
1643            if !model.is_empty() {
1644                let own = (c.id.clone(), model.to_owned());
1645                // `extend_siblings` above already took a live own key to the credential's
1646                // deadline, unless it is longer, the model's own bounded window, or the
1647                // credential's window is bounded: then the own deadline stands.
1648                let own_deadline = self
1649                    .cooldowns
1650                    .get(&own)
1651                    .filter(|s| s.deadline > now)
1652                    .map_or(deadline, |s| s.deadline);
1653                let own_trust = self.cooldowns.get(&own).map(|s| s.trust).unwrap_or_default();
1654                let own_window = own_trust.window;
1655                let mut own_state = state(own_deadline, own_reason);
1656                if own_deadline != deadline {
1657                    // The model's own window (longer, or bounded) wins: its provenance
1658                    // goes with it.
1659                    own_state.trust = Trust {
1660                        probe: None,
1661                        ..own_trust
1662                    };
1663                } else {
1664                    own_state.trust = provenance(deadline);
1665                }
1666                own_state.trust.window = own_state.trust.window.max(own_window);
1667                self.cooldowns.insert(own, own_state);
1668            }
1669        }
1670    }
1671
1672    /// Go's credential-scoped 429: live sibling model states become quota cooldowns
1673    /// (`credential_quota`) lasting at least as long as the credential. Under a bounded
1674    /// credential window, and for a model's own bounded window under any, a model's own
1675    /// deadline is left as it is: the credential's key holds the model until the
1676    /// credential's probe anyway, and a copied deadline would follow the credential's
1677    /// when an earlier reset moves it up, before the model's own deadline (Go never
1678    /// shortens one). Only copies take the credential's deadline: the copies of the
1679    /// window this one `renews` (the credential's previous deadline) move onto it even
1680    /// when they ended, so no copy outlives the account's recovery.
1681    fn extend_siblings(&mut self, c: &Credential, deadline: Instant, renews: Option<Instant>, now: Instant) {
1682        let (level, trust, status) = self
1683            .cooldowns
1684            .get(&(c.id.clone(), String::new()))
1685            .map_or((0, Trust::default(), 429), |s| (s.level, s.trust, s.status));
1686        for ((id, model), state) in &mut self.cooldowns {
1687            if *id != c.id || model.is_empty() {
1688                continue;
1689            }
1690            let copy = state.trust.bounded && (state.deadline == deadline || Some(state.deadline) == renews);
1691            if state.deadline > now || copy {
1692                let window = state.trust.window.max(trust.window);
1693                if copy || !(state.trust.bounded || trust.bounded) {
1694                    if deadline >= state.deadline {
1695                        state.trust = Trust { probe: None, ..trust };
1696                    }
1697                    state.deadline = state.deadline.max(deadline);
1698                }
1699                // A model further along its own escalation keeps its count.
1700                state.trust.window = window;
1701                state.quota = true;
1702                state.credential = true;
1703                state.level = level;
1704                state.quota_kind = Some(QuotaKind {
1705                    status,
1706                    reason: "credential_quota",
1707                });
1708            }
1709        }
1710    }
1711
1712    /// Clears every cooldown of one credential (Go `ResetQuota`), returning the models
1713    /// that were cooling.
1714    pub fn reset(&mut self, id: &str) -> Vec<String> {
1715        let mut models: Vec<String> = self
1716            .cooldowns
1717            .keys()
1718            .filter(|(cid, _)| cid == id)
1719            .map(|(_, m)| m.clone())
1720            .collect();
1721        self.cooldowns.retain(|(cid, _), _| cid != id);
1722        self.affinity.invalidate(id);
1723        self.lcp.invalidate(id);
1724        self.probes.remove(id);
1725        models.sort();
1726        models
1727    }
1728
1729    /// Go `cooldownStateRecordsForAuthLocked`: one record per live cooldown of `c`, by
1730    /// model (the credential-wide quota is the model-less record).
1731    pub fn records(&self, c: &Credential, now: Instant, wall: SystemTime) -> Vec<crate::cooldown_store::Record> {
1732        use crate::cooldown_store::{LastError, Quota, Record};
1733        let mut out: Vec<Record> = self
1734            .cooldowns
1735            .iter()
1736            .filter(|((id, _), state)| *id == c.id && state.deadline > now)
1737            .map(|((_, model), state)| {
1738                let at = wall + (state.deadline - now);
1739                // Go's recovery time is the stated reset; the probe time is the retry.
1740                let recover = state
1741                    .trust
1742                    .stated
1743                    .filter(|r| state.trust.bounded && *r > state.deadline)
1744                    .map_or(at, |r| wall + (r - now));
1745                let reason = match (model.is_empty() || state.credential, state.quota, state.status) {
1746                    (true, true, _) => "credential_quota".to_owned(),
1747                    (false, true, 429) => "quota".to_owned(),
1748                    (false, true, _) => "cloudflare challenge".to_owned(),
1749                    (_, false, _) => state.error.clone(),
1750                };
1751                Record {
1752                    provider: c.provider.trim().to_owned(),
1753                    auth_id: c.id.clone(),
1754                    model: model.clone(),
1755                    status: "cooling".into(),
1756                    next_retry_after: Some(at),
1757                    quota: if state.quota {
1758                        Quota {
1759                            exceeded: true,
1760                            reason: reason.clone(),
1761                            next_recover_at: Some(recover),
1762                            backoff_level: state.level,
1763                            observed_at: None,
1764                            trust_windows: Some(state.trust.window.map_or(0, |w| w.saturating_add(1))),
1765                        }
1766                    } else {
1767                        Quota::default()
1768                    },
1769                    reason,
1770                    last_error: Some(LastError {
1771                        message: state.error.clone(),
1772                        http_status: u32::from(state.status),
1773                        ..LastError::default()
1774                    }),
1775                    updated_at: Some(state.since),
1776                    auth_file: match &c.source {
1777                        cpa_core::credential::Source::File(path) => Some(path.clone()),
1778                        _ => None,
1779                    },
1780                }
1781            })
1782            .collect();
1783        out.sort_by(|a, b| a.model.cmp(&b.model));
1784        out
1785    }
1786
1787    /// Go `restoreCooldownRecordLocked` for one live record of `c`. A model-less record
1788    /// restores the credential-wide quota; other model-less records aggregate the model
1789    /// records (Go recomputes them) and apply only when `c` has no model records.
1790    pub fn restore(
1791        &mut self,
1792        c: &Credential,
1793        record: &crate::cooldown_store::Record,
1794        has_model_records: bool,
1795        cap: Duration,
1796        now: Instant,
1797        wall: SystemTime,
1798    ) -> bool {
1799        // Go blocks until the later of the retry deadline and, for a quota state, the
1800        // quota recovery time (`availabilityBlock`); one deadline here.
1801        // Go restores only records whose retry deadline is still ahead.
1802        let Some(retry) = record.next_retry_after.filter(|at| *at > wall) else {
1803            return false;
1804        };
1805        let recover = record.quota.next_recover_at.filter(|_| record.quota.exceeded);
1806        let saved = record.quota.trust_windows.map(|n| n.min(MAX_TRUST_WINDOW + 1));
1807        let model = record.model.trim();
1808        if model.is_empty() && has_model_records && record.quota.reason != "credential_quota" {
1809            return false;
1810        }
1811        let since = |at: SystemTime| at.duration_since(wall).unwrap_or_default();
1812        let (remaining, trust) = match (record.quota.exceeded && !cap.is_zero(), saved) {
1813            // Written by cliproxy-rs: the saved retry is the deadline, as it is, whatever
1814            // set it (a bounded window, a trusted reset, or a longer cooldown from another
1815            // cause that a credential-wide quota marked), so a restart never cuts or
1816            // escalates it. A stated reset after the retry marks a bounded window.
1817            (true, Some(spent)) => {
1818                let stated = recover.filter(|r| *r > retry).map(|r| now + since(r));
1819                let trust = Trust {
1820                    window: spent.checked_sub(1),
1821                    generation: 0,
1822                    bounded: stated.is_some(),
1823                    accepted: false,
1824                    stated,
1825                    probe: None,
1826                };
1827                (since(retry), trust)
1828            }
1829            // A Go record, or one from before the bound: Go's deadline gets the first
1830            // bound. Go's `backoff_level` keeps its meaning and does not raise it.
1831            (true, None) => {
1832                let at = recover.map_or(retry, |r| retry.max(r));
1833                match bound(since(at), cap, None) {
1834                    Some((bound, window)) => (
1835                        bound,
1836                        Trust {
1837                            window: Some(window),
1838                            generation: 0,
1839                            bounded: true,
1840                            accepted: false,
1841                            stated: Some(now + since(at)),
1842                            probe: None,
1843                        },
1844                    ),
1845                    None => (since(at), Trust::default()),
1846                }
1847            }
1848            (false, _) => (since(recover.map_or(retry, |r| retry.max(r))), Trust::default()),
1849        };
1850        if remaining.is_zero() {
1851            return false;
1852        }
1853        let deadline = now + remaining;
1854        let error = record.last_error.clone().unwrap_or_default();
1855        let status = match error.http_status {
1856            0 if record.quota.exceeded => 429,
1857            s => u16::try_from(s).unwrap_or(0),
1858        };
1859        let key = (c.id.clone(), canonical_model(model).to_owned());
1860        let recorded = self.cooldowns.get(&key).map_or(0, |prev| prev.recorded);
1861        let prev = self.cooldowns.get(&key).filter(|prev| prev.deadline > now);
1862        let prev_wins = prev.is_some_and(|p| p.deadline > deadline);
1863        let deadline = prev.map_or(deadline, |prev| prev.deadline.max(deadline));
1864        let mut quota = (
1865            record.quota.exceeded,
1866            status,
1867            !model.is_empty() && record.quota.reason == "credential_quota",
1868        );
1869        let reason = match record.quota.reason.as_str() {
1870            "credential_quota" => "credential_quota",
1871            "cloudflare challenge" => "cloudflare challenge",
1872            _ => "quota",
1873        };
1874        let mut quota_kind = record.quota.exceeded.then_some(QuotaKind { status, reason });
1875        // A live quota block under the entry this merges with stays: a record saved while
1876        // a 401 answered a quota is the 401's alone.
1877        let quota_under = prev.and_then(|p| p.quota_block(now));
1878        let mut trust = trust;
1879        // With bounded trust on either side, provenance and the quota classification come
1880        // from the entry that supplies the deadline, and the counts merge. Without it the
1881        // merge is Go's.
1882        if let Some(prev) = prev.filter(|p| p.trust != Trust::default() || trust != Trust::default()) {
1883            let window = prev.trust.window.max(trust.window);
1884            if prev_wins {
1885                trust = Trust {
1886                    probe: None,
1887                    ..prev.trust
1888                };
1889                quota = (prev.quota, prev.status, prev.credential);
1890                quota_kind = prev.quota_kind;
1891            }
1892            trust.window = window;
1893        }
1894        // A bounded window this restore installs is newer than every lease in flight (a
1895        // restore also runs while serving, when `save-cooldown-status` turns on or its
1896        // directory moves); a live window it keeps keeps its generation.
1897        if trust.bounded && !prev_wins {
1898            self.generation = self.generation.wrapping_add(1);
1899            trust.generation = self.generation;
1900        }
1901        let (quota, status, credential) = quota;
1902        self.cooldowns.insert(
1903            key,
1904            Cooldown {
1905                deadline,
1906                level: record.quota.backoff_level,
1907                status,
1908                quota,
1909                error: if error.message.is_empty() {
1910                    record.reason.clone()
1911                } else {
1912                    error.message
1913                },
1914                since: record.updated_at.unwrap_or(wall),
1915                credential,
1916                trust,
1917                quota_kind: quota_kind.filter(|_| quota),
1918                quota_under: quota_under.filter(|_| !quota),
1919                // Go `mergeModelState` keeps the live state's watermark.
1920                recorded,
1921            },
1922        );
1923        true
1924    }
1925
1926    /// Go `clearDisabledCooldownStates`: drops cooldowns of credentials that are disabled
1927    /// or whose cooling the policy disables. Returns whether anything was cleared.
1928    pub fn clear_disabled(&mut self, credentials: &[std::sync::Arc<Credential>], policy: &Policy) -> bool {
1929        let before = self.cooldowns.len();
1930        self.cooldowns.retain(|(id, _), _| {
1931            credentials
1932                .iter()
1933                .find(|c| c.id == *id)
1934                .is_none_or(|c| !c.disabled && !policy.cooling_disabled(c))
1935        });
1936        self.cooldowns.len() != before
1937    }
1938
1939    pub fn reconcile(&mut self, credentials: &[std::sync::Arc<Credential>]) {
1940        self.cooldowns
1941            .retain(|(id, _), _| credentials.iter().any(|c| c.id == *id));
1942        self.affinity.retain(|id| credentials.iter().any(|c| c.id == id));
1943        self.lcp.retain(|id| credentials.iter().any(|c| c.id == id));
1944        self.probes.retain(|id, _| credentials.iter().any(|c| c.id == *id));
1945        self.rejected.retain(|id, _| credentials.iter().any(|c| c.id == *id));
1946    }
1947}
1948
1949/// Time left until a bounded cooldown's stated reset, while it is ahead.
1950fn recover_after(s: &Cooldown, now: Instant) -> Option<Duration> {
1951    let stated = s
1952        .trust
1953        .stated
1954        .filter(|r| s.trust.bounded && *r > now && *r > s.deadline)?;
1955    Some(stated - now)
1956}
1957
1958/// One active cooldown for management views (additive read API). `model` is empty for
1959/// a credential-wide cooldown.
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961pub struct CooldownState {
1962    pub model: String,
1963    /// Until the next attempt (zero once a bounded window ended and the next request
1964    /// probes the account).
1965    pub remaining: Duration,
1966    /// Until the upstream's stated reset, when the bound cut it (cliproxy-rs only).
1967    pub recover_in: Option<Duration>,
1968    pub level: u32,
1969    pub status: u16,
1970    pub quota: bool,
1971}
1972
1973impl Scheduler {
1974    /// Unexpired cooldowns of one credential, model keys sorted, credential-wide first.
1975    pub(crate) fn cooldowns_of(&self, id: &str, now: Instant) -> Vec<CooldownState> {
1976        let mut out: Vec<CooldownState> = self
1977            .cooldowns
1978            .iter()
1979            .filter(|((cid, _), s)| cid == id && (s.deadline > now || recover_after(s, now).is_some()))
1980            .map(|((_, model), s)| CooldownState {
1981                model: model.clone(),
1982                remaining: s.deadline.saturating_duration_since(now),
1983                recover_in: recover_after(s, now),
1984                level: s.level,
1985                status: s.status,
1986                quota: s.quota,
1987            })
1988            .collect();
1989        out.sort_by(|a, b| a.model.cmp(&b.model));
1990        out
1991    }
1992
1993    /// Clears every cooldown of one credential; returns the model keys that had one.
1994    pub(crate) fn reset_cooldowns(&mut self, id: &str) -> Vec<String> {
1995        let mut models: Vec<String> = self
1996            .cooldowns
1997            .keys()
1998            .filter(|(cid, model)| cid == id && !model.is_empty())
1999            .map(|(_, model)| model.clone())
2000            .collect();
2001        models.sort();
2002        self.cooldowns.retain(|(cid, _), _| cid != id);
2003        // A reset credential is probed again under `soonest-reset`; affinity is kept.
2004        self.probes.remove(id);
2005        models
2006    }
2007
2008    /// The unauthorized part of Go `BindCodexSIWC`: the 401 cooldowns of one
2009    /// credential are dropped, except that a quota block still live under one stays, as
2010    /// a quota cooldown again with its quota reason (Go clears only `LastError` and keeps
2011    /// the retry time). Returns whether anything changed.
2012    ///
2013    /// Model keys match like `clearUnauthorizedModelStates` (a 401, or "unauthorized" in
2014    /// the status message, which is the error here); the credential-wide key like
2015    /// `isUnauthorizedError`. Go also matches the error code `unauthorized`, which
2016    /// cooldowns do not keep.
2017    pub(crate) fn clear_unauthorized(&mut self, id: &str, now: Instant) -> bool {
2018        self.rejected.remove(id);
2019        let mut changed = false;
2020        self.cooldowns.retain(|(cid, model), c| {
2021            let error = c.error.to_ascii_lowercase();
2022            let unauthorized = c.status == 401
2023                || error.contains("status 401")
2024                || error.contains("401 unauthorized")
2025                || (!model.is_empty() && error.contains("unauthorized"));
2026            if cid != id || !unauthorized {
2027                return true;
2028            }
2029            changed = true;
2030            let Some((end, kind)) = c.quota_block(now) else {
2031                return false;
2032            };
2033            c.quota = true;
2034            c.quota_kind = Some(kind);
2035            c.quota_under = None;
2036            c.deadline = c.deadline.max(end);
2037            c.status = kind.status;
2038            // Go clears `LastError` and puts the quota reason in a status message that
2039            // names the failure.
2040            if error.contains("unauthorized") {
2041                c.error = kind.reason.into();
2042            }
2043            true
2044        });
2045        changed
2046    }
2047}
2048
2049#[cfg(test)]
2050mod tests {
2051    use super::*;
2052    use std::path::Path;
2053
2054    /// The quota a recovery keeps under a 401 survives a restore while serving (a saved
2055    /// record is the 401's alone) and keeps its own kind: a Cloudflare challenge that
2056    /// answered 429 stays one, and a credential-wide quota that reached a model's 401
2057    /// keeps the quota's status. Error text that does not name the 401 stays.
2058    #[test]
2059    fn a_quota_under_a_401_keeps_its_kind_through_restore_and_recovery() {
2060        let (c, d) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
2061        let p = Policy {
2062            max_trusted_cooldown: Duration::ZERO,
2063            ..Policy::default()
2064        };
2065        let (start, wall) = (Instant::now(), SystemTime::now());
2066        let unauthorized = |text: &str| Outcome::Failure(ExecError::local(401, FailureScope::Credential, text));
2067        let mut challenge = ExecError::local(429, FailureScope::Model, "<title>Just a moment...</title> cloudflare");
2068        challenge.retry_after = Some(secs(2 * H));
2069        let mut s = Scheduler::default();
2070        s.record(&c, "restored", &hinted(FailureScope::Model, secs(2 * H)), &p, start);
2071        s.record(&c, "cloudflare", &Outcome::Failure(challenge), &p, start);
2072        s.record(&c, "plain", &hinted(FailureScope::Model, secs(2 * H)), &p, start);
2073        s.record(&d, "sibling", &unauthorized("unauthorized"), &p, start);
2074        // Inside the challenge's 10s cooldown.
2075        let answered = start + secs(2);
2076        for model in ["restored", "cloudflare"] {
2077            s.record(&c, model, &unauthorized("unauthorized"), &p, answered);
2078        }
2079        s.record(&c, "plain", &unauthorized("status 401"), &p, answered);
2080        s.record(
2081            &d,
2082            "other",
2083            &hinted(FailureScope::Credential, secs(2 * H)),
2084            &p,
2085            answered,
2086        );
2087        assert!(
2088            entry(&s, &d, "sibling").quota,
2089            "the credential-wide quota reached the 401"
2090        );
2091        let restored = s
2092            .records(&c, answered, wall)
2093            .into_iter()
2094            .find(|r| r.model == "restored")
2095            .unwrap();
2096        assert!(!restored.quota.exceeded, "the saved record is the 401's");
2097        assert!(s.restore(&c, &restored, true, Duration::ZERO, answered, wall));
2098        let cases = [
2099            (&c, "restored", "quota"),
2100            (&c, "cloudflare", "cloudflare challenge"),
2101            (&c, "plain", "status 401"),
2102            (&d, "sibling", "credential_quota"),
2103        ];
2104        let blocked: Vec<_> = cases.iter().map(|(x, m, _)| s.wait(x, m, answered)).collect();
2105        assert!(s.clear_unauthorized("a", answered));
2106        assert!(s.clear_unauthorized("b", answered));
2107        for (x, model, error) in cases {
2108            let e = entry(&s, x, model);
2109            assert_eq!((e.quota, e.status, e.error.as_str()), (true, 429, error), "{model}");
2110        }
2111        let after: Vec<_> = cases.iter().map(|(x, m, _)| s.wait(x, m, answered)).collect();
2112        assert_eq!(after, blocked, "each stays blocked until the same time");
2113        assert!(after.iter().all(Option::is_some));
2114    }
2115
2116    /// Which cooldowns the SIWC recovery clears: model keys match a 401 or
2117    /// "unauthorized" in the error (Go's status-message check), the credential-wide key
2118    /// only Go's `isUnauthorizedError`; other causes and other credentials stay.
2119    #[test]
2120    fn clear_unauthorized_matches_go_per_key_kind() {
2121        let now = Instant::now();
2122        let cooldown = |status: u16, error: &str| Cooldown {
2123            deadline: now + Duration::from_secs(600),
2124            level: 0,
2125            status,
2126            quota: false,
2127            error: error.into(),
2128            since: SystemTime::now(),
2129            credential: false,
2130            trust: Trust::default(),
2131            quota_kind: None,
2132            quota_under: None,
2133            recorded: 0,
2134        };
2135        let mut s = Scheduler::default();
2136        for (id, model, c) in [
2137            ("a", "401", cooldown(401, "upstream said no")),
2138            ("a", "403-text", cooldown(403, "unauthorized operation")),
2139            ("a", "403", cooldown(403, "forbidden")),
2140            ("a", "", cooldown(403, "unauthorized operation")),
2141            ("b", "401", cooldown(401, "upstream said no")),
2142        ] {
2143            s.cooldowns.insert((id.into(), model.into()), c);
2144        }
2145        assert!(s.clear_unauthorized("a", now));
2146        let mut left: Vec<_> = s.cooldowns.keys().map(|(id, m)| (id.as_str(), m.as_str())).collect();
2147        left.sort();
2148        assert_eq!(left, [("a", ""), ("a", "403"), ("b", "401")]);
2149        for (status, error) in [(401, "x"), (403, "Status 401: revoked"), (403, "HTTP 401 Unauthorized")] {
2150            s.cooldowns.insert(("a".into(), String::new()), cooldown(status, error));
2151            assert!(s.clear_unauthorized("a", now), "{status} {error}");
2152            assert!(
2153                !s.cooldowns.contains_key(&("a".into(), String::new())),
2154                "{status} {error}"
2155            );
2156        }
2157        assert!(!s.clear_unauthorized("a", now), "nothing left to clear");
2158    }
2159
2160    /// Go keeps `ModelState.Quota` when a 401 lands on a model under a live quota, and
2161    /// the SIWC recovery keeps that block (with its quota reason) while it lasts: the
2162    /// model is not freed early. A quota that ended before the recovery goes with the 401.
2163    #[test]
2164    fn a_401_over_a_live_quota_keeps_the_quota_through_recovery() {
2165        let c = cred("a", serde_json::json!({}));
2166        let start = Instant::now();
2167        let unauthorized = Outcome::Failure(ExecError::local(401, FailureScope::Credential, "unauthorized"));
2168        // Go's own cooldowns (the 401 replaces the entry) and a bounded window (the
2169        // window stands and takes the 401's error).
2170        for (cap, replaced) in [(Duration::ZERO, 401), (Policy::default().max_trusted_cooldown, 429)] {
2171            let p = Policy {
2172                max_trusted_cooldown: cap,
2173                ..Policy::default()
2174            };
2175            let mut s = Scheduler::default();
2176            s.record(&c, "long", &hinted(FailureScope::Model, secs(2 * H)), &p, start);
2177            s.record(&c, "short", &hinted(FailureScope::Model, secs(MIN)), &p, start);
2178            let answered = start + secs(MIN / 2);
2179            s.record(&c, "long", &unauthorized, &p, answered);
2180            s.record(&c, "short", &unauthorized, &p, answered);
2181            assert_eq!(entry(&s, &c, "long").status, replaced, "{cap:?}");
2182            let blocked = s.wait(&c, "long", answered);
2183            let bind = start + secs(5 * MIN);
2184            assert!(s.clear_unauthorized("a", bind), "{cap:?}");
2185            let long = entry(&s, &c, "long");
2186            assert_eq!(
2187                (long.quota, long.status, long.error.as_str()),
2188                (true, 429, "quota"),
2189                "{cap:?}"
2190            );
2191            assert_eq!(
2192                s.wait(&c, "long", bind),
2193                blocked.map(|b| b - (bind - answered)),
2194                "{cap:?}: still blocked until the same time"
2195            );
2196            assert!(s.wait(&c, "long", bind).is_some_and(|w| w > secs(30 * MIN)), "{cap:?}");
2197            assert_eq!(s.wait(&c, "short", bind), None, "{cap:?}: its quota had ended");
2198        }
2199    }
2200
2201    fn tag<'a>(creds: &[&'a Credential]) -> Vec<(&'a Credential, &'static str)> {
2202        creds.iter().map(|c| (*c, "claude")).collect()
2203    }
2204
2205    fn cred(id: &str, extra: Value) -> Credential {
2206        let mut metadata = extra.as_object().unwrap().clone();
2207        metadata.insert("type".into(), "claude".into());
2208        Credential::from_file(Path::new("/mock"), &Path::new("/mock").join(id), metadata).unwrap()
2209    }
2210
2211    fn selection(model: &str) -> Selection {
2212        Selection {
2213            provider: "claude".into(),
2214            model: model.into(),
2215            ..Default::default()
2216        }
2217    }
2218
2219    #[test]
2220    fn normalization_matches_go_runtime_not_example_config() {
2221        for (raw, expected) in [
2222            (" WRR ", Strategy::WeightedRoundRobin),
2223            ("weightedroundrobin", Strategy::WeightedRoundRobin),
2224            ("weighted-round-robin", Strategy::WeightedRoundRobin),
2225            (" FF", Strategy::FillFirst),
2226            ("fillfirst", Strategy::FillFirst),
2227            ("fill-first", Strategy::FillFirst),
2228            ("soonest-reset", Strategy::SoonestReset),
2229            (" Reset-First ", Strategy::SoonestReset),
2230            ("resetfirst", Strategy::SoonestReset),
2231            ("", Strategy::SoonestReset),
2232            (" ", Strategy::SoonestReset),
2233            ("round-robin", Strategy::RoundRobin),
2234            ("anything", Strategy::RoundRobin),
2235        ] {
2236            assert_eq!(
2237                normalize(RawRouting {
2238                    strategy: raw,
2239                    ..Default::default()
2240                })
2241                .strategy,
2242                expected
2243            );
2244        }
2245        for (ttl, seconds) in [
2246            ("", 3600),
2247            ("bad", 3600),
2248            ("-1h", 3600),
2249            ("0", 3600),
2250            ("0.5s", 1),
2251            ("0.0000000001s", 3600),
2252            ("1h30m", 5400),
2253            (" 2m ", 120),
2254        ] {
2255            assert_eq!(
2256                normalize(RawRouting {
2257                    session_affinity_ttl: ttl,
2258                    ..Default::default()
2259                })
2260                .session_affinity_ttl,
2261                Duration::from_secs(seconds),
2262                "{ttl}"
2263            );
2264        }
2265        let policy = normalize(RawRouting {
2266            request_retry: -1,
2267            max_retry_credentials: -5,
2268            max_retry_interval: -3,
2269            session_affinity_subagents: Some(false),
2270            ..Default::default()
2271        });
2272        assert_eq!(policy.request_retry, 0);
2273        assert_eq!(policy.max_retry_credentials, 0);
2274        assert_eq!(policy.max_retry_interval, Duration::ZERO);
2275        assert!(policy.session_affinity_subagents, "ignored when affinity disabled");
2276        assert!(
2277            !normalize(RawRouting {
2278                session_affinity: true,
2279                session_affinity_subagents: Some(false),
2280                ..Default::default()
2281            })
2282            .session_affinity_subagents
2283        );
2284        assert!(
2285            normalize(RawRouting {
2286                session_affinity: true,
2287                ..Default::default()
2288            })
2289            .session_affinity_subagents
2290        );
2291    }
2292
2293    #[test]
2294    fn go_duration_decimal_compound_units_and_overflow() {
2295        for (raw, expected) in [
2296            ("1h2m3.4s", 3_723_400_000_000),
2297            (".5s", 500_000_000),
2298            ("+1.000000001s", 1_000_000_001),
2299            ("1.s", 1_000_000_000),
2300            ("1µs2μs3us", 6000),
2301            ("0.3333333333333333333h", 1_200_000_000_000),
2302            ("0.100000000000000000000h", 360_000_000_000),
2303            ("-9223372036854775808ns", i64::MIN),
2304            ("9223372036854775807ns", i64::MAX),
2305            ("0.9ns", 0),
2306        ] {
2307            assert_eq!(cpa_core::config::parse_duration(raw), Some(expected), "{raw}");
2308        }
2309        for raw in [
2310            "1d",
2311            ".s",
2312            "1",
2313            "1h 2m",
2314            "1s-2s",
2315            "1e3s",
2316            "9223372036854775808ns",
2317            "-9223372036854775809ns",
2318        ] {
2319            assert_eq!(cpa_core::config::parse_duration(raw), None, "{raw}");
2320        }
2321    }
2322
2323    #[test]
2324    fn rotation_tracks_previous_identity_and_provider_model_keys() {
2325        let a = cred("a", serde_json::json!({}));
2326        let b = cred("b", serde_json::json!({}));
2327        let c = cred("c", serde_json::json!({}));
2328        let p = Policy::default();
2329        let now = Instant::now();
2330        let mut s = Scheduler::default();
2331        assert_eq!(s.pick(&tag(&[&c, &b, &a]), &selection("m"), &p, now).unwrap().id, "a");
2332        assert_eq!(s.pick(&tag(&[&a, &c]), &selection("m"), &p, now).unwrap().id, "c");
2333        assert_eq!(s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now).unwrap().id, "a");
2334        assert_eq!(s.pick(&tag(&[&a, &b]), &selection("other"), &p, now).unwrap().id, "a");
2335        assert_eq!(s.pick(&tag(&[&a, &b]), &selection("m(high)"), &p, now).unwrap().id, "b");
2336    }
2337
2338    #[test]
2339    fn smooth_weights_keep_signed_credits_across_temporary_exclusions() {
2340        let a = cred("a", serde_json::json!({"weight":5}));
2341        let b = cred("b", serde_json::json!({"weight":1}));
2342        let c = cred("c", serde_json::json!({"weight":1}));
2343        let p = Policy {
2344            strategy: Strategy::WeightedRoundRobin,
2345            ..Default::default()
2346        };
2347        let now = Instant::now();
2348        let mut s = Scheduler::default();
2349        let picks: Vec<_> = (0..7)
2350            .map(|_| {
2351                s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now)
2352                    .unwrap()
2353                    .id
2354                    .clone()
2355            })
2356            .collect();
2357        assert_eq!(picks, ["a", "a", "b", "a", "c", "a", "a"]);
2358        let mut s = Scheduler::default();
2359        assert_eq!(s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now).unwrap().id, "a");
2360        assert_eq!(s.pick(&tag(&[&b, &c]), &selection("m"), &p, now).unwrap().id, "b");
2361        assert_eq!(s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now).unwrap().id, "a");
2362        assert_eq!(s.pick(&tag(&[&a, &b, &c]), &selection("m"), &p, now).unwrap().id, "c");
2363        for v in [
2364            serde_json::json!(0),
2365            serde_json::json!(-3),
2366            serde_json::json!(1_000_001),
2367            serde_json::json!(1.5),
2368            serde_json::json!("bad"),
2369        ] {
2370            assert!(!s.admits(&cred("off", serde_json::json!({"weight":v})), &p));
2371        }
2372        assert_eq!(weight(&cred("max", serde_json::json!({"weight":1_000_000}))), 1_000_000);
2373        assert_eq!(weight(&cred("float", serde_json::json!({"weight":2.0}))), 2);
2374    }
2375
2376    #[test]
2377    fn affinity_beats_recovered_priority_until_unusable_or_expired() {
2378        let low = cred("a", serde_json::json!({"priority":-1}));
2379        let high = cred("b", serde_json::json!({"priority":9}));
2380        let p = Policy {
2381            strategy: Strategy::FillFirst,
2382            session_affinity: true,
2383            session_affinity_ttl: Duration::from_secs(10),
2384            ..Default::default()
2385        };
2386        let sel = Selection {
2387            session: Some("session".into()),
2388            ..selection("m")
2389        };
2390        let now = Instant::now();
2391        let mut s = Scheduler::default();
2392        assert_eq!(s.pick(&tag(&[&low]), &sel, &p, now).unwrap().id, "a");
2393        assert_eq!(
2394            s.pick(&tag(&[&low, &high]), &sel, &p, now + Duration::from_secs(5))
2395                .unwrap()
2396                .id,
2397            "a"
2398        );
2399        assert_eq!(
2400            s.pick(&tag(&[&low, &high]), &sel, &p, now + Duration::from_secs(15))
2401                .unwrap()
2402                .id,
2403            "b"
2404        );
2405        assert_eq!(
2406            s.pick(&tag(&[&low]), &sel, &p, now + Duration::from_secs(16))
2407                .unwrap()
2408                .id,
2409            "a"
2410        );
2411        assert_eq!(s.pick(&tag(&[&low, &high]), &selection("m"), &p, now).unwrap().id, "b");
2412    }
2413
2414    #[test]
2415    fn max_trusted_cooldown_reads_durations_seconds_and_off() {
2416        let read = |raw| {
2417            normalize(RawRouting {
2418                max_trusted_cooldown: raw,
2419                ..Default::default()
2420            })
2421            .max_trusted_cooldown
2422        };
2423        assert_eq!(read(""), Duration::from_secs(3600));
2424        assert_eq!(read("90m"), Duration::from_secs(5400));
2425        assert_eq!(read("120"), Duration::from_secs(120));
2426        assert_eq!(read("0"), Duration::ZERO);
2427        assert_eq!(read("0s"), Duration::ZERO);
2428        assert_eq!(read("-5"), Duration::ZERO);
2429        for unreadable in ["soon", "1d", "false", "1.5"] {
2430            assert_eq!(read(unreadable), Duration::from_secs(3600), "{unreadable}");
2431        }
2432        assert_eq!(read("3"), Duration::from_secs(10), "clamped to Go's floor");
2433        assert_eq!(read("2s"), Duration::from_secs(10), "clamped to Go's floor");
2434    }
2435
2436    const H: u64 = 3600;
2437    const MIN: u64 = 60;
2438
2439    fn secs(n: u64) -> Duration {
2440        Duration::from_secs(n)
2441    }
2442
2443    /// A 429 whose stated reset is `stated` from `at`.
2444    fn hinted(scope: FailureScope, stated: Duration) -> Outcome {
2445        let mut e = ExecError::local(429, scope, "usage_limit_reached");
2446        e.retry_after = Some(stated);
2447        Outcome::Failure(e)
2448    }
2449
2450    fn unhinted(scope: FailureScope) -> Outcome {
2451        Outcome::Failure(ExecError::local(429, scope, "rate limited"))
2452    }
2453
2454    fn entry<'a>(s: &'a Scheduler, c: &Credential, model: &str) -> &'a Cooldown {
2455        &s.cooldowns[&(c.id.clone(), model.to_owned())]
2456    }
2457
2458    /// A stated reset six days away is trusted for one hour, then for doubling windows
2459    /// while the account keeps answering 429, and never past the stated reset. Success
2460    /// clears the escalation; `0` trusts the stated reset as Go does.
2461    #[test]
2462    fn quota_resets_are_trusted_up_to_a_doubling_bound() {
2463        let c = cred("a", serde_json::json!({}));
2464        let p = Policy::default();
2465        let start = Instant::now();
2466        let reset = start + secs(6 * 24 * H);
2467        for (scope, key) in [(FailureScope::Model, "m"), (FailureScope::Credential, "")] {
2468            let mut s = Scheduler::default();
2469            let mut now = start;
2470            for window in [1, 2, 4, 8, 16, 32, 64] {
2471                s.record(&c, "m", &hinted(scope, reset - now), &p, now);
2472                assert_eq!(s.wait(&c, "m", now), Some(secs(window * H)), "{scope:?}");
2473                assert_eq!(entry(&s, &c, key).trust.stated, Some(reset), "{scope:?}: stated kept");
2474                assert_eq!(entry(&s, &c, key).level, 0, "{scope:?}: Go's level untouched");
2475                now += secs(window * H);
2476                assert!(s.retry_eligible(&c, "m", now));
2477            }
2478            // 127 hours have passed; the next window would be 128 hours, past the reset.
2479            s.record(&c, "m", &hinted(scope, reset - now), &p, now);
2480            assert_eq!(
2481                s.wait(&c, "m", now),
2482                Some(reset - now),
2483                "{scope:?}: capped by the stated reset"
2484            );
2485            // The account came back early: success clears the escalation.
2486            let mut s = Scheduler::default();
2487            s.record(&c, "m", &hinted(scope, reset - start), &p, start);
2488            let then = start + secs(H);
2489            s.record(&c, "m", &hinted(scope, reset - then), &p, then);
2490            assert_eq!(s.wait(&c, "m", then), Some(secs(2 * H)));
2491            let later = start + secs(3 * H);
2492            s.record(&c, "m", &Outcome::Success, &p, later);
2493            s.record(&c, "m", &hinted(scope, reset - later), &p, later);
2494            assert_eq!(
2495                s.wait(&c, "m", later),
2496                Some(secs(H)),
2497                "{scope:?}: back to the first bound"
2498            );
2499        }
2500        // A reset shorter than the bound is kept as stated.
2501        let mut s = Scheduler::default();
2502        s.record(&c, "m", &hinted(FailureScope::Model, secs(30 * MIN)), &p, start);
2503        assert_eq!(s.wait(&c, "m", start), Some(secs(30 * MIN)));
2504        assert!(!entry(&s, &c, "m").trust.bounded);
2505        // `0` is Go: the whole stated reset.
2506        let go = Policy {
2507            max_trusted_cooldown: Duration::ZERO,
2508            ..Policy::default()
2509        };
2510        let mut s = Scheduler::default();
2511        s.record(&c, "m", &hinted(FailureScope::Model, reset - start), &go, start);
2512        assert_eq!(s.wait(&c, "m", start), Some(reset - start));
2513    }
2514
2515    /// The trust count is its own: Go's no-hint backoff level neither raises the first
2516    /// bound nor is changed by a cut window.
2517    #[test]
2518    fn trust_count_and_go_backoff_level_are_separate() {
2519        let c = cred("a", serde_json::json!({}));
2520        let p = Policy::default();
2521        let days = secs(6 * 24 * H);
2522        // No-hint 429s first: Go's level climbs (1 s, 2 s, 4 s, ...).
2523        let mut s = Scheduler::default();
2524        let mut now = Instant::now();
2525        for _ in 0..9 {
2526            s.record(&c, "m", &unhinted(FailureScope::Model), &p, now);
2527            now += secs(600);
2528        }
2529        assert_eq!(entry(&s, &c, "m").level, 9);
2530        s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
2531        assert_eq!(s.wait(&c, "m", now), Some(secs(H)), "the first bound, not 2^9 hours");
2532        assert_eq!(entry(&s, &c, "m").level, 9, "Go's level keeps its meaning");
2533        // Hinted first, then no-hint 429s after the window: Go's backoff runs from its
2534        // own level, and the next cut window still doubles.
2535        let mut s = Scheduler::default();
2536        let mut now = Instant::now();
2537        s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
2538        now += secs(H);
2539        s.record(&c, "m", &unhinted(FailureScope::Model), &p, now);
2540        assert_eq!(s.wait(&c, "m", now), Some(secs(1)), "Go's first backoff step");
2541        now += secs(1);
2542        s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
2543        assert_eq!(s.wait(&c, "m", now), Some(secs(2 * H)), "the trust count survived");
2544    }
2545
2546    /// A stated reset that lands during a cooldown the bound did not set (a 5xx, a
2547    /// 401, a no-hint 429 backoff) opens a bounded window instead of probing at once.
2548    #[test]
2549    fn a_reset_inside_another_cooldown_gets_the_bound() {
2550        let c = cred("a", serde_json::json!({}));
2551        let p = Policy::default();
2552        let days = secs(6 * 24 * H);
2553        let start = Instant::now();
2554        let first = |status: u16, scope| {
2555            let mut s = Scheduler::default();
2556            s.record(
2557                &c,
2558                "m",
2559                &Outcome::Failure(ExecError::local(status, scope, "earlier")),
2560                &p,
2561                start,
2562            );
2563            s
2564        };
2565        for (status, scope, earlier) in [
2566            (503, FailureScope::Credential, secs(60)),
2567            (429, FailureScope::Model, secs(1)),
2568        ] {
2569            let mut s = first(status, scope);
2570            assert_eq!(s.wait(&c, "m", start), Some(earlier));
2571            let at = start + Duration::from_millis(500);
2572            s.record(&c, "m", &hinted(FailureScope::Model, days), &p, at);
2573            assert_eq!(s.wait(&c, "m", at), Some(secs(H)), "after {status}");
2574            assert!(entry(&s, &c, "m").trust.bounded, "after {status}");
2575            // A concurrent 6-day answer inside that window neither extends nor escalates.
2576            let later = at + secs(60);
2577            s.record(&c, "m", &hinted(FailureScope::Model, days), &p, later);
2578            assert_eq!(s.wait(&c, "m", at), Some(secs(H)), "after {status}");
2579            assert_eq!(entry(&s, &c, "m").trust.window, Some(0), "after {status}");
2580        }
2581        // A longer cooldown from another cause stands, and is not marked bounded.
2582        let mut s = first(401, FailureScope::Credential);
2583        let cap = Policy {
2584            max_trusted_cooldown: secs(600),
2585            ..Policy::default()
2586        };
2587        s.record(&c, "m", &hinted(FailureScope::Model, days), &cap, start);
2588        assert_eq!(s.wait(&c, "m", start), Some(secs(30 * MIN)));
2589        assert!(!entry(&s, &c, "m").trust.bounded);
2590    }
2591
2592    /// Inside a live bounded window, a concurrent stated reset keeps the probe deadline
2593    /// unless it is earlier; with the setting off Go's merge applies.
2594    #[test]
2595    fn concurrent_resets_keep_or_shorten_the_window() {
2596        let c = cred("a", serde_json::json!({}));
2597        let p = Policy::default();
2598        let start = Instant::now();
2599        let window = |s: &mut Scheduler, policy: &Policy| {
2600            s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), policy, start);
2601        };
2602        // A 45-minute reset at minute 30 does not push the probe to minute 75.
2603        let mut s = Scheduler::default();
2604        window(&mut s, &p);
2605        let at = start + secs(30 * MIN);
2606        s.record(&c, "m", &hinted(FailureScope::Model, secs(45 * MIN)), &p, at);
2607        assert_eq!(s.wait(&c, "m", start), Some(secs(H)));
2608        // A 30-minute reset at minute 1 moves it up to minute 31.
2609        let mut s = Scheduler::default();
2610        window(&mut s, &p);
2611        let at = start + secs(MIN);
2612        s.record(&c, "m", &hinted(FailureScope::Model, secs(30 * MIN)), &p, at);
2613        assert_eq!(s.wait(&c, "m", start), Some(secs(31 * MIN)));
2614        assert_eq!(entry(&s, &c, "m").trust.window, Some(0), "not a new window");
2615        // Setting off: Go keeps the later deadline.
2616        let go = Policy {
2617            max_trusted_cooldown: Duration::ZERO,
2618            ..Policy::default()
2619        };
2620        let mut s = Scheduler::default();
2621        s.record(&c, "m", &hinted(FailureScope::Model, secs(H)), &go, start);
2622        let at = start + secs(30 * MIN);
2623        s.record(&c, "m", &hinted(FailureScope::Model, secs(45 * MIN)), &go, at);
2624        assert_eq!(s.wait(&c, "m", start), Some(secs(75 * MIN)));
2625    }
2626
2627    /// The first pick after a bounded window reserves the probe: the window keeps
2628    /// cooling until the probe's answer, and is never waited for by retries.
2629    #[test]
2630    fn one_probe_per_ended_window() {
2631        let c = cred("a", serde_json::json!({}));
2632        let p = Policy::default();
2633        let start = Instant::now();
2634        for (scope, key) in [(FailureScope::Model, "m"), (FailureScope::Credential, "")] {
2635            let mut s = Scheduler::default();
2636            s.record(&c, "m", &hinted(scope, secs(6 * 24 * H)), &p, start);
2637            let near = start + secs(H) - secs(5);
2638            assert!(
2639                !s.retry_eligible(&c, "m", near),
2640                "{scope:?}: a bounded window is not waited for"
2641            );
2642            let end = start + secs(H);
2643            assert_eq!(s.wait(&c, "m", end), None, "{scope:?}: the probe is due");
2644            s.reserve_probe(&c, "m", end);
2645            assert_eq!(s.wait(&c, "m", end), Some(PROBE_HOLD), "{scope:?}: reserved");
2646            assert!(s.quota_cooling(&c, "m", end), "{scope:?}");
2647            assert!(!s.retry_eligible(&c, "m", end), "{scope:?}");
2648            // An unrelated model of the same credential waits too when the window is the
2649            // credential's.
2650            assert_eq!(s.wait(&c, "other", end).is_some(), key.is_empty(), "{scope:?}");
2651            // The probe's 429 opens the next window; its success would clear all.
2652            s.record(&c, "m", &hinted(scope, secs(5 * 24 * H)), &p, end + secs(2));
2653            assert_eq!(s.wait(&c, "m", end + secs(2)), Some(secs(2 * H)), "{scope:?}");
2654            assert_eq!(entry(&s, &c, key).trust.probe, None);
2655            // A cancelled probe frees the window at once; a lost one after PROBE_HOLD.
2656            let next = end + secs(2) + secs(2 * H);
2657            s.reserve_probe(&c, "m", next);
2658            s.record(&c, "m", &Outcome::Cancelled, &p, next + secs(1));
2659            assert_eq!(s.wait(&c, "m", next + secs(1)), None, "{scope:?}");
2660            s.reserve_probe(&c, "m", next + secs(1));
2661            assert_eq!(s.wait(&c, "m", next + secs(1) + PROBE_HOLD), None, "{scope:?}");
2662        }
2663        // Only bounded windows reserve: an ended Go backoff does not.
2664        let mut s = Scheduler::default();
2665        s.record(&c, "m", &unhinted(FailureScope::Model), &p, start);
2666        s.reserve_probe(&c, "m", start + secs(2));
2667        assert_eq!(s.wait(&c, "m", start + secs(2)), None);
2668    }
2669
2670    /// A credential-wide window escalates the model keys too; when the credential
2671    /// recovers through a success on another model, they forget the count.
2672    #[test]
2673    fn a_success_on_another_model_clears_the_escalation() {
2674        let c = cred("a", serde_json::json!({}));
2675        let p = Policy::default();
2676        let start = Instant::now();
2677        let days = secs(6 * 24 * H);
2678        let mut s = Scheduler::default();
2679        s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, start);
2680        assert_eq!(entry(&s, &c, "a").trust.window, Some(0));
2681        let end = start + secs(H);
2682        s.record(&c, "b", &Outcome::Success, &p, end);
2683        s.record(&c, "a", &hinted(FailureScope::Model, days), &p, end);
2684        assert_eq!(s.wait(&c, "a", end), Some(secs(H)), "the first bound again");
2685    }
2686
2687    /// A probe the upstream accepted (a stream started) ends the window's question at
2688    /// once: the next pick goes through while the stream runs, and a later hinted 429
2689    /// still doubles. A credential-wide window's copies on model keys are disarmed too;
2690    /// a newer live window is left alone.
2691    #[test]
2692    fn an_accepted_probe_frees_the_account_and_keeps_the_count() {
2693        let c = cred("a", serde_json::json!({}));
2694        let p = Policy::default();
2695        let days = secs(6 * 24 * H);
2696        let start = Instant::now();
2697        let end = start + secs(H);
2698        // Model scope.
2699        let mut s = Scheduler::default();
2700        s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
2701        s.reserve_probe(&c, "m", end);
2702        assert!(s.wait(&c, "m", end).is_some(), "reserved");
2703        s.accept_probe(&c, "m", end);
2704        assert_eq!(s.wait(&c, "m", end), None, "the next pick goes through");
2705        s.reserve_probe(&c, "m", end + secs(1));
2706        assert_eq!(s.wait(&c, "m", end + secs(1)), None, "no second reservation");
2707        let later = end + secs(60);
2708        s.record(&c, "m", &hinted(FailureScope::Model, days), &p, later);
2709        assert_eq!(s.wait(&c, "m", later), Some(secs(2 * H)), "the count survived");
2710        // A live window (another model's newer one) is untouched.
2711        s.accept_probe(&c, "m", later + secs(1));
2712        assert!(entry(&s, &c, "m").trust.bounded);
2713        // Credential scope: B's accepted probe disarms A's copy as well.
2714        let mut s = Scheduler::default();
2715        s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, start);
2716        s.reserve_probe(&c, "b", end);
2717        s.accept_probe(&c, "b", end);
2718        assert_eq!(s.wait(&c, "b", end), None);
2719        s.reserve_probe(&c, "a", end);
2720        assert_eq!(s.wait(&c, "a", end), None, "A's copy does not reserve again");
2721        assert_eq!(entry(&s, &c, "a").trust.window, Some(0), "count kept");
2722    }
2723
2724    /// The credential's probe answers its window and the window's copies on model keys,
2725    /// not a model's own window that ended unprobed before it: that model still gets a
2726    /// probe of its own, after a streamed probe and after a plain success alike, and the
2727    /// credential probe's later success leaves that model's newer probe state alone.
2728    #[test]
2729    fn a_credential_probe_leaves_a_models_own_ended_window() {
2730        let c = cred("a", serde_json::json!({}));
2731        let p = Policy::default();
2732        let days = secs(6 * 24 * H);
2733        let start = Instant::now();
2734        for streamed in [true, false] {
2735            let mut s = Scheduler::default();
2736            s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
2737            let opened = start + secs(H) + secs(MIN);
2738            s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, opened);
2739            let end = opened + secs(H);
2740            s.reserve_probe(&c, "b", end);
2741            if streamed {
2742                s.accept_probe(&c, "b", end);
2743            } else {
2744                s.record(&c, "b", &Outcome::Success, &p, end);
2745            }
2746            s.reserve_probe(&c, "a", end);
2747            assert_eq!(s.wait(&c, "a", end), None, "streamed {streamed}: A's copy is answered");
2748            s.reserve_probe(&c, "m", end);
2749            assert_eq!(
2750                s.wait(&c, "m", end),
2751                Some(PROBE_HOLD),
2752                "streamed {streamed}: m reserves its own probe"
2753            );
2754        }
2755        // The credential's stream started, then m's probe was taken over by a later one
2756        // that was accepted too. The credential stream's success comes from an older
2757        // lease: it leaves m's accepted window alone, so the superseded m probe's late
2758        // 429 is still ignored and the later probe's success frees m.
2759        let mut s = Scheduler::default();
2760        s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
2761        let opened = start + secs(H) + secs(MIN);
2762        s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, opened);
2763        let end = opened + secs(H);
2764        let credential_probe = s.reserve_probe(&c, "b", end);
2765        s.accept_probe_picked(&c, "b", credential_probe, end);
2766        let first = s.reserve_probe(&c, "m", end);
2767        let lapsed = end + PROBE_HOLD + secs(1);
2768        let second = s.reserve_probe(&c, "m", lapsed);
2769        assert!(credential_probe != first && first != second, "real generations");
2770        s.accept_probe_picked(&c, "m", second, lapsed);
2771        let done = lapsed + secs(1);
2772        s.record_picked(&c, "b", &Outcome::Success, &p, credential_probe, done);
2773        s.record_picked(&c, "m", &hinted(FailureScope::Model, days), &p, first, done);
2774        assert_eq!(s.wait(&c, "m", done), None, "no window from the superseded probe");
2775        s.record_picked(&c, "m", &Outcome::Success, &p, second, done + secs(1));
2776        assert!(
2777            !s.cooldowns.contains_key(&(c.id.clone(), "m".to_owned())),
2778            "m recovered"
2779        );
2780    }
2781
2782    /// A probe whose hold lapsed was taken over by a later probe: its late 429 neither
2783    /// opens the next window nor touches the later probe's reservation, also once the
2784    /// later probe was accepted, so the later probe's success frees the account.
2785    #[test]
2786    fn a_superseded_probes_429_leaves_the_window_to_the_current_probe() {
2787        let c = cred("a", serde_json::json!({}));
2788        let p = Policy::default();
2789        let days = secs(6 * 24 * H);
2790        let start = Instant::now();
2791        for scope in [FailureScope::Model, FailureScope::Credential] {
2792            // The second probe is still unanswered, or already accepted (its stream's
2793            // first event arrived) when the first one's 429 comes back.
2794            for accepted in [false, true] {
2795                let mut s = Scheduler::default();
2796                s.record(&c, "m", &hinted(scope, days), &p, start);
2797                let end = start + secs(H);
2798                let first = s.reserve_probe(&c, "m", end);
2799                let lapsed = end + PROBE_HOLD + secs(1);
2800                let second = s.reserve_probe(&c, "m", lapsed);
2801                if accepted {
2802                    s.accept_probe_picked(&c, "m", second, lapsed);
2803                }
2804                let late = lapsed + secs(1);
2805                s.record_picked(&c, "m", &hinted(scope, days - secs(H)), &p, first, late);
2806                assert_eq!(
2807                    s.wait(&c, "m", late),
2808                    (!accepted).then(|| PROBE_HOLD - secs(1)),
2809                    "{scope:?}, accepted {accepted}: still the second probe's"
2810                );
2811                s.record_picked(&c, "m", &Outcome::Success, &p, second, late + secs(1));
2812                assert_eq!(
2813                    s.wait(&c, "m", late + secs(1)),
2814                    None,
2815                    "{scope:?}, accepted {accepted}: recovered"
2816                );
2817            }
2818        }
2819    }
2820
2821    /// A credential-wide window that its probe's 429 renews carries its copies on model
2822    /// keys along, ended ones included: when the next probe recovers the account, no copy
2823    /// is left to ask for a probe of its own or to keep the count.
2824    #[test]
2825    fn a_renewed_credential_window_carries_its_ended_copies() {
2826        let c = cred("a", serde_json::json!({}));
2827        let p = Policy::default();
2828        let days = secs(6 * 24 * H);
2829        let start = Instant::now();
2830        for streamed in [true, false] {
2831            let mut s = Scheduler::default();
2832            s.record(&c, "a", &hinted(FailureScope::Credential, days), &p, start);
2833            let first_end = start + secs(H);
2834            s.reserve_probe(&c, "b", first_end);
2835            let renewed = first_end + secs(1);
2836            s.record(&c, "b", &hinted(FailureScope::Credential, days - secs(H)), &p, renewed);
2837            let second_end = renewed + secs(2 * H);
2838            s.reserve_probe(&c, "c", second_end);
2839            if streamed {
2840                s.accept_probe(&c, "c", second_end);
2841            } else {
2842                s.record(&c, "c", &Outcome::Success, &p, second_end);
2843            }
2844            s.reserve_probe(&c, "a", second_end);
2845            assert_eq!(s.wait(&c, "a", second_end), None, "streamed {streamed}: no probe for A");
2846            if !streamed {
2847                s.record(&c, "a", &hinted(FailureScope::Model, days), &p, second_end);
2848                assert_eq!(s.wait(&c, "a", second_end), Some(secs(H)), "the count is cleared");
2849            }
2850        }
2851    }
2852
2853    /// A lease picked outside selection (the credits fallback) on a credential whose
2854    /// ended window has a probe in flight neither frees that probe nor ends the window,
2855    /// whatever its outcome; nor does it answer a window opened after it.
2856    #[test]
2857    fn outside_selection_leases_answer_no_bounded_window() {
2858        let c = cred("a", serde_json::json!({}));
2859        let p = Policy::default();
2860        let start = Instant::now();
2861        let mut s = Scheduler::default();
2862        s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, start);
2863        let ended = start + secs(H);
2864        let probe = s.reserve_probe(&c, "m", ended);
2865        let credits = s.outside_selection_generation();
2866        for outcome in [Outcome::Cancelled, Outcome::Success] {
2867            s.record_picked(&c, "m", &outcome, &p, credits, ended);
2868            assert!(s.wait(&c, "m", ended).is_some(), "the reserved probe holds the window");
2869        }
2870        s.accept_probe_picked(&c, "m", credits, ended);
2871        assert!(s.wait(&c, "m", ended).is_some(), "a credits stream accepts nothing");
2872        s.record_picked(&c, "m", &Outcome::Success, &p, probe, ended);
2873        assert_eq!(s.wait(&c, "m", ended), None, "the probe's own success ends it");
2874        // A window opened after the credits lease was picked is newer than it too.
2875        let credits = s.outside_selection_generation();
2876        let later = ended + secs(MIN);
2877        s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, later);
2878        let reopened = later + secs(H);
2879        s.reserve_probe(&c, "m", reopened);
2880        s.record_picked(&c, "m", &Outcome::Cancelled, &p, credits, reopened);
2881        assert!(s.wait(&c, "m", reopened).is_some(), "the later probe still holds");
2882    }
2883
2884    /// Generations compare in serial-number order: a window opened just after the counter
2885    /// wraps is newer than a lease picked just before, and older than one picked after.
2886    #[test]
2887    fn generations_compare_across_the_wrap() {
2888        let c = cred("a", serde_json::json!({}));
2889        let p = Policy::default();
2890        let start = Instant::now();
2891        let mut s = Scheduler {
2892            generation: u32::MAX,
2893            ..Scheduler::default()
2894        };
2895        let before = s.reserve_probe(&c, "m", start);
2896        assert_eq!(before, u32::MAX);
2897        s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, start);
2898        assert_eq!(entry(&s, &c, "m").trust.generation, 0, "wrapped");
2899        let at = start + secs(MIN);
2900        s.record_picked(&c, "m", &Outcome::Success, &p, before, at);
2901        assert_eq!(s.wait(&c, "m", at), Some(secs(H - MIN)), "an older success leaves it");
2902        s.record_picked(&c, "m", &Outcome::Success, &p, 0, at);
2903        assert_eq!(s.wait(&c, "m", at), None, "a newer success ends it");
2904    }
2905
2906    /// An earlier credential-wide reset moves up the model keys that inherited the
2907    /// window; an independent longer cooldown stands.
2908    #[test]
2909    fn an_earlier_credential_reset_shortens_inherited_model_windows() {
2910        let c = cred("a", serde_json::json!({}));
2911        let p = Policy::default();
2912        let start = Instant::now();
2913        let mut s = Scheduler::default();
2914        let support = ExecError::local(400, FailureScope::Model, "model is not supported");
2915        s.record(&c, "support", &Outcome::Failure(support), &p, start);
2916        s.record(&c, "sibling", &unhinted(FailureScope::Model), &p, start);
2917        s.record(&c, "a", &hinted(FailureScope::Credential, secs(6 * 24 * H)), &p, start);
2918        let at = start + secs(MIN);
2919        s.record(&c, "a", &hinted(FailureScope::Credential, secs(30 * MIN)), &p, at);
2920        for model in ["a", "sibling", "unrelated"] {
2921            assert_eq!(s.wait(&c, model, start), Some(secs(31 * MIN)), "{model}");
2922        }
2923        assert_eq!(s.wait(&c, "support", start), Some(secs(12 * H)));
2924    }
2925
2926    /// A model's own bounded window that a credential-wide window covers keeps its own
2927    /// probe time: an earlier credential reset moves the credential's probe up, never
2928    /// the model's before its own bound, whether the credential's 429 came from another
2929    /// model or from that one.
2930    #[test]
2931    fn an_earlier_credential_reset_never_moves_a_models_own_probe_earlier() {
2932        let c = cred("a", serde_json::json!({}));
2933        let p = Policy::default();
2934        let days = secs(6 * 24 * H);
2935        let start = Instant::now();
2936        let own_end = start + secs(H);
2937        for failing in ["a", "m"] {
2938            let mut s = Scheduler::default();
2939            s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
2940            let opened = start + secs(30 * MIN);
2941            s.record(&c, failing, &hinted(FailureScope::Credential, days), &p, opened);
2942            let at = opened + secs(MIN);
2943            s.record(&c, failing, &hinted(FailureScope::Credential, secs(10 * MIN)), &p, at);
2944            let moved = at + secs(10 * MIN);
2945            assert_eq!(
2946                s.wait(&c, "other", moved),
2947                None,
2948                "{failing}: the credential's probe is due"
2949            );
2950            assert_eq!(
2951                s.wait(&c, "m", moved),
2952                Some(own_end - moved),
2953                "{failing}: m waits for its own bound"
2954            );
2955        }
2956    }
2957
2958    /// A model's own cooldown of another cause (a 401's 30 minutes) under a bounded
2959    /// credential window keeps its own deadline, as Go keeps the later one: an earlier
2960    /// credential reset brings the credential's probe forward, never the model back
2961    /// before its own deadline, whether the credential's 429 came from another model or
2962    /// from that one.
2963    #[test]
2964    fn an_earlier_credential_reset_never_shortens_a_models_own_cooldown() {
2965        let c = cred("a", serde_json::json!({}));
2966        let p = Policy::default();
2967        let start = Instant::now();
2968        let unauthorized = Outcome::Failure(ExecError::local(401, FailureScope::Credential, "unauthorized"));
2969        for failing in ["a", "m"] {
2970            let mut s = Scheduler::default();
2971            s.record(&c, "m", &unauthorized, &p, start);
2972            assert_eq!(s.wait(&c, "m", start), Some(secs(30 * MIN)));
2973            let opened = start + secs(MIN);
2974            s.record(
2975                &c,
2976                failing,
2977                &hinted(FailureScope::Credential, secs(6 * 24 * H)),
2978                &p,
2979                opened,
2980            );
2981            let at = start + secs(2 * MIN);
2982            s.record(&c, failing, &hinted(FailureScope::Credential, secs(MIN)), &p, at);
2983            let moved = start + secs(3 * MIN);
2984            assert_eq!(
2985                s.wait(&c, "other", moved),
2986                None,
2987                "{failing}: the credential's probe is due"
2988            );
2989            assert_eq!(
2990                s.wait(&c, "m", moved),
2991                Some(secs(27 * MIN)),
2992                "{failing}: m keeps its own 30 minutes"
2993            );
2994        }
2995    }
2996
2997    /// Provenance follows the winning deadline: the failing model's own longer bounded
2998    /// window keeps its flag and stated reset through a credential-wide merge.
2999    #[test]
3000    fn the_own_key_keeps_its_window_through_a_credential_merge() {
3001        let c = cred("a", serde_json::json!({}));
3002        let p = Policy::default();
3003        let days = secs(6 * 24 * H);
3004        let start = Instant::now();
3005        let mut s = Scheduler::default();
3006        s.record(&c, "m", &hinted(FailureScope::Model, days), &p, start);
3007        s.record(
3008            &c,
3009            "m",
3010            &hinted(FailureScope::Model, days - secs(H)),
3011            &p,
3012            start + secs(H),
3013        );
3014        let own_end = start + secs(3 * H);
3015        let at = start + secs(H) + secs(MIN);
3016        s.record(&c, "m", &hinted(FailureScope::Credential, days), &p, at);
3017        let own = entry(&s, &c, "m");
3018        assert_eq!(own.deadline, own_end);
3019        assert!(own.trust.bounded && own.trust.stated == Some(start + days));
3020        s.reserve_probe(&c, "m", own_end);
3021        assert!(
3022            entry(&s, &c, "m").trust.probe.is_some(),
3023            "it reserves a probe at its end"
3024        );
3025    }
3026
3027    /// A restore into a scheduler that already holds a later deadline keeps that
3028    /// entry's provenance and classification, merging the counts.
3029    #[test]
3030    fn a_restore_keeps_the_winner_trust() {
3031        use crate::cooldown_store::{Quota, Record};
3032        let c = cred("a", serde_json::json!({}));
3033        let p = Policy::default();
3034        let (now, wall) = (Instant::now(), SystemTime::now());
3035        let days = secs(6 * 24 * H);
3036        let mut s = Scheduler::default();
3037        s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
3038        let live = entry(&s, &c, "m").trust;
3039        let shorter = Record {
3040            auth_id: "a".into(),
3041            model: "m".into(),
3042            next_retry_after: Some(wall + secs(10 * MIN)),
3043            quota: Quota {
3044                exceeded: true,
3045                reason: "quota".into(),
3046                next_recover_at: Some(wall + secs(10 * MIN)),
3047                trust_windows: Some(3),
3048                ..Default::default()
3049            },
3050            ..Default::default()
3051        };
3052        assert!(s.restore(&c, &shorter, true, secs(H), now, wall));
3053        let merged = entry(&s, &c, "m");
3054        assert_eq!(merged.deadline, now + secs(H));
3055        assert!(merged.trust.bounded && merged.trust.stated == live.stated);
3056        assert_eq!(merged.trust.window, Some(2), "counts merge");
3057        assert_eq!(merged.trust.generation, live.generation, "the live window's generation");
3058    }
3059
3060    /// Go 35af74f9 (conductor_issue6643_test.go): a success dispatched before a live
3061    /// cooldown was recorded leaves it, model-keyed or credential-level, and across a
3062    /// restore while serving; one dispatched after it, or one without a stamp, clears it.
3063    #[test]
3064    fn a_success_dispatched_before_a_cooldown_keeps_it() {
3065        let c = cred("a", serde_json::json!({}));
3066        let p = Policy {
3067            max_trusted_cooldown: Duration::ZERO,
3068            ..Policy::default()
3069        };
3070        let (now, wall) = (Instant::now(), SystemTime::now());
3071        let g = 0;
3072        let quota = || hinted(FailureScope::Model, secs(H));
3073        // Here a credential-level 429 is the credential's quota, which no success clears;
3074        // the credential-level case uses a 503 with a retry hint.
3075        let unavailable = || {
3076            let mut e = ExecError::local(503, FailureScope::Model, "unavailable");
3077            e.retry_after = Some(secs(H));
3078            Outcome::Failure(e)
3079        };
3080        for (model, failure) in [("m", quota()), ("", unavailable())] {
3081            let mut s = Scheduler::default();
3082            let stale = s.dispatch();
3083            let failed = s.dispatch();
3084            s.record_dispatched(&c, model, &failure, &p, g, failed, now);
3085            s.record_dispatched(&c, model, &Outcome::Success, &p, g, stale, now);
3086            s.record_dispatched(&c, model, &Outcome::Success, &p, g, failed, now);
3087            assert!(s.cooling(&c, model, now), "{model:?}: the stale successes keep it");
3088            let after = s.dispatch();
3089            s.record_dispatched(&c, model, &Outcome::Success, &p, g, after, now);
3090            assert!(!s.cooling(&c, model, now), "{model:?}: a later dispatch clears it");
3091
3092            let again = s.dispatch();
3093            s.record_dispatched(&c, model, &failure, &p, g, again, now);
3094            s.record_dispatched(&c, model, &Outcome::Success, &p, g, 0, now);
3095            assert!(!s.cooling(&c, model, now), "{model:?}: an unstamped success clears it");
3096        }
3097        // A skipped credential-level failure (a connection drop) still moves the
3098        // credential's watermark past a success dispatched before it.
3099        let mut s = Scheduler::default();
3100        let first = s.dispatch();
3101        s.record_dispatched(&c, "", &unavailable(), &p, g, first, now);
3102        let success = s.dispatch();
3103        let dropped = Outcome::Failure(ExecError::local(0, FailureScope::Model, "http2: client conn is closed"));
3104        let third = s.dispatch();
3105        s.record_dispatched(&c, "", &dropped, &p, g, third, now);
3106        s.record_dispatched(&c, "", &Outcome::Success, &p, g, success, now);
3107        assert!(s.cooling(&c, "", now), "the skipped failure stamped the credential");
3108        // Go `mergeModelState` keeps the live watermark through a restore; a cooldown
3109        // restored after a restart has none, and any success clears it.
3110        let mut s = Scheduler::default();
3111        let stale = s.dispatch();
3112        let failed = s.dispatch();
3113        s.record_dispatched(&c, "m", &quota(), &p, g, failed, now);
3114        let saved = s.records(&c, now, wall);
3115        assert!(s.restore(&c, &saved[0], true, Duration::ZERO, now, wall));
3116        s.record_dispatched(&c, "m", &Outcome::Success, &p, g, stale, now);
3117        assert!(s.cooling(&c, "m", now), "the restore kept the watermark");
3118        let mut restarted = Scheduler::default();
3119        assert!(restarted.restore(&c, &saved[0], true, Duration::ZERO, now, wall));
3120        let first = restarted.dispatch();
3121        restarted.record_dispatched(&c, "m", &Outcome::Success, &p, g, first, now);
3122        assert!(!restarted.cooling(&c, "m", now), "no watermark after a restart");
3123        // A credential-wide 429 stamps every sibling it extends.
3124        let mut s = Scheduler::default();
3125        let other = s.dispatch();
3126        s.record_dispatched(&c, "other", &quota(), &p, g, other, now);
3127        let stale = s.dispatch();
3128        let wide = s.dispatch();
3129        s.record_dispatched(&c, "m", &hinted(FailureScope::Credential, secs(H)), &p, g, wide, now);
3130        s.cooldowns.remove(&(c.id.clone(), String::new()));
3131        s.record_dispatched(&c, "other", &Outcome::Success, &p, g, stale, now);
3132        assert!(
3133            s.cooling(&c, "other", now),
3134            "the sibling carries the credential's watermark"
3135        );
3136    }
3137
3138    /// A restore while serving (`save-cooldown-status` turned on) that reinstalls a
3139    /// bounded window gives it a fresh generation: a stream picked before the window
3140    /// opened still cannot end it.
3141    #[test]
3142    fn a_restore_while_serving_keeps_the_window_newer_than_older_leases() {
3143        let c = cred("a", serde_json::json!({}));
3144        let p = Policy::default();
3145        let (now, wall) = (Instant::now(), SystemTime::now());
3146        let mut s = Scheduler::default();
3147        let stream = s.reserve_probe(&c, "m", now);
3148        s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, now);
3149        let saved = s.records(&c, now, wall);
3150        assert!(s.restore(&c, &saved[0], true, secs(H), now, wall));
3151        assert!(entry(&s, &c, "m").trust.bounded);
3152        let at = now + secs(MIN);
3153        s.record_picked(&c, "m", &Outcome::Success, &p, stream, at);
3154        assert_eq!(s.wait(&c, "m", at), Some(secs(H - MIN)), "the window stands");
3155    }
3156
3157    /// A 503 from a request already in flight inside a live bounded window: the window
3158    /// stands, and its saved record keeps the stated reset and count.
3159    #[test]
3160    fn a_503_inside_a_bounded_window_keeps_it() {
3161        let c = cred("a", serde_json::json!({}));
3162        let p = Policy::default();
3163        let (now, wall) = (Instant::now(), SystemTime::now());
3164        let days = secs(6 * 24 * H);
3165        let mut s = Scheduler::default();
3166        s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
3167        let at = now + secs(30 * MIN);
3168        let unavailable = ExecError::local(503, FailureScope::Model, "overloaded");
3169        s.record(&c, "m", &Outcome::Failure(unavailable), &p, at);
3170        let saved = &s.records(&c, at, wall + secs(30 * MIN))[0];
3171        assert!(saved.quota.exceeded);
3172        assert_eq!(saved.quota.next_recover_at, Some(wall + days));
3173        assert_eq!(saved.quota.trust_windows, Some(1));
3174        assert_eq!(saved.reason, "quota");
3175    }
3176
3177    /// A Cloudflare challenge from a request already in flight inside a live bounded
3178    /// window leaves the window as it is but still raises Go's level, as Go does.
3179    #[test]
3180    fn a_cloudflare_challenge_inside_a_bounded_window_raises_the_level() {
3181        let c = cred("a", serde_json::json!({}));
3182        let p = Policy::default();
3183        let start = Instant::now();
3184        let mut s = Scheduler::default();
3185        s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, start);
3186        let challenge = Outcome::Failure(ExecError::local(403, FailureScope::Model, "cloudflare challenge"));
3187        for (n, level) in [(1, 1), (2, 2)] {
3188            s.record(&c, "m", &challenge, &p, start + secs(n * MIN));
3189            assert_eq!(entry(&s, &c, "m").level, level);
3190            assert_eq!(s.wait(&c, "m", start), Some(secs(H)), "the window stands");
3191            assert!(entry(&s, &c, "m").trust.bounded);
3192        }
3193    }
3194
3195    /// `.cds` records: the probe time is the retry, the stated reset Go's recovery time,
3196    /// and the count is its own field. Restores never escalate, and Go's records get
3197    /// the first bound whatever their backoff level.
3198    #[test]
3199    fn restored_quota_resets_are_bounded_too() {
3200        use crate::cooldown_store::{Quota, Record};
3201        let c = cred("a", serde_json::json!({}));
3202        let p = Policy::default();
3203        let (now, wall) = (Instant::now(), SystemTime::now());
3204        let cap = secs(H);
3205        let days = secs(6 * 24 * H);
3206        let mut s = Scheduler::default();
3207        s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
3208        let saved = s.records(&c, now, wall);
3209        assert_eq!(saved[0].next_retry_after, Some(wall + secs(H)));
3210        assert_eq!(saved[0].quota.next_recover_at, Some(wall + days));
3211        assert_eq!(
3212            (saved[0].quota.trust_windows, saved[0].quota.backoff_level),
3213            (Some(1), 0)
3214        );
3215        // Our own record restores as it was: one hour, the same window.
3216        let mut back = Scheduler::default();
3217        assert!(back.restore(&c, &saved[0], true, cap, now, wall));
3218        assert_eq!(back.wait(&c, "m", now), Some(secs(H)));
3219        assert_eq!(back.records(&c, now, wall), saved);
3220        let record_at = |retry: Duration, trust_windows, backoff_level| Record {
3221            auth_id: "a".into(),
3222            model: "m".into(),
3223            next_retry_after: Some(wall + retry),
3224            quota: Quota {
3225                exceeded: true,
3226                reason: "quota".into(),
3227                next_recover_at: Some(wall + days),
3228                backoff_level,
3229                trust_windows,
3230                ..Default::default()
3231            },
3232            ..Default::default()
3233        };
3234        // Go's record with backoff level 2: the first bound; the level stays Go's.
3235        let mut s = Scheduler::default();
3236        assert!(s.restore(&c, &record_at(days, None, 2), true, cap, now, wall));
3237        assert_eq!(s.wait(&c, "m", now), Some(secs(H)));
3238        assert_eq!(entry(&s, &c, "m").level, 2);
3239        // Our record in its third window (4 h, count 2): restored as it is, and the
3240        // count survives, so the next cut window is 8 h.
3241        let mut s = Scheduler::default();
3242        assert!(s.restore(&c, &record_at(secs(4 * H), Some(3), 0), true, cap, now, wall));
3243        assert_eq!(s.wait(&c, "m", now), Some(secs(4 * H)));
3244        let trust = entry(&s, &c, "m").trust;
3245        assert!(trust.bounded && trust.window == Some(2));
3246        assert_eq!(trust.stated, Some(now + days));
3247        let later = now + secs(4 * H);
3248        s.record(&c, "m", &hinted(FailureScope::Model, days - secs(4 * H)), &p, later);
3249        assert_eq!(s.wait(&c, "m", later), Some(secs(8 * H)));
3250        // A reset trusted as stated restores as it was, unbounded, keeping the count.
3251        let ninety = secs(90 * MIN);
3252        let trusted = Record {
3253            quota: Quota {
3254                next_recover_at: Some(wall + ninety),
3255                ..record_at(ninety, Some(1), 0).quota
3256            },
3257            ..record_at(ninety, Some(1), 0)
3258        };
3259        let mut s = Scheduler::default();
3260        assert!(s.restore(&c, &trusted, true, cap, now, wall));
3261        assert_eq!(s.wait(&c, "m", now), Some(ninety));
3262        let back = &s.records(&c, now, wall)[0];
3263        assert_eq!(
3264            (
3265                back.next_retry_after,
3266                back.quota.next_recover_at,
3267                back.quota.trust_windows
3268            ),
3269            (trusted.next_retry_after, trusted.quota.next_recover_at, Some(1))
3270        );
3271        assert!(!entry(&s, &c, "m").trust.bounded);
3272        // An absurd saved count is clamped, not overflowed.
3273        let mut s = Scheduler::default();
3274        assert!(s.restore(&c, &record_at(secs(H), Some(u32::MAX), 0), true, cap, now, wall));
3275        assert_eq!(entry(&s, &c, "m").trust.window, Some(MAX_TRUST_WINDOW));
3276        // Setting off: Go's six days.
3277        let mut s = Scheduler::default();
3278        assert!(s.restore(&c, &record_at(days, Some(3), 0), true, Duration::ZERO, now, wall));
3279        assert_eq!(s.wait(&c, "m", now), Some(days));
3280    }
3281
3282    /// A longer cooldown of another cause that a credential-wide quota marked as quota
3283    /// (model B's 12 h model-support cooldown during a six-day 429 on model A) keeps its
3284    /// own deadline through a save and a restore: it is cliproxy-rs's record, not Go's.
3285    #[test]
3286    fn a_marked_sibling_restores_its_own_deadline() {
3287        let c = cred("a", serde_json::json!({}));
3288        let p = Policy::default();
3289        let (now, wall) = (Instant::now(), SystemTime::now());
3290        let mut s = Scheduler::default();
3291        let support = ExecError::local(400, FailureScope::Model, "model is not supported");
3292        s.record(&c, "b", &Outcome::Failure(support), &p, now);
3293        assert_eq!(s.wait(&c, "b", now), Some(secs(12 * H)), "Go's model-support cooldown");
3294        s.record(&c, "a", &hinted(FailureScope::Credential, secs(6 * 24 * H)), &p, now);
3295        let records = s.records(&c, now, wall);
3296        let b = records.iter().find(|r| r.model == "b").unwrap();
3297        assert!(b.quota.exceeded && b.quota.trust_windows.is_some(), "{b:?}");
3298        let mut back = Scheduler::default();
3299        for r in &records {
3300            back.restore(&c, r, true, secs(H), now, wall);
3301        }
3302        assert_eq!(back.wait(&c, "b", now), Some(secs(12 * H)));
3303        assert!(!entry(&back, &c, "b").trust.bounded, "not cut, so not a bounded window");
3304    }
3305
3306    /// A credential-wide window does not lower a model's own count: a model already in
3307    /// its fourth window (8 h) keeps it when a credential-wide 1 h window outlasts its
3308    /// own, so its next cut window is 16 h, not 2 h. Both the sibling and the failing
3309    /// model's own key.
3310    #[test]
3311    fn credential_wide_windows_keep_the_larger_model_count() {
3312        let c = cred("a", serde_json::json!({}));
3313        let p = Policy::default();
3314        let days = secs(6 * 24 * H);
3315        for failing in ["other", "m"] {
3316            let mut now = Instant::now();
3317            let mut s = Scheduler::default();
3318            for window in [1, 2, 4, 8] {
3319                s.record(&c, "m", &hinted(FailureScope::Model, days), &p, now);
3320                assert_eq!(s.wait(&c, "m", now), Some(secs(window * H)));
3321                if window < 8 {
3322                    now += secs(window * H);
3323                }
3324            }
3325            // Ten minutes before m's 8 h window ends, a credential-wide hint whose 1 h
3326            // window outlasts it.
3327            let late = now + secs(8 * H) - secs(10 * MIN);
3328            s.record(&c, failing, &hinted(FailureScope::Credential, days), &p, late);
3329            assert_eq!(entry(&s, &c, "m").trust.window, Some(3), "{failing}: count kept");
3330            let end = late + secs(H);
3331            s.record(&c, "m", &hinted(FailureScope::Model, days), &p, end);
3332            assert_eq!(s.wait(&c, "m", end), Some(secs(16 * H)), "{failing}");
3333        }
3334    }
3335
3336    /// Go's level climbs under Cloudflare challenges too, even inside a live window; a
3337    /// stated reset there still gets the first bound.
3338    #[test]
3339    fn a_reset_inside_a_cloudflare_window_gets_the_first_bound() {
3340        let c = cred("a", serde_json::json!({}));
3341        let p = Policy::default();
3342        let mut now = Instant::now();
3343        let mut s = Scheduler::default();
3344        for _ in 0..5 {
3345            let challenge = ExecError::local(403, FailureScope::Model, "cloudflare challenge");
3346            s.record(&c, "m", &Outcome::Failure(challenge), &p, now);
3347            now += secs(1);
3348        }
3349        assert!(entry(&s, &c, "m").level >= 5);
3350        assert!(s.wait(&c, "m", now).is_some(), "inside the challenge window");
3351        s.record(&c, "m", &hinted(FailureScope::Model, secs(6 * 24 * H)), &p, now);
3352        assert_eq!(s.wait(&c, "m", now), Some(secs(H)));
3353    }
3354
3355    #[test]
3356    fn cooldown_deadlines_floor_backoff_and_sibling_success() {
3357        let c = cred("a", serde_json::json!({}));
3358        let p = Policy::default();
3359        let now = Instant::now();
3360        let mut s = Scheduler::default();
3361        let fail = |status, scope, hint| {
3362            let mut e = ExecError::local(status, scope, "failure");
3363            e.retry_after = hint;
3364            Outcome::Failure(e)
3365        };
3366        s.record(&c, "sibling", &fail(404, FailureScope::Model, None), &p, now);
3367        s.record(
3368            &c,
3369            "m",
3370            &fail(429, FailureScope::Credential, Some(Duration::from_secs(2))),
3371            &p,
3372            now,
3373        );
3374        assert_eq!(s.wait(&c, "m", now), Some(Duration::from_secs(10)));
3375        assert_eq!(s.wait(&c, "new", now), Some(Duration::from_secs(10)));
3376        assert_eq!(
3377            s.wait(&c, "sibling", now),
3378            Some(Duration::from_secs(43200)),
3379            "quota must not shorten sibling"
3380        );
3381        s.record(&c, "sibling", &Outcome::Success, &p, now + Duration::from_secs(1));
3382        assert_eq!(
3383            s.wait(&c, "sibling", now),
3384            Some(Duration::from_secs(43200)),
3385            "inflight success cannot clear live quota"
3386        );
3387        assert_eq!(
3388            s.wait(&c, "new", now + Duration::from_secs(10)),
3389            None,
3390            "deadline boundary"
3391        );
3392        s.record(&c, "new", &Outcome::Success, &p, now + Duration::from_secs(10));
3393        assert!(s.wait(&c, "sibling", now + Duration::from_secs(10)).is_some());
3394        let mut s = Scheduler::default();
3395        s.record(&c, "m", &fail(429, FailureScope::Model, None), &p, now);
3396        assert_eq!(s.wait(&c, "m(high)", now), Some(Duration::from_secs(1)));
3397        s.record(&c, "m", &fail(429, FailureScope::Model, None), &p, now);
3398        assert_eq!(
3399            s.wait(&c, "m", now),
3400            Some(Duration::from_secs(1)),
3401            "active quota reused"
3402        );
3403        s.record(
3404            &c,
3405            "m",
3406            &fail(429, FailureScope::Model, None),
3407            &p,
3408            now + Duration::from_secs(1),
3409        );
3410        assert_eq!(
3411            s.wait(&c, "m", now + Duration::from_secs(1)),
3412            Some(Duration::from_secs(2))
3413        );
3414        assert_eq!(s.wait(&c, "other", now), None, "model failure stays local");
3415        for (status, seconds) in [
3416            (401, 1800),
3417            (402, 1800),
3418            (403, 1800),
3419            (404, 43200),
3420            (408, 60),
3421            (500, 60),
3422            (526, 60),
3423        ] {
3424            let mut s = Scheduler::default();
3425            s.record(&c, "m", &fail(status, FailureScope::Model, None), &p, now);
3426            assert_eq!(
3427                s.wait(&c, "m", now),
3428                Some(Duration::from_secs(seconds)),
3429                "status {status}"
3430            );
3431        }
3432    }
3433
3434    /// Go `Manager.MarkResult` cooldowns: goldens from tests/reference/server/main.go,
3435    /// replayed with upstream-shaped errors (headers present, executor scopes).
3436    #[test]
3437    fn cooldowns_match_go_mark_result() {
3438        let fixture: Value = serde_json::from_str(include_str!("../tests/fixtures/server_go.json")).unwrap();
3439        let cases = fixture["cooldown"].as_array().unwrap();
3440        assert_eq!(cases.len(), 25);
3441        // Go trusts every stated reset (docs/DIFFERENCES-FROM-GO.md).
3442        let policy = Policy {
3443            max_trusted_cooldown: Duration::ZERO,
3444            ..Policy::default()
3445        };
3446        for case in cases {
3447            let name = case["name"].as_str().unwrap();
3448            let c = cred(&format!("{name}.json"), serde_json::json!({}));
3449            let mut s = Scheduler::default();
3450            let now = Instant::now();
3451            for step in case["steps"].as_array().unwrap() {
3452                let status = step["status"].as_u64().unwrap() as u16;
3453                let scope = if step["credential_scope"].as_bool().unwrap() {
3454                    FailureScope::Credential
3455                } else {
3456                    match status {
3457                        429 => FailureScope::Model,
3458                        401..=403 | 408 | 500.. => FailureScope::Credential,
3459                        _ => FailureScope::Request,
3460                    }
3461                };
3462                let mut error = ExecError::local(status, scope, step["message"].as_str().unwrap());
3463                error
3464                    .headers
3465                    .insert("content-type", "application/json".parse().unwrap());
3466                let hint = step["retry_after_ms"].as_i64().unwrap();
3467                error.retry_after = (hint >= 0).then(|| Duration::from_millis(hint as u64));
3468                s.record(
3469                    &c,
3470                    step["model"].as_str().unwrap(),
3471                    &Outcome::Failure(error),
3472                    &policy,
3473                    now,
3474                );
3475            }
3476            for (model, seconds) in case["seconds"].as_object().unwrap() {
3477                let wait = s.wait(&c, model, now).unwrap_or_default();
3478                assert_eq!(
3479                    wait.as_secs_f64().round() as u64,
3480                    seconds.as_u64().unwrap(),
3481                    "{name} {model} wait"
3482                );
3483                let quota = case["quota"][model].as_bool().unwrap();
3484                // Go keeps `Quota.Exceeded` on an expired state; only a live one matters.
3485                assert_eq!(
3486                    s.quota_cooling(&c, model, now),
3487                    quota && wait > Duration::ZERO,
3488                    "{name} {model} quota"
3489                );
3490            }
3491        }
3492    }
3493
3494    /// Go `SessionAffinitySelector` (`Enrich`, `Pick` with fill-first, `OnResult`):
3495    /// goldens from tests/reference/server/main.go.
3496    #[test]
3497    fn session_affinity_matches_go_selector() {
3498        let fixture: Value = serde_json::from_str(include_str!("../tests/fixtures/server_go.json")).unwrap();
3499        let cases = fixture["affinity"].as_array().unwrap();
3500        assert_eq!(cases.len(), 25);
3501        for case in cases {
3502            let name = case["name"].as_str().unwrap();
3503            let lcp_case = case["lcp"].as_bool().unwrap_or_default();
3504            let creds: Vec<Credential> = ["a", "b", "c"]
3505                .iter()
3506                .map(|id| {
3507                    let mut c = cred(id, serde_json::json!({}));
3508                    if let Some(p) = case["priorities"][*id].as_str() {
3509                        c.attributes.insert("priority".into(), p.into());
3510                    }
3511                    c
3512                })
3513                .collect();
3514            let policy = Policy {
3515                strategy: Strategy::FillFirst,
3516                session_affinity: true,
3517                session_affinity_subagents: case["subagent_affinity"].as_bool().unwrap(),
3518                ..Default::default()
3519            };
3520            let mut s = Scheduler::default();
3521            let now = Instant::now();
3522            for (i, step) in case["steps"].as_array().unwrap().iter().enumerate() {
3523                let mut headers = axum::http::HeaderMap::new();
3524                for pair in step["headers"].as_array().into_iter().flatten() {
3525                    headers.append(
3526                        axum::http::HeaderName::from_bytes(pair[0].as_str().unwrap().as_bytes()).unwrap(),
3527                        pair[1].as_str().unwrap().parse().unwrap(),
3528                    );
3529                }
3530                let payload = step["payload"].as_str().unwrap_or_default().as_bytes();
3531                let format = cpa_core::format::Format::parse(case["format"].as_str().unwrap_or("openai")).unwrap();
3532                let caller = match step["caller"].as_str() {
3533                    Some(c) => c,
3534                    None => case["caller"].as_str().unwrap_or_default(),
3535                };
3536                let caller = if lcp_case { caller } else { "" };
3537                let session = crate::session::resolve(format, &headers, payload, None, caller);
3538                // dispatch::run's LCP step.
3539                let lcp = (lcp_case && !session.explicit)
3540                    .then(|| crate::lcp::Request::new(format.as_str(), payload, caller))
3541                    .flatten()
3542                    .map(std::sync::Arc::new);
3543                let sel = Selection {
3544                    session: session.id,
3545                    session_parent: session.parent,
3546                    session_fork: session.fork,
3547                    lcp,
3548                    ..selection("m")
3549                };
3550                let find = |id: &str| creds.iter().find(|c| c.id == id).unwrap();
3551                let outcome = |op: &str| {
3552                    if op.ends_with("ok") {
3553                        Outcome::Success
3554                    } else {
3555                        let status = step["status"].as_u64().unwrap() as u16;
3556                        let scope = if status >= 500 {
3557                            FailureScope::Credential
3558                        } else {
3559                            FailureScope::Request
3560                        };
3561                        let mut e = ExecError::local(status, scope, step["message"].as_str().unwrap());
3562                        e.headers.insert("content-type", "text/plain".parse().unwrap());
3563                        Outcome::Failure(e)
3564                    }
3565                };
3566                match step["op"].as_str().unwrap() {
3567                    op @ ("pick" | "pick_ok" | "pick_fail") => {
3568                        let available: Vec<&Credential> = step["available"]
3569                            .as_array()
3570                            .unwrap()
3571                            .iter()
3572                            .map(|id| find(id.as_str().unwrap()))
3573                            .collect();
3574                        let (picked, bound) = s
3575                            .pick_session(&tag(&available), &sel, &policy, &|_| Windows::default(), now)
3576                            .unwrap();
3577                        assert_eq!(picked.id, step["picked"].as_str().unwrap(), "{name} step {i}");
3578                        // The metadata Go's pickLCP wrote, as the attempt reports it.
3579                        let got = bound.as_ref().map(|m| {
3580                            let (node_kind, fork, compaction) = m.node();
3581                            serde_json::json!({
3582                                "session": m.session, "parent": m.parent, "node_kind": node_kind,
3583                                "fork": fork, "compaction": compaction, "generation": m.access,
3584                            })
3585                        });
3586                        assert_eq!(got.as_ref(), step.get("lcp"), "{name} step {i} lcp");
3587                        if op != "pick" {
3588                            let picked = picked.clone();
3589                            s.session_result(&picked, &sel, &outcome(op), &policy, bound.as_ref(), now);
3590                        }
3591                    }
3592                    op => {
3593                        s.session_result(
3594                            find(step["auth"].as_str().unwrap()),
3595                            &sel,
3596                            &outcome(op),
3597                            &policy,
3598                            None,
3599                            now,
3600                        );
3601                    }
3602                }
3603            }
3604        }
3605    }
3606
3607    /// The failure of a request picked before a policy change removes nothing bound
3608    /// after it, though the change replaced the matcher.
3609    #[test]
3610    fn delayed_failure_keeps_binding_made_after_policy_change() {
3611        let a = cred("a", serde_json::json!({}));
3612        let p = Policy {
3613            session_affinity: true,
3614            ..Default::default()
3615        };
3616        let next = Policy {
3617            session_affinity_ttl: Duration::from_secs(600),
3618            ..p.clone()
3619        };
3620        let now = Instant::now();
3621        let mut s = Scheduler::default();
3622        let body = br#"{"messages":[{"role":"user","content":"hello"}]}"#;
3623        let request = crate::lcp::Request::new("openai", body, "client-key").unwrap();
3624        let sel = Selection {
3625            lcp: Some(std::sync::Arc::new(request.clone())),
3626            ..selection("m")
3627        };
3628        let ranks = |_: &Credential| Windows::default();
3629        let mut old = None;
3630        for _ in 0..5 {
3631            old = s.pick_session(&tag(&[&a]), &sel, &p, &ranks, now).unwrap().1;
3632        }
3633        let old = old.unwrap();
3634        s.configure(&p, &next);
3635        let fresh = s
3636            .pick_session(&tag(&[&a]), &sel, &next, &ranks, now)
3637            .unwrap()
3638            .1
3639            .unwrap();
3640        assert!(fresh.access > old.access, "{} after {}", fresh.access, old.access);
3641        let failure = Outcome::Failure(ExecError::local(500, FailureScope::Credential, "boom"));
3642        s.session_result(&a, &sel, &failure, &p, Some(&old), now);
3643        let namespace = request.namespace("m");
3644        let kept = s.lcp.find(&namespace, request.prepared(), now);
3645        assert_eq!(kept.map(|m| m.auth), Some("a".to_owned()));
3646    }
3647
3648    /// After session affinity is turned off, a request picked while it was on binds
3649    /// nothing: neither its result nor a retry under its policy.
3650    #[test]
3651    fn leases_from_before_disabling_affinity_bind_nothing() {
3652        let a = cred("a", serde_json::json!({}));
3653        let on = Policy {
3654            session_affinity: true,
3655            ..Default::default()
3656        };
3657        let off = Policy::default();
3658        let now = Instant::now();
3659        let mut s = Scheduler::default();
3660        s.configure(&off, &on);
3661        let body = br#"{"messages":[{"role":"user","content":"hello"}]}"#;
3662        let request = crate::lcp::Request::new("openai", body, "client-key").unwrap();
3663        let namespace = request.namespace("m");
3664        let lcp = Selection {
3665            lcp: Some(std::sync::Arc::new(request.clone())),
3666            ..selection("m")
3667        };
3668        let explicit = Selection {
3669            session: Some("s1".into()),
3670            ..selection("m")
3671        };
3672        let ranks = |_: &Credential| Windows::default();
3673        let bound = s.pick_session(&tag(&[&a]), &lcp, &on, &ranks, now).unwrap().1;
3674        assert!(bound.is_some());
3675        s.configure(&on, &off);
3676        assert!(s.lcp.find(&namespace, request.prepared(), now).is_none());
3677        s.session_result(&a, &lcp, &Outcome::Success, &on, bound.as_ref(), now);
3678        s.session_result(&a, &explicit, &Outcome::Success, &on, None, now);
3679        let retry = s.pick_session(&tag(&[&a]), &lcp, &on, &ranks, now).unwrap().1;
3680        assert!(retry.is_none());
3681        s.pick_session(&tag(&[&a]), &explicit, &on, &ranks, now);
3682        assert!(s.lcp.find(&namespace, request.prepared(), now).is_none());
3683        assert!(s.affinity.get(&session_keys(&explicit).unwrap().primary, now).is_none());
3684        // Turning it back on binds again.
3685        s.configure(&off, &on);
3686        assert!(s.pick_session(&tag(&[&a]), &lcp, &on, &ranks, now).unwrap().1.is_some());
3687    }
3688
3689    #[test]
3690    fn policy_change_resets_rotation_and_affinity_but_keeps_cooldowns() {
3691        let a = cred("a", serde_json::json!({}));
3692        let b = cred("b", serde_json::json!({}));
3693        let p = Policy {
3694            session_affinity: true,
3695            ..Default::default()
3696        };
3697        let now = Instant::now();
3698        let mut s = Scheduler::default();
3699        let mut selection = selection("m");
3700        selection.session = Some("session".into());
3701        assert_eq!(s.pick(&tag(&[&a, &b]), &selection, &p, now).unwrap().id, "a");
3702        s.record(
3703            &a,
3704            "other",
3705            &Outcome::Failure(ExecError::local(429, FailureScope::Model, "quota")),
3706            &p,
3707            now,
3708        );
3709        let mut next = p.clone();
3710        next.request_retry = 2;
3711        s.configure(&p, &next);
3712        assert!(!s.rotations.is_empty(), "retry-only change preserves selector state");
3713        assert!(!s.affinity.is_empty());
3714        next.session_affinity_ttl = Duration::from_secs(5);
3715        s.configure(&p, &next);
3716        assert!(s.rotations.is_empty());
3717        assert!(s.affinity.is_empty());
3718        assert_eq!(s.wait(&a, "other", now), Some(Duration::from_secs(1)));
3719    }
3720
3721    #[test]
3722    fn forced_cooling_uses_fallback_and_quota_does_not_reuse_transient_state() {
3723        let now = Instant::now();
3724        // Go: a forced cooldown re-enables the normal policy despite disable_cooling
3725        // (conductor_cooldown.go MarkResult), so the status decides the duration.
3726        for (status, seconds) in [(401, 1800), (404, 43200), (429, 1), (503, 60)] {
3727            let c = cred(
3728                "a",
3729                serde_json::json!({"disable_cooling":true,
3730                "request_scoped_errors":[{"status":status,"match":["forced"],"action":"stop-and-cooldown"}]}),
3731            );
3732            let mut s = Scheduler::default();
3733            s.record(
3734                &c,
3735                "m",
3736                &Outcome::Failure(ExecError::local(status, FailureScope::Credential, "forced")),
3737                &Policy::default(),
3738                now,
3739            );
3740            assert_eq!(
3741                s.wait(&c, "m", now),
3742                Some(Duration::from_secs(seconds)),
3743                "status {status}"
3744            );
3745        }
3746        // With transient cooldowns disabled, the forced cooldown falls back to one minute.
3747        let c = cred(
3748            "a",
3749            serde_json::json!({"request_scoped_errors":[{"status":503,"match":["forced"],"action":"continue-and-cooldown"}]}),
3750        );
3751        let no_transient = Policy {
3752            transient_error_cooldown_seconds: -1,
3753            ..Policy::default()
3754        };
3755        let mut s = Scheduler::default();
3756        let forced = Outcome::Failure(ExecError::local(503, FailureScope::Credential, "forced"));
3757        s.record(&c, "m", &forced, &no_transient, now);
3758        assert_eq!(s.wait(&c, "m", now), Some(Duration::from_secs(60)));
3759        let plain = Outcome::Failure(ExecError::local(503, FailureScope::Credential, "plain"));
3760        let mut s = Scheduler::default();
3761        s.record(&c, "m", &plain, &no_transient, now);
3762        assert_eq!(
3763            s.wait(&c, "m", now),
3764            None,
3765            "negative transient seconds disable 503 cooldowns"
3766        );
3767        let c = cred("a", serde_json::json!({}));
3768        let p = Policy::default();
3769        let mut s = Scheduler::default();
3770        s.record(
3771            &c,
3772            "m",
3773            &Outcome::Failure(ExecError::local(503, FailureScope::Model, "transient")),
3774            &p,
3775            now,
3776        );
3777        s.record(
3778            &c,
3779            "m",
3780            &Outcome::Failure(ExecError::local(429, FailureScope::Model, "quota")),
3781            &p,
3782            now,
3783        );
3784        assert!(
3785            s.quota_cooling(&c, "m", now),
3786            "quota must replace transient classification"
3787        );
3788        assert_eq!(
3789            s.wait(&c, "m", now),
3790            Some(Duration::from_secs(60)),
3791            "live deadline never shortened"
3792        );
3793        let mut hint = ExecError::local(418, FailureScope::Model, "other status");
3794        hint.retry_after = Some(Duration::from_secs(900));
3795        s.record(&c, "other", &Outcome::Failure(hint), &p, now);
3796        assert_eq!(
3797            s.wait(&c, "other", now),
3798            Some(Duration::from_secs(60)),
3799            "unlisted statuses ignore hint"
3800        );
3801    }
3802
3803    /// Go matches openai-compatibility groups by `EqualFold` against the credential's
3804    /// compat name, provider key and provider (`resolveOpenAICompatConfig`), where σ and ς
3805    /// are equal. Group `ΟΣ` now registers `openai-compatible-οσ` (Go `strings.ToLower`),
3806    /// and the earlier group `openai-compatible-ος` still supplies its proxy and its
3807    /// disable-cooling setting.
3808    #[test]
3809    fn compat_groups_match_with_go_equal_fold() {
3810        let cfg = cpa_core::config::Config::parse(
3811            "openai-compatibility:\n  - name: openai-compatible-ος\n    base-url: http://127.0.0.1:9\n    disable-cooling: true\n    api-key-entries: [{api-key: k, proxy-url: 'http://127.0.0.1:8080'}]\n  - name: ΟΣ\n    base-url: http://127.0.0.1:9\n    api-key-entries: [{api-key: k}]\n",
3812        )
3813        .unwrap();
3814        let creds = cpa_core::config::credentials::from_config(&cfg);
3815        let c = creds
3816            .iter()
3817            .find(|c| c.attributes.get("compat_name").is_some_and(|n| n == "ΟΣ"))
3818            .unwrap();
3819        assert_eq!(cpa_core::registry::dynamic::provider_key(c), "openai-compatible-οσ");
3820        assert_eq!(
3821            cpa_core::config::credentials::api_key_config_proxy(&cfg, c),
3822            "http://127.0.0.1:8080"
3823        );
3824        let mut policy = crate::management::policy(&cfg);
3825        policy.compat_disable_cooling = compat_cooling(&cfg);
3826        assert_eq!(policy.provider_cooling(c), Some(true));
3827    }
3828
3829    /// OAuth request-scoped-error channels and the credential provider they are looked up
3830    /// by are both Go `strings.ToLower` (U+0130 becomes a plain `i`): rules keyed `HİGH`
3831    /// apply to a `"type":"HİGH"` file credential.
3832    #[test]
3833    fn oauth_rule_channels_lowercase_like_go() {
3834        let cfg = cpa_core::config::Config::parse(
3835            "oauth:\n  request-scoped-errors:\n    HİGH:\n      - {status: 503, match: [boom], action: stop}\n",
3836        )
3837        .unwrap();
3838        let c = cpa_core::config::credentials::from_file(
3839            &cfg,
3840            std::path::Path::new("/auth"),
3841            std::path::Path::new("/auth/a.json"),
3842            r#"{"type":"HİGH","access_token":"t"}"#.as_bytes(),
3843        )
3844        .unwrap()
3845        .unwrap();
3846        let action =
3847            crate::management::policy(&cfg).error_action(&c, &ExecError::local(503, FailureScope::Request, "boom"));
3848        assert!(action.matched && action.stop);
3849    }
3850
3851    /// Go matches rules against a local missing-thread error's structured body, which
3852    /// names `not_found_error`; its message does not.
3853    #[test]
3854    fn rules_match_the_structured_body_of_a_missing_thread() {
3855        let c = cred(
3856            "a",
3857            serde_json::json!({"request_scoped_errors":[{"status":404,"match":["not_found_error"],"action":"continue"}]}),
3858        );
3859        let e = ExecError::local(404, FailureScope::Request, cpa_exec::claude::THREAD_NOT_FOUND);
3860        let action = Policy::default().error_action(&c, &e);
3861        assert!(action.matched && !action.stop);
3862        // Any other 404 is matched by its text, including an upstream answer with the
3863        // same words.
3864        let other = ExecError::local(404, FailureScope::Request, "not found");
3865        assert!(!Policy::default().error_action(&c, &other).matched);
3866        let mut upstream = e.clone();
3867        upstream
3868            .headers
3869            .insert(axum::http::header::CONTENT_TYPE, "text/plain".parse().unwrap());
3870        assert!(!Policy::default().error_action(&c, &upstream).matched);
3871    }
3872
3873    #[test]
3874    fn rules_are_ordered_status_and_body_matches_with_canonical_precedence() {
3875        let c = cred(
3876            "a",
3877            serde_json::json!({"disable_cooling":true, "request_scoped_errors":[
3878            {"status":429,"match":[],"action":"stop"},
3879            {"status":500,"match":["quota"],"action":"stop"},
3880            {"status":429,"match-regexr":["[", "quota [0-9]+"],"action":" CONTINUE-AND-COOLDOWN "},
3881            {"status":429,"match":["quota"],"action":"stop"}]}),
3882        );
3883        let p = Policy::default();
3884        let e = ExecError::local(429, FailureScope::Request, "quota 12");
3885        let action = p.error_action(&c, &e);
3886        assert!(!action.stop);
3887        assert!(action.cooldown && action.force_cooldown);
3888        let now = Instant::now();
3889        let mut s = Scheduler::default();
3890        s.record(&c, "m", &Outcome::Failure(e), &p, now);
3891        assert!(
3892            s.wait(&c, "m", now).is_some(),
3893            "force cooling bypasses disabled cooling"
3894        );
3895        let c = cred(
3896            "a",
3897            serde_json::json!({"request_retry":0, "request-retry":5,
3898            "disable_cooling":false,"disable-cooling":true}),
3899        );
3900        assert_eq!(p.retry_limit(&c), 0);
3901        assert!(!p.cooling_disabled(&c));
3902        let legacy = cred(
3903            "legacy",
3904            serde_json::json!({"request_retry":null,"request-retry":2,
3905            "disable_cooling":"TrUe","disable-cooling":1}),
3906        );
3907        assert_eq!(
3908            p.retry_limit(&legacy),
3909            2,
3910            "invalid canonical override permits legacy fallback"
3911        );
3912        assert!(
3913            p.cooling_disabled(&legacy),
3914            "Go ParseBool rejects mixed case; numeric legacy accepted"
3915        );
3916        assert!(!p.cooling_disabled(&cred("numeric", serde_json::json!({"disable_cooling":0}))));
3917        assert_eq!(
3918            Policy {
3919                request_retry: 3,
3920                ..Default::default()
3921            }
3922            .retry_limit(&cred("b", serde_json::json!({"request_retry":"-1"}))),
3923            3
3924        );
3925        // Go `providerCoolingOverrideForAuth`: the first enabled compat entry named by
3926        // compat_name, provider_key or provider; credential metadata still wins.
3927        let cfg = cpa_core::config::Config::parse(
3928            "openai-compatibility:\n  - {name: skipped, base-url: '', disable-cooling: false}\n  - {name: Router, base-url: http://r.invalid, disable-cooling: true}\n  - {name: plain, base-url: http://p.invalid}\n",
3929        )
3930        .unwrap();
3931        let mut p = Policy {
3932            compat_disable_cooling: compat_cooling(&cfg),
3933            ..Default::default()
3934        };
3935        assert_eq!(
3936            p.compat_disable_cooling,
3937            [("Router".to_owned(), Some(true)), ("plain".to_owned(), None)],
3938            "legacy openai-compatibility is read through the v8 document"
3939        );
3940        let compat = |attrs: &[(&str, &str)], meta: Value| {
3941            let mut c = cred("compat", meta);
3942            c.provider = "openai-compatibility".into();
3943            for (k, v) in attrs {
3944                c.attributes.insert((*k).into(), (*v).into());
3945            }
3946            c
3947        };
3948        assert!(p.cooling_disabled(&compat(&[("compat_name", "router")], serde_json::json!({}))));
3949        assert!(p.cooling_disabled(&compat(&[("provider_key", "ROUTER")], serde_json::json!({}))));
3950        assert!(!p.cooling_disabled(&compat(
3951            &[("compat_name", "router")],
3952            serde_json::json!({"disable_cooling": false})
3953        )));
3954        p.disable_cooling = true;
3955        assert!(
3956            p.cooling_disabled(&compat(&[("compat_name", "plain")], serde_json::json!({}))),
3957            "an entry without disable-cooling defers to global"
3958        );
3959        p.disable_cooling = false;
3960        assert!(
3961            !p.cooling_disabled(&cred("claude-file", serde_json::json!({}))),
3962            "credentials outside OpenAI compatibility ignore entries"
3963        );
3964        for scope in [FailureScope::Request, FailureScope::Transport] {
3965            let mut s = Scheduler::default();
3966            s.record(
3967                &c,
3968                "m",
3969                &Outcome::Failure(ExecError::local(502, scope, "fault")),
3970                &p,
3971                now,
3972            );
3973            assert_eq!(s.wait(&c, "m", now), None);
3974        }
3975    }
3976
3977    // ---- soonest-reset (cliproxy-rs addition) ----
3978
3979    const DAY: u64 = 24 * 3600;
3980
3981    fn epoch(t: SystemTime) -> String {
3982        t.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs().to_string()
3983    }
3984
3985    /// Claude's measured headers observed at `now`: the 7-day reset `weekly` seconds
3986    /// later and the 5-hour window's status and reset.
3987    fn claude(now: SystemTime, weekly: u64, five_hour: &str, five_hour_reset: SystemTime) -> Windows {
3988        claude_at(now, now, weekly, five_hour, five_hour_reset)
3989    }
3990
3991    /// [`claude`] observed at `observed` and read at `now`.
3992    fn claude_at(
3993        observed: SystemTime,
3994        now: SystemTime,
3995        weekly: u64,
3996        five_hour: &str,
3997        five_hour_reset: SystemTime,
3998    ) -> Windows {
3999        let signals: std::collections::BTreeMap<String, String> = [
4000            ("Anthropic-Ratelimit-Unified-7d-Status", "allowed".to_owned()),
4001            ("Anthropic-Ratelimit-Unified-7d-Utilization", "0.40".to_owned()),
4002            (
4003                "Anthropic-Ratelimit-Unified-7d-Reset",
4004                epoch(observed + Duration::from_secs(weekly)),
4005            ),
4006            ("Anthropic-Ratelimit-Unified-5h-Status", five_hour.to_owned()),
4007            ("Anthropic-Ratelimit-Unified-5h-Reset", epoch(five_hour_reset)),
4008        ]
4009        .into_iter()
4010        .map(|(k, v)| (k.to_owned(), v))
4011        .collect();
4012        Windows::observed("claude", &signals, observed, now)
4013    }
4014
4015    fn soonest() -> Policy {
4016        Policy {
4017            strategy: Strategy::SoonestReset,
4018            ..Default::default()
4019        }
4020    }
4021
4022    /// The asymmetric case: `a` resets in four days and `b` in one, so fill-first (id
4023    /// order) would take `a` and round-robin would alternate.
4024    #[test]
4025    fn soonest_reset_spends_the_earliest_weekly_window_first() {
4026        let wall = SystemTime::now();
4027        let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4028        let in_an_hour = wall + Duration::from_secs(3600);
4029        // Both observed at `wall`, read at `now`; `b`'s 5-hour window in `b_five_hour`.
4030        let ranks_at = |b_five_hour: &'static str, now: SystemTime| {
4031            let (wa, wb) = (
4032                claude_at(wall, now, 4 * DAY, "allowed", in_an_hour),
4033                claude_at(wall, now, DAY, b_five_hour, in_an_hour),
4034            );
4035            move |cred: &Credential| if cred.id == "a" { wa } else { wb }
4036        };
4037        let policy = soonest();
4038        let mut s = Scheduler::default();
4039        let now = Instant::now();
4040        let ranks = ranks_at("allowed", wall);
4041        for _ in 0..3 {
4042            let picked = s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &policy, &ranks, now);
4043            assert_eq!(picked.unwrap().id, "b", "the sooner weekly reset is used up first");
4044        }
4045        // `b` cools on a 429: the next soonest takes over.
4046        s.record(
4047            &b,
4048            "m",
4049            &Outcome::Failure(ExecError::local(429, FailureScope::Credential, "quota")),
4050            &policy,
4051            now,
4052        );
4053        let ready: Vec<&Credential> = [&a, &b].into_iter().filter(|x| s.wait(x, "m", now).is_none()).collect();
4054        assert_eq!(ready.iter().map(|x| x.id.as_str()).collect::<Vec<_>>(), ["a"]);
4055        assert_eq!(
4056            s.pick_ranked(&tag(&ready), &selection("m"), &policy, &ranks, now)
4057                .unwrap()
4058                .id,
4059            "a"
4060        );
4061        // Without the cooldown, `b`'s used-up 5-hour window also hands over to `a`...
4062        let mut s = Scheduler::default();
4063        let exhausted = ranks_at("rejected", wall);
4064        for _ in 0..2 {
4065            assert_eq!(
4066                s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &policy, &exhausted, now)
4067                    .unwrap()
4068                    .id,
4069                "a"
4070            );
4071        }
4072        // ...until that window resets, an hour later.
4073        let reset = ranks_at("rejected", in_an_hour + Duration::from_secs(1));
4074        assert_eq!(
4075            s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &policy, &reset, now)
4076                .unwrap()
4077                .id,
4078            "b"
4079        );
4080    }
4081
4082    /// Accounts with no known reset get one probe request each, then rank by what they
4083    /// reported; one that reported nothing comes after the known resets.
4084    #[test]
4085    fn soonest_reset_probes_unknown_accounts_once_then_ranks_them() {
4086        let wall = SystemTime::now();
4087        let creds: Vec<Credential> = ["a", "b", "c", "d"]
4088            .iter()
4089            .map(|id| cred(id, serde_json::json!({})))
4090            .collect();
4091        let [a, b, c, d] = [&creds[0], &creds[1], &creds[2], &creds[3]];
4092        let in_an_hour = wall + Duration::from_secs(3600);
4093        // `d` resets in 3 days, `c` is used up, `a` and `b` were never observed.
4094        let known = |x: &Credential| match x.id.as_str() {
4095            "c" => claude(wall, DAY, "rejected", in_an_hour),
4096            "d" => claude(wall, 3 * DAY, "allowed", in_an_hour),
4097            _ => Windows::default(),
4098        };
4099        let policy = soonest();
4100        let mut s = Scheduler::default();
4101        let now = Instant::now();
4102        let pick = |s: &mut Scheduler, set: &[&Credential], ranks: &Ranks<'_>| {
4103            s.pick_ranked(&tag(set), &selection("m"), &policy, ranks, now)
4104                .unwrap()
4105                .id
4106                .clone()
4107        };
4108        // One probe each for the unknown accounts, then the known reset.
4109        let order: Vec<String> = (0..4).map(|_| pick(&mut s, &[a, b, c, d], &known)).collect();
4110        assert_eq!(order, ["a", "b", "d", "d"]);
4111        // `a`'s probe answered with a reset in 12 hours: it now ranks by it, ahead of
4112        // `d`; `b`'s answer reported nothing, so it stays behind the known resets.
4113        let learned = |x: &Credential| match x.id.as_str() {
4114            "a" => claude(wall + Duration::from_secs(1), 12 * 3600, "allowed", in_an_hour),
4115            _ => known(x),
4116        };
4117        assert_eq!(pick(&mut s, &[a, b, c, d], &learned), "a");
4118        assert_eq!(pick(&mut s, &[b, c, d], &learned), "d");
4119        assert_eq!(
4120            pick(&mut s, &[b, c], &learned),
4121            "b",
4122            "probed and unknown, still before used-up"
4123        );
4124        assert_eq!(
4125            pick(&mut s, &[c], &learned),
4126            "c",
4127            "a used-up account is the last resort"
4128        );
4129        // When `d`'s weekly reset passes, it is unknown again and gets a new probe.
4130        let passed = |x: &Credential| match x.id.as_str() {
4131            "d" => claude_at(
4132                wall,
4133                wall + Duration::from_secs(3 * DAY + 1),
4134                3 * DAY,
4135                "allowed",
4136                in_an_hour,
4137            ),
4138            _ => learned(x),
4139        };
4140        assert_eq!(pick(&mut s, &[a, b, d], &passed), "d");
4141        assert_eq!(pick(&mut s, &[a, b, d], &passed), "a");
4142    }
4143
4144    #[test]
4145    fn soonest_reset_rotates_equal_ranks() {
4146        let wall = SystemTime::now();
4147        let creds: Vec<Credential> = ["a", "b"].iter().map(|id| cred(id, serde_json::json!({}))).collect();
4148        let in_an_hour = wall + Duration::from_secs(3600);
4149        // The same reset for both.
4150        let ranks = |_: &Credential| claude(wall, 2 * DAY, "allowed", in_an_hour);
4151        let mut s = Scheduler::default();
4152        let now = Instant::now();
4153        let turns: Vec<String> = (0..4)
4154            .map(|_| {
4155                s.pick_ranked(&tag(&[&creds[0], &creds[1]]), &selection("m"), &soonest(), &ranks, now)
4156                    .unwrap()
4157                    .id
4158                    .clone()
4159            })
4160            .collect();
4161        assert_eq!(turns, ["a", "b", "a", "b"]);
4162    }
4163
4164    #[test]
4165    fn soonest_reset_never_moves_a_session_pin() {
4166        let wall = SystemTime::now();
4167        let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4168        let in_an_hour = wall + Duration::from_secs(3600);
4169        let ranks = |x: &Credential| claude(wall, if x.id == "a" { 4 * DAY } else { DAY }, "allowed", in_an_hour);
4170        let policy = Policy {
4171            session_affinity: true,
4172            ..soonest()
4173        };
4174        let mut s = Scheduler::default();
4175        let now = Instant::now();
4176        let mut pinned = selection("m");
4177        pinned.session = Some("claude:thread-1".into());
4178        // The thread starts while only `a` is available, and keeps `a` afterwards.
4179        assert_eq!(
4180            s.pick_ranked(&tag(&[&a]), &pinned, &policy, &ranks, now).unwrap().id,
4181            "a"
4182        );
4183        for _ in 0..2 {
4184            assert_eq!(
4185                s.pick_ranked(&tag(&[&a, &b]), &pinned, &policy, &ranks, now)
4186                    .unwrap()
4187                    .id,
4188                "a"
4189            );
4190        }
4191        // A new thread starts on the soonest reset.
4192        let mut other = selection("m");
4193        other.session = Some("claude:thread-2".into());
4194        assert_eq!(
4195            s.pick_ranked(&tag(&[&a, &b]), &other, &policy, &ranks, now).unwrap().id,
4196            "b"
4197        );
4198    }
4199
4200    #[test]
4201    fn default_strategy_stays_round_robin_and_ignores_resets() {
4202        assert_eq!(Policy::default().strategy, Strategy::RoundRobin);
4203        let wall = SystemTime::now();
4204        let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4205        let ranks = |x: &Credential| claude(wall, if x.id == "a" { 4 * DAY } else { DAY }, "allowed", wall);
4206        let mut s = Scheduler::default();
4207        let now = Instant::now();
4208        let turns: Vec<String> = (0..4)
4209            .map(|_| {
4210                s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &Policy::default(), &ranks, now)
4211                    .unwrap()
4212                    .id
4213                    .clone()
4214            })
4215            .collect();
4216        assert_eq!(turns, ["a", "b", "a", "b"]);
4217    }
4218
4219    #[test]
4220    fn soonest_reset_in_mixed_providers_stays_in_the_first_group() {
4221        let wall = SystemTime::now();
4222        let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4223        let x = cred("x", serde_json::json!({}));
4224        let in_an_hour = wall + Duration::from_secs(3600);
4225        // The second provider's `x` resets soonest, but fill-first order picks the group.
4226        let ranks = |c: &Credential| {
4227            let days = match c.id.as_str() {
4228                "a" => 4,
4229                "b" => 2,
4230                _ => 1,
4231            };
4232            claude(wall, days * DAY, "allowed", in_an_hour)
4233        };
4234        let sel = Selection {
4235            providers: vec!["claude".into(), "codex".into()],
4236            model: "m".into(),
4237            ..Default::default()
4238        };
4239        let candidates = [(&a, "claude"), (&b, "claude"), (&x, "codex")];
4240        let mut s = Scheduler::default();
4241        let picked = s.pick_ranked(&candidates, &sel, &soonest(), &ranks, Instant::now());
4242        assert_eq!(picked.unwrap().id, "b");
4243    }
4244
4245    #[test]
4246    fn windows_read_claude_and_codex_signals() {
4247        let now = SystemTime::now();
4248        let map = |pairs: &[(&str, String)]| -> std::collections::BTreeMap<String, String> {
4249            pairs.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect()
4250        };
4251        // A weekly reset already behind is unknown, not "soonest".
4252        let stale = map(&[(
4253            "Anthropic-Ratelimit-Unified-7d-Reset",
4254            epoch(now - Duration::from_secs(60)),
4255        )]);
4256        let w = Windows::observed("claude", &stale, now, now);
4257        assert_eq!(
4258            (
4259                w.weekly_reset,
4260                w.rolled_over.map(|r| (r.window, epoch(r.at))),
4261                w.exhausted
4262            ),
4263            (
4264                None,
4265                Some((WindowId::Claude7d, epoch(now - Duration::from_secs(60)))),
4266                false
4267            )
4268        );
4269        // Utilization 1.0 exhausts the 5-hour window until its reset.
4270        let full = map(&[
4271            ("Anthropic-Ratelimit-Unified-5h-Utilization", "1.0".into()),
4272            (
4273                "Anthropic-Ratelimit-Unified-5h-Reset",
4274                epoch(now + Duration::from_secs(600)),
4275            ),
4276        ]);
4277        assert!(Windows::observed("claude", &full, now, now).exhausted);
4278        assert!(!Windows::observed("claude", &full, now, now + Duration::from_secs(601)).exhausted);
4279        // Codex: the longer window is weekly whichever header carries it; reset-after
4280        // counts from the observation.
4281        let observed = now - Duration::from_secs(100);
4282        let codex = map(&[
4283            ("X-Codex-Primary-Used-Percent", "100".into()),
4284            ("X-Codex-Primary-Window-Minutes", "300".into()),
4285            ("X-Codex-Primary-Reset-After-Seconds", "3600".into()),
4286            ("X-Codex-Secondary-Used-Percent", "40".into()),
4287            ("X-Codex-Secondary-Window-Minutes", "10080".into()),
4288            ("X-Codex-Secondary-Reset-At", epoch(now + Duration::from_secs(2 * DAY))),
4289        ]);
4290        let w = Windows::observed("codex", &codex, observed, now);
4291        assert!(w.exhausted, "the 5-hour primary is used up");
4292        assert_eq!(
4293            w.weekly_reset.map(epoch),
4294            Some(epoch(now + Duration::from_secs(2 * DAY)))
4295        );
4296        assert!(!Windows::observed("codex", &codex, observed, observed + Duration::from_secs(3601)).exhausted);
4297        let weekly_primary = map(&[
4298            ("X-Codex-Primary-Used-Percent", "51".into()),
4299            ("X-Codex-Primary-Window-Minutes", "10080".into()),
4300            ("X-Codex-Primary-Reset-At", epoch(now + Duration::from_secs(3 * DAY))),
4301        ]);
4302        let w = Windows::observed("codex", &weekly_primary, now, now);
4303        assert_eq!(
4304            (w.weekly_reset.map(epoch), w.exhausted),
4305            (Some(epoch(now + Duration::from_secs(3 * DAY))), false)
4306        );
4307        let reached = map(&[("X-Codex-Limit-Reached", "True".into())]);
4308        assert!(Windows::observed("codex", &reached, now, now).exhausted);
4309        // Another provider's headers mean nothing here.
4310        let w = Windows::observed("gemini", &codex, now, now);
4311        assert_eq!((w.weekly_reset, w.exhausted), (None, false));
4312    }
4313
4314    // ---- soonest-reset review fixes ----
4315
4316    fn signals(pairs: &[(&str, String)]) -> std::collections::BTreeMap<String, String> {
4317        pairs.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect()
4318    }
4319
4320    /// P1: an account answering with quota signals but no weekly reset is probed once;
4321    /// each new answer (a newer observation) must not re-arm the probe, or it would take
4322    /// every request ahead of an account with a known reset.
4323    #[test]
4324    fn soonest_reset_answers_without_a_reset_do_not_re_arm_the_probe() {
4325        let wall = SystemTime::now();
4326        let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4327        let policy = soonest();
4328        let mut s = Scheduler::default();
4329        let now = Instant::now();
4330        let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
4331        let mut picks = Vec::new();
4332        for i in 0..5u64 {
4333            // `a` has answered `i` times, each time with a fresh 5-hour status only.
4334            let a_seen = (i > 0).then(|| {
4335                let seen = wall + Duration::from_secs(i);
4336                Windows::observed(
4337                    "claude",
4338                    &signals(&[("Anthropic-Ratelimit-Unified-5h-Status", "allowed".into())]),
4339                    seen,
4340                    seen,
4341                )
4342            });
4343            let ranks = |x: &Credential| {
4344                if x.id == "a" {
4345                    a_seen.unwrap_or_default()
4346                } else {
4347                    b_known
4348                }
4349            };
4350            picks.push(
4351                s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &policy, &ranks, now)
4352                    .unwrap()
4353                    .id
4354                    .clone(),
4355            );
4356        }
4357        assert_eq!(picks, ["a", "b", "b", "b", "b"]);
4358    }
4359
4360    /// The probe is reserved when picked: one that fails before any header arrives (no
4361    /// new observation) is spent, so the next picks go to the known reset.
4362    #[test]
4363    fn soonest_reset_a_probe_that_fails_before_headers_is_spent() {
4364        let wall = SystemTime::now();
4365        let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4366        let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
4367        let ranks = |x: &Credential| if x.id == "a" { Windows::default() } else { b_known };
4368        let mut s = Scheduler::default();
4369        let now = Instant::now();
4370        let mut pick = || {
4371            s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
4372                .unwrap()
4373                .id
4374                .clone()
4375        };
4376        assert_eq!(pick(), "a", "the probe");
4377        assert_eq!([pick(), pick()], ["b", "b"]);
4378    }
4379
4380    /// A known reset that passes rolls the window over and re-arms exactly one probe.
4381    #[test]
4382    fn soonest_reset_re_arms_only_on_a_rollover() {
4383        let wall = SystemTime::now();
4384        let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4385        let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
4386        let a_reset = wall + Duration::from_secs(60);
4387        // `a` reported a weekly reset in 60 s; read 61 s later it has rolled over.
4388        let a_rolled = |seen: SystemTime| {
4389            Windows::observed(
4390                "claude",
4391                &signals(&[("Anthropic-Ratelimit-Unified-7d-Reset", epoch(a_reset))]),
4392                seen,
4393                wall + Duration::from_secs(61),
4394            )
4395        };
4396        let mut s = Scheduler::default();
4397        let now = Instant::now();
4398        let mut picks = Vec::new();
4399        for i in 0..3u64 {
4400            let a_now = a_rolled(wall + Duration::from_secs(i));
4401            let ranks = |x: &Credential| if x.id == "a" { a_now } else { b_known };
4402            picks.push(
4403                s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
4404                    .unwrap()
4405                    .id
4406                    .clone(),
4407            );
4408        }
4409        assert_eq!(picks, ["a", "b", "b"], "one probe per rollover");
4410        // The next weekly rollover re-arms it once more.
4411        let next = a_reset + Duration::from_secs(7 * DAY);
4412        let later = Windows::observed(
4413            "claude",
4414            &signals(&[("Anthropic-Ratelimit-Unified-7d-Reset", epoch(next))]),
4415            wall,
4416            next + Duration::from_secs(1),
4417        );
4418        let ranks = |x: &Credential| if x.id == "a" { later } else { b_known };
4419        let mut pick = || {
4420            s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
4421                .unwrap()
4422                .id
4423                .clone()
4424        };
4425        assert_eq!([pick(), pick()], ["a", "b"]);
4426    }
4427
4428    /// Codex answers with `Reset-After-Seconds: 0` and no `Reset-At`: the reset is the
4429    /// observation time itself, which must not count as a new rollover on every answer.
4430    #[test]
4431    fn soonest_reset_zero_relative_resets_do_not_re_arm_the_probe() {
4432        let wall = SystemTime::now();
4433        let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4434        let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
4435        let zero = signals(&[
4436            ("X-Codex-Secondary-Used-Percent", "0".into()),
4437            ("X-Codex-Secondary-Window-Minutes", "10080".into()),
4438            ("X-Codex-Secondary-Reset-After-Seconds", "0".into()),
4439        ]);
4440        let mut s = Scheduler::default();
4441        let now = Instant::now();
4442        let mut picks = Vec::new();
4443        for i in 0..4u64 {
4444            // `a` has answered `i` times, each identical answer observed a second later
4445            // and read a second after that.
4446            let a_now = (i > 0).then(|| {
4447                let seen = wall + Duration::from_secs(2 * i);
4448                Windows::observed("codex", &zero, seen, seen + Duration::from_secs(1))
4449            });
4450            let ranks = |x: &Credential| {
4451                if x.id == "a" {
4452                    a_now.unwrap_or_default()
4453                } else {
4454                    b_known
4455                }
4456            };
4457            picks.push(
4458                s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
4459                    .unwrap()
4460                    .id
4461                    .clone(),
4462            );
4463        }
4464        assert_eq!(picks, ["a", "b", "b", "b"]);
4465        // The zero delay still ends a used-up window now.
4466        let used_up = signals(&[
4467            ("X-Codex-Primary-Used-Percent", "100".into()),
4468            ("X-Codex-Primary-Reset-After-Seconds", "0".into()),
4469        ]);
4470        assert!(!Windows::observed("codex", &used_up, wall, wall).exhausted);
4471    }
4472
4473    /// Relative resets read around a boundary differ by rounding; rollovers less than
4474    /// half a window apart are the same one, so they never re-arm the probe twice.
4475    #[test]
4476    fn soonest_reset_rollovers_within_half_a_window_are_one_rollover() {
4477        let wall = SystemTime::now();
4478        let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4479        let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
4480        // `a` reports its weekly reset `after` seconds from an observation at `seen`.
4481        let codex = |seen: SystemTime, after: u64, read: SystemTime| {
4482            Windows::observed(
4483                "codex",
4484                &signals(&[
4485                    ("X-Codex-Secondary-Window-Minutes", "10080".into()),
4486                    ("X-Codex-Secondary-Reset-After-Seconds", after.to_string()),
4487                ]),
4488                seen,
4489                read,
4490            )
4491        };
4492        let mut s = Scheduler::default();
4493        let now = Instant::now();
4494        let mut pick = |a_now: Windows| {
4495            let ranks = |x: &Credential| if x.id == "a" { a_now } else { b_known };
4496            s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
4497                .unwrap()
4498                .id
4499                .clone()
4500        };
4501        let boundary = wall + Duration::from_secs(600);
4502        // Reset reached: probed once.
4503        assert_eq!(pick(codex(wall, 600, boundary + Duration::from_secs(1))), "a");
4504        // Lagging answers keep reporting one second left, each from a later observation.
4505        for i in 1..4u64 {
4506            let seen = boundary + Duration::from_secs(i);
4507            assert_eq!(pick(codex(seen, 1, seen + Duration::from_secs(2))), "b", "answer {i}");
4508        }
4509        // A week later the window really rolls over again: one more probe.
4510        let next = boundary + Duration::from_secs(7 * DAY);
4511        assert_eq!(
4512            pick(codex(next - Duration::from_secs(5), 5, next + Duration::from_secs(1))),
4513            "a"
4514        );
4515        assert_eq!(
4516            pick(codex(next - Duration::from_secs(5), 5, next + Duration::from_secs(1))),
4517            "b"
4518        );
4519    }
4520
4521    /// The oracle's sequence: a probe made for one window's rollover must not suppress a
4522    /// later rollover of another window.
4523    #[test]
4524    fn soonest_reset_probe_markers_are_kept_per_window() {
4525        let wall = SystemTime::now();
4526        let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4527        let b_known = claude(wall, 5 * DAY, "allowed", wall + Duration::from_secs(3600));
4528        let r = wall + Duration::from_secs(600);
4529        let after = |t: SystemTime, secs: u64| t + Duration::from_secs(secs);
4530        // 1. Only `Secondary-Reset-At: R` and no lengths: secondary is taken as weekly.
4531        let partial = signals(&[("X-Codex-Secondary-Reset-At", epoch(r))]);
4532        // 2. The probe's answer is complete: the weekly window is primary, resetting at
4533        //    R + 1 day; secondary is the 5-hour window.
4534        let complete = signals(&[
4535            ("X-Codex-Primary-Used-Percent", "10".into()),
4536            ("X-Codex-Primary-Window-Minutes", "10080".into()),
4537            ("X-Codex-Primary-Reset-At", epoch(after(r, DAY))),
4538            ("X-Codex-Secondary-Used-Percent", "10".into()),
4539            ("X-Codex-Secondary-Window-Minutes", "300".into()),
4540            ("X-Codex-Secondary-Reset-At", epoch(after(r, 5 * 3600))),
4541        ]);
4542        let mut s = Scheduler::default();
4543        let now = Instant::now();
4544        let mut pick = |a_now: Windows| {
4545            let ranks = |x: &Credential| if x.id == "a" { a_now } else { b_known };
4546            s.pick_ranked(&tag(&[&a, &b]), &selection("m"), &soonest(), &ranks, now)
4547                .unwrap()
4548                .id
4549                .clone()
4550        };
4551        // Just after R, `a`'s secondary rolled over: its probe.
4552        let w = Windows::observed("codex", &partial, wall, after(r, 1));
4553        assert_eq!(w.rolled_over.map(|r| r.window), Some(WindowId::CodexSecondary));
4554        assert_eq!(pick(w), "a");
4555        // The answer reports primary resetting in a day, sooner than `b`: `a` keeps it.
4556        assert_eq!(
4557            pick(Windows::observed("codex", &complete, after(r, 1), after(r, 2))),
4558            "a"
4559        );
4560        // 3. Just after R + 1 day, primary rolls over: one probe, not suppressed by the
4561        //    secondary's marker R (R + 1 day is within half of primary's week of R).
4562        let w = Windows::observed("codex", &complete, after(r, 1), after(r, DAY + 1));
4563        assert_eq!(w.rolled_over.map(|r| r.window), Some(WindowId::CodexPrimary));
4564        assert_eq!(pick(w), "a");
4565        assert_eq!(pick(w), "b", "probed for that rollover");
4566    }
4567
4568    /// P2: a management reset clears the probe record (the account is probed again) and
4569    /// leaves session affinity alone.
4570    #[test]
4571    fn soonest_reset_management_reset_re_arms_the_probe_and_keeps_affinity() {
4572        let wall = SystemTime::now();
4573        let (a, b) = (cred("a", serde_json::json!({})), cred("b", serde_json::json!({})));
4574        let b_known = claude(wall, 2 * DAY, "allowed", wall + Duration::from_secs(3600));
4575        let ranks = |x: &Credential| if x.id == "a" { Windows::default() } else { b_known };
4576        let policy = Policy {
4577            session_affinity: true,
4578            ..soonest()
4579        };
4580        let mut s = Scheduler::default();
4581        let now = Instant::now();
4582        let mut thread = selection("m");
4583        thread.session = Some("claude:thread-1".into());
4584        let mut pick = |sel: &Selection, set: &[&Credential]| {
4585            s.pick_ranked(&tag(set), sel, &policy, &ranks, now).unwrap().id.clone()
4586        };
4587        // The thread starts on `a` while it is the only account: that was `a`'s probe.
4588        assert_eq!(pick(&thread, &[&a]), "a");
4589        let fresh = selection("m");
4590        assert_eq!(pick(&fresh, &[&a, &b]), "b");
4591        s.reset_cooldowns("a");
4592        let mut pick = |sel: &Selection, set: &[&Credential]| {
4593            s.pick_ranked(&tag(set), sel, &policy, &ranks, now).unwrap().id.clone()
4594        };
4595        assert_eq!(pick(&fresh, &[&a, &b]), "a", "probed again");
4596        assert_eq!(pick(&fresh, &[&a, &b]), "b");
4597        assert_eq!(pick(&thread, &[&a, &b]), "a", "the pin is kept");
4598    }
4599
4600    /// P2: an oversized finite window length neither panics nor holds a used-up window
4601    /// past the missing-length policy (five hours).
4602    #[test]
4603    fn windows_survive_unrepresentable_window_lengths() {
4604        let now = SystemTime::now();
4605        for (used, exhausted) in [("0", false), ("100", true)] {
4606            let huge = signals(&[
4607                ("X-Codex-Primary-Used-Percent", used.into()),
4608                ("X-Codex-Primary-Window-Minutes", "1e308".into()),
4609            ]);
4610            assert_eq!(
4611                Windows::observed("codex", &huge, now, now).exhausted,
4612                exhausted,
4613                "{used}%"
4614            );
4615            let later = now + Duration::from_secs(5 * 3600 + 1);
4616            assert!(
4617                !Windows::observed("codex", &huge, now, later).exhausted,
4618                "{used}% after five hours"
4619            );
4620        }
4621        // Long but representable lengths still count.
4622        let long = signals(&[
4623            ("X-Codex-Primary-Used-Percent", "100".into()),
4624            ("X-Codex-Primary-Window-Minutes", "1e9".into()),
4625        ]);
4626        assert!(Windows::observed("codex", &long, now, now + Duration::from_secs(DAY)).exhausted);
4627    }
4628
4629    /// P2: `X-Codex-Limit-Reached` ends with the used-up window's own reset; the
4630    /// five-hour fallback applies only when no window reports a reset.
4631    #[test]
4632    fn codex_limit_reached_expires_with_the_window_reset() {
4633        let now = SystemTime::now();
4634        let flagged = signals(&[
4635            ("X-Codex-Limit-Reached", "true".into()),
4636            ("X-Codex-Primary-Used-Percent", "100".into()),
4637            ("X-Codex-Primary-Window-Minutes", "300".into()),
4638            ("X-Codex-Primary-Reset-At", epoch(now + Duration::from_secs(60))),
4639        ]);
4640        assert!(Windows::observed("codex", &flagged, now, now).exhausted);
4641        assert!(!Windows::observed("codex", &flagged, now, now + Duration::from_secs(61)).exhausted);
4642        // No window used up: the flag holds until the soonest reported reset.
4643        let below = signals(&[
4644            ("X-Codex-Limit-Reached", "true".into()),
4645            ("X-Codex-Primary-Used-Percent", "40".into()),
4646            ("X-Codex-Primary-Reset-At", epoch(now + Duration::from_secs(600))),
4647        ]);
4648        assert!(Windows::observed("codex", &below, now, now + Duration::from_secs(599)).exhausted);
4649        assert!(!Windows::observed("codex", &below, now, now + Duration::from_secs(601)).exhausted);
4650        // No reset anywhere: five hours.
4651        let bare = signals(&[("X-Codex-Limit-Reached", "true".into())]);
4652        assert!(Windows::observed("codex", &bare, now, now + Duration::from_secs(5 * 3600 - 1)).exhausted);
4653        assert!(!Windows::observed("codex", &bare, now, now + Duration::from_secs(5 * 3600 + 1)).exhausted);
4654    }
4655
4656    /// P2: Codex usage, reset and length are parsed independently, so a window whose
4657    /// usage is missing still reports its reset.
4658    #[test]
4659    fn codex_windows_parse_usage_reset_and_length_independently() {
4660        let now = SystemTime::now();
4661        let weekly = now + Duration::from_secs(3 * DAY);
4662        let no_usage = signals(&[
4663            ("X-Codex-Secondary-Window-Minutes", "10080".into()),
4664            ("X-Codex-Secondary-Reset-At", epoch(weekly)),
4665        ]);
4666        let w = Windows::observed("codex", &no_usage, now, now);
4667        assert_eq!((w.weekly_reset.map(epoch), w.exhausted), (Some(epoch(weekly)), false));
4668        // Reset only, no usage or length: secondary is weekly.
4669        let reset_only = signals(&[("X-Codex-Secondary-Reset-After-Seconds", "7200".into())]);
4670        let w = Windows::observed("codex", &reset_only, now, now);
4671        assert_eq!(w.weekly_reset.map(epoch), Some(epoch(now + Duration::from_secs(7200))));
4672    }
4673
4674    /// Both Codex windows, the primary the longer one: the primary is weekly.
4675    #[test]
4676    fn codex_primary_longer_than_secondary_is_the_weekly_window() {
4677        let now = SystemTime::now();
4678        let weekly = now + Duration::from_secs(3 * DAY);
4679        let both = signals(&[
4680            ("X-Codex-Primary-Used-Percent", "51".into()),
4681            ("X-Codex-Primary-Window-Minutes", "10080".into()),
4682            ("X-Codex-Primary-Reset-At", epoch(weekly)),
4683            ("X-Codex-Secondary-Used-Percent", "100".into()),
4684            ("X-Codex-Secondary-Window-Minutes", "300".into()),
4685            ("X-Codex-Secondary-Reset-After-Seconds", "3600".into()),
4686        ]);
4687        let w = Windows::observed("codex", &both, now, now);
4688        assert_eq!((w.weekly_reset.map(epoch), w.exhausted), (Some(epoch(weekly)), true));
4689        let w = Windows::observed("codex", &both, now, now + Duration::from_secs(3601));
4690        assert_eq!((w.weekly_reset.map(epoch), w.exhausted), (Some(epoch(weekly)), false));
4691    }
4692}