Skip to main content

firstpass_core/
rollout.rs

1//! Percentage rollout: deciding which slice of matched traffic actually enforces (ADR 0009 D1).
2//!
3//! The whole mechanism is one pure function, and the reason it is a *function of a stable key*
4//! rather than a random draw is the point of the module. Drawing per request would:
5//!
6//!   * flip a multi-turn conversation between enforced and observed mid-thread, so a user sees
7//!     the model change for no reason they can perceive; and, worse,
8//!   * make the served population unstable. The distribution-free bound Firstpass publishes is
9//!     computed over *the population that was served* — if membership is re-drawn every request,
10//!     that population is not a stable sample of anything and the bound describes a cohort that
11//!     never existed.
12//!
13//! So assignment is `hash(salt || key_kind || key_value)`: deterministic for a fixed salt,
14//! identical across restarts and across replicas sharing config, needing no coordination, and
15//! leaking nothing — the same salted-hash discipline [`crate::features::repo_fingerprint`] uses.
16
17use serde::Deserialize;
18
19/// Buckets per rollout space. 10 000 gives two decimal places of percent, which is finer than
20/// any operator needs and keeps the modulo bias negligible against a 32-bit draw.
21pub const BUCKETS: u32 = 10_000;
22
23/// Which stable identity a rollout holds constant.
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
25#[serde(rename_all = "lowercase")]
26pub enum RolloutKey {
27    /// The `x-firstpass-session` header — a whole agent run moves together.
28    ///
29    /// The default, because a conversation is the unit a user experiences, so it is the unit
30    /// that should be held constant. Falls back to [`RolloutKey::Request`] when absent.
31    #[default]
32    Session,
33    /// The request's own identity. For stateless single-shot traffic with no session.
34    Request,
35    /// The tenant — for hosted deployments ramping tenant by tenant.
36    Tenant,
37}
38
39impl RolloutKey {
40    /// Domain-separator tag mixed into the hash, so the same value under two different key kinds
41    /// lands in unrelated buckets (a tenant id that happens to equal a session id must not
42    /// inherit its assignment).
43    #[must_use]
44    pub const fn tag(self) -> &'static str {
45        match self {
46            Self::Session => "session",
47            Self::Request => "request",
48            Self::Tenant => "tenant",
49        }
50    }
51}
52
53/// Rollout settings for a route. Absent means "all matched traffic", i.e. today's behaviour.
54#[derive(Debug, Clone, Copy, PartialEq, Deserialize)]
55#[serde(deny_unknown_fields)]
56pub struct Rollout {
57    /// Percent of matched traffic that actually enforces, in `[0, 100]`. The remainder takes the
58    /// ordinary observe path. Rollout only ever *subtracts* from what `mode` already allows.
59    pub percent: f64,
60    /// Which identity is held constant.
61    #[serde(default)]
62    pub key: RolloutKey,
63}
64
65impl Rollout {
66    /// Whether this rollout is configured coherently.
67    ///
68    /// # Errors
69    /// If `percent` is not finite or falls outside `[0, 100]`.
70    pub fn validate(&self) -> Result<(), String> {
71        if !self.percent.is_finite() || !(0.0..=100.0).contains(&self.percent) {
72            return Err(format!(
73                "route.rollout.percent must be finite and within [0, 100], got {}",
74                self.percent
75            ));
76        }
77        Ok(())
78    }
79}
80
81/// Shadow scoring settings for a route (ADR 0009 D2).
82///
83/// Observe mode records what traffic looked like; it never runs the ladder, so it cannot answer
84/// the one question it exists for — *what would Firstpass have served, and what would it have
85/// cost?* Shadow answers that by scoring a sample of observed requests off the hot path.
86///
87/// It makes real model calls, so there is deliberately no default-on sample rate and a hard daily
88/// ceiling is required. A measurement that quietly runs up a bill is worse than no measurement.
89#[derive(Debug, Clone, Copy, PartialEq, Deserialize)]
90#[serde(deny_unknown_fields)]
91pub struct Shadow {
92    /// Fraction of observed requests scored counterfactually, in `[0, 1]`.
93    pub sample_rate: f64,
94    /// Hard ceiling on shadow spend per UTC day, in USD. Shadow stops when exhausted, and the
95    /// fact that it stopped is recorded — silently degrading a measurement is its own bug.
96    pub max_usd_per_day: f64,
97}
98
99impl Shadow {
100    /// Whether this shadow config is coherent.
101    ///
102    /// # Errors
103    /// If `sample_rate` is outside `[0, 1]` or `max_usd_per_day` is negative or non-finite.
104    pub fn validate(&self) -> Result<(), String> {
105        if !self.sample_rate.is_finite() || !(0.0..=1.0).contains(&self.sample_rate) {
106            return Err(format!(
107                "route.shadow.sample_rate must be finite and within [0, 1], got {}",
108                self.sample_rate
109            ));
110        }
111        if !self.max_usd_per_day.is_finite() || self.max_usd_per_day < 0.0 {
112            return Err(format!(
113                "route.shadow.max_usd_per_day must be finite and >= 0, got {}",
114                self.max_usd_per_day
115            ));
116        }
117        Ok(())
118    }
119
120    /// Whether this request is in the shadow sample.
121    ///
122    /// Uses the same salted bucketing as rollout, under its own domain tag, so shadow sampling is
123    /// deterministic and — critically — *independent* of the rollout arm. If the two shared a
124    /// hash, the shadow sample would be a biased subset of one arm and the projection it produces
125    /// would not describe the traffic an operator is about to enforce on.
126    #[must_use]
127    pub fn sampled(&self, salt: &str, key_value: &str) -> bool {
128        if self.sample_rate <= 0.0 {
129            return false;
130        }
131        if self.sample_rate >= 1.0 {
132            return true;
133        }
134        use sha2::{Digest, Sha256};
135        let mut h = Sha256::new();
136        h.update(salt.as_bytes());
137        h.update([0u8]);
138        h.update(b"shadow"); // distinct tag: shadow sampling must not correlate with the arm
139        h.update([0u8]);
140        h.update(key_value.as_bytes());
141        let d = h.finalize();
142        let bucket = u32::from_be_bytes([d[0], d[1], d[2], d[3]]) % BUCKETS;
143        #[expect(
144            clippy::cast_possible_truncation,
145            clippy::cast_sign_loss,
146            reason = "sample_rate is validated finite in [0,1]"
147        )]
148        let cutoff = (self.sample_rate * f64::from(BUCKETS)) as u32;
149        bucket < cutoff
150    }
151}
152
153/// Whether a challenger runs as a live arm or as a paired shadow (ADR 0014).
154#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
155#[serde(rename_all = "snake_case")]
156pub enum ChallengerMode {
157    /// (Default) A randomized fraction of *sessions* is served by the challenger ladder through
158    /// the normal enforce path. Task outcomes arrive through the feedback API. Only valid on
159    /// routes where the task outcome is a meaningful signal; not suitable for agent sessions
160    /// when no per-session outcome gate is available.
161    #[default]
162    Live,
163    /// The challenger ladder runs on the **same request** in a detached task after the incumbent
164    /// response has been returned, comparing gate verdicts rather than task outcomes. No session
165    /// is ever served by the challenger, so this is safe for single-turn routes. Requires at
166    /// least one gate other than `non-empty` (a pass-everything gate makes gate-pass
167    /// non-inferiority meaningless — finding 1 of ADR 0014) and a daily spend ceiling
168    /// (`max_usd_per_day`). Certifies **gate-pass** non-inferiority, not outcome.
169    Paired,
170}
171
172/// A challenger ladder under evaluation on live traffic (ADR 0014).
173///
174/// A randomized `percent` of *sessions* is served by `ladder` instead of the route's own ladder,
175/// through the ordinary enforce path (gates still run). Task outcomes arrive through the feedback
176/// API and a sequential non-inferiority test ([`crate::select::Experiment`]) decides whether the
177/// challenger is certified, halted, or still undecided. Certification never promotes anything by
178/// itself: adopting the ladder is a human edit of the config.
179#[derive(Debug, Clone, PartialEq, Deserialize)]
180#[serde(deny_unknown_fields)]
181pub struct Challenger {
182    /// The ladder under evaluation, cheapest first, as `provider/model` strings. Same shape as
183    /// `Route.ladder`; every rung must resolve to a price and a provider.
184    pub ladder: Vec<String>,
185    /// **Live mode**: percent of *sessions* served by the challenger, in `(0, 100)` exclusive.
186    /// The rest are the incumbent arm; at 100 there would be no control and the test could never
187    /// form a pair. **Paired mode**: unused — set to `0` or omit (defaults to `0`).
188    #[serde(default)]
189    pub percent: f64,
190    /// Non-inferiority margin: how much worse the challenger may be and still be certified.
191    /// See [`crate::select::DEFAULT_MARGIN`].
192    #[serde(default = "default_challenger_margin")]
193    pub margin: f64,
194    /// Error rate of each conclusion (certify / halt). See [`crate::select::DEFAULT_ALPHA`].
195    #[serde(default = "default_challenger_alpha")]
196    pub alpha: f64,
197    /// How this challenger collects evidence. `"live"` (default) assigns sessions to an arm by
198    /// salted hash. `"paired"` runs the challenger off-path on every (sampled) incumbent request.
199    #[serde(default)]
200    pub mode: ChallengerMode,
201    /// **Paired mode only**: fraction of enforce requests sampled for pairing, in `[0, 1]`.
202    /// `1.0` (default) pairs every enforce request. Values below `1.0` reduce challenger spend
203    /// at the cost of slower convergence. Ignored in `"live"` mode.
204    #[serde(default = "default_sample_rate")]
205    pub sample_rate: f64,
206    /// **Paired mode only**: hard ceiling on challenger spend per UTC day, in USD. Required when
207    /// `mode = "paired"` (the challenger makes real model calls off the hot path; a silent spend
208    /// would be worse than a visible one). Ignored in `"live"` mode.
209    #[serde(default)]
210    pub max_usd_per_day: Option<f64>,
211    /// Optional Redis URL for sharing experiment state across replicas.
212    ///
213    /// When set, all replicas append outcomes / pairs to a shared Redis list and rebuild their
214    /// local `Experiment` incrementally from it, so the halt/certify decision is global rather
215    /// than per-process. Requires the `redis-cache` build feature in `firstpass-proxy`; a URL
216    /// without that feature is rejected at startup rather than silently ignored.
217    #[serde(default)]
218    pub redis_url: Option<String>,
219}
220
221fn default_challenger_margin() -> f64 {
222    crate::select::DEFAULT_MARGIN
223}
224
225fn default_challenger_alpha() -> f64 {
226    crate::select::DEFAULT_ALPHA
227}
228
229fn default_sample_rate() -> f64 {
230    1.0
231}
232
233impl Challenger {
234    /// Whether this challenger is in paired mode.
235    #[must_use]
236    pub fn is_paired(&self) -> bool {
237        self.mode == ChallengerMode::Paired
238    }
239
240    /// Whether this challenger config is coherent on its own. Cross-checks against the route and
241    /// the price table happen in `Config::parse`.
242    ///
243    /// # Errors
244    /// - Live mode: if `percent` is not in `(0, 100)`.
245    /// - Paired mode: if `sample_rate` is not in `[0, 1]`, or `max_usd_per_day` is absent / invalid.
246    /// - Either mode: if the ladder is empty, or `margin`/`alpha` are refused by
247    ///   [`crate::select::Experiment::new`].
248    pub fn validate(&self) -> Result<(), String> {
249        if self.ladder.is_empty() {
250            return Err("route.challenger.ladder must not be empty".to_owned());
251        }
252        match self.mode {
253            ChallengerMode::Live => {
254                if !self.percent.is_finite() || self.percent <= 0.0 || self.percent >= 100.0 {
255                    return Err(format!(
256                        "route.challenger.percent must be finite and within (0, 100) exclusive — \
257                         at 100 there is no incumbent control and the test can never pair — got {}",
258                        self.percent
259                    ));
260                }
261            }
262            ChallengerMode::Paired => {
263                if !self.sample_rate.is_finite() || !(0.0..=1.0).contains(&self.sample_rate) {
264                    return Err(format!(
265                        "route.challenger.sample_rate must be finite and within [0, 1], got {}",
266                        self.sample_rate
267                    ));
268                }
269                let max = self.max_usd_per_day.ok_or_else(|| {
270                    "route.challenger.max_usd_per_day is required in paired mode — the challenger \
271                     makes real model calls off the hot path; a ceiling prevents silent spend"
272                        .to_owned()
273                })?;
274                if !max.is_finite() || max < 0.0 {
275                    return Err(format!(
276                        "route.challenger.max_usd_per_day must be finite and >= 0, got {max}"
277                    ));
278                }
279            }
280        }
281        crate::select::Experiment::new(self.margin, self.alpha)
282            .map(|_| ())
283            .map_err(|e| format!("route.challenger: {e}"))
284    }
285
286    /// Whether a given request falls in the paired sample (paired mode only).
287    ///
288    /// Uses the same salted-bucketing discipline as shadow sampling, under a `"paired"` domain tag
289    /// so the sample is independent of rollout and shadow membership. `request_id` should be the
290    /// trace id of the served request (a stable, per-request identity that does not repeat).
291    #[must_use]
292    pub fn paired_sampled(&self, salt: &str, request_id: &str) -> bool {
293        if self.sample_rate <= 0.0 {
294            return false;
295        }
296        if self.sample_rate >= 1.0 {
297            return true;
298        }
299        use sha2::{Digest, Sha256};
300        let mut h = Sha256::new();
301        h.update(salt.as_bytes());
302        h.update([0u8]);
303        h.update(b"paired"); // distinct tag: independent of rollout and shadow sampling
304        h.update([0u8]);
305        h.update(request_id.as_bytes());
306        let d = h.finalize();
307        let bucket = u32::from_be_bytes([d[0], d[1], d[2], d[3]]) % BUCKETS;
308        #[expect(
309            clippy::cast_possible_truncation,
310            clippy::cast_sign_loss,
311            reason = "sample_rate is validated finite in [0,1]"
312        )]
313        let cutoff = (self.sample_rate * f64::from(BUCKETS)) as u32;
314        bucket < cutoff
315    }
316
317    /// Identity of this experiment's configuration: the first 16 hex of SHA-256 over the canonical
318    /// JSON `[incumbent_ladder, challenger_ladder, margin, alpha]`. Stamped on every receipt so
319    /// that evidence gathered under a different configuration is never pooled with the current
320    /// one — editing either ladder, the margin or alpha starts a fresh experiment.
321    #[must_use]
322    pub fn fingerprint(&self, incumbent_ladder: &[String]) -> String {
323        use sha2::{Digest, Sha256};
324        let canonical = serde_json::json!([incumbent_ladder, self.ladder, self.margin, self.alpha]);
325        let digest = Sha256::digest(canonical.to_string().as_bytes());
326        hex::encode(&digest[..8])
327    }
328
329    /// The bucket `session_key` falls in, within `0..`[`BUCKETS`], under the `challenger` domain
330    /// tag.
331    #[must_use]
332    pub fn bucket(salt: &str, session_key: &str) -> u32 {
333        use sha2::{Digest, Sha256};
334        let mut h = Sha256::new();
335        h.update(salt.as_bytes());
336        h.update([0u8]);
337        // Distinct tag: arm assignment must be independent of rollout and shadow sampling, or
338        // the challenger arm would be a biased subset of whatever those select.
339        h.update(b"challenger");
340        h.update([0u8]);
341        h.update(session_key.as_bytes());
342        let d = h.finalize();
343        u32::from_be_bytes([d[0], d[1], d[2], d[3]]) % BUCKETS
344    }
345
346    /// The arm for a bucket under this config's `percent`.
347    #[must_use]
348    pub fn arm_of(&self, bucket: u32) -> crate::select::Arm {
349        if in_rollout(bucket, self.percent) {
350            crate::select::Arm::Challenger
351        } else {
352            crate::select::Arm::Incumbent
353        }
354    }
355
356    /// Which arm a session is in. Deterministic for a fixed `salt`, so a whole session stays in
357    /// one arm — outcomes are one per session and pairing assumes the arm never changes mid-run.
358    #[must_use]
359    pub fn arm(&self, salt: &str, session_key: &str) -> crate::select::Arm {
360        self.arm_of(Self::bucket(salt, session_key))
361    }
362}
363
364/// The bucket `key_value` falls in, within `0..`[`BUCKETS`].
365///
366/// Deterministic for a fixed `salt`. The salt is the per-deployment secret already used for
367/// feature fingerprints, so bucket assignment cannot be reconstructed — or gamed — by a caller
368/// who can choose their own session id.
369#[must_use]
370pub fn bucket_of(salt: &str, key: RolloutKey, key_value: &str) -> u32 {
371    use sha2::{Digest, Sha256};
372    let mut h = Sha256::new();
373    h.update(salt.as_bytes());
374    h.update([0u8]); // separator: salt||tag must not collide with a different split
375    h.update(key.tag().as_bytes());
376    h.update([0u8]);
377    h.update(key_value.as_bytes());
378    let d = h.finalize();
379    u32::from_be_bytes([d[0], d[1], d[2], d[3]]) % BUCKETS
380}
381
382/// Whether a bucket is inside a `percent` rollout.
383///
384/// Strictly `<` so that `percent = 0` enforces nothing and `percent = 100` enforces everything —
385/// both endpoints exact, which matters because "ramp to 0" is how an operator backs out.
386#[must_use]
387pub fn in_rollout(bucket: u32, percent: f64) -> bool {
388    if percent <= 0.0 {
389        return false;
390    }
391    if percent >= 100.0 {
392        return true;
393    }
394    #[expect(
395        clippy::cast_possible_truncation,
396        clippy::cast_sign_loss,
397        reason = "percent is validated finite in [0,100]; the product is within u32"
398    )]
399    let cutoff = (percent / 100.0 * f64::from(BUCKETS)) as u32;
400    bucket < cutoff
401}
402
403/// A stable identity for one request, for use as bucketing input when there is no session to key
404/// on. Hashes the request bytes, so the same request always lands in the same arm.
405///
406/// The digest is truncated and the input is never retained — this exists so rollout can bucket
407/// stateless traffic without Firstpass holding on to prompt content.
408#[must_use]
409pub fn request_identity(body: &[u8]) -> String {
410    use sha2::{Digest, Sha256};
411    let mut h = Sha256::new();
412    h.update(body);
413    hex::encode(&h.finalize()[..8])
414}
415
416/// The decision recorded on the trace, so an auditor can re-derive which arm a request was in.
417///
418/// Without this a mixed-population receipt log cannot be split back into the enforced and
419/// observed arms, and every savings or failure figure computed over it silently blends two
420/// different regimes.
421#[derive(Debug, Clone, Copy, PartialEq)]
422pub struct RolloutDecision {
423    /// Configured percent at decision time.
424    pub percent: f64,
425    /// Which identity was held constant.
426    pub key: RolloutKey,
427    /// The bucket this request landed in.
428    pub bucket: u32,
429    /// Whether it enforced.
430    pub enforced: bool,
431}
432
433/// Decide the arm for one request.
434#[must_use]
435pub fn decide(salt: &str, rollout: &Rollout, key_value: &str) -> RolloutDecision {
436    let bucket = bucket_of(salt, rollout.key, key_value);
437    RolloutDecision {
438        percent: rollout.percent,
439        key: rollout.key,
440        bucket,
441        enforced: in_rollout(bucket, rollout.percent),
442    }
443}
444
445#[cfg(test)]
446mod tests {
447    use super::*;
448
449    /// The property the whole design rests on: the same key always lands in the same arm, so a
450    /// conversation cannot flip mid-thread and the served population stays a stable sample.
451    #[test]
452    fn assignment_is_stable_for_a_key() {
453        let r = Rollout {
454            percent: 5.0,
455            key: RolloutKey::Session,
456        };
457        let first = decide("salt-a", &r, "session-123");
458        for _ in 0..1000 {
459            assert_eq!(decide("salt-a", &r, "session-123"), first);
460        }
461    }
462
463    /// Different salts must reshuffle, or two deployments would enforce on exactly the same
464    /// sessions and a caller who learned one deployment's arm would know the other's.
465    #[test]
466    fn salt_reshuffles_assignment() {
467        let differing = (0..200)
468            .filter(|i| {
469                let k = format!("session-{i}");
470                bucket_of("salt-a", RolloutKey::Session, &k)
471                    != bucket_of("salt-b", RolloutKey::Session, &k)
472            })
473            .count();
474        assert!(
475            differing > 190,
476            "salt barely changed assignment: {differing}/200"
477        );
478    }
479
480    /// The same value under two key kinds must not inherit one assignment.
481    #[test]
482    fn key_kind_is_domain_separated() {
483        let v = "same-identifier";
484        let s = bucket_of("salt", RolloutKey::Session, v);
485        let t = bucket_of("salt", RolloutKey::Tenant, v);
486        let q = bucket_of("salt", RolloutKey::Request, v);
487        assert!(s != t || t != q, "key kinds collapsed to one bucket");
488    }
489
490    /// A skewed hash would silently ramp to the wrong percentage — the operator would ask for 10%
491    /// and get 3%, and only notice through a savings number that never moved.
492    #[test]
493    fn buckets_are_roughly_uniform() {
494        for pct in [1.0, 5.0, 10.0, 50.0] {
495            let r = Rollout {
496                percent: pct,
497                key: RolloutKey::Session,
498            };
499            let n = 20_000;
500            let hits = (0..n)
501                .filter(|i| decide("salt", &r, &format!("s{i}")).enforced)
502                .count();
503            let observed = hits as f64 / f64::from(n) * 100.0;
504            assert!(
505                (observed - pct).abs() < 1.0,
506                "{pct}% rollout produced {observed:.2}% of {n} keys"
507            );
508        }
509    }
510
511    /// Both endpoints must be exact: 0 is how an operator backs out, 100 is full enforcement.
512    #[test]
513    fn endpoints_are_exact() {
514        for i in 0..500 {
515            let k = format!("s{i}");
516            assert!(
517                !decide(
518                    "salt",
519                    &Rollout {
520                        percent: 0.0,
521                        key: RolloutKey::Session
522                    },
523                    &k
524                )
525                .enforced
526            );
527            assert!(
528                decide(
529                    "salt",
530                    &Rollout {
531                        percent: 100.0,
532                        key: RolloutKey::Session
533                    },
534                    &k
535                )
536                .enforced
537            );
538        }
539    }
540
541    /// Ramping up must never eject a key that was already enforcing — an operator going 5% → 25%
542    /// expects to add traffic, not to reshuffle who is in the experiment.
543    #[test]
544    fn ramping_up_is_monotonic() {
545        let keys: Vec<String> = (0..2000).map(|i| format!("s{i}")).collect();
546        let arm = |pct: f64| -> Vec<bool> {
547            keys.iter()
548                .map(|k| {
549                    decide(
550                        "salt",
551                        &Rollout {
552                            percent: pct,
553                            key: RolloutKey::Session,
554                        },
555                        k,
556                    )
557                    .enforced
558                })
559                .collect()
560        };
561        let (small, large) = (arm(5.0), arm(25.0));
562        for (i, (s, l)) in small.iter().zip(&large).enumerate() {
563            assert!(!*s || *l, "key {i} enforced at 5% but not at 25%");
564        }
565    }
566
567    #[test]
568    fn validate_rejects_impossible_percentages() {
569        for bad in [-0.1, 100.1, f64::NAN, f64::INFINITY] {
570            assert!(
571                Rollout {
572                    percent: bad,
573                    key: RolloutKey::Session
574                }
575                .validate()
576                .is_err()
577            );
578        }
579        for good in [0.0, 0.01, 50.0, 100.0] {
580            assert!(
581                Rollout {
582                    percent: good,
583                    key: RolloutKey::Session
584                }
585                .validate()
586                .is_ok()
587            );
588        }
589    }
590
591    /// Shadow sampling must be INDEPENDENT of the rollout arm. If both hashed the same input
592    /// under the same tag, every shadow-sampled request would sit in the same arm, and the
593    /// projection would describe a biased subset of traffic rather than the traffic an operator
594    /// is about to enforce on.
595    #[test]
596    fn shadow_sampling_is_independent_of_the_rollout_arm() {
597        let roll = Rollout {
598            percent: 50.0,
599            key: RolloutKey::Session,
600        };
601        let shadow = Shadow {
602            sample_rate: 0.5,
603            max_usd_per_day: 5.0,
604        };
605        let (mut both, mut only_shadow, mut only_arm, mut neither) = (0, 0, 0, 0);
606        for i in 0..4000 {
607            let k = format!("s{i}");
608            match (
609                decide("salt", &roll, &k).enforced,
610                shadow.sampled("salt", &k),
611            ) {
612                (true, true) => both += 1,
613                (false, true) => only_shadow += 1,
614                (true, false) => only_arm += 1,
615                (false, false) => neither += 1,
616            }
617        }
618        // Independent 50/50 splits put ~25% in each cell; a shared hash would empty two of them.
619        for (name, n) in [
620            ("both", both),
621            ("shadow-only", only_shadow),
622            ("arm-only", only_arm),
623            ("neither", neither),
624        ] {
625            let pct = f64::from(n) / 4000.0 * 100.0;
626            assert!(
627                (pct - 25.0).abs() < 3.0,
628                "{name} cell held {pct:.1}% — shadow sampling correlates with the rollout arm"
629            );
630        }
631    }
632
633    #[test]
634    fn shadow_sample_rate_endpoints_are_exact() {
635        for i in 0..300 {
636            let k = format!("s{i}");
637            assert!(
638                !Shadow {
639                    sample_rate: 0.0,
640                    max_usd_per_day: 1.0
641                }
642                .sampled("salt", &k)
643            );
644            assert!(
645                Shadow {
646                    sample_rate: 1.0,
647                    max_usd_per_day: 1.0
648                }
649                .sampled("salt", &k)
650            );
651        }
652    }
653
654    #[test]
655    fn shadow_validate_rejects_impossible_settings() {
656        for bad in [-0.1, 1.1, f64::NAN] {
657            assert!(
658                Shadow {
659                    sample_rate: bad,
660                    max_usd_per_day: 1.0
661                }
662                .validate()
663                .is_err()
664            );
665        }
666        for bad in [-1.0, f64::INFINITY] {
667            assert!(
668                Shadow {
669                    sample_rate: 0.1,
670                    max_usd_per_day: bad
671                }
672                .validate()
673                .is_err()
674            );
675        }
676        assert!(
677            Shadow {
678                sample_rate: 0.1,
679                max_usd_per_day: 5.0
680            }
681            .validate()
682            .is_ok()
683        );
684    }
685
686    fn challenger(percent: f64) -> Challenger {
687        Challenger {
688            ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
689            percent,
690            margin: crate::select::DEFAULT_MARGIN,
691            alpha: crate::select::DEFAULT_ALPHA,
692            mode: ChallengerMode::Live,
693            sample_rate: 1.0,
694            max_usd_per_day: None,
695            redis_url: None,
696        }
697    }
698
699    fn paired_challenger() -> Challenger {
700        Challenger {
701            ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
702            percent: 0.0,
703            margin: crate::select::DEFAULT_MARGIN,
704            alpha: crate::select::DEFAULT_ALPHA,
705            mode: ChallengerMode::Paired,
706            sample_rate: 1.0,
707            max_usd_per_day: Some(5.0),
708            redis_url: None,
709        }
710    }
711
712    #[test]
713    fn challenger_arm_is_deterministic_and_matches_percent() {
714        use crate::select::Arm;
715        let c = challenger(20.0);
716        for i in 0..200 {
717            let k = format!("s{i}");
718            assert_eq!(c.arm("salt", &k), c.arm("salt", &k));
719        }
720        let n = 10_000;
721        let hits = (0..n)
722            .filter(|i| c.arm("salt", &format!("s{i}")) == Arm::Challenger)
723            .count();
724        let observed = hits as f64 / f64::from(n) * 100.0;
725        assert!((observed - 20.0).abs() < 1.5, "got {observed:.2}%");
726        // The pure function still handles the endpoint (validation is what forbids it).
727        assert!(
728            (0..300).all(|i| challenger(100.0).arm("salt", &format!("s{i}")) == Arm::Challenger)
729        );
730    }
731
732    /// Arm assignment must be independent of shadow sampling at the same salt.
733    #[test]
734    fn challenger_arm_is_uncorrelated_with_shadow_sampling() {
735        use crate::select::Arm;
736        let c = challenger(50.0);
737        let shadow = Shadow {
738            sample_rate: 0.5,
739            max_usd_per_day: 1.0,
740        };
741        let mut cells = [0u32; 4];
742        for i in 0..4000 {
743            let k = format!("s{i}");
744            let a = usize::from(c.arm("salt", &k) == Arm::Challenger);
745            let b = usize::from(shadow.sampled("salt", &k));
746            cells[a * 2 + b] += 1;
747        }
748        for (i, n) in cells.iter().enumerate() {
749            let pct = f64::from(*n) / 4000.0 * 100.0;
750            assert!((pct - 25.0).abs() < 3.0, "cell {i} held {pct:.1}%");
751        }
752    }
753
754    #[test]
755    fn challenger_validate_rejects_incoherent_settings() {
756        for bad in [0.0, -1.0, 100.0, 100.1, f64::NAN, f64::INFINITY] {
757            assert!(challenger(bad).validate().is_err(), "percent {bad}");
758        }
759        for good in [0.01, 10.0, 99.99] {
760            assert!(challenger(good).validate().is_ok(), "percent {good}");
761        }
762        let mut c = challenger(10.0);
763        c.ladder.clear();
764        assert!(c.validate().is_err());
765        let mut c = challenger(10.0);
766        c.margin = 0.0;
767        assert!(c.validate().is_err());
768        let mut c = challenger(10.0);
769        c.alpha = 0.5;
770        assert!(c.validate().is_err());
771    }
772
773    #[test]
774    fn paired_challenger_validate_rejects_incoherent_settings() {
775        assert!(paired_challenger().validate().is_ok(), "base paired ok");
776        // Missing max_usd_per_day in paired mode.
777        let mut c = paired_challenger();
778        c.max_usd_per_day = None;
779        assert!(c.validate().is_err(), "missing max_usd_per_day");
780        // Negative max_usd_per_day.
781        let mut c = paired_challenger();
782        c.max_usd_per_day = Some(-1.0);
783        assert!(c.validate().is_err(), "negative max_usd_per_day");
784        // sample_rate out of range.
785        let mut c = paired_challenger();
786        c.sample_rate = 1.5;
787        assert!(c.validate().is_err(), "sample_rate > 1");
788        let mut c = paired_challenger();
789        c.sample_rate = -0.1;
790        assert!(c.validate().is_err(), "sample_rate < 0");
791        // Empty ladder.
792        let mut c = paired_challenger();
793        c.ladder.clear();
794        assert!(c.validate().is_err(), "empty ladder");
795    }
796
797    #[test]
798    fn paired_sampled_is_deterministic_and_approximately_rate() {
799        let c = paired_challenger();
800        // sample_rate = 1.0: all sampled.
801        for i in 0..100 {
802            assert!(c.paired_sampled("salt", &format!("r{i}")));
803        }
804        // sample_rate = 0.0: none sampled.
805        let mut c0 = paired_challenger();
806        c0.sample_rate = 0.0;
807        for i in 0..100 {
808            assert!(!c0.paired_sampled("salt", &format!("r{i}")));
809        }
810        // sample_rate = 0.5: approximately 50%.
811        let mut c5 = paired_challenger();
812        c5.sample_rate = 0.5;
813        let n = 10_000u32;
814        let hits = (0..n)
815            .filter(|i| c5.paired_sampled("salt", &format!("r{i}")))
816            .count();
817        let observed = hits as f64 / f64::from(n) * 100.0;
818        assert!((observed - 50.0).abs() < 2.0, "got {observed:.2}%");
819        // Deterministic: same result on repeated calls.
820        for i in 0..100 {
821            let key = format!("req-{i}");
822            assert_eq!(
823                c5.paired_sampled("salt", &key),
824                c5.paired_sampled("salt", &key)
825            );
826        }
827    }
828
829    #[test]
830    fn fingerprint_tracks_ladders_margin_and_alpha() {
831        let inc = vec!["anthropic/claude-sonnet-5".to_owned()];
832        let base = challenger(10.0);
833        let fp = base.fingerprint(&inc);
834        assert_eq!(fp.len(), 16);
835        assert_eq!(fp, challenger(10.0).fingerprint(&inc), "deterministic");
836        // percent is not part of the identity: ramping must not reset the evidence.
837        assert_eq!(fp, challenger(25.0).fingerprint(&inc));
838        let mut c = challenger(10.0);
839        c.ladder = vec!["anthropic/claude-opus-4-8".to_owned()];
840        assert_ne!(fp, c.fingerprint(&inc));
841        let mut c = challenger(10.0);
842        c.margin = 0.04;
843        assert_ne!(fp, c.fingerprint(&inc));
844        let mut c = challenger(10.0);
845        c.alpha = 0.01;
846        assert_ne!(fp, c.fingerprint(&inc));
847        assert_ne!(
848            fp,
849            base.fingerprint(&["anthropic/claude-opus-4-8".to_owned()])
850        );
851    }
852}