firstpass_core/rollout.rs
1//! Percentage rollout: deciding which slice of matched traffic actually enforces (ADR 0009 D1).
2//!
3//! The whole mechanism is one pure function, and the reason it is a *function of a stable key*
4//! rather than a random draw is the point of the module. Drawing per request would:
5//!
6//! * flip a multi-turn conversation between enforced and observed mid-thread, so a user sees
7//! the model change for no reason they can perceive; and, worse,
8//! * make the served population unstable. The distribution-free bound Firstpass publishes is
9//! computed over *the population that was served* — if membership is re-drawn every request,
10//! that population is not a stable sample of anything and the bound describes a cohort that
11//! never existed.
12//!
13//! So assignment is `hash(salt || key_kind || key_value)`: deterministic for a fixed salt,
14//! identical across restarts and across replicas sharing config, needing no coordination, and
15//! leaking nothing — the same salted-hash discipline [`crate::features::repo_fingerprint`] uses.
16
17use serde::Deserialize;
18
19/// Buckets per rollout space. 10 000 gives two decimal places of percent, which is finer than
20/// any operator needs and keeps the modulo bias negligible against a 32-bit draw.
21pub const BUCKETS: u32 = 10_000;
22
23/// Which stable identity a rollout holds constant.
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
25#[serde(rename_all = "lowercase")]
26pub enum RolloutKey {
27 /// The `x-firstpass-session` header — a whole agent run moves together.
28 ///
29 /// The default, because a conversation is the unit a user experiences, so it is the unit
30 /// that should be held constant. Falls back to [`RolloutKey::Request`] when absent.
31 #[default]
32 Session,
33 /// The request's own identity. For stateless single-shot traffic with no session.
34 Request,
35 /// The tenant — for hosted deployments ramping tenant by tenant.
36 Tenant,
37}
38
39impl RolloutKey {
40 /// Domain-separator tag mixed into the hash, so the same value under two different key kinds
41 /// lands in unrelated buckets (a tenant id that happens to equal a session id must not
42 /// inherit its assignment).
43 #[must_use]
44 pub const fn tag(self) -> &'static str {
45 match self {
46 Self::Session => "session",
47 Self::Request => "request",
48 Self::Tenant => "tenant",
49 }
50 }
51}
52
53/// Rollout settings for a route. Absent means "all matched traffic", i.e. today's behaviour.
54#[derive(Debug, Clone, Copy, PartialEq, Deserialize)]
55#[serde(deny_unknown_fields)]
56pub struct Rollout {
57 /// Percent of matched traffic that actually enforces, in `[0, 100]`. The remainder takes the
58 /// ordinary observe path. Rollout only ever *subtracts* from what `mode` already allows.
59 pub percent: f64,
60 /// Which identity is held constant.
61 #[serde(default)]
62 pub key: RolloutKey,
63}
64
65impl Rollout {
66 /// Whether this rollout is configured coherently.
67 ///
68 /// # Errors
69 /// If `percent` is not finite or falls outside `[0, 100]`.
70 pub fn validate(&self) -> Result<(), String> {
71 if !self.percent.is_finite() || !(0.0..=100.0).contains(&self.percent) {
72 return Err(format!(
73 "route.rollout.percent must be finite and within [0, 100], got {}",
74 self.percent
75 ));
76 }
77 Ok(())
78 }
79}
80
81/// Shadow scoring settings for a route (ADR 0009 D2).
82///
83/// Observe mode records what traffic looked like; it never runs the ladder, so it cannot answer
84/// the one question it exists for — *what would Firstpass have served, and what would it have
85/// cost?* Shadow answers that by scoring a sample of observed requests off the hot path.
86///
87/// It makes real model calls, so there is deliberately no default-on sample rate and a hard daily
88/// ceiling is required. A measurement that quietly runs up a bill is worse than no measurement.
89#[derive(Debug, Clone, Copy, PartialEq, Deserialize)]
90#[serde(deny_unknown_fields)]
91pub struct Shadow {
92 /// Fraction of observed requests scored counterfactually, in `[0, 1]`.
93 pub sample_rate: f64,
94 /// Hard ceiling on shadow spend per UTC day, in USD. Shadow stops when exhausted, and the
95 /// fact that it stopped is recorded — silently degrading a measurement is its own bug.
96 pub max_usd_per_day: f64,
97}
98
99impl Shadow {
100 /// Whether this shadow config is coherent.
101 ///
102 /// # Errors
103 /// If `sample_rate` is outside `[0, 1]` or `max_usd_per_day` is negative or non-finite.
104 pub fn validate(&self) -> Result<(), String> {
105 if !self.sample_rate.is_finite() || !(0.0..=1.0).contains(&self.sample_rate) {
106 return Err(format!(
107 "route.shadow.sample_rate must be finite and within [0, 1], got {}",
108 self.sample_rate
109 ));
110 }
111 if !self.max_usd_per_day.is_finite() || self.max_usd_per_day < 0.0 {
112 return Err(format!(
113 "route.shadow.max_usd_per_day must be finite and >= 0, got {}",
114 self.max_usd_per_day
115 ));
116 }
117 Ok(())
118 }
119
120 /// Whether this request is in the shadow sample.
121 ///
122 /// Uses the same salted bucketing as rollout, under its own domain tag, so shadow sampling is
123 /// deterministic and — critically — *independent* of the rollout arm. If the two shared a
124 /// hash, the shadow sample would be a biased subset of one arm and the projection it produces
125 /// would not describe the traffic an operator is about to enforce on.
126 #[must_use]
127 pub fn sampled(&self, salt: &str, key_value: &str) -> bool {
128 if self.sample_rate <= 0.0 {
129 return false;
130 }
131 if self.sample_rate >= 1.0 {
132 return true;
133 }
134 use sha2::{Digest, Sha256};
135 let mut h = Sha256::new();
136 h.update(salt.as_bytes());
137 h.update([0u8]);
138 h.update(b"shadow"); // distinct tag: shadow sampling must not correlate with the arm
139 h.update([0u8]);
140 h.update(key_value.as_bytes());
141 let d = h.finalize();
142 let bucket = u32::from_be_bytes([d[0], d[1], d[2], d[3]]) % BUCKETS;
143 #[expect(
144 clippy::cast_possible_truncation,
145 clippy::cast_sign_loss,
146 reason = "sample_rate is validated finite in [0,1]"
147 )]
148 let cutoff = (self.sample_rate * f64::from(BUCKETS)) as u32;
149 bucket < cutoff
150 }
151}
152
153/// Whether a challenger runs as a live arm or as a paired shadow (ADR 0014).
154#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
155#[serde(rename_all = "snake_case")]
156pub enum ChallengerMode {
157 /// (Default) A randomized fraction of *sessions* is served by the challenger ladder through
158 /// the normal enforce path. Task outcomes arrive through the feedback API. Only valid on
159 /// routes where the task outcome is a meaningful signal; not suitable for agent sessions
160 /// when no per-session outcome gate is available.
161 #[default]
162 Live,
163 /// The challenger ladder runs on the **same request** in a detached task after the incumbent
164 /// response has been returned, comparing gate verdicts rather than task outcomes. No session
165 /// is ever served by the challenger, so this is safe for single-turn routes. Requires at
166 /// least one gate other than `non-empty` (a pass-everything gate makes gate-pass
167 /// non-inferiority meaningless — finding 1 of ADR 0014) and a daily spend ceiling
168 /// (`max_usd_per_day`). Certifies **gate-pass** non-inferiority, not outcome.
169 Paired,
170}
171
172/// A challenger ladder under evaluation on live traffic (ADR 0014).
173///
174/// A randomized `percent` of *sessions* is served by `ladder` instead of the route's own ladder,
175/// through the ordinary enforce path (gates still run). Task outcomes arrive through the feedback
176/// API and a sequential non-inferiority test ([`crate::select::Experiment`]) decides whether the
177/// challenger is certified, halted, or still undecided. Certification never promotes anything by
178/// itself: adopting the ladder is a human edit of the config.
179#[derive(Debug, Clone, PartialEq, Deserialize)]
180#[serde(deny_unknown_fields)]
181pub struct Challenger {
182 /// The ladder under evaluation, cheapest first, as `provider/model` strings. Same shape as
183 /// `Route.ladder`; every rung must resolve to a price and a provider.
184 pub ladder: Vec<String>,
185 /// **Live mode**: percent of *sessions* served by the challenger, in `(0, 100)` exclusive.
186 /// The rest are the incumbent arm; at 100 there would be no control and the test could never
187 /// form a pair. **Paired mode**: unused — set to `0` or omit (defaults to `0`).
188 #[serde(default)]
189 pub percent: f64,
190 /// Non-inferiority margin: how much worse the challenger may be and still be certified.
191 /// See [`crate::select::DEFAULT_MARGIN`].
192 #[serde(default = "default_challenger_margin")]
193 pub margin: f64,
194 /// Error rate of each conclusion (certify / halt). See [`crate::select::DEFAULT_ALPHA`].
195 #[serde(default = "default_challenger_alpha")]
196 pub alpha: f64,
197 /// How this challenger collects evidence. `"live"` (default) assigns sessions to an arm by
198 /// salted hash. `"paired"` runs the challenger off-path on every (sampled) incumbent request.
199 #[serde(default)]
200 pub mode: ChallengerMode,
201 /// **Paired mode only**: fraction of enforce requests sampled for pairing, in `[0, 1]`.
202 /// `1.0` (default) pairs every enforce request. Values below `1.0` reduce challenger spend
203 /// at the cost of slower convergence. Ignored in `"live"` mode.
204 #[serde(default = "default_sample_rate")]
205 pub sample_rate: f64,
206 /// **Paired mode only**: hard ceiling on challenger spend per UTC day, in USD. Required when
207 /// `mode = "paired"` (the challenger makes real model calls off the hot path; a silent spend
208 /// would be worse than a visible one). Ignored in `"live"` mode.
209 #[serde(default)]
210 pub max_usd_per_day: Option<f64>,
211 /// Optional Redis URL for sharing experiment state across replicas.
212 ///
213 /// When set, all replicas append outcomes / pairs to a shared Redis list and rebuild their
214 /// local `Experiment` incrementally from it, so the halt/certify decision is global rather
215 /// than per-process. Requires the `redis-cache` build feature in `firstpass-proxy`; a URL
216 /// without that feature is rejected at startup rather than silently ignored.
217 #[serde(default)]
218 pub redis_url: Option<String>,
219}
220
221fn default_challenger_margin() -> f64 {
222 crate::select::DEFAULT_MARGIN
223}
224
225fn default_challenger_alpha() -> f64 {
226 crate::select::DEFAULT_ALPHA
227}
228
229fn default_sample_rate() -> f64 {
230 1.0
231}
232
233impl Challenger {
234 /// Whether this challenger is in paired mode.
235 #[must_use]
236 pub fn is_paired(&self) -> bool {
237 self.mode == ChallengerMode::Paired
238 }
239
240 /// Whether this challenger config is coherent on its own. Cross-checks against the route and
241 /// the price table happen in `Config::parse`.
242 ///
243 /// # Errors
244 /// - Live mode: if `percent` is not in `(0, 100)`.
245 /// - Paired mode: if `sample_rate` is not in `[0, 1]`, or `max_usd_per_day` is absent / invalid.
246 /// - Either mode: if the ladder is empty, or `margin`/`alpha` are refused by
247 /// [`crate::select::Experiment::new`].
248 pub fn validate(&self) -> Result<(), String> {
249 if self.ladder.is_empty() {
250 return Err("route.challenger.ladder must not be empty".to_owned());
251 }
252 match self.mode {
253 ChallengerMode::Live => {
254 if !self.percent.is_finite() || self.percent <= 0.0 || self.percent >= 100.0 {
255 return Err(format!(
256 "route.challenger.percent must be finite and within (0, 100) exclusive — \
257 at 100 there is no incumbent control and the test can never pair — got {}",
258 self.percent
259 ));
260 }
261 }
262 ChallengerMode::Paired => {
263 if !self.sample_rate.is_finite() || !(0.0..=1.0).contains(&self.sample_rate) {
264 return Err(format!(
265 "route.challenger.sample_rate must be finite and within [0, 1], got {}",
266 self.sample_rate
267 ));
268 }
269 let max = self.max_usd_per_day.ok_or_else(|| {
270 "route.challenger.max_usd_per_day is required in paired mode — the challenger \
271 makes real model calls off the hot path; a ceiling prevents silent spend"
272 .to_owned()
273 })?;
274 if !max.is_finite() || max < 0.0 {
275 return Err(format!(
276 "route.challenger.max_usd_per_day must be finite and >= 0, got {max}"
277 ));
278 }
279 }
280 }
281 crate::select::Experiment::new(self.margin, self.alpha)
282 .map(|_| ())
283 .map_err(|e| format!("route.challenger: {e}"))
284 }
285
286 /// Whether a given request falls in the paired sample (paired mode only).
287 ///
288 /// Uses the same salted-bucketing discipline as shadow sampling, under a `"paired"` domain tag
289 /// so the sample is independent of rollout and shadow membership. `request_id` should be the
290 /// trace id of the served request (a stable, per-request identity that does not repeat).
291 #[must_use]
292 pub fn paired_sampled(&self, salt: &str, request_id: &str) -> bool {
293 if self.sample_rate <= 0.0 {
294 return false;
295 }
296 if self.sample_rate >= 1.0 {
297 return true;
298 }
299 use sha2::{Digest, Sha256};
300 let mut h = Sha256::new();
301 h.update(salt.as_bytes());
302 h.update([0u8]);
303 h.update(b"paired"); // distinct tag: independent of rollout and shadow sampling
304 h.update([0u8]);
305 h.update(request_id.as_bytes());
306 let d = h.finalize();
307 let bucket = u32::from_be_bytes([d[0], d[1], d[2], d[3]]) % BUCKETS;
308 #[expect(
309 clippy::cast_possible_truncation,
310 clippy::cast_sign_loss,
311 reason = "sample_rate is validated finite in [0,1]"
312 )]
313 let cutoff = (self.sample_rate * f64::from(BUCKETS)) as u32;
314 bucket < cutoff
315 }
316
317 /// Identity of this experiment's configuration: the first 16 hex of SHA-256 over the canonical
318 /// JSON `[incumbent_ladder, challenger_ladder, margin, alpha]`. Stamped on every receipt so
319 /// that evidence gathered under a different configuration is never pooled with the current
320 /// one — editing either ladder, the margin or alpha starts a fresh experiment.
321 #[must_use]
322 pub fn fingerprint(&self, incumbent_ladder: &[String]) -> String {
323 use sha2::{Digest, Sha256};
324 let canonical = serde_json::json!([incumbent_ladder, self.ladder, self.margin, self.alpha]);
325 let digest = Sha256::digest(canonical.to_string().as_bytes());
326 hex::encode(&digest[..8])
327 }
328
329 /// The bucket `session_key` falls in, within `0..`[`BUCKETS`], under the `challenger` domain
330 /// tag.
331 #[must_use]
332 pub fn bucket(salt: &str, session_key: &str) -> u32 {
333 use sha2::{Digest, Sha256};
334 let mut h = Sha256::new();
335 h.update(salt.as_bytes());
336 h.update([0u8]);
337 // Distinct tag: arm assignment must be independent of rollout and shadow sampling, or
338 // the challenger arm would be a biased subset of whatever those select.
339 h.update(b"challenger");
340 h.update([0u8]);
341 h.update(session_key.as_bytes());
342 let d = h.finalize();
343 u32::from_be_bytes([d[0], d[1], d[2], d[3]]) % BUCKETS
344 }
345
346 /// The arm for a bucket under this config's `percent`.
347 #[must_use]
348 pub fn arm_of(&self, bucket: u32) -> crate::select::Arm {
349 if in_rollout(bucket, self.percent) {
350 crate::select::Arm::Challenger
351 } else {
352 crate::select::Arm::Incumbent
353 }
354 }
355
356 /// Which arm a session is in. Deterministic for a fixed `salt`, so a whole session stays in
357 /// one arm — outcomes are one per session and pairing assumes the arm never changes mid-run.
358 #[must_use]
359 pub fn arm(&self, salt: &str, session_key: &str) -> crate::select::Arm {
360 self.arm_of(Self::bucket(salt, session_key))
361 }
362}
363
364/// The bucket `key_value` falls in, within `0..`[`BUCKETS`].
365///
366/// Deterministic for a fixed `salt`. The salt is the per-deployment secret already used for
367/// feature fingerprints, so bucket assignment cannot be reconstructed — or gamed — by a caller
368/// who can choose their own session id.
369#[must_use]
370pub fn bucket_of(salt: &str, key: RolloutKey, key_value: &str) -> u32 {
371 use sha2::{Digest, Sha256};
372 let mut h = Sha256::new();
373 h.update(salt.as_bytes());
374 h.update([0u8]); // separator: salt||tag must not collide with a different split
375 h.update(key.tag().as_bytes());
376 h.update([0u8]);
377 h.update(key_value.as_bytes());
378 let d = h.finalize();
379 u32::from_be_bytes([d[0], d[1], d[2], d[3]]) % BUCKETS
380}
381
382/// Whether a bucket is inside a `percent` rollout.
383///
384/// Strictly `<` so that `percent = 0` enforces nothing and `percent = 100` enforces everything —
385/// both endpoints exact, which matters because "ramp to 0" is how an operator backs out.
386#[must_use]
387pub fn in_rollout(bucket: u32, percent: f64) -> bool {
388 if percent <= 0.0 {
389 return false;
390 }
391 if percent >= 100.0 {
392 return true;
393 }
394 #[expect(
395 clippy::cast_possible_truncation,
396 clippy::cast_sign_loss,
397 reason = "percent is validated finite in [0,100]; the product is within u32"
398 )]
399 let cutoff = (percent / 100.0 * f64::from(BUCKETS)) as u32;
400 bucket < cutoff
401}
402
403/// A stable identity for one request, for use as bucketing input when there is no session to key
404/// on. Hashes the request bytes, so the same request always lands in the same arm.
405///
406/// The digest is truncated and the input is never retained — this exists so rollout can bucket
407/// stateless traffic without Firstpass holding on to prompt content.
408#[must_use]
409pub fn request_identity(body: &[u8]) -> String {
410 use sha2::{Digest, Sha256};
411 let mut h = Sha256::new();
412 h.update(body);
413 hex::encode(&h.finalize()[..8])
414}
415
416/// The decision recorded on the trace, so an auditor can re-derive which arm a request was in.
417///
418/// Without this a mixed-population receipt log cannot be split back into the enforced and
419/// observed arms, and every savings or failure figure computed over it silently blends two
420/// different regimes.
421#[derive(Debug, Clone, Copy, PartialEq)]
422pub struct RolloutDecision {
423 /// Configured percent at decision time.
424 pub percent: f64,
425 /// Which identity was held constant.
426 pub key: RolloutKey,
427 /// The bucket this request landed in.
428 pub bucket: u32,
429 /// Whether it enforced.
430 pub enforced: bool,
431}
432
433/// Decide the arm for one request.
434#[must_use]
435pub fn decide(salt: &str, rollout: &Rollout, key_value: &str) -> RolloutDecision {
436 let bucket = bucket_of(salt, rollout.key, key_value);
437 RolloutDecision {
438 percent: rollout.percent,
439 key: rollout.key,
440 bucket,
441 enforced: in_rollout(bucket, rollout.percent),
442 }
443}
444
445#[cfg(test)]
446mod tests {
447 use super::*;
448
449 /// The property the whole design rests on: the same key always lands in the same arm, so a
450 /// conversation cannot flip mid-thread and the served population stays a stable sample.
451 #[test]
452 fn assignment_is_stable_for_a_key() {
453 let r = Rollout {
454 percent: 5.0,
455 key: RolloutKey::Session,
456 };
457 let first = decide("salt-a", &r, "session-123");
458 for _ in 0..1000 {
459 assert_eq!(decide("salt-a", &r, "session-123"), first);
460 }
461 }
462
463 /// Different salts must reshuffle, or two deployments would enforce on exactly the same
464 /// sessions and a caller who learned one deployment's arm would know the other's.
465 #[test]
466 fn salt_reshuffles_assignment() {
467 let differing = (0..200)
468 .filter(|i| {
469 let k = format!("session-{i}");
470 bucket_of("salt-a", RolloutKey::Session, &k)
471 != bucket_of("salt-b", RolloutKey::Session, &k)
472 })
473 .count();
474 assert!(
475 differing > 190,
476 "salt barely changed assignment: {differing}/200"
477 );
478 }
479
480 /// The same value under two key kinds must not inherit one assignment.
481 #[test]
482 fn key_kind_is_domain_separated() {
483 let v = "same-identifier";
484 let s = bucket_of("salt", RolloutKey::Session, v);
485 let t = bucket_of("salt", RolloutKey::Tenant, v);
486 let q = bucket_of("salt", RolloutKey::Request, v);
487 assert!(s != t || t != q, "key kinds collapsed to one bucket");
488 }
489
490 /// A skewed hash would silently ramp to the wrong percentage — the operator would ask for 10%
491 /// and get 3%, and only notice through a savings number that never moved.
492 #[test]
493 fn buckets_are_roughly_uniform() {
494 for pct in [1.0, 5.0, 10.0, 50.0] {
495 let r = Rollout {
496 percent: pct,
497 key: RolloutKey::Session,
498 };
499 let n = 20_000;
500 let hits = (0..n)
501 .filter(|i| decide("salt", &r, &format!("s{i}")).enforced)
502 .count();
503 let observed = hits as f64 / f64::from(n) * 100.0;
504 assert!(
505 (observed - pct).abs() < 1.0,
506 "{pct}% rollout produced {observed:.2}% of {n} keys"
507 );
508 }
509 }
510
511 /// Both endpoints must be exact: 0 is how an operator backs out, 100 is full enforcement.
512 #[test]
513 fn endpoints_are_exact() {
514 for i in 0..500 {
515 let k = format!("s{i}");
516 assert!(
517 !decide(
518 "salt",
519 &Rollout {
520 percent: 0.0,
521 key: RolloutKey::Session
522 },
523 &k
524 )
525 .enforced
526 );
527 assert!(
528 decide(
529 "salt",
530 &Rollout {
531 percent: 100.0,
532 key: RolloutKey::Session
533 },
534 &k
535 )
536 .enforced
537 );
538 }
539 }
540
541 /// Ramping up must never eject a key that was already enforcing — an operator going 5% → 25%
542 /// expects to add traffic, not to reshuffle who is in the experiment.
543 #[test]
544 fn ramping_up_is_monotonic() {
545 let keys: Vec<String> = (0..2000).map(|i| format!("s{i}")).collect();
546 let arm = |pct: f64| -> Vec<bool> {
547 keys.iter()
548 .map(|k| {
549 decide(
550 "salt",
551 &Rollout {
552 percent: pct,
553 key: RolloutKey::Session,
554 },
555 k,
556 )
557 .enforced
558 })
559 .collect()
560 };
561 let (small, large) = (arm(5.0), arm(25.0));
562 for (i, (s, l)) in small.iter().zip(&large).enumerate() {
563 assert!(!*s || *l, "key {i} enforced at 5% but not at 25%");
564 }
565 }
566
567 #[test]
568 fn validate_rejects_impossible_percentages() {
569 for bad in [-0.1, 100.1, f64::NAN, f64::INFINITY] {
570 assert!(
571 Rollout {
572 percent: bad,
573 key: RolloutKey::Session
574 }
575 .validate()
576 .is_err()
577 );
578 }
579 for good in [0.0, 0.01, 50.0, 100.0] {
580 assert!(
581 Rollout {
582 percent: good,
583 key: RolloutKey::Session
584 }
585 .validate()
586 .is_ok()
587 );
588 }
589 }
590
591 /// Shadow sampling must be INDEPENDENT of the rollout arm. If both hashed the same input
592 /// under the same tag, every shadow-sampled request would sit in the same arm, and the
593 /// projection would describe a biased subset of traffic rather than the traffic an operator
594 /// is about to enforce on.
595 #[test]
596 fn shadow_sampling_is_independent_of_the_rollout_arm() {
597 let roll = Rollout {
598 percent: 50.0,
599 key: RolloutKey::Session,
600 };
601 let shadow = Shadow {
602 sample_rate: 0.5,
603 max_usd_per_day: 5.0,
604 };
605 let (mut both, mut only_shadow, mut only_arm, mut neither) = (0, 0, 0, 0);
606 for i in 0..4000 {
607 let k = format!("s{i}");
608 match (
609 decide("salt", &roll, &k).enforced,
610 shadow.sampled("salt", &k),
611 ) {
612 (true, true) => both += 1,
613 (false, true) => only_shadow += 1,
614 (true, false) => only_arm += 1,
615 (false, false) => neither += 1,
616 }
617 }
618 // Independent 50/50 splits put ~25% in each cell; a shared hash would empty two of them.
619 for (name, n) in [
620 ("both", both),
621 ("shadow-only", only_shadow),
622 ("arm-only", only_arm),
623 ("neither", neither),
624 ] {
625 let pct = f64::from(n) / 4000.0 * 100.0;
626 assert!(
627 (pct - 25.0).abs() < 3.0,
628 "{name} cell held {pct:.1}% — shadow sampling correlates with the rollout arm"
629 );
630 }
631 }
632
633 #[test]
634 fn shadow_sample_rate_endpoints_are_exact() {
635 for i in 0..300 {
636 let k = format!("s{i}");
637 assert!(
638 !Shadow {
639 sample_rate: 0.0,
640 max_usd_per_day: 1.0
641 }
642 .sampled("salt", &k)
643 );
644 assert!(
645 Shadow {
646 sample_rate: 1.0,
647 max_usd_per_day: 1.0
648 }
649 .sampled("salt", &k)
650 );
651 }
652 }
653
654 #[test]
655 fn shadow_validate_rejects_impossible_settings() {
656 for bad in [-0.1, 1.1, f64::NAN] {
657 assert!(
658 Shadow {
659 sample_rate: bad,
660 max_usd_per_day: 1.0
661 }
662 .validate()
663 .is_err()
664 );
665 }
666 for bad in [-1.0, f64::INFINITY] {
667 assert!(
668 Shadow {
669 sample_rate: 0.1,
670 max_usd_per_day: bad
671 }
672 .validate()
673 .is_err()
674 );
675 }
676 assert!(
677 Shadow {
678 sample_rate: 0.1,
679 max_usd_per_day: 5.0
680 }
681 .validate()
682 .is_ok()
683 );
684 }
685
686 fn challenger(percent: f64) -> Challenger {
687 Challenger {
688 ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
689 percent,
690 margin: crate::select::DEFAULT_MARGIN,
691 alpha: crate::select::DEFAULT_ALPHA,
692 mode: ChallengerMode::Live,
693 sample_rate: 1.0,
694 max_usd_per_day: None,
695 redis_url: None,
696 }
697 }
698
699 fn paired_challenger() -> Challenger {
700 Challenger {
701 ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
702 percent: 0.0,
703 margin: crate::select::DEFAULT_MARGIN,
704 alpha: crate::select::DEFAULT_ALPHA,
705 mode: ChallengerMode::Paired,
706 sample_rate: 1.0,
707 max_usd_per_day: Some(5.0),
708 redis_url: None,
709 }
710 }
711
712 #[test]
713 fn challenger_arm_is_deterministic_and_matches_percent() {
714 use crate::select::Arm;
715 let c = challenger(20.0);
716 for i in 0..200 {
717 let k = format!("s{i}");
718 assert_eq!(c.arm("salt", &k), c.arm("salt", &k));
719 }
720 let n = 10_000;
721 let hits = (0..n)
722 .filter(|i| c.arm("salt", &format!("s{i}")) == Arm::Challenger)
723 .count();
724 let observed = hits as f64 / f64::from(n) * 100.0;
725 assert!((observed - 20.0).abs() < 1.5, "got {observed:.2}%");
726 // The pure function still handles the endpoint (validation is what forbids it).
727 assert!(
728 (0..300).all(|i| challenger(100.0).arm("salt", &format!("s{i}")) == Arm::Challenger)
729 );
730 }
731
732 /// Arm assignment must be independent of shadow sampling at the same salt.
733 #[test]
734 fn challenger_arm_is_uncorrelated_with_shadow_sampling() {
735 use crate::select::Arm;
736 let c = challenger(50.0);
737 let shadow = Shadow {
738 sample_rate: 0.5,
739 max_usd_per_day: 1.0,
740 };
741 let mut cells = [0u32; 4];
742 for i in 0..4000 {
743 let k = format!("s{i}");
744 let a = usize::from(c.arm("salt", &k) == Arm::Challenger);
745 let b = usize::from(shadow.sampled("salt", &k));
746 cells[a * 2 + b] += 1;
747 }
748 for (i, n) in cells.iter().enumerate() {
749 let pct = f64::from(*n) / 4000.0 * 100.0;
750 assert!((pct - 25.0).abs() < 3.0, "cell {i} held {pct:.1}%");
751 }
752 }
753
754 #[test]
755 fn challenger_validate_rejects_incoherent_settings() {
756 for bad in [0.0, -1.0, 100.0, 100.1, f64::NAN, f64::INFINITY] {
757 assert!(challenger(bad).validate().is_err(), "percent {bad}");
758 }
759 for good in [0.01, 10.0, 99.99] {
760 assert!(challenger(good).validate().is_ok(), "percent {good}");
761 }
762 let mut c = challenger(10.0);
763 c.ladder.clear();
764 assert!(c.validate().is_err());
765 let mut c = challenger(10.0);
766 c.margin = 0.0;
767 assert!(c.validate().is_err());
768 let mut c = challenger(10.0);
769 c.alpha = 0.5;
770 assert!(c.validate().is_err());
771 }
772
773 #[test]
774 fn paired_challenger_validate_rejects_incoherent_settings() {
775 assert!(paired_challenger().validate().is_ok(), "base paired ok");
776 // Missing max_usd_per_day in paired mode.
777 let mut c = paired_challenger();
778 c.max_usd_per_day = None;
779 assert!(c.validate().is_err(), "missing max_usd_per_day");
780 // Negative max_usd_per_day.
781 let mut c = paired_challenger();
782 c.max_usd_per_day = Some(-1.0);
783 assert!(c.validate().is_err(), "negative max_usd_per_day");
784 // sample_rate out of range.
785 let mut c = paired_challenger();
786 c.sample_rate = 1.5;
787 assert!(c.validate().is_err(), "sample_rate > 1");
788 let mut c = paired_challenger();
789 c.sample_rate = -0.1;
790 assert!(c.validate().is_err(), "sample_rate < 0");
791 // Empty ladder.
792 let mut c = paired_challenger();
793 c.ladder.clear();
794 assert!(c.validate().is_err(), "empty ladder");
795 }
796
797 #[test]
798 fn paired_sampled_is_deterministic_and_approximately_rate() {
799 let c = paired_challenger();
800 // sample_rate = 1.0: all sampled.
801 for i in 0..100 {
802 assert!(c.paired_sampled("salt", &format!("r{i}")));
803 }
804 // sample_rate = 0.0: none sampled.
805 let mut c0 = paired_challenger();
806 c0.sample_rate = 0.0;
807 for i in 0..100 {
808 assert!(!c0.paired_sampled("salt", &format!("r{i}")));
809 }
810 // sample_rate = 0.5: approximately 50%.
811 let mut c5 = paired_challenger();
812 c5.sample_rate = 0.5;
813 let n = 10_000u32;
814 let hits = (0..n)
815 .filter(|i| c5.paired_sampled("salt", &format!("r{i}")))
816 .count();
817 let observed = hits as f64 / f64::from(n) * 100.0;
818 assert!((observed - 50.0).abs() < 2.0, "got {observed:.2}%");
819 // Deterministic: same result on repeated calls.
820 for i in 0..100 {
821 let key = format!("req-{i}");
822 assert_eq!(
823 c5.paired_sampled("salt", &key),
824 c5.paired_sampled("salt", &key)
825 );
826 }
827 }
828
829 #[test]
830 fn fingerprint_tracks_ladders_margin_and_alpha() {
831 let inc = vec!["anthropic/claude-sonnet-5".to_owned()];
832 let base = challenger(10.0);
833 let fp = base.fingerprint(&inc);
834 assert_eq!(fp.len(), 16);
835 assert_eq!(fp, challenger(10.0).fingerprint(&inc), "deterministic");
836 // percent is not part of the identity: ramping must not reset the evidence.
837 assert_eq!(fp, challenger(25.0).fingerprint(&inc));
838 let mut c = challenger(10.0);
839 c.ladder = vec!["anthropic/claude-opus-4-8".to_owned()];
840 assert_ne!(fp, c.fingerprint(&inc));
841 let mut c = challenger(10.0);
842 c.margin = 0.04;
843 assert_ne!(fp, c.fingerprint(&inc));
844 let mut c = challenger(10.0);
845 c.alpha = 0.01;
846 assert_ne!(fp, c.fingerprint(&inc));
847 assert_ne!(
848 fp,
849 base.fingerprint(&["anthropic/claude-opus-4-8".to_owned()])
850 );
851 }
852}