use serde::Deserialize;
pub const BUCKETS: u32 = 10_000;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum RolloutKey {
#[default]
Session,
Request,
Tenant,
}
impl RolloutKey {
#[must_use]
pub const fn tag(self) -> &'static str {
match self {
Self::Session => "session",
Self::Request => "request",
Self::Tenant => "tenant",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Rollout {
pub percent: f64,
#[serde(default)]
pub key: RolloutKey,
}
impl Rollout {
pub fn validate(&self) -> Result<(), String> {
if !self.percent.is_finite() || !(0.0..=100.0).contains(&self.percent) {
return Err(format!(
"route.rollout.percent must be finite and within [0, 100], got {}",
self.percent
));
}
Ok(())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Shadow {
pub sample_rate: f64,
pub max_usd_per_day: f64,
}
impl Shadow {
pub fn validate(&self) -> Result<(), String> {
if !self.sample_rate.is_finite() || !(0.0..=1.0).contains(&self.sample_rate) {
return Err(format!(
"route.shadow.sample_rate must be finite and within [0, 1], got {}",
self.sample_rate
));
}
if !self.max_usd_per_day.is_finite() || self.max_usd_per_day < 0.0 {
return Err(format!(
"route.shadow.max_usd_per_day must be finite and >= 0, got {}",
self.max_usd_per_day
));
}
Ok(())
}
#[must_use]
pub fn sampled(&self, salt: &str, key_value: &str) -> bool {
if self.sample_rate <= 0.0 {
return false;
}
if self.sample_rate >= 1.0 {
return true;
}
use sha2::{Digest, Sha256};
let mut h = Sha256::new();
h.update(salt.as_bytes());
h.update([0u8]);
h.update(b"shadow"); h.update([0u8]);
h.update(key_value.as_bytes());
let d = h.finalize();
let bucket = u32::from_be_bytes([d[0], d[1], d[2], d[3]]) % BUCKETS;
#[expect(
clippy::cast_possible_truncation,
clippy::cast_sign_loss,
reason = "sample_rate is validated finite in [0,1]"
)]
let cutoff = (self.sample_rate * f64::from(BUCKETS)) as u32;
bucket < cutoff
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ChallengerMode {
#[default]
Live,
Paired,
}
#[derive(Debug, Clone, PartialEq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Challenger {
pub ladder: Vec<String>,
#[serde(default)]
pub percent: f64,
#[serde(default = "default_challenger_margin")]
pub margin: f64,
#[serde(default = "default_challenger_alpha")]
pub alpha: f64,
#[serde(default)]
pub mode: ChallengerMode,
#[serde(default = "default_sample_rate")]
pub sample_rate: f64,
#[serde(default)]
pub max_usd_per_day: Option<f64>,
#[serde(default)]
pub redis_url: Option<String>,
}
fn default_challenger_margin() -> f64 {
crate::select::DEFAULT_MARGIN
}
fn default_challenger_alpha() -> f64 {
crate::select::DEFAULT_ALPHA
}
fn default_sample_rate() -> f64 {
1.0
}
impl Challenger {
#[must_use]
pub fn is_paired(&self) -> bool {
self.mode == ChallengerMode::Paired
}
pub fn validate(&self) -> Result<(), String> {
if self.ladder.is_empty() {
return Err("route.challenger.ladder must not be empty".to_owned());
}
match self.mode {
ChallengerMode::Live => {
if !self.percent.is_finite() || self.percent <= 0.0 || self.percent >= 100.0 {
return Err(format!(
"route.challenger.percent must be finite and within (0, 100) exclusive — \
at 100 there is no incumbent control and the test can never pair — got {}",
self.percent
));
}
}
ChallengerMode::Paired => {
if !self.sample_rate.is_finite() || !(0.0..=1.0).contains(&self.sample_rate) {
return Err(format!(
"route.challenger.sample_rate must be finite and within [0, 1], got {}",
self.sample_rate
));
}
let max = self.max_usd_per_day.ok_or_else(|| {
"route.challenger.max_usd_per_day is required in paired mode — the challenger \
makes real model calls off the hot path; a ceiling prevents silent spend"
.to_owned()
})?;
if !max.is_finite() || max < 0.0 {
return Err(format!(
"route.challenger.max_usd_per_day must be finite and >= 0, got {max}"
));
}
}
}
crate::select::Experiment::new(self.margin, self.alpha)
.map(|_| ())
.map_err(|e| format!("route.challenger: {e}"))
}
#[must_use]
pub fn paired_sampled(&self, salt: &str, request_id: &str) -> bool {
if self.sample_rate <= 0.0 {
return false;
}
if self.sample_rate >= 1.0 {
return true;
}
use sha2::{Digest, Sha256};
let mut h = Sha256::new();
h.update(salt.as_bytes());
h.update([0u8]);
h.update(b"paired"); h.update([0u8]);
h.update(request_id.as_bytes());
let d = h.finalize();
let bucket = u32::from_be_bytes([d[0], d[1], d[2], d[3]]) % BUCKETS;
#[expect(
clippy::cast_possible_truncation,
clippy::cast_sign_loss,
reason = "sample_rate is validated finite in [0,1]"
)]
let cutoff = (self.sample_rate * f64::from(BUCKETS)) as u32;
bucket < cutoff
}
#[must_use]
pub fn fingerprint(&self, incumbent_ladder: &[String]) -> String {
use sha2::{Digest, Sha256};
let canonical = serde_json::json!([incumbent_ladder, self.ladder, self.margin, self.alpha]);
let digest = Sha256::digest(canonical.to_string().as_bytes());
hex::encode(&digest[..8])
}
#[must_use]
pub fn bucket(salt: &str, session_key: &str) -> u32 {
use sha2::{Digest, Sha256};
let mut h = Sha256::new();
h.update(salt.as_bytes());
h.update([0u8]);
h.update(b"challenger");
h.update([0u8]);
h.update(session_key.as_bytes());
let d = h.finalize();
u32::from_be_bytes([d[0], d[1], d[2], d[3]]) % BUCKETS
}
#[must_use]
pub fn arm_of(&self, bucket: u32) -> crate::select::Arm {
if in_rollout(bucket, self.percent) {
crate::select::Arm::Challenger
} else {
crate::select::Arm::Incumbent
}
}
#[must_use]
pub fn arm(&self, salt: &str, session_key: &str) -> crate::select::Arm {
self.arm_of(Self::bucket(salt, session_key))
}
}
#[must_use]
pub fn bucket_of(salt: &str, key: RolloutKey, key_value: &str) -> u32 {
use sha2::{Digest, Sha256};
let mut h = Sha256::new();
h.update(salt.as_bytes());
h.update([0u8]); h.update(key.tag().as_bytes());
h.update([0u8]);
h.update(key_value.as_bytes());
let d = h.finalize();
u32::from_be_bytes([d[0], d[1], d[2], d[3]]) % BUCKETS
}
#[must_use]
pub fn in_rollout(bucket: u32, percent: f64) -> bool {
if percent <= 0.0 {
return false;
}
if percent >= 100.0 {
return true;
}
#[expect(
clippy::cast_possible_truncation,
clippy::cast_sign_loss,
reason = "percent is validated finite in [0,100]; the product is within u32"
)]
let cutoff = (percent / 100.0 * f64::from(BUCKETS)) as u32;
bucket < cutoff
}
#[must_use]
pub fn request_identity(body: &[u8]) -> String {
use sha2::{Digest, Sha256};
let mut h = Sha256::new();
h.update(body);
hex::encode(&h.finalize()[..8])
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct RolloutDecision {
pub percent: f64,
pub key: RolloutKey,
pub bucket: u32,
pub enforced: bool,
}
#[must_use]
pub fn decide(salt: &str, rollout: &Rollout, key_value: &str) -> RolloutDecision {
let bucket = bucket_of(salt, rollout.key, key_value);
RolloutDecision {
percent: rollout.percent,
key: rollout.key,
bucket,
enforced: in_rollout(bucket, rollout.percent),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn assignment_is_stable_for_a_key() {
let r = Rollout {
percent: 5.0,
key: RolloutKey::Session,
};
let first = decide("salt-a", &r, "session-123");
for _ in 0..1000 {
assert_eq!(decide("salt-a", &r, "session-123"), first);
}
}
#[test]
fn salt_reshuffles_assignment() {
let differing = (0..200)
.filter(|i| {
let k = format!("session-{i}");
bucket_of("salt-a", RolloutKey::Session, &k)
!= bucket_of("salt-b", RolloutKey::Session, &k)
})
.count();
assert!(
differing > 190,
"salt barely changed assignment: {differing}/200"
);
}
#[test]
fn key_kind_is_domain_separated() {
let v = "same-identifier";
let s = bucket_of("salt", RolloutKey::Session, v);
let t = bucket_of("salt", RolloutKey::Tenant, v);
let q = bucket_of("salt", RolloutKey::Request, v);
assert!(s != t || t != q, "key kinds collapsed to one bucket");
}
#[test]
fn buckets_are_roughly_uniform() {
for pct in [1.0, 5.0, 10.0, 50.0] {
let r = Rollout {
percent: pct,
key: RolloutKey::Session,
};
let n = 20_000;
let hits = (0..n)
.filter(|i| decide("salt", &r, &format!("s{i}")).enforced)
.count();
let observed = hits as f64 / f64::from(n) * 100.0;
assert!(
(observed - pct).abs() < 1.0,
"{pct}% rollout produced {observed:.2}% of {n} keys"
);
}
}
#[test]
fn endpoints_are_exact() {
for i in 0..500 {
let k = format!("s{i}");
assert!(
!decide(
"salt",
&Rollout {
percent: 0.0,
key: RolloutKey::Session
},
&k
)
.enforced
);
assert!(
decide(
"salt",
&Rollout {
percent: 100.0,
key: RolloutKey::Session
},
&k
)
.enforced
);
}
}
#[test]
fn ramping_up_is_monotonic() {
let keys: Vec<String> = (0..2000).map(|i| format!("s{i}")).collect();
let arm = |pct: f64| -> Vec<bool> {
keys.iter()
.map(|k| {
decide(
"salt",
&Rollout {
percent: pct,
key: RolloutKey::Session,
},
k,
)
.enforced
})
.collect()
};
let (small, large) = (arm(5.0), arm(25.0));
for (i, (s, l)) in small.iter().zip(&large).enumerate() {
assert!(!*s || *l, "key {i} enforced at 5% but not at 25%");
}
}
#[test]
fn validate_rejects_impossible_percentages() {
for bad in [-0.1, 100.1, f64::NAN, f64::INFINITY] {
assert!(
Rollout {
percent: bad,
key: RolloutKey::Session
}
.validate()
.is_err()
);
}
for good in [0.0, 0.01, 50.0, 100.0] {
assert!(
Rollout {
percent: good,
key: RolloutKey::Session
}
.validate()
.is_ok()
);
}
}
#[test]
fn shadow_sampling_is_independent_of_the_rollout_arm() {
let roll = Rollout {
percent: 50.0,
key: RolloutKey::Session,
};
let shadow = Shadow {
sample_rate: 0.5,
max_usd_per_day: 5.0,
};
let (mut both, mut only_shadow, mut only_arm, mut neither) = (0, 0, 0, 0);
for i in 0..4000 {
let k = format!("s{i}");
match (
decide("salt", &roll, &k).enforced,
shadow.sampled("salt", &k),
) {
(true, true) => both += 1,
(false, true) => only_shadow += 1,
(true, false) => only_arm += 1,
(false, false) => neither += 1,
}
}
for (name, n) in [
("both", both),
("shadow-only", only_shadow),
("arm-only", only_arm),
("neither", neither),
] {
let pct = f64::from(n) / 4000.0 * 100.0;
assert!(
(pct - 25.0).abs() < 3.0,
"{name} cell held {pct:.1}% — shadow sampling correlates with the rollout arm"
);
}
}
#[test]
fn shadow_sample_rate_endpoints_are_exact() {
for i in 0..300 {
let k = format!("s{i}");
assert!(
!Shadow {
sample_rate: 0.0,
max_usd_per_day: 1.0
}
.sampled("salt", &k)
);
assert!(
Shadow {
sample_rate: 1.0,
max_usd_per_day: 1.0
}
.sampled("salt", &k)
);
}
}
#[test]
fn shadow_validate_rejects_impossible_settings() {
for bad in [-0.1, 1.1, f64::NAN] {
assert!(
Shadow {
sample_rate: bad,
max_usd_per_day: 1.0
}
.validate()
.is_err()
);
}
for bad in [-1.0, f64::INFINITY] {
assert!(
Shadow {
sample_rate: 0.1,
max_usd_per_day: bad
}
.validate()
.is_err()
);
}
assert!(
Shadow {
sample_rate: 0.1,
max_usd_per_day: 5.0
}
.validate()
.is_ok()
);
}
fn challenger(percent: f64) -> Challenger {
Challenger {
ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
percent,
margin: crate::select::DEFAULT_MARGIN,
alpha: crate::select::DEFAULT_ALPHA,
mode: ChallengerMode::Live,
sample_rate: 1.0,
max_usd_per_day: None,
redis_url: None,
}
}
fn paired_challenger() -> Challenger {
Challenger {
ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
percent: 0.0,
margin: crate::select::DEFAULT_MARGIN,
alpha: crate::select::DEFAULT_ALPHA,
mode: ChallengerMode::Paired,
sample_rate: 1.0,
max_usd_per_day: Some(5.0),
redis_url: None,
}
}
#[test]
fn challenger_arm_is_deterministic_and_matches_percent() {
use crate::select::Arm;
let c = challenger(20.0);
for i in 0..200 {
let k = format!("s{i}");
assert_eq!(c.arm("salt", &k), c.arm("salt", &k));
}
let n = 10_000;
let hits = (0..n)
.filter(|i| c.arm("salt", &format!("s{i}")) == Arm::Challenger)
.count();
let observed = hits as f64 / f64::from(n) * 100.0;
assert!((observed - 20.0).abs() < 1.5, "got {observed:.2}%");
assert!(
(0..300).all(|i| challenger(100.0).arm("salt", &format!("s{i}")) == Arm::Challenger)
);
}
#[test]
fn challenger_arm_is_uncorrelated_with_shadow_sampling() {
use crate::select::Arm;
let c = challenger(50.0);
let shadow = Shadow {
sample_rate: 0.5,
max_usd_per_day: 1.0,
};
let mut cells = [0u32; 4];
for i in 0..4000 {
let k = format!("s{i}");
let a = usize::from(c.arm("salt", &k) == Arm::Challenger);
let b = usize::from(shadow.sampled("salt", &k));
cells[a * 2 + b] += 1;
}
for (i, n) in cells.iter().enumerate() {
let pct = f64::from(*n) / 4000.0 * 100.0;
assert!((pct - 25.0).abs() < 3.0, "cell {i} held {pct:.1}%");
}
}
#[test]
fn challenger_validate_rejects_incoherent_settings() {
for bad in [0.0, -1.0, 100.0, 100.1, f64::NAN, f64::INFINITY] {
assert!(challenger(bad).validate().is_err(), "percent {bad}");
}
for good in [0.01, 10.0, 99.99] {
assert!(challenger(good).validate().is_ok(), "percent {good}");
}
let mut c = challenger(10.0);
c.ladder.clear();
assert!(c.validate().is_err());
let mut c = challenger(10.0);
c.margin = 0.0;
assert!(c.validate().is_err());
let mut c = challenger(10.0);
c.alpha = 0.5;
assert!(c.validate().is_err());
}
#[test]
fn paired_challenger_validate_rejects_incoherent_settings() {
assert!(paired_challenger().validate().is_ok(), "base paired ok");
let mut c = paired_challenger();
c.max_usd_per_day = None;
assert!(c.validate().is_err(), "missing max_usd_per_day");
let mut c = paired_challenger();
c.max_usd_per_day = Some(-1.0);
assert!(c.validate().is_err(), "negative max_usd_per_day");
let mut c = paired_challenger();
c.sample_rate = 1.5;
assert!(c.validate().is_err(), "sample_rate > 1");
let mut c = paired_challenger();
c.sample_rate = -0.1;
assert!(c.validate().is_err(), "sample_rate < 0");
let mut c = paired_challenger();
c.ladder.clear();
assert!(c.validate().is_err(), "empty ladder");
}
#[test]
fn paired_sampled_is_deterministic_and_approximately_rate() {
let c = paired_challenger();
for i in 0..100 {
assert!(c.paired_sampled("salt", &format!("r{i}")));
}
let mut c0 = paired_challenger();
c0.sample_rate = 0.0;
for i in 0..100 {
assert!(!c0.paired_sampled("salt", &format!("r{i}")));
}
let mut c5 = paired_challenger();
c5.sample_rate = 0.5;
let n = 10_000u32;
let hits = (0..n)
.filter(|i| c5.paired_sampled("salt", &format!("r{i}")))
.count();
let observed = hits as f64 / f64::from(n) * 100.0;
assert!((observed - 50.0).abs() < 2.0, "got {observed:.2}%");
for i in 0..100 {
let key = format!("req-{i}");
assert_eq!(
c5.paired_sampled("salt", &key),
c5.paired_sampled("salt", &key)
);
}
}
#[test]
fn fingerprint_tracks_ladders_margin_and_alpha() {
let inc = vec!["anthropic/claude-sonnet-5".to_owned()];
let base = challenger(10.0);
let fp = base.fingerprint(&inc);
assert_eq!(fp.len(), 16);
assert_eq!(fp, challenger(10.0).fingerprint(&inc), "deterministic");
assert_eq!(fp, challenger(25.0).fingerprint(&inc));
let mut c = challenger(10.0);
c.ladder = vec!["anthropic/claude-opus-4-8".to_owned()];
assert_ne!(fp, c.fingerprint(&inc));
let mut c = challenger(10.0);
c.margin = 0.04;
assert_ne!(fp, c.fingerprint(&inc));
let mut c = challenger(10.0);
c.alpha = 0.01;
assert_ne!(fp, c.fingerprint(&inc));
assert_ne!(
fp,
base.fingerprint(&["anthropic/claude-opus-4-8".to_owned()])
);
}
}