use std::collections::HashMap;
use serde::{Deserialize, Serialize};
pub type ExperimentId = u64;
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum ExperimentVariable {
ReminderLeadMinutes,
SurfacingMode,
SurfacingFrequency,
InformationDensity,
SurfacingTimeOfDay,
Custom(String),
}
impl ExperimentVariable {
pub fn as_str(&self) -> &str {
match self {
Self::ReminderLeadMinutes => "reminder_lead_minutes",
Self::SurfacingMode => "surfacing_mode",
Self::SurfacingFrequency => "surfacing_frequency",
Self::InformationDensity => "information_density",
Self::SurfacingTimeOfDay => "surfacing_time_of_day",
Self::Custom(s) => s,
}
}
pub fn is_safe(&self) -> bool {
!matches!(self, Self::Custom(_))
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum VariantValue {
Number(f64),
Label(String),
}
impl VariantValue {
pub fn as_number(&self) -> Option<f64> {
match self {
Self::Number(n) => Some(*n),
_ => None,
}
}
pub fn as_label(&self) -> Option<&str> {
match self {
Self::Label(s) => Some(s),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum ExperimentStatus {
Designed,
Running,
Concluded,
Aborted,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum TrialOutcome {
Positive,
Negative,
Neutral,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum SafetyBound {
MaxConsecutiveRejections(u32),
MaxDuration(f64),
MinAcceptanceRate(f64),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Experiment {
pub id: ExperimentId,
pub hypothesis: String,
pub variable: ExperimentVariable,
pub variants: Vec<VariantValue>,
pub sample_size_target: u32,
pub safety_bounds: Vec<SafetyBound>,
pub status: ExperimentStatus,
pub created_at: f64,
pub started_at: Option<f64>,
pub ended_at: Option<f64>,
pub variant_results: Vec<BetaPosterior>,
pub last_variant_idx: usize,
pub consecutive_negatives: Vec<u32>,
pub winner: Option<usize>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BetaPosterior {
pub alpha: f64,
pub beta: f64,
pub trials: u32,
}
impl BetaPosterior {
pub fn new() -> Self {
Self {
alpha: 1.0,
beta: 1.0,
trials: 0,
}
}
pub fn record_positive(&mut self) {
self.alpha += 1.0;
self.trials += 1;
}
pub fn record_negative(&mut self) {
self.beta += 1.0;
self.trials += 1;
}
pub fn record_neutral(&mut self) {
self.trials += 1;
}
pub fn mean(&self) -> f64 {
self.alpha / (self.alpha + self.beta)
}
pub fn variance(&self) -> f64 {
let s = self.alpha + self.beta;
(self.alpha * self.beta) / (s * s * (s + 1.0))
}
pub fn credible_interval_width(&self) -> f64 {
2.0 * 1.96 * self.variance().sqrt()
}
pub fn acceptance_rate(&self) -> f64 {
if self.trials == 0 {
return 0.5; }
(self.alpha - 1.0) / self.trials as f64
}
pub fn thompson_score(&self, jitter: f64) -> f64 {
let mode = if self.alpha > 1.0 && self.beta > 1.0 {
(self.alpha - 1.0) / (self.alpha + self.beta - 2.0)
} else {
self.mean()
};
(mode + jitter).clamp(0.0, 1.0)
}
}
impl Default for BetaPosterior {
fn default() -> Self {
Self::new()
}
}
impl Experiment {
pub fn new(
id: ExperimentId,
hypothesis: String,
variable: ExperimentVariable,
variants: Vec<VariantValue>,
sample_size_target: u32,
safety_bounds: Vec<SafetyBound>,
now: f64,
) -> Self {
let n = variants.len();
Self {
id,
hypothesis,
variable,
variants,
sample_size_target,
safety_bounds,
status: ExperimentStatus::Designed,
created_at: now,
started_at: None,
ended_at: None,
variant_results: (0..n).map(|_| BetaPosterior::new()).collect(),
last_variant_idx: 0,
consecutive_negatives: vec![0; n],
winner: None,
}
}
pub fn start(&mut self, now: f64) {
self.status = ExperimentStatus::Running;
self.started_at = Some(now);
}
pub fn is_active(&self) -> bool {
self.status == ExperimentStatus::Running
}
pub fn variant_count(&self) -> usize {
self.variants.len()
}
pub fn total_trials(&self) -> u32 {
self.variant_results.iter().map(|v| v.trials).sum()
}
pub fn is_sample_complete(&self) -> bool {
self.variant_results
.iter()
.all(|v| v.trials >= self.sample_size_target)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExperimentRegistry {
pub experiments: Vec<Experiment>,
pub max_concurrent: usize,
next_id: u64,
pub total_created: u64,
pub total_concluded: u64,
pub total_aborted: u64,
}
impl ExperimentRegistry {
pub fn new() -> Self {
Self {
experiments: Vec::new(),
max_concurrent: 3,
next_id: 1,
total_created: 0,
total_concluded: 0,
total_aborted: 0,
}
}
fn alloc_id(&mut self) -> ExperimentId {
let id = self.next_id;
self.next_id += 1;
id
}
pub fn active_count(&self) -> usize {
self.experiments
.iter()
.filter(|e| e.is_active())
.count()
}
pub fn can_start_new(&self) -> bool {
self.active_count() < self.max_concurrent
}
pub fn active_experiments(&self) -> Vec<&Experiment> {
self.experiments
.iter()
.filter(|e| e.is_active())
.collect()
}
pub fn concluded_experiments(&self) -> Vec<&Experiment> {
self.experiments
.iter()
.filter(|e| e.status == ExperimentStatus::Concluded)
.collect()
}
pub fn find(&self, id: ExperimentId) -> Option<&Experiment> {
self.experiments.iter().find(|e| e.id == id)
}
pub fn find_mut(&mut self, id: ExperimentId) -> Option<&mut Experiment> {
self.experiments.iter_mut().find(|e| e.id == id)
}
}
impl Default for ExperimentRegistry {
fn default() -> Self {
Self::new()
}
}
pub fn design_experiment(
registry: &mut ExperimentRegistry,
hypothesis: String,
variable: ExperimentVariable,
variants: Vec<VariantValue>,
sample_size_target: u32,
safety_bounds: Vec<SafetyBound>,
now: f64,
) -> Option<ExperimentId> {
if !variable.is_safe() {
return None;
}
if !registry.can_start_new() {
return None;
}
let has_existing = registry
.experiments
.iter()
.any(|e| e.is_active() && e.variable == variable);
if has_existing {
return None;
}
if variants.len() < 2 {
return None;
}
let id = registry.alloc_id();
let mut experiment = Experiment::new(
id, hypothesis, variable, variants, sample_size_target, safety_bounds, now,
);
experiment.start(now);
registry.experiments.push(experiment);
registry.total_created += 1;
Some(id)
}
pub fn assign_variant(experiment: &mut Experiment, jitter_seed: f64) -> Option<(usize, &VariantValue)> {
if !experiment.is_active() {
return None;
}
let best_idx = experiment
.variant_results
.iter()
.enumerate()
.max_by(|(i, a), (j, b)| {
let ja = jitter_seed * (1.0 + *i as f64 * 0.1) % 0.05;
let jb = jitter_seed * (1.0 + *j as f64 * 0.1) % 0.05;
a.thompson_score(ja)
.partial_cmp(&b.thompson_score(jb))
.unwrap()
})
.map(|(i, _)| i)
.unwrap_or(0);
experiment.last_variant_idx = best_idx;
Some((best_idx, &experiment.variants[best_idx]))
}
pub fn assign_variant_round_robin(experiment: &mut Experiment) -> Option<(usize, &VariantValue)> {
if !experiment.is_active() {
return None;
}
let idx = (experiment.last_variant_idx + 1) % experiment.variant_count();
experiment.last_variant_idx = idx;
Some((idx, &experiment.variants[idx]))
}
pub fn record_trial(
experiment: &mut Experiment,
variant_idx: usize,
outcome: TrialOutcome,
now: f64,
) -> bool {
if variant_idx >= experiment.variant_count() {
return false;
}
match outcome {
TrialOutcome::Positive => {
experiment.variant_results[variant_idx].record_positive();
experiment.consecutive_negatives[variant_idx] = 0;
}
TrialOutcome::Negative => {
experiment.variant_results[variant_idx].record_negative();
experiment.consecutive_negatives[variant_idx] += 1;
}
TrialOutcome::Neutral => {
experiment.variant_results[variant_idx].record_neutral();
}
}
if !check_safety_bounds(experiment, now) {
experiment.status = ExperimentStatus::Aborted;
experiment.ended_at = Some(now);
return false;
}
if experiment.is_sample_complete() {
return false; }
true
}
fn check_safety_bounds(experiment: &Experiment, now: f64) -> bool {
for bound in &experiment.safety_bounds {
match bound {
SafetyBound::MaxConsecutiveRejections(max) => {
if experiment
.consecutive_negatives
.iter()
.any(|&c| c >= *max)
{
return false;
}
}
SafetyBound::MaxDuration(max_secs) => {
if let Some(started) = experiment.started_at {
if now - started > *max_secs {
return false;
}
}
}
SafetyBound::MinAcceptanceRate(min_rate) => {
for result in &experiment.variant_results {
if result.trials >= 5 && result.acceptance_rate() < *min_rate {
return false;
}
}
}
}
}
true
}
pub fn conclude_experiment(experiment: &mut Experiment, now: f64) -> Option<(usize, VariantValue)> {
if experiment.status != ExperimentStatus::Running {
return None;
}
let winner_idx = experiment
.variant_results
.iter()
.enumerate()
.max_by(|(_, a), (_, b)| a.mean().partial_cmp(&b.mean()).unwrap())
.map(|(i, _)| i)?;
experiment.winner = Some(winner_idx);
experiment.status = ExperimentStatus::Concluded;
experiment.ended_at = Some(now);
Some((winner_idx, experiment.variants[winner_idx].clone()))
}
pub fn check_experiments(registry: &mut ExperimentRegistry, now: f64) -> Vec<ExperimentId> {
let mut concluded_ids = Vec::new();
let active_ids: Vec<ExperimentId> = registry
.experiments
.iter()
.filter(|e| e.is_active())
.map(|e| e.id)
.collect();
for id in active_ids {
let experiment = match registry.find_mut(id) {
Some(e) => e,
None => continue,
};
if !check_safety_bounds(experiment, now) {
experiment.status = ExperimentStatus::Aborted;
experiment.ended_at = Some(now);
registry.total_aborted += 1;
concluded_ids.push(id);
continue;
}
if experiment.is_sample_complete() {
conclude_experiment(experiment, now);
registry.total_concluded += 1;
concluded_ids.push(id);
}
}
concluded_ids
}
pub fn variant_superiority(a: &BetaPosterior, b: &BetaPosterior) -> f64 {
let mean_a = a.mean();
let mean_b = b.mean();
let var_a = a.variance();
let var_b = b.variance();
let combined_std = (var_a + var_b).sqrt();
if combined_std < 1e-10 {
return if mean_a > mean_b { 1.0 } else { 0.0 };
}
let z = (mean_a - mean_b) / combined_std;
1.0 / (1.0 + (-1.7 * z).exp())
}
#[cfg(test)]
mod tests {
use super::*;
fn ts(offset: f64) -> f64 {
86400.0 * 100.0 + offset
}
fn default_bounds() -> Vec<SafetyBound> {
vec![
SafetyBound::MaxConsecutiveRejections(5),
SafetyBound::MaxDuration(86400.0), SafetyBound::MinAcceptanceRate(0.1),
]
}
#[test]
fn test_beta_posterior_new() {
let beta = BetaPosterior::new();
assert_eq!(beta.alpha, 1.0);
assert_eq!(beta.beta, 1.0);
assert!((beta.mean() - 0.5).abs() < 0.01);
}
#[test]
fn test_beta_posterior_updates() {
let mut beta = BetaPosterior::new();
for _ in 0..8 {
beta.record_positive();
}
for _ in 0..2 {
beta.record_negative();
}
assert_eq!(beta.trials, 10);
assert!((beta.mean() - 0.75).abs() < 0.01);
}
#[test]
fn test_beta_credible_interval() {
let mut small = BetaPosterior::new();
small.record_positive();
let mut large = BetaPosterior::new();
for _ in 0..100 {
large.record_positive();
}
assert!(
large.credible_interval_width() < small.credible_interval_width(),
"More data → narrower CI",
);
}
#[test]
fn test_design_experiment() {
let mut registry = ExperimentRegistry::new();
let id = design_experiment(
&mut registry,
"30min lead > 15min lead".to_string(),
ExperimentVariable::ReminderLeadMinutes,
vec![VariantValue::Number(15.0), VariantValue::Number(30.0)],
10,
default_bounds(),
ts(0.0),
);
assert!(id.is_some());
assert_eq!(registry.active_count(), 1);
}
#[test]
fn test_design_rejects_unsafe() {
let mut registry = ExperimentRegistry::new();
let id = design_experiment(
&mut registry,
"Test custom".to_string(),
ExperimentVariable::Custom("dangerous".to_string()),
vec![VariantValue::Label("a".to_string()), VariantValue::Label("b".to_string())],
10,
default_bounds(),
ts(0.0),
);
assert!(id.is_none());
}
#[test]
fn test_design_rejects_duplicate() {
let mut registry = ExperimentRegistry::new();
design_experiment(
&mut registry,
"First".to_string(),
ExperimentVariable::SurfacingMode,
vec![VariantValue::Label("whisper".to_string()), VariantValue::Label("nudge".to_string())],
10,
default_bounds(),
ts(0.0),
);
let id2 = design_experiment(
&mut registry,
"Duplicate".to_string(),
ExperimentVariable::SurfacingMode,
vec![VariantValue::Label("a".to_string()), VariantValue::Label("b".to_string())],
10,
default_bounds(),
ts(1.0),
);
assert!(id2.is_none());
}
#[test]
fn test_max_concurrent() {
let mut registry = ExperimentRegistry::new();
registry.max_concurrent = 2;
for i in 0..2 {
design_experiment(
&mut registry,
format!("Exp {}", i),
if i == 0 {
ExperimentVariable::ReminderLeadMinutes
} else {
ExperimentVariable::SurfacingFrequency
},
vec![VariantValue::Number(1.0), VariantValue::Number(2.0)],
10,
default_bounds(),
ts(i as f64),
);
}
let id3 = design_experiment(
&mut registry,
"Third".to_string(),
ExperimentVariable::InformationDensity,
vec![VariantValue::Label("brief".to_string()), VariantValue::Label("detailed".to_string())],
10,
default_bounds(),
ts(2.0),
);
assert!(id3.is_none(), "Should reject 3rd experiment (max 2)");
}
#[test]
fn test_round_robin_assignment() {
let mut registry = ExperimentRegistry::new();
let id = design_experiment(
&mut registry,
"Test".to_string(),
ExperimentVariable::ReminderLeadMinutes,
vec![
VariantValue::Number(15.0),
VariantValue::Number(30.0),
VariantValue::Number(45.0),
],
5,
default_bounds(),
ts(0.0),
)
.unwrap();
let exp = registry.find_mut(id).unwrap();
let (idx0, _) = assign_variant_round_robin(exp).unwrap();
let (idx1, _) = assign_variant_round_robin(exp).unwrap();
let (idx2, _) = assign_variant_round_robin(exp).unwrap();
let (idx3, _) = assign_variant_round_robin(exp).unwrap();
assert_eq!(idx0, 1); assert_eq!(idx1, 2);
assert_eq!(idx2, 0);
assert_eq!(idx3, 1);
}
#[test]
fn test_record_trial() {
let mut registry = ExperimentRegistry::new();
let id = design_experiment(
&mut registry,
"Test".to_string(),
ExperimentVariable::SurfacingMode,
vec![VariantValue::Label("whisper".to_string()), VariantValue::Label("nudge".to_string())],
5,
default_bounds(),
ts(0.0),
)
.unwrap();
let exp = registry.find_mut(id).unwrap();
assert!(record_trial(exp, 0, TrialOutcome::Positive, ts(1.0)));
assert_eq!(exp.variant_results[0].trials, 1);
assert!((exp.variant_results[0].mean() - 0.667).abs() < 0.01);
}
#[test]
fn test_safety_max_rejections() {
let mut registry = ExperimentRegistry::new();
let id = design_experiment(
&mut registry,
"Test".to_string(),
ExperimentVariable::SurfacingFrequency,
vec![VariantValue::Number(1.0), VariantValue::Number(2.0)],
100,
vec![SafetyBound::MaxConsecutiveRejections(3)],
ts(0.0),
)
.unwrap();
let exp = registry.find_mut(id).unwrap();
assert!(record_trial(exp, 0, TrialOutcome::Negative, ts(1.0)));
assert!(record_trial(exp, 0, TrialOutcome::Negative, ts(2.0)));
assert!(!record_trial(exp, 0, TrialOutcome::Negative, ts(3.0)));
assert_eq!(exp.status, ExperimentStatus::Aborted);
}
#[test]
fn test_safety_max_duration() {
let mut registry = ExperimentRegistry::new();
let id = design_experiment(
&mut registry,
"Test".to_string(),
ExperimentVariable::InformationDensity,
vec![VariantValue::Label("brief".to_string()), VariantValue::Label("detailed".to_string())],
100,
vec![SafetyBound::MaxDuration(60.0)], ts(0.0),
)
.unwrap();
let exp = registry.find_mut(id).unwrap();
assert!(record_trial(exp, 0, TrialOutcome::Positive, ts(30.0)));
assert!(!record_trial(exp, 0, TrialOutcome::Positive, ts(120.0)));
assert_eq!(exp.status, ExperimentStatus::Aborted);
}
#[test]
fn test_conclude_experiment() {
let mut registry = ExperimentRegistry::new();
let id = design_experiment(
&mut registry,
"Test lead time".to_string(),
ExperimentVariable::ReminderLeadMinutes,
vec![VariantValue::Number(15.0), VariantValue::Number(30.0)],
3,
default_bounds(),
ts(0.0),
)
.unwrap();
let exp = registry.find_mut(id).unwrap();
record_trial(exp, 0, TrialOutcome::Positive, ts(1.0));
record_trial(exp, 0, TrialOutcome::Negative, ts(2.0));
record_trial(exp, 0, TrialOutcome::Negative, ts(3.0));
record_trial(exp, 1, TrialOutcome::Positive, ts(4.0));
record_trial(exp, 1, TrialOutcome::Positive, ts(5.0));
record_trial(exp, 1, TrialOutcome::Positive, ts(6.0));
let result = conclude_experiment(exp, ts(7.0));
assert!(result.is_some());
let (winner_idx, winner_value) = result.unwrap();
assert_eq!(winner_idx, 1);
assert_eq!(winner_value, VariantValue::Number(30.0));
assert_eq!(exp.status, ExperimentStatus::Concluded);
}
#[test]
fn test_variant_superiority() {
let mut a = BetaPosterior::new();
let mut b = BetaPosterior::new();
for _ in 0..80 {
a.record_positive();
}
for _ in 0..20 {
a.record_negative();
}
for _ in 0..40 {
b.record_positive();
}
for _ in 0..60 {
b.record_negative();
}
let prob = variant_superiority(&a, &b);
assert!(
prob > 0.95,
"P(A > B) = {:.3} should be > 0.95",
prob,
);
}
#[test]
fn test_check_experiments_auto_conclude() {
let mut registry = ExperimentRegistry::new();
let id = design_experiment(
&mut registry,
"Auto conclude".to_string(),
ExperimentVariable::SurfacingTimeOfDay,
vec![VariantValue::Number(9.0), VariantValue::Number(14.0)],
2,
default_bounds(),
ts(0.0),
)
.unwrap();
let exp = registry.find_mut(id).unwrap();
record_trial(exp, 0, TrialOutcome::Positive, ts(1.0));
record_trial(exp, 0, TrialOutcome::Positive, ts(2.0));
record_trial(exp, 1, TrialOutcome::Positive, ts(3.0));
record_trial(exp, 1, TrialOutcome::Negative, ts(4.0));
let concluded = check_experiments(&mut registry, ts(5.0));
assert_eq!(concluded.len(), 1);
assert_eq!(concluded[0], id);
let exp = registry.find(id).unwrap();
assert_eq!(exp.status, ExperimentStatus::Concluded);
assert_eq!(exp.winner, Some(0)); }
}