Skip to main content

fetchium_core/evolve/
ab_test.rs

1//! A/B testing for ranking variants (PRD §39).
2//!
3//! Uses a deterministic hash-based assignment so the same user (identified
4//! by a stable session ID) always gets the same variant.
5
6use crate::error::{FetchiumError, FetchiumResult};
7use serde::{Deserialize, Serialize};
8use sha2::{Digest, Sha256};
9
10/// An A/B test definition.
11#[derive(Debug, Clone, Serialize, Deserialize)]
12pub struct AbTest {
13    pub name: String,
14    pub description: String,
15    /// Traffic split: 0.0–1.0 fraction going to `variant_b`.
16    pub split_ratio: f64,
17    pub variant_a: String,
18    pub variant_b: String,
19    pub is_active: bool,
20    pub created_at: String,
21}
22
23/// The variant assigned to a session.
24#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
25pub enum Variant {
26    A,
27    B,
28}
29
30impl Variant {
31    pub fn label(&self) -> &str {
32        match self {
33            Variant::A => "A",
34            Variant::B => "B",
35        }
36    }
37}
38
39impl AbTest {
40    pub fn new(
41        name: &str,
42        description: &str,
43        variant_a: &str,
44        variant_b: &str,
45        split_ratio: f64,
46    ) -> Self {
47        Self {
48            name: name.to_string(),
49            description: description.to_string(),
50            split_ratio: split_ratio.clamp(0.0, 1.0),
51            variant_a: variant_a.to_string(),
52            variant_b: variant_b.to_string(),
53            is_active: true,
54            created_at: chrono::Utc::now().to_rfc3339(),
55        }
56    }
57
58    /// Deterministically assign a variant to a session ID.
59    ///
60    /// Uses SHA-256(session_id + test_name) to produce a stable hash,
61    /// then maps it to [0, 1) and compares against `split_ratio`.
62    pub fn assign(&self, session_id: &str) -> Variant {
63        let mut hasher = Sha256::new();
64        hasher.update(session_id.as_bytes());
65        hasher.update(b":");
66        hasher.update(self.name.as_bytes());
67        let digest = hasher.finalize();
68        // Take first 4 bytes as u32 and map to [0, 1).
69        let bucket = u32::from_be_bytes([digest[0], digest[1], digest[2], digest[3]]);
70        let fraction = bucket as f64 / u32::MAX as f64;
71        if fraction < self.split_ratio {
72            Variant::B
73        } else {
74            Variant::A
75        }
76    }
77
78    /// Return the variant label name (e.g., "control" or "semantic-boost").
79    pub fn variant_name(&self, variant: &Variant) -> &str {
80        match variant {
81            Variant::A => &self.variant_a,
82            Variant::B => &self.variant_b,
83        }
84    }
85}
86
87/// A result observed during an A/B test run.
88#[derive(Debug, Clone, Serialize, Deserialize)]
89pub struct AbResult {
90    pub test_name: String,
91    pub session_id: String,
92    pub variant: Variant,
93    pub metric: String,
94    pub value: f64,
95    pub recorded_at: String,
96}
97
98impl AbResult {
99    pub fn new(
100        test_name: &str,
101        session_id: &str,
102        variant: Variant,
103        metric: &str,
104        value: f64,
105    ) -> Self {
106        Self {
107            test_name: test_name.to_string(),
108            session_id: session_id.to_string(),
109            variant,
110            metric: metric.to_string(),
111            value,
112            recorded_at: chrono::Utc::now().to_rfc3339(),
113        }
114    }
115}
116
117/// Aggregate statistics for one variant of a test.
118#[derive(Debug, Clone, Serialize, Deserialize)]
119pub struct VariantStats {
120    pub variant: Variant,
121    pub n: usize,
122    pub mean: f64,
123    pub std_dev: f64,
124}
125
126/// Compute per-variant statistics for a set of results.
127pub fn compute_stats(
128    results: &[AbResult],
129    metric: &str,
130) -> FetchiumResult<(VariantStats, VariantStats)> {
131    let values_a: Vec<f64> = results
132        .iter()
133        .filter(|r| r.metric == metric && r.variant == Variant::A)
134        .map(|r| r.value)
135        .collect();
136    let values_b: Vec<f64> = results
137        .iter()
138        .filter(|r| r.metric == metric && r.variant == Variant::B)
139        .map(|r| r.value)
140        .collect();
141
142    if values_a.is_empty() || values_b.is_empty() {
143        return Err(FetchiumError::Config(format!(
144            "insufficient data for metric '{metric}' in A/B analysis"
145        )));
146    }
147
148    Ok((
149        stats_for(Variant::A, &values_a),
150        stats_for(Variant::B, &values_b),
151    ))
152}
153
154fn stats_for(variant: Variant, values: &[f64]) -> VariantStats {
155    let n = values.len();
156    let mean = values.iter().sum::<f64>() / n as f64;
157    let variance = values.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / n as f64;
158    VariantStats {
159        variant,
160        n,
161        mean,
162        std_dev: variance.sqrt(),
163    }
164}
165
166#[cfg(test)]
167mod tests {
168    use super::*;
169
170    #[test]
171    fn test_deterministic_assignment() {
172        let test = AbTest::new("ranking-v2", "test", "control", "semantic-boost", 0.5);
173        let v1 = test.assign("session-abc");
174        let v2 = test.assign("session-abc");
175        assert_eq!(v1, v2, "assignment must be deterministic");
176    }
177
178    #[test]
179    fn test_split_ratio_zero_always_a() {
180        let test = AbTest::new("test", "desc", "A", "B", 0.0);
181        for i in 0..20 {
182            assert_eq!(test.assign(&format!("session-{i}")), Variant::A);
183        }
184    }
185
186    #[test]
187    fn test_split_ratio_one_always_b() {
188        let test = AbTest::new("test", "desc", "A", "B", 1.0);
189        for i in 0..20 {
190            assert_eq!(test.assign(&format!("session-{i}")), Variant::B);
191        }
192    }
193}