kaizen/experiment/
types.rs

1// SPDX-License-Identifier: AGPL-3.0-or-later
2//! Pure data for experiments. See `docs/experiments.md`.
3
4use serde::{Deserialize, Serialize};
5
6/// Variant a session falls into under a binding.
7#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
8pub enum Classification {
9    Control,
10    Treatment,
11    Excluded,
12}
13
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
15pub enum Metric {
16    TokensPerSession,
17    CostPerSession,
18    SuccessRate,
19    ToolLoops,
20    DurationMinutes,
21    FilesPerSession,
22    SuccessRateByPrompt,
23    CostByPrompt,
24}
25
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
27pub enum Binding {
28    GitCommit {
29        control_commit: String,
30        treatment_commit: String,
31    },
32    Branch {
33        control_branch: String,
34        treatment_branch: String,
35    },
36    ManualTag {
37        variant_field: String,
38    },
39}
40
41#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
42pub enum Direction {
43    Decrease,
44    Increase,
45}
46
47#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
48pub enum Criterion {
49    Delta {
50        direction: Direction,
51        target_pct: f64,
52    },
53    Absolute {
54        metric_value: f64,
55    },
56}
57
58/// Lifecycle state. `Archived` is terminal.
59#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
60pub enum State {
61    Draft,
62    Running,
63    Concluded,
64    Archived,
65}
66
67#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
68pub struct Experiment {
69    pub id: String,
70    pub name: String,
71    pub hypothesis: String,
72    pub change_description: String,
73    pub metric: Metric,
74    pub binding: Binding,
75    pub duration_days: u32,
76    pub success_criterion: Criterion,
77    pub state: State,
78    pub created_at_ms: u64,
79    pub concluded_at_ms: Option<u64>,
80}
81
82impl Metric {
83    pub fn as_str(&self) -> &'static str {
84        match self {
85            Metric::TokensPerSession => "tokens_per_session",
86            Metric::CostPerSession => "cost_per_session",
87            Metric::SuccessRate => "success_rate",
88            Metric::ToolLoops => "tool_loops",
89            Metric::DurationMinutes => "duration_minutes",
90            Metric::FilesPerSession => "files_per_session",
91            Metric::SuccessRateByPrompt => "success_rate_by_prompt",
92            Metric::CostByPrompt => "cost_by_prompt",
93        }
94    }
95
96    pub fn parse(s: &str) -> Option<Metric> {
97        Some(match s {
98            "tokens_per_session" => Metric::TokensPerSession,
99            "cost_per_session" => Metric::CostPerSession,
100            "success_rate" => Metric::SuccessRate,
101            "tool_loops" => Metric::ToolLoops,
102            "duration_minutes" => Metric::DurationMinutes,
103            "files_per_session" => Metric::FilesPerSession,
104            "success_rate_by_prompt" => Metric::SuccessRateByPrompt,
105            "cost_by_prompt" => Metric::CostByPrompt,
106            _ => return None,
107        })
108    }
109}
110
111/// Pure state-machine transition. Returns `Some(next)` when `action` is enabled.
112pub fn transition(state: State, action: &str) -> Option<State> {
113    Some(match (state, action) {
114        (State::Draft, "start") => State::Running,
115        (State::Running, "conclude") => State::Concluded,
116        (State::Concluded, "archive") => State::Archived,
117        _ => return None,
118    })
119}
120
121#[cfg(test)]
122mod tests {
123    use super::*;
124
125    #[test]
126    fn transitions_follow_spec_order() {
127        assert_eq!(transition(State::Draft, "start"), Some(State::Running));
128        assert_eq!(
129            transition(State::Running, "conclude"),
130            Some(State::Concluded)
131        );
132        assert_eq!(
133            transition(State::Concluded, "archive"),
134            Some(State::Archived)
135        );
136    }
137
138    #[test]
139    fn archived_is_terminal() {
140        assert_eq!(transition(State::Archived, "start"), None);
141        assert_eq!(transition(State::Archived, "conclude"), None);
142        assert_eq!(transition(State::Archived, "archive"), None);
143    }
144
145    #[test]
146    fn no_backward_transitions() {
147        assert_eq!(transition(State::Concluded, "start"), None);
148        assert_eq!(transition(State::Running, "archive"), None);
149    }
150
151    #[test]
152    fn metric_round_trip() {
153        for m in [
154            Metric::TokensPerSession,
155            Metric::CostPerSession,
156            Metric::SuccessRate,
157            Metric::ToolLoops,
158            Metric::DurationMinutes,
159            Metric::FilesPerSession,
160        ] {
161            assert_eq!(Metric::parse(m.as_str()), Some(m));
162        }
163    }
164}
kaizen/experiment/types.rs

kaizen/experiment/
types.rs
