cruxx_improve/
comparison.rs

1use crate::metrics::TraceMetrics;
2use cruxx_types::crux_value::Crux;
3use serde::{Deserialize, Serialize};
4
5#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
6#[serde(rename_all = "snake_case")]
7pub enum Verdict {
8    Improved,
9    Regressed,
10    Neutral,
11}
12
13#[derive(Debug, Clone, Serialize, Deserialize)]
14pub struct Comparison {
15    pub verdict: Verdict,
16    pub delta: f32,
17    pub old_metrics: TraceMetrics,
18    pub new_metrics: TraceMetrics,
19}
20
21/// Compare two execution traces and produce a verdict.
22///
23/// This is crux-domain logic: it knows what steps mean, how to weight
24/// success rate vs confidence, and what constitutes a meaningful delta.
25pub fn replay_compare(old: &Crux<serde_json::Value>, new: &Crux<serde_json::Value>) -> Comparison {
26    let old_metrics = TraceMetrics::extract(old);
27    let new_metrics = TraceMetrics::extract(new);
28    let delta = new_metrics.score - old_metrics.score;
29
30    let verdict = if delta > 0.05 {
31        Verdict::Improved
32    } else if delta < -0.05 {
33        Verdict::Regressed
34    } else {
35        Verdict::Neutral
36    };
37
38    Comparison {
39        verdict,
40        delta,
41        old_metrics,
42        new_metrics,
43    }
44}
45
46#[cfg(test)]
47mod tests {
48    use super::*;
49    use crate::metrics::test_helpers::{step, trace};
50    use cruxx_types::step::{StepKind, StepStatus};
51
52    #[test]
53    fn detects_improvement() {
54        let old = trace(
55            vec![step("a", StepStatus::Err, 0.3, StepKind::Plain)],
56            vec![],
57        );
58        let new = trace(
59            vec![step("a", StepStatus::Ok, 0.8, StepKind::Plain)],
60            vec![],
61        );
62        let cmp = replay_compare(&old, &new);
63        assert_eq!(cmp.verdict, Verdict::Improved);
64        assert!(cmp.delta > 0.0);
65    }
66
67    #[test]
68    fn detects_regression() {
69        let old = trace(
70            vec![step("a", StepStatus::Ok, 0.9, StepKind::Plain)],
71            vec![],
72        );
73        let new = trace(
74            vec![step("a", StepStatus::Err, 0.2, StepKind::Plain)],
75            vec![],
76        );
77        let cmp = replay_compare(&old, &new);
78        assert_eq!(cmp.verdict, Verdict::Regressed);
79    }
80
81    #[test]
82    fn detects_neutral() {
83        let old = trace(
84            vec![step("a", StepStatus::Ok, 0.7, StepKind::Plain)],
85            vec![],
86        );
87        let new = trace(
88            vec![step("a", StepStatus::Ok, 0.72, StepKind::Plain)],
89            vec![],
90        );
91        let cmp = replay_compare(&old, &new);
92        assert_eq!(cmp.verdict, Verdict::Neutral);
93    }
94
95    #[test]
96    fn comparison_includes_metric_deltas() {
97        let old = trace(
98            vec![
99                step("a", StepStatus::Ok, 0.5, StepKind::Plain),
100                step("b", StepStatus::Err, 0.3, StepKind::Plain),
101            ],
102            vec![],
103        );
104        let new = trace(
105            vec![
106                step("a", StepStatus::Ok, 0.9, StepKind::Plain),
107                step("b", StepStatus::Ok, 0.8, StepKind::Plain),
108            ],
109            vec![],
110        );
111        let cmp = replay_compare(&old, &new);
112        assert!(cmp.new_metrics.success_rate > cmp.old_metrics.success_rate);
113        assert!(cmp.new_metrics.avg_confidence > cmp.old_metrics.avg_confidence);
114    }
115}
cruxx_improve/comparison.rs

cruxx_improve/
comparison.rs
