scientific_method/
criterion.rs

1// Copyright © 2026 James Hunt. All rights reserved.
2
3use crate::{ObservationRef, Observations, Scalar};
4use std::fmt;
5
6/// Numeric direction for an improvement or regression criterion.
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
9#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))]
10pub enum Direction {
11    HigherIsBetter,
12    LowerIsBetter,
13}
14
15/// A comparison operator.
16#[derive(Debug, Clone, Copy, PartialEq, Eq)]
17#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
18#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))]
19pub enum Comparator {
20    Equal,
21    NotEqual,
22    GreaterThan,
23    GreaterThanOrEqual,
24    LessThan,
25    LessThanOrEqual,
26}
27
28/// A literal value or a reference to an observed measurement.
29#[derive(Debug, Clone, PartialEq)]
30#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
31#[cfg_attr(feature = "serde", serde(tag = "kind", rename_all = "snake_case"))]
32pub enum Operand {
33    Literal { value: Scalar },
34    Observation { reference: ObservationRef },
35}
36
37impl Operand {
38    pub fn literal(value: impl Into<Scalar>) -> Self {
39        Self::Literal {
40            value: value.into(),
41        }
42    }
43
44    pub fn observation(condition: impl Into<String>, measurement: impl Into<String>) -> Self {
45        Self::Observation {
46            reference: ObservationRef::new(condition, measurement),
47        }
48    }
49}
50
51/// A machine-evaluable acceptance rule.
52#[derive(Debug, Clone, PartialEq)]
53#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
54#[cfg_attr(feature = "serde", serde(tag = "kind", rename_all = "snake_case"))]
55pub enum Criterion {
56    Compare {
57        left: Operand,
58        comparator: Comparator,
59        right: Operand,
60    },
61    Improvement {
62        baseline: ObservationRef,
63        candidate: ObservationRef,
64        direction: Direction,
65        minimum_fraction: f64,
66    },
67    RegressionLimit {
68        baseline: ObservationRef,
69        candidate: ObservationRef,
70        direction: Direction,
71        maximum_fraction: f64,
72    },
73    All {
74        criteria: Vec<Criterion>,
75    },
76    Any {
77        criteria: Vec<Criterion>,
78    },
79    Not {
80        criterion: Box<Criterion>,
81    },
82}
83
84impl Criterion {
85    pub fn compare(left: Operand, comparator: Comparator, right: Operand) -> Self {
86        Self::Compare {
87            left,
88            comparator,
89            right,
90        }
91    }
92
93    pub fn improvement(
94        baseline: ObservationRef,
95        candidate: ObservationRef,
96        direction: Direction,
97        minimum_fraction: f64,
98    ) -> Self {
99        Self::Improvement {
100            baseline,
101            candidate,
102            direction,
103            minimum_fraction,
104        }
105    }
106
107    pub fn regression_limit(
108        baseline: ObservationRef,
109        candidate: ObservationRef,
110        direction: Direction,
111        maximum_fraction: f64,
112    ) -> Self {
113        Self::RegressionLimit {
114            baseline,
115            candidate,
116            direction,
117            maximum_fraction,
118        }
119    }
120
121    pub fn all(criteria: impl IntoIterator<Item = Criterion>) -> Self {
122        Self::All {
123            criteria: criteria.into_iter().collect(),
124        }
125    }
126
127    pub fn any(criteria: impl IntoIterator<Item = Criterion>) -> Self {
128        Self::Any {
129            criteria: criteria.into_iter().collect(),
130        }
131    }
132
133    pub fn negate(criterion: Criterion) -> Self {
134        Self::Not {
135            criterion: Box::new(criterion),
136        }
137    }
138
139    pub fn evaluate(&self, observations: &Observations) -> Result<CriterionResult, CriterionError> {
140        match self {
141            Self::Compare {
142                left,
143                comparator,
144                right,
145            } => {
146                let left = resolve(left, observations)?;
147                let right = resolve(right, observations)?;
148                let passed = compare(left, *comparator, right)?;
149                Ok(CriterionResult::leaf(
150                    passed,
151                    format!("comparison {:?}", comparator),
152                ))
153            }
154            Self::Improvement {
155                baseline,
156                candidate,
157                direction,
158                minimum_fraction,
159            } => {
160                require_fraction("minimum_fraction", *minimum_fraction)?;
161                let (before, after) = numeric_pair(baseline, candidate, observations)?;
162                let fraction = improvement_fraction(before, after, *direction)?;
163                Ok(CriterionResult::leaf(
164                    fraction >= *minimum_fraction,
165                    format!("improvement {fraction:.6} >= {minimum_fraction:.6}"),
166                ))
167            }
168            Self::RegressionLimit {
169                baseline,
170                candidate,
171                direction,
172                maximum_fraction,
173            } => {
174                require_fraction("maximum_fraction", *maximum_fraction)?;
175                let (before, after) = numeric_pair(baseline, candidate, observations)?;
176                let improvement = improvement_fraction(before, after, *direction)?;
177                let regression = (-improvement).max(0.0);
178                Ok(CriterionResult::leaf(
179                    regression <= *maximum_fraction,
180                    format!("regression {regression:.6} <= {maximum_fraction:.6}"),
181                ))
182            }
183            Self::All { criteria } => {
184                if criteria.is_empty() {
185                    return Err(CriterionError::EmptyComposite("all"));
186                }
187                let children = criteria
188                    .iter()
189                    .map(|c| c.evaluate(observations))
190                    .collect::<Result<Vec<_>, _>>()?;
191                Ok(CriterionResult::branch(
192                    children.iter().all(|r| r.passed),
193                    "all",
194                    children,
195                ))
196            }
197            Self::Any { criteria } => {
198                if criteria.is_empty() {
199                    return Err(CriterionError::EmptyComposite("any"));
200                }
201                let children = criteria
202                    .iter()
203                    .map(|c| c.evaluate(observations))
204                    .collect::<Result<Vec<_>, _>>()?;
205                Ok(CriterionResult::branch(
206                    children.iter().any(|r| r.passed),
207                    "any",
208                    children,
209                ))
210            }
211            Self::Not { criterion } => {
212                let child = criterion.evaluate(observations)?;
213                Ok(CriterionResult::branch(!child.passed, "not", vec![child]))
214            }
215        }
216    }
217}
218
219/// The structured result of evaluating a criterion.
220#[derive(Debug, Clone, PartialEq, Eq)]
221#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
222pub struct CriterionResult {
223    pub passed: bool,
224    pub detail: String,
225    pub children: Vec<CriterionResult>,
226}
227
228impl CriterionResult {
229    fn leaf(passed: bool, detail: impl Into<String>) -> Self {
230        Self {
231            passed,
232            detail: detail.into(),
233            children: Vec::new(),
234        }
235    }
236
237    fn branch(passed: bool, detail: impl Into<String>, children: Vec<CriterionResult>) -> Self {
238        Self {
239            passed,
240            detail: detail.into(),
241            children,
242        }
243    }
244}
245
246/// Why a criterion could not be evaluated.
247#[derive(Debug, Clone, PartialEq)]
248pub enum CriterionError {
249    MissingObservation(ObservationRef),
250    NonNumericObservation(ObservationRef),
251    NonFiniteNumber,
252    UndefinedRelativeChange,
253    InvalidFraction { name: &'static str, value: f64 },
254    IncompatibleOperands,
255    UnsupportedOrdering,
256    EmptyComposite(&'static str),
257}
258
259impl fmt::Display for CriterionError {
260    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
261        match self {
262            Self::MissingObservation(r) => {
263                write!(f, "missing observation {}:{}", r.condition, r.measurement)
264            }
265            Self::NonNumericObservation(r) => write!(
266                f,
267                "observation {}:{} is not numeric",
268                r.condition, r.measurement
269            ),
270            Self::NonFiniteNumber => write!(f, "a numeric value is not finite"),
271            Self::UndefinedRelativeChange => {
272                write!(f, "relative change is undefined for a zero baseline")
273            }
274            Self::InvalidFraction { name, value } => {
275                write!(f, "{name} must be finite and non-negative, got {value}")
276            }
277            Self::IncompatibleOperands => write!(f, "the operands have incompatible scalar types"),
278            Self::UnsupportedOrdering => write!(f, "ordering comparisons require numeric operands"),
279            Self::EmptyComposite(kind) => {
280                write!(f, "{kind} criterion must contain at least one criterion")
281            }
282        }
283    }
284}
285
286impl std::error::Error for CriterionError {}
287
288fn require_fraction(name: &'static str, value: f64) -> Result<(), CriterionError> {
289    if value.is_finite() && value >= 0.0 {
290        Ok(())
291    } else {
292        Err(CriterionError::InvalidFraction { name, value })
293    }
294}
295
296fn resolve<'a>(
297    operand: &'a Operand,
298    observations: &'a Observations,
299) -> Result<&'a Scalar, CriterionError> {
300    match operand {
301        Operand::Literal { value } => Ok(value),
302        Operand::Observation { reference } => observations
303            .get(reference)
304            .map(|m| &m.value)
305            .ok_or_else(|| CriterionError::MissingObservation(reference.clone())),
306    }
307}
308
309fn numeric_pair(
310    baseline: &ObservationRef,
311    candidate: &ObservationRef,
312    observations: &Observations,
313) -> Result<(f64, f64), CriterionError> {
314    let a = observations
315        .get(baseline)
316        .ok_or_else(|| CriterionError::MissingObservation(baseline.clone()))?;
317    let b = observations
318        .get(candidate)
319        .ok_or_else(|| CriterionError::MissingObservation(candidate.clone()))?;
320    let before = match &a.value {
321        Scalar::Number(v) if v.is_finite() => *v,
322        Scalar::Number(_) => return Err(CriterionError::NonFiniteNumber),
323        _ => return Err(CriterionError::NonNumericObservation(baseline.clone())),
324    };
325    let after = match &b.value {
326        Scalar::Number(v) if v.is_finite() => *v,
327        Scalar::Number(_) => return Err(CriterionError::NonFiniteNumber),
328        _ => return Err(CriterionError::NonNumericObservation(candidate.clone())),
329    };
330    Ok((before, after))
331}
332
333fn compare(left: &Scalar, comparator: Comparator, right: &Scalar) -> Result<bool, CriterionError> {
334    use Comparator::*;
335    match comparator {
336        Equal | NotEqual => {
337            let equal = match (left, right) {
338                (Scalar::Number(a), Scalar::Number(b)) => {
339                    if !a.is_finite() || !b.is_finite() {
340                        return Err(CriterionError::NonFiniteNumber);
341                    }
342                    a == b
343                }
344                (Scalar::Boolean(a), Scalar::Boolean(b)) => a == b,
345                (Scalar::Text(a), Scalar::Text(b)) => a == b,
346                _ => return Err(CriterionError::IncompatibleOperands),
347            };
348            Ok(if comparator == Equal { equal } else { !equal })
349        }
350        GreaterThan | GreaterThanOrEqual | LessThan | LessThanOrEqual => {
351            let (a, b) = match (left, right) {
352                (Scalar::Number(a), Scalar::Number(b)) => {
353                    if !a.is_finite() || !b.is_finite() {
354                        return Err(CriterionError::NonFiniteNumber);
355                    }
356                    (*a, *b)
357                }
358                (Scalar::Number(_), _) | (_, Scalar::Number(_)) => {
359                    return Err(CriterionError::IncompatibleOperands);
360                }
361                _ => return Err(CriterionError::UnsupportedOrdering),
362            };
363            Ok(match comparator {
364                GreaterThan => a > b,
365                GreaterThanOrEqual => a >= b,
366                LessThan => a < b,
367                LessThanOrEqual => a <= b,
368                Equal | NotEqual => unreachable!(),
369            })
370        }
371    }
372}
373
374fn improvement_fraction(
375    before: f64,
376    after: f64,
377    direction: Direction,
378) -> Result<f64, CriterionError> {
379    if before == 0.0 {
380        return Err(CriterionError::UndefinedRelativeChange);
381    }
382    let magnitude = before.abs();
383    Ok(match direction {
384        Direction::HigherIsBetter => (after - before) / magnitude,
385        Direction::LowerIsBetter => (before - after) / magnitude,
386    })
387}
388
389#[cfg(test)]
390mod tests {
391    use super::*;
392    use crate::{Measurement, Observation};
393
394    fn observations(before: f64, after: f64) -> Observations {
395        Observations::new()
396            .with(Observation::new(
397                "baseline",
398                Measurement::new("latency", before),
399            ))
400            .with(Observation::new(
401                "candidate",
402                Measurement::new("latency", after),
403            ))
404    }
405
406    #[test]
407    fn lower_is_better_improvement_passes() {
408        let c = Criterion::improvement(
409            ObservationRef::new("baseline", "latency"),
410            ObservationRef::new("candidate", "latency"),
411            Direction::LowerIsBetter,
412            0.05,
413        );
414        assert!(c.evaluate(&observations(100.0, 90.0)).unwrap().passed);
415    }
416
417    #[test]
418    fn lower_is_better_improvement_fails() {
419        let c = Criterion::improvement(
420            ObservationRef::new("baseline", "latency"),
421            ObservationRef::new("candidate", "latency"),
422            Direction::LowerIsBetter,
423            0.05,
424        );
425        assert!(!c.evaluate(&observations(100.0, 97.0)).unwrap().passed);
426    }
427
428    #[test]
429    fn regression_limit_passes_on_improvement() {
430        let c = Criterion::regression_limit(
431            ObservationRef::new("baseline", "latency"),
432            ObservationRef::new("candidate", "latency"),
433            Direction::LowerIsBetter,
434            0.02,
435        );
436        assert!(c.evaluate(&observations(100.0, 90.0)).unwrap().passed);
437    }
438
439    #[test]
440    fn regression_limit_catches_regression() {
441        let c = Criterion::regression_limit(
442            ObservationRef::new("baseline", "latency"),
443            ObservationRef::new("candidate", "latency"),
444            Direction::LowerIsBetter,
445            0.02,
446        );
447        assert!(!c.evaluate(&observations(100.0, 103.0)).unwrap().passed);
448    }
449
450    #[test]
451    fn all_composes() {
452        let o = observations(100.0, 90.0).with(Observation::new(
453            "candidate",
454            Measurement::new("tests_pass", true),
455        ));
456        let c = Criterion::all([
457            Criterion::improvement(
458                ObservationRef::new("baseline", "latency"),
459                ObservationRef::new("candidate", "latency"),
460                Direction::LowerIsBetter,
461                0.05,
462            ),
463            Criterion::compare(
464                Operand::observation("candidate", "tests_pass"),
465                Comparator::Equal,
466                Operand::literal(true),
467            ),
468        ]);
469        assert!(c.evaluate(&o).unwrap().passed);
470    }
471
472    #[test]
473    fn missing_observation_is_error() {
474        let c = Criterion::compare(
475            Operand::observation("candidate", "missing"),
476            Comparator::Equal,
477            Operand::literal(true),
478        );
479        assert!(matches!(
480            c.evaluate(&Observations::new()),
481            Err(CriterionError::MissingObservation(_))
482        ));
483    }
484
485    #[test]
486    fn zero_baseline_refuses_relative_change() {
487        let c = Criterion::improvement(
488            ObservationRef::new("baseline", "latency"),
489            ObservationRef::new("candidate", "latency"),
490            Direction::LowerIsBetter,
491            0.05,
492        );
493        assert_eq!(
494            c.evaluate(&observations(0.0, 1.0)).unwrap_err(),
495            CriterionError::UndefinedRelativeChange
496        );
497    }
498}
scientific_method/criterion.rs

scientific_method/
criterion.rs
