sofie_cli/bench/
baseline.rs

1use std::collections::BTreeMap;
2
3use serde_json::{Map, Value, json};
4
5const LATENCY_MARGIN: f64 = 0.2;
6const ERROR_RATE_MARGIN: f64 = 0.01;
7
8#[derive(Clone, Copy, Debug, PartialEq)]
9pub struct Measurement {
10    pub p95_ms: f64,
11    pub error_rate: f64,
12}
13
14pub type Measurements = BTreeMap<String, Measurement>;
15
16#[derive(Clone, Debug, PartialEq, Eq)]
17pub enum Verdict {
18    New,
19    Held,
20    Lowered,
21    Regressed(Vec<String>),
22}
23
24#[derive(Clone, Debug, PartialEq)]
25pub struct RouteResult {
26    pub endpoint: String,
27    pub measured: Measurement,
28    pub baseline: Option<Measurement>,
29    pub verdict: Verdict,
30}
31
32#[derive(Clone, Debug, PartialEq)]
33pub struct Comparison {
34    pub routes: Vec<RouteResult>,
35    pub next_baseline: Measurements,
36}
37
38impl Comparison {
39    #[must_use]
40    pub fn regressed(&self) -> bool {
41        self.routes
42            .iter()
43            .any(|route| matches!(route.verdict, Verdict::Regressed(_)))
44    }
45}
46
47/// A ceiling: a regression beyond the margin fails and keeps the old value,
48/// an improvement lowers it, an endpoint no longer measured drops out.
49#[must_use]
50pub fn compare(baseline: &Measurements, measured: &Measurements) -> Comparison {
51    let mut routes = Vec::new();
52    let mut next_baseline = Measurements::new();
53    for (endpoint, &measured) in measured {
54        let previous = baseline.get(endpoint).copied();
55        let (verdict, ceiling) = previous.map_or((Verdict::New, measured), |previous| {
56            judge(previous, measured)
57        });
58        next_baseline.insert(endpoint.clone(), ceiling);
59        routes.push(RouteResult {
60            endpoint: endpoint.clone(),
61            measured,
62            baseline: previous,
63            verdict,
64        });
65    }
66    Comparison {
67        routes,
68        next_baseline,
69    }
70}
71
72fn judge(previous: Measurement, measured: Measurement) -> (Verdict, Measurement) {
73    let mut regressions = Vec::new();
74    if measured.p95_ms > previous.p95_ms * (1.0 + LATENCY_MARGIN) {
75        regressions.push(format!(
76            "p95 {:.2} ms is above {:.2} ms plus {:.0}%",
77            measured.p95_ms,
78            previous.p95_ms,
79            LATENCY_MARGIN * 100.0
80        ));
81    }
82    if measured.error_rate > previous.error_rate + ERROR_RATE_MARGIN {
83        regressions.push(format!(
84            "error rate {:.2}% is above {:.2}% plus {:.0} points",
85            measured.error_rate * 100.0,
86            previous.error_rate * 100.0,
87            ERROR_RATE_MARGIN * 100.0
88        ));
89    }
90    if !regressions.is_empty() {
91        return (Verdict::Regressed(regressions), previous);
92    }
93    let ceiling = Measurement {
94        p95_ms: previous.p95_ms.min(measured.p95_ms),
95        error_rate: previous.error_rate.min(measured.error_rate),
96    };
97    let verdict = if ceiling == previous {
98        Verdict::Held
99    } else {
100        Verdict::Lowered
101    };
102    (verdict, ceiling)
103}
104
105/// The shape of both `bench/baseline.json` and the summary the k6 script
106/// writes: `{"routes": {"<endpoint>": {"p95_ms": …, "error_rate": …}}}`.
107pub fn parse(text: &str) -> Result<Measurements, String> {
108    let document: Value = serde_json::from_str(text).map_err(|error| error.to_string())?;
109    let routes = document
110        .get("routes")
111        .and_then(Value::as_object)
112        .ok_or("no `routes` object")?;
113    routes
114        .iter()
115        .map(|(endpoint, values)| {
116            let number = |key: &str| {
117                values
118                    .get(key)
119                    .and_then(Value::as_f64)
120                    .ok_or_else(|| format!("`{endpoint}` has no number `{key}`"))
121            };
122            let measurement = Measurement {
123                p95_ms: round(number("p95_ms")?, 100.0),
124                error_rate: round(number("error_rate")?, 10_000.0),
125            };
126            Ok((endpoint.clone(), measurement))
127        })
128        .collect()
129}
130
131#[must_use]
132pub fn render(measurements: &Measurements) -> String {
133    let routes: Map<String, Value> = measurements
134        .iter()
135        .map(|(endpoint, measurement)| {
136            let values =
137                json!({ "p95_ms": measurement.p95_ms, "error_rate": measurement.error_rate });
138            (endpoint.clone(), values)
139        })
140        .collect();
141    let text = serde_json::to_string_pretty(&json!({ "routes": routes }))
142        .expect("a map of numbers serializes");
143    format!("{text}\n")
144}
145
146fn round(value: f64, scale: f64) -> f64 {
147    (value * scale).round() / scale
148}
149
150#[cfg(test)]
151mod tests {
152    use super::{Measurement, Measurements, Verdict, compare, parse, render};
153
154    fn measurements(entries: &[(&str, f64, f64)]) -> Measurements {
155        entries
156            .iter()
157            .map(|&(endpoint, p95_ms, error_rate)| {
158                let measurement = Measurement { p95_ms, error_rate };
159                (endpoint.to_owned(), measurement)
160            })
161            .collect()
162    }
163
164    #[test]
165    fn a_regression_beyond_the_margin_fails_and_keeps_the_old_ceiling() {
166        let baseline = measurements(&[("list html", 10.0, 0.0)]);
167
168        let comparison = compare(&baseline, &measurements(&[("list html", 12.5, 0.0)]));
169
170        assert!(comparison.regressed());
171        assert!(
172            matches!(&comparison.routes[0].verdict, Verdict::Regressed(reasons) if reasons[0].contains("p95 12.50 ms"))
173        );
174        assert_eq!(comparison.next_baseline, baseline);
175    }
176
177    #[test]
178    fn an_error_rate_above_the_margin_is_a_regression() {
179        let baseline = measurements(&[("list json", 10.0, 0.0)]);
180
181        let comparison = compare(&baseline, &measurements(&[("list json", 9.0, 0.02)]));
182
183        assert!(comparison.regressed());
184    }
185
186    #[test]
187    fn noise_within_the_margin_holds_and_an_improvement_lowers_the_ceiling() {
188        let baseline = measurements(&[("list html", 10.0, 0.0), ("list json", 8.0, 0.01)]);
189
190        let comparison = compare(
191            &baseline,
192            &measurements(&[("list html", 11.5, 0.0), ("list json", 6.0, 0.0)]),
193        );
194
195        assert!(!comparison.regressed());
196        assert_eq!(comparison.routes[0].verdict, Verdict::Held);
197        assert_eq!(comparison.routes[1].verdict, Verdict::Lowered);
198        assert_eq!(
199            comparison.next_baseline,
200            measurements(&[("list html", 10.0, 0.0), ("list json", 6.0, 0.0)])
201        );
202    }
203
204    #[test]
205    fn a_new_endpoint_sets_its_ceiling_and_a_removed_one_drops_out() {
206        let baseline = measurements(&[("gone json", 3.0, 0.0)]);
207
208        let comparison = compare(&baseline, &measurements(&[("list json", 4.0, 0.0)]));
209
210        assert_eq!(comparison.routes[0].verdict, Verdict::New);
211        assert_eq!(
212            comparison.next_baseline,
213            measurements(&[("list json", 4.0, 0.0)])
214        );
215    }
216
217    #[test]
218    fn the_baseline_file_reads_back_what_it_wrote_rounded() {
219        let summary = r#"{"routes": {"list html": {"p95_ms": 12.3456, "error_rate": 0.000049}}}"#;
220
221        let read = parse(summary).unwrap();
222
223        assert_eq!(read, measurements(&[("list html", 12.35, 0.0)]));
224        assert_eq!(parse(&render(&read)).unwrap(), read);
225    }
226}
sofie_cli/bench/baseline.rs

sofie_cli/bench/
baseline.rs
