git_perf/
audit.rs

1use crate::{
2    data::MeasurementData,
3    measurement_retrieval::{self, summarize_measurements},
4    stats,
5};
6use anyhow::{anyhow, bail, Result};
7use git_perf_cli_types::ReductionFunc;
8use itertools::Itertools;
9use log::error;
10use sparklines::spark;
11use std::cmp::Ordering;
12use std::iter;
13
14#[derive(Debug, PartialEq)]
15struct AuditResult {
16    message: String,
17    passed: bool,
18}
19
20pub fn audit_multiple(
21    measurements: &[String],
22    max_count: usize,
23    min_count: u16,
24    selectors: &[(String, String)],
25    summarize_by: ReductionFunc,
26    sigma: f64,
27) -> Result<()> {
28    let mut failed = false;
29
30    for measurement in measurements {
31        let result = audit(
32            measurement,
33            max_count,
34            min_count,
35            selectors,
36            summarize_by,
37            sigma,
38        )?;
39
40        println!("{}", result.message);
41
42        if !result.passed {
43            failed = true;
44        }
45    }
46
47    if failed {
48        bail!("One or more measurements failed audit.");
49    }
50
51    Ok(())
52}
53
54fn audit(
55    measurement: &str,
56    max_count: usize,
57    min_count: u16,
58    selectors: &[(String, String)],
59    summarize_by: ReductionFunc,
60    sigma: f64,
61) -> Result<AuditResult> {
62    let all = measurement_retrieval::walk_commits(max_count)?;
63
64    let filter_by = |m: &MeasurementData| {
65        m.name == measurement
66            && selectors
67                .iter()
68                .all(|s| m.key_values.get(&s.0).map(|v| *v == s.1).unwrap_or(false))
69    };
70
71    let mut aggregates = measurement_retrieval::take_while_same_epoch(summarize_measurements(
72        all,
73        &summarize_by,
74        &filter_by,
75    ));
76
77    let head = aggregates
78        .next()
79        .ok_or(anyhow!("No commit at HEAD"))
80        .and_then(|s| {
81            s.and_then(|cs| {
82                cs.measurement
83                    .map(|m| m.val)
84                    .ok_or(anyhow!("No measurement for HEAD."))
85            })
86        })?;
87
88    let tail: Vec<_> = aggregates
89        .filter_map_ok(|cs| cs.measurement.map(|m| m.val))
90        .take(max_count)
91        .try_collect()?;
92
93    let head_summary = stats::aggregate_measurements(iter::once(&head));
94    let tail_summary = stats::aggregate_measurements(tail.iter());
95
96    if tail_summary.len < min_count.into() {
97        let number_measurements = tail_summary.len;
98        let plural_s = if number_measurements > 1 { "s" } else { "" };
99        error!("Only {number_measurements} measurement{plural_s} found. Less than requested min_measurements of {min_count}. Skipping test.");
100        return Ok(AuditResult {
101            message: format!("Only {number_measurements} measurement{plural_s} found. Less than requested min_measurements of {min_count}. Skipping test."),
102            passed: true,
103        });
104    }
105
106    let direction = match head_summary.mean.partial_cmp(&tail_summary.mean).unwrap() {
107        Ordering::Greater => "↑",
108        Ordering::Less => "↓",
109        Ordering::Equal => "→",
110    };
111
112    let all_measurements = tail.into_iter().chain(iter::once(head)).collect::<Vec<_>>();
113    let average = all_measurements.iter().sum::<f64>() / all_measurements.len() as f64;
114    let relative_min = all_measurements
115        .iter()
116        .min_by(|a, b| a.partial_cmp(b).unwrap())
117        .unwrap()
118        / average
119        - 1.0;
120    let relative_max = all_measurements
121        .iter()
122        .max_by(|a, b| a.partial_cmp(b).unwrap())
123        .unwrap()
124        / average
125        - 1.0;
126
127    let text_summary = format!(
128        "z-score: {direction} {:.2}\nHead: {}\nTail: {}\n [{:+.1}% – {:+.1}%] {}",
129        head_summary.z_score(&tail_summary),
130        &head_summary,
131        &tail_summary,
132        (relative_min * 100.0),
133        (relative_max * 100.0),
134        spark(all_measurements.as_slice()),
135    );
136
137    if head_summary.z_score(&tail_summary) > sigma {
138        return Ok(AuditResult {
139            message: format!(
140                "❌ '{measurement}'\nHEAD differs significantly from tail measurements.\n{text_summary}"
141            ),
142            passed: false,
143        });
144    }
145
146    Ok(AuditResult {
147        message: format!("✅ '{measurement}'\n{text_summary}"),
148        passed: true,
149    })
150}
git_perf/audit.rs

git_perf/
audit.rs
