iai_callgrind_runner/runner/dhat/
regression.rs

1//! Module containing the dhat specific regression check configuration
2use indexmap::{IndexMap, IndexSet};
3
4use crate::api::{self, DhatMetric};
5use crate::runner::metrics::{Metric, MetricKind, MetricsSummary};
6use crate::runner::summary::ToolRegression;
7use crate::runner::tool::regression::RegressionConfig;
8
9/// The dhat regression check configuration
10#[derive(Debug, Clone, PartialEq)]
11pub struct DhatRegressionConfig {
12    /// True if benchmarks should fail on first encountered failed regression check
13    pub fail_fast: bool,
14    /// The hard limits
15    pub hard_limits: Vec<(DhatMetric, Metric)>,
16    /// The soft limits
17    pub soft_limits: Vec<(DhatMetric, f64)>,
18}
19
20impl Default for DhatRegressionConfig {
21    fn default() -> Self {
22        Self {
23            soft_limits: vec![(DhatMetric::TotalBytes, 10f64)],
24            hard_limits: Vec::default(),
25            fail_fast: Default::default(),
26        }
27    }
28}
29
30impl RegressionConfig<DhatMetric> for DhatRegressionConfig {
31    fn check(&self, metrics_summary: &MetricsSummary<DhatMetric>) -> Vec<ToolRegression> {
32        self.check_regressions(metrics_summary)
33            .into_iter()
34            .map(|regressions| ToolRegression::with(MetricKind::Dhat, regressions))
35            .collect()
36    }
37
38    fn get_soft_limits(&self) -> &[(DhatMetric, f64)] {
39        &self.soft_limits
40    }
41
42    fn get_hard_limits(&self) -> &[(DhatMetric, Metric)] {
43        &self.hard_limits
44    }
45}
46
47impl TryFrom<api::DhatRegressionConfig> for DhatRegressionConfig {
48    type Error = String;
49
50    fn try_from(value: api::DhatRegressionConfig) -> std::result::Result<Self, Self::Error> {
51        let api::DhatRegressionConfig {
52            soft_limits,
53            hard_limits,
54            fail_fast,
55        } = value;
56
57        let (soft_limits, hard_limits) = if soft_limits.is_empty() && hard_limits.is_empty() {
58            (
59                IndexMap::from([(DhatMetric::TotalBytes, 10f64)]),
60                IndexMap::new(),
61            )
62        } else {
63            let hard_limits = hard_limits
64                .into_iter()
65                .flat_map(|(dhat_metrics, metric)| {
66                    IndexSet::from(dhat_metrics)
67                        .into_iter()
68                        .map(move |metric_kind| {
69                            Metric::from(metric)
70                                .try_convert(metric_kind)
71                                .ok_or_else(|| {
72                                    format!(
73                                        "Invalid hard limit for \
74                                         '{metric_kind:?}/{dhat_metrics:?}': Expected a 'Int' but \
75                                         found '{metric:?}'"
76                                    )
77                                })
78                        })
79                })
80                .collect::<Result<IndexMap<DhatMetric, Metric>, String>>()?;
81
82            let soft_limits = soft_limits
83                .into_iter()
84                .flat_map(|(m, l)| IndexSet::from(m).into_iter().map(move |e| (e, l)))
85                .collect::<IndexMap<_, _>>();
86
87            (soft_limits, hard_limits)
88        };
89        Ok(Self {
90            soft_limits: soft_limits.into_iter().collect(),
91            hard_limits: hard_limits.into_iter().collect(),
92            fail_fast: fail_fast.unwrap_or(false),
93        })
94    }
95}
96
97#[cfg(test)]
98mod tests {
99    use pretty_assertions::assert_eq;
100    use rstest::rstest;
101    use DhatMetric::*;
102
103    use super::*;
104    use crate::runner::metrics::Metrics;
105    use crate::runner::tool::regression::RegressionMetrics;
106    use crate::util::EitherOrBoth;
107
108    fn costs_fixture(costs: [u64; 2]) -> Metrics<DhatMetric> {
109        Metrics::with_metric_kinds([(TotalBytes, costs[0]), (TotalBlocks, costs[1])])
110    }
111
112    #[rstest]
113    #[case::all_zero_no_regression(
114        vec![(TotalBytes, 0)],
115        [0, 0],
116        vec![]
117    )]
118    #[case::total_bytes_regression_by_one(
119        vec![(TotalBytes, 0)],
120        [1, 0],
121        vec![(TotalBytes, 1, 1, 0)]
122    )]
123    #[case::total_bytes_regression_by_two(
124        vec![(TotalBytes, 0)],
125        [2, 0],
126        vec![(TotalBytes, 2, 2, 0)]
127    )]
128    #[case::total_bytes_regression_some_value(
129        vec![(TotalBytes, 10)],
130        [11, 0],
131        vec![(TotalBytes, 11, 1, 10)]
132    )]
133    #[case::total_bytes_and_block_regression(
134        vec![(TotalBytes, 9), (TotalBlocks, 1)],
135        [10, 4],
136        vec![(TotalBytes, 10, 1, 9), (TotalBlocks, 4, 3, 1)]
137    )]
138    fn test_regression_check_when_hard<U>(
139        #[case] limits: Vec<(DhatMetric, U)>,
140        #[case] new: [u64; 2],
141        #[case] expected: Vec<(DhatMetric, u64, u64, u64)>,
142    ) where
143        U: Into<Metric>,
144    {
145        let regression = DhatRegressionConfig {
146            hard_limits: limits.into_iter().map(|(x, y)| (x, y.into())).collect(),
147            soft_limits: vec![],
148            ..Default::default()
149        };
150
151        let new_costs = costs_fixture(new);
152
153        let summary = MetricsSummary::new(EitherOrBoth::Left(new_costs));
154        let expected = expected
155            .iter()
156            .map(|(e, n, d, l)| ToolRegression::Hard {
157                metric: MetricKind::Dhat(*e),
158                new: (*n).into(),
159                diff: (*d).into(),
160                limit: (*l).into(),
161            })
162            .collect::<Vec<ToolRegression>>();
163
164        assert_eq!(regression.check(&summary), expected);
165    }
166
167    #[test]
168    fn test_regression_check_when_hard_and_soft() {
169        let config = DhatRegressionConfig {
170            hard_limits: vec![(DhatMetric::TotalBytes, 2.into())],
171            soft_limits: vec![(DhatMetric::TotalBlocks, 20f64)],
172            ..Default::default()
173        };
174
175        let new_costs = costs_fixture([3, 4]);
176        let old_costs = costs_fixture([1, 2]);
177
178        let summary = MetricsSummary::new(EitherOrBoth::Both(new_costs, old_costs));
179        let expected = vec![
180            ToolRegression::with(
181                MetricKind::Dhat,
182                RegressionMetrics::Soft(DhatMetric::TotalBlocks, 4.into(), 2.into(), 100f64, 20f64),
183            ),
184            ToolRegression::with(
185                MetricKind::Dhat,
186                RegressionMetrics::Hard(DhatMetric::TotalBytes, 3.into(), 1.into(), 2.into()),
187            ),
188        ];
189
190        assert_eq!(config.check(&summary), expected);
191    }
192}
iai_callgrind_runner/runner/dhat/regression.rs

iai_callgrind_runner/runner/dhat/
regression.rs
