iai_callgrind_runner/runner/callgrind/
regression.rs

1//! Module containing the callgrind specific regression check configuration
2use indexmap::{IndexMap, IndexSet};
3
4use crate::api::{self, EventKind};
5use crate::runner::metrics::{Metric, MetricKind, MetricsSummary};
6use crate::runner::summary::ToolRegression;
7use crate::runner::tool::regression::RegressionConfig;
8
9/// The callgrind regression check configuration
10#[derive(Debug, Clone, PartialEq)]
11pub struct CallgrindRegressionConfig {
12    /// True if benchmarks should fail on first encountered failed regression check
13    pub fail_fast: bool,
14    /// The hard limits
15    pub hard_limits: Vec<(EventKind, Metric)>,
16    /// The soft limits
17    pub soft_limits: Vec<(EventKind, f64)>,
18}
19
20impl Default for CallgrindRegressionConfig {
21    fn default() -> Self {
22        Self {
23            soft_limits: vec![(EventKind::Ir, 10f64)],
24            hard_limits: Vec::default(),
25            fail_fast: Default::default(),
26        }
27    }
28}
29
30impl RegressionConfig<EventKind> for CallgrindRegressionConfig {
31    /// Check the `MetricsSummary` for regressions.
32    ///
33    /// The limits for event kinds which are not present in the `MetricsSummary` are ignored.
34    fn check(&self, metrics_summary: &MetricsSummary) -> Vec<ToolRegression> {
35        self.check_regressions(metrics_summary)
36            .into_iter()
37            .map(|regressions| ToolRegression::with(MetricKind::Callgrind, regressions))
38            .collect()
39    }
40
41    fn get_soft_limits(&self) -> &[(EventKind, f64)] {
42        &self.soft_limits
43    }
44
45    fn get_hard_limits(&self) -> &[(EventKind, Metric)] {
46        &self.hard_limits
47    }
48}
49
50impl TryFrom<api::CallgrindRegressionConfig> for CallgrindRegressionConfig {
51    type Error = String;
52
53    fn try_from(value: api::CallgrindRegressionConfig) -> Result<Self, Self::Error> {
54        let api::CallgrindRegressionConfig {
55            soft_limits,
56            hard_limits,
57            fail_fast,
58        } = value;
59
60        let (soft_limits, hard_limits) = if soft_limits.is_empty() && hard_limits.is_empty() {
61            (IndexMap::from([(EventKind::Ir, 10f64)]), IndexMap::new())
62        } else {
63            let hard_limits = hard_limits
64                .into_iter()
65                .flat_map(|(callgrind_metrics, metric)| {
66                    IndexSet::from(callgrind_metrics)
67                        .into_iter()
68                        .map(move |metric_kind| {
69                            Metric::from(metric)
70                                .try_convert(metric_kind)
71                                .ok_or_else(|| {
72                                    format!(
73                                        "Invalid hard limit for \
74                                         '{metric_kind:?}/{callgrind_metrics:?}': Expected a \
75                                         'Int' but found '{metric:?}'"
76                                    )
77                                })
78                        })
79                })
80                .collect::<Result<IndexMap<EventKind, Metric>, String>>()?;
81
82            let soft_limits = soft_limits
83                .into_iter()
84                .flat_map(|(m, l)| IndexSet::from(m).into_iter().map(move |e| (e, l)))
85                .collect::<IndexMap<_, _>>();
86
87            (soft_limits, hard_limits)
88        };
89
90        Ok(Self {
91            soft_limits: soft_limits.into_iter().collect(),
92            hard_limits: hard_limits.into_iter().collect(),
93            fail_fast: fail_fast.unwrap_or(false),
94        })
95    }
96}
97
98#[cfg(test)]
99mod tests {
100    use rstest::rstest;
101    use EventKind::*;
102
103    use super::*;
104    use crate::api::{CallgrindMetrics, Limit};
105    use crate::runner::callgrind::model::Metrics;
106    use crate::runner::metrics::{Metric, TypeChecker};
107    use crate::util::EitherOrBoth;
108
109    fn cachesim_costs(costs: [u64; 9]) -> Metrics {
110        Metrics::with_metric_kinds([
111            (Ir, Metric::Int(costs[0])),
112            (Dr, Metric::Int(costs[1])),
113            (Dw, Metric::Int(costs[2])),
114            (I1mr, Metric::Int(costs[3])),
115            (D1mr, Metric::Int(costs[4])),
116            (D1mw, Metric::Int(costs[5])),
117            (ILmr, Metric::Int(costs[6])),
118            (DLmr, Metric::Int(costs[7])),
119            (DLmw, Metric::Int(costs[8])),
120        ])
121    }
122
123    #[rstest]
124    fn test_regression_check_when_old_is_none() {
125        let regression = CallgrindRegressionConfig::default();
126        let new = cachesim_costs([0, 0, 0, 0, 0, 0, 0, 0, 0]);
127        let summary = MetricsSummary::new(EitherOrBoth::Left(new));
128
129        assert!(regression.check(&summary).is_empty());
130    }
131
132    #[rstest]
133    #[case::ir_all_zero(
134        vec![(Ir, 0f64)],
135        [0, 0, 0, 0, 0, 0, 0, 0, 0],
136        [0, 0, 0, 0, 0, 0, 0, 0, 0],
137        vec![]
138    )]
139    #[case::ir_when_regression(
140        vec![(Ir, 0f64)],
141        [2, 0, 0, 0, 0, 0, 0, 0, 0],
142        [1, 0, 0, 0, 0, 0, 0, 0, 0],
143        vec![(Ir, 2, 1, 100f64, 0f64)]
144    )]
145    #[case::ir_when_improved(
146        vec![(Ir, 0f64)],
147        [1, 0, 0, 0, 0, 0, 0, 0, 0],
148        [2, 0, 0, 0, 0, 0, 0, 0, 0],
149        vec![]
150    )]
151    #[case::ir_when_negative_limit(
152        vec![(Ir, -49f64)],
153        [1, 0, 0, 0, 0, 0, 0, 0, 0],
154        [2, 0, 0, 0, 0, 0, 0, 0, 0],
155        vec![(Ir, 1, 2, -50f64, -49f64)]
156    )]
157    #[case::derived_all_zero(
158        vec![(EstimatedCycles, 0f64)],
159        [0, 0, 0, 0, 0, 0, 0, 0, 0],
160        [0, 0, 0, 0, 0, 0, 0, 0, 0],
161        vec![]
162    )]
163    #[case::derived_when_regression(
164        vec![(EstimatedCycles, 0f64)],
165        [2, 0, 0, 0, 0, 0, 0, 0, 0],
166        [1, 0, 0, 0, 0, 0, 0, 0, 0],
167        vec![(EstimatedCycles, 2, 1, 100f64, 0f64)]
168    )]
169    #[case::derived_when_regression_multiple(
170        vec![(EstimatedCycles, 5f64), (Ir, 10f64)],
171        [2, 0, 0, 0, 0, 0, 0, 0, 0],
172        [1, 0, 0, 0, 0, 0, 0, 0, 0],
173        vec![(EstimatedCycles, 2, 1, 100f64, 5f64), (Ir, 2, 1, 100f64, 10f64)]
174    )]
175    #[case::derived_when_improved(
176        vec![(EstimatedCycles, 0f64)],
177        [1, 0, 0, 0, 0, 0, 0, 0, 0],
178        [2, 0, 0, 0, 0, 0, 0, 0, 0],
179        vec![]
180    )]
181    #[case::derived_when_regression_mixed(
182        vec![(EstimatedCycles, 0f64)],
183        [96, 24, 18, 6, 0, 2, 6, 0, 2],
184        [48, 12, 9, 3, 0, 1, 3, 0, 1],
185        vec![(EstimatedCycles, 410, 205, 100f64, 0f64)]
186    )]
187    fn test_regression_check_when_soft_and_old_is_some(
188        #[case] soft_limits: Vec<(EventKind, f64)>,
189        #[case] new: [u64; 9],
190        #[case] old: [u64; 9],
191        #[case] expected: Vec<(EventKind, u64, u64, f64, f64)>,
192    ) {
193        let regression = CallgrindRegressionConfig {
194            soft_limits,
195            ..Default::default()
196        };
197
198        let new = cachesim_costs(new);
199        let old = cachesim_costs(old);
200        let summary = MetricsSummary::new(EitherOrBoth::Both(new, old));
201        let expected = expected
202            .iter()
203            .map(|(e, n, o, d, l)| ToolRegression::Soft {
204                metric: MetricKind::Callgrind(*e),
205                new: (*n).into(),
206                old: (*o).into(),
207                diff_pct: *d,
208                limit: *l,
209            })
210            .collect::<Vec<ToolRegression>>();
211
212        assert_eq!(regression.check(&summary), expected);
213    }
214
215    #[rstest]
216    #[case::empty_then_default(Vec::<(EventKind, f64)>::new(), vec![(EventKind::Ir, 10f64)])]
217    #[case::single(vec![(Ir, 0f64)], vec![(Ir, 0f64)])]
218    #[case::two(vec![(Ir, 0f64), (Dr, 10f64)], vec![(Ir, 0f64), (Dr, 10f64)])]
219    #[case::duplicate(vec![(Ir, 0f64), (Ir, 10f64)], vec![(Ir, 10f64)])]
220    #[case::group(
221        vec![(CallgrindMetrics::WriteBackBehaviour, 10f64)],
222        vec![(ILdmr, 10f64), (DLdmr, 10f64), (DLdmw, 10f64)],
223    )]
224    #[case::group_overwrite_keeps_order(
225        vec![(CallgrindMetrics::WriteBackBehaviour, 10f64), (ILdmr.into(), 20f64)],
226        vec![(ILdmr, 20f64), (DLdmr, 10f64), (DLdmw, 10f64)],
227    )]
228    fn test_try_from_regression_config_for_soft_limits<T>(
229        #[case] soft_limits: Vec<(T, f64)>,
230        #[case] expected_soft_limits: Vec<(EventKind, f64)>,
231    ) where
232        T: Into<CallgrindMetrics>,
233    {
234        let expected = CallgrindRegressionConfig {
235            soft_limits: expected_soft_limits,
236            hard_limits: Vec::default(),
237            fail_fast: false,
238        };
239        let api_regression_config = api::CallgrindRegressionConfig {
240            soft_limits: soft_limits
241                .into_iter()
242                .map(|(m, l)| (m.into(), l))
243                .collect(),
244            hard_limits: Vec::default(),
245            fail_fast: Option::default(),
246        };
247
248        assert_eq!(
249            CallgrindRegressionConfig::try_from(api_regression_config).unwrap(),
250            expected
251        );
252    }
253
254    #[rstest]
255    #[case::empty_then_default(Vec::<(EventKind, f64)>::new(), Vec::<(EventKind, f64)>::new())]
256    #[case::single(vec![(Ir, 0)], vec![(Ir, 0)])]
257    #[case::single_convert(vec![(L1HitRate, 1)], vec![(L1HitRate, 1f64)])]
258    #[case::two(vec![(Ir, 0), (Dr, 2)], vec![(Ir, 0), (Dr, 2)])]
259    #[case::duplicate_overwrite( vec![(Ir, 0), (Ir, 20)], vec![(Ir, 20)])]
260    #[case::integer_group(
261        vec![(CallgrindMetrics::WriteBackBehaviour, 10)],
262        vec![(ILdmr, 10), (DLdmr, 10), (DLdmw, 10)],
263    )]
264    #[case::float_group(
265        vec![(CallgrindMetrics::CacheHitRates, 10f64)],
266        vec![(L1HitRate, 10f64), (LLHitRate, 10f64), (RamHitRate, 10f64)],
267    )]
268    #[case::float_group_convert(
269        vec![(CallgrindMetrics::CacheHitRates, 10)],
270        vec![(L1HitRate, 10f64), (LLHitRate, 10f64), (RamHitRate, 10f64)],
271    )]
272    #[case::mixed_group(
273        vec![(CallgrindMetrics::CacheSim, 10)],
274        IndexSet::from(CallgrindMetrics::CacheSim)
275            .into_iter()
276            .map(|m| {
277                if m.is_int() {
278                    (m, Metric::Int(10))
279                } else {
280                    (m, Metric::Float(10.0))
281                }
282           }).collect()
283    )]
284    #[case::group_overwrite_keeps_order(
285        vec![(CallgrindMetrics::WriteBackBehaviour, 10), (ILdmr.into(), 20)],
286        vec![(ILdmr, 20), (DLdmr, 10), (DLdmw, 10)],
287    )]
288    fn test_try_from_regression_config_for_hard_limits<T, U, V>(
289        #[case] hard_limits: Vec<(T, U)>,
290        #[case] expected_hard_limits: Vec<(EventKind, V)>,
291    ) where
292        T: Into<CallgrindMetrics>,
293        U: Into<Limit>,
294        V: Into<Metric>,
295    {
296        let expected = CallgrindRegressionConfig {
297            soft_limits: if hard_limits.is_empty() {
298                vec![(EventKind::Ir, 10f64)]
299            } else {
300                Vec::default()
301            },
302            hard_limits: expected_hard_limits
303                .into_iter()
304                .map(|(m, l)| (m, l.into()))
305                .collect::<Vec<(EventKind, Metric)>>(),
306            fail_fast: false,
307        };
308        let api_regression_config = api::CallgrindRegressionConfig {
309            soft_limits: Vec::default(),
310            hard_limits: hard_limits
311                .into_iter()
312                .map(|(m, l)| (m.into(), l.into()))
313                .collect(),
314            fail_fast: Option::default(),
315        };
316
317        assert_eq!(
318            CallgrindRegressionConfig::try_from(api_regression_config).unwrap(),
319            expected
320        );
321    }
322
323    #[test]
324    fn test_try_from_regression_config_for_hard_limits_then_error() {
325        let api_regression_config = api::CallgrindRegressionConfig {
326            soft_limits: Vec::default(),
327            hard_limits: vec![(EventKind::Ir.into(), Limit::Float(10f64))],
328            fail_fast: Option::default(),
329        };
330
331        CallgrindRegressionConfig::try_from(api_regression_config).unwrap_err();
332
333        let api_regression_config = api::CallgrindRegressionConfig {
334            soft_limits: Vec::default(),
335            hard_limits: vec![(CallgrindMetrics::All, Limit::Float(10f64))],
336            fail_fast: Option::default(),
337        };
338
339        CallgrindRegressionConfig::try_from(api_regression_config).unwrap_err();
340    }
341}
iai_callgrind_runner/runner/callgrind/regression.rs

iai_callgrind_runner/runner/callgrind/
regression.rs
