git_perf/
reporting.rs

1use std::{
2    fs::File,
3    io::{self, ErrorKind, Write},
4    path::{Path, PathBuf},
5};
6
7use anyhow::anyhow;
8use anyhow::{bail, Result};
9use itertools::Itertools;
10use plotly::{
11    common::{Font, LegendGroupTitle, Title},
12    layout::{Axis, Legend},
13    Configuration, Layout, Plot,
14};
15
16// TODO(kaihowl) find central place for the data structures
17use crate::{
18    data::{MeasurementData, MeasurementSummary, ReductionFunc},
19    measurement_retrieval::{self, Commit, ReductionFuncIterator},
20    serialization::{serialize_single, DELIMITER},
21};
22
23trait Reporter<'a> {
24    fn add_commits(&mut self, hashes: &'a [Commit]);
25    fn add_trace(
26        &mut self,
27        indexed_measurements: Vec<(usize, &'a MeasurementData)>,
28        measurement_name: &str,
29        group_value: Option<&String>,
30    );
31    fn add_summarized_trace(
32        &mut self,
33        indexed_measurements: Vec<(usize, MeasurementSummary)>,
34        measurement_name: &str,
35        group_value: Option<&String>,
36    );
37    fn as_bytes(&self) -> Vec<u8>;
38}
39
40struct PlotlyReporter {
41    plot: Plot,
42    // TODO(kaihowl) hack until we can auto_range 'reverse' the axis in plotly directly
43    size: usize,
44}
45
46impl PlotlyReporter {
47    fn new() -> PlotlyReporter {
48        let config = Configuration::default().responsive(true).fill_frame(true);
49        let mut plot = Plot::new();
50        plot.set_configuration(config);
51        PlotlyReporter { plot, size: 0 }
52    }
53
54    fn convert_to_x_y(&self, indexed_measurements: Vec<(usize, f64)>) -> (Vec<usize>, Vec<f64>) {
55        indexed_measurements
56            .iter()
57            .map(|(i, m)| (self.size - i - 1, m))
58            .unzip()
59    }
60}
61
62impl<'a> Reporter<'a> for PlotlyReporter {
63    fn add_commits(&mut self, commits: &'a [Commit]) {
64        let enumerated_commits = commits.iter().rev().enumerate();
65        self.size = commits.len();
66
67        let (commit_nrs, short_hashes): (Vec<_>, Vec<_>) = enumerated_commits
68            .map(|(n, c)| (n as f64, c.commit[..6].to_owned()))
69            .unzip();
70        let x_axis = Axis::new()
71            .tick_values(commit_nrs)
72            .tick_text(short_hashes)
73            .tick_angle(45.0)
74            .tick_font(Font::new().family("monospace"));
75        let layout = Layout::new()
76            .title(Title::new("Performance Measurements"))
77            .x_axis(x_axis)
78            .legend(
79                Legend::new()
80                    .group_click(plotly::layout::GroupClick::ToggleItem)
81                    .orientation(plotly::common::Orientation::Horizontal),
82            );
83
84        self.plot.set_layout(layout);
85    }
86
87    fn add_trace(
88        &mut self,
89        indexed_measurements: Vec<(usize, &'a MeasurementData)>,
90        measurement_name: &str,
91        group_value: Option<&String>,
92    ) {
93        let (x, y) = self.convert_to_x_y(
94            indexed_measurements
95                .into_iter()
96                .map(|(i, m)| (i, m.val))
97                .collect_vec(),
98        );
99
100        let trace = plotly::BoxPlot::new_xy(x, y);
101
102        let trace = if let Some(group_value) = group_value {
103            trace
104                .name(group_value)
105                .legend_group(measurement_name)
106                .legend_group_title(LegendGroupTitle::new(measurement_name))
107        } else {
108            trace.name(measurement_name)
109        };
110
111        self.plot.add_trace(trace);
112    }
113
114    fn add_summarized_trace(
115        &mut self,
116        indexed_measurements: Vec<(usize, MeasurementSummary)>,
117        measurement_name: &str,
118        group_value: Option<&String>,
119    ) {
120        let (x, y) = self.convert_to_x_y(
121            indexed_measurements
122                .into_iter()
123                .map(|(i, m)| (i, m.val))
124                .collect_vec(),
125        );
126
127        let trace = plotly::Scatter::new(x, y).name(measurement_name);
128
129        let trace = if let Some(group_value) = group_value {
130            trace
131                .name(group_value)
132                .legend_group(measurement_name)
133                .legend_group_title(LegendGroupTitle::new(measurement_name))
134        } else {
135            trace.name(measurement_name)
136        };
137
138        self.plot.add_trace(trace);
139    }
140
141    fn as_bytes(&self) -> Vec<u8> {
142        self.plot.to_html().as_bytes().to_vec()
143    }
144}
145
146struct CsvReporter<'a> {
147    hashes: Vec<String>,
148    indexed_measurements: Vec<(usize, &'a MeasurementData)>,
149}
150
151impl CsvReporter<'_> {
152    fn new() -> Self {
153        CsvReporter {
154            hashes: Vec::new(),
155            indexed_measurements: Vec::new(),
156        }
157    }
158}
159
160impl<'a> Reporter<'a> for CsvReporter<'a> {
161    fn add_commits(&mut self, hashes: &'a [Commit]) {
162        self.hashes = hashes.iter().map(|c| c.commit.to_owned()).collect();
163    }
164
165    fn add_trace(
166        &mut self,
167        indexed_measurements: Vec<(usize, &'a MeasurementData)>,
168        _measurement_name: &str,
169        _group_value: Option<&String>,
170    ) {
171        self.indexed_measurements
172            .extend_from_slice(indexed_measurements.as_slice());
173    }
174
175    fn as_bytes(&self) -> Vec<u8> {
176        // TODO(kaihowl) write to path directly instead?
177
178        self.indexed_measurements
179            .iter()
180            .map(|(index, measurement_data)| {
181                let ser_measurement = serialize_single(measurement_data, "\t");
182                let commit = &self.hashes[*index];
183                format!("{commit}{DELIMITER}{ser_measurement}")
184            })
185            .join("")
186            .into_bytes()
187    }
188
189    fn add_summarized_trace(
190        &mut self,
191        _indexed_measurements: Vec<(usize, MeasurementSummary)>,
192        _measurement_name: &str,
193        _group_value: Option<&String>,
194    ) {
195        todo!()
196    }
197}
198
199struct ReporterFactory {}
200
201impl ReporterFactory {
202    fn from_file_name(path: &Path) -> Option<Box<dyn Reporter + '_>> {
203        if path == Path::new("-") {
204            return Some(Box::new(CsvReporter::new()) as Box<dyn Reporter>);
205        }
206        let mut res = None;
207        if let Some(ext) = path.extension() {
208            let extension = ext.to_ascii_lowercase().into_string().unwrap();
209            res = match extension.as_str() {
210                "html" => Some(Box::new(PlotlyReporter::new()) as Box<dyn Reporter>),
211                "csv" => Some(Box::new(CsvReporter::new()) as Box<dyn Reporter>),
212                _ => None,
213            }
214        }
215        res
216    }
217}
218// TODO(kaihowl) needs more fine grained output e2e tests
219pub fn report(
220    output: PathBuf,
221    separate_by: Option<String>,
222    num_commits: usize,
223    measurement_names: &[String],
224    key_values: &[(String, String)],
225    aggregate_by: Option<ReductionFunc>,
226) -> Result<()> {
227    let commits: Vec<Commit> = measurement_retrieval::walk_commits(num_commits)?.try_collect()?;
228
229    let mut plot =
230        ReporterFactory::from_file_name(&output).ok_or(anyhow!("Could not infer output format"))?;
231
232    plot.add_commits(&commits);
233
234    let relevant = |m: &MeasurementData| {
235        if !measurement_names.is_empty() && !measurement_names.contains(&m.name) {
236            return false;
237        }
238        // TODO(kaihowl) express this and the audit-fn equivalent as subset relations
239        key_values
240            .iter()
241            .all(|(k, v)| m.key_values.get(k).map(|mv| v == mv).unwrap_or(false))
242    };
243
244    let relevant_measurements = commits
245        .iter()
246        .map(|commit| commit.measurements.iter().filter(|m| relevant(m)));
247
248    let unique_measurement_names: Vec<_> = relevant_measurements
249        .clone()
250        .flat_map(|m| m.map(|m| &m.name))
251        .unique()
252        .collect();
253
254    if unique_measurement_names.is_empty() {
255        bail!("No performance measurements found.")
256    }
257
258    for measurement_name in unique_measurement_names {
259        let filtered_measurements = relevant_measurements
260            .clone()
261            .map(|ms| ms.filter(|m| m.name == *measurement_name));
262
263        let group_values = if let Some(separate_by) = &separate_by {
264            filtered_measurements
265                .clone()
266                .flat_map(|ms| {
267                    ms.flat_map(|m| {
268                        m.key_values
269                            .iter()
270                            .filter(|(k, _v)| *k == separate_by)
271                            .map(|(_k, v)| v)
272                    })
273                })
274                .unique()
275                .map(|val| (Some(separate_by), Some(val)))
276                .collect_vec()
277        } else {
278            vec![(None, None)]
279        };
280
281        if group_values.is_empty() {
282            bail!("Invalid separator supplied, no measurements.")
283        }
284
285        for (group_key, group_value) in group_values {
286            let group_measurements = filtered_measurements.clone().map(|ms| {
287                ms.filter(|m| {
288                    group_key
289                        .map(|gk| m.key_values.get(gk) == group_value)
290                        .unwrap_or(true)
291                })
292            });
293
294            if let Some(reduction_func) = aggregate_by {
295                let trace_measurements = group_measurements
296                    .clone()
297                    .enumerate()
298                    .flat_map(move |(i, ms)| {
299                        ms.reduce_by(reduction_func)
300                            .into_iter()
301                            .map(move |m| (i, m))
302                    })
303                    .collect_vec();
304                plot.add_summarized_trace(trace_measurements, measurement_name, group_value);
305            } else {
306                let trace_measurements: Vec<_> = group_measurements
307                    .clone()
308                    .enumerate()
309                    .flat_map(|(i, ms)| ms.map(move |m| (i, m)))
310                    .collect();
311                plot.add_trace(trace_measurements, measurement_name, group_value);
312            }
313        }
314    }
315
316    // TODO(kaihowl) fewer than the -n specified measurements appear in plot (old problem, even in
317    // python)
318
319    if output == Path::new("-") {
320        match io::stdout().write_all(&plot.as_bytes()) {
321            Err(e) if e.kind() == ErrorKind::BrokenPipe => Ok(()),
322            res => res,
323        }?;
324    } else {
325        File::create(&output)?.write_all(&plot.as_bytes())?;
326    }
327
328    Ok(())
329}
git_perf/reporting.rs

git_perf/
reporting.rs
