stuffr_cli/bench/
score.rs

1//! Scoring and winners for `stuffr bench`. Pure: no I/O, no timing.
2//!
3//! Each metric (`bytes_out`, `compress_ns`, `decompress_ns`) is scored
4//! `best / value` over the rows that succeeded, a value of 0 counting as 1.
5//! `balance` is the mean of the three scores. Failed rows score `None` and
6//! never win. Ties go to the smaller `bytes_out`, then to the earlier row.
7
8/// What one successful format/level run produced.
9#[derive(Clone, Copy, Debug, PartialEq)]
10pub struct Measured {
11    pub bytes_out: u64,
12    pub compress_ns: u64,
13    pub decompress_ns: u64,
14    /// The compressed output went to the temp file, so its writes are
15    /// inside `compress_ns`; the reports mark the row.
16    pub on_disk: bool,
17}
18
19/// The result of one row: measured, or failed with a reason.
20#[derive(Clone, Debug, PartialEq)]
21pub enum Outcome {
22    Ok(Measured),
23    Failed(String),
24}
25
26/// Peak resident memory of one row, from the untimed memory runs. A number is
27/// `None` when its run failed; `error` then says why (a refusal by
28/// `--memory-limit` reads differently from a crash).
29#[derive(Clone, Debug, Default, PartialEq)]
30pub struct Memory {
31    pub compress: Option<u64>,
32    pub decompress: Option<u64>,
33    pub error: Option<String>,
34}
35
36impl Memory {
37    /// The RAM a machine needs to use the row's format: the larger of the
38    /// two peaks. `None` unless both were measured.
39    fn needed(&self) -> Option<u64> {
40        Some(self.compress?.max(self.decompress?))
41    }
42}
43
44/// One line of the table. `level` is `None` for codecs without levels
45/// (lz4, snappy). `backend` is `"c"` or `"pure"`: which implementation of
46/// the format this build ran (`levels::backend_of`).
47#[derive(Clone, Debug, PartialEq)]
48pub struct Row {
49    pub format: &'static str,
50    pub level: Option<i32>,
51    pub backend: &'static str,
52    pub outcome: Outcome,
53    /// `None`: not measured (`--no-memory`, or a build without `getrusage`).
54    pub memory: Option<Memory>,
55}
56
57/// Indices into the rows slice, one per column; `None` when no row succeeded.
58#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
59pub struct Winners {
60    pub smallest: Option<usize>,
61    pub fastest_compress: Option<usize>,
62    pub fastest_decompress: Option<usize>,
63    pub balance: Option<usize>,
64    /// Smallest `max(compress, decompress)` memory, among rows with both.
65    pub least_memory: Option<usize>,
66}
67
68fn measured(row: &Row) -> Option<&Measured> {
69    match &row.outcome {
70        Outcome::Ok(m) => Some(m),
71        Outcome::Failed(_) => None,
72    }
73}
74
75/// `best / value`, a zero counting as 1 on both sides.
76fn ratio(best: u64, value: u64) -> f64 {
77    best.max(1) as f64 / value.max(1) as f64
78}
79
80/// The balance score of every row: the mean of the three per-metric scores,
81/// `None` for a failed row.
82pub fn balance_scores(rows: &[Row]) -> Vec<Option<f64>> {
83    let best = |f: fn(&Measured) -> u64| rows.iter().filter_map(measured).map(f).min();
84    let (Some(b_out), Some(b_c), Some(b_d)) = (
85        best(|m| m.bytes_out),
86        best(|m| m.compress_ns),
87        best(|m| m.decompress_ns),
88    ) else {
89        return vec![None; rows.len()];
90    };
91    rows.iter()
92        .map(|row| {
93            measured(row).map(|m| {
94                (ratio(b_out, m.bytes_out)
95                    + ratio(b_c, m.compress_ns)
96                    + ratio(b_d, m.decompress_ns))
97                    / 3.0
98            })
99        })
100        .collect()
101}
102
103/// The winner of each column. Ties go to the smaller `bytes_out`, then to
104/// the earlier row. `least_memory` considers only rows with both numbers.
105pub fn winners(rows: &[Row]) -> Winners {
106    let scores = balance_scores(rows);
107    let by_min = |key: fn(&Measured) -> u64| {
108        rows.iter()
109            .enumerate()
110            .filter_map(|(i, r)| measured(r).map(|m| (i, m)))
111            .min_by_key(|(i, m)| (key(m), m.bytes_out, *i))
112            .map(|(i, _)| i)
113    };
114    let balance = rows
115        .iter()
116        .enumerate()
117        .filter_map(|(i, r)| Some((i, measured(r)?, scores[i]?)))
118        .reduce(|best, cand| {
119            // Replace only on a strictly higher score, or an equal score
120            // with strictly smaller output; otherwise the earlier row stays.
121            if cand.2 > best.2 || (cand.2 == best.2 && cand.1.bytes_out < best.1.bytes_out) {
122                cand
123            } else {
124                best
125            }
126        })
127        .map(|(i, _, _)| i);
128    let least_memory = rows
129        .iter()
130        .enumerate()
131        .filter_map(|(i, r)| Some((i, measured(r)?, r.memory.as_ref()?.needed()?)))
132        .min_by_key(|(i, m, need)| (*need, m.bytes_out, *i))
133        .map(|(i, _, _)| i);
134    Winners {
135        smallest: by_min(|m| m.bytes_out),
136        fastest_compress: by_min(|m| m.compress_ns),
137        fastest_decompress: by_min(|m| m.decompress_ns),
138        balance,
139        least_memory,
140    }
141}
142
143#[cfg(test)]
144mod tests {
145    use super::*;
146
147    fn ok(bytes_out: u64, compress_ns: u64, decompress_ns: u64) -> Row {
148        Row {
149            format: "x",
150            level: Some(1),
151            backend: "pure",
152            outcome: Outcome::Ok(Measured {
153                bytes_out,
154                compress_ns,
155                decompress_ns,
156                on_disk: false,
157            }),
158            memory: None,
159        }
160    }
161
162    fn with_mem(mut row: Row, c: Option<u64>, d: Option<u64>) -> Row {
163        row.memory = Some(Memory {
164            compress: c,
165            decompress: d,
166            error: None,
167        });
168        row
169    }
170
171    fn failed() -> Row {
172        Row {
173            format: "x",
174            level: None,
175            backend: "pure",
176            outcome: Outcome::Failed("boom".into()),
177            memory: None,
178        }
179    }
180
181    fn abc() -> Vec<Row> {
182        vec![ok(100, 10, 10), ok(50, 20, 20), ok(200, 5, 40)]
183    }
184
185    fn close(got: Option<f64>, want: f64) {
186        let got = got.expect("a score");
187        assert!((got - want).abs() < 1e-9, "{got} vs {want}");
188    }
189
190    #[test]
191    fn balance_weights_the_three_metrics_equally() {
192        let rows = abc();
193        let s = balance_scores(&rows);
194        close(s[0], (0.5 + 0.5 + 1.0) / 3.0);
195        close(s[1], (1.0 + 0.25 + 0.5) / 3.0);
196        close(s[2], (0.25 + 1.0 + 0.25) / 3.0);
197        assert_eq!(winners(&rows).balance, Some(0));
198    }
199
200    #[test]
201    fn winners_name_each_column() {
202        let w = winners(&abc());
203        assert_eq!(w.smallest, Some(1));
204        assert_eq!(w.fastest_compress, Some(2));
205        assert_eq!(w.fastest_decompress, Some(0));
206    }
207
208    #[test]
209    fn failed_rows_never_score_or_win() {
210        let mut rows = abc();
211        rows.insert(0, failed());
212        assert_eq!(balance_scores(&rows)[0], None);
213        let w = winners(&rows);
214        for idx in [
215            w.smallest,
216            w.fastest_compress,
217            w.fastest_decompress,
218            w.balance,
219        ] {
220            assert!(idx.expect("a winner") > 0);
221        }
222    }
223
224    #[test]
225    fn ties_go_to_the_smaller_output() {
226        let rows = vec![ok(100, 10, 10), ok(60, 10, 10)];
227        assert_eq!(winners(&rows).fastest_compress, Some(1));
228        assert_eq!(winners(&rows).fastest_decompress, Some(1));
229        assert_eq!(winners(&rows).balance, Some(1));
230    }
231
232    /// Equal mean scores reached by different metrics: the later row wins
233    /// on its smaller output, where "earlier row" alone would pick the first.
234    #[test]
235    fn a_balance_tie_goes_to_the_smaller_output() {
236        let rows = vec![ok(100, 10, 40), ok(50, 20, 40)];
237        let s = balance_scores(&rows);
238        assert_eq!(s[0], s[1], "a real tie");
239        assert_eq!(winners(&rows).balance, Some(1));
240    }
241
242    #[test]
243    fn ties_on_everything_go_to_the_earlier_row() {
244        let rows = vec![ok(7, 7, 7), ok(7, 7, 7)];
245        let w = winners(&rows);
246        assert_eq!(w.smallest, Some(0));
247        assert_eq!(w.balance, Some(0));
248    }
249
250    #[test]
251    fn all_failed_gives_no_winners() {
252        let rows = vec![failed(), failed()];
253        assert_eq!(winners(&rows), Winners::default());
254        assert_eq!(balance_scores(&rows), vec![None, None]);
255        assert_eq!(winners(&[]), Winners::default());
256    }
257
258    #[test]
259    fn zero_values_do_not_divide_by_zero() {
260        let rows = vec![ok(10, 10, 10), ok(0, 0, 0)];
261        for s in balance_scores(&rows) {
262            assert!(s.expect("a score").is_finite());
263        }
264        // Best in every column, a zero counting as 1 on both sides.
265        assert_eq!(balance_scores(&rows)[1], Some(1.0));
266        let w = winners(&rows);
267        assert_eq!(w.smallest, Some(1));
268        assert_eq!(w.fastest_compress, Some(1));
269        assert_eq!(w.fastest_decompress, Some(1));
270        assert_eq!(w.balance, Some(1));
271    }
272
273    #[test]
274    fn least_memory_takes_the_smaller_maximum() {
275        let rows = vec![
276            with_mem(ok(1, 1, 1), Some(10), Some(90)),
277            with_mem(ok(1, 1, 1), Some(50), Some(50)),
278            with_mem(ok(1, 1, 1), Some(80), Some(5)),
279        ];
280        assert_eq!(winners(&rows).least_memory, Some(1));
281    }
282
283    #[test]
284    fn least_memory_ignores_rows_missing_a_number() {
285        let rows = vec![
286            with_mem(ok(1, 1, 1), Some(1), None),
287            with_mem(ok(1, 1, 1), None, Some(1)),
288            ok(1, 1, 1),
289            with_mem(failed(), Some(1), Some(1)),
290            with_mem(ok(1, 1, 1), Some(70), Some(60)),
291        ];
292        assert_eq!(winners(&rows).least_memory, Some(4));
293        let none = vec![with_mem(ok(1, 1, 1), Some(1), None), ok(1, 1, 1)];
294        assert_eq!(winners(&none).least_memory, None);
295    }
296
297    #[test]
298    fn least_memory_tie_goes_to_the_smaller_output() {
299        let rows = vec![
300            with_mem(ok(100, 1, 1), Some(5), Some(9)),
301            with_mem(ok(60, 1, 1), Some(9), Some(2)),
302            with_mem(ok(60, 1, 1), Some(9), Some(9)),
303        ];
304        assert_eq!(winners(&rows).least_memory, Some(1));
305    }
306}
stuffr_cli/bench/score.rs

stuffr_cli/bench/
score.rs
