stuffr_cli/bench/
mod.rs

1//! `stuffr bench`: compares codecs on the user's own files.
2//!
3//! [`run`] wires the pieces together in order: expand the operands, check
4//! `--formats`, refuse a debug build, load and warm the input once, then
5//! measure every row strictly one after another (there is one process-wide
6//! signal-cleanup slot, so two `measure` calls must never overlap) and
7//! render the result.
8
9pub mod levels;
10pub mod measure;
11pub mod report;
12pub mod score;
13
14use std::io::{self, IsTerminal, Write};
15use std::path::{Path, PathBuf};
16
17use stuffr::{Error, FormatKind};
18
19use crate::cli::BenchArgs;
20use levels::{backend_of, benchable_codecs, levels_of};
21use measure::{Input, Mode, Plan, RealClock, governor_for, load_input, measure, warm};
22use report::Conditions;
23use score::Row;
24
25/// The refusal `main` prints, verbatim, for [`Ran::RefusedDebugBuild`] (exit
26/// 2). Not an `Error::Usage`, whose display would prefix "usage error:".
27pub const DEBUG_REFUSAL: &str =
28    "stuffr: bench: timings from a debug build are meaningless; build with --release";
29
30/// How a [`run`] that raised no error ended.
31#[derive(Clone, Copy, Debug, PartialEq, Eq)]
32pub enum Ran {
33    /// The rows ran and the report reached stdout.
34    Measured,
35    /// The arguments were valid, but this is a debug build, so nothing was
36    /// timed, written or created.
37    RefusedDebugBuild,
38}
39
40/// Runs `stuffr bench`. Usage problems are `Error::Usage` (exit 2); a format
41/// that fails is a failed row, never an error here.
42pub fn run(args: BenchArgs) -> stuffr::Result<Ran> {
43    let paths = expand_inputs(&args.inputs)?;
44    let codecs = chosen_codecs(&args.formats)?;
45    let memory_limit = match &args.memory_limit {
46        Some(s) => crate::size::parse_size(s).map_err(Error::Usage)?,
47        None => stuffr::default_memory_limit(),
48    };
49    // After validation, so a debug build still reports usage errors.
50    if cfg!(debug_assertions) {
51        return Ok(Ran::RefusedDebugBuild);
52    }
53
54    let (input, bytes_in) = load_input(&paths, memory_limit)?;
55    warm(&input)?;
56    let plan = Plan {
57        mode: if args.per_file {
58            Mode::PerFile
59        } else {
60            Mode::OneStream
61        },
62        repeat: args.repeat,
63        threads: args.threads,
64        memory_limit,
65    };
66    let planned = plan_rows(&codecs, args.levels);
67    let mut meter = Meter {
68        on: io::stderr().is_terminal() && !(args.json || args.csv || args.table_only),
69        shown: false,
70    };
71    let mut clock = RealClock::new();
72    let mut rows = Vec::with_capacity(planned.len());
73    for (k, p) in planned.iter().enumerate() {
74        // Between rows only: `measure` owns every timed region.
75        meter.show(k + 1, planned.len(), p.format, p.label);
76        let outcome = measure(p.format, p.level, &input, &plan, &mut clock);
77        rows.push(Row {
78            format: p.format,
79            level: p.label,
80            backend: backend_of(p.format),
81            outcome,
82        });
83    }
84    meter.clear();
85
86    let conditions = Conditions {
87        files: paths.len(),
88        bytes_in,
89        mode: plan.mode,
90        threads: plan.threads,
91        // The budget every row's encoder is given; no governor is one.
92        workers: governor_for(&plan).map_or(1, |g| g.workers()),
93        repeat: plan.repeat,
94        streamed: matches!(input, Input::Files(_)),
95    };
96    let mut out = io::stdout().lock();
97    if args.json {
98        report::json(&conditions, &rows, &mut out)?;
99    } else if args.csv {
100        report::csv(&conditions, &rows, &mut out)?;
101    } else if args.table_only {
102        report::table_only(&rows, conditions.bytes_in, &mut out)?;
103    } else {
104        report::human(&conditions, &rows, &mut out)?;
105    }
106    out.flush()?;
107    Ok(Ran::Measured)
108}
109
110/// The row-count progress meter on stderr: `[k/N] <format> <level>` before
111/// each row, cleared before anything reaches stdout.
112struct Meter {
113    on: bool,
114    shown: bool,
115}
116
117impl Meter {
118    fn show(&mut self, k: usize, n: usize, format: &str, level: Option<i32>) {
119        if !self.on {
120            return;
121        }
122        let level = level.map_or_else(|| "-".to_string(), |l| l.to_string());
123        // `\x1b[K` after the text clears what a longer previous line left.
124        eprint!("\r[{k}/{n}] {format} {level}\x1b[K");
125        self.shown = true;
126    }
127
128    fn clear(&mut self) {
129        if self.shown {
130            eprint!("\r\x1b[K");
131            self.shown = false;
132        }
133    }
134}
135
136/// One row to run: `level` is what `measure` gets (`None`: the encoder's
137/// own default), `label` what the table shows.
138#[derive(Clone, Copy, Debug, PartialEq, Eq)]
139struct Planned {
140    format: &'static str,
141    level: Option<i32>,
142    label: Option<i32>,
143}
144
145/// One default row per codec, or with `sweep` one row per sweep level; a
146/// codec without levels gets its one row either way.
147fn plan_rows(codecs: &[&'static str], sweep: bool) -> Vec<Planned> {
148    let mut rows = Vec::new();
149    for &format in codecs {
150        let lv = levels_of(format);
151        match lv.map(|l| l.sweep).filter(|s| sweep && !s.is_empty()) {
152            Some(levels) => rows.extend(levels.iter().map(|&l| Planned {
153                format,
154                level: Some(l),
155                label: Some(l),
156            })),
157            None => rows.push(Planned {
158                format,
159                level: None,
160                label: lv.and_then(|l| l.default),
161            }),
162        }
163    }
164    rows
165}
166
167/// The codecs to run: `names` in order without repeats, each checked, or
168/// every benchable codec when none are named.
169fn chosen_codecs(names: &[String]) -> stuffr::Result<Vec<&'static str>> {
170    let benchable = benchable_codecs();
171    if names.is_empty() {
172        return Ok(benchable);
173    }
174    let mut chosen = Vec::new();
175    for name in names {
176        let id = crate::format_by_name(name)?;
177        let Some(&codec) = benchable.iter().find(|c| **c == id.as_str()) else {
178            return Err(Error::Usage(format!(
179                "{name} cannot be benchmarked in this build ({})",
180                why_not_benchable(name)
181            )));
182        };
183        if !chosen.contains(&codec) {
184            chosen.push(codec);
185        }
186    }
187    Ok(chosen)
188}
189
190/// Why a registered format is not in `benchable_codecs`.
191fn why_not_benchable(name: &str) -> &'static str {
192    let row = stuffr::registry()
193        .matrix()
194        .into_iter()
195        .find(|r| r.id.as_str() == name);
196    match row {
197        Some(r) => why_not(r.kind, r.write),
198        None => why_not(FormatKind::Codec, true),
199    }
200}
201
202/// The reason, from the matrix facts: a container, a codec that cannot
203/// write, or (`benchable_codecs`' remaining filter) a weak encoder.
204fn why_not(kind: FormatKind, write: bool) -> &'static str {
205    match (kind, write) {
206        (FormatKind::Container, _) => "container",
207        (_, false) => "read-only",
208        _ => "weak fallback encoder",
209    }
210}
211
212fn has_glob_chars(s: &str) -> bool {
213    s.contains(['*', '?', '['])
214}
215
216/// The operands as files, in order. An existing path is used as is; a
217/// missing one with glob characters is expanded (sorted, dot files only
218/// when the pattern spells the dot, as a shell does). Directories a pattern
219/// matches are passed over; one named outright, or the only kind of match a
220/// pattern has, is refused. A FIFO, device or socket, named or matched, is
221/// refused from its metadata (it is never opened): bench needs each file's
222/// size up front. A symlink counts as what it points to.
223fn expand_inputs(operands: &[String]) -> stuffr::Result<Vec<PathBuf>> {
224    let mut files = Vec::new();
225    for op in operands {
226        let path = Path::new(op);
227        if path.exists() {
228            if path.is_dir() {
229                return Err(directory_error(path));
230            }
231            files.push(regular(path)?);
232            continue;
233        }
234        if !has_glob_chars(op) {
235            return Err(no_match(op));
236        }
237        let opts = glob::MatchOptions {
238            require_literal_leading_dot: true,
239            ..Default::default()
240        };
241        let matches = glob::glob_with(op, opts)
242            .map_err(|e| Error::Usage(format!("bad pattern {op}: {e}")))?;
243        let mut found = Vec::new();
244        let mut first_dir = None;
245        for m in matches {
246            let m = m.map_err(|e| Error::Io(e.into()))?;
247            if m.is_dir() {
248                first_dir.get_or_insert(m);
249            } else {
250                found.push(regular(&m)?);
251            }
252        }
253        if found.is_empty() {
254            return Err(first_dir.map_or_else(|| no_match(op), |d| directory_error(&d)));
255        }
256        found.sort();
257        files.extend(found);
258    }
259    Ok(files)
260}
261
262/// `path` if it is (or links to) a regular file, else the usage error.
263fn regular(path: &Path) -> stuffr::Result<PathBuf> {
264    if std::fs::metadata(path)?.is_file() {
265        Ok(path.to_path_buf())
266    } else {
267        Err(Error::Usage(format!(
268            "{} is not a regular file; bench needs files whose size it can read",
269            path.display()
270        )))
271    }
272}
273
274fn no_match(op: &str) -> Error {
275    Error::Usage(format!("no input matched {op}"))
276}
277
278fn directory_error(path: &Path) -> Error {
279    let p = path.display();
280    Error::Usage(format!("{p} is a directory; name its files (e.g. {p}/*)"))
281}
282
283#[cfg(test)]
284mod tests {
285    use super::*;
286    use std::fs;
287    use std::sync::atomic::{AtomicU32, Ordering};
288
289    /// A scratch directory `stuffr-benchrun-<pid>-<n>`, removed on drop.
290    struct Scratch(PathBuf);
291
292    impl Scratch {
293        fn new() -> Self {
294            static N: AtomicU32 = AtomicU32::new(0);
295            let n = N.fetch_add(1, Ordering::Relaxed);
296            let dir =
297                std::env::temp_dir().join(format!("stuffr-benchrun-{}-{n}", std::process::id()));
298            fs::create_dir_all(&dir).unwrap();
299            Scratch(dir)
300        }
301
302        fn file(&self, name: &str) -> PathBuf {
303            let p = self.0.join(name);
304            fs::write(&p, b"x").unwrap();
305            p
306        }
307
308        fn s(&self, rest: &str) -> String {
309            format!("{}/{rest}", self.0.display())
310        }
311    }
312
313    impl Drop for Scratch {
314        fn drop(&mut self) {
315            let _ = fs::remove_dir_all(&self.0);
316        }
317    }
318
319    fn usage(r: stuffr::Result<Vec<PathBuf>>) -> String {
320        match r {
321            Err(Error::Usage(m)) => m,
322            other => panic!("expected a usage error, got {other:?}"),
323        }
324    }
325
326    #[test]
327    fn a_default_row_runs_the_encoders_default_labelled_with_the_table_default() {
328        let rows = plan_rows(&["bzip2", "compress", "lz4"], false);
329        assert_eq!(
330            rows,
331            vec![
332                Planned {
333                    format: "bzip2",
334                    level: None,
335                    label: Some(6)
336                },
337                Planned {
338                    format: "compress",
339                    level: None,
340                    label: Some(16)
341                },
342                Planned {
343                    format: "lz4",
344                    level: None,
345                    label: None
346                },
347            ]
348        );
349    }
350
351    #[test]
352    fn a_sweep_runs_every_level_and_one_row_for_a_codec_without_levels() {
353        let rows = plan_rows(&["compress", "snappy"], true);
354        let got: Vec<_> = rows.iter().map(|r| (r.format, r.level, r.label)).collect();
355        let mut want: Vec<_> = (9..=16).map(|l| ("compress", Some(l), Some(l))).collect();
356        want.push(("snappy", None, None));
357        assert_eq!(got, want);
358    }
359
360    #[test]
361    fn a_glob_expands_sorted_and_skips_dot_files_and_directories() {
362        let s = Scratch::new();
363        s.file("b.log");
364        s.file("a.log");
365        s.file(".hidden.log");
366        fs::create_dir(s.0.join("dir.log")).unwrap();
367        let got = expand_inputs(&[s.s("*.log")]).unwrap();
368        assert_eq!(got, vec![s.0.join("a.log"), s.0.join("b.log")]);
369    }
370
371    #[test]
372    fn an_existing_path_is_used_as_is_even_with_glob_characters() {
373        let s = Scratch::new();
374        let odd = s.file("a[1].log");
375        assert_eq!(expand_inputs(&[s.s("a[1].log")]).unwrap(), vec![odd]);
376    }
377
378    #[test]
379    fn no_match_and_a_missing_path_are_usage_errors() {
380        let s = Scratch::new();
381        let pat = s.s("nomatch*.log");
382        assert_eq!(
383            usage(expand_inputs(std::slice::from_ref(&pat))),
384            format!("no input matched {pat}")
385        );
386        let missing = s.s("absent.log");
387        assert!(usage(expand_inputs(&[missing])).starts_with("no input matched"));
388    }
389
390    #[test]
391    fn a_directory_is_refused_named_or_as_a_patterns_only_match() {
392        let s = Scratch::new();
393        fs::create_dir(s.0.join("logs")).unwrap();
394        let named = usage(expand_inputs(&[s.s("logs")]));
395        let dir = s.s("logs");
396        assert_eq!(
397            named,
398            format!("{dir} is a directory; name its files (e.g. {dir}/*)")
399        );
400        assert!(usage(expand_inputs(&[s.s("lo*")])).contains("is a directory"));
401    }
402
403    #[cfg(unix)]
404    #[test]
405    fn a_device_is_refused_and_a_symlink_to_a_file_is_not() {
406        let s = Scratch::new();
407        let want = "/dev/null is not a regular file; bench needs files whose size it can read";
408        assert_eq!(usage(expand_inputs(&["/dev/null".into()])), want);
409        assert_eq!(usage(expand_inputs(&["/dev/nul?".into()])), want);
410        let target = s.file("real.log");
411        let link = s.0.join("link.log");
412        std::os::unix::fs::symlink(&target, &link).unwrap();
413        assert_eq!(expand_inputs(&[s.s("link.log")]).unwrap(), vec![link]);
414    }
415
416    #[test]
417    fn formats_are_checked_and_deduplicated() {
418        let all = benchable_codecs();
419        assert_eq!(chosen_codecs(&[]).unwrap(), all);
420        let gz = vec!["gzip".to_string(), "bzip2".into(), "gzip".into()];
421        assert_eq!(chosen_codecs(&gz).unwrap(), vec!["gzip", "bzip2"]);
422        let msg = |n: &str| match chosen_codecs(&[n.to_string()]) {
423            Err(Error::Usage(m)) => m,
424            other => panic!("{other:?}"),
425        };
426        assert_eq!(
427            msg("tar"),
428            "tar cannot be benchmarked in this build (container)"
429        );
430        assert!(msg("nosuch").starts_with("unknown format `nosuch`"));
431        if !all.contains(&"zstd") {
432            assert_eq!(
433                msg("zstd"),
434                "zstd cannot be benchmarked in this build (weak fallback encoder)"
435            );
436        }
437    }
438
439    #[test]
440    fn the_reason_follows_the_matrix_facts() {
441        // No codec in today's builds is read-only, so the classifier is
442        // tested on the facts rather than on a registered format.
443        assert_eq!(why_not(FormatKind::Container, true), "container");
444        assert_eq!(why_not(FormatKind::Container, false), "container");
445        assert_eq!(why_not(FormatKind::Codec, false), "read-only");
446        assert_eq!(why_not(FormatKind::Codec, true), "weak fallback encoder");
447        assert_eq!(why_not_benchable("tar"), "container");
448    }
449}
stuffr_cli/bench/mod.rs

stuffr_cli/bench/
mod.rs
