iai_callgrind_runner/runner/
args.rs

1use std::path::PathBuf;
2use std::process::{Command, Stdio};
3use std::str::FromStr;
4
5use clap::builder::BoolishValueParser;
6use clap::{ArgAction, Parser};
7
8use super::format::OutputFormatKind;
9use super::summary::{BaselineName, SummaryFormat};
10use crate::api::{EventKind, RawArgs, RegressionConfig};
11
12/// A filter for benchmarks
13///
14/// # Developer Notes
15///
16/// This enum is used instead of a plain `String` for possible future usages to filter by benchmark
17/// ids, group name, file name etc.
18#[derive(Debug, Clone)]
19pub enum BenchmarkFilter {
20    /// The name of the benchmark
21    Name(String),
22}
23
24impl BenchmarkFilter {
25    /// Return true if the haystack contains the filter
26    pub fn apply(&self, haystack: &str) -> bool {
27        let Self::Name(name) = self;
28        haystack.contains(name)
29    }
30}
31
32impl FromStr for BenchmarkFilter {
33    type Err = String;
34
35    fn from_str(s: &str) -> Result<Self, Self::Err> {
36        Ok(BenchmarkFilter::Name(s.to_owned()))
37    }
38}
39
40#[derive(Debug, Clone, Copy, PartialEq, Eq)]
41pub enum NoCapture {
42    True,
43    False,
44    Stderr,
45    Stdout,
46}
47
48impl NoCapture {
49    pub fn apply(self, command: &mut Command) {
50        match self {
51            NoCapture::True | NoCapture::False => {}
52            NoCapture::Stderr => {
53                command.stdout(Stdio::null()).stderr(Stdio::inherit());
54            }
55            NoCapture::Stdout => {
56                command.stdout(Stdio::inherit()).stderr(Stdio::null());
57            }
58        };
59    }
60}
61
62/// The command line arguments the user provided after `--` when running cargo bench
63///
64/// These arguments are not the command line arguments passed to `iai-callgrind-runner`. We collect
65/// the command line arguments in the `iai-callgrind::main!` macro without the binary as first
66/// argument, that's why `no_binary_name` is set to `true`.
67#[allow(clippy::partial_pub_fields)]
68#[derive(Parser, Debug, Clone)]
69#[command(
70    author,
71    version,
72    about = "High-precision and consistent benchmarking framework/harness for Rust
73
74Boolish command line arguments take also one of `y`, `yes`, `t`, `true`, `on`, `1`
75instead of `true` and one of `n`, `no`, `f`, `false`, `off`, and `0` instead of
76`false`",
77    long_about = None,
78    no_binary_name = true,
79    override_usage= "cargo bench ... [BENCHNAME] -- [OPTIONS]"
80)]
81pub struct CommandLineArgs {
82    /// `--bench` usually shows up as last argument set by cargo and not by us.
83    ///
84    /// This argument is useless, so we sort it out and never make use of it.
85    #[arg(long = "bench", hide = true, action = ArgAction::SetTrue, required = false)]
86    _bench: bool,
87
88    /// If specified, only run benches containing this string in their names
89    ///
90    /// Note that a benchmark name might differ from the benchmark file name.
91    #[arg(name = "BENCHNAME", num_args = 0..=1, env = "IAI_CALLGRIND_FILTER")]
92    pub filter: Option<BenchmarkFilter>,
93
94    /// The raw arguments to pass through to Callgrind
95    ///
96    /// This is a space separated list of command-line-arguments specified as if they were
97    /// passed directly to valgrind.
98    ///
99    /// Examples:
100    ///   * --callgrind-args=--dump-instr=yes
101    ///   * --callgrind-args='--dump-instr=yes --collect-systime=yes'
102    #[arg(
103        long = "callgrind-args",
104        value_parser = parse_args,
105        num_args = 1,
106        verbatim_doc_comment,
107        env = "IAI_CALLGRIND_CALLGRIND_ARGS"
108    )]
109    pub callgrind_args: Option<RawArgs>,
110
111    /// Save a machine-readable summary of each benchmark run in json format next to the usual
112    /// benchmark output
113    #[arg(
114        long = "save-summary",
115        value_enum,
116        num_args = 0..=1,
117        require_equals = true,
118        default_missing_value = "json",
119        env = "IAI_CALLGRIND_SAVE_SUMMARY"
120    )]
121    pub save_summary: Option<SummaryFormat>,
122
123    /// Allow ASLR (Address Space Layout Randomization)
124    ///
125    /// If possible, ASLR is disabled on platforms that support it (linux, freebsd) because ASLR
126    /// could noise up the callgrind cache simulation results a bit. Setting this option to true
127    /// runs all benchmarks with ASLR enabled.
128    ///
129    /// See also <https://docs.kernel.org/admin-guide/sysctl/kernel.html?highlight=randomize_va_space#randomize-va-space>
130    #[arg(
131        long = "allow-aslr",
132        default_missing_value = "true",
133        num_args = 0..=1,
134        require_equals = true,
135        value_parser = BoolishValueParser::new(),
136        env = "IAI_CALLGRIND_ALLOW_ASLR",
137    )]
138    pub allow_aslr: Option<bool>,
139
140    /// Set performance regression limits for specific `EventKinds`
141    ///
142    /// This is a `,` separate list of EventKind=limit (key=value) pairs with the limit being a
143    /// positive or negative percentage. If positive, a performance regression check for this
144    /// `EventKind` fails if the limit is exceeded. If negative, the regression check fails if the
145    /// value comes below the limit. The `EventKind` is matched case-insensitive. For a list of
146    /// valid `EventKinds` see the docs: <https://docs.rs/iai-callgrind/latest/iai_callgrind/enum.EventKind.html>
147    ///
148    /// Examples: --regression='ir=0.0' or --regression='ir=0, EstimatedCycles=10'
149    #[arg(
150        long = "regression",
151        num_args = 1,
152        value_parser = parse_regression_config,
153        env = "IAI_CALLGRIND_REGRESSION",
154    )]
155    pub regression: Option<RegressionConfig>,
156
157    /// If true, the first failed performance regression check fails the whole benchmark run
158    ///
159    /// This option requires `--regression=...` or `IAI_CALLGRIND_REGRESSION=...` to be present.
160    #[arg(
161        long = "regression-fail-fast",
162        requires = "regression",
163        default_missing_value = "true",
164        num_args = 0..=1,
165        require_equals = true,
166        value_parser = BoolishValueParser::new(),
167        env = "IAI_CALLGRIND_REGRESSION_FAIL_FAST",
168    )]
169    pub regression_fail_fast: Option<bool>,
170
171    /// Compare against this baseline if present and then overwrite it
172    #[arg(
173        long = "save-baseline",
174        default_missing_value = "default",
175        num_args = 0..=1,
176        require_equals = true,
177        conflicts_with_all = &["baseline", "LOAD_BASELINE"],
178        env = "IAI_CALLGRIND_SAVE_BASELINE",
179    )]
180    pub save_baseline: Option<BaselineName>,
181
182    /// Compare against this baseline if present but do not overwrite it
183    #[arg(
184        long = "baseline",
185        default_missing_value = "default",
186        num_args = 0..=1,
187        require_equals = true,
188        env = "IAI_CALLGRIND_BASELINE"
189    )]
190    pub baseline: Option<BaselineName>,
191
192    /// Load this baseline as the new data set instead of creating a new one
193    #[clap(
194        id = "LOAD_BASELINE",
195        long = "load-baseline",
196        requires = "baseline",
197        num_args = 0..=1,
198        require_equals = true,
199        default_missing_value = "default",
200        env = "IAI_CALLGRIND_LOAD_BASELINE"
201    )]
202    pub load_baseline: Option<BaselineName>,
203
204    /// The terminal output format in default human-readable format or in machine-readable json
205    /// format
206    ///
207    /// # The JSON Output Format
208    ///
209    /// The json terminal output schema is the same as the schema with the `--save-summary`
210    /// argument when saving to a `summary.json` file. All other output than the json output goes
211    /// to stderr and only the summary output goes to stdout. When not printing pretty json, each
212    /// line is a dictionary summarizing a single benchmark. You can combine all lines
213    /// (benchmarks) into an array for example with `jq`
214    ///
215    /// `cargo bench -- --output-format=json | jq -s`
216    ///
217    /// which transforms `{...}\n{...}` into `[{...},{...}]`
218    #[arg(
219        long = "output-format",
220        value_enum,
221        required = false,
222        default_value = "default",
223        num_args = 1,
224        env = "IAI_CALLGRIND_OUTPUT_FORMAT"
225    )]
226    pub output_format: OutputFormatKind,
227
228    /// Separate iai-callgrind benchmark output files by target
229    ///
230    /// The default output path for files created by iai-callgrind and valgrind during the
231    /// benchmark is
232    ///
233    /// `target/iai/$PACKAGE_NAME/$BENCHMARK_FILE/$GROUP/$BENCH_FUNCTION.$BENCH_ID`.
234    ///
235    /// This can be problematic if you're running the benchmarks not only for a
236    /// single target because you end up comparing the benchmark runs with the wrong targets.
237    /// Setting this option changes the default output path to
238    ///
239    /// `target/iai/$TARGET/$PACKAGE_NAME/$BENCHMARK_FILE/$GROUP/$BENCH_FUNCTION.$BENCH_ID`
240    ///
241    /// Although not as comfortable and strict, you could achieve a separation by target also with
242    /// baselines and a combination of `--save-baseline=$TARGET` and `--baseline=$TARGET` if you
243    /// prefer having all files of a single $BENCH in the same directory.
244    #[arg(
245        long = "separate-targets",
246        default_missing_value = "true",
247        default_value = "false",
248        num_args = 0..=1,
249        require_equals = true,
250        value_parser = BoolishValueParser::new(),
251        action = ArgAction::Set,
252        env = "IAI_CALLGRIND_SEPARATE_TARGETS",
253    )]
254    pub separate_targets: bool,
255
256    /// Specify the home directory of iai-callgrind benchmark output files
257    ///
258    /// All output files are per default stored under the `$PROJECT_ROOT/target/iai` directory.
259    /// This option lets you customize this home directory, and it will be created if it
260    /// doesn't exist.
261    #[arg(long = "home", num_args = 1, env = "IAI_CALLGRIND_HOME")]
262    pub home: Option<PathBuf>,
263
264    /// Don't capture terminal output of benchmarks
265    ///
266    /// Possible values are one of [true, false, stdout, stderr].
267    ///
268    /// This option is currently restricted to the `callgrind` run of benchmarks. The output of
269    /// additional tool runs like DHAT, Memcheck, ... is still captured, to prevent showing the
270    /// same output of benchmarks multiple times. Use `IAI_CALLGRIND_LOG=info` to also show
271    /// captured and logged output.
272    ///
273    /// If no value is given, the default missing value is `true` and doesn't capture stdout and
274    /// stderr. Besides `true` or `false` you can specify the special values `stdout` or `stderr`.
275    /// If `--nocapture=stdout` is given, the output to `stdout` won't be captured and the output
276    /// to `stderr` will be discarded. Likewise, if `--nocapture=stderr` is specified, the
277    /// output to `stderr` won't be captured and the output to `stdout` will be discarded.
278    #[arg(
279        long = "nocapture",
280        required = false,
281        default_missing_value = "true",
282        default_value = "false",
283        num_args = 0..=1,
284        require_equals = true,
285        value_parser = parse_nocapture,
286        env = "IAI_CALLGRIND_NOCAPTURE"
287    )]
288    pub nocapture: NoCapture,
289}
290
291/// This function parses a space separated list of raw argument strings into [`crate::api::RawArgs`]
292fn parse_args(value: &str) -> Result<RawArgs, String> {
293    shlex::split(value)
294        .ok_or_else(|| "Failed to split callgrind args".to_owned())
295        .map(RawArgs::new)
296}
297
298fn parse_regression_config(value: &str) -> Result<RegressionConfig, String> {
299    let value = value.trim();
300    if value.is_empty() {
301        return Err("No limits found: At least one limit must be specified".to_owned());
302    }
303
304    let regression_config = if value.eq_ignore_ascii_case("default") {
305        RegressionConfig::default()
306    } else {
307        let mut limits = vec![];
308
309        for split in value.split(',') {
310            let split = split.trim();
311
312            if let Some((key, value)) = split.split_once('=') {
313                let (key, value) = (key.trim(), value.trim());
314                let event_kind = EventKind::from_str_ignore_case(key)
315                    .ok_or_else(|| -> String { format!("Unknown event kind: '{key}'") })?;
316
317                let pct = value.parse::<f64>().map_err(|error| -> String {
318                    format!("Invalid percentage for '{key}': {error}")
319                })?;
320                limits.push((event_kind, pct));
321            } else {
322                return Err(format!("Invalid format of key/value pair: '{split}'"));
323            }
324        }
325
326        RegressionConfig {
327            limits,
328            ..Default::default()
329        }
330    };
331
332    Ok(regression_config)
333}
334
335impl From<&CommandLineArgs> for Option<RegressionConfig> {
336    fn from(value: &CommandLineArgs) -> Self {
337        let mut config = value.regression.clone();
338        if let Some(config) = config.as_mut() {
339            config.fail_fast = value.regression_fail_fast;
340        }
341        config
342    }
343}
344
345fn parse_nocapture(value: &str) -> Result<NoCapture, String> {
346    // Taken from clap source code
347    const TRUE_LITERALS: [&str; 6] = ["y", "yes", "t", "true", "on", "1"];
348    const FALSE_LITERALS: [&str; 6] = ["n", "no", "f", "false", "off", "0"];
349
350    let lowercase: String = value.to_lowercase();
351
352    if TRUE_LITERALS.contains(&lowercase.as_str()) {
353        Ok(NoCapture::True)
354    } else if FALSE_LITERALS.contains(&lowercase.as_str()) {
355        Ok(NoCapture::False)
356    } else if lowercase == "stdout" {
357        Ok(NoCapture::Stdout)
358    } else if lowercase == "stderr" {
359        Ok(NoCapture::Stderr)
360    } else {
361        Err(format!("Invalid value: {value}"))
362    }
363}
364
365#[cfg(test)]
366mod tests {
367    use rstest::rstest;
368
369    use super::*;
370    use crate::api::EventKind::*;
371    use crate::api::RawArgs;
372
373    #[rstest]
374    #[case::empty("", &[])]
375    #[case::single_key_value("--some=yes", &["--some=yes"])]
376    #[case::two_key_value("--some=yes --other=no", &["--some=yes", "--other=no"])]
377    #[case::single_escaped("--some='yes and no'", &["--some=yes and no"])]
378    #[case::double_escaped("--some='\"yes and no\"'", &["--some=\"yes and no\""])]
379    #[case::multiple_escaped("--some='yes and no' --other='no and yes'", &["--some=yes and no", "--other=no and yes"])]
380    fn test_parse_callgrind_args(#[case] value: &str, #[case] expected: &[&str]) {
381        let actual = parse_args(value).unwrap();
382        assert_eq!(actual, RawArgs::from_iter(expected));
383    }
384
385    #[rstest]
386    #[case::regression_default("default", vec![])]
387    #[case::regression_default_case_insensitive("DefAulT", vec![])]
388    #[case::regression_only("Ir=10", vec![(Ir, 10f64)])]
389    #[case::regression_case_insensitive("EstIMATedCycles=10", vec![(EstimatedCycles, 10f64)])]
390    #[case::multiple_regression("Ir=10,EstimatedCycles=5", vec![(Ir, 10f64), (EstimatedCycles, 5f64)])]
391    #[case::multiple_regression_with_whitespace("Ir= 10 ,  EstimatedCycles = 5", vec![(Ir, 10f64), (EstimatedCycles, 5f64)])]
392    fn test_parse_regression_config(
393        #[case] regression_var: &str,
394        #[case] expected_limits: Vec<(EventKind, f64)>,
395    ) {
396        let expected = RegressionConfig {
397            limits: expected_limits,
398            fail_fast: None,
399        };
400
401        let actual = parse_regression_config(regression_var).unwrap();
402        assert_eq!(actual, expected);
403    }
404
405    #[rstest]
406    #[case::regression_wrong_format_of_key_value_pair(
407        "Ir:10",
408        "Invalid format of key/value pair: 'Ir:10'"
409    )]
410    #[case::regression_unknown_event_kind("WRONG=10", "Unknown event kind: 'WRONG'")]
411    #[case::regression_invalid_percentage(
412        "Ir=10.0.0",
413        "Invalid percentage for 'Ir': invalid float literal"
414    )]
415    #[case::regression_empty_limits("", "No limits found: At least one limit must be specified")]
416    fn test_try_regression_config_from_env_then_error(
417        #[case] regression_var: &str,
418        #[case] expected_reason: &str,
419    ) {
420        assert_eq!(
421            &parse_regression_config(regression_var).unwrap_err(),
422            expected_reason,
423        );
424    }
425
426    #[test]
427    #[serial_test::serial]
428    fn test_callgrind_args_env() {
429        let test_arg = "--just-testing=yes";
430        std::env::set_var("IAI_CALLGRIND_CALLGRIND_ARGS", test_arg);
431        let result = CommandLineArgs::parse_from::<[_; 0], &str>([]);
432        assert_eq!(
433            result.callgrind_args,
434            Some(RawArgs::new(vec![test_arg.to_owned()]))
435        );
436    }
437
438    #[test]
439    fn test_callgrind_args_not_env() {
440        let test_arg = "--just-testing=yes";
441        let result = CommandLineArgs::parse_from([format!("--callgrind-args={test_arg}")]);
442        assert_eq!(
443            result.callgrind_args,
444            Some(RawArgs::new(vec![test_arg.to_owned()]))
445        );
446    }
447
448    #[test]
449    #[serial_test::serial]
450    fn test_callgrind_args_cli_takes_precedence_over_env() {
451        let test_arg_yes = "--just-testing=yes";
452        let test_arg_no = "--just-testing=no";
453        std::env::set_var("IAI_CALLGRIND_CALLGRIND_ARGS", test_arg_yes);
454        let result = CommandLineArgs::parse_from([format!("--callgrind-args={test_arg_no}")]);
455        assert_eq!(
456            result.callgrind_args,
457            Some(RawArgs::new(vec![test_arg_no.to_owned()]))
458        );
459    }
460
461    #[test]
462    #[serial_test::serial]
463    fn test_save_summary_env() {
464        std::env::set_var("IAI_CALLGRIND_SAVE_SUMMARY", "json");
465        let result = CommandLineArgs::parse_from::<[_; 0], &str>([]);
466        assert_eq!(result.save_summary, Some(SummaryFormat::Json));
467    }
468
469    #[rstest]
470    #[case::default("", SummaryFormat::Json)]
471    #[case::json("json", SummaryFormat::Json)]
472    #[case::pretty_json("pretty-json", SummaryFormat::PrettyJson)]
473    fn test_save_summary_cli(#[case] value: &str, #[case] expected: SummaryFormat) {
474        let result = if value.is_empty() {
475            CommandLineArgs::parse_from(["--save-summary".to_owned()])
476        } else {
477            CommandLineArgs::parse_from([format!("--save-summary={value}")])
478        };
479        assert_eq!(result.save_summary, Some(expected));
480    }
481
482    #[test]
483    #[serial_test::serial]
484    fn test_allow_aslr_env() {
485        std::env::set_var("IAI_CALLGRIND_ALLOW_ASLR", "yes");
486        let result = CommandLineArgs::parse_from::<[_; 0], &str>([]);
487        assert_eq!(result.allow_aslr, Some(true));
488    }
489
490    #[rstest]
491    #[case::default("", true)]
492    #[case::yes("yes", true)]
493    #[case::no("no", false)]
494    fn test_allow_aslr_cli(#[case] value: &str, #[case] expected: bool) {
495        let result = if value.is_empty() {
496            CommandLineArgs::parse_from(["--allow-aslr".to_owned()])
497        } else {
498            CommandLineArgs::parse_from([format!("--allow-aslr={value}")])
499        };
500        assert_eq!(result.allow_aslr, Some(expected));
501    }
502
503    #[test]
504    #[serial_test::serial]
505    fn test_separate_targets_env() {
506        std::env::set_var("IAI_CALLGRIND_SEPARATE_TARGETS", "yes");
507        let result = CommandLineArgs::parse_from::<[_; 0], &str>([]);
508        assert!(result.separate_targets);
509    }
510
511    #[rstest]
512    #[case::default("", true)]
513    #[case::yes("yes", true)]
514    #[case::no("no", false)]
515    fn test_separate_targets_cli(#[case] value: &str, #[case] expected: bool) {
516        let result = if value.is_empty() {
517            CommandLineArgs::parse_from(["--separate-targets".to_owned()])
518        } else {
519            CommandLineArgs::parse_from([format!("--separate-targets={value}")])
520        };
521        assert_eq!(result.separate_targets, expected);
522    }
523
524    #[test]
525    #[serial_test::serial]
526    fn test_home_env() {
527        std::env::set_var("IAI_CALLGRIND_HOME", "/tmp/my_iai_home");
528        let result = CommandLineArgs::parse_from::<[_; 0], &str>([]);
529        assert_eq!(result.home, Some(PathBuf::from("/tmp/my_iai_home")));
530    }
531
532    #[test]
533    fn test_home_cli() {
534        let result = CommandLineArgs::parse_from(["--home=/test_me".to_owned()]);
535        assert_eq!(result.home, Some(PathBuf::from("/test_me")));
536    }
537
538    #[test]
539    fn test_home_cli_when_no_value_then_error() {
540        let result = CommandLineArgs::try_parse_from(["--home=".to_owned()]);
541        assert!(result.is_err());
542    }
543
544    #[rstest]
545    #[case::default("", NoCapture::True)]
546    #[case::yes("true", NoCapture::True)]
547    #[case::no("false", NoCapture::False)]
548    #[case::stdout("stdout", NoCapture::Stdout)]
549    #[case::stderr("stderr", NoCapture::Stderr)]
550    fn test_nocapture_cli(#[case] value: &str, #[case] expected: NoCapture) {
551        let result = if value.is_empty() {
552            CommandLineArgs::parse_from(["--nocapture".to_owned()])
553        } else {
554            CommandLineArgs::parse_from([format!("--nocapture={value}")])
555        };
556        assert_eq!(result.nocapture, expected);
557    }
558
559    #[test]
560    #[serial_test::serial]
561    fn test_nocapture_env() {
562        std::env::set_var("IAI_CALLGRIND_NOCAPTURE", "true");
563        let result = CommandLineArgs::parse_from::<[_; 0], &str>([]);
564        assert_eq!(result.nocapture, NoCapture::True);
565    }
566
567    #[rstest]
568    #[case::y("y", true)]
569    #[case::yes("yes", true)]
570    #[case::t("t", true)]
571    #[case::true_value("true", true)]
572    #[case::on("on", true)]
573    #[case::one("1", true)]
574    #[case::n("n", false)]
575    #[case::no("no", false)]
576    #[case::f("f", false)]
577    #[case::false_value("false", false)]
578    #[case::off("off", false)]
579    #[case::zero("0", false)]
580    fn test_boolish(#[case] value: &str, #[case] expected: bool) {
581        let result = CommandLineArgs::parse_from(&[format!("--allow-aslr={value}")]);
582        assert_eq!(result.allow_aslr, Some(expected));
583    }
584}
iai_callgrind_runner/runner/args.rs

iai_callgrind_runner/runner/
args.rs
