iai_callgrind_runner/runner/

bin_bench.rs

use std::collections::HashMap;
use std::ffi::OsString;
use std::io::stderr;
use std::io::ErrorKind::WouldBlock;
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpStream, UdpSocket};
use std::path::{Path, PathBuf};
use std::sync::mpsc::{self, RecvTimeoutError};
use std::time::Duration;
use std::{panic, thread};

use anyhow::{anyhow, Context, Result};
use log::{debug, warn};

use super::args::NoCapture;
use super::callgrind::args::Args;
use super::callgrind::flamegraph::{
    BaselineFlamegraphGenerator, Config as FlamegraphConfig, Flamegraph, FlamegraphGenerator,
    LoadBaselineFlamegraphGenerator, SaveBaselineFlamegraphGenerator,
};
use super::callgrind::parser::CallgrindParser;
use super::callgrind::summary_parser::SummaryParser;
use super::callgrind::{RegressionConfig, Summaries};
use super::common::{Assistant, AssistantKind, Config, ModulePath, Sandbox};
use super::format::{BinaryBenchmarkHeader, Formatter, OutputFormat, VerticalFormatter};
use super::meta::Metadata;
use super::summary::{
    BaselineKind, BaselineName, BenchmarkKind, BenchmarkSummary, CallgrindSummary, MetricsSummary,
    SummaryOutput, ToolRun,
};
use super::tool::{
    RunOptions, ToolCommand, ToolConfig, ToolConfigs, ToolOutputPath, ToolOutputPathKind,
    ValgrindTool,
};
use crate::api::{
    self, BinaryBenchmarkBench, BinaryBenchmarkConfig, BinaryBenchmarkGroups, DelayKind, Stdin,
};
use crate::error::Error;
use crate::runner::format;

mod defaults {
    use crate::api::Stdin;

    pub const COMPARE_BY_ID: bool = false;
    pub const ENV_CLEAR: bool = true;
    pub const REGRESSION_FAIL_FAST: bool = false;
    pub const STDIN: Stdin = Stdin::Pipe;
    pub const WORKSPACE_ROOT_ENV: &str = "_WORKSPACE_ROOT";
}

#[derive(Debug)]
struct BaselineBenchmark {
    baseline_kind: BaselineKind,
}

#[derive(Debug)]
pub struct BinBench {
    pub id: Option<String>,
    pub args: Option<String>,
    pub function_name: String,
    pub command: Command,
    pub run_options: RunOptions,
    pub callgrind_args: Args,
    pub flamegraph_config: Option<FlamegraphConfig>,
    pub regression_config: Option<RegressionConfig>,
    pub tools: ToolConfigs,
    pub setup: Option<Assistant>,
    pub teardown: Option<Assistant>,
    pub sandbox: Option<api::Sandbox>,
    pub module_path: ModulePath,
    pub output_format: OutputFormat,
}

/// The Command we derive from the `api::Command`
///
/// If the path is relative we convert it to an absolute path relative to the workspace root.
/// `stdin`, `stdout`, `stderr` of the `api::Command` are part of the `RunOptions` and not part of
/// this `Command`
#[derive(Debug, Clone)]
pub struct Command {
    pub path: PathBuf,
    pub args: Vec<OsString>,
    pub delay: Option<Delay>,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Delay {
    pub poll: Duration,
    pub timeout: Duration,
    pub kind: DelayKind,
}

#[derive(Debug)]
struct Group {
    /// This name is the name from the `library_benchmark_group!` macro
    ///
    /// Due to the way we expand the `library_benchmark_group!` macro, we can safely assume that
    /// this name is unique.
    name: String,
    /// The module path so far which should be `file_name::group_name`
    module_path: ModulePath,
    benches: Vec<BinBench>,
    setup: Option<Assistant>,
    teardown: Option<Assistant>,
    compare_by_id: bool,
}

#[derive(Debug)]
struct Groups(Vec<Group>);

#[derive(Debug)]
struct LoadBaselineBenchmark {
    loaded_baseline: BaselineName,
    baseline: BaselineName,
}

#[derive(Debug)]
struct Runner {
    groups: Groups,
    config: Config,
    benchmark: Box<dyn Benchmark>,
    setup: Option<Assistant>,
    teardown: Option<Assistant>,
}

#[derive(Debug)]
struct SaveBaselineBenchmark {
    baseline: BaselineName,
}

trait Benchmark: std::fmt::Debug {
    fn output_path(&self, bin_bench: &BinBench, config: &Config, group: &Group) -> ToolOutputPath;
    fn baselines(&self) -> (Option<String>, Option<String>);
    fn run(&self, bin_bench: &BinBench, config: &Config, group: &Group)
        -> Result<BenchmarkSummary>;
}

impl Benchmark for BaselineBenchmark {
    fn output_path(&self, bin_bench: &BinBench, config: &Config, group: &Group) -> ToolOutputPath {
        ToolOutputPath::new(
            ToolOutputPathKind::Out,
            ValgrindTool::Callgrind,
            &self.baseline_kind,
            &config.meta.target_dir,
            &group.module_path,
            &bin_bench.name(),
        )
    }

    fn baselines(&self) -> (Option<String>, Option<String>) {
        match &self.baseline_kind {
            BaselineKind::Old => (None, None),
            BaselineKind::Name(name) => (None, Some(name.to_string())),
        }
    }

    fn run(
        &self,
        bin_bench: &BinBench,
        config: &Config,
        group: &Group,
    ) -> Result<BenchmarkSummary> {
        let header = BinaryBenchmarkHeader::new(&config.meta, bin_bench);
        header.print();

        let callgrind_command = ToolCommand::new(
            ValgrindTool::Callgrind,
            &config.meta,
            config.meta.args.nocapture,
        );

        let tool_config = ToolConfig::new(
            ValgrindTool::Callgrind,
            true,
            bin_bench.callgrind_args.clone(),
            None,
        );

        let out_path = self.output_path(bin_bench, config, group);
        out_path.init()?;
        out_path.shift()?;

        let old_path = out_path.to_base_path();
        let log_path = out_path.to_log_output();
        log_path.shift()?;

        for path in bin_bench.tools.output_paths(&out_path) {
            path.shift()?;
            path.to_log_output().shift()?;
        }

        let mut benchmark_summary = bin_bench.create_benchmark_summary(
            config,
            &out_path,
            &bin_bench.function_name,
            header.description(),
        )?;

        // We're implicitly applying the default here: In the absence of a user provided sandbox we
        // don't run the benchmarks in a sandbox. Everything from here on runs with the current
        // directory set to the sandbox directory until the sandbox is reset.
        let sandbox = bin_bench
            .sandbox
            .as_ref()
            .map(|sandbox| Sandbox::setup(sandbox, &config.meta))
            .transpose()?;

        let mut child = bin_bench
            .setup
            .as_ref()
            .map_or(Ok(None), |setup| setup.run(config, &bin_bench.module_path))?;

        if let Some(delay) = &bin_bench.command.delay {
            if let Err(error) = delay.run() {
                if let Some(mut child) = child.take() {
                    // To avoid zombies
                    child.kill()?;
                    return Err(error);
                }
            }
        }

        let output = callgrind_command.run(
            tool_config,
            &bin_bench.command.path,
            &bin_bench.command.args,
            bin_bench.run_options.clone(),
            &out_path,
            &bin_bench.module_path,
            child,
        )?;

        if let Some(teardown) = &bin_bench.teardown {
            teardown.run(config, &bin_bench.module_path)?;
        }

        // We print the no capture footer after the teardown to keep the output consistent with
        // library benchmarks.
        bin_bench.print_nocapture_footer(config.meta.args.nocapture);

        if let Some(sandbox) = sandbox {
            sandbox.reset()?;
        }

        let parsed_new = SummaryParser.parse(&out_path)?;
        let parsed_old = old_path
            .exists()
            .then(|| SummaryParser.parse(&old_path))
            .transpose()?;

        let summaries = Summaries::new(parsed_new, parsed_old);
        VerticalFormatter::new(bin_bench.output_format).print(
            config,
            self.baselines(),
            &ToolRun::from(&summaries),
        )?;

        output.dump_log(log::Level::Info);
        log_path.dump_log(log::Level::Info, &mut stderr())?;

        let regressions = bin_bench.check_and_print_regressions(&summaries.total);

        let callgrind_summary = benchmark_summary
            .callgrind_summary
            .insert(CallgrindSummary::new(
                log_path.real_paths()?,
                out_path.real_paths()?,
            ));

        callgrind_summary.add_summaries(
            &bin_bench.command.path,
            &bin_bench.command.args,
            &self.baselines(),
            summaries,
            regressions,
        );

        if let Some(flamegraph_config) = bin_bench.flamegraph_config.clone() {
            callgrind_summary.flamegraphs = BaselineFlamegraphGenerator {
                baseline_kind: self.baseline_kind.clone(),
            }
            .create(
                &Flamegraph::new(header.to_title(), flamegraph_config),
                &out_path,
                None,
                &config.meta.project_root,
            )?;
        }

        benchmark_summary.tool_summaries = bin_bench.tools.run(
            config,
            &bin_bench.command.path,
            &bin_bench.command.args,
            &bin_bench.run_options,
            &out_path,
            false,
            &bin_bench.module_path,
            bin_bench.sandbox.as_ref(),
            bin_bench.setup.as_ref(),
            bin_bench.teardown.as_ref(),
            bin_bench.command.delay.as_ref(),
            &bin_bench.output_format,
        )?;

        Ok(benchmark_summary)
    }
}

impl BinBench {
    fn new(
        meta: &Metadata,
        group: &Group,
        config: BinaryBenchmarkConfig,
        group_index: usize,
        bench_index: usize,
        meta_callgrind_args: &api::RawArgs,
        binary_benchmark_bench: BinaryBenchmarkBench,
    ) -> Result<Self> {
        let module_path = group
            .module_path
            .join(&binary_benchmark_bench.function_name);

        let api::Command {
            path,
            args,
            stdin,
            stdout,
            stderr,
            delay,
            ..
        } = binary_benchmark_bench.command;

        let command = Command::new(&module_path, path, args, delay.map(Into::into))?;

        let callgrind_args = Args::try_from_raw_args(&[
            &config.valgrind_args,
            &config.callgrind_args,
            meta_callgrind_args,
        ])?;

        let mut assistant_envs = config.collect_envs();
        assistant_envs.push((
            OsString::from(defaults::WORKSPACE_ROOT_ENV),
            meta.project_root.clone().into(),
        ));

        let command_envs = config.resolve_envs();
        let flamegraph_config = config.flamegraph_config.map(Into::into);
        let mut output_format = config
            .output_format
            .map_or_else(OutputFormat::default, Into::into);
        output_format.kind = meta.args.output_format;

        Ok(Self {
            id: binary_benchmark_bench.id,
            args: binary_benchmark_bench.args,
            function_name: binary_benchmark_bench.function_name,
            callgrind_args,
            flamegraph_config,
            regression_config: api::update_option(
                &config.regression_config,
                &meta.regression_config,
            )
            .map(Into::into),
            tools: ToolConfigs(
                config
                    .tools
                    .0
                    .into_iter()
                    .map(|mut t| {
                        if !config.valgrind_args.is_empty() {
                            let mut new_args = config.valgrind_args.clone();
                            new_args.extend_ignore_flag(t.raw_args.0.iter());
                            t.raw_args = new_args;
                        }
                        t.try_into()
                    })
                    .collect::<Result<Vec<_>, _>>()?,
            ),
            setup: binary_benchmark_bench
                .has_setup
                .then_some(Assistant::new_bench_assistant(
                    AssistantKind::Setup,
                    &group.name,
                    (group_index, bench_index),
                    stdin.as_ref().and_then(|s| {
                        if let Stdin::Setup(p) = s {
                            Some(*p)
                        } else {
                            None
                        }
                    }),
                    assistant_envs.clone(),
                    config.setup_parallel.unwrap_or(false),
                )),
            teardown: binary_benchmark_bench.has_teardown.then_some(
                Assistant::new_bench_assistant(
                    AssistantKind::Teardown,
                    &group.name,
                    (group_index, bench_index),
                    None,
                    assistant_envs,
                    false,
                ),
            ),
            run_options: RunOptions {
                env_clear: config.env_clear.unwrap_or(defaults::ENV_CLEAR),
                envs: command_envs,
                stdin: stdin.or(Some(defaults::STDIN)),
                stdout,
                stderr,
                exit_with: config.exit_with,
                current_dir: config.current_dir,
            },
            sandbox: config.sandbox,
            module_path,
            command,
            output_format,
        })
    }

    fn name(&self) -> String {
        if let Some(bench_id) = &self.id {
            format!("{}.{}", self.function_name, bench_id)
        } else {
            self.function_name.clone()
        }
    }

    fn print_nocapture_footer(&self, nocapture: NoCapture) {
        format::print_no_capture_footer(
            nocapture,
            self.run_options.stdout.as_ref(),
            self.run_options.stderr.as_ref(),
        );
    }

    fn create_benchmark_summary(
        &self,
        config: &Config,
        output_path: &ToolOutputPath,
        function_name: &str,
        description: Option<String>,
    ) -> Result<BenchmarkSummary> {
        let summary_output = if let Some(format) = config.meta.args.save_summary {
            let output = SummaryOutput::new(format, &output_path.dir);
            output.init()?;
            Some(output)
        } else {
            None
        };

        Ok(BenchmarkSummary::new(
            BenchmarkKind::BinaryBenchmark,
            config.meta.project_root.clone(),
            config.package_dir.clone(),
            config.bench_file.clone(),
            self.command.path.clone(),
            &self.module_path,
            function_name,
            self.id.clone(),
            description,
            summary_output,
        ))
    }

    fn check_and_print_regressions(
        &self,
        metrics_summary: &MetricsSummary,
    ) -> Vec<super::summary::CallgrindRegression> {
        if let Some(regression_config) = &self.regression_config {
            regression_config.check_and_print(metrics_summary)
        } else {
            vec![]
        }
    }
}

impl Command {
    fn new(
        module_path: &ModulePath,
        path: PathBuf,
        args: Vec<OsString>,
        delay: Option<Delay>,
    ) -> Result<Self> {
        if path.as_os_str().is_empty() {
            return Err(anyhow!("{module_path}: Empty path in command",));
        }

        Ok(Self { path, args, delay })
    }
}

impl From<api::Delay> for Delay {
    fn from(value: api::Delay) -> Self {
        let (poll, timeout) = if let DelayKind::DurationElapse(_) = value.kind {
            if value.poll.is_some() {
                warn!("Ignoring poll setting. Not supported for {:?}", value.kind);
            }
            if value.timeout.is_some() {
                warn!(
                    "Ignoring timeout setting. Not supported for {:?}",
                    value.kind
                );
            }
            (Duration::ZERO, Duration::ZERO)
        } else {
            let mut poll = value.poll.unwrap_or_else(|| Duration::from_millis(10));
            let timeout = value.timeout.map_or_else(
                || Duration::from_secs(600),
                |t| {
                    if t < Duration::from_millis(10) {
                        warn!("The minimum timeout setting is 10ms");
                        Duration::from_millis(10)
                    } else {
                        t
                    }
                },
            );

            if poll >= timeout {
                warn!(
                    "Poll duration is equal to or greater than the timeout duration ({:?} >= \
                     {:?}).",
                    poll, timeout
                );
                poll = timeout - Duration::from_millis(5);
                warn!("Using poll duration {:?} instead", poll);
            }
            (poll, timeout)
        };

        Self {
            poll,
            timeout,
            kind: value.kind,
        }
    }
}

impl Delay {
    pub fn new(poll: Duration, timeout: Duration, kind: DelayKind) -> Self {
        Self {
            poll,
            timeout,
            kind,
        }
    }

    pub fn run(&self) -> Result<()> {
        if let DelayKind::DurationElapse(_) = self.kind {
            self.exec_delay_fn()
        } else {
            let (tx, rx) = mpsc::channel::<std::result::Result<(), anyhow::Error>>();

            let delay = self.clone();
            let handle = thread::spawn(move || {
                tx.send(delay.exec_delay_fn()).map_err(|error| {
                    anyhow!("Command::Delay MPSC channel send error. Error: {error:?}")
                })
            });

            match rx.recv_timeout(self.timeout) {
                Ok(result) => {
                    // These unwraps are safe
                    handle.join().unwrap().unwrap();
                    result.map(|()| debug!("Command::Delay successfully executed."))
                }
                Err(RecvTimeoutError::Timeout) => {
                    Err(anyhow!("Timeout of '{:?}' reached", self.timeout))
                }
                Err(RecvTimeoutError::Disconnected) => {
                    // The disconnect is caused by a panic in the thread, so the `unwrap_err` is
                    // safe. We propagate the panic as is.
                    panic::resume_unwind(handle.join().unwrap_err())
                }
            }
        }
    }

    fn exec_delay_fn(&self) -> Result<()> {
        match &self.kind {
            DelayKind::DurationElapse(duration) => {
                thread::sleep(*duration);
            }
            DelayKind::TcpConnect(addr) => {
                while let Err(_err) = TcpStream::connect(addr) {
                    thread::sleep(self.poll);
                }
            }
            DelayKind::UdpResponse(remote, req) => {
                let socket = match remote {
                    SocketAddr::V4(_) => {
                        UdpSocket::bind(SocketAddr::new(Ipv4Addr::UNSPECIFIED.into(), 0))
                            .context("Could not bind local IPv4 UDP socket.")?
                    }
                    SocketAddr::V6(_) => {
                        UdpSocket::bind(SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 0))
                            .context("Could not bind local IPv6 UDP socket.")?
                    }
                };

                socket.set_read_timeout(Some(self.poll))?;
                socket.set_write_timeout(Some(self.poll))?;

                loop {
                    while let Err(_err) = socket.send_to(req.as_slice(), remote) {
                        thread::sleep(self.poll);
                    }

                    let mut buf = [0; 1];
                    match socket.recv(&mut buf) {
                        Ok(_size) => break,
                        Err(e) => {
                            if e.kind() != WouldBlock {
                                thread::sleep(self.poll);
                            }
                        }
                    }
                }
            }
            DelayKind::PathExists(path) => {
                let wait_for_path = std::path::PathBuf::from(Path::new(path));
                while !wait_for_path.exists() {
                    thread::sleep(self.poll);
                }
            }
        }

        Ok(())
    }
}

impl Group {
    fn run(
        &self,
        benchmark: &dyn Benchmark,
        is_regressed: &mut bool,
        config: &Config,
    ) -> Result<()> {
        let mut summaries: HashMap<String, Vec<BenchmarkSummary>> =
            HashMap::with_capacity(self.benches.len());
        for bench in &self.benches {
            let fail_fast = bench
                .regression_config
                .as_ref()
                .map_or(defaults::REGRESSION_FAIL_FAST, |r| r.fail_fast);

            let summary = benchmark.run(bench, config, self)?;
            summary.print_and_save(&config.meta.args.output_format)?;
            summary.check_regression(is_regressed, fail_fast)?;

            if self.compare_by_id && bench.output_format.is_default() {
                if let Some(id) = &summary.id {
                    if let Some(sums) = summaries.get_mut(id) {
                        for sum in sums.iter() {
                            sum.compare_and_print(id, &summary, &bench.output_format)?;
                        }
                        sums.push(summary);
                    } else {
                        summaries.insert(id.clone(), vec![summary]);
                    }
                }
            }
        }

        Ok(())
    }
}

impl Groups {
    fn from_binary_benchmark(
        module: &ModulePath,
        benchmark_groups: BinaryBenchmarkGroups,
        meta: &Metadata,
    ) -> Result<Self> {
        let global_config = benchmark_groups.config;
        let meta_callgrind_args = meta.args.callgrind_args.clone().unwrap_or_default();

        let mut groups = vec![];
        for binary_benchmark_group in benchmark_groups.groups {
            let group_module_path = module.join(&binary_benchmark_group.id);
            let group_config = global_config
                .clone()
                .update_from_all([binary_benchmark_group.config.as_ref()]);

            let setup = binary_benchmark_group
                .has_setup
                .then_some(Assistant::new_group_assistant(
                    AssistantKind::Setup,
                    &binary_benchmark_group.id,
                    group_config.collect_envs(),
                    false,
                ));
            let teardown =
                binary_benchmark_group
                    .has_teardown
                    .then_some(Assistant::new_group_assistant(
                        AssistantKind::Teardown,
                        &binary_benchmark_group.id,
                        group_config.collect_envs(),
                        false,
                    ));

            let mut group = Group {
                name: binary_benchmark_group.id,
                module_path: group_module_path,
                benches: vec![],
                setup,
                teardown,
                compare_by_id: binary_benchmark_group
                    .compare_by_id
                    .unwrap_or(defaults::COMPARE_BY_ID),
            };

            for (group_index, binary_benchmark_benches) in binary_benchmark_group
                .binary_benchmarks
                .into_iter()
                .enumerate()
            {
                for (bench_index, binary_benchmark_bench) in
                    binary_benchmark_benches.benches.into_iter().enumerate()
                {
                    let config = group_config.clone().update_from_all([
                        binary_benchmark_benches.config.as_ref(),
                        binary_benchmark_bench.config.as_ref(),
                        Some(&binary_benchmark_bench.command.config),
                    ]);

                    let bin_bench = BinBench::new(
                        meta,
                        &group,
                        config,
                        group_index,
                        bench_index,
                        &meta_callgrind_args,
                        binary_benchmark_bench,
                    )?;
                    group.benches.push(bin_bench);
                }
            }

            groups.push(group);
        }
        Ok(Self(groups))
    }

    /// Run all [`Group`] benchmarks
    ///
    /// # Errors
    ///
    /// Return an [`anyhow::Error`] with sources:
    ///
    /// * [`Error::RegressionError`] if a regression occurred.
    fn run(&self, benchmark: &dyn Benchmark, config: &Config) -> Result<()> {
        let mut is_regressed = false;
        for group in &self.0 {
            if let Some(setup) = &group.setup {
                setup.run(config, &group.module_path)?;
            }

            group.run(benchmark, &mut is_regressed, config)?;

            if let Some(teardown) = &group.teardown {
                teardown.run(config, &group.module_path)?;
            }
        }

        if is_regressed {
            Err(Error::RegressionError(false).into())
        } else {
            Ok(())
        }
    }
}

impl Benchmark for LoadBaselineBenchmark {
    fn output_path(&self, bin_bench: &BinBench, config: &Config, group: &Group) -> ToolOutputPath {
        ToolOutputPath::new(
            ToolOutputPathKind::Base(self.loaded_baseline.to_string()),
            ValgrindTool::Callgrind,
            &BaselineKind::Name(self.baseline.clone()),
            &config.meta.target_dir,
            &group.module_path,
            &bin_bench.name(),
        )
    }

    fn baselines(&self) -> (Option<String>, Option<String>) {
        (
            Some(self.loaded_baseline.to_string()),
            Some(self.baseline.to_string()),
        )
    }

    fn run(
        &self,
        bin_bench: &BinBench,
        config: &Config,
        group: &Group,
    ) -> Result<BenchmarkSummary> {
        let header = BinaryBenchmarkHeader::new(&config.meta, bin_bench);
        header.print();

        let out_path = self.output_path(bin_bench, config, group);
        let old_path = out_path.to_base_path();
        let log_path = out_path.to_log_output();

        let mut benchmark_summary = bin_bench.create_benchmark_summary(
            config,
            &out_path,
            &bin_bench.function_name,
            header.description(),
        )?;

        let parsed_new = SummaryParser.parse(&out_path)?;
        let parsed_old = Some(SummaryParser.parse(&old_path)?);
        let summaries = Summaries::new(parsed_new, parsed_old);

        VerticalFormatter::new(bin_bench.output_format).print(
            config,
            self.baselines(),
            &ToolRun::from(&summaries),
        )?;

        let regressions = bin_bench.check_and_print_regressions(&summaries.total);

        let callgrind_summary = benchmark_summary
            .callgrind_summary
            .insert(CallgrindSummary::new(
                log_path.real_paths()?,
                out_path.real_paths()?,
            ));

        callgrind_summary.add_summaries(
            &bin_bench.command.path,
            &bin_bench.command.args,
            &self.baselines(),
            summaries,
            regressions,
        );

        if let Some(flamegraph_config) = bin_bench.flamegraph_config.clone() {
            callgrind_summary.flamegraphs = LoadBaselineFlamegraphGenerator {
                loaded_baseline: self.loaded_baseline.clone(),
                baseline: self.baseline.clone(),
            }
            .create(
                &Flamegraph::new(header.to_title(), flamegraph_config),
                &out_path,
                None,
                &config.meta.project_root,
            )?;
        }

        benchmark_summary.tool_summaries =
            bin_bench
                .tools
                .run_loaded_vs_base(config, &out_path, &bin_bench.output_format)?;

        Ok(benchmark_summary)
    }
}

impl Runner {
    fn new(benchmark_groups: BinaryBenchmarkGroups, config: Config) -> Result<Self> {
        let setup = benchmark_groups
            .has_setup
            .then_some(Assistant::new_main_assistant(
                AssistantKind::Setup,
                benchmark_groups.config.collect_envs(),
                false,
            ));
        let teardown = benchmark_groups
            .has_teardown
            .then_some(Assistant::new_main_assistant(
                AssistantKind::Teardown,
                benchmark_groups.config.collect_envs(),
                false,
            ));

        let groups =
            Groups::from_binary_benchmark(&config.module_path, benchmark_groups, &config.meta)?;

        let benchmark: Box<dyn Benchmark> =
            if let Some(baseline_name) = &config.meta.args.save_baseline {
                Box::new(SaveBaselineBenchmark {
                    baseline: baseline_name.clone(),
                })
            } else if let Some(baseline_name) = &config.meta.args.load_baseline {
                Box::new(LoadBaselineBenchmark {
                    loaded_baseline: baseline_name.clone(),
                    baseline: config
                        .meta
                        .args
                        .baseline
                        .as_ref()
                        .expect("A baseline should be present")
                        .clone(),
                })
            } else {
                Box::new(BaselineBenchmark {
                    baseline_kind: config
                        .meta
                        .args
                        .baseline
                        .as_ref()
                        .map_or(BaselineKind::Old, |name| BaselineKind::Name(name.clone())),
                })
            };

        Ok(Self {
            groups,
            config,
            benchmark,
            setup,
            teardown,
        })
    }

    fn run(&self) -> Result<()> {
        if let Some(setup) = &self.setup {
            setup.run(&self.config, &self.config.module_path)?;
        }

        self.groups.run(self.benchmark.as_ref(), &self.config)?;

        if let Some(teardown) = &self.teardown {
            teardown.run(&self.config, &self.config.module_path)?;
        }
        Ok(())
    }
}

impl Benchmark for SaveBaselineBenchmark {
    fn output_path(&self, bin_bench: &BinBench, config: &Config, group: &Group) -> ToolOutputPath {
        ToolOutputPath::new(
            ToolOutputPathKind::Base(self.baseline.to_string()),
            ValgrindTool::Callgrind,
            &BaselineKind::Name(self.baseline.clone()),
            &config.meta.target_dir,
            &group.module_path,
            &bin_bench.name(),
        )
    }

    fn baselines(&self) -> (Option<String>, Option<String>) {
        (
            Some(self.baseline.to_string()),
            Some(self.baseline.to_string()),
        )
    }

    fn run(
        &self,
        bin_bench: &BinBench,
        config: &Config,
        group: &Group,
    ) -> Result<BenchmarkSummary> {
        let header = BinaryBenchmarkHeader::new(&config.meta, bin_bench);
        header.print();

        let callgrind_command = ToolCommand::new(
            ValgrindTool::Callgrind,
            &config.meta,
            config.meta.args.nocapture,
        );

        let tool_config = ToolConfig::new(
            ValgrindTool::Callgrind,
            true,
            bin_bench.callgrind_args.clone(),
            None,
        );

        let out_path = self.output_path(bin_bench, config, group);
        out_path.init()?;

        let parsed_old = out_path
            .exists()
            .then(|| {
                SummaryParser
                    .parse(&out_path)
                    .and_then(|parsed| out_path.clear().map(|()| parsed))
            })
            .transpose()?;

        let log_path = out_path.to_log_output();
        log_path.clear()?;

        let mut benchmark_summary = bin_bench.create_benchmark_summary(
            config,
            &out_path,
            &bin_bench.function_name,
            header.description(),
        )?;

        let sandbox = bin_bench
            .sandbox
            .as_ref()
            .map(|sandbox| Sandbox::setup(sandbox, &config.meta))
            .transpose()?;

        let mut child = bin_bench
            .setup
            .as_ref()
            .map_or(Ok(None), |setup| setup.run(config, &bin_bench.module_path))?;

        if let Some(delay) = &bin_bench.command.delay {
            if let Err(error) = delay.run() {
                if let Some(mut child) = child.take() {
                    // To avoid zombies
                    child.kill()?;
                    return Err(error);
                }
            }
        }

        let output = callgrind_command.run(
            tool_config,
            &bin_bench.command.path,
            &bin_bench.command.args,
            bin_bench.run_options.clone(),
            &out_path,
            &bin_bench.module_path,
            child,
        )?;

        if let Some(teardown) = &bin_bench.teardown {
            teardown.run(config, &bin_bench.module_path)?;
        }

        bin_bench.print_nocapture_footer(config.meta.args.nocapture);

        if let Some(sandbox) = sandbox {
            sandbox.reset()?;
        }

        let parsed_new = SummaryParser.parse(&out_path)?;
        let summaries = Summaries::new(parsed_new, parsed_old);
        VerticalFormatter::new(bin_bench.output_format).print(
            config,
            self.baselines(),
            &ToolRun::from(&summaries),
        )?;

        output.dump_log(log::Level::Info);
        log_path.dump_log(log::Level::Info, &mut stderr())?;

        let regressions = bin_bench.check_and_print_regressions(&summaries.total);

        let callgrind_summary = benchmark_summary
            .callgrind_summary
            .insert(CallgrindSummary::new(
                log_path.real_paths()?,
                out_path.real_paths()?,
            ));

        callgrind_summary.add_summaries(
            &bin_bench.command.path,
            &bin_bench.command.args,
            &self.baselines(),
            summaries,
            regressions,
        );

        if let Some(flamegraph_config) = bin_bench.flamegraph_config.clone() {
            callgrind_summary.flamegraphs = SaveBaselineFlamegraphGenerator {
                baseline: self.baseline.clone(),
            }
            .create(
                &Flamegraph::new(header.to_title(), flamegraph_config),
                &out_path,
                None,
                &config.meta.project_root,
            )?;
        }

        benchmark_summary.tool_summaries = bin_bench.tools.run(
            config,
            &bin_bench.command.path,
            &bin_bench.command.args,
            &bin_bench.run_options,
            &out_path,
            true,
            &bin_bench.module_path,
            bin_bench.sandbox.as_ref(),
            bin_bench.setup.as_ref(),
            bin_bench.teardown.as_ref(),
            bin_bench.command.delay.as_ref(),
            &bin_bench.output_format,
        )?;

        Ok(benchmark_summary)
    }
}

pub fn run(benchmark_groups: BinaryBenchmarkGroups, config: Config) -> Result<()> {
    Runner::new(benchmark_groups, config)?.run()
}

#[cfg(test)]
mod tests {
    use std::fs::File;
    use std::net::TcpListener;

    use pretty_assertions::assert_eq;
    use rstest::rstest;
    use tempfile::tempdir;

    use super::*;

    fn api_delay_fixture<T, U>(poll: T, timeout: U, kind: DelayKind) -> api::Delay
    where
        T: Into<Option<u64>>,
        U: Into<Option<u64>>,
    {
        api::Delay {
            poll: poll.into().map(Duration::from_millis),
            timeout: timeout.into().map(Duration::from_millis),
            kind,
        }
    }

    #[rstest]
    #[case::duration_elapse_when_no_poll_no_timeout(
        api_delay_fixture(None, None, DelayKind::DurationElapse(Duration::from_millis(100))),
        Duration::ZERO,
        Duration::ZERO
    )]
    #[case::duration_elapse_when_poll_no_timeout(
        api_delay_fixture(10, None, DelayKind::DurationElapse(Duration::from_millis(100))),
        Duration::ZERO,
        Duration::ZERO
    )]
    #[case::duration_elapse_when_no_poll_but_timeout(
        api_delay_fixture(None, 10, DelayKind::DurationElapse(Duration::from_millis(100))),
        Duration::ZERO,
        Duration::ZERO
    )]
    #[case::duration_elapse_when_poll_and_timeout(
        api_delay_fixture(10, 100, DelayKind::DurationElapse(Duration::from_millis(100))),
        Duration::ZERO,
        Duration::ZERO
    )]
    #[case::path_when_no_poll_no_timeout(
        api_delay_fixture(None, None, DelayKind::PathExists(PathBuf::from("/some/path"))),
        Duration::from_millis(10),
        Duration::from_secs(600)
    )]
    #[case::path_when_poll_no_timeout(
        api_delay_fixture(20, None, DelayKind::PathExists(PathBuf::from("/some/path"))),
        Duration::from_millis(20),
        Duration::from_secs(600)
    )]
    #[case::path_when_no_poll_but_timeout(
        api_delay_fixture(None, 200, DelayKind::PathExists(PathBuf::from("/some/path"))),
        Duration::from_millis(10),
        Duration::from_millis(200)
    )]
    #[case::path_when_poll_and_timeout(
        api_delay_fixture(20, 200, DelayKind::PathExists(PathBuf::from("/some/path"))),
        Duration::from_millis(20),
        Duration::from_millis(200)
    )]
    #[case::path_when_poll_equal_to_timeout(
        api_delay_fixture(200, 200, DelayKind::PathExists(PathBuf::from("/some/path"))),
        Duration::from_millis(195),
        Duration::from_millis(200)
    )]
    #[case::path_when_poll_higher_than_timeout(
        api_delay_fixture(201, 200, DelayKind::PathExists(PathBuf::from("/some/path"))),
        Duration::from_millis(195),
        Duration::from_millis(200)
    )]
    #[case::path_when_poll_equal_to_timeout_smaller_than_10(
        api_delay_fixture(10, 9, DelayKind::PathExists(PathBuf::from("/some/path"))),
        Duration::from_millis(5),
        Duration::from_millis(10)
    )]
    #[case::path_when_poll_lower_than_timeout_smaller_than_10(
        api_delay_fixture(7, 9, DelayKind::PathExists(PathBuf::from("/some/path"))),
        Duration::from_millis(7),
        Duration::from_millis(10)
    )]
    fn test_from_api_delay_for_delay(
        #[case] delay: api::Delay,
        #[case] poll: Duration,
        #[case] timeout: Duration,
    ) {
        let expected = Delay::new(poll, timeout, delay.kind.clone());
        assert_eq!(Delay::from(delay), expected);
    }

    #[test]
    fn test_delay_path() {
        let dir = tempdir().unwrap();
        let file_path = dir.path().join("file.pid");

        let delay = Delay {
            poll: Duration::from_millis(50),
            timeout: Duration::from_millis(200),
            kind: DelayKind::PathExists(file_path.clone()),
        };
        let handle = thread::spawn(move || {
            delay.run().unwrap();
        });

        thread::sleep(Duration::from_millis(100));
        File::create(file_path).unwrap();

        handle.join().unwrap();
        drop(dir);
    }

    #[test]
    fn test_delay_tcp_connect() {
        let addr = "127.0.0.1:32000".parse::<SocketAddr>().unwrap();
        let _listener = TcpListener::bind(addr).unwrap();

        let delay = Delay {
            poll: Duration::from_millis(20),
            timeout: Duration::from_secs(1),
            kind: DelayKind::TcpConnect(addr),
        };
        delay.run().unwrap();
    }

    #[test]
    fn test_delay_tcp_connect_poll() {
        let addr = "127.0.0.1:32001".parse::<SocketAddr>().unwrap();

        let check_addr = addr;
        let handle = thread::spawn(move || {
            let delay = Delay {
                poll: Duration::from_millis(20),
                timeout: Duration::from_secs(1),
                kind: DelayKind::TcpConnect(check_addr),
            };
            delay.run().unwrap();
        });

        thread::sleep(Duration::from_millis(100));
        let _listener = TcpListener::bind(addr).unwrap();

        handle.join().unwrap();
    }

    #[test]
    fn test_delay_tcp_connect_timeout() {
        let addr = "127.0.0.1:32002".parse::<SocketAddr>().unwrap();
        let delay = Delay {
            poll: Duration::from_millis(20),
            timeout: Duration::from_secs(1),
            kind: DelayKind::TcpConnect(addr),
        };

        let result = delay.run();
        assert!(result.is_err());
        assert_eq!(result.unwrap_err().to_string(), "Timeout of '1s' reached");
    }

    #[test]
    fn test_delay_udp_response() {
        let addr = "127.0.0.1:34000".parse::<SocketAddr>().unwrap();

        thread::spawn(move || {
            let server = UdpSocket::bind(addr).unwrap();
            server
                .set_read_timeout(Some(Duration::from_millis(100)))
                .unwrap();
            server
                .set_write_timeout(Some(Duration::from_millis(100)))
                .unwrap();

            loop {
                let mut buf = [0; 1];

                match server.recv_from(&mut buf) {
                    Ok((_size, from)) => {
                        server.send_to(&[2], from).unwrap();
                    }
                    Err(_e) => {}
                }
            }
        });

        let delay = Delay {
            poll: Duration::from_millis(20),
            timeout: Duration::from_millis(100),
            kind: DelayKind::UdpResponse(addr, vec![1]),
        };

        delay.run().unwrap();
    }

    #[test]
    fn test_delay_udp_response_poll() {
        let addr = "127.0.0.1:34001".parse::<SocketAddr>().unwrap();

        thread::spawn(move || {
            let delay = Delay {
                poll: Duration::from_millis(20),
                timeout: Duration::from_millis(100),
                kind: DelayKind::UdpResponse(addr, vec![1]),
            };
            delay.run().unwrap();
        });

        let server = UdpSocket::bind(addr).unwrap();
        server
            .set_read_timeout(Some(Duration::from_millis(100)))
            .unwrap();
        server
            .set_write_timeout(Some(Duration::from_millis(100)))
            .unwrap();

        loop {
            let mut buf = [0; 1];

            thread::sleep(Duration::from_millis(70));

            match server.recv_from(&mut buf) {
                Ok((_size, from)) => {
                    server.send_to(&[2], from).unwrap();
                    break;
                }
                Err(_e) => {}
            }
        }
    }

    #[test]
    fn test_delay_udp_response_timeout() {
        let addr = "127.0.0.1:34002".parse::<SocketAddr>().unwrap();
        let delay = Delay {
            poll: Duration::from_millis(20),
            timeout: Duration::from_millis(100),
            kind: DelayKind::UdpResponse(addr, vec![1]),
        };
        let result = delay.run();
        assert!(result.is_err());
        assert_eq!(
            result.unwrap_err().to_string(),
            "Timeout of '100ms' reached"
        );
    }
}