use crate::analysis;
use crate::benchmark::PartialBenchmarkConfig;
use crate::connection::OutgoingMessage;
use crate::measurement::Measurement;
use crate::report::BenchmarkId as InternalBenchmarkId;
use crate::report::Report;
use crate::report::ReportContext;
use crate::routine::{Function, Routine};
use crate::{Bencher, Criterion, DurationExt, Mode, PlotConfiguration, SamplingMode, Throughput};
use std::time::Duration;
/// Structure used to group together a set of related benchmarks, along with custom configuration
/// settings for groups of benchmarks. All benchmarks performed using a benchmark group will be
/// grouped together in the final report.
///
/// # Examples:
///
/// ```no_run
/// #[macro_use] extern crate criterion;
/// use self::criterion::*;
/// use std::time::Duration;
///
/// fn bench_simple(c: &mut Criterion) {
/// let mut group = c.benchmark_group("My Group");
///
/// // Now we can perform benchmarks with this group
/// group.bench_function("Bench 1", |b| b.iter(|| 1 ));
/// group.bench_function("Bench 2", |b| b.iter(|| 2 ));
///
/// // It's recommended to call group.finish() explicitly at the end, but if you don't it will
/// // be called automatically when the group is dropped.
/// group.finish();
/// }
///
/// fn bench_nested(c: &mut Criterion) {
/// let mut group = c.benchmark_group("My Second Group");
/// // We can override the configuration on a per-group level
/// group.measurement_time(Duration::from_secs(1));
///
/// // We can also use loops to define multiple benchmarks, even over multiple dimensions.
/// for x in 0..3 {
/// for y in 0..3 {
/// let point = (x, y);
/// let parameter_string = format!("{} * {}", x, y);
/// group.bench_with_input(BenchmarkId::new("Multiply", parameter_string), &point,
/// |b, (p_x, p_y)| b.iter(|| p_x * p_y));
/// }
/// }
///
/// group.finish();
/// }
///
/// fn bench_throughput(c: &mut Criterion) {
/// let mut group = c.benchmark_group("Summation");
///
/// for size in [1024, 2048, 4096].iter() {
/// // Generate input of an appropriate size...
/// let input = vec![1u64, *size];
///
/// // We can use the throughput function to tell Criterion.rs how large the input is
/// // so it can calculate the overall throughput of the function. If we wanted, we could
/// // even change the benchmark configuration for different inputs (eg. to reduce the
/// // number of samples for extremely large and slow inputs) or even different functions.
/// group.throughput(Throughput::Elements(*size as u64));
///
/// group.bench_with_input(BenchmarkId::new("sum", *size), &input,
/// |b, i| b.iter(|| i.iter().sum::<u64>()));
/// group.bench_with_input(BenchmarkId::new("fold", *size), &input,
/// |b, i| b.iter(|| i.iter().fold(0u64, |a, b| a + b)));
/// }
///
/// group.finish();
/// }
///
/// criterion_group!(benches, bench_simple, bench_nested, bench_throughput);
/// criterion_main!(benches);
/// ```
pub struct BenchmarkGroup<'a, M: Measurement> {
criterion: &'a mut Criterion<M>,
group_name: String,
all_ids: Vec<InternalBenchmarkId>,
any_matched: bool,
partial_config: PartialBenchmarkConfig,
throughput: Option<Throughput>,
}
impl<'a, M: Measurement> BenchmarkGroup<'a, M> {
/// Changes the size of the sample for this benchmark
///
/// A bigger sample should yield more accurate results if paired with a sufficiently large
/// measurement time.
///
/// Sample size must be at least 10.
///
/// # Panics
///
/// Panics if n < 10.
pub fn sample_size(&mut self, n: usize) -> &mut Self {
assert!(n >= 10);
self.partial_config.sample_size = Some(n);
self
}
/// Changes the warm up time for this benchmark
///
/// # Panics
///
/// Panics if the input duration is zero
pub fn warm_up_time(&mut self, dur: Duration) -> &mut Self {
assert!(dur.to_nanos() > 0);
self.partial_config.warm_up_time = Some(dur);
self
}
/// Changes the target measurement time for this benchmark group.
///
/// Criterion will attempt to spent approximately this amount of time measuring each
/// benchmark on a best-effort basis. If it is not possible to perform the measurement in
/// the requested time (eg. because each iteration of the benchmark is long) then Criterion
/// will spend as long as is needed to collect the desired number of samples. With a longer
/// time, the measurement will become more resilient to interference from other programs.
///
/// # Panics
///
/// Panics if the input duration is zero
pub fn measurement_time(&mut self, dur: Duration) -> &mut Self {
assert!(dur.to_nanos() > 0);
self.partial_config.measurement_time = Some(dur);
self
}
/// Changes the number of resamples for this benchmark group
///
/// Number of resamples to use for the
/// [bootstrap](http://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Case_resampling)
///
/// A larger number of resamples reduces the random sampling errors which are inherent to the
/// bootstrap method, but also increases the analysis time.
///
/// # Panics
///
/// Panics if the number of resamples is set to zero
pub fn nresamples(&mut self, n: usize) -> &mut Self {
assert!(n > 0);
if n <= 1000 {
println!("\nWarning: It is not recommended to reduce nresamples below 1000.");
}
self.partial_config.nresamples = Some(n);
self
}
/// Changes the noise threshold for benchmarks in this group. The noise threshold
/// is used to filter out small changes in performance from one run to the next, even if they
/// are statistically significant. Sometimes benchmarking the same code twice will result in
/// small but statistically significant differences solely because of noise. This provides a way
/// to filter out some of these false positives at the cost of making it harder to detect small
/// changes to the true performance of the benchmark.
///
/// The default is 0.01, meaning that changes smaller than 1% will be ignored.
///
/// # Panics
///
/// Panics if the threshold is set to a negative value
pub fn noise_threshold(&mut self, threshold: f64) -> &mut Self {
assert!(threshold >= 0.0);
self.partial_config.noise_threshold = Some(threshold);
self
}
/// Changes the confidence level for benchmarks in this group. The confidence
/// level is the desired probability that the true runtime lies within the estimated
/// [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval). The default is
/// 0.95, meaning that the confidence interval should capture the true value 95% of the time.
///
/// # Panics
///
/// Panics if the confidence level is set to a value outside the `(0, 1)` range
pub fn confidence_level(&mut self, cl: f64) -> &mut Self {
assert!(cl > 0.0 && cl < 1.0);
if cl < 0.5 {
println!("\nWarning: It is not recommended to reduce confidence level below 0.5.");
}
self.partial_config.confidence_level = Some(cl);
self
}
/// Changes the [significance level](https://en.wikipedia.org/wiki/Statistical_significance)
/// for benchmarks in this group. This is used to perform a
/// [hypothesis test](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) to see if
/// the measurements from this run are different from the measured performance of the last run.
/// The significance level is the desired probability that two measurements of identical code
/// will be considered 'different' due to noise in the measurements. The default value is 0.05,
/// meaning that approximately 5% of identical benchmarks will register as different due to
/// noise.
///
/// This presents a trade-off. By setting the significance level closer to 0.0, you can increase
/// the statistical robustness against noise, but it also weakens Criterion.rs' ability to
/// detect small but real changes in the performance. By setting the significance level
/// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also
/// report more spurious differences.
///
/// See also the noise threshold setting.
///
/// # Panics
///
/// Panics if the significance level is set to a value outside the `(0, 1)` range
pub fn significance_level(&mut self, sl: f64) -> &mut Self {
assert!(sl > 0.0 && sl < 1.0);
self.partial_config.significance_level = Some(sl);
self
}
/// Changes the plot configuration for this benchmark group.
pub fn plot_config(&mut self, new_config: PlotConfiguration) -> &mut Self {
self.partial_config.plot_config = new_config;
self
}
/// Set the input size for this benchmark group. Used for reporting the
/// throughput.
pub fn throughput(&mut self, throughput: Throughput) -> &mut Self {
self.throughput = Some(throughput);
self
}
/// Set the sampling mode for this benchmark group.
pub fn sampling_mode(&mut self, new_mode: SamplingMode) -> &mut Self {
self.partial_config.sampling_mode = Some(new_mode);
self
}
pub(crate) fn new(criterion: &mut Criterion<M>, group_name: String) -> BenchmarkGroup<'_, M> {
BenchmarkGroup {
criterion,
group_name,
all_ids: vec![],
any_matched: false,
partial_config: PartialBenchmarkConfig::default(),
throughput: None,
}
}
/// Benchmark the given parameterless function inside this benchmark group.
pub fn bench_function<ID: IntoBenchmarkId, F>(&mut self, id: ID, mut f: F) -> &mut Self
where
F: FnMut(&mut Bencher<'_, M>),
{
self.run_bench(id.into_benchmark_id(), &(), |b, _| f(b));
self
}
/// Benchmark the given parameterized function inside this benchmark group.
pub fn bench_with_input<ID: IntoBenchmarkId, F, I>(
&mut self,
id: ID,
input: &I,
f: F,
) -> &mut Self
where
F: FnMut(&mut Bencher<'_, M>, &I),
I: ?Sized,
{
self.run_bench(id.into_benchmark_id(), input, f);
self
}
fn run_bench<F, I>(&mut self, id: BenchmarkId, input: &I, f: F)
where
F: FnMut(&mut Bencher<'_, M>, &I),
I: ?Sized,
{
let config = self.partial_config.to_complete(&self.criterion.config);
let report_context = ReportContext {
output_directory: self.criterion.output_directory.clone(),
plot_config: self.partial_config.plot_config.clone(),
};
let mut id = InternalBenchmarkId::new(
self.group_name.clone(),
id.function_name,
id.parameter,
self.throughput.clone(),
);
assert!(
!self.all_ids.contains(&id),
"Benchmark IDs must be unique within a group."
);
id.ensure_directory_name_unique(&self.criterion.all_directories);
self.criterion
.all_directories
.insert(id.as_directory_name().to_owned());
id.ensure_title_unique(&self.criterion.all_titles);
self.criterion.all_titles.insert(id.as_title().to_owned());
let do_run = self.criterion.filter_matches(id.id());
self.any_matched |= do_run;
let mut func = Function::new(f);
match self.criterion.mode {
Mode::Benchmark => {
if let Some(conn) = &self.criterion.connection {
if do_run {
conn.send(&OutgoingMessage::BeginningBenchmark { id: (&id).into() })
.unwrap();
} else {
conn.send(&OutgoingMessage::SkippingBenchmark { id: (&id).into() })
.unwrap();
}
}
if do_run {
analysis::common(
&id,
&mut func,
&config,
self.criterion,
&report_context,
input,
self.throughput.clone(),
);
}
}
Mode::List => {
if do_run {
println!("{}: bench", id);
}
}
Mode::Test => {
if do_run {
// In test mode, run the benchmark exactly once, then exit.
self.criterion.report.test_start(&id, &report_context);
func.test(&self.criterion.measurement, input);
self.criterion.report.test_pass(&id, &report_context);
}
}
Mode::Profile(duration) => {
if do_run {
func.profile(
&self.criterion.measurement,
&id,
self.criterion,
&report_context,
duration,
input,
);
}
}
}
self.all_ids.push(id);
}
/// Consume the benchmark group and generate the summary reports for the group.
///
/// It is recommended to call this explicitly, but if you forget it will be called when the
/// group is dropped.
pub fn finish(self) {
::std::mem::drop(self);
}
}
impl<'a, M: Measurement> Drop for BenchmarkGroup<'a, M> {
fn drop(&mut self) {
// I don't really like having a bunch of non-trivial code in drop, but this is the only way
// to really write linear types like this in Rust...
if let Some(conn) = &mut self.criterion.connection {
conn.send(&OutgoingMessage::FinishedBenchmarkGroup {
group: &self.group_name,
})
.unwrap();
conn.serve_value_formatter(self.criterion.measurement.formatter())
.unwrap();
}
if self.all_ids.len() > 1 && self.any_matched && self.criterion.mode.is_benchmark() {
let report_context = ReportContext {
output_directory: self.criterion.output_directory.clone(),
plot_config: self.partial_config.plot_config.clone(),
};
self.criterion.report.summarize(
&report_context,
&self.all_ids,
self.criterion.measurement.formatter(),
);
}
if self.any_matched {
self.criterion.report.group_separator();
}
}
}
/// Simple structure representing an ID for a benchmark. The ID must be unique within a benchmark
/// group.
#[derive(Clone, Eq, PartialEq, Hash)]
pub struct BenchmarkId {
pub(crate) function_name: Option<String>,
pub(crate) parameter: Option<String>,
}
impl BenchmarkId {
/// Construct a new benchmark ID from a string function name and a parameter value.
///
/// Note that the parameter value need not be the same as the parameter passed to your
/// actual benchmark. For instance, you might have a benchmark that takes a 1MB string as
/// input. It would be impractical to embed the whole string in the benchmark ID, so instead
/// your parameter value might be a descriptive string like "1MB Alphanumeric".
///
/// # Examples
/// ```
/// # use criterion::{BenchmarkId, Criterion};
/// // A basic benchmark ID is typically constructed from a constant string and a simple
/// // parameter
/// let basic_id = BenchmarkId::new("my_id", 5);
///
/// // The function name can be a string
/// let function_name = "test_string".to_string();
/// let string_id = BenchmarkId::new(function_name, 12);
///
/// // Benchmark IDs are passed to benchmark groups:
/// let mut criterion = Criterion::default();
/// let mut group = criterion.benchmark_group("My Group");
/// // Generate a very large input
/// let input : String = ::std::iter::repeat("X").take(1024 * 1024).collect();
///
/// // Note that we don't have to use the input as the parameter in the ID
/// group.bench_with_input(BenchmarkId::new("Test long string", "1MB X's"), &input, |b, i| {
/// b.iter(|| i.len())
/// });
/// ```
pub fn new<S: Into<String>, P: ::std::fmt::Display>(
function_name: S,
parameter: P,
) -> BenchmarkId {
BenchmarkId {
function_name: Some(function_name.into()),
parameter: Some(format!("{}", parameter)),
}
}
/// Construct a new benchmark ID from just a parameter value. Use this when benchmarking a
/// single function with a variety of different inputs.
pub fn from_parameter<P: ::std::fmt::Display>(parameter: P) -> BenchmarkId {
BenchmarkId {
function_name: None,
parameter: Some(format!("{}", parameter)),
}
}
pub(crate) fn no_function() -> BenchmarkId {
BenchmarkId {
function_name: None,
parameter: None,
}
}
pub(crate) fn no_function_with_input<P: ::std::fmt::Display>(parameter: P) -> BenchmarkId {
BenchmarkId {
function_name: None,
parameter: Some(format!("{}", parameter)),
}
}
}
mod private {
pub trait Sealed {}
impl Sealed for super::BenchmarkId {}
impl<S: Into<String>> Sealed for S {}
}
/// Sealed trait which allows users to automatically convert strings to benchmark IDs.
pub trait IntoBenchmarkId: private::Sealed {
fn into_benchmark_id(self) -> BenchmarkId;
}
impl IntoBenchmarkId for BenchmarkId {
fn into_benchmark_id(self) -> BenchmarkId {
self
}
}
impl<S: Into<String>> IntoBenchmarkId for S {
fn into_benchmark_id(self) -> BenchmarkId {
let function_name = self.into();
if function_name.is_empty() {
panic!("Function name must not be empty.");
}
BenchmarkId {
function_name: Some(function_name),
parameter: None,
}
}
}