criterion_polyglot/

lib.rs

#![warn(missing_docs)]

// Copyright (c) 2022 Ross Williams

//! `criterion-polyglot` provides an extension trait for [`criterion::Criterion`] and
//! [`criterion::BenchmarkGroup`] that provides benchmark methods for non-Rust programming
//! languages.
//!
//! `criterion-polyglot` currently supports the following languages:
//! * [Python 3](CriterionPolyglotExt::python_benchmark)
//! * [Ruby](CriterionPolyglotExt::ruby_benchmark)
//! * [Go](CriterionPolyglotExt::go_benchmark)
//! * [Zig](CriterionPolyglotExt::zig_benchmark)
//! * [C](CriterionPolyglotExt::c_benchmark)
//!
//! See [`CriterionPolyglotExt`] for detailed documentation on constructing benchmarks.
//!
//! # Examples
//!
//! #### Rust `Vec` versus Python `list`
//! ```
//! use criterion::{black_box, criterion_group, criterion_main, Criterion};
//! use criterion_polyglot::{BenchSpec, CriterionPolyglotExt};
//!
//! fn bench(c: &mut Criterion) {
//!     const LENGTH: u64 = 65_536;
//!     let mut source = Vec::with_capacity(LENGTH as usize);
//!     source.extend(0..LENGTH);
//!
//!     c.bench_function("Vec", |b| b.iter(||
//!         Vec::new().extend(black_box(source.iter().copied()))
//!     ));
//!
//!     c.python_benchmark("list",
//!         BenchSpec::new(r#"
//!             l = list()
//!             l += source
//!         "#).with_global_init(r#"
//!             source = [n for n in range(0, 65536)]
//!         "#)
//!     );
//! }
//!
//! # // The following two macros are cosmetic only and generate
//! # // functions that are not invoked by the doctest
//! criterion_group!(benches, bench);
//! criterion_main!(benches);
//! #
//! # // This helper function actually runs the benchmark
//! # criterion_polyglot::test_helpers::run_short_bench(bench);
//! ```
//!
//! #### Rust vs. Zig Hypothetical Tree Structure
//! ```
//! # mod some_tree {
//! #     use std::marker::PhantomData;
//! #     pub struct SomeTree<T>(PhantomData<T>);
//! #     impl<T> SomeTree<T> {
//! #         pub fn new() -> Self {
//! #             Self(PhantomData)
//! #         }
//! #         pub fn insert(&mut self, _: T) {}
//! #     }
//! # }
//! # #[derive(Clone)]
//! # struct Element;
//! use criterion::{black_box, BatchSize, Criterion};
//! use criterion_polyglot::{BenchSpec, CriterionPolyglotExt};
//! use some_tree::SomeTree;
//! 
//! fn make_elements() -> Vec<Element> {
//!     /* */
//!     # Vec::new()
//! }
//! 
//! fn bench(c: &mut Criterion) {
//!     let mut g = c.benchmark_group("some_tree");
//! 
//!     g.bench_function("Rust", |b| {
//!         let mut tree = SomeTree::new();
//!         let elements = make_elements();
//!         b.iter_batched(
//!             || elements.clone(),
//!             |elements| for element in elements { tree.insert(black_box(element)); },
//!             BatchSize::SmallInput,
//!         );
//!     });
//!     g.zig_benchmark("Zig",
//!         BenchSpec::new(r#"
//! #           // Hidden lines obscure nonsense code from visible documentation
//! #           // that allows this doctest to compile and run
//!             for (elements) |element| {
//!                 tree.insert(element);
//!             }
//!         "#).with_sample_init(r#"
//!             const elements = make_elements(allocator);
//!             var tree = new_tree(allocator);
//!         "#).with_global_init(r#"
//!             var arena = ArenaAllocator.init(std.heap.page_allocator);
//!             defer arena.deinit();
//!             const allocator = arena.allocator();
//!         "#)
//!         .with_imports(r#"
//!             const std = @import("std");
//! #           const ArrayList = std.ArrayList;
//!             const Allocator = std.mem.Allocator;
//!             const ArenaAllocator = std.heap.ArenaAllocator;
//!         "#)
//!         .with_declarations(r#"
//!             const SomeTree = struct {
//!                 pub fn insert(self: *SomeTree, value: u8) void {
//!                     // ... tree implementation ...
//! #                   _ = self;
//! #                   _ = value;
//!                 }
//!                 // ...
//!             };
//!             fn make_elements(allocator: Allocator) []u8 {
//!                 // ...
//! #               var list = ArrayList(u8).init(allocator);
//! #               return list.toOwnedSlice();
//!             }
//!             fn new_tree(allocator: Allocator) SomeTree {
//!                 // ...
//! #               _ = allocator;
//! #               return SomeTree { };
//!             }
//!         "#)
//!     );
//! 
//! }
//! # criterion_polyglot::test_helpers::run_short_bench(bench);
//! ```
//!
mod helper;
mod lang;
mod runner;
mod spec;

#[cfg(feature = "c")]
#[doc(no_inline)]
pub use cc;
use criterion::{measurement::WallTime, Criterion, BenchmarkGroup};

pub use spec::BenchSpec;

// This should be possible to do with #[cfg(doctest)],
// but it's broken: https://github.com/rust-lang/rust/issues/67295
#[doc(hidden)]
#[path = "../tests/helpers.rs"]
pub mod test_helpers;

#[doc = include_str!("../README.md")]
#[cfg(doctest)]
pub struct ReadmeDoctests;

/// `CriterionPolyglotExt` is an extension trait for [`criterion::Criterion`] and
/// [`criterion::BenchmarkGroup`] that provides benchmark methods for various non-Rust programming
/// languages.
///
/// ### Function Parameters
///
/// * `id: &str` - identifies the benchmark in Criterion's output
/// * `spec: `[`BenchSpec`] - defines the benchmark code, including non-benchmarked initialization
///                           code; see the [`BenchSpec`] documentation for detailed usage
pub trait CriterionPolyglotExt {
    /// Benchmark Python 3 code
    ///
    /// See [trait-level documentation](Self#function-parameters) for argument descriptions.
    fn python_benchmark(&mut self, id: &str, spec: BenchSpec) -> &mut Self;

    /// Benchmark Ruby code
    ///
    /// See [trait-level documentation](Self#function-parameters) for argument descriptions.
    fn ruby_benchmark(&mut self, id: &str, spec: BenchSpec) -> &mut Self;

    /// Benchmark Go code
    ///
    /// See [trait-level documentation](Self#function-parameters) for argument descriptions.
    fn go_benchmark(&mut self, id: &str, spec: BenchSpec) -> &mut Self;

    /// Benchmark Zig code
    ///
    /// See [trait-level documentation](Self#function-parameters) for argument descriptions.
    fn zig_benchmark(&mut self, id: &str, spec: BenchSpec) -> &mut Self;

    #[cfg(feature = "c")]
    /// Benchmark C code, using a custom compiler
    ///
    /// See [trait-level documentation](Self#function-parameters) for details on the `id` and `spec`
    /// arguments.
    ///
    /// The `builder` argument takes an instance of [`cc::Build`] that can be
    /// constructed to customize which C compiler is used to build the benchmark code and with
    /// which options. To ensure you construct an instance of `cc::Build` that is compatible with
    /// the version `criterion-polyglot` depends on, you may want to use the `cc` module
    /// re-exported as [`criterion_polyglot::cc`](crate#reexport.cc).
    fn c_benchmark_with_builder(
        &mut self,
        id: &str,
        spec: BenchSpec,
        builder: cc::Build,
    ) -> &mut Self;

    #[cfg(feature = "c")]
    /// Benchmark C code
    ///
    /// See [trait-level documentation](Self#function-parameters) for argument descriptions.
    ///
    /// Uses the C compiler as configured by [`cc::Build::default()`], then set to the same
    /// optimization level as `rustc` (determined by the `OPT_LEVEL` environment variable)
    ///
    /// Header files already included by the benchmarking harness are:
    /// ```c
    /// #include <stdio.h>
    /// #include <stdlib.h>
    /// #include <string.h>
    /// #include <limits.h>
    /// #include <sys/errno.h>
    /// #include <time.h>
    /// #include <sys/time.h>
    /// ```
    /// and on macOS:
    /// ```c
    /// #include <mach/mach.h>
    /// #include <mach/clock.h>
    /// ```
    fn c_benchmark(
        &mut self,
        id: &str,
        spec: BenchSpec,
    ) -> &mut Self {
        self.c_benchmark_with_builder(id, spec, cc::Build::default())
    }
}

macro_rules! spawn_bench {
    ($lang:ident, $this:ident, $id:expr, $target:expr) => {
        let mut bench = lang::$lang::spawn($target);
        $this.bench_function($id, |b| b.iter_custom(|n| bench.run(n)));
        bench.terminate();
    };
    ($lang:ident, $this:ident, $id:expr, $target:expr, $($extra:expr),+) => {
        let mut bench = lang::$lang::spawn($target, $($extra),+);
        $this.bench_function($id, |b| b.iter_custom(|n| bench.run(n)));
        bench.terminate();
    };
}

macro_rules! impl_interpreted_lang {
    ($lang:ident) => {
    ::paste::paste! {
        fn [<$lang _benchmark>](&mut self, id: &str, spec: BenchSpec) -> &mut Self {
            spawn_bench!($lang, self, id, spec);
            self
        }
    }
    };
}

macro_rules! impl_compiled_lang {
    ($lang:ident) => {
    ::paste::paste! {
        fn [<$lang _benchmark>](&mut self, id: &str, spec: BenchSpec) -> &mut Self {
            let bench_executable = lang::$lang::compile(spec);

            spawn_bench!($lang, self, id, &bench_executable);
            self
        }
    }
    };
}

macro_rules! impl_compiled_lang_with_params {
    ($lang:ident, $fn_name:ident, compile = $($param_ident:ident: $param_ty:ty),+) => {
    ::paste::paste! {
        fn [<$lang _benchmark_ $fn_name>](&mut self, id: &str, spec: BenchSpec, $($param_ident: $param_ty),+) -> &mut Self {
            let bench_executable = lang::$lang::compile(spec, $($param_ident),+);

            spawn_bench!($lang, self, id, &bench_executable);
            self
        }
    }
    };
    ($lang:ident, $fn_name:ident, spawn = $($param_ident:ident: $param_ty:ty),+) => {
    ::paste::paste! {
        fn [<$lang _benchmark_ $fn_name>](&mut self, id: &str, spec: BenchSpec, $($param_ident: $param_ty),+) -> &mut Self {
            let bench_executable = lang::$lang::compile(spec);

            spawn_bench!($lang, self, id, &bench_executable, $($param_ident),+);
            self
        }
    }
    };
}

macro_rules! impl_for {
    ($($for:ty)|+ => $impls:tt) => {
$(
        impl CriterionPolyglotExt for $for
            $impls
    )+
    };
}


impl_for! {
    Criterion<WallTime> | BenchmarkGroup<'_, WallTime> => {
        impl_interpreted_lang!(python);
        impl_interpreted_lang!(ruby);
        impl_compiled_lang!(zig);
        impl_compiled_lang!(go);

        #[cfg(feature = "c")]
        impl_compiled_lang_with_params!(c, with_builder, compile = builder: cc::Build);
    }
}