protest_criterion/lib.rs
1//! Property-Based Benchmarking with Criterion
2//!
3//! This crate provides integration between [Protest](https://crates.io/crates/protest)
4//! property-based testing and [Criterion](https://crates.io/crates/criterion) benchmarking.
5//!
6//! # Overview
7//!
8//! Property-based benchmarking allows you to:
9//! - Benchmark functions with **diverse, generated inputs**
10//! - Measure performance across the **full input space**
11//! - Detect **performance regressions** with statistical analysis
12//! - Understand **worst-case and average-case** performance
13//!
14//! # Quick Start
15//!
16//! ```rust,no_run
17//! use criterion::{criterion_group, criterion_main, Criterion};
18//! use protest_criterion::PropertyBencher;
19//! use protest::primitives::IntGenerator;
20//!
21//! fn bench_abs(c: &mut Criterion) {
22//! c.bench_function_over_inputs(
23//! "i32::abs",
24//! |b, &input: &i32| b.iter(|| input.abs()),
25//! IntGenerator::new(-1000, 1000),
26//! 100, // sample count
27//! );
28//! }
29//!
30//! criterion_group!(benches, bench_abs);
31//! criterion_main!(benches);
32//! ```
33//!
34//! # Features
35//!
36//! - **Generator Integration**: Use any Protest generator for benchmark inputs
37//! - **Statistical Analysis**: Leverage Criterion's statistical analysis
38//! - **Regression Detection**: Automatic detection of performance changes
39//! - **Input Distribution**: Benchmark across diverse input distributions
40//! - **Reproducible**: Seed-based generation for consistent benchmarks
41//!
42//! # Examples
43//!
44//! ## Benchmarking Sorting Algorithms
45//!
46//! ```rust,no_run
47//! use criterion::{criterion_group, criterion_main, Criterion, BenchmarkId};
48//! use protest_criterion::PropertyBencher;
49//! use protest::primitives::VecGenerator;
50//! use protest::{Generator, config::GeneratorConfig};
51//!
52//! fn bench_sort_algorithms(c: &mut Criterion) {
53//! let mut group = c.benchmark_group("sorting");
54//!
55//! for size in [10, 100, 1000].iter() {
56//! let generator = VecGenerator::new(
57//! protest::primitives::IntGenerator::new(0, 1000),
58//! *size,
59//! *size
60//! );
61//!
62//! group.bench_with_input(
63//! BenchmarkId::new("sort", size),
64//! &generator,
65//! |b, generator| {
66//! let config = GeneratorConfig::default();
67//! b.iter_batched(
68//! || generator.generate(&mut rand::thread_rng(), &config),
69//! |mut v| v.sort(),
70//! criterion::BatchSize::SmallInput
71//! )
72//! }
73//! );
74//! }
75//!
76//! group.finish();
77//! }
78//!
79//! criterion_group!(benches, bench_sort_algorithms);
80//! criterion_main!(benches);
81//! ```
82//!
83//! ## Benchmarking with Properties
84//!
85//! ```rust,no_run
86//! use criterion::{criterion_group, criterion_main, Criterion};
87//! use protest_criterion::PropertyBencher;
88//! use protest::primitives::VecGenerator;
89//! use protest::Generator;
90//!
91//! fn bench_property(c: &mut Criterion) {
92//! c.bench_property(
93//! "vec reversal is involutive",
94//! VecGenerator::new(
95//! protest::primitives::IntGenerator::new(0, 100),
96//! 0,
97//! 1000
98//! ),
99//! |v: &Vec<i32>| {
100//! let mut reversed = v.clone();
101//! reversed.reverse();
102//! reversed.reverse();
103//! assert_eq!(v, &reversed);
104//! },
105//! 100 // sample count
106//! );
107//! }
108//!
109//! criterion_group!(benches, bench_property);
110//! criterion_main!(benches);
111//! ```
112
113use criterion::{BenchmarkGroup, BenchmarkId, Criterion, measurement::WallTime};
114use protest::{Generator, config::GeneratorConfig};
115use rand::SeedableRng;
116use rand::rngs::StdRng;
117
118/// Extension trait for Criterion to enable property-based benchmarking
119pub trait PropertyBencher {
120 /// Benchmark a function over inputs generated by a property-based generator
121 ///
122 /// # Arguments
123 ///
124 /// * `name` - Name of the benchmark
125 /// * `bench_fn` - Function to benchmark (takes immutable reference to input)
126 /// * `generator` - Generator for creating diverse test inputs
127 /// * `sample_count` - Number of samples to generate and benchmark
128 ///
129 /// # Example
130 ///
131 /// ```rust,no_run
132 /// use criterion::{criterion_group, criterion_main, Criterion};
133 /// use protest_criterion::PropertyBencher;
134 /// use protest::primitives::IntGenerator;
135 ///
136 /// fn bench_abs(c: &mut Criterion) {
137 /// c.bench_function_over_inputs(
138 /// "i32::abs",
139 /// |b, &input: &i32| b.iter(|| input.abs()),
140 /// IntGenerator::new(-1000, 1000),
141 /// 100,
142 /// );
143 /// }
144 ///
145 /// criterion_group!(benches, bench_abs);
146 /// criterion_main!(benches);
147 /// ```
148 fn bench_function_over_inputs<I, G, F>(
149 &mut self,
150 name: &str,
151 bench_fn: F,
152 generator: G,
153 sample_count: usize,
154 ) where
155 I: Clone + 'static,
156 G: Generator<I>,
157 F: FnMut(&mut criterion::Bencher, &I);
158
159 /// Benchmark a property test (function that takes ownership and may assert)
160 ///
161 /// # Arguments
162 ///
163 /// * `name` - Name of the benchmark
164 /// * `generator` - Generator for creating diverse test inputs
165 /// * `property` - Property function to benchmark
166 /// * `sample_count` - Number of samples to generate and benchmark
167 ///
168 /// # Example
169 ///
170 /// ```rust,no_run
171 /// use criterion::{criterion_group, criterion_main, Criterion};
172 /// use protest_criterion::PropertyBencher;
173 /// use protest::primitives::VecGenerator;
174 /// use protest::Generator;
175 ///
176 /// fn bench_reverse_involutive(c: &mut Criterion) {
177 /// c.bench_property(
178 /// "vec reverse is involutive",
179 /// VecGenerator::new(
180 /// protest::primitives::IntGenerator::new(0, 100),
181 /// 0,
182 /// 100
183 /// ),
184 /// |v: &Vec<i32>| {
185 /// let mut reversed = v.clone();
186 /// reversed.reverse();
187 /// reversed.reverse();
188 /// },
189 /// 50,
190 /// );
191 /// }
192 ///
193 /// criterion_group!(benches, bench_reverse_involutive);
194 /// criterion_main!(benches);
195 /// ```
196 fn bench_property<I, G, P>(
197 &mut self,
198 name: &str,
199 generator: G,
200 property: P,
201 sample_count: usize,
202 ) where
203 I: Clone + 'static,
204 G: Generator<I>,
205 P: Fn(&I) + 'static;
206}
207
208impl PropertyBencher for Criterion {
209 fn bench_function_over_inputs<I, G, F>(
210 &mut self,
211 name: &str,
212 mut bench_fn: F,
213 generator: G,
214 sample_count: usize,
215 ) where
216 I: Clone + 'static,
217 G: Generator<I>,
218 F: FnMut(&mut criterion::Bencher, &I),
219 {
220 let mut rng = StdRng::from_entropy();
221 let config = GeneratorConfig::default();
222 let inputs: Vec<I> = (0..sample_count)
223 .map(|_| generator.generate(&mut rng, &config))
224 .collect();
225
226 let mut group = self.benchmark_group(name);
227
228 for (idx, input) in inputs.iter().enumerate() {
229 group.bench_with_input(BenchmarkId::from_parameter(idx), input, &mut bench_fn);
230 }
231
232 group.finish();
233 }
234
235 fn bench_property<I, G, P>(
236 &mut self,
237 name: &str,
238 generator: G,
239 property: P,
240 sample_count: usize,
241 ) where
242 I: Clone + 'static,
243 G: Generator<I>,
244 P: Fn(&I) + 'static,
245 {
246 let mut rng = StdRng::from_entropy();
247 let config = GeneratorConfig::default();
248 let inputs: Vec<I> = (0..sample_count)
249 .map(|_| generator.generate(&mut rng, &config))
250 .collect();
251
252 let mut group = self.benchmark_group(name);
253
254 for (idx, input) in inputs.iter().enumerate() {
255 group.bench_with_input(BenchmarkId::from_parameter(idx), &input, |b, input| {
256 b.iter(|| property(input));
257 });
258 }
259
260 group.finish();
261 }
262}
263
264/// Helper for creating benchmark groups with property-based inputs
265///
266/// This provides a more ergonomic API for benchmarking multiple configurations
267/// with generated inputs.
268pub trait PropertyBenchmarkGroup<'a> {
269 /// Benchmark over generated inputs with a specific parameter
270 ///
271 /// # Example
272 ///
273 /// ```rust,no_run
274 /// use criterion::{criterion_group, criterion_main, Criterion};
275 /// use protest_criterion::PropertyBenchmarkGroup;
276 /// use protest::primitives::VecGenerator;
277 /// use protest::Generator;
278 ///
279 /// fn bench_by_size(c: &mut Criterion) {
280 /// let mut group = c.benchmark_group("sort_by_size");
281 ///
282 /// for size in [10, 100, 1000] {
283 /// let generator = VecGenerator::new(
284 /// protest::primitives::IntGenerator::new(0, 1000),
285 /// size,
286 /// size
287 /// );
288 ///
289 /// group.bench_generated(&size.to_string(), generator, |v: &Vec<i32>| {
290 /// let mut sorted = v.clone();
291 /// sorted.sort();
292 /// });
293 /// }
294 ///
295 /// group.finish();
296 /// }
297 ///
298 /// criterion_group!(benches, bench_by_size);
299 /// criterion_main!(benches);
300 /// ```
301 fn bench_generated<I, G, F>(&mut self, id: &str, generator: G, f: F)
302 where
303 I: Clone + 'static,
304 G: Generator<I>,
305 F: FnMut(&I) + 'static;
306}
307
308impl<'a> PropertyBenchmarkGroup<'a> for BenchmarkGroup<'a, WallTime> {
309 fn bench_generated<I, G, F>(&mut self, id: &str, generator: G, mut f: F)
310 where
311 I: Clone + 'static,
312 G: Generator<I>,
313 F: FnMut(&I) + 'static,
314 {
315 self.bench_function(id, move |b| {
316 let mut rng = StdRng::from_entropy();
317 let config = GeneratorConfig::default();
318 b.iter_batched(
319 || generator.generate(&mut rng, &config),
320 |input| f(&input),
321 criterion::BatchSize::SmallInput,
322 );
323 });
324 }
325}
326
327#[cfg(test)]
328mod tests {
329 use super::*;
330 use protest::primitives::{IntGenerator, VecGenerator};
331
332 #[test]
333 fn test_generates_inputs() {
334 let generator = IntGenerator::new(0, 100);
335 let mut rng = StdRng::from_seed([42; 32]);
336 let config = GeneratorConfig::default();
337
338 let inputs: Vec<i32> = (0..10)
339 .map(|_| generator.generate(&mut rng, &config))
340 .collect();
341
342 assert_eq!(inputs.len(), 10);
343 for input in inputs {
344 assert!((0..=100).contains(&input));
345 }
346 }
347
348 #[test]
349 fn test_vec_generator() {
350 let generator = VecGenerator::new(IntGenerator::new(0, 10), 5, 5);
351 let mut rng = StdRng::from_seed([42; 32]);
352 let config = GeneratorConfig::default();
353
354 let input = generator.generate(&mut rng, &config);
355 assert_eq!(input.len(), 5);
356 for &val in &input {
357 assert!((0..=10).contains(&val));
358 }
359 }
360}