benchmark_rs/lib.rs
1//! A benchmarking library for Rust package authors.
2//!
3//! Benchmark-rs crate provides tools for Rust package authors to evaluate performance of their implementation for
4//! varying workloads with varying configurations, find performance regressions between versions and also some crude
5//! facilities to measure execution times of code blocks.
6//!
7//! For example, if you implement a concurrent algorithm with an expectation of reduced execution time when more
8//! CPU cores are added, benchmark-rs will help validate that the concurrency has the desired effect and that there
9//! is no anomaly for varying data sets, that is, the algorithm performs consistently within boundaries defined for it.
10//!
11//! However, this doesn't have to be a fancy algorithm that we want to test, some banal looking code can exhibit
12//! radically different behavior for different data sets, depending on database queries, buffer sizes, number of
13//! open files, etc. We may want to compare the behavior of our code with different querying strategies or different
14//! buffer sizes for a range of data sets to find the best fit for our use-case.
15//!
16//!
17//! Another common use-case is to verify that there is no regression in performance introduced by the new code. Benchmark-rs
18//! supports comparison to previous results with user defined equality threshold.
19//!
20//! # Design
21//!
22//! Benchmark-rs is designed to evaluate performance of subjects processing a set of workloads with modifiable configurations.
23//! For example if we write a protocol parser we may define our workload points as 10MB, 20MB, ..., 100MB input sizes. Workloads
24//! are user defined in benchmark-rs. Any type `W: Clone + Display` can be used to specify workloads. It could
25//! be an integer that specifies the size of the workload, a path to a file or a key that we can use
26//! to fetch the workload from the benchmark configuration. The `Display` is required
27//! for the workload point to produce the key in result series, so it is recommended to keep it short and descriptive.
28//! See examples below.
29//!
30//! Each benchmark is repeated `repeat` times for every workload point after it was ramped up for that point. Upon
31//! completion the summary is available as JSON or as CSV.
32//!
33//! # Issues
34//! Issues are welcome and appreciated. Please submit to https://github.com/navigatorsguild/benchmark-rs/issues
35//!
36//! # Examples
37//! A real world example can be found at
38//! [command-executor](https://github.com/navigatorsguild/command-executor) project
39//! [blocking_queue.rs](https://github.com/navigatorsguild/command-executor/blob/main/benches/blocking_queue.rs)
40//! benchmark and at [Benchmarks](https://github.com/navigatorsguild/command-executor/wiki/Benchmarks)
41//! wiki page which was built from the generated data.
42//!
43//! 
44//!
45//! ## Simple Benchmark
46//! A simple benchmark that measures execution time for increasing workloads. In this case the workload is simulated by
47//! by a `u64` value passed to `thread::sleep` function
48//! ```
49//! use std::thread;
50//! use std::time::Duration;
51//! use benchmark_rs::benchmarks::Benchmarks;
52//! use benchmark_rs::stopwatch::StopWatch;
53//!
54//! fn example(_stop_watch: &mut StopWatch, _config: &str, work: u64) -> Result<(), anyhow::Error> {
55//! thread::sleep(Duration::from_millis(work));
56//! Ok(())
57//! }
58//!
59//! fn main() -> Result<(), anyhow::Error> {
60//! let mut benchmarks = Benchmarks::new("Example");
61//! benchmarks.add("A Simple Benchmark", example, "No Configuration", (1..=10).collect(), 2, 1)?;
62//! benchmarks.run()?;
63//!
64//! let summary = benchmarks.summary_as_json();
65//! println!("Summary: {summary}");
66//! Ok(())
67//! }
68//! ```
69//!
70//! ## Benchmark Workloads
71//!
72//! A more complex example that shows how to use Benchmark configuration and how to control the
73//! stopwatch from within the benchmark to avoid measuring the housekeeping tasks.
74//!
75//! <details>
76//! <summary>Benchmark Workloads Example</summary>
77//!
78//! ```
79//! use std::collections::BTreeMap;
80//! use std::fmt::{Display, Formatter};
81//! use std::thread;
82//! use std::time::Duration;
83//!
84//! use benchmark_rs::benchmarks::Benchmarks;
85//! use benchmark_rs::stopwatch::StopWatch;
86//!
87//! #[derive(Clone)]
88//! struct Config {
89//! // simulate available resources - CPU cores, memory buffers, etc.
90//! pub resources: u32,
91//! pub workloads: BTreeMap<u64, Duration>,
92//! }
93//!
94//! impl Display for Config {
95//! fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
96//! let keys: Vec<String> = self.workloads.keys().map(|k| k.to_string()).collect();
97//! write!(f, "{}", keys.join(", "))
98//! }
99//! }
100//!
101//! fn example(stop_watch: &mut StopWatch, config: Config, work: u64) -> Result<(), anyhow::Error> {
102//! stop_watch.pause();
103//! // perform potentially lengthy preparation that will not reflect in the measurement
104//! // ...
105//! // fetch the workload definition from configuration associated with the 'work' point
106//! let sleep_time = config.workloads.get(&work).unwrap().clone();
107//! // resume the stopwatch to measure the actual work
108//! stop_watch.resume();
109//! // perform measured computation
110//! thread::sleep(sleep_time / config.resources);
111//! stop_watch.pause();
112//! // perform potentially lengthy cleanup
113//! // ...
114//! Ok(())
115//! }
116//!
117//! fn main() -> Result<(), anyhow::Error> {
118//! let mut benchmarks = Benchmarks::new("benchmark-workloads");
119//! let workloads: BTreeMap<u64, Duration> =
120//! (0..=10).map(|i| (i, Duration::from_millis(i))).collect();
121//! benchmarks.add(
122//! "benchmark-workload-1",
123//! example,
124//! Config {
125//! resources: 1,
126//! workloads: workloads.clone(),
127//! },
128//! (1..=10).collect(),
129//! 2,
130//! 1,
131//! )?;
132//!
133//! benchmarks.add(
134//! "benchmark-workload-2",
135//! example,
136//! Config {
137//! resources: 2,
138//! workloads: workloads.clone(),
139//! },
140//! (1..=10).collect(),
141//! 2,
142//! 1,
143//! )?;
144//!
145//! benchmarks.run()?;
146//!
147//! let summary = benchmarks.summary_as_json();
148//! println!("Benchmark summary in JSON format.");
149//! println!("Summary:");
150//! println!("{summary}");
151//! println!();
152//!
153//! println!("Benchmark series in CSV format.");
154//! let csv_data = benchmarks.summary_as_csv(true, false);
155//! for (k, v) in csv_data {
156//! println!("Benchmark name: {k}");
157//! for line in v {
158//! println!("{line}")
159//! }
160//! println!();
161//! }
162//!
163//! Ok(())
164//! }
165//! ```
166//! </details>
167//!
168//! <details>
169//! <summary>Benchmark Workloads Summary</summary>
170//!
171//! ```json
172//! {
173//! "name": "benchmark-workloads",
174//! "created_at": "2023-08-13 04:09:13.923036",
175//! "series": {
176//! "benchmark-workload-2": {
177//! "name": "benchmark-workload-2",
178//! "config": "0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10",
179//! "runs": [
180//! [
181//! "1",
182//! {
183//! "name": "benchmark-workload-2",
184//! "ramp_up": 1,
185//! "repeat": 2,
186//! "min_nanos": 639791,
187//! "min_sec": 0.000639791,
188//! "min_str": "00:00:00.000",
189//! "max_nanos": 644250,
190//! "max_sec": 0.00064425,
191//! "max_str": "00:00:00.000",
192//! "median_nanos": 642020,
193//! "median_sec": 0.00064202,
194//! "median_str": "00:00:00.000",
195//! "std_dev": 3152.9891373108153,
196//! "std_dev_sec": 3.1529891373108154e-6,
197//! "std_dev_str": "00:00:00.000"
198//! }
199//! ],
200//! [
201//! "2",
202//! {
203//! "name": "benchmark-workload-2",
204//! "ramp_up": 1,
205//! "repeat": 2,
206//! "min_nanos": 1264124,
207//! "min_sec": 0.001264124,
208//! "min_str": "00:00:00.001",
209//! "max_nanos": 1264250,
210//! "max_sec": 0.00126425,
211//! "max_str": "00:00:00.001",
212//! "median_nanos": 1264187,
213//! "median_sec": 0.001264187,
214//! "median_str": "00:00:00.001",
215//! "std_dev": 89.09545442950498,
216//! "std_dev_sec": 8.909545442950498e-8,
217//! "std_dev_str": "00:00:00.000"
218//! }
219//! ],
220//! [
221//! "3",
222//! {
223//! "name": "benchmark-workload-2",
224//! "ramp_up": 1,
225//! "repeat": 2,
226//! "min_nanos": 1891833,
227//! "min_sec": 0.001891833,
228//! "min_str": "00:00:00.001",
229//! "max_nanos": 1892875,
230//! "max_sec": 0.001892875,
231//! "max_str": "00:00:00.001",
232//! "median_nanos": 1892354,
233//! "median_sec": 0.001892354,
234//! "median_str": "00:00:00.001",
235//! "std_dev": 736.8052659963826,
236//! "std_dev_sec": 7.368052659963826e-7,
237//! "std_dev_str": "00:00:00.000"
238//! }
239//! ],
240//! [
241//! "4",
242//! {
243//! "name": "benchmark-workload-2",
244//! "ramp_up": 1,
245//! "repeat": 2,
246//! "min_nanos": 2566417,
247//! "min_sec": 0.002566417,
248//! "min_str": "00:00:00.002",
249//! "max_nanos": 2576416,
250//! "max_sec": 0.002576416,
251//! "max_str": "00:00:00.002",
252//! "median_nanos": 2571416,
253//! "median_sec": 0.002571416,
254//! "median_str": "00:00:00.002",
255//! "std_dev": 7070.360705084288,
256//! "std_dev_sec": 7.0703607050842884e-6,
257//! "std_dev_str": "00:00:00.000"
258//! }
259//! ],
260//! [
261//! "5",
262//! {
263//! "name": "benchmark-workload-2",
264//! "ramp_up": 1,
265//! "repeat": 2,
266//! "min_nanos": 2928666,
267//! "min_sec": 0.002928666,
268//! "min_str": "00:00:00.002",
269//! "max_nanos": 3174917,
270//! "max_sec": 0.003174917,
271//! "max_str": "00:00:00.003",
272//! "median_nanos": 3051791,
273//! "median_sec": 0.003051791,
274//! "median_str": "00:00:00.003",
275//! "std_dev": 174125.75197396852,
276//! "std_dev_sec": 0.00017412575197396852,
277//! "std_dev_str": "00:00:00.000"
278//! }
279//! ],
280//! [
281//! "6",
282//! {
283//! "name": "benchmark-workload-2",
284//! "ramp_up": 1,
285//! "repeat": 2,
286//! "min_nanos": 3377791,
287//! "min_sec": 0.003377791,
288//! "min_str": "00:00:00.003",
289//! "max_nanos": 3784625,
290//! "max_sec": 0.003784625,
291//! "max_str": "00:00:00.003",
292//! "median_nanos": 3581208,
293//! "median_sec": 0.003581208,
294//! "median_str": "00:00:00.003",
295//! "std_dev": 287675.0802172479,
296//! "std_dev_sec": 0.00028767508021724787,
297//! "std_dev_str": "00:00:00.000"
298//! }
299//! ],
300//! [
301//! "7",
302//! {
303//! "name": "benchmark-workload-2",
304//! "ramp_up": 1,
305//! "repeat": 2,
306//! "min_nanos": 4404499,
307//! "min_sec": 0.004404499,
308//! "min_str": "00:00:00.004",
309//! "max_nanos": 4915541,
310//! "max_sec": 0.004915541,
311//! "max_str": "00:00:00.004",
312//! "median_nanos": 4660020,
313//! "median_sec": 0.00466002,
314//! "median_str": "00:00:00.004",
315//! "std_dev": 361361.26367113565,
316//! "std_dev_sec": 0.00036136126367113564,
317//! "std_dev_str": "00:00:00.000"
318//! }
319//! ],
320//! [
321//! "8",
322//! {
323//! "name": "benchmark-workload-2",
324//! "ramp_up": 1,
325//! "repeat": 2,
326//! "min_nanos": 4882584,
327//! "min_sec": 0.004882584,
328//! "min_str": "00:00:00.004",
329//! "max_nanos": 5029209,
330//! "max_sec": 0.005029209,
331//! "max_str": "00:00:00.005",
332//! "median_nanos": 4955896,
333//! "median_sec": 0.004955896,
334//! "median_str": "00:00:00.004",
335//! "std_dev": 103679.53179147754,
336//! "std_dev_sec": 0.00010367953179147753,
337//! "std_dev_str": "00:00:00.000"
338//! }
339//! ],
340//! [
341//! "9",
342//! {
343//! "name": "benchmark-workload-2",
344//! "ramp_up": 1,
345//! "repeat": 2,
346//! "min_nanos": 5638959,
347//! "min_sec": 0.005638959,
348//! "min_str": "00:00:00.005",
349//! "max_nanos": 5643416,
350//! "max_sec": 0.005643416,
351//! "max_str": "00:00:00.005",
352//! "median_nanos": 5641187,
353//! "median_sec": 0.005641187,
354//! "median_str": "00:00:00.005",
355//! "std_dev": 3151.5749237484424,
356//! "std_dev_sec": 3.1515749237484423e-6,
357//! "std_dev_str": "00:00:00.000"
358//! }
359//! ],
360//! [
361//! "10",
362//! {
363//! "name": "benchmark-workload-2",
364//! "ramp_up": 1,
365//! "repeat": 2,
366//! "min_nanos": 5086250,
367//! "min_sec": 0.00508625,
368//! "min_str": "00:00:00.005",
369//! "max_nanos": 5587292,
370//! "max_sec": 0.005587292,
371//! "max_str": "00:00:00.005",
372//! "median_nanos": 5336771,
373//! "median_sec": 0.005336771,
374//! "median_str": "00:00:00.005",
375//! "std_dev": 354290.19585927017,
376//! "std_dev_sec": 0.00035429019585927015,
377//! "std_dev_str": "00:00:00.000"
378//! }
379//! ]
380//! ]
381//! },
382//! "benchmark-workload-1": {
383//! "name": "benchmark-workload-1",
384//! "config": "0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10",
385//! "runs": [
386//! [
387//! "1",
388//! {
389//! "name": "benchmark-workload-1",
390//! "ramp_up": 1,
391//! "repeat": 2,
392//! "min_nanos": 1259375,
393//! "min_sec": 0.001259375,
394//! "min_str": "00:00:00.001",
395//! "max_nanos": 1276125,
396//! "max_sec": 0.001276125,
397//! "max_str": "00:00:00.001",
398//! "median_nanos": 1267750,
399//! "median_sec": 0.00126775,
400//! "median_str": "00:00:00.001",
401//! "std_dev": 11844.038584874672,
402//! "std_dev_sec": 0.000011844038584874672,
403//! "std_dev_str": "00:00:00.000"
404//! }
405//! ],
406//! [
407//! "2",
408//! {
409//! "name": "benchmark-workload-1",
410//! "ramp_up": 1,
411//! "repeat": 2,
412//! "min_nanos": 2513584,
413//! "min_sec": 0.002513584,
414//! "min_str": "00:00:00.002",
415//! "max_nanos": 2520875,
416//! "max_sec": 0.002520875,
417//! "max_str": "00:00:00.002",
418//! "median_nanos": 2517229,
419//! "median_sec": 0.002517229,
420//! "median_str": "00:00:00.002",
421//! "std_dev": 5155.515541631118,
422//! "std_dev_sec": 5.155515541631118e-6,
423//! "std_dev_str": "00:00:00.000"
424//! }
425//! ],
426//! [
427//! "3",
428//! {
429//! "name": "benchmark-workload-1",
430//! "ramp_up": 1,
431//! "repeat": 2,
432//! "min_nanos": 3755750,
433//! "min_sec": 0.00375575,
434//! "min_str": "00:00:00.003",
435//! "max_nanos": 3784000,
436//! "max_sec": 0.003784,
437//! "max_str": "00:00:00.003",
438//! "median_nanos": 3769875,
439//! "median_sec": 0.003769875,
440//! "median_str": "00:00:00.003",
441//! "std_dev": 19975.766568519968,
442//! "std_dev_sec": 0.00001997576656851997,
443//! "std_dev_str": "00:00:00.000"
444//! }
445//! ],
446//! [
447//! "4",
448//! {
449//! "name": "benchmark-workload-1",
450//! "ramp_up": 1,
451//! "repeat": 2,
452//! "min_nanos": 5003833,
453//! "min_sec": 0.005003833,
454//! "min_str": "00:00:00.005",
455//! "max_nanos": 5030084,
456//! "max_sec": 0.005030084,
457//! "max_str": "00:00:00.005",
458//! "median_nanos": 5016958,
459//! "median_sec": 0.005016958,
460//! "median_str": "00:00:00.005",
461//! "std_dev": 18562.26011292806,
462//! "std_dev_sec": 0.00001856226011292806,
463//! "std_dev_str": "00:00:00.000"
464//! }
465//! ],
466//! [
467//! "5",
468//! {
469//! "name": "benchmark-workload-1",
470//! "ramp_up": 1,
471//! "repeat": 2,
472//! "min_nanos": 5518500,
473//! "min_sec": 0.0055185,
474//! "min_str": "00:00:00.005",
475//! "max_nanos": 6283084,
476//! "max_sec": 0.006283084,
477//! "max_str": "00:00:00.006",
478//! "median_nanos": 5900792,
479//! "median_sec": 0.005900792,
480//! "median_str": "00:00:00.005",
481//! "std_dev": 540642.5311867353,
482//! "std_dev_sec": 0.0005406425311867353,
483//! "std_dev_str": "00:00:00.000"
484//! }
485//! ],
486//! [
487//! "6",
488//! {
489//! "name": "benchmark-workload-1",
490//! "ramp_up": 1,
491//! "repeat": 2,
492//! "min_nanos": 6942292,
493//! "min_sec": 0.006942292,
494//! "min_str": "00:00:00.006",
495//! "max_nanos": 7541458,
496//! "max_sec": 0.007541458,
497//! "max_str": "00:00:00.007",
498//! "median_nanos": 7241875,
499//! "median_sec": 0.007241875,
500//! "median_str": "00:00:00.007",
501//! "std_dev": 423674.3416564189,
502//! "std_dev_sec": 0.00042367434165641895,
503//! "std_dev_str": "00:00:00.000"
504//! }
505//! ],
506//! [
507//! "7",
508//! {
509//! "name": "benchmark-workload-1",
510//! "ramp_up": 1,
511//! "repeat": 2,
512//! "min_nanos": 8784417,
513//! "min_sec": 0.008784417,
514//! "min_str": "00:00:00.008",
515//! "max_nanos": 8812875,
516//! "max_sec": 0.008812875,
517//! "max_str": "00:00:00.008",
518//! "median_nanos": 8798646,
519//! "median_sec": 0.008798646,
520//! "median_str": "00:00:00.008",
521//! "std_dev": 20122.84477900677,
522//! "std_dev_sec": 0.00002012284477900677,
523//! "std_dev_str": "00:00:00.000"
524//! }
525//! ],
526//! [
527//! "8",
528//! {
529//! "name": "benchmark-workload-1",
530//! "ramp_up": 1,
531//! "repeat": 2,
532//! "min_nanos": 9426875,
533//! "min_sec": 0.009426875,
534//! "min_str": "00:00:00.009",
535//! "max_nanos": 11596125,
536//! "max_sec": 0.011596125,
537//! "max_str": "00:00:00.011",
538//! "median_nanos": 10511500,
539//! "median_sec": 0.0105115,
540//! "median_str": "00:00:00.010",
541//! "std_dev": 1533891.3850889183,
542//! "std_dev_sec": 0.0015338913850889183,
543//! "std_dev_str": "00:00:00.001"
544//! }
545//! ],
546//! [
547//! "9",
548//! {
549//! "name": "benchmark-workload-1",
550//! "ramp_up": 1,
551//! "repeat": 2,
552//! "min_nanos": 9498333,
553//! "min_sec": 0.009498333,
554//! "min_str": "00:00:00.009",
555//! "max_nanos": 9953333,
556//! "max_sec": 0.009953333,
557//! "max_str": "00:00:00.009",
558//! "median_nanos": 9725833,
559//! "median_sec": 0.009725833,
560//! "median_str": "00:00:00.009",
561//! "std_dev": 321733.5854398791,
562//! "std_dev_sec": 0.0003217335854398791,
563//! "std_dev_str": "00:00:00.000"
564//! }
565//! ],
566//! [
567//! "10",
568//! {
569//! "name": "benchmark-workload-1",
570//! "ramp_up": 1,
571//! "repeat": 2,
572//! "min_nanos": 11674000,
573//! "min_sec": 0.011674,
574//! "min_str": "00:00:00.011",
575//! "max_nanos": 12627167,
576//! "max_sec": 0.012627167,
577//! "max_str": "00:00:00.012",
578//! "median_nanos": 12150583,
579//! "median_sec": 0.012150583,
580//! "median_str": "00:00:00.012",
581//! "std_dev": 673990.849303238,
582//! "std_dev_sec": 0.0006739908493032379,
583//! "std_dev_str": "00:00:00.000"
584//! }
585//! ]
586//! ]
587//! }
588//! }
589//! }
590//! ```
591//! </details>
592//!
593//! <details>
594//! <summary>Benchmark Workloads Series</summary>
595//!
596//! Benchmark name: benchmark-workload-2
597//! ```csv
598//! point,ramp_up,repeat,min_sec,max_sec,median_sec,std_dev_sec
599//! 1,1,2,0.00063575,0.000637376,0.000636563,0.0000011497556262093261
600//! 2,1,2,0.001265333,0.001269584,0.001267458,0.0000030059109268240135
601//! 3,1,2,0.001890958,0.001892333,0.001891645,0.0000009722718241315028
602//! 4,1,2,0.002512042,0.002604791,0.002558416,0.0000655834468482711
603//! 5,1,2,0.003129084,0.00314525,0.003137167,0.000011431088224661727
604//! 6,1,2,0.003092583,0.003767833,0.003430208,0.0004774738539962162
605//! 7,1,2,0.004055333,0.004084166,0.004069749,0.000020388009821951726
606//! 8,1,2,0.004171542,0.005756541,0.004964041,0.0011207635410738967
607//! 9,1,2,0.004549334,0.005149792,0.004849563,0.0004245879236177119
608//! 10,1,2,0.006351,0.007404083,0.006877541,0.000744642130452273
609//! ```
610//!
611//! Benchmark name: benchmark-workload-1
612//! ```csv
613//! point,ramp_up,repeat,min_sec,max_sec,median_sec,std_dev_sec
614//! 1,1,2,0.001265417,0.001266709,0.001266063,0.0000009135819612930194
615//! 2,1,2,0.002518501,0.002534999,0.00252675,0.000011665847676015662
616//! 3,1,2,0.003762958,0.004072459,0.003917708,0.00021885025588401765
617//! 4,1,2,0.004471833,0.004559375,0.004515604,0.00006190154183863275
618//! 5,1,2,0.005676917,0.005765751,0.005721334,0.00006281512379992577
619//! 6,1,2,0.006610875,0.007539833,0.007075354,0.0006568725012374928
620//! 7,1,2,0.007831792,0.008021959,0.007926875,0.0001344683752579022
621//! 8,1,2,0.009668583,0.009832959,0.009750771,0.00011623138426431993
622//! 9,1,2,0.009677333,0.011178501,0.010427917,0.0010614860725002473
623//! 10,1,2,0.011440417,0.012526,0.011983208,0.0007676231008408358
624//! ```
625//!
626//! </details>
627//!
628//!
629//! <details>
630//! <summary>Find Regressions Example</summary>
631//!
632//! ```
633//! use std::collections::BTreeMap;
634//! use std::fmt::{Display, Formatter};
635//! use std::thread;
636//! use std::time::Duration;
637//!
638//! use benchmark_rs::benchmarks::Benchmarks;
639//! use benchmark_rs::stopwatch::StopWatch;
640//! use rand::Rng;
641//!
642//! #[derive(Clone)]
643//! struct Config {
644//! // simulate available resources - CPU cores, memory buffers, etc.
645//! pub resources: u32,
646//! pub workloads: BTreeMap<u64, Duration>,
647//! }
648//!
649//! impl Display for Config {
650//! fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
651//! for k in self.workloads.keys() {
652//! write!(f, "{k} ")?;
653//! }
654//! Ok(())
655//! }
656//! }
657//!
658//! fn example(_stop_watch: &mut StopWatch, config: Config, work: u64) -> Result<(), anyhow::Error> {
659//! let sleep_time = config.workloads.get(&work).unwrap().clone();
660//! thread::sleep(sleep_time / config.resources);
661//! Ok(())
662//! }
663//!
664//! fn modified_example(
665//! _stop_watch: &mut StopWatch,
666//! config: Config,
667//! work: u64,
668//! ) -> Result<(), anyhow::Error> {
669//! // we introduce a random deviation in modified example to simulate
670//! // regression introduced by a code change
671//! let deviation = Duration::from_millis(rand::thread_rng().gen_range(0..10));
672//! let sleep_time = config.workloads.get(&work).unwrap().clone();
673//! thread::sleep(sleep_time / config.resources + deviation);
674//! Ok(())
675//! }
676//!
677//! fn main() -> Result<(), anyhow::Error> {
678//! let mut previous_benchmarks = Benchmarks::new("benchmarks");
679//! let workloads: BTreeMap<u64, Duration> = (1..=10)
680//! .map(|i| (i, Duration::from_millis(i * 25)))
681//! .collect();
682//! previous_benchmarks.add(
683//! "benchmark-1",
684//! example,
685//! Config {
686//! resources: 1,
687//! workloads: workloads.clone(),
688//! },
689//! (1..=10).collect(),
690//! 5,
691//! 3,
692//! )?;
693//! previous_benchmarks.run()?;
694//! let previous_summary = previous_benchmarks.summary_as_json();
695//!
696//! let mut current_benchmarks = Benchmarks::new("benchmarks");
697//! current_benchmarks.add(
698//! "benchmark-1",
699//! modified_example,
700//! Config {
701//! resources: 1,
702//! workloads: workloads.clone(),
703//! },
704//! (1..=10).collect(),
705//! 5,
706//! 3,
707//! )?;
708//! current_benchmarks.add(
709//! "benchmark-2",
710//! modified_example,
711//! Config {
712//! resources: 2,
713//! workloads: workloads.clone(),
714//! },
715//! (1..=10).collect(),
716//! 5,
717//! 3,
718//! )?;
719//! current_benchmarks.run()?;
720//!
721//! // compare results of this run with the results of previous runs with threshold of 5 percent
722//! let analysis_result = current_benchmarks.analyze(Some(previous_summary), 5.0)?;
723//! println!("Analysis result:");
724//! println!("{}", analysis_result.to_string());
725//! assert!(!analysis_result.divergent_series().is_empty());
726//! Ok(())
727//! }
728//! ```
729//!
730//! </details>
731//!
732//!
733//! <details>
734//! <summary>Find Regressions Output</summary>
735//!
736//! Analysis result. Current and previous values are in nanosecond units, the change is in percents.
737//! ```json
738//! {
739//! "name": "benchmarks",
740//! "new_series": [
741//! "benchmark-2"
742//! ],
743//! "equal_series": {},
744//! "divergent_series": {
745//! "benchmark-1": {
746//! "8": {
747//! "Equal": {
748//! "point": "8",
749//! "previous": 200686042,
750//! "current": 206411375,
751//! "change": 2.852880520709064
752//! }
753//! },
754//! "7": {
755//! "Equal": {
756//! "point": "7",
757//! "previous": 176700166,
758//! "current": 183395333,
759//! "change": 3.788998704166474
760//! }
761//! },
762//! "10": {
763//! "Equal": {
764//! "point": "10",
765//! "previous": 251701083,
766//! "current": 256856000,
767//! "change": 2.0480313149864315
768//! }
769//! },
770//! "2": {
771//! "Equal": {
772//! "point": "2",
773//! "previous": 52684667,
774//! "current": 55052334,
775//! "change": 4.494034288951653
776//! }
777//! },
778//! "5": {
779//! "Equal": {
780//! "point": "5",
781//! "previous": 127510583,
782//! "current": 130817709,
783//! "change": 2.5936090340046434
784//! }
785//! },
786//! "6": {
787//! "Equal": {
788//! "point": "6",
789//! "previous": 152803084,
790//! "current": 158141750,
791//! "change": 3.4938208446106955
792//! }
793//! },
794//! "9": {
795//! "Equal": {
796//! "point": "9",
797//! "previous": 225706250,
798//! "current": 229522083,
799//! "change": 1.6906191122310474
800//! }
801//! },
802//! "1": {
803//! "Greater": {
804//! "point": "1",
805//! "previous": 26413209,
806//! "current": 35264166,
807//! "change": 33.50958605597674
808//! }
809//! },
810//! "4": {
811//! "Equal": {
812//! "point": "4",
813//! "previous": 104201208,
814//! "current": 109166958,
815//! "change": 4.7655397622645665
816//! }
817//! },
818//! "3": {
819//! "Equal": {
820//! "point": "3",
821//! "previous": 79801875,
822//! "current": 84060834,
823//! "change": 5.336915955922095
824//! }
825//! }
826//! }
827//! }
828//! }
829//! ```
830//! </details>
831//!
832//! # Similar Projects
833//! * [criterion](https://crates.io/crates/criterion)
834//! * [iai](https://crates.io/crates/iai)
835//!
836//!
837pub mod analysis_result;
838pub mod benchmark;
839pub mod benchmark_comparison;
840pub mod benchmarks;
841pub mod disk_usage;
842pub mod run_summary;
843pub mod series_summary;
844pub mod stopwatch;
845pub mod summary;