pub use self::TestFn::*;
use self::TestResult::*;
use self::TestEvent::*;
use self::NamePadding::*;
use self::OutputLocation::*;
use std::borrow::Cow;
use std::cmp;
use std::fmt;
use std::fs::File;
use std::io::prelude::*;
use std::io;
use std::iter::repeat;
use std::mem::forget;
use std::path::PathBuf;
use std::ptr;
use std::time::{Instant, Duration};
pub mod stats;
mod macros;
pub type TestName = Cow<'static, str>;
#[derive(Clone, Copy, PartialEq, Eq)]
enum NamePadding {
PadOnRight,
}
impl TestDesc {
fn padded_name(&self, column_count: usize, align: NamePadding) -> String {
let mut name = self.name.to_string();
let fill = column_count.saturating_sub(name.len());
let pad = repeat(" ").take(fill).collect::<String>();
match align {
PadOnRight => {
name.push_str(&pad);
name
}
}
}
}
pub trait TDynBenchFn: Send {
fn run(&self, harness: &mut Bencher);
}
pub enum TestFn {
StaticBenchFn(fn(&mut Bencher)),
DynBenchFn(Box<TDynBenchFn + 'static>),
}
impl TestFn {
fn padding(&self) -> NamePadding {
match *self {
StaticBenchFn(..) |
DynBenchFn(..) => PadOnRight,
}
}
}
impl fmt::Debug for TestFn {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str(match *self {
StaticBenchFn(..) => "StaticBenchFn(..)",
DynBenchFn(..) => "DynBenchFn(..)",
})
}
}
#[derive(Copy, Clone)]
pub struct Bencher {
iterations: u64,
dur: Duration,
pub bytes: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct TestDesc {
pub name: TestName,
pub ignore: bool,
}
#[derive(Clone)]
pub struct TestPaths {
pub file: PathBuf, pub base: PathBuf, pub relative_dir: PathBuf, }
#[derive(Debug)]
pub struct TestDescAndFn {
pub desc: TestDesc,
pub testfn: TestFn,
}
#[derive(Default)]
pub struct TestOpts {
pub filter: Option<String>,
pub run_ignored: bool,
pub logfile: Option<PathBuf>,
pub quiet: bool,
pub test_threads: Option<usize>,
}
#[derive(Clone, PartialEq)]
pub struct BenchSamples {
ns_iter_summ: stats::Summary,
mb_s: usize,
}
#[derive(Clone, PartialEq)]
enum TestResult {
TrIgnored,
TrBench(BenchSamples),
}
unsafe impl Send for TestResult {}
enum OutputLocation<T> {
Raw(T),
}
struct ConsoleTestState<T> {
log_out: Option<File>,
out: OutputLocation<T>,
quiet: bool,
total: usize,
passed: usize,
failed: usize,
ignored: usize,
measured: usize,
failures: Vec<(TestDesc, Vec<u8>)>,
max_name_len: usize, }
impl ConsoleTestState<()> {
pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestState<io::Stdout>> {
let log_out = match opts.logfile {
Some(ref path) => Some(try!(File::create(path))),
None => None,
};
let out = Raw(io::stdout());
Ok(ConsoleTestState {
out: out,
log_out: log_out,
quiet: opts.quiet,
total: 0,
passed: 0,
failed: 0,
ignored: 0,
measured: 0,
failures: Vec::new(),
max_name_len: 0,
})
}
}
impl<T: Write> ConsoleTestState<T> {
pub fn write_ignored(&mut self) -> io::Result<()> {
self.write_short_result("ignored", "i")
}
pub fn write_bench(&mut self) -> io::Result<()> {
self.write_pretty("bench")
}
pub fn write_short_result(&mut self, verbose: &str, quiet: &str)
-> io::Result<()> {
if self.quiet {
self.write_pretty(quiet)
} else {
try!(self.write_pretty(verbose));
self.write_plain("\n")
}
}
pub fn write_pretty(&mut self, word: &str) -> io::Result<()> {
match self.out {
Raw(ref mut stdout) => {
try!(stdout.write_all(word.as_bytes()));
stdout.flush()
}
}
}
pub fn write_plain(&mut self, s: &str) -> io::Result<()> {
match self.out {
Raw(ref mut stdout) => {
try!(stdout.write_all(s.as_bytes()));
stdout.flush()
}
}
}
pub fn write_run_start(&mut self, len: usize) -> io::Result<()> {
self.total = len;
let noun = if len != 1 {
"tests"
} else {
"test"
};
self.write_plain(&format!("\nrunning {} {}\n", len, noun))
}
pub fn write_test_start(&mut self, test: &TestDesc, align: NamePadding) -> io::Result<()> {
if self.quiet && align != PadOnRight {
Ok(())
} else {
let name = test.padded_name(self.max_name_len, align);
self.write_plain(&format!("test {} ... ", name))
}
}
pub fn write_result(&mut self, result: &TestResult) -> io::Result<()> {
match *result {
TrIgnored => self.write_ignored(),
TrBench(ref bs) => {
try!(self.write_bench());
self.write_plain(&format!(": {}\n", fmt_bench_samples(bs)))
}
}
}
pub fn write_log(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> {
match self.log_out {
None => Ok(()),
Some(ref mut o) => {
let s = format!("{} {}\n",
match *result {
TrIgnored => "ignored".to_owned(),
TrBench(ref bs) => fmt_bench_samples(bs),
},
test.name);
o.write_all(s.as_bytes())
}
}
}
pub fn write_failures(&mut self) -> io::Result<()> {
try!(self.write_plain("\nfailures:\n"));
let mut failures = Vec::new();
let mut fail_out = String::new();
for &(ref f, ref stdout) in &self.failures {
failures.push(f.name.to_string());
if !stdout.is_empty() {
fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name));
let output = String::from_utf8_lossy(stdout);
fail_out.push_str(&output);
fail_out.push_str("\n");
}
}
if !fail_out.is_empty() {
try!(self.write_plain("\n"));
try!(self.write_plain(&fail_out));
}
try!(self.write_plain("\nfailures:\n"));
failures.sort();
for name in &failures {
try!(self.write_plain(&format!(" {}\n", name)));
}
Ok(())
}
pub fn write_run_finish(&mut self) -> io::Result<bool> {
assert_eq!(self.passed + self.failed + self.ignored + self.measured, self.total);
let success = self.failed == 0;
if !success {
try!(self.write_failures());
}
try!(self.write_plain("\ntest result: "));
if success {
try!(self.write_pretty("ok"));
} else {
try!(self.write_pretty("FAILED"));
}
let s = format!(". {} passed; {} failed; {} ignored; {} measured\n\n",
self.passed,
self.failed,
self.ignored,
self.measured);
try!(self.write_plain(&s));
Ok(success)
}
}
fn fmt_thousands_sep(mut n: usize, sep: char) -> String {
use std::fmt::Write;
let mut output = String::new();
let mut trailing = false;
for &pow in &[9, 6, 3, 0] {
let base = 10_usize.pow(pow);
if pow == 0 || trailing || n / base != 0 {
if !trailing {
output.write_fmt(format_args!("{}", n / base)).unwrap();
} else {
output.write_fmt(format_args!("{:03}", n / base)).unwrap();
}
if pow != 0 {
output.push(sep);
}
trailing = true;
}
n %= base;
}
output
}
pub fn fmt_bench_samples(bs: &BenchSamples) -> String {
use std::fmt::Write;
let mut output = String::new();
let median = bs.ns_iter_summ.median as usize;
let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;
output.write_fmt(format_args!("{:>11} ns/iter (+/- {})",
fmt_thousands_sep(median, ','),
fmt_thousands_sep(deviation, ',')))
.unwrap();
if bs.mb_s != 0 {
output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap();
}
output
}
pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> {
fn callback<T: Write>(event: &TestEvent, st: &mut ConsoleTestState<T>) -> io::Result<()> {
match (*event).clone() {
TeFiltered(ref filtered_tests) => st.write_run_start(filtered_tests.len()),
TeWait(ref test, padding) => st.write_test_start(test, padding),
TeResult(test, result, _) => {
try!(st.write_log(&test, &result));
try!(st.write_result(&result));
match result {
TrIgnored => st.ignored += 1,
TrBench(_) => {
st.measured += 1
}
}
Ok(())
}
}
}
let mut st = try!(ConsoleTestState::new(opts));
fn len_if_padded(t: &TestDescAndFn) -> usize {
match t.testfn.padding() {
PadOnRight => t.desc.name.len(),
}
}
if let Some(t) = tests.iter().max_by_key(|t| len_if_padded(*t)) {
let n = &t.desc.name;
st.max_name_len = n.len();
}
try!(run_tests(opts, tests, |x| callback(&x, &mut st)));
st.write_run_finish()
}
#[test]
fn should_sort_failures_before_printing_them() {
let test_a = TestDesc {
name: Cow::from("a"),
ignore: false,
};
let test_b = TestDesc {
name: Cow::from("b"),
ignore: false,
};
let mut st = ConsoleTestState {
log_out: None,
out: Raw(Vec::new()),
quiet: false,
total: 0,
passed: 0,
failed: 0,
ignored: 0,
measured: 0,
max_name_len: 10,
failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
};
st.write_failures().unwrap();
let s = match st.out {
Raw(ref m) => String::from_utf8_lossy(&m[..]),
};
let apos = s.find("a").unwrap();
let bpos = s.find("b").unwrap();
assert!(apos < bpos);
}
#[derive(Clone)]
enum TestEvent {
TeFiltered(Vec<TestDesc>),
TeWait(TestDesc, NamePadding),
TeResult(TestDesc, TestResult, Vec<u8>),
}
type MonitorMsg = (TestDesc, TestResult, Vec<u8>);
fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()>
where F: FnMut(TestEvent) -> io::Result<()>
{
let filtered_tests = filter_tests(opts, tests);
let filtered_descs = filtered_tests.iter()
.map(|t| t.desc.clone())
.collect();
try!(callback(TeFiltered(filtered_descs)));
let filtered_benchs_and_metrics = filtered_tests;
for b in filtered_benchs_and_metrics {
try!(callback(TeWait(b.desc.clone(), b.testfn.padding())));
let (test, result, stdout) = run_test(opts, false, b);
try!(callback(TeResult(test, result, stdout)));
}
Ok(())
}
fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
let mut filtered = tests;
filtered = match opts.filter {
None => filtered,
Some(ref filter) => {
filtered.into_iter()
.filter(|test| test.desc.name.contains(&filter[..]))
.collect()
}
};
filtered = if !opts.run_ignored {
filtered
} else {
fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> {
if test.desc.ignore {
let TestDescAndFn {desc, testfn} = test;
Some(TestDescAndFn {
desc: TestDesc { ignore: false, ..desc },
testfn: testfn,
})
} else {
None
}
}
filtered.into_iter().filter_map(filter).collect()
};
filtered.sort_by(|t1, t2| t1.desc.name.cmp(&t2.desc.name));
filtered
}
fn run_test(_opts: &TestOpts,
force_ignore: bool,
test: TestDescAndFn) -> MonitorMsg
{
let TestDescAndFn {desc, testfn} = test;
if force_ignore || desc.ignore {
return (desc, TrIgnored, Vec::new());
}
match testfn {
DynBenchFn(bencher) => {
let bs = ::bench::benchmark(|harness| bencher.run(harness));
(desc, TrBench(bs), Vec::new())
}
StaticBenchFn(benchfn) => {
let bs = ::bench::benchmark(|harness| benchfn(harness));
(desc, TrBench(bs), Vec::new())
}
}
}
pub fn black_box<T>(dummy: T) -> T {
unsafe {
let ret = ptr::read_volatile(&dummy);
forget(dummy);
ret
}
}
impl Bencher {
pub fn iter<T, F>(&mut self, mut inner: F)
where F: FnMut() -> T
{
let start = Instant::now();
let k = self.iterations;
for _ in 0..k {
black_box(inner());
}
self.dur = start.elapsed();
}
pub fn ns_elapsed(&mut self) -> u64 {
self.dur.as_secs() * 1_000_000_000 + (self.dur.subsec_nanos() as u64)
}
pub fn ns_per_iter(&mut self) -> u64 {
if self.iterations == 0 {
0
} else {
self.ns_elapsed() / cmp::max(self.iterations, 1)
}
}
pub fn bench_n<F>(&mut self, n: u64, f: F)
where F: FnOnce(&mut Bencher)
{
self.iterations = n;
f(self);
}
pub fn auto_bench<F>(&mut self, mut f: F) -> stats::Summary
where F: FnMut(&mut Bencher)
{
let mut n = 1;
self.bench_n(n, |x| f(x));
if self.ns_per_iter() == 0 {
n = 1_000_000;
} else {
n = 1_000_000 / cmp::max(self.ns_per_iter(), 1);
}
if n == 0 {
n = 1;
}
let mut total_run = Duration::new(0, 0);
let samples: &mut [f64] = &mut [0.0_f64; 50];
loop {
let loop_start = Instant::now();
for p in &mut *samples {
self.bench_n(n, |x| f(x));
*p = self.ns_per_iter() as f64;
}
stats::winsorize(samples, 5.0);
let summ = stats::Summary::new(samples);
for p in &mut *samples {
self.bench_n(5 * n, |x| f(x));
*p = self.ns_per_iter() as f64;
}
stats::winsorize(samples, 5.0);
let summ5 = stats::Summary::new(samples);
let loop_run = loop_start.elapsed();
if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 &&
summ.median - summ5.median < summ5.median_abs_dev {
return summ5;
}
total_run += loop_run;
if total_run > Duration::from_secs(3) {
return summ5;
}
n = match n.checked_mul(10) {
Some(_) => n * 2,
None => return summ5,
};
}
}
}
pub mod bench {
use std::cmp;
use std::time::Duration;
use super::{Bencher, BenchSamples};
pub fn benchmark<F>(f: F) -> BenchSamples
where F: FnMut(&mut Bencher)
{
let mut bs = Bencher {
iterations: 0,
dur: Duration::new(0, 0),
bytes: 0,
};
let ns_iter_summ = bs.auto_bench(f);
let ns_iter = cmp::max(ns_iter_summ.median as u64, 1);
let mb_s = bs.bytes * 1000 / ns_iter;
BenchSamples {
ns_iter_summ: ns_iter_summ,
mb_s: mb_s as usize,
}
}
pub fn run_once<F>(f: F)
where F: FnOnce(&mut Bencher)
{
let mut bs = Bencher {
iterations: 0,
dur: Duration::new(0, 0),
bytes: 0,
};
bs.bench_n(1, f);
}
}