use crate::parse_deqp::DeqpStatus;
use anyhow::{Context, Result};
use regex::Regex;
use std::collections::HashMap;
use std::fmt;
use std::io::prelude::*;
use std::io::{BufReader, BufWriter};
use std::str::FromStr;
use std::time::{Duration, Instant};
use structopt::StructOpt;
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum RunnerStatus {
Pass,
Fail,
Skip,
Crash,
Flake,
Warn,
Missing,
ExpectedFail,
UnexpectedPass,
Timeout,
}
impl FromStr for RunnerStatus {
type Err = anyhow::Error;
fn from_str(input: &str) -> Result<RunnerStatus, Self::Err> {
match input {
"Pass" => Ok(RunnerStatus::Pass),
"Fail" => Ok(RunnerStatus::Fail),
"Crash" => Ok(RunnerStatus::Crash),
"Skip" => Ok(RunnerStatus::Skip),
"Flake" => Ok(RunnerStatus::Flake),
"Warn" => Ok(RunnerStatus::Warn),
"Missing" => Ok(RunnerStatus::Missing),
"ExpectedFail" => Ok(RunnerStatus::ExpectedFail),
"UnexpectedPass" => Ok(RunnerStatus::UnexpectedPass),
"Timeout" => Ok(RunnerStatus::Timeout),
_ => anyhow::bail!("unknown runner status '{}'", input),
}
}
}
impl RunnerStatus {
pub fn is_success(&self) -> bool {
match self {
RunnerStatus::Pass
| RunnerStatus::Skip
| RunnerStatus::Warn
| RunnerStatus::Flake
| RunnerStatus::ExpectedFail => true,
RunnerStatus::Fail
| RunnerStatus::Crash
| RunnerStatus::Missing
| RunnerStatus::UnexpectedPass
| RunnerStatus::Timeout => false,
}
}
pub fn from_deqp(status: DeqpStatus) -> RunnerStatus {
match status {
DeqpStatus::Pass => RunnerStatus::Pass,
DeqpStatus::Fail
| DeqpStatus::ResourceError
| DeqpStatus::InternalError
| DeqpStatus::Pending => RunnerStatus::Fail,
DeqpStatus::Crash => RunnerStatus::Crash,
DeqpStatus::NotSupported => RunnerStatus::Skip,
DeqpStatus::CompatibilityWarning | DeqpStatus::QualityWarning | DeqpStatus::Waiver => {
RunnerStatus::Warn
}
DeqpStatus::Timeout => RunnerStatus::Timeout,
}
}
pub fn with_baseline(self, baseline: Option<RunnerStatus>) -> RunnerStatus {
use RunnerStatus::*;
if let Some(baseline) = baseline {
match self {
Fail => match baseline {
Fail | ExpectedFail => ExpectedFail,
Crash => UnexpectedPass,
_ => self,
},
Pass => match baseline {
Fail | Crash => UnexpectedPass,
_ => Pass,
},
Crash => match baseline {
Crash | ExpectedFail => ExpectedFail,
_ => Crash,
},
Warn => match baseline {
Fail | Crash => UnexpectedPass,
_ => Warn,
},
Skip => {
self
}
Flake => Flake,
Timeout => Timeout,
Missing | ExpectedFail | UnexpectedPass => {
unreachable!("can't appear from DeqpStatus")
}
}
} else {
self
}
}
}
impl fmt::Display for RunnerStatus {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str(match self {
RunnerStatus::Pass => "Pass",
RunnerStatus::Fail => "Fail",
RunnerStatus::Skip => "Skip",
RunnerStatus::Crash => "Crash",
RunnerStatus::Warn => "Warn",
RunnerStatus::Flake => "Flake",
RunnerStatus::Missing => "Missing",
RunnerStatus::ExpectedFail => "ExpectedFail",
RunnerStatus::UnexpectedPass => "UnexpectedPass",
RunnerStatus::Timeout => "Timeout",
})
}
}
#[derive(Debug)]
pub struct RunnerResult {
pub test: String,
pub status: RunnerStatus,
pub duration: Duration,
}
impl PartialEq for RunnerResult {
fn eq(&self, other: &Self) -> bool {
self.status == other.status
}
}
pub struct CaselistState {
pub caselist_id: u32,
pub run_id: u32,
}
#[derive(Default, PartialEq, Debug)]
pub struct ResultCounts {
pub pass: u32,
pub fail: u32,
pub skip: u32,
pub crash: u32,
pub warn: u32,
pub flake: u32,
pub missing: u32,
pub expected_fail: u32,
pub unexpected_pass: u32,
pub timeout: u32,
pub total: u32,
}
impl ResultCounts {
pub fn new() -> ResultCounts {
Default::default()
}
pub fn increment(&mut self, s: RunnerStatus) {
match s {
RunnerStatus::Pass => self.pass += 1,
RunnerStatus::Fail => self.fail += 1,
RunnerStatus::Skip => self.skip += 1,
RunnerStatus::Crash => self.crash += 1,
RunnerStatus::Warn => self.warn += 1,
RunnerStatus::Flake => self.flake += 1,
RunnerStatus::Missing => self.missing += 1,
RunnerStatus::ExpectedFail => self.expected_fail += 1,
RunnerStatus::UnexpectedPass => self.unexpected_pass += 1,
RunnerStatus::Timeout => self.timeout += 1,
}
self.total += 1;
}
pub fn get_count(&self, status: RunnerStatus) -> u32 {
use RunnerStatus::*;
match status {
Pass => self.pass,
Fail => self.fail,
Skip => self.skip,
Crash => self.crash,
Warn => self.warn,
Flake => self.flake,
Missing => self.missing,
ExpectedFail => self.expected_fail,
UnexpectedPass => self.unexpected_pass,
Timeout => self.timeout,
}
}
}
impl fmt::Display for ResultCounts {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
use RunnerStatus::*;
write!(f, "Pass: {}", self.pass)?;
for status in &[
Fail,
Crash,
UnexpectedPass,
ExpectedFail,
Warn,
Skip,
Timeout,
Missing,
Flake,
] {
let count = self.get_count(*status);
if count != 0 {
write!(f, ", {}: {}", status, count)?;
}
}
Ok(())
}
}
pub struct RunnerResults {
pub tests: HashMap<String, RunnerResult>,
pub result_counts: ResultCounts,
pub time: Instant,
}
impl RunnerResults {
pub fn new() -> RunnerResults {
Default::default()
}
pub fn record_result(&mut self, result: RunnerResult) {
self.result_counts.increment(result.status);
self.tests.insert(result.test.clone(), result);
}
pub fn is_success(&self) -> bool {
self.tests
.iter()
.all(|(_, result)| result.status.is_success())
}
pub fn write_results<W: Write>(&self, writer: &mut W) -> Result<()> {
let mut sorted: Vec<_> = self.tests.iter().collect();
sorted.sort_by(|x, y| (*x).0.cmp((*y).0));
let mut writer = BufWriter::new(writer);
for (test, result) in sorted {
writeln!(
writer,
"{},{},{}",
test,
result.status,
result.duration.as_secs_f32()
)?;
}
Ok(())
}
pub fn write_failures<W>(&self, writer: &mut W) -> Result<()>
where
W: Write,
{
let mut sorted: Vec<_> = self.tests.iter().collect();
sorted.sort_by(|x, y| (*x).0.cmp((*y).0));
let mut writer = BufWriter::new(writer);
for (test, result) in sorted {
if !result.status.is_success() {
writeln!(writer, "{},{}", test, result.status,)?;
}
}
Ok(())
}
pub fn write_junit_failures<W>(
&self,
writer: &mut W,
options: &JunitGeneratorOptions,
) -> Result<()>
where
W: Write,
{
use junit_report::*;
let mut sorted: Vec<_> = self.tests.iter().collect();
sorted.sort_by(|x, y| (*x).0.cmp((*y).0));
let limit = if options.limit == 0 {
std::usize::MAX
} else {
options.limit
};
let mut testcases = Vec::new();
for (test, result) in sorted.iter().take(limit) {
let tc = if !result.status.is_success() {
let message = options.template.replace("{{testcase}}", test);
let type_ = format!("{}", result.status);
junit_report::TestCase::failure(test, Duration::seconds(0), &type_, &message)
} else {
junit_report::TestCase::success(test, Duration::seconds(0))
};
testcases.push(tc);
}
let ts = junit_report::TestSuite::new(&options.testsuite).add_testcases(testcases);
junit_report::Report::new()
.add_testsuite(ts)
.write_xml(BufWriter::new(writer))
.context("writing XML output")
}
pub fn from_csv<R: Read>(r: &mut R) -> Result<RunnerResults>
where
R: Read,
{
lazy_static! {
static ref CSV_RE: Regex = Regex::new("^([^,]+),([^,]+)").unwrap();
}
let mut results = RunnerResults::new();
let r = BufReader::new(r);
for line in r.lines() {
let line = line.context("Reading CSV")?;
if let Some(cap) = CSV_RE.captures(&line) {
let result = RunnerResult {
test: cap[1].to_string(),
status: cap[2].parse()?,
duration: Duration::default(),
};
results.record_result(result);
}
}
Ok(results)
}
pub fn print_summary(&self, summary_limit: usize) {
if self.tests.is_empty() {
return;
}
let mut slowest: Vec<_> = self
.tests
.iter()
.map(|(test, result)| (test, result.duration))
.collect();
slowest.sort_by_key(|x| x.1);
slowest.reverse();
println!();
println!("Slowest tests:");
for test in slowest.iter().take(5) {
println!(" {} ({:.02}s)", test.0, test.1.as_secs_f32());
}
let mut flakes: Vec<_> = self
.tests
.iter()
.map(|(name, result)| (name, result.status))
.filter(|(_, status)| *status == RunnerStatus::Flake)
.collect();
if !flakes.is_empty() {
flakes.sort_by_key(|x| x.0);
println!();
println!("Some flaky tests found:");
for test in flakes.iter().take(summary_limit) {
println!(" {}", test.0)
}
if flakes.len() > summary_limit {
println!(" ... and more (see results.csv)");
}
}
let mut fails: Vec<_> = self
.tests
.iter()
.map(|(name, result)| (name, result.status))
.filter(|(_, status)| !status.is_success())
.collect();
if !fails.is_empty() {
fails.sort_by_key(|x| x.0);
println!();
println!("Some failures found:");
for test in fails.iter().take(summary_limit) {
println!(" {},{}", test.0, test.1)
}
if fails.len() > summary_limit {
println!(" ... and more (see failures.csv)");
}
}
}
}
impl Default for RunnerResults {
fn default() -> RunnerResults {
RunnerResults {
tests: Default::default(),
result_counts: Default::default(),
time: Instant::now(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn status_map() {
use RunnerStatus::*;
assert_eq!(
Pass,
RunnerStatus::from_deqp(DeqpStatus::Pass).with_baseline(None)
);
assert_eq!(
Fail,
RunnerStatus::from_deqp(DeqpStatus::Fail).with_baseline(None)
);
assert_eq!(
Crash,
RunnerStatus::from_deqp(DeqpStatus::Crash).with_baseline(None)
);
assert_eq!(
Warn,
RunnerStatus::from_deqp(DeqpStatus::CompatibilityWarning).with_baseline(None)
);
assert_eq!(
Warn,
RunnerStatus::from_deqp(DeqpStatus::QualityWarning).with_baseline(None)
);
assert_eq!(
ExpectedFail,
RunnerStatus::from_deqp(DeqpStatus::Fail).with_baseline(Some(Fail))
);
assert_eq!(
ExpectedFail,
RunnerStatus::from_deqp(DeqpStatus::Crash).with_baseline(Some(Crash))
);
assert_eq!(
UnexpectedPass,
RunnerStatus::from_deqp(DeqpStatus::Pass).with_baseline(Some(Fail))
);
assert_eq!(
UnexpectedPass,
RunnerStatus::from_deqp(DeqpStatus::Fail).with_baseline(Some(Crash))
);
assert_eq!(
ExpectedFail,
RunnerStatus::from_deqp(DeqpStatus::Fail).with_baseline(Some(ExpectedFail))
);
assert_eq!(
ExpectedFail,
RunnerStatus::from_deqp(DeqpStatus::Crash).with_baseline(Some(ExpectedFail))
);
}
}
#[derive(Debug, StructOpt)]
pub struct JunitGeneratorOptions {
#[structopt(long, help = "Testsuite name for junit")]
testsuite: String,
#[structopt(
long,
default_value = "",
help = "Failure message template (with {{testcase}} replaced with the test name)"
)]
template: String,
#[structopt(
long,
default_value = "0",
help = "Number of junit cases to list (or 0 for unlimited)"
)]
limit: usize,
}