#[macro_use]
extern crate lazy_static;
mod parse_deqp;
pub mod parse_piglit;
mod runner_results;
pub use runner_results::*;
use anyhow::{Context, Result};
use log::*;
use parse_deqp::{DeqpStatus, DeqpTestResult};
use rand::rngs::StdRng;
use rand::seq::SliceRandom;
use rand::{Rng, SeedableRng};
use rayon::prelude::*;
use regex::RegexSet;
use std::fs::File;
use std::io::prelude::*;
use std::io::{BufReader, BufWriter};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::mpsc::{channel, Receiver};
use std::time::Duration;
use std::{fmt, time::Instant};
struct HMSDuration(Duration);
impl fmt::Display for HMSDuration {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let mut secs = self.0.as_secs();
let mut has_hours = false;
if secs >= 3600 {
write!(f, "{}:", secs / 3600)?;
secs %= 3600;
has_hours = true;
}
if secs >= 60 {
if has_hours {
write!(f, "{:02}:", secs / 60)?;
} else {
write!(f, "{}:", secs / 60)?;
}
secs %= 60;
write!(f, "{:02}", secs)
} else {
write!(f, "{}", secs)
}
}
}
pub trait Deqp {
fn skips(&self) -> Option<&RegexSet>;
fn flakes(&self) -> Option<&RegexSet>;
fn run<S: AsRef<TestCase>, I: IntoIterator<Item = S>>(
&self,
caselist_state: &CaselistState,
tests: I,
) -> Result<Vec<RunnerResult>>;
fn see_more(&self, _caselist_state: &CaselistState) -> String {
"".to_string()
}
fn status_update(&self, results: &RunnerResults, total_tests: u32) {
let duration = results.time.elapsed();
print!(
"{}, Duration: {duration}",
results.result_counts,
duration = HMSDuration(duration),
);
let duration = duration.as_secs_f32();
if results.result_counts.total != 0 {
let average_test_time = duration / results.result_counts.total as f32;
let remaining = average_test_time * (total_tests - results.result_counts.total) as f32;
print!(
", Remaining: {}",
HMSDuration(Duration::from_secs_f32(remaining))
);
}
println!();
}
fn baseline(&self) -> &RunnerResults;
fn min_tests_per_group(&self) -> usize {
20
}
fn tests_per_group(&self) -> usize {
500
}
fn qpa_to_xml(&self) -> Option<&PathBuf> {
None
}
fn baseline_status<S: AsRef<str>>(&self, test: S) -> Option<RunnerStatus> {
self.baseline().tests.get(test.as_ref()).map(|x| x.status)
}
fn translate_result(
&self,
result: DeqpTestResult,
caselist_state: &CaselistState,
tests: &[TestCase],
) -> RunnerResult {
let test = match (&tests).iter().find(|x| result.name.eq(x.name())) {
Some(v) => v.clone(),
None => panic!("Can't find test {} in test cases", result.name),
};
let mut status = RunnerStatus::from_deqp(result.status)
.with_baseline(self.baseline_status(&test.name()));
if let Some(flakes) = self.flakes() {
if !status.is_success() && flakes.is_match(&test.name()) {
status = RunnerStatus::Flake;
}
}
if !status.is_success() || status == RunnerStatus::Flake {
error!(
"Test {}: {}: {}",
test.name(),
status,
self.see_more(&caselist_state)
);
}
RunnerResult {
test,
status,
duration: result.duration,
}
}
fn results_collection(
&self,
run_results: &mut RunnerResults,
total_tests: u32,
receiver: Receiver<Result<Vec<RunnerResult>>>,
) {
let update_interval = Duration::new(2, 0);
self.status_update(run_results, total_tests);
let mut last_status_update = Instant::now();
for group_results in receiver {
match group_results {
Ok(group_results) => {
for result in group_results {
assert!(!run_results.tests.contains_key(result.test.name()));
run_results.record_result(result);
}
}
Err(e) => {
println!("Error: {}", e);
}
}
if last_status_update.elapsed() >= update_interval {
self.status_update(run_results, total_tests);
last_status_update = Instant::now();
}
}
self.status_update(run_results, total_tests);
}
fn skip_test(&self, test: &str) -> bool {
if let Some(skips) = self.skips() {
skips.is_match(test)
} else {
false
}
}
fn run_caselist_and_flake_detect(
&self,
caselist: &[TestCase],
caselist_state: &mut CaselistState,
) -> Result<Vec<RunnerResult>> {
let mut caselist: Vec<_> = caselist.iter().collect();
caselist.sort_by(|x, y| x.name().cmp(y.name()));
caselist_state.run_id += 1;
let mut results = self.run(&caselist_state, &caselist)?;
if results.is_empty() {
anyhow::bail!("No results parsed");
}
if results.iter().any(|x| !x.status.is_success()) {
caselist_state.run_id += 1;
let retest_results = self.run(&caselist_state, &caselist)?;
for pair in results.iter_mut().zip(retest_results.iter()) {
if pair.0.status != pair.1.status {
pair.0.status = RunnerStatus::Flake;
}
}
}
Ok(results)
}
fn process_caselist<S: AsRef<TestCase>, I: IntoIterator<Item = S>>(
&self,
tests: I,
caselist_id: u32,
) -> Result<Vec<RunnerResult>> {
let mut caselist_results: Vec<RunnerResult> = Vec::new();
let mut remaining_tests = Vec::new();
for test in tests {
let test = test.as_ref().clone();
if self.skip_test(&test.name()) {
caselist_results.push(RunnerResult {
test,
status: RunnerStatus::Skip,
duration: Default::default(),
});
} else {
remaining_tests.push(test);
}
}
let mut caselist_state = CaselistState {
caselist_id,
run_id: 0,
};
while !remaining_tests.is_empty() {
let results = self.run_caselist_and_flake_detect(&remaining_tests, &mut caselist_state);
match results {
Ok(results) => {
for result in results {
remaining_tests.swap_remove(
remaining_tests
.iter()
.position(|x| *x == result.test)
.with_context(|| {
format!("Finding test name {} in test list", result.test.name())
})?,
);
caselist_results.push(result);
}
}
Err(e) => {
error!(
"Failure getting run results: {:#} ({})",
e,
self.see_more(&caselist_state)
);
for test in remaining_tests {
caselist_results.push(RunnerResult {
test,
status: RunnerStatus::Missing,
duration: Default::default(),
});
}
break;
}
}
caselist_state.caselist_id += 1;
}
Ok(caselist_results)
}
fn split_tests_to_groups(&self, mut tests: Vec<TestCase>) -> Vec<(u32, Vec<TestCase>)> {
tests.shuffle(&mut StdRng::from_seed([0x3bu8; 32]));
let mut test_groups: Vec<(u32, Vec<TestCase>)> = Vec::new();
let mut remaining = tests.len();
let mut i = 0u32;
while remaining != 0 {
let min = usize::min(self.min_tests_per_group(), remaining);
let group_len = usize::min(usize::max(remaining / 32, min), self.tests_per_group());
remaining -= group_len;
test_groups.push((i, tests.split_off(remaining)));
i += 1;
}
test_groups
}
}
pub struct DeqpCommand {
pub deqp: PathBuf,
pub args: Vec<String>,
pub output_dir: PathBuf,
pub skips: Option<RegexSet>,
pub flakes: Option<RegexSet>,
pub qpa_to_xml: Option<PathBuf>,
pub baseline: RunnerResults,
pub timeout: Duration,
pub tests_per_group: usize,
pub min_tests_per_group: usize,
}
pub struct PiglitCommand {
pub piglit_folder: PathBuf,
pub profile: String,
pub output_dir: PathBuf,
pub skips: Option<RegexSet>,
pub flakes: Option<RegexSet>,
pub baseline: RunnerResults,
pub timeout: Duration,
}
#[derive(Debug, Eq, PartialEq, Clone)]
pub struct PiglitTest {
pub name: String,
pub binary: String,
pub args: Vec<String>,
}
#[derive(Clone, Debug, PartialEq)]
pub enum TestCase {
Deqp(String),
Piglit(PiglitTest),
}
impl TestCase {
fn name(&self) -> &str {
match self {
TestCase::Deqp(name) => &name,
TestCase::Piglit(test) => &test.name,
}
}
}
impl AsRef<str> for TestCase {
fn as_ref(&self) -> &str {
match self {
TestCase::Deqp(name) => &name,
TestCase::Piglit(test) => &test.name,
}
}
}
impl AsRef<TestCase> for TestCase {
fn as_ref(&self) -> &TestCase {
self
}
}
fn write_caselist_file(filename: &Path, tests: &[TestCase]) -> Result<()> {
let file = File::create(filename)
.with_context(|| format!("creating temp caselist file {}", filename.display()))?;
let mut file = BufWriter::new(file);
for test in (&tests).iter() {
file.write(test.name().as_bytes())
.context("writing temp caselist")?;
file.write(b"\n").context("writing temp caselist")?;
}
Ok(())
}
fn add_filename_arg(args: &mut Vec<String>, arg: &str, path: &Path) -> Result<()> {
args.push(format!(
"{}={}",
arg,
path.to_str()
.with_context(|| format!("filename to utf8 for {}", path.display()))?
));
Ok(())
}
fn filter_qpa<R: Read, S: AsRef<str>>(reader: R, test: S) -> Result<String> {
let lines = BufReader::new(reader).lines();
let start = format!("#beginTestCaseResult {}", test.as_ref());
let mut found_case = false;
let mut including = true;
let mut output = String::new();
for line in lines {
let line = line.context("reading QPA")?;
if line == start {
found_case = true;
including = true;
}
if including {
output.push_str(&line);
output.push('\n');
}
if line == "#beginSession" {
including = false;
}
if including && line == "#endTestCaseResult" {
break;
}
}
if !found_case {
anyhow::bail!("Failed to find {} in QPA", test.as_ref());
}
Ok(output)
}
impl DeqpCommand {
fn caselist_file_path(&self, caselist_state: &CaselistState, suffix: &str) -> Result<PathBuf> {
let output_dir = self.output_dir.canonicalize()?;
Ok(output_dir.join(format!("c{}.{}", caselist_state.caselist_id, suffix)))
}
fn try_extract_qpa<S: AsRef<str>, P: AsRef<Path>>(&self, test: S, qpa_path: P) -> Result<()> {
let qpa_path = qpa_path.as_ref();
let test = test.as_ref();
let output = filter_qpa(
File::open(qpa_path).with_context(|| format!("Opening {}", qpa_path.display()))?,
test,
)?;
if !output.is_empty() {
let out_path = qpa_path.parent().unwrap().join(format!("{}.qpa", test));
{
let mut out_qpa = BufWriter::new(File::create(&out_path).with_context(|| {
format!("Opening output QPA file {:?}", qpa_path.display())
})?);
out_qpa.write_all(output.as_bytes())?;
}
if let Some(qpa_to_xml) = self.qpa_to_xml() {
let xml_path = out_path.with_extension("xml");
let convert_output = Command::new(qpa_to_xml)
.current_dir(self.deqp.parent().unwrap_or_else(|| Path::new("/")))
.arg(&out_path)
.arg(xml_path)
.output()
.with_context(|| format!("Failed to spawn {}", qpa_to_xml.display()))?;
if !convert_output.status.success() {
anyhow::bail!(
"Failed to run {}: {}",
qpa_to_xml.display(),
String::from_utf8_lossy(&convert_output.stderr)
);
} else {
std::fs::remove_file(&out_path).context("removing converted QPA")?;
}
}
}
Ok(())
}
}
impl Deqp for DeqpCommand {
fn run<S: AsRef<TestCase>, I: IntoIterator<Item = S>>(
&self,
caselist_state: &CaselistState,
tests_o: I,
) -> Result<Vec<RunnerResult>> {
let caselist_path = self
.caselist_file_path(
&caselist_state,
format!("r{}.caselist.txt", caselist_state.run_id).as_str(),
)
.context("caselist path")?;
let qpa_path = self
.caselist_file_path(
&caselist_state,
format!("r{}.qpa", caselist_state.run_id).as_str(),
)
.context("qpa path")?;
let cache_path = self
.output_dir
.canonicalize()
.context("cache path")?
.join(format!("t{}.shader_cache", thread_id::get()));
let mut tests: Vec<TestCase> = Vec::new();
for test in tests_o {
tests.push(test.as_ref().clone());
}
write_caselist_file(&caselist_path, &tests).context("writing caselist file")?;
let mut args: Vec<String> = Vec::new();
add_filename_arg(&mut args, "--deqp-caselist-file", &caselist_path)
.context("adding caselist to args")?;
add_filename_arg(&mut args, "--deqp-log-filename", &qpa_path)
.context("adding log to args")?;
args.push("--deqp-log-flush=disable".to_string());
add_filename_arg(&mut args, "--deqp-shadercache-filename", &cache_path)
.context("adding cache to args")?;
args.push("--deqp-shadercache-truncate=disable".to_string());
for arg in &self.args {
args.push(arg.clone());
}
let mut child = Command::new(&self.deqp)
.current_dir(self.deqp.parent().unwrap_or_else(|| Path::new("/")))
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.stdin(Stdio::null())
.args(args)
.env("MESA_DEBUG", "silent")
.spawn()
.with_context(|| format!("Failed to spawn {}", &self.deqp.display()))?;
let stdout = child.stdout.take().context("opening stdout")?;
let deqp_results = parse_deqp::parse_deqp_results_with_timeout(stdout, self.timeout);
let _ = child.kill();
child.wait().context("waiting for child")?;
let mut deqp_results = deqp_results.context("parsing results")?;
let stderr = BufReader::new(child.stderr.as_mut().context("opening stderr")?);
for line in stderr.lines() {
if let Ok(line) = line {
if line.contains("ERROR: LeakSanitizer: detected memory leaks") {
error!(
"deqp-runner: Leak detected, marking caselist as failed ({})",
self.see_more(caselist_state)
);
for result in deqp_results.iter_mut() {
result.status = DeqpStatus::Fail;
}
}
error!("dEQP error: {}", line);
}
}
let mut results: Vec<RunnerResult> = Vec::new();
for result in deqp_results {
let result = self.translate_result(result, &caselist_state, &tests);
if !result.status.is_success() {
if let Err(e) = self.try_extract_qpa(&result.test, &qpa_path) {
warn!("Failed to extract QPA resuls: {}", e)
}
}
results.push(result);
}
if results.iter().all(|x| !x.status.should_save_logs()) {
std::fs::remove_file(caselist_path).context("removing caselist")?;
}
std::fs::remove_file(qpa_path).context("removing qpa")?;
Ok(results)
}
fn see_more(&self, caselist_state: &CaselistState) -> String {
let qpa_path = self.output_dir.join(
format!(
"c{}.r{}.caselist.txt",
caselist_state.caselist_id, caselist_state.run_id
)
.as_str(),
);
format!("See {:?}", qpa_path)
}
fn skips(&self) -> Option<&RegexSet> {
self.skips.as_ref()
}
fn flakes(&self) -> Option<&RegexSet> {
self.flakes.as_ref()
}
fn baseline(&self) -> &RunnerResults {
&self.baseline
}
fn tests_per_group(&self) -> usize {
self.tests_per_group
}
fn min_tests_per_group(&self) -> usize {
self.min_tests_per_group
}
fn qpa_to_xml(&self) -> Option<&PathBuf> {
self.qpa_to_xml.as_ref()
}
}
impl Deqp for PiglitCommand {
fn run<S: AsRef<TestCase>, I: IntoIterator<Item = S>>(
&self,
caselist_state: &CaselistState,
tests: I,
) -> Result<Vec<RunnerResult>> {
let mut bin_path = self.piglit_folder.clone();
bin_path.push("bin");
let tests: Vec<TestCase> = tests.into_iter().map(|x| x.as_ref().clone()).collect();
assert!(self.tests_per_group() == 1);
let test = &tests[0];
let test = match test {
TestCase::Piglit(t) => t,
_ => panic!("Invalid case"),
};
let log_path = self.output_dir.join(
format!(
"c{}.r{}.log",
caselist_state.caselist_id, caselist_state.run_id
)
.as_str(),
);
let mut command = Command::new(bin_path.join(Path::new(&test.binary)));
command
.current_dir(&self.piglit_folder)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.stdin(Stdio::null())
.args(&test.args)
.env("MESA_DEBUG", "silent")
.env("PIGLIT_SOURCE_DIR", &self.piglit_folder);
let command_line = format!("{:?}", command);
let mut stderr = Vec::new();
let mut status = None;
let (piglit_result, stdout) = match command
.spawn()
.with_context(|| format!("Failed to spawn {}", &test.binary))
{
Ok(mut child) => {
let stdout = child.stdout.take().context("opening stdout")?;
let r = parse_piglit::parse_piglit_results_with_timeout(
&test.name,
stdout,
self.timeout,
);
let _ = child.kill();
if let Ok(s) = child.wait() {
status = Some(s);
}
for line in BufReader::new(child.stderr.as_mut().context("opening stderr")?).lines()
{
if let Ok(line) = line {
stderr.push(line);
}
}
r.context("parsing results")?
}
Err(e) => (
DeqpTestResult {
name: test.name.to_string(),
status: DeqpStatus::Fail,
duration: std::time::Duration::new(0, 0),
},
vec![format!("Error spawning piglit command: {:?}", e)],
),
};
let mut results = Vec::new();
let translated_result = self.translate_result(piglit_result, &caselist_state, &tests);
if translated_result.status.should_save_logs() {
let mut file = File::create(log_path).context("opening log file")?;
fn write_output(file: &mut File, name: &str, out: Vec<String>) -> Result<()> {
if out.is_empty() {
writeln!(file, "{}: (empty)", name)?;
} else {
writeln!(file, "{}:", name)?;
writeln!(file, "-------")?;
for line in out {
writeln!(file, "{}", line)?;
}
}
Ok(())
};
|| -> Result<()> {
writeln!(file, "test: {}", test.name)?;
writeln!(file, "command: {}", command_line)?;
if let Some(status) = status {
writeln!(file, "exit status: {}", status)?;
}
write_output(&mut file, "stdout", stdout)?;
write_output(&mut file, "stderr", stderr)?;
Ok(())
}()
.context("writing log file")?;
}
results.push(translated_result);
Ok(results)
}
fn see_more(&self, caselist_state: &CaselistState) -> String {
let log_path = self.output_dir.join(
format!(
"c{}.r{}.log",
caselist_state.caselist_id, caselist_state.run_id
)
.as_str(),
);
format!("See {:?}", log_path)
}
fn skips(&self) -> Option<&RegexSet> {
self.skips.as_ref()
}
fn flakes(&self) -> Option<&RegexSet> {
self.flakes.as_ref()
}
fn baseline(&self) -> &RunnerResults {
&self.baseline
}
fn tests_per_group(&self) -> usize {
1
}
}
pub fn parallel_deqp<D>(deqp: &D, tests: Vec<TestCase>) -> Result<RunnerResults>
where
D: Deqp,
D: Sync,
{
let test_count = tests.len();
let test_groups = deqp.split_tests_to_groups(tests);
let mut run_results = RunnerResults::new();
let (sender, receiver) = channel::<Result<Vec<RunnerResult>>>();
crossbeam_utils::thread::scope(|s| {
s.spawn(|_| deqp.results_collection(&mut run_results, test_count as u32, receiver));
test_groups
.into_iter()
.par_bridge()
.try_for_each_with(sender, |sender, (i, tests)| {
sender.send(deqp.process_caselist(tests, i))
})
.unwrap();
})
.unwrap();
Ok(run_results)
}
pub fn parse_regex_set<I, S>(exprs: I) -> Result<RegexSet>
where
S: AsRef<str>,
I: IntoIterator<Item = S>,
{
RegexSet::new(
exprs
.into_iter()
.filter(|x| !x.as_ref().is_empty() && !x.as_ref().starts_with('#')),
)
.context("Parsing regex set")
}
#[derive(Default)]
pub struct DeqpMock {
pub skips: Option<RegexSet>,
pub flakes: Option<RegexSet>,
pub baseline: RunnerResults,
}
impl DeqpMock {
pub fn new() -> DeqpMock {
Default::default()
}
pub fn skips(&self) -> Option<&RegexSet> {
self.skips.as_ref()
}
pub fn flakes(&self) -> Option<&RegexSet> {
self.flakes.as_ref()
}
pub fn with_skips<S>(&mut self, skips: S) -> &mut DeqpMock
where
S: AsRef<str>,
{
self.skips = Some(parse_regex_set(skips.as_ref().lines()).unwrap());
self
}
pub fn with_flakes<S>(&mut self, flakes: S) -> &mut DeqpMock
where
S: AsRef<str>,
{
self.flakes = Some(parse_regex_set(flakes.as_ref().lines()).unwrap());
self
}
pub fn with_baseline<S>(&mut self, baseline: S) -> &mut DeqpMock
where
S: AsRef<str>,
{
self.baseline =
RunnerResults::from_csv(&mut std::io::Cursor::new(baseline.as_ref())).unwrap();
self
}
}
impl Deqp for DeqpMock {
fn run<S: AsRef<TestCase>, I: IntoIterator<Item = S>>(
&self,
_caselist_state: &CaselistState,
tests: I,
) -> Result<Vec<RunnerResult>> {
let mut tests: Vec<TestCase> = tests.into_iter().map(|x| x.as_ref().clone()).collect();
tests.sort_by(|x, y| x.name().cmp(y.name()));
let mut output = String::from("dEQP Mock starting\n");
let mut timeout_test = None;
for test in (&tests).iter() {
let test_name = test.name();
if test_name.contains("dEQP-GLES2.test.m.") {
continue;
}
if test_name.contains(".timeout.") {
timeout_test = Some(test);
break;
}
output = format!("{}Test case '{}'..\n", output, test_name);
if test_name.contains(".p.") {
output += " Pass (success case)\n";
} else if test_name.contains(".f.") {
output += " Fail (failure case)\n";
} else if test_name.contains(".flaky") {
if rand::thread_rng().gen::<bool>() {
output += " Fail (failure case)\n";
} else {
output += " Pass (success)\n";
}
} else if test_name.contains(".s.") {
output += " NotSupported (skip case)\n";
} else if test_name.contains(".c.") {
break;
} else {
unimplemented!("unknown mock test name {}", test_name)
}
}
let mut deqp_results = parse_deqp::parse_deqp_results(output.as_bytes())?;
if let Some(test) = timeout_test {
deqp_results.push(DeqpTestResult {
name: test.name().to_string(),
status: DeqpStatus::Timeout,
duration: Duration::new(0, 0),
});
}
let caselist_state = CaselistState {
caselist_id: 0,
run_id: 0,
};
Ok(deqp_results
.into_iter()
.map(|x| self.translate_result(x, &caselist_state, &tests))
.collect())
}
fn skips(&self) -> Option<&RegexSet> {
self.skips.as_ref()
}
fn flakes(&self) -> Option<&RegexSet> {
self.flakes.as_ref()
}
fn baseline(&self) -> &RunnerResults {
&self.baseline
}
fn status_update(&self, _results: &RunnerResults, _total_tests: u32) {}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
fn mocked_parallel_deqp(tests: Vec<TestCase>) -> RunnerResults {
let deqp = DeqpMock::new();
parallel_deqp(&deqp, tests).unwrap()
}
fn result_status<S: AsRef<str>>(results: &RunnerResults, test: S) -> RunnerStatus {
results.tests.get(test.as_ref()).unwrap().status
}
#[test]
fn many_passes() {
let mut tests = Vec::new();
for i in 0..1000 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.p.{}", i)));
}
let results = mocked_parallel_deqp(tests);
assert_eq!(results.result_counts.pass, 1000);
}
#[test]
fn many_passes_and_a_fail() {
let mut tests = Vec::new();
for i in 0..1000 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.p.{}", i)));
}
tests.push(TestCase::Deqp("dEQP-GLES2.test.f.foo".to_string()));
let results = mocked_parallel_deqp(tests);
assert_eq!(results.result_counts.pass, 1000);
assert_eq!(results.result_counts.fail, 1);
assert_eq!(
result_status(&results, "dEQP-GLES2.test.f.foo"),
RunnerStatus::Fail
);
}
#[test]
fn crash() {
let mut tests = Vec::new();
for i in 0..50 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.{}.p.foo", i)));
}
tests.push(TestCase::Deqp("dEQP-GLES2.test.50.c.foo".to_string()));
for i in 51..100 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.{}.p.foo", i)));
}
let results = mocked_parallel_deqp(tests);
assert_eq!(results.result_counts.pass, 99);
assert_eq!(results.result_counts.crash, 1);
assert_eq!(
result_status(&results, "dEQP-GLES2.test.50.c.foo"),
RunnerStatus::Crash
);
}
#[test]
fn timeout() {
let mut tests = Vec::new();
for i in 0..20 {
if i % 7 == 1 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.{}.timeout.foo", i)));
} else {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.{}.p.foo", i)));
}
}
let results = mocked_parallel_deqp(tests);
assert_eq!(results.result_counts.pass, 17);
assert_eq!(results.result_counts.timeout, 3);
assert_eq!(
result_status(&results, "dEQP-GLES2.test.1.timeout.foo"),
RunnerStatus::Timeout
);
}
#[test]
fn skip_crash() {
let mut tests = Vec::new();
for i in 0..100 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.p.{}", i)));
}
tests.push(TestCase::Deqp("dEQP-GLES2.test.c.foo".to_string()));
let results = parallel_deqp(
DeqpMock::new().with_skips(
"
# Skip all crashing tests
dEQP-GLES2.test.c.*
",
),
tests,
)
.unwrap();
assert_eq!(results.result_counts.pass, 100);
assert_eq!(results.result_counts.crash, 0);
assert_eq!(results.result_counts.skip, 1);
assert_eq!(
result_status(&results, "dEQP-GLES2.test.c.foo"),
RunnerStatus::Skip
);
}
#[test]
fn flake_handling() {
let mut tests = Vec::new();
for i in 0..100 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.p.{}", i)));
}
for i in 0..2 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.flaky.{}", i)));
}
{
let mut found_pass = false;
let mut found_fail = false;
let mut found_flake = false;
while !(found_fail && found_pass && found_flake) {
let results = mocked_parallel_deqp(tests.clone());
match result_status(&results, "dEQP-GLES2.test.flaky.0") {
RunnerStatus::Pass => found_pass = true,
RunnerStatus::Fail => found_fail = true,
RunnerStatus::Flake => found_flake = true,
_ => unreachable!("bad test result"),
}
}
}
{
let mut found_flake = false;
let mut found_pass = false;
let mut found_xfail = false;
while !(found_flake && found_pass && found_xfail) {
let results = parallel_deqp(
DeqpMock::new()
.with_flakes("dEQP-GLES2.test.flaky.*\n")
.with_baseline("dEQP-GLES2.test.flaky.1,Fail"),
tests.clone(),
)
.unwrap();
match result_status(&results, "dEQP-GLES2.test.flaky.0") {
RunnerStatus::Pass => found_pass = true,
RunnerStatus::Flake => {
found_flake = true;
assert!(results.result_counts.flake >= 1);
}
_ => unreachable!("bad test result"),
}
match result_status(&results, "dEQP-GLES2.test.flaky.1") {
RunnerStatus::ExpectedFail => found_xfail = true,
RunnerStatus::Flake => {
found_flake = true;
assert!(results.result_counts.flake >= 1);
}
_ => unreachable!("bad test result"),
}
}
}
}
#[test]
fn baseline() {
let mut tests = Vec::new();
for i in 0..10 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.p.{}", i)));
}
for i in 0..4 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.f.{}", i)));
}
for i in 0..2 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.c.{}", i)));
}
let results = parallel_deqp(
DeqpMock::new().with_baseline(
"
dEQP-GLES2.test.p.1,Fail
dEQP-GLES2.test.f.2,Fail
dEQP-GLES2.test.f.3,Fail
dEQP-GLES2.test.c.1,Crash",
),
tests,
)
.unwrap();
assert_eq!(results.result_counts.pass, 9);
assert_eq!(results.result_counts.unexpected_pass, 1);
assert_eq!(results.result_counts.crash, 1);
assert_eq!(results.result_counts.fail, 2);
assert_eq!(results.result_counts.expected_fail, 3);
}
#[test]
fn missing() {
let mut tests = Vec::new();
for i in 0..100 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.p.{}", i)));
}
tests.push(TestCase::Deqp("dEQP-GLES2.test.m.foo".to_string()));
let results = mocked_parallel_deqp(tests);
assert_eq!(results.result_counts.pass, 100);
assert_eq!(results.result_counts.missing, 1);
assert_eq!(
result_status(&results, "dEQP-GLES2.test.m.foo"),
RunnerStatus::Missing
);
}
fn add_result(results: &mut RunnerResults, test: &str, status: RunnerStatus) {
results.record_result(RunnerResult {
test: TestCase::Deqp(test.to_string()),
status,
duration: Duration::new(0, 0),
});
}
#[test]
fn results_is_success() {
let mut results = RunnerResults::new();
add_result(&mut results, "pass1", RunnerStatus::Pass);
add_result(&mut results, "pass2", RunnerStatus::Pass);
assert_eq!(results.is_success(), true);
add_result(&mut results, "Crash", RunnerStatus::Crash);
assert_eq!(results.is_success(), false);
}
#[test]
fn hms_display() {
assert_eq!(format!("{}", HMSDuration(Duration::new(15, 20))), "15");
assert_eq!(format!("{}", HMSDuration(Duration::new(0, 20))), "0");
assert_eq!(format!("{}", HMSDuration(Duration::new(70, 20))), "1:10");
assert_eq!(format!("{}", HMSDuration(Duration::new(69, 20))), "1:09");
assert_eq!(
format!("{}", HMSDuration(Duration::new(3735, 20))),
"1:02:15"
);
}
#[test]
fn results_serialization() {
let mut tests = Vec::new();
for i in 0..50 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.p.{}", i)));
}
for i in 0..30 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.f.{}", i)));
}
for i in 0..20 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.s.{}", i)));
}
for i in 0..10 {
tests.push(TestCase::Deqp(format!("dEQP-GLES2.test.m.{}", i)));
}
tests.push(TestCase::Deqp("dEQP-GLES2.test.c.foo".to_string()));
let results = mocked_parallel_deqp(tests);
let mut results_file = Cursor::new(Vec::new());
results.write_results(&mut results_file).unwrap();
results_file.set_position(0);
let read_results = RunnerResults::from_csv(&mut results_file).unwrap();
assert_eq!(results.result_counts, read_results.result_counts);
let mut results_file = Cursor::new(Vec::new());
results.write_failures(&mut results_file).unwrap();
results_file.set_position(0);
let read_results = RunnerResults::from_csv(&mut results_file).unwrap();
assert_eq!(0, read_results.result_counts.pass);
assert_eq!(0, read_results.result_counts.skip);
assert_eq!(results.result_counts.fail, read_results.result_counts.fail);
assert_eq!(
results.result_counts.crash,
read_results.result_counts.crash
);
}
#[test]
fn filter_qpa_success() {
assert_eq!(
include_str!("test_data/deqp-gles2-renderer.qpa"),
filter_qpa(
Cursor::new(include_str!("test_data/deqp-gles2-info.qpa")),
"dEQP-GLES2.info.renderer"
)
.unwrap(),
);
}
#[test]
fn filter_qpa_no_results() {
assert!(filter_qpa(
Cursor::new(include_str!("test_data/deqp-empty.qpa")),
"dEQP-GLES2.info.version"
)
.is_err());
}
}