#[macro_use]
extern crate lazy_static;
mod parse_deqp;
mod runner_results;

pub use runner_results::*;

use anyhow::{Context, Result};
use log::*;
use parse_deqp::{DeqpStatus, DeqpTestResult};
use rand::rngs::StdRng;
use rand::seq::SliceRandom;
use rand::{Rng, SeedableRng};
use rayon::prelude::*;
use regex::RegexSet;
use std::fmt;
use std::fs::File;
use std::io::prelude::*;
use std::io::{BufReader, BufWriter};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::mpsc::{channel, Receiver};
use std::time::Duration;

// Wrapper for displaying a duration in h:m:s (integer seconds, rounded down)
struct HMSDuration(Duration);
impl fmt::Display for HMSDuration {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let mut secs = self.0.as_secs();
        let mut has_hours = false;
        if secs >= 3600 {
            write!(f, "{}:", secs / 3600)?;
            secs %= 3600;
            has_hours = true;
        }
        if secs >= 60 {
            if has_hours {
                write!(f, "{:02}:", secs / 60)?;
            } else {
                write!(f, "{}:", secs / 60)?;
            }
            secs %= 60;
            write!(f, "{:02}", secs)
        } else {
            write!(f, "{}", secs)
        }
    }
}

pub trait Deqp {
    fn skips(&self) -> Option<&RegexSet>;
    fn flakes(&self) -> Option<&RegexSet>;
    fn run<S: AsRef<str>, I: IntoIterator<Item = S>>(
        &self,
        caselist_state: &CaselistState,
        tests: I,
    ) -> Result<Vec<RunnerResult>>;

    fn see_more(&self, _caselist_state: &CaselistState) -> String {
        "".to_string()
    }

    fn status_update(&self, _results: &RunnerResults, _total_tests: u32) {}

    fn baseline(&self) -> &RunnerResults;

    fn min_tests_per_group(&self) -> usize {
        1
    }
    fn tests_per_group(&self) -> usize {
        500
    }

    fn qpa_to_xml(&self) -> Option<&PathBuf> {
        None
    }

    fn baseline_status<S: AsRef<str>>(&self, test: S) -> Option<RunnerStatus> {
        self.baseline().tests.get(test.as_ref()).map(|x| x.status)
    }

    fn translate_result(
        &self,
        result: DeqpTestResult,
        caselist_state: &CaselistState,
    ) -> RunnerResult {
        let test = result.name;
        let mut status =
            RunnerStatus::from_deqp(result.status).with_baseline(self.baseline_status(&test));

        if let Some(flakes) = self.flakes() {
            if !status.is_success() && flakes.is_match(&test) {
                status = RunnerStatus::Flake;
            }
        }

        if !status.is_success() || status == RunnerStatus::Flake {
            error!(
                "Test {}: {}: {}",
                test,
                status,
                self.see_more(&caselist_state)
            );
        }

        RunnerResult {
            test,
            status,
            duration: result.duration,
        }
    }

    fn results_collection(
        &self,
        run_results: &mut RunnerResults,
        total_tests: u32,
        receiver: Receiver<Result<Vec<RunnerResult>>>,
    ) {
        self.status_update(run_results, total_tests);
        for group_results in receiver {
            match group_results {
                Ok(group_results) => {
                    for result in group_results {
                        assert!(!run_results.tests.contains_key(&result.test));

                        run_results.record_result(result);
                    }
                }
                Err(e) => {
                    println!("Error: {}", e);
                }
            }
            self.status_update(run_results, total_tests);
        }
    }

    fn skip_test(&self, test: &str) -> bool {
        if let Some(skips) = self.skips() {
            skips.is_match(test)
        } else {
            false
        }
    }

    fn run_caselist_and_flake_detect(
        &self,
        caselist: &[String],
        caselist_state: &mut CaselistState,
    ) -> Result<Vec<RunnerResult>> {
        // Sort the caselists within test groups.  dEQP runs tests in sorted order, and when one
        // is debugging a failure in one case in a caselist, it can be nice to be able to easily trim
        // all of the caselist appearing after the failure, to reduce runtime.
        let mut caselist: Vec<_> = caselist.iter().collect();
        caselist.sort();

        caselist_state.run_id += 1;
        let mut results = self.run(&caselist_state, &caselist)?;
        // If we made no more progress on the whole caselist,
        // then dEQP doesn't know about some of our tests and they'll report Missing.
        if results.is_empty() {
            anyhow::bail!("No results parsed");
        }

        // If any results came back with an unexpected failure, run the caselist again
        // to see if we get the same results, and mark any changing results as flaky tests.
        if results.iter().any(|x| !x.status.is_success()) {
            caselist_state.run_id += 1;
            let retest_results = self.run(&caselist_state, &caselist)?;
            for pair in results.iter_mut().zip(retest_results.iter()) {
                if pair.0.status != pair.1.status {
                    pair.0.status = RunnerStatus::Flake;
                }
            }
        }

        Ok(results)
    }

    fn process_caselist<S: AsRef<str>, I: IntoIterator<Item = S>>(
        &self,
        tests: I,
        caselist_id: u32,
    ) -> Result<Vec<RunnerResult>> {
        let mut caselist_results: Vec<RunnerResult> = Vec::new();
        let mut remaining_tests = Vec::new();
        for test in tests {
            let test = test.as_ref().to_string();
            if self.skip_test(&test) {
                caselist_results.push(RunnerResult {
                    test,
                    status: RunnerStatus::Skip,
                    duration: Default::default(),
                });
            } else {
                remaining_tests.push(test);
            }
        }

        let mut caselist_state = CaselistState {
            caselist_id,
            run_id: 0,
        };

        while !remaining_tests.is_empty() {
            let results = self.run_caselist_and_flake_detect(&remaining_tests, &mut caselist_state);

            match results {
                Ok(results) => {
                    for result in results {
                        remaining_tests.swap_remove(
                            remaining_tests
                                .iter()
                                .position(|x| *x == result.test)
                                .context("Finding deqp test name in test list")?,
                        );
                        caselist_results.push(result);
                    }
                }
                Err(e) => {
                    error!(
                        "Failure getting dEQP run results: {:#} ({})",
                        e,
                        self.see_more(&caselist_state)
                    );

                    for test in remaining_tests {
                        caselist_results.push(RunnerResult {
                            test,
                            status: RunnerStatus::Missing,
                            duration: Default::default(),
                        });
                    }
                    break;
                }
            }

            caselist_state.caselist_id += 1;
        }

        Ok(caselist_results)
    }

    fn split_tests_to_groups(&self, mut test_names: Vec<String>) -> Vec<(u32, Vec<String>)> {
        // Shuffle the test groups using a deterministic RNG so that every run gets the same shuffle.
        test_names.shuffle(&mut StdRng::from_seed([0x3bu8; 32]));

        // Make test groups of tests_per_group() (512) tests, or if
        // min_tests_per_group() is lower than that, then 1/32nd of the
        // remaining tests down to that limit.
        let mut test_groups: Vec<(u32, Vec<String>)> = Vec::new();
        let mut remaining = test_names.len();
        let mut i = 0u32;
        while remaining != 0 {
            let min = usize::min(self.min_tests_per_group(), remaining);
            let group_len = usize::min(usize::max(remaining / 32, min), self.tests_per_group());
            remaining -= group_len;

            test_groups.push((i, test_names.split_off(remaining)));
            i += 1;
        }

        test_groups
    }
}

pub struct DeqpCommand {
    pub deqp: PathBuf,
    pub args: Vec<String>,
    pub output_dir: PathBuf,
    pub skips: Option<RegexSet>,
    pub flakes: Option<RegexSet>,
    pub qpa_to_xml: Option<PathBuf>,
    pub baseline: RunnerResults,
    pub timeout: Duration,
    pub tests_per_group: usize,
    pub min_tests_per_group: usize,
}

fn write_caselist_file<S: AsRef<str>, I: IntoIterator<Item = S>>(
    filename: &Path,
    tests: I,
) -> Result<()> {
    let file = File::create(filename)
        .with_context(|| format!("creating temp caselist file {}", filename.display()))?;
    let mut file = BufWriter::new(file);

    for test in tests {
        file.write(test.as_ref().as_bytes())
            .context("writing temp caselist")?;
        file.write(b"\n").context("writing temp caselist")?;
    }
    Ok(())
}

fn add_filename_arg(args: &mut Vec<String>, arg: &str, path: &Path) -> Result<()> {
    args.push(format!(
        "{}={}",
        arg,
        path.to_str()
            .with_context(|| format!("filename to utf8 for {}", path.display()))?
    ));
    Ok(())
}

fn filter_qpa<R: Read, S: AsRef<str>>(reader: R, test: S) -> Result<String> {
    let lines = BufReader::new(reader).lines();

    let start = format!("#beginTestCaseResult {}", test.as_ref());

    let mut found_case = false;
    let mut including = true;
    let mut output = String::new();
    for line in lines {
        let line = line.context("reading QPA")?;
        if line == start {
            found_case = true;
            including = true;
        }

        if including {
            output.push_str(&line);
            output.push('\n');
        }

        if line == "#beginSession" {
            including = false;
        }

        if including && line == "#endTestCaseResult" {
            break;
        }
    }

    if !found_case {
        anyhow::bail!("Failed to find {} in QPA", test.as_ref());
    }

    Ok(output)
}

impl DeqpCommand {
    fn caselist_file_path(&self, caselist_state: &CaselistState, suffix: &str) -> Result<PathBuf> {
        // deqp must be run from its directory, so make sure all the filenames we pass in are absolute.
        let output_dir = self.output_dir.canonicalize()?;

        Ok(output_dir.join(format!("c{}.{}", caselist_state.caselist_id, suffix)))
    }

    fn try_extract_qpa<S: AsRef<str>, P: AsRef<Path>>(&self, test: S, qpa_path: P) -> Result<()> {
        let qpa_path = qpa_path.as_ref();
        let test = test.as_ref();
        let output = filter_qpa(
            File::open(qpa_path).with_context(|| format!("Opening {}", qpa_path.display()))?,
            test,
        )?;

        if !output.is_empty() {
            let out_path = qpa_path.parent().unwrap().join(format!("{}.qpa", test));
            // Write the extracted QPA contents to an individual file.
            {
                let mut out_qpa = BufWriter::new(File::create(&out_path).with_context(|| {
                    format!("Opening output QPA file {:?}", qpa_path.display())
                })?);
                out_qpa.write_all(output.as_bytes())?;
            }

            // Now that the QPA file is written (and flushed, note the separate
            // block!), call out to testlog-to-xml to convert it to an XML file
            // for display.
            if let Some(qpa_to_xml) = self.qpa_to_xml() {
                let xml_path = out_path.with_extension("xml");
                let convert_output = Command::new(qpa_to_xml)
                    .current_dir(self.deqp.parent().unwrap_or_else(|| Path::new("/")))
                    .arg(&out_path)
                    .arg(xml_path)
                    .output()
                    .with_context(|| format!("Failed to spawn {}", qpa_to_xml.display()))?;
                if !convert_output.status.success() {
                    anyhow::bail!(
                        "Failed to run {}: {}",
                        qpa_to_xml.display(),
                        String::from_utf8_lossy(&convert_output.stderr)
                    );
                } else {
                    std::fs::remove_file(&out_path).context("removing converted QPA")?;
                }
            }
        }

        Ok(())
    }
}

impl Deqp for DeqpCommand {
    fn run<S: AsRef<str>, I: IntoIterator<Item = S>>(
        &self,
        caselist_state: &CaselistState,
        tests: I,
    ) -> Result<Vec<RunnerResult>> {
        let caselist_path = self.caselist_file_path(
            &caselist_state,
            format!("r{}.caselist.txt", caselist_state.run_id).as_str(),
        )?;
        let qpa_path = self.caselist_file_path(
            &caselist_state,
            format!("r{}.qpa", caselist_state.run_id).as_str(),
        )?;
        let cache_path = self
            .output_dir
            .canonicalize()?
            .join(format!("t{}.shader_cache", thread_id::get()));

        self.caselist_file_path(&caselist_state, "shader_cache")?;

        write_caselist_file(&caselist_path, tests)?;

        let mut args: Vec<String> = Vec::new();

        add_filename_arg(&mut args, "--deqp-caselist-file", &caselist_path)?;
        add_filename_arg(&mut args, "--deqp-log-filename", &qpa_path)?;
        args.push("--deqp-log-flush=disable".to_string());

        // The shader cache is not multiprocess safe, use one per
        // caselist_state.  However, since we're spawning lots of separate dEQP
        // runs, disable truncation (which would otherwise mean we only
        // get caching within a single run_block(), which is pretty
        // small).
        add_filename_arg(&mut args, "--deqp-shadercache-filename", &cache_path)?;
        args.push("--deqp-shadercache-truncate=disable".to_string());

        // Add on the user's specified deqp arguments.
        for arg in &self.args {
            args.push(arg.clone());
        }

        let mut child = Command::new(&self.deqp)
            .current_dir(self.deqp.parent().unwrap_or_else(|| Path::new("/")))
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .stdin(Stdio::null())
            .args(args)
            // Disable MESA_DEBUG output by default for debug Mesa builds, which
            // otherwise fills the logs with warnings about GL errors that are
            // thrown (you're running deqp!  Of course it makes GL errors!)
            .env("MESA_DEBUG", "silent")
            .spawn()
            .with_context(|| format!("Failed to spawn {}", &self.deqp.display()))?;

        let stdout = child.stdout.take().context("opening stdout")?;
        let mut deqp_results = parse_deqp::parse_deqp_results_with_timeout(stdout, self.timeout)
            .context("parsing results")?;

        // The child should have run to completion based on parse_deqp_results() consuming its output,
        // but if we had a timeout then we want to kill this run.
        let _ = child.kill();

        // Make sure we reap the child process.
        child.wait()?;

        let stderr = BufReader::new(child.stderr.as_mut().context("opening stderr")?);
        for line in stderr.lines() {
            if let Ok(line) = line {
                // If the driver has ASan enabled and it detected leaks, then mark
                // all the tests in the caselist as failed (since we don't know who
                // to assign the failure to).
                if line.contains("ERROR: LeakSanitizer: detected memory leaks") {
                    error!(
                        "deqp-runner: Leak detected, marking caselist as failed ({})",
                        self.see_more(caselist_state)
                    );
                    for result in deqp_results.iter_mut() {
                        result.status = DeqpStatus::Fail;
                    }
                }
                error!("dEQP error: {}", line);
            }
        }

        let mut results: Vec<RunnerResult> = Vec::new();
        for result in deqp_results {
            let result = self.translate_result(result, &caselist_state);

            if !result.status.is_success() {
                if let Err(e) = self.try_extract_qpa(&result.test, &qpa_path) {
                    warn!("Failed to extract QPA resuls: {}", e)
                }
            }
            results.push(result);
        }

        if results
            .iter()
            .all(|x| x.status.is_success() && x.status != RunnerStatus::Flake)
        {
            std::fs::remove_file(caselist_path)?;
        }
        std::fs::remove_file(qpa_path)?;

        Ok(results)
    }

    fn see_more(&self, caselist_state: &CaselistState) -> String {
        // This is the same as run() did, so we should be safe to unwrap.
        let qpa_path = self.output_dir.join(
            format!(
                "c{}.r{}.caselist.txt",
                caselist_state.caselist_id, caselist_state.run_id
            )
            .as_str(),
        );
        format!("See {:?}", qpa_path)
    }

    fn status_update(&self, results: &RunnerResults, total_tests: u32) {
        let duration = results.time.elapsed();

        print!(
            "{}, Duration: {duration}",
            results.result_counts,
            duration = HMSDuration(duration),
        );

        // If we have some tests completed, use that to estimate remaining runtime.
        let duration = duration.as_secs_f32();
        if results.result_counts.total != 0 {
            let average_test_time = duration / results.result_counts.total as f32;
            let remaining = average_test_time * (total_tests - results.result_counts.total) as f32;

            print!(
                ", Remaining: {}",
                HMSDuration(Duration::from_secs_f32(remaining))
            );
        }

        println!();
    }

    fn skips(&self) -> Option<&RegexSet> {
        self.skips.as_ref()
    }

    fn flakes(&self) -> Option<&RegexSet> {
        self.flakes.as_ref()
    }

    fn baseline(&self) -> &RunnerResults {
        &self.baseline
    }

    fn tests_per_group(&self) -> usize {
        self.tests_per_group
    }

    fn min_tests_per_group(&self) -> usize {
        self.min_tests_per_group
    }

    fn qpa_to_xml(&self) -> Option<&PathBuf> {
        self.qpa_to_xml.as_ref()
    }
}

// Splits the list of tests to groups and parallelize them across all cores, collecting results in
// a separate thread
pub fn parallel_deqp<D>(deqp: &D, tests: Vec<String>) -> Result<RunnerResults>
where
    D: Deqp,
    D: Sync,
{
    let test_count = tests.len();
    let test_groups = deqp.split_tests_to_groups(tests);

    let mut run_results = RunnerResults::new();

    // Make a channel for the parallel iterator to send results to, which is what will be
    // printing the console status output but also computing the run_results.
    let (sender, receiver) = channel::<Result<Vec<RunnerResult>>>();

    rayon::scope(|s| {
        // Spin up the thread collecting and displaying our results.
        s.spawn(|_| deqp.results_collection(&mut run_results, test_count as u32, receiver));

        // Rayon parallel iterator takes our vector and spawns threads processing the test groups in it.
        test_groups
            .into_iter()
            .par_bridge()
            .try_for_each_with(sender, |sender, (i, tests)| {
                sender.send(deqp.process_caselist(tests, i))
            })
            .unwrap();

        // As we leave this scope, rayon will join all our threads.
    });

    Ok(run_results)
}

// Parses a deqp-runner regex set list.  We ignore empty lines and lines starting with "#", so you can
// leave notes in your skips/flakes lists about why.
pub fn parse_regex_set<I, S>(exprs: I) -> Result<RegexSet>
where
    S: AsRef<str>,
    I: IntoIterator<Item = S>,
{
    RegexSet::new(
        exprs
            .into_iter()
            .filter(|x| !x.as_ref().is_empty() && !x.as_ref().starts_with('#')),
    )
    .context("Parsing regex set")
}

#[derive(Default)]
// Mock dEQP runner with behavior based on the test name.
pub struct DeqpMock {
    pub skips: Option<RegexSet>,
    pub flakes: Option<RegexSet>,
    pub baseline: RunnerResults,
}

impl DeqpMock {
    pub fn new() -> DeqpMock {
        Default::default()
    }
    pub fn skips(&self) -> Option<&RegexSet> {
        self.skips.as_ref()
    }
    pub fn flakes(&self) -> Option<&RegexSet> {
        self.flakes.as_ref()
    }

    pub fn with_skips<S>(&mut self, skips: S) -> &mut DeqpMock
    where
        S: AsRef<str>,
    {
        self.skips = Some(parse_regex_set(skips.as_ref().lines()).unwrap());
        self
    }

    pub fn with_flakes<S>(&mut self, flakes: S) -> &mut DeqpMock
    where
        S: AsRef<str>,
    {
        self.flakes = Some(parse_regex_set(flakes.as_ref().lines()).unwrap());
        self
    }

    pub fn with_baseline<S>(&mut self, baseline: S) -> &mut DeqpMock
    where
        S: AsRef<str>,
    {
        self.baseline =
            RunnerResults::from_csv(&mut std::io::Cursor::new(baseline.as_ref())).unwrap();
        self
    }
}

impl Deqp for DeqpMock {
    fn run<S: AsRef<str>, I: IntoIterator<Item = S>>(
        &self,
        _caselist_state: &CaselistState,
        tests: I,
    ) -> Result<Vec<RunnerResult>> {
        // Sort the tests so we print results in order, same as real deqp.
        let mut tests: Vec<String> = tests.into_iter().map(|x| x.as_ref().to_string()).collect();
        tests.sort();

        let mut output = String::from("dEQP Mock starting\n");

        for test in tests {
            // Missing tests won't appear in the output at all.
            if test.contains("dEQP-GLES2.test.m.") {
                continue;
            }

            output = format!("{}Test case '{}'..\n", output, test);

            if test.contains("dEQP-GLES2.test.p.") {
                output += "  Pass (success case)\n";
            } else if test.contains("dEQP-GLES2.test.f.") {
                output += "  Fail (failure case)\n";
            } else if test.contains("dEQP-GLES2.test.flaky.") {
                if rand::thread_rng().gen::<bool>() {
                    output += "  Fail (failure case)\n";
                } else {
                    output += "  Pass (success)\n";
                }
            } else if test.contains("dEQP-GLES2.test.s.") {
                output += "  NotSupported (skip case)\n";
            } else if test.contains("dEQP-GLES2.test.c.") {
                // In a crash, the output just stops before we get a result.
                break;
            } else {
                unimplemented!("unknown mock test name {}", test)
            }
        }

        let deqp_results = parse_deqp::parse_deqp_results(output.as_bytes())?;

        let caselist_state = CaselistState {
            caselist_id: 0,
            run_id: 0,
        };
        Ok(deqp_results
            .into_iter()
            .map(|x| self.translate_result(x, &caselist_state))
            .collect())
    }

    fn skips(&self) -> Option<&RegexSet> {
        self.skips.as_ref()
    }

    fn flakes(&self) -> Option<&RegexSet> {
        self.flakes.as_ref()
    }

    fn baseline(&self) -> &RunnerResults {
        &self.baseline
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::Cursor;

    fn mocked_parallel_deqp(tests: Vec<String>) -> RunnerResults {
        let deqp = DeqpMock::new();
        parallel_deqp(&deqp, tests).unwrap()
    }

    fn result_status<S: AsRef<str>>(results: &RunnerResults, test: S) -> RunnerStatus {
        results.tests.get(test.as_ref()).unwrap().status
    }

    #[test]
    fn many_passes() {
        let mut tests = Vec::new();
        for i in 0..1000 {
            tests.push(format!("dEQP-GLES2.test.p.{}", i));
        }

        let results = mocked_parallel_deqp(tests);
        assert_eq!(results.result_counts.pass, 1000);
    }
    #[test]
    fn many_passes_and_a_fail() {
        let mut tests = Vec::new();
        for i in 0..1000 {
            tests.push(format!("dEQP-GLES2.test.p.{}", i));
        }
        tests.push("dEQP-GLES2.test.f.foo".to_string());

        let results = mocked_parallel_deqp(tests);
        assert_eq!(results.result_counts.pass, 1000);
        assert_eq!(results.result_counts.fail, 1);
        assert_eq!(
            result_status(&results, "dEQP-GLES2.test.f.foo"),
            RunnerStatus::Fail
        );
    }

    #[test]
    fn crash() {
        let mut tests = Vec::new();
        for i in 0..100 {
            tests.push(format!("dEQP-GLES2.test.p.{}", i));
        }
        tests.push("dEQP-GLES2.test.c.foo".to_string());

        let results = mocked_parallel_deqp(tests);
        assert_eq!(results.result_counts.pass, 100);
        assert_eq!(results.result_counts.crash, 1);
        assert_eq!(
            result_status(&results, "dEQP-GLES2.test.c.foo"),
            RunnerStatus::Crash
        );
    }

    // Tests a run with a skips list like we might actually write in Mesa CI
    #[test]
    fn skip_crash() {
        let mut tests = Vec::new();
        for i in 0..100 {
            tests.push(format!("dEQP-GLES2.test.p.{}", i));
        }
        tests.push("dEQP-GLES2.test.c.foo".to_string());

        let results = parallel_deqp(
            DeqpMock::new().with_skips(
                "
# Skip all crashing tests

dEQP-GLES2.test.c.*
",
            ),
            tests,
        )
        .unwrap();

        assert_eq!(results.result_counts.pass, 100);
        assert_eq!(results.result_counts.crash, 0);
        assert_eq!(results.result_counts.skip, 1);
        assert_eq!(
            result_status(&results, "dEQP-GLES2.test.c.foo"),
            RunnerStatus::Skip
        );
    }

    // Tests a run with a flakes list like we might actually write in Mesa CI
    #[test]
    fn flake_handling() {
        let mut tests = Vec::new();
        for i in 0..100 {
            tests.push(format!("dEQP-GLES2.test.p.{}", i));
        }
        for i in 0..2 {
            tests.push(format!("dEQP-GLES2.test.flaky.{}", i));
        }

        {
            // Verify that our mocked flaky test actually flakes, and that
            // the default flake handling detects it!
            let mut found_pass = false;
            let mut found_fail = false;
            let mut found_flake = false;
            while !(found_fail && found_pass && found_flake) {
                let results = mocked_parallel_deqp(tests.clone());
                match result_status(&results, "dEQP-GLES2.test.flaky.0") {
                    RunnerStatus::Pass => found_pass = true,
                    RunnerStatus::Fail => found_fail = true,
                    RunnerStatus::Flake => found_flake = true,
                    _ => unreachable!("bad test result"),
                }
            }
        }

        {
            // Verify that we can handle known flakes
            let mut found_flake = false;
            let mut found_pass = false;
            let mut found_xfail = false;
            while !(found_flake && found_pass && found_xfail) {
                let results = parallel_deqp(
                    DeqpMock::new()
                        .with_flakes("dEQP-GLES2.test.flaky.*\n")
                        .with_baseline("dEQP-GLES2.test.flaky.1,Fail"),
                    tests.clone(),
                )
                .unwrap();

                match result_status(&results, "dEQP-GLES2.test.flaky.0") {
                    RunnerStatus::Pass => found_pass = true,
                    RunnerStatus::Flake => {
                        found_flake = true;
                        assert!(results.result_counts.flake >= 1);
                    }
                    _ => unreachable!("bad test result"),
                }

                match result_status(&results, "dEQP-GLES2.test.flaky.1") {
                    RunnerStatus::ExpectedFail => found_xfail = true,
                    RunnerStatus::Flake => {
                        found_flake = true;
                        assert!(results.result_counts.flake >= 1);
                    }
                    _ => unreachable!("bad test result"),
                }
            }
        }
    }

    #[test]
    fn baseline() {
        let mut tests = Vec::new();
        for i in 0..10 {
            tests.push(format!("dEQP-GLES2.test.p.{}", i));
        }
        for i in 0..4 {
            tests.push(format!("dEQP-GLES2.test.f.{}", i));
        }
        for i in 0..2 {
            tests.push(format!("dEQP-GLES2.test.c.{}", i));
        }

        let results = parallel_deqp(
            DeqpMock::new().with_baseline(
                "
dEQP-GLES2.test.p.1,Fail
dEQP-GLES2.test.f.2,Fail
dEQP-GLES2.test.f.3,Fail
dEQP-GLES2.test.c.1,Crash",
            ),
            tests,
        )
        .unwrap();

        assert_eq!(results.result_counts.pass, 9);
        assert_eq!(results.result_counts.unexpected_pass, 1);
        assert_eq!(results.result_counts.crash, 1);
        assert_eq!(results.result_counts.fail, 2);
        assert_eq!(results.result_counts.expected_fail, 3);
    }

    #[test]
    fn missing() {
        let mut tests = Vec::new();
        for i in 0..100 {
            tests.push(format!("dEQP-GLES2.test.p.{}", i));
        }
        tests.push("dEQP-GLES2.test.m.foo".to_string());

        let results = mocked_parallel_deqp(tests);
        assert_eq!(results.result_counts.pass, 100);
        assert_eq!(results.result_counts.missing, 1);
        assert_eq!(
            result_status(&results, "dEQP-GLES2.test.m.foo"),
            RunnerStatus::Missing
        );
    }

    fn add_result(results: &mut RunnerResults, test: &str, status: RunnerStatus) {
        results.record_result(RunnerResult {
            test: test.to_string(),
            status,
            duration: Duration::new(0, 0),
        });
    }

    #[test]
    fn results_is_success() {
        let mut results = RunnerResults::new();

        add_result(&mut results, "pass1", RunnerStatus::Pass);
        add_result(&mut results, "pass2", RunnerStatus::Pass);

        assert_eq!(results.is_success(), true);

        add_result(&mut results, "Crash", RunnerStatus::Crash);

        assert_eq!(results.is_success(), false);
    }

    #[test]
    fn hms_display() {
        assert_eq!(format!("{}", HMSDuration(Duration::new(15, 20))), "15");
        assert_eq!(format!("{}", HMSDuration(Duration::new(0, 20))), "0");
        assert_eq!(format!("{}", HMSDuration(Duration::new(70, 20))), "1:10");
        assert_eq!(format!("{}", HMSDuration(Duration::new(69, 20))), "1:09");
        assert_eq!(
            format!("{}", HMSDuration(Duration::new(3735, 20))),
            "1:02:15"
        );
    }

    // Tests round-tripping some results through csv formatting.
    #[test]
    fn results_serialization() {
        let mut tests = Vec::new();
        for i in 0..50 {
            tests.push(format!("dEQP-GLES2.test.p.{}", i));
        }
        for i in 0..30 {
            tests.push(format!("dEQP-GLES2.test.f.{}", i));
        }
        for i in 0..20 {
            tests.push(format!("dEQP-GLES2.test.s.{}", i));
        }
        for i in 0..10 {
            tests.push(format!("dEQP-GLES2.test.m.{}", i));
        }
        tests.push("dEQP-GLES2.test.c.foo".to_string());
        let results = mocked_parallel_deqp(tests);

        let mut results_file = Cursor::new(Vec::new());
        results.write_results(&mut results_file).unwrap();
        results_file.set_position(0);
        let read_results = RunnerResults::from_csv(&mut results_file).unwrap();
        assert_eq!(results.result_counts, read_results.result_counts);

        let mut results_file = Cursor::new(Vec::new());
        results.write_failures(&mut results_file).unwrap();
        results_file.set_position(0);
        let read_results = RunnerResults::from_csv(&mut results_file).unwrap();
        assert_eq!(0, read_results.result_counts.pass);
        assert_eq!(0, read_results.result_counts.skip);
        assert_eq!(results.result_counts.fail, read_results.result_counts.fail);
        assert_eq!(
            results.result_counts.crash,
            read_results.result_counts.crash
        );
    }

    #[test]
    fn filter_qpa_success() {
        assert_eq!(
            include_str!("test_data/deqp-gles2-renderer.qpa"),
            filter_qpa(
                Cursor::new(include_str!("test_data/deqp-gles2-info.qpa")),
                "dEQP-GLES2.info.renderer"
            )
            .unwrap(),
        );
    }

    #[test]
    fn filter_qpa_no_results() {
        assert!(filter_qpa(
            Cursor::new(include_str!("test_data/deqp-empty.qpa")),
            "dEQP-GLES2.info.version"
        )
        .is_err());
    }
}